Copy from RequesterPays buckets to your bucket

rp-copy.py

import boto3
import argparse
import logging
from multiprocessing.pool import ThreadPool
from functools import partial

resource = boto3.client('s3')
client = boto3.client('s3')


def splitbucket(key):
    first, second = key.split('//')

    # bucket is first item, key is the rest
    keys = second.split('/')
    bucket = keys[0]
    key = '/'.join(keys[1:])

    if args.verbose:
        logging.debug("Split bucket '%s' with key '%s'" % (bucket, key))

    return (bucket, key)

def get_objects(args):

    if args.verbose:
        logging.debug("Listing bucket '%s' with key '%s'" % (args.source_bucket, args.source_key))
    paginator = client.get_paginator('list_objects')
    pageresponse = paginator.paginate(Bucket=args.source_bucket,
                                      Prefix=args.source_key,
                                      RequestPayer='requester')
    keys = []
    for pageobject in pageresponse:
        for obj in pageobject["Contents"]:
            keys.append(obj)

    return keys

def copy(obj, args):

    source_key = obj['Key']

    fname = source_key.split('/')[-1]
    destination_key = args.destination_key + '/' + fname

    if args.verbose:
        logging.debug("Copy from '%s' with key '%s' to '%s' with key '%s' " % (args.source_bucket, source_key, args.destination_bucket, args.destination_key))
    client.copy_object(CopySource={'Bucket':args.source_bucket, 'Key':source_key},
                       Bucket=args.destination_bucket,
                       Key = destination_key,
                       RequestPayer='requester')



def main(args):
    if args.verbose:
        logging.debug("Fetching bucket '%s' to '%s'" % (args.source, args.destination))

    args.source_bucket, args.source_key = splitbucket(args.source)
    args.destination_bucket, args.destination_key = splitbucket(args.destination)
    keys = get_objects(args)
    pool = ThreadPool(args.processes)
    results = pool.map(partial(copy, args=args), keys)

#     for key in keys:
#         copy(key, args)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description = 'Copy a RequesterPays bucket to another bucket')
    parser.add_argument('source', help='RequesterPays bucket to copy')
    parser.add_argument('destination', help='Destination bucket')
    parser.add_argument('--processes', help='Number of processes to use', type=int)
    parser.add_argument('-v', '--verbose', help='Verbose', action="store_true")
    parser.set_defaults(processes=1, type=int)
    args = parser.parse_args()

    if args.verbose:
        logging.basicConfig(level=logging.DEBUG)

    # rp-copy.py s3://my-rp-bucket/key s3://my-own-bucket/key -v --processes 5
    main(args)