Multi-part Upload

gistfile1.txt

import sys
import argparse
import glob
import subprocess

import boto

parser = argparse.ArgumentParser(description='Upload large (5GB+) file to Amazon S3.')

parser.add_argument('--aws_access_key_id', required=True,
                    help='Your AWS Access Key ID')
parser.add_argument('--aws_secret_access_key', required=True,
                    help='Your AWS Secret Access ID')
parser.add_argument('--s3_bucket', required=True,
                    help='S3 bucket you want the file added to.')
parser.add_argument('--key', required=True,
                    help='Key for S3 endpoint.')
parser.add_argument('--file', required=True,
                    help='Full path to file to be uploaded.')

args = parser.parse_args()

s3 = boto.connect_s3(aws_access_key_id=args.aws_access_key_id,
                       aws_secret_access_key=args.aws_secret_access_key)

bucket = s3.get_bucket(args.s3_bucket)

print bucket

mp = bucket.initiate_multipart_upload(args.key)

files = glob.glob('multi-part-upload-*')

if len(files) == 0:
    sys.stdout.write('Splitting file into 500m chunks...\n')
    ret = subprocess.call('split -a3 -b500m {file} multi-part-upload-'.format(file=args.file), shell=True)
    sys.stdout.write('..finished splitting into {num} files. Uploading..'.format(num=len(files)))

files = glob.glob('multi-part-upload-*')

for part_number, piece in enumerate(files, start=1):
    with open(piece, 'rb') as fp:
        mp.upload_part_from_file(fp, part_number)

mp.complete_upload()