Glacier Multipart Uploader

gmu.py

#!/usr/bin/python

import os
import sys
import getopt
import boto3
from botocore.utils import calculate_tree_hash


def read_in_chunks(file_object, chunk_size):
    while True:
        data = file_object.read(chunk_size)
        if not data:
            break
        yield data

        
def upload(argv):

    vault_name = ''
    file_name = ''
    chunk_size = 1048576

    try:
        opts, args = getopt.getopt(argv, "v:f:c:")
    except getopt.GetoptError:
        print 'gmu.py -v <vault_name> -f <file_name> [-c <chunk_size>]'
        sys.exit(2)

    for opt, arg in opts:
        if opt == '-v':
            vault_name = arg
        elif opt == '-f':
            file_name = arg
        elif opt == '-c':
            chunk_size = int(arg)

    glacier = boto3.client('glacier')
    init_upload_resp = glacier.initiate_multipart_upload(
        vaultName=vault_name,
        partSize=str(chunk_size)
    )
    rng_start = 0
    f = open(file_name, 'rb')
    f_cs = calculate_tree_hash(open(file_name, 'rb'))
    f_size = os.path.getsize(file_name)
    for piece in read_in_chunks(f, chunk_size):
        rng_end = rng_start + len(piece) - 1
        rng = 'bytes {0}-{1}/*'.format(rng_start, rng_end)
        upload_resp = glacier.upload_multipart_part(
            vaultName=vault_name,
            uploadId=init_upload_resp[u'uploadId'],
            range=rng,
            body=piece
        )['ResponseMetadata']['HTTPStatusCode']
        if upload_resp >= 300:
            print 'error'
            sys.exit(2)

        rng_start += chunk_size
        sys.stdout.write('.')
        sys.stdout.flush()

    resp = glacier.complete_multipart_upload(
        vaultName=vault_name,
        uploadId=init_upload_resp[u'uploadId'],
        checksum=f_cs,
        archiveSize=str(f_size)
    )
    print resp

if __name__ == "__main__":
    upload(sys.argv[1:])