Upload large files in multi part

boto3_multipartupload.py

import boto3
import os
import sys
import threading
from boto3.s3.transfer import TransferConfig

s3_client = boto3.client('s3')

# S3 transfer documentation: https://boto3.amazonaws.com/v1/documentation/api/1.9.156/reference/customizations/s3.html?highlight=transferconfig#boto3.s3.transfer.TransferConfig

# From boto3.s3.transfer: source code https://boto3.amazonaws.com/v1/documentation/api/latest/_modules/boto3/s3/transfer.html
class ProgressPercentage(object):
    def __init__(self, filename):
        self._filename = filename
        self._size = float(os.path.getsize(filename))
        self._seen_so_far = 0
        self._lock = threading.Lock()

    def __call__(self, bytes_amount):
        # To simplify we'll assume this is hooked up
        # to a single filename.
        with self._lock:
            self._seen_so_far += bytes_amount
            percentage = (self._seen_so_far / self._size) * 100
            sys.stdout.write(
                "\r%s  %s / %s  (%.2f%%)" % (
                    self._filename, self._seen_so_far, self._size,
                    percentage))
            sys.stdout.flush()

def uploadFileS3(filepath, bucket, keypath=''):
    config = TransferConfig(multipart_threshold=1024*25,
                          max_concurrency=10,
                          multipart_chunksize=1024*25,
                          use_threads=True)

    head, tail = os.path.split(filepath)
    key = keypath + tail
    # https://boto3.amazonaws.com/v1/documentation/api/1.9.185/guide/s3-uploading-files.html
    s3_client.upload_file(filepath, bucket, key,
        Config = config,
        Callback=ProgressPercentage(filepath)
    )

uploadFileS3('/tmp/500mfile', 'nivonh-poc')