blob2s3.py

import os
import tempfile
import uuid
import argparse

import boto3
import azure.storage.blob as blob
import tqdm


# I cheated and generated this list from azure storage explorer 
# by selecting all the files I wanted and copying the url and adding keys for the object name on s3
uri_dict = {}


BUCKET_NAME = "<bucket-name>"


# https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html
def upload_file(file_name, bucket, object_name=None):
    """Upload a file to an S3 bucket

    :param file_name: File to upload
    :param bucket: Bucket to upload to
    :param object_name: S3 object name. If not specified then file_name is used
    :return: True if file was uploaded, else False
    """

    # If S3 object_name was not specified, use file_name
    if object_name is None:
        object_name = file_name

    # Upload the file
    s3_client = boto3.client('s3')
    try:
        response = s3_client.upload_file(file_name, bucket, object_name)
    except ClientError as e:
        logging.error(e)
        return False
    return True


for object_name, uri in tqdm.tqdm(uri_dict.items()):
    # print("### DRY RUN")
    filename = f"store/{object_name[-2]}"
    # print(f"Download {uri}\nAs {filename}\nUpload as {object_name}.gz")
    # https://docs.microsoft.com/en-us/python/api/azure-storage-blob/azure.storage.blob?view=azure-python#download-blob-from-url-blob-url--output--credential-none----kwargs-
    blob.download_blob_from_url(uri, filename, overwrite=True)
    upload_file(filename, BUCKET_NAME, f"{object_name}.gz")
    os.remove(filename)