-
-
Save manuelcoppotelli/afc9c3a5b1337e7a34e71d0ae6f58e33 to your computer and use it in GitHub Desktop.
boto3 aws s3 sync
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import boto3 | |
def pull_s3_prefix(dst_dir, bucket, prefix): | |
client = boto3.client('s3') | |
resource = boto3.resource('s3') | |
download_dir(client, resource, prefix, prefix, dst_dir, bucket) | |
def download_dir(client, resource, prefix, start_prefix, local, bucket ): | |
paginator = client.get_paginator('list_objects') | |
for result in paginator.paginate(Bucket=bucket, Delimiter='/', Prefix=prefix): | |
if result.get('CommonPrefixes') is not None: | |
for subdir in result.get('CommonPrefixes'): | |
download_dir(client, resource, subdir.get('Prefix'), start_prefix, local, bucket) | |
if result.get('Contents') is not None: | |
for file in result.get('Contents'): | |
key_relative = file.get('Key').replace(start_prefix, '') | |
if not os.path.exists(os.path.dirname(local + os.sep + key_relative)): | |
os.makedirs(os.path.dirname(local + os.sep + key_relative)) | |
local_path = local + os.sep + key_relative | |
s3_path = file.get('Key') | |
print(f"Downloading {s3_path} -> {local_path}") | |
resource.meta.client.download_file(bucket, s3_path, local_path) | |
def upload_dir_s3(source_dir, dst_bucket, dst_prefix=''): | |
client = boto3.client('s3') | |
# enumerate local files recursively | |
for root, dirs, files in os.walk(source_dir): | |
for filename in files: | |
# construct the full local path | |
local_path = os.path.join(root, filename) | |
# construct the full Dropbox path | |
relative_path = os.path.relpath(local_path, source_dir) | |
s3_path = os.path.join(dst_prefix, relative_path) | |
# s3_path = "{dst_prefix}{s3_path}" | |
# relative_path = os.path.relpath(os.path.join(root, filename)) | |
try: | |
print("Uploading %s..." % s3_path) | |
client.upload_file(local_path, dst_bucket, s3_path) | |
# try: | |
# client.delete_object(Bucket=bucket, Key=s3_path) | |
# except: | |
# print "Unable to delete %s..." % s3_path | |
except Exception as e: | |
logger.exception(f"Failed to upload {local_path} to {s3_path}") |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment