download_from_s3.py

# console:
# export AWS_ACCESS_KEY_ID=AWS_KEY
# export AWS_SECRET_ACCESS_KEY=AWS_SECRET


from tqdm import tqdm
import boto3
import os

s3 = boto3.client('s3')
s3_bucket = 'bucket_name'
cache_dir = 'local_dir'
file_names = []

for f in s3.list_objects(Bucket=s3_bucket)['Contents']:
  file_names.append(f['Key'])
  
for file_name in file_names:
  local_path = os.path.join(cache_dir, file_name)
  kwargs = {"Bucket": s3_bucket, "Key": file_name}
  object_size = s3.head_object(**kwargs)["ContentLength"]
  with tqdm(total=object_size, unit="B", unit_scale=True, desc=file_name) as pbar:
    s3.download_file(
      s3_bucket,
      file_name,
      local_path,
      Callback=lambda bytes_transferred: pbar.update(bytes_transferred)
    )