Skip to content

Instantly share code, notes, and snippets.

@rdemorais
Created July 23, 2022 12:43
Show Gist options
  • Save rdemorais/1854b63cf03f9fb9a7e83e03a9c22f58 to your computer and use it in GitHub Desktop.
Save rdemorais/1854b63cf03f9fb9a7e83e03a9c22f58 to your computer and use it in GitHub Desktop.
Get download progress with boto3
import sys
import threading
class ProgressPercentage(object):
''' Progress Class
Class for calculating and displaying download progress
'''
def __init__(self, client, bucket, filename):
''' Initialize
initialize with: file name, file size and lock.
Set seen_so_far to 0. Set progress bar length
'''
self._filename = filename
self._size = client.head_object(Bucket=bucket, Key=filename)['ContentLength']
self._seen_so_far = 0
self._lock = threading.Lock()
self.prog_bar_len = 80
def __call__(self, bytes_amount):
''' Call
When called, increments seen_so_far by bytes_amount,
calculates percentage of seen_so_far/total file size
and prints progress bar.
'''
# To simplify we'll assume this is hooked up to a single filename.
with self._lock:
self._seen_so_far += bytes_amount
ratio = round((float(self._seen_so_far) / float(self._size)) * (self.prog_bar_len - 6), 1)
current_length = int(round(ratio))
percentage = round(100 * ratio / (self.prog_bar_len - 6), 1)
bars = '+' * current_length
output = bars + ' ' * (self.prog_bar_len - current_length - len(str(percentage)) - 1) + str(percentage) + '%'
if self._seen_so_far != self._size:
sys.stdout.write(output + '\r')
else:
sys.stdout.write(output + '\n')
sys.stdout.flush()
s3 = boto3.client('s3')
progress = ProgressPercentage(s3, bucket, f'{model_name}.tar.gz')
s3.download_file(bucket, f'{model_name}.tar.gz', f'{model_name}.tar.gz', Callback=progress)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment