Created
July 23, 2022 12:43
-
-
Save rdemorais/1854b63cf03f9fb9a7e83e03a9c22f58 to your computer and use it in GitHub Desktop.
Get download progress with boto3
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import sys | |
import threading | |
class ProgressPercentage(object): | |
''' Progress Class | |
Class for calculating and displaying download progress | |
''' | |
def __init__(self, client, bucket, filename): | |
''' Initialize | |
initialize with: file name, file size and lock. | |
Set seen_so_far to 0. Set progress bar length | |
''' | |
self._filename = filename | |
self._size = client.head_object(Bucket=bucket, Key=filename)['ContentLength'] | |
self._seen_so_far = 0 | |
self._lock = threading.Lock() | |
self.prog_bar_len = 80 | |
def __call__(self, bytes_amount): | |
''' Call | |
When called, increments seen_so_far by bytes_amount, | |
calculates percentage of seen_so_far/total file size | |
and prints progress bar. | |
''' | |
# To simplify we'll assume this is hooked up to a single filename. | |
with self._lock: | |
self._seen_so_far += bytes_amount | |
ratio = round((float(self._seen_so_far) / float(self._size)) * (self.prog_bar_len - 6), 1) | |
current_length = int(round(ratio)) | |
percentage = round(100 * ratio / (self.prog_bar_len - 6), 1) | |
bars = '+' * current_length | |
output = bars + ' ' * (self.prog_bar_len - current_length - len(str(percentage)) - 1) + str(percentage) + '%' | |
if self._seen_so_far != self._size: | |
sys.stdout.write(output + '\r') | |
else: | |
sys.stdout.write(output + '\n') | |
sys.stdout.flush() | |
s3 = boto3.client('s3') | |
progress = ProgressPercentage(s3, bucket, f'{model_name}.tar.gz') | |
s3.download_file(bucket, f'{model_name}.tar.gz', f'{model_name}.tar.gz', Callback=progress) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment