Skip to content

Instantly share code, notes, and snippets.

@AtmaMani
Created September 11, 2017 23:55
Show Gist options
  • Save AtmaMani/865c4f4c10ba944935fe3970c9c9a3c8 to your computer and use it in GitHub Desktop.
Save AtmaMani/865c4f4c10ba944935fe3970c9c9a3c8 to your computer and use it in GitHub Desktop.
Script to download images in parallel
import requests
from pathlib import Path
import multiprocessing
#read the download file
with open('./download_links.txt','r') as fhandle:
files = fhandle.readlines()
def downloader(url):
img_name = Path(url).name.rstrip('\n')
print('downloading '+ img_name, end=" # ")
response = requests.get(url.rstrip('\n'))
print(" writing stream ", end=" # ")
with open('./' + img_name,"wb") as img_handle:
img_handle.write(response.content)
print("done")
return (url, "done")
#print banner
print("============== DOWNLOADING HURRICANE IRMA DATA =================")
print("Number of images: " + str(len(files)))
print()
#download in parallel
if __name__ == '__main__':
pool = multiprocessing.Pool(processes=7)
output = pool.map(downloader, files)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment