Skip to content

Instantly share code, notes, and snippets.

@sanfx
Created June 14, 2014 15:41
Show Gist options
  • Select an option

  • Save sanfx/aed0b65cfa1e5318abb0 to your computer and use it in GitHub Desktop.

Select an option

Save sanfx/aed0b65cfa1e5318abb0 to your computer and use it in GitHub Desktop.
Download multiple files parallel with multiple threads for each download.
import urllib2
import os
import sys
import time
import threading
urls = ["http://broadcast.lds.org/churchmusic/MP3/1/2/nowords/271.mp3",
"http://s1.fans.ge/mp3/201109/08/John_Legend_So_High_Remix(fans_ge).mp3",
"http://megaboon.com/common/preview/track/786203.mp3"]
url = urls[1]
def downloadFile(url, saveTo=None):
file_name = url.split('/')[-1]
if not saveTo:
saveTo = '/Users/userName/Desktop'
try:
u = urllib2.urlopen(url)
except urllib2.URLError , er:
print("%s for %s failed to download." % (er.reason, os.path.basename(url)))
else:
f = open(os.path.join(saveTo, file_name), 'wb')
meta = u.info()
file_size = int(meta.getheaders("Content-Length")[0])
print "Downloading: %s Bytes: %s" % (file_name, file_size)
file_size_dl = 0
block_sz = 8192
while True:
buffer = u.read(block_sz)
if not buffer:
break
file_size_dl += len(buffer)
f.write(buffer)
status = r"%10d [%3.2f%%]" % (file_size_dl, file_size_dl * 100. / file_size)
status = status + chr(8)*(len(status)+1)
sys.stdout.write('%s\r' % status)
time.sleep(.2)
sys.stdout.flush()
if file_size_dl == file_size:
print r"Download Completed %s%% for file %s, saved to %s" % (file_size_dl * 100. / file_size, file_name, saveTo,)
f.close()
return
def synchronusDownload():
urls_saveTo = {urls[0]: None, urls[1]: None, urls[2]: None}
for url, saveTo in urls_saveTo.iteritems():
th = threading.Thread(target=downloadFile, args=(url, saveTo), name="%s_Download_Thread" % os.path.basename(url))
th.start()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment