Skip to content

Instantly share code, notes, and snippets.

@dukn
Last active July 23, 2018 10:06
Show Gist options
  • Save dukn/06ecc12d45b327d16aa6dfc272e27f7d to your computer and use it in GitHub Desktop.
Save dukn/06ecc12d45b327d16aa6dfc272e27f7d to your computer and use it in GitHub Desktop.
Parallel download image from list of links
from __future__ import print_function
import os
import sys
import Queue
import threading
import urllib
import time
PARALLEL = True
if not os.path.exists("parallels"):
os.makedirs("parallels")
if len(sys.argv) < 2:
print ('\tAdd path to file containing the link.')
print ('\tEx: `$ python2 parallels.py hanoi.links`')
exit()
if os.path.isfile(sys.argv[1]):
f = open(sys.argv[1])
print ("\tProcessing file "+sys.argv[1])
else:
print ("\tFile not found error!")
exit()
theurls = []
for line in f:
line = line.replace("\n", "")
theurls.append(line)
f.close()
print ('\t',len(theurls), "links")
print ()
#####################################
def get_url(q, url, index):
filename = url.split('/')[-1]
try:
q.put(urllib.urlretrieve(url,"parallels/"+str(index)+". "+filename))
except Exception as e:
pass
#####################################
if PARALLEL:
q = Queue.Queue()
index = 0
for u in theurls:
index += 1
t = threading.Thread(target=get_url, args = (q,u, index))
t.daemon = False
t.start()
try:
s = q.get()
except Exception as e:
pass
else:
for index, url in enumerate(theurls):
filename = url.split('/')[-1]
try:
urllib.urlretrieve(url,"parallels/"+str(index)+". "+filename)
except Exception as e:
pass
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment