Last active
July 23, 2018 10:06
-
-
Save dukn/06ecc12d45b327d16aa6dfc272e27f7d to your computer and use it in GitHub Desktop.
Parallel download image from list of links
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from __future__ import print_function | |
import os | |
import sys | |
import Queue | |
import threading | |
import urllib | |
import time | |
PARALLEL = True | |
if not os.path.exists("parallels"): | |
os.makedirs("parallels") | |
if len(sys.argv) < 2: | |
print ('\tAdd path to file containing the link.') | |
print ('\tEx: `$ python2 parallels.py hanoi.links`') | |
exit() | |
if os.path.isfile(sys.argv[1]): | |
f = open(sys.argv[1]) | |
print ("\tProcessing file "+sys.argv[1]) | |
else: | |
print ("\tFile not found error!") | |
exit() | |
theurls = [] | |
for line in f: | |
line = line.replace("\n", "") | |
theurls.append(line) | |
f.close() | |
print ('\t',len(theurls), "links") | |
print () | |
##################################### | |
def get_url(q, url, index): | |
filename = url.split('/')[-1] | |
try: | |
q.put(urllib.urlretrieve(url,"parallels/"+str(index)+". "+filename)) | |
except Exception as e: | |
pass | |
##################################### | |
if PARALLEL: | |
q = Queue.Queue() | |
index = 0 | |
for u in theurls: | |
index += 1 | |
t = threading.Thread(target=get_url, args = (q,u, index)) | |
t.daemon = False | |
t.start() | |
try: | |
s = q.get() | |
except Exception as e: | |
pass | |
else: | |
for index, url in enumerate(theurls): | |
filename = url.split('/')[-1] | |
try: | |
urllib.urlretrieve(url,"parallels/"+str(index)+". "+filename) | |
except Exception as e: | |
pass | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment