download url parallel

download_url_parallel.py

import eventlet,sys
# note: this urllib import doesn't work in Python2
from eventlet.green.urllib.request import urlopen

file = sys.argv[1] # list of urls
with open(file,'r') as f:
    urls = [x.rstrip() for x in f.readlines()]
urls = urls + urls
urls = urls + urls
urls = urls + urls 

def fetch(url):
    try:
        return urlopen(url).read()
    except:
        return [1]

pool = eventlet.GreenPool(8)

for body in pool.imap(fetch, urls):
    print(f"got body {len(body)}")
print(len(urls))