Curl Pool DL

pooldl.py

from multiprocessing import Pool
import fileinput


def download(url):
    from subprocess import check_call

    check_call(["curl", "--silent", "-O", url])
    return url


if __name__ == '__main__':
    with Pool(20) as p:
        urls = [l.strip() for l in fileinput.input()]
        print("Read", len(urls), "urls", urls[0])
        for url in p.imap(download, urls):
            print("Downloaded", url)