Download files from list parallel

multithread_downloads.py

import os
import sys
import logging
import argparse
import requests
from multiprocessing.pool import ThreadPool


def main(args):
    links = open(args.filename, "r")
    results = ThreadPool(6).imap_unordered(fetch_url, links)

    for path in results:
        print(path)


def fetch_url(link):
    link = link.strip()
    name = link.rsplit("/", 1)[-1]
    filename = os.path.join(args.target, name)

    if not os.path.isdir(args.target):
        os.mkdir(args.target)

    print("Downloading: " + filename)
    try:
        r = requests.get(link)
        open(filename, "wb").write(r.content)
    except Exception as inst:
        print(inst)
        print("Encountered unknown error. Continuing.")

    return filename


if __name__ == "__main__":

    logging.basicConfig(stream=sys.stdout, level=logging.INFO)
    parser = argparse.ArgumentParser(description="Process module factors.")
    parser.add_argument("filename", type=str, help="Add a valid filename")
    parser.add_argument("target", type=str, help="Add a valid target")
    args = parser.parse_args()
    main(args)