download image form link farallel

download_image_parallel.py

import os
import sys 
import wget
import pandas as pd 
from multiprocessing import Pool

FILE_PATH = sys.argv[1]

def get_file_paths(FILE_PATH):
    df = pd.read_csv(FILE_PATH, header=None)
    res = df[0].tolist()
    print (len(res))
    return res 

def downloader(link):
    try:
        wget.download(link, 'data/' + link.split('/')[-1])
        return 1
    except Exception as e:
        print (e)
        return 0


def main():
    if not os.path.isdir('data'):
        os.mkdir('data')
    list_file = list(get_file_paths(FILE_PATH))
    p = Pool(8)
    data = p.map(downloader, list_file)
    print (sum(data)/float(len(data)))

if __name__ == '__main__':
    #get_file_paths(FILE_PATH)
    main()