alternative code for http://stackoverflow.com/questions/24952918

stackoverflow24952918tmp.py

import sys, os
import json
import urllib2
import tempfile
import multiprocessing as mp

def worker(url):
    """ reading lots of data """
    print "# Process %s running on file %s"%(mp.current_process(), url)
    raw = json.load(urllib2.urlopen(url))
    _, tmpath = tempfile.mkstemp()
    file = open(tmpath, "w")
    for r in raw:
        file.write("%s\n"%r)
    return tmpath

# example urls to read
url = "http://eu.battle.net/auction-data/258993a3c6b974ef3e6f22ea6f822720/auctions.json"
urls_to_parse = (url, url, url), url, url, url, url, url, url, url, url, \
        url, url, url, url, url, url, url, url, url, url, url, url, url, \
        url, url, url, url, url, url, url, url, url, url, url, url, url, \
        url, url, url, url, url, url, url, url, url, url, url, url, url, \
        url, url, url, url, url, url, url, url, url, url, url, url, url, \
        url, url, url, url, url, url, url, url, url, url, url, url, url)

# multiprocessing options
nproc = mp.cpu_count()
pool = mp.Pool(processes=nproc)
temps = pool.map(worker, urls_to_parse)