Skip to content

Instantly share code, notes, and snippets.

@alexmic
Created March 2, 2012 00:26
Show Gist options
  • Save alexmic/1954246 to your computer and use it in GitHub Desktop.
Save alexmic/1954246 to your computer and use it in GitHub Desktop.
Multi-threaded map()
def mtmap(fn, iterable):
""" A multi-threaded map(). Must be used with care - large
iterables might cause a machine to crash from too many
threads. If an exception is thrown in a thread,
the iteration stops and that exception is thrown.
"""
result = {}
threads = []
exception_q = Queue.Queue()
def sort_dict(d):
keys = d.keys()
keys.sort()
return [d[key] for key in keys]
def exc(fn, i, item):
try:
result[i] = fn(item)
except Exception as e:
result[i] = None
exception_q.put(e)
for i, item in enumerate(iterable):
t = threading.Thread(target=exc, args=(fn, i, item))
threads.append(t)
t.start()
for t in threads:
t.join()
try:
ex = exception_q.get(block=False)
exception_q.task_done()
raise ex
except Queue.Empty:
continue
return sort_dict(result)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment