Skip to content

Instantly share code, notes, and snippets.

@svalleru
Last active February 18, 2017 17:19
Show Gist options
  • Save svalleru/6c0bcba836dfe8e783e9b2e94b7d115a to your computer and use it in GitHub Desktop.
Save svalleru/6c0bcba836dfe8e783e9b2e94b7d115a to your computer and use it in GitHub Desktop.
Python Pool Multiprocessing
from multiprocessing import Pool
import functools
def _myfunc_with_one_arg(n):
return n ** 2
def _myfunc_with_two_args(arg1, arg2, s):
return arg1 + arg2 + s
def ramifier():
"""Simple parallel processing in Python"""
pool = Pool(processes=3)
_input = [-3, 1, 4]
func_results = pool.map(_myfunc_with_one_arg, _input)
pool.terminate()
print "myfunc_results: ", func_results
_my_new_input = ["Don't blink",
"The Doctor lies",
"Never knowingly be serious",
"Never run when you're scared",
"Time is not the boss of you",
"Never ignore a coincidence..unless you're busy",
"Bow ties are cool!"]
pool = Pool(processes=min(len(_my_new_input) if len(_my_new_input) > 0 else 1, 30))
myfunc_results = pool.map(functools.partial(_myfunc_with_two_args, "Doctor ", "says: "), _my_new_input)
pool.terminate()
print "myfunc_with_two_args_results:\n", '\n'.join(myfunc_results)
ramifier()
# Output
# myfunc_results: [9, 1, 16]
# myfunc_with_two_args_results:
# Doctor says: Don't blink
# Doctor says: The Doctor lies
# Doctor says: Never knowingly be serious
# Doctor says: Never run when you're scared
# Doctor says: Time is not the boss of you
# Doctor says: Never ignore a coincidence..unless you're busy
# Doctor says: Bow ties are cool!
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment