Last active
February 18, 2017 17:19
-
-
Save svalleru/6c0bcba836dfe8e783e9b2e94b7d115a to your computer and use it in GitHub Desktop.
Python Pool Multiprocessing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from multiprocessing import Pool | |
| import functools | |
| def _myfunc_with_one_arg(n): | |
| return n ** 2 | |
| def _myfunc_with_two_args(arg1, arg2, s): | |
| return arg1 + arg2 + s | |
| def ramifier(): | |
| """Simple parallel processing in Python""" | |
| pool = Pool(processes=3) | |
| _input = [-3, 1, 4] | |
| func_results = pool.map(_myfunc_with_one_arg, _input) | |
| pool.terminate() | |
| print "myfunc_results: ", func_results | |
| _my_new_input = ["Don't blink", | |
| "The Doctor lies", | |
| "Never knowingly be serious", | |
| "Never run when you're scared", | |
| "Time is not the boss of you", | |
| "Never ignore a coincidence..unless you're busy", | |
| "Bow ties are cool!"] | |
| pool = Pool(processes=min(len(_my_new_input) if len(_my_new_input) > 0 else 1, 30)) | |
| myfunc_results = pool.map(functools.partial(_myfunc_with_two_args, "Doctor ", "says: "), _my_new_input) | |
| pool.terminate() | |
| print "myfunc_with_two_args_results:\n", '\n'.join(myfunc_results) | |
| ramifier() | |
| # Output | |
| # myfunc_results: [9, 1, 16] | |
| # myfunc_with_two_args_results: | |
| # Doctor says: Don't blink | |
| # Doctor says: The Doctor lies | |
| # Doctor says: Never knowingly be serious | |
| # Doctor says: Never run when you're scared | |
| # Doctor says: Time is not the boss of you | |
| # Doctor says: Never ignore a coincidence..unless you're busy | |
| # Doctor says: Bow ties are cool! |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment