Last active
January 21, 2017 19:55
-
-
Save messa/38c45fcf20b6bca3d5e1 to your computer and use it in GitHub Desktop.
Python multiprocessing demo
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python3 | |
| import logging | |
| import multiprocessing | |
| from time import sleep | |
| logger = logging.getLogger(__name__) | |
| logging.basicConfig(format='%(asctime)s [%(process)d]: %(message)s', level=logging.DEBUG) | |
| def fce(x): | |
| logger.info('fce(%s) started', x) | |
| sleep(.2) | |
| logger.info('fce(%s) done', x) | |
| return x**2 | |
| with multiprocessing.Pool(3) as pool: | |
| logger.info('Running map') | |
| result = pool.map(fce, [0, 1, 2, 3, 4]) | |
| assert result == [0, 1, 4, 9, 16] | |
| logger.info('Map done') | |
| logger.info('-----------------------') | |
| results = [] | |
| for x in [0, 1, 2, 3, 4]: | |
| logger.info('Calling apply with %s', x) | |
| result = pool.apply(fce, (x, )) | |
| logger.info('apply with %s done: %s', x, result) | |
| results.append(result) | |
| assert results == [0, 1, 4, 9, 16] | |
| logger.info('-----------------------') | |
| results = [] | |
| for x in [0, 1, 2, 3, 4]: | |
| logger.info('Calling apply_async with %s', x) | |
| result = pool.apply_async(fce, (x, )) | |
| logger.info('apply_async with %s done: %s', x, result) | |
| results.append(result) | |
| logger.info('Waiting for results...') | |
| results = [r.get() for r in results] | |
| logger.info('Got results') | |
| assert sorted(results) == [0, 1, 4, 9, 16] | |
| logger.info('Done') |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 2015-12-20 19:32:51,867 [48261]: Running map | |
| 2015-12-20 19:32:51,870 [48262]: fce(0) started | |
| 2015-12-20 19:32:51,873 [48263]: fce(1) started | |
| 2015-12-20 19:32:51,873 [48264]: fce(2) started | |
| 2015-12-20 19:32:52,073 [48262]: fce(0) done | |
| 2015-12-20 19:32:52,074 [48262]: fce(3) started | |
| 2015-12-20 19:32:52,076 [48263]: fce(1) done | |
| 2015-12-20 19:32:52,076 [48264]: fce(2) done | |
| 2015-12-20 19:32:52,077 [48263]: fce(4) started | |
| 2015-12-20 19:32:52,274 [48262]: fce(3) done | |
| 2015-12-20 19:32:52,278 [48263]: fce(4) done | |
| 2015-12-20 19:32:52,278 [48261]: Map done | |
| 2015-12-20 19:32:52,279 [48261]: ----------------------- | |
| 2015-12-20 19:32:52,279 [48261]: Calling apply with 0 | |
| 2015-12-20 19:32:52,280 [48264]: fce(0) started | |
| 2015-12-20 19:32:52,482 [48264]: fce(0) done | |
| 2015-12-20 19:32:52,483 [48261]: apply with 0 done: 0 | |
| 2015-12-20 19:32:52,483 [48261]: Calling apply with 1 | |
| 2015-12-20 19:32:52,483 [48262]: fce(1) started | |
| 2015-12-20 19:32:52,684 [48262]: fce(1) done | |
| 2015-12-20 19:32:52,685 [48261]: apply with 1 done: 1 | |
| 2015-12-20 19:32:52,685 [48261]: Calling apply with 2 | |
| 2015-12-20 19:32:52,686 [48263]: fce(2) started | |
| 2015-12-20 19:32:52,887 [48263]: fce(2) done | |
| 2015-12-20 19:32:52,888 [48261]: apply with 2 done: 4 | |
| 2015-12-20 19:32:52,888 [48261]: Calling apply with 3 | |
| 2015-12-20 19:32:52,889 [48264]: fce(3) started | |
| 2015-12-20 19:32:53,090 [48264]: fce(3) done | |
| 2015-12-20 19:32:53,091 [48261]: apply with 3 done: 9 | |
| 2015-12-20 19:32:53,092 [48261]: Calling apply with 4 | |
| 2015-12-20 19:32:53,092 [48262]: fce(4) started | |
| 2015-12-20 19:32:53,296 [48262]: fce(4) done | |
| 2015-12-20 19:32:53,297 [48261]: apply with 4 done: 16 | |
| 2015-12-20 19:32:53,297 [48261]: ----------------------- | |
| 2015-12-20 19:32:53,297 [48261]: Calling apply_async with 0 | |
| 2015-12-20 19:32:53,298 [48261]: apply_async with 0 done: <multiprocessing.pool.ApplyResult object at 0x106a45588> | |
| 2015-12-20 19:32:53,298 [48261]: Calling apply_async with 1 | |
| 2015-12-20 19:32:53,299 [48261]: apply_async with 1 done: <multiprocessing.pool.ApplyResult object at 0x106ffdb70> | |
| 2015-12-20 19:32:53,299 [48261]: Calling apply_async with 2 | |
| 2015-12-20 19:32:53,299 [48263]: fce(0) started | |
| 2015-12-20 19:32:53,299 [48261]: apply_async with 2 done: <multiprocessing.pool.ApplyResult object at 0x106ffdc18> | |
| 2015-12-20 19:32:53,299 [48261]: Calling apply_async with 3 | |
| 2015-12-20 19:32:53,300 [48261]: apply_async with 3 done: <multiprocessing.pool.ApplyResult object at 0x106ffdd30> | |
| 2015-12-20 19:32:53,300 [48261]: Calling apply_async with 4 | |
| 2015-12-20 19:32:53,300 [48261]: apply_async with 4 done: <multiprocessing.pool.ApplyResult object at 0x106ffddd8> | |
| 2015-12-20 19:32:53,300 [48261]: Waiting for results... | |
| 2015-12-20 19:32:53,301 [48264]: fce(1) started | |
| 2015-12-20 19:32:53,301 [48262]: fce(2) started | |
| 2015-12-20 19:32:53,500 [48263]: fce(0) done | |
| 2015-12-20 19:32:53,501 [48263]: fce(3) started | |
| 2015-12-20 19:32:53,501 [48262]: fce(2) done | |
| 2015-12-20 19:32:53,501 [48264]: fce(1) done | |
| 2015-12-20 19:32:53,502 [48262]: fce(4) started | |
| 2015-12-20 19:32:53,702 [48263]: fce(3) done | |
| 2015-12-20 19:32:53,704 [48262]: fce(4) done | |
| 2015-12-20 19:32:53,704 [48261]: Got results | |
| 2015-12-20 19:32:53,745 [48261]: Done |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment