Last active
January 3, 2016 09:29
-
-
Save sgillies/8442728 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| MapBox-FC:multi sean$ time parallel wc ::: /Users/sean/Library/Logs/HipChat/*.txt | |
| 5103 38662 420689 /Users/sean/Library/Logs/HipChat/log-2014-01-02-1.txt | |
| 2109 15175 170369 /Users/sean/Library/Logs/HipChat/log-2014-01-08-1.txt | |
| 4917 36306 400980 /Users/sean/Library/Logs/HipChat/log-2014-01-07-1.txt | |
| 294 2364 26589 /Users/sean/Library/Logs/HipChat/log-2013-12-20-1.txt | |
| 5482 41656 448271 /Users/sean/Library/Logs/HipChat/log-2014-01-10-1.txt | |
| 4160 31137 336279 /Users/sean/Library/Logs/HipChat/log-2014-01-13-1.txt | |
| real 0m0.185s | |
| user 0m0.135s | |
| sys 0m0.072s | |
| MapBox-FC:multi sean$ time ./pp.py wc ::: /Users/sean/Library/Logs/HipChat/*.txt 294 2364 26589 /Users/sean/Library/Logs/HipChat/log-2013-12-20-1.txt | |
| 5103 38662 420689 /Users/sean/Library/Logs/HipChat/log-2014-01-02-1.txt | |
| 4917 36306 400980 /Users/sean/Library/Logs/HipChat/log-2014-01-07-1.txt | |
| 2109 15175 170369 /Users/sean/Library/Logs/HipChat/log-2014-01-08-1.txt | |
| 5482 41656 448271 /Users/sean/Library/Logs/HipChat/log-2014-01-10-1.txt | |
| 4160 31137 336279 /Users/sean/Library/Logs/HipChat/log-2014-01-13-1.txt | |
| real 0m0.159s | |
| user 0m0.059s | |
| sys 0m0.043s |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python | |
| import argparse | |
| from multiprocessing import Pool | |
| import logging | |
| import subprocess | |
| import sys | |
| logging.basicConfig(stream=sys.stderr, level=logging.INFO) | |
| logger = logging.getLogger('pp') | |
| parser = argparse.ArgumentParser( | |
| description="Emulates GNU parallel") | |
| parser.add_argument( | |
| 'program', | |
| metavar='PROGRAM', | |
| help="Program to run jobs") | |
| known, unknown = parser.parse_known_args() | |
| # find the parallel args | |
| i = unknown.index(':::') | |
| assert i >= 0 | |
| pargs = unknown[i+1:] | |
| def runner(*args): | |
| return subprocess.check_output([known.program] + list(args)) | |
| if __name__ == '__main__': | |
| pool = Pool() | |
| results = pool.map(runner, pargs) | |
| for r in results: | |
| print(r.rstrip()) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment