Created
June 5, 2014 08:33
-
-
Save mtorromeo/b95e1545396c1a63b421 to your computer and use it in GitHub Desktop.
Parallelize shell processes enforcing a concurrency limit using python3 and asyncio
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import asyncio | |
concurrency_sem = asyncio.Semaphore(3) | |
@asyncio.coroutine | |
def run(process): | |
yield from concurrency_sem.acquire() | |
print("Running {}...".format(process)) | |
proc = yield from asyncio.create_subprocess_shell(process) | |
yield from proc.communicate() | |
print("Process {} terminated.".format(process)) | |
concurrency_sem.release() | |
tasks = [ | |
asyncio.Task(run("sleep 1")), | |
asyncio.Task(run("sleep 2")), | |
asyncio.Task(run("sleep 1")), | |
asyncio.Task(run("sleep 2")), | |
asyncio.Task(run("sleep 1")), | |
asyncio.Task(run("sleep 2")), | |
asyncio.Task(run("sleep 1")), | |
asyncio.Task(run("sleep 2")), | |
] | |
loop = asyncio.get_event_loop() | |
loop.run_until_complete(asyncio.wait(tasks)) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment