Created
December 28, 2020 12:55
-
-
Save jaymecd/fcf15ea4481b6049c58d068807a3f190 to your computer and use it in GitHub Desktop.
parallel async python3 with timeout
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python3 | |
import asyncio | |
from typing import List, Awaitable | |
async def slow_job(delay: int): | |
print(f"running slow_job with {delay}s delay ...") | |
await asyncio.sleep(delay) | |
print(f"slow_job with {delay}s delay is done") | |
if delay == 3: | |
raise RuntimeError("ouch, something happened") | |
return delay * 2 | |
async def parallel(*awaitables: List[Awaitable], timeout: int = None, return_exc: bool = True): | |
gathered = asyncio.gather(*awaitables, return_exceptions=return_exc) | |
try: | |
results = await asyncio.wait_for(gathered, timeout=timeout) | |
except asyncio.TimeoutError as ex: | |
try: | |
raise gathered.exception() | |
except asyncio.CancelledError: | |
pass | |
raise ex | |
return results | |
if __name__ == "__main__": | |
try: | |
r1, r2, r3 = asyncio.run(parallel( | |
slow_job(3), | |
slow_job(1), | |
slow_job(5), | |
timeout=7, # try to reduce timeout or remove it at all | |
# return_exc=False, | |
)) | |
except asyncio.TimeoutError: | |
print(f"Error: timed out") | |
except Exception as ex: | |
print(f"{ex.__class__.__qualname__}: {ex}") | |
exit(7) | |
print(f"r1: {r1!r}") | |
print(f"r2: {r2!r}") | |
print(f"r3: {r3!r}") |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment