Created
June 15, 2021 04:06
-
-
Save jonluca/3ac654bea8b0a8d2ca0c2b149cfdf2f9 to your computer and use it in GitHub Desktop.
Fast AsyncIO HTTP requests
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import http.cookies | |
http.cookies._is_legal_key = lambda _: True | |
import asyncio | |
import aiohttp | |
import nest_asyncio | |
import time | |
nest_asyncio.apply() | |
async def gather_with_concurrency(n, *tasks): | |
semaphore = asyncio.Semaphore(n) | |
async def sem_task(task): | |
async with semaphore: | |
return await task | |
return await asyncio.gather(*(sem_task(task) for task in tasks)) | |
conn = aiohttp.TCPConnector(limit=None, ttl_dns_cache=300) | |
session = aiohttp.ClientSession(connector=conn) | |
async def get_async(url): | |
async with session.get(url, ssl=False) as response: | |
obj = await response.read() | |
results[url] = obj | |
urls = [f"https://jsonplaceholder.typicode.com/todos/{i}" for i in range(4000)] | |
conc_req = 60 | |
s = time.perf_counter() | |
await gather_with_concurrency(conc_req, *[get_async(i) for i in urls]) | |
elapsed = time.perf_counter() - s |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Outside Jupyter context, this worked for me (on v3.9.5) when
results = {}
gather_with_concurrency()
into anasyncio
event loop:get_async()
like so:.. which makes this a great implementation, crazy fast 🦊