README.md

Python HTTP clients

Results:

➤ uv run with_httpx.py  
time taken to make 10000 requests: 20.2337 seconds
Counter({200: 10000})

➤ uv run with_thread_pool.py 
time taken to make 10000 requests: 6.9722 seconds
Counter({200: 10000})

➤ uv run with_aiohttp.py    
time taken to make 10000 requests: 4.0490 seconds
Counter({200: 10000})

with_aiohttp.py

# /// script
# requires-python = ">=3.12"
# dependencies = [
#     "aiohttp",
# ]
# ///
import asyncio
import time
from collections import Counter

from aiohttp import ClientSession

status_counter: Counter[int] = Counter()


async def worker(client: ClientSession, queue: asyncio.Queue[int]):
    while True:
        i = await queue.get()
        async with client.get(f'https://cloudflare.com/cdn-cgi/trace?v={i}') as r:
            status_counter[r.status] += 1
        queue.task_done()


async def main():
    queue: asyncio.Queue[int] = asyncio.Queue()
    requests = 10_000

    for i in range(requests):
        queue.put_nowait(i)

    tasks: list[asyncio.Task[None]] = []
    async with ClientSession() as client:
        for i in range(100):
            tasks.append(asyncio.create_task(worker(client, queue)))

        started_at = time.perf_counter()
        await queue.join()
        time_taken = time.perf_counter() - started_at

        for task in tasks:
            task.cancel()
        try:
            await asyncio.gather(*tasks)
        except asyncio.CancelledError:
            pass

    print(f'time taken to make {requests} requests: {time_taken:.4f} seconds')
    print(status_counter)


if __name__ == '__main__':
    asyncio.run(main())

with_httpx.py

# /// script
# requires-python = ">=3.12"
# dependencies = [
#     "httpx",
# ]
# ///
import asyncio
import time
from collections import Counter

from httpx import AsyncClient

status_counter: Counter[int] = Counter()


async def worker(client: AsyncClient, queue: asyncio.Queue[int]):
    while True:
        i = await queue.get()
        r = await client.get(
            f'https://cloudflare.com/cdn-cgi/trace?v={i}',
        )
        status_counter[r.status_code] += 1
        queue.task_done()


async def main():
    queue: asyncio.Queue[int] = asyncio.Queue()
    requests = 10_000

    for i in range(requests):
        queue.put_nowait(i)

    tasks: list[asyncio.Task[None]] = []
    async with AsyncClient() as client:
        for i in range(100):
            tasks.append(asyncio.create_task(worker(client, queue)))

        started_at = time.perf_counter()
        await queue.join()
        time_taken = time.perf_counter() - started_at

        for task in tasks:
            task.cancel()
        try:
            await asyncio.gather(*tasks)
        except asyncio.CancelledError:
            pass

    print(f'time taken to make {requests} requests: {time_taken:.4f} seconds')
    print(status_counter)


if __name__ == '__main__':
    asyncio.run(main())

with_thread_pool_requests.py

# /// script
# requires-python = ">=3.12"
# dependencies = [
#     "requests",
# ]
# ///
import asyncio
import time
from collections import Counter
from concurrent.futures import ThreadPoolExecutor

from requests import Session

status_counter: Counter[int] = Counter()


async def worker(thread_pool: ThreadPoolExecutor, client: Session, queue: asyncio.Queue[int]):
    loop = asyncio.get_event_loop()
    while True:
        i = await queue.get()
        r = await loop.run_in_executor(thread_pool, client.get, f'https://cloudflare.com/cdn-cgi/trace?v={i}')
        status_counter[r.status_code] += 1
        queue.task_done()


async def main():
    queue: asyncio.Queue[int] = asyncio.Queue()
    requests = 10_000

    for i in range(requests):
        queue.put_nowait(i)

    tasks: list[asyncio.Task[None]] = []
    with ThreadPoolExecutor() as thread_pool:
        with Session() as client:
            for i in range(100):
                tasks.append(asyncio.create_task(worker(thread_pool, client, queue)))

            started_at = time.perf_counter()
            await queue.join()
            time_taken = time.perf_counter() - started_at

            for task in tasks:
                task.cancel()
            try:
                await asyncio.gather(*tasks)
            except asyncio.CancelledError:
                pass

    print(f'time taken to make {requests} requests: {time_taken:.4f} seconds')
    print(status_counter)


if __name__ == '__main__':
    asyncio.run(main())