Fast asyncio HTTP requests

file.py

import sys
import os
import json
import asyncio
import aiohttp


# Initialize connection pool
conn = aiohttp.TCPConnector(limit_per_host=100, limit=0, ttl_dns_cache=300)
PARALLEL_REQUESTS = 100
results = []
urls = ['https://jsonplaceholder.typicode.com/todos/1' for i in range(10)] #array of urls

async def gather_with_concurrency(n):
    semaphore = asyncio.Semaphore(n)
    session = aiohttp.ClientSession(connector=conn)

    # heres the logic for the generator
    async def get(url):
        async with semaphore:
            async with session.get(url, ssl=False) as response:
                obj = json.loads(await response.read())
                results.append(obj)
    await asyncio.gather(*(get(url) for url in urls))
    await session.close()

loop = asyncio.get_event_loop()
loop.run_until_complete(gather_with_concurrency(PARALLEL_REQUESTS))
conn.close()

print(f"Completed {len(urls)} requests with {len(results)} results")

Is PARALLEL_REQUESTS set to None in your code?

File "C:\Users\jbard\AppData\Local\Programs\Python\Python39\lib\concurrent\futures\thread.py", line 140, in init
if max_workers <= 0:
TypeError: '<=' not supported between instances of 'NoneType' and 'int'

max_workers is None - you need to pass a valid number to loop.run_until_complete(gather_with_concurrency(PARALLEL_REQUESTS))

Ta- I think I did something funny with one of the threading;/future or some other module - to allow me to increase max threads - because I'm running exact same code on other machine at it works beautifully - I'll get there - thanks so much for swift response. your code is awesome!!! thumbs up - take care, J

Why I run this slower than solution 2 & 3? This costs over 1 sec, but solution 2&3 cost 0.8sec, it looks like async is always slower than multi-thread

strange, now when I run this, I consistently get:

Traceback (most recent call last):
File "C:\Users\jbard\AppData\Local\Programs\Python\Python312\Lib\site-packages\IPython\core\interactiveshell.py", line 3577, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "", line 9, in
conn = aiohttp.TCPConnector(limit_per_host=100, limit=0, ttl_dns_cache=300)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\jbard\AppData\Local\Programs\Python\Python312\Lib\site-packages\aiohttp\connector.py", line 805, in init
super().init(
File "C:\Users\jbard\AppData\Local\Programs\Python\Python312\Lib\site-packages\aiohttp\connector.py", line 252, in init
loop = loop or asyncio.get_running_loop()
^^^^^^^^^^^^^^^^^^^^^^^^^^
RuntimeError: no running event loop