aiohttp_example

aiohttp_example.py

import asyncio
import aiohttp
import logging


async def fetchurl(session, url: str):
    async with session.get(url) as response:
        return len(await response.text())


async def urlfetcher(session, queue):
    items = []
    while True:
        try:
            x = await queue.get()
            print(x)
            items.append(await fetchurl(session, x))
            queue.task_done()
        except asyncio.exceptions.CancelledError as e:
            return items



async def run():
    # simple
    l = ["https://google.com" for x in [1,2,3,4,5,6]]
    async with aiohttp.ClientSession() as session:
        tasks = [fetchurl(session, url) for url in l]
        lx = await asyncio.gather(*tasks)
        print(lx)

    # worker 
    queue = asyncio.Queue()
    for x in l:
        queue.put_nowait(x)

    tasks = []
    async with aiohttp.ClientSession() as session:
        for i in range(3):
            task = asyncio.create_task(urlfetcher(session, queue))
            tasks.append(task)

        await queue.join()

        for task in tasks:
            task.cancel()

        l = await asyncio.gather(*tasks)
        print(l)

print(asyncio.run(run()))