Fetch JSON data from URLs (async) and store in container - python

multiFetchJSON.py

# Install required libraries tqdm, aiohttp

# from tqdm import tqdm
# import aiohttp

import typing

async def multiFetchJSON(indices,
                         container: dict = {},
                         urlFormat: str = '',
                         urlGenerator: typing.Callable = None,
                         returnData: bool = False,
                         **urlParams
                        ):

    async with aiohttp.ClientSession() as session:

        for id in tqdm(indices):
            # Don't fetch for the ones already done
            if id in container.keys():
                continue

            if urlGenerator is not None:
                url = urlGenerator(id = id)
            else:
                url = urlFormat.format(id=id, **urlParams)
            async with session.get(url) as resp:
                try:
                    resp.raise_for_status()
                    response_obj = await resp.json()
                    container[id] = response_obj
                except aiohttp.ClientResponseError as e:
                    print(f'Error: {e.status} for URL: {e.request_info.real_url}')
    return container if returnData else None

Sample implementation

data = {}

await multiFetchJSON(
    range(1, 151),
    urlFormat='https://pokeapi.co/api/v2/pokemon/{id}',
    container=data
)

With callback:

d2 = await multiFetchJSON(
    map(str,range(1, 4)),  # compliance with JSON standard key type
    urlGenerator = lambda id: f'https://pokeapi.co/api/v2/pokemon/{id}',
    returnData = True
)