Concurrency and Parallel Requests

concurrent_requests.py

import requests
import concurrent.futures


def get_todo(i, optional_arg=None):
  if i < 1: raise ValueError
  r = requests.get(f'https://jsonplaceholder.typicode.com/todos/{i}')
  return r.json()


def get_todos() -> tuple:
    """
    returns: list[dict]
    uses a threadPoolExecutor instead of asyncio
    """
    todos = []
    errors = []
    # play arround with the max_workers. I used 35 for one of my scripts but 
    #  35 was overkill for this proof of concept.
    # ProcessPoolExecutor is useful for CPU intensive tasks like compiling and processing lots of data
    # see https://stackoverflow.com/a/64450523/7732434 for an example than can use ProcessPoolExecutor in place of ThreadPool..
    with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:
        future_to_todo = {executor.submit(get_todo, i, optional_arg=True): i for i in range(500)}
        # Order is not guaranteed even if you use a list. Use the value part above as an index
        for future in concurrent.futures.as_completed(future_to_todo):
            try:
                todos.append(future.result())
            except ValueError as e:
                errors.append(e)
    return todos