async python http requests using exponential backoff, jitter, and event logging

http_requests.py

"""
This module provides functionality for making HTTP requests. It leverages the `aiohttp` 
library for asynchronous HTTP requests, and the `backoff` library to implement exponential 
backoff in case of failed requests.

The module defines a child logger for logging purposes and implements two methods, `on_backoff` 
and `on_giveup`, which log information about the retry attempts and when the retry attempts are 
given up respectively.

The `http_request` function is the primary function of the module, making an HTTP request with the 
provided parameters. If the request fails due to an `aiohttp.ClientError`, the function will retry 
the request using an exponential backoff strategy, up to a maximum of 5 times or a total of 60 
seconds.

The function logs the result of the HTTP request, either indicating success and the status code of 
the response or raising an exception if the response cannot be parsed as JSON or if the response 
status code indicates a client error.
"""

import json
import logging
import aiohttp
import backoff

# set up child logger for module
logger = logging.getLogger(__name__)


def on_backoff(backoff_event):
    """
    Logs a warning message whenever a backoff event occurs due to a failed HTTP request.

    Parameters
    ----------
    backoff_event : dict
        A dictionary containing detailed information about the backoff event.
        It includes the following keys:
        'wait' (the delay before the next retry),
        'tries' (the number of attempts made),
        'exception' (the exception that caused the backoff),
        'target' (the function where the exception was raised),
        'args' (the arguments passed to the target function),
        'kwargs' (the keyword arguments passed to the target function),
        and 'elapsed' (the time elapsed since the first attempt).
    """
    logger.warning(
        "http backoff event",
        extra={
            "Retrying in, seconds": {backoff_event["wait"]},
            "Attempt number": {backoff_event["tries"]},
            "Exception": {backoff_event["exception"]},
            "Target function": {backoff_event["target"].__name__},
            "Args": {backoff_event["args"]},
            "Kwargs": {backoff_event["kwargs"]},
            "Elapsed time": {backoff_event["elapsed"]},
        },
    )


def on_giveup(giveup_event):
    """
    Logs an error message when a series of HTTP requests fail and the retry attempts are given up.

    Parameters
    ----------
    giveup_event : dict
        A dictionary containing detailed information about the event when retries are given up.
        It includes the following keys:
        'tries' (the number of attempts made),
        'exception' (the exception that caused the retries to be given up),
        'target' (the function where the exception was raised),
        'args' (the arguments passed to the target function),
        'kwargs' (the keyword arguments passed to the target function),
        and 'elapsed' (the time elapsed since the first attempt).
    """
    logger.error(
        "http giveup event",
        extra={
            "Giving up after retries exceeded": giveup_event["tries"],
            "Exception": giveup_event["exception"],
            "Target function": giveup_event["target"].__name__,
            "Args": giveup_event["args"],
            "Kwargs": giveup_event["kwargs"],
            "Elapsed time": giveup_event["elapsed"],
        },
    )


@backoff.on_exception(
    backoff.expo,
    aiohttp.ClientError,
    jitter=backoff.random_jitter,
    max_tries=5,
    max_time=60,
    on_backoff=on_backoff,
    on_giveup=on_giveup,
)
async def http_request(verb, url, query_params, headers, payload):
    """
    Performs an HTTP request, retrying on `aiohttp.ClientError` exceptions with an exponential
    backoff strategy.

    Parameters
    ----------
    verb : str
        The HTTP method for the request, such as 'GET', 'POST', etc.
    url : str
        The URL for the HTTP request.
    query_params : dict
        The query parameters to be included in the request.
    headers : dict
        The headers to be included in the request.
    payload : dict
        The payload (body) of the request, which will be sent as JSON.

    Returns
    -------
    None

    Raises
    ------
    ValueError
        If the response from the HTTP request cannot be parsed as JSON.
    aiohttp.ClientResponseError
        If the HTTP request returns a response with a status code indicating a client error.

    Note
    ----
    The function will automatically retry the request if an `aiohttp.ClientError` is raised.
    It uses an exponential backoff strategy with a maximum of 5 tries and a total retry duration
    of 60 seconds. The 'on_backoff' function will be called after each failed attempt, and the
    'on_giveup' function will be called if all retry attempts fail.
    """
    async with aiohttp.ClientSession() as session:
        async with session.request(
            method=verb, url=url, params=query_params, headers=headers, json=payload
        ) as response:
            if response.ok:
                try:
                    data = await response.json()
                    return data
                except json.JSONDecodeError as json_decode_error:
                    raise ValueError(
                        "Failed to parse response JSON"
                    ) from json_decode_error
            else:
                raise aiohttp.ClientResponseError(
                    response.request_info,
                    response.history,
                    status=response.status,
                    message=response.reason,
                )