requests_retry.py

Python requests with retries configured

# see https://findwork.dev/blog/advanced-usage-python-requests-timeouts-retries-hooks/

import requests

from requests.adapters import HTTPAdapter, Retry

class TimeoutHTTPAdapter(HTTPAdapter):
    DEFAULT_TIMEOUT = 30

    def __init__(self, *args, **kwargs):
        self.timeout = self.DEFAULT_TIMEOUT
        if "timeout" in kwargs:
            self.timeout = kwargs["timeout"]
            del kwargs["timeout"]
        super().__init__(*args, **kwargs)

    def send(self, request, **kwargs):
        timeout = kwargs.get("timeout")
        if timeout is None:
            kwargs["timeout"] = self.timeout
        return super().send(request, **kwargs)


retry_strategy = Retry(
    total=10,
    backoff_factor=0.05,  # [(.05)*2**i for i in range(10)]
    respect_retry_after_header=True,
    status_forcelist=list(Retry.RETRY_AFTER_STATUS_CODES) + [408, 425, 500, 502, 504],
)
adapter = TimeoutHTTPAdapter(max_retries=retry_strategy)
http = requests.Session()
http.mount("https://", adapter)
http.mount("http://", adapter)