Exponential backoff

ratemgr.py

def backoff(
    url: str,
    waitfor: Optional[datetime.timedelta] = None,
    waituntil: Optional[datetime.datetime] = None,
    base: int = 2,
    method: str = 'GET',
    bad_status_codes: Optional[Sequence] = None,
    session: Optional[requests.Session] = None,
    begin_timeout: int = 10,
    _now=datetime.datetime.now,
    **kwargs
) -> requests.Response:
    """HTTP request with exponential backoff.

    Parameters
    ----------
    url: str
        Passed to requests.request
    waitfor: datetime.timedelta or None
        Keep making request retries for this amount of time
    waituntil: datetime.datetime or None
        Keep making request retries until this time deadline
        If neither `waitfor` or `waituntil` is specified, default to 1 minute
    base: int, default 2
        The base used for calculating the sleeptime for next request.
        It is raised to `tries`, the current number of attempts.
        If `base` is 2, sleep-times will be for 2**1, 2**2, 2**3, ...
    method: str, default 'GET'
        Passed to requests.request
    **kwargs
        Passed to `requests.request()`

    Example
    -------
    >>> import datetime
    >>> base = 'http://www.reddit.com/api/info.json?url=%s'
    >>> url = 'https://www.zerohedge.com/news/2018-09-14/deluded-bankers-tale-lehmans-last-days'  # noqa
    >>> resp = backoff(url=base % url, waitfor=waitfor)
    """

    _okay_to_close = False
    if session is None:
        _okay_to_close = True
        session = make_session()
    if waitfor and waituntil:
        raise ValueError('Specify one of `waitfor`/`waituntil`, not both.')
    elif waituntil:
        pass
    elif waitfor:
        waituntil = _now() + waitfor
    else:
        # Default to 1 minute
        waituntil = _now() + datetime.timedelta(minutes=1)

    # Pass random user-agent if not provided
    headers = kwargs.pop('headers', {})
    ua = headers.get('User-Agent', None)

    try:
        tries = 1
        while waituntil >= _now():
            timeout = max(3, begin_timeout - tries)
            if ua:
                _headers = headers
            else:
                _headers = {**headers,
                            **{'User-Agent': make_random_useragent()}}
            try:
                resp = session.request(
                    method=method, url=url,
                    headers=_headers,
                    timeout=timeout,
                    **kwargs
                )
            except requests.exceptions.SSLError:
                logger.warning(
                    'Cert. verify failed.  Making unverified request to %s.',
                    url
                )
                resp = session.request(
                    method=method, url=url,
                    headers=_headers,
                    timeout=timeout,
                    verify=False,
                    **kwargs
                )
            except:
                # This is not a bad response status; it's that we couldn't get
                # a response at all and indicates a more serious problem
                raise

            if bad_status_codes:
                if resp.status_code in bad_status_codes:
                    # Some status codes may be deemed unrecoverable and
                    # not worth a retry, such as a 404 or 410
                    #
                    # Again, we let the caller determine what to actually
                    # do with the response rather than raising
                    logger.info('Bad status code [%s] for %s.  Returning.',
                                resp.status_code, url)
                    return resp
            if resp.ok:
                return resp

            # first sleeptime is just `base`
            sleeptime = base ** tries
            logger.debug('Bad request #%s: [%s] status for %s.'
                         '  Sleeping for %d seconds.',
                         tries, resp.status_code, url, sleeptime)
            time.sleep(sleeptime)
            tries += 1

        # Don't actually raise; let the caller determine what they want to
        # do if the status code is still bad
        logger.info('Exhausted backoff cycle for %s.  Returning.', url)
        return resp

    finally:
        if _okay_to_close:
            # If the caller is sending its own session object, don't close
            # it without asking.
            session.close()