generic_get.py

def get_url_or_retry(url, retry_in=(), wait=1, json=False, header={}, **params):
    """
    Fetch an url using Requests or retry fetching it if the server is
     complaining with retry_in error.
    :param url: url to be fetched as a string
    :param wait: sleeping between tries in seconds
    :param params: request.get kwargs.
    :return: url content or url content in json data structure.
    """
    if json:
        header.update({"Content-Type": "application/json"})
    request = requests.get(url, headers=header, params=params)
    if request.ok:
        if json:
            return request.json()
        else:
            return request.content
    elif request.status_code in retry_in:
        time.sleep(wait)
        return request_info_url(
            url, retry_in, wait, json, header, **params)
    else:
        logger.error(request.status_code)
        request.raise_for_status()