Skip to content

Instantly share code, notes, and snippets.

@elena-roff
Created March 12, 2019 13:21
Show Gist options
  • Save elena-roff/b7bab43ceb65b3203af921dd07495666 to your computer and use it in GitHub Desktop.
Save elena-roff/b7bab43ceb65b3203af921dd07495666 to your computer and use it in GitHub Desktop.
Get data from the api with an offset update and limit extended
def api_get_data() -> List[Mapping]:
""" Gets data from the API. """
url = "{}/v1/booking".format(os.environ['CT_BOOKING_API_URI'])
params: Dict = {
...
'limit': 1000,
'offset': 0,
}
headers = {
'Authorization': get_s2s_token(env='API_TOKEN'),
'Accept-Encoding': "gzip, deflate"
}
responses: List[Mapping] = []
try:
while True:
response = requests.get(url, headers=headers, params=params)
if response.status_code != 200:
logger.error('Failed to get data: {}'.format(
response.text))
return responses
else:
data = response.json()
responses.extend(data)
if len(data) == 1000:
params['offset'] += len(data)
else:
return responses
except (requests.exceptions.RequestException,
json.decoder.JSONDecodeError, TypeError) as e:
logger.exception(e)
return responses
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment