Created
June 10, 2023 03:54
-
-
Save gchristian/7462f9758bd9331958d610a7bb9a5116 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def make_request(self, id="", type="deck", username="TMind"): | |
endpoint = f"{self.base_url}/{type}/{id}" | |
self.params.update({"username" : username}) | |
print(f"Requesting Data from Website: {','.join([username,type,id])}") | |
response = requests.get(endpoint, params=self.params) | |
data = json.loads(response.content) | |
#insert pagination code | |
if "LastEvaluatedKey" in data: | |
paginatationRequired = True | |
else: | |
paginatationRequired = False | |
while paginatationRequired == True: | |
self.params["exclusiveStartKey"] = json.dumps(data['LastEvaluatedKey']) | |
page_response = requests.get(endpoint, params=self.params) | |
page_data = json.loads(page_response.content) | |
if 'error' in page_data: | |
print(f"Error in response: {page_data['error']}") | |
return [] | |
data["Items"].extend(page_data["Items"]) | |
if "LastEvaluatedKey" in page_data: | |
data["LastEvaluatedKey"] = page_data["LastEvaluatedKey"] | |
else: | |
paginatationRequired = False | |
#end pagination code | |
if 'error' in data: | |
print(f"Error in response: {data['error']}") | |
return [] | |
with open('data/online_request.json', 'w') as f: | |
json.dump(data, f) | |
print("Loading Data...") | |
decks_data = self.ucl.load_data('data/online_request.json') | |
return decks_data |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment