Created
March 27, 2011 19:49
-
-
Save dmitric/889533 to your computer and use it in GitHub Desktop.
A nice way to handle twitter API errors
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def make_api_call(self,f,**kwargs): | |
if f is None: | |
raise ValueError("No function passed in!") | |
response = [] | |
wait_period = 1 | |
while True: | |
try: | |
#your api call | |
response = f(**kwargs) | |
break | |
except twitter.api.TwitterHttpError, e: | |
wait_period = self.handle_twitter_error(e,wait_period) #get new wait period | |
if wait_period is None: #if its none, we can skip it | |
break | |
return response |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def handle_twitter_error(self,e,wait_period=1): | |
if wait_period > 3600: | |
logger.error('Too many retries') | |
raise e | |
if e.e.code == 401: | |
logger.warning('Not Authorized To View This Information. Skipping...') | |
return None | |
elif e.e.code in (502,503): | |
logger.warning(str(e.e.code)+' error. Waiting for '+str(wait_period)+' seconds...') | |
time.sleep(wait_period) | |
return wait_period + 2 #they suggest adding time linearly | |
elif e.e.code == 404: | |
logger.warning('404 Error. Page not found.') | |
return None | |
elif self.api.account.rate_limit_status()['remaining_hits'] == 0: | |
now = time.time() | |
sleep_for = self.api.account.rate_limit_status()['reset_time_in_seconds']-now | |
logger.warning('Rate limited. Sleeping for '+str(sleep_for)+' seconds') | |
time.sleep(sleep_for) | |
return 1 #start over at 1 second wait time | |
else: | |
logger.error("Uh oh, something bad happened...") | |
raise e |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#created comma delimited list of first 100 names and ids passed in. | |
params = { | |
'screen_name': ','.join(screen_names[:100]), | |
'user_id': ','.join([str(user_id) for user_id in user_ids[:100]]) | |
} | |
#make the call using api.users.lookup function, this handles all errors | |
response = self.api_call(f=api.users.lookup,**params) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment