Last active
May 4, 2016 23:21
-
-
Save gdsaxton/0dbfa236a7fadb52c93e to your computer and use it in GitHub Desktop.
Loop over list of Twitter user screen_names, create dictionary of users' follower IDs and friend IDs.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
for id in screen_names_unique[42:44]: | |
screen_name = id | |
print '\n', screen_name | |
cursor = "-1" | |
print 'followers list first cursor', cursor | |
d = get_followers_ids (screen_name, cursor) | |
limits = t.get_application_rate_limit_status() | |
follower_limit = limits['resources']['followers']['/followers/ids']['remaining'] | |
print '# followers/ids API calls remaining', follower_limit | |
if not d: | |
followers_list = 'protected' | |
friends_list = 'protected' | |
org_dict_protected = {'screen_name': screen_name, 'followers_list': followers_list, 'friends_list': friends_list} | |
#org_list.append(org_dict_protected) | |
if screen_name not in org_dict: | |
org_dict[screen_name] = {'followers_list': followers_list, 'friends_list': friends_list} | |
continue | |
else: | |
pass | |
followers_list = d['ids'] | |
print 'len(followers_list)', len(followers_list) | |
cursor = d['next_cursor'] | |
print 'followers list second cursor', cursor | |
count = 2 | |
while cursor != 0 : | |
print "SLEEPING FOR 75 SECONDS................................" | |
time.sleep(75) | |
print 'trying to get another set of 5,000 for page %s' % count | |
limits = t.get_application_rate_limit_status() | |
follower_limit = limits['resources']['followers']['/followers/ids']['remaining'] | |
if follower_limit > 1: | |
print "follower_limit OK...continuing" | |
#continue #RETURN TO START OF WHILE LOOP | |
else: | |
time.sleep(450) #5 minute sleep | |
#time.sleep(900) #15 minute sleep | |
#continue | |
d = get_followers_ids (screen_name, cursor) | |
followers_list.extend(d['ids']) | |
cursor = d['next_cursor'] | |
print 'followers list cursor %s' % count, cursor | |
print 'len(followers_list)', len(followers_list) | |
count +=1 | |
limits = t.get_application_rate_limit_status() | |
follower_limit = limits['resources']['followers']['/followers/ids']['remaining'] | |
print '# followers/ids API calls remaining', follower_limit | |
if follower_limit > 1: | |
print "follower_limit OK...continuing" | |
#continue #RETURN TO START OF WHILE LOOP | |
else: | |
time.sleep(450) #5 minute sleep | |
#time.sleep(900) #15 minute sleep | |
#continue | |
limits = t.get_application_rate_limit_status() | |
follower_limit = limits['resources']['followers']['/followers/ids']['remaining'] | |
if follower_limit > 0: | |
print "follower_limit OK...continuing" | |
#continue #RETURN TO START OF WHILE LOOP | |
else: | |
print "SLEEPING FOR 10 MINUTES....." | |
time.sleep(600) #10 minute sleep | |
#continue | |
print "DONE WITH FOLLOWERS LIST, NOW LET'S GET FRIENDS LIST" | |
cursor = "-1" | |
print 'friends list first cursor', cursor | |
d = get_friends_ids (screen_name, cursor) | |
if not d: | |
friends_list = 'no friends' | |
org_dict_protected = {'screen_name': screen_name, 'followers_list': followers_list, 'friends_list': friends_list} | |
org_list.append(org_dict_protected) | |
if screen_name not in org_dict: | |
org_dict[screen_name] = {'followers_list': followers_list, 'friends_list': friends_list} | |
continue | |
else: | |
pass | |
friends_list = d['ids'] | |
print 'len(friends_list)', len(friends_list) | |
cursor = d['next_cursor'] | |
print 'friends list second cursor', cursor | |
count = 2 | |
while cursor != 0 : | |
print 'trying to get another set of 5,000 for page %s' % count | |
d = get_friends_ids (screen_name, cursor) | |
friends_list.extend(d['ids']) | |
cursor = d['next_cursor'] | |
print 'friends list cursor %s' % count, cursor | |
print 'len(friends_list)', len(friends_list) | |
count +=1 | |
limit = t.get_lastfunction_header('x-rate-limit-remaining') | |
friend_limit = limits['resources']['friends']['/friends/ids']['remaining'] | |
print '# friends/ids API calls remaining', limit, friend_limit | |
if friend_limit > 1: | |
print "friend_limit OK...continuing" | |
#continue | |
else: | |
time.sleep(900) #15 minute sleep | |
"WAKING UP.....NOW TRYING TO CONTINUE" | |
continue | |
#org_dict_for_list = {'screen_name': screen_name, 'followers_list': followers_list, 'friends_list': friends_list} | |
#org_list.append(org_dict_for_list) | |
if screen_name not in org_dict: | |
org_dict[screen_name] = {'followers_list': followers_list, 'friends_list': friends_list} | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Hi! Total neophyte here, but I am hoping to use your code snippets to scrape some data. However, in this instance, I can't seem to figure out where to actually input the main screen name(s) that I want to gather data for. Any guidance is appreciated. Thanks for making all of this public!