twitter_streaming_api_users.py

'''Tweet Streaming API consumer'''
import twitter
import csv

# == OAuth Authentication from your homepage on the twitter developer web site ==
CONSUMER_KEY = ""
CONSUMER_SECRET = ""
ACCESS_TOKEN = ""
ACCESS_TOKEN_SECRET = ""


AUTH = twitter.oauth.OAuth(ACCESS_TOKEN, ACCESS_TOKEN_SECRET, CONSUMER_KEY, CONSUMER_SECRET)
TWITTER_API = twitter.Twitter(auth=AUTH)

csvfile = open('all_users_tweets.csv', 'w')
csvwriter = csv.writer(csvfile)
csvwriter.writerow(['created_at',
                    'user-screen_name',
                    'text',
                    'coordinates lng',
                    'coordinates lat',
                    'place',
                    'user-location',
                    'user-geo_enabled',
                    'user-lang',
                    'user-time_zone',
                    'user-statuses_count',
                    'user-followers_count',
                    'user-created_at'])

# Get user ID's here http://gettwitterid.com/
# Independent, BBCNews, DailyMailUK, Telegraph, guardiannews, TheSun, DailyMirror, thetimes
TWITTER_USER_IDS = '16973333,612473,111556423,16343974,788524, 34655603, 16887175, 6107422'

print 'Following user IDs="%s"' % (TWITTER_USER_IDS)
twitter_stream = twitter.TwitterStream(auth=TWITTER_API.auth)
stream = twitter_stream.statuses.filter(_method="POST", follow=TWITTER_USER_IDS)

''' helper functions for tweet data, clean up utf data, unpack dictionaries '''
def getVal(val):
    clean = ""
    if isinstance(val, bool):
        return val
    if isinstance(val, int):
        return val
    if val:
        clean = val.encode('utf-8') 
    return clean

def getLng(val):
    if isinstance(val, dict):
        return val['coordinates'][0]

def getLat(val):
    if isinstance(val, dict):
        return val['coordinates'][1]

def getPlace(val):
    if isinstance(val, dict):
        return val['full_name'].encode('utf-8')


# main loop, we grab one tweet from the stream and dig out the bits of data we want
# and then write them into a CSV file
for tweet in stream:
    try:
        csvwriter.writerow([tweet['created_at'],
                            getVal(tweet['user']['screen_name']),
                            getVal(tweet['text']),
                            getLng(tweet['coordinates']),
                            getLat(tweet['coordinates']),
                            getPlace(tweet['place']),
                            getVal(tweet['user']['location']),
                            getVal(tweet['user']['geo_enabled']),
                            getVal(tweet['user']['lang']),
                            getVal(tweet['user']['time_zone']),
                            getVal(tweet['user']['statuses_count']),
                            getVal(tweet['user']['followers_count']),
                            getVal(tweet['user']['created_at'])
                            ])
        csvfile.flush()
        print getVal(tweet['user']['screen_name']), getVal(tweet['text']), tweet['coordinates'], getPlace(tweet['place'])
    except Exception as e:
        print e