Skip to content

Instantly share code, notes, and snippets.

@mchow01
Last active July 6, 2020 21:12
Show Gist options
  • Save mchow01/af931385b40fd9fe2931 to your computer and use it in GitHub Desktop.
Save mchow01/af931385b40fd9fe2931 to your computer and use it in GitHub Desktop.
Twitter Stream Tracker + Inserting Data into (MySQL) Database
import tweepy
import MySQLdb
import sys
import os
import logging
# Tweepy API doc here: http://pythonhosted.org/tweepy/html/api.html
# Keys
consumer_key = os.getenv("TWITTER_CONSUMER_KEY")
consumer_secret = os.getenv("TWITTER_CONSUMER_SECRET")
access_token = os.getenv("TWITTER_TOKEN")
access_token_secret = os.getenv("TWITTER_TOKEN_SECRET")
# Twitter initialization
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)
# MySQL initialization
connection = MySQLdb.connect(host= "localhost",
user="twtr",
passwd="twtr",
db="twitter")
cursor = connection.cursor()
# The table schema: CREATE TABLE tweets (id INT PRIMARY KEY AUTO_INCREMENT, tweet_id BIGINT NOT NULL, tweet_text VARCHAR(160) NOT NULL, screen_name VARCHAR(160) NOT NULL, author_id BIGINT, created_at DATETIME NOT NULL, inserted_at DATETIME NOT NULL)
# For streaming
class TwitterStreamListener(tweepy.StreamListener):
def on_status(self, status):
try:
cursor.execute("INSERT INTO tweets (tweet_id, tweet_text, screen_name, author_id, created_at, inserted_at) VALUES (%s, %s, %s, %s, %s, NOW());", (status.id, status.text, status.author.screen_name, status.author.id, status.created_at))
connection.commit()
except:
pass
def on_error(self, status_code):
# Rate limit reached
if status_code == 420:
logging.exception("Rate limit reached") # disconnects stream
try:
streamListener = TwitterStreamListener()
twitterStream = tweepy.Stream(auth = api.auth, listener=streamListener)
twitterStream.filter(track=['ransomware'])
except tweepy.error.TweepError as e:
logging.exception(e)
except UnicodeEncodeError as e:
logging.exception(e)
except:
logging.exception(sys.exc_info()[0])
raise
finally:
cursor.close()
connection.close()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment