Last active
May 7, 2020 17:36
-
-
Save mchow01/7910c873f6a2094b24bf to your computer and use it in GitHub Desktop.
Store tweets from a Twitter timeline into a MySQL database using Tweepy
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import tweepy | |
import MySQLdb | |
import sys | |
# Tweepy API doc here: http://pythonhosted.org/tweepy/html/api.html | |
# Keys | |
consumer_key = '' | |
consumer_secret = '' | |
access_token = '' | |
access_token_secret = '' | |
# Twitter initialization | |
auth = tweepy.OAuthHandler(consumer_key, consumer_secret) | |
auth.set_access_token(access_token, access_token_secret) | |
api = tweepy.API(auth) | |
# MySQL initialization | |
connection = MySQLdb.connect(host= "localhost", | |
user="?", | |
passwd="?", | |
db="?") | |
cursor = connection.cursor() | |
# The table schema: CREATE TABLE tweets (id INT PRIMARY KEY AUTO_INCREMENT, tweet_id BIGINT NOT NULL, tweet_text VARCHAR(160) NOT NULL, screen_name VARCHAR(160) NOT NULL, author_id INT, created_at DATETIME NOT NULL, inserted_at DATETIME NOT NULL) | |
try: | |
statuses = api.list_timeline(api.me().screen_name, '<NAME OF TIMELINE?>') | |
for s in statuses: | |
# To remove duplicate entries | |
cursor.execute("SELECT id FROM tweets WHERE tweet_text = %s;", [s.text]) | |
if cursor.rowcount == 0: | |
cursor.execute("INSERT INTO tweets (tweet_id, tweet_text, screen_name, author_id, created_at, inserted_at) VALUES (%s, %s, %s, %s, %s, NOW());", (s.id, s.text, s.author.screen_name, s.author.id, s.created_at)) | |
connection.commit() | |
except tweepy.error.TweepError: | |
print "Whoops, could not fetch news!" | |
except UnicodeEncodeError: | |
pass | |
except: | |
print "Unexpected error: ", sys.exc_info()[0] | |
raise | |
finally: | |
cursor.close() | |
connection.close() |
Getting this error while running it
cursor.execute("INSERT INTO tweets (tweet_id, tweet_text, screen_name, author_id, created_at, inserted_at) VALUES (%s, %s, %s, %s, %s, NOW());", (s.id, s.text, s.author.screen_name, s.author.id, s.created_at))
UnicodeEncodeError: 'latin-1' codec can't encode character u'\u2026' in position 139: ordinal not in range(256)
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
hi, this code is useful, how can i use this to get followers of an account and store their screen names in data base