Tornado Twitter Stream

gistfile1.txt

# not pretty, I quickly moved to another idea
# not involving the Twitter stream. Good starting
# point.

import re
import base64
import socket
import asyncmongo
from tornado import ioloop
from tornado import iostream
from tornado import escape

#substitue your Twitter username/password below
auth = base64.b64encode("USERNAME:PASSWORD")

db = asyncmongo.Client(pool_id='mydb', host='127.0.0.1', port=27017, maxcached=10, maxconnections=50, dbname='trends')

def send_request():
    stream.write("GET /1/statuses/sample.json HTTP/1.0\r\nHost: stream.twitter.com\r\nAuthorization: Basic %s\r\n\r\n" % auth)
    stream.read_until("\r\n\r\n", on_headers)

def on_headers(data):
    headers = {}
    for line in data.split("\r\n"):
       parts = line.split(":")
       if len(parts) == 2:
           headers[parts[0].strip()] = parts[1].strip()
    stream.read_until("\n", on_body)

def quit(*args, **kwargs):
    stream.close()
    ioloop.IOLoop.instance().stop()

def cb_pass(*args, **kwargs):
    pass

def on_body(data):
    tweet = escape.json_decode(data)
    text = re.sub("@\w+", "", tweet.get("text", ""))
    text = re.sub("RT", "", text)
    text = re.sub("http\w+", "", text)
    if text != "":
        #print text
        db.tweets.insert({"text": text}, callback=cb_pass)
    stream.read_until("\n", on_body)


s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
stream = iostream.IOStream(s)
stream.connect(("stream.twitter.com", 80), send_request)
ioloop.IOLoop.instance().start()