zeromq + twitter staus filter

gistfile1.py

import requests
import time
import json
import zmq
from multiprocessing import Process
import couchdbkit


s = couchdbkit.Server("https://usr:[email protected]")
db = couchdbkit.Database("https://usr:[email protected]/db")

def worker():
    context = zmq.Context()
    work_receiver = context.socket(zmq.PULL)
    work_receiver.connect("tcp://127.0.0.1:5557")
    while 1:
        d = json.loads(work_receiver.recv())
        try:
            db.save_doc(doc=d)
        except Exception as e:
            with open('sachin_omq_log.txt','a') as f:
                msg = " ".join([e.message, " ", time.ctime(), "\n"])
                f.write(msg)
        

def store_live_tweets():
    Process(target=worker, args=()).start()
    context = zmq.Context()
    send = context.socket(zmq.PUSH)
    send.bind("tcp://127.0.0.1:5557")
    track = ['sachin', 'sachinism', 'Sachin', 'Sachinism', 'tendulkar', 'Tendulkar', 'sachintendulkar','SachinTendulkar']
    r = requests.post('https://stream.twitter.com/1/statuses/filter.json',
        data={'track': track}, auth=('usr', 'passwd'))
    for line in r.iter_lines():
        if line:
            print line
            send.send(line)

if __name__ == '__main__':
    try:
        store_live_tweets()
    except Exception as e:
        with open('sachin_omq_log.txt', 'a') as f:
            msg = " ".join([e.message, " ", time.ctime(), "\n"])
            f.write(msg)