Python example view of Gnip v2.5 stream output

viewStream.py

#!/usr/bin/env python

import urllib2
import base64
import zlib
import threading
from threading import Lock
from cStringIO import StringIO
import json
import sys
from pprint import pprint

# tune these as needed
CHUNKSIZE = 1024
MAXBUFFSIZE = 20*CHUNKSIZE

NEWLINE = '\n'

url = "https://stream.gnip.com:443/accounts/client/publishers/twitter/streams/link/ls.json"
UN = 'un'
PWD = 'pwd'

print_lock = Lock()

class procEntry(threading.Thread):
        def __init__(self, buf):
                self.bufList = [x.strip() for x in buf.split(NEWLINE) if x.strip() <> '']
                threading.Thread.__init__(self)

        def run(self):
                try:
                        self.output(self.bufList)
                except Exception, e:  # v hard to debug, catches everything!
                        sys.stderr.write("thread failed: (%s)\n"%e)

        def output(self, bufList):
                for rec in bufList:
                        jrec = json.loads(rec.strip())
                        with print_lock:
                                pprint(jrec)
                                print "="*40

def get():
        headers = {
        'Accept': 'application/json',
        'Connection': 'keep-alive',
        'Accept-Encoding' : 'gzip',
        'Authorization' : 'Basic %s' % base64.encodestring('%s:%s' % (UN, PWD))
        }
        req = urllib2.Request(url, headers=headers)
        response = urllib2.urlopen(req)

        d = zlib.decompressobj(16+zlib.MAX_WBITS)

        ldata = StringIO()            
        bufSize = 0
        while True:
                tmpString = d.decompress(response.read(CHUNKSIZE))
                bufSize += len(tmpString)
                ldata.write(tmpString)
                if tmpString.endswith(NEWLINE) and bufSize > MAXBUFFSIZE:
                        procEntry(ldata.getvalue()).start()
                        ldata = StringIO()
                        bufSize = 0

if __name__ == "__main__":
        get()