webtail

demo.py

import eventlet
from eventlet import GreenPool
from eventlet.green import urllib2
import logging

log = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)

class Reader(object):
    def __init__(self, i):
        self.i = i

    def read_stream(self):
        log.info("Starting listener #%s" % (self.i, ))
        fh = urllib2.urlopen("http://localhost:8000/")

        while True:
            print fh.read(10),


# Create a pool of 1000 coroutines
pool_size = 1000
pool = GreenPool(pool_size)
for i in range(pool_size):
    pool.spawn(Reader(i).read_stream)
pool.waitall()

webtail.py

import eventlet
from eventlet import Queue, wsgi, GreenPool
from eventlet.green import os
import logging

log = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

# Create a set to hold the request queue objects
queue_set = set()


# This is the file reader thread
def follow(filename):
    fh = open(filename, "r")

    # Seek to the end of the file
    fh.seek(0, 2)

    while True:
        line = fh.readline()
        if line:
            log.info(line)
            # Push the line into the queue list
            for queue in queue_set:
                queue.put(line)
        eventlet.sleep(0.001)

def wsgi_app(environ, start_response):
    # This is the generator that streams the lines out HTTP
    def line_gen():
        # Create a new queue object for this request
        queue = Queue()
        queue_set.add(queue)

        log.debug("Created queue: %s" % (id(queue), ))

        try:
            while True:
                log.debug("Reading queue")
                line = queue.get()
                yield line
        finally:
            log.debug("Removing queue: %s" % (id(queue), ))
            queue_set.remove(queue)

    start_response("200 OK", [("Content-Type", "text/plain")])
    return line_gen()

if __name__ == '__main__':
    import sys

    # Spawn the follow thread
    filenames = sys.argv[1:]

    if not filenames:
        print "provide some log files on the command line"
        sys.exit()

    for filename in filenames:
        eventlet.spawn(follow, filename)

    # Serve up the WSGI app
    wsgi.server(eventlet.listen(("", 8000)), wsgi_app, minimum_chunk_size=1)