multiline logstash to flumelogger

flume-logstash.py

#!/usr/bin/python
import re
import time
import logging
from os import stat
from stat import ST_SIZE
from flumelogger.handler import FlumeHandler

FILE_INPUT = "/var/log/jenkins/jenkins.log"
MULTILINE_PATTERN_RE = r"^\w+ \d+, \d+ (\d+?:){1,2}\d\d \w\w "

def follow(filename):
    fd = open(filename, 'r')
    file_len = stat(filename)[ST_SIZE]
    fd.seek(file_len)
    while True:
        pos = fd.tell()
        line = fd.readline()
        if not line:
            if stat(filename)[ST_SIZE] < pos:
                fd.close()
                fd = open(filename, 'r')
                continue
            time.sleep(0.1)
            fd.seek(pos)
            continue
        yield line

fh = FlumeHandler(type='og', headers={'application': 'jenkins'})
logger = logging.getLogger("jenkins")
logger.setLevel(logging.DEBUG)
logger.addHandler(fh)

buffer = []
loglines = follow(FILE_INPUT)
for line in loglines:
    if re.match(MULTILINE_PATTERN_RE, line):
        if buffer:
            event = "".join(buffer)
            event_no_timestamp = re.sub(MULTILINE_PATTERN_RE, '', event)
            #print "------------------------------LOG EVENT---------------------------"
            #print event_no_timestamp
            #print "------------------------------------------------------------------"
            logger.info(event_no_timestamp)
        buffer = [line]
        continue
    if buffer:
        buffer.append(line)