log-filter.py

gistfile1.py

#!/usr/bin/python

from collections import namedtuple
from dateutil.parser import parse as date_parse


LineFormat = namedtuple('LineFormat', ['separator', 'datecol', 'levelcol'])


class LogLine(object):
    def __init__(self, line, lineformat):
        splitline = line.split(lineformat.separator)
        self.datetime = date_parse(splitline[lineformat.datecol])
        self.loglevel = splitline[lineformat.levelcol]
        self.line = line

    def __str__(self):
        return str(self.line)


def read_logs(filename, lineformat):
    with open(filename, 'r') as logfile:
        for line in logfile:
            yield LogLine(line, lineformat)


def filter_daterange(start, end, stream):
    """assume sequential datetimes in the stream"""

    start_dt = date_parse(start)
    end_dt = date_parse(end)

    for line in stream:
        if line.datetime > end_dt:
            return
        if line.datetime > start_dt:
            yield line


def filter_loglevel(level, stream):
    for line in stream:
        if line.loglevel == level:
            yield line


line_format = LineFormat(separator='|', datecol=1, levelcol=2)
start_time = '2012-12-13T10:00:00 +0400'
end_time = '2012-12-13T11:00:00 +0400'
level = 'INFO'
logfile = 'sample.log'

for line in filter_loglevel(level, filter_daterange(start_time, end_time, read_logs(logfile, line_format))):
    print(line)