Log file tailer to send events to statsd

logstats.py

#!/usr/bin/env python

from __future__ import with_statement

import random
import re
import select
import socket
import time
from optparse import OptionParser

class StatsD(object):
    def __init__(self, host='localhost', port=8125):
        self.host = host
        self.port = port
        self.udp_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    
    def timing(self, stat, time, sample_rate=1):
        stats = {stat: "%d|ms" % time}
        self.send(stats, sample_rate)

    def increment(self, stats, sample_rate=1):
        self.update_stats(stats, 1, sample_rate)

    def decrement(self, stats, sample_rate=1):
        self.update_stats(stats, -1, sample_rate)
    
    def update_stats(self, stats, delta=1, sampleRate=1):
        if (type(stats) is not list):
            stats = [stats]
        data = {}
        for stat in stats:
            data[stat] = "%s|c" % delta

        self.send(data, sampleRate)
    
    def send(self, data, sample_rate=1):
        addr = (self.host, self.port)
        
        sampled_data = {}
        
        if sample_rate < 1:
            if random.random() <= sample_rate:
                for stat, value in data.iteritems():
                    value = data[stat]
                    sampled_data[stat] = "%s|@%s" %(value, sample_rate)
        else:
            sampled_data = data
        
        try:
            for stat, value in sampled_data.iteritems():
                send_data = "%s:%s" % (stat, value)
                self.udp_sock.sendto(send_data, addr)
        except:
            pass # we don't care

def build_parser():
    parser = OptionParser(usage="Usage: %prog [options] logfile")
    parser.add_option("-a", "--host", dest="host", help="Host of statsd", default="127.0.0.1")
    parser.add_option("-d", "--debug", dest="debug", help="Print to standard out rather than sending to statsd", default=False, action="store_true")
    parser.add_option("-n", "--lines", dest="lines", type="int", help="Start this many lines from end of file (default 0)", default=0)
    parser.add_option("-p", "--port", dest="port", type="int", help="Port of statsd", default=8125)
    parser.add_option("-s", "--stat", dest="stats", help="Stat name=regularexpression (can have multiple)", action="append", default=[])
    return parser

def main():
    parser = build_parser()
    options, args = parser.parse_args()
    if not options.stats:
        parser.error("must provide at least one stat (-s)")
    if not args:
        parser.error("must provide at least one log file path")

    stats = []
    for s in options.stats:
        name, restr = s.split('=')
        stats.append((name, re.compile(restr)))
    
    statsd = StatsD(options.host, options.port)
    
    with open(args[0], "r") as fp:
        # Start at the end of the log
        if options.lines:
            try:
                fp.seek(-2048, 2)
            except IOError:
                # File smaller than requested
                pass
            end = fp.read()
            start = sum(len(x)+1 for x in end.split('\n')[-options.lines-1:])-1
            fp.seek(-start, 2)
        else:
            fp.seek(0, 2)

        while True:
            line = fp.readline()
            if not line:
                time.sleep(1)
                continue

            line = line.strip()
            for name, reo in stats:
                m = reo.search(line)
                if m:
                    fullname = name.format(*m.groups(), **m.groupdict())
                    if options.debug:
                        print fullname, line
                    else:
                        statsd.increment(fullname)

if __name__ == "__main__":
    main()