parse_perf_log.py

import re
import sys


# [03/Jun/2010:21:58:08 -0700] "GET /ja/thunderbird/downloads/file/75384/xpi/lightning-1.0b1-tb+sm-win.xpi?src=category HTTP/1.1" 200 9


def avg(l):
    return sum(l) / len(l)

def stddev(l):
    a = avg(l)
    return (sum((i - a) ** 2 for i in l) / len(l)) ** .5


perf_log_re = re.compile('\[\d\d/\w{3}/\d{4}(?::\d\d){3} -0700\] "\S+ (\S+) \S+" (\d+) (\d+)')

entries = {}
for l in sys.stdin:
    m = perf_log_re.search(l)

    if m:
        url, status, time = m.groups()
        entries.setdefault(url, []).append(int(time))


print "count, max, min, avg, stddev, url"
for k, v in sorted(entries.iteritems(), cmp=lambda x, y: cmp(avg(x[1]), avg(y[1]))):

    if len(v) > 3 and avg(v) > 0:
        print "%d, %d, %d, %0.2f, %0.2f, %s" % (
            len(v),
            max(v),
            min(v),
            avg(v),
            stddev(v),
            k)