gistfile1.txt

import re, sys, os
from datetime import datetime
from collections import defaultdict


class Request():
	def __init__(self, id, start):
		self.id = id
		
		self.start = start
		self.end = None
		
		self.hh_session = None
		self.currencyList = None
		self.other_start = None
		self.other = []
		
		self.start_xsl = None
		self.end = None
		
	def processHttpLine(self, timestamp, text):
		guts = text.split()
		status = guts[1]
		url = guts[2].split('?')[0] 
		if 'hh-session' in url:
			self.hh_session = timestamp
		elif 'currencyList' in url:
			self.currencyList = timestamp
			self.other_start = timestamp
		else:
			self.other.append((timestamp, url))
			
loglineTag = re.compile('\[\d+\] (\d+-\d+-\d+ \d+:\d+:\d+,\d+) DEBUG \S+: {(\w+)} (.+)')
	
requests = {}

def parse(f):
	for l in f:
		g = loglineTag.search(l)
		
		if g is None: 
			continue
		
		timestamp = datetime.strptime(g.group(1), "%Y-%m-%d %H:%M:%S,%f")
		id = g.group(2)
		text = g.group(3)
		if 'started GET /page/applicant/searchvacancyresult/' in text:
			requests[id] = Request(id, timestamp)
		elif text[:3] == 'got' :
			try:
				requests[id].processHttpLine(timestamp, text)
			except KeyError:
				continue
		elif 'finishing with xsl' in text:
			try:
				requests[id].start_xsl = timestamp
			except KeyError:
				continue
		elif text == 'done':
			try:
				requests[id].end = timestamp
			except KeyError:
				continue