munin script for apache response time

apache_response_time.py

#!/usr/bin/env python
import sys
import os.path
import re
from subprocess import *

LAST_N_REQUESTS = 50000
LOG_PATH = '/usr/local/apache/logs/access.log'
LOG_OPEN_COMMANDS = [
  ('tail', '-n %d' % LAST_N_REQUESTS, LOG_PATH),
]
LOG_DELIM = '\t'
URL_INDEX = 4
RESPONSE_TIME_INDEX = 9

DOMAIN = 'kei2100.gist.github.com'
REQUEST_TYPES = [
  # api
  {
    'munin_fields': {
      'label': 'api',
      'min': 0,
      'max': 30 * 1000 * 1000,
    },
    'url_pattern': r'/api/',
    'response_time_total': 0,
    'call_count': 0,
  },
  # other
  {
    'munin_fields': {
      'label': 'other',
      'min': 0,
      'max': 30 * 1000 * 1000,
    },
    'url_pattern': r'^(?!.*/api/)',
    'response_time_total': 0,
    'call_count': 0,
  },
]


def autoconf():
  """
  autconf for munin

  """
  if os.path.exists(LOG_PATH):
    print 'yes'
  else:
    print 'no (log file not found. %s)' % LOG_PATH

def config():
  """
  config for munin

  """
  print 'graph_title Apache response time avrg last %s (%s)' % (LAST_N_REQUESTS, DOMAIN)
  print 'graph_args --lower-limit 0'
  print 'graph_category apache'
  print 'graph_vlabel micro sec'
  for req_type in REQUEST_TYPES:
    label = req_type['munin_fields']['label']
    min = req_type['munin_fields']['min']
    max = req_type['munin_fields']['max']
    print '%s.label %s' % (label, label)
    print '%s.min %d' % (label, min)
    print '%s.max %d' % (label, max)

def main():
  """
  calculate average response time per url patterns from last n requests

  """
  pipes = []
  try:
    pipes = create_pipelines()
    for line in pipes[-1].readlines():
      line_splited = line.split(LOG_DELIM)
      if len(line_splited) < URL_INDEX + 1: continue
      if len(line_splited) < RESPONSE_TIME_INDEX + 1: continue

      url = line_splited[URL_INDEX].strip()
      response_time = line_splited[RESPONSE_TIME_INDEX].strip()
      if not re.match(r'^\d+$', response_time): continue
      response_time = int(response_time)

      for req_type in REQUEST_TYPES:
        if re.search(req_type['url_pattern'], url):
          req_type['response_time_total'] += response_time
          req_type['call_count'] += 1
          break
  finally:
    for pipe in pipes: pipe.close()

  for req_type in REQUEST_TYPES:
    label = req_type['munin_fields']['label']
    response_time_total = req_type['response_time_total']
    call_count = req_type['call_count']
    if call_count > 0:
      print '%s.value %d' % (label, response_time_total / call_count)
    else:
      print '%s.value U' % (label)

def create_pipelines():
  pipes = []
  for command in LOG_OPEN_COMMANDS:
    if len(pipes) == 0:
      pipes.append(Popen(list(command), stdout=PIPE).stdout)
    else:
      pipe_before = pipes.pop()
      pipe_next = (Popen(list(command), stdin=pipe_before, stdout=PIPE).stdout)
      pipes.extend([pipe_before, pipe_next])
  if len(pipes) > 1:
    for pipe in pipes[0:-1]: pipe.close()
  return pipes

argv = sys.argv
if len(argv) > 1:
  if argv[1] == 'autoconf':
    autoconf()
    sys.exit()
  elif argv[1] == 'config':
    config()
    sys.exit()

main()