http-top.rb

#!/usr/bin/env ruby

require 'file/tail'
require 'http_log_parser'
require 'time'

parser = HttpLogParser.new

UPDATE_INTERVAL = 10 #Sec
ALERT_INTERVAL = 120 #Sec
ALERT_THRESHOLD = 10 #RPS
SECTION_CNT = 5

def print_stats ( sections, hits, hps, total_hits, alerts=[] )
  system "clear"
  puts '### HTTP Stats ###'
  puts "Current Time: #{Time.now}"
  puts
  puts 'Top Page Sections:'
  sections.each { |x| puts "#{x[0]}: #{x[1]}" }
  puts
  puts "Hits: #{hits}"
  puts "H/s: #{hps}"
  puts "Total Hits: #{total_hits}"
  puts
  alerts.each { |x| puts x } unless alerts.empty?
end

events = {}

if ( ARGV.length != 1 )
  puts 'Usage: http-top.sh <logfile>'
  exit 1
end

# Ingestion Thread
thread1 = Thread.new do
  File::Tail::Logfile.open(ARGV.first) do |log|
    log.backward(0).tail do |line|
      parsed_data = parser.parse_line(line)
      /(?<section>(http:\/\/\w+\.?\w*)?(\/[\w0-9]*))/ =~ parsed_data[:request]
      time = Time.strptime(parsed_data[:datetime],
        "%d/%b/%Y:%H:%M:%S %z").to_i
      events[time] ||= []
      events[time] << section
    end
  end
end

# Reporting Thread
thread2 = Thread.new do
  total_hits = 0
  alerts = []
  alert_state = false
  hps_hist = []

  loop do
    sleep UPDATE_INTERVAL
    range = events.select { |key, value| Time.now.to_i - key <= UPDATE_INTERVAL }
    sections = {}
    hits, hps = 0

    if range.empty? && alert_state
      alerts << "Traffic back to normal, triggered at #{Time.now}"
      print_stats sections, hits, hps, total_hits, alerts
    else
      range.each_value do |value|
        value.each do |x|
          sections[x] ||= 0
          sections[x] += 1
          hits += 1
          total_hits += 1
        end
      end
      sections = sections.sort_by { |key, value| -value }.first SECTION_CNT
      hps = hits.to_f / UPDATE_INTERVAL

      hps_hist << hps
      iterations = ALERT_INTERVAL / UPDATE_INTERVAL
      if hps_hist.length >= iterations
        hps_total = 0
        hps_hist.last(iterations).each { |x| hps_total += x }
        hps_avg = hps_total / iterations.to_f
        # Only alert first crossing of threshold
        if ( !alert_state && hps_avg >= ALERT_THRESHOLD )
          alert_state = true
          alerts << "High traffic generated an alert - avg H/s = #{hps_avg.to_i}, triggered at #{Time.now}"
        elsif ( alert_state && hps_avg < ALERT_THRESHOLD )
          alert_state = false
          alerts << "Traffic back to normal, triggered at #{Time.now}"
        end
      end

      print_stats sections, hits, hps, total_hits, alerts

      sections = {}
      hits = 0
      hps = 0
    end
    # Keep events at a reasonable size
    events.select! { |key, value| Time.now.to_i - key <= ALERT_INTERVAL }
  end
end

thread1.join
thread2.join