process_logs.rb

#!/usr/bin/env ruby

# Output:
# TSV
# hits_date.tsv
# date, count, status, host, path

dates = [
  "2013-10-19",
  "2013-10-20",
  "2013-10-21",
  "2013-10-22",
  "2013-10-23",
  "2013-10-24",
  "2013-10-25",
  "2013-10-26",
  "2013-10-27",
  "2013-10-28",
  "2013-10-29",
  "2013-10-30",
  "2013-10-31",
  "2013-11-01",
  "2013-11-02",
  "2013-11-03",
  "2013-11-04",
  "2013-11-05",
  "2013-11-06",
  "2013-11-07",
  "2013-11-08",
  "2013-11-09",
  "2013-11-10",
  "2013-11-11",
  "2013-11-12"
]

dates.each do |date|

  FILENAME = "#{date}-fastly.redirector.log"

  puts FILENAME

  logs = File.open(FILENAME).read.split("\n")

  processed = {}

  logs.each do |log|
    parts = log.split(" ")

    if parts.length < 6
      next
    end

    host = parts[3].strip
    status = parts[5].strip
    path = parts[4].strip

    processed[host] = {} if not processed[host]
    processed[host][status] = {} if not processed[host][status]
    processed[host][status][path] = 0 if not processed[host][status][path]

    processed[host][status][path] += 1
  end

  string = ""

  processed.each do |host, meta|
    meta.each do |status, meta2|
      meta2.each do |path, count|
        if status.length != 3
          puts "    Bad line: #{count} #{status} #{host} #{path}"
          next
        end
        if count >= 10
          string += "#{date}\t#{count}\t#{status}\t#{host}\t#{path}\n"
        end
      end
    end
  end

  out = File.open("hits_" + FILENAME, "w")
  out.write(string)
  out.close

  # Sort file
  `sort -n -k2 -r hits_#{FILENAME} -o hits_#{FILENAME}`

  # Prepend header row
  File.open("new.hits_" + FILENAME, 'w') do |fo|
    fo.puts "date\tcount\tstatus\thost\tpath\n"
    File.foreach("hits_" + FILENAME) do |li|
      fo.puts li
    end
  end
  File.rename("new.hits_"+FILENAME, "hits_"+FILENAME)

end

separate_days.sh

awk '/^[^#]/ && /^2013-11-12/ { print $0 }' fastly.redirector.log > 2013-11-12-fastly.redirector.log