gistfile1.txt

#!/usr/bin/env ruby

require 'pp'

urls = {}
users = {}

explained = {
  "/admin_additions/recommend_more_items" => {
    function: "styling",
    ok_for_now: true,
  },
  "/admin_additions/shipment_add_item/" => {
    function: "styling",
    ok_for_now: true,
  },
  "/admin_additions/shipment_remove_item/" => {
    function: "styling",
    ok_for_now: true,
  },
  "/admin_additions/inventory/" => {
    function: "inbound",
    ok_for_now: true,
  },
  "/admin_additions/sku_and_item_json/" => {
    function: "inbound",
    ok_for_now: true,
  },
  "/admin_additions/shipment_mark_packed/" => {
    function: "styling",
    ok_for_now: true,
  },
  "/admin_additions/save_inventory/" => {
    function: "inbound",
    ok_for_now: true,
  },
  "/admin_additions/shipment_mark_received/" => {
    function: "returns",
    ok_for_now: false,
  },
  "/admin_additions/shipment_print_labels/" => {
    function: "labels",
    ok_for_now: false,
  },
  "/admin_additions/shipment_change_item_statuses/" => {
    function: "unknown",
    ok_for_now: false,
  },
  "/admin_additions/shipment_mark_shipped/" => {
    function: "labels",
    ok_for_now: false,
  },
  "/admin_additions/style_attributes/" => {
    function: "unknown",
    ok_for_now: false,
  },
  "/admin_additions/shipment_mark_processed/" => {
    function: "returns",
    ok_for_now: false,
  },
  "/admin_additions/shipment_mark_cancelled/" => {
    function: "unknown",
    ok_for_now: false,
  },
  "/admin_additions/shipment_add_item_by_id" => {
    function: "styling",
    ok_for_now: true,
  },
  "/admin_additions/rate_shipment/" => {
    function: "styling",
    ok_for_now: true,
  },
  "/admin_additions/shipment_print_packingslip/" => {
    function: "pick_slip",
    ok_for_now: false,
  },
}

STDIN.each do |line|
  next if line =~ /Data collector is not contactable/
  next if line =~ /Data collector is not unavailable/
  next if line =~ /New Relic Python Agent/i
  parts = line.chomp.split(/\s+/)
  element = parts[8]
  if element != 'heroku/router'
    who,url,method = if parts[14] == 'GET'
                       [nil,parts[17],:get]
                     elsif parts[15] == 'GET'
                       [parts[11],parts[18],:get]
                     elsif parts[15] == 'POST'
                       [parts[11],parts[18],:post]
                     elsif parts[9] == 'Stopping' || parts[13] == "Worker" || parts[13] == "Handling" || parts[16] = "Relic"
                       :ignore
                     else
                       readable = ""
                       parts.each_with_index { |part,index|
                         readable += "[#{index}]: #{part}\n"
                       }
                       raise "Can't parse line:\n#{readable}"
                     end
    #if url =~ /(\/admin\/core\/[^\/]+)\/[\d]+\/?$/
    #  url = $1
    #end
    #if url =~ /(\/admin\/core\/[^\/]+)\/[\d]+\/history\/?$/
    #  url = $1
    #end
    #if url =~ /(\/admin_additions\/[^\/]+)\/[\d]+\/?$/
    #  url = $1
    #end
    wackies = []
    unless url == :ignore || url.nil? || url == '/' || url == '/admin/jsi18n/' || url == '/favicon.ico' || url =~ /\/ajax_select\//
      original_url = url
      url = url.gsub(/\/shipment\/\d+\./,'/shipment/').gsub(/\d+/,'').gsub(/\/\/+/,'/')
      if url =~/\/\.\//
        wackies << original_url
      end
      urls[method] ||= {}
      urls[method][url] ||= 0
      urls[method][url] += 1
      users[url] ||= {}
      users[url][who] ||= 0
      users[url][who] += 1
    end
    raise wackies.join("\n") if wackies.size > 0
  end
end
(
  (urls[:get].to_a.map { |(url,count)| [url,count,:get] }) +
  (urls[:post].to_a.map { |(url,count)| [url,count,:post] })
).sort { |a,b| a[1] <=> b[1] }.map { |(url,count,method)| [count,url,method] }.reverse.each do |(count,url,method)|
  if (explained[url] || {})[:ok_for_now]
  else
    printf("%5d : %5s %s\n",count,method.to_s.upcase,url)
  end
end

# 2013-06-23 16:55:31  a71944c2f14934044b51bceb651e59e8  1                                INFO GET  Request to      /  
# 9          10        11                                12                               13   14   15      16      17 18
# 2013-06-23 15:50:43  slee                              2d7ab7fad98d1dfea3e853c15aba094e 2547 INFO POST    Request to /admin/core/shipment/83941/ POST Params: {comments:,_save:Save,tax_collected:,fun:,internal_rating:,fix_num:,experience_was_delightful:,stylist_notes:,experience_was_fun:,got_my_style_right:,experience_was_convenient:,style:,request_notes:,delightful:,keep_all_email_date_1:,keep_all_email_date_0:,csrfmiddlewaretoken:292132a5393d76a62225f19bde097f9e,outbound_tracking_service:,convenient:,outbound_tracking:,client_text:,internal_comments:,algorithm_version:,shipment_id:83941,client:3119512,inbound_tracking:,status:7,requested_date:2013-06-27,got_my_size_right:,inbound_tracking_service:,shipment_status:Cancelled,sf_received_date_1:,sf_received_date_0:,size:,reminder_email_date_0:,reminder_email_date_1:,ship_date_1:,ship_date_0:,client_received_date_1:,client_received_date_0:}
#