gistfile1.rb

require "mongo"
require 'json'

connection = Mongo::Connection.new("localhost", 27017)
db = connection.db('log_201208')
col = db.collection('d_01')
col.create_index([['ts', Mongo::ASCENDING], ['pid', Mongo::DESCENDING]])
col.create_index('status')


ARGF.each_line do |line|
  next if line =~ /nagios/ or line =~ /stage/ or line =~ /soap/
  target_line = line.chomp!().gsub(/,\s/, ',').gsub(/,\t/, ',')

  begin
    list = target_line.split(/\t/)
    user_agent = list[14..list.length]
    if user_agent != nil
      user_agent = user_agent.join(',')
      next if user_agent =~ /bot/i
    end

    doc = {
        tld: list[1].gsub(/www\.trivago\./, ""),
        ip: list[2].split(','),
        ts: list[4].to_i,
        meth: list[5],
        f: list[6],
        st: list[8].to_i,
        ttr: list[10].to_i,
        pid: list[11],
        ref: list[12],
        ua: user_agent,
    }

    puts doc.to_json

  rescue
    next
  end
end