fetch td-agent log.

fetch_log.rb

require 'rubygems'
require 'pp'
require 'json'

def init
  `mkdir -p ./fetch_log_tmp`
end

def finalize 
  `rm -f ./fetch_log_tmp/*`
end

def ungzip(filename, pattern = '')
  pattern = "| grep #{pattern}" unless pattern.empty?
  `gzip -dc #{filename} #{pattern} > ./fetch_log_tmp/#{ungzipedname(filename)} ` 
end

def parse(filename)
  parsed_log_file = File::open("./#{parsedname(filename)}", "a") 
  File::open("./fetch_log_tmp/#{ungzipedname(filename)}").each do |line|
    parsed_log    = line.split("\t")

    parsed_log[2] = JSON.parse(parsed_log[2])
    parsed        = "#{parsed_log[0]}\t#{parsed_log[2]['_hostname']}\t#{parsed_log[2]['message']}\n"
    parsed_log_file.puts(parsed)
  end
  parsed_log_file.close
end

def gzip(filename)
  `gzip #{parsedname(filename)}`
end

def ungzipedname(filename)
  filename.gsub(/.gz$/, "")
end

def parsedname(filename)
  filename.gsub(/.gz$/, ".parsed")
end

pattern   = ARGV.shift
logfiles  = ARGV

init

logfiles.each do |filename|
  ungzip(filename, pattern)
  parse(filename)
  gzip(filename)
end

finalize