log parse

logparse.rb

# For turning heroku DB logs into a CSV 
# getting the logs from papertrail:
# you need your papertrail token available
# echo "token: xxxxxxxx" > ~/.papertrail.yml
# papertrail -S pg_load --min-time "30 days ago" > logs.txt
# where `pg_load` is a saved search which finds the postgres status logs 

txt = File.read("./logs.txt")
require 'csv'
CSV.open("logs.csv", "wb") do |csv|
  csv << ["active_connections", "waiting_connections", "load1m", "load5m", "load15m", "tmp_disk_used", "tmp_disk_avail", "mem_free"]
  txt.split("\n").each do |line|

    active_connections = line.match(/sample#active-connections=([0-9]+)/)[1].to_i
    waiting_connections = line.match(/sample#waiting-connections=([0-9]+)/)[1].to_i
    load1m = line.match(/sample#load-avg-1m=([0-9\.]+)/)[1].to_f
    load5m = line.match(/sample#load-avg-5m=([0-9\.]+)/)[1].to_f
    load15m = line.match(/sample#load-avg-15m=([0-9\.]+)/)[1].to_f
    tmp_disk_used = line.match(/sample#tmp-disk-used=([0-9]+)/)[1].to_i
    tmp_disk_avail = line.match(/sample#tmp-disk-available=([0-9]+)/)[1].to_i
    mem_free = line.match(/sample#memory-free=([0-9]+)kB/)[1].to_i
    csv << [active_connections, waiting_connections, load1m, load5m, load15m, tmp_disk_used,tmp_disk_avail, mem_free]
  end
end