Loghugger

loghugger.rb

require 'google/api_client'
require 'json'

@configs = {
  "development" => {
    project_id: 'squar-staging',
    dataset: 'logs_staging',
    table_id: 'squar_server_development',
    log_file: 'log/development.log',
    service_account: '87068494222-0cggkqe51418os2nvsi8gqut67r52mr3@developer.gserviceaccount.com',
    key: 'private.p12'
  },
  "beta" => {
    project_id: 'squar-staging',
    dataset: 'logs_staging',
    table_id: 'squar_server',
    log_file: 'log/unicorn.stdout.log',
    service_account: '87068494222-0cggkqe51418os2nvsi8gqut67r52mr3@developer.gserviceaccount.com',
    key: 'private.p12'
  },
  "production" => {
    project_id: 'acquired-vector-747',
    dataset: 'logs',
    table_id: 'squar_server',
    log_file: 'log/unicorn.stdout.log',
    service_account: '557460464813-68nq6c0lu3g3t23kns9nct1h3l7vkr6v@developer.gserviceaccount.com',
    key: 'private-production.p12'
  }
}

def config
  @config ||= @configs[ENV['ENV']]
end

client = Google::APIClient.new(
  :application_name => 'RAILS LOG',
  :application_version => '1.0.0'
)
key = Google::APIClient::KeyUtils.load_from_pkcs12(config[:key], 'notasecret')
client.authorization = Signet::OAuth2::Client.new(
  :token_credential_uri => 'https://accounts.google.com/o/oauth2/token',
  :audience => 'https://accounts.google.com/o/oauth2/token',
  :scope => 'https://www.googleapis.com/auth/bigquery',
  :issuer => config[:service_account],
  :signing_key => key)
client.authorization.fetch_access_token!

bigquery = client.discovered_api('bigquery', 'v2')

exit_requested = false
Kernel.trap( "INT" ) { exit_requested = true }
log_file = File.open(config[:log_file])
log_file.seek(0, IO::SEEK_END)
while !exit_requested
  rows = []
  while (line = log_file.readline rescue nil)
    if line
      begin
        row = JSON::parse(line)
      rescue => e
        # puts "Cannot parse #{line}, error #{e.inspect}"
        row = nil
      end
    end

    rows << {"json" => row} if row
  end
  unless rows.empty?
    result = client.execute(:api_method => bigquery.tabledata.insert_all,
                            :parameters => {datasetId: config[:dataset], projectId: config[:project_id], tableId: config[:table_id]},
                            :body_object => { "rows" => rows})
    puts result.inspect
  end
  sleep 1
end