stackdriver log converts to goaccess format

_stackdriver-log-to-goaccess.md

1. Use gcloud logging read to pull logs
2. Convert it to csv style via json2csv.rb

ruby json2csv.rb

3. Use goaccess to generate html output

goaccess -p goaccess.format -f output.csv -o output.html --hour-spec=min

goaccess.format

log-format %dT%t+%^,%h,%v,%m,%U,%L,%s
date-format %Y-%m-%d
time-format %T

json2csv.rb

require 'json'

class JsonFileReader

  def initialize(file_path, &obj_callback)
    @file         = File.open(file_path)
    @obj_callback = obj_callback
  end

  def run
    brace = 0
    buffer = []
    read_size = 0
    total_size = @file.size
    prev_progress = 0

    while true
      c = @file.readchar
      read_size += 1

      progress = ((read_size * 100.0)/ total_size).to_i
      if prev_progress != progress
        prev_progress = progress
        puts "progress: #{progress}%"
      end

      # only ignore the outmost []
      next if ['[', ']'].include?(c) && brace == 0
      next if c == ',' && brace == 0

      buffer << c

      case c
      when '{' then brace += 1
      when '}'
        brace -= 1
        if brace == 0
          str = buffer.join
          @obj_callback.call(JSON.parse(str))
          buffer = []
        end
      end
    end
  rescue EOFError
  ensure
    @file.close
  end
end

out_path = "output.csv"
File.open(out_path, "w") do |f|
  jr = JsonFileReader.new('server.log') do |o|
    o = o['jsonPayload']

    buf = [
      o['timestamp'],
      o['remote_ip'],
      o['host'],
      o['method'],
      o['path'],
      o['duration'],
      o['status']
    ].join(',') + "\n"

    f.write(buf)
  end
  jr.run
end

puts "fini: ", out_path