CloudWatch LogsのGroupから一括でLogを取得するスクリプト

Readme.md

使い方

$ bundle exec ruby test.rb --help
Options:
     --group-name, -g <s>:   log group name
     --start-time, -s <s>:   log start time
       --end-time, -e <s>:   log end time
  --output-format, -o <s>:   output format [tsv,csv] (default: tsv)
               --help, -h:   Show this message

$ bundle exec ruby test.rb --group-name /var/log/httpd/access_log --start-time "2014/07/31 05:00:00 +09:00"
i-123456789	2014-07-31 01:59:40 +0000	2014-07-31 01:59:52 +0000	"port:80	x-forwarded-for:""-""	host:127.0.0.1	ident:-	user:-	time:[31/Jul/2014:01:59:40 +0000]	request-first-line:""GET / HTTP/1.0""	status:200	response-size:-	refer:""-""	user-agent:""ApacheBench/2.3""	request-time:96	server-ip:127.0.0.1"

Gemfile

# A sample Gemfile
source "https://rubygems.org"

gem 'trollop'
gem "json"
gem "aws-sdk-core"

get_cw_log_by_group.rb

require 'aws-sdk-core'
require 'trollop'

output_formats = ['tsv', 'csv']
opts = Trollop::options do
  opt :group_name, 'log group name', type: :string, require: true
  opt :start_time, 'log start time', type: :string
  opt :end_time, 'log end time', type: :string
  opt :output_format, "output format [#{output_formats.join(",")}]", type: :string, default: output_formats.first
end

module CloudWatchLogs
  def self.get_group_log(group_name, options = {})
    cloudwatchlogs = Aws::CloudWatchLogs::Client.new(region: 'us-east-1')
    res = cloudwatchlogs.describe_log_streams(log_group_name: group_name)
    log_streams = res.log_streams.map { |x| x.log_stream_name }
    events = []
    log_streams.each do |log_stream|
      res = nil
      base_request = { log_group_name: group_name, log_stream_name: log_stream }
      base_request.merge!(start_time: time_to_i(options[:start_time])) if options[:start_time]
      base_request.merge!(end_time: time_to_i(options[:end_time])) if options[:end_time]
      begin
        if res
          events += res.events.map do |x|
            Struct.new(:log_stream,
                       :timestamp,
                       :ingestion_time,
                       :message).new(log_stream,
                                     i_to_time(x.timestamp),
                                     i_to_time(x.ingestion_time),
                                     x.message)
          end
        end
        request = res ? base_request.merge(next_token: res.next_forward_token) : base_request
        res = cloudwatchlogs.get_log_events(request)
      end while !res.events.empty?
    end
    events.sort { |x, y| x.timestamp <=> y.timestamp }
  end

  def self.time_to_i(time)
    time = Time.parse(time) if time.class == String
    time.to_i * 1000 + (time.usec / 1000.0).round
  end

  def self.i_to_time(i)
    Time.at(i / 1000, (i % 1000) * 1000)
  end
end

logs = CloudWatchLogs.get_group_log(opts[:group_name], opts)
case opts[:output_format]
when "tsv"
  require 'csv'
  logs.each do |log|
    puts CSV.generate_line(log.to_h.values, col_sep: "\t")
  end
when "csv"
  require 'csv'
  logs.each do |log|
    puts log.to_h.values.to_csv
  end
end