Backup old papertrail archive to s3

papertrail-to-s3.rb

require "active_support/all"
require "aws-sdk"
require "concurrent"

papertrail_token = "..."

s3 = Aws::S3::Client.new(region: "us-east-1")
s3_bucket = "..."
s3_prefix = "papertrail/logs/[account-id]"

pool = Concurrent::FixedThreadPool.new(8)

(365 * 24).times.each do |i|
  pool.post do
    date = i.hours.ago
    datename = date.strftime("%Y-%m-%d")
    hourname = date.strftime("%Y-%m-%d-%H")
    filename = "#{hourname}.tsv.gz"
    papertrail_url = "https://papertrailapp.com/api/v1/archives/#{hourname}/download"
    s3_key = "#{s3_prefix}/dt=#{datename}/#{filename}"
    #s3_url = "s3://#{s3_bucket}/#{s3_key}"

    begin
      s3.head_object(bucket: s3_bucket, key: s3_key)
      puts "#{hourname}: already on s3"
    rescue Aws::S3::Errors::NotFound
      puts "#{hourname}: downloading from papertrail"
      system "wget", "--quiet",
                     "--header", "X-Papertrail-Token: #{papertrail_token}",
                     "--output-document", filename,
                     "--continue",
                     papertrail_url
      unless $?.success?
        puts "#{hourname}: no such archive, skipping"
        next
      end

      puts "#{hourname}: uploading to s3"
      File.open(filename, "rb") do |file|
        s3.put_object(bucket: s3_bucket, key: s3_key, body: file)
      end
    end

    if File.exists?(filename)
      puts "#{hourname}: removing local file"
      File.unlink(filename)
    end
  end
end

pool.wait_for_termination