CI downloader script

cidl.rb

#!/usr/bin/env ruby

require 'uri'
require 'optparse'

def process_gcp(dir)
  puts "processing gcp artifacts"
  `mv #{dir}/artifacts/e2e-gcp #{dir}/artifacts/must-gather` if Dir.exist?("#{dir}/artifacts/e2e-gcp")
  `mv #{dir}/artifacts/e2e-gcp-upgrade #{dir}/artifacts/must-gather` if Dir.exist?("#{dir}/artifacts/e2e-gcp-upgrade")
  `tar xf #{dir}/artifacts/must-gather/must-gather.tar -C #{dir}/artifacts/must-gather --strip-components 2`
  `find #{dir}/artifacts/must-gather/audit_logs -name '*.gz' -exec gunzip {} \\;`
end

def process_aws(dir)
  puts "processing aws artifacts"
  `tar xf #{dir}/artifacts/e2e-aws/gather-must-gather/must-gather.tar -C #{dir}/artifacts/e2e-aws/gather-must-gather --strip-components 2`
  `tar xf #{dir}/artifacts/e2e-aws/gather-audit-logs/audit-logs.tar -C #{dir}/artifacts/e2e-aws/gather-audit-logs --strip-components 2`
  `find #{dir}/artifacts/e2e-aws/gather-audit-logs -name '*.gz' -exec gunzip {} \\;`
end

def process_azure(dir)
  puts "processing azure artifacts"
  `tar xf #{dir}/artifacts/e2e-azure/must-gather.tar -C #{dir}/artifacts/e2e-azure --strip-components 2`
  `find #{dir} -name workers-journal -exec gunzip -f --suffix '' {} \\;`
  `find #{dir} -name masters-journal -exec gunzip -f --suffix '' {} \\;`
  `find #{dir}/artifacts/e2e-azure/audit_logs -name '*.gz' -exec gunzip {} \\;`
end

force = false

opts = {}
OptionParser.new do |parser|
  parser.on('-f', '--force') do
    force = true
  end
end.parse!

# This should be a prow URL, e.g.
# https://prow.svc.ci.openshift.org/view/gcs/origin-ci-test/logs/release-openshift-ocp-installer-e2e-azure-4.3/209
in_url = ARGV.pop

path = in_url.split("/")[5..-1].join("/")
url = URI.join("https://gcsweb-ci.svc.ci.openshift.org/gcs/", path)
name = [path.split("/")[-2..-1].join("_")].join("/")

ci_dir = File.join(ENV['HOME'], 'ci-runs')
global_cache_dir = File.join(ci_dir, '.cache')
cache_dir = File.join(global_cache_dir, name)
dir = File.join(ci_dir, name)

if Dir.exist?(dir)
  if force
    trash = File.join(`mktemp -d`.chomp, name)
    `mv #{dir} #{trash}`
    puts "moved existing ci run to trash #{trash}"
  else
    raise "dir already exists at #{dir}" if Dir.exist?(dir)
  end
end

if Dir.exist?(cache_dir)
  puts "using cached download from #{cache_dir}"
else
  `mkdir -p #{cache_dir}`
  puts "downloading #{name} to #{cache_dir}"
  puts `gsutil -q -m cp -r gs://#{path} #{cache_dir}`
  artifacts = File.dirname(`find #{cache_dir} -name e2e.log`.chomp).chomp
  unless File.exists?(artifacts)
    puts "couldn't find artifacts dir in #{cache_dir}, aborting"
    exit 1
  end
end

src_dir = File.dirname(`find #{cache_dir} -name started.json`).chomp
raise "couldn't find started.json in #{cache_dir}" unless Dir.exist?(src_dir)
`cp -R #{src_dir} #{dir}`
puts "created #{dir} from #{src_dir}"


if Dir.exist?(File.join(dir,  "artifacts/e2e-gcp")) or Dir.exist?(File.join(dir,  "artifacts/e2e-gcp-upgrade"))
  process_gcp(dir)
elsif Dir.exist?(File.join(dir, "artifacts/e2e-aws"))
  process_aws(dir)
elsif Dir.exist?(File.join(dir, "artifacts/e2e-azure"))
  process_azure(dir)
end

puts "browseable URL: #{url}"
puts "dir: #{dir}"

exec("code #{dir}") if fork.nil?