failed-test-finder.rb

require 'json'
require 'net/http'
require 'optparse'
require 'set'
require 'uri'

options = {}
OptionParser.new do |opt|
  opt.on('-s', '--start BUILD_NUMBER', 'Require start') { |o| options[:start] = o }
  opt.on('-e', '--end BUILD_NUMBER', 'Require end') { |o| options[:end] = o }
end.parse!

puts "Will crawl builds from #{options[:start]} to #{options[:end]}"
all_failed_tests = []
(options[:start]..options[:end]).each do |job_number|
  if job_number.to_i % 10 == 0
    puts "job_number: #{job_number}"
  end
  base_job_uri = "https://build.ci.opensearch.org/job/gradle-check/#{job_number}"
  result = JSON.parse(Net::HTTP.get_response(URI.parse(base_job_uri + '/api/json')).body)['result']

  if result == 'FAILURE'
    uri = URI.parse(base_job_uri + '/testReport/api/json?tree=suites[cases[status,className,name]]')
    begin
      json = JSON.parse(Net::HTTP.get_response(uri).body)
    rescue JSON::ParserError => e
      next
    end
    # 'FAILED' means the test failed, just like the previous run.
    # 'REGRESSION' means the test failed, but previously passed.
    # See https://javadoc.jenkins.io/plugin/junit/hudson/tasks/junit/CaseResult.Status.html
    failed_cases = json['suites'].map do |s|
      s['cases'].select do |c|
        c['status'] == 'REGRESSION' || c['status'] == 'FAILED'
      end
    end.flatten
    failed_tests = failed_cases.map { |c| {'name' => "#{c['className']}.#{c['name']}", 'build' => job_number}}
    all_failed_tests.push(failed_tests)
  end
end

puts '------------------'

count = {}
all_failed_tests.flatten.each do |test|
  unless count.include?(test['name'])
    count[test['name']] = {'count' => 0, 'builds' => []}
  end
  count[test['name']]['count'] += 1
  count[test['name']]['builds'].push(test['build'])
end
count.to_a.sort {|a,b| b[1]['count'] <=> a[1]['count'] }.each {|a| puts "#{a[1]['count']} #{a[0]} (#{a[1]['builds'].join(',')})" }