parse_rspec_json.rb

KNOWN_EXAMPLE_FILE_PATH_LN = [
  './spec/services/form1010cg/auditor_spec.rb:301',
  './spec/lib/common/exceptions/detailed_schema_errors_spec.rb:223',
  './spec/jobs/va_notify_email_job_spec.rb:18',
  './spec/controllers/v0/caregivers_assistance_claims_controller_spec.rb:49',
  './spec/controllers/v0/caregivers_assistance_claims_controller_spec.rb:25',
  './spec/requests/v0/form1010cg/attachments_request_spec.rb:74',
  './spec/requests/v0/form1010cg/attachments_request_spec.rb:49',
  './spec/controllers/inherited_proofing_controller_spec.rb:195',
  './spec/controllers/v1/sessions_controller_spec.rb:421',
  './spec/controllers/v1/sessions_controller_spec.rb:427',
  './spec/controllers/v1/sessions_controller_spec.rb:447',
  './spec/controllers/v1/sessions_controller_spec.rb:393',
  './spec/controllers/v1/sessions_controller_spec.rb:1008',
  './spec/controllers/v1/sessions_controller_spec.rb:1035',
  './spec/jobs/decision_review/submit_upload_spec.rb:31',
  './spec/requests/swagger_spec.rb:3489'
]

require 'json'

json_file = File.read 'rspec-errors.json'
json_data = JSON.parse json_file

error_types = {}
unknown_examples = []
known_examples = []

error_examples = json_data['examples'].select { |example| example['status'] == 'failed' }

error_examples.each do |example|
  message = example['exception']['message']

  if message.include?('wrong number of arguments') || message.include?('ArgumentError')
    type = 'wrong number of arguments'
    error_types[type] ||= 0
    error_types[type] += 1
  else
    file_path_ln = [example['file_path'], example['line_number']].join(':')
    if KNOWN_EXAMPLE_FILE_PATH_LN.include? file_path_ln
      known_examples << file_path_ln
      error_types['known'] ||= 0
      error_types['known'] += 1
    else
      unknown_examples << file_path_ln
      error_types['unknown'] ||= 0
      error_types['unknown'] += 1
    end
  end
end

puts error_types
puts
puts unknown_examples

Run rspec via rspec -f json -o rspec-errors.json and then run this file to get a list of error types. You'll need to define them yourself by modifying this gist with if conditionals.