Dirty simple order dependencies detector

dep_detect.rb

## This script can be used to detect simple order dependencies if one of tests is failing when some other test(or tests, independently) is being run before it.
## Script will not give reliable results if the dependency is complex - e.g. two specific tests must be run before for affected one to fail.
## Script will refine results with each rspec run: more runs will result smaller UNSAFE lists. Files from previous script runs are used too

## Ensure that you have config.order = 'random' in RSpec.configure block .

## Running rspec with different seeds to collect data
RSPEC_RUN_COUNT = 5 #set to 0 to analyze already existing rspec_*.txt files

RSPEC_RUN_COUNT.times do
  `rspec --format progress --format documentation --out rspec.txt`

  seed = open('rspec.txt') {|f| f.grep(/^Randomized with seed/).first.gsub /[^0-9]/, '' }
  puts "seed: #{seed}"

  succeeded = open('rspec.txt') {|f| f.grep(/^Failed examples:/).first.nil? }
  puts "succeeded: #{succeeded}"

  `mv rspec.txt rspec_#{succeeded ? 'ok' : 'fail'}_#{seed}.txt`
end

## Parsing collected data
AFFECTED_TEST = 'Your affected test name'

LINES_TO_SKIP = 2 #See "First 5 lines" output to adjust
ENDLINES_TO_SKIP = 3  #See "Last 5 lines" output to adjust
EXCLUDES = ['Pending:', 'Failures:', AFFECTED_TEST] # exclude by line value

require 'set'

$excluded_lines = Set.new

def extract_test_names(fname)

  open(fname) do |f|
    lines = f.grep(/^[A-Z]/).map{|line| line.gsub "\n", ''}
    if LINES_TO_SKIP > 0
      $excluded_lines += lines[0..LINES_TO_SKIP-1]
    end
    if ENDLINES_TO_SKIP > 0
      $excluded_lines += lines[lines.length-ENDLINES_TO_SKIP..-1]
    end
    lines[LINES_TO_SKIP..-1-ENDLINES_TO_SKIP]
  end
end

safe_tests = Set.new
probably_safe_tests = Set.new
unsafe_tests = nil

Dir.foreach(".") do |fname|
  next unless  /rspec_.*\.txt/ =~ fname
  puts "Analyzing file #{fname}"
  test_names = extract_test_names fname
  puts
  puts "First 5 lines:"
  puts test_names[0..5]
  puts
  puts "Last 5 lines:"
  puts test_names[-5..-1]
  test_index = test_names.index AFFECTED_TEST

  if test_index.nil?
    puts
    puts "Could not find \"#{ AFFECTED_TEST}\" on the list, select one from the list below and set it in AFFECTED_TEST const, then rerun script: "
    puts test_names
    exit
  end

  tests_before_affected = Set.new(test_index > 0 ? test_names[0..test_index-1] : [])
  tests_after_affected = Set.new(test_index+1 < test_names.length ? test_names[test_index+1..-1] : [])

  unsafe_tests ||= Set.new test_names

  if /rspec_fail.*\.txt/ =~ fname #failed
    probably_safe_tests += tests_after_affected
  else #passed
    safe_tests += tests_before_affected
  end

end

safe_tests -= EXCLUDES

unsafe_tests -= safe_tests
unsafe_tests -= EXCLUDES

probably_safe_tests -= safe_tests
probably_safe_tests -= EXCLUDES

puts
puts
puts "RESULTS:"
puts
puts
puts 'SAFE TESTS (In successful runs at least once were run before affected test):'
puts safe_tests.to_a.join("\n")
puts
puts 'UNSAFE TESTS (All tests except SAFE):'
puts unsafe_tests.to_a.join("\n")
puts
puts 'PROBABLY SAFE TESTS (In failed runs at least once were run after affected test except the SAFE ones listed above):'
puts probably_safe_tests.to_a.join("\n")
puts
puts 'MOST UNSAFE TESTS (All tests except SAFE and PROBABLY SAFE):'
puts (unsafe_tests-probably_safe_tests).to_a.join("\n")
puts
puts
puts 'SKIPPED LINES (Check that there are no test names and modify skip settings if there are):'
puts $excluded_lines.to_a