Script to measure and compare speed of running tests in parallel versus sequentially

measure.rb

require "benchmark"

# https://github.com/rails/rails/pull/42761
class Execution
  TEST_FILE_PATH = "test/threshold_test.rb"
  SINGLE_TEST_DURATION = 0.1

  attr_reader :threshold, :parallel

  def initialize(threshold, parallel: false)
    @threshold = threshold
    @parallel = parallel
  end

  def time_in_seconds
    write_test_file
    configure_threshold
    stop_spring

    Benchmark.measure { run_tests }.real
  end

  private
    def write_test_file
      File.write(TEST_FILE_PATH, test_file_body)
    end

    def test_file_body
      test_methods = threshold.times.collect do |i|
        <<~ruby
          test "threshold test #{i}" do
            sleep #{SINGLE_TEST_DURATION}
          end
        ruby
      end.join("\n\n")

      <<~ruby
        require "test_helper"
        
        class ThresholdTest < ActiveSupport::TestCase
          #{test_methods}
        end
      ruby
    end

    def configure_threshold
      test_file_content = File.read(test_env_file_path)

      test_threshold = if parallel
        0
      else
        threshold + 1
      end

      test_file_content.gsub! /config\.active_support\.test_parallelization_minimum_number_of_tests = \d+/,
        "config.active_support.test_parallelization_minimum_number_of_tests = #{test_threshold}"

      File.write(test_env_file_path, test_file_content)
    end

    def stop_spring
      system "spring stop"
    end

    def test_env_file_path
      "config/environments/test.rb"
    end

    def run_tests
      puts "*" * 100
      puts "Running with threshold = #{threshold}, parallel = #{parallel}"
      puts "*" * 100

      system "touch tmp/restart.txt"
      system run_tests_command
    end

    def run_tests_command
      "rails test #{TEST_FILE_PATH}"
    end
end

(75..).find do |threshold|
  time_in_parallel = Execution.new(threshold, parallel: true).time_in_seconds
  time_in_sequential = Execution.new(threshold, parallel: false).time_in_seconds
  
  puts "\n\n\tTime_in_parallel = #{time_in_parallel}, time_in_sequential = #{time_in_sequential}\n\n"
  
  if time_in_parallel < time_in_sequential
    puts "Threshold = #{threshold}!"
    exit 0
  end
end