Ruby Testing Framework Benchmark

ruby_testing_framework_benchmark.rb

require "fileutils"
require "ostruct"

class Runner
  class << self
    attr_accessor :type

    def run(tests_per_file, max_number_of_files, method = nil)
      method ||= :total_time
      tupels = (max_number_of_files / 50).times.map do |i|
        50 * (i + 1)
      end
      puts "# running with #{tests_per_file} per file"
      print ([nil] + tupels).join("\t")
      [RSpecRunner, MinitestRunner, MinitestSpecRunner].each do |clazz|
        print "\n#{clazz.type}\t"
        $stdout.flush
        rows = [clazz.type]
        rows += tupels.each do |files|
          print "#{clazz.new.benchmark(files, tests_per_file).send(method)}\t"
          $stdout.flush
        end
      end
      puts "\n"
    end
  end

  def type
    self.class.type
  end

  def each_file(test_files)
    FileUtils.mkdir_p(type)
    test_files.times do |i|
      file_name = "#{type}/test_#{i.to_s(36)}_#{type.gsub("/", "_")}.rb"
      File.open(file_name, "w") do |f|
        yield(f, i)
      end
    end
  end

  def benchmark(test_files, tests_per_file)
    write_files(test_files, tests_per_file)
    started = Time.now
    result = run_tests
    finished_in = result[/Finished in ([\d+\.]+)/, 1].to_f
    total_time = Time.now - started
    cleanup!
    OpenStruct.new(:finished_in => finished_in, :total_time => total_time, :type => type)
  end

  def cleanup!
    FileUtils.rm_rf(type)
  end

  def run_tests
    `#{cmd} #{type}`
  end
end

class RSpecRunner < Runner
  self.type = "spec"

  def write_files(tests_per_file, test_files)
    each_file(test_files) do |f, i|
      f.puts(%(describe "test #{i}" do))
      tests_per_file.times do |j|
        f.puts(%(  it "runs test #{j}" do))
        f.puts(%(    #{j}.should == #{j}))
        f.puts(%(  end))
      end
      f.puts("end")
    end
  end

  def cmd
    "rspec"
  end
end

class MinitestRunner < Runner
  self.type = "minitest/unit"

  def write_files(tests_per_file, test_files)
    each_file(test_files) do |f, i|
      f.puts(%(require "minitest/autorun"))
      f.puts(%(class TestCase#{i} < MiniTest::Unit::TestCase))
      tests_per_file.times do |j|
        f.puts(%(  def test_#{j}))
        f.puts(%(    assert_equal #{j}, #{j}))
        f.puts(%(  end))
      end
      f.puts("end")
    end
  end

  def cmd
    "testrb"
  end

  def cleanup!
    super
    FileUtils.rmdir("minitest")
  end
end

class MinitestSpecRunner < MinitestRunner
  self.type = "minitest/spec"

  def write_files(tests_per_file, test_files)
    each_file(test_files) do |f, i|
      f.puts(%(require "minitest/autorun"))
      f.puts(%(describe "TestCase#{i}" do))
      tests_per_file.times do |j|
        f.puts(%(  it "runs test #{j}" do))
        f.puts(%(    #{j}.must_equal #{j}))
        f.puts(%(  end))
      end
      f.puts("end")
    end
  end
end

Runner.run(ARGV.at(0).to_i, ARGV.at(1).to_i, ARGV.at(2))