Proof of concept for an idiomatic RSpec generative testing library

generative-poc.rb

# TODO:
# 
# Allow specifying number of runs.
# Ensure child contexts get run multiple times.
# Check what happens on failure in one of the runs.
# Different color for dots.
# Shrinking.


module RandomlyGenerated
  def self.respond_to?(method_name)
    class_name = method_name.to_s.split("_").map(&:capitalize).join
    Object.const_defined?("::RandomlyGenerated::#{class_name}")
  end

  def self.method_missing(method_name, *args, &block)
    if respond_to?(method_name)
      class_name = method_name.to_s.split("_").map(&:capitalize).join
      class_const = Object.const_get("::RandomlyGenerated::#{class_name}")
      class_const.new(*args, &block)
    else
      super
    end
  end
end


class RandomlyGenerated::Object
  attr_reader :seed
  attr_reader :rand

  def initialize(options={})
    @seed = options.fetch(:seed) { Random.new_seed }
    @rand = Random.new(seed)
  end

  # Returns the generated object.
  def call
    raise NotImplementedError
  end

  # Returns an array of "shrunken" proper subsets of the object.
  # These subsets are intended to find simpler cases that will reproduce a test failure.
  def shrunken_subsets
    # By default, assume that the object is atomic and cannot be simplified.
    []
  end
end


# From https://gist.github.com/pithyless/9738125
class Integer
  N_BYTES = [42].pack('i').size
  N_BITS = N_BYTES * 16
  MAX = 2 ** (N_BITS - 2) - 1
  MIN = -MAX - 1
end

class RandomlyGenerated::Integer < RandomlyGenerated::Object
  attr_reader :minimum
  attr_reader :maximum

  def initialize(options={})
    super
    options[:range] ||= Integer::MIN..Integer::MAX
    @minimum = options.fetch(:minimum) { options.fetch(:range).first }
    @maximum = options.fetch(:maximum) { options.fetch(:range).last }
  end

  def call
    # TODO: We should weight this to make more "special edge-case" results show up -- like 0, 1, Integer::MAX, etc.
    @value ||= rand.rand(minimum..maximum)
  end
end


class RandomlyGenerated::String < RandomlyGenerated::Object
  attr_reader :length

  def initialize(options={})
    super
    @length = options.fetch(:length) { (1..5000) }
    @length = rand.rand(@length) if @length.is_a?(Range)
  end

  def call
    @value ||= rand.bytes(length) # TODO: This doesn't handle Unicode characters, only code points 0-255.
  end
end


require "rspec"


module RSpec::Generative
  NUMBER_OF_RUNS = 1000
end


module RSpec::Generative::Let
  def let(name, &block)
    # NOTE: This probably breaks the use of `return` and `super` in the `let` body.
    # See https://github.com/rspec/rspec-core/blob/master/lib/rspec/core/memoized_helpers.rb#L232-L233
    bigger_block = lambda {
                      result = block.call # TODO: Use `tap`.
                      if result.is_a?(RandomlyGenerated::Object)
                        self.class.metadata[:generative] = true
                        result.call
                      else
                        result
                      end
                   }
    super(name, &bigger_block)
  end
end


class RSpec::Core::ExampleGroup
  extend RSpec::Generative::Let
end


RSpec.configure do |config|
  config.around(:example) do |example|
    example.run
    if example.example_group.metadata[:generative]
      (RSpec::Generative::NUMBER_OF_RUNS - 1).times do
        example.run
      end
    end
  end
end



RSpec.describe "String#reverse" do

  let(:string) { "Craig" }

  specify "this should only run once" do
    puts "this should only run once"
  end

  # If a `describe` block contains a `let` that returns a `RandomlyGenerated::Object`,
  # then it gets run 1000 times instead of 1.
  describe "inner" do
    # If a `let` returns a `RandomlyGenerated::Object`, then it calls `call` on the value.
    # Otherwise it just returns the value, as a normal `let` would.
    # It also sets a flag for the immediately enclosing `describe` block to run 999 more times.
    let(:string) { RandomlyGenerated.string(length: 0..1000) }

    it "has the same length as the string" do
      expect(string.reverse.length).to eq(string.length)
    end

    it "can be round-tripped back to the string" do
      expect(string.reverse.reverse).to eq(string)
    end
  end

end