Helper to provide alternate configuration syntax for Split, the A/B testing tool.

a_b_helper.rb

module ABHelper
  def split_test(name)
    experiment = YAML.load_file("config/experiments.yml")[name]

    given_probability, num_with_probability = experiment[:variants].inject([0,0]) do |a,v|
      p, n = a
      if v.kind_of?(Hash) && v[:percent]
        [p + v[:percent], n + 1]
      else
        a
      end
    end


    num_without_probability = experiment[:variants].length - num_with_probability
    unassigned_probability = ((100.0 - given_probability) / num_without_probability / 100.0)

    if num_with_probability.nonzero?
      experiment[:variants] = experiment[:variants].map do |v|
        if v.kind_of?(Hash) && v[:name] && v[:percent]
          { v[:name] => v[:percent] / 100.0 }
        elsif v.kind_of?(Hash) && v[:name]
          { v[:name] => unassigned_probability }
        else
          { v => unassigned_probability }
        end
      end
      ab_test name, experiment[:variants].shift, experiment[:variants].inject({}, :merge)
    else
      ab_test name, *experiment[:variants]
    end

  end

  def track_metric(name, opts={})
    YAML.load_file("config/experiments.yml").each do |experiment_name,experiment|
      if experiment[:metric] == name

        opts = {}
        unless experiment[:resettable].nil?
          opts[:reset] = experiment[:resettable]
        end

        finished(experiment_name, opts)
      end
    end
  end
end

a_b_helper_spec.rb

require 'spec_helper'

describe ABHelper do
  def config
    # This is wacky, but I like the syntax
    Class.new do
      def []=(name, hash)
        YAML.stub(:load_file).and_return({name => hash})
      end
    end.new
  end

  def config_full(hash)
    YAML.stub(:load_file).and_return(hash)
  end

  context "begin test" do
    it "pulls options from config file" do
      config[:my_experiment] = {
        :variants => [ "control_opt", "other_opt" ],
      }
      should_receive(:ab_test).with(:my_experiment, "control_opt", "other_opt")
      split_test :my_experiment
    end

    it "accepts multiple variants" do
      config[:my_experiment] = {
        :variants => [ "control_opt", "second_opt", "third_opt" ],
      }
      should_receive(:ab_test).with(:my_experiment, "control_opt", "second_opt", "third_opt")
      split_test :my_experiment
    end

    it "accepts probability on variants" do
      config[:my_experiment] = {
        :variants => [
          { :name => "control_opt", :percent => 67 },
          { :name => "second_opt", :percent => 10 },
          { :name => "third_opt", :percent => 23 },
        ],
      }
      should_receive(:ab_test).with(:my_experiment, {"control_opt" => 0.67}, {"second_opt" => 0.1, "third_opt" => 0.23})
      split_test :my_experiment
    end

    it "accepts probability on some variants" do
      config[:my_experiment] = {
        :variants => [
          { :name => "control_opt", :percent => 34 },
          "second_opt",
          { :name => "third_opt", :percent => 23 },
          "fourth_opt",
        ],
      }
      should_receive(:ab_test).with(:my_experiment, {"control_opt" => 0.34}, {"second_opt" => 0.215, "third_opt" => 0.23, "fourth_opt" => 0.215})
      split_test :my_experiment
    end

    it "allows name param without probability" do
      config[:my_experiment] = {
        :variants => [
          { :name => "control_opt" },
          "second_opt",
          { :name => "third_opt", :percent => 64 },
        ],
      }
      should_receive(:ab_test).with(:my_experiment, {"control_opt" => 0.18}, {"second_opt" => 0.18, "third_opt" => 0.64})
      split_test :my_experiment
    end
  end

  context "track metric" do
    it "completes the test" do
      config[:my_experiment] = {
        :variants => [ "control_opt", "other_opt" ],
        :metric => :my_metric
      }
      should_receive(:finished).with(:my_experiment, {})
      track_metric :my_metric
    end

    it "completes all relevant tests" do
      config_full({
        :exp_1 => {
          :variants => [ "1-1", "1-2" ],
          :metric => :my_metric
        },
        :exp_2 => {
          :variants => [ "2-1", "2-2" ],
          :metric => :another_metric
        },
        :exp_3 => {
          :variants => [ "3-1", "3-2" ],
          :metric => :my_metric
        },
      })
      should_receive(:finished).with(:exp_1, {})
      should_not_receive(:finished).with(:exp_2, {})
      should_receive(:finished).with(:exp_3, {})
      track_metric :my_metric
    end

    it "passes reset option" do
      config[:my_experiment] = {
        :variants => [ "control_opt", "other_opt" ],
        :metric => :my_metric,
        :resettable => false,
      }
      should_receive(:finished).with(:my_experiment, :reset => false)
      track_metric :my_metric
    end
  end
end

experiments.example.yml

:call_to_action_text:
  :variants:
    - "Create An Account"
    - "Get Started"
  :metric: :conversion
  :resettable: false

:default_home:
  :variants:
    - :name: "old_home"
    - :name: "new_home"
      :percent: 10
  :metric: :default_home_retention
  :resettable: false