Little script to simulate A/B test payloads

simulate_payloads.rb

experiments = [1]
# 0 control
# 1 experiment
probabilities = {1 => [0.01, 0.011]}

$goal_reservation={1 => [0,0]}
$belong_to_branch={1 => [0,0]}

def abFunction(conversions, sampleSize)
  p = conversions/sampleSize.to_f
  q = 1.0 - p
  se = Math.sqrt(p*q/sampleSize.to_f)
  return [p, se]
end

def progress(index)
  meane,see = abFunction($goal_reservation[1][1], $belong_to_branch[1][1])
  meanc,sec = abFunction($goal_reservation[1][0], $belong_to_branch[1][0])
  puts "%s\t%s\t%s\t%s\t%s" % [index, meane, see*1.645, meanc, sec*1.645]
end

simulation = File.open("./simulation.tsv", "w")
1.upto(1_000_000) do |user|
  visitor_id = SecureRandom.uuid
  toss_belongs_to_experiment = rand >= 0.5
  if toss_belongs_to_experiment
    $belong_to_branch[1][1] += 1
    Analytics.track(
      user_id: 0,
      anonymous_id: visitor_id,
      event: 'track_experiment',
      properties: { experiment_name: 'web::login_flow', variant: 1  }
    )
    if (rand <= probabilities[1][1])
      $goal_reservation[1][1] += 1
      Analytics.track(
        user_id: 0,
        anonymous_id: visitor_id,
        event: 'track_goal',
        properties: { goal_name: 'reservation' }
      )
    end
  else
    $belong_to_branch[1][0] += 1
    Analytics.track(
      user_id: 0,
      anonymous_id: visitor_id,
      event: 'track_experiment',
      properties: { experiment_name: 'web::login_flow', variant: 0  }
    )
    if (rand <= probabilities[1][0])
      $goal_reservation[1][0] += 1
      Analytics.track(
        user_id: 0,
        anonymous_id: visitor_id,
        event: 'track_goal',
        properties: { goal_name: 'reservation' }
      )
    end
  end
  #progress(user) if user%1000 == 0
  if (user%1000 == 0)
    meane,see = abFunction($goal_reservation[1][1], $belong_to_branch[1][1])
    meanc,sec = abFunction($goal_reservation[1][0], $belong_to_branch[1][0])
    simulation.write("%s\t%s\t%s\t%s\t%s\n" % [user, meane, see*1.645, meanc, sec*1.645])
  end
end

simulation.close