eustin

for i, rate in enumerate(simulated_diffs_in_rates[:10]):
    print(f"simulated diff in rates {i+1}: {rate:.1%}")
    
#> simulated diff in rates 1: -50.0%
#> simulated diff in rates 2: -16.7%
#> simulated diff in rates 3: 16.7%
#> simulated diff in rates 4: 16.7%
#> simulated diff in rates 5: 50.0%
#> simulated diff in rates 6: 50.0%
#> simulated diff in rates 7: -16.7%

eustin

NUM_SIMULATIONS = 10_000
simulated_diffs_in_rates = []

for _ in range(NUM_SIMULATIONS):
    rng.shuffle(all_users)
    control_conversion_rate = all_users[:NUM_CONTROL_USERS].mean()
    variant_conversion_rate = all_users[NUM_CONTROL_USERS:].mean()
    simulated_diffs_in_rates.append(variant_conversion_rate - control_conversion_rate)

simulated_diffs_in_rates = np.array(simulated_diffs_in_rates)

eustin

print(f"simulated control conversion rate: {simulated_control_group.mean():.1%}")
print(f"simulated variant conversion rate: {simulated_variant_group.mean():.1%}")

#> simulated control conversion rate: 66.7%
#> simulated variant conversion rate: 50.0%

simulated_diff_in_rates = simulated_variant_group.mean() - simulated_control_group.mean()

print(f"simulated difference in conversion rates: {simulated_diff_in_rates:.1%}")

eustin

simulated_control_group = all_users[:NUM_CONTROL_USERS]
simulated_variant_group = all_users[NUM_CONTROL_USERS:]

simulated_control_group, simulated_variant_group

#> (array([0., 1., 1., 0., 1., 1.]), array([1., 0., 0., 0., 1., 1.]))

eustin

rng.shuffle(all_users)

#> array([0., 1., 1., 0., 1., 1., 1., 0., 0., 0., 1., 1.])

eustin

all_users = np.hstack([control_users, variant_users])

#> array([1., 1., 1., 0., 0., 0., 1., 1., 1., 1., 0., 0.])

eustin

observed_diff_in_rates = variant_conversion_rate - control_conversion_rate
print(f"observed difference in conversion rates: {observed_diff_in_rates:.1%}")

#> observed difference in conversion rates: 16.7%

eustin

NUM_VARIANT_USERS = 6
NUM_CONVERTING_VARIANT_USERS = 4

variant_users = np.zeros(NUM_VARIANT_USERS)
variant_users[:NUM_CONVERTING_VARIANT_USERS] = 1.0

variant_conversion_rate = variant_users.mean()
print(f"variant conversion rate: {variant_conversion_rate:.1%}")

#> variant conversion rate: 66.7%

eustin

control_conversion_rate = control_users.mean()
print(f"control conversion rate: {control_conversion_rate:.1%}")

#> control conversion rate: 50.0%

eustin

NUM_CONVERTING_CONTROL_USERS = 3
control_users[:NUM_CONVERTING_CONTROL_USERS] = 1.0

#> array([1., 1., 1., 0., 0., 0.])