stat_sig_random_14

stat_sig_random_14.py

NUM_CONTROL_USERS = 1_000_000
NUM_CONVERTING_CONTROL_USERS = 26_000

NUM_VARIANT_USERS = 1_000_000
NUM_CONVERTING_VARIANT_USERS = 26_400

# create our arrays of users
control_users = np.zeros(NUM_CONTROL_USERS)
control_users[:NUM_CONVERTING_CONTROL_USERS] = 1.0
control_conversion_rate = control_users.mean()

variant_users = np.zeros(NUM_VARIANT_USERS)
variant_users[:NUM_CONVERTING_VARIANT_USERS] = 1.0
variant_conversion_rate = variant_users.mean()

# calculate our experiment result
observed_diff_in_rates = variant_conversion_rate - control_conversion_rate

# chuck our users into a bag
all_users = np.hstack([control_users, variant_users])

print(f"control conversion rate = {control_conversion_rate:.2%}")
print(f"variant conversion rate = {variant_conversion_rate:.2%}")
print(f"observed absolute difference in conversion rates = {observed_diff_in_rates:.2%}")
print(f"observed relative difference in conversion rates = {observed_diff_in_rates / control_conversion_rate:.2%}")

#> control conversion rate = 2.60%
#> variant conversion rate = 2.64%
#> observed absolute difference in conversion rates = 0.04%
#> observed relative difference in conversion rates = 1.54%