Javascript Chicago Lightning Talk Outline (WIP)

binomial.js

// A pure Javscript implementation of binomial distribution
// This is for educational purposes.  These numbers can become unreliable
// when the numerators and denominators get too big.  This is because
// of floating point arithmetic.
//
// Use a more robust statistical library for mission critical calculations

function factorial(n) {
  let total = 1;
  for(let i=n; i>1; i--) {
    total = total * i;
  }

  return total;
}

function choose(n, k) {
  return factorial(n) / (factorial(k) * factorial(n - k))
}

function binomial(n, k, p) {
  return choose(n, k) * Math.pow(p, k) * Math.pow(1-p, n-k);
}

brute_force_monte_carlo.js

// Run 10_000 trials
// In each trial, 100 people visit site A and 100 people visit site B
// In the simulation, both sites have the same joint rate
// The final output of each trial will be the click through rate on page A and the click through rate on page B

function click_rate(total_visits=100, base_rate=0.115) {
  let visit_count = 0;
  for (let visit=0; visit<=total_visits; visit++) {
    if (Math.random() < base_rate) {
      visit_count += 1;
    }
  }

  return visit_count / total_visits;
}

function percent_difference(rate_a, rate_b) {
  if (rate_a === 0 && rate_b === 0) {
    return 1;
  } else {
    return rate_b / rate_a;
  }
}

function run_trials(options = {}) {
  const DEFAULTS = {
    total_trials: 100_000,
    total_visits: 100,
    base_rate: 0.115,
    target_difference: 1.30,
  };

  const {
    total_trials,
    total_visits,
    base_rate,
    target_difference
  } = { ...DEFAULTS, ...options};

  let success_count = 0;
  for (let trial=0; trial<total_trials; trial++) {
    let click_rate_a = click_rate(total_visits, base_rate);
    let click_rate_b = click_rate(total_visits, base_rate);

    // Remember this is a two sided test
    let ratio_left  = percent_difference(click_rate_a, click_rate_b);
    let ratio_right = percent_difference(click_rate_b, click_rate_a);
    if (ratio_left > target_difference || ratio_right > target_difference) {
      success_count += 1;
    }
  }

  return success_count / total_trials;
}

flaky_test_monte_carlo.js

function scenario(run_count=5, p=5/6) {
  let total = 0;
  for (let i=0; i<run_count; i++) {
    if (Math.random() < p) {
      total += 1;
    }
  }
  return total / run_count;
}

function run_trials(options = {}) {
  const defaults = { trial_count: 100_000, run_count: 5, p: 5/6 };
  const { trial_count, run_count, p } = Object.assign({}, defaults, options);
  let total = 0;
  for (let trial=0; trial<trial_count; trial++) {
    if (scenario(run_count, p) === 1) {
      total += 1;
    }
  }
  return total / trial_count;
}

javascript_chicago_lightning_talk_outline.md

Probability and Stats For Software Engineers

Learn just enough to be dangerous.

Real world problem in programming

You're running an A/B test on a website. Version A of the page has been visited 100 times, and 9 of those times the user submitted a form. Version B of the page has been visited 100 times and 13 of those times a user submitted a form. Does version B have a higher conversion rate or is this just random chance?

Goal of this talk:

In the next 20 minutes we're going to come back to this story problem and show the first principles behind how we solve it.

Attempt 1: Brute Force

If we want to answer the story problem, we first assume that there's no difference between page A and page B. Then we see how likely it is by pure chance that page B would have this much higher of a click-through rate. The lower that probability, the more certain we are that Page B is in fact better than Page A.

One-Sided Vs Two Sided Test

When I start my A/B test, I don't know which version of the page I expect will have a higher conversion rate. So this means that when I see a difference in conversion rates, I have to check the possibility that the results are this lopsided in EITHER direction.

If you CANNOT anticipate ahead of time the direction of the effect, use a two sided test.

If you CAN anticipate the direction of the effect ahead of time, use a one sided test.

This is an important topic to dwell on. Let's say I'm testing whether Ozempic causes people to lose weight. At the beginning I'm going to assume one of two possibilities: A) The drug helps people lose weight, B) the drug has no effect. I'm not considering the possibility that the drug causes weight gain. Sure, anything is possible, but it's way more likely that a weight loss drug will help a patient lose 20 pounds than gain 20 pounds.

Brute Force Monte Carlo Simulation

A couple of points with this simulation:

If users have only visited each page 100 times, we're only 75% sure that page B actually has a higher click through rate than page A. If we increase the number of trials from 10,000 to 100,000 the results don't change. If we change the number of vists per page in the simulation, that DOES increase our certainty rate. In fact, how many visits per page would we need in order to know for sure?

Let's play around with the numbers.

page_visits	p-value (%)
100	49.89%
200	34.19%
300	24.97%
400	18.55%
500	13.69%
600	10.33%
700	7.86%
800	6.03%
900	4.70%
1000	3.56%

As more people on average visit the page, then our certainty gets higher and higher. In statitistics we generally want to want this p-value to be 5% or smaller. That number is just a convention.

Brute Force Is A Perfectly Valid Mathematical Tool

There are too many statistical formulas to remember, some of which I'll get into with today's lightning talk. But today's computers are powerful enough that doing a brute force monte-carlo simulation is often the quickest way to get to the right answer.

Tinkering With Parameters Is A Perfectly Valid Learning Tool

Formulas are great shortcuts once you grasp the underlying concepts. If you don't know why the formula is the way it is, tinkering with parameters and seeing what happens is a great way to grasp the fundamental concepts. Here are a few:

• As we increase the number of page visits, the final p-value becomes smaller
• As we increase the number of trials, the final p-value doesn't become bigger or smaller, just more stable. This is the "law of large numbers" if you're curious.
• If we increase the difference in click through rates between pages A and B, we need fewer page visits in order to be confident in our assertion
• If the overall click-through rate is small for BOTH pages, we have to wait longer before we're certain of our conclusion

In Statsistics You Assume The Opposite And Then Try To Disprove It

What's the thing I'm trying to prove? I want to prove that page B has a higher click through rate than page A. What's the opposite of that statement? In this case we assume page B and Page A have the same click through rate.

Parallels to TDD and QA

This is actually similar to the way testing and QA works. In order to show that your code is correct you first assume that the code is flawed and then you try to prove that the code is flawed. Once you've tried and failed to prove that the code is flawed, then you can assume that your code is correct.|

Bernoulli/Binomial Distribution Explained

Bernoulli distribution is a fancy term for a coin flip. The coin doesn't have to be fair. It could be a coin that's heads 99% of the time and tails 1% of the time. Binomial distribution says "If you flip the coin X times, what's the probability that it's heads Y times." For example, "If you make 30% of your free throws and you take 20 free throws in a game, what's the probability you make exactly 9 of them?"

Formula For Binomial Distrbution

The probability mass function (PMF) of the binomial distribution is given by:

$$ P(X = k) = \binom{n}{k} p^k (1 - p)^{n - k} $$

where:

• ( n ) is the number of trials,
• ( k ) is the number of successes,
• ( p ) is the probability of success in each trial

Binomial Distribution In Pure JS

Poisson Distribution Explained

Imagine the following scenario among basketball players:

• Player 1 makes 50% of their free throws. How many shots do they make on average if they take 10 shots? (Answer: 5)
• Player 2 makes 25% of their free throws. How many shots do they make on average if they take 20 shots? (Answer: 5)
• Player 3 makes 1% of their free throws. How many shots do they make on average if they take 500 shots? (Answer: 5)

Imagine this sequence continues. Each successive player makes a smaller percentage of their free throws but takes more shots so that they make 5 shots on average. As the number of shots approaches infinity, this is the Poisson distribution.

The Poisson distribution is a great approximation for rare events that occur at random. Examples:

• Car crashes (rare for any one driver, but there are a lot of drivers in any given city)
• Server outages
• Click-through rates on pages if the rate is low

Use a 3rd party statistical library

• Throw in some examples with j-stat here
• Also encourage users to explore the vast world outside javascript
• (Google Sheets, Microsoft Excel)

Normal Approximation Explained

Details TBA

Attempt #2, Using Normal Distribution

$$\widehat{\mu} = \frac{k_a + k_b}{n_a + n_b} = \frac{10 + 13}{100 + 100} = \frac{23}{200} = 0.115 $$

$$\widehat{\sigma}^2 = 0.115 * (1 - 0.115) = 0.101775$$

$$SE^2 = SE_a^2 + SE_b^2 = \frac{\sigma_a^2}{n_a} + \frac{\sigma_b^2}{n_b}$$

$$SE^2 = \frac{\widehat{\sigma}^2}{n_a} + \frac{\widehat{\sigma}^2}{n_b}$$

$$SE^2 = \widehat{\sigma}^2 \left( \frac{1}{n_a} + \frac{1}{n_b} \right) $$

$$SE = \widehat{\sigma} \sqrt{\frac{1}{n_a} + \frac{1}{n_b}} $$

$$SE = 0.045116515822922316$$

$$z = \frac{\mu_a - \mu_b}{SE}$$

$$z = \frac{0.13 - 0.10}{0.045116515822922316} $$

$$ z = 0.664944964228774 $$

$$ Final = P[Z>z] + P[Z<-z]$$

$$ Final = 1 - CDF(z) + CDF(-z) = 2 * \left(1 - CDF(z)\right) $$

$$ Final = 2 * \left(1 - CDF(z)\right) $$

$$ Final = 0.5060856949 $$

poisson_simulation.js

function random_poisson_approximation(mu) {
  // Simulates it with a very large binomial distribution

  const n = Math.floor(mu * 100);
  const p = mu / n;

  let total = 0;
  for (let i=0; i<n; i++) {
    if (Math.random() < p) {
      total += 1;
    }
  }

  return total;
}

function run_trials(options={}) {
  const defaults = { mu: 10, trial_count: 100_000, target: 8 };
  const { mu, trial_count, target } = { ...defaults, ...options };

  let success_count = 0;

  for (let trial=0; trial<trial_count; trial++) {
    if (random_poisson_approximation(mu) <= target) {
      success_count += 1;
    }
  }

  return success_count / trial_count;
}

standard_error.js

function mean(values) {
  let total = 0;
  let count = 0;

  for (const value of values) {
    total += value;
    count += 1;
  }

  return total / count;
}

function variance(values) {
  let s_1 = 0, s_2 = 0, count = 0;

  for (const value of values) {
    count += 1;
    s_1 += value;
    s_2 += (value * value);
  }

  const mu = s_1 / count;

  return s_2 / count - mu * mu;
}

function stdev(values) {
  return Math.sqrt(variance(values));
}

function get_observed_rate(visit_count=100, click_rate=0.115) {
  let click_count = 0;

  for (let visit=0; visit<visit_count; visit++) {
    if (Math.random() < click_rate) {
      click_count += 1;
    }
  }

  return click_count / visit_count;
}

function run_trials(overrides = {}) {
  const defaults = {
    visit_count: 100,
    click_rate: 0.115,
    trial_count: 100_000,
  }

  const {
    visit_count,
    click_rate,
    trial_count
  } = Object.assign({}, defaults, overrides);

  const observed_rates = [];
  for (let trial=0; trial<=trial_count; trial++) {
    const rate = get_observed_rate(visit_count, click_rate);
    observed_rates.push(rate);
  }

  return {
    mean: mean(observed_rates),
    stdev: stdev(observed_rates),
    values: observed_rates
  }
}