Early test termination This is a simple script to illustrate the danger of early termination of a trial (a form of selection bias). The script takes a fair coin, and flips it *flips* times. Throughout the test, it watches to see if the current counts are ever "statistically significant (p <0.05)" to be more likely heads than tails. This is to em…

coinflips.py

import math
import random

pdf_memo = {}


def pdf(h, n):
    h = min(h, n-h)
    if (h, n) not in pdf_memo:
        pdf_memo[(h, n)] = math.factorial(n)/(math.factorial(h)*math.factorial(n-h))*(0.5**h)*(0.5**(n-h))
    return pdf_memo[(h, n)]


def check_coin(iters):
    heads = 0
    for num_flips in range(1, iters+1):
        heads += random.randint(0,1)
        cdf = 0
        for i in range(heads, num_flips + 1):
            cdf += pdf(i, num_flips)
        # print "%s/%s (%.2f)" % (heads, num_flips, cdf)
        if cdf < 0.05:
            print "Unfair coin!  %s/%s (p=%.2f)" % (heads, num_flips, cdf)
            return False
    print "Coin not proven to be unfair."
    return True

fair_counts = 0
checks = 100
flips = 100
for x in range(checks):
    if check_coin(flips):
        fair_counts += 1

print "Coin found to be unfair with p=0.05 in the first %s flips %s times out of %s" % (
    flips, (checks-fair_counts), checks
)