100000 loops. Every loop creates 1000 sessions. Max deviation was 7.5%. For 3000 sessions: goes down to 4.1%

testing-random-ab-test.js

const secureRand = (() => {
    const crypto = globalThis.crypto || require('crypto').webcrypto;
    const maxUint32 = 0xFFFFFFFF;
    const buffer = new Uint32Array(1);

    return () => {
        // Fill the array with random values
        crypto.getRandomValues(buffer);

        // Convert the random value into a float between 0 and 1
        return buffer[0] / maxUint32;
    };
})();

const simpleRand = () => Math.random();

const mockedRand = (() => {
    let seed = 1;
    const m = 2 ** 31 - 1;
    const a = 48271;
    const c = 0;
    const func = () => {
        seed = (a * seed + c) % m;
        return seed / m;
    };
    func.reset = () => { seed = 1; };
    return func;
})();

const mockedRand2 = (() => {
    let seed;
    const func = () => {
        seed = (1664525 * seed + 1013904223) % 4294967296;
        return seed / 4294967296;
    };
    func.reset = () => {
        seed = 12345;
    };
    func.reset();
    return func;
})();

const cases = [
    { name: 'case A', frequency: 1/3 },
    { name: 'case B', frequency: 1/5 },
    { name: 'case C', frequency: 1/30 },
    { name: 'case D', frequency: 1/30 },
];

const DEFAULT_CASE = {
    name: 'default',
    frequency: 1 - cases.reduce((acc, { frequency }) => { return frequency + acc }, 0),
};

let maxDeviation = 0;
for (let _ = 0; _ < 100000; _++) {
    /**
    * Possible case selectors: selectABTestCase1, selectABTestCase2, selectABTestCase3
    * Possible randomGenerators: secureRand, simpleRand, mockedRand, mockedRand2
    */
    maxDeviation = Math.max(
        testCase(1000, selectABTestCase1, mockedRand2),
        maxDeviation
    );
}

console.log('maxDeviation', maxDeviation);


/* HELPERS */

// Performs a cumulative sum and compares with a random value.
// Returns maxDeviation
function testCase (SESSIONS_SAMPLE_NUMBER, caseSelector, randomGenerator) {
    if (randomGenerator.reset) {
        randomGenerator.reset();
    }

    const results = {};
    // selecting case and assigning number of sesions in results object.
    // Example {"case A": 333, "case B": 200}
    for (let _ = 0; _ < SESSIONS_SAMPLE_NUMBER; _++) {
        const selectedCase = caseSelector(cases, randomGenerator);
        results[selectedCase.name] = (results[selectedCase.name] || 0) + 1;
    }
    // calculating how much deviation we get after 1000 sessions from the intended frequency.
    // for frequency 0.2 we expect 200 sessions. 204 sessions means 2% deviation
    let maxDeviation = 0;
    Object.entries(results).forEach(([ caseName, timesTriggered ]) => {
        let caseFrequency = cases.find(({ name }) => name === caseName);
        caseFrequency = caseFrequency
            ? caseFrequency.frequency
            : DEFAULT_CASE.frequency;

     //   console.log(caseName, timesTriggered / SESSIONS_SAMPLE_NUMBER, 'of', caseFrequency);

        maxDeviation = Math.max(
            Math.abs((timesTriggered / SESSIONS_SAMPLE_NUMBER) - caseFrequency),
            maxDeviation,
        );
    });

    return maxDeviation;
}

/**
 * Selects an A/B testing case based on independent frequencies (probabilities).
 * Frequencies are between 0 and 1.
 * @param {Object} cases - Object where keys are case names and values are probabilities.
 * @returns {Object} - Selected case name or default case if none triggered.
 *
 * NOTE: this selection methods seems to have the least deviation score
 */
function selectABTestCase1(cases, randomGenerator) {
    const random = randomGenerator();
    let cumulative = 0;

    for (const currCase of cases) {
        if (currCase.frequency < 0 || currCase.frequency > 1) {
            throw new Error(`Invalid frequency for ${currCase.name}: must be between 0 and 1`);
        }

        const start = cumulative;
        const end = cumulative + currCase.frequency;
        
        if (random >= start && random < end) {
            return currCase;
        }

        cumulative = end;
    }

    return DEFAULT_CASE;
}

// Duplicates cases according to their frequency, then picks a random index (memory hungry).
function selectABTestCase2(cases, randomGenerator) {
    const random = randomGenerator();
    const weightedCases = cases.flatMap(testCase => Array(Math.floor(1 / testCase.frequency)).fill(testCase));
    const randomIndex = Math.floor(random * weightedCases.length);
    return weightedCases[randomIndex] || DEFAULT_CASE;
}

// Uses cumulative frequencies and performs a binary search.
function selectABTestCase3(cases, randomGenerator) {
    const cumulativeFrequencies = [];
    let total = 0;

    // Calculate cumulative frequencies
    cases.forEach(testCase => {
        total += testCase.frequency;
        cumulativeFrequencies.push(total);
    });

    const random = randomGenerator() * total;

    // Binary search for the test case
    for (let i = 0; i < cumulativeFrequencies.length; i++) {
        if (random <= cumulativeFrequencies[i]) {
            return cases[i];
        }
    }
    return DEFAULT_CASE; // Fallback case
}