attempt at an all-in-one jest performance expectation

jest.d.ts

declare global {
  namespace jest {
    interface Matchers<R> {
      toBeFasterThan(target: number): Promise<CustomMatcherResult>;
    }
  }
}

toBeFasterThan.ts

import { matcherHint, printDiffOrStringify } from 'jest-matcher-utils';

// This is an adaptation of https://github.com/nickheal/test-performance and
// https://github.com/nickheal/jest-test-performance for use with our test
// suite.

/**
 * This runs a single performance test of a function
 */
async function perfTest(func: Function): Promise<number> {
  const start = performance.now();
  await func();
  const end = performance.now();
  return end - start;
}

async function getScore(
  func: Function,
  numberOfTests: number,
): Promise<number> {
  const tests = new Array(numberOfTests)
    .fill(undefined)
    .map(async () => perfTest(func));
  const results = await Promise.all(tests);
  return results.reduce((acc, val) => acc + val, 0) / numberOfTests;
}

// can change based on system performance
const EXPECTED_BASELINE_RUNTIME = 12;

function baselineFunc(dummyParam?: number) {
  function test() {}

  const a = dummyParam || Math.random();

  if (a > 0.5) {
    test();
  } else {
    test();
  }
}

async function runTests(
  baselineFunc: Function,
  targetFunc: Function,
  numberOfTests: number,
): Promise<[number, number]> {
  /**
   * Tests deliberately run in sequence to catch performance changes over time
   */
  const initialBaselineScore = await getScore(baselineFunc, numberOfTests);
  const initialTargetScore = await getScore(targetFunc, numberOfTests);
  const midwayBaselineScore = await getScore(baselineFunc, numberOfTests);
  const endTargetScore = await getScore(targetFunc, numberOfTests);
  const endBaselineScore = await getScore(baselineFunc, numberOfTests);

  const totalBaselineScore =
    (initialBaselineScore + midwayBaselineScore + endBaselineScore) / 3;
  const totalTargetScore = (initialTargetScore + endTargetScore) / 2;

  return [totalBaselineScore, totalTargetScore];
}

/**
 * This calculates the expected performance of the function, based on comparison
 * to a baseline.
 */
function calculateExpectedPerformance(
  baselineExpected: number,
  baselineActual: number,
  target: number,
): number {
  const performanceRatio = baselineActual / baselineExpected;
  return target / performanceRatio;
}

async function getPerformanceScore(
  func: Function,
  numberOfTests: number,
): Promise<number> {
  if (typeof func !== 'function') throw new Error(`${func} is not a function`);
  const [baseline, target] = await runTests(baselineFunc, func, numberOfTests);
  return calculateExpectedPerformance(
    EXPECTED_BASELINE_RUNTIME,
    baseline,
    target,
  );
}

async function predicate(func: Function, time: number, numberOfTests: number) {
  const result = await getPerformanceScore(func, numberOfTests);
  return [result < time, result];
}

const passMessage =
  (func: Function, target: number, result: number | boolean) => () =>
    `
  ${matcherHint('.not.toBeFasterThan', 'received', '')}
  Function '${func}' ran faster than expected:
  ${printDiffOrStringify(target, result, 'Expected', 'Received', true)}
`;

const failMessage =
  (func: Function, target: number, result: number | boolean) => () =>
    `
  ${matcherHint('.toBeFasterThan', 'received', '')}
  Function '${func}' ran slower than expected:
  ${printDiffOrStringify(target, result, 'Expected', 'Received', true)}
`;

const matcher = {
  toBeFasterThan: async (
    func: Function,
    target: number,
    times: number = 25,
  ) => {
    const [pass, result] = await predicate(func, target, times);
    if (pass) {
      return { pass: true, message: passMessage(func, target, result) };
    }

    return { pass: false, message: failMessage(func, target, result) };
  },
};

expect.extend(matcher);