run-website-performance-ab

compare-rendered-pages.mjs

#!/usr/bin/env node

import { mkdir, writeFile } from 'node:fs/promises';
import path from 'node:path';
import {
  ARM_NAMES,
  artifactName,
  assertExpectedPage,
  booleanArgument,
  createContext,
  hash,
  loadChromium,
  loadRoutes,
  normalizeOrigin,
  parseArguments,
  parseExpectedHeader,
  parseExpectedMeta,
  parseViewports,
  positiveNumber,
  writeArtifacts,
} from './website-ab-common.mjs';

const chromium = loadChromium();

const argumentsMap = parseArguments(process.argv.slice(2));
if (argumentsMap.has('help')) {
  printUsage();
  process.exit(0);
}

const options = {
  controlUrl: requiredOrigin('control-url'),
  candidateUrl: requiredOrigin('candidate-url'),
  routes: await loadRoutes(argumentsMap),
  viewports: parseViewports(argumentsMap.get('viewports') ?? 'desktop,mobile'),
  screenshots: booleanArgument(argumentsMap.get('screenshots'), true),
  timeoutMs: positiveNumber(argumentsMap.get('timeout-ms'), 30_000),
  settleMs: positiveNumber(argumentsMap.get('settle-ms'), 500),
  headless: booleanArgument(argumentsMap.get('headless'), true),
  output: path.resolve(argumentsMap.get('output') ?? '/tmp/rendered-pages-ab'),
  ignoreSelector: argumentsMap.get('ignore-selector') ?? '',
  expected: Object.fromEntries(
    ARM_NAMES.map((arm) => [
      arm,
      {
        header: parseExpectedHeader(argumentsMap.get(`expected-${arm}-header`)),
        meta: parseExpectedMeta(
          argumentsMap.get(`expected-${arm}-meta-selector`),
          argumentsMap.get(`expected-${arm}-meta-value`),
        ),
      },
    ]),
  ),
};

const browser = await chromium.launch({ headless: options.headless });
const comparisons = [];
let fatalError;

try {
  await preflight('control', options.routes[0], options.viewports[0]);
  await preflight('candidate', options.routes[0], options.viewports[0]);
  for (const route of options.routes) {
    const viewportReports = [];
    for (const viewport of options.viewports) {
      const snapshots = {};
      for (const arm of ARM_NAMES) {
        snapshots[arm] = await capture(arm, route, viewport);
      }
      const differences = compareSnapshots(
        snapshots.control,
        snapshots.candidate,
      );
      if (differences.length > 0) {
        await writeFailure(route, viewport, snapshots);
      }
      viewportReports.push({
        name: viewport.name,
        ok: differences.length === 0,
        differences,
        sources: Object.fromEntries(
          ARM_NAMES.map((arm) => [arm, summarizeSnapshot(snapshots[arm])]),
        ),
      });
    }
    const differences = viewportReports.flatMap((viewport) =>
      viewport.differences.map(
        (difference) => `${viewport.name}: ${difference}`,
      ),
    );
    comparisons.push({
      path: route,
      ok: differences.length === 0,
      differences,
      viewports: viewportReports,
    });
    console.log(`${differences.length === 0 ? '✓' : '✗'} ${route}`);
  }
} catch (error) {
  fatalError = error instanceof Error ? error.message : String(error);
  console.error(fatalError);
} finally {
  await browser.close();
}

const report = {
  schemaVersion: 1,
  generatedAt: new Date().toISOString(),
  settings: {
    controlUrl: options.controlUrl,
    candidateUrl: options.candidateUrl,
    routes: options.routes,
    viewports: options.viewports,
    screenshots: options.screenshots,
    separateContexts: true,
    contextConfiguration: {
      control: Boolean(process.env.WEBSITE_AB_CONTROL_CONTEXT_JSON),
      candidate: Boolean(process.env.WEBSITE_AB_CANDIDATE_CONTEXT_JSON),
      adapter: Boolean(process.env.WEBSITE_AB_CONTEXT_ADAPTER_MODULE),
    },
  },
  routes: comparisons,
  complete: !fatalError && comparisons.length === options.routes.length,
  failedRoutes: comparisons.filter((comparison) => !comparison.ok).length,
  failedChecks: comparisons.reduce(
    (sum, comparison) => sum + comparison.differences.length,
    0,
  ),
  fatalError,
};
await writeArtifacts(options.output, report, paritySummary(report));
console.log(`Report: ${path.join(options.output, 'report.json')}`);
if (!report.complete || report.failedRoutes > 0) process.exitCode = 1;

async function preflight(arm, route, viewport) {
  const baseUrl = arm === 'control' ? options.controlUrl : options.candidateUrl;
  const context = await createContext(browser, arm, baseUrl, viewport);
  const page = await context.newPage();
  try {
    const response = await page.goto(new URL(route, baseUrl).toString(), {
      waitUntil: 'domcontentloaded',
      timeout: options.timeoutMs,
    });
    await assertExpectedPage({
      arm,
      baseUrl,
      expectedHeader: options.expected[arm].header,
      expectedMeta: options.expected[arm].meta,
      page,
      response,
    });
  } finally {
    await context.close();
  }
}

async function capture(arm, route, viewport) {
  const baseUrl = arm === 'control' ? options.controlUrl : options.candidateUrl;
  const context = await createContext(browser, arm, baseUrl, viewport);
  const page = await context.newPage();
  const consoleErrors = [];
  const failedRequests = [];
  page.on('console', (message) => {
    if (message.type() === 'error') consoleErrors.push(message.text());
  });
  page.on('requestfailed', (request) => {
    failedRequests.push(`${request.method()} ${request.url()}`);
  });
  try {
    const response = await page.goto(new URL(route, baseUrl).toString(), {
      waitUntil: 'domcontentloaded',
      timeout: options.timeoutMs,
    });
    await assertExpectedPage({
      arm,
      baseUrl,
      expectedHeader: options.expected[arm].header,
      expectedMeta: options.expected[arm].meta,
      page,
      response,
    });
    await page
      .waitForLoadState('networkidle', { timeout: options.timeoutMs })
      .catch(() => {});
    await page.evaluate(async () => {
      await document.fonts?.ready;
      await Promise.all(
        [...document.images]
          .filter((image) => !image.complete)
          .map(
            (image) =>
              new Promise((resolve) => {
                image.addEventListener('load', resolve, { once: true });
                image.addEventListener('error', resolve, { once: true });
              }),
          ),
      );
    });
    await page.waitForTimeout(options.settleMs);
    const snapshot = await extractSnapshot(page);
    const screenshot = options.screenshots
      ? await page.screenshot({ fullPage: true, animations: 'disabled' })
      : undefined;
    return {
      ...snapshot,
      status: response?.status() ?? null,
      finalPath: new URL(page.url()).pathname,
      consoleErrors: [...new Set(consoleErrors)].sort(),
      failedRequests: [...new Set(failedRequests)].sort(),
      screenshot,
    };
  } finally {
    await context.close();
  }
}

async function extractSnapshot(page) {
  return await page.evaluate(
    ({ comparableOrigins, ignoreSelector }) => {
      const origins = new Set(comparableOrigins);
      const normalizeText = (value) =>
        (value ?? '').replace(/\s+/gu, ' ').trim();
      const normalizeUrl = (value) => {
        try {
          const url = new URL(value, window.location.href);
          if (origins.has(url.origin)) {
            return `${url.pathname}${url.search}${url.hash}`;
          }
          return url.toString();
        } catch {
          return value;
        }
      };
      const ignored = (element) =>
        ignoreSelector ? Boolean(element.closest(ignoreSelector)) : false;
      let visibleText = normalizeText(document.body.innerText);
      if (ignoreSelector) {
        for (const element of document.querySelectorAll(ignoreSelector)) {
          const ignoredText = normalizeText(element.innerText);
          if (ignoredText) visibleText = visibleText.replace(ignoredText, '');
        }
        visibleText = normalizeText(visibleText);
      }
      const root = document.body.cloneNode(true);
      root
        .querySelectorAll(
          ['script', 'style', 'noscript', 'template', ignoreSelector]
            .filter(Boolean)
            .join(','),
        )
        .forEach((element) => element.remove());
      root.querySelectorAll('*').forEach((element) => {
        for (const attribute of [...element.attributes]) {
          if (
            attribute.name === 'nonce' ||
            attribute.name === 'style' ||
            attribute.name === 'data-reactroot' ||
            attribute.name === 'data-rsc'
          ) {
            element.removeAttribute(attribute.name);
          } else if (
            ['href', 'src', 'poster', 'action'].includes(attribute.name)
          ) {
            element.setAttribute(attribute.name, normalizeUrl(attribute.value));
          } else if (attribute.name === 'srcset') {
            element.setAttribute(
              attribute.name,
              attribute.value
                .split(',')
                .map((candidate) => {
                  const [url, ...descriptor] = candidate.trim().split(/\s+/u);
                  return [normalizeUrl(url), ...descriptor].join(' ');
                })
                .join(', '),
            );
          }
        }
      });
      const metadata = (selector) => {
        const values = {};
        for (const element of document.querySelectorAll(selector)) {
          const key =
            element.getAttribute('name') ?? element.getAttribute('property');
          if (!key) continue;
          const content = normalizeText(element.getAttribute('content'));
          let normalizedContent = content;
          try {
            normalizedContent = normalizeUrl(new URL(content).toString());
          } catch {}
          values[key] ??= [];
          values[key].push(normalizedContent);
        }
        return Object.fromEntries(
          Object.entries(values)
            .map(([key, contents]) => [key, contents.sort()])
            .sort(([left], [right]) => left.localeCompare(right)),
        );
      };
      return {
        title: normalizeText(document.title),
        description:
          document
            .querySelector('meta[name="description"]')
            ?.getAttribute('content')
            ?.trim() ?? null,
        robots:
          document
            .querySelector('meta[name="robots"]')
            ?.getAttribute('content')
            ?.trim() ?? null,
        canonical: document.querySelector('link[rel="canonical"]')?.href
          ? normalizeUrl(document.querySelector('link[rel="canonical"]').href)
          : null,
        language: document.documentElement.getAttribute('lang'),
        visibleText,
        headings: [...document.querySelectorAll('h1,h2,h3,h4,h5,h6')]
          .filter((element) => !ignored(element))
          .map((element) => normalizeText(element.textContent)),
        links: [...document.querySelectorAll('a[href]')]
          .filter((element) => !ignored(element))
          .map((element) => ({
            href: normalizeUrl(element.href),
            text: normalizeText(element.textContent),
            rel: normalizeText(element.getAttribute('rel')),
          }))
          .sort((left, right) =>
            JSON.stringify(left).localeCompare(JSON.stringify(right)),
          ),
        media: [...document.querySelectorAll('img,video[src]')]
          .filter((element) => !ignored(element))
          .map((element) => ({
            tag: element.tagName.toLowerCase(),
            src: normalizeUrl(element.currentSrc || element.src),
            alt: normalizeText(element.getAttribute('alt')),
          }))
          .sort((left, right) =>
            JSON.stringify(left).localeCompare(JSON.stringify(right)),
          ),
        openGraph: metadata('meta[property^="og:"]'),
        twitter: metadata('meta[name^="twitter:"]'),
        structuredData: [
          ...document.querySelectorAll('script[type="application/ld+json"]'),
        ]
          .map((element) => normalizeText(element.textContent))
          .sort(),
        dom: root.innerHTML.replace(/\s+/gu, ' ').trim(),
      };
    },
    {
      comparableOrigins: [
        new URL(options.controlUrl).origin,
        new URL(options.candidateUrl).origin,
      ],
      ignoreSelector: options.ignoreSelector,
    },
  );
}

function compareSnapshots(control, candidate) {
  const differences = [];
  for (const key of [
    'status',
    'finalPath',
    'title',
    'description',
    'robots',
    'canonical',
    'language',
    'visibleText',
    'headings',
    'links',
    'media',
    'openGraph',
    'twitter',
    'structuredData',
    'dom',
    'consoleErrors',
    'failedRequests',
  ]) {
    if (JSON.stringify(control[key]) !== JSON.stringify(candidate[key])) {
      differences.push(key);
    }
  }
  if (
    control.screenshot &&
    candidate.screenshot &&
    hash(control.screenshot) !== hash(candidate.screenshot)
  ) {
    differences.push('screenshot');
  }
  return differences;
}

function summarizeSnapshot(snapshot) {
  return {
    status: snapshot.status,
    finalPath: snapshot.finalPath,
    title: snapshot.title,
    visibleTextHash: hash(snapshot.visibleText),
    domHash: hash(snapshot.dom),
    screenshotHash: snapshot.screenshot ? hash(snapshot.screenshot) : null,
    consoleErrors: snapshot.consoleErrors,
    failedRequests: snapshot.failedRequests,
  };
}

async function writeFailure(route, viewport, snapshots) {
  const directory = path.join(
    options.output,
    'failures',
    `${artifactName(route)}-${viewport.name}`,
  );
  await mkdir(directory, { recursive: true });
  for (const arm of ARM_NAMES) {
    const { screenshot, ...serializable } = snapshots[arm];
    await writeFile(
      path.join(directory, `${arm}.json`),
      `${JSON.stringify(serializable, null, 2)}\n`,
    );
    if (screenshot) {
      await writeFile(path.join(directory, `${arm}.png`), screenshot);
    }
  }
}

function requiredOrigin(name) {
  const value = argumentsMap.get(name);
  if (!value) throw new Error(`--${name} is required`);
  return normalizeOrigin(value);
}

function paritySummary(report) {
  const lines = [
    '# Rendered page A/B',
    '',
    `Control: ${report.settings.controlUrl}`,
    `Candidate: ${report.settings.candidateUrl}`,
    `Result: ${report.failedRoutes === 0 && report.complete ? 'pass' : 'fail'}`,
    '',
    '| Route | Result | Differences |',
    '| --- | --- | --- |',
  ];
  for (const route of report.routes) {
    lines.push(
      `| ${route.path} | ${route.ok ? 'pass' : 'fail'} | ${route.differences.join(', ') || 'none'} |`,
    );
  }
  return lines.join('\n');
}

function printUsage() {
  console.log(`Usage:
  node compare-rendered-pages.mjs --control-url=<origin> --candidate-url=<origin> [options]

Options:
  --routes=/,/example or --routes-file=/absolute/routes.txt
  --viewports=desktop,mobile|<width>x<height>
  --screenshots=true --settle-ms=500 --timeout-ms=30000
  --ignore-selector=<css>
  --expected-<arm>-header=name:value
  --expected-<arm>-meta-selector=<css> --expected-<arm>-meta-value=<value>
  --output=/tmp/rendered-pages-ab --headless=true

Optional environment configuration:
  WEBSITE_AB_CONTROL_CONTEXT_JSON
  WEBSITE_AB_CANDIDATE_CONTEXT_JSON
  WEBSITE_AB_CONTEXT_ADAPTER_MODULE=/absolute/path/to/context-adapter.mjs

Each JSON object can contain cookies, extraHTTPHeaders, and ignoreHTTPSErrors.
A context adapter must default-export a function or export configureContext.
It receives { context, arm, baseUrl } before the first navigation.
Keep secrets in environment variables; do not put them in arguments or reports.`);
}

compare-website-performance.mjs

#!/usr/bin/env node

import path from 'node:path';
import {
  ARM_NAMES,
  assertExpectedPage,
  blockMatchingRequests,
  booleanArgument,
  createContext,
  flushRumTelemetry,
  formatNumber,
  loadInteractionAdapter,
  loadChromium,
  loadRoutes,
  normalizeOrigin,
  optionalRegExp,
  parseArguments,
  parseExpectedHeader,
  parseExpectedMeta,
  parseViewports,
  positiveNumber,
  summarizeValues,
  writeArtifacts,
} from './website-ab-common.mjs';

const chromium = loadChromium();

const METRICS = [
  'ttfbMs',
  'fcpMs',
  'lcpMs',
  'cls',
  'loadMs',
  'resourceCompletionMs',
  'longTaskCount',
  'longTaskTotalMs',
  'requestCount',
  'htmlTransferBytes',
  'resourceTransferBytes',
];

const argumentsMap = parseArguments(process.argv.slice(2));
if (argumentsMap.has('help')) {
  printUsage();
  process.exit(0);
}

const options = {
  controlUrl: requiredOrigin('control-url'),
  candidateUrl: requiredOrigin('candidate-url'),
  routes: await loadRoutes(argumentsMap),
  viewport: parseViewports(argumentsMap.get('viewports'))[0],
  runs: positiveNumber(argumentsMap.get('runs'), 7),
  warmups: positiveNumber(argumentsMap.get('warmups'), 2),
  settleMs: positiveNumber(argumentsMap.get('settle-ms'), 750),
  timeoutMs: positiveNumber(argumentsMap.get('timeout-ms'), 30_000),
  headless: booleanArgument(argumentsMap.get('headless'), true),
  rumFlush: booleanArgument(argumentsMap.get('flush-rum'), false),
  rumFlushTimeoutMs: positiveNumber(
    argumentsMap.get('rum-flush-timeout-ms'),
    5_000,
  ),
  rumRequestPattern: optionalRegExp(
    argumentsMap.get('rum-request-pattern') ??
      process.env.WEBSITE_AB_RUM_REQUEST_PATTERN,
    'RUM request pattern',
  ),
  output: path.resolve(
    argumentsMap.get('output') ?? '/tmp/website-performance-ab',
  ),
  expected: Object.fromEntries(
    ARM_NAMES.map((arm) => [
      arm,
      {
        header: parseExpectedHeader(argumentsMap.get(`expected-${arm}-header`)),
        meta: parseExpectedMeta(
          argumentsMap.get(`expected-${arm}-meta-selector`),
          argumentsMap.get(`expected-${arm}-meta-value`),
        ),
      },
    ]),
  ),
};
options.interactionAdapter = await loadInteractionAdapter(
  process.env.WEBSITE_AB_INTERACTION_MODULE,
);
if (options.rumFlush && !options.rumRequestPattern) {
  throw new Error(
    '--flush-rum=true requires --rum-request-pattern or WEBSITE_AB_RUM_REQUEST_PATTERN',
  );
}
if (options.interactionAdapter && !options.rumFlush) {
  throw new Error('WEBSITE_AB_INTERACTION_MODULE requires --flush-rum=true');
}

const browser = await chromium.launch({ headless: options.headless });
const routeReports = [];
let failure;

try {
  await preflight('control');
  await preflight('candidate');
  for (const route of options.routes) {
    const runs = { control: [], candidate: [] };
    const totalSamples = options.warmups + options.runs;
    for (let sample = 0; sample < totalSamples; sample += 1) {
      const order = sample % 2 === 0 ? ARM_NAMES : [...ARM_NAMES].reverse();
      for (const arm of order) {
        const measured = sample >= options.warmups;
        const result = await capture(arm, route, measured);
        if (sample >= options.warmups) runs[arm].push(result);
      }
    }
    routeReports.push({
      path: route,
      sources: Object.fromEntries(
        ARM_NAMES.map((arm) => [arm, summarizeRuns(runs[arm])]),
      ),
    });
    console.log(`✓ ${route}`);
  }
} catch (error) {
  failure = error instanceof Error ? error.message : String(error);
  console.error(failure);
} finally {
  await browser.close();
}

const report = {
  schemaVersion: 1,
  generatedAt: new Date().toISOString(),
  evidenceMode: 'synthetic-lab',
  settings: {
    controlUrl: options.controlUrl,
    candidateUrl: options.candidateUrl,
    routes: options.routes,
    viewport: options.viewport,
    runs: options.runs,
    warmups: options.warmups,
    serial: true,
    alternatingOrder: true,
    separateContexts: true,
    rumTelemetry: {
      flushConfigured: options.rumFlush,
      interactionConfigured: Boolean(options.interactionAdapter),
    },
    contextConfiguration: {
      control: Boolean(process.env.WEBSITE_AB_CONTROL_CONTEXT_JSON),
      candidate: Boolean(process.env.WEBSITE_AB_CANDIDATE_CONTEXT_JSON),
      adapter: Boolean(process.env.WEBSITE_AB_CONTEXT_ADAPTER_MODULE),
    },
  },
  routes: routeReports,
  complete: !failure && routeReports.length === options.routes.length,
  failure,
};
await writeArtifacts(options.output, report, performanceSummary(report));
console.log(`Report: ${path.join(options.output, 'report.json')}`);
if (!report.complete) process.exitCode = 1;

async function preflight(arm) {
  const baseUrl = arm === 'control' ? options.controlUrl : options.candidateUrl;
  const context = await createContext(browser, arm, baseUrl, options.viewport);
  const page = await context.newPage();
  try {
    await blockMatchingRequests(
      context,
      options.rumFlush ? options.rumRequestPattern : undefined,
    );
    const response = await page.goto(
      new URL(options.routes[0], baseUrl).toString(),
      {
        waitUntil: 'domcontentloaded',
        timeout: options.timeoutMs,
      },
    );
    await assertExpectedPage({
      arm,
      baseUrl,
      expectedHeader: options.expected[arm].header,
      expectedMeta: options.expected[arm].meta,
      page,
      response,
    });
  } finally {
    await context.close();
  }
}

async function capture(arm, route, measured) {
  const baseUrl = arm === 'control' ? options.controlUrl : options.candidateUrl;
  const context = await createContext(browser, arm, baseUrl, options.viewport);
  const page = await context.newPage();
  if (options.rumFlush && !measured) {
    await blockMatchingRequests(context, options.rumRequestPattern);
  }
  await page.addInitScript(() => {
    const state = { cls: 0, lcp: 0, longTasks: [] };
    Object.defineProperty(window, '__websiteAbPerformance', { value: state });
    new PerformanceObserver((list) => {
      for (const entry of list.getEntries()) state.lcp = entry.startTime;
    }).observe({ type: 'largest-contentful-paint', buffered: true });
    new PerformanceObserver((list) => {
      for (const entry of list.getEntries()) {
        if (!entry.hadRecentInput) state.cls += entry.value;
      }
    }).observe({ type: 'layout-shift', buffered: true });
    new PerformanceObserver((list) => {
      state.longTasks.push(...list.getEntries().map((entry) => entry.duration));
    }).observe({ type: 'longtask', buffered: true });
  });
  try {
    const response = await page.goto(new URL(route, baseUrl).toString(), {
      waitUntil: 'domcontentloaded',
      timeout: options.timeoutMs,
    });
    await assertExpectedPage({
      arm,
      baseUrl,
      expectedHeader: options.expected[arm].header,
      expectedMeta: options.expected[arm].meta,
      page,
      response,
    });
    await page
      .waitForLoadState('load', { timeout: options.timeoutMs })
      .catch(() => {});
    await page
      .waitForLoadState('networkidle', { timeout: options.settleMs })
      .catch(() => {});
    await page.waitForTimeout(options.settleMs);
    const metrics = await page.evaluate(() => {
      const navigation = performance.getEntriesByType('navigation')[0];
      const resources = performance.getEntriesByType('resource');
      const fcp =
        performance.getEntriesByName('first-contentful-paint').at(-1)
          ?.startTime ?? 0;
      const state = window.__websiteAbPerformance;
      return {
        ttfbMs: navigation.responseStart,
        fcpMs: fcp,
        lcpMs: state.lcp,
        cls: state.cls,
        loadMs: navigation.loadEventEnd,
        resourceCompletionMs: Math.max(
          navigation.loadEventEnd,
          ...resources.map((entry) => entry.responseEnd),
        ),
        longTaskCount: state.longTasks.length,
        longTaskTotalMs: state.longTasks.reduce(
          (sum, duration) => sum + duration,
          0,
        ),
        requestCount: resources.length + 1,
        htmlTransferBytes: navigation.transferSize,
        resourceTransferBytes: resources.reduce(
          (sum, entry) => sum + entry.transferSize,
          0,
        ),
        resources: resources.map((entry) => ({
          name: entry.name,
          initiatorType: entry.initiatorType,
          transferBytes: entry.transferSize,
          encodedBodyBytes: entry.encodedBodySize,
          responseEndMs: entry.responseEnd,
        })),
      };
    });
    const rumTelemetry =
      measured && options.rumFlush
        ? await flushRumTelemetry({
            adapter: options.interactionAdapter,
            arm,
            page,
            requestPattern: options.rumRequestPattern,
            route,
            timeoutMs: options.rumFlushTimeoutMs,
          })
        : { interactionPerformed: false, requestObserved: false };
    return { ...metrics, rumTelemetry };
  } finally {
    await context.close();
  }
}

function summarizeRuns(runs) {
  return {
    metrics: Object.fromEntries(
      METRICS.map((metric) => [
        metric,
        summarizeValues(runs.map((run) => run[metric])),
      ]),
    ),
    runs,
  };
}

function requiredOrigin(name) {
  const value = argumentsMap.get(name);
  if (!value) throw new Error(`--${name} is required`);
  return normalizeOrigin(value);
}

function performanceSummary(report) {
  const lines = [
    '# Website performance A/B',
    '',
    `Control: ${report.settings.controlUrl}`,
    `Candidate: ${report.settings.candidateUrl}`,
    '',
    '| Route | Metric | Control p50 / p75 / p95 | Candidate p50 / p75 / p95 |',
    '| --- | --- | ---: | ---: |',
  ];
  for (const route of report.routes) {
    for (const metric of METRICS) {
      const control = route.sources.control.metrics[metric];
      const candidate = route.sources.candidate.metrics[metric];
      lines.push(
        `| ${route.path} | ${metric} | ${[control.p50, control.p75, control.p95].map(formatNumber).join(' / ')} | ${[candidate.p50, candidate.p75, candidate.p95].map(formatNumber).join(' / ')} |`,
      );
    }
  }
  lines.push(
    '',
    'These are lab Web Vitals measurements from scripted browser visits, not field or representative-user percentiles.',
  );
  return lines.join('\n');
}

function printUsage() {
  console.log(`Usage:
  node compare-website-performance.mjs --control-url=<origin> --candidate-url=<origin> [options]

Options:
  --routes=/,/example or --routes-file=/absolute/routes.txt
  --viewports=desktop|mobile|<width>x<height> (one viewport per run)
  --warmups=2 --runs=7 --settle-ms=750 --timeout-ms=30000
  --flush-rum=true --rum-request-pattern=<regex> --rum-flush-timeout-ms=5000
  --expected-<arm>-header=name:value
  --expected-<arm>-meta-selector=<css> --expected-<arm>-meta-value=<value>
  --output=/tmp/website-performance-ab --headless=true

Optional environment configuration:
  WEBSITE_AB_CONTROL_CONTEXT_JSON
  WEBSITE_AB_CANDIDATE_CONTEXT_JSON
  WEBSITE_AB_CONTEXT_ADAPTER_MODULE=/absolute/path/to/context-adapter.mjs
  WEBSITE_AB_INTERACTION_MODULE=/absolute/path/to/interaction.mjs
  WEBSITE_AB_RUM_REQUEST_PATTERN=<regex>

Each JSON object can contain cookies, extraHTTPHeaders, and ignoreHTTPSErrors.
A context adapter must default-export a function or export configureContext.
It receives { context, arm, baseUrl } before the first navigation.
An interaction module must default-export a function or export runInteraction.
It receives { page, arm, route }. RUM flushing applies only to measured visits;
preflights and warm-ups block matching telemetry requests.
Keep secrets in environment variables; do not put them in arguments or reports.`);
}

SKILL.md

name

run-website-performance-ab

description

Build, publish, and evaluate website performance changes with controlled browser A/B tests, representative-route discovery, rendered parity checks, and evidence-based keep-or-revert decisions. Supports matched Preview or staging deployments with no user traffic and production rollouts with field telemetry. Use for web performance experiments, bundle or module-graph changes, runtime variants, matched deployments, or requests to compare candidate and control websites without assuming a repository, framework, hosting provider, CMS, authentication system, or fixed route list.

Run Website Performance A/B

Use an exact control and one independently reversible candidate. Preserve unrelated work and never expose credentials, cookies, tokens, environment values, or protected deployment configuration.

1. Discover the website and representative routes

Locate the affected application, local instructions, route inventory or sitemap, performance harness, parity harness, deployment workflow, source or variant marker, and available field monitoring. Record how each item was discovered.

Map the candidate's imports, layouts, middleware, route entries, renderer families, client boundaries, and generated manifests to its public impact surface. Select the smallest route set that covers:

• every changed route family, layout, client boundary, or renderer;
• a simple and a component-heavy page when bundle breadth is the hypothesis;
• relevant LCP forms such as text, image, video, or an interactive hero;
• a high-traffic or monitored page when field data is available;
• a page outside the expected impact surface when shared code can regress;
• affected locales and static or dynamic route shapes when routing changes.

Record the reason and affected mechanism for every selected route. Treat routes named in old reports as evidence from those runs, not defaults. If no trustworthy inventory, source marker, or variant marker exists, document the discovery gap before calling the experiment controlled.

2. Choose the evidence mode

Select the mode from the traffic that will actually reach the candidate. Do not wait for field telemetry on a deployment that has no representative users.

Situation	Primary evidence	Recommended use
Matched Preview or staging deployments with no user traffic	Controlled browser lab A/B plus parity	Immediate pre-release comparison of exact builds, bundles, rendering, and resources
Preview or staging used by a known test cohort	Lab A/B plus cohort telemetry	Directional validation; report the cohort and do not generalize it to production users
Production rollout with stable control/candidate assignment	Variant-aware real-user monitoring plus the pre-release lab and parity gates	Release decision and field Core Web Vitals confirmation

For an untrafficked Preview comparison:

• drive both exact deployments with the browser harness;
• verify the revision and source or variant on every run;
• measure requests, bytes, resources, long tasks, rendering milestones, and parity immediately;
• report LCP and CLS from scripted visits as lab Core Web Vitals data, even if a real-user monitoring service ingests it;
• report INP only when the harness performs a representative scripted interaction, and label it a lab estimate;
• do not present lab measurements as field Core Web Vitals or representative-user percentiles.

For a production rollout:

• pass the lab and parity gates before exposing users;
• assign users stably and concurrently to control or candidate;
• ensure every field event can be attributed to its resolved variant or exact deployment;
• define the minimum sample count, observation window, segmentation, error guardrails, and rollback threshold before rollout;
• compare p75 LCP, INP, and CLS on equivalent routes, devices, regions, and time windows;
• retain the lab resource evidence because field Core Web Vitals do not explain request, byte, or main-thread changes.

If field telemetry cannot distinguish the variants, use it only as an overall rollout guardrail. Do not present it as an A/B result.

Do not assume that every visit produces every field metric. Some real-user monitoring systems finalize layout, interaction, or paint metrics only after an interaction, visibility change, navigation, or page exit. Query the accepted sample count for each metric and segment, and ensure a scripted cohort completes the lifecycle event required by its telemetry provider. Telemetry generated by scripted visits remains lab evidence even when a real-user monitoring backend stores it.

3. Freeze the experiment

Record the control revision, candidate hypothesis, selected routes and reasons, expected source or variant, and acceptance criteria before editing.

Choose the isolation method:

• Use one deployment with separate deterministic contexts for runtime-only behavior that does not change emitted assets.
• Use matched deployments for dependency, module-graph, route-entry, or bundling changes. A request-time override cannot recreate another build's assets.

Keep content revision, project, environment, region, viewport, authentication, cache policy, browser, and network settings equal unless the hypothesis intentionally changes one of them.

4. Prepare and publish

Inspect the dirty worktree and stage only experiment-owned files. Follow the closest repository instructions. Run the scoped lint, typecheck, tests, unused-code checks, formatting, generated-artifact checks, and build required by the affected application.

For bundle work, capture a current build manifest or analyzer artifact plus browser request and transferred-byte evidence. Do not use stale build output as final evidence.

Commit an immutable control when one does not exist. Implement one reversible candidate at a time. Publish control and candidate through the website's normal test-deployment workflow. Filter deployment output to non-secret identifiers, URLs, states, and errors.

Fail before measurement if a smoke route reaches authentication, an error page, the wrong revision, or the wrong source or variant. Never create, rotate, or reveal a protection secret merely to run the test.

5. Configure controlled contexts

Use separate fresh browser contexts for control and candidate. Apply deterministic runtime overrides before navigation when the experiment uses them. Verify the resolved revision, source, or variant on every accepted run.

Alternate order to reduce warm-cache and temporal bias. Run serially unless concurrency is itself the hypothesis. Use the same viewport and browser profile for both sources. Start with two warm-ups and seven measured samples per source; increase the sample count when noise or the decision threshold requires it.

Use the portable comparison engines

The skill includes two neutral Playwright engines:

• scripts/compare-website-performance.mjs for controlled lab performance;
• scripts/compare-rendered-pages.mjs for rendered parity.

They use only control and candidate arm names. They do not assume a framework, repository, CMS, host, deployment provider, authentication system, route list, or feature-flag product. Run them from a project that provides @playwright/test or playwright:

node /path/to/compare-website-performance.mjs \
  --control-url=https://control.example \
  --candidate-url=https://candidate.example \
  --routes-file=/absolute/path/routes.txt \
  --warmups=2 \
  --runs=7 \
  --output=/tmp/website-performance-ab

node /path/to/compare-rendered-pages.mjs \
  --control-url=https://control.example \
  --candidate-url=https://candidate.example \
  --routes-file=/absolute/path/routes.txt \
  --viewports=desktop,mobile \
  --output=/tmp/rendered-pages-ab

To deliberately send only measured visits to a real-user monitoring collector, enable the optional lifecycle mode and provide a request matcher:

WEBSITE_AB_RUM_REQUEST_PATTERN='(?:/rum/|/vitals(?:[/?#]|$))' \
node /path/to/compare-website-performance.mjs \
  --control-url=https://control.example \
  --candidate-url=https://candidate.example \
  --routes-file=/absolute/path/routes.txt \
  --flush-rum=true

The engine blocks matching telemetry during preflight and warm-up visits. For each measured visit, it captures lab metrics, navigates to about:blank to finalize the page lifecycle, waits for a matching outbound RUM request, and only then closes the context. A missing request fails that sample. The request proves that the browser attempted delivery; verify collector acceptance and ingestion from provider sample counts. This mode creates a scripted cohort; it does not turn the results into representative field evidence.

For an INP-producing workflow, set WEBSITE_AB_INTERACTION_MODULE to an absolute ESM module path. The module must default-export a function or export runInteraction:

export async function runInteraction({ page, arm, route }) {
  await page.getByRole('button', { name: 'Open menu' }).click();
  await page.getByRole('menu').waitFor();
}

The adapter receives the Playwright page, control or candidate arm, and route. Keep site-specific selectors and behavior in this caller-owned module, not in the neutral engine.

Use WEBSITE_AB_CONTROL_CONTEXT_JSON and WEBSITE_AB_CANDIDATE_CONTEXT_JSON to supply Playwright context cookies, headers, or ignoreHTTPSErrors without putting values in command arguments or reports. Use WEBSITE_AB_CONTEXT_ADAPTER_MODULE when context setup needs provider APIs, scoped request interception, or another behavior that plain context JSON cannot express. The absolute ESM module path must default-export a function or export configureContext({ context, arm, baseUrl }). Both portable engines call it before the first navigation.

Do not put a deployment-protection secret in Playwright extraHTTPHeaders when a page can request third-party origins. Those headers apply too broadly. A context adapter must scope the secret to the protected origin.

Optional Vercel Deployment Protection adapter

The portable engines include scripts/vercel-protection-context-adapter.mjs. It can use an explicitly supplied Protection Bypass for Automation secret, or retrieve an existing secret through the authenticated Vercel CLI:

WEBSITE_AB_CONTEXT_ADAPTER_MODULE=/path/to/vercel-protection-context-adapter.mjs \
WEBSITE_AB_VERCEL_BYPASS_FROM_CLI=true \
WEBSITE_AB_VERCEL_PROJECT=example-project \
WEBSITE_AB_VERCEL_SCOPE=example-team \
WEBSITE_AB_VERCEL_PROTECTED_ARMS=candidate \
node /path/to/compare-website-performance.mjs \
  --control-url=https://control.example \
  --candidate-url=https://candidate.example \
  --routes-file=/absolute/path/routes.txt

The optional scope can be omitted. Use WEBSITE_AB_VERCEL_PROTECTED_ARMS=control,candidate when both deployments are protected. WEBSITE_AB_VERCEL_BYPASS_SECRET takes precedence over CLI retrieval when a secret is already available in the environment.

CLI retrieval runs only the read-only project API command. It extracts an existing automation-bypass secret in memory and refuses to create or rotate one. The adapter sends the bypass headers only to requests whose origin matches that arm's configured base URL. Scripts and reports record only that a context adapter was configured; they do not include its module path, project, scope, or secret.

Use --expected-<arm>-header=name:value or the paired --expected-<arm>-meta-selector and --expected-<arm>-meta-value options to reject the wrong revision, source, or variant.

An adapter skill can discover routes, create context JSON for deterministic flags, select a provider context adapter, choose expected markers, and run these engines. Keep provider-specific retrieval and secret handling in context adapters. Do not add provider or product logic to the neutral engines.

6. Measure performance

Report p50, p75, and p95 for:

• LCP, FCP, CLS, and INP or a documented lab proxy;
• TTFB, load, and resource completion;
• long-task count and duration;
• request count and transferred resources.

For bundle candidates, require a deterministic request, byte, parse, or main-thread reduction in addition to timing. Inspect which resources disappeared; aggregate browser timing can undercount cross-origin transfers.

Call the synthetic LCP and CLS results lab Core Web Vitals data. They measure the same rendering and layout signals in a controlled browser, but describe only the scripted environment and sample. A scripted interaction can produce a lab INP estimate; a navigation-only run cannot. Field Core Web Vitals require real-user measurements across eligible page loads and use the field population's p75. Query sample counts with the percentiles and mark a result inconclusive when a route, device class, or variant has insufficient data.

The portable performance engine measures one viewport per invocation. Run it once for each required device profile so samples and summaries remain independent.

7. Verify rendered parity

Run the discovered parity harness against the same revisions, routes, contexts, and expected markers. Include desktop and mobile profiles. Compare the candidate with its exact control build.

Require zero unexplained differences in status, redirects, metadata, visible text, headings, links, media, normalized DOM, structured data, and browser errors. Review screenshot differences separately because animation, font rasterization, and antialiasing can change hashes without changing content.

8. Decide and iterate

For an untrafficked Preview, keep a candidate only when the lab improvement is substantial, repeatable, mechanism-consistent, and parity-safe, without a material tail regression. Deterministic byte or main-thread reductions can justify retention when timings are noisy, but document that the result is an efficiency improvement pending field confirmation.

For a production rollout, require both the pre-release gates and enough variant-attributed field data for the predefined decision. Roll back when error guardrails or material field regressions trigger, even when the lab result was positive.

Reject and revert a candidate that adds errors, changes content, only reshuffles resources, or trades a small median gain for a material p75 or p95 regression. Use a normal revert after publishing shared history.

Use each retained state as the next control. Repeat the workflow for the next independent candidate.

9. Handoff

Document:

• exact control and candidate revisions and test URLs;
• isolation method and verified revision, source, or variant;
• route-selection reasons, warm-ups, samples, ordering, viewport, and concurrency;
• metric percentiles and deterministic resource differences;
• parity result and artifact paths;
• keep or revert decision and its mechanism;
• limitations, including cache state, deployment noise, and synthetic versus field evidence.

State the evidence mode explicitly. For Preview-only results, say that no representative field traffic was available. For production results, report variant sample counts and attribution method with the p75 metrics.

Commit and publish the decision record only with user authorization. Verify the remote revision and preserve unrelated work.

vercel-protection-context-adapter.mjs

import { execFile } from 'node:child_process';
import { promisify } from 'node:util';

const execFileAsync = promisify(execFile);
let cachedSecretPromise;

export function extractVercelAutomationBypassSecret(projectSettings) {
  if (!isRecord(projectSettings)) return undefined;
  const protectionBypass = projectSettings.protectionBypass;
  if (!isRecord(protectionBypass)) return undefined;

  for (const [secret, settings] of Object.entries(protectionBypass)) {
    if (
      secret.trim() &&
      isRecord(settings) &&
      settings.scope === 'automation-bypass'
    ) {
      return secret;
    }
  }
  return undefined;
}

export async function loadVercelAutomationBypassSecret({
  cwd,
  project,
  runCli = runVercelCli,
  scope,
}) {
  const arguments_ = [
    'api',
    `/v9/projects/${encodeURIComponent(project)}`,
    '--raw',
    '--no-color',
    '--non-interactive',
  ];
  if (scope) arguments_.push('--scope', scope);

  let projectSettings;
  try {
    projectSettings = JSON.parse(await runCli(arguments_, cwd));
  } catch {
    throw new Error(
      'Could not read the Vercel project through the authenticated Vercel CLI',
    );
  }

  const secret = extractVercelAutomationBypassSecret(projectSettings);
  if (!secret) {
    throw new Error(
      'The Vercel project has no existing Protection Bypass for Automation secret; retrieval is read-only and will not create or rotate one',
    );
  }
  return secret;
}

export async function installVercelProtectionBypass({
  baseUrl,
  context,
  secret,
}) {
  const protectedOrigin = new URL(baseUrl).origin;
  await context.route('**/*', async (route) => {
    if (new URL(route.request().url()).origin !== protectedOrigin) {
      await route.fallback();
      return;
    }
    await route.fallback({
      headers: {
        ...route.request().headers(),
        'x-vercel-protection-bypass': secret,
        'x-vercel-set-bypass-cookie': 'true',
      },
    });
  });
}

export async function configureContext({ arm, baseUrl, context }) {
  const protectedArms = new Set(
    (process.env.WEBSITE_AB_VERCEL_PROTECTED_ARMS ?? 'candidate')
      .split(',')
      .map((value) => value.trim())
      .filter(Boolean),
  );
  if (!protectedArms.has(arm)) return;

  cachedSecretPromise ??= resolveConfiguredSecret();
  await installVercelProtectionBypass({
    baseUrl,
    context,
    secret: await cachedSecretPromise,
  });
}

async function resolveConfiguredSecret() {
  const explicitSecret = process.env.WEBSITE_AB_VERCEL_BYPASS_SECRET;
  if (explicitSecret) return explicitSecret;
  if (process.env.WEBSITE_AB_VERCEL_BYPASS_FROM_CLI !== 'true') {
    throw new Error(
      'Configure an existing Vercel bypass secret or opt in to read-only CLI retrieval',
    );
  }
  const project = process.env.WEBSITE_AB_VERCEL_PROJECT;
  if (!project) {
    throw new Error(
      'WEBSITE_AB_VERCEL_PROJECT is required for read-only CLI retrieval',
    );
  }
  return loadVercelAutomationBypassSecret({
    cwd: process.cwd(),
    project,
    scope: process.env.WEBSITE_AB_VERCEL_SCOPE,
  });
}

async function runVercelCli(arguments_, cwd) {
  const { stdout } = await execFileAsync('vercel', arguments_, {
    cwd,
    encoding: 'utf8',
    maxBuffer: 2 * 1024 * 1024,
  });
  return stdout;
}

function isRecord(value) {
  return typeof value === 'object' && value !== null && !Array.isArray(value);
}

website-ab-common.mjs

import { createHash } from 'node:crypto';
import { mkdir, readFile, writeFile } from 'node:fs/promises';
import { createRequire } from 'node:module';
import path from 'node:path';
import { pathToFileURL } from 'node:url';

export const ARM_NAMES = ['control', 'candidate'];

export function loadChromium() {
  const projectRequire = createRequire(
    path.join(process.cwd(), 'package.json'),
  );
  for (const packageName of ['@playwright/test', 'playwright']) {
    try {
      return projectRequire(packageName).chromium;
    } catch (error) {
      if (error?.code !== 'MODULE_NOT_FOUND') throw error;
    }
  }
  throw new Error(
    'Install @playwright/test or playwright in the project that runs this script',
  );
}

export function parseArguments(argv) {
  const values = new Map();
  for (const argument of argv) {
    if (!argument.startsWith('--')) continue;
    const separator = argument.indexOf('=');
    values.set(
      argument.slice(2, separator === -1 ? undefined : separator),
      separator === -1 ? 'true' : argument.slice(separator + 1),
    );
  }
  return values;
}

export function booleanArgument(value, fallback) {
  if (value === undefined) return fallback;
  if (value === 'true') return true;
  if (value === 'false') return false;
  throw new Error(`Expected true or false, received ${value}`);
}

export function positiveNumber(value, fallback) {
  const parsed = Number(value);
  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
}

export function optionalRegExp(value, label = 'regular expression') {
  if (!value) return undefined;
  try {
    return new RegExp(value, 'u');
  } catch {
    throw new Error(`Invalid ${label}`);
  }
}

export async function loadInteractionAdapter(modulePath) {
  if (!modulePath) return undefined;
  const module = await import(pathToFileURL(path.resolve(modulePath)).href);
  const adapter = module.runInteraction ?? module.default;
  if (typeof adapter !== 'function') {
    throw new Error(
      'Interaction module must export runInteraction or a default function',
    );
  }
  return adapter;
}

export async function loadContextAdapter(modulePath) {
  if (!modulePath) return undefined;
  const module = await import(pathToFileURL(path.resolve(modulePath)).href);
  const adapter = module.configureContext ?? module.default;
  if (typeof adapter !== 'function') {
    throw new Error(
      'Context module must export configureContext or a default function',
    );
  }
  return adapter;
}

export async function blockMatchingRequests(context, pattern) {
  if (!pattern) return;
  await context.route('**/*', async (route) => {
    pattern.lastIndex = 0;
    if (pattern.test(route.request().url())) {
      await route.abort('blockedbyclient');
      return;
    }
    await route.fallback();
  });
}

export async function flushRumTelemetry({
  adapter,
  arm,
  page,
  requestPattern,
  route,
  timeoutMs,
}) {
  if (!requestPattern) {
    return { interactionPerformed: false, requestObserved: false };
  }
  const requestPromise = page.context().waitForEvent('request', {
    predicate: (request) => {
      requestPattern.lastIndex = 0;
      return requestPattern.test(request.url());
    },
    timeout: timeoutMs,
  });
  try {
    if (adapter) await adapter({ arm, page, route });
    await page.goto('about:blank', { timeout: timeoutMs, waitUntil: 'commit' });
    await requestPromise;
  } catch (error) {
    await requestPromise.catch(() => undefined);
    throw error;
  }
  return {
    interactionPerformed: Boolean(adapter),
    requestObserved: true,
  };
}

export function normalizeOrigin(value) {
  const url = new URL(value);
  url.pathname = '/';
  url.search = '';
  url.hash = '';
  return url.toString().replace(/\/$/u, '');
}

export async function loadRoutes(argumentsMap) {
  const routes = [];
  const inline = argumentsMap.get('routes');
  if (inline) routes.push(...inline.split(','));
  const routesFile = argumentsMap.get('routes-file');
  if (routesFile) {
    routes.push(...(await readFile(routesFile, 'utf8')).split(/\r?\n/u));
  }
  const normalized = routes
    .map((route) => route.trim())
    .filter(Boolean)
    .map((route) => {
      try {
        return new URL(route).pathname;
      } catch {
        return route.startsWith('/') ? route : `/${route}`;
      }
    })
    .map((route) => route.replace(/\/+$/u, '') || '/');
  return [...new Set(normalized.length > 0 ? normalized : ['/'])].sort();
}

export function parseViewports(value = 'desktop') {
  const presets = {
    desktop: { name: 'desktop', width: 1440, height: 900 },
    mobile: { name: 'mobile', width: 390, height: 844 },
  };
  return value.split(',').map((entry) => {
    const name = entry.trim();
    if (presets[name]) return presets[name];
    const match = name.match(/^(?<width>\d+)x(?<height>\d+)$/u);
    if (!match?.groups) throw new Error(`Unknown viewport ${name}`);
    return {
      name,
      width: Number(match.groups.width),
      height: Number(match.groups.height),
    };
  });
}

export function readContextConfiguration(arm) {
  const key = `WEBSITE_AB_${arm.toUpperCase()}_CONTEXT_JSON`;
  const raw = process.env[key];
  if (!raw) return {};
  let parsed;
  try {
    parsed = JSON.parse(raw);
  } catch {
    throw new Error(`${key} must contain valid JSON`);
  }
  if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
    throw new Error(`${key} must contain a JSON object`);
  }
  return parsed;
}

export async function createContext(browser, arm, baseUrl, viewport) {
  const configuration = readContextConfiguration(arm);
  const context = await browser.newContext({
    ignoreHTTPSErrors: Boolean(configuration.ignoreHTTPSErrors),
    viewport: { width: viewport.width, height: viewport.height },
    ...(configuration.extraHTTPHeaders
      ? { extraHTTPHeaders: configuration.extraHTTPHeaders }
      : {}),
  });
  try {
    if (
      Array.isArray(configuration.cookies) &&
      configuration.cookies.length > 0
    ) {
      await context.addCookies(
        configuration.cookies.map((cookie) => ({
          ...cookie,
          ...(cookie.url || cookie.domain
            ? {}
            : { url: new URL(baseUrl).origin }),
        })),
      );
    }
    const adapter = await loadContextAdapter(
      process.env.WEBSITE_AB_CONTEXT_ADAPTER_MODULE,
    );
    if (adapter) await adapter({ arm, baseUrl, context });
    return context;
  } catch (error) {
    await context.close().catch(() => undefined);
    throw error;
  }
}

export function parseExpectedHeader(value) {
  if (!value) return undefined;
  const separator = value.indexOf(':');
  if (separator < 1) throw new Error('Expected header must use name:value');
  return {
    name: value.slice(0, separator).trim().toLowerCase(),
    value: value.slice(separator + 1).trim(),
  };
}

export function parseExpectedMeta(selector, value) {
  if (!selector && !value) return undefined;
  if (!selector || value === undefined) {
    throw new Error('Expected meta verification requires selector and value');
  }
  return { selector, value };
}

export async function assertExpectedPage(options) {
  const { arm, baseUrl, expectedHeader, expectedMeta, page, response } =
    options;
  const status = response?.status() ?? null;
  if (status === null || status < 200 || status >= 400) {
    throw new Error(`${arm} returned HTTP ${status ?? 'unknown'}`);
  }
  if (new URL(page.url()).origin !== new URL(baseUrl).origin) {
    throw new Error(`${arm} navigation left its configured origin`);
  }
  if (expectedHeader) {
    const actual = await response?.headerValue(expectedHeader.name);
    if (actual !== expectedHeader.value) {
      throw new Error(`${arm} resolved an unexpected response marker`);
    }
  }
  if (expectedMeta) {
    const actual = await page
      .locator(expectedMeta.selector)
      .getAttribute('content');
    if (actual !== expectedMeta.value) {
      throw new Error(`${arm} resolved an unexpected document marker`);
    }
  }
}

export function percentile(values, fraction) {
  if (values.length === 0) return null;
  const sorted = [...values].sort((left, right) => left - right);
  return sorted[Math.ceil(fraction * sorted.length) - 1] ?? sorted.at(-1);
}

export function summarizeValues(values) {
  const finite = values.filter((value) => Number.isFinite(value));
  return {
    samples: finite.length,
    min: finite.length ? Math.min(...finite) : null,
    p50: percentile(finite, 0.5),
    p75: percentile(finite, 0.75),
    p95: percentile(finite, 0.95),
    max: finite.length ? Math.max(...finite) : null,
  };
}

export function hash(value) {
  return createHash('sha256').update(value).digest('hex');
}

export function artifactName(route) {
  const label = route.replace(/[^a-zA-Z0-9]+/gu, '_').replace(/^_|_$/gu, '');
  return `${label || 'home'}-${hash(route).slice(0, 8)}`;
}

export async function writeArtifacts(outputDirectory, report, summary) {
  await mkdir(outputDirectory, { recursive: true });
  await Promise.all([
    writeFile(
      path.join(outputDirectory, 'report.json'),
      `${JSON.stringify(report, null, 2)}\n`,
    ),
    writeFile(path.join(outputDirectory, 'summary.md'), `${summary.trim()}\n`),
  ]);
}

export function formatNumber(value) {
  return value === null ? 'n/a' : String(Math.round(value * 100) / 100);
}