ESLint--v9.34-Concurrency--Benchmarks.md

ESLint Concurrency Benchmark

Overview

• Benchmarks ESLint v9.34+ --concurrency across one or more targets
• Runs each concurrency setting multiple times and aggregates stats
• Prints a fastest-per-target table and a per-target breakdown (★ marks best)
• Optional verbose mode prints ESLint commands, stderr, and one stylish preview

Defaults

• concurrency: off,4,8,auto
• runs: 3
• outDir: ./eslint-perf

Flags

• --targets=dir[,dir...]: Comma-separated target directories
• --concurrency=v[,v...]: values in {off,auto,1..N}
• --runs=N: Runs per configuration (default 3)
• --verbose: Show ESLint commands/stderr and one stylish preview per target
• --outDir=path: Directory for JSON + summary outputs
• --patterns=glob[,glob...]: (bench.ts) Explicit file globs to lint (e.g. packages/utils/**/*.ts,packages/utils/**/*.tsx)

Examples

• Minimal (defaults):

node --import tsx bench.ts

• Custom targets + concurrency:

node --import tsx bench.ts \
  --targets=packages/lib-a,packages/lib-b \
  --concurrency=off,2,4,auto \
  --runs=5

• Custom patterns (bench.ts):

node --import tsx bench.ts \
  --targets=packages/lib-a \
  --patterns="packages/lib-a/**/*.ts,packages/lib-a/**/*.tsx" \
  --concurrency=off,4,auto

Output

• Raw JSON: ./eslint-perf/system-info-*.json and ./eslint-perf/eslint-benchmark-*.json
• Summary: ./eslint-perf/*.summary.md
• Terminal: fastest table + per-target breakdown (★ marks best)

Output Example

> node --import tsx tools/eslint-perf/
--targets=packages/lib-a --concurrency=off,1,2,4,6,8,auto --runs=3 --verbose

📁 Target: packages/lib-a (files: 106, ts: 101, js: 5)
  🔧 Concurrency: off
    Run 1/3 ...
      $ npx eslint --config="packages/lib-a/eslint.config.js" --concurrency=off --format=json "packages/lib-a/**/*.ts"
      Time: 8.764s, Files: 101, Errors: 0, Warnings: 0
    Run 2/3 ...
    ✅ Avg: 9.49s (min 9.385s, max 9.615s, ±0.095s)
...

✅ Benchmark complete
Raw results: tools/eslint-perf/results/eslint-benchmark-2025-08-23T01-39-46-558Z.json
Summary: ./eslint-perf/eslint-benchmark-2025-08-23T01-39-46-558Z.summary.md

Fastest per target:
Target                        Best    Avg(s)    Baseline(s)   Speedup
---------------------------------------------------------------------
packages/lib-a                off     8.849     8.849         1.00x

Per-target breakdown (best marked with ★):

packages/lib-a
Concurrency   Avg(s)    StdDev    Speedup   Mark
------------------------------------------------
off           8.849     0.078     1.00x     ★
1             9.024     0.094     0.98x     
2             9.615     0.107     0.92x     
4             9.742     0.190     0.91x     
6             11.652    0.439     0.76x     
8             13.749    0.195     0.64x     
auto          9.490     0.095     0.93x     

CPU Measures

# Profile CPU usage while running ESLint with explicit config and patterns
npx @push-based/cpu-prof@latest -- \
  npx eslint -c packages/lib-a/eslint.config.js \
  "packages/lib-a/**/*.ts" \
  --concurrency=4

What this does:

• Starts ESLint with Node CPU profiling enabled and collects .cpuprofile files
• Merges them into a single Chrome trace JSON for easy inspection

Expected output:

• Profiles folder: ./profiles (or a printed absolute path)
• Individual .cpuprofile files for the run
• Merged trace JSON like Trace-<timestamp>.json
• Open the merged JSON in Chrome DevTools → Performance → Load profile to view threads, tasks, and CPU usage with --concurrency=4

bench.ts

import { exec as execCb } from 'node:child_process';
import { mkdir, readdir, stat, writeFile } from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import { promisify } from 'node:util';

const exec = promisify(execCb);

type C = 'off' | 'auto' | `${number}`;
type Run = {
  target: string;
  concurrency: C;
  run: number;
  timing: {
    durationSeconds: number;
    realTimeSeconds: null;
    userTimeSeconds: null;
    sysTimeSeconds: null;
  };
  eslintResults: {
    filesProcessed: number;
    errors: number;
    warnings: number;
    eslintWarnings: string[];
  };
  timestamp: string;
};
type Stats = {
  runs: number;
  min: number;
  max: number;
  avg: number;
  stdDev: number;
};
type TI = {
  target: string;
  tsFiles: number;
  jsFiles: number;
  totalFiles: number;
};

const DFLT_TARGETS = [
  'packages/models',
  'packages/plugin-eslint',
  'packages/cli',
  'packages/core',
  'packages/utils',
];
const DFLT_CONC: C[] = ['off', '4', '8', 'auto'];
const DFLT_RUNS = 3;
const pad = (s: string, n: number) =>
  s.length >= n ? s : s + ' '.repeat(n - s.length);
const exists = async (p: string) => !!(await stat(p).catch(() => null));
const ensureDir = (d: string) => mkdir(d, { recursive: true });

function args() {
  const a = process.argv.slice(2);
  const get = (n: string) =>
    a.find(x => x.startsWith(`--${n}=`))?.split('=')[1];
  const targets = get('targets')?.split(',').filter(Boolean) ?? DFLT_TARGETS;
  const conc =
    (get('concurrency')?.split(',').filter(Boolean) as C[] | undefined) ??
    DFLT_CONC;
  const runs = Number.parseInt(get('runs') ?? `${DFLT_RUNS}`, 10);
  const outDir = get('outDir') ?? path.join('tools', 'eslint-perf', 'results');
  const verbose = a.includes('--verbose') || get('verbose') === 'true';
  const useLocalConfig = get('useLocalConfig') === 'false' ? false : true;
  const tsOnly = get('tsOnly') === 'false' ? false : true;
  return { targets, conc, runs, outDir, verbose, useLocalConfig, tsOnly };
}

async function sys() {
  const nx = await exec('npx nx --version')
    .then(r => r.stdout.trim().split('\n')[0])
    .catch(() => null);
  return {
    timestamp: new Date().toISOString(),
    system: {
      os: os.platform(),
      osVersion: os.release(),
      architecture: os.arch(),
      cpuCores: os.cpus()?.length ?? 0,
      totalMemGb: Math.round((os.totalmem() / 1024 ** 3) * 100) / 100,
      hostname: os.hostname(),
    },
    software: {
      nodeVersion: process.version,
      npmVersion: (await exec('npm --version')).stdout.trim(),
      eslintVersion: (await exec('npx eslint --version')).stdout.trim(),
      nxVersion: nx,
    },
  };
}

async function count(dir: string, exts: string[]) {
  let n = 0;
  const w = async (d: string): Promise<void> => {
    const es = await readdir(d, { withFileTypes: true });
    for (const e of es) {
      const p = path.join(d, e.name);
      if (e.isDirectory()) {
        if (['node_modules', 'dist', 'coverage'].includes(e.name)) continue;
        await w(p);
      } else if (e.isFile() && exts.some(ext => e.name.endsWith(ext))) n++;
    }
  };
  await w(dir);
  return n;
}
async function info(t: string): Promise<TE> {
  const abs = path.resolve(t);
  if (!(await exists(abs)))
    return { target: t, tsFiles: 0, jsFiles: 0, totalFiles: 0 };
  const ts = await count(abs, ['.ts', '.tsx']);
  const js = await count(abs, ['.js', '.jsx', '.mjs', '.cjs']);
  return { target: t, tsFiles: ts, jsFiles: js, totalFiles: ts + js };
}
type TE = TI;

const cmd = (
  t: string,
  c: C,
  o: { local: boolean; ts: boolean; json: boolean },
) =>
  `npx eslint${o.local ? ` --config=${JSON.stringify(path.join(t, 'eslint.config.js'))}` : ''} --concurrency=${c} ${o.json ? '--format=json' : '--format=stylish'} ${[o.ts ? path.join(t, '**', '*.ts') : t].map(p => JSON.stringify(p)).join(' ')}`;

async function once(
  t: string,
  c: C,
  i: number,
  v: boolean,
  local: boolean,
  ts: boolean,
): Promise<Run> {
  const s = process.hrtime.bigint();
  const k = cmd(t, c, { local, ts, json: true });
  let out = '',
    err = '';
  try {
    const r = await exec(k, { maxBuffer: 1024 * 1024 * 200 });
    out = r.stdout;
    err = r.stderr;
  } catch (e: any) {
    out = e.stdout ?? '';
    err = e.stderr ?? '';
  }
  if (v) {
    console.log(`      $ ${k}`);
    if (err.trim()) {
      console.log('      [stderr]');
      console.log(
        err
          .split('\n')
          .map(l => `      ${l}`)
          .join('\n'),
      );
    }
  }
  const d = Number(process.hrtime.bigint() - s) / 1_000_000_000;
  let files = 0,
    errors = 0,
    warnings = 0;
  try {
    const j = JSON.parse(out) as Array<{
      errorCount: number;
      warningCount: number;
    }>;
    files = j.length;
    for (const r of j) {
      errors += r.errorCount || 0;
      warnings += r.warningCount || 0;
    }
  } catch {}
  const warns = err.includes('ESLintPoorConcurrencyWarning')
    ? ['ESLintPoorConcurrencyWarning']
    : [];
  return {
    target: t,
    concurrency: c,
    run: i,
    timing: {
      durationSeconds: d,
      realTimeSeconds: null,
      userTimeSeconds: null,
      sysTimeSeconds: null,
    },
    eslintResults: {
      filesProcessed: files,
      errors,
      warnings,
      eslintWarnings: warns,
    },
    timestamp: new Date().toISOString(),
  };
}

const statz = (xs: number[]): Stats => {
  const n = xs.length,
    min = Math.min(...xs),
    max = Math.max(...xs),
    avg = xs.reduce((a, b) => a + b, 0) / n,
    sd = Math.sqrt(xs.reduce((a, v) => a + (v - avg) ** 2, 0) / n);
  return {
    runs: n,
    min,
    max,
    avg: Number(avg.toFixed(3)),
    stdDev: Number(sd.toFixed(3)),
  };
};

async function stylish(t: string) {
  const k = cmd(t, 'off', { local: true, ts: true, json: false });
  try {
    const { stdout, stderr } = await exec(k, { maxBuffer: 1024 * 1024 * 200 });
    if (stdout.trim()) {
      console.log('    [stylish output]');
      console.log(
        stdout
          .split('\n')
          .slice(0, 300)
          .map(l => `    ${l}`)
          .join('\n'),
      );
    }
    if (stderr.trim()) {
      console.log('    [stylish stderr]');
      console.log(
        stderr
          .split('\n')
          .map(l => `    ${l}`)
          .join('\n'),
      );
    }
  } catch (e: any) {
    const o = e.stdout ?? '',
      er = e.stderr ?? '';
    if (o.trim()) {
      console.log('    [stylish output]');
      console.log(
        o
          .split('\n')
          .slice(0, 300)
          .map((l: string) => `    ${l}`)
          .join('\n'),
      );
    }
    if (er.trim()) {
      console.log('    [stylish stderr]');
      console.log(
        er
          .split('\n')
          .map((l: string) => `    ${l}`)
          .join('\n'),
      );
    }
  }
}

async function main() {
  const { targets, conc, runs, outDir, verbose, useLocalConfig, tsOnly } =
    args();
  await ensureDir(outDir);
  const si = await sys(),
    ts = new Date().toISOString().replace(/[:.]/g, '-');
  const resPath = path.join(outDir, `eslint-benchmark-${ts}.json`),
    sumPath = path.join(outDir, `eslint-benchmark-${ts}.summary.md`);
  const results: any[] = [];
  await writeFile(
    path.join(outDir, `system-info-${ts}.json`),
    JSON.stringify(si, null, 2),
  );
  for (const t of targets) {
    if (!(await exists(path.resolve(t)))) {
      console.warn(`Skipping missing target: ${t}`);
      continue;
    }
    const ti = await info(t);
    console.log(
      `\n📁 Target: ${t} (files: ${ti.totalFiles}, ts: ${ti.tsFiles}, js: ${ti.jsFiles})`,
    );
    if (verbose) await stylish(t);
    for (const c of conc) {
      console.log(`  🔧 Concurrency: ${c}`);
      const rs: Run[] = [];
      for (let i = 1; i <= runs; i++) {
        console.log(`    Run ${i}/${runs} ...`);
        const r = await once(t, c, i, verbose, useLocalConfig, tsOnly);
        rs.push(r);
        console.log(
          `      Time: ${r.timing.durationSeconds.toFixed(3)}s, Files: ${r.eslintResults.filesProcessed}, Errors: ${r.eslintResults.errors}, Warnings: ${r.eslintResults.warnings}${r.eslintResults.eslintWarnings.length ? ' ⚠️ ' + r.eslintResults.eslintWarnings.join(',') : ''}`,
        );
      }
      const s = statz(rs.map(r => r.timing.durationSeconds));
      results.push({
        target: t,
        targetInfo: ti,
        concurrency: c,
        statistics: s,
        runs: rs,
      });
      console.log(
        `    ✅ Avg: ${s.avg}s (min ${s.min.toFixed(3)}s, max ${s.max.toFixed(3)}s, ±${s.stdDev}s)`,
      );
    }
  }
  await writeFile(
    resPath,
    JSON.stringify({ systemInfo: si, results }, null, 2),
  );

  // Build Markdown summary (system info, fastest table, per-target breakdown)
  const tset = Array.from(new Set(results.map((r: any) => r.target)));
  const rows = tset.map(t => {
    const xs = results.filter((r: any) => r.target === t);
    const best = xs.reduce((a: any, b: any) =>
      a.statistics.avg <= b.statistics.avg ? a : b,
    );
    const base =
      xs.find((r: any) => r.concurrency === 'off')?.statistics.avg ?? null;
    const speed = base ? (base / best.statistics.avg).toFixed(2) + 'x' : 'n/a';
    return {
      t,
      bestConc: best.concurrency as C,
      bestAvg: Number(best.statistics.avg.toFixed(3)),
      base,
      speed,
    };
  });

  const md: string[] = [];
  md.push('# ESLint Concurrency Benchmark Report');
  md.push('');
  md.push(`Generated: ${new Date().toString()}`);
  md.push('');
  md.push('## System');
  md.push(`- **OS**: ${si.system.os} ${si.system.osVersion}`);
  md.push(`- **CPU Cores**: ${si.system.cpuCores}`);
  md.push(`- **Memory**: ${si.system.totalMemGb} GB`);
  md.push(`- **Node**: \\`${si.software.nodeVersion}\\``);
  md.push(`- **ESLint**: \\`${si.software.eslintVersion}\\``);
  if (si.software.nxVersion) md.push(`- **Nx**: \\`${si.software.nxVersion}\\``);
  md.push('');

  md.push('## Fastest per target');
  md.push('');
  md.push('| Target | Best | Avg (s) | Baseline (s) | Speedup |');
  md.push('|---|:---:|---:|---:|---:|');
  for (const r of rows) {
    md.push(`| \\`${r.t}\\` | **${r.bestConc}** | **${r.bestAvg.toFixed(3)}** | ${r.base ? r.base.toFixed(3) : 'n/a'} | **${r.speed}** |`);
  }
  md.push('');

  md.push('## Per-target breakdown');
  md.push('Best rows are marked with ★.');
  for (const t of tset) {
    const xs = results.filter((r: any) => r.target === t);
    const base = xs.find((r: any) => r.concurrency === 'off')?.statistics.avg ?? null;
    const bestAvg = Math.min(...xs.map((r: any) => r.statistics.avg));
    md.push('');
    md.push(`### \\`${t}\\``);
    md.push('');
    md.push('| Concurrency | Avg (s) | StdDev | Speedup |');
    md.push('|:-----------:|---:|---:|---:|');
    for (const c of Array.from(new Set(xs.map((r: any) => r.concurrency)))) {
      const r = xs.find((q: any) => q.concurrency === c)!;
      const isBest = Math.abs(r.statistics.avg - bestAvg) < 1e-6;
      const sp = base ? `${(base / r.statistics.avg).toFixed(2)}x` : 'n/a';
      const row = `| \\`${String(c)}\\` | ${r.statistics.avg.toFixed(3)} | ${r.statistics.stdDev.toFixed(3)} | ${sp} | ${isBest ? '★' : ''}`;
      md.push(isBest ? row.replace(/\| ([^|]+) \|/, match => match.replace(/([^|]+)/, '**$1**')) : row);
    }
  }

  await writeFile(sumPath, md.join('\n'));
  console.log('\n✅ Benchmark complete');
  console.log(`Raw results: ${resPath}`);
  console.log(`Summary: ${sumPath}`);
  // Reuse computed tset/rows for terminal tables
  const header = `${pad('Target', 28)}  ${pad('Best', 6)}  ${pad('Avg(s)', 8)}  ${pad('Baseline(s)', 12)}  Speedup`;
  console.log('\nFastest per target:');
  console.log(header);
  console.log('-'.repeat(header.length));
  for (const r of rows)
    console.log(
      `${pad(r.t, 28)}  ${pad(String(r.bestConc), 6)}  ${pad(r.bestAvg.toFixed(3), 8)}  ${pad(r.base ? r.base.toFixed(3) : 'n/a', 12)}  ${r.speed}`,
    );
  console.log('\nPer-target breakdown (best marked with ★):');
  for (const t of tset) {
    const xs = results.filter((r: any) => r.target === t);
    const base =
      xs.find((r: any) => r.concurrency === 'off')?.statistics.avg ?? null;
    const best = Math.min(...xs.map((r: any) => r.statistics.avg));
    console.log(`\n${t}`);
    const h2 = `${pad('Concurrency', 12)}  ${pad('Avg(s)', 8)}  ${pad('StdDev', 8)}  ${pad('Speedup', 8)}  Mark`;
    console.log(h2);
    console.log('-'.repeat(h2.length));
    for (const c of Array.from(new Set(xs.map((r: any) => r.concurrency)))) {
      const r = xs.find((q: any) => q.concurrency === c)!;
      const sp = base ? `${(base / r.statistics.avg).toFixed(2)}x` : 'n/a';
      console.log(
        `${pad(String(c), 12)}  ${pad(r.statistics.avg.toFixed(3), 8)}  ${pad(r.statistics.stdDev.toFixed(3), 8)}  ${pad(sp, 8)}  ${Math.abs(r.statistics.avg - best) < 1e-6 ? '★' : ''}`,
      );
    }
  }
}

main().catch(e => {
  console.error(e);
  process.exitCode = 1;
});