fsharplint pathological benchmark

bench-fsharplint.sh

#!/usr/bin/env bash
# Benchmark four FSharpLint variants on a target repo's solution, head-to-head:
#
#   1. released      the published dotnet-fsharplint tool        (default 0.27.0)
#   2. master        FSharpLint master                           (default origin/master)
#   3. +#846         master + parallel per-file type-checking    (PR #846, variant C)
#   4. +share        #846 + cross-project WorkspaceLoader/FSharpChecker sharing
#
# Emits a PR-comment-style markdown table (solution lint time + speedup vs master).
#
# Self-contained: clones the target repo, installs the released tool, and builds
# the three local refs from a FSharpLint checkout via git worktrees. Everything
# lands under $WORKDIR; nothing is assumed to already exist on the machine.
#
# Requirements: bash, git, hyperfine, and a .NET install (see DOTNET_BUILD / DOTNET_RUN).
#
# The built CLI targets the TFM in FSharpLint's global.json (net9), but the target
# repo may target a newer runtime, so the CLI runs with DOTNET_ROLL_FORWARD=LatestMajor
# and project-cracking uses whatever SDK $DOTNET_RUN exposes. If one `dotnet` has every
# needed SDK, leave defaults; if build/run SDKs live in separate roots (mise/asdf),
# point DOTNET_BUILD and DOTNET_RUN at each.
#
# Configuration — all optional, override via environment:
#   FSLINT_REPO     FSharpLint checkout to build from   (default: git root of this script)
#   REF_MASTER      master ref                          (default: origin/master)
#   REF_846         parallel-per-file ref               (default: ee218763)
#   REF_SHARE       sharing ref (stack tip)             (default: 78f134b3)
#   RELEASED_VER    released tool version               (default: 0.27.0)
#   TARGET_URL      git URL of the repo to lint         (default: FsHotWatch)
#   TARGET_REF      ref to check out in the target      (default: its default branch)
#   TARGET_CONFIG   fsharplint config, relative to repo (default: fsharplint.json)
#   TARGET_SOLUTION solution to lint, relative path     (default: FsHotWatch.slnx)
#   WORKDIR         scratch dir                         (default: ./.fsharplint-bench)
#   DOTNET_BUILD    dotnet used to BUILD the CLIs       (default: $DOTNET or `dotnet`)
#   DOTNET_RUN      dotnet used to RUN the benchmarks   (default: $DOTNET or `dotnet`)
#   SLOW_RUNS       hyperfine runs for slow variants    (default: 2  — released & master)
#   FAST_RUNS       hyperfine runs for fast variants    (default: 3  — #846 & share)
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
FSLINT_REPO="${FSLINT_REPO:-$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel 2>/dev/null || echo "$SCRIPT_DIR")}"
REF_MASTER="${REF_MASTER:-origin/master}"
REF_846="${REF_846:-ee218763}"
REF_SHARE="${REF_SHARE:-78f134b3}"
RELEASED_VER="${RELEASED_VER:-0.27.0}"
TARGET_URL="${TARGET_URL:-https://github.com/michaelglass/FsHotWatch.git}"
TARGET_REF="${TARGET_REF:-}"
TARGET_CONFIG="${TARGET_CONFIG:-fsharplint.json}"
TARGET_SOLUTION="${TARGET_SOLUTION:-FsHotWatch.slnx}"
WORKDIR="${WORKDIR:-$PWD/.fsharplint-bench}"
DOTNET_BUILD="${DOTNET_BUILD:-${DOTNET:-dotnet}}"
DOTNET_RUN="${DOTNET_RUN:-${DOTNET:-dotnet}}"
SLOW_RUNS="${SLOW_RUNS:-2}"
FAST_RUNS="${FAST_RUNS:-3}"
export DOTNET_ROLL_FORWARD="${DOTNET_ROLL_FORWARD:-LatestMajor}"

for tool in git hyperfine "$DOTNET_BUILD" "$DOTNET_RUN"; do
    command -v "$tool" >/dev/null 2>&1 || { echo "error: '$tool' not found on PATH" >&2; exit 1; }
done
DOTNET_RUN_ROOT="$(cd "$(dirname "$(command -v "$DOTNET_RUN")")" && pwd)"

mkdir -p "$WORKDIR"
CONSOLE="src/FSharpLint.Console/FSharpLint.Console.fsproj"
RESULTS="$WORKDIR/results.md"

# --- target repo -------------------------------------------------------------
TARGET="$WORKDIR/target"
if [ ! -e "$TARGET/.git" ]; then
    git clone "$TARGET_URL" "$TARGET"
    [ -n "$TARGET_REF" ] && git -C "$TARGET" checkout "$TARGET_REF"
fi
SLN="$TARGET/$TARGET_SOLUTION"
CFG="$TARGET/$TARGET_CONFIG"

# --- released tool -----------------------------------------------------------
TOOLDIR="$WORKDIR/released-$RELEASED_VER"
if [ ! -x "$TOOLDIR/dotnet-fsharplint" ]; then
    "$DOTNET_RUN" tool install dotnet-fsharplint --version "$RELEASED_VER" --tool-path "$TOOLDIR"
fi
RELEASED_BIN="$TOOLDIR/dotnet-fsharplint"

# --- build one FSharpLint CLI from a git ref into a worktree, echo its dll ----
build_cli () {
    local ref="$1" name="$2" wt="$WORKDIR/build-$2"
    if [ ! -e "$wt/.git" ]; then
        git -C "$FSLINT_REPO" worktree add --force --detach "$wt" "$ref" >&2
    fi
    ( cd "$wt" && DOTNET_ROOT="$(cd "$(dirname "$(command -v "$DOTNET_BUILD")")" && pwd)" \
        "$DOTNET_BUILD" build -c Release "$CONSOLE" \
        -p:EnableSourceLink=false -p:EnableSourceControlManagerQueries=false \
        -p:DeterministicSourcePaths=false -p:EmbedUntrackedSources=false >&2 )
    ls "$wt"/src/FSharpLint.Console/bin/Release/*/dotnet-fsharplint.dll | head -1
}
DLL_MASTER="$(build_cli "$REF_MASTER" master)"
DLL_846="$(build_cli "$REF_846" pr846)"
DLL_SHARE="$(build_cli "$REF_SHARE" share)"

# --- benchmark one variant on the solution, echo "name<TAB>seconds" ----------
export PATH="$DOTNET_RUN_ROOT:$PATH"
bench () {
    local name="$1" runs="$2"; shift 2
    local json="$WORKDIR/hf-$name.json"
    hyperfine --warmup 0 --runs "$runs" --export-json "$json" \
        --command-name "$name" "$* lint -l '$CFG' '$SLN'" >&2
    printf '%s\t%s\n' "$name" \
        "$(python3 -c "import json;print(round(json.load(open('$json'))['results'][0]['mean'],1))")"
}

{
  echo "| stage | solution lint | vs master |"
  echo "|---|---:|---:|"
} > "$RESULTS"

declare -A T
T[released]="$(bench released "$SLOW_RUNS" "$RELEASED_BIN")"
T[master]="$(bench master    "$SLOW_RUNS" "$DOTNET_RUN $DLL_MASTER")"
T[846]="$(bench 846          "$FAST_RUNS" "$DOTNET_RUN $DLL_846")"
T[share]="$(bench share      "$FAST_RUNS" "$DOTNET_RUN $DLL_SHARE")"

sec () { echo "${1#*$'\t'}"; }
M="$(sec "${T[master]}")"
row () { # label, time
    awk -v l="$1" -v t="$2" -v m="$M" 'BEGIN{printf "| %s | ~%s s | ~%.1f× |\n", l, t, m/t}'
}
{
  row "released $RELEASED_VER"                    "$(sec "${T[released]}")"
  row "master (\`$REF_MASTER\`)"                  "$(sec "${T[master]}")"
  row "+ #846 parallel per-file (\`$REF_846\`)"   "$(sec "${T[846]}")"
  row "+ cross-project sharing (\`$REF_SHARE\`)"  "$(sec "${T[share]}")"
} >> "$RESULTS"

echo >&2; echo "=== results table ($RESULTS) ===" >&2
cat "$RESULTS"

bench-fshotwatch.sh

#!/usr/bin/env bash
# Benchmark FSharpLint PR #846 (parallel per-file type checking) head-to-head
# against the released tool and master, on the FsHotWatch repo, with hyperfine.
# Emits PR-comment-style tables (the Relative column is slowdown vs the fastest).
#
# Self-contained: it clones the target repo, installs the released
# dotnet-fsharplint tool, and builds `master` and the PR ref from a FSharpLint
# checkout via git worktrees. Everything lands under $WORKDIR; nothing is assumed
# to already exist on the machine.
#
# Requirements: bash, git, hyperfine, and a .NET install (see DOTNET_BUILD /
# DOTNET_RUN below).
#
# The built CLI targets an older TFM (per FSharpLint's global.json) but the target
# repo may target a newer one, so the CLI is run with DOTNET_ROLL_FORWARD=LatestMajor
# (a net9 app on a net10 runtime) and project-cracking uses whatever SDK $DOTNET_RUN
# exposes. If one `dotnet` has every needed SDK, leave the defaults. If build- and
# run-time SDKs live in separate roots (e.g. under mise/asdf), point DOTNET_BUILD and
# DOTNET_RUN at each.
#
# Configuration — all optional, override via environment:
#   FSLINT_REPO    FSharpLint checkout to build from   (default: git root of this script)
#   REF_MASTER     master ref to build                 (default: origin/master)
#   REF_PR         PR ref to build                     (default: the PR #846 branch)
#   RELEASED_VER   released tool version to install    (default: 0.27.0)
#   TARGET_URL     git URL of the repo to lint         (default: FsHotWatch)
#   TARGET_REF     ref to check out in the target      (default: its default branch)
#   TARGET_CONFIG  fsharplint config, relative to repo (default: fsharplint.json)
#   TARGET_SOLUTION  solution to lint, relative path   (default: FsHotWatch.slnx)
#   TARGET_PROJECT   project to lint, relative path    (default: src/FsHotWatch/FsHotWatch.fsproj)
#   WORKDIR        scratch dir for clones/builds/output (default: ./.fsharplint-bench)
#   DOTNET_BUILD   dotnet used to build the CLIs       (default: $DOTNET or `dotnet`)
#   DOTNET_RUN     dotnet used to run the benchmarks   (default: $DOTNET or `dotnet`)
#   PROJECT_RUNS   hyperfine runs for the project      (default: 5)
#   SOLUTION_RUNS  hyperfine runs for the solution     (default: 1, it is slow)
#   SKIP_PROJECT / SKIP_SOLUTION   set to skip a target
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

FSLINT_REPO="${FSLINT_REPO:-$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel 2>/dev/null || echo "$SCRIPT_DIR")}"
REF_MASTER="${REF_MASTER:-origin/master}"
REF_PR="${REF_PR:-perf/skip-type-check-and-fix-async-naming-overhead}"
RELEASED_VER="${RELEASED_VER:-0.27.0}"

TARGET_URL="${TARGET_URL:-https://github.com/michaelglass/FsHotWatch.git}"
TARGET_REF="${TARGET_REF:-}"
TARGET_CONFIG="${TARGET_CONFIG:-fsharplint.json}"
TARGET_SOLUTION="${TARGET_SOLUTION:-FsHotWatch.slnx}"
TARGET_PROJECT="${TARGET_PROJECT:-src/FsHotWatch/FsHotWatch.fsproj}"

WORKDIR="${WORKDIR:-$PWD/.fsharplint-bench}"
DOTNET_BUILD="${DOTNET_BUILD:-${DOTNET:-dotnet}}"
DOTNET_RUN="${DOTNET_RUN:-${DOTNET:-dotnet}}"
PROJECT_RUNS="${PROJECT_RUNS:-5}"
SOLUTION_RUNS="${SOLUTION_RUNS:-1}"

export DOTNET_ROLL_FORWARD="${DOTNET_ROLL_FORWARD:-LatestMajor}"

for tool in git hyperfine "$DOTNET_BUILD" "$DOTNET_RUN"; do
    command -v "$tool" >/dev/null 2>&1 || { echo "error: '$tool' not found on PATH"; exit 1; }
done

# .NET resolves the SDK from the CWD's global.json and finds the runtime/SDK under
# DOTNET_ROOT (the dir holding the dotnet binary). Builds need the SDK pinned by
# FSharpLint's global.json; the benchmark needs the target's SDK on PATH so
# FSharpLint's project-cracking can resolve it. Derive a root for each.
DOTNET_BUILD_ROOT="$(cd "$(dirname "$(command -v "$DOTNET_BUILD")")" && pwd)"
DOTNET_RUN_ROOT="$(cd "$(dirname "$(command -v "$DOTNET_RUN")")" && pwd)"

mkdir -p "$WORKDIR"
OUT_DIR="$WORKDIR/results"; mkdir -p "$OUT_DIR"
CONSOLE="src/FSharpLint.Console/FSharpLint.Console.fsproj"

# --- build one FSharpLint CLI from a git ref into a worktree, echo its dll -----
build_cli () {
    local ref="$1" name="$2" wt="$WORKDIR/build-$2"
    if [ ! -e "$wt/.git" ]; then
        git -C "$FSLINT_REPO" worktree add --force --detach "$wt" "$ref" >&2
    fi
    # first TFM declared by the Console project (e.g. net9.0)
    local tfm
    tfm="$(grep -oE '<TargetFrameworks?>[^<]+' "$wt/$CONSOLE" | head -1 \
            | sed -E 's/.*>//; s/;.*//')"
    # build from inside the worktree so its global.json governs SDK selection
    ( cd "$wt" && DOTNET_ROOT="$DOTNET_BUILD_ROOT" PATH="$DOTNET_BUILD_ROOT:$PATH" \
        "$DOTNET_BUILD" build -c Release -f "$tfm" \
            -p:EnableSourceLink=false -p:EnableSourceControlManagerQueries=false \
            -p:DeterministicSourcePaths=false -p:EmbedUntrackedSources=false \
            "$CONSOLE" >&2 )
    echo "$wt/src/FSharpLint.Console/bin/Release/$tfm/dotnet-fsharplint.dll"
}

echo "==> building variants under $WORKDIR" >&2
CLI_MASTER="$(build_cli "$REF_MASTER" master)"
CLI_PR="$(build_cli "$REF_PR" pr846)"

# --- released tool via NuGet -------------------------------------------------
TOOLDIR="$WORKDIR/released-$RELEASED_VER"
if [ ! -x "$TOOLDIR/dotnet-fsharplint" ]; then
    # install from WORKDIR (no global.json) so any available SDK can run the command
    ( cd "$WORKDIR" && DOTNET_ROOT="$DOTNET_BUILD_ROOT" PATH="$DOTNET_BUILD_ROOT:$PATH" \
        "$DOTNET_BUILD" tool install dotnet-fsharplint --version "$RELEASED_VER" --tool-path "$TOOLDIR" >&2 )
fi
# prefer the newest TFM the tool ships (e.g. net9.0 over net8.0)
CLI_RELEASED="$(find "$TOOLDIR/.store" -path '*tools*' -name 'dotnet-fsharplint.dll' | sort | tail -1)"

# --- target repo -------------------------------------------------------------
TARGET_DIR="$WORKDIR/$(basename "$TARGET_URL" .git)"
if [ ! -e "$TARGET_DIR/.git" ]; then
    git clone "$TARGET_URL" "$TARGET_DIR" >&2
fi
[ -n "$TARGET_REF" ] && git -C "$TARGET_DIR" checkout "$TARGET_REF" >&2

CONFIG="$TARGET_DIR/$TARGET_CONFIG"
SOLUTION="$TARGET_DIR/$TARGET_SOLUTION"
PROJECT="$TARGET_DIR/$TARGET_PROJECT"

# variant list as "label<TAB>dll"
VARIANTS=$(printf '%s\t%s\n' \
    "$RELEASED_VER (released)" "$CLI_RELEASED" \
    "master"                   "$CLI_MASTER" \
    "PR#846"                   "$CLI_PR")

echo
echo "dotnet (run): $("$DOTNET_RUN" --version)  (roll-forward $DOTNET_ROLL_FORWARD)"
echo "config      : $CONFIG"
echo

bench_target () {
    local kind="$1" target="$2" runs="$3" warmup="$4"
    local out="$OUT_DIR/${kind}.md"
    local -a args=(--warmup "$warmup" --runs "$runs" --ignore-failure --export-markdown "$out")
    while IFS=$'\t' read -r label dll; do
        args+=(--command-name "$label" "$DOTNET_RUN $dll lint -l $CONFIG $target")
    done <<< "$VARIANTS"
    echo "==> $kind  (runs=$runs warmup=$warmup): $target"
    # run from the target repo with the run-SDK on PATH so project-cracking resolves it
    ( cd "$TARGET_DIR" && DOTNET_ROOT="$DOTNET_RUN_ROOT" PATH="$DOTNET_RUN_ROOT:$PATH" \
        hyperfine "${args[@]}" )
    echo; echo "--- $out ---"; cat "$out"; echo
}

if [ -z "${SKIP_PROJECT:-}" ];  then bench_target "project"  "$PROJECT"  "$PROJECT_RUNS"  1; fi
if [ -z "${SKIP_SOLUTION:-}" ]; then bench_target "solution" "$SOLUTION" "$SOLUTION_RUNS" 0; fi