Prowl build benchmark scripts for PR #478 evaluation

benchmark-build.sh

#!/bin/bash
set -euo pipefail

# Benchmark the full `make build-app` and `make test-app` pipelines,
# with per-phase timing for deps vs xcodebuild.
#
# Usage:
#   ./scripts/benchmark-build.sh <label>
#
# Examples:
#   ./scripts/benchmark-build.sh baseline
#   ./scripts/benchmark-build.sh pr478

LABEL="${1:?Usage: $0 <label>}"
set +u

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
cd "$PROJECT_DIR"

RESULTS_DIR="$PROJECT_DIR/benchmark-results"
RESULTS_FILE="$RESULTS_DIR/${LABEL}.txt"
MUTATION_FILE="$PROJECT_DIR/supacode/App/supacodeApp.swift"

NOOP_RUNS=3
MUTATE_RUNS=3

mkdir -p "$RESULTS_DIR"

now() {
  python3 -c 'import time; print(f"{time.time():.3f}")'
}

elapsed() {
  python3 -c "import time; print(f'{time.time() - $1:.2f}')"
}

log() {
  echo "$1" | tee -a "$RESULTS_FILE"
}

separator() {
  log "────────────────────────────────────────────────────────────"
}

MUTATION_ANCHOR="func applicationDidFinishLaunching"
MUTATION_LINE='    let _ = 42 /* BENCHMARK_MUTATION_MARKER */'

mutate_file() {
  if grep -q "BENCHMARK_MUTATION_MARKER" "$MUTATION_FILE"; then
    grep -v "BENCHMARK_MUTATION_MARKER" "$MUTATION_FILE" > "${MUTATION_FILE}.tmp" && mv "${MUTATION_FILE}.tmp" "$MUTATION_FILE"
  else
    local line_num
    line_num=$(grep -n "$MUTATION_ANCHOR" "$MUTATION_FILE" | head -1 | cut -d: -f1)
    if [ -z "$line_num" ]; then
      echo "ERROR: mutation anchor not found" >&2
      return 1
    fi
    local insert_at=$((line_num + 2))
    head -n "$insert_at" "$MUTATION_FILE" > "${MUTATION_FILE}.tmp"
    echo "$MUTATION_LINE" >> "${MUTATION_FILE}.tmp"
    tail -n +"$((insert_at + 1))" "$MUTATION_FILE" >> "${MUTATION_FILE}.tmp"
    mv "${MUTATION_FILE}.tmp" "$MUTATION_FILE"
  fi
}

restore_file() {
  if grep -q "BENCHMARK_MUTATION_MARKER" "$MUTATION_FILE" 2>/dev/null; then
    grep -v "BENCHMARK_MUTATION_MARKER" "$MUTATION_FILE" > "${MUTATION_FILE}.tmp" && mv "${MUTATION_FILE}.tmp" "$MUTATION_FILE"
  fi
}

clean_derived_data() {
  rm -rf ~/Library/Developer/Xcode/DerivedData/supacode-* 2>/dev/null || true
}

clean_build_settings_cache() {
  rm -f "$PROJECT_DIR/.build_settings_cache.json" 2>/dev/null || true
}

# ── Header ──
: > "$RESULTS_FILE"
log "=== Make Pipeline Benchmark: $LABEL ==="
log "Date: $(date '+%Y-%m-%d %H:%M:%S')"
log "Commit: $(git rev-parse --short HEAD) ($(git log -1 --format=%s))"
log "Branch: $(git branch --show-current)"
log "Mutation target: $(basename "$MUTATION_FILE")"
log ""

# ════════════════════════════════════════════════════════════════
# SECTION A: Dependency Phase Timing
# ════════════════════════════════════════════════════════════════
separator
log "SECTION A: Dependency Targets (individual timing)"
separator
log ""

# Time each dep target individually (sequential)
for target in ensure-ghostty embed-cli-debug embed-docs; do
  start=$(now)
  make -C "$PROJECT_DIR" "$target" >/dev/null 2>&1
  dur=$(elapsed "$start")
  log "  $target: ${dur}s"
done
log ""

# Time deps via make build-app's dep mechanism
# On baseline: sequential (ensure-ghostty embed-cli-debug embed-docs)
# On PR: parallel (_build-app-all-deps with -j3)
if make -C "$PROJECT_DIR" -n _build-app-all-deps >/dev/null 2>&1; then
  log "  Parallel dep target (_build-app-all-deps) detected."
  start=$(now)
  make -C "$PROJECT_DIR" _build-app-all-deps >/dev/null 2>&1
  dur=$(elapsed "$start")
  log "  _build-app-all-deps (parallel): ${dur}s"
else
  log "  No parallel dep target; deps run sequentially in build-app."
  start=$(now)
  make -C "$PROJECT_DIR" ensure-ghostty embed-cli-debug embed-docs >/dev/null 2>&1
  dur=$(elapsed "$start")
  log "  sequential deps total: ${dur}s"
fi
log ""

# ════════════════════════════════════════════════════════════════
# SECTION B: make build-app (full pipeline)
# ════════════════════════════════════════════════════════════════
separator
log "SECTION B: make build-app (full pipeline)"
separator
log ""

# B1. Cold build
log "B1. Cold build"
clean_derived_data
clean_build_settings_cache

start=$(now)
make -C "$PROJECT_DIR" build-app >/dev/null 2>&1
dur=$(elapsed "$start")
log "  make build-app (cold): ${dur}s"
log ""

# B2. No-change builds
log "B2. No-change builds ($NOOP_RUNS runs)"
for i in $(seq 1 $NOOP_RUNS); do
  start=$(now)
  make -C "$PROJECT_DIR" build-app >/dev/null 2>&1
  dur=$(elapsed "$start")
  log "  noop_${i}: ${dur}s"
done
log ""

# B3. Single-file mutation builds
log "B3. Single-file mutation builds ($MUTATE_RUNS runs)"
for i in $(seq 1 $MUTATE_RUNS); do
  mutate_file
  start=$(now)
  make -C "$PROJECT_DIR" build-app >/dev/null 2>&1
  dur=$(elapsed "$start")
  log "  mutate_${i}: ${dur}s"
done
restore_file
log ""

# ════════════════════════════════════════════════════════════════
# SECTION C: make test-app (test pipeline)
# ════════════════════════════════════════════════════════════════
separator
log "SECTION C: make test-app"
separator
log ""

# C1. No-change test (DerivedData warm from Section B)
log "C1. No-change test ($NOOP_RUNS runs)"
for i in $(seq 1 $NOOP_RUNS); do
  start=$(now)
  make -C "$PROJECT_DIR" test-app >/dev/null 2>&1 || true
  dur=$(elapsed "$start")
  log "  test_noop_${i}: ${dur}s"
done
log ""

# C2. Single-file mutation test
log "C2. Single-file mutation test ($MUTATE_RUNS runs)"
for i in $(seq 1 $MUTATE_RUNS); do
  mutate_file
  start=$(now)
  make -C "$PROJECT_DIR" test-app >/dev/null 2>&1 || true
  dur=$(elapsed "$start")
  log "  test_mutate_${i}: ${dur}s"
done
restore_file
log ""

# ════════════════════════════════════════════════════════════════
separator
log "Results written to: $RESULTS_FILE"
log "=== Done ==="

benchmark-cold-cache.sh

#!/bin/bash
set -euo pipefail

# Verify cold-cache hypothesis: purge filesystem cache, then run
# cold → 3 noops → mutation for wholemodule vs incremental.
# Requires sudo for purge.

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
cd "$PROJECT_DIR"

SPM_CACHE_DIR="$HOME/Library/Caches/supacode-spm-cache/SourcePackages"
TARGET_FILE="$PROJECT_DIR/supacode/App/supacodeApp.swift"
RESULTS_DIR="$PROJECT_DIR/benchmark-results"
RESULTS_FILE="$RESULTS_DIR/cold-cache.txt"
mkdir -p "$RESULTS_DIR"

now() { python3 -c 'import time; print(f"{time.time():.3f}")'; }
elapsed() { python3 -c "import time; print(f'{time.time() - $1:.2f}')"; }
log() { echo "$1" | tee -a "$RESULTS_FILE"; }

XCODE_ARGS=(-project supacode.xcodeproj -scheme supacode -configuration Debug build
  -skipMacroValidation -clonedSourcePackagesDirPath "$SPM_CACHE_DIR")

run_build() {
  local mode="${1:-}"
  local args=("${XCODE_ARGS[@]}")
  if [ -n "$mode" ]; then
    args+=("SWIFT_COMPILATION_MODE=$mode")
  fi
  xcodebuild "${args[@]}" >/dev/null 2>&1
}

apply_mutation() {
  local anchor="func applicationDidFinishLaunching"
  if grep -q "BENCHMARK_MUTATION_MARKER" "$TARGET_FILE"; then
    grep -v "BENCHMARK_MUTATION_MARKER" "$TARGET_FILE" > "${TARGET_FILE}.tmp"
    mv "${TARGET_FILE}.tmp" "$TARGET_FILE"
  else
    local line_num
    line_num=$(grep -n "$anchor" "$TARGET_FILE" | head -1 | cut -d: -f1)
    local insert_at=$((line_num + 2))
    head -n "$insert_at" "$TARGET_FILE" > "${TARGET_FILE}.tmp"
    echo '    let _ = 42 /* BENCHMARK_MUTATION_MARKER */' >> "${TARGET_FILE}.tmp"
    tail -n +"$((insert_at + 1))" "$TARGET_FILE" >> "${TARGET_FILE}.tmp"
    mv "${TARGET_FILE}.tmp" "$TARGET_FILE"
  fi
}

restore() {
  if grep -q "BENCHMARK_MUTATION_MARKER" "$TARGET_FILE" 2>/dev/null; then
    grep -v "BENCHMARK_MUTATION_MARKER" "$TARGET_FILE" > "${TARGET_FILE}.tmp"
    mv "${TARGET_FILE}.tmp" "$TARGET_FILE"
  fi
}

run_sequence() {
  local label="$1"
  local mode="$2"

  log ""
  log "── $label (mode=${mode:-wholemodule}) ──"

  # Purge + clean DerivedData
  log "  Cleaning DerivedData..."
  rm -rf ~/Library/Developer/Xcode/DerivedData/supacode-* 2>/dev/null || true
  if sudo -n purge 2>/dev/null; then
    log "  Filesystem cache purged."
  else
    log "  WARNING: sudo purge failed (password not cached). Run: sudo -v"
    exit 1
  fi
  sleep 2

  # Cold build
  local start dur
  start=$(now); run_build "$mode"; dur=$(elapsed "$start")
  log "  cold:     ${dur}s"

  # 3 noops
  for i in 1 2 3; do
    start=$(now); run_build "$mode"; dur=$(elapsed "$start")
    log "  noop_$i:   ${dur}s"
  done

  # 1 mutation
  apply_mutation
  start=$(now); run_build "$mode"; dur=$(elapsed "$start")
  log "  mutate_1: ${dur}s"
  restore
}

: > "$RESULTS_FILE"
log "=== Cold Cache Benchmark ==="
log "Date: $(date '+%Y-%m-%d %H:%M:%S')"
log ""

# Ensure deps (before purge)
make -C "$PROJECT_DIR" ensure-ghostty embed-cli-debug embed-docs >/dev/null 2>&1

run_sequence "TEST 1: Cold cache + wholemodule" ""
run_sequence "TEST 2: Cold cache + incremental" "incremental"

# Control: warm cache (no purge before this one)
log ""
log "── CONTROL: Warm cache + wholemodule (no purge) ──"
rm -rf ~/Library/Developer/Xcode/DerivedData/supacode-* 2>/dev/null || true

start=$(now); run_build; dur=$(elapsed "$start")
log "  cold:     ${dur}s"
for i in 1 2 3; do
  start=$(now); run_build; dur=$(elapsed "$start")
  log "  noop_$i:   ${dur}s"
done
apply_mutation
start=$(now); run_build; dur=$(elapsed "$start")
log "  mutate_1: ${dur}s"
restore

log ""
log "=== Done ==="

benchmark-investigate.sh

#!/bin/bash
set -euo pipefail

# Controlled 2x2 investigation: mutation type × compilation mode
# All use direct xcodebuild (no make, no xcsift) to isolate variables.

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
cd "$PROJECT_DIR"

SPM_CACHE_DIR="$HOME/Library/Caches/supacode-spm-cache/SourcePackages"
TARGET_FILE="$PROJECT_DIR/supacode/App/supacodeApp.swift"
RESULTS_DIR="$PROJECT_DIR/benchmark-results"
RESULTS_FILE="$RESULTS_DIR/investigation.txt"
mkdir -p "$RESULTS_DIR"

now() { python3 -c 'import time; print(f"{time.time():.3f}")'; }
elapsed() { python3 -c "import time; print(f'{time.time() - $1:.2f}')"; }

log() { echo "$1" | tee -a "$RESULTS_FILE"; }

run_xcodebuild() {
  local mode="${1:-}"
  local args=(-project supacode.xcodeproj -scheme supacode -configuration Debug build
    -skipMacroValidation -clonedSourcePackagesDirPath "$SPM_CACHE_DIR")
  if [ -n "$mode" ]; then
    args+=("SWIFT_COMPILATION_MODE=$mode")
  fi
  xcodebuild "${args[@]}" >/dev/null 2>&1
}

clean() {
  rm -rf ~/Library/Developer/Xcode/DerivedData/supacode-* 2>/dev/null || true
}

apply_toplevel_mutation() {
  if grep -q "BENCHMARK_MUTATION_MARKER" "$TARGET_FILE"; then
    grep -v "BENCHMARK_MUTATION_MARKER" "$TARGET_FILE" > "${TARGET_FILE}.tmp"
    mv "${TARGET_FILE}.tmp" "$TARGET_FILE"
  else
    echo "private let BENCHMARK_MUTATION_MARKER = true" >> "$TARGET_FILE"
  fi
}

apply_funcbody_mutation() {
  local anchor="func applicationDidFinishLaunching"
  if grep -q "BENCHMARK_MUTATION_MARKER" "$TARGET_FILE"; then
    grep -v "BENCHMARK_MUTATION_MARKER" "$TARGET_FILE" > "${TARGET_FILE}.tmp"
    mv "${TARGET_FILE}.tmp" "$TARGET_FILE"
  else
    local line_num
    line_num=$(grep -n "$anchor" "$TARGET_FILE" | head -1 | cut -d: -f1)
    local insert_at=$((line_num + 2))
    head -n "$insert_at" "$TARGET_FILE" > "${TARGET_FILE}.tmp"
    echo '    let _ = 42 /* BENCHMARK_MUTATION_MARKER */' >> "${TARGET_FILE}.tmp"
    tail -n +"$((insert_at + 1))" "$TARGET_FILE" >> "${TARGET_FILE}.tmp"
    mv "${TARGET_FILE}.tmp" "$TARGET_FILE"
  fi
}

restore() {
  if grep -q "BENCHMARK_MUTATION_MARKER" "$TARGET_FILE" 2>/dev/null; then
    grep -v "BENCHMARK_MUTATION_MARKER" "$TARGET_FILE" > "${TARGET_FILE}.tmp"
    mv "${TARGET_FILE}.tmp" "$TARGET_FILE"
  fi
}

run_test() {
  local label="$1"
  local mode="$2"
  local mutate_fn="$3"

  log ""
  log "── $label ──"
  log "  Compilation mode: ${mode:-default(wholemodule)}"
  log "  Mutation type: $mutate_fn"

  # Clean + cold build
  clean
  log "  Cleaning DerivedData..."
  local start dur
  start=$(now)
  run_xcodebuild "$mode"
  dur=$(elapsed "$start")
  log "  cold_build: ${dur}s"

  # Single noop to establish baseline
  start=$(now)
  run_xcodebuild "$mode"
  dur=$(elapsed "$start")
  log "  noop: ${dur}s"

  # Mutation build (3 runs)
  for i in 1 2 3; do
    $mutate_fn
    start=$(now)
    run_xcodebuild "$mode"
    dur=$(elapsed "$start")
    log "  mutation_${i}: ${dur}s"
  done
  restore
}

: > "$RESULTS_FILE"
log "=== Controlled Investigation: Mutation Type × Compilation Mode ==="
log "Date: $(date '+%Y-%m-%d %H:%M:%S')"
log "Commit: $(git rev-parse --short HEAD)"
log ""

# Ensure deps are satisfied (not timed)
log "Preparing deps..."
make -C "$PROJECT_DIR" ensure-ghostty embed-cli-debug embed-docs >/dev/null 2>&1
log "Deps ready."

# 4 tests: 2 mutation types × 2 compilation modes
run_test "TEST 1: wholemodule + top-level private let" "" "apply_toplevel_mutation"
run_test "TEST 2: incremental + top-level private let" "incremental" "apply_toplevel_mutation"
run_test "TEST 3: wholemodule + function-body let" "" "apply_funcbody_mutation"
run_test "TEST 4: incremental + function-body let" "incremental" "apply_funcbody_mutation"

log ""
log "=== Investigation Complete ==="

benchmark-make-only.sh

#!/bin/bash
set -euo pipefail

# Reproduce the exact make build-app sequence that showed 25.78s mutation.
# Also measures xcodebuild output volume per step to check back-pressure.

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
cd "$PROJECT_DIR"

TARGET_FILE="$PROJECT_DIR/supacode/App/supacodeApp.swift"
RESULTS_DIR="$PROJECT_DIR/benchmark-results"
RESULTS_FILE="$RESULTS_DIR/make-reproduce.txt"
mkdir -p "$RESULTS_DIR"

now() { python3 -c 'import time; print(f"{time.time():.3f}")'; }
elapsed() { python3 -c "import time; print(f'{time.time() - $1:.2f}')"; }
log() { echo "$1" | tee -a "$RESULTS_FILE"; }

apply_mutation() {
  local anchor="func applicationDidFinishLaunching"
  if grep -q "BENCHMARK_MUTATION_MARKER" "$TARGET_FILE"; then
    grep -v "BENCHMARK_MUTATION_MARKER" "$TARGET_FILE" > "${TARGET_FILE}.tmp"
    mv "${TARGET_FILE}.tmp" "$TARGET_FILE"
  else
    local line_num
    line_num=$(grep -n "$anchor" "$TARGET_FILE" | head -1 | cut -d: -f1)
    local insert_at=$((line_num + 2))
    head -n "$insert_at" "$TARGET_FILE" > "${TARGET_FILE}.tmp"
    echo '    let _ = 42 /* BENCHMARK_MUTATION_MARKER */' >> "${TARGET_FILE}.tmp"
    tail -n +"$((insert_at + 1))" "$TARGET_FILE" >> "${TARGET_FILE}.tmp"
    mv "${TARGET_FILE}.tmp" "$TARGET_FILE"
  fi
}

restore() {
  if grep -q "BENCHMARK_MUTATION_MARKER" "$TARGET_FILE" 2>/dev/null; then
    grep -v "BENCHMARK_MUTATION_MARKER" "$TARGET_FILE" > "${TARGET_FILE}.tmp"
    mv "${TARGET_FILE}.tmp" "$TARGET_FILE"
  fi
}

: > "$RESULTS_FILE"
log "=== make build-app Reproduction ==="
log "Date: $(date '+%Y-%m-%d %H:%M:%S')"
log ""

# Clean
rm -rf ~/Library/Developer/Xcode/DerivedData/supacode-* 2>/dev/null || true

# Cold
start=$(now)
make -C "$PROJECT_DIR" build-app >/dev/null 2>&1
log "cold:     $(elapsed "$start")s"

# 3 noops
for i in 1 2 3; do
  start=$(now)
  make -C "$PROJECT_DIR" build-app >/dev/null 2>&1
  log "noop_$i:   $(elapsed "$start")s"
done

# 3 mutations
for i in 1 2 3; do
  apply_mutation
  start=$(now)
  make -C "$PROJECT_DIR" build-app >/dev/null 2>&1
  log "mutate_$i: $(elapsed "$start")s"
done
restore

log ""

# Now measure xcodebuild output volume
log "── Output volume (lines from xcodebuild) ──"
SPM_CACHE_DIR="$HOME/Library/Caches/supacode-spm-cache/SourcePackages"
XCODE_ARGS="-project supacode.xcodeproj -scheme supacode -configuration Debug build -skipMacroValidation -clonedSourcePackagesDirPath $SPM_CACHE_DIR"

lines=$(xcodebuild $XCODE_ARGS 2>&1 | wc -l)
log "noop output: $lines lines"

apply_mutation
lines=$(xcodebuild $XCODE_ARGS 2>&1 | wc -l)
log "mutation output: $lines lines"
restore

log ""
log "=== Done ==="

benchmark-pipe.sh

#!/bin/bash
set -euo pipefail

# Test whether piping xcodebuild through xcsift causes the slowdown.
# Sequence: cold → 1 noop → mutation, repeated 3 ways:
#   A) xcodebuild > /dev/null
#   B) xcodebuild | xcsift > /dev/null
#   C) xcodebuild | mise exec -- xcsift > /dev/null

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
cd "$PROJECT_DIR"

SPM_CACHE_DIR="$HOME/Library/Caches/supacode-spm-cache/SourcePackages"
TARGET_FILE="$PROJECT_DIR/supacode/App/supacodeApp.swift"
RESULTS_DIR="$PROJECT_DIR/benchmark-results"
RESULTS_FILE="$RESULTS_DIR/pipe-test.txt"
mkdir -p "$RESULTS_DIR"

XCSIFT_PATH="$(mise which xcsift 2>/dev/null)"

now() { python3 -c 'import time; print(f"{time.time():.3f}")'; }
elapsed() { python3 -c "import time; print(f'{time.time() - $1:.2f}')"; }
log() { echo "$1" | tee -a "$RESULTS_FILE"; }

apply_mutation() {
  local anchor="func applicationDidFinishLaunching"
  if grep -q "BENCHMARK_MUTATION_MARKER" "$TARGET_FILE"; then
    grep -v "BENCHMARK_MUTATION_MARKER" "$TARGET_FILE" > "${TARGET_FILE}.tmp"
    mv "${TARGET_FILE}.tmp" "$TARGET_FILE"
  else
    local line_num
    line_num=$(grep -n "$anchor" "$TARGET_FILE" | head -1 | cut -d: -f1)
    local insert_at=$((line_num + 2))
    head -n "$insert_at" "$TARGET_FILE" > "${TARGET_FILE}.tmp"
    echo '    let _ = 42 /* BENCHMARK_MUTATION_MARKER */' >> "${TARGET_FILE}.tmp"
    tail -n +"$((insert_at + 1))" "$TARGET_FILE" >> "${TARGET_FILE}.tmp"
    mv "${TARGET_FILE}.tmp" "$TARGET_FILE"
  fi
}

restore() {
  if grep -q "BENCHMARK_MUTATION_MARKER" "$TARGET_FILE" 2>/dev/null; then
    grep -v "BENCHMARK_MUTATION_MARKER" "$TARGET_FILE" > "${TARGET_FILE}.tmp"
    mv "${TARGET_FILE}.tmp" "$TARGET_FILE"
  fi
}

clean() { rm -rf ~/Library/Developer/Xcode/DerivedData/supacode-* 2>/dev/null || true; }

XCODE_ARGS=(-project supacode.xcodeproj -scheme supacode -configuration Debug build
  -skipMacroValidation -clonedSourcePackagesDirPath "$SPM_CACHE_DIR")

: > "$RESULTS_FILE"
log "=== Pipe Back-Pressure Test ==="
log "Date: $(date '+%Y-%m-%d %H:%M:%S')"
log "xcsift path: $XCSIFT_PATH"
log ""

# Ensure deps
make -C "$PROJECT_DIR" ensure-ghostty embed-cli-debug embed-docs >/dev/null 2>&1

# ── Test A: direct to /dev/null ──
log "── A: xcodebuild > /dev/null ──"
clean
start=$(now)
xcodebuild "${XCODE_ARGS[@]}" >/dev/null 2>&1
log "  cold: $(elapsed "$start")s"

start=$(now)
xcodebuild "${XCODE_ARGS[@]}" >/dev/null 2>&1
log "  noop: $(elapsed "$start")s"

apply_mutation
start=$(now)
xcodebuild "${XCODE_ARGS[@]}" >/dev/null 2>&1
log "  mutation: $(elapsed "$start")s"
restore
log ""

# ── Test B: pipe through xcsift (direct path) ──
log "── B: xcodebuild | xcsift > /dev/null ──"
clean
start=$(now)
bash -o pipefail -c "xcodebuild ${XCODE_ARGS[*]} 2>&1 | $XCSIFT_PATH -w --format toon" >/dev/null 2>&1
log "  cold: $(elapsed "$start")s"

start=$(now)
bash -o pipefail -c "xcodebuild ${XCODE_ARGS[*]} 2>&1 | $XCSIFT_PATH -w --format toon" >/dev/null 2>&1
log "  noop: $(elapsed "$start")s"

apply_mutation
start=$(now)
bash -o pipefail -c "xcodebuild ${XCODE_ARGS[*]} 2>&1 | $XCSIFT_PATH -w --format toon" >/dev/null 2>&1
log "  mutation: $(elapsed "$start")s"
restore
log ""

# ── Test C: pipe through mise exec -- xcsift ──
log "── C: xcodebuild | mise exec -- xcsift > /dev/null ──"
clean
start=$(now)
bash -o pipefail -c "xcodebuild ${XCODE_ARGS[*]} 2>&1 | mise exec -- xcsift -w --format toon" >/dev/null 2>&1
log "  cold: $(elapsed "$start")s"

start=$(now)
bash -o pipefail -c "xcodebuild ${XCODE_ARGS[*]} 2>&1 | mise exec -- xcsift -w --format toon" >/dev/null 2>&1
log "  noop: $(elapsed "$start")s"

apply_mutation
start=$(now)
bash -o pipefail -c "xcodebuild ${XCODE_ARGS[*]} 2>&1 | mise exec -- xcsift -w --format toon" >/dev/null 2>&1
log "  mutation: $(elapsed "$start")s"
restore
log ""

log "=== Done ==="

benchmark-reproduce.sh

#!/bin/bash
set -euo pipefail

# Attempt to reproduce the ~25s mutation_1 by replicating the exact
# sequence from the earlier full benchmark: cold → 3 noops → 3 mutations.
# Includes CPU clock and thermal info between each step.

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
cd "$PROJECT_DIR"

SPM_CACHE_DIR="$HOME/Library/Caches/supacode-spm-cache/SourcePackages"
TARGET_FILE="$PROJECT_DIR/supacode/App/supacodeApp.swift"
RESULTS_DIR="$PROJECT_DIR/benchmark-results"
RESULTS_FILE="$RESULTS_DIR/reproduce.txt"
mkdir -p "$RESULTS_DIR"

now() { python3 -c 'import time; print(f"{time.time():.3f}")'; }
elapsed() { python3 -c "import time; print(f'{time.time() - $1:.2f}')"; }

log() { echo "$1" | tee -a "$RESULTS_FILE"; }

cpu_info() {
  local freq pstate
  freq=$(sysctl -n hw.cpufrequency_max 2>/dev/null || echo "N/A")
  pstate=$(pmset -g therm 2>/dev/null | grep -i "cpu_speed_limit" || echo "N/A")
  echo "CPU freq_max=$freq $pstate"
}

run_xcodebuild() {
  local mode="${1:-}"
  local args=(-project supacode.xcodeproj -scheme supacode -configuration Debug build
    -skipMacroValidation -clonedSourcePackagesDirPath "$SPM_CACHE_DIR")
  if [ -n "$mode" ]; then
    args+=("SWIFT_COMPILATION_MODE=$mode")
  fi
  xcodebuild "${args[@]}" >/dev/null 2>&1
}

apply_funcbody_mutation() {
  local anchor="func applicationDidFinishLaunching"
  if grep -q "BENCHMARK_MUTATION_MARKER" "$TARGET_FILE"; then
    grep -v "BENCHMARK_MUTATION_MARKER" "$TARGET_FILE" > "${TARGET_FILE}.tmp"
    mv "${TARGET_FILE}.tmp" "$TARGET_FILE"
  else
    local line_num
    line_num=$(grep -n "$anchor" "$TARGET_FILE" | head -1 | cut -d: -f1)
    local insert_at=$((line_num + 2))
    head -n "$insert_at" "$TARGET_FILE" > "${TARGET_FILE}.tmp"
    echo '    let _ = 42 /* BENCHMARK_MUTATION_MARKER */' >> "${TARGET_FILE}.tmp"
    tail -n +"$((insert_at + 1))" "$TARGET_FILE" >> "${TARGET_FILE}.tmp"
    mv "${TARGET_FILE}.tmp" "$TARGET_FILE"
  fi
}

restore() {
  if grep -q "BENCHMARK_MUTATION_MARKER" "$TARGET_FILE" 2>/dev/null; then
    grep -v "BENCHMARK_MUTATION_MARKER" "$TARGET_FILE" > "${TARGET_FILE}.tmp"
    mv "${TARGET_FILE}.tmp" "$TARGET_FILE"
  fi
}

: > "$RESULTS_FILE"
log "=== Reproduction Test ==="
log "Date: $(date '+%Y-%m-%d %H:%M:%S')"
log "Sequence: cold → 3 noops → 3 mutations (same as full benchmark)"
log "Mode: wholemodule (default)"
log ""

# Ensure deps
make -C "$PROJECT_DIR" ensure-ghostty embed-cli-debug embed-docs >/dev/null 2>&1

# Clean + cold build
log "── Cleaning DerivedData ──"
rm -rf ~/Library/Developer/Xcode/DerivedData/supacode-* 2>/dev/null || true
log "  $(cpu_info)"

start=$(now); run_xcodebuild; dur=$(elapsed "$start")
log "  cold_build: ${dur}s"
log "  $(cpu_info)"
log ""

# 3 noop builds
log "── 3 Noop Builds ──"
for i in 1 2 3; do
  start=$(now); run_xcodebuild; dur=$(elapsed "$start")
  log "  noop_${i}: ${dur}s"
done
log "  $(cpu_info)"
log ""

# 3 mutation builds (wholemodule)
log "── 3 Mutation Builds (wholemodule) ──"
for i in 1 2 3; do
  apply_funcbody_mutation
  log "  $(cpu_info)"
  start=$(now); run_xcodebuild; dur=$(elapsed "$start")
  log "  mutation_${i}: ${dur}s"
done
restore
log ""

# Now the same sequence but incremental
log "════════════════════════════════════════"
log "── Same sequence with INCREMENTAL ──"
log ""

# Clean + cold build
log "── Cleaning DerivedData ──"
rm -rf ~/Library/Developer/Xcode/DerivedData/supacode-* 2>/dev/null || true
sleep 3  # brief CPU cooldown
log "  $(cpu_info)"

start=$(now); run_xcodebuild "incremental"; dur=$(elapsed "$start")
log "  cold_build: ${dur}s"
log "  $(cpu_info)"
log ""

# 3 noop builds
log "── 3 Noop Builds ──"
for i in 1 2 3; do
  start=$(now); run_xcodebuild "incremental"; dur=$(elapsed "$start")
  log "  noop_${i}: ${dur}s"
done
log "  $(cpu_info)"
log ""

# 3 mutation builds (incremental)
log "── 3 Mutation Builds (incremental) ──"
for i in 1 2 3; do
  apply_funcbody_mutation
  log "  $(cpu_info)"
  start=$(now); run_xcodebuild "incremental"; dur=$(elapsed "$start")
  log "  mutation_${i}: ${dur}s"
done
restore
log ""

log "=== Done ==="