Savvy vs Bazel format benchmark script for services-pilot

benchmark.sh

#!/usr/bin/env bash
# Benchmark: savvy format vs bazel run //:format
#
# Creates a temporary worktree, runs both formatters on test files
# under different cache conditions, and prints a comparison table.
#
# Usage: ./tools/savvy/benchmark.sh [-n RUNS]
set -euo pipefail

RUNS=3
SETS=""
while getopts "n:s:" opt; do
  case "$opt" in
  n) RUNS="$OPTARG" ;;
  s) SETS="$OPTARG" ;;
  *)
    echo "Usage: $0 [-n RUNS] [-s SETS]" >&2
    echo "  -n RUNS   Number of runs per scenario (default: 3)" >&2
    echo "  -s SETS   Comma-separated file sets to run: SMALL,LARGE,LARGE_BAZEL" >&2
    echo "            (default: all three)" >&2
    exit 1
    ;;
  esac
done

if [[ -z "$SETS" ]]; then
  SETS="SMALL,LARGE,LARGE_BAZEL"
fi
IFS=',' read -ra SET_LIST <<< "$SETS"

SOURCE_DIR="$(cd "$(dirname "$0")/../.." && pwd)"
BENCH_DIR="${TMPDIR:-/tmp}/savvy-bench-$$"
OLD_COMMIT="42ed2e7755ebb" # ~2 weeks ago (2026-05-08)

# --- File sets ---

# Small: typical quick fix, no bazel-dependent formatters
SMALL_FILES=(
  "backend-golden-path/be210-smurray/src/main/java/com/spotify/be210smurray/Main.java"
  "tools/errorprone/patch-all-the-things.py"
  "ci/functions.sh"
  "crane-config.yaml"
)

# Large: realistic PR, no bazel-dependent formatters (.conf/.go excluded)
# shellcheck disable=SC2034  # used via nameref
LARGE_FILES=(
  "backend-golden-path/be210-smurray/src/main/java/com/spotify/be210smurray/Main.java"
  "backend-golden-path/be210-smurray/src/main/java/com/spotify/be210smurray/ConcatBusinessLogic.java"
  "backend-golden-path/be210-smurray/src/main/java/com/spotify/be210smurray/ConcatServiceImpl.java"
  "backend-golden-path/be210-smurray/src/main/java/com/spotify/be210smurray/ConcatServiceHttpImpl.java"
  "backend-golden-path/andreyg-golden-path-service/src/main/java/com/spotify/andreyggoldenpathservice/GoldenPathGrpcResource.java"
  "content-understanding/user-prompts/local/user_info.proto"
  "library-tags/tags-admin-backend/src/main/proto/tags-service.proto"
  "home-logging/home-logging-conductor/scripts/user-info.proto"
  "crane-config.yaml"
  "service-info.yaml"
  "youx-unknown/catalog-export-request-processor/src/main/resources/logback.xml"
  "docs/getting-support.md"
  "docs/limitations.md"
  "tools/errorprone/patch-all-the-things.py"
  "tools/exceptions/extend_exceptions.py"
  "ci/functions.sh"
  "common-config/BUILD.bazel"
)

# Large + bazel deps: same as LARGE but adds .conf files (hocon uses bazel run)
# Comparing LARGE vs LARGE_BAZEL isolates the cost of bazel-dependent formatters.
# shellcheck disable=SC2034  # used via nameref
LARGE_BAZEL_FILES=(
  "backend-golden-path/be210-smurray/src/main/java/com/spotify/be210smurray/Main.java"
  "backend-golden-path/be210-smurray/src/main/java/com/spotify/be210smurray/ConcatBusinessLogic.java"
  "backend-golden-path/be210-smurray/src/main/java/com/spotify/be210smurray/ConcatServiceImpl.java"
  "backend-golden-path/be210-smurray/src/main/java/com/spotify/be210smurray/ConcatServiceHttpImpl.java"
  "backend-golden-path/andreyg-golden-path-service/src/main/java/com/spotify/andreyggoldenpathservice/GoldenPathGrpcResource.java"
  "content-understanding/user-prompts/local/user_info.proto"
  "library-tags/tags-admin-backend/src/main/proto/tags-service.proto"
  "home-logging/home-logging-conductor/scripts/user-info.proto"
  "crane-config.yaml"
  "service-info.yaml"
  "media-platform/media-platform-registration-user.conf"
  "message-scheduling/event-scheduler-user.conf"
  "youx-unknown/catalog-export-request-processor/src/main/resources/logback.xml"
  "docs/getting-support.md"
  "docs/limitations.md"
  "tools/errorprone/patch-all-the-things.py"
  "ci/functions.sh"
  "common-config/BUILD.bazel"
)

# --- Helpers ---

# Perl startup is ~15ms vs ~34ms for python3. Since this is called
# twice per timing measurement, the overhead matters for sub-second runs.
now_ms() {
  perl -MTime::HiRes=time -e 'printf "%d\n", time()*1000'
}

# Time a command silently. Sets LAST_ELAPSED_MS and LAST_EXIT_CODE.
LAST_ELAPSED_MS=0
LAST_EXIT_CODE=0
time_cmd_quiet() {
  local stderr_file
  stderr_file=$(mktemp)
  local start end
  start=$(now_ms)
  set +e
  "$@" >/dev/null 2>"$stderr_file"
  LAST_EXIT_CODE=$?
  set -e
  end=$(now_ms)
  LAST_ELAPSED_MS=$((end - start))
  if [[ $LAST_EXIT_CODE -ne 0 ]]; then
    local err
    err=$(tail -3 "$stderr_file" | sed 's/^/    stderr: /')
    if [[ -n "$err" ]]; then
      echo "$err" >&2
    fi
  fi
  rm -f "$stderr_file"
}

fmt_s() {
  printf "%d.%01ds" $(($1 / 1000)) $((($1 % 1000) / 100))
}

# Run a tool $RUNS times. Print one line: label, individual times, median.
time_cmd_Nx() {
  local label="$1"
  shift
  local dir="$1"
  shift
  local tool_fn="$1"
  shift
  local files=("$@")
  local times=()

  for ((i = 1; i <= RUNS; i++)); do
    backup_files "$dir" "${files[@]}"
    break_formatting "$dir" "${files[@]}"
    time_cmd_quiet "$tool_fn" "$dir" "${files[@]}"
    times+=("$LAST_ELAPSED_MS")
    restore_files "$dir" "${files[@]}"
  done

  local sorted
  sorted=($(printf '%s\n' "${times[@]}" | sort -n))
  local median="${sorted[$((RUNS / 2))]}"

  local status=""
  if [[ $LAST_EXIT_CODE -ne 0 ]]; then
    status=" [exit $LAST_EXIT_CODE]"
  fi

  local run_strs=""
  for t in "${times[@]}"; do
    run_strs+="  $(fmt_s "$t")"
  done

  printf "  %-30s%s  → %s median%s\n" "$label" "$run_strs" "$(fmt_s "$median")" "$status"
}

backup_files() {
  local dir="$1"
  shift
  for f in "$@"; do
    cp "$dir/$f" "$dir/$f.bak" 2>/dev/null || true
  done
}

restore_files() {
  local dir="$1"
  shift
  for f in "$@"; do
    if [[ -f "$dir/$f.bak" ]]; then
      mv "$dir/$f.bak" "$dir/$f"
    fi
  done
}

break_formatting() {
  local dir="$1"
  shift
  for f in "$@"; do
    [[ -f "$dir/$f" ]] || continue
    local ext="${f##*.}"
    case "$ext" in
    java) sed -i '' 's/^import/import  /' "$dir/$f" ;;
    py) sed -i '' 's/^import/import  /' "$dir/$f" ;;
    sh | savvy) ;; # skip shell — shfmt .editorconfig may differ
    yaml) sed -i '' '1 s/$/ /' "$dir/$f" ;;
    conf) sed -i '' '1 s/$/ /' "$dir/$f" ;;
    proto) sed -i '' 's/^syntax = /syntax=/' "$dir/$f" ;;
    xml) sed -i '' '2 s/$/ /' "$dir/$f" ;;
    md) sed -i '' '1 s/$/ /' "$dir/$f" ;;
    esac
  done
}

run_bazel() {
  local dir="$1"
  shift
  (cd "$dir" && bazel run //:format --config=quiet_output -- "$@")
}

run_savvy() {
  local dir="$1"
  shift
  (cd "$dir" && tools/savvy/savvy format "$@")
}

run_scenario() {
  local label="$1"
  local dir="$2"
  shift 2
  local files=("$@")

  time_cmd_Nx "bazel ($label)" "$dir" run_bazel "${files[@]}"
  time_cmd_Nx "savvy ($label)" "$dir" run_savvy "${files[@]}"
}

# Like run_scenario but calls a setup function before each run of both tools.
run_scenario_with_setup() {
  local label="$1"
  local dir="$2"
  local setup_fn="$3"
  shift 3
  local files=("$@")

  for tool_label in bazel savvy; do
    local tool_fn="run_$tool_label"
    local times=()
    for ((i = 1; i <= RUNS; i++)); do
      "$setup_fn" "$dir"
      backup_files "$dir" "${files[@]}"
      break_formatting "$dir" "${files[@]}"
      time_cmd_quiet "$tool_fn" "$dir" "${files[@]}"
      times+=("$LAST_ELAPSED_MS")
      restore_files "$dir" "${files[@]}"
    done
    local sorted
    sorted=($(printf '%s\n' "${times[@]}" | sort -n))
    local median="${sorted[$((RUNS / 2))]}"
    local status=""
    if [[ $LAST_EXIT_CODE -ne 0 ]]; then
      status=" [exit $LAST_EXIT_CODE]"
    fi
    local run_strs=""
    for t in "${times[@]}"; do
      run_strs+="  $(fmt_s "$t")"
    done
    printf "  %-30s%s  → %s median%s\n" \
      "$tool_label ($label)" "$run_strs" "$(fmt_s "$median")" "$status"
  done
}

# --- Main ---

echo ""
echo "  Savvy vs Bazel Format Benchmark"
echo "  ================================"
echo ""
echo "Started: $(date '+%Y-%m-%d %H:%M:%S %Z')"
echo "Commit:  $(git -C "$SOURCE_DIR" rev-parse --short HEAD) ($(git -C "$SOURCE_DIR" rev-parse --abbrev-ref HEAD))"
echo "Runs per scenario: $RUNS (median reported)"
echo ""

# Clean up stale py.sh cache dirs from before the cache consolidation
stale_count=0
for d in "$HOME/Library/Caches"/backend-monorepo-savvy-bootstrap-* \
  "$HOME/Library/Caches"/backend-monorepo-savvy-venv-* \
  "$HOME/Library/Caches"/services-pilot-savvy-bootstrap-* \
  "$HOME/Library/Caches"/services-pilot-savvy-venv-*; do
  if [[ -d "$d" ]]; then
    rm -rf "$d"
    stale_count=$((stale_count + 1))
  fi
done
if [[ $stale_count -gt 0 ]]; then
  echo "Cleaned up $stale_count stale py.sh cache directories"
fi

# Create worktree (clean up any stale one first)
echo "Setting up benchmark worktree at $BENCH_DIR..."
git -C "$SOURCE_DIR" worktree remove "$BENCH_DIR" --force 2>/dev/null || true
git -C "$SOURCE_DIR" worktree add "$BENCH_DIR" HEAD --detach -q
trap 'echo ""; echo "Cleaning up worktree..."; git -C "$SOURCE_DIR" worktree remove "$BENCH_DIR" --force 2>/dev/null || true' EXIT

# Verify files exist in the worktree
missing=0
for f in "${LARGE_FILES[@]}"; do
  if [[ ! -f "$BENCH_DIR/$f" ]]; then
    echo "WARNING: $f not found in worktree"
    missing=$((missing + 1))
  fi
done
if [[ $missing -gt 0 ]]; then
  echo "($missing files missing — results may be incomplete)"
fi

# Quick sanity check: does bazel run work in the worktree?
echo ""
echo "Sanity check: bazel run //:format in worktree..."
if run_bazel "$BENCH_DIR" "${SMALL_FILES[0]}" >/dev/null 2>&1; then
  echo "  bazel: OK"
else
  echo "  bazel: FAILED (exit $?) — bazel results will be unreliable"
fi
if run_savvy "$BENCH_DIR" "${SMALL_FILES[0]}" >/dev/null 2>&1; then
  echo "  savvy: OK"
else
  echo "  savvy: FAILED (exit $?) — savvy results will be unreliable"
fi

setup_cold_server() {
  (cd "$1" && bazel shutdown) >/dev/null 2>&1 || true
  sleep 2
}

setup_stale_analysis() {
  (
    cd "$1"
    local_backup="${TMPDIR:-/tmp}/savvy-bench-backup-$$"
    cp -r tools/savvy "$local_backup"
    git checkout "$OLD_COMMIT" -q 2>/dev/null || true
    bazel build //:format --config=quiet_output >/dev/null 2>&1 || true
    git checkout - -q 2>/dev/null
    cp -r "$local_backup"/* tools/savvy/ 2>/dev/null || true
    rm -rf "$local_backup"
  )
}

ETNA_CACHE="$HOME/Library/Caches/etna"
SAVVY_CACHE="$HOME/Library/Caches/backend-monorepo-savvy"

run_bazel_expunged() {
  local dir="$1"
  shift
  (cd "$dir" && bazel run //:format --config=quiet_output --repository_cache= --disk_cache= -- "$@")
}

TOTAL_SCENARIOS=$(( ${#SET_LIST[@]} * 4 ))
scenario_num=0

for set_name in "${SET_LIST[@]}"; do
  declare -n files_ref="${set_name}_FILES"
  echo ""
  echo "━━━ ${set_name} set (${#files_ref[@]} files) ━━━"
  echo ""

  # --- Warm ---
  echo "Warming caches..."
  run_bazel "$BENCH_DIR" "${files_ref[@]}" >/dev/null 2>&1 || true
  run_savvy "$BENCH_DIR" "${files_ref[@]}" >/dev/null 2>&1 || true

  scenario_num=$((scenario_num + 1))
  echo "[$scenario_num/$TOTAL_SCENARIOS] warm"
  run_scenario "warm" "$BENCH_DIR" "${files_ref[@]}"

  # --- Cold bazel server ---
  scenario_num=$((scenario_num + 1))
  echo "[$scenario_num/$TOTAL_SCENARIOS] cold server — after bazel shutdown"
  run_scenario_with_setup "cold server" "$BENCH_DIR" setup_cold_server "${files_ref[@]}"

  # --- Stale analysis cache ---
  scenario_num=$((scenario_num + 1))
  echo "[$scenario_num/$TOTAL_SCENARIOS] stale analysis — checkout 2-week-old commit then back"
  run_scenario_with_setup "stale analysis" "$BENCH_DIR" setup_stale_analysis "${files_ref[@]}"

  # --- Prior state expunged (first-ever run) ---
  scenario_num=$((scenario_num + 1))
  echo "[$scenario_num/$TOTAL_SCENARIOS] prior state expunged — all caches wiped"
  BENCH_CACHE_BACKUP="${TMPDIR:-/tmp}/savvy-cache-backup-$$"

  # bazel Nx
  bazel_times=()
  for ((i = 1; i <= RUNS; i++)); do
    (cd "$BENCH_DIR" && bazel clean --expunge) >/dev/null 2>&1 || true
    backup_files "$BENCH_DIR" "${files_ref[@]}"
    break_formatting "$BENCH_DIR" "${files_ref[@]}"
    time_cmd_quiet run_bazel_expunged "$BENCH_DIR" "${files_ref[@]}"
    bazel_times+=("$LAST_ELAPSED_MS")
    restore_files "$BENCH_DIR" "${files_ref[@]}"
  done
  sorted=($(printf '%s\n' "${bazel_times[@]}" | sort -n))
  run_strs=""
  for t in "${bazel_times[@]}"; do run_strs+="  $(fmt_s "$t")"; done
  printf "  %-30s%s  → %s median\n" "bazel (prior state expunged)" "$run_strs" "$(fmt_s "${sorted[$((RUNS / 2))]}")"

  # savvy Nx
  savvy_times=()
  for ((i = 1; i <= RUNS; i++)); do
    mkdir -p "$BENCH_CACHE_BACKUP"
    [[ -d "$ETNA_CACHE" ]] && mv "$ETNA_CACHE" "$BENCH_CACHE_BACKUP/etna-$i"
    [[ -d "$SAVVY_CACHE" ]] && mv "$SAVVY_CACHE" "$BENCH_CACHE_BACKUP/savvy-$i"
    backup_files "$BENCH_DIR" "${files_ref[@]}"
    break_formatting "$BENCH_DIR" "${files_ref[@]}"
    time_cmd_quiet run_savvy "$BENCH_DIR" "${files_ref[@]}"
    savvy_times+=("$LAST_ELAPSED_MS")
    restore_files "$BENCH_DIR" "${files_ref[@]}"
  done
  sorted=($(printf '%s\n' "${savvy_times[@]}" | sort -n))
  run_strs=""
  for t in "${savvy_times[@]}"; do run_strs+="  $(fmt_s "$t")"; done
  printf "  %-30s%s  → %s median\n" "savvy (prior state expunged)" "$run_strs" "$(fmt_s "${sorted[$((RUNS / 2))]}")"

  rm -rf "$BENCH_CACHE_BACKUP" 2>/dev/null || true
done

echo ""
echo "Finished: $(date '+%Y-%m-%d %H:%M:%S %Z')"
echo "Done."