Maven 4 vs Maven 3 performance benchmark (resolver LayeredMap + model builder optimizations)

mvn4-bench.sh

#!/usr/bin/env bash
#
# Maven 4 Performance Benchmark
# ==============================
# Self-contained script to benchmark Maven 3.9.x vs Maven 4 RC6 vs optimized Maven 4 SNAPSHOT
# on both a small (36-module) and large (4383-module) reactor project.
#
# Prerequisites: JDK 21+, git, curl/wget
# Usage:        ./mvn4-bench.sh [workdir]
#               Default workdir: /tmp/mvn4-bench
#
set -euo pipefail

# macOS default is 256 — far too low for a 4383-module reactor.
# If both ulimit calls fail, the build WILL fail on the big project.
# Fix: sudo launchctl limit maxfiles 65536 200000
ulimit -n 65536 2>/dev/null || ulimit -n 10240 2>/dev/null || true
EFFECTIVE_ULIMIT=$(ulimit -n)
echo "Effective ulimit -n: $EFFECTIVE_ULIMIT"
if [ "$EFFECTIVE_ULIMIT" -lt 10240 ]; then
  echo "⚠️  WARNING: ulimit -n is $EFFECTIVE_ULIMIT — too low for the 4383-module project!"
  echo "   Run: sudo launchctl limit maxfiles 65536 200000"
  echo "   Then re-run this script."
fi

WORKDIR="${1:-/tmp/mvn4-bench}"
RESULTS="$WORKDIR/results.txt"

# --- Maven versions ---
MVN3_VERSION="3.9.16"
MVN4_RC6_VERSION="4.0.0-rc-6"

# --- Colors ---
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m'

info()  { echo -e "${BLUE}[INFO]${NC}  $*"; }
ok()    { echo -e "${GREEN}[OK]${NC}    $*"; }
warn()  { echo -e "${YELLOW}[WARN]${NC}  $*"; }
err()   { echo -e "${RED}[ERROR]${NC} $*"; }
banner(){ echo -e "\n${GREEN}════════════════════════════════════════${NC}"; echo -e "${GREEN}  $*${NC}"; echo -e "${GREEN}════════════════════════════════════════${NC}\n"; }

# ─────────────────────────────────────────────────────
# Phase 0: Setup
# ─────────────────────────────────────────────────────
banner "Phase 0: Setup"
mkdir -p "$WORKDIR"

# Detect OS for download URLs
OS=$(uname -s)
ARCH=$(uname -m)
case "$OS" in
  Darwin) ARCHIVE_TYPE="bin.tar.gz" ;;
  Linux)  ARCHIVE_TYPE="bin.tar.gz" ;;
  *)      err "Unsupported OS: $OS"; exit 1 ;;
esac

# Check JDK
JAVA_VER=$(java -version 2>&1 | head -1 | grep -oE '[0-9]+' | head -1)
if [ "$JAVA_VER" -lt 21 ]; then
  err "JDK 21+ required (found $JAVA_VER)"
  exit 1
fi
ok "JDK $JAVA_VER detected"

# ─────────────────────────────────────────────────────
# Phase 1: Download Maven 3.9.x
# ─────────────────────────────────────────────────────
banner "Phase 1: Download Maven 3.9.x"
MVN3_DIR="$WORKDIR/apache-maven-$MVN3_VERSION"
if [ -x "$MVN3_DIR/bin/mvn" ]; then
  ok "Maven 3 already present"
else
  info "Downloading Maven $MVN3_VERSION..."
  MVN3_URL="https://archive.apache.org/dist/maven/maven-3/$MVN3_VERSION/binaries/apache-maven-$MVN3_VERSION-$ARCHIVE_TYPE"
  curl -fsSL "$MVN3_URL" | tar xz -C "$WORKDIR"
  ok "Maven $MVN3_VERSION installed"
fi
MVN3="$MVN3_DIR/bin/mvn"

# ─────────────────────────────────────────────────────
# Phase 2: Download Maven 4 RC6
# ─────────────────────────────────────────────────────
banner "Phase 2: Download Maven 4 RC6"
MVN4RC6_DIR="$WORKDIR/apache-maven-$MVN4_RC6_VERSION"
if [ -x "$MVN4RC6_DIR/bin/mvn" ]; then
  ok "Maven 4 RC6 already present"
else
  info "Downloading Maven $MVN4_RC6_VERSION..."
  MVN4RC6_URL="https://dist.apache.org/repos/dist/dev/maven/maven-4/$MVN4_RC6_VERSION/binaries/apache-maven-$MVN4_RC6_VERSION-$ARCHIVE_TYPE"
  curl -fsSL "$MVN4RC6_URL" | tar xz -C "$WORKDIR"
  ok "Maven $MVN4_RC6_VERSION installed"
fi
MVN4RC6="$MVN4RC6_DIR/bin/mvn"

# ─────────────────────────────────────────────────────
# Phase 3: Build optimized Maven 4 SNAPSHOT
# ─────────────────────────────────────────────────────
banner "Phase 3: Build optimized Maven 4 SNAPSHOT (resolver + maven)"

# Included optimizations:
#   Resolver (branch fix/2013):
#     - LayeredMap: O(1) zero-copy mergeAncestors (replaces HashMap copying)
#     - Multi-entry memoization in deriveChildManager (4-entry ring buffer)
#     - Cumulative ancestor maps for O(1) lookups
#     - Bounded LRU idle-channel pool in FileLockNamedLockFactory (FD leak fix)
#   Maven (branch bench/resolver-2.0.22, cherry-picked from master PRs):
#     - PR #12652: O(1) reactor sort via index map (replaces O(n) indexOf)
#     - PR #12652: PhaseComparator optimization
#     - PR #12655: Wire isLocationTracking() to XML parser (skips location tracking overhead)

# 3a. Build optimized resolver (2.0.22-SNAPSHOT with LayeredMap + FD fix)
RESOLVER_DIR="$WORKDIR/maven-resolver"
if [ -d "$RESOLVER_DIR" ]; then
  info "Resolver repo already cloned, updating..."
  cd "$RESOLVER_DIR" && git fetch origin && git checkout fix/2013 && git pull origin fix/2013 || true
else
  info "Cloning maven-resolver (branch fix/2013)..."
  git clone --branch fix/2013 --single-branch https://github.com/apache/maven-resolver.git "$RESOLVER_DIR"
fi

RESOLVER_SNAPSHOT_VER="2.0.22-SNAPSHOT"
# Check if already installed in local repo
if [ -f "$HOME/.m2/repository/org/apache/maven/resolver/maven-resolver-util/$RESOLVER_SNAPSHOT_VER/maven-resolver-util-$RESOLVER_SNAPSHOT_VER.jar" ]; then
  ok "Resolver $RESOLVER_SNAPSHOT_VER already in local Maven repo"
else
  info "Building resolver $RESOLVER_SNAPSHOT_VER..."
  cd "$RESOLVER_DIR"
  "$MVN3" clean install -DskipTests -B -q
  ok "Resolver $RESOLVER_SNAPSHOT_VER built and installed"
fi

# 3b. Build Maven 4 SNAPSHOT from bench branch (maven-4.0.x + resolver 2.0.22-SNAPSHOT API fix)
MAVEN_DIR="$WORKDIR/maven"
if [ -d "$MAVEN_DIR" ]; then
  info "Maven repo already cloned, updating..."
  cd "$MAVEN_DIR"
  git fetch origin
  git checkout -B bench/resolver-2.0.22 origin/bench/resolver-2.0.22
else
  info "Cloning maven (branch bench/resolver-2.0.22)..."
  git clone --branch bench/resolver-2.0.22 https://github.com/gnodet/maven.git "$MAVEN_DIR"
fi

MVN4SNAP_DIST="$WORKDIR/mvn4-snapshot"
if [ -x "$MVN4SNAP_DIST/bin/mvn" ]; then
  # Verify it has the right resolver
  if ls "$MVN4SNAP_DIST/lib/maven-resolver-util-$RESOLVER_SNAPSHOT_VER.jar" >/dev/null 2>&1; then
    ok "Optimized Maven 4 SNAPSHOT already built with resolver $RESOLVER_SNAPSHOT_VER"
  else
    warn "Stale snapshot found, rebuilding..."
    rm -rf "$MVN4SNAP_DIST"
  fi
fi

if [ ! -x "$MVN4SNAP_DIST/bin/mvn" ]; then
  info "Building Maven 4 SNAPSHOT..."
  cd "$MAVEN_DIR"
  "$MVN3" clean install -DskipTests -B -q -Dformatter.skip=true -Dimpsort.skip=true 2>&1 | tail -5

  # Extract distribution
  DIST_TAR=$(ls "$MAVEN_DIR"/apache-maven/target/apache-maven-*-bin.tar.gz 2>/dev/null | head -1)
  if [ -z "$DIST_TAR" ]; then
    err "Maven distribution tar.gz not found — build may have failed"
    err "Check: ls $MAVEN_DIR/apache-maven/target/"
    exit 1
  fi
  mkdir -p "$MVN4SNAP_DIST"
  tar xzf "$DIST_TAR" --strip-components=1 -C "$MVN4SNAP_DIST"

  # Verify resolver version
  if ls "$MVN4SNAP_DIST/lib/maven-resolver-util-$RESOLVER_SNAPSHOT_VER.jar" >/dev/null 2>&1; then
    ok "Maven 4 SNAPSHOT built with resolver $RESOLVER_SNAPSHOT_VER ✓"
  else
    err "WRONG RESOLVER VERSION in snapshot!"
    ls "$MVN4SNAP_DIST/lib/maven-resolver-util-"*
    exit 1
  fi
fi
MVN4SNAP="$MVN4SNAP_DIST/bin/mvn"

# ─────────────────────────────────────────────────────
# Phase 4: Set up test projects
# ─────────────────────────────────────────────────────
banner "Phase 4: Set up test projects"

# 4a. Small project = Maven 4 RC6 source (36 modules)
SMALL_DIR="$WORKDIR/small-project"
if [ -d "$SMALL_DIR/.git" ]; then
  ok "Small project (Maven RC6 source, 36 modules) already present"
else
  info "Cloning Maven RC6 source as small project..."
  git clone --branch maven-$MVN4_RC6_VERSION --depth 1 https://github.com/apache/maven.git "$SMALL_DIR"
  ok "Small project ready"
fi

# 4b. Big project = generated 4383-module diamond reactor
BIG_DIR="$WORKDIR/big-project"
if [ -d "$BIG_DIR" ] && [ -f "$BIG_DIR/pom.xml" ]; then
  MODULE_COUNT=$(grep -c '<module>' "$BIG_DIR/pom.xml" 2>/dev/null || echo 0)
  if [ "$MODULE_COUNT" -gt 1000 ]; then
    ok "Big project ($MODULE_COUNT modules) already present"
  else
    warn "Big project looks incomplete ($MODULE_COUNT modules), regenerating..."
    rm -rf "$BIG_DIR"
  fi
fi

if [ ! -d "$BIG_DIR" ] || [ ! -f "$BIG_DIR/pom.xml" ]; then
  GEN_DIR="$WORKDIR/maven-multiproject-generator"
  if [ ! -d "$GEN_DIR" ]; then
    info "Cloning multiproject generator..."
    git clone https://github.com/maven-turbo-reactor/maven-multiproject-generator.git "$GEN_DIR"
  fi
  info "Building generator..."
  cd "$GEN_DIR"
  "$MVN3" clean install -DskipTests -B -q
  info "Generating 4383-module project (this takes a minute)..."
  java -jar target/maven-multiproject-generator-exec.jar
  mv "$GEN_DIR/generated" "$BIG_DIR"
  ok "Big project generated"
fi

# ─────────────────────────────────────────────────────
# Phase 5: Benchmark
# ─────────────────────────────────────────────────────
banner "Phase 5: Benchmark"

timed_run() {
  local label=$1 mvn=$2 dir=$3 cmd=$4
  cd "$dir"
  local start end ms rc
  start=$(python3 -c 'import time; print(int(time.time()*1000))' 2>/dev/null || perl -MTime::HiRes=time -e 'printf "%.0f\n", time()*1000' 2>/dev/null || echo $(( $(date +%s) * 1000 )))
  $mvn $cmd -q > /tmp/mvn-bench-last.log 2>&1
  rc=$?
  end=$(python3 -c 'import time; print(int(time.time()*1000))' 2>/dev/null || perl -MTime::HiRes=time -e 'printf "%.0f\n", time()*1000' 2>/dev/null || echo $(( $(date +%s) * 1000 )))
  ms=$(( end - start ))
  local status="OK"
  if [ $rc -ne 0 ]; then
    status="FAILED(rc=$rc)"
    # Capture last 5 lines of error for diagnosis
    local errtail
    errtail=$(tail -5 /tmp/mvn-bench-last.log 2>/dev/null | tr '\n' ' ')
    warn "  $label FAILED (rc=$rc): $errtail"
  fi
  echo "${label}: ${ms}ms  [${status}]"
  echo "${label}: ${ms}ms  [${status}]" >> "$RESULTS"
}

warmup() {
  local label=$1 mvn=$2 dir=$3 cmd=$4
  info "  warmup: $label"
  cd "$dir"
  $mvn $cmd -q > /dev/null 2>&1 || true
}

# Clear results
echo "==========================================" > "$RESULTS"
echo "BENCHMARK RESULTS — $(date)" >> "$RESULTS"
echo "Machine: $(uname -srm)" >> "$RESULTS"
echo "JDK: $(java -version 2>&1 | head -1)" >> "$RESULTS"
echo "CPU: $(sysctl -n machdep.cpu.brand_string 2>/dev/null || grep 'model name' /proc/cpuinfo 2>/dev/null | head -1 || echo unknown)" >> "$RESULTS"
echo "ulimit -n: $EFFECTIVE_ULIMIT" >> "$RESULTS"
echo "==========================================" >> "$RESULTS"

# ── 5a: mvn foo (lifecycle resolution only, no compilation) ──
banner "5a: mvn foo — lifecycle resolution"
echo "" >> "$RESULTS"
echo "--- mvn foo (lifecycle resolution, no compilation) ---" >> "$RESULTS"

info "Warming up foo on small project..."
warmup "mvn3"  "$MVN3"     "$SMALL_DIR" "foo"
warmup "rc6"   "$MVN4RC6"  "$SMALL_DIR" "foo"
warmup "snap"  "$MVN4SNAP" "$SMALL_DIR" "foo"

info "Timed: foo on small project"
timed_run "mvn3-foo-small"     "$MVN3"     "$SMALL_DIR" "foo"
timed_run "mvn4rc6-foo-small"  "$MVN4RC6"  "$SMALL_DIR" "foo"
timed_run "mvn4snap-foo-small" "$MVN4SNAP" "$SMALL_DIR" "foo"

info "Warming up foo on big project..."
warmup "mvn3"  "$MVN3"     "$BIG_DIR" "foo"
warmup "rc6"   "$MVN4RC6"  "$BIG_DIR" "foo"
warmup "snap"  "$MVN4SNAP" "$BIG_DIR" "foo"

info "Timed: foo on big project"
timed_run "mvn3-foo-big"     "$MVN3"     "$BIG_DIR" "foo"
timed_run "mvn4rc6-foo-big"  "$MVN4RC6"  "$BIG_DIR" "foo"
timed_run "mvn4snap-foo-big" "$MVN4SNAP" "$BIG_DIR" "foo"

# ── 5b: mvn clean install -DskipTests ──
banner "5b: mvn clean install -DskipTests"
echo "" >> "$RESULTS"
echo "--- mvn clean install -DskipTests ---" >> "$RESULTS"

info "Warming up install on small project..."
warmup "mvn3"  "$MVN3"     "$SMALL_DIR" "clean install -DskipTests"
warmup "rc6"   "$MVN4RC6"  "$SMALL_DIR" "clean install -DskipTests"
warmup "snap"  "$MVN4SNAP" "$SMALL_DIR" "clean install -DskipTests"

info "Timed: install on small project"
timed_run "mvn3-install-small"     "$MVN3"     "$SMALL_DIR" "clean install -DskipTests"
timed_run "mvn4rc6-install-small"  "$MVN4RC6"  "$SMALL_DIR" "clean install -DskipTests"
timed_run "mvn4snap-install-small" "$MVN4SNAP" "$SMALL_DIR" "clean install -DskipTests"

info "Warming up install on big project..."
warmup "mvn3"  "$MVN3"     "$BIG_DIR" "clean install -DskipTests"
warmup "rc6"   "$MVN4RC6"  "$BIG_DIR" "clean install -DskipTests"
warmup "snap"  "$MVN4SNAP" "$BIG_DIR" "clean install -DskipTests"

info "Timed: install on big project"
timed_run "mvn3-install-big"     "$MVN3"     "$BIG_DIR" "clean install -DskipTests"
timed_run "mvn4rc6-install-big"  "$MVN4RC6"  "$BIG_DIR" "clean install -DskipTests"
timed_run "mvn4snap-install-big" "$MVN4SNAP" "$BIG_DIR" "clean install -DskipTests"

# ─────────────────────────────────────────────────────
# Phase 6: Summary
# ─────────────────────────────────────────────────────
banner "Results"

echo "" >> "$RESULTS"
echo "==========================================" >> "$RESULTS"
echo "DONE — $(date)" >> "$RESULTS"
echo "==========================================" >> "$RESULTS"

cat "$RESULTS"

echo ""
info "Full results saved to: $RESULTS"
info "Copy-paste the file contents to share."