Hermes Agent — Python version benchmark (3.11/3.12/3.13/3.14 full test suite)

bench_versions.sh

#!/usr/bin/env bash
# Hermes Agent — Python version benchmark
# Runs the full test suite on Python 3.11, 3.12, 3.13, 3.14
# Each version: fresh venv, delete uv.lock, regenerate lock file, fresh install, run all tests
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$SCRIPT_DIR"

VERSIONS=("3.11" "3.12" "3.13" "3.14")
N_WORKERS=$(nproc)
RESULTS=()

unset VIRTUAL_ENV
cleanup_venv() { rm -rf .venv uv.lock; }
rm -f uv.lock

echo "======================================================"
echo " Hermes Agent — Python Version Benchmark"
echo " Host: $(nproc) cores, $(free -h | awk '/^Mem:/{print $2}') RAM"
echo " Test workers: $N_WORKERS"
echo "======================================================"
echo ""

# Pre-cache all Python versions
for ver in "${VERSIONS[@]}"; do
    echo "→ Caching Python $ver..."
    uv python install "$ver" 2>&1 | tail -1
done
echo ""

for ver in "${VERSIONS[@]}"; do
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    echo "  Python $ver"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

    cleanup_venv
    unset VIRTUAL_ENV

    # Step 1: Fresh venv
    echo "  [1/5] Creating fresh venv..."
    uv venv --python "$ver" .venv 2>&1 | tail -1
    PY_VER=$(.venv/bin/python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}')")
    echo "  → Python $PY_VER"

    # Step 2: Regenerate lock file from scratch
    echo "  [2/5] Regenerating lock file..."
    LOCK_START=$(python3 -c "import time; print(time.time())")
    uv lock 2>&1 || true
    LOCK_END=$(python3 -c "import time; print(time.time())")

    if [ ! -f uv.lock ]; then
        LOCK_TIME=$(python3 -c "print(f'{$LOCK_END - $LOCK_START:.2f}')")
        echo "  ⚠  uv.lock not generated — incompatible"
        RESULTS+=("$ver|FAIL|$PY_VER|$LOCK_TIME|N/A")
        cleanup_venv
        echo ""
        continue
    fi

    # Step 3: Install deps including dev extras (pytest etc.)
    echo "  [3/5] Installing dependencies (including dev extras)..."
    uv sync --frozen --extra dev 2>&1 | tail -3

    # Step 4: Verify pytest
    echo "  [4/5] Verifying pytest..."
    .venv/bin/python -m pytest --version 2>&1

    # Step 5: Run tests
    echo "  [5/5] Running test suite (workers=$N_WORKERS)..."
    TEST_START=$(python3 -c "import time; print(time.time())")
    HERMES_TEST_WORKERS=$N_WORKERS \
    .venv/bin/python scripts/run_tests_parallel.py 2>&1 || true
    TEST_END=$(python3 -c "import time; print(time.time())")

    LOCK_TIME=$(python3 -c "print(f'{$LOCK_END - $LOCK_START:.2f}')")
    TEST_TIME=$(python3 -c "print(f'{$TEST_END - $TEST_START:.2f}')")
    echo "  → Lock: ${LOCK_TIME}s  |  Tests: ${TEST_TIME}s"

    RESULTS+=("$ver|OK|$PY_VER|$LOCK_TIME|$TEST_TIME")
    cleanup_venv
    echo ""
done

# Report
echo ""
echo "======================================================"
echo "  RESULTS"
echo "======================================================"
printf "%-10s %-10s %-12s %-10s %-12s %s\n" "Python" "Status" "Version" "Lock(s)" "Test(s)" "Speedup vs 3.11"
echo "----------------------------------------------------------------"
BASELINE=""
for result in "${RESULTS[@]}"; do
    IFS='|' read -r ver status py_ver lock_time test_time <<< "$result"
    if [ "$status" = "FAIL" ]; then
        printf "%-10s %-10s %-12s %-10s %-12s %s\n" "$ver" "FAILED" "$py_ver" "$lock_time" "N/A" "incompatible"
        continue
    fi
    if [ -z "$BASELINE" ]; then
        GAIN="(baseline)"
        BASELINE="$test_time"
    else
        SPEEDUP=$(python3 -c "
b = float('$BASELINE'); t = float('$test_time')
ratio = b / t if t else 0
pct = (1 - t/b) * 100 if b else 0
print(f'{ratio:.2f}x  ({pct:+.1f}%)')
")
        GAIN="$SPEEDUP"
    fi
    printf "%-10s %-10s %-12s %-10s %-12s %s\n" "$ver" "$status" "$py_ver" "$lock_time" "$test_time" "$GAIN"
done
echo "======================================================"

# Restore lock file
cd "$SCRIPT_DIR"
cleanup_venv
uv lock 2>&1 | tail -1