eSlider · August 21, 2026 19:50
diff --git a/gemma4-edge-rocm.md b/gemma4-edge-rocm.md
diff --git a/qwen38-2b-perf.md b/qwen38-2b-perf.md
diff --git a/serve-gemma4-e2b.sh b/serve-gemma4-e2b.sh
 #!/usr/bin/env bash
 # Serve Gemma 4 E2B QAT Q4_0 (google/gemma-4-E2B-it-qat-q4_0-gguf) via llama.cpp (Vulkan/RADV) on gfx90c.
 #
 # Measured: 26.15 t/s decode (llama-bench tg64), 312.6 t/s prefill (pp128). Native ctx 131072.
 # Tool calling VERIFIED on this host (get_weather -> tool_calls, 2026-08-17).
 #
 # Default: max native context window (131072, single slot via --parallel 1).
 # Uses the same port as opencode's `llamacpp` provider (8081), so stop any other server first.
 #
 # Usage:
 #   ./utils/serve-gemma4-e2b.sh start        # start in background (daemon), wait for readiness
 #   ./utils/serve-gemma4-e2b.sh stop         # stop the daemon
 #   ./utils/serve-gemma4-e2b.sh status       # show pid/health
 #   ./utils/serve-gemma4-e2b.sh              # foreground (same as `start` but stays attached)
 #   SERV_CTX=8192 ./utils/serve-gemma4-e2b.sh start   # smaller window / multi-user (4 slots)
 #
 # Speculative decoding:
 #   SERV_SPEC=ngram-map-k,ngram-cache     # default: ngram-map-k + ngram-cache (recommended)
 #   SERV_SPEC=""                          # disable speculative decoding entirely
 #   SERV_SPEC="draft-dflash,ngram-map-k"  # dflash + ngram (requires dflash.block_size model metadata —
 #                                          # NOT AVAILABLE for gemma-4-E2B/E4B; may error or fall back to ngram only)
 #   SERV_NGRAM_N=12  SERV_NGRAM_M=48  SERV_NGRAM_MIN_HITS=1   # ngram-map-k tuning
 # Note: dflash requires 'dflash.block_size' GGUF metadata — gemma-4-E2B/Q4_0 lacks this.
 # If used, llama-server will attempt to load dflash draft but may error/fall back since the
 # Gemma E2B model has no dflash.block_size metadata. ngram-only combos work without extra metadata.
 #
 # opencode: provider `llamacpp` / model `gemma4-e2b` -> http://127.0.0.1:8081/v1,
 # configured with limit.context = 131072 to match this server.
 set -euo pipefail

 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 BUILD="${LLAMACPP_BUILD:-$ROOT/build-work/llama.cpp/build-vulkan2}"
 MODEL="${SERV_MODEL:-$ROOT/build-work/gemma-4-E2B_q4_0-it.gguf}"
 BIN="$BUILD/bin"

 PORT="${SERV_PORT:-8081}"
 HOST="${SERV_HOST:-127.0.0.1}"
 CTX="${SERV_CTX:-131072}"
 PARALLEL="${SERV_PARALLEL:-1}"
 NGL="${SERV_NGL:-99}"
 N_THREADS="${SERV_N_THREADS:-1}"
 TEMP="${SERV_TEMP:-0.2}"
 SPEC="${SERV_SPEC:-ngram-map-k,ngram-cache}"
 SPEC_N_MAX="${SERV_SPEC_N_MAX:-16}"
 NGRAM_N="${SERV_NGRAM_N:-12}"
 NGRAM_M="${SERV_NGRAM_M:-48}"
 NGRAM_MIN_HITS="${SERV_NGRAM_MIN_HITS:-1}"

 NAME="gemma4-e2b"
 PIDFILE="${SERV_PIDFILE:-/tmp/opencode/serve-${NAME}.pid}"
 LOGFILE="${SERV_LOGFILE:-/tmp/opencode/serve-${NAME}.log}"

 if [[ ! -x "$BIN/llama-server" ]]; then
  echo "Missing $BIN/llama-server. Build it first:" >&2
  echo "  cmake -S $ROOT/build-work/llama.cpp -B $BUILD -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_VULKAN=ON -DGGML_HIP=OFF" >&2
  echo "  cmake --build $BUILD -j\"\$(nproc)\" --target llama-server" >&2
  exit 1
 fi
 if [[ ! -f "$MODEL" ]]; then
  echo "Missing model: $MODEL (download first: hf download google/gemma-4-E2B-it-qat-q4_0-gguf gemma-4-E2B_q4_0-it.gguf --local-dir $ROOT/build-work)" >&2
  exit 1
 fi

 export LD_LIBRARY_PATH="$BIN${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"

 build_args() {
  args=(
    -m "$MODEL"
    -ngl "$NGL"
    -t "$N_THREADS"
    -fa off
    -c "$CTX"
    --parallel "$PARALLEL"
    --temp "$TEMP"
    --jinja
    --host "$HOST"
    --port "$PORT"
  )
  if [ -n "$SPEC" ]; then
    args+=(--spec-type "$SPEC")
    args+=(--spec-draft-n-max "$SPEC_N_MAX")
    if [[ ",$SPEC," == *",ngram-map-k,"* ]]; then
      args+=(--spec-ngram-map-k-size-n "$NGRAM_N")
      args+=(--spec-ngram-map-k-size-m "$NGRAM_M")
      args+=(--spec-ngram-map-k-min-hits "$NGRAM_MIN_HITS")
    fi
    if [[ ",$SPEC," == *",draft-dflash,"* ]]; then
      echo "WARNING: draft-dflash selected but gemma-4-E2B model has no dflash.block_size metadata." >&2
      echo "         llama-server may error or fall back to ngram-only decode." >&2
    fi
  fi
 }

 health() { curl -s -m 2 "http://$HOST:$PORT/health" 2>/dev/null; }

 daemon_alive() {
  local pid; pid=$(cat "$PIDFILE" 2>/dev/null || true)
  [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null
 }

 cmd_start() {
  if daemon_alive; then
    echo "$NAME already running (pid $(cat "$PIDFILE")) on http://$HOST:$PORT"
    return 0
  fi
  mkdir -p "$(dirname "$PIDFILE")"
  build_args
  echo "Starting $NAME on http://$HOST:$PORT (ctx $CTX, parallel $PARALLEL, ngl $NGL)… log: $LOGFILE"
  setsid nohup "$BIN/llama-server" "${args[@]}" > "$LOGFILE" 2>&1 < /dev/null &
  echo $! > "$PIDFILE"
  for _ in $(seq 1 60); do
    [ "$(health)" = '{"status":"ok"}' ] && break
    sleep 2
  done
  if [ "$(health)" = '{"status":"ok"}' ]; then
    echo "Ready: $(health) (pid $(cat "$PIDFILE"))"
  else
    echo "WARNING: no healthy response yet — tail $LOGFILE" >&2
    exit 1
  fi
 }

 cmd_stop() {
  if ! daemon_alive; then
    echo "$NAME not running"
    rm -f "$PIDFILE"
    return 0
  fi
  echo "Stopping $NAME (pid $(cat "$PIDFILE"))"
  kill "$(cat "$PIDFILE")"
  sleep 2
  if daemon_alive; then
    kill -9 "$(cat "$PIDFILE")" 2>/dev/null || true
  fi
  rm -f "$PIDFILE"
  echo "Stopped"
 }

 cmd_status() {
  if daemon_alive; then
    echo "$NAME: RUNNING (pid $(cat "$PIDFILE")) — health: $(health)"
  else
    echo "$NAME: stopped"
  fi
 }

 case "${1:-}" in
  start)  shift || true; cmd_start ;;
  stop)   cmd_stop ;;
  status) cmd_status ;;
  *)      cmd_start ;;
 esac
diff --git a/serve-qwen38.sh b/serve-qwen38.sh
 #!/usr/bin/env bash
 # Serve empero-ai/Qwen3.8-2B-GGUF (Q4_K_M) via llama.cpp (Vulkan/RADV) on gfx90c.
 #
 # Fastest measured config on this host: stock llama.cpp Vulkan, -fa off, no MTP.
 # MTP/speculative decoding is a NET LOSS on this bandwidth-limited iGPU (see docs/qwen38-2b-perf.md).
 #
 # Default: max native context window (262144 tokens, single slot via --parallel 1).
 # Measured: 25.7 t/s decode at 262144 ctx (native max), 25.4 t/s at 81920. Context size
 # does not slow decode on this bandwidth-bound iGPU.
 #
 # temp=0.2: opencode does NOT send temperature for reasoning models (llama-server default
 # 0.8 makes this 2B model drift into prose instead of tool_calls). Low temp = reliable tools.
 #
 # Usage:
 #   ./utils/serve-qwen38.sh start        # start in background (daemon), wait for readiness
 #   ./utils/serve-qwen38.sh stop         # stop the daemon
 #   ./utils/serve-qwen38.sh status       # show pid/health
 #   ./utils/serve-qwen38.sh              # foreground (same as `start` but stays attached)
 #   SERV_CTX=81920 ./utils/serve-qwen38.sh start   # 80k window (4 slots, multi-user)
 #   ./utils/serve-qwen38.sh start --mtp  # also load the MTP draft head (draft-mtp; SLOWER)
 #   SERV_SPEC=ngram-map-k,ngram-cache ./utils/serve-qwen38.sh start   # draft-free ngram spec decode
 #   SERV_SPEC="" ./utils/serve-qwen38.sh start --mtp   # MTP without ngram
 #   SERV_SPEC_N_MAX=16 SERV_NGRAM_N=12 SERV_NGRAM_M=48 SERV_NGRAM_MIN_HITS=1   # ngram-map-k tuning
 #
 # opencode: provider `llamacpp` / model `qwen3.8-2b` -> http://127.0.0.1:8081/v1,
 # configured with limit.context = 262144 to match this server.
 set -euo pipefail

 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 BUILD="${LLAMACPP_BUILD:-$ROOT/build-work/llama.cpp/build-vulkan2}"
 MODEL="${SERV_MODEL:-$ROOT/build-work/Qwen3.8-2B-Q4_K_M.gguf}"
 MTP_HEAD="${SERV_MTP_HEAD:-$ROOT/build-work/mtp-Qwen3.8-2B-Q4_K_M.gguf}"
 BIN="$BUILD/bin"

 PORT="${SERV_PORT:-8081}"
 HOST="${SERV_HOST:-127.0.0.1}"
 CTX="${SERV_CTX:-262144}"
 PARALLEL="${SERV_PARALLEL:-1}"
 NGL="${SERV_NGL:-99}"
 N_THREADS="${SERV_N_THREADS:-1}"
 TEMP="${SERV_TEMP:-0.2}"
 SPEC="${SERV_SPEC:-}"
 SPEC_N_MAX="${SERV_SPEC_N_MAX:-16}"
 NGRAM_N="${SERV_NGRAM_N:-12}"
 NGRAM_M="${SERV_NGRAM_M:-48}"
 NGRAM_MIN_HITS="${SERV_NGRAM_MIN_HITS:-1}"

 NAME="qwen3.8-2b"
 PIDFILE="${SERV_PIDFILE:-/tmp/opencode/serve-${NAME}.pid}"
 LOGFILE="${SERV_LOGFILE:-/tmp/opencode/serve-${NAME}.log}"

 if [[ ! -x "$BIN/llama-server" ]]; then
  echo "Missing $BIN/llama-server. Build it first:" >&2
  echo "  cmake -S $ROOT/build-work/llama.cpp -B $BUILD -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_VULKAN=ON -DGGML_HIP=OFF" >&2
  echo "  cmake --build $BUILD -j\"\$(nproc)\" --target llama-server" >&2
  exit 1
 fi
 if [[ ! -f "$MODEL" ]]; then
  echo "Missing model: $MODEL (download first: hf download empero-ai/Qwen3.8-2B-GGUF Qwen3.8-2B-Q4_K_M.gguf --local-dir $ROOT/build-work)" >&2
  exit 1
 fi

 export LD_LIBRARY_PATH="$BIN${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"

 build_args() {
  args=(
    -m "$MODEL"
    -ngl "$NGL"
    -t "$N_THREADS"
    -fa off
    -c "$CTX"
    --parallel "$PARALLEL"
    --temp "$TEMP"
    --jinja
    --host "$HOST"
    --port "$PORT"
  )
  local spec_type="$SPEC"
  if [[ "${1:-}" == "--mtp" ]]; then
    if [[ ! -f "$MTP_HEAD" ]]; then
      echo "Missing MTP head: $MTP_HEAD" >&2
      exit 1
    fi
    args+=(-md "$MTP_HEAD")
    spec_type="draft-mtp${SPEC:+,$SPEC}"
    echo "Serving with MTP draft ($MTP_HEAD) — NOTE: slower than plain on this GPU." >&2
  fi
  if [ -n "$spec_type" ]; then
    args+=(--spec-type "$spec_type")
    args+=(--spec-draft-n-max "$SPEC_N_MAX")
    if [[ ",$spec_type," == *",ngram-map-k,"* ]]; then
      args+=(--spec-ngram-map-k-size-n "$NGRAM_N")
      args+=(--spec-ngram-map-k-size-m "$NGRAM_M")
      args+=(--spec-ngram-map-k-min-hits "$NGRAM_MIN_HITS")
    fi
  fi
 }

 health() { curl -s -m 2 "http://$HOST:$PORT/health" 2>/dev/null; }

 daemon_alive() {
  local pid; pid=$(cat "$PIDFILE" 2>/dev/null || true)
  [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null
 }

 cmd_start() {
  if daemon_alive; then
    echo "$NAME already running (pid $(cat "$PIDFILE")) on http://$HOST:$PORT"
    return 0
  fi
  mkdir -p "$(dirname "$PIDFILE")"
  build_args "${1:-}"
  echo "Starting $NAME on http://$HOST:$PORT (ctx $CTX, parallel $PARALLEL, ngl $NGL)… log: $LOGFILE"
  setsid nohup "$BIN/llama-server" "${args[@]}" > "$LOGFILE" 2>&1 < /dev/null &
  echo $! > "$PIDFILE"
  for _ in $(seq 1 60); do
    [ "$(health)" = '{"status":"ok"}' ] && break
    sleep 2
  done
  if [ "$(health)" = '{"status":"ok"}' ]; then
    echo "Ready: $(health) (pid $(cat "$PIDFILE"))"
  else
    echo "WARNING: no healthy response yet — tail $LOGFILE" >&2
    exit 1
  fi
 }

 cmd_stop() {
  if ! daemon_alive; then
    echo "$NAME not running"
    rm -f "$PIDFILE"
    return 0
  fi
  echo "Stopping $NAME (pid $(cat "$PIDFILE"))"
  kill "$(cat "$PIDFILE")"
  sleep 2
  if daemon_alive; then
    kill -9 "$(cat "$PIDFILE")" 2>/dev/null || true
  fi
  rm -f "$PIDFILE"
  echo "Stopped"
 }

 cmd_status() {
  if daemon_alive; then
    echo "$NAME: RUNNING (pid $(cat "$PIDFILE")) — health: $(health)"
  else
    echo "$NAME: stopped"
  fi
 }

 case "${1:-}" in
  start) shift || true; cmd_start "${1:-}" ;;
  stop)  cmd_stop ;;
  status) cmd_status ;;
  *)     cmd_start "${1:-}" ;;   # no/first-arg==--mtp -> foreground-style start (detached)
 esac
Model	Source	File	On-disk size
Gemma 4 E2B IT QAT Q4_0	`google/gemma-4-E2B-it-qat-q4_0-gguf`	`gemma-4-E2B_q4_0-it.gguf`	3.10 GiB (4.63B params reported)
Gemma 4 E4B IT QAT Q4_0	`google/gemma-4-E4B-it-qat-q4_0-gguf`	`gemma-4-E4B_q4_0-it.gguf`	4.79 GiB (7.46B params reported)
Qwen3.6 35B-A3B MoE IQ2_M	`HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive`	`...-IQ2_M.gguf`	10.85 GiB (34.66B total / ~3B active)
Baseline	local	`Qwen3-0.6B-Q4_0.gguf`	359 MiB
Model	backend	ngl	pp128 t/s	tg64 t/s
Qwen3 0.6B Q4_0	Vulkan	99	1083.44 ± 12.26	88.75 ± 0.26
Gemma 4 E2B Q4_0	Vulkan	99	312.56 ± 7.30	26.15 ± 0.02
Gemma 4 E4B Q4_0	Vulkan	99	156.73 ± 1.46	12.84 ± 0.02
Model	pp	tg
E2B (`pp64`/`tg32`)	11.30	8.48
E4B (`pp64`/`tg32`)	5.59	4.40
Qwen3-0.6B (`pp32`/`tg16`)	41.44	28.11
Qwen3.6-35B-A3B IQ2_M (`pp32`/`tg16`)	3.73	3.21
Goal	Pick
Edge GPU on this gfx90c host	Gemma 4 E2B QAT Q4_0 + Vulkan/RADV (~26 tg t/s)
Stronger edge, still local	Gemma 4 E4B QAT Q4_0 + Vulkan (~13 tg t/s)
MoE mechanics demo	Qwen3.6-35B-A3B IQ2_M (CPU mmap)
ROCm HIP on this iGPU	Blocked — use supported GPU targets instead
Turn	prompt tok	gen tok	secs	gen t/s
decide + emit tool call	95	81	6.5	12.5
grounded answer from result	205	225	14.0	16.1
Model	Repo	Params	Q4 on-disk	Arch	Native ctx	MTP head
Qwen3 0.6B Q4_0	local (baseline)	0.6B	359 MiB	qwen3	32768	no
Qwen3.8-2B Q4_K_M	`empero-ai/Qwen3.8-2B-GGUF`	1.94B	1.21 GiB	qwen35 (Gated DeltaNet)	262144	yes (`nextn`, mtp_layers=1)
Qwen3.8-4B Q4_K_M	`empero-ai/Qwen3.8-4B-GGUF`	4.33B	2.58 GiB	qwen35 (Gated DeltaNet)	262144	yes (`nextn_predict_layers=1`, blk.32.nextn)
Gemma 4 E2B QAT Q4_0	`google/gemma-4-E2B-it-qat-q4_0-gguf`	4.63B	3.10 GiB	gemma4 (PLE, hybrid attn)	131072	no
Gemma 4 E4B QAT Q4_0	`google/gemma-4-E4B-it-qat-q4_0-gguf`	7.46B	4.79 GiB	gemma4	131072	no
Qwen3.6-35B-A3B MoE IQ2_M	`HauhauCS/...`	34.66B / ~3B active	10.85 GiB	qwen35moe	262144	no
Runtime	Version/build	Backend	Tool API
llama.cpp (stock)	commit `0005475` (b10696), `build-vulkan2`	Vulkan/RADV + CPU	yes (OpenAI, verified)
Atomic fork (`atomic-llama-cpp-turboquant`)	commit `cd56093` (b10269)	Vulkan/RADV	yes
Atomic Chat app	AppImage 2.0.0	— (empty engine stubs, needs GUI)	n/a
Zinc (`zolotukhin/zinc`)	zig 0.15.2 ReleaseFast	Vulkan/RADV	no (headless)
gguf-runner	v0.5.0 (Rust, `-C target-cpu=native`)	CPU	yes (REPL `--allowed-tools`)
ROCm TheRock HIP	repo `build`	HIP	broken on gfx90c (see below)
#	Model	Runtime	Backend	pp128 t/s	tg t/s	Tool calling	Notes
1	Qwen3-0.6B Q4_0	llama.cpp	Vulkan	1083.4	88.75	-	tiny baseline
2	Qwen3.8-2B	Zinc	Vulkan	38-43	26.6-33.9	no	fastest decode measured; short-ctx run 33.9
3	Qwen3.8-2B	llama.cpp stock	Vulkan	382.0	27.2	yes	server 256t A/B: 25.15; `-fa off`
4	Gemma 4 E2B	llama.cpp	Vulkan	312.6	26.15	yes (best)	biggest capable model at full speed; ctx 131072
5	Qwen3.8-2B	Atomic fork	Vulkan	342.6	24.9	yes	turbo4 KV: 22.31
6	Qwen3.8-2B	llama.cpp + MTP (draft-mtp n3)	Vulkan	179.3	22.7	yes	256t A/B: 14.7 (accept 0.41)
7	Qwen3.8-2B	llama.cpp CPU-only	CPU 8T	232.1	19.83	no	`-ngl 0`
8	Gemma 4 E4B	llama.cpp	Vulkan	156.7	12.84	-	bigger, half of E2B speed
9	Qwen3.8-2B	gguf-runner	CPU	-	12.4	yes (REPL)
10	Qwen3.8-4B	llama.cpp stock	Vulkan	141.5	12.14	yes (3/3)	reliable tools, 2x slower than E2B
11	Qwen3.8-4B	Atomic fork turbo4	Vulkan	28.0 (live)	11.8	yes	turbo4 ≈ stock; 128k ctx
12	Qwen3.8-4B	Atomic fork turbo4 + draft-mtp n6	Vulkan	21.6 (live)	6.8	yes	MTP = net loss (draft=full model)
13	Qwen3.8-2B	Atomic fork + MTP + turbo3	Vulkan	-	8.6	yes	worst (accept 0.11)
14	Gemma 4 E2B	llama.cpp CPU	CPU	11.3	8.48	-	ref
15	Gemma 4 E4B	llama.cpp CPU	CPU	5.59	4.40	-	ref
16	Qwen3.6-35B-A3B MoE	llama.cpp CPU mmap	CPU	3.73	3.21	-	MoE smoke only
Model	simple tool (`get_weather`)	coding tools (`list dir` → bash)	Full tool-result loop	Verdict for opencode
Qwen3.8-2B	yes (5/5)	NO (answers in prose/```bash block)	no	not for tools — plain reasoning / 262k window only
Qwen3.8-4B	yes	yes (3/3)	yes	yes — but 12 t/s
Gemma 4 E2B	yes	yes	yes	yes — recommended (26 t/s)
Gemma 4 E4B	not tested	not tested	-	likely yes (same template)
Zinc	n/a	n/a	-	no tool API yet
gguf-runner	yes (REPL `--allowed-tools`)	-	-	CLI only
Model	ctx	slots	tg t/s	VmPeak	loads
Qwen3.8-2B	81920 (80k)	4	25.4	5.8 GB	~15 s
Qwen3.8-2B	131072	4	24.8	7.4 GB	ok
Qwen3.8-2B	262144 (native max)	1 (`--parallel 1`)	25.7	~5.7 GB	~6 s
Qwen3.8-4B	131072	1	11.5	ok	~40 s
Qwen3.8-4B	262144	1	—	—	OOM (`ErrorOutOfDeviceMemory`/DeviceLost at load) — only the 2B fits 262k; 4B practical max = 131072
Gemma 4 E2B	8192 (default)	4	23.1	5.5 GB (RSS 383 MiB)	~10 s
Gemma 4 E2B	131072 (native max)	1	22.9 (live) / 26.15 (bench)	10.0 GB (RSS 404 MiB)	~15 s
Gemma 4 E2B	>131072 (e.g. 262144)	1	—	—	silently capped to native 131072 (slot ctx capped to n_ctx_train)