sparkrun recipe: Qwen3.8-27B NVFP4 + DFlash2

qwen38-27b-dflash2-amasu.yaml

# Qwen3.8-27B NVFP4 + DFlash2 — faithful port of amasu/dgx-spark-qwen38
# docker-compose.yml (current production stack, SGLang + NVFP4 + DFlash2,
# greedy default) to a sparkrun recipe for a single DGX Spark.
#
# Origin:
# https://github.com/amasu/dgx-spark-qwen38
#
# NOTE: this stack uses the PLAIN NVFP4 checkpoint — the
# repo A/B tested RadixArk/Qwen3.8-27B-NVFP4-BF16-LMHead on 2026-08-29 and
# REJECTED it (decode regressed ~20-25%, no quality gain). It also does NOT
# set --kv-cache-dtype fp8_e4m3, --context-length, --mamba-full-memory-ratio,
# --sampling-defaults, or the metrics flags; instead it enables torch.compile,
# cuda-graph/compile batch caps, unquant draft, auto-truncate, a patched chat
# template, and sleep-on-idle.
#
# Prerequisites (once per host — see repo README "Obtaining the serving image"):
#   * qwen38-dflash2:v1.2.2 is a LOCAL image tag, NOT pullable from any
#     registry. Build it via hasso5703/dgx-spark-qwen38:
#       git clone https://github.com/hasso5703/dgx-spark-qwen38 && cd dgx-spark-qwen38
#       ./install.sh --no-service
#     (= lmsysorg/sglang:qwen38-27b @ sha256:febfb971c735... + the 5-file
#     sha256-verified DFlash2 overlay from SGLang commit 1cf2b8c / PR #35496 —
#     the quantized-lm_head selector the NVFP4 checkpoint requires. Without it,
#     boot fails with "requires a dense FP16/BF16/FP32 target lm_head".)
#   * The patched chat template (reasoning_effort max/high->xhigh + mid-
#     conversation system messages as <system-reminder> blocks) is generated
#     at launch by pre_exec below, inside the container, from the model's own
#     chat_template.jinja in the mounted HF cache — no host-side staging
#     needed. Requires network at launch (curl of the pinned patch script).
#
# Usage:
#   sparkrun run qwen38-27b-dflash2-amasu.yaml --rootful --no-follow
#
# First boot ~9 min (torch.compile + CUDA graph capture). Smoke test:
#   curl -s -o /dev/null -w "%{http_code}\n" http://localhost:8000/health   # → 200
#
# Sampler default is greedy (temp 0) per the repo's engine × sampler A/B —
# that is a client-request convention, not a server flag.
recipe_version: "2"
model: RadixArk/Qwen3.8-27B-NVFP4
runtime: sglang
container: qwen38-dflash2:v1.2.2

max_nodes: 1

metadata:
  model_dtype: nvfp4
  description: >-
    Qwen3.8-27B NVFP4 + DFlash2 (amasu production compose port) —
    single-Spark, plain NVFP4 lm_head, torch.compile, patched chat template

defaults:
  port: 8000
  host: 0.0.0.0
  tensor_parallel: 1
  # Do NOT raise above 0.50 — SGLang under-counts GB10 unified-memory
  # transients; >0.50 risks a host freeze (repo README).
  gpu_memory_utilization: 0.50
  served_model_name: qwen3.8-27b
  attention_backend: flashinfer
  # Compose default is 8192, but production adopted 2048 via the host .env
  # (2026-08-29 A/B: decode within noise, long-context TTFT win at 512K ctx).
  chunked_prefill_size: 2048
  cuda_graph_max_bs: 8
  torch_compile_max_bs: 4
  max_mamba_cache_size: 96
  max_running_requests: 8
  speculative_algorithm: DFLASH
  speculative_draft_model_path: z-lab/Qwen3.8-27B-DFlash2
  speculative_draft_model_revision: 50307d4c4cde6860d4eee73e2547cd786fe8e8a4
  speculative_num_draft_tokens: 8
  chat_template: /out/chat-template-sglang.jinja

env:
  HF_HUB_OFFLINE: "0"
  TORCHINDUCTOR_CACHE_DIR: /cache/inductor
  HF_HOME: /cache/huggingface
  HF_HUB_CACHE: /cache/huggingface/hub

executor_config:
  # Compose sets its own entrypoint; the local image's entrypoint must not
  # consume sparkrun's serve command.
  entrypoint: ""
  # Compose: mem_limit/memswap_limit 100g, shm_size 16g, restart unless-stopped
  # (sparkrun has no memswap knob; the real guard is --mem-fraction-static).
  memory_limit: 100g
  shm_size: 16g
  restart_policy: unless-stopped
  # ipc: host and network: host are sparkrun's Docker defaults (match compose).

pre_exec:
  # Generate the patched chat template in-container. patch-template.py is
  # hasso5703/dgx-spark-qwen38's stdlib-only patcher (pinned commit = v1.5.9);
  # it reads chat_template.jinja from the target model's snapshot in the HF
  # cache (mounted by sparkrun at /cache/huggingface, model already synced)
  # and writes the patched copy. Idempotent; fails loudly if upstream changed.
  #
  # Offline alternative: if the host already has the install.sh-written
  # template, replace this entry with a directory copy (source must be a
  # DIRECTORY containing only the .jinja file; sparkrun runs
  # `docker cp <src>/. <container>:/out/`):
  #   - copy: ~/sparkrun/assets/qwen38-out
  #     dest: /out
  - >-
    mkdir -p /out &&
    curl -fsSL https://raw.githubusercontent.com/hasso5703/dgx-spark-qwen38/17e7e2280e632b0a3ab91839c8c7522b256937ac/patch-template.py
    -o /tmp/patch-template.py &&
    python3 /tmp/patch-template.py /cache/huggingface /out/chat-template-sglang.jinja

command: |
  python3 -m sglang.launch_server \
    --trust-remote-code \
    --model-path {model} \
    --tp-size {tensor_parallel} \
    --served-model-name {served_model_name} \
    --mem-fraction-static {gpu_memory_utilization} \
    --attention-backend {attention_backend} \
    --chunked-prefill-size {chunked_prefill_size} \
    --disable-prefill-cuda-graph \
    --cuda-graph-max-bs {cuda_graph_max_bs} \
    --disable-flashinfer-autotune \
    --speculative-algorithm {speculative_algorithm} \
    --speculative-draft-model-path {speculative_draft_model_path} \
    --speculative-draft-model-revision {speculative_draft_model_revision} \
    --speculative-num-draft-tokens {speculative_num_draft_tokens} \
    --speculative-draft-model-quantization unquant \
    --mamba-radix-cache-strategy extra_buffer \
    --mamba-ssm-dtype bfloat16 \
    --max-mamba-cache-size {max_mamba_cache_size} \
    --max-running-requests {max_running_requests} \
    --enable-torch-compile \
    --torch-compile-max-bs {torch_compile_max_bs} \
    --num-continuous-decode-steps 2 \
    --allow-auto-truncate \
    --reasoning-parser qwen3 \
    --tool-call-parser qwen3_coder \
    --chat-template {chat_template} \
    --sleep-on-idle \
    --host {host} \
    --port {port}