Laguna S 2.1 NVFP4 + DFlash + vLLM

start-dflash-laguna-118B-A8B-vllm.bash

#!/usr/bin/env bash
# vLLM Docker launcher for Poolside Laguna S 2.1 NVFP4 + DFlash on one GB10.

set -euo pipefail

TARGET_SNAPSHOT="${HOME:?}/.cache/huggingface/hub/models--poolside--Laguna-S-2.1-NVFP4/snapshots/216d1f13878dd4e715bc7412848d0f330e95bba6"
DRAFT_SNAPSHOT="${HOME:?}/.cache/huggingface/hub/models--poolside--Laguna-S-2.1-DFlash-NVFP4/snapshots/723794750422b3efbf3a7b3af76dffb4ba035943"
HF_CACHE_HOST="${HF_CACHE_HOST:-${HOME}/.cache/huggingface}"
HF_CACHE_CONTAINER="/root/.cache/huggingface"
TARGET_PATH="${HF_CACHE_CONTAINER}/hub/models--poolside--Laguna-S-2.1-NVFP4/snapshots/216d1f13878dd4e715bc7412848d0f330e95bba6"
DRAFT_PATH="${HF_CACHE_CONTAINER}/hub/models--poolside--Laguna-S-2.1-DFlash-NVFP4/snapshots/723794750422b3efbf3a7b3af76dffb4ba035943"
DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:nightly}"

PORT="${PORT:-8000}"
CONTAINER_NAME="${CONTAINER_NAME:-laguna-s21-dflash-vllm}"
SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-poolside/Laguna-S-2.1-NVFP4}"
SERVED_MODEL_ALIAS="${SERVED_MODEL_ALIAS:-poolside/Laguna-S-2.1}"
CONTEXT_LENGTH="${CONTEXT_LENGTH:-262144}"
GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.85}"
MAX_NUM_SEQS="${MAX_NUM_SEQS:-32}"
MAX_NUM_BATCHED_TOKENS="${MAX_NUM_BATCHED_TOKENS:-16384}"
# Live GB10 measurements at K=15 accepted only about 2.0-2.5 tokens per verify
# step, with acceptance almost exhausted after positions 5-8. K=8 retains the
# useful early proposals while avoiding seven consistently rejected tokens.
NUM_SPECULATIVE_TOKENS="${NUM_SPECULATIVE_TOKENS:-8}"
ENABLE_DFLASH="${ENABLE_DFLASH:-1}"

# vLLM otherwise sizes the KV pool from --gpu-memory-utilization. On GB10's
# unified-memory system that auto-allocation consumed 28.04 GiB, leaving only
# about 3 GiB available to the host and pushing it into swap. 14 GiB projects
# to about 325K cache tokens from the measured 651K-token / 28.04-GiB pool,
# enough for one complete 262,144-token request plus modest headroom.
# An explicit cache size overrides GPU_MEMORY_UTILIZATION for KV allocation.
KV_CACHE_MEMORY_BYTES="${KV_CACHE_MEMORY_BYTES:-15032385536}" # 14 GiB

# Leave KV_CACHE_DTYPE unset to use vLLM's model-aware default, which is the
# model card's recommended path. Set KV_CACHE_DTYPE=bfloat16 for an explicit
# precision experiment.
KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-}"

# CUTE DSL needs the GB10-specific architecture string for its FP4 JIT kernels.
# Cap compiler fan-out because a cold FlashInfer cache can otherwise exhaust the
# 128 GB unified-memory pool.
export CUTE_DSL_ARCH="${CUTE_DSL_ARCH:-sm_121a}"
export MAX_JOBS="${MAX_JOBS:-4}"
export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}"
export VLLM_ENGINE_READY_TIMEOUT_S="${VLLM_ENGINE_READY_TIMEOUT_S:-1800}"

# Under speculative decoding, vLLM's strict structural grammar has historically
# been fragile for automatic tool calls. This leaves auto tool calls parseable
# from model text; named/required tool calls should be tested without DFlash.
export VLLM_ENFORCE_STRICT_TOOL_CALLING="${VLLM_ENFORCE_STRICT_TOOL_CALLING:-false}"

for required_path in "${TARGET_SNAPSHOT}" "${DRAFT_SNAPSHOT}"; do
    if [[ ! -d "${required_path}" ]]; then
        echo "Error: required Laguna snapshot not found: ${required_path}" >&2
        exit 1
    fi
done

if [[ "${ENABLE_DFLASH}" != "0" && "${ENABLE_DFLASH}" != "1" ]]; then
    echo "Error: ENABLE_DFLASH must be 0 or 1, got ${ENABLE_DFLASH}." >&2
    exit 1
fi

SPEC_ARGS=()
if [[ "${ENABLE_DFLASH}" == "1" ]]; then
    SPEC_ARGS=(
        --speculative-config
        "{\"method\":\"dflash\",\"model\":\"${DRAFT_PATH}\",\"num_speculative_tokens\":${NUM_SPECULATIVE_TOKENS}}"
    )
fi

KV_CACHE_ARGS=()
if [[ -n "${KV_CACHE_DTYPE}" ]]; then
    KV_CACHE_ARGS=(--kv-cache-dtype "${KV_CACHE_DTYPE}")
fi

echo "Launching Laguna S 2.1 NVFP4 with official vLLM Docker..."
echo "  Image                  : ${DOCKER_IMAGE}"
echo "  Container              : ${CONTAINER_NAME}"
echo "  Port                   : ${PORT}"
echo "  Served model names     : ${SERVED_MODEL_NAME}, ${SERVED_MODEL_ALIAS}"
echo "  Context length         : ${CONTEXT_LENGTH}"
echo "  GPU memory utilization : ${GPU_MEMORY_UTILIZATION}"
echo "  KV cache memory cap    : ${KV_CACHE_MEMORY_BYTES} bytes"
echo "  Max sequences          : ${MAX_NUM_SEQS}"
echo "  Max batched tokens     : ${MAX_NUM_BATCHED_TOKENS}"
echo "  KV cache dtype         : ${KV_CACHE_DTYPE:-model default}"
if [[ "${ENABLE_DFLASH}" == "1" ]]; then
    echo "  Speculation            : DFlash (${NUM_SPECULATIVE_TOKENS} tokens)"
else
    echo "  Speculation            : disabled"
fi
echo "  Thinking               : enabled by default"
echo "  Tool calls             : poolside_v1 parser, auto mode"

# Fail before the 118B load if a stale nightly image lacks Laguna DFlash.
docker run --rm --gpus all --entrypoint python3 "${DOCKER_IMAGE}" -c '
import torch
from vllm.model_executor.models.registry import ModelRegistry

supported = ModelRegistry.get_supported_archs()
assert torch.cuda.get_device_capability() == (12, 1)
assert "LagunaForCausalLM" in supported
assert "DFlashLagunaForCausalLM" in supported
print("Official vLLM GB10 and Laguna DFlash preflight: OK")
'

exec docker run --rm --name "${CONTAINER_NAME}" --gpus all \
    --shm-size 32g \
    --ipc=host \
    -e HF_HUB_OFFLINE=1 \
    -e TRANSFORMERS_OFFLINE=1 \
    -e CUTE_DSL_ARCH="${CUTE_DSL_ARCH}" \
    -e MAX_JOBS="${MAX_JOBS}" \
    -e PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF}" \
    -e VLLM_ENGINE_READY_TIMEOUT_S="${VLLM_ENGINE_READY_TIMEOUT_S}" \
    -e VLLM_ENFORCE_STRICT_TOOL_CALLING="${VLLM_ENFORCE_STRICT_TOOL_CALLING}" \
    -v "${HF_CACHE_HOST}:${HF_CACHE_CONTAINER}" \
    -v "${HOME}/.cache/vllm:/root/.cache/vllm" \
    -v "${HOME}/.cache/vllm_compilers/triton:/root/.triton" \
    -v "${HOME}/.cache/vllm_compilers/nv:/root/.nv" \
    -v "${HOME}/.cache/vllm_compilers/flashinfer:/root/.cache/flashinfer" \
    -v "${HOME}/.cache/vllm_compilers/torch:/root/.cache/torch" \
    -p "${PORT}:${PORT}" \
    "${DOCKER_IMAGE}" \
    --model "${TARGET_PATH}" \
    --served-model-name "${SERVED_MODEL_NAME}" "${SERVED_MODEL_ALIAS}" \
    --host 0.0.0.0 \
    --port "${PORT}" \
    --tensor-parallel-size 1 \
    --trust-remote-code \
    --gpu-memory-utilization "${GPU_MEMORY_UTILIZATION}" \
    --kv-cache-memory-bytes "${KV_CACHE_MEMORY_BYTES}" \
    --max-model-len "${CONTEXT_LENGTH}" \
    --max-num-seqs "${MAX_NUM_SEQS}" \
    --max-num-batched-tokens "${MAX_NUM_BATCHED_TOKENS}" \
    --enable-prefix-caching \
    --chat-template "${TARGET_PATH}/chat_template.jinja" \
    --default-chat-template-kwargs '{"enable_thinking":true}' \
    --override-generation-config '{"temperature":0.7,"top_p":0.95}' \
    --enable-auto-tool-choice \
    --tool-call-parser poolside_v1 \
    --reasoning-parser poolside_v1 \
    "${KV_CACHE_ARGS[@]}" \
    "${SPEC_ARGS[@]}"

Prerequisites:

docker pull vllm/vllm-openai:nightly
hf download poolside/Laguna-S-2.1-DFlash-NVFP4
hf download poolside/Laguna-S-2.1-NVFP4