vllm locally on 5060Ti 16GB x 2

compose.yaml

services:
  vllm-server:
    image: vllm/vllm-openai:latest
    container_name: vllm-server
    restart: unless-stopped
    ports:
      - "${VLLM_PORT}:8000"
    environment:
      - VLLM_FP4_GEMM_BACKEND=cutlass
      - NVIDIA_VISIBLE_DEVICES=all
      - NVIDIA_DRIVER_CAPABILITIES=compute,utility
    volumes:
      - ${HF_CACHE_DIR}:/root/.cache/huggingface
    ipc: host
    shm_size: '16gb'
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    command: >
      --model ${VLLM_MODEL}
      --served-model-name ${VLLM_SERVED_NAME}
      --tensor-parallel-size ${VLLM_TP_SIZE}
      --gpu-memory-utilization ${VLLM_MEMORY_UTIL}
      --max-model-len ${VLLM_MAX_MODEL_LEN}
      --max-num-seqs ${VLLM_MAX_NUM_SEQS}
      --api-key ${API_KEY}
      --language-model-only
      --enable-auto-tool-choice
      --generation-config vllm
      --trust-remote-code
      --kv-cache-dtype fp8
      --reasoning-parser ${VLLM_REASON_PARSER}
      --tool-call-parser ${VLLM_TOOL_PARSER}
      --speculative-config '{"method": "mtp", "num_speculative_tokens": 2}'
      --default-chat-template-kwargs '{"preserve_thinking": true}'
      --host 0.0.0.0

qwen3.6.env

VLLM_PORT=8000

VLLM_MODEL=unsloth/Qwen3.6-35B-A3B-NVFP4
VLLM_SERVED_NAME=Qwen3.6-35B-A3B-NVFP4

VLLM_REASON_PARSER=qwen3
VLLM_TOOL_PARSER=qwen3_coder

# number of GPUs
VLLM_TP_SIZE=2
VLLM_MEMORY_UTIL=0.9

# context length 
VLLM_MAX_MODEL_LEN=131072
VLLM_MAX_NUM_SEQS=8

HF_CACHE_DIR=~/.cache/huggingface
API_KEY=replace_with_your_secret