run-vllm.sh

#!/bin/sh

type=cuda
type=rocm
model=google/gemma-4-26B-A4B-it
u=0.9

if [ "$#" -eq 0 ]; then
  set -- -d --restart=unless-stopped
fi

if [ "$type" = "cuda" ]; then
  image=vllm/vllm-openai:nightly
  set -- "$@" --gpus all
elif [ "$type" = "rocm" ]; then
  image=vllm/vllm-openai-rocm:nightly
  set -- "$@" --device /dev/kfd --device /dev/dri \
    --security-opt seccomp=unconfined --group-add video
fi

if command -v hf > /dev/null 2>&1; then
  hf_token=$(hf auth token 2>/dev/null)
  if ! [ -z "$hf_token" ]; then
    set -- "$@" -e "HF_TOKEN=$hf_token"
  fi
fi

set -- "$@" --privileged --ipc=host \
  --network host \
  --name vllm \
  -v vllm-cache:/root/.cache/huggingface \
  $image \
  $model \
  --gpu-memory-utilization $u \
  --host 0.0.0.0 --port 8013 \
  --tensor-parallel-size 1 \
  --enable-auto-tool-choice \
  --tool-call-parser gemma4 \
  --max-model-len -1 \
  --async-scheduling \
  --chat-template vllm/examples/tool_chat_template_gemma4.jinja \
  --reasoning-parser gemma4

docker pull "$image"
docker run "$@"