config.toml

[model_providers.qwen]
name = "qwen"
base_url = "http://.../v1"
wire_api = "responses"
bearer_token_env_var = "EMPTY"

[profiles.qwen]
model_provider = "qwen"
model = "Qwen/Qwen3.6-27B"
model_reasoning_effort = "high"
model_context_window = 128000

start-qwen-double.sh

#!/usr/bin/env bash

set -euo pipefail

export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1}"
export TP_SIZE="${TP_SIZE:-2}"
export CTM_VLLM_ENV_DIR="${CTM_VLLM_ENV_DIR:-/opt/ctm-vllm/envs/vllm-stable}"

export MAX_MODEL_LEN="${MAX_MODEL_LEN:-262144}"
export GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9075}"
export KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-auto}"
export MAX_NUM_BATCHED_TOKENS="${MAX_NUM_BATCHED_TOKENS:-16384}"
export MAX_NUM_SEQS="${MAX_NUM_SEQS:-512}"
export LANGUAGE_MODEL_ONLY="${LANGUAGE_MODEL_ONLY:-0}"
LIMIT_MM_PER_PROMPT="${LIMIT_MM_PER_PROMPT:-}"
if [[ -z "$LIMIT_MM_PER_PROMPT" ]]; then
  LIMIT_MM_PER_PROMPT='{"image":4,"video":0}'
fi
export LIMIT_MM_PER_PROMPT

export VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS="${VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS:-1}"
export ENFORCE_EAGER="${ENFORCE_EAGER:-0}"
export DISABLE_CUSTOM_ALL_REDUCE="${DISABLE_CUSTOM_ALL_REDUCE:-1}"
export ENABLE_PREFIX_CACHING="${ENABLE_PREFIX_CACHING:-1}"
export ENABLE_MTP="${ENABLE_MTP:-1}"
export MTP_METHOD="${MTP_METHOD:-mtp}"
export MTP_NUM_SPECULATIVE_TOKENS="${MTP_NUM_SPECULATIVE_TOKENS:-2}"

# This host's GPUs are PCIe-attached without a fast peer interconnect.
export NCCL_P2P_DISABLE="${NCCL_P2P_DISABLE:-1}"
export NCCL_IB_DISABLE="${NCCL_IB_DISABLE:-1}"

exec "$(dirname "${BASH_SOURCE[0]}")/start-qwen.sh" "$@"