vllm test

000.memo.md

Memo

mkdir -p /data/tmp/vllm

podman container run \
 --rm \
 --name vllm \
 --init \
 --read-only \
  -ti --entrypoint /bin/bash \
 --user 1000:1000 \
 --env=HOME=/var/cache/vllm \
 --env=XDG_CACHE_HOME=/var/cache/vllm \
 --device nvidia.com/gpu=all \
 --publish 18000:8000 \
 --volume /data/models/qwen/Qwen3-Embedding-0.6B:/mnt/model:ro \
 --volume /data/tmp/vllm:/var/cache/vllm \
 docker.io/vllm/vllm-openai:v0.19.1-cu130 \
;

export PYTHONHASHSEED=1
export HF_HUB_OFFLINE=1
export DO_NOT_TRACK=1

export VLLM_DISABLE_LOG_LOGO=1
export VLLM_LOGGING_LEVEL=INFO
export VLLM_LOG_STATS_INTERVAL=1


vllm serve \
 /mnt/model --served-model-name Qwen/Qwen3-Embedding-0.6B current --hf_overrides='{"is_matryoshka": true}' \
 --max-num-seqs=128 \
 --max-num-batched-tokens=16384 \
 --no-enable-prefix-caching \
 --shutdown-timeout=30 \
 --fail-on-environ-validation \
 --enable-offline-docs \
 --disable-access-log-for-endpoints="/health,/metrics,/ping" \
 --enable-request-id-headers \
 --no-use-tqdm-on-load \
;

Benchmark

mkdir -p /data/tmp/vllm-bench

podman container run \
 --rm \
 --name vllm-cpu \
 --init \
 --read-only \
 -ti --entrypoint /bin/bash \
 --volume /data/models/qwen/Qwen3-Embedding-0.6B:/mnt/model:ro \
 --volume /data/tmp/vllm-bench:/vllm-workspace \
 docker.io/vllm/vllm-openai-cpu:v0.19.1 \
;

vllm bench serve \
  --backend=openai-embeddings \
  --base-url=http://host.containers.internal:18000 \
  --endpoint=/v1/embeddings \
  --result-dir=/vllm-workspace \
  --save-result --save-detailed --plot-timeline \
  --metric-percentiles=80,90,95,99 \
  --num-warmups=10 \
  --dataset-name=random \
  --random-batch-size=5 \
  --random-range-ratio=0.995 --input-len=2048 \
  --num-prompts=1000 --max-concurrency=10 \
;

Backup

export NO_COLOR=1
export XDG_CACHE_HOME=/var/cache/vllm
export VLLM_CACHE_ROOT="$XDG_CACHE_HOME"

--env=XDG_CACHE_HOME=/var/cache/vllm --env=VLLM_CACHE_ROOT=/var/cache/vllm --env=FLASHINFER_WORKSPACE_DIR=/var/cache/vllm --env=TORCHINDUCTOR_CACHE_DIR=/var/c
--enable-log-requests 
--max-num-batched-tokens=2048 --max-cudagraph-capture-size=2048 
--max-cudagraph-capture-size=16384
 --enable-server-load-tracking --cudagraph-metrics --enable-mfu-metrics --max-num-batched-tokens=16384 --max-num-seqs=128 --cudagraph-capture-sizes 16 32 64 128 256 384 512 768 1024 1536 2048 2560 3072 4096 6144 7680 8192 10240 16384 20480 25600 32768

export VLLM_LOGGING_LEVEL=DEBUG
pip intall orjson

(EngineCore pid=879) INFO 04-19 00:10:03 [core.py:105] Initializing a V1 LLM engine (v0.19.1) with config: model='/mnt/model', speculative_config=None, tokenizer='/mnt/model', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=32768, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=Qwen/Qwen3-Embedding-0.6B, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=PoolerConfig(task=None, pooling_type=None, seq_pooling_type='LAST', tok_pooling_type='ALL', use_activation=True, dimensions=None, enable_chunked_processing=False, max_embed_len=None, logit_bias=None, step_tag_id=None, returned_token_ids=None), compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_images_per_batch': 0, 'compile_sizes': [], 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.PIECEWISE: 1>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 512, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': True, 'static_all_moe_layers': []}

help-bench-serve.txt

https://docs.vllm.ai/en/latest/cli/bench/serve/
root@730d3846f091:/vllm-workspace# vllm bench serve --help=all
usage: vllm bench serve [options]

Benchmark the online serving throughput.

options:
  --append-result       Append the benchmark result to the existing json file. (default: False)
  --backend {vllm,openai,openai-chat,openai-audio,openai-embeddings,openai-embeddings-chat,openai-embeddings-clip,openai-embeddings-vlm2vec,infinity-embeddings,infinity-embeddings-clip,vllm-pooling,vllm-rerank}
                        The type of backend or endpoint to use for the benchmark. (default: openai)
  --base-url BASE_URL   Server or API base url if not using http host and port. (default: None)
  --burstiness BURSTINESS
                        Burstiness factor of the request generation. Only take effect when request_rate is not inf. Default value is 1, which follows Poisson process. Otherwise, the request intervals follow a gamma distribution. A lower burstiness value (0 <
                        burstiness < 1) results in more bursty requests. A higher burstiness value (burstiness > 1) results in a more uniform arrival of requests. (default: 1.0)
  --dataset-name {sharegpt,burstgpt,sonnet,random,random-mm,random-rerank,hf,custom,custom_mm,prefix_repetition,spec_bench}
                        Name of the dataset to benchmark on. (default: random)
  --dataset-path DATASET_PATH
                        Path to the sharegpt/sonnet dataset. Or the huggingface dataset ID if using HF dataset. (default: None)
  --disable-shuffle     Disable shuffling of dataset samples for deterministic ordering. (default: False)
  --disable-tqdm        Specify to disable tqdm progress bar. (default: False)
  --enable-multimodal-chat
                        Enable multimodal chat transformation for datasets that support it. (default: False)
  --endpoint ENDPOINT   API endpoint. (default: /v1/completions)
  --extra-body EXTRA_BODY
                        A JSON string representing extra body parameters to include in each request.Example: '{"chat_template_kwargs":{"enable_thinking":false}}' (default: None)
  --goodput GOODPUT [GOODPUT ...]
                        Specify service level objectives for goodput as "KEY:VALUE" pairs, where the key is a metric name, and the value is in milliseconds. Multiple "KEY:VALUE" pairs can be provided, separated by spaces. Allowed request level metric names are "ttft",
                        "tpot", "e2el". For more context on the definition of goodput, refer to DistServe paper: https://arxiv.org/pdf/2401.09670 and the blog: https://hao-ai-lab.github.io/blogs/distserve (default: None)
  --header [KEY=VALUE ...]
                        Key-value pairs (e.g, --header x-additional-info=0.3.3) for headers to be passed with each request. These headers override per backend constants and values set via environment variable, and will be overridden by other arguments (such as request
                        ids). (default: None)
  --host HOST
  --ignore-eos          Set ignore_eos flag when sending the benchmark request.Warning: ignore_eos is not supported in deepspeed_mii and tgi. (default: False)
  --input-len INPUT_LEN
                        General input length for datasets. Maps to dataset-specific input length arguments (e.g., --random-input-len, --sonnet-input-len). If not specified, uses dataset defaults. (default: None)
  --insecure            Disable SSL certificate verification. Use this option when connecting to servers with self-signed certificates. (default: False)
  --label LABEL         The label (prefix) of the benchmark results. If not specified, the value of '--backend' will be used as the label. (default: None)
  --logprobs LOGPROBS   Number of logprobs-per-token to compute & return as part of the request. If unspecified, then either (1) if beam search is disabled, no logprobs are computed & a single dummy logprob is returned for each token; or (2) if beam search is enabled
                        1 logprob per token is computed (default: None)
  --lora-assignment {random,round-robin}
                        Strategy for assigning LoRA modules to requests. 'random' (default) selects a LoRA at random for each request. 'round-robin' cycles through LoRA modules deterministically. (default: random)
  --lora-modules LORA_MODULES [LORA_MODULES ...]
                        A subset of LoRA module names passed in when launching the server. For each request, the script chooses a LoRA module at random by default. Use --lora-assignment to control selection strategy. (default: None)
  --max-concurrency MAX_CONCURRENCY
                        Maximum number of concurrent requests. This can be used to help simulate an environment where a higher level component is enforcing a maximum number of concurrent requests. While the --request-rate argument controls the rate at which requests
                        are initiated, this argument will control how many are actually allowed to execute at a time. This means that when used in combination, the actual request rate may be lower than specified with --request-rate, if the server is not processing
                        requests fast enough to keep up. (default: None)
  --metadata [KEY=VALUE ...]
                        Key-value pairs (e.g, --metadata version=0.3.3 tp=1) for metadata of this run to be saved in the result JSON file for record keeping purposes. (default: None)
  --metric-percentiles METRIC_PERCENTILES
                        Comma-separated list of percentiles for selected metrics. To report 25-th, 50-th, and 75-th percentiles, use "25,50,75". Default value is "99".Use "--percentile-metrics" to select metrics. (default: 99)
  --model MODEL         Name of the model. If not specified, will fetch the first model from the server's /v1/models endpoint. (default: None)
  --no-oversample       Do not oversample if the dataset has fewer samples than num-prompts. (default: False)
  --no-stream           Do not load the dataset in streaming mode. (default: False)
  --num-prompts NUM_PROMPTS
                        Number of prompts to process. (default: 1000)
  --num-warmups NUM_WARMUPS
                        Number of warmup requests. (default: 0)
  --output-len OUTPUT_LEN
                        General output length for datasets. Maps to dataset-specific output length arguments (e.g., --random-output-len, --sonnet-output-len). If not specified, uses dataset defaults. (default: None)
  --percentile-metrics PERCENTILE_METRICS
                        Comma-separated list of selected metrics to report percentiles. This argument specifies the metrics to report percentiles. Allowed metric names are "ttft", "tpot", "itl", "e2el". If not specified, defaults to "ttft,tpot,itl" for generative
                        models and "e2el" for pooling models. (default: None)
  --plot-dataset-stats  Generate a matplotlib figure with dataset statistics showing prompt tokens, output tokens, and combined token distributions. (default: False)
  --plot-timeline       Generate an HTML timeline plot showing request execution. The plot will be saved alongside the results JSON file. (default: False)
  --port PORT
  --profile             Use vLLM Profiling. --profiler-config must be provided on the server. (default: False)
  --ramp-up-end-rps RAMP_UP_END_RPS
                        The ending request rate for ramp-up (RPS). Needs to be specified when --ramp-up-strategy is used. (default: None)
  --ramp-up-start-rps RAMP_UP_START_RPS
                        The starting request rate for ramp-up (RPS). Needs to be specified when --ramp-up-strategy is used. (default: None)
  --ramp-up-strategy {linear,exponential}
                        The ramp-up strategy. This would be used to ramp up the request rate from initial RPS to final RPS rate (specified by --ramp-up-start-rps and --ramp-up-end-rps.) over the duration of the benchmark. (default: None)
  --ready-check-timeout-sec READY_CHECK_TIMEOUT_SEC
                        Maximum time to wait for the endpoint to become ready in seconds. Ready check will be skipped by default. (default: 0)
  --request-id-prefix REQUEST_ID_PREFIX
                        Specify the prefix of request id. (default: bench-7cc80001-)
  --request-rate REQUEST_RATE
                        Number of requests per second. If this is inf, then all the requests are sent at time 0. Otherwise, we use Poisson process or gamma distribution to synthesize the request arrival times. (default: inf)
  --result-dir RESULT_DIR
                        Specify directory to save benchmark json results.If not specified, results are saved in the current directory. (default: None)
  --result-filename RESULT_FILENAME
                        Specify the filename to save benchmark json results.If not specified, results will be saved in {label}-{args.request_rate}qps-{base_model_id}-{current_dt}.json format. (default: None)
  --save-detailed       When saving the results, whether to include per request information such as response, error, ttfts, tpots, etc. (default: False)
  --save-result         Specify to save benchmark results to a json file (default: False)
  --seed SEED
  --served-model-name SERVED_MODEL_NAME
                        The model name used in the API. If not specified, the model name will be the same as the `--model` argument.  (default: None)
  --skip-chat-template  Skip applying chat template to prompt for datasets that support it. (default: False)
  --skip-tokenizer-init
                        Skip initialization of tokenizer and detokenizer (default: False)
  --timeline-itl-thresholds THRESHOLD1 THRESHOLD2
                        ITL thresholds in milliseconds for timeline plot coloring. Specify two values to categorize inter-token latencies into three groups: below first threshold (green), between thresholds (orange), and above second threshold (red). Default: 25 50
                        (milliseconds). (default: [25.0, 50.0])
  --tokenizer TOKENIZER
                        Name or path of the tokenizer, if not using the default tokenizer. (default: None)
  --tokenizer-mode TOKENIZER_MODE
                        Tokenizer mode:
                                - "auto" will use the tokenizer from `mistral_common` for Mistral models if available, otherwise it will use the "hf" tokenizer.
                                - "hf" will use the fast tokenizer if available.
                                - "slow" will always use the slow tokenizer.
                                - "mistral" will always use the tokenizer from `mistral_common`.
                                - "deepseek_v32" will always use the tokenizer from `deepseek_v32`.
                                - "qwen_vl" will always use the tokenizer from `qwen_vl`.
                                - Other custom values can be supported via plugins. (default: auto)
  --trust-remote-code   Trust remote code from huggingface (default: False)
  --use-beam-search
  -h, --help            show this help message and exit

custom dataset options:
  --custom-output-len CUSTOM_OUTPUT_LEN
                        Number of output tokens per request. Unless it is set to -1, the value overrides potential output length loaded from the dataset. It is used only for custom dataset. (default: 256)

spec bench dataset options:
  --spec-bench-category SPEC_BENCH_CATEGORY
                        Category for spec bench dataset. If None, use all categories. (default: None)
  --spec-bench-output-len SPEC_BENCH_OUTPUT_LEN
                        Num of output tokens per request, used only for spec bench dataset. (default: 256)

sonnet dataset options:
  --sonnet-input-len SONNET_INPUT_LEN
                        Number of input tokens per request, used only for sonnet dataset. (default: 550)
  --sonnet-output-len SONNET_OUTPUT_LEN
                        Number of output tokens per request, used only for sonnet dataset. (default: 150)
  --sonnet-prefix-len SONNET_PREFIX_LEN
                        Number of prefix tokens per request, used only for sonnet dataset. (default: 200)

sharegpt dataset options:
  --sharegpt-output-len SHAREGPT_OUTPUT_LEN
                        Output length for each request. Overrides the output length from the ShareGPT dataset. (default: None)

blazedit dataset options:
  --blazedit-max-distance BLAZEDIT_MAX_DISTANCE
                        Maximum distance for blazedit dataset. Min: 0, Max: 1.0 (default: 1.0)
  --blazedit-min-distance BLAZEDIT_MIN_DISTANCE
                        Minimum distance for blazedit dataset. Min: 0, Max: 1.0 (default: 0.0)

asr dataset options:
  --asr-max-audio-len-sec ASR_MAX_AUDIO_LEN_SEC
                        Maximum audio length in seconds for ASR dataset. (default: inf)
  --asr-min-audio-len-sec ASR_MIN_AUDIO_LEN_SEC
                        Minimum audio length in seconds for ASR dataset. (default: 0.0)

random dataset options:
  --no-reranker         Whether the model supports reranking natively. Only used for reranker benchmark. (default: False)
  --random-batch-size RANDOM_BATCH_SIZE
                        Batch size for random sampling. Only used for embeddings benchmark. (default: 1)
  --random-input-len RANDOM_INPUT_LEN
                        Number of input tokens per request, used only for random sampling. (default: 1024)
  --random-output-len RANDOM_OUTPUT_LEN
                        Number of output tokens per request, used only for random sampling. (default: 128)
  --random-prefix-len RANDOM_PREFIX_LEN
                        Number of fixed prefix tokens before the random context in a request. The total input length is the sum of `random-prefix-len` and a random context length sampled from [input_len * (1 - range_ratio), input_len * (1 + range_ratio)]. (default: 0)
  --random-range-ratio RANDOM_RANGE_RATIO
                        Range ratio for sampling input/output length, used only for random sampling. Must be in the range [0, 1) to define a symmetric sampling range[length * (1 - range_ratio), length * (1 + range_ratio)]. (default: 0.0)

random multimodal dataset options extended from random dataset:
  --random-mm-base-items-per-request RANDOM_MM_BASE_ITEMS_PER_REQUEST
                        Base number of multimodal items per request for random-mm. Actual per-request count is sampled around this base using --random-mm-num-mm-items-range-ratio. (default: 1)
  --random-mm-bucket-config RANDOM_MM_BUCKET_CONFIG
                        The bucket config is a dictionary mapping a multimodal itemsampling configuration to a probability.Currently allows for 2 modalities: images and videos. An bucket key is a tuple of (height, width, num_frames)The value is the probability of
                        sampling that specific item. Example: --random-mm-bucket-config {(256, 256, 1): 0.5, (720, 1280, 1): 0.4, (720, 1280, 16): 0.10} First item: images with resolution 256x256 w.p. 0.5Second item: images with resolution 720x1280 w.p. 0.4 Third
                        item: videos with resolution 720x1280 and 16 frames w.p. 0.1OBS.: If the probabilities do not sum to 1, they are normalized.OBS bis.: Only image sampling is supported for now. (default: {(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16):
                        0.0})
  --random-mm-limit-mm-per-prompt RANDOM_MM_LIMIT_MM_PER_PROMPT
                        Per-modality hard caps for items attached per request, e.g. '{"image": 3, "video": 0}'. The sampled per-request item count is clamped to the sum of these limits. When a modality reaches its cap, its buckets are excluded and probabilities are
                        renormalized.OBS.: Only image sampling is supported for now. (default: {'image': 255, 'video': 1})
  --random-mm-num-mm-items-range-ratio RANDOM_MM_NUM_MM_ITEMS_RANGE_RATIO
                        Range ratio r in [0, 1] for sampling items per request. We sample uniformly from the closed integer range [floor(n*(1-r)), ceil(n*(1+r))] where n is the base items per request. r=0 keeps it fixed; r=1 allows 0 items. The maximum is clamped to
                        the sum of per-modality limits from --random-mm-limit-mm-per-prompt. An error is raised if the computed min exceeds the max. (default: 0.0)

hf dataset options:
  --hf-name HF_NAME     Name of the dataset on HuggingFace (e.g., 'lmarena-ai/VisionArena-Chat'). Specify this if your dataset-path is a local path. (default: None)
  --hf-output-len HF_OUTPUT_LEN
                        Output length for each request. Overrides the output lengths from the sampled HF dataset. (default: None)
  --hf-split HF_SPLIT   Split of the HF dataset. (default: None)
  --hf-subset HF_SUBSET
                        Subset of the HF dataset. (default: None)

prefix repetition dataset options:
  --prefix-repetition-num-prefixes PREFIX_REPETITION_NUM_PREFIXES
                        Number of prefixes to generate, used only for prefix repetition dataset. Prompts per prefix is num_requests // num_prefixes. (default: 10)
  --prefix-repetition-output-len PREFIX_REPETITION_OUTPUT_LEN
                        Number of output tokens per request, used only for prefix repetition dataset. (default: 128)
  --prefix-repetition-prefix-len PREFIX_REPETITION_PREFIX_LEN
                        Number of prefix tokens per request, used only for prefix repetition dataset. (default: 256)
  --prefix-repetition-suffix-len PREFIX_REPETITION_SUFFIX_LEN
                        Number of suffix tokens per request, used only for prefix repetition dataset. Total input length is prefix_len + suffix_len. (default: 256)

sampling parameters:
  --frequency-penalty FREQUENCY_PENALTY
                        Frequency penalty sampling parameter. Only has effect on openai-compatible backends. (default: None)
  --min-p MIN_P         Min-p sampling parameter. Only has effect on openai-compatible backends. (default: None)
  --presence-penalty PRESENCE_PENALTY
                        Presence penalty sampling parameter. Only has effect on openai-compatible backends. (default: None)
  --repetition-penalty REPETITION_PENALTY
                        Repetition penalty sampling parameter. Only has effect on openai-compatible backends. (default: None)
  --temperature TEMPERATURE
                        Temperature sampling parameter. Only has effect on openai-compatible backends. (default: None)
  --top-k TOP_K         Top-k sampling parameter. Only has effect on openai-compatible backends. (default: None)
  --top-p TOP_P         Top-p sampling parameter. Only has effect on openai-compatible backends. (default: None)

When passing JSON CLI arguments, the following sets of arguments are equivalent:
   --json-arg '{"key1": "value1", "key2": {"key3": "value2"}}'
   --json-arg.key1 value1 --json-arg.key2.key3 value2

Additionally, list elements can be passed individually using +:
   --json-arg '{"key4": ["value3", "value4", "value5"]}'
   --json-arg.key4+ value3 --json-arg.key4+='value4,value5'

help-serve.txt

https://docs.vllm.ai/en/latest/cli/serve/

usage: vllm serve [model_tag] [options]

Launch a local OpenAI-compatible API server to serve LLM
completions via HTTP. Defaults to Qwen/Qwen3-0.6B if no model is specified.

Search by using: `--help=<ConfigGroup>` to explore options by section (e.g.,
--help=ModelConfig, --help=Frontend)
  Use `--help=all` to show all available flags at once.

positional arguments:
  model_tag             The model tag to serve (optional if specified in config) (default: None)

options:
  --aggregate-engine-logging
                        Log aggregate rather than per-engine statistics when using data parallelism. (default: False)
  --api-server-count API_SERVER_COUNT, -asc API_SERVER_COUNT
                        How many API server processes to run. Defaults to data_parallel_size if not specified. (default: None)
  --config CONFIG       Read CLI options from a config file. Must be a YAML with the following options: https://docs.vllm.ai/en/latest/configuration/serve_args.html (default: None)
  --disable-log-stats   Disable logging statistics. (default: False)
  --enable-log-requests, --no-enable-log-requests
                        Enable logging request information, dependent on log level:
                        - INFO: Request ID, parameters and LoRA request.
                        - DEBUG: Prompt inputs (e.g: text, token IDs). You can set the minimum log level via `VLLM_LOGGING_LEVEL`. (default: False)
  --fail-on-environ-validation, --no-fail-on-environ-validation
                        If set, the engine will raise an error if environment validation fails. (default: False)
  --gdn-prefill-backend {flashinfer,triton}
                        Select GDN prefill backend. (default: None)
  --grpc                Launch a gRPC server instead of the HTTP OpenAI-compatible server. Requires: pip install vllm[grpc]. (default: False)
  --headless            Run in headless mode. See multi-node data parallel documentation for more details. (default: False)
  --shutdown-timeout SHUTDOWN_TIMEOUT
                        Shutdown timeout in seconds. 0 = abort, >0 = wait. (default: 0)
  -h, --help            show this help message and exit

Frontend:
  Arguments for the OpenAI-compatible frontend server.

  --allow-credentials, --no-allow-credentials
                        Allow credentials. (default: False)
  --allowed-headers ALLOWED_HEADERS
                        Allowed headers. (default: ['*'])
  --allowed-methods ALLOWED_METHODS
                        Allowed methods. (default: ['*'])
  --allowed-origins ALLOWED_ORIGINS
                        Allowed origins. (default: ['*'])
  --api-key API_KEY [API_KEY ...]
                        If provided, the server will require one of these keys to be presented in the header. (default: None)
  --chat-template CHAT_TEMPLATE
  --chat-template-content-format {auto,openai,string}
  --default-chat-template-kwargs DEFAULT_CHAT_TEMPLATE_KWARGS
                        Should either be a valid JSON string or JSON keys passed individually. (default: None)
  --disable-access-log-for-endpoints DISABLE_ACCESS_LOG_FOR_ENDPOINTS
                        Comma-separated list of endpoint paths to exclude from uvicorn access logs. This is useful to reduce log noise from high-frequency endpoints like health checks. Example: "/health,/metrics,/ping". When set, access logs for requests to these
                        paths will be suppressed while keeping logs for other endpoints. (default: None)
  --disable-fastapi-docs, --no-disable-fastapi-docs
                        Disable FastAPI's OpenAPI schema, Swagger UI, and ReDoc endpoint. (default: False)
  --disable-uvicorn-access-log, --no-disable-uvicorn-access-log
                        Disable uvicorn access log. (default: False)
  --enable-auto-tool-choice, --no-enable-auto-tool-choice
  --enable-force-include-usage, --no-enable-force-include-usage
  --enable-log-deltas, --no-enable-log-deltas
  --enable-log-outputs, --no-enable-log-outputs
  --enable-offline-docs, --no-enable-offline-docs
                        Enable offline FastAPI documentation for air-gapped environments. Uses vendored static assets bundled with vLLM. (default: False)
  --enable-prompt-tokens-details, --no-enable-prompt-tokens-details
  --enable-request-id-headers, --no-enable-request-id-headers
                        If specified, API server will add X-Request-Id header to responses. (default: False)
  --enable-server-load-tracking, --no-enable-server-load-tracking
  --enable-ssl-refresh, --no-enable-ssl-refresh
                        Refresh SSL Context when SSL certificate files change (default: False)
  --enable-tokenizer-info-endpoint, --no-enable-tokenizer-info-endpoint
  --exclude-tools-when-tool-choice-none, --no-exclude-tools-when-tool-choice-none
  --h11-max-header-count H11_MAX_HEADER_COUNT
                        Maximum number of HTTP headers allowed in a request for h11 parser. Helps mitigate header abuse. Default: 256. (default: 256)
  --h11-max-incomplete-event-size H11_MAX_INCOMPLETE_EVENT_SIZE
                        Maximum size (bytes) of an incomplete HTTP event (header or body) for h11 parser. Helps mitigate header abuse. Default: 4194304 (4 MB). (default: 4194304)
  --host HOST           Host name. (default: None)
  --log-config-file LOG_CONFIG_FILE
  --log-error-stack, --no-log-error-stack
  --lora-modules LORA_MODULES [LORA_MODULES ...]
  --max-log-len MAX_LOG_LEN
  --middleware MIDDLEWARE
                        Additional ASGI middleware to apply to the app. We accept multiple
                        --middleware arguments. The value should be an import path. If a function is provided, vLLM will add it to the server using `@app.middleware('http')`. If a class is provided, vLLM will add it to the server using `app.add_middleware()`.
                        (default: [])
  --port PORT           Port number. (default: 8000)
  --response-role RESPONSE_ROLE
  --return-tokens-as-token-ids, --no-return-tokens-as-token-ids
  --root-path ROOT_PATH
                        FastAPI root_path when app is behind a path based routing proxy. (default: None)
  --ssl-ca-certs SSL_CA_CERTS
                        The CA certificates file. (default: None)
  --ssl-cert-reqs SSL_CERT_REQS
                        Whether client certificate is required (see stdlib ssl module's). (default: 0)
  --ssl-certfile SSL_CERTFILE
                        The file path to the SSL cert file. (default: None)
  --ssl-ciphers SSL_CIPHERS
                        SSL cipher suites for HTTPS (TLS 1.2 and below only). Example: 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305' (default: None)
  --ssl-keyfile SSL_KEYFILE
                        The file path to the SSL key file. (default: None)
  --tokens-only, --no-tokens-only
  --tool-call-parser {deepseek_v3,deepseek_v31,deepseek_v32,ernie45,functiongemma,gemma4,gigachat3,glm45,glm47,granite,granite-20b-fc,granite4,hermes,hunyuan_a13b,internlm,jamba,kimi_k2,llama3_json,llama4_json,llama4_pythonic,longcat,minimax,minimax_m2,mistral,olmo3,openai,phi4_mini_json,pythonic,qwen3_coder,qwen3_xml,seed_oss,step3,step3p5,xlam} or name registered in --tool-parser-plugin
  --tool-parser-plugin TOOL_PARSER_PLUGIN
  --tool-server TOOL_SERVER
  --trust-request-chat-template, --no-trust-request-chat-template
  --uds UDS             Unix domain socket path. If set, host and port arguments are ignored. (default: None)
  --uvicorn-log-level {critical,debug,error,info,trace,warning}
                        Log level for uvicorn. (default: info)

ModelConfig:
  Configuration for the model.

  --allow-deprecated-quantization, --no-allow-deprecated-quantization
                        Whether to allow deprecated quantization methods. (default: False)
  --allowed-local-media-path ALLOWED_LOCAL_MEDIA_PATH
                        Allowing API requests to read local images or videos from directories specified by the server file system. This is a security risk. Should only be enabled in trusted environments. (default: )
  --allowed-media-domains ALLOWED_MEDIA_DOMAINS [ALLOWED_MEDIA_DOMAINS ...]
                        If set, only media URLs that belong to this domain can be used for multi-modal inputs. (default: None)
  --code-revision CODE_REVISION
                        The specific revision to use for the model code on the Hugging Face Hub. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version. (default: None)
  --config-format ['auto', 'hf', 'mistral']
                        The format of the model config to load:
                        - "auto" will try to load the config in hf format if available after trying to load in mistral format.
                        - "hf" will load the config in hf format.
                        - "mistral" will load the config in mistral format. (default: auto)
  --convert {auto,classify,embed,none}
                        Convert the model using adapters defined in [vllm.model_executor.models.adapters][]. The most common use case is to adapt a text generation model to be used for pooling tasks. (default: auto)
  --disable-cascade-attn, --no-disable-cascade-attn
                        Disable cascade attention for V1. While cascade attention does not change the mathematical correctness, disabling it could be useful for preventing potential numerical issues. This defaults to True, so users must opt in to cascade attention by
                        setting this to False. Even when this is set to False, cascade attention will only be used when the heuristic tells that it's beneficial. (default: True)
  --disable-sliding-window, --no-disable-sliding-window
                        Whether to disable sliding window. If True, we will disable the sliding window functionality of the model, capping to sliding window size. If the model does not support sliding window, this argument is ignored. (default: False)
  --dtype {auto,bfloat16,float,float16,float32,half}
                        Data type for model weights and activations:
                        - "auto" will use FP16 precision for FP32 and FP16 models, and BF16 precision for BF16 models.
                        - "half" for FP16. Recommended for AWQ quantization.
                        - "float16" is the same as "half".
                        - "bfloat16" for a balance between precision and range.
                        - "float" is shorthand for FP32 precision.
                        - "float32" for FP32 precision. (default: auto)
  --enable-prompt-embeds, --no-enable-prompt-embeds
                        If `True`, enables passing text embeddings as inputs via the `prompt_embeds` key.
                        WARNING: The vLLM engine may crash if incorrect shape of embeddings is passed. Only enable this flag for trusted users! (default: False)
  --enable-return-routed-experts, --no-enable-return-routed-experts
                        Whether to return routed experts. (default: False)
  --enable-sleep-mode, --no-enable-sleep-mode
                        Enable sleep mode for the engine (only cuda and hip platforms are supported). (default: False)
  --enforce-eager, --no-enforce-eager
                        Whether to always use eager-mode PyTorch. If True, we will disable CUDA graph and always execute the model in eager mode. If False, we will use CUDA graph and eager execution in hybrid for maximal performance and flexibility. (default: False)
  --generation-config GENERATION_CONFIG
                        The folder path to the generation config. Defaults to `"auto"`, the generation config will be loaded from model path. If set to `"vllm"`, no generation config is loaded, vLLM defaults will be used. If set to a folder path, the generation config
                        will be loaded from the specified folder path. If `max_new_tokens` is specified in generation config, then it sets a server-wide limit on the number of output tokens for all requests. (default: auto)
  --hf-config-path HF_CONFIG_PATH
                        Name or path of the Hugging Face config to use. If unspecified, model name or path will be used. (default: None)
  --hf-overrides HF_OVERRIDES
                        If a dictionary, contains arguments to be forwarded to the Hugging Face config. If a callable, it is called to update the HuggingFace config. (default: {})
  --hf-token [HF_TOKEN]
                        The token to use as HTTP bearer authorization for remote files . If `True`, will use the token generated when running `hf auth login` (stored in `~/.cache/huggingface/token`). (default: None)
  --io-processor-plugin IO_PROCESSOR_PLUGIN
                        IOProcessor plugin name to load at model startup (default: None)
  --logits-processors LOGITS_PROCESSORS [LOGITS_PROCESSORS ...]
                        One or more logits processors' fully-qualified class names or class definitions (default: None)
  --logprobs-mode {processed_logits,processed_logprobs,raw_logits,raw_logprobs}
                        Indicates the content returned in the logprobs and prompt_logprobs. Supported mode: 1) raw_logprobs, 2) processed_logprobs, 3) raw_logits, 4) processed_logits. Raw means the values before applying any logit processors, like bad words. Processed
                        means the values after applying all processors, including temperature and top_k/top_p. (default: raw_logprobs)
  --max-logprobs MAX_LOGPROBS
                        Maximum number of log probabilities to return when `logprobs` is specified in `SamplingParams`. The default value comes the default for the OpenAI Chat Completions API. -1 means no cap, i.e. all (output_length * vocab_size) logprobs are allowed
                        to be returned and it may cause OOM. (default: 20)
  --max-model-len MAX_MODEL_LEN
                        Model context length (prompt and output). If unspecified, will be automatically derived from the model config.
                        When passing via `--max-model-len`, supports k/m/g/K/M/G in human-readable format. Examples:
                        - 1k -> 1000
                        - 1K -> 1024
                        - 25.6k -> 25,600
                        - -1 or 'auto' -> Automatically choose the maximum model length that fits in GPU memory. This will use the model's maximum context length if it fits, otherwise it will find the largest length that can be accommodated.
                        Parse human-readable integers like '1k', '2M', etc. Including decimal values with decimal multipliers. Also accepts -1 or 'auto' as a special value for auto-detection.
                            Examples:
                            - '1k' -> 1,000
                            - '1K' -> 1,024
                            - '25.6k' -> 25,600
                            - '-1' or 'auto' -> -1 (special value for auto-detection) (default: None)
  --model MODEL         Name or path of the Hugging Face model to use. It is also used as the content for `model_name` tag in metrics output when `served_model_name` is not specified. (default: Qwen/Qwen3-0.6B)
  --model-impl ['auto', 'terratorch', 'transformers', 'vllm']
                        Which implementation of the model to use:
                        - "auto" will try to use the vLLM implementation, if it exists, and fall back to the Transformers implementation if no vLLM implementation is available.
                        - "vllm" will use the vLLM model implementation.
                        - "transformers" will use the Transformers model implementation.
                        - "terratorch" will use the TerraTorch model implementation. (default: auto)
  --override-attention-dtype OVERRIDE_ATTENTION_DTYPE
                        Override dtype for attention (default: None)
  --override-generation-config OVERRIDE_GENERATION_CONFIG
                        Overrides or sets generation config. e.g. `{"temperature": 0.5}`. If used with `--generation-config auto`, the override parameters will be merged with the default config from the model. If used with `--generation-config vllm`, only the override
                        parameters are used.
                        Should either be a valid JSON string or JSON keys passed individually. (default: {})
  --pooler-config POOLER_CONFIG
                        Pooler config which controls the behaviour of output pooling in pooling models.
                        API docs: https://docs.vllm.ai/en/v0.19.0/api/vllm/config/#vllm.config.PoolerConfig
                        Should either be a valid JSON string or JSON keys passed individually. (default: None)
  --quantization QUANTIZATION, -q QUANTIZATION
                        Method used to quantize the weights. If `None`, we first check the `quantization_config` attribute in the model config file. If that is `None`, we assume the model weights are not quantized and use `dtype` to determine the data type of the
                        weights. (default: None)
  --renderer-num-workers RENDERER_NUM_WORKERS
                        Number of worker threads in the renderer thread pool. This pool handles async tokenization, chat template rendering, and multimodal preprocessing. (default: 1)
  --revision REVISION   The specific model version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version. (default: None)
  --runner {auto,draft,generate,pooling}
                        The type of model runner to use. Each vLLM instance only supports one model runner, even if the same model can be used for multiple types. (default: auto)
  --seed SEED           Random seed for reproducibility.
                        We must set the global seed because otherwise, different tensor parallel workers would sample different tokens, leading to inconsistent results. (default: 0)
  --served-model-name SERVED_MODEL_NAME [SERVED_MODEL_NAME ...]
                        The model name(s) used in the API. If multiple names are provided, the server will respond to any of the provided names. The model name in the model field of a response will be the first name in this list. If not specified, the model name will
                        be the same as the `--model` argument. Noted that this name(s) will also be used in `model_name` tag content of prometheus metrics, if multiple names provided, metrics tag will take the first one. (default: None)
  --skip-tokenizer-init, --no-skip-tokenizer-init
                        Skip initialization of tokenizer and detokenizer. Expects valid `prompt_token_ids` and `None` for prompt from the input. The generated output will contain token ids. (default: False)
  --tokenizer TOKENIZER
                        Name or path of the Hugging Face tokenizer to use. If unspecified, model name or path will be used. (default: None)
  --tokenizer-mode ['auto', 'deepseek_v32', 'hf', 'mistral', 'slow']
                        Tokenizer mode:
                        - "auto" will use the tokenizer from `mistral_common` for Mistral models if available, otherwise it will use the "hf" tokenizer.
                        - "hf" will use the fast tokenizer if available.
                        - "slow" will always use the slow tokenizer.
                        - "mistral" will always use the tokenizer from `mistral_common`.
                        - "deepseek_v32" will always use the tokenizer from `deepseek_v32`.
                        - "qwen_vl" will always use the tokenizer from `qwen_vl`.
                        - Other custom values can be supported via plugins. (default: auto)
  --tokenizer-revision TOKENIZER_REVISION
                        The specific revision to use for the tokenizer on the Hugging Face Hub. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version. (default: None)
  --trust-remote-code, --no-trust-remote-code
                        Trust remote code (e.g., from HuggingFace) when downloading the model and tokenizer. (default: False)

LoadConfig:
  Configuration for loading the model weights.

  --download-dir DOWNLOAD_DIR
                        Directory to download and load the weights, default to the default cache directory of Hugging Face. (default: None)
  --ignore-patterns IGNORE_PATTERNS [IGNORE_PATTERNS ...]
                        The list of patterns to ignore when loading the model. Default to "original/**/*" to avoid repeated loading of llama's checkpoints. (default: ['original/**/*'])
  --load-format LOAD_FORMAT
                        The format of the model weights to load.
                        - "auto" will try to load the weights in the safetensors format and fall back to the pytorch bin format if safetensors format is not available.
                        - "pt" will load the weights in the pytorch bin format.
                        - "safetensors" will load the weights in the safetensors format.
                        - "instanttensor" will load the Safetensors weights on CUDA devices using InstantTensor, which enables distributed loading with pipelined prefetching and fast direct I/O.
                        - "npcache" will load the weights in pytorch format and store a numpy cache to speed up the loading.
                        - "dummy" will initialize the weights with random values, which is mainly for profiling.
                        - "tensorizer" will use CoreWeave's tensorizer library for fast weight loading. See the Tensorize vLLM Model script in the Examples section for more information.
                        - "runai_streamer" will load the Safetensors weights using Run:ai Model Streamer.
                        - "runai_streamer_sharded" will load weights from pre-sharded checkpoint files using Run:ai Model Streamer.
                        - "bitsandbytes" will load the weights using bitsandbytes quantization.
                        - "sharded_state" will load weights from pre-sharded checkpoint files, supporting efficient loading of tensor-parallel models.
                        - "gguf" will load weights from GGUF format files (details specified in https://github.com/ggml-org/ggml/blob/master/docs/gguf.md).
                        - "mistral" will load weights from consolidated safetensors files used by Mistral models.
                        - Other custom values can be supported via plugins. (default: auto)
  --model-loader-extra-config MODEL_LOADER_EXTRA_CONFIG
                        Extra config for model loader. This will be passed to the model loader corresponding to the chosen load_format. (default: {})
  --pt-load-map-location PT_LOAD_MAP_LOCATION
                        The map location for loading pytorch checkpoint, to support loading checkpoints can only be loaded on certain devices like "cuda", this is equivalent to `{"": "cuda"}`. Another supported format is mapping from different devices like from GPU 1
                        to GPU 0: `{"cuda:1": "cuda:0"}`. Note that when passed from command line, the strings in dictionary need to be double quoted for json parsing. For more details, see the original doc for `map_location` parameter in [`torch.load`][] parameter.
                        (default: cpu)
  --safetensors-load-strategy SAFETENSORS_LOAD_STRATEGY
                        Specifies the loading strategy for safetensors weights.
                        - None (default): Uses memory-mapped (lazy) loading. When an NFS filesystem is detected and the total checkpoint size fits within 90%% of available RAM, prefetching is enabled automatically.
                        - "lazy": Weights are memory-mapped from the file. This enables on-demand loading and is highly efficient for models on local storage. Unlike the default (None), auto-prefetch on NFS is not performed.
                        - "eager": The entire file is read into CPU memory upfront before loading. This is recommended for models on network filesystems (e.g., Lustre, NFS) as it avoids inefficient random reads, significantly speeding up model initialization. However,
                        it uses more CPU RAM.
                        - "prefetch": Checkpoint files are read into the OS page cache before workers load them, speeding up the model loading phase. Useful on network or high-latency storage.
                        - "torchao": Weights are loaded in upfront and then reconstructed into torchao tensor subclasses. This is used when the checkpoint was quantized using torchao and saved using safetensors. Needs `torchao >= 0.14.0`. (default: None)
  --use-tqdm-on-load, --no-use-tqdm-on-load
                        Whether to enable tqdm for showing progress bar when loading model weights. (default: True)

AttentionConfig:
  Configuration for attention mechanisms in vLLM.

  --attention-backend ATTENTION_BACKEND
                        Attention backend to use. Use "auto" or None for automatic selection. (default: None)

StructuredOutputsConfig:
  Dataclass which contains structured outputs config for the engine.

  --reasoning-parser REASONING_PARSER
                        Select the reasoning parser depending on the model that you're using. This is used to parse the reasoning content into OpenAI API format. (default: )
  --reasoning-parser-plugin REASONING_PARSER_PLUGIN
                        Path to a dynamically reasoning parser plugin that can be dynamically loaded and registered. (default: )

ParallelConfig:
  Configuration for the distributed execution.

  --all2all-backend {allgather_reducescatter,deepep_high_throughput,deepep_low_latency,flashinfer_all2allv,flashinfer_nvlink_one_sided,flashinfer_nvlink_two_sided,mori,naive,nixl_ep,pplx}
                        All2All backend for MoE expert parallel communication. Available options:
                        - "allgather_reducescatter": All2all based on allgather and reducescatter
                        - "deepep_high_throughput": Use deepep high-throughput kernels
                        - "deepep_low_latency": Use deepep low-latency kernels
                        - "mori": Use mori kernels
                        - "nixl_ep": Use nixl-ep kernels
                        - "flashinfer_nvlink_two_sided": Use flashinfer two-sided kernels for mnnvl
                        - "flashinfer_nvlink_one_sided": Use flashinfer high-throughput a2a kernels (default: allgather_reducescatter)
  --cp-kv-cache-interleave-size CP_KV_CACHE_INTERLEAVE_SIZE
                        Interleave size of kv_cache storage while using DCP or PCP. For `total_cp_rank = pcp_rank * dcp_world_size + dcp_rank`, and `total_cp_world_size = pcp_world_size * dcp_world_size`. store interleave_size tokens on total_cp_rank i, then store
                        next interleave_size tokens on total_cp_rank i+1. Interleave_size=1: token-level alignment, where token `i` is stored on total_cp_rank `i % total_cp_world_size`. Interleave_size=block_size: block-level alignment, where tokens are first
                        populated to the preceding ranks. Tokens are then stored in (rank i+1, block j) only after (rank i, block j) is fully occupied. Block_size should be greater than or equal to cp_kv_cache_interleave_size. Block_size should be divisible by
                        cp_kv_cache_interleave_size. (default: 1)
  --data-parallel-address DATA_PARALLEL_ADDRESS, -dpa DATA_PARALLEL_ADDRESS
                        Address of data parallel cluster head-node. (default: None)
  --data-parallel-backend DATA_PARALLEL_BACKEND, -dpb DATA_PARALLEL_BACKEND
                        Backend for data parallel, either "mp" or "ray". (default: mp)
  --data-parallel-external-lb, --no-data-parallel-external-lb, -dpe
                        Whether to use "external" DP LB mode. Applies only to online serving and when data_parallel_size > 0. This is useful for a "one-pod-per-rank" wide-EP setup in Kubernetes. Set implicitly when --data-parallel-rank is provided explicitly to vllm
                        serve. (default: False)
  --data-parallel-hybrid-lb, --no-data-parallel-hybrid-lb, -dph
                        Whether to use "hybrid" DP LB mode. Applies only to online serving and when data_parallel_size > 0. Enables running an AsyncLLM and API server on a "per-node" basis where vLLM load balances between local data parallel ranks, but an external LB
                        balances between vLLM nodes/replicas. Set explicitly in conjunction with
                        --data-parallel-start-rank. (default: False)
  --data-parallel-rank DATA_PARALLEL_RANK, -dpn DATA_PARALLEL_RANK
                        Data parallel rank of this instance. When set, enables external load balancer mode. (default: None)
  --data-parallel-rpc-port DATA_PARALLEL_RPC_PORT, -dpp DATA_PARALLEL_RPC_PORT
                        Port for data parallel RPC communication. (default: None)
  --data-parallel-size DATA_PARALLEL_SIZE, -dp DATA_PARALLEL_SIZE
                        Number of data parallel groups. MoE layers will be sharded according to the product of the tensor parallel size and data parallel size. (default: 1)
  --data-parallel-size-local DATA_PARALLEL_SIZE_LOCAL, -dpl DATA_PARALLEL_SIZE_LOCAL
                        Number of data parallel replicas to run on this node. (default: None)
  --data-parallel-start-rank DATA_PARALLEL_START_RANK, -dpr DATA_PARALLEL_START_RANK
                        Starting data parallel rank for secondary nodes. (default: None)
  --dbo-decode-token-threshold DBO_DECODE_TOKEN_THRESHOLD
                        The threshold for dual batch overlap for batches only containing decodes. If the number of tokens in the request is greater than this threshold, microbatching will be used. Otherwise, the request will be processed in a single batch. (default:
                        32)
  --dbo-prefill-token-threshold DBO_PREFILL_TOKEN_THRESHOLD
                        The threshold for dual batch overlap for batches that contain one or more prefills. If the number of tokens in the request is greater than this threshold, microbatching will be used. Otherwise, the request will be processed in a single batch.
                        (default: 512)
  --dcp-comm-backend {a2a,ag_rs}
                        Communication backend for Decode Context Parallel (DCP).
                        - "ag_rs": AllGather + ReduceScatter (default, existing behavior)
                        - "a2a": All-to-All exchange of partial outputs + LSE, then combine with Triton kernel. Reduces NCCL calls from 3 to 2 per layer for MLA models. (default: ag_rs)
  --dcp-kv-cache-interleave-size DCP_KV_CACHE_INTERLEAVE_SIZE
                        Interleave size of kv_cache storage while using DCP. dcp_kv_cache_interleave_size has been replaced by cp_kv_cache_interleave_size, and will be deprecated when PCP is fully supported. (default: 1)
  --decode-context-parallel-size DECODE_CONTEXT_PARALLEL_SIZE, -dcp DECODE_CONTEXT_PARALLEL_SIZE
                        Number of decode context parallel groups, because the world size does not change by dcp, it simply reuse the GPUs of TP group, and tp_size needs to be divisible by dcp_size. (default: 1)
  --disable-custom-all-reduce, --no-disable-custom-all-reduce
                        Disable the custom all-reduce kernel and fall back to NCCL. (default: False)
  --disable-nccl-for-dp-synchronization, --no-disable-nccl-for-dp-synchronization
                        Forces the dp synchronization logic in vllm/v1/worker/dp_utils.py  to use Gloo instead of NCCL for its all reduce.
                        Defaults to True when async scheduling is enabled, False otherwise. (default: None)
  --distributed-executor-backend ['external_launcher', 'mp', 'ray', 'uni']
                        Backend to use for distributed model workers, either "ray" or "mp" (multiprocessing). If the product of pipeline_parallel_size and tensor_parallel_size is less than or equal to the number of GPUs available, "mp" will be used to keep processing
                        on a single host. Otherwise, an error will be raised. To use "mp" you must also set nnodes, and to use "ray" you must manually set distributed_executor_backend to "ray".
                        Note that tpu only support Ray for distributed inference. (default: None)
  --distributed-timeout-seconds DISTRIBUTED_TIMEOUT_SECONDS
                        Timeout in seconds for distributed operations (e.g., init_process_group). If set, this value is passed to torch.distributed.init_process_group as the timeout parameter. If None, PyTorch's default timeout is used (600s for NCCL). Increase this
                        for multi-node setups where model downloads may be slow. (default: None)
  --enable-dbo, --no-enable-dbo
                        Enable dual batch overlap for the model executor. (default: False)
  --enable-elastic-ep, --no-enable-elastic-ep
                        Enable elastic expert parallelism with stateless NCCL groups for DP/EP. (default: False)
  --enable-ep-weight-filter, --no-enable-ep-weight-filter
                        Skip non-local expert weights during model loading when expert parallelism is active.  Each rank only reads its own expert shard from disk, which can drastically reduce storage I/O for MoE models with per-expert weight tensors (e.g. DeepSeek,
                        Mixtral, Kimi-K2.5).  Has no effect on 3D fused-expert checkpoints (e.g. GPT-OSS) or non-MoE models. (default: False)
  --enable-eplb, --no-enable-eplb
                        Enable expert parallelism load balancing for MoE layers. (default: False)
  --enable-expert-parallel, --no-enable-expert-parallel, -ep
                        Use expert parallelism instead of tensor parallelism for MoE layers. (default: False)
  --eplb-config EPLB_CONFIG
                        Expert parallelism configuration.
                        API docs: https://docs.vllm.ai/en/v0.19.0/api/vllm/config/#vllm.config.EPLBConfig
                        Should either be a valid JSON string or JSON keys passed individually. (default: EPLBConfig(window_size=1000, step_interval=3000, num_redundant_experts=0, log_balancedness=False, log_balancedness_interval=1, use_async=False, policy='default'))
  --expert-placement-strategy {linear,round_robin}
                        The expert placement strategy for MoE layers:
                        - "linear": Experts are placed in a contiguous manner. For example, with 4 experts and 2 ranks, rank 0 will have experts [0, 1] and rank 1 will have experts [2, 3].
                        - "round_robin": Experts are placed in a round-robin manner. For example, with 4 experts and 2 ranks, rank 0 will have experts [0, 2] and rank 1 will have experts [1, 3]. This strategy can help improve load balancing for grouped expert models
                        with no redundant experts. (default: linear)
  --master-addr MASTER_ADDR
                        distributed master address for multi-node distributed  inference when distributed_executor_backend is mp. (default: 127.0.0.1)
  --master-port MASTER_PORT
                        distributed master port for multi-node distributed  inference when distributed_executor_backend is mp. (default: 29501)
  --max-parallel-loading-workers MAX_PARALLEL_LOADING_WORKERS
                        Maximum number of parallel loading workers when loading model sequentially in multiple batches. To avoid RAM OOM when using tensor parallel and large models. (default: None)
  --nnodes NNODES, -n NNODES
                        num of nodes for multi-node distributed inference when distributed_executor_backend is mp. (default: 1)
  --node-rank NODE_RANK, -r NODE_RANK
                        distributed node rank for multi-node distributed  inference when distributed_executor_backend is mp. (default: 0)
  --pipeline-parallel-size PIPELINE_PARALLEL_SIZE, -pp PIPELINE_PARALLEL_SIZE
                        Number of pipeline parallel groups. (default: 1)
  --prefill-context-parallel-size PREFILL_CONTEXT_PARALLEL_SIZE, -pcp PREFILL_CONTEXT_PARALLEL_SIZE
                        Number of prefill context parallel groups. (default: 1)
  --ray-workers-use-nsight, --no-ray-workers-use-nsight
                        Whether to profile Ray workers with nsight, see https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html#profiling-nsight-profiler. (default: False)
  --tensor-parallel-size TENSOR_PARALLEL_SIZE, -tp TENSOR_PARALLEL_SIZE
                        Number of tensor parallel groups. (default: 1)
  --ubatch-size UBATCH_SIZE
                        Number of ubatch size. (default: 0)
  --worker-cls WORKER_CLS
                        The full name of the worker class to use. If "auto", the worker class will be determined based on the platform. (default: auto)
  --worker-extension-cls WORKER_EXTENSION_CLS
                        The full name of the worker extension class to use. The worker extension class is dynamically inherited by the worker class. This is used to inject new attributes and methods to the worker class for use in collective_rpc calls. (default: )

CacheConfig:
  Configuration for the KV cache.

  --block-size BLOCK_SIZE
                        Size of a contiguous cache block in number of tokens. Accepts None (meaning "use default"). After construction, always int. (default: None)
  --calculate-kv-scales, --no-calculate-kv-scales
                        Deprecated: This option is deprecated and will be removed in v0.19. It enables dynamic calculation of `k_scale` and `v_scale` when kv_cache_dtype is fp8. If `False`, the scales will be loaded from the model checkpoint if available. Otherwise,
                        the scales will default to 1.0. (default: False)
  --enable-prefix-caching, --no-enable-prefix-caching
                        Whether to enable prefix caching. (default: None)
  --gpu-memory-utilization GPU_MEMORY_UTILIZATION
                        The fraction of GPU memory to be used for the model executor, which can range from 0 to 1. For example, a value of 0.5 would imply 50% GPU memory utilization. If unspecified, will use the default value of 0.9. This is a per-instance limit, and
                        only applies to the current vLLM instance. It does not matter if you have another vLLM instance running on the same GPU. For example, if you have two vLLM instances running on the same GPU, you can set the GPU memory utilization to 0.5 for each
                        instance. (default: 0.9)
  --kv-cache-dtype {auto,bfloat16,float16,fp8,fp8_ds_mla,fp8_e4m3,fp8_e5m2,fp8_inc}
                        Data type for kv cache storage. If "auto", will use model data type. CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. ROCm (AMD GPU) supports fp8 (=fp8_e4m3). Intel Gaudi (HPU) supports fp8 (using fp8_inc). Some models (namely DeepSeekV3.2)
                        default to fp8, set to bfloat16 to use bfloat16 instead, this is an invalid option for models that do not default to fp8. (default: auto)
  --kv-cache-dtype-skip-layers KV_CACHE_DTYPE_SKIP_LAYERS [KV_CACHE_DTYPE_SKIP_LAYERS ...]
                        Layer patterns to skip KV cache quantization. Accepts layer indices (e.g., '0', '2', '4') or attention type names (e.g., 'sliding_window'). (default: [])
  --kv-cache-memory-bytes KV_CACHE_MEMORY_BYTES
                        Size of KV Cache per GPU in bytes. By default, this is set to None and vllm can automatically infer the kv cache size based on gpu_memory_utilization. However, users may want to manually specify the kv cache memory size. kv_cache_memory_bytes
                        allows more fine-grain control of how much memory gets used when compared with using gpu_memory_utilization. Note that kv_cache_memory_bytes (when not-None) ignores gpu_memory_utilization
                        Parse human-readable integers like '1k', '2M', etc. Including decimal values with decimal multipliers.
                            Examples:
                            - '1k' -> 1,000
                            - '1K' -> 1,024
                            - '25.6k' -> 25,600 (default: None)
  --kv-offloading-backend {lmcache,native}
                        The backend to use for KV cache offloading. Supported backends include 'native' (vLLM native CPU offloading), 'lmcache'. KV offloading is only activated when kv_offloading_size is set. (default: native)
  --kv-offloading-size KV_OFFLOADING_SIZE
                        Size of the KV cache offloading buffer in GiB. When TP > 1, this is the total buffer size summed across all TP ranks. By default, this is set to None, which means no KV offloading is enabled. When set, vLLM will enable KV cache offloading to
                        CPU using the kv_offloading_backend. (default: None)
  --kv-sharing-fast-prefill, --no-kv-sharing-fast-prefill
                        This feature is work in progress and no prefill optimization takes place with this flag enabled currently.
                        In some KV sharing setups, e.g. YOCO (https://arxiv.org/abs/2405.05254), some layers can skip tokens corresponding to prefill. This flag enables attention metadata for eligible layers to be overridden with metadata necessary for implementing
                        this optimization in some models (e.g. Gemma3n) (default: False)
  --mamba-block-size MAMBA_BLOCK_SIZE
                        Size of a contiguous cache block in number of tokens for mamba cache. Can be set only when prefix caching is enabled. Value must be a multiple of 8 to align with causal_conv1d kernel. (default: None)
  --mamba-cache-dtype {auto,float16,float32}
                        The data type to use for the Mamba cache (both the conv as well as the ssm state). If set to 'auto', the data type will be inferred from the model config. (default: auto)
  --mamba-cache-mode {align,all,none}
                        The cache strategy for Mamba layers.
                        - "none": set when prefix caching is disabled.
                        - "all": cache the mamba state of all tokens at position i * block_size. This is  the default behavior (for models that support it) when prefix caching is enabled.
                        - "align": only cache the mamba state of the last token of each scheduler step and when the token is at position i * block_size. (default: none)
  --mamba-ssm-cache-dtype {auto,float16,float32}
                        The data type to use for the Mamba cache (ssm state only, conv state will still be controlled by mamba_cache_dtype). If set to 'auto', the data type for the ssm state will be determined by mamba_cache_dtype. (default: auto)
  --num-gpu-blocks-override NUM_GPU_BLOCKS_OVERRIDE
                        Number of GPU blocks to use. This overrides the profiled `num_gpu_blocks` if specified. Does nothing if `None`. Used for testing preemption. (default: None)
  --prefix-caching-hash-algo {sha256,sha256_cbor,xxhash,xxhash_cbor}
                        Set the hash algorithm for prefix caching:
                        - "sha256" uses Pickle for object serialization before hashing. This is the current default, as SHA256 is the most secure choice to avoid potential hash collisions.
                        - "sha256_cbor" provides a reproducible, cross-language compatible hash. It serializes objects using canonical CBOR and hashes them with SHA-256.
                        - "xxhash" uses Pickle serialization with xxHash (128-bit) for faster, non-cryptographic hashing. Requires the optional ``xxhash`` package. IMPORTANT: Use of a hashing algorithm that is not considered  cryptographically secure theoretically
                        increases the risk of hash collisions, which can cause undefined behavior or even leak private information in multi-tenant environments. Even if collisions are still very unlikely, it is important to consider your security risk tolerance
                        against the performance benefits before turning this on.
                        - "xxhash_cbor" combines canonical CBOR serialization with xxHash for reproducible hashing. Requires the optional ``xxhash`` package. (default: sha256)

OffloadConfig:
  Configuration for model weight offloading to reduce GPU memory usage.

  --cpu-offload-gb CPU_OFFLOAD_GB
                        The space in GiB to offload to CPU, per GPU. Default is 0, which means no offloading. Intuitively, this argument can be seen as a virtual way to increase the GPU memory size. For example, if you have one 24 GB GPU and set this to 10, virtually
                        you can think of it as a 34 GB GPU. Then you can load a 13B model with BF16 weight, which requires at least 26GB GPU memory. Note that this requires fast CPU-GPU interconnect, as part of the model is loaded from CPU memory to GPU memory on the
                        fly in each model forward pass. This uses UVA (Unified Virtual Addressing) for zero-copy access. (default: 0)
  --cpu-offload-params CPU_OFFLOAD_PARAMS [CPU_OFFLOAD_PARAMS ...]
                        The set of parameter name segments to target for CPU offloading. Unmatched parameters are not offloaded. If this set is empty, parameters are offloaded non-selectively until the memory limit defined by `cpu_offload_gb` is reached. Examples:
                            - For parameter name "mlp.experts.w2_weight":
                                - "experts" or "experts.w2_weight" will match.
                                - "expert" or "w2" will NOT match (must be exact segments). This allows distinguishing parameters like "w2_weight" and "w2_weight_scale". (default: set())
  --offload-backend {auto,prefetch,uva}
                        The backend for weight offloading. Options:
                        - "auto": Selects based on which sub-config has non-default values (prefetch if offload_group_size > 0, uva if cpu_offload_gb > 0).
                        - "uva": UVA (Unified Virtual Addressing) zero-copy offloading.
                        - "prefetch": Async prefetch with group-based layer offloading. (default: auto)
  --offload-group-size OFFLOAD_GROUP_SIZE
                        Group every N layers together. Offload last `offload_num_in_group` layers of each group. Default is 0 (disabled). Example: group_size=8, num_in_group=2 offloads layers 6,7,14,15,22,23,... Unlike cpu_offload_gb, this uses explicit async
                        prefetching to hide transfer latency. (default: 0)
  --offload-num-in-group OFFLOAD_NUM_IN_GROUP
                        Number of layers to offload per group. Must be <= offload_group_size. Default is 1. (default: 1)
  --offload-params OFFLOAD_PARAMS [OFFLOAD_PARAMS ...]
                        The set of parameter name segments to target for prefetch offloading. Unmatched parameters are not offloaded. If this set is empty, ALL parameters of each offloaded layer are offloaded. Uses segment matching: "w13_weight" matches
                        "mlp.experts.w13_weight" but not "mlp.experts.w13_weight_scale". (default: set())
  --offload-prefetch-step OFFLOAD_PREFETCH_STEP
                        Number of layers to prefetch ahead. Higher values hide more latency but use more GPU memory. Default is 1. (default: 1)

MultiModalConfig:
  Controls the behavior of multimodal models.

  --enable-mm-embeds, --no-enable-mm-embeds
                        If `True`, enables passing multimodal embeddings: for `LLM` class, this refers to tensor inputs under `multi_modal_data`; for the OpenAI-compatible server, this refers to chat messages with content `"type": "*_embeds"`.
                        When enabled with `--limit-mm-per-prompt` set to 0 for a modality, precomputed embeddings skip count validation for that modality,  saving memory by not loading encoder modules while still enabling  embeddings as an input. Limits greater than 0
                        still apply to embeddings.
                        WARNING: The vLLM engine may crash if incorrect shape of embeddings is passed. Only enable this flag for trusted users! (default: False)
  --interleave-mm-strings, --no-interleave-mm-strings
                        Enable fully interleaved support for multimodal prompts, while using
                        --chat-template-content-format=string. (default: False)
  --language-model-only, --no-language-model-only
                        If True, disables all multimodal inputs by setting all modality limits to 0. Equivalent to setting `--limit-mm-per-prompt` to 0 for every modality. (default: False)
  --limit-mm-per-prompt LIMIT_MM_PER_PROMPT
                        The maximum number of input items and options allowed per prompt for each modality.
                        Defaults to 999 for each modality.
                        Legacy format (count only): {"image": 16, "video": 2}
                        Configurable format (with options): {"video": {"count": 1, "num_frames": 32, "width": 512, "height": 512}, "image": {"count": 5, "width": 512, "height": 512}}
                        Mixed format (combining both): {"image": 16, "video": {"count": 1, "num_frames": 32, "width": 512, "height": 512}}
                        Should either be a valid JSON string or JSON keys passed individually. (default: {})
  --media-io-kwargs MEDIA_IO_KWARGS
                        Additional args passed to process media inputs, keyed by modalities. For example, to set num_frames for video, set `--media-io-kwargs '{"video": {"num_frames": 40} }'`
                        Should either be a valid JSON string or JSON keys passed individually. (default: {})
  --mm-encoder-attn-backend MM_ENCODER_ATTN_BACKEND
                        Optional override for the multi-modal encoder attention backend when using vision transformers. Accepts any value from `vllm.v1.attention.backends.registry.AttentionBackendEnum` (e.g. `FLASH_ATTN`). (default: None)
  --mm-encoder-only, --no-mm-encoder-only
                        When enabled, skips the language component of the model.
                        This is usually only valid in disaggregated Encoder process. (default: False)
  --mm-encoder-tp-mode {data,weights}
                        Indicates how to optimize multi-modal encoder inference using tensor parallelism (TP).
                        - `"weights"`: Within the same vLLM engine, split the weights of each layer across TP ranks. (default TP behavior)
                        - `"data"`: Within the same vLLM engine, split the batched input data across TP ranks to process the data in parallel, while hosting the full weights on each TP rank. This batch-level DP is not to be confused with API request-level DP (which is
                        controlled by `--data-parallel-size`). This is only supported on a per-model basis and falls back to `"weights"` if the encoder does not support DP. (default: weights)
  --mm-processor-cache-gb MM_PROCESSOR_CACHE_GB
                        The size (in GiB) of the multi-modal processor cache, which is used to avoid re-processing past multi-modal inputs.
                        This cache is duplicated for each API process and engine core process, resulting in a total memory usage of `mm_processor_cache_gb * (api_server_count + data_parallel_size)`.
                        Set to `0` to disable this cache completely (not recommended). (default: 4)
  --mm-processor-cache-type {lru,shm}
                        Type of cache to use for the multi-modal preprocessor/mapper. If `shm`, use shared memory FIFO cache. If `lru`, use mirrored LRU cache. (default: lru)
  --mm-processor-kwargs MM_PROCESSOR_KWARGS
                        Arguments to be forwarded to the model's processor for multi-modal data, e.g., image processor. Overrides for the multi-modal processor obtained from `transformers.AutoProcessor.from_pretrained`.
                        The available overrides depend on the model that is being run.
                        For example, for Phi-3-Vision: `{"num_crops": 4}`.
                        Should either be a valid JSON string or JSON keys passed individually. (default: None)
  --mm-shm-cache-max-object-size-mb MM_SHM_CACHE_MAX_OBJECT_SIZE_MB
                        Size limit (in MiB) for each object stored in the multi-modal processor shared memory cache. Only effective when `mm_processor_cache_type` is `"shm"`. (default: 128)
  --mm-tensor-ipc {direct_rpc,torch_shm}
                        IPC (inter-process communication) method for multimodal tensors.
                        - "direct_rpc": Use msgspec serialization via RPC
                        - "torch_shm": Use torch.multiprocessing shared memory for zero-copy IPC Defaults to "direct_rpc". (default: direct_rpc)
  --skip-mm-profiling, --no-skip-mm-profiling
                        When enabled, skips multimodal memory profiling and only profiles with language backbone model during engine initialization.
                        This reduces engine startup time but shifts the responsibility to users for estimating the peak memory usage of the activation of multimodal encoder and embedding cache. (default: False)
  --video-pruning-rate VIDEO_PRUNING_RATE
                        Sets pruning rate for video pruning via Efficient Video Sampling. Value sits in range [0;1) and determines fraction of media tokens from each video to be pruned. (default: None)

LoRAConfig:
  Configuration for LoRA.

  --default-mm-loras DEFAULT_MM_LORAS
                        Dictionary mapping specific modalities to LoRA model paths; this field is only applicable to multimodal models and should be leveraged when a model always expects a LoRA to be active when a given modality is present. Note that currently, if a
                        request provides multiple additional modalities, each of which have their own LoRA, we do NOT apply default_mm_loras because we currently only support one lora adapter per prompt. When run in offline mode, the lora IDs for n modalities will be
                        automatically assigned to 1-n with the names of the modalities in alphabetic order.
                        Should either be a valid JSON string or JSON keys passed individually. (default: None)
  --enable-lora, --no-enable-lora
                        If True, enable handling of LoRA adapters. (default: None)
  --enable-tower-connector-lora, --no-enable-tower-connector-lora
                        If `True`, LoRA support for the tower (vision encoder) and connector  of multimodal models will be enabled. This is an experimental feature and  currently only supports some MM models such as the Qwen VL series. The default  is False. (default:
                        False)
  --fully-sharded-loras, --no-fully-sharded-loras
                        By default, only half of the LoRA computation is sharded with tensor parallelism. Enabling this will use the fully sharded layers. At high sequence length, max rank or tensor parallel size, this is likely faster. (default: False)
  --lora-dtype {auto,bfloat16,float16}
                        Data type for LoRA. If auto, will default to base model dtype. (default: auto)
  --lora-target-modules LORA_TARGET_MODULES [LORA_TARGET_MODULES ...]
                        Restrict LoRA to specific module suffixes (e.g., ["o_proj", "qkv_proj"]). If None, all supported LoRA modules are used. This allows deployment-time control over which modules have LoRA applied, useful for performance tuning. (default: None)
  --max-cpu-loras MAX_CPU_LORAS
                        Maximum number of LoRAs to store in CPU memory. Must be >= than `max_loras`. (default: None)
  --max-lora-rank {1,8,16,32,64,128,256,320,512}
                        Max LoRA rank. (default: 16)
  --max-loras MAX_LORAS
                        Max number of LoRAs in a single batch. (default: 1)
  --specialize-active-lora, --no-specialize-active-lora
                        Whether to construct lora kernel grid by the number of active LoRA adapters. When set to True, separate cuda graphs will be captured for different counts of active LoRAs (powers of 2 up to max_loras), which can improve performance for variable
                        LoRA usage patterns at the cost of increased startup time and memory usage. Only takes effect when cudagraph_specialize_lora is True. (default: False)

ObservabilityConfig:
  Configuration for observability - metrics and tracing.

  --collect-detailed-traces {all,model,worker,None} [{all,model,worker,None} ...]
                        It makes sense to set this only if `--otlp-traces-endpoint` is set. If set, it will collect detailed traces for the specified modules. This involves use of possibly costly and or blocking operations and hence might have a performance impact.
                        Note that collecting detailed timing information for each request can be expensive. (default: None)
  --cudagraph-metrics, --no-cudagraph-metrics
                        Enable CUDA graph metrics (number of padded/unpadded tokens, runtime cudagraph dispatch modes, and their observed frequencies at every logging interval). (default: False)
  --enable-layerwise-nvtx-tracing, --no-enable-layerwise-nvtx-tracing
                        Enable layerwise NVTX tracing. This traces the execution of each layer or module in the model and attach information such as input/output shapes to nvtx range markers. Noted that this doesn't work with CUDA graphs enabled. (default: False)
  --enable-logging-iteration-details, --no-enable-logging-iteration-details
                        Enable detailed logging of iteration details. If set, vllm EngineCore will log iteration details This includes number of context/generation requests and tokens and the elapsed cpu time for the iteration. (default: False)
  --enable-mfu-metrics, --no-enable-mfu-metrics
                        Enable Model FLOPs Utilization (MFU) metrics. (default: False)
  --kv-cache-metrics, --no-kv-cache-metrics
                        Enable KV cache residency metrics (lifetime, idle time, reuse gaps). Uses sampling to minimize overhead. Requires log stats to be enabled (i.e., --disable-log-stats not set). (default: False)
  --kv-cache-metrics-sample KV_CACHE_METRICS_SAMPLE
                        Sampling rate for KV cache metrics (0.0, 1.0]. Default 0.01 = 1% of blocks. (default: 0.01)
  --otlp-traces-endpoint OTLP_TRACES_ENDPOINT
                        Target URL to which OpenTelemetry traces will be sent. (default: None)
  --show-hidden-metrics-for-version SHOW_HIDDEN_METRICS_FOR_VERSION
                        Enable deprecated Prometheus metrics that have been hidden since the specified version. For example, if a previously deprecated metric has been hidden since the v0.7.0 release, you use `--show-hidden-metrics-for-version=0.7` as a temporary
                        escape hatch while you migrate to new metrics. The metric is likely to be removed completely in an upcoming release. (default: None)

SchedulerConfig:
  Scheduler configuration.

  --async-scheduling, --no-async-scheduling
                        If set to False, disable async scheduling. Async scheduling helps to avoid gaps in GPU utilization, leading to better latency and throughput. (default: None)
  --disable-chunked-mm-input, --no-disable-chunked-mm-input
                        If set to true and chunked prefill is enabled, we do not want to partially schedule a multimodal item. Only used in V1 This ensures that if a request has a mixed prompt (like text tokens TTTT followed by image tokens IIIIIIIIII) where only some
                        image tokens can be scheduled (like TTTTIIIII, leaving IIIII), it will be scheduled as TTTT in one step and IIIIIIIIII in the next. (default: False)
  --disable-hybrid-kv-cache-manager, --no-disable-hybrid-kv-cache-manager
                        If set to True, KV cache manager will allocate the same size of KV cache for all attention layers even if there are multiple type of attention layers like full attention and sliding window attention. If set to None, the default value will be
                        determined based on the environment and starting configuration. (default: None)
  --enable-chunked-prefill, --no-enable-chunked-prefill
                        If True, prefill requests can be chunked based on the remaining `max_num_batched_tokens`.
                        The default value here is mainly for convenience when testing. In real usage, this should be set in `EngineArgs.create_engine_config`. (default: None)
  --long-prefill-token-threshold LONG_PREFILL_TOKEN_THRESHOLD
                        For chunked prefill, a request is considered long if the prompt is longer than this number of tokens. (default: 0)
  --max-long-partial-prefills MAX_LONG_PARTIAL_PREFILLS
                        For chunked prefill, the maximum number of prompts longer than long_prefill_token_threshold that will be prefilled concurrently. Setting this less than max_num_partial_prefills will allow shorter prompts to jump the queue in front of longer
                        prompts in some cases, improving latency. (default: 1)
  --max-num-batched-tokens MAX_NUM_BATCHED_TOKENS
                        Maximum number of tokens that can be processed in a single iteration.
                        The default value here is mainly for convenience when testing. In real usage, this should be set in `EngineArgs.create_engine_config`.
                        Parse human-readable integers like '1k', '2M', etc. Including decimal values with decimal multipliers.
                            Examples:
                            - '1k' -> 1,000
                            - '1K' -> 1,024
                            - '25.6k' -> 25,600 (default: None)
  --max-num-partial-prefills MAX_NUM_PARTIAL_PREFILLS
                        For chunked prefill, the maximum number of sequences that can be partially prefilled concurrently. (default: 1)
  --max-num-seqs MAX_NUM_SEQS
                        Maximum number of sequences to be processed in a single iteration.
                        The default value here is mainly for convenience when testing. In real usage, this should be set in `EngineArgs.create_engine_config`. (default: None)
  --scheduler-cls SCHEDULER_CLS
                        The scheduler class to use. "vllm.v1.core.sched.scheduler.Scheduler" is the default scheduler. Can be a class directly or the path to a class of form "mod.custom_class". (default: None)
  --scheduler-reserve-full-isl, --no-scheduler-reserve-full-isl
                        If True, the scheduler checks whether the full input sequence length fits in the KV cache before admitting a new request, rather than only checking the first chunk. Prevents over-admission and KV cache thrashing with chunked prefill. (default:
                        True)
  --scheduling-policy {fcfs,priority}
                        The scheduling policy to use:
                        - "fcfs" means first come first served, i.e. requests are handled in order  of arrival.
                        - "priority" means requests are handled based on given priority (lower value means earlier handling) and time of arrival deciding any ties). (default: fcfs)
  --stream-interval STREAM_INTERVAL
                        The interval (or buffer size) for streaming in terms of token length. A smaller value (1) makes streaming smoother by sending each token immediately, while a larger value (e.g., 10) reduces host overhead and may increase throughput by batching
                        multiple tokens before sending. (default: 1)

CompilationConfig:
  Configuration for compilation.
  
      You must pass CompilationConfig to VLLMConfig constructor.
      VLLMConfig's post_init does further initialization. If used outside of the
      VLLMConfig, some fields will be left in an improper state.
  
      It contains PassConfig, which controls the custom fusion/transformation passes.
      The rest has three parts:
  
      - Top-level Compilation control:
          - [`mode`][vllm.config.CompilationConfig.mode]
          - [`debug_dump_path`][vllm.config.CompilationConfig.debug_dump_path]
          - [`cache_dir`][vllm.config.CompilationConfig.cache_dir]
          - [`backend`][vllm.config.CompilationConfig.backend]
          - [`custom_ops`][vllm.config.CompilationConfig.custom_ops]
          - [`splitting_ops`][vllm.config.CompilationConfig.splitting_ops]
          - [`compile_mm_encoder`][vllm.config.CompilationConfig.compile_mm_encoder]
      - CudaGraph capture:
          - [`cudagraph_mode`][vllm.config.CompilationConfig.cudagraph_mode]
          - [`cudagraph_capture_sizes`]
          [vllm.config.CompilationConfig.cudagraph_capture_sizes]
          - [`max_cudagraph_capture_size`]
          [vllm.config.CompilationConfig.max_cudagraph_capture_size]
          - [`cudagraph_num_of_warmups`]
          [vllm.config.CompilationConfig.cudagraph_num_of_warmups]
          - [`cudagraph_copy_inputs`]
          [vllm.config.CompilationConfig.cudagraph_copy_inputs]
      - Inductor compilation:
          - [`compile_sizes`][vllm.config.CompilationConfig.compile_sizes]
          - [`compile_ranges_endpoints`]
              [vllm.config.CompilationConfig.compile_ranges_endpoints]
          - [`inductor_compile_config`]
          [vllm.config.CompilationConfig.inductor_compile_config]
          - [`inductor_passes`][vllm.config.CompilationConfig.inductor_passes]
          - custom inductor passes
  
      Why we have different sizes for cudagraph and inductor:
      - cudagraph: a cudagraph captured for a specific size can only be used
          for the same size. We need to capture all the sizes we want to use.
      - inductor: a graph compiled by inductor for a general shape can be used
          for different sizes. Inductor can also compile for specific sizes,
          where it can have more information to optimize the graph with fully
          static shapes. However, we find the general shape compilation is
          sufficient for most cases. It might be beneficial to compile for
          certain small batchsizes, where inductor is good at optimizing.
      

  --cudagraph-capture-sizes CUDAGRAPH_CAPTURE_SIZES [CUDAGRAPH_CAPTURE_SIZES ...]
                        Sizes to capture cudagraph.
                        - None (default): capture sizes are inferred from vllm config.
                        - list[int]: capture sizes are specified as given. (default: None)
  --max-cudagraph-capture-size MAX_CUDAGRAPH_CAPTURE_SIZE
                        The maximum cudagraph capture size.
                        If cudagraph_capture_sizes is specified, this will be set to the largest size in that list (or checked for consistency if specified). If cudagraph_capture_sizes is not specified, the list of sizes is generated automatically following the
                        pattern:
                            [1, 2, 4] + list(range(8, 256, 8)) + list( range(256, max_cudagraph_capture_size + 1, 16))
                        If not specified, max_cudagraph_capture_size is set to min(max_num_seqs*2, 512) by default. This voids OOM in tight memory scenarios with small max_num_seqs, and prevents capture of many large graphs (>512) that would greatly increase startup
                        time with limited performance benefit. (default: None)

KernelConfig:
  Configuration for kernel selection and warmup behavior.

  --enable-flashinfer-autotune, --no-enable-flashinfer-autotune
                        If True, run FlashInfer autotuning during kernel warmup. (default: None)
  --moe-backend {aiter,auto,cutlass,deep_gemm,flashinfer_cutedsl,flashinfer_cutlass,flashinfer_trtllm,marlin,triton}
                        Backend for MoE expert computation kernels. Available options:
                        - "auto": Automatically select the best backend based on model and hardware
                        - "triton": Use Triton-based fused MoE kernels
                        - "deep_gemm": Use DeepGEMM kernels (FP8 block-quantized only)
                        - "cutlass": Use vLLM CUTLASS kernels
                        - "flashinfer_trtllm": Use FlashInfer with TRTLLM-GEN kernels
                        - "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels
                        - "flashinfer_cutedsl": Use FlashInfer with CuteDSL kernels (FP4 only)
                        - "marlin": Use Marlin kernels (weight-only quantization)
                        - "aiter": Use AMD AITer kernels (ROCm only) (default: auto)

VllmConfig:
  Dataclass which contains all vllm-related configuration. This
      simplifies passing around the distinct configurations in the codebase.
      

  --additional-config ADDITIONAL_CONFIG
                        Additional config for specified platform. Different platforms may support different configs. Make sure the configs are valid for the platform you are using. Contents must be hashable. (default: {})
  --attention-config ATTENTION_CONFIG, -ac ATTENTION_CONFIG
                        Attention configuration.
                        API docs: https://docs.vllm.ai/en/v0.19.0/api/vllm/config/#vllm.config.AttentionConfig
                        Should either be a valid JSON string or JSON keys passed individually. (default: AttentionConfig(backend=None, flash_attn_version=None, use_prefill_decode_attention=False, flash_attn_max_num_splits_for_cuda_graph=32, use_cudnn_prefill=False,
                        use_trtllm_ragged_deepseek_prefill=False, use_trtllm_attention=None, disable_flashinfer_prefill=True, disable_flashinfer_q_quantization=False, use_prefill_query_quantization=False))
  --compilation-config COMPILATION_CONFIG, -cc COMPILATION_CONFIG
                        `torch.compile` and cudagraph capture configuration for the model.
                        As a shorthand, one can append compilation arguments via
                        -cc.parameter=argument such as `-cc.mode=3` (same as `-cc='{"mode":3}'`).
                        You can specify the full compilation config like so: `{"mode": 3, "cudagraph_capture_sizes": [1, 2, 4, 8]}`
                        API docs: https://docs.vllm.ai/en/v0.19.0/api/vllm/config/#vllm.config.CompilationConfig
                        Should either be a valid JSON string or JSON keys passed individually. (default: {'mode': None, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': [], 'splitting_ops': None,
                        'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_images_per_batch': 0, 'compile_sizes': None, 'compile_ranges_endpoints': None, 'inductor_compile_config':
                        {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': None,
                        'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': None, 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': None, 'pass_config': {}, 'max_cudagraph_capture_size': None,
                        'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': None, 'static_all_moe_layers': []})
  --ec-transfer-config EC_TRANSFER_CONFIG
                        The configurations for distributed EC cache transfer.
                        API docs: https://docs.vllm.ai/en/v0.19.0/api/vllm/config/#vllm.config.ECTransferConfig
                        Should either be a valid JSON string or JSON keys passed individually. (default: None)
  --kernel-config KERNEL_CONFIG
                        Kernel configuration.
                        API docs: https://docs.vllm.ai/en/v0.19.0/api/vllm/config/#vllm.config.KernelConfig
                        Should either be a valid JSON string or JSON keys passed individually. (default: KernelConfig(enable_flashinfer_autotune=None, moe_backend='auto'))
  --kv-events-config KV_EVENTS_CONFIG
                        The configurations for event publishing.
                        API docs: https://docs.vllm.ai/en/v0.19.0/api/vllm/config/#vllm.config.KVEventsConfig
                        Should either be a valid JSON string or JSON keys passed individually. (default: None)
  --kv-transfer-config KV_TRANSFER_CONFIG
                        The configurations for distributed KV cache transfer.
                        API docs: https://docs.vllm.ai/en/v0.19.0/api/vllm/config/#vllm.config.KVTransferConfig
                        Should either be a valid JSON string or JSON keys passed individually. (default: None)
  --optimization-level OPTIMIZATION_LEVEL
                        The optimization level. These levels trade startup time cost for performance, with -O0 having the best startup time and -O3 having the best performance. -O2 is used by default. See OptimizationLevel for full description. (default: 2)
  --performance-mode {balanced,interactivity,throughput}
                        Performance mode for runtime behavior, 'balanced' is the default. 'interactivity' favors low end-to-end per-request latency at small batch sizes (fine-grained CUDA graphs, latency-oriented kernels). 'throughput' favors aggregate tokens/sec at
                        high concurrency (larger CUDA graphs, more aggressive batching, throughput-oriented kernels). (default: balanced)
  --profiler-config PROFILER_CONFIG
                        Profiling configuration.
                        API docs: https://docs.vllm.ai/en/v0.19.0/api/vllm/config/#vllm.config.ProfilerConfig
                        Should either be a valid JSON string or JSON keys passed individually. (default: ProfilerConfig(profiler=None, torch_profiler_dir='', torch_profiler_with_stack=True, torch_profiler_with_flops=False, torch_profiler_use_gzip=True,
                        torch_profiler_dump_cuda_time_total=True, torch_profiler_record_shapes=False, torch_profiler_with_memory=False, ignore_frontend=False, delay_iterations=0, max_iterations=0, warmup_iterations=0, active_iterations=5, wait_iterations=0))
  --reasoning-config REASONING_CONFIG
                        The configurations for reasoning model.
                        API docs: https://docs.vllm.ai/en/v0.19.0/api/vllm/config/#vllm.config.ReasoningConfig
                        Should either be a valid JSON string or JSON keys passed individually. (default: None)
  --speculative-config SPECULATIVE_CONFIG, -sc SPECULATIVE_CONFIG
                        Speculative decoding configuration.
                        API docs: https://docs.vllm.ai/en/v0.19.0/api/vllm/config/#vllm.config.SpeculativeConfig
                        Should either be a valid JSON string or JSON keys passed individually. (default: None)
  --structured-outputs-config STRUCTURED_OUTPUTS_CONFIG
                        Structured outputs configuration.
                        API docs: https://docs.vllm.ai/en/v0.19.0/api/vllm/config/#vllm.config.StructuredOutputsConfig
                        Should either be a valid JSON string or JSON keys passed individually. (default: StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='',
                        enable_in_reasoning=False))
  --weight-transfer-config WEIGHT_TRANSFER_CONFIG
                        The configurations for weight transfer during RL training.
                        API docs: https://docs.vllm.ai/en/v0.19.0/api/vllm/config/#vllm.config.WeightTransferConfig
                        Should either be a valid JSON string or JSON keys passed individually. (default: None)

When passing JSON CLI arguments, the following sets of arguments are equivalent:
   --json-arg '{"key1": "value1", "key2": {"key3": "value2"}}'
   --json-arg.key1 value1 --json-arg.key2.key3 value2

Additionally, list elements can be passed individually using +:
   --json-arg '{"key4": ["value3", "value4", "value5"]}'
   --json-arg.key4+ value3 --json-arg.key4+='value4,value5'