Skip to content

Instantly share code, notes, and snippets.

@sshleifer

sshleifer/baseten_dedicated_bench.json

Last active July 30, 2026 02:49
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save sshleifer/20d655db9cc516bd403c658b856ba1ab to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save sshleifer/20d655db9cc516bd403c658b856ba1ab to your computer and use it in GitHub Desktop.
Download ZIP
GLM-5.2 serving: TML internal SGLang config (4xGB300) + bench comparison vs Baseten dedicated/serverless (2026-07-29)
Raw
baseten_dedicated_bench.json
[
{
"phase": "single",
"batch_size": 1,
"input_len": 256,
"output_len": 256,
"completed": 1,
"errors": 0,
"wall_sec": 1.8,
"lat_med": 1.81,
"lat_p99": 1.81,
"ttft_med": 0.15,
"ttft_p99": 0.15,
"tpot_med": 0.0065,
"in_tput": 142.8,
"out_tput": 141.7
},
{
"phase": "decode_medium",
"batch_size": 512,
"input_len": 256,
"output_len": 256,
"completed": 512,
"errors": 0,
"wall_sec": 61.5,
"lat_med": 32.12,
"lat_p99": 61.19,
"ttft_med": 28.76,
"ttft_p99": 59.15,
"tpot_med": 0.0196,
"in_tput": 2153.5,
"out_tput": 1659.7
},
{
"phase": "decode_long",
"batch_size": 16,
"input_len": 8192,
"output_len": 256,
"completed": 16,
"errors": 0,
"wall_sec": 8.7,
"lat_med": 8.16,
"lat_p99": 8.65,
"ttft_med": 3.93,
"ttft_p99": 6.67,
"tpot_med": 0.0238,
"in_tput": 15174.5,
"out_tput": 294.1
},
{
"phase": "throughput_8k",
"batch_size": 929,
"input_len": 8192,
"output_len": 1024,
"completed": 492,
"errors": 437,
"wall_sec": 1804.5,
"lat_med": 129.72,
"lat_p99": 259.1,
"ttft_med": 114.5,
"ttft_p99": 253.88,
"tpot_med": 0.0598,
"in_tput": 2237.8,
"out_tput": 83.3,
"error_kinds": {
"HTTP 429: b'{\\n\\t\"error\": {\\n\\t\\t\"code\": 429,\\n\\t\\t\"message\"": 398,
"ReadTimeout('')": 39
}
}
]
Raw
baseten_k3_bench.json
[
{
"phase": "single",
"batch_size": 1,
"input_len": 256,
"output_len": 256,
"completed": 1,
"errors": 0,
"wall_sec": 4.3,
"lat_med": 4.33,
"lat_p99": 4.33,
"ttft_med": 1.03,
"ttft_p99": 1.03,
"tpot_med": 0.0141,
"in_tput": 59.9,
"out_tput": 54.3
},
{
"phase": "decode_medium",
"batch_size": 512,
"input_len": 256,
"output_len": 256,
"completed": 226,
"errors": 286,
"wall_sec": 49.4,
"lat_med": 10.86,
"lat_p99": 45.17,
"ttft_med": 6.3,
"ttft_p99": 19.34,
"tpot_med": 0.0206,
"in_tput": 1184.6,
"out_tput": 1054.7,
"error_kinds": {
"HTTP 429: b'{\"error\":\"Rate limit exceeded\"}\\n'": 286
}
},
{
"phase": "decode_long",
"batch_size": 16,
"input_len": 8192,
"output_len": 256,
"completed": 16,
"errors": 0,
"wall_sec": 39.9,
"lat_med": 5.43,
"lat_p99": 39.93,
"ttft_med": 2.11,
"ttft_p99": 3.54,
"tpot_med": 0.0142,
"in_tput": 3283.1,
"out_tput": 90.9
},
{
"phase": "throughput_8k",
"batch_size": 929,
"input_len": 8192,
"output_len": 1024,
"completed": 329,
"errors": 600,
"wall_sec": 77.3,
"lat_med": 15.28,
"lat_p99": 61.53,
"ttft_med": 9.14,
"ttft_p99": 12.5,
"tpot_med": 0.0279,
"in_tput": 34855.3,
"out_tput": 1069.6,
"error_kinds": {
"HTTP 429: b'{\"error\":\"Rate limit exceeded\"}\\n'": 600
}
}
]
Raw
bench_endpoints.py
#!/usr/bin/env python3
"""Throughput/latency bench of the Baseten dedicated GLM-5.2 endpoint,
mirroring the internal sgl-bench batch_full shapes (4xGB300 baseline:
bench_glm5_2_nvfp4_kv_cache_dtype_fp8_e4m3_20260729_150320).
Phases (same batch sizes as the internal report):
single bs=1 input=256 output=256
decode_medium bs=512 input=256 output=256
decode_long bs=16 input=8192 output=256
throughput_8k bs=929 input=8192 output=1024 (capped by --phase-cap-sec)
Uses /v1/chat/completions with streaming to measure TTFT and TPOT per request;
in/out token counts come from usage (stream_options.include_usage) when
available, else estimated. Prompts are built from a repeated filler word and
calibrated against usage.prompt_tokens with one probe request.
Usage:
BASETEN_API_KEY=... python3 bench_baseten_dedicated.py [--phases single,decode_medium,...]
"""
import argparse
import asyncio
import json
import os
import statistics
import sys
import time
try:
import httpx
except ImportError:
print("needs httpx (run under uv run / a venv with httpx)", file=sys.stderr)
raise
BASE = os.environ.get("BENCH_BASE_URL", "https://model-womkd9kq.api.baseten.co/environments/production/sync/v1")
MODEL = os.environ.get("BENCH_MODEL", "zai-org/GLM-5.2")
PHASES = {
# bs, input_len, output_len
"single": (1, 256, 256),
"decode_medium": (512, 256, 256),
"decode_long": (16, 8192, 256),
"throughput_8k": (929, 8192, 1024),
}
FILLER = " the"
def pct(vals, p):
if not vals:
return float("nan")
vals = sorted(vals)
return vals[min(len(vals) - 1, int(p / 100 * len(vals)))]
async def calibrate_prompt(client, key, target_tokens):
"""Find the filler-repeat count whose prompt tokenizes to ~target_tokens."""
n = target_tokens
for _ in range(4):
body = {"model": MODEL, "messages": [{"role": "user", "content": FILLER * n}], "max_tokens": 1}
r = await client.post(f"{BASE}/chat/completions", json=body,
headers={"Authorization": f"Bearer {key}"}, timeout=120)
r.raise_for_status()
got = r.json()["usage"]["prompt_tokens"]
if abs(got - target_tokens) <= max(4, target_tokens // 100):
return n, got
n = max(1, int(n * target_tokens / max(got, 1)))
return n, got
async def one_request(client, key, prompt, output_len, deadline):
body = {
"model": MODEL,
"messages": [{"role": "user", "content": prompt}],
"max_tokens": output_len,
"stream": True,
"stream_options": {"include_usage": True},
}
if os.environ.get("BENCH_TEMPERATURE"):
body["temperature"] = float(os.environ["BENCH_TEMPERATURE"])
t0 = time.monotonic()
ttft = None
n_chunks = 0
usage = None
try:
async with client.stream(
"POST", f"{BASE}/chat/completions", json=body,
headers={"Authorization": f"Bearer {key}",
**({"x-session-id": f"bench-{os.getpid()}-{id(body)}"} if os.environ.get("BENCH_SESSION_HEADER") else {})},
timeout=httpx.Timeout(connect=30, read=max(600, deadline), write=60, pool=None),
) as resp:
if resp.status_code != 200:
text = (await resp.aread())[:120]
return {"error": f"HTTP {resp.status_code}: {text!r}", "seconds": time.monotonic() - t0}
async for line in resp.aiter_lines():
if not line.startswith("data:"):
continue
payload = line[5:].strip()
if payload == "[DONE]":
break
try:
chunk = json.loads(payload)
except json.JSONDecodeError:
continue
if chunk.get("usage"):
usage = chunk["usage"]
if chunk.get("choices"):
delta = chunk["choices"][0].get("delta") or {}
if delta.get("content") or delta.get("reasoning_content"):
n_chunks += 1
if ttft is None:
ttft = time.monotonic() - t0
except Exception as e:
return {"error": repr(e)[:150], "seconds": time.monotonic() - t0}
dur = time.monotonic() - t0
out_toks = (usage or {}).get("completion_tokens") or n_chunks
in_toks = (usage or {}).get("prompt_tokens")
tpot = (dur - (ttft or 0)) / max(out_toks - 1, 1)
return {"seconds": dur, "ttft": ttft, "tpot": tpot, "out_tokens": out_toks, "in_tokens": in_toks}
async def run_phase(name, bs, input_len, output_len, key, cap_sec):
limits = httpx.Limits(max_connections=bs + 16, max_keepalive_connections=bs + 16)
async with httpx.AsyncClient(limits=limits) as client:
n_filler, got = await calibrate_prompt(client, key, input_len)
print(f"[{name}] calibrated prompt: {got} tokens (target {input_len})")
t0 = time.monotonic()
# each request gets a distinct prefix token to defeat prefix caching
tasks = [
one_request(client, key, f"{i:06d}" + FILLER * n_filler, output_len, cap_sec)
for i in range(bs)
]
results = await asyncio.wait_for(asyncio.gather(*tasks), timeout=cap_sec + 120)
wall = time.monotonic() - t0
ok = [r for r in results if "error" not in r]
errs = [r for r in results if "error" in r]
ttfts = [r["ttft"] for r in ok if r.get("ttft")]
tpots = [r["tpot"] for r in ok if r.get("tpot")]
lats = [r["seconds"] for r in ok]
out_total = sum(r.get("out_tokens") or 0 for r in ok)
in_total = sum(r.get("in_tokens") or 0 for r in ok)
row = {
"phase": name, "batch_size": bs, "input_len": input_len, "output_len": output_len,
"completed": len(ok), "errors": len(errs), "wall_sec": round(wall, 1),
"lat_med": round(statistics.median(lats), 2) if lats else None,
"lat_p99": round(pct(lats, 99), 2) if lats else None,
"ttft_med": round(statistics.median(ttfts), 2) if ttfts else None,
"ttft_p99": round(pct(ttfts, 99), 2) if ttfts else None,
"tpot_med": round(statistics.median(tpots), 4) if tpots else None,
"in_tput": round(in_total / wall, 1),
"out_tput": round(out_total / wall, 1),
}
if errs:
from collections import Counter
row["error_kinds"] = dict(Counter(e["error"][:60] for e in errs).most_common(3))
print(f"[{name}] {json.dumps(row)}")
return row
async def main():
ap = argparse.ArgumentParser()
ap.add_argument("--phases", default="single,decode_medium,decode_long,throughput_8k")
ap.add_argument("--phase-cap-sec", type=int, default=1800)
ap.add_argument("--api-key", default=os.environ.get("BASETEN_API_KEY"))
args = ap.parse_args()
rows = []
for name in args.phases.split(","):
bs, il, ol = PHASES[name]
rows.append(await run_phase(name, bs, il, ol, args.api_key, args.phase_cap_sec))
out_name = os.environ.get("BENCH_OUT", "baseten_dedicated_bench.json")
with open(out_name, "w") as f:
json.dump(rows, f, indent=1)
print("\n== summary ==")
for r in rows:
print(f"{r['phase']:>14} bs={r['batch_size']:<4} ok={r['completed']}/{r['batch_size']}"
f" ttft_med={r['ttft_med']}s tpot_med={r['tpot_med']}s"
f" out_tput={r['out_tput']} tok/s in_tput={r['in_tput']} tok/s")
if __name__ == "__main__":
asyncio.run(main())
Raw
results.md

GLM-5.2 serving comparison: TML internal (SGLang, 4×GB300) vs Baseten endpoints — 2026-07-29

We benchmark identical request shapes against three deployments. Our internal serving config is attached in full (sglang_server_config.json — resolved SGLang ServerArgs); we'd like the Baseten dedicated deployment to match it.

Internal serving config (the thing to copy)

  • SGLang, 4×GB300, tp=4 dp=4 ep=1, enable_dp_attention=true
  • model: GLM-5.2-NVFP4 (quantization=modelopt_fp4), kv_cache_dtype=fp8_e4m3
  • attention_backend=nsa, moe_runner_backend=flashinfer_trtllm
  • chunked_prefill_size=4096, max_running_requests=1024, mem_fraction_static=0.9
  • full resolved args: sglang_server_config.json

Throughput / latency (bench_endpoints.py, streaming chat completions)

phase (bs / in / out) TML 4×GB300 SGLang Baseten dedicated GLM-5.2 (womkd9kq) Baseten serverless Kimi-K3
single (1 / 256 / 256) tpot 12.8ms (78 tok/s), ttft 0.24s tpot 6.5ms (154 tok/s), ttft 0.15s tpot 14.1ms, ttft 1.0s
decode_medium (512 / 256 / 256) out 11,052 tok/s, ttft 1.9s out 1,660 tok/s, ttft_med 28.8s 226/512 ok (286×429), out 1,055
decode_long (16 / 8k / 256) out 532, in 17.0k tok/s out 294, in 15.2k out 91, in 3.3k (16/16 ok)
throughput_8k_sat (929 / 8k / 1k) out 4,136 tok/s, all complete 492/929 complete, 398×429 + 39 requests hung >30min, ttft_med 114s 329/929 ok (600×429), ttft_med 9.1s

Observations:

  • Dedicated single-stream decode is excellent (2× ours) but aggregate capacity is ~6.7× below one internal 4-GPU node at bs512, and the deployment sheds/hangs at bs929 despite "no rate limits" (please explain the 429/queue limiter we hit).
  • Serverless K3 sheds early via 429 (TPM/RPM caps) but serves accepted requests fast.

Quality reference (internal, same server config)

eval internal 4×GB300
wikitext-loss ppl 2.923 (nll/tok 1.0727)
long-decode-ppl ppl 2.417 (worst_seg 4.560)
pt-mmlu-category 5-shot 87.9%
trivia_qa_pt 5-shot 1k 86.2%

We could not run the ppl/logprob evals against Baseten endpoints: POST /v1/completions with echo:true returns 500 on BOTH serverless and dedicated (ValidationError.__new__() missing 1 required positional argument: 'line_errors' on dedicated). logprobs without echo works. Prompt logprobs (echo) are required for ppl scoring.

Repro

BASETEN_API_KEY=... BENCH_BASE_URL=<endpoint>/v1 BENCH_MODEL=<model> \
  python3 bench_endpoints.py --phases single,decode_medium,decode_long,throughput_8k
Raw
sglang_server_config.json
{
"model_path": "gs://tml-data-projects-us-east4/base_models/hf_original/GLM-5.2-NVFP4",
"tokenizer_path": "zai-org/GLM-5.2",
"tokenizer_mode": "auto",
"tokenizer_backend": "huggingface",
"tokenizer_worker_num": 1,
"detokenizer_worker_num": 1,
"skip_tokenizer_init": false,
"load_format": "runai_streamer",
"model_loader_extra_config": "{}",
"trust_remote_code": true,
"context_length": null,
"is_embedding": false,
"enable_multimodal": null,
"revision": null,
"model_impl": "auto",
"model_config_parser": "auto",
"json_model_override_args": "{}",
"dtype": "auto",
"quantization": "modelopt_fp4",
"quantization_param_path": null,
"kv_cache_dtype": "fp8_e4m3",
"enable_fp32_lm_head": false,
"modelopt_quant": null,
"modelopt_checkpoint_restore_path": null,
"modelopt_checkpoint_save_path": null,
"modelopt_export_path": null,
"quantize_and_serve": false,
"rl_quant_profile": null,
"enable_tf32_matmul": false,
"mem_fraction_static": 0.9,
"max_running_requests": 1024,
"max_queued_requests": null,
"max_total_tokens": null,
"chunked_prefill_size": 4096,
"enable_dynamic_chunking": false,
"max_prefill_tokens": 16384,
"prefill_max_requests": 128,
"schedule_policy": "lpm",
"enable_priority_scheduling": false,
"disable_priority_preemption": false,
"default_priority_value": null,
"abort_on_priority_when_disabled": false,
"schedule_low_priority_values_first": false,
"priority_scheduling_preemption_threshold": 100,
"retraction_policy": "length",
"schedule_conservativeness": 0.18,
"page_size": 64,
"swa_full_tokens_ratio": 0.3,
"disable_hybrid_swa_memory": false,
"radix_eviction_policy": "lru",
"prefill_only_disable_kv_cache": false,
"disable_radix_cache": false,
"enable_page_major_kv_layout": false,
"enable_unified_memory": false,
"disable_chunked_prefix_cache": false,
"disable_overlap_schedule": false,
"num_continuous_decode_steps": 1,
"scheduler_recv_interval": 1,
"enable_mixed_chunk": false,
"nccl_port": null,
"dist_timeout": null,
"dist_init_addr": null,
"nnodes": 1,
"node_rank": 0,
"tp_size": 4,
"dcp_size": 1,
"pp_size": 1,
"pp_max_micro_batch_size": null,
"pp_async_batch_depth": 0,
"dp_size": 4,
"load_balance_method": "round_robin",
"attn_cp_size": 1,
"moe_dp_size": 1,
"dwdp_size": 1,
"enable_prefill_cp": false,
"cp_strategy": null,
"enable_dsa_cache_layer_split": false,
"enable_dsa_prefill_context_parallel": false,
"dsa_prefill_cp_mode": "round-robin-split",
"enable_prefill_context_parallel": false,
"prefill_cp_mode": "in-seq-split",
"enable_dp_attention": true,
"enable_dp_attention_local_control_broadcast": false,
"enable_dp_lm_head": false,
"enable_attn_tp_input_scattered": false,
"disable_attn_tp_gather": false,
"enable_p2p_check": false,
"device": "cuda",
"base_gpu_id": 0,
"gpu_id_step": 1,
"random_seed": 42,
"watchdog_timeout": 300,
"soft_watchdog_timeout": null,
"sleep_on_idle": false,
"use_ray": false,
"custom_sigquit_handler": null,
"numa_node": null,
"gc_threshold": null,
"host": "0.0.0.0",
"port": 30000,
"fastapi_root_path": "",
"smg_grpc_mode": false,
"grpc_mode": false,
"grpc_port": null,
"skip_server_warmup": false,
"warmups": null,
"enable_http2": false,
"ssl_keyfile": null,
"ssl_certfile": null,
"ssl_ca_certs": null,
"ssl_keyfile_password": null,
"enable_ssl_refresh": false,
"api_key": null,
"admin_api_key": null,
"served_model_name": "base_models/hf_original/GLM-5.2-NVFP4",
"weight_version": "default",
"chat_template": null,
"hf_chat_template_name": null,
"completion_template": null,
"file_storage_path": "sglang_storage",
"enable_cache_report": false,
"reasoning_parser": "glm45",
"default_chat_template_kwargs": null,
"strip_thinking_cache": false,
"enable_strict_thinking": false,
"tool_call_parser": "glm47",
"tool_server": null,
"sampling_defaults": "openai",
"asr_max_buffer_seconds": 60,
"asr_max_concurrent_sessions": 32,
"preferred_sampling_params": null,
"allow_auto_truncate": false,
"stream_interval": 8,
"batch_notify_size": 16,
"stream_response_default_include_usage": false,
"incremental_streaming_output": true,
"enable_streaming_session": true,
"enable_session_radix_cache": false,
"log_level": "info",
"log_level_http": "warning",
"log_requests": false,
"log_requests_level": 2,
"log_requests_format": "text",
"log_requests_target": null,
"uvicorn_access_log_exclude_prefixes": [],
"crash_dump_folder": null,
"show_time_cost": false,
"enable_metrics": true,
"smg_http_sidecar_port": null,
"enable_mfu_metrics": true,
"enable_metrics_for_all_schedulers": false,
"load_snapshot_publish_interval": 15,
"tokenizer_metrics_custom_labels_header": "x-custom-labels",
"tokenizer_metrics_allowed_custom_labels": null,
"extra_metric_labels": {
"slurm_job_id": "15538838",
"actor_label": "sampling",
"replica_id": "0"
},
"bucket_time_to_first_token": null,
"bucket_inter_token_latency": null,
"bucket_e2e_request_latency": null,
"prompt_tokens_buckets": null,
"generation_tokens_buckets": null,
"gc_warning_threshold_secs": 0.0,
"decode_log_interval": 40,
"enable_request_time_stats_logging": false,
"kv_events_config": null,
"enable_forward_pass_metrics": false,
"forward_pass_metrics_worker_id": "",
"forward_pass_metrics_ipc_name": null,
"enable_trace": false,
"trace_modules": "request",
"otlp_traces_endpoint": "172.22.6.219:4162",
"export_metrics_to_file": false,
"export_metrics_to_file_dir": null,
"stat_loggers": null,
"constrained_json_whitespace_pattern": null,
"constrained_json_disable_any_whitespace": false,
"attention_backend": "nsa",
"decode_attention_backend": null,
"prefill_attention_backend": null,
"sampling_backend": "pytorch",
"grammar_backend": "xgrammar",
"radix_cache_backend": null,
"mm_attention_backend": null,
"fp8_gemm_runner_backend": "auto",
"fp4_gemm_runner_backend": "flashinfer_trtllm",
"bf16_gemm_backend": "auto",
"dsa_prefill_backend": "trtllm",
"dsa_decode_backend": "trtllm",
"dsa_paged_mqa_logits_backend": "auto",
"dsa_topk_backend": "sgl-kernel",
"disable_flashinfer_autotune": false,
"mamba_backend": "triton",
"cuda_graph_config": {
"decode": {
"backend": "full",
"max_bs": 1024,
"bs": [
1,
2,
4,
8,
16,
32,
64,
128,
256,
384,
512,
640,
768,
896,
1024
],
"tc_compiler": "eager",
"full_prefill_max_req": null
},
"prefill": {
"backend": "disabled",
"max_bs": 2048,
"bs": [
8,
16,
32,
64,
128,
256,
512,
1024,
1536,
2048,
2560,
3072,
3584,
4096,
4608,
5120,
5632,
6144,
6656,
7168,
7680,
8192,
8704,
9216,
9728,
10240,
10752,
11264,
11776,
12288,
12800,
13312,
13824,
14336,
14848,
15360,
15872,
16384
],
"tc_compiler": "eager",
"full_prefill_max_req": null
}
},
"cuda_graph_backend_decode": null,
"cuda_graph_backend_prefill": null,
"cuda_graph_max_bs_decode": null,
"cuda_graph_max_bs_prefill": null,
"cuda_graph_bs_decode": [
1,
2,
4,
8,
16,
32,
64,
128,
256,
384,
512,
640,
768,
896,
1024
],
"cuda_graph_bs_prefill": [
8,
16,
32,
64,
128,
256,
512,
1024,
1536,
2048,
2560,
3072,
3584,
4096,
4608,
5120,
5632,
6144,
6656,
7168,
7680,
8192,
8704,
9216,
9728,
10240,
10752,
11264,
11776,
12288,
12800,
13312,
13824,
14336,
14848,
15360,
15872,
16384
],
"cuda_graph_tc_compiler": null,
"disable_prefill_cuda_graph": false,
"disable_decode_cuda_graph": false,
"disable_cuda_graph": false,
"disable_cuda_graph_padding": false,
"enable_profile_cuda_graph": false,
"enable_cudagraph_gc": false,
"debug_cuda_graph": false,
"enable_layerwise_nvtx_marker": false,
"enable_nccl_nvls": false,
"enable_symm_mem": false,
"triton_attention_reduce_in_fp32": false,
"triton_attention_num_kv_splits": 8,
"triton_attention_split_tile_size": null,
"flashinfer_mla_disable_ragged": false,
"enable_fused_qk_norm_rope": false,
"enable_precise_embedding_interpolation": false,
"enable_fused_moe_sum_all_reduce": false,
"enable_deepseek_v4_fp4_indexer": false,
"disable_custom_all_reduce": true,
"enable_mscclpp": false,
"enable_torch_symm_mem": false,
"enable_scattered_sconv": false,
"pre_warm_nccl": false,
"enable_quant_communications": false,
"enable_flashinfer_allreduce_fusion": false,
"enforce_disable_flashinfer_allreduce_fusion": false,
"flashinfer_allreduce_fusion_backend": null,
"enable_aiter_allreduce_fusion": false,
"enable_torch_compile": false,
"enable_torch_compile_debug_mode": false,
"torch_compile_max_bs": 32,
"torchao_config": "",
"speculative_algorithm": null,
"speculative_draft_model_path": null,
"speculative_draft_model_revision": null,
"speculative_draft_load_format": null,
"speculative_num_steps": null,
"speculative_eagle_topk": null,
"speculative_num_draft_tokens": null,
"speculative_dflash_block_size": null,
"speculative_dspark_block_size": null,
"speculative_dspark_sps_table_path": null,
"speculative_dspark_confidence_sts_path": null,
"speculative_dspark_align_verify_tokens_to_graph_tier": false,
"speculative_accept_threshold_single": 1.0,
"speculative_accept_threshold_acc": 1.0,
"speculative_use_rejection_sampling": false,
"speculative_token_map": null,
"speculative_attention_mode": "prefill",
"speculative_draft_attention_backend": null,
"speculative_draft_window_size": null,
"speculative_moe_runner_backend": "flashinfer_trtllm",
"speculative_moe_a2a_backend": null,
"speculative_draft_model_quantization": "modelopt_fp4",
"speculative_skip_dp_mlp_sync": false,
"enable_multi_layer_eagle": false,
"speculative_adaptive": false,
"speculative_adaptive_config": null,
"decoupled_spec_bind_endpoint": null,
"decoupled_spec_connect_endpoints": null,
"decoupled_spec_rank": null,
"decoupled_spec_role": "null",
"spec_trace_dir": null,
"speculative_ngram_min_bfs_breadth": 1,
"speculative_ngram_max_bfs_breadth": 10,
"speculative_ngram_match_type": "BFS",
"speculative_ngram_max_trie_depth": 18,
"speculative_ngram_capacity": 10000000,
"speculative_ngram_external_corpus_path": null,
"speculative_ngram_external_sam_budget": 0,
"speculative_ngram_external_corpus_max_tokens": 10000000,
"ep_size": 1,
"moe_a2a_backend": "none",
"moe_runner_backend": "flashinfer_trtllm",
"flashinfer_mxfp4_moe_precision": "default",
"deepep_mode": "auto",
"fuseep_mode": 2,
"deepep_dispatcher_output_dtype": "auto",
"ep_num_redundant_experts": 0,
"ep_dispatch_algorithm": "dynamic",
"init_expert_location": "trivial",
"enable_eplb": false,
"eplb_algorithm": "deepseek",
"eplb_rebalance_num_iterations": 1000,
"eplb_rebalance_layers_per_chunk": null,
"eplb_min_rebalancing_utilization_threshold": 1.0,
"expert_distribution_recorder_mode": null,
"expert_distribution_recorder_buffer_size": 1000,
"enable_expert_distribution_metrics": false,
"deepep_config": null,
"moe_dense_tp_size": null,
"elastic_ep_backend": null,
"enable_elastic_expert_backup": false,
"mooncake_ib_device": null,
"enable_waterfill": false,
"ep_join_mode": null,
"ep_join_rank_offset": 0,
"elastic_ep_initial_size": null,
"max_ep_size": null,
"elastic_ep_scale_timeout": 600,
"elastic_ep_rejoin": false,
"disable_flashinfer_cutlass_moe_fp4_allgather": false,
"disable_shared_experts_fusion": true,
"enforce_shared_experts_fusion": false,
"max_mamba_cache_size": null,
"mamba_ssm_dtype": null,
"enable_mamba_cache_stochastic_rounding": false,
"mamba_cache_philox_rounds": 0,
"mamba_full_memory_ratio": 0.3,
"mamba_radix_cache_strategy": "auto",
"uses_mamba_radix_cache": false,
"mamba_track_interval": 256,
"disable_finished_mamba_cache": false,
"enable_int8_mamba_checkpoint": false,
"int8_mamba_ckpt_size": null,
"linear_attn_backend": "triton",
"linear_attn_decode_backend": null,
"linear_attn_prefill_backend": null,
"enable_linear_replayssm": false,
"linear_replayssm_cache_len": 16,
"enable_gdn_replayssm_spec": false,
"enable_hierarchical_cache": false,
"hicache_ratio": 2.0,
"hicache_size": 0,
"hicache_write_policy": "write_through",
"hicache_io_backend": "kernel",
"hicache_mem_layout": "page_first",
"hicache_storage_backend": null,
"hicache_storage_prefetch_policy": "timeout",
"hicache_storage_backend_extra_config": null,
"enable_hisparse": false,
"hisparse_config": null,
"enable_broadcast_mm_inputs_process": false,
"enable_prefix_mm_cache": false,
"mm_enable_dp_encoder": false,
"mm_process_config": {},
"mm_processor_worker_num": 0,
"mm_io_worker_num": 0,
"limit_mm_data_per_request": null,
"enable_mm_global_cache": false,
"disable_fast_image_processor": false,
"mm_feature_transport": "cpu",
"keep_mm_feature_on_device": false,
"enable_lora": null,
"enable_lora_overlap_loading": null,
"max_lora_rank": 64,
"lora_target_modules": null,
"lora_paths": null,
"max_loaded_loras": null,
"max_loras_per_batch": 4,
"lora_eviction_policy": "lru",
"lora_backend": "triton",
"max_lora_chunk_size": 16,
"experts_shared_outer_loras": false,
"lora_use_virtual_experts": false,
"lora_strict_loading": true,
"lora_drain_wait_threshold": 0.0,
"enable_two_batch_overlap": false,
"enable_single_batch_overlap": false,
"tbo_token_distribution_threshold": 0.48,
"cpu_offload_gb": 0,
"offload_group_size": -1,
"offload_num_in_group": 1,
"offload_prefetch_step": 1,
"offload_mode": "cpu",
"enable_lmcache": false,
"lmcache_config_file": null,
"enable_flexkv": false,
"flexkv_config_file": null,
"kt_weight_path": null,
"kt_method": "AMXINT4",
"kt_cpuinfer": null,
"kt_threadpool_count": 2,
"kt_num_gpu_experts": null,
"kt_max_deferred_experts_per_token": null,
"dllm_algorithm": null,
"dllm_algorithm_config": null,
"dllm_fdfo": true,
"disaggregation_mode": "null",
"disaggregation_transfer_backend": "nixl",
"disaggregation_bootstrap_port": 30137,
"disaggregation_ib_device": null,
"disaggregation_decode_enable_radix_cache": false,
"disaggregation_decode_enable_offload_kvcache": false,
"num_reserved_decode_tokens": 512,
"disaggregation_decode_extra_slots": null,
"disaggregation_decode_polling_interval": 1,
"optimistic_prefill_attempts": 0,
"encoder_only": false,
"language_only": false,
"encoder_transfer_backend": "zmq_to_scheduler",
"encoder_urls": [],
"encoder_bootstrap_port": 8997,
"encoder_register_urls": [],
"enable_adaptive_dispatch_to_encoder": false,
"enable_pdmux": false,
"pdmux_config_path": null,
"sm_group_num": 8,
"custom_weight_loader": [],
"weight_loader_disable_mmap": false,
"weight_loader_prefetch_checkpoints": false,
"weight_loader_prefetch_num_threads": 4,
"weight_loader_drop_cache_after_load": false,
"remote_instance_weight_loader_seed_instance_ip": null,
"remote_instance_weight_loader_seed_instance_service_port": null,
"remote_instance_weight_loader_send_weights_group_ports": null,
"remote_instance_weight_loader_backend": "nccl",
"remote_instance_weight_loader_start_seed_via_transfer_engine": false,
"engine_info_bootstrap_port": 6789,
"modelexpress_config": null,
"download_dir": null,
"model_checksum": null,
"delete_ckpt_after_loading": false,
"decrypted_config_file": null,
"decrypted_draft_config_file": null,
"checkpoint_engine_wait_weights_before_ready": false,
"enable_prefill_delayer": true,
"prefill_delayer_max_delay_passes": 30,
"prefill_delayer_token_usage_low_watermark": 0.1,
"prefill_delayer_forward_passes_buckets": null,
"prefill_delayer_wait_seconds_buckets": null,
"prefill_delayer_queue_min_ratio": null,
"prefill_delayer_max_delay_ms": null,
"min_free_slots_delay": null,
"enable_deterministic_inference": false,
"rl_on_policy_target": null,
"kv_canary": "none",
"kv_canary_real_data": "none",
"kv_canary_sweep_interval": 0,
"enable_dynamic_batch_tokenizer": false,
"dynamic_batch_tokenizer_batch_size": 32,
"dynamic_batch_tokenizer_batch_timeout": 0.002,
"enable_tokenizer_batch_encode": false,
"disable_tokenizer_batch_decode": false,
"debug_tensor_dump_output_folder": null,
"debug_tensor_dump_layers": null,
"debug_tensor_dump_input_file": null,
"enable_memory_saver": false,
"enable_weights_cpu_backup": false,
"enable_draft_weights_cpu_backup": false,
"enable_custom_logit_processor": false,
"enable_return_hidden_states": false,
"enable_return_routed_experts": false,
"enable_return_indexer_topk": false,
"disable_outlines_disk_cache": false,
"enable_mis": false,
"forward_hooks": null,
"msprobe_dump_config": null,
"use_token_span": true,
"tml_mm_process_mode": false,
"tml_colocate_hydration": false,
"tml_rescale_image_frac": null,
"tml_rescale_image_max_upscaled_long_edge": null,
"async_detokenizer_send": false,
"remote_model_weights_path": null,
"enable_low_latency_kernels": false,
"enable_ll_moe_v2": true,
"mxfp8_kv_roundtrip": false,
"enable_prefill_cuda_graph": false,
"prefill_cuda_graph_sizes": [
1,
2,
4,
8,
12,
16,
24,
32,
40,
48,
56,
64,
96,
128
],
"prefill_cuda_graph_max_size": 128,
"decode_attention_num_splits": null,
"enable_dp_mixed_cuda_graph": false,
"mxfp8_moe_compute": false,
"ckpt_has_mxfp8_scales": false,
"nvfp4_use_calibrated_w2_input_scales": false,
"nvfp4_use_per_token_activation_scales": false,
"force_shared_experts_bf16": false,
"nvfp4_act_4over6": false,
"nvfp4_quantize_on_load": false,
"disable_sample_audio": false,
"use_moe_v2": false,
"tokenizer_batch_max_size": 1,
"tokenizer_batch_timeout_ms": 50,
"tokenizer_vocab_mask_size": 154856,
"use_tml_lora_backend": true,
"allow_full_non_token_span_fallback": true,
"audio_flow_max_steps": null,
"audio_flow_sampler_dtype": null,
"default_alter_topk": 0,
"min_prefill_batch_size": 0,
"max_prefill_skip_count": 10,
"send_continuous_mel": false,
"enable_moe_straggler_metrics": true,
"enable_gpu_process_marker": false,
"sconv_rms_preserving": null,
"dsconv_rms_preserving": null,
"enable_distributed_gumbel_sampling": false,
"coupled_reject_sampling": false,
"cg_verify_scheme": "shared_gumbel",
"status": "ready",
"max_total_num_tokens": 2328128,
"max_req_input_len": 1048570,
"scheduler_pids": [
309895,
309897,
309899,
309901
],
"internal_states": [
{
"model_path": "gs://tml-data-projects-us-east4/base_models/hf_original/GLM-5.2-NVFP4",
"tokenizer_path": "zai-org/GLM-5.2",
"tokenizer_mode": "auto",
"tokenizer_backend": "huggingface",
"tokenizer_worker_num": 1,
"detokenizer_worker_num": 1,
"skip_tokenizer_init": false,
"load_format": "runai_streamer",
"model_loader_extra_config": "{}",
"trust_remote_code": true,
"context_length": null,
"is_embedding": false,
"enable_multimodal": null,
"revision": null,
"model_impl": "auto",
"model_config_parser": "auto",
"json_model_override_args": "{}",
"dtype": "auto",
"quantization": "modelopt_fp4",
"quantization_param_path": null,
"kv_cache_dtype": "fp8_e4m3",
"enable_fp32_lm_head": false,
"modelopt_quant": null,
"modelopt_checkpoint_restore_path": null,
"modelopt_checkpoint_save_path": null,
"modelopt_export_path": null,
"quantize_and_serve": false,
"rl_quant_profile": null,
"enable_tf32_matmul": false,
"mem_fraction_static": 0.9,
"max_running_requests": 1024,
"max_queued_requests": null,
"max_total_tokens": null,
"chunked_prefill_size": 4096,
"enable_dynamic_chunking": false,
"max_prefill_tokens": 16384,
"prefill_max_requests": 128,
"schedule_policy": "lpm",
"enable_priority_scheduling": false,
"disable_priority_preemption": false,
"default_priority_value": null,
"abort_on_priority_when_disabled": false,
"schedule_low_priority_values_first": false,
"priority_scheduling_preemption_threshold": 100,
"retraction_policy": "length",
"schedule_conservativeness": 0.18,
"page_size": 64,
"swa_full_tokens_ratio": 0.3,
"disable_hybrid_swa_memory": false,
"radix_eviction_policy": "lru",
"prefill_only_disable_kv_cache": false,
"disable_radix_cache": false,
"enable_page_major_kv_layout": false,
"enable_unified_memory": false,
"disable_chunked_prefix_cache": true,
"disable_overlap_schedule": false,
"num_continuous_decode_steps": 1,
"scheduler_recv_interval": 1,
"enable_mixed_chunk": false,
"nccl_port": null,
"dist_timeout": null,
"dist_init_addr": null,
"nnodes": 1,
"node_rank": 0,
"tp_size": 4,
"dcp_size": 1,
"pp_size": 1,
"pp_max_micro_batch_size": 256,
"pp_async_batch_depth": 0,
"dp_size": 4,
"load_balance_method": "round_robin",
"attn_cp_size": 1,
"moe_dp_size": 1,
"dwdp_size": 1,
"enable_prefill_cp": false,
"cp_strategy": null,
"enable_dsa_cache_layer_split": false,
"enable_dsa_prefill_context_parallel": false,
"dsa_prefill_cp_mode": "round-robin-split",
"enable_prefill_context_parallel": false,
"prefill_cp_mode": "in-seq-split",
"enable_dp_attention": true,
"enable_dp_attention_local_control_broadcast": false,
"enable_dp_lm_head": false,
"enable_attn_tp_input_scattered": false,
"disable_attn_tp_gather": false,
"enable_p2p_check": false,
"device": "cuda",
"base_gpu_id": 0,
"gpu_id_step": 1,
"random_seed": 42,
"watchdog_timeout": 300,
"soft_watchdog_timeout": null,
"sleep_on_idle": false,
"use_ray": false,
"numa_node": null,
"gc_threshold": null,
"host": "0.0.0.0",
"port": 30000,
"fastapi_root_path": "",
"smg_grpc_mode": false,
"grpc_mode": false,
"grpc_port": null,
"skip_server_warmup": false,
"warmups": null,
"enable_http2": false,
"ssl_keyfile": null,
"ssl_certfile": null,
"ssl_ca_certs": null,
"ssl_keyfile_password": null,
"enable_ssl_refresh": false,
"api_key": null,
"admin_api_key": null,
"served_model_name": "base_models/hf_original/GLM-5.2-NVFP4",
"weight_version": "default",
"chat_template": null,
"hf_chat_template_name": null,
"completion_template": null,
"file_storage_path": "sglang_storage",
"enable_cache_report": false,
"reasoning_parser": "glm45",
"default_chat_template_kwargs": null,
"strip_thinking_cache": false,
"enable_strict_thinking": false,
"tool_call_parser": "glm47",
"tool_server": null,
"sampling_defaults": "openai",
"asr_max_buffer_seconds": 60,
"asr_max_concurrent_sessions": 32,
"preferred_sampling_params": null,
"allow_auto_truncate": false,
"stream_interval": 8,
"batch_notify_size": 16,
"stream_response_default_include_usage": false,
"incremental_streaming_output": true,
"enable_streaming_session": true,
"enable_session_radix_cache": false,
"log_level": "info",
"log_level_http": "warning",
"log_requests": false,
"log_requests_level": 2,
"log_requests_format": "text",
"log_requests_target": null,
"uvicorn_access_log_exclude_prefixes": [],
"crash_dump_folder": null,
"show_time_cost": false,
"enable_metrics": true,
"smg_http_sidecar_port": null,
"enable_mfu_metrics": true,
"enable_metrics_for_all_schedulers": false,
"load_snapshot_publish_interval": 15,
"tokenizer_metrics_custom_labels_header": "x-custom-labels",
"tokenizer_metrics_allowed_custom_labels": null,
"extra_metric_labels": {
"slurm_job_id": "15538838",
"actor_label": "sampling",
"replica_id": "0"
},
"bucket_time_to_first_token": null,
"bucket_inter_token_latency": null,
"bucket_e2e_request_latency": null,
"prompt_tokens_buckets": null,
"generation_tokens_buckets": null,
"gc_warning_threshold_secs": 0.0,
"decode_log_interval": 40,
"enable_request_time_stats_logging": false,
"kv_events_config": null,
"enable_forward_pass_metrics": false,
"forward_pass_metrics_worker_id": "",
"forward_pass_metrics_ipc_name": null,
"enable_trace": false,
"trace_modules": "request",
"otlp_traces_endpoint": "172.22.6.219:4162",
"export_metrics_to_file": false,
"export_metrics_to_file_dir": null,
"stat_loggers": null,
"constrained_json_whitespace_pattern": null,
"constrained_json_disable_any_whitespace": false,
"attention_backend": "nsa",
"decode_attention_backend": null,
"prefill_attention_backend": null,
"sampling_backend": "pytorch",
"grammar_backend": "xgrammar",
"radix_cache_backend": null,
"mm_attention_backend": null,
"fp8_gemm_runner_backend": "auto",
"fp4_gemm_runner_backend": "flashinfer_trtllm",
"bf16_gemm_backend": "auto",
"dsa_prefill_backend": "trtllm",
"dsa_decode_backend": "trtllm",
"dsa_paged_mqa_logits_backend": "auto",
"dsa_topk_backend": "sgl-kernel",
"disable_flashinfer_autotune": false,
"mamba_backend": "triton",
"cuda_graph_config": {
"decode": {
"backend": "full",
"max_bs": 1024,
"bs": [
1,
2,
4,
8,
16,
32,
64,
128,
256,
384,
512,
640,
768,
896,
1024
],
"tc_compiler": "eager",
"full_prefill_max_req": null
},
"prefill": {
"backend": "disabled",
"max_bs": 2048,
"bs": [
8,
16,
32,
64,
128,
256,
512,
1024,
1536,
2048,
2560,
3072,
3584,
4096,
4608,
5120,
5632,
6144,
6656,
7168,
7680,
8192,
8704,
9216,
9728,
10240,
10752,
11264,
11776,
12288,
12800,
13312,
13824,
14336,
14848,
15360,
15872,
16384
],
"tc_compiler": "eager",
"full_prefill_max_req": null
}
},
"cuda_graph_backend_decode": null,
"cuda_graph_backend_prefill": null,
"cuda_graph_max_bs_decode": null,
"cuda_graph_max_bs_prefill": null,
"cuda_graph_bs_decode": [
1,
2,
4,
8,
16,
32,
64,
128,
256,
384,
512,
640,
768,
896,
1024
],
"cuda_graph_bs_prefill": [
8,
16,
32,
64,
128,
256,
512,
1024,
1536,
2048,
2560,
3072,
3584,
4096,
4608,
5120,
5632,
6144,
6656,
7168,
7680,
8192,
8704,
9216,
9728,
10240,
10752,
11264,
11776,
12288,
12800,
13312,
13824,
14336,
14848,
15360,
15872,
16384
],
"cuda_graph_tc_compiler": null,
"disable_prefill_cuda_graph": false,
"disable_decode_cuda_graph": false,
"disable_cuda_graph": false,
"disable_cuda_graph_padding": false,
"enable_profile_cuda_graph": false,
"enable_cudagraph_gc": false,
"debug_cuda_graph": false,
"enable_layerwise_nvtx_marker": false,
"enable_nccl_nvls": false,
"enable_symm_mem": false,
"triton_attention_reduce_in_fp32": false,
"triton_attention_num_kv_splits": 8,
"triton_attention_split_tile_size": null,
"flashinfer_mla_disable_ragged": false,
"enable_fused_qk_norm_rope": false,
"enable_precise_embedding_interpolation": false,
"enable_fused_moe_sum_all_reduce": false,
"enable_deepseek_v4_fp4_indexer": false,
"disable_custom_all_reduce": true,
"enable_mscclpp": false,
"enable_torch_symm_mem": false,
"enable_scattered_sconv": false,
"pre_warm_nccl": false,
"enable_quant_communications": false,
"enable_flashinfer_allreduce_fusion": false,
"enforce_disable_flashinfer_allreduce_fusion": false,
"flashinfer_allreduce_fusion_backend": null,
"enable_aiter_allreduce_fusion": false,
"enable_torch_compile": false,
"enable_torch_compile_debug_mode": false,
"torch_compile_max_bs": 32,
"torchao_config": "",
"speculative_algorithm": null,
"speculative_draft_model_path": null,
"speculative_draft_model_revision": null,
"speculative_draft_load_format": null,
"speculative_num_steps": null,
"speculative_eagle_topk": null,
"speculative_num_draft_tokens": null,
"speculative_dflash_block_size": null,
"speculative_dspark_block_size": null,
"speculative_dspark_sps_table_path": null,
"speculative_dspark_confidence_sts_path": null,
"speculative_dspark_align_verify_tokens_to_graph_tier": false,
"speculative_accept_threshold_single": 1.0,
"speculative_accept_threshold_acc": 1.0,
"speculative_use_rejection_sampling": false,
"speculative_token_map": null,
"speculative_attention_mode": "prefill",
"speculative_draft_attention_backend": null,
"speculative_draft_window_size": null,
"speculative_moe_runner_backend": "flashinfer_trtllm",
"speculative_moe_a2a_backend": null,
"speculative_draft_model_quantization": "modelopt_fp4",
"speculative_skip_dp_mlp_sync": false,
"enable_multi_layer_eagle": false,
"speculative_adaptive": false,
"speculative_adaptive_config": null,
"decoupled_spec_bind_endpoint": null,
"decoupled_spec_connect_endpoints": null,
"decoupled_spec_rank": null,
"decoupled_spec_role": "null",
"spec_trace_dir": null,
"speculative_ngram_min_bfs_breadth": 1,
"speculative_ngram_max_bfs_breadth": 10,
"speculative_ngram_match_type": "BFS",
"speculative_ngram_max_trie_depth": 18,
"speculative_ngram_capacity": 10000000,
"speculative_ngram_external_corpus_path": null,
"speculative_ngram_external_sam_budget": 0,
"speculative_ngram_external_corpus_max_tokens": 10000000,
"ep_size": 1,
"moe_a2a_backend": "none",
"moe_runner_backend": "flashinfer_trtllm",
"flashinfer_mxfp4_moe_precision": "default",
"deepep_mode": "auto",
"fuseep_mode": 2,
"deepep_dispatcher_output_dtype": "auto",
"ep_num_redundant_experts": 0,
"ep_dispatch_algorithm": "dynamic",
"init_expert_location": "trivial",
"enable_eplb": false,
"eplb_algorithm": "deepseek",
"eplb_rebalance_num_iterations": 1000,
"eplb_rebalance_layers_per_chunk": null,
"eplb_min_rebalancing_utilization_threshold": 1.0,
"expert_distribution_recorder_mode": null,
"expert_distribution_recorder_buffer_size": 1000,
"enable_expert_distribution_metrics": false,
"deepep_config": null,
"moe_dense_tp_size": null,
"elastic_ep_backend": null,
"enable_elastic_expert_backup": false,
"mooncake_ib_device": null,
"enable_waterfill": false,
"ep_join_mode": null,
"ep_join_rank_offset": 0,
"elastic_ep_initial_size": null,
"max_ep_size": null,
"elastic_ep_scale_timeout": 600,
"elastic_ep_rejoin": false,
"disable_flashinfer_cutlass_moe_fp4_allgather": false,
"disable_shared_experts_fusion": true,
"enforce_shared_experts_fusion": false,
"max_mamba_cache_size": null,
"mamba_ssm_dtype": null,
"enable_mamba_cache_stochastic_rounding": false,
"mamba_cache_philox_rounds": 0,
"mamba_full_memory_ratio": 0.3,
"mamba_radix_cache_strategy": "auto",
"uses_mamba_radix_cache": false,
"mamba_track_interval": 256,
"disable_finished_mamba_cache": false,
"enable_int8_mamba_checkpoint": false,
"int8_mamba_ckpt_size": null,
"linear_attn_backend": "triton",
"linear_attn_decode_backend": null,
"linear_attn_prefill_backend": null,
"enable_linear_replayssm": false,
"linear_replayssm_cache_len": 16,
"enable_gdn_replayssm_spec": false,
"enable_hierarchical_cache": false,
"hicache_ratio": 2.0,
"hicache_size": 0,
"hicache_write_policy": "write_through",
"hicache_io_backend": "kernel",
"hicache_mem_layout": "page_first",
"hicache_storage_backend": null,
"hicache_storage_prefetch_policy": "timeout",
"hicache_storage_backend_extra_config": null,
"enable_hisparse": false,
"hisparse_config": null,
"enable_broadcast_mm_inputs_process": false,
"enable_prefix_mm_cache": false,
"mm_enable_dp_encoder": false,
"mm_process_config": {},
"mm_processor_worker_num": 0,
"mm_io_worker_num": 0,
"limit_mm_data_per_request": null,
"enable_mm_global_cache": false,
"disable_fast_image_processor": false,
"mm_feature_transport": "cpu",
"keep_mm_feature_on_device": false,
"enable_lora": null,
"enable_lora_overlap_loading": null,
"max_lora_rank": 64,
"lora_target_modules": null,
"lora_paths": null,
"max_loaded_loras": null,
"max_loras_per_batch": 4,
"lora_eviction_policy": "lru",
"lora_backend": "triton",
"max_lora_chunk_size": 16,
"experts_shared_outer_loras": false,
"lora_use_virtual_experts": false,
"lora_strict_loading": true,
"lora_drain_wait_threshold": 0.0,
"enable_two_batch_overlap": false,
"enable_single_batch_overlap": false,
"tbo_token_distribution_threshold": 0.48,
"cpu_offload_gb": 0,
"offload_group_size": -1,
"offload_num_in_group": 1,
"offload_prefetch_step": 1,
"offload_mode": "cpu",
"enable_lmcache": false,
"lmcache_config_file": null,
"enable_flexkv": false,
"flexkv_config_file": null,
"kt_weight_path": null,
"kt_method": "AMXINT4",
"kt_cpuinfer": null,
"kt_threadpool_count": 2,
"kt_num_gpu_experts": null,
"kt_max_deferred_experts_per_token": null,
"dllm_algorithm": null,
"dllm_algorithm_config": null,
"dllm_fdfo": true,
"disaggregation_mode": "null",
"disaggregation_transfer_backend": "nixl",
"disaggregation_bootstrap_port": 30137,
"disaggregation_ib_device": null,
"disaggregation_decode_enable_radix_cache": false,
"disaggregation_decode_enable_offload_kvcache": false,
"num_reserved_decode_tokens": 512,
"disaggregation_decode_extra_slots": null,
"disaggregation_decode_polling_interval": 1,
"optimistic_prefill_attempts": 0,
"encoder_only": false,
"language_only": false,
"encoder_transfer_backend": "zmq_to_scheduler",
"encoder_urls": [],
"encoder_bootstrap_port": 8997,
"encoder_register_urls": [],
"enable_adaptive_dispatch_to_encoder": false,
"enable_pdmux": false,
"pdmux_config_path": null,
"sm_group_num": 8,
"custom_weight_loader": [],
"weight_loader_disable_mmap": false,
"weight_loader_prefetch_checkpoints": false,
"weight_loader_prefetch_num_threads": 4,
"weight_loader_drop_cache_after_load": false,
"remote_instance_weight_loader_seed_instance_ip": null,
"remote_instance_weight_loader_seed_instance_service_port": null,
"remote_instance_weight_loader_send_weights_group_ports": null,
"remote_instance_weight_loader_backend": "nccl",
"remote_instance_weight_loader_start_seed_via_transfer_engine": false,
"engine_info_bootstrap_port": 6789,
"modelexpress_config": null,
"download_dir": null,
"model_checksum": null,
"delete_ckpt_after_loading": false,
"decrypted_config_file": null,
"decrypted_draft_config_file": null,
"checkpoint_engine_wait_weights_before_ready": false,
"enable_prefill_delayer": true,
"prefill_delayer_max_delay_passes": 30,
"prefill_delayer_token_usage_low_watermark": 0.1,
"prefill_delayer_forward_passes_buckets": null,
"prefill_delayer_wait_seconds_buckets": null,
"prefill_delayer_queue_min_ratio": null,
"prefill_delayer_max_delay_ms": null,
"min_free_slots_delay": null,
"enable_deterministic_inference": false,
"rl_on_policy_target": null,
"kv_canary": "none",
"kv_canary_real_data": "none",
"kv_canary_sweep_interval": 0,
"enable_dynamic_batch_tokenizer": false,
"dynamic_batch_tokenizer_batch_size": 32,
"dynamic_batch_tokenizer_batch_timeout": 0.002,
"enable_tokenizer_batch_encode": false,
"disable_tokenizer_batch_decode": false,
"debug_tensor_dump_output_folder": null,
"debug_tensor_dump_layers": null,
"debug_tensor_dump_input_file": null,
"enable_memory_saver": false,
"enable_weights_cpu_backup": false,
"enable_draft_weights_cpu_backup": false,
"enable_custom_logit_processor": false,
"enable_return_hidden_states": false,
"enable_return_routed_experts": false,
"enable_return_indexer_topk": false,
"disable_outlines_disk_cache": false,
"enable_mis": false,
"forward_hooks": null,
"msprobe_dump_config": null,
"use_token_span": true,
"tml_mm_process_mode": false,
"tml_colocate_hydration": false,
"tml_rescale_image_frac": null,
"tml_rescale_image_max_upscaled_long_edge": null,
"async_detokenizer_send": false,
"remote_model_weights_path": null,
"enable_low_latency_kernels": false,
"enable_ll_moe_v2": true,
"mxfp8_kv_roundtrip": false,
"enable_prefill_cuda_graph": false,
"prefill_cuda_graph_sizes": [
1,
2,
4,
8,
12,
16,
24,
32,
40,
48,
56,
64,
96,
128
],
"prefill_cuda_graph_max_size": 128,
"decode_attention_num_splits": null,
"enable_dp_mixed_cuda_graph": false,
"mxfp8_moe_compute": false,
"ckpt_has_mxfp8_scales": false,
"nvfp4_use_calibrated_w2_input_scales": false,
"nvfp4_use_per_token_activation_scales": false,
"force_shared_experts_bf16": false,
"nvfp4_act_4over6": false,
"nvfp4_quantize_on_load": false,
"disable_sample_audio": false,
"use_moe_v2": false,
"tokenizer_batch_max_size": 1,
"tokenizer_batch_timeout_ms": 50,
"tokenizer_vocab_mask_size": 154856,
"use_tml_lora_backend": true,
"allow_full_non_token_span_fallback": true,
"audio_flow_max_steps": null,
"audio_flow_sampler_dtype": null,
"default_alter_topk": 0,
"min_prefill_batch_size": 0,
"max_prefill_skip_count": 10,
"send_continuous_mel": false,
"enable_moe_straggler_metrics": true,
"enable_gpu_process_marker": false,
"sconv_rms_preserving": null,
"dsconv_rms_preserving": null,
"enable_distributed_gumbel_sampling": false,
"coupled_reject_sampling": false,
"cg_verify_scheme": "shared_gumbel",
"_resolved_overrides": [
[
"_deepseek_family_overrides",
{
"page_size": 64
}
],
[
"_dsa_split_backend_resolution",
{
"dsa_prefill_backend": "trtllm",
"dsa_decode_backend": "trtllm"
}
],
[
"_moe_runner_fusion_disable",
{
"disable_shared_experts_fusion": true
}
],
[
"_speculative_moe_runner_default",
{
"speculative_moe_runner_backend": "flashinfer_trtllm"
}
]
],
"grpc_worker_threads": 4,
"_quantization_explicitly_unset": false,
"_cuda_graph_config_locked": [
[
"prefill",
"bs"
],
[
"decode",
"bs"
]
],
"_declarations_materialized": true,
"_in_override": false,
"tml_weight_version_state": {},
"_runtime_mutations": [
[
"model_runner.chunked_prefix_cache_gate",
{
"disable_chunked_prefix_cache": true
}
],
[
"scheduler.pp_max_micro_batch_size_default",
{
"pp_max_micro_batch_size": 256
}
]
],
"_mx_config_cache": {},
"max_speculative_num_draft_tokens": null,
"last_gen_throughput": 0.0,
"memory_usage": {
"weight": 127.43,
"kvcache": 119.74,
"token_capacity": 2328128,
"graph": 2.26
},
"effective_max_running_requests_per_dp": 256
},
{
"model_path": "gs://tml-data-projects-us-east4/base_models/hf_original/GLM-5.2-NVFP4",
"tokenizer_path": "zai-org/GLM-5.2",
"tokenizer_mode": "auto",
"tokenizer_backend": "huggingface",
"tokenizer_worker_num": 1,
"detokenizer_worker_num": 1,
"skip_tokenizer_init": false,
"load_format": "runai_streamer",
"model_loader_extra_config": "{}",
"trust_remote_code": true,
"context_length": null,
"is_embedding": false,
"enable_multimodal": null,
"revision": null,
"model_impl": "auto",
"model_config_parser": "auto",
"json_model_override_args": "{}",
"dtype": "auto",
"quantization": "modelopt_fp4",
"quantization_param_path": null,
"kv_cache_dtype": "fp8_e4m3",
"enable_fp32_lm_head": false,
"modelopt_quant": null,
"modelopt_checkpoint_restore_path": null,
"modelopt_checkpoint_save_path": null,
"modelopt_export_path": null,
"quantize_and_serve": false,
"rl_quant_profile": null,
"enable_tf32_matmul": false,
"mem_fraction_static": 0.9,
"max_running_requests": 1024,
"max_queued_requests": null,
"max_total_tokens": null,
"chunked_prefill_size": 4096,
"enable_dynamic_chunking": false,
"max_prefill_tokens": 16384,
"prefill_max_requests": 128,
"schedule_policy": "lpm",
"enable_priority_scheduling": false,
"disable_priority_preemption": false,
"default_priority_value": null,
"abort_on_priority_when_disabled": false,
"schedule_low_priority_values_first": false,
"priority_scheduling_preemption_threshold": 100,
"retraction_policy": "length",
"schedule_conservativeness": 0.18,
"page_size": 64,
"swa_full_tokens_ratio": 0.3,
"disable_hybrid_swa_memory": false,
"radix_eviction_policy": "lru",
"prefill_only_disable_kv_cache": false,
"disable_radix_cache": false,
"enable_page_major_kv_layout": false,
"enable_unified_memory": false,
"disable_chunked_prefix_cache": true,
"disable_overlap_schedule": false,
"num_continuous_decode_steps": 1,
"scheduler_recv_interval": 1,
"enable_mixed_chunk": false,
"nccl_port": null,
"dist_timeout": null,
"dist_init_addr": null,
"nnodes": 1,
"node_rank": 0,
"tp_size": 4,
"dcp_size": 1,
"pp_size": 1,
"pp_max_micro_batch_size": 256,
"pp_async_batch_depth": 0,
"dp_size": 4,
"load_balance_method": "round_robin",
"attn_cp_size": 1,
"moe_dp_size": 1,
"dwdp_size": 1,
"enable_prefill_cp": false,
"cp_strategy": null,
"enable_dsa_cache_layer_split": false,
"enable_dsa_prefill_context_parallel": false,
"dsa_prefill_cp_mode": "round-robin-split",
"enable_prefill_context_parallel": false,
"prefill_cp_mode": "in-seq-split",
"enable_dp_attention": true,
"enable_dp_attention_local_control_broadcast": false,
"enable_dp_lm_head": false,
"enable_attn_tp_input_scattered": false,
"disable_attn_tp_gather": false,
"enable_p2p_check": false,
"device": "cuda",
"base_gpu_id": 0,
"gpu_id_step": 1,
"random_seed": 42,
"watchdog_timeout": 300,
"soft_watchdog_timeout": null,
"sleep_on_idle": false,
"use_ray": false,
"numa_node": null,
"gc_threshold": null,
"host": "0.0.0.0",
"port": 30000,
"fastapi_root_path": "",
"smg_grpc_mode": false,
"grpc_mode": false,
"grpc_port": null,
"skip_server_warmup": false,
"warmups": null,
"enable_http2": false,
"ssl_keyfile": null,
"ssl_certfile": null,
"ssl_ca_certs": null,
"ssl_keyfile_password": null,
"enable_ssl_refresh": false,
"api_key": null,
"admin_api_key": null,
"served_model_name": "base_models/hf_original/GLM-5.2-NVFP4",
"weight_version": "default",
"chat_template": null,
"hf_chat_template_name": null,
"completion_template": null,
"file_storage_path": "sglang_storage",
"enable_cache_report": false,
"reasoning_parser": "glm45",
"default_chat_template_kwargs": null,
"strip_thinking_cache": false,
"enable_strict_thinking": false,
"tool_call_parser": "glm47",
"tool_server": null,
"sampling_defaults": "openai",
"asr_max_buffer_seconds": 60,
"asr_max_concurrent_sessions": 32,
"preferred_sampling_params": null,
"allow_auto_truncate": false,
"stream_interval": 8,
"batch_notify_size": 16,
"stream_response_default_include_usage": false,
"incremental_streaming_output": true,
"enable_streaming_session": true,
"enable_session_radix_cache": false,
"log_level": "info",
"log_level_http": "warning",
"log_requests": false,
"log_requests_level": 2,
"log_requests_format": "text",
"log_requests_target": null,
"uvicorn_access_log_exclude_prefixes": [],
"crash_dump_folder": null,
"show_time_cost": false,
"enable_metrics": true,
"smg_http_sidecar_port": null,
"enable_mfu_metrics": true,
"enable_metrics_for_all_schedulers": false,
"load_snapshot_publish_interval": 15,
"tokenizer_metrics_custom_labels_header": "x-custom-labels",
"tokenizer_metrics_allowed_custom_labels": null,
"extra_metric_labels": {
"slurm_job_id": "15538838",
"actor_label": "sampling",
"replica_id": "0"
},
"bucket_time_to_first_token": null,
"bucket_inter_token_latency": null,
"bucket_e2e_request_latency": null,
"prompt_tokens_buckets": null,
"generation_tokens_buckets": null,
"gc_warning_threshold_secs": 0.0,
"decode_log_interval": 40,
"enable_request_time_stats_logging": false,
"kv_events_config": null,
"enable_forward_pass_metrics": false,
"forward_pass_metrics_worker_id": "",
"forward_pass_metrics_ipc_name": null,
"enable_trace": false,
"trace_modules": "request",
"otlp_traces_endpoint": "172.22.6.219:4162",
"export_metrics_to_file": false,
"export_metrics_to_file_dir": null,
"stat_loggers": null,
"constrained_json_whitespace_pattern": null,
"constrained_json_disable_any_whitespace": false,
"attention_backend": "nsa",
"decode_attention_backend": null,
"prefill_attention_backend": null,
"sampling_backend": "pytorch",
"grammar_backend": "xgrammar",
"radix_cache_backend": null,
"mm_attention_backend": null,
"fp8_gemm_runner_backend": "auto",
"fp4_gemm_runner_backend": "flashinfer_trtllm",
"bf16_gemm_backend": "auto",
"dsa_prefill_backend": "trtllm",
"dsa_decode_backend": "trtllm",
"dsa_paged_mqa_logits_backend": "auto",
"dsa_topk_backend": "sgl-kernel",
"disable_flashinfer_autotune": false,
"mamba_backend": "triton",
"cuda_graph_config": {
"decode": {
"backend": "full",
"max_bs": 1024,
"bs": [
1,
2,
4,
8,
16,
32,
64,
128,
256,
384,
512,
640,
768,
896,
1024
],
"tc_compiler": "eager",
"full_prefill_max_req": null
},
"prefill": {
"backend": "disabled",
"max_bs": 2048,
"bs": [
8,
16,
32,
64,
128,
256,
512,
1024,
1536,
2048,
2560,
3072,
3584,
4096,
4608,
5120,
5632,
6144,
6656,
7168,
7680,
8192,
8704,
9216,
9728,
10240,
10752,
11264,
11776,
12288,
12800,
13312,
13824,
14336,
14848,
15360,
15872,
16384
],
"tc_compiler": "eager",
"full_prefill_max_req": null
}
},
"cuda_graph_backend_decode": null,
"cuda_graph_backend_prefill": null,
"cuda_graph_max_bs_decode": null,
"cuda_graph_max_bs_prefill": null,
"cuda_graph_bs_decode": [
1,
2,
4,
8,
16,
32,
64,
128,
256,
384,
512,
640,
768,
896,
1024
],
"cuda_graph_bs_prefill": [
8,
16,
32,
64,
128,
256,
512,
1024,
1536,
2048,
2560,
3072,
3584,
4096,
4608,
5120,
5632,
6144,
6656,
7168,
7680,
8192,
8704,
9216,
9728,
10240,
10752,
11264,
11776,
12288,
12800,
13312,
13824,
14336,
14848,
15360,
15872,
16384
],
"cuda_graph_tc_compiler": null,
"disable_prefill_cuda_graph": false,
"disable_decode_cuda_graph": false,
"disable_cuda_graph": false,
"disable_cuda_graph_padding": false,
"enable_profile_cuda_graph": false,
"enable_cudagraph_gc": false,
"debug_cuda_graph": false,
"enable_layerwise_nvtx_marker": false,
"enable_nccl_nvls": false,
"enable_symm_mem": false,
"triton_attention_reduce_in_fp32": false,
"triton_attention_num_kv_splits": 8,
"triton_attention_split_tile_size": null,
"flashinfer_mla_disable_ragged": false,
"enable_fused_qk_norm_rope": false,
"enable_precise_embedding_interpolation": false,
"enable_fused_moe_sum_all_reduce": false,
"enable_deepseek_v4_fp4_indexer": false,
"disable_custom_all_reduce": true,
"enable_mscclpp": false,
"enable_torch_symm_mem": false,
"enable_scattered_sconv": false,
"pre_warm_nccl": false,
"enable_quant_communications": false,
"enable_flashinfer_allreduce_fusion": false,
"enforce_disable_flashinfer_allreduce_fusion": false,
"flashinfer_allreduce_fusion_backend": null,
"enable_aiter_allreduce_fusion": false,
"enable_torch_compile": false,
"enable_torch_compile_debug_mode": false,
"torch_compile_max_bs": 32,
"torchao_config": "",
"speculative_algorithm": null,
"speculative_draft_model_path": null,
"speculative_draft_model_revision": null,
"speculative_draft_load_format": null,
"speculative_num_steps": null,
"speculative_eagle_topk": null,
"speculative_num_draft_tokens": null,
"speculative_dflash_block_size": null,
"speculative_dspark_block_size": null,
"speculative_dspark_sps_table_path": null,
"speculative_dspark_confidence_sts_path": null,
"speculative_dspark_align_verify_tokens_to_graph_tier": false,
"speculative_accept_threshold_single": 1.0,
"speculative_accept_threshold_acc": 1.0,
"speculative_use_rejection_sampling": false,
"speculative_token_map": null,
"speculative_attention_mode": "prefill",
"speculative_draft_attention_backend": null,
"speculative_draft_window_size": null,
"speculative_moe_runner_backend": "flashinfer_trtllm",
"speculative_moe_a2a_backend": null,
"speculative_draft_model_quantization": "modelopt_fp4",
"speculative_skip_dp_mlp_sync": false,
"enable_multi_layer_eagle": false,
"speculative_adaptive": false,
"speculative_adaptive_config": null,
"decoupled_spec_bind_endpoint": null,
"decoupled_spec_connect_endpoints": null,
"decoupled_spec_rank": null,
"decoupled_spec_role": "null",
"spec_trace_dir": null,
"speculative_ngram_min_bfs_breadth": 1,
"speculative_ngram_max_bfs_breadth": 10,
"speculative_ngram_match_type": "BFS",
"speculative_ngram_max_trie_depth": 18,
"speculative_ngram_capacity": 10000000,
"speculative_ngram_external_corpus_path": null,
"speculative_ngram_external_sam_budget": 0,
"speculative_ngram_external_corpus_max_tokens": 10000000,
"ep_size": 1,
"moe_a2a_backend": "none",
"moe_runner_backend": "flashinfer_trtllm",
"flashinfer_mxfp4_moe_precision": "default",
"deepep_mode": "auto",
"fuseep_mode": 2,
"deepep_dispatcher_output_dtype": "auto",
"ep_num_redundant_experts": 0,
"ep_dispatch_algorithm": "dynamic",
"init_expert_location": "trivial",
"enable_eplb": false,
"eplb_algorithm": "deepseek",
"eplb_rebalance_num_iterations": 1000,
"eplb_rebalance_layers_per_chunk": null,
"eplb_min_rebalancing_utilization_threshold": 1.0,
"expert_distribution_recorder_mode": null,
"expert_distribution_recorder_buffer_size": 1000,
"enable_expert_distribution_metrics": false,
"deepep_config": null,
"moe_dense_tp_size": null,
"elastic_ep_backend": null,
"enable_elastic_expert_backup": false,
"mooncake_ib_device": null,
"enable_waterfill": false,
"ep_join_mode": null,
"ep_join_rank_offset": 0,
"elastic_ep_initial_size": null,
"max_ep_size": null,
"elastic_ep_scale_timeout": 600,
"elastic_ep_rejoin": false,
"disable_flashinfer_cutlass_moe_fp4_allgather": false,
"disable_shared_experts_fusion": true,
"enforce_shared_experts_fusion": false,
"max_mamba_cache_size": null,
"mamba_ssm_dtype": null,
"enable_mamba_cache_stochastic_rounding": false,
"mamba_cache_philox_rounds": 0,
"mamba_full_memory_ratio": 0.3,
"mamba_radix_cache_strategy": "auto",
"uses_mamba_radix_cache": false,
"mamba_track_interval": 256,
"disable_finished_mamba_cache": false,
"enable_int8_mamba_checkpoint": false,
"int8_mamba_ckpt_size": null,
"linear_attn_backend": "triton",
"linear_attn_decode_backend": null,
"linear_attn_prefill_backend": null,
"enable_linear_replayssm": false,
"linear_replayssm_cache_len": 16,
"enable_gdn_replayssm_spec": false,
"enable_hierarchical_cache": false,
"hicache_ratio": 2.0,
"hicache_size": 0,
"hicache_write_policy": "write_through",
"hicache_io_backend": "kernel",
"hicache_mem_layout": "page_first",
"hicache_storage_backend": null,
"hicache_storage_prefetch_policy": "timeout",
"hicache_storage_backend_extra_config": null,
"enable_hisparse": false,
"hisparse_config": null,
"enable_broadcast_mm_inputs_process": false,
"enable_prefix_mm_cache": false,
"mm_enable_dp_encoder": false,
"mm_process_config": {},
"mm_processor_worker_num": 0,
"mm_io_worker_num": 0,
"limit_mm_data_per_request": null,
"enable_mm_global_cache": false,
"disable_fast_image_processor": false,
"mm_feature_transport": "cpu",
"keep_mm_feature_on_device": false,
"enable_lora": null,
"enable_lora_overlap_loading": null,
"max_lora_rank": 64,
"lora_target_modules": null,
"lora_paths": null,
"max_loaded_loras": null,
"max_loras_per_batch": 4,
"lora_eviction_policy": "lru",
"lora_backend": "triton",
"max_lora_chunk_size": 16,
"experts_shared_outer_loras": false,
"lora_use_virtual_experts": false,
"lora_strict_loading": true,
"lora_drain_wait_threshold": 0.0,
"enable_two_batch_overlap": false,
"enable_single_batch_overlap": false,
"tbo_token_distribution_threshold": 0.48,
"cpu_offload_gb": 0,
"offload_group_size": -1,
"offload_num_in_group": 1,
"offload_prefetch_step": 1,
"offload_mode": "cpu",
"enable_lmcache": false,
"lmcache_config_file": null,
"enable_flexkv": false,
"flexkv_config_file": null,
"kt_weight_path": null,
"kt_method": "AMXINT4",
"kt_cpuinfer": null,
"kt_threadpool_count": 2,
"kt_num_gpu_experts": null,
"kt_max_deferred_experts_per_token": null,
"dllm_algorithm": null,
"dllm_algorithm_config": null,
"dllm_fdfo": true,
"disaggregation_mode": "null",
"disaggregation_transfer_backend": "nixl",
"disaggregation_bootstrap_port": 30137,
"disaggregation_ib_device": null,
"disaggregation_decode_enable_radix_cache": false,
"disaggregation_decode_enable_offload_kvcache": false,
"num_reserved_decode_tokens": 512,
"disaggregation_decode_extra_slots": null,
"disaggregation_decode_polling_interval": 1,
"optimistic_prefill_attempts": 0,
"encoder_only": false,
"language_only": false,
"encoder_transfer_backend": "zmq_to_scheduler",
"encoder_urls": [],
"encoder_bootstrap_port": 8997,
"encoder_register_urls": [],
"enable_adaptive_dispatch_to_encoder": false,
"enable_pdmux": false,
"pdmux_config_path": null,
"sm_group_num": 8,
"custom_weight_loader": [],
"weight_loader_disable_mmap": false,
"weight_loader_prefetch_checkpoints": false,
"weight_loader_prefetch_num_threads": 4,
"weight_loader_drop_cache_after_load": false,
"remote_instance_weight_loader_seed_instance_ip": null,
"remote_instance_weight_loader_seed_instance_service_port": null,
"remote_instance_weight_loader_send_weights_group_ports": null,
"remote_instance_weight_loader_backend": "nccl",
"remote_instance_weight_loader_start_seed_via_transfer_engine": false,
"engine_info_bootstrap_port": 6789,
"modelexpress_config": null,
"download_dir": null,
"model_checksum": null,
"delete_ckpt_after_loading": false,
"decrypted_config_file": null,
"decrypted_draft_config_file": null,
"checkpoint_engine_wait_weights_before_ready": false,
"enable_prefill_delayer": true,
"prefill_delayer_max_delay_passes": 30,
"prefill_delayer_token_usage_low_watermark": 0.1,
"prefill_delayer_forward_passes_buckets": null,
"prefill_delayer_wait_seconds_buckets": null,
"prefill_delayer_queue_min_ratio": null,
"prefill_delayer_max_delay_ms": null,
"min_free_slots_delay": null,
"enable_deterministic_inference": false,
"rl_on_policy_target": null,
"kv_canary": "none",
"kv_canary_real_data": "none",
"kv_canary_sweep_interval": 0,
"enable_dynamic_batch_tokenizer": false,
"dynamic_batch_tokenizer_batch_size": 32,
"dynamic_batch_tokenizer_batch_timeout": 0.002,
"enable_tokenizer_batch_encode": false,
"disable_tokenizer_batch_decode": false,
"debug_tensor_dump_output_folder": null,
"debug_tensor_dump_layers": null,
"debug_tensor_dump_input_file": null,
"enable_memory_saver": false,
"enable_weights_cpu_backup": false,
"enable_draft_weights_cpu_backup": false,
"enable_custom_logit_processor": false,
"enable_return_hidden_states": false,
"enable_return_routed_experts": false,
"enable_return_indexer_topk": false,
"disable_outlines_disk_cache": false,
"enable_mis": false,
"forward_hooks": null,
"msprobe_dump_config": null,
"use_token_span": true,
"tml_mm_process_mode": false,
"tml_colocate_hydration": false,
"tml_rescale_image_frac": null,
"tml_rescale_image_max_upscaled_long_edge": null,
"async_detokenizer_send": false,
"remote_model_weights_path": null,
"enable_low_latency_kernels": false,
"enable_ll_moe_v2": true,
"mxfp8_kv_roundtrip": false,
"enable_prefill_cuda_graph": false,
"prefill_cuda_graph_sizes": [
1,
2,
4,
8,
12,
16,
24,
32,
40,
48,
56,
64,
96,
128
],
"prefill_cuda_graph_max_size": 128,
"decode_attention_num_splits": null,
"enable_dp_mixed_cuda_graph": false,
"mxfp8_moe_compute": false,
"ckpt_has_mxfp8_scales": false,
"nvfp4_use_calibrated_w2_input_scales": false,
"nvfp4_use_per_token_activation_scales": false,
"force_shared_experts_bf16": false,
"nvfp4_act_4over6": false,
"nvfp4_quantize_on_load": false,
"disable_sample_audio": false,
"use_moe_v2": false,
"tokenizer_batch_max_size": 1,
"tokenizer_batch_timeout_ms": 50,
"tokenizer_vocab_mask_size": 154856,
"use_tml_lora_backend": true,
"allow_full_non_token_span_fallback": true,
"audio_flow_max_steps": null,
"audio_flow_sampler_dtype": null,
"default_alter_topk": 0,
"min_prefill_batch_size": 0,
"max_prefill_skip_count": 10,
"send_continuous_mel": false,
"enable_moe_straggler_metrics": true,
"enable_gpu_process_marker": false,
"sconv_rms_preserving": null,
"dsconv_rms_preserving": null,
"enable_distributed_gumbel_sampling": false,
"coupled_reject_sampling": false,
"cg_verify_scheme": "shared_gumbel",
"_resolved_overrides": [
[
"_deepseek_family_overrides",
{
"page_size": 64
}
],
[
"_dsa_split_backend_resolution",
{
"dsa_prefill_backend": "trtllm",
"dsa_decode_backend": "trtllm"
}
],
[
"_moe_runner_fusion_disable",
{
"disable_shared_experts_fusion": true
}
],
[
"_speculative_moe_runner_default",
{
"speculative_moe_runner_backend": "flashinfer_trtllm"
}
]
],
"grpc_worker_threads": 4,
"_quantization_explicitly_unset": false,
"_cuda_graph_config_locked": [
[
"decode",
"bs"
],
[
"prefill",
"bs"
]
],
"_declarations_materialized": true,
"_in_override": false,
"tml_weight_version_state": {},
"_runtime_mutations": [
[
"model_runner.chunked_prefix_cache_gate",
{
"disable_chunked_prefix_cache": true
}
],
[
"scheduler.pp_max_micro_batch_size_default",
{
"pp_max_micro_batch_size": 256
}
]
],
"_mx_config_cache": {},
"max_speculative_num_draft_tokens": null,
"last_gen_throughput": 0.0,
"memory_usage": {
"weight": 127.43,
"kvcache": 119.74,
"token_capacity": 2328128,
"graph": 2.26
},
"effective_max_running_requests_per_dp": 256
},
{
"model_path": "gs://tml-data-projects-us-east4/base_models/hf_original/GLM-5.2-NVFP4",
"tokenizer_path": "zai-org/GLM-5.2",
"tokenizer_mode": "auto",
"tokenizer_backend": "huggingface",
"tokenizer_worker_num": 1,
"detokenizer_worker_num": 1,
"skip_tokenizer_init": false,
"load_format": "runai_streamer",
"model_loader_extra_config": "{}",
"trust_remote_code": true,
"context_length": null,
"is_embedding": false,
"enable_multimodal": null,
"revision": null,
"model_impl": "auto",
"model_config_parser": "auto",
"json_model_override_args": "{}",
"dtype": "auto",
"quantization": "modelopt_fp4",
"quantization_param_path": null,
"kv_cache_dtype": "fp8_e4m3",
"enable_fp32_lm_head": false,
"modelopt_quant": null,
"modelopt_checkpoint_restore_path": null,
"modelopt_checkpoint_save_path": null,
"modelopt_export_path": null,
"quantize_and_serve": false,
"rl_quant_profile": null,
"enable_tf32_matmul": false,
"mem_fraction_static": 0.9,
"max_running_requests": 1024,
"max_queued_requests": null,
"max_total_tokens": null,
"chunked_prefill_size": 4096,
"enable_dynamic_chunking": false,
"max_prefill_tokens": 16384,
"prefill_max_requests": 128,
"schedule_policy": "lpm",
"enable_priority_scheduling": false,
"disable_priority_preemption": false,
"default_priority_value": null,
"abort_on_priority_when_disabled": false,
"schedule_low_priority_values_first": false,
"priority_scheduling_preemption_threshold": 100,
"retraction_policy": "length",
"schedule_conservativeness": 0.18,
"page_size": 64,
"swa_full_tokens_ratio": 0.3,
"disable_hybrid_swa_memory": false,
"radix_eviction_policy": "lru",
"prefill_only_disable_kv_cache": false,
"disable_radix_cache": false,
"enable_page_major_kv_layout": false,
"enable_unified_memory": false,
"disable_chunked_prefix_cache": true,
"disable_overlap_schedule": false,
"num_continuous_decode_steps": 1,
"scheduler_recv_interval": 1,
"enable_mixed_chunk": false,
"nccl_port": null,
"dist_timeout": null,
"dist_init_addr": null,
"nnodes": 1,
"node_rank": 0,
"tp_size": 4,
"dcp_size": 1,
"pp_size": 1,
"pp_max_micro_batch_size": 256,
"pp_async_batch_depth": 0,
"dp_size": 4,
"load_balance_method": "round_robin",
"attn_cp_size": 1,
"moe_dp_size": 1,
"dwdp_size": 1,
"enable_prefill_cp": false,
"cp_strategy": null,
"enable_dsa_cache_layer_split": false,
"enable_dsa_prefill_context_parallel": false,
"dsa_prefill_cp_mode": "round-robin-split",
"enable_prefill_context_parallel": false,
"prefill_cp_mode": "in-seq-split",
"enable_dp_attention": true,
"enable_dp_attention_local_control_broadcast": false,
"enable_dp_lm_head": false,
"enable_attn_tp_input_scattered": false,
"disable_attn_tp_gather": false,
"enable_p2p_check": false,
"device": "cuda",
"base_gpu_id": 0,
"gpu_id_step": 1,
"random_seed": 42,
"watchdog_timeout": 300,
"soft_watchdog_timeout": null,
"sleep_on_idle": false,
"use_ray": false,
"numa_node": null,
"gc_threshold": null,
"host": "0.0.0.0",
"port": 30000,
"fastapi_root_path": "",
"smg_grpc_mode": false,
"grpc_mode": false,
"grpc_port": null,
"skip_server_warmup": false,
"warmups": null,
"enable_http2": false,
"ssl_keyfile": null,
"ssl_certfile": null,
"ssl_ca_certs": null,
"ssl_keyfile_password": null,
"enable_ssl_refresh": false,
"api_key": null,
"admin_api_key": null,
"served_model_name": "base_models/hf_original/GLM-5.2-NVFP4",
"weight_version": "default",
"chat_template": null,
"hf_chat_template_name": null,
"completion_template": null,
"file_storage_path": "sglang_storage",
"enable_cache_report": false,
"reasoning_parser": "glm45",
"default_chat_template_kwargs": null,
"strip_thinking_cache": false,
"enable_strict_thinking": false,
"tool_call_parser": "glm47",
"tool_server": null,
"sampling_defaults": "openai",
"asr_max_buffer_seconds": 60,
"asr_max_concurrent_sessions": 32,
"preferred_sampling_params": null,
"allow_auto_truncate": false,
"stream_interval": 8,
"batch_notify_size": 16,
"stream_response_default_include_usage": false,
"incremental_streaming_output": true,
"enable_streaming_session": true,
"enable_session_radix_cache": false,
"log_level": "info",
"log_level_http": "warning",
"log_requests": false,
"log_requests_level": 2,
"log_requests_format": "text",
"log_requests_target": null,
"uvicorn_access_log_exclude_prefixes": [],
"crash_dump_folder": null,
"show_time_cost": false,
"enable_metrics": true,
"smg_http_sidecar_port": null,
"enable_mfu_metrics": true,
"enable_metrics_for_all_schedulers": false,
"load_snapshot_publish_interval": 15,
"tokenizer_metrics_custom_labels_header": "x-custom-labels",
"tokenizer_metrics_allowed_custom_labels": null,
"extra_metric_labels": {
"slurm_job_id": "15538838",
"actor_label": "sampling",
"replica_id": "0"
},
"bucket_time_to_first_token": null,
"bucket_inter_token_latency": null,
"bucket_e2e_request_latency": null,
"prompt_tokens_buckets": null,
"generation_tokens_buckets": null,
"gc_warning_threshold_secs": 0.0,
"decode_log_interval": 40,
"enable_request_time_stats_logging": false,
"kv_events_config": null,
"enable_forward_pass_metrics": false,
"forward_pass_metrics_worker_id": "",
"forward_pass_metrics_ipc_name": null,
"enable_trace": false,
"trace_modules": "request",
"otlp_traces_endpoint": "172.22.6.219:4162",
"export_metrics_to_file": false,
"export_metrics_to_file_dir": null,
"stat_loggers": null,
"constrained_json_whitespace_pattern": null,
"constrained_json_disable_any_whitespace": false,
"attention_backend": "nsa",
"decode_attention_backend": null,
"prefill_attention_backend": null,
"sampling_backend": "pytorch",
"grammar_backend": "xgrammar",
"radix_cache_backend": null,
"mm_attention_backend": null,
"fp8_gemm_runner_backend": "auto",
"fp4_gemm_runner_backend": "flashinfer_trtllm",
"bf16_gemm_backend": "auto",
"dsa_prefill_backend": "trtllm",
"dsa_decode_backend": "trtllm",
"dsa_paged_mqa_logits_backend": "auto",
"dsa_topk_backend": "sgl-kernel",
"disable_flashinfer_autotune": false,
"mamba_backend": "triton",
"cuda_graph_config": {
"decode": {
"backend": "full",
"max_bs": 1024,
"bs": [
1,
2,
4,
8,
16,
32,
64,
128,
256,
384,
512,
640,
768,
896,
1024
],
"tc_compiler": "eager",
"full_prefill_max_req": null
},
"prefill": {
"backend": "disabled",
"max_bs": 2048,
"bs": [
8,
16,
32,
64,
128,
256,
512,
1024,
1536,
2048,
2560,
3072,
3584,
4096,
4608,
5120,
5632,
6144,
6656,
7168,
7680,
8192,
8704,
9216,
9728,
10240,
10752,
11264,
11776,
12288,
12800,
13312,
13824,
14336,
14848,
15360,
15872,
16384
],
"tc_compiler": "eager",
"full_prefill_max_req": null
}
},
"cuda_graph_backend_decode": null,
"cuda_graph_backend_prefill": null,
"cuda_graph_max_bs_decode": null,
"cuda_graph_max_bs_prefill": null,
"cuda_graph_bs_decode": [
1,
2,
4,
8,
16,
32,
64,
128,
256,
384,
512,
640,
768,
896,
1024
],
"cuda_graph_bs_prefill": [
8,
16,
32,
64,
128,
256,
512,
1024,
1536,
2048,
2560,
3072,
3584,
4096,
4608,
5120,
5632,
6144,
6656,
7168,
7680,
8192,
8704,
9216,
9728,
10240,
10752,
11264,
11776,
12288,
12800,
13312,
13824,
14336,
14848,
15360,
15872,
16384
],
"cuda_graph_tc_compiler": null,
"disable_prefill_cuda_graph": false,
"disable_decode_cuda_graph": false,
"disable_cuda_graph": false,
"disable_cuda_graph_padding": false,
"enable_profile_cuda_graph": false,
"enable_cudagraph_gc": false,
"debug_cuda_graph": false,
"enable_layerwise_nvtx_marker": false,
"enable_nccl_nvls": false,
"enable_symm_mem": false,
"triton_attention_reduce_in_fp32": false,
"triton_attention_num_kv_splits": 8,
"triton_attention_split_tile_size": null,
"flashinfer_mla_disable_ragged": false,
"enable_fused_qk_norm_rope": false,
"enable_precise_embedding_interpolation": false,
"enable_fused_moe_sum_all_reduce": false,
"enable_deepseek_v4_fp4_indexer": false,
"disable_custom_all_reduce": true,
"enable_mscclpp": false,
"enable_torch_symm_mem": false,
"enable_scattered_sconv": false,
"pre_warm_nccl": false,
"enable_quant_communications": false,
"enable_flashinfer_allreduce_fusion": false,
"enforce_disable_flashinfer_allreduce_fusion": false,
"flashinfer_allreduce_fusion_backend": null,
"enable_aiter_allreduce_fusion": false,
"enable_torch_compile": false,
"enable_torch_compile_debug_mode": false,
"torch_compile_max_bs": 32,
"torchao_config": "",
"speculative_algorithm": null,
"speculative_draft_model_path": null,
"speculative_draft_model_revision": null,
"speculative_draft_load_format": null,
"speculative_num_steps": null,
"speculative_eagle_topk": null,
"speculative_num_draft_tokens": null,
"speculative_dflash_block_size": null,
"speculative_dspark_block_size": null,
"speculative_dspark_sps_table_path": null,
"speculative_dspark_confidence_sts_path": null,
"speculative_dspark_align_verify_tokens_to_graph_tier": false,
"speculative_accept_threshold_single": 1.0,
"speculative_accept_threshold_acc": 1.0,
"speculative_use_rejection_sampling": false,
"speculative_token_map": null,
"speculative_attention_mode": "prefill",
"speculative_draft_attention_backend": null,
"speculative_draft_window_size": null,
"speculative_moe_runner_backend": "flashinfer_trtllm",
"speculative_moe_a2a_backend": null,
"speculative_draft_model_quantization": "modelopt_fp4",
"speculative_skip_dp_mlp_sync": false,
"enable_multi_layer_eagle": false,
"speculative_adaptive": false,
"speculative_adaptive_config": null,
"decoupled_spec_bind_endpoint": null,
"decoupled_spec_connect_endpoints": null,
"decoupled_spec_rank": null,
"decoupled_spec_role": "null",
"spec_trace_dir": null,
"speculative_ngram_min_bfs_breadth": 1,
"speculative_ngram_max_bfs_breadth": 10,
"speculative_ngram_match_type": "BFS",
"speculative_ngram_max_trie_depth": 18,
"speculative_ngram_capacity": 10000000,
"speculative_ngram_external_corpus_path": null,
"speculative_ngram_external_sam_budget": 0,
"speculative_ngram_external_corpus_max_tokens": 10000000,
"ep_size": 1,
"moe_a2a_backend": "none",
"moe_runner_backend": "flashinfer_trtllm",
"flashinfer_mxfp4_moe_precision": "default",
"deepep_mode": "auto",
"fuseep_mode": 2,
"deepep_dispatcher_output_dtype": "auto",
"ep_num_redundant_experts": 0,
"ep_dispatch_algorithm": "dynamic",
"init_expert_location": "trivial",
"enable_eplb": false,
"eplb_algorithm": "deepseek",
"eplb_rebalance_num_iterations": 1000,
"eplb_rebalance_layers_per_chunk": null,
"eplb_min_rebalancing_utilization_threshold": 1.0,
"expert_distribution_recorder_mode": null,
"expert_distribution_recorder_buffer_size": 1000,
"enable_expert_distribution_metrics": false,
"deepep_config": null,
"moe_dense_tp_size": null,
"elastic_ep_backend": null,
"enable_elastic_expert_backup": false,
"mooncake_ib_device": null,
"enable_waterfill": false,
"ep_join_mode": null,
"ep_join_rank_offset": 0,
"elastic_ep_initial_size": null,
"max_ep_size": null,
"elastic_ep_scale_timeout": 600,
"elastic_ep_rejoin": false,
"disable_flashinfer_cutlass_moe_fp4_allgather": false,
"disable_shared_experts_fusion": true,
"enforce_shared_experts_fusion": false,
"max_mamba_cache_size": null,
"mamba_ssm_dtype": null,
"enable_mamba_cache_stochastic_rounding": false,
"mamba_cache_philox_rounds": 0,
"mamba_full_memory_ratio": 0.3,
"mamba_radix_cache_strategy": "auto",
"uses_mamba_radix_cache": false,
"mamba_track_interval": 256,
"disable_finished_mamba_cache": false,
"enable_int8_mamba_checkpoint": false,
"int8_mamba_ckpt_size": null,
"linear_attn_backend": "triton",
"linear_attn_decode_backend": null,
"linear_attn_prefill_backend": null,
"enable_linear_replayssm": false,
"linear_replayssm_cache_len": 16,
"enable_gdn_replayssm_spec": false,
"enable_hierarchical_cache": false,
"hicache_ratio": 2.0,
"hicache_size": 0,
"hicache_write_policy": "write_through",
"hicache_io_backend": "kernel",
"hicache_mem_layout": "page_first",
"hicache_storage_backend": null,
"hicache_storage_prefetch_policy": "timeout",
"hicache_storage_backend_extra_config": null,
"enable_hisparse": false,
"hisparse_config": null,
"enable_broadcast_mm_inputs_process": false,
"enable_prefix_mm_cache": false,
"mm_enable_dp_encoder": false,
"mm_process_config": {},
"mm_processor_worker_num": 0,
"mm_io_worker_num": 0,
"limit_mm_data_per_request": null,
"enable_mm_global_cache": false,
"disable_fast_image_processor": false,
"mm_feature_transport": "cpu",
"keep_mm_feature_on_device": false,
"enable_lora": null,
"enable_lora_overlap_loading": null,
"max_lora_rank": 64,
"lora_target_modules": null,
"lora_paths": null,
"max_loaded_loras": null,
"max_loras_per_batch": 4,
"lora_eviction_policy": "lru",
"lora_backend": "triton",
"max_lora_chunk_size": 16,
"experts_shared_outer_loras": false,
"lora_use_virtual_experts": false,
"lora_strict_loading": true,
"lora_drain_wait_threshold": 0.0,
"enable_two_batch_overlap": false,
"enable_single_batch_overlap": false,
"tbo_token_distribution_threshold": 0.48,
"cpu_offload_gb": 0,
"offload_group_size": -1,
"offload_num_in_group": 1,
"offload_prefetch_step": 1,
"offload_mode": "cpu",
"enable_lmcache": false,
"lmcache_config_file": null,
"enable_flexkv": false,
"flexkv_config_file": null,
"kt_weight_path": null,
"kt_method": "AMXINT4",
"kt_cpuinfer": null,
"kt_threadpool_count": 2,
"kt_num_gpu_experts": null,
"kt_max_deferred_experts_per_token": null,
"dllm_algorithm": null,
"dllm_algorithm_config": null,
"dllm_fdfo": true,
"disaggregation_mode": "null",
"disaggregation_transfer_backend": "nixl",
"disaggregation_bootstrap_port": 30137,
"disaggregation_ib_device": null,
"disaggregation_decode_enable_radix_cache": false,
"disaggregation_decode_enable_offload_kvcache": false,
"num_reserved_decode_tokens": 512,
"disaggregation_decode_extra_slots": null,
"disaggregation_decode_polling_interval": 1,
"optimistic_prefill_attempts": 0,
"encoder_only": false,
"language_only": false,
"encoder_transfer_backend": "zmq_to_scheduler",
"encoder_urls": [],
"encoder_bootstrap_port": 8997,
"encoder_register_urls": [],
"enable_adaptive_dispatch_to_encoder": false,
"enable_pdmux": false,
"pdmux_config_path": null,
"sm_group_num": 8,
"custom_weight_loader": [],
"weight_loader_disable_mmap": false,
"weight_loader_prefetch_checkpoints": false,
"weight_loader_prefetch_num_threads": 4,
"weight_loader_drop_cache_after_load": false,
"remote_instance_weight_loader_seed_instance_ip": null,
"remote_instance_weight_loader_seed_instance_service_port": null,
"remote_instance_weight_loader_send_weights_group_ports": null,
"remote_instance_weight_loader_backend": "nccl",
"remote_instance_weight_loader_start_seed_via_transfer_engine": false,
"engine_info_bootstrap_port": 6789,
"modelexpress_config": null,
"download_dir": null,
"model_checksum": null,
"delete_ckpt_after_loading": false,
"decrypted_config_file": null,
"decrypted_draft_config_file": null,
"checkpoint_engine_wait_weights_before_ready": false,
"enable_prefill_delayer": true,
"prefill_delayer_max_delay_passes": 30,
"prefill_delayer_token_usage_low_watermark": 0.1,
"prefill_delayer_forward_passes_buckets": null,
"prefill_delayer_wait_seconds_buckets": null,
"prefill_delayer_queue_min_ratio": null,
"prefill_delayer_max_delay_ms": null,
"min_free_slots_delay": null,
"enable_deterministic_inference": false,
"rl_on_policy_target": null,
"kv_canary": "none",
"kv_canary_real_data": "none",
"kv_canary_sweep_interval": 0,
"enable_dynamic_batch_tokenizer": false,
"dynamic_batch_tokenizer_batch_size": 32,
"dynamic_batch_tokenizer_batch_timeout": 0.002,
"enable_tokenizer_batch_encode": false,
"disable_tokenizer_batch_decode": false,
"debug_tensor_dump_output_folder": null,
"debug_tensor_dump_layers": null,
"debug_tensor_dump_input_file": null,
"enable_memory_saver": false,
"enable_weights_cpu_backup": false,
"enable_draft_weights_cpu_backup": false,
"enable_custom_logit_processor": false,
"enable_return_hidden_states": false,
"enable_return_routed_experts": false,
"enable_return_indexer_topk": false,
"disable_outlines_disk_cache": false,
"enable_mis": false,
"forward_hooks": null,
"msprobe_dump_config": null,
"use_token_span": true,
"tml_mm_process_mode": false,
"tml_colocate_hydration": false,
"tml_rescale_image_frac": null,
"tml_rescale_image_max_upscaled_long_edge": null,
"async_detokenizer_send": false,
"remote_model_weights_path": null,
"enable_low_latency_kernels": false,
"enable_ll_moe_v2": true,
"mxfp8_kv_roundtrip": false,
"enable_prefill_cuda_graph": false,
"prefill_cuda_graph_sizes": [
1,
2,
4,
8,
12,
16,
24,
32,
40,
48,
56,
64,
96,
128
],
"prefill_cuda_graph_max_size": 128,
"decode_attention_num_splits": null,
"enable_dp_mixed_cuda_graph": false,
"mxfp8_moe_compute": false,
"ckpt_has_mxfp8_scales": false,
"nvfp4_use_calibrated_w2_input_scales": false,
"nvfp4_use_per_token_activation_scales": false,
"force_shared_experts_bf16": false,
"nvfp4_act_4over6": false,
"nvfp4_quantize_on_load": false,
"disable_sample_audio": false,
"use_moe_v2": false,
"tokenizer_batch_max_size": 1,
"tokenizer_batch_timeout_ms": 50,
"tokenizer_vocab_mask_size": 154856,
"use_tml_lora_backend": true,
"allow_full_non_token_span_fallback": true,
"audio_flow_max_steps": null,
"audio_flow_sampler_dtype": null,
"default_alter_topk": 0,
"min_prefill_batch_size": 0,
"max_prefill_skip_count": 10,
"send_continuous_mel": false,
"enable_moe_straggler_metrics": true,
"enable_gpu_process_marker": false,
"sconv_rms_preserving": null,
"dsconv_rms_preserving": null,
"enable_distributed_gumbel_sampling": false,
"coupled_reject_sampling": false,
"cg_verify_scheme": "shared_gumbel",
"_resolved_overrides": [
[
"_deepseek_family_overrides",
{
"page_size": 64
}
],
[
"_dsa_split_backend_resolution",
{
"dsa_prefill_backend": "trtllm",
"dsa_decode_backend": "trtllm"
}
],
[
"_moe_runner_fusion_disable",
{
"disable_shared_experts_fusion": true
}
],
[
"_speculative_moe_runner_default",
{
"speculative_moe_runner_backend": "flashinfer_trtllm"
}
]
],
"grpc_worker_threads": 4,
"_quantization_explicitly_unset": false,
"_cuda_graph_config_locked": [
[
"decode",
"bs"
],
[
"prefill",
"bs"
]
],
"_declarations_materialized": true,
"_in_override": false,
"tml_weight_version_state": {},
"_runtime_mutations": [
[
"model_runner.chunked_prefix_cache_gate",
{
"disable_chunked_prefix_cache": true
}
],
[
"scheduler.pp_max_micro_batch_size_default",
{
"pp_max_micro_batch_size": 256
}
]
],
"_mx_config_cache": {},
"max_speculative_num_draft_tokens": null,
"last_gen_throughput": 0.0,
"memory_usage": {
"weight": 127.43,
"kvcache": 119.74,
"token_capacity": 2328128,
"graph": 2.26
},
"effective_max_running_requests_per_dp": 256
},
{
"model_path": "gs://tml-data-projects-us-east4/base_models/hf_original/GLM-5.2-NVFP4",
"tokenizer_path": "zai-org/GLM-5.2",
"tokenizer_mode": "auto",
"tokenizer_backend": "huggingface",
"tokenizer_worker_num": 1,
"detokenizer_worker_num": 1,
"skip_tokenizer_init": false,
"load_format": "runai_streamer",
"model_loader_extra_config": "{}",
"trust_remote_code": true,
"context_length": null,
"is_embedding": false,
"enable_multimodal": null,
"revision": null,
"model_impl": "auto",
"model_config_parser": "auto",
"json_model_override_args": "{}",
"dtype": "auto",
"quantization": "modelopt_fp4",
"quantization_param_path": null,
"kv_cache_dtype": "fp8_e4m3",
"enable_fp32_lm_head": false,
"modelopt_quant": null,
"modelopt_checkpoint_restore_path": null,
"modelopt_checkpoint_save_path": null,
"modelopt_export_path": null,
"quantize_and_serve": false,
"rl_quant_profile": null,
"enable_tf32_matmul": false,
"mem_fraction_static": 0.9,
"max_running_requests": 1024,
"max_queued_requests": null,
"max_total_tokens": null,
"chunked_prefill_size": 4096,
"enable_dynamic_chunking": false,
"max_prefill_tokens": 16384,
"prefill_max_requests": 128,
"schedule_policy": "lpm",
"enable_priority_scheduling": false,
"disable_priority_preemption": false,
"default_priority_value": null,
"abort_on_priority_when_disabled": false,
"schedule_low_priority_values_first": false,
"priority_scheduling_preemption_threshold": 100,
"retraction_policy": "length",
"schedule_conservativeness": 0.18,
"page_size": 64,
"swa_full_tokens_ratio": 0.3,
"disable_hybrid_swa_memory": false,
"radix_eviction_policy": "lru",
"prefill_only_disable_kv_cache": false,
"disable_radix_cache": false,
"enable_page_major_kv_layout": false,
"enable_unified_memory": false,
"disable_chunked_prefix_cache": true,
"disable_overlap_schedule": false,
"num_continuous_decode_steps": 1,
"scheduler_recv_interval": 1,
"enable_mixed_chunk": false,
"nccl_port": null,
"dist_timeout": null,
"dist_init_addr": null,
"nnodes": 1,
"node_rank": 0,
"tp_size": 4,
"dcp_size": 1,
"pp_size": 1,
"pp_max_micro_batch_size": 256,
"pp_async_batch_depth": 0,
"dp_size": 4,
"load_balance_method": "round_robin",
"attn_cp_size": 1,
"moe_dp_size": 1,
"dwdp_size": 1,
"enable_prefill_cp": false,
"cp_strategy": null,
"enable_dsa_cache_layer_split": false,
"enable_dsa_prefill_context_parallel": false,
"dsa_prefill_cp_mode": "round-robin-split",
"enable_prefill_context_parallel": false,
"prefill_cp_mode": "in-seq-split",
"enable_dp_attention": true,
"enable_dp_attention_local_control_broadcast": false,
"enable_dp_lm_head": false,
"enable_attn_tp_input_scattered": false,
"disable_attn_tp_gather": false,
"enable_p2p_check": false,
"device": "cuda",
"base_gpu_id": 0,
"gpu_id_step": 1,
"random_seed": 42,
"watchdog_timeout": 300,
"soft_watchdog_timeout": null,
"sleep_on_idle": false,
"use_ray": false,
"numa_node": null,
"gc_threshold": null,
"host": "0.0.0.0",
"port": 30000,
"fastapi_root_path": "",
"smg_grpc_mode": false,
"grpc_mode": false,
"grpc_port": null,
"skip_server_warmup": false,
"warmups": null,
"enable_http2": false,
"ssl_keyfile": null,
"ssl_certfile": null,
"ssl_ca_certs": null,
"ssl_keyfile_password": null,
"enable_ssl_refresh": false,
"api_key": null,
"admin_api_key": null,
"served_model_name": "base_models/hf_original/GLM-5.2-NVFP4",
"weight_version": "default",
"chat_template": null,
"hf_chat_template_name": null,
"completion_template": null,
"file_storage_path": "sglang_storage",
"enable_cache_report": false,
"reasoning_parser": "glm45",
"default_chat_template_kwargs": null,
"strip_thinking_cache": false,
"enable_strict_thinking": false,
"tool_call_parser": "glm47",
"tool_server": null,
"sampling_defaults": "openai",
"asr_max_buffer_seconds": 60,
"asr_max_concurrent_sessions": 32,
"preferred_sampling_params": null,
"allow_auto_truncate": false,
"stream_interval": 8,
"batch_notify_size": 16,
"stream_response_default_include_usage": false,
"incremental_streaming_output": true,
"enable_streaming_session": true,
"enable_session_radix_cache": false,
"log_level": "info",
"log_level_http": "warning",
"log_requests": false,
"log_requests_level": 2,
"log_requests_format": "text",
"log_requests_target": null,
"uvicorn_access_log_exclude_prefixes": [],
"crash_dump_folder": null,
"show_time_cost": false,
"enable_metrics": true,
"smg_http_sidecar_port": null,
"enable_mfu_metrics": true,
"enable_metrics_for_all_schedulers": false,
"load_snapshot_publish_interval": 15,
"tokenizer_metrics_custom_labels_header": "x-custom-labels",
"tokenizer_metrics_allowed_custom_labels": null,
"extra_metric_labels": {
"slurm_job_id": "15538838",
"actor_label": "sampling",
"replica_id": "0"
},
"bucket_time_to_first_token": null,
"bucket_inter_token_latency": null,
"bucket_e2e_request_latency": null,
"prompt_tokens_buckets": null,
"generation_tokens_buckets": null,
"gc_warning_threshold_secs": 0.0,
"decode_log_interval": 40,
"enable_request_time_stats_logging": false,
"kv_events_config": null,
"enable_forward_pass_metrics": false,
"forward_pass_metrics_worker_id": "",
"forward_pass_metrics_ipc_name": null,
"enable_trace": false,
"trace_modules": "request",
"otlp_traces_endpoint": "172.22.6.219:4162",
"export_metrics_to_file": false,
"export_metrics_to_file_dir": null,
"stat_loggers": null,
"constrained_json_whitespace_pattern": null,
"constrained_json_disable_any_whitespace": false,
"attention_backend": "nsa",
"decode_attention_backend": null,
"prefill_attention_backend": null,
"sampling_backend": "pytorch",
"grammar_backend": "xgrammar",
"radix_cache_backend": null,
"mm_attention_backend": null,
"fp8_gemm_runner_backend": "auto",
"fp4_gemm_runner_backend": "flashinfer_trtllm",
"bf16_gemm_backend": "auto",
"dsa_prefill_backend": "trtllm",
"dsa_decode_backend": "trtllm",
"dsa_paged_mqa_logits_backend": "auto",
"dsa_topk_backend": "sgl-kernel",
"disable_flashinfer_autotune": false,
"mamba_backend": "triton",
"cuda_graph_config": {
"decode": {
"backend": "full",
"max_bs": 1024,
"bs": [
1,
2,
4,
8,
16,
32,
64,
128,
256,
384,
512,
640,
768,
896,
1024
],
"tc_compiler": "eager",
"full_prefill_max_req": null
},
"prefill": {
"backend": "disabled",
"max_bs": 2048,
"bs": [
8,
16,
32,
64,
128,
256,
512,
1024,
1536,
2048,
2560,
3072,
3584,
4096,
4608,
5120,
5632,
6144,
6656,
7168,
7680,
8192,
8704,
9216,
9728,
10240,
10752,
11264,
11776,
12288,
12800,
13312,
13824,
14336,
14848,
15360,
15872,
16384
],
"tc_compiler": "eager",
"full_prefill_max_req": null
}
},
"cuda_graph_backend_decode": null,
"cuda_graph_backend_prefill": null,
"cuda_graph_max_bs_decode": null,
"cuda_graph_max_bs_prefill": null,
"cuda_graph_bs_decode": [
1,
2,
4,
8,
16,
32,
64,
128,
256,
384,
512,
640,
768,
896,
1024
],
"cuda_graph_bs_prefill": [
8,
16,
32,
64,
128,
256,
512,
1024,
1536,
2048,
2560,
3072,
3584,
4096,
4608,
5120,
5632,
6144,
6656,
7168,
7680,
8192,
8704,
9216,
9728,
10240,
10752,
11264,
11776,
12288,
12800,
13312,
13824,
14336,
14848,
15360,
15872,
16384
],
"cuda_graph_tc_compiler": null,
"disable_prefill_cuda_graph": false,
"disable_decode_cuda_graph": false,
"disable_cuda_graph": false,
"disable_cuda_graph_padding": false,
"enable_profile_cuda_graph": false,
"enable_cudagraph_gc": false,
"debug_cuda_graph": false,
"enable_layerwise_nvtx_marker": false,
"enable_nccl_nvls": false,
"enable_symm_mem": false,
"triton_attention_reduce_in_fp32": false,
"triton_attention_num_kv_splits": 8,
"triton_attention_split_tile_size": null,
"flashinfer_mla_disable_ragged": false,
"enable_fused_qk_norm_rope": false,
"enable_precise_embedding_interpolation": false,
"enable_fused_moe_sum_all_reduce": false,
"enable_deepseek_v4_fp4_indexer": false,
"disable_custom_all_reduce": true,
"enable_mscclpp": false,
"enable_torch_symm_mem": false,
"enable_scattered_sconv": false,
"pre_warm_nccl": false,
"enable_quant_communications": false,
"enable_flashinfer_allreduce_fusion": false,
"enforce_disable_flashinfer_allreduce_fusion": false,
"flashinfer_allreduce_fusion_backend": null,
"enable_aiter_allreduce_fusion": false,
"enable_torch_compile": false,
"enable_torch_compile_debug_mode": false,
"torch_compile_max_bs": 32,
"torchao_config": "",
"speculative_algorithm": null,
"speculative_draft_model_path": null,
"speculative_draft_model_revision": null,
"speculative_draft_load_format": null,
"speculative_num_steps": null,
"speculative_eagle_topk": null,
"speculative_num_draft_tokens": null,
"speculative_dflash_block_size": null,
"speculative_dspark_block_size": null,
"speculative_dspark_sps_table_path": null,
"speculative_dspark_confidence_sts_path": null,
"speculative_dspark_align_verify_tokens_to_graph_tier": false,
"speculative_accept_threshold_single": 1.0,
"speculative_accept_threshold_acc": 1.0,
"speculative_use_rejection_sampling": false,
"speculative_token_map": null,
"speculative_attention_mode": "prefill",
"speculative_draft_attention_backend": null,
"speculative_draft_window_size": null,
"speculative_moe_runner_backend": "flashinfer_trtllm",
"speculative_moe_a2a_backend": null,
"speculative_draft_model_quantization": "modelopt_fp4",
"speculative_skip_dp_mlp_sync": false,
"enable_multi_layer_eagle": false,
"speculative_adaptive": false,
"speculative_adaptive_config": null,
"decoupled_spec_bind_endpoint": null,
"decoupled_spec_connect_endpoints": null,
"decoupled_spec_rank": null,
"decoupled_spec_role": "null",
"spec_trace_dir": null,
"speculative_ngram_min_bfs_breadth": 1,
"speculative_ngram_max_bfs_breadth": 10,
"speculative_ngram_match_type": "BFS",
"speculative_ngram_max_trie_depth": 18,
"speculative_ngram_capacity": 10000000,
"speculative_ngram_external_corpus_path": null,
"speculative_ngram_external_sam_budget": 0,
"speculative_ngram_external_corpus_max_tokens": 10000000,
"ep_size": 1,
"moe_a2a_backend": "none",
"moe_runner_backend": "flashinfer_trtllm",
"flashinfer_mxfp4_moe_precision": "default",
"deepep_mode": "auto",
"fuseep_mode": 2,
"deepep_dispatcher_output_dtype": "auto",
"ep_num_redundant_experts": 0,
"ep_dispatch_algorithm": "dynamic",
"init_expert_location": "trivial",
"enable_eplb": false,
"eplb_algorithm": "deepseek",
"eplb_rebalance_num_iterations": 1000,
"eplb_rebalance_layers_per_chunk": null,
"eplb_min_rebalancing_utilization_threshold": 1.0,
"expert_distribution_recorder_mode": null,
"expert_distribution_recorder_buffer_size": 1000,
"enable_expert_distribution_metrics": false,
"deepep_config": null,
"moe_dense_tp_size": null,
"elastic_ep_backend": null,
"enable_elastic_expert_backup": false,
"mooncake_ib_device": null,
"enable_waterfill": false,
"ep_join_mode": null,
"ep_join_rank_offset": 0,
"elastic_ep_initial_size": null,
"max_ep_size": null,
"elastic_ep_scale_timeout": 600,
"elastic_ep_rejoin": false,
"disable_flashinfer_cutlass_moe_fp4_allgather": false,
"disable_shared_experts_fusion": true,
"enforce_shared_experts_fusion": false,
"max_mamba_cache_size": null,
"mamba_ssm_dtype": null,
"enable_mamba_cache_stochastic_rounding": false,
"mamba_cache_philox_rounds": 0,
"mamba_full_memory_ratio": 0.3,
"mamba_radix_cache_strategy": "auto",
"uses_mamba_radix_cache": false,
"mamba_track_interval": 256,
"disable_finished_mamba_cache": false,
"enable_int8_mamba_checkpoint": false,
"int8_mamba_ckpt_size": null,
"linear_attn_backend": "triton",
"linear_attn_decode_backend": null,
"linear_attn_prefill_backend": null,
"enable_linear_replayssm": false,
"linear_replayssm_cache_len": 16,
"enable_gdn_replayssm_spec": false,
"enable_hierarchical_cache": false,
"hicache_ratio": 2.0,
"hicache_size": 0,
"hicache_write_policy": "write_through",
"hicache_io_backend": "kernel",
"hicache_mem_layout": "page_first",
"hicache_storage_backend": null,
"hicache_storage_prefetch_policy": "timeout",
"hicache_storage_backend_extra_config": null,
"enable_hisparse": false,
"hisparse_config": null,
"enable_broadcast_mm_inputs_process": false,
"enable_prefix_mm_cache": false,
"mm_enable_dp_encoder": false,
"mm_process_config": {},
"mm_processor_worker_num": 0,
"mm_io_worker_num": 0,
"limit_mm_data_per_request": null,
"enable_mm_global_cache": false,
"disable_fast_image_processor": false,
"mm_feature_transport": "cpu",
"keep_mm_feature_on_device": false,
"enable_lora": null,
"enable_lora_overlap_loading": null,
"max_lora_rank": 64,
"lora_target_modules": null,
"lora_paths": null,
"max_loaded_loras": null,
"max_loras_per_batch": 4,
"lora_eviction_policy": "lru",
"lora_backend": "triton",
"max_lora_chunk_size": 16,
"experts_shared_outer_loras": false,
"lora_use_virtual_experts": false,
"lora_strict_loading": true,
"lora_drain_wait_threshold": 0.0,
"enable_two_batch_overlap": false,
"enable_single_batch_overlap": false,
"tbo_token_distribution_threshold": 0.48,
"cpu_offload_gb": 0,
"offload_group_size": -1,
"offload_num_in_group": 1,
"offload_prefetch_step": 1,
"offload_mode": "cpu",
"enable_lmcache": false,
"lmcache_config_file": null,
"enable_flexkv": false,
"flexkv_config_file": null,
"kt_weight_path": null,
"kt_method": "AMXINT4",
"kt_cpuinfer": null,
"kt_threadpool_count": 2,
"kt_num_gpu_experts": null,
"kt_max_deferred_experts_per_token": null,
"dllm_algorithm": null,
"dllm_algorithm_config": null,
"dllm_fdfo": true,
"disaggregation_mode": "null",
"disaggregation_transfer_backend": "nixl",
"disaggregation_bootstrap_port": 30137,
"disaggregation_ib_device": null,
"disaggregation_decode_enable_radix_cache": false,
"disaggregation_decode_enable_offload_kvcache": false,
"num_reserved_decode_tokens": 512,
"disaggregation_decode_extra_slots": null,
"disaggregation_decode_polling_interval": 1,
"optimistic_prefill_attempts": 0,
"encoder_only": false,
"language_only": false,
"encoder_transfer_backend": "zmq_to_scheduler",
"encoder_urls": [],
"encoder_bootstrap_port": 8997,
"encoder_register_urls": [],
"enable_adaptive_dispatch_to_encoder": false,
"enable_pdmux": false,
"pdmux_config_path": null,
"sm_group_num": 8,
"custom_weight_loader": [],
"weight_loader_disable_mmap": false,
"weight_loader_prefetch_checkpoints": false,
"weight_loader_prefetch_num_threads": 4,
"weight_loader_drop_cache_after_load": false,
"remote_instance_weight_loader_seed_instance_ip": null,
"remote_instance_weight_loader_seed_instance_service_port": null,
"remote_instance_weight_loader_send_weights_group_ports": null,
"remote_instance_weight_loader_backend": "nccl",
"remote_instance_weight_loader_start_seed_via_transfer_engine": false,
"engine_info_bootstrap_port": 6789,
"modelexpress_config": null,
"download_dir": null,
"model_checksum": null,
"delete_ckpt_after_loading": false,
"decrypted_config_file": null,
"decrypted_draft_config_file": null,
"checkpoint_engine_wait_weights_before_ready": false,
"enable_prefill_delayer": true,
"prefill_delayer_max_delay_passes": 30,
"prefill_delayer_token_usage_low_watermark": 0.1,
"prefill_delayer_forward_passes_buckets": null,
"prefill_delayer_wait_seconds_buckets": null,
"prefill_delayer_queue_min_ratio": null,
"prefill_delayer_max_delay_ms": null,
"min_free_slots_delay": null,
"enable_deterministic_inference": false,
"rl_on_policy_target": null,
"kv_canary": "none",
"kv_canary_real_data": "none",
"kv_canary_sweep_interval": 0,
"enable_dynamic_batch_tokenizer": false,
"dynamic_batch_tokenizer_batch_size": 32,
"dynamic_batch_tokenizer_batch_timeout": 0.002,
"enable_tokenizer_batch_encode": false,
"disable_tokenizer_batch_decode": false,
"debug_tensor_dump_output_folder": null,
"debug_tensor_dump_layers": null,
"debug_tensor_dump_input_file": null,
"enable_memory_saver": false,
"enable_weights_cpu_backup": false,
"enable_draft_weights_cpu_backup": false,
"enable_custom_logit_processor": false,
"enable_return_hidden_states": false,
"enable_return_routed_experts": false,
"enable_return_indexer_topk": false,
"disable_outlines_disk_cache": false,
"enable_mis": false,
"forward_hooks": null,
"msprobe_dump_config": null,
"use_token_span": true,
"tml_mm_process_mode": false,
"tml_colocate_hydration": false,
"tml_rescale_image_frac": null,
"tml_rescale_image_max_upscaled_long_edge": null,
"async_detokenizer_send": false,
"remote_model_weights_path": null,
"enable_low_latency_kernels": false,
"enable_ll_moe_v2": true,
"mxfp8_kv_roundtrip": false,
"enable_prefill_cuda_graph": false,
"prefill_cuda_graph_sizes": [
1,
2,
4,
8,
12,
16,
24,
32,
40,
48,
56,
64,
96,
128
],
"prefill_cuda_graph_max_size": 128,
"decode_attention_num_splits": null,
"enable_dp_mixed_cuda_graph": false,
"mxfp8_moe_compute": false,
"ckpt_has_mxfp8_scales": false,
"nvfp4_use_calibrated_w2_input_scales": false,
"nvfp4_use_per_token_activation_scales": false,
"force_shared_experts_bf16": false,
"nvfp4_act_4over6": false,
"nvfp4_quantize_on_load": false,
"disable_sample_audio": false,
"use_moe_v2": false,
"tokenizer_batch_max_size": 1,
"tokenizer_batch_timeout_ms": 50,
"tokenizer_vocab_mask_size": 154856,
"use_tml_lora_backend": true,
"allow_full_non_token_span_fallback": true,
"audio_flow_max_steps": null,
"audio_flow_sampler_dtype": null,
"default_alter_topk": 0,
"min_prefill_batch_size": 0,
"max_prefill_skip_count": 10,
"send_continuous_mel": false,
"enable_moe_straggler_metrics": true,
"enable_gpu_process_marker": false,
"sconv_rms_preserving": null,
"dsconv_rms_preserving": null,
"enable_distributed_gumbel_sampling": false,
"coupled_reject_sampling": false,
"cg_verify_scheme": "shared_gumbel",
"_resolved_overrides": [
[
"_deepseek_family_overrides",
{
"page_size": 64
}
],
[
"_dsa_split_backend_resolution",
{
"dsa_prefill_backend": "trtllm",
"dsa_decode_backend": "trtllm"
}
],
[
"_moe_runner_fusion_disable",
{
"disable_shared_experts_fusion": true
}
],
[
"_speculative_moe_runner_default",
{
"speculative_moe_runner_backend": "flashinfer_trtllm"
}
]
],
"grpc_worker_threads": 4,
"_quantization_explicitly_unset": false,
"_cuda_graph_config_locked": [
[
"decode",
"bs"
],
[
"prefill",
"bs"
]
],
"_declarations_materialized": true,
"_in_override": false,
"tml_weight_version_state": {},
"_runtime_mutations": [
[
"model_runner.chunked_prefix_cache_gate",
{
"disable_chunked_prefix_cache": true
}
],
[
"scheduler.pp_max_micro_batch_size_default",
{
"pp_max_micro_batch_size": 256
}
]
],
"_mx_config_cache": {},
"max_speculative_num_draft_tokens": null,
"last_gen_throughput": 0.0,
"memory_usage": {
"weight": 127.43,
"kvcache": 119.74,
"token_capacity": 2328128,
"graph": 2.26
},
"effective_max_running_requests_per_dp": 256
}
],
"version": "0.5.6",
"kv_events": null
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment