Skip to content

Instantly share code, notes, and snippets.

@KastanDay
Created July 27, 2026 18:33
Show Gist options
  • Select an option

  • Save KastanDay/a2346f10d3a3e67a7f5870a5d8eb8b21 to your computer and use it in GitHub Desktop.

Select an option

Save KastanDay/a2346f10d3a3e67a7f5870a5d8eb8b21 to your computer and use it in GitHub Desktop.
SGLang PR #25 multi-node Mooncake transfer-quiescence validation

PR #25 multi-node Mooncake validation

Revision and deployment

  • Current PR head: 7db762ddf260df8ff6b102a9f20616a09c0d0457
  • Deployed source snapshot: 8361cb5705493de9489cc57cc7536279ae3b4de0
  • python/ and test/ are byte-for-byte identical between those revisions. The only full-tree differences are two unrelated Kimi-K3 documentation files introduced by the newer base.
  • Runtime: lmsysorg/sglang:latest@sha256:984699c298a95b73c469b2191403ddc85fd780506e13c39c4afff3845e27bc6c
  • Topology: two TP8 prefill engines and one TP8 decode engine on three separate H200 nodes, 24 GPUs total.
  • Barrier policy: SGLANG_DISAGGREGATION_TRANSFER_BARRIER=2 (STRICT).

Accepted race matrix

  • 16 synchronized waves: widths 8 and 16, two repeats, abort delays 0.25 and 2 seconds, victim gaps 0 and 20 ms.
  • Donor shape: 49,152 input tokens and 8,192 requested output tokens.
  • Victim shape: 8,192 input tokens and 256 requested output tokens.
  • 192 donors intentionally aborted while prefill-to-decode work was active.
  • 192/192 immediate victim reuse probes completed successfully.
  • Zero transport errors, semantic failures, hard failures, critical failures, or foreign canaries.
  • All acceptance checks passed.

Telemetry contained 339 successful scrapes with no scrape errors. The decode transfer queue peaked at 20 requests; each prefill inflight queue peaked at 4. There were zero retractions and pauses, and all three model pods remained Ready with zero restarts.

An earlier wider screen also included a 10-second abort delay. Eight of 144 victims received explicit proxy admission responses (HTTP 429, internal code 3040, "Service temporarily at capacity"). Those failures occurred only in the two width-16/10-second cells and were not model transport failures. The accepted run retained width 16 and the same transfer-queue peak while removing that admission-confounded hold time.

Preserved artifact hashes

38eb8e6748047d476035483b95800d875bfb021403bbbba2f0a9c1bf0e002067  metadata.json
82a80e5654338ed54b787c48be35863a068d422f46eb81237c1ab0615241ef98  race_matrix.json
1933a6bad799ed10e5c92cf47a8415a06a68c9f848112544219de230ab8b5e7d  responses.jsonl
856f0b08667f9d3a358adcb04e58b8dfea8b93c000166b94f8abb5e6d8df8c03  summary.json
c218cce1ed04805e8aa23b74a549845a4059df9511245a01f2c100ba25af3971  telemetry.jsonl
7a41a4393c806fc1b1e9e1870f9df11d2d06e5f803dce87bd10834df5556ed7b  telemetry_summary.json
72f8f1326a496f95d0a7769710d098e0652bfb97dc571dd7633bbf29ce6d3cbb  wave_events.jsonl
{
"cells": {
"width=16:delay=0.25:gap=0.02:offset=0": {
"donor_timeout-abort": 32,
"foreign_canaries": 0,
"victims": 32
},
"width=16:delay=0.25:gap=0.0:offset=0": {
"donor_timeout-abort": 32,
"foreign_canaries": 0,
"victims": 32
},
"width=16:delay=2.0:gap=0.02:offset=0": {
"donor_timeout-abort": 32,
"foreign_canaries": 0,
"victims": 32
},
"width=16:delay=2.0:gap=0.0:offset=0": {
"donor_timeout-abort": 32,
"foreign_canaries": 0,
"victims": 32
},
"width=8:delay=0.25:gap=0.02:offset=0": {
"donor_timeout-abort": 16,
"foreign_canaries": 0,
"victims": 16
},
"width=8:delay=0.25:gap=0.0:offset=0": {
"donor_timeout-abort": 16,
"foreign_canaries": 0,
"victims": 16
},
"width=8:delay=2.0:gap=0.02:offset=0": {
"donor_timeout-abort": 16,
"foreign_canaries": 0,
"victims": 16
},
"width=8:delay=2.0:gap=0.0:offset=0": {
"donor_timeout-abort": 16,
"foreign_canaries": 0,
"victims": 16
}
},
"overall": {
"donor_timeout-abort": 192,
"foreign_canaries": 0,
"victims": 192
}
}
{
"acceptance": {
"checks": {
"cache_observed": true,
"hard_failure_rate": true,
"transport_error_rate": true,
"zero_critical_failures": true,
"zero_semantic_failures": true
},
"passed": true
},
"families": {
"donor": {
"critical_failure_rate": 0.0,
"critical_failures": 0,
"hard_failure_rate": 0.0,
"hard_failures": 0,
"intentional_aborts": 192,
"mean_cached_tokens": 0,
"ordinary_attempts": 0,
"records": 192,
"responses": 0,
"responses_with_cached_tokens": 0,
"signal_counts": {},
"transport_error_rate": 0.0,
"transport_errors": 0
},
"isolation": {
"critical_failure_rate": 0.0,
"critical_failures": 0,
"hard_failure_rate": 0.0,
"hard_failures": 0,
"intentional_aborts": 0,
"mean_cached_tokens": 0.0,
"ordinary_attempts": 192,
"records": 192,
"responses": 192,
"responses_with_cached_tokens": 0,
"signal_counts": {},
"transport_error_rate": 0.0,
"transport_errors": 0
}
},
"overall": {
"critical_failure_rate": 0.0,
"critical_failures": 0,
"hard_failure_rate": 0.0,
"hard_failures": 0,
"intentional_aborts": 192,
"mean_cached_tokens": 0.0,
"ordinary_attempts": 192,
"records": 384,
"responses": 192,
"responses_with_cached_tokens": 0,
"signal_counts": {},
"transport_error_rate": 0.0,
"transport_errors": 0
},
"phases": {
"donor-abort-reuse": {
"critical_failure_rate": 0.0,
"critical_failures": 0,
"hard_failure_rate": 0.0,
"hard_failures": 0,
"intentional_aborts": 192,
"mean_cached_tokens": 0,
"ordinary_attempts": 0,
"records": 192,
"responses": 0,
"responses_with_cached_tokens": 0,
"signal_counts": {},
"transport_error_rate": 0.0,
"transport_errors": 0
},
"victim-reuse-probe": {
"critical_failure_rate": 0.0,
"critical_failures": 0,
"hard_failure_rate": 0.0,
"hard_failures": 0,
"intentional_aborts": 0,
"mean_cached_tokens": 0.0,
"ordinary_attempts": 192,
"records": 192,
"responses": 192,
"responses_with_cached_tokens": 0,
"signal_counts": {},
"transport_error_rate": 0.0,
"transport_errors": 0
}
},
"stage": "quiesce-strict-8361cb5705-valid-pressure"
}
{
"each_role_reached_target": false,
"max_simultaneous_min_token_usage": 0.2,
"peak_by_role": {
"decode-0": {
"max_total_num_tokens": 679744.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 20.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 0.0,
"num_queue_reqs": 0.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 4.0,
"num_used_tokens": 374592.0,
"token_usage": 0.55
},
"prefill-0": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 4.0,
"num_queue_reqs": 8.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 81920.0,
"token_usage": 0.2
},
"prefill-1": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 4.0,
"num_queue_reqs": 10.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 81920.0,
"token_usage": 0.2
}
},
"per_wave": {
"0": {
"abort_after_s": 0.25,
"boundary_offset": 0,
"each_role_reached_target": false,
"max_simultaneous_min_token_usage": 0.16,
"peak_by_role": {
"decode-0": {
"max_total_num_tokens": 679744.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 8.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 0.0,
"num_queue_reqs": 0.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 2.0,
"num_used_tokens": 162752.0,
"token_usage": 0.24
},
"prefill-0": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 1.0,
"num_queue_reqs": 3.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 2.0,
"num_used_tokens": 65536.0,
"token_usage": 0.16
},
"prefill-1": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 3.0,
"num_queue_reqs": 4.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 81920.0,
"token_usage": 0.2
}
},
"repeat": 0,
"samples": 15,
"scrape_errors": {},
"simultaneous_target_reached": false,
"target_token_usage": 0.98,
"victim_gap_s": 0.0,
"wave_index": 0,
"wave_width": 8
},
"1": {
"abort_after_s": 0.25,
"boundary_offset": 0,
"each_role_reached_target": false,
"max_simultaneous_min_token_usage": 0.2,
"peak_by_role": {
"decode-0": {
"max_total_num_tokens": 679744.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 10.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 0.0,
"num_queue_reqs": 0.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 2.0,
"num_used_tokens": 260992.0,
"token_usage": 0.38
},
"prefill-0": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 1.0,
"num_queue_reqs": 4.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 2.0,
"num_used_tokens": 81920.0,
"token_usage": 0.2
},
"prefill-1": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 1.0,
"num_queue_reqs": 4.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 2.0,
"num_used_tokens": 81920.0,
"token_usage": 0.2
}
},
"repeat": 1,
"samples": 20,
"scrape_errors": {},
"simultaneous_target_reached": false,
"target_token_usage": 0.98,
"victim_gap_s": 0.0,
"wave_index": 1,
"wave_width": 8
},
"10": {
"abort_after_s": 0.25,
"boundary_offset": 0,
"each_role_reached_target": false,
"max_simultaneous_min_token_usage": 0.16,
"peak_by_role": {
"decode-0": {
"max_total_num_tokens": 679744.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 20.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 0.0,
"num_queue_reqs": 0.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 2.0,
"num_used_tokens": 374528.0,
"token_usage": 0.55
},
"prefill-0": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 4.0,
"num_queue_reqs": 4.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 65536.0,
"token_usage": 0.16
},
"prefill-1": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 4.0,
"num_queue_reqs": 9.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 81920.0,
"token_usage": 0.2
}
},
"repeat": 0,
"samples": 27,
"scrape_errors": {},
"simultaneous_target_reached": false,
"target_token_usage": 0.98,
"victim_gap_s": 0.02,
"wave_index": 10,
"wave_width": 16
},
"11": {
"abort_after_s": 0.25,
"boundary_offset": 0,
"each_role_reached_target": false,
"max_simultaneous_min_token_usage": 0.16,
"peak_by_role": {
"decode-0": {
"max_total_num_tokens": 679744.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 19.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 0.0,
"num_queue_reqs": 0.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 2.0,
"num_used_tokens": 325440.0,
"token_usage": 0.48
},
"prefill-0": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 4.0,
"num_queue_reqs": 4.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 65536.0,
"token_usage": 0.16
},
"prefill-1": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 4.0,
"num_queue_reqs": 7.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 81920.0,
"token_usage": 0.2
}
},
"repeat": 1,
"samples": 25,
"scrape_errors": {},
"simultaneous_target_reached": false,
"target_token_usage": 0.98,
"victim_gap_s": 0.02,
"wave_index": 11,
"wave_width": 16
},
"12": {
"abort_after_s": 2.0,
"boundary_offset": 0,
"each_role_reached_target": false,
"max_simultaneous_min_token_usage": 0.16,
"peak_by_role": {
"decode-0": {
"max_total_num_tokens": 679744.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 18.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 0.0,
"num_queue_reqs": 0.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 276352.0,
"token_usage": 0.41
},
"prefill-0": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 4.0,
"num_queue_reqs": 2.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 81920.0,
"token_usage": 0.2
},
"prefill-1": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 4.0,
"num_queue_reqs": 9.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 65600.0,
"token_usage": 0.16
}
},
"repeat": 0,
"samples": 28,
"scrape_errors": {},
"simultaneous_target_reached": false,
"target_token_usage": 0.98,
"victim_gap_s": 0.0,
"wave_index": 12,
"wave_width": 16
},
"13": {
"abort_after_s": 2.0,
"boundary_offset": 0,
"each_role_reached_target": false,
"max_simultaneous_min_token_usage": 0.2,
"peak_by_role": {
"decode-0": {
"max_total_num_tokens": 679744.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 19.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 0.0,
"num_queue_reqs": 0.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 2.0,
"num_used_tokens": 374592.0,
"token_usage": 0.55
},
"prefill-0": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 4.0,
"num_queue_reqs": 2.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 81920.0,
"token_usage": 0.2
},
"prefill-1": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 4.0,
"num_queue_reqs": 10.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 81920.0,
"token_usage": 0.2
}
},
"repeat": 1,
"samples": 27,
"scrape_errors": {},
"simultaneous_target_reached": false,
"target_token_usage": 0.98,
"victim_gap_s": 0.0,
"wave_index": 13,
"wave_width": 16
},
"14": {
"abort_after_s": 2.0,
"boundary_offset": 0,
"each_role_reached_target": false,
"max_simultaneous_min_token_usage": 0.2,
"peak_by_role": {
"decode-0": {
"max_total_num_tokens": 679744.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 18.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 0.0,
"num_queue_reqs": 0.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 4.0,
"num_used_tokens": 325504.0,
"token_usage": 0.48
},
"prefill-0": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 4.0,
"num_queue_reqs": 4.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 81920.0,
"token_usage": 0.2
},
"prefill-1": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 4.0,
"num_queue_reqs": 7.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 81920.0,
"token_usage": 0.2
}
},
"repeat": 0,
"samples": 26,
"scrape_errors": {},
"simultaneous_target_reached": false,
"target_token_usage": 0.98,
"victim_gap_s": 0.02,
"wave_index": 14,
"wave_width": 16
},
"15": {
"abort_after_s": 2.0,
"boundary_offset": 0,
"each_role_reached_target": false,
"max_simultaneous_min_token_usage": 0.2,
"peak_by_role": {
"decode-0": {
"max_total_num_tokens": 679744.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 18.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 0.0,
"num_queue_reqs": 0.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 325504.0,
"token_usage": 0.48
},
"prefill-0": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 4.0,
"num_queue_reqs": 4.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 81920.0,
"token_usage": 0.2
},
"prefill-1": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 4.0,
"num_queue_reqs": 7.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 81920.0,
"token_usage": 0.2
}
},
"repeat": 1,
"samples": 24,
"scrape_errors": {},
"simultaneous_target_reached": false,
"target_token_usage": 0.98,
"victim_gap_s": 0.02,
"wave_index": 15,
"wave_width": 16
},
"2": {
"abort_after_s": 0.25,
"boundary_offset": 0,
"each_role_reached_target": false,
"max_simultaneous_min_token_usage": 0.16,
"peak_by_role": {
"decode-0": {
"max_total_num_tokens": 679744.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 8.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 0.0,
"num_queue_reqs": 0.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 1.0,
"num_used_tokens": 113664.0,
"token_usage": 0.17
},
"prefill-0": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 1.0,
"num_queue_reqs": 3.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 2.0,
"num_used_tokens": 65536.0,
"token_usage": 0.16
},
"prefill-1": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 1.0,
"num_queue_reqs": 3.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 2.0,
"num_used_tokens": 65536.0,
"token_usage": 0.16
}
},
"repeat": 0,
"samples": 14,
"scrape_errors": {},
"simultaneous_target_reached": false,
"target_token_usage": 0.98,
"victim_gap_s": 0.02,
"wave_index": 2,
"wave_width": 8
},
"3": {
"abort_after_s": 0.25,
"boundary_offset": 0,
"each_role_reached_target": false,
"max_simultaneous_min_token_usage": 0.16,
"peak_by_role": {
"decode-0": {
"max_total_num_tokens": 679744.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 8.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 0.0,
"num_queue_reqs": 0.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 2.0,
"num_used_tokens": 162816.0,
"token_usage": 0.24
},
"prefill-0": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 1.0,
"num_queue_reqs": 3.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 2.0,
"num_used_tokens": 65536.0,
"token_usage": 0.16
},
"prefill-1": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 1.0,
"num_queue_reqs": 3.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 2.0,
"num_used_tokens": 65536.0,
"token_usage": 0.16
}
},
"repeat": 1,
"samples": 14,
"scrape_errors": {},
"simultaneous_target_reached": false,
"target_token_usage": 0.98,
"victim_gap_s": 0.02,
"wave_index": 3,
"wave_width": 8
},
"4": {
"abort_after_s": 2.0,
"boundary_offset": 0,
"each_role_reached_target": false,
"max_simultaneous_min_token_usage": 0.2,
"peak_by_role": {
"decode-0": {
"max_total_num_tokens": 679744.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 10.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 0.0,
"num_queue_reqs": 0.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 2.0,
"num_used_tokens": 260992.0,
"token_usage": 0.38
},
"prefill-0": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 3.0,
"num_queue_reqs": 3.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 2.0,
"num_used_tokens": 81920.0,
"token_usage": 0.2
},
"prefill-1": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 3.0,
"num_queue_reqs": 4.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 2.0,
"num_used_tokens": 81920.0,
"token_usage": 0.2
}
},
"repeat": 0,
"samples": 19,
"scrape_errors": {},
"simultaneous_target_reached": false,
"target_token_usage": 0.98,
"victim_gap_s": 0.0,
"wave_index": 4,
"wave_width": 8
},
"5": {
"abort_after_s": 2.0,
"boundary_offset": 0,
"each_role_reached_target": false,
"max_simultaneous_min_token_usage": 0.2,
"peak_by_role": {
"decode-0": {
"max_total_num_tokens": 679744.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 11.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 0.0,
"num_queue_reqs": 0.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 260992.0,
"token_usage": 0.38
},
"prefill-0": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 1.0,
"num_queue_reqs": 3.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 2.0,
"num_used_tokens": 81920.0,
"token_usage": 0.2
},
"prefill-1": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 3.0,
"num_queue_reqs": 2.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 81920.0,
"token_usage": 0.2
}
},
"repeat": 1,
"samples": 21,
"scrape_errors": {},
"simultaneous_target_reached": false,
"target_token_usage": 0.98,
"victim_gap_s": 0.0,
"wave_index": 5,
"wave_width": 8
},
"6": {
"abort_after_s": 2.0,
"boundary_offset": 0,
"each_role_reached_target": false,
"max_simultaneous_min_token_usage": 0.0,
"peak_by_role": {
"decode-0": {
"max_total_num_tokens": 679744.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 8.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 0.0,
"num_queue_reqs": 0.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 162688.0,
"token_usage": 0.24
},
"prefill-0": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 3.0,
"num_queue_reqs": 2.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 81920.0,
"token_usage": 0.2
},
"prefill-1": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 1.0,
"num_queue_reqs": 2.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 2.0,
"num_used_tokens": 65536.0,
"token_usage": 0.16
}
},
"repeat": 0,
"samples": 13,
"scrape_errors": {},
"simultaneous_target_reached": false,
"target_token_usage": 0.98,
"victim_gap_s": 0.02,
"wave_index": 6,
"wave_width": 8
},
"7": {
"abort_after_s": 2.0,
"boundary_offset": 0,
"each_role_reached_target": false,
"max_simultaneous_min_token_usage": 0.16,
"peak_by_role": {
"decode-0": {
"max_total_num_tokens": 679744.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 8.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 0.0,
"num_queue_reqs": 0.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 2.0,
"num_used_tokens": 162752.0,
"token_usage": 0.24
},
"prefill-0": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 1.0,
"num_queue_reqs": 2.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 2.0,
"num_used_tokens": 65536.0,
"token_usage": 0.16
},
"prefill-1": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 1.0,
"num_queue_reqs": 2.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 2.0,
"num_used_tokens": 65536.0,
"token_usage": 0.16
}
},
"repeat": 1,
"samples": 14,
"scrape_errors": {},
"simultaneous_target_reached": false,
"target_token_usage": 0.98,
"victim_gap_s": 0.02,
"wave_index": 7,
"wave_width": 8
},
"8": {
"abort_after_s": 0.25,
"boundary_offset": 0,
"each_role_reached_target": false,
"max_simultaneous_min_token_usage": 0.16,
"peak_by_role": {
"decode-0": {
"max_total_num_tokens": 679744.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 17.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 0.0,
"num_queue_reqs": 0.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 227264.0,
"token_usage": 0.33
},
"prefill-0": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 4.0,
"num_queue_reqs": 8.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 65600.0,
"token_usage": 0.16
},
"prefill-1": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 4.0,
"num_queue_reqs": 7.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 81920.0,
"token_usage": 0.2
}
},
"repeat": 0,
"samples": 26,
"scrape_errors": {},
"simultaneous_target_reached": false,
"target_token_usage": 0.98,
"victim_gap_s": 0.0,
"wave_index": 8,
"wave_width": 16
},
"9": {
"abort_after_s": 0.25,
"boundary_offset": 0,
"each_role_reached_target": false,
"max_simultaneous_min_token_usage": 0.2,
"peak_by_role": {
"decode-0": {
"max_total_num_tokens": 679744.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 19.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 0.0,
"num_queue_reqs": 0.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 2.0,
"num_used_tokens": 374592.0,
"token_usage": 0.55
},
"prefill-0": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 4.0,
"num_queue_reqs": 5.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 81856.0,
"token_usage": 0.2
},
"prefill-1": {
"max_total_num_tokens": 411520.0,
"num_decode_prealloc_queue_reqs": 0.0,
"num_decode_transfer_queue_reqs": 0.0,
"num_paused_reqs": 0.0,
"num_prefill_inflight_queue_reqs": 4.0,
"num_queue_reqs": 8.0,
"num_retracted_reqs": 0.0,
"num_running_reqs": 3.0,
"num_used_tokens": 81920.0,
"token_usage": 0.2
}
},
"repeat": 1,
"samples": 25,
"scrape_errors": {},
"simultaneous_target_reached": false,
"target_token_usage": 0.98,
"victim_gap_s": 0.0,
"wave_index": 9,
"wave_width": 16
}
},
"samples": 339,
"scrape_errors": {},
"simultaneous_target_reached": false,
"target_token_usage": 0.98
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment