diff --git a/benchmarks/single_node/agentic/README.md b/benchmarks/single_node/agentic/README.md index c2c0df7d7..57de401b4 100644 --- a/benchmarks/single_node/agentic/README.md +++ b/benchmarks/single_node/agentic/README.md @@ -20,7 +20,7 @@ currently either `none` or `dram`; when it is `dram`, the backend must be set: ```yaml - dram-utilization: 0.80 search-space: - - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [16, 32] } + - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-native }, conc-list: [16, 32] } - { tp: 8, kv-offloading: none, conc-list: [16, 32] } ``` diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh index 5ebb6765e..e6cc14cb7 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh @@ -22,7 +22,7 @@ set -x # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR # # KV_OFFLOADING=dram requires one of these. -# KV_OFFLOAD_BACKEND=native. +# KV_OFFLOAD_BACKEND=vllm-native. # KV_OFFLOAD_BACKEND=mooncake. # KV_OFFLOAD_BACKEND=lmcache. # KV_OFFLOAD_BACKEND=hicache. @@ -102,8 +102,8 @@ OFFLOAD_ARGS=() if agentic_kv_offload_enabled; then case "${KV_OFFLOAD_BACKEND:-}" in - native) - require_agentic_kv_offload_backend native + vllm-native) + require_agentic_kv_offload_backend vllm-native # ---- vLLM native config ---------------------------------------------------------- unset VLLM_USE_SIMPLE_KV_OFFLOAD # MI355X nodes have ~2.7 TiB of host DRAM available for offload; @@ -111,7 +111,7 @@ case "${KV_OFFLOAD_BACKEND:-}" in # worker RSS / page cache / slurm cgroup). TOTAL_CPU_DRAM_PARTITION_GB="$((TOTAL_CPU_DRAM_GB / (8 / TP)))" # Use vLLM's regular native KV-offload path (OffloadingConnector), - # NOT the SimpleCPUOffloadConnector. The "native" backend resolves to + # NOT the SimpleCPUOffloadConnector. The "vllm-native" backend resolves to # OffloadingConnector by default; setting VLLM_USE_SIMPLE_KV_OFFLOAD=1 # would switch it to SimpleCPUOffloadConnector. We intentionally leave # that env var UNSET here so the regular OffloadingConnector path is @@ -277,7 +277,7 @@ EOF git clone https://github.com/LMCache/LMCache.git cd LMCache # https://github.com/LMCache/LMCache/pull/3853 - git checkout dev + git checkout 9229067cec0b3a63bb8a39368d101db7ac0bc3c1 pip install -r requirements/build.txt pip install grpcio==1.78.0 CXX=hipcc BUILD_WITH_HIP=1 pip install -e . --no-build-isolation @@ -345,7 +345,7 @@ EOF ) ;; *) - echo "Error: unsupported KV_OFFLOAD_BACKEND '${KV_OFFLOAD_BACKEND:-}' (expected: native, mooncake, lmcache)" >&2 + echo "Error: unsupported KV_OFFLOAD_BACKEND '${KV_OFFLOAD_BACKEND:-}' (expected: vllm-native, mooncake, lmcache)" >&2 exit 1 ;; esac diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_b200.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_b200.sh index b4ac77f2b..ce0e75fbb 100755 --- a/benchmarks/single_node/agentic/kimik2.5_fp4_b200.sh +++ b/benchmarks/single_node/agentic/kimik2.5_fp4_b200.sh @@ -96,7 +96,7 @@ if require_agentic_kv_offload_backend lmcache; then { set +x; } 2>/dev/null unset VLLM_USE_SIMPLE_KV_OFFLOAD - agentic_pip_install --quiet --no-cache-dir lmcache + agentic_pip_install --quiet --no-cache-dir 'lmcache==0.5.1' python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null # MP mode owns the configured CPU pool in the external LMCache diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh index b99299b12..5fc46ab76 100755 --- a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh +++ b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh @@ -7,7 +7,7 @@ set -x # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR # -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=native. +# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=vllm-simple. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -42,7 +42,7 @@ mkdir -p "$RESULT_DIR" OFFLOAD_ARGS=() PREFIX_CACHE_ARGS=() -if require_agentic_kv_offload_backend native; then +if require_agentic_kv_offload_backend vllm-simple; then export VLLM_USE_SIMPLE_KV_OFFLOAD=1 OFFLOAD_ARGS=( --kv_offloading_backend native diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_mi355x.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_mi355x.sh index 8a525d623..3ac229c2e 100755 --- a/benchmarks/single_node/agentic/kimik2.5_fp4_mi355x.sh +++ b/benchmarks/single_node/agentic/kimik2.5_fp4_mi355x.sh @@ -7,7 +7,7 @@ set -x # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR # -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=native. +# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=vllm-native. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -64,10 +64,10 @@ mkdir -p "$RESULT_DIR" OFFLOAD_ARGS=() PREFIX_CACHE_ARGS=() -if require_agentic_kv_offload_backend native; then +if require_agentic_kv_offload_backend vllm-native; then unset VLLM_USE_SIMPLE_KV_OFFLOAD # Use vLLM's regular native KV-offload path (OffloadingConnector), - # NOT the SimpleCPUOffloadConnector. The "native" backend resolves to + # NOT the SimpleCPUOffloadConnector. The "vllm-native" backend resolves to # OffloadingConnector by default; setting VLLM_USE_SIMPLE_KV_OFFLOAD=1 # would switch it to SimpleCPUOffloadConnector. We intentionally leave # that env var UNSET here so the regular OffloadingConnector path is diff --git a/benchmarks/single_node/agentic/kimik2.5_int4_b200.sh b/benchmarks/single_node/agentic/kimik2.5_int4_b200.sh index 8c5ab5328..f9d170e06 100755 --- a/benchmarks/single_node/agentic/kimik2.5_int4_b200.sh +++ b/benchmarks/single_node/agentic/kimik2.5_int4_b200.sh @@ -38,7 +38,7 @@ SERVER_LOG="$RESULT_DIR/server.log" mkdir -p "$RESULT_DIR" OFFLOAD_ARGS="" -if require_agentic_kv_offload_backend native; then +if require_agentic_kv_offload_backend vllm-simple; then export VLLM_USE_SIMPLE_KV_OFFLOAD=1 OFFLOAD_ARGS="--kv_offloading_backend native --kv_offloading_size $TOTAL_CPU_DRAM_GB --disable-hybrid-kv-cache-manager" fi diff --git a/benchmarks/single_node/agentic/kimik2.5_int4_h100.sh b/benchmarks/single_node/agentic/kimik2.5_int4_h100.sh index d5ade390b..f7753b4df 100755 --- a/benchmarks/single_node/agentic/kimik2.5_int4_h100.sh +++ b/benchmarks/single_node/agentic/kimik2.5_int4_h100.sh @@ -38,7 +38,7 @@ SERVER_LOG="$RESULT_DIR/server.log" mkdir -p "$RESULT_DIR" OFFLOAD_ARGS="" -if require_agentic_kv_offload_backend native; then +if require_agentic_kv_offload_backend vllm-simple; then export VLLM_USE_SIMPLE_KV_OFFLOAD=1 OFFLOAD_ARGS="--kv_offloading_backend native --kv_offloading_size $TOTAL_CPU_DRAM_GB --disable-hybrid-kv-cache-manager" fi diff --git a/benchmarks/single_node/agentic/kimik2.5_int4_h200.sh b/benchmarks/single_node/agentic/kimik2.5_int4_h200.sh index 3b9411d1c..1e1d881d5 100755 --- a/benchmarks/single_node/agentic/kimik2.5_int4_h200.sh +++ b/benchmarks/single_node/agentic/kimik2.5_int4_h200.sh @@ -38,7 +38,7 @@ SERVER_LOG="$RESULT_DIR/server.log" mkdir -p "$RESULT_DIR" OFFLOAD_ARGS="" -if require_agentic_kv_offload_backend native; then +if require_agentic_kv_offload_backend vllm-simple; then # Kimi K2.5 is pure TP (no DP-attn): single engine, world_size=TP. # SimpleCPUOffloadConnector internally divides cpu_bytes_to_use by # world_size, so pass the full TOTAL_CPU_DRAM_GB; TP-shared mmap diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 39ca4d8d3..4b1ea4aed 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -355,6 +355,7 @@ qwen3.5-fp8-mi355x-sglang-disagg: runner: mi355x-disagg precision: fp8 framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } kv-p2p-transfer: mori multinode: true disagg: true @@ -483,6 +484,7 @@ qwen3.5-fp4-mi355x-sglang-disagg: runner: mi355x-disagg precision: fp4 framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } kv-p2p-transfer: mori multinode: true disagg: true @@ -597,6 +599,7 @@ glm5-fp8-mi355x-sglang-disagg: runner: mi355x-disagg precision: fp8 framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } kv-p2p-transfer: mori multinode: true disagg: true @@ -803,13 +806,13 @@ kimik2.5-fp4-mi355x-vllm-agentic: # DRAM offload only above the KV cliff. Lower concurrencies fit # entirely on-GPU, so paying the offload-path overhead there would # just slow them down without measuring anything new. - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [32, 40, 48, 56] } + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-native }, conc-list: [32, 40, 48, 56] } # TP=4 probe: half-node layout doubles per-GPU weight footprint # (~62 GB on MI355X's 288 GB HBM, plenty of headroom). Restrict to # cliff-region concurrencies on both offload modes so we can directly # compare TP=4 vs TP=8 at the same conc points. - { tp: 4, kv-offloading: none, conc-list: [16, 24, 32, 40] } - - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [16, 24, 32, 40] } + - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-native }, conc-list: [16, 24, 32, 40] } kimik2.5-fp4-mi355x-atom: image: rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.4_202607091539 @@ -876,6 +879,7 @@ dsr1-fp8-mi355x-sglang-disagg: runner: mi355x-disagg precision: fp8 framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } kv-p2p-transfer: mori multinode: true disagg: true @@ -1031,6 +1035,7 @@ dsr1-fp8-mi355x-sglang-disagg-mtp: runner: mi355x-disagg precision: fp8 framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } kv-p2p-transfer: mori multinode: true disagg: true @@ -1186,6 +1191,7 @@ kimik2.5-fp4-mi355x-vllm-disagg: runner: mi355x-disagg precision: fp4 framework: vllm-disagg + router: { name: vllm-router, version: "0.1.14" } kv-p2p-transfer: moriio multinode: true disagg: true @@ -1239,6 +1245,7 @@ dsr1-fp4-mi355x-sglang-disagg: runner: mi355x-disagg precision: fp4 framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } kv-p2p-transfer: mori multinode: true disagg: true @@ -1468,6 +1475,7 @@ dsr1-fp4-mi355x-sglang-disagg-1k1k-mtp: runner: mi355x-disagg precision: fp4 framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } kv-p2p-transfer: mori multinode: true disagg: true @@ -1579,6 +1587,7 @@ dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp: runner: mi355x-disagg precision: fp4 framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } kv-p2p-transfer: mori multinode: true disagg: true @@ -1748,6 +1757,7 @@ dsv4-fp4-mi355x-sglang-disagg: runner: mi355x-disagg precision: fp4 framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } kv-p2p-transfer: mori multinode: true disagg: true @@ -2093,8 +2103,8 @@ dsv4-fp4-mi355x-vllm-agentic: - dram-utilization: 0.60 search-space: - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 40, 48] } - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, conc-list: [64, 72] } - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: lmcache }, conc-list: [32, 40, 48] } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, conc-list: [64, 72], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "9229067cec0b3a63bb8a39368d101db7ac0bc3c1" }, conc-list: [32, 40, 48] } # DSv4-Pro FP4 on MI355X via SGLang. Uses a rocm720 mi35x image built off the # amd/deepseek_v4 branch in sgl-project/sglang; the SHA is encoded in the @@ -2108,6 +2118,7 @@ dsr1-fp4-mi355x-sglang-disagg-mtp: runner: mi355x-disagg precision: fp4 framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } kv-p2p-transfer: mori multinode: true disagg: true @@ -2359,6 +2370,7 @@ dsv4-fp4-mi355x-atom-disagg: runner: mi355x precision: fp4 framework: atom-disagg + router: { name: atomesh, version: "087b82d9c1f630e79149ba37e6213257ec9a76f8" } kv-p2p-transfer: mooncake multinode: true disagg: true @@ -2479,6 +2491,7 @@ minimaxm3-fp4-mi355x-vllm-disagg: runner: mi355x-disagg precision: fp4 framework: vllm-disagg + router: { name: vllm-router, version: "0.1.14" } kv-p2p-transfer: moriio multinode: true disagg: true @@ -2668,6 +2681,7 @@ minimaxm3-fp8-mi355x-atom-disagg: runner: mi355x-disagg precision: fp8 framework: atom-disagg + router: { name: atomesh, version: "04b120d3040e0dd1a6915e88a3a3c1f588e6684a" } kv-p2p-transfer: mooncake multinode: true disagg: true @@ -2739,6 +2753,7 @@ minimaxm3-fp4-mi355x-atom-disagg: runner: mi355x-disagg precision: fp4 framework: atom-disagg + router: { name: atomesh, version: "04b120d3040e0dd1a6915e88a3a3c1f588e6684a" } kv-p2p-transfer: mooncake multinode: true disagg: true @@ -2943,6 +2958,7 @@ minimaxm3-fp8-mi355x-vllm-disagg: runner: mi355x-disagg precision: fp8 framework: vllm-disagg + router: { name: vllm-router, version: "0.1.14" } kv-p2p-transfer: moriio multinode: true disagg: true @@ -3050,8 +3066,8 @@ minimaxm3-fp8-mi300x-vllm-agentic: search-space: - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } - { tp: 8, ep: 8, kv-offloading: none, conc-list: [2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake }, conc-list: [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } - - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake }, conc-list: [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } + - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } minimaxm3-fp8-mi325x-vllm-agentic: image: vllm/vllm-openai-rocm:nightly-04c2a8deac44fdb1ca3e2b5ec3e6bf16f3f6a914 @@ -3074,7 +3090,7 @@ minimaxm3-fp8-mi325x-vllm-agentic: - { tp: 4, ep: 4, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } - { tp: 8, ep: 8, kv-offloading: none, conc-list: [2, 4, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80] } - - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [24, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80, 96] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } + - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [24, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80, 96], router: { name: vllm-router, version: "0.1.14" } } diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 91679af05..f90c51018 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -5,6 +5,7 @@ dsr1-fp4-b200-dynamo-trt: runner: b200-multinode precision: fp4 framework: dynamo-trt + router: { name: dynamo-router, version: "0.8.1.post1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -392,6 +393,7 @@ dsr1-fp8-b200-dynamo-trt: runner: b200-multinode precision: fp8 framework: dynamo-trt + router: { name: dynamo-router, version: "0.8.1.post2" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -843,6 +845,7 @@ dsr1-fp4-b300-dynamo-trt: runner: b300 precision: fp4 framework: dynamo-trt + router: { name: dynamo-router, version: "0.8.1.post1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -1257,6 +1260,7 @@ dsr1-fp8-b300-dynamo-trt: runner: b300 precision: fp8 framework: dynamo-trt + router: { name: dynamo-router, version: "0.8.1.post1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -1782,10 +1786,10 @@ dsv4-fp4-b200-vllm-agentic: - { tp: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5] } # Sample the useful MooncakeStore range without repeating its collapsed # high-concurrency tail. - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake }, conc-list: [8, 10, 16] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 38, 44, 50] } + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [8, 10, 16] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 38, 44, 50], router: { name: vllm-router, version: "0.1.14" } } # Retain the external-cache transition and peak-throughput region. - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake }, conc-list: [16, 38, 44, 56, 64, 66, 68] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [16, 38, 44, 56, 64, 66, 68], router: { name: vllm-router, version: "0.1.14" } } dsv4-fp4-b200-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 @@ -2341,6 +2345,7 @@ glm5-fp4-gb300-dynamo-trt: runner: gb300-nv precision: fp4 framework: dynamo-trt + router: { name: dynamo-router, version: "v1.3.0-dev.1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -2815,7 +2820,7 @@ kimik2.5-int4-b200-vllm-agentic: - dram-utilization: 0.80 search-space: - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [32, 64, 96, 128] } + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [32, 64, 96, 128] } kimik2.5-int4-b300-vllm: image: vllm/vllm-openai:v0.24.0 @@ -2870,7 +2875,7 @@ kimik2.5-int4-h200-vllm-agentic: - dram-utilization: 0.80 search-space: - { tp: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [6, 7, 8, 9, 10, 11, 12, 13, 14] } + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [6, 7, 8, 9, 10, 11, 12, 13, 14] } # NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html # does not have a B300-specific recipe, so this config reuses the existing @@ -2982,7 +2987,7 @@ kimik2.5-fp4-b300-vllm-agentic: - dram-utilization: 0.80 search-space: - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32, 40, 48, 56, 64] } - - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 2, 4, 8, 16, 32, 40, 48, 56, 64] } + - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2, 4, 8, 16, 32, 40, 48, 56, 64] } dsr1-fp8-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 @@ -3226,14 +3231,14 @@ dsv4-fp4-b300-vllm-agentic: search-space: # Compare native GPU-cache and MooncakeStore CPU-offload cliffs. - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 16] } - - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: mooncake }, conc-list: [16, 18, 20, 24] } + - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [16, 18, 20, 24] } # TP8 remains cache-resident through conc 52. - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 16, 32, 40, 48, 52] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake }, conc-list: [52] } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, conc-list: [8, 16] } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake }, conc-list: [32] } + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [52] } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, conc-list: [8, 16], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [32], router: { name: vllm-router, version: "0.1.14" } } # TP8 DEP retains representative low, peak, and transition points. - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [52, 72, 100, 128, 144] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [52, 72, 100, 128, 144], router: { name: vllm-router, version: "0.1.14" } } dsv4-fp4-b300-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 @@ -3435,6 +3440,7 @@ dsr1-fp8-h200-dynamo-trt: runner: h200-multinode precision: fp8 framework: dynamo-trt + router: { name: dynamo-router, version: "0.8.1.post1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -3980,6 +3986,7 @@ dsr1-fp8-h100-dynamo-trt: runner: h100-multinode precision: fp8 framework: dynamo-trt + router: { name: dynamo-router, version: "0.8.1.post3" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -4452,6 +4459,7 @@ dsr1-fp8-h100-dynamo-sglang: runner: h100-multinode precision: fp8 framework: dynamo-sglang + router: { name: dynamo-router, version: "0.8.0" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -4614,6 +4622,7 @@ dsr1-fp4-gb200-dynamo-trt: runner: gb200 precision: fp4 framework: dynamo-trt + router: { name: dynamo-router, version: "0.8.1.post2" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -4972,6 +4981,7 @@ dsr1-fp8-gb200-dynamo-trt: runner: gb200 precision: fp8 framework: dynamo-trt + router: { name: dynamo-router, version: "0.8.1.post2" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -5401,6 +5411,7 @@ dsr1-fp8-gb200-dynamo-sglang: runner: gb200 precision: fp8 framework: dynamo-sglang + router: { name: dynamo-router, version: "0.8.1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -5531,6 +5542,7 @@ dsr1-fp8-gb300-dynamo-sglang: runner: gb300 precision: fp8 framework: dynamo-sglang + router: { name: dynamo-router, version: "0.8.0" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -5645,6 +5657,7 @@ dsr1-fp4-gb200-dynamo-sglang: runner: gb200 precision: fp4 framework: dynamo-sglang + router: { name: dynamo-router, version: "0.8.1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -5761,6 +5774,7 @@ dsr1-fp4-gb300-dynamo-trt: runner: gb300 precision: fp4 framework: dynamo-trt + router: { name: dynamo-router, version: "0.8.1.post2" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -6189,6 +6203,7 @@ dsr1-fp4-gb300-dynamo-sglang: runner: gb300 precision: fp4 framework: dynamo-sglang + router: { name: dynamo-router, version: "0.8.1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -6305,6 +6320,7 @@ dsr1-fp8-gb300-dynamo-trt: runner: gb300-nv precision: fp8 framework: dynamo-trt + router: { name: dynamo-router, version: "v1.3.0-dev.1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -6718,6 +6734,7 @@ dsr1-fp8-h200-dynamo-sglang: runner: h200-multinode precision: fp8 framework: dynamo-sglang + router: { name: dynamo-router, version: "0.8.0" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -6976,6 +6993,7 @@ dsr1-fp4-b200-dynamo-sglang: runner: b200-multinode precision: fp4 framework: dynamo-sglang + router: { name: dynamo-router, version: "0.8.0" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -7130,6 +7148,7 @@ dsr1-fp8-b200-dynamo-sglang: dp-attn: false additional-settings: - "CONFIG_FILE=recipes/b200-fp8/1k1k.yaml:zip_override_stp_lowlat[0]" + router: { name: dynamo-router, version: "0.8.0" } decode: num-worker: 1 tp: 8 @@ -7143,6 +7162,7 @@ dsr1-fp8-b200-dynamo-sglang: dp-attn: false additional-settings: - "CONFIG_FILE=recipes/b200-fp8/1k1k.yaml:zip_override_stp_lowlat[1]" + router: { name: dynamo-router, version: "0.8.0" } decode: num-worker: 3 tp: 8 @@ -7156,6 +7176,7 @@ dsr1-fp8-b200-dynamo-sglang: dp-attn: true additional-settings: - "CONFIG_FILE=recipes/b200-fp8/1k1k.yaml:zip_override_stp_maxtpt[0]" + router: { name: dynamo-router, version: "0.8.0" } decode: num-worker: 5 tp: 8 @@ -7169,6 +7190,7 @@ dsr1-fp8-b200-dynamo-sglang: dp-attn: true additional-settings: - "CONFIG_FILE=recipes/b200-fp8/1k1k.yaml:zip_override_stp_maxtpt[1]" + router: { name: dynamo-router, version: "0.8.0" } decode: num-worker: 5 tp: 8 @@ -7187,6 +7209,7 @@ dsr1-fp8-b200-dynamo-sglang: additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_lowlat_0.yaml - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_lowlat_0.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 3 tp: 8 @@ -7201,6 +7224,7 @@ dsr1-fp8-b200-dynamo-sglang: additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_lowlat_1.yaml - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_lowlat_1.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 4 tp: 8 @@ -7215,6 +7239,7 @@ dsr1-fp8-b200-dynamo-sglang: additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_lowlat_2.yaml - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_lowlat_2.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 6 tp: 8 @@ -7230,6 +7255,7 @@ dsr1-fp8-b200-dynamo-sglang: additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_0.yaml - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_maxtpt_0.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 2 tp: 8 @@ -7244,6 +7270,7 @@ dsr1-fp8-b200-dynamo-sglang: additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_1.yaml - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_maxtpt_1.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 1 tp: 8 @@ -7258,6 +7285,7 @@ dsr1-fp8-b200-dynamo-sglang: additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_2.yaml - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_maxtpt_2.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 1 tp: 8 @@ -7272,6 +7300,7 @@ dsr1-fp8-b200-dynamo-sglang: additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_3.yaml - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_maxtpt_3.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 1 tp: 8 @@ -7303,6 +7332,7 @@ dsr1-fp8-b200-dynamo-sglang-mtp: dp-attn: false additional-settings: - "CONFIG_FILE=recipes/b200-fp8/1k1k.yaml:zip_override_mtp_lowlat[0]" + router: { name: dynamo-router, version: "0.8.0" } decode: num-worker: 1 tp: 8 @@ -7318,6 +7348,7 @@ dsr1-fp8-b200-dynamo-sglang-mtp: dp-attn: false additional-settings: - "CONFIG_FILE=recipes/b200-fp8/1k1k.yaml:zip_override_mtp_lowlat[1]" + router: { name: dynamo-router, version: "0.8.0" } decode: num-worker: 3 tp: 8 @@ -7333,6 +7364,7 @@ dsr1-fp8-b200-dynamo-sglang-mtp: dp-attn: true additional-settings: - "CONFIG_FILE=recipes/b200-fp8/1k1k.yaml:zip_override_mtp_maxtpt[1]" + router: { name: dynamo-router, version: "0.8.0" } decode: num-worker: 5 tp: 8 @@ -7348,6 +7380,7 @@ dsr1-fp8-b200-dynamo-sglang-mtp: dp-attn: true additional-settings: - "CONFIG_FILE=recipes/b200-fp8/1k1k.yaml:zip_override_mtp_maxtpt[2]" + router: { name: dynamo-router, version: "0.8.0" } decode: num-worker: 5 tp: 8 @@ -7363,6 +7396,7 @@ dsr1-fp8-b200-dynamo-sglang-mtp: dp-attn: true additional-settings: - "CONFIG_FILE=recipes/b200-fp8/1k1k.yaml:override_mtp_maxtpt_1p2d" + router: { name: dynamo-router, version: "0.8.0" } decode: num-worker: 2 tp: 8 @@ -7382,6 +7416,7 @@ dsr1-fp8-b200-dynamo-sglang-mtp: additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_lowlat_0.yaml - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_lowlat_0.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 3 tp: 8 @@ -7397,6 +7432,7 @@ dsr1-fp8-b200-dynamo-sglang-mtp: additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_lowlat_1.yaml - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_lowlat_1.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 4 tp: 8 @@ -7412,6 +7448,7 @@ dsr1-fp8-b200-dynamo-sglang-mtp: additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_lowlat_2.yaml - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_lowlat_2.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 6 tp: 8 @@ -7428,6 +7465,7 @@ dsr1-fp8-b200-dynamo-sglang-mtp: additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_0.yaml - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_maxtpt_0.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 2 tp: 8 @@ -7443,6 +7481,7 @@ dsr1-fp8-b200-dynamo-sglang-mtp: additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_1.yaml - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_maxtpt_1.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 1 tp: 8 @@ -7458,6 +7497,7 @@ dsr1-fp8-b200-dynamo-sglang-mtp: additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_2.yaml - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_maxtpt_2.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 1 tp: 8 @@ -7473,6 +7513,7 @@ dsr1-fp8-b200-dynamo-sglang-mtp: additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_3.yaml - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_maxtpt_3.yaml" + router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 1 tp: 8 @@ -7486,6 +7527,7 @@ dsr1-fp4-b200-dynamo-sglang-mtp: runner: b200-multinode precision: fp4 framework: dynamo-sglang + router: { name: dynamo-router, version: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -7697,6 +7739,7 @@ kimik2.5-fp4-gb200-dynamo-trt: runner: gb200 precision: fp4 framework: dynamo-trt + router: { name: dynamo-router, version: "v1.3.0-dev.1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -8069,6 +8112,7 @@ kimik2.5-fp4-gb300-dynamo-trt: runner: gb300 precision: fp4 framework: dynamo-trt + router: { name: dynamo-router, version: "v1.3.0-dev.1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -8413,6 +8457,7 @@ kimik2.5-fp4-gb200-dynamo-vllm: runner: gb200 precision: fp4 framework: dynamo-vllm + router: { name: dynamo-router, version: "1.2.1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -8536,6 +8581,7 @@ dsv4-fp4-b200-dynamo-vllm: runner: b200-multinode precision: fp4 framework: dynamo-vllm + router: { name: dynamo-router, version: "1.2.0.dev20260426" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -8619,6 +8665,7 @@ dsv4-fp4-gb200-dynamo-vllm: runner: gb200 precision: fp4 framework: dynamo-vllm + router: { name: dynamo-router, version: "1.2.0.dev20260426" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -8720,6 +8767,7 @@ dsv4-fp4-gb200-llmd-vllm: runner: gb200 precision: fp4 framework: llmd-vllm + router: { name: llm-d-router, version: "0.9.0" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -8845,6 +8893,7 @@ dsv4-fp4-gb200-dynamo-vllm-mtp2: runner: gb200 precision: fp4 framework: dynamo-vllm + router: { name: dynamo-router, version: "1.2.0.dev20260426" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -8926,6 +8975,7 @@ dsv4-fp4-gb200-dynamo-sglang: runner: gb200 precision: fp4 framework: dynamo-sglang + router: { name: dynamo-router, version: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" } kv-p2p-transfer: mooncake multinode: true disagg: true @@ -9054,6 +9104,7 @@ qwen3.5-fp8-gb200-dynamo-sglang: runner: gb200 precision: fp8 framework: dynamo-sglang + router: { name: dynamo-router, version: "46520ca59afe992fb5ef61b3197b2316f8df9b2b" } kv-p2p-transfer: mooncake multinode: true disagg: true @@ -9164,6 +9215,7 @@ dsv4-fp4-gb200-dynamo-sglang-mtp: runner: gb200 precision: fp4 framework: dynamo-sglang + router: { name: dynamo-router, version: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" } kv-p2p-transfer: mooncake multinode: true disagg: true @@ -9301,6 +9353,7 @@ glm5-fp4-gb200-dynamo-sglang-mtp: runner: gb200 precision: fp4 framework: dynamo-sglang + router: { name: dynamo-router, version: "1.2.1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -9585,6 +9638,7 @@ dsv4-fp4-b300-dynamo-vllm: runner: b300 precision: fp4 framework: dynamo-vllm + router: { name: dynamo-router, version: "1.2.0.dev20260426" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -9642,6 +9696,7 @@ dsv4-fp4-gb300-dynamo-vllm: runner: gb300-nv precision: fp4 framework: dynamo-vllm + router: { name: dynamo-router, version: "1.2.0.dev20260426" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -9736,6 +9791,7 @@ dsv4-fp4-gb300-dynamo-trt: runner: gb300-nv precision: fp4 framework: dynamo-trt + router: { name: dynamo-router, version: "v1.3.0-deepseek-v4-dev.1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -10133,6 +10189,7 @@ dsv4-fp4-gb300-dynamo-trt-mtp: runner: gb300-nv precision: fp4 framework: dynamo-trt + router: { name: dynamo-router, version: "v1.3.0-deepseek-v4-dev.1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -10557,6 +10614,7 @@ dsv4-fp4-gb300-dynamo-sglang: runner: gb300 precision: fp4 framework: dynamo-sglang + router: { name: dynamo-router, version: "81d0555ee23519cea80a42b4fe824e30368b7300" } kv-p2p-transfer: mooncake multinode: true disagg: true @@ -10672,6 +10730,7 @@ glm5-fp8-b200-dynamo-sglang: runner: b200-dgxc precision: fp8 framework: dynamo-sglang + router: { name: dynamo-router, version: "0.8.0" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -10887,6 +10946,7 @@ dsv4-fp4-gb300-dynamo-sglang-mtp: runner: gb300 precision: fp4 framework: dynamo-sglang + router: { name: dynamo-router, version: "81d0555ee23519cea80a42b4fe824e30368b7300" } kv-p2p-transfer: mooncake multinode: true disagg: true @@ -11032,7 +11092,7 @@ kimik2.5-int4-h100-vllm: - dram-utilization: 0.80 search-space: - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 16, 20] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 2, 4, 8, 12, 16, 20] } + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2, 4, 8, 12, 16, 20] } qwen3.5-fp8-h100-sglang: image: lmsysorg/sglang:v0.5.14-cu130 @@ -11081,6 +11141,7 @@ glm5-fp4-gb200-dynamo-sglang: runner: gb200 precision: fp4 framework: dynamo-sglang + router: { name: dynamo-router, version: "1.2.1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -11348,6 +11409,7 @@ qwen3.5-fp4-gb300-dynamo-sglang: runner: gb300 precision: fp4 framework: dynamo-sglang + router: { name: dynamo-router, version: "1.1.0" } multinode: true disagg: true scenarios: @@ -11450,6 +11512,7 @@ glm5-fp4-gb300-dynamo-sglang: dp-attn: true additional-settings: - "CONFIG_FILE=recipes/gb300-fp4/glm5.yaml:zip_override_8k1k_hightpt[0]" + router: { name: dynamo-router, version: "0.8.0" } decode: num-worker: 1 tp: 32 @@ -11464,6 +11527,7 @@ glm5-fp4-gb300-dynamo-sglang: dp-attn: true additional-settings: - "CONFIG_FILE=recipes/gb300-fp4/glm5.yaml:zip_override_8k1k_hightpt[1]" + router: { name: dynamo-router, version: "0.8.0" } decode: num-worker: 1 tp: 32 @@ -11478,6 +11542,7 @@ glm5-fp4-gb300-dynamo-sglang: dp-attn: true additional-settings: - "CONFIG_FILE=recipes/gb300-fp4/glm5.yaml:zip_override_8k1k_hightpt[2]" + router: { name: dynamo-router, version: "0.8.0" } decode: num-worker: 1 tp: 32 @@ -11496,6 +11561,7 @@ glm5-fp4-gb300-dynamo-sglang: dp-attn: true additional-settings: - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml" + router: { name: dynamo-router, version: "1.1.0" } decode: num-worker: 3 tp: 4 @@ -11510,6 +11576,7 @@ glm5-fp4-gb300-dynamo-sglang: dp-attn: true additional-settings: - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml" + router: { name: dynamo-router, version: "1.1.0" } decode: num-worker: 5 tp: 4 @@ -11524,6 +11591,7 @@ glm5-fp4-gb300-dynamo-sglang: dp-attn: true additional-settings: - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml" + router: { name: dynamo-router, version: "1.1.0" } decode: num-worker: 9 tp: 4 @@ -11538,6 +11606,7 @@ glm5-fp4-gb300-dynamo-sglang: dp-attn: true additional-settings: - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_3.yaml" + router: { name: dynamo-router, version: "1.1.0" } decode: num-worker: 15 tp: 4 @@ -11552,6 +11621,7 @@ glm5-fp4-gb300-dynamo-sglang: dp-attn: true additional-settings: - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_4.yaml" + router: { name: dynamo-router, version: "1.1.0" } decode: num-worker: 17 tp: 4 @@ -11570,6 +11640,7 @@ glm5-fp4-gb300-dynamo-sglang: dp-attn: true additional-settings: - "CONFIG_FILE=recipes/gb300-fp4/glm5.yaml:zip_override_1k1k_hightpt[0]" + router: { name: dynamo-router, version: "0.8.0" } decode: num-worker: 1 tp: 32 @@ -11584,6 +11655,7 @@ glm5-fp4-gb300-dynamo-sglang: dp-attn: true additional-settings: - "CONFIG_FILE=recipes/gb300-fp4/glm5.yaml:zip_override_1k1k_hightpt[1]" + router: { name: dynamo-router, version: "0.8.0" } decode: num-worker: 1 tp: 32 @@ -11598,6 +11670,7 @@ glm5-fp4-gb300-dynamo-sglang: dp-attn: true additional-settings: - "CONFIG_FILE=recipes/gb300-fp4/glm5.yaml:zip_override_1k1k_hightpt[2]" + router: { name: dynamo-router, version: "0.8.0" } decode: num-worker: 1 tp: 32 @@ -11616,6 +11689,7 @@ glm5-fp4-gb300-dynamo-sglang: dp-attn: true additional-settings: - "CONFIG_FILE=recipes/gb300-fp4/glm5.yaml:zip_override_1k1k_lowlat[0]" + router: { name: dynamo-router, version: "0.8.0" } decode: num-worker: 17 tp: 4 @@ -11630,6 +11704,7 @@ glm5-fp4-gb300-dynamo-sglang: dp-attn: true additional-settings: - "CONFIG_FILE=recipes/gb300-fp4/glm5.yaml:zip_override_1k1k_lowlat[1]" + router: { name: dynamo-router, version: "0.8.0" } decode: num-worker: 17 tp: 4 @@ -11643,6 +11718,7 @@ glm5-fp8-gb200-dynamo-sglang: runner: gb200 precision: fp8 framework: dynamo-sglang + router: { name: dynamo-router, version: "1.2.1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -11868,6 +11944,7 @@ glm5-fp4-gb300-dynamo-sglang-mtp: runner: gb300-nv precision: fp4 framework: dynamo-sglang + router: { name: dynamo-router, version: "0.8.0" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -12137,6 +12214,7 @@ glm5-fp8-gb300-dynamo-sglang: runner: gb300-nv precision: fp8 framework: dynamo-sglang + router: { name: dynamo-router, version: "0.8.0" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -12348,6 +12426,7 @@ glm5-fp8-gb300-dynamo-sglang-mtp: runner: gb300-nv precision: fp8 framework: dynamo-sglang + router: { name: dynamo-router, version: "1.2.1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -12559,9 +12638,9 @@ kimik2.5-fp4-b200-vllm-agentic-lmcache: - dram-utilization: 0.80 search-space: - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 24] } - - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: lmcache }, conc-list: [16, 24, 32, 36] } + - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.1" }, conc-list: [16, 24, 32, 36] } - { tp: 4, ep: 1, kv-offloading: none, conc-list: [8, 12, 14, 16, 18, 20] } - - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: { name: lmcache }, conc-list: [12, 14, 16, 18, 20, 22, 24, 32] } + - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.1" }, conc-list: [12, 14, 16, 18, 20, 22, 24, 32] } # CONC range conservative for H100's 80 GB HBM3 under the long-ISL with- # subagents corpus. hicache arm capped at conc 16 since high-conc + hicache @@ -12574,6 +12653,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic: runner: cluster:gb300-nv precision: fp4 framework: dynamo-vllm + router: { name: dynamo-router, version: "1.2.0.dev20260426" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -12657,6 +12737,7 @@ minimaxm3-fp8-b300-dynamo-vllm: runner: b300 precision: fp8 framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260614" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -12874,6 +12955,7 @@ minimaxm3-fp4-b300-dynamo-vllm: runner: b300 precision: fp4 framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260614" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -13091,6 +13173,7 @@ minimaxm3-fp8-gb300-dynamo-vllm: runner: gb300-nv precision: fp8 framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260614" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -13365,6 +13448,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: runner: gb200 precision: fp8 framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260614" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -13932,6 +14016,7 @@ kimik2.5-fp4-gb300-dynamo-vllm: runner: gb300-nv precision: fp4 framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260601" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -14105,8 +14190,8 @@ minimaxm3-fp8-h100-vllm-agentic: search-space: - { tp: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - { tp: 8, ep: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake }, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake }, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } + - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } minimaxm3-fp8-h200-vllm-agentic: image: vllm/vllm-openai:nightly-04c2a8deac44fdb1ca3e2b5ec3e6bf16f3f6a914 @@ -14126,7 +14211,7 @@ minimaxm3-fp8-h200-vllm-agentic: search-space: - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } - { tp: 8, ep: 8, kv-offloading: none, conc-list: [2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } - - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake }, conc-list: [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } + - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } dsv4-fp4-b200-sglang-agentic-hicache: image: lmsysorg/sglang:nightly-dev-cu13-20260707-b4155233 @@ -14144,8 +14229,8 @@ dsv4-fp4-b200-sglang-agentic-hicache: - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [8, 10, 16, 32, 40, 44] } # DEP without HiCache is already degraded at conc 52 (2,098 tok/s/GPU), # so resolve its pre-52 cliff instead of repeating the collapsed tail. - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 24, 32, 38, 44, 48, 50, 52] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 32, 38, 44, 50, 56, 64, 66, 68] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 24, 32, 38, 44, 48, 50, 52], router: { name: sglang-router, version: "0.3.2" } } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 32, 38, 44, 50, 56, 64, 66, 68], router: { name: sglang-router, version: "0.3.2" } } # GB200 DeepSeek-V4 disaggregated AgentX frontier. The 3P/2D TEP8/TP8 curve # covers the middle/high-interactivity range omitted by the one-decode DEP @@ -14164,9 +14249,9 @@ dsv4-fp4-b300-sglang-agentic-hicache: search-space: - { tp: 4, kv-offloading: none, conc-list: [1, 4, 8, 16, 20, 24, 32] } - { tp: 8, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 40, 48, 52, 56, 60, 64, 72] } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, conc-list: [8, 16, 24, 32, 40, 64] } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 40, 48, 56, 64, 72, 80, 88, 96, 128] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [52, 72, 100, 128, 144, 196, 512] } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, conc-list: [8, 16, 24, 32, 40, 64], router: { name: sglang-router, version: "0.3.2" } } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 40, 48, 56, 64, 72, 80, 88, 96, 128], router: { name: sglang-router, version: "0.3.2" } } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [52, 72, 100, 128, 144, 196, 512], router: { name: sglang-router, version: "0.3.2" } } # DEP8 prefill uses an 8K batch because 16K OOMs in the FP4 MoE intermediate; # decode uses FULL_DECODE_ONLY after the controlled graph test restored decode @@ -14179,6 +14264,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-3p2d-tep8-tp8: runner: cluster:gb200-nv precision: fp4 framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260618" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -14236,6 +14322,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep8: runner: cluster:gb200-nv precision: fp4 framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260618" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -14293,6 +14380,7 @@ glm5-fp4-gb300-dynamo-trt-mtp: runner: gb300-nv precision: fp4 framework: dynamo-trt + router: { name: dynamo-router, version: "v1.3.0-dev.1" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -14621,6 +14709,7 @@ glm5-fp4-gb300-dynamo-trt-mtp: tp: 16 ep: 16 dp-attn: true + qwen3.5-fp8-gb300-dynamo-sglang: image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 model: Qwen/Qwen3.5-397B-A17B-FP8 @@ -14628,9 +14717,10 @@ qwen3.5-fp8-gb300-dynamo-sglang: runner: gb300 precision: fp8 framework: dynamo-sglang + router: { name: dynamo-router, version: "46520ca59afe992fb5ef61b3197b2316f8df9b2b" } + kv-p2p-transfer: mooncake multinode: true disagg: true - kv-p2p-transfer: mooncake scenarios: fixed-seq-len: - isl: 1024