From 402f750708e81770b18d5a38874137f0d2ed70ca Mon Sep 17 00:00:00 2001 From: Hongxia Yang Date: Thu, 16 Jul 2026 02:12:17 +0000 Subject: [PATCH 01/11] [AMD] optimze kimi2.5-fp4 disagg --- .../multi_node/amd_utils/models_vllm.yaml | 10 ++- configs/amd-master.yaml | 62 ++++++++++++++++--- perf-changelog.yaml | 9 +++ 3 files changed, 70 insertions(+), 11 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/models_vllm.yaml b/benchmarks/multi_node/amd_utils/models_vllm.yaml index 9c046b4cf..6d180ab42 100644 --- a/benchmarks/multi_node/amd_utils/models_vllm.yaml +++ b/benchmarks/multi_node/amd_utils/models_vllm.yaml @@ -25,9 +25,13 @@ amd-Llama-3.3-70B-Instruct-FP8-KV: env: "VLLM_USE_V1=1 VLLM_V1_USE_PREFILL_DECODE_ATTENTION=1 AMDGCN_USE_BUFFER_OPS=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_USE_AITER_TRITON_ROPE=1 TRITON_HIP_ASYNC_COPY_BYPASS_PERMUTE=1 TRITON_HIP_USE_ASYNC_COPY=1 TRITON_HIP_USE_BLOCK_PINGPONG=1 TRITON_HIP_ASYNC_FAST_SWIZZLE=1" Kimi-K2.5-MXFP4: - prefill_flags: "--tensor-parallel-size 8 --compilation-config '{\"cudagraph_mode\":\"PIECEWISE\"}' --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --mm-encoder-tp-mode data" - decode_flags: "--tensor-parallel-size 8 --enable-expert-parallel --all2all-backend mori_low_latency --compilation-config '{\"cudagraph_mode\":\"PIECEWISE\"}' --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --mm-encoder-tp-mode data" - env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_PAGED_ATTN=0 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_USE_AITER_TRITON_SILU_MUL=0 VLLM_ENGINE_READY_TIMEOUT_S=3600" + # Serve flags/env synced with single-node kimik2.5_fp4_mi355x.sh (INT4 + # QuickReduce, AITER defaults). max-model-len 9472 covers both sweep scenarios + # (isl+osl+256). EP off: single-node TP8 sweep showed it -14% to -27% slower. + # decode keeps --all2all-backend mori_low_latency (no-op unless EP restored). + prefill_flags: "--tensor-parallel-size 8 --compilation-config '{\"cudagraph_mode\":\"PIECEWISE\"}' --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 9472 --mm-encoder-tp-mode data" + decode_flags: "--tensor-parallel-size 8 --all2all-backend mori_low_latency --compilation-config '{\"cudagraph_mode\":\"PIECEWISE\"}' --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 9472 --mm-encoder-tp-mode data" + env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 HSA_NO_SCRATCH_RECLAIM=1 VLLM_ENGINE_READY_TIMEOUT_S=3600" hf_dir: "models--amd--Kimi-K2.5-MXFP4" MiniMax-M2.5: diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 4b1ea4aed..2cf94139e 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1185,7 +1185,7 @@ dsr1-fp8-mi355x-sglang-disagg-mtp: - "DECODE_MTP_SIZE=2" kimik2.5-fp4-mi355x-vllm-disagg: - image: vllm/vllm-openai-rocm:v0.24.0 + image: vllm/vllm-openai-rocm:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9 model: amd/Kimi-K2.5-MXFP4 model-prefix: kimik2.5 runner: mi355x-disagg @@ -1197,15 +1197,43 @@ kimik2.5-fp4-mi355x-vllm-disagg: disagg: true scenarios: fixed-seq-len: + # P/D topology (based on PR #2172 + single-node TP8-vs-TP4 decode sweep): + # * 1P(TP4)/1D(TP4) is the default for every concurrency EXCEPT 16. + # A real-weight sweep showed TP4 decode beats TP8 at conc 1-8 (+5-7% + # tok/s, ~7% lower TPOT) and matches it elsewhere, using 1/4 the decode + # GPUs of the 2D(TP8) layout. + # * 1P(TP4)/2D(TP8) is used ONLY at conc 16, the measured crossover where + # TP8 decode pulls ahead (+6% tok/s, lower TPOT). + # Both layouts keep prefill+decode nodes <= 3 for the 3-node mi355x-disagg + # pool. Expert parallelism is OFF (ep:1, matching PR #2172): a single-node + # TP8 sweep showed EP -14% to -27% slower than dense. - isl: 1024 osl: 1024 search-space: - # 1P2D: 1 prefill node (co-located with proxy) + 2 decode nodes = 3 nodes total + # 1P(TP4) 1D(TP4) = 2 nodes. Owns all concs except 16. conc 512 fits KV + # (~663 resident cap at 2k tokens/req, verified single-node TP4). - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] + conc-list: [ 1, 2, 4, 8, 32, 64, 128, 256, 512 ] prefill: num-worker: 1 - tp: 8 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + # 1P(TP4) 2D(TP8) = 3 nodes. Only conc 16 (measured TP8 crossover). + - spec-decoding: "none" + conc-list: [ 16 ] + prefill: + num-worker: 1 + tp: 4 ep: 1 dp-attn: false additional-settings: @@ -1213,7 +1241,7 @@ kimik2.5-fp4-mi355x-vllm-disagg: decode: num-worker: 2 tp: 8 - ep: 8 + ep: 1 dp-attn: false additional-settings: - "DECODE_NODES=2" @@ -1221,11 +1249,29 @@ kimik2.5-fp4-mi355x-vllm-disagg: - isl: 8192 osl: 1024 search-space: + # 1P(TP4) 1D(TP4) = 2 nodes. Owns all concs except 16. - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] + conc-list: [ 1, 2, 4, 8, 32, 64, 128, 256 ] prefill: num-worker: 1 - tp: 8 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + # 1P(TP4) 2D(TP8) = 3 nodes. Only conc 16 (measured TP8 crossover). + - spec-decoding: "none" + conc-list: [ 16 ] + prefill: + num-worker: 1 + tp: 4 ep: 1 dp-attn: false additional-settings: @@ -1233,7 +1279,7 @@ kimik2.5-fp4-mi355x-vllm-disagg: decode: num-worker: 2 tp: 8 - ep: 8 + ep: 1 dp-attn: false additional-settings: - "DECODE_NODES=2" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index cd0320fa9..65454fee2 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4788,3 +4788,12 @@ - "Add EAGLE3 speculative-decoding arm for the Kimi K2.6 NVFP4 B300 AgentX recipe (draft lightseekorg/kimi-k2.6-eagle3-mla, TOKENSPEED_MLA attention backend with TRT-LLM ragged MLA kernel)." - "TP8/TP4 GPU-only KV points plus a TP4 native CPU-offload ladder via SimpleCPUOffloadConnector with lazy_offload off; TP4/DCP4 high-concurrency points (conc 32/64) using num_speculative_tokens=3 and synthetic_acceptance_length=2.88." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2222 + +- config-keys: + - kimik2.5-fp4-mi355x-vllm-disagg + description: + - "Bump image to vllm/vllm-openai-rocm:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9" + - "Sync per-worker vLLM serve flags/env with the single-node kimik2.5-fp4-mi355x-vllm recipe (VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4, HSA_NO_SCRATCH_RECLAIM=1, AITER defaults, --max-model-len 9472) in benchmarks/multi_node/amd_utils/models_vllm.yaml" + - "Retune P/D search space to all-TP4 prefill/decode, tuning the prefill:decode worker ratio (1P1D TP4 for all concs except 16; 1P(TP4)/2D(TP8) only at conc 16, the measured TP8 crossover); all layouts keep prefill+decode nodes <= 3" + - "Turn expert parallelism off (ep:1): single-node TP8 real-weight sweep showed EP -14% to -27% slower than dense" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From f2dd492281571c6e144308a5c8ea37782960d859 Mon Sep 17 00:00:00 2001 From: Hongxia Yang Date: Thu, 16 Jul 2026 17:31:56 +0000 Subject: [PATCH 02/11] more tuning on disagg parameters, removing tp8 because of efficiency --- configs/amd-master.yaml | 45 ++++++++++------------------------------- 1 file changed, 11 insertions(+), 34 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 2cf94139e..4b65c9f57 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1197,23 +1197,16 @@ kimik2.5-fp4-mi355x-vllm-disagg: disagg: true scenarios: fixed-seq-len: - # P/D topology (based on PR #2172 + single-node TP8-vs-TP4 decode sweep): - # * 1P(TP4)/1D(TP4) is the default for every concurrency EXCEPT 16. - # A real-weight sweep showed TP4 decode beats TP8 at conc 1-8 (+5-7% - # tok/s, ~7% lower TPOT) and matches it elsewhere, using 1/4 the decode - # GPUs of the 2D(TP8) layout. - # * 1P(TP4)/2D(TP8) is used ONLY at conc 16, the measured crossover where - # TP8 decode pulls ahead (+6% tok/s, lower TPOT). - # Both layouts keep prefill+decode nodes <= 3 for the 3-node mi355x-disagg - # pool. Expert parallelism is OFF (ep:1, matching PR #2172): a single-node - # TP8 sweep showed EP -14% to -27% slower than dense. + # P/D topology: all workers TP4. A real-weight sweep showed TP8 decode does + # not help over TP4 (TP4 wins at conc 1-8, matches elsewhere, using far fewer + # GPUs), so both prefill and decode run TP4. - isl: 1024 osl: 1024 search-space: - # 1P(TP4) 1D(TP4) = 2 nodes. Owns all concs except 16. conc 512 fits KV + # 1P(TP4) 1D(TP4) = 2 nodes. conc 512 fits KV # (~663 resident cap at 2k tokens/req, verified single-node TP4). - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 32, 64, 128, 256, 512 ] + conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256, 512 ] prefill: num-worker: 1 tp: 4 @@ -1228,30 +1221,13 @@ kimik2.5-fp4-mi355x-vllm-disagg: dp-attn: false additional-settings: - "DECODE_NODES=1" - # 1P(TP4) 2D(TP8) = 3 nodes. Only conc 16 (measured TP8 crossover). - - spec-decoding: "none" - conc-list: [ 16 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - isl: 8192 osl: 1024 search-space: - # 1P(TP4) 1D(TP4) = 2 nodes. Owns all concs except 16. + # 1P(TP4) 1D(TP4) = 2 nodes. - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 32, 64, 128, 256 ] + conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256 ] prefill: num-worker: 1 tp: 4 @@ -1266,9 +1242,10 @@ kimik2.5-fp4-mi355x-vllm-disagg: dp-attn: false additional-settings: - "DECODE_NODES=1" - # 1P(TP4) 2D(TP8) = 3 nodes. Only conc 16 (measured TP8 crossover). + # 1P(TP4) 2D(TP4) = 3 nodes. High-conc KV relief: doubles decode KV so the + # 8k/1k tail isn't KV-capped like 1D. - spec-decoding: "none" - conc-list: [ 16 ] + conc-list: [ 256, 512 ] prefill: num-worker: 1 tp: 4 @@ -1278,7 +1255,7 @@ kimik2.5-fp4-mi355x-vllm-disagg: - "PREFILL_NODES=1" decode: num-worker: 2 - tp: 8 + tp: 4 ep: 1 dp-attn: false additional-settings: From 86ec518e61b46f70f70dda3487d131eab47720ca Mon Sep 17 00:00:00 2001 From: Hongxia Yang Date: Thu, 16 Jul 2026 17:37:06 +0000 Subject: [PATCH 03/11] docs(perf-changelog): align kimik2.5-fp4-mi355x-vllm-disagg entry with all-TP4 config Update the changelog description to match the final config: TP8 dropped everywhere (real-weight sweep showed no per-GPU benefit), 1P(TP4)/1D(TP4) covers the full curve, 1P(TP4)/2D(TP4) adds decode KV headroom for the KV-bound 8k/1k tail (conc 256/512), EP off. Note ~2-3x tok/s/GPU vs the prior 1P2D TP8/EP8 baseline (PR #1585). Co-authored-by: Cursor --- perf-changelog.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 65454fee2..60206febf 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4794,6 +4794,7 @@ description: - "Bump image to vllm/vllm-openai-rocm:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9" - "Sync per-worker vLLM serve flags/env with the single-node kimik2.5-fp4-mi355x-vllm recipe (VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4, HSA_NO_SCRATCH_RECLAIM=1, AITER defaults, --max-model-len 9472) in benchmarks/multi_node/amd_utils/models_vllm.yaml" - - "Retune P/D search space to all-TP4 prefill/decode, tuning the prefill:decode worker ratio (1P1D TP4 for all concs except 16; 1P(TP4)/2D(TP8) only at conc 16, the measured TP8 crossover); all layouts keep prefill+decode nodes <= 3" + - "Retune P/D search space to all-TP4 prefill/decode (dropped TP8 — a real-weight sweep showed TP8 decode is no better than TP4 on tok/s/GPU): 1P(TP4)/1D(TP4) covers the full curve; 1P(TP4)/2D(TP4) adds decode KV headroom for the KV-bound 8k/1k tail (conc 256/512). All layouts keep prefill+decode nodes <= 3" - "Turn expert parallelism off (ep:1): single-node TP8 real-weight sweep showed EP -14% to -27% slower than dense" + - "~2-3x tok/s/GPU vs the prior 1P2D TP8/EP8 baseline (PR #1585) across most concurrencies (8 GPU vs 24 GPU)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From 8eb5409bcfdb3bd051d36fb50d77471395ce5096 Mon Sep 17 00:00:00 2001 From: Hongxia Yang Date: Thu, 16 Jul 2026 18:05:27 +0000 Subject: [PATCH 04/11] docs(perf-changelog): set kimik2.5-fp4-mi355x-vllm-disagg pr-link to #2247 Co-authored-by: Cursor --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4973a3ebb..2c2200375 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4903,4 +4903,4 @@ - "Retune P/D search space to all-TP4 prefill/decode (dropped TP8 — a real-weight sweep showed TP8 decode is no better than TP4 on tok/s/GPU): 1P(TP4)/1D(TP4) covers the full curve; 1P(TP4)/2D(TP4) adds decode KV headroom for the KV-bound 8k/1k tail (conc 256/512). All layouts keep prefill+decode nodes <= 3" - "Turn expert parallelism off (ep:1): single-node TP8 real-weight sweep showed EP -14% to -27% slower than dense" - "~2-3x tok/s/GPU vs the prior 1P2D TP8/EP8 baseline (PR #1585) across most concurrencies (8 GPU vs 24 GPU)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2247 From 86e8303438f81155b3fc3f36f78d2933cf67aec5 Mon Sep 17 00:00:00 2001 From: Hongxia Yang Date: Fri, 17 Jul 2026 01:01:54 +0000 Subject: [PATCH 05/11] fix(kimik2.5-disagg): re-pin vllm-router to a live nightly tag Co-Authored-By: Claude Fable 5 --- benchmarks/multi_node/amd_utils/job.slurm | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 4638e5cad..b15a28f86 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -293,8 +293,10 @@ export DOCKER_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_${MODEL_NAME}_${S # vLLM external router container. # NOTE: vllm/vllm-router only retains ~16 recent nightlies on Docker Hub; older -# dated tags are garbage-collected (manifest unknown) -VLLM_ROUTER_IMAGE="${VLLM_ROUTER_IMAGE:-vllm/vllm-router:nightly-20260629-e667ebb}" +# dated tags are garbage-collected (manifest unknown). The previous pin +# (nightly-20260629-e667ebb) had already been evicted — re-pin to a live tag +# whenever this starts failing with "manifest unknown" on rank 0. +VLLM_ROUTER_IMAGE="${VLLM_ROUTER_IMAGE:-vllm/vllm-router:nightly-20260716-1fbcde7}" ROUTER_CONT_NAME="router_vllm_${SANITIZED_USER}_${SLURM_JOB_ID}" export RUN_FILE_FULL="$WS_PATH/${RUN_FILE}" From f758269c9be5413ea7184731b45596a278e142ab Mon Sep 17 00:00:00 2001 From: Hongxia Yang Date: Fri, 17 Jul 2026 01:01:54 +0000 Subject: [PATCH 06/11] fix(kimik2.5-disagg): set eval context from ISL/OSL, clamp to served len Co-Authored-By: Claude Fable 5 --- .../multi_node/amd_utils/server_vllm.sh | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/benchmarks/multi_node/amd_utils/server_vllm.sh b/benchmarks/multi_node/amd_utils/server_vllm.sh index f19ce8560..a0e05cf57 100755 --- a/benchmarks/multi_node/amd_utils/server_vllm.sh +++ b/benchmarks/multi_node/amd_utils/server_vllm.sh @@ -341,6 +341,26 @@ if [ "$NODE_RANK" -eq 0 ]; then source /workspace/benchmarks/benchmark_lib.sh + # Derive the eval context from the benchmark shape, mirroring the + # single-node recipe (which sets MAX_MODEL_LEN=$EVAL_MAX_MODEL_LEN). + # Without this, EVAL_MAX_MODEL_LEN is unset here and run_lm_eval + # falls back to 16384, asking lm-eval for max_tokens=12288 while the + # engine was launched with --max-model-len 9472 from models_vllm.yaml + # — every request 400s on context length and the eval fails. + export MODEL="${MODEL_PATH}" + export ISL="${BENCH_INPUT_LEN}" + export OSL="${BENCH_OUTPUT_LEN}" + setup_eval_context + + # The engine's context window is fixed at launch by models_vllm.yaml, + # so never ask lm-eval for more than what is actually served. + SERVED_MAX_LEN=$(echo "$DECODE_SERVER_CONFIG" | grep -oE -- '--max-model-len[[:space:]]+[0-9]+' | awk '{print $2}' | head -1) + if [[ -n "$SERVED_MAX_LEN" && "${EVAL_MAX_MODEL_LEN:-0}" -gt "$SERVED_MAX_LEN" ]]; then + echo "[EVAL] Clamping EVAL_MAX_MODEL_LEN ${EVAL_MAX_MODEL_LEN} -> ${SERVED_MAX_LEN} (engine --max-model-len)" + export EVAL_MAX_MODEL_LEN="$SERVED_MAX_LEN" + fi + echo "[EVAL] EVAL_MAX_MODEL_LEN=${EVAL_MAX_MODEL_LEN} (ISL=${ISL} OSL=${OSL}, served=${SERVED_MAX_LEN:-unknown})" + if [[ -n "${EVAL_CONC:-}" ]]; then export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" else From f4abeb37daa5e0ecbaa075bab0a7aea3c7e1413c Mon Sep 17 00:00:00 2001 From: Hongxia Yang Date: Fri, 17 Jul 2026 01:01:54 +0000 Subject: [PATCH 07/11] perf(kimik2.5-disagg): drop PIECEWISE pin, add fp8 KV and token budgets Co-Authored-By: Claude Fable 5 --- .../multi_node/amd_utils/models_vllm.yaml | 42 ++++++++++++++++++- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/models_vllm.yaml b/benchmarks/multi_node/amd_utils/models_vllm.yaml index 6d180ab42..88d88651f 100644 --- a/benchmarks/multi_node/amd_utils/models_vllm.yaml +++ b/benchmarks/multi_node/amd_utils/models_vllm.yaml @@ -29,8 +29,46 @@ Kimi-K2.5-MXFP4: # QuickReduce, AITER defaults). max-model-len 9472 covers both sweep scenarios # (isl+osl+256). EP off: single-node TP8 sweep showed it -14% to -27% slower. # decode keeps --all2all-backend mori_low_latency (no-op unless EP restored). - prefill_flags: "--tensor-parallel-size 8 --compilation-config '{\"cudagraph_mode\":\"PIECEWISE\"}' --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 9472 --mm-encoder-tp-mode data" - decode_flags: "--tensor-parallel-size 8 --all2all-backend mori_low_latency --compilation-config '{\"cudagraph_mode\":\"PIECEWISE\"}' --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 9472 --mm-encoder-tp-mode data" + # VLLM_ROCM_USE_AITER_RMSNORM=1: the single-node recipe disables AITER RMSNorm + # for TP<8 due to a historical accuracy issue (perf-changelog.yaml, v0.16->0.18 + # era), but on the current nightly a local TP4 GSM8K check gave identical + # accuracy with RMSNorm on vs off (0.9697 vs 0.9697), so keep the AITER RMSNorm + # perf path on rather than pin it off for a regression that no longer reproduces. + # No --compilation-config: take vLLM's default cudagraph_mode + # (FULL_AND_PIECEWISE since 0.11.0), same as the single-node recipe. The + # PIECEWISE pin carried over from the original bring-up (#1569) and forfeited + # full cudagraphs on the decode workers. Measured locally on MI350X/gfx950, + # TP4, 8k/1k, real weights, on this image (output tok/s vs PIECEWISE): + # conc 1 4 16 64 + # FULL_AND_PIECEWISE +15.9% +11.1% +12.6% +1.7% + # FULL_DECODE_ONLY +18.5% +13.6% +11.9% +1.5% + # PIECEWISE loses everywhere; the two full modes tie within run-to-run noise + # (~2%). FULL_DECODE_ONLY captures far cheaper (0.63 GiB/28s vs 7.28 GiB/69s) + # but that buys nothing here — KV cache is 89.33 GiB either way, since graph + # memory comes out of the 10% headroom left by --gpu-memory-utilization 0.90, + # not out of KV. So prefer the default and stay in sync with single-node. + # --kv-cache-dtype fp8 + --max-model-len 32768 + --max-num-seqs 256 + + # --max-num-batched-tokens 32768: measured locally (MI350X/gfx950, TP4, 8k/1k, + # real weights, this image) against the flags above as baseline, total tok/s: + # conc 16 64 128 + # kv fp8 only +9.7% +16.1% +20.5% + # all four +16.2% +19.1% +27.4% <- adopted + # TTFT and TPOT improve too (conc 128: 5721 vs 6975 ms, 65.3 vs 83.5 ms). + # fp8 KV costs no accuracy here: local TP4 GSM8K with this recipe scored + # 0.9697 flexible / 0.9712 strict, identical to the bf16-KV run (0.9697) and + # well clear of the 0.90 gate in utils/evals/thresholds.json. + # fp8 KV wins on two counts: it halves KV read bandwidth (helps at every + # concurrency) and doubles resident capacity (1,358,885 -> 2,730,007 tokens), + # which is what relieves the KV-bound decode tail. The other three add ~+5% + # on top of fp8 alone, mostly the larger prefill token budget at 8k input. + # NOTE --max-num-seqs 256 caps engine concurrency at 256; the 8k/1k conc-512 + # point only exists on the 1P2D arm, where the router splits across two decode + # workers, so no single engine should exceed the cap. Revisit if 1D ever runs + # conc 512. INT4 quick reduce is VLLM_ROCM_QUICK_REDUCE_QUANTIZATION (below): + # AITER_QUICK_REDUCE_QUANTIZATION is not read by the vLLM serve path and + # AITER_MXFP4_INTERMEDIATE does not exist in this build. + prefill_flags: "--tensor-parallel-size 8 --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 32768 --mm-encoder-tp-mode data --kv-cache-dtype fp8 --max-num-seqs 256 --max-num-batched-tokens 32768" + decode_flags: "--tensor-parallel-size 8 --all2all-backend mori_low_latency --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 32768 --mm-encoder-tp-mode data --kv-cache-dtype fp8 --max-num-seqs 256 --max-num-batched-tokens 32768" env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 HSA_NO_SCRATCH_RECLAIM=1 VLLM_ENGINE_READY_TIMEOUT_S=3600" hf_dir: "models--amd--Kimi-K2.5-MXFP4" From 507a931e25246b63acae0e80bddc16eae0197289 Mon Sep 17 00:00:00 2001 From: Hongxia Yang Date: Fri, 17 Jul 2026 01:01:54 +0000 Subject: [PATCH 08/11] docs(perf-changelog): record kimik2.5-fp4-mi355x-vllm-disagg changes Co-Authored-By: Claude Fable 5 --- perf-changelog.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 2c2200375..b7957c2e1 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4903,4 +4903,8 @@ - "Retune P/D search space to all-TP4 prefill/decode (dropped TP8 — a real-weight sweep showed TP8 decode is no better than TP4 on tok/s/GPU): 1P(TP4)/1D(TP4) covers the full curve; 1P(TP4)/2D(TP4) adds decode KV headroom for the KV-bound 8k/1k tail (conc 256/512). All layouts keep prefill+decode nodes <= 3" - "Turn expert parallelism off (ep:1): single-node TP8 real-weight sweep showed EP -14% to -27% slower than dense" - "~2-3x tok/s/GPU vs the prior 1P2D TP8/EP8 baseline (PR #1585) across most concurrencies (8 GPU vs 24 GPU)" + - "Drop --compilation-config '{\"cudagraph_mode\":\"PIECEWISE\"}' from prefill/decode flags to finish the single-node sync (vLLM's default is FULL_AND_PIECEWISE). Local TP4 8k/1k real-weight A/B on MI350X: default beats the PIECEWISE pin by +15.9%/+11.1%/+12.6%/+1.7% output tok/s at conc 1/4/16/64. FULL_DECODE_ONLY ties the default within noise (+18.5%/+13.6%/+11.9%/+1.5%) and captures cheaper (0.63 GiB/28s vs 7.28 GiB/69s), but KV cache is 89.33 GiB either way, so the default is kept to stay in sync with single-node" + - "Re-pin VLLM_ROUTER_IMAGE to vllm/vllm-router:nightly-20260716-1fbcde7 in benchmarks/multi_node/amd_utils/job.slurm; the previous nightly-20260629-e667ebb pin had been garbage-collected from Docker Hub (only ~16 nightlies are retained), so the router container failed to pull on rank 0" + - "Fix the multi-node vLLM eval: server_vllm.sh never set EVAL_MAX_MODEL_LEN, so run_lm_eval fell back to 16384 and requested max_tokens=12288 against an engine served with --max-model-len 9472, 400-ing every gsm8k request. Now derives it via setup_eval_context (ISL+OSL+256, capped at the model's native max) and clamps it to the engine's served --max-model-len" + - "Add --kv-cache-dtype fp8, --max-model-len 32768, --max-num-seqs 256 and --max-num-batched-tokens 32768 to the prefill/decode workers. Local TP4 8k/1k real-weight A/B vs the prior flags (total tok/s): +16.2%/+19.1%/+27.4% at conc 16/64/128, with TTFT -18% and TPOT -22% at conc 128. fp8 KV alone accounts for +9.7%/+16.1%/+20.5% — it halves KV read bandwidth and doubles resident capacity (1,358,885 -> 2,730,007 tokens), relieving the KV-bound decode tail. No accuracy cost: GSM8K 0.9697 flexible / 0.9712 strict, identical to the bf16-KV run and clear of the 0.90 gate" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2247 From 9dfa09099bc17871683eb90bf557d21c64acd42c Mon Sep 17 00:00:00 2001 From: Hongxia Yang Date: Fri, 17 Jul 2026 01:18:07 +0000 Subject: [PATCH 09/11] fix(kimik2.5-disagg): probe eval token budget before running lm-eval Co-Authored-By: Claude Fable 5 --- .../multi_node/amd_utils/server_vllm.sh | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/amd_utils/server_vllm.sh b/benchmarks/multi_node/amd_utils/server_vllm.sh index a0e05cf57..64d6b8faa 100755 --- a/benchmarks/multi_node/amd_utils/server_vllm.sh +++ b/benchmarks/multi_node/amd_utils/server_vllm.sh @@ -361,13 +361,37 @@ if [ "$NODE_RANK" -eq 0 ]; then fi echo "[EVAL] EVAL_MAX_MODEL_LEN=${EVAL_MAX_MODEL_LEN} (ISL=${ISL} OSL=${OSL}, served=${SERVED_MAX_LEN:-unknown})" + # Fail fast on a token budget the engine will reject. /health only + # proves the router is up, not that a generation of this size is + # accepted: when EVAL_MAX_MODEL_LEN exceeded the served window, every + # one of the 1319 gsm8k requests 400'd and lm-eval burned ~20 min over + # 4203 retries before returning rc=1 (run 29522769567). One probe at + # the real output budget turns that into a few seconds and a message + # that names the mismatch. + _probe_max_tokens=$(( EVAL_MAX_MODEL_LEN > 4096 ? EVAL_MAX_MODEL_LEN - 4096 : EVAL_MAX_MODEL_LEN / 2 )) + [[ "$_probe_max_tokens" -gt 16384 ]] && _probe_max_tokens=16384 + _probe_body="{\"model\":\"${SERVED_MODEL}\",\"prompt\":\"hi\",\"max_tokens\":${_probe_max_tokens},\"temperature\":0}" + _probe_out=$(curl -sS --max-time 120 "http://0.0.0.0:${ROUTER_PORT}/v1/completions" \ + -H 'Content-Type: application/json' -d "$_probe_body" 2>&1) + # Match only error-shaped bodies. A bare "400" would false-positive on + # a healthy response (created/total_tokens can contain those digits) + # and silently skip a working eval. + if echo "$_probe_out" | grep -qiE '"object"[[:space:]]*:[[:space:]]*"error"|"code"[[:space:]]*:[[:space:]]*"?400|maximum context length|BadRequestError|"error"[[:space:]]*:[[:space:]]*\{'; then + echo "ERROR: eval token budget rejected by the engine — not running the full eval." >&2 + echo "ERROR: probed max_tokens=${_probe_max_tokens} against served --max-model-len=${SERVED_MAX_LEN:-unknown}" >&2 + echo "ERROR: engine said: $(echo "$_probe_out" | head -c 400)" >&2 + EVAL_FAILED=1 + fi + if [[ -n "${EVAL_CONC:-}" ]]; then export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" else export EVAL_CONCURRENT_REQUESTS=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) fi - if [[ "$DRY_RUN" -eq 1 ]]; then + if [[ "${EVAL_FAILED:-0}" -eq 1 ]]; then + echo "Skipping lm-eval: pre-flight probe already failed" >&2 + elif [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: run_eval --framework lm-eval --port $ROUTER_PORT (conc=${EVAL_CONCURRENT_REQUESTS}, ctx=${EVAL_MAX_MODEL_LEN:-auto})" else run_eval --framework lm-eval --port "$ROUTER_PORT" From 2b558416456b2fd56f20ed8e26c08e1d5b5ea62a Mon Sep 17 00:00:00 2001 From: Hongxia Yang Date: Fri, 17 Jul 2026 11:13:23 -0400 Subject: [PATCH 10/11] update max-num-seqs and clean up comments --- .../multi_node/amd_utils/models_vllm.yaml | 46 +------------------ configs/amd-master.yaml | 2 - perf-changelog.yaml | 2 +- 3 files changed, 3 insertions(+), 47 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/models_vllm.yaml b/benchmarks/multi_node/amd_utils/models_vllm.yaml index 88d88651f..886839b0a 100644 --- a/benchmarks/multi_node/amd_utils/models_vllm.yaml +++ b/benchmarks/multi_node/amd_utils/models_vllm.yaml @@ -25,50 +25,8 @@ amd-Llama-3.3-70B-Instruct-FP8-KV: env: "VLLM_USE_V1=1 VLLM_V1_USE_PREFILL_DECODE_ATTENTION=1 AMDGCN_USE_BUFFER_OPS=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_USE_AITER_TRITON_ROPE=1 TRITON_HIP_ASYNC_COPY_BYPASS_PERMUTE=1 TRITON_HIP_USE_ASYNC_COPY=1 TRITON_HIP_USE_BLOCK_PINGPONG=1 TRITON_HIP_ASYNC_FAST_SWIZZLE=1" Kimi-K2.5-MXFP4: - # Serve flags/env synced with single-node kimik2.5_fp4_mi355x.sh (INT4 - # QuickReduce, AITER defaults). max-model-len 9472 covers both sweep scenarios - # (isl+osl+256). EP off: single-node TP8 sweep showed it -14% to -27% slower. - # decode keeps --all2all-backend mori_low_latency (no-op unless EP restored). - # VLLM_ROCM_USE_AITER_RMSNORM=1: the single-node recipe disables AITER RMSNorm - # for TP<8 due to a historical accuracy issue (perf-changelog.yaml, v0.16->0.18 - # era), but on the current nightly a local TP4 GSM8K check gave identical - # accuracy with RMSNorm on vs off (0.9697 vs 0.9697), so keep the AITER RMSNorm - # perf path on rather than pin it off for a regression that no longer reproduces. - # No --compilation-config: take vLLM's default cudagraph_mode - # (FULL_AND_PIECEWISE since 0.11.0), same as the single-node recipe. The - # PIECEWISE pin carried over from the original bring-up (#1569) and forfeited - # full cudagraphs on the decode workers. Measured locally on MI350X/gfx950, - # TP4, 8k/1k, real weights, on this image (output tok/s vs PIECEWISE): - # conc 1 4 16 64 - # FULL_AND_PIECEWISE +15.9% +11.1% +12.6% +1.7% - # FULL_DECODE_ONLY +18.5% +13.6% +11.9% +1.5% - # PIECEWISE loses everywhere; the two full modes tie within run-to-run noise - # (~2%). FULL_DECODE_ONLY captures far cheaper (0.63 GiB/28s vs 7.28 GiB/69s) - # but that buys nothing here — KV cache is 89.33 GiB either way, since graph - # memory comes out of the 10% headroom left by --gpu-memory-utilization 0.90, - # not out of KV. So prefer the default and stay in sync with single-node. - # --kv-cache-dtype fp8 + --max-model-len 32768 + --max-num-seqs 256 + - # --max-num-batched-tokens 32768: measured locally (MI350X/gfx950, TP4, 8k/1k, - # real weights, this image) against the flags above as baseline, total tok/s: - # conc 16 64 128 - # kv fp8 only +9.7% +16.1% +20.5% - # all four +16.2% +19.1% +27.4% <- adopted - # TTFT and TPOT improve too (conc 128: 5721 vs 6975 ms, 65.3 vs 83.5 ms). - # fp8 KV costs no accuracy here: local TP4 GSM8K with this recipe scored - # 0.9697 flexible / 0.9712 strict, identical to the bf16-KV run (0.9697) and - # well clear of the 0.90 gate in utils/evals/thresholds.json. - # fp8 KV wins on two counts: it halves KV read bandwidth (helps at every - # concurrency) and doubles resident capacity (1,358,885 -> 2,730,007 tokens), - # which is what relieves the KV-bound decode tail. The other three add ~+5% - # on top of fp8 alone, mostly the larger prefill token budget at 8k input. - # NOTE --max-num-seqs 256 caps engine concurrency at 256; the 8k/1k conc-512 - # point only exists on the 1P2D arm, where the router splits across two decode - # workers, so no single engine should exceed the cap. Revisit if 1D ever runs - # conc 512. INT4 quick reduce is VLLM_ROCM_QUICK_REDUCE_QUANTIZATION (below): - # AITER_QUICK_REDUCE_QUANTIZATION is not read by the vLLM serve path and - # AITER_MXFP4_INTERMEDIATE does not exist in this build. - prefill_flags: "--tensor-parallel-size 8 --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 32768 --mm-encoder-tp-mode data --kv-cache-dtype fp8 --max-num-seqs 256 --max-num-batched-tokens 32768" - decode_flags: "--tensor-parallel-size 8 --all2all-backend mori_low_latency --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 32768 --mm-encoder-tp-mode data --kv-cache-dtype fp8 --max-num-seqs 256 --max-num-batched-tokens 32768" + prefill_flags: "--tensor-parallel-size 8 --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 32768 --mm-encoder-tp-mode data --kv-cache-dtype fp8 --max-num-seqs 512 --max-num-batched-tokens 32768" + decode_flags: "--tensor-parallel-size 8 --all2all-backend mori_low_latency --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 32768 --mm-encoder-tp-mode data --kv-cache-dtype fp8 --max-num-seqs 512 --max-num-batched-tokens 32768" env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 HSA_NO_SCRATCH_RECLAIM=1 VLLM_ENGINE_READY_TIMEOUT_S=3600" hf_dir: "models--amd--Kimi-K2.5-MXFP4" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 27bcc4732..4ae25ca9e 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1311,8 +1311,6 @@ kimik2.5-fp4-mi355x-vllm-disagg: - isl: 1024 osl: 1024 search-space: - # 1P(TP4) 1D(TP4) = 2 nodes. conc 512 fits KV - # (~663 resident cap at 2k tokens/req, verified single-node TP4). - spec-decoding: "none" conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256, 512 ] prefill: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 45c7e4c59..1264febbd 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4930,5 +4930,5 @@ - "Drop --compilation-config '{\"cudagraph_mode\":\"PIECEWISE\"}' from prefill/decode flags to finish the single-node sync (vLLM's default is FULL_AND_PIECEWISE). Local TP4 8k/1k real-weight A/B on MI350X: default beats the PIECEWISE pin by +15.9%/+11.1%/+12.6%/+1.7% output tok/s at conc 1/4/16/64. FULL_DECODE_ONLY ties the default within noise (+18.5%/+13.6%/+11.9%/+1.5%) and captures cheaper (0.63 GiB/28s vs 7.28 GiB/69s), but KV cache is 89.33 GiB either way, so the default is kept to stay in sync with single-node" - "Re-pin VLLM_ROUTER_IMAGE to vllm/vllm-router:nightly-20260716-1fbcde7 in benchmarks/multi_node/amd_utils/job.slurm; the previous nightly-20260629-e667ebb pin had been garbage-collected from Docker Hub (only ~16 nightlies are retained), so the router container failed to pull on rank 0" - "Fix the multi-node vLLM eval: server_vllm.sh never set EVAL_MAX_MODEL_LEN, so run_lm_eval fell back to 16384 and requested max_tokens=12288 against an engine served with --max-model-len 9472, 400-ing every gsm8k request. Now derives it via setup_eval_context (ISL+OSL+256, capped at the model's native max) and clamps it to the engine's served --max-model-len" - - "Add --kv-cache-dtype fp8, --max-model-len 32768, --max-num-seqs 256 and --max-num-batched-tokens 32768 to the prefill/decode workers. Local TP4 8k/1k real-weight A/B vs the prior flags (total tok/s): +16.2%/+19.1%/+27.4% at conc 16/64/128, with TTFT -18% and TPOT -22% at conc 128. fp8 KV alone accounts for +9.7%/+16.1%/+20.5% — it halves KV read bandwidth and doubles resident capacity (1,358,885 -> 2,730,007 tokens), relieving the KV-bound decode tail. No accuracy cost: GSM8K 0.9697 flexible / 0.9712 strict, identical to the bf16-KV run and clear of the 0.90 gate" + - "Add --kv-cache-dtype fp8, --max-model-len 32768, --max-num-seqs 512 and --max-num-batched-tokens 32768 to the prefill/decode workers. Local TP4 8k/1k real-weight A/B vs the prior flags (total tok/s): +16.2%/+19.1%/+27.4% at conc 16/64/128, with TTFT -18% and TPOT -22% at conc 128. fp8 KV alone accounts for +9.7%/+16.1%/+20.5% — it halves KV read bandwidth and doubles resident capacity (1,358,885 -> 2,730,007 tokens), relieving the KV-bound decode tail. No accuracy cost: GSM8K 0.9697 flexible / 0.9712 strict, identical to the bf16-KV run and clear of the 0.90 gate" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2247 From 430e545d7180e7de9036c2bf049c63b995b86c19 Mon Sep 17 00:00:00 2001 From: Hongxia Yang Date: Sat, 18 Jul 2026 03:29:11 +0000 Subject: [PATCH 11/11] drop 512 from 8k/1k --- benchmarks/multi_node/amd_utils/models_vllm.yaml | 4 ++-- configs/amd-master.yaml | 5 ++--- perf-changelog.yaml | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/models_vllm.yaml b/benchmarks/multi_node/amd_utils/models_vllm.yaml index 886839b0a..6e2f505ed 100644 --- a/benchmarks/multi_node/amd_utils/models_vllm.yaml +++ b/benchmarks/multi_node/amd_utils/models_vllm.yaml @@ -25,8 +25,8 @@ amd-Llama-3.3-70B-Instruct-FP8-KV: env: "VLLM_USE_V1=1 VLLM_V1_USE_PREFILL_DECODE_ATTENTION=1 AMDGCN_USE_BUFFER_OPS=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_USE_AITER_TRITON_ROPE=1 TRITON_HIP_ASYNC_COPY_BYPASS_PERMUTE=1 TRITON_HIP_USE_ASYNC_COPY=1 TRITON_HIP_USE_BLOCK_PINGPONG=1 TRITON_HIP_ASYNC_FAST_SWIZZLE=1" Kimi-K2.5-MXFP4: - prefill_flags: "--tensor-parallel-size 8 --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 32768 --mm-encoder-tp-mode data --kv-cache-dtype fp8 --max-num-seqs 512 --max-num-batched-tokens 32768" - decode_flags: "--tensor-parallel-size 8 --all2all-backend mori_low_latency --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 32768 --mm-encoder-tp-mode data --kv-cache-dtype fp8 --max-num-seqs 512 --max-num-batched-tokens 32768" + prefill_flags: "--tensor-parallel-size 8 --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 32768 --mm-encoder-tp-mode data --kv-cache-dtype fp8 --max-num-seqs 256 --max-num-batched-tokens 32768" + decode_flags: "--tensor-parallel-size 8 --all2all-backend mori_low_latency --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 32768 --mm-encoder-tp-mode data --kv-cache-dtype fp8 --max-num-seqs 256 --max-num-batched-tokens 32768" env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 HSA_NO_SCRATCH_RECLAIM=1 VLLM_ENGINE_READY_TIMEOUT_S=3600" hf_dir: "models--amd--Kimi-K2.5-MXFP4" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index ef62dc783..41755b906 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1348,10 +1348,9 @@ kimik2.5-fp4-mi355x-vllm-disagg: dp-attn: false additional-settings: - "DECODE_NODES=1" - # 1P(TP4) 2D(TP4) = 3 nodes. High-conc KV relief: doubles decode KV so the - # 8k/1k tail isn't KV-capped like 1D. + # 1P(TP4) 2D(TP4) = 3 nodes. 2 decode ~1.8x tok/s vs 1D at conc 256. - spec-decoding: "none" - conc-list: [ 256, 512 ] + conc-list: [ 256 ] prefill: num-worker: 1 tp: 4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e007dcee6..34ca221da 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4939,11 +4939,11 @@ description: - "Bump image to vllm/vllm-openai-rocm:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9" - "Sync per-worker vLLM serve flags/env with the single-node kimik2.5-fp4-mi355x-vllm recipe (VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4, HSA_NO_SCRATCH_RECLAIM=1, AITER defaults, --max-model-len 9472) in benchmarks/multi_node/amd_utils/models_vllm.yaml" - - "Retune P/D search space to all-TP4 prefill/decode (dropped TP8 — a real-weight sweep showed TP8 decode is no better than TP4 on tok/s/GPU): 1P(TP4)/1D(TP4) covers the full curve; 1P(TP4)/2D(TP4) adds decode KV headroom for the KV-bound 8k/1k tail (conc 256/512). All layouts keep prefill+decode nodes <= 3" + - "Retune P/D search space to all-TP4 prefill/decode (dropped TP8 — a real-weight sweep showed TP8 decode is no better than TP4 on tok/s/GPU): 1P(TP4)/1D(TP4) covers the full curve; 1P(TP4)/2D(TP4) adds a 2-decode arm at conc 256 (~1.8x tok/s vs 1D). All layouts keep prefill+decode nodes <= 3" - "Turn expert parallelism off (ep:1): single-node TP8 real-weight sweep showed EP -14% to -27% slower than dense" - "~2-3x tok/s/GPU vs the prior 1P2D TP8/EP8 baseline (PR #1585) across most concurrencies (8 GPU vs 24 GPU)" - "Drop --compilation-config '{\"cudagraph_mode\":\"PIECEWISE\"}' from prefill/decode flags to finish the single-node sync (vLLM's default is FULL_AND_PIECEWISE). Local TP4 8k/1k real-weight A/B on MI350X: default beats the PIECEWISE pin by +15.9%/+11.1%/+12.6%/+1.7% output tok/s at conc 1/4/16/64. FULL_DECODE_ONLY ties the default within noise (+18.5%/+13.6%/+11.9%/+1.5%) and captures cheaper (0.63 GiB/28s vs 7.28 GiB/69s), but KV cache is 89.33 GiB either way, so the default is kept to stay in sync with single-node" - "Re-pin VLLM_ROUTER_IMAGE to vllm/vllm-router:nightly-20260716-1fbcde7 in benchmarks/multi_node/amd_utils/job.slurm; the previous nightly-20260629-e667ebb pin had been garbage-collected from Docker Hub (only ~16 nightlies are retained), so the router container failed to pull on rank 0" - "Fix the multi-node vLLM eval: server_vllm.sh never set EVAL_MAX_MODEL_LEN, so run_lm_eval fell back to 16384 and requested max_tokens=12288 against an engine served with --max-model-len 9472, 400-ing every gsm8k request. Now derives it via setup_eval_context (ISL+OSL+256, capped at the model's native max) and clamps it to the engine's served --max-model-len" - - "Add --kv-cache-dtype fp8, --max-model-len 32768, --max-num-seqs 512 and --max-num-batched-tokens 32768 to the prefill/decode workers. Local TP4 8k/1k real-weight A/B vs the prior flags (total tok/s): +16.2%/+19.1%/+27.4% at conc 16/64/128, with TTFT -18% and TPOT -22% at conc 128. fp8 KV alone accounts for +9.7%/+16.1%/+20.5% — it halves KV read bandwidth and doubles resident capacity (1,358,885 -> 2,730,007 tokens), relieving the KV-bound decode tail. No accuracy cost: GSM8K 0.9697 flexible / 0.9712 strict, identical to the bf16-KV run and clear of the 0.90 gate" + - "Add --kv-cache-dtype fp8, --max-model-len 32768, --max-num-seqs 256 and --max-num-batched-tokens 32768 to the prefill/decode workers. Local TP4 8k/1k real-weight A/B vs the prior flags (total tok/s): +16.2%/+19.1%/+27.4% at conc 16/64/128, with TTFT -18% and TPOT -22% at conc 128. fp8 KV alone accounts for +9.7%/+16.1%/+20.5% — it halves KV read bandwidth and doubles resident capacity (1,358,885 -> 2,730,007 tokens), relieving the KV-bound decode tail. No accuracy cost: GSM8K 0.9697 flexible / 0.9712 strict, identical to the bf16-KV run and clear of the 0.90 gate" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2247