[AMD] Optimize kimik2.5-fp4-mi355x-vllm-disagg: image bump + serve sync + all-TP4 P/D (EP off) / 优化 kimik2.5-fp4-mi355x-vllm-disagg：镜像升级 + serve 参数对齐 + 全 TP4 P/D（关闭 EP）

benchmarks/multi_node/amd_utils/job.slurm

@@ -293,8 +293,10 @@ export DOCKER_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_${MODEL_NAME}_${S
 
 # vLLM external router container.
 # NOTE: vllm/vllm-router only retains ~16 recent nightlies on Docker Hub; older
-# dated tags are garbage-collected (manifest unknown)
-VLLM_ROUTER_IMAGE="${VLLM_ROUTER_IMAGE:-vllm/vllm-router:nightly-20260629-e667ebb}"
+# dated tags are garbage-collected (manifest unknown). The previous pin
+# (nightly-20260629-e667ebb) had already been evicted — re-pin to a live tag
+# whenever this starts failing with "manifest unknown" on rank 0.
+VLLM_ROUTER_IMAGE="${VLLM_ROUTER_IMAGE:-vllm/vllm-router:nightly-20260716-1fbcde7}"
 ROUTER_CONT_NAME="router_vllm_${SANITIZED_USER}_${SLURM_JOB_ID}"
 export RUN_FILE_FULL="$WS_PATH/${RUN_FILE}"
 

benchmarks/multi_node/amd_utils/models_vllm.yaml

@@ -25,9 +25,9 @@ amd-Llama-3.3-70B-Instruct-FP8-KV:
   env: "VLLM_USE_V1=1 VLLM_V1_USE_PREFILL_DECODE_ATTENTION=1 AMDGCN_USE_BUFFER_OPS=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_USE_AITER_TRITON_ROPE=1 TRITON_HIP_ASYNC_COPY_BYPASS_PERMUTE=1 TRITON_HIP_USE_ASYNC_COPY=1 TRITON_HIP_USE_BLOCK_PINGPONG=1 TRITON_HIP_ASYNC_FAST_SWIZZLE=1"
 
 Kimi-K2.5-MXFP4:
-  prefill_flags: "--tensor-parallel-size 8 --compilation-config '{\"cudagraph_mode\":\"PIECEWISE\"}' --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --mm-encoder-tp-mode data"
-  decode_flags: "--tensor-parallel-size 8 --enable-expert-parallel --all2all-backend mori_low_latency --compilation-config '{\"cudagraph_mode\":\"PIECEWISE\"}' --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --mm-encoder-tp-mode data"
-  env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_PAGED_ATTN=0 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_USE_AITER_TRITON_SILU_MUL=0 VLLM_ENGINE_READY_TIMEOUT_S=3600"
+  prefill_flags: "--tensor-parallel-size 8 --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 32768 --mm-encoder-tp-mode data --kv-cache-dtype fp8 --max-num-seqs 256 --max-num-batched-tokens 32768"
+  decode_flags: "--tensor-parallel-size 8 --all2all-backend mori_low_latency --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 32768 --mm-encoder-tp-mode data --kv-cache-dtype fp8 --max-num-seqs 256 --max-num-batched-tokens 32768"
+  env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 HSA_NO_SCRATCH_RECLAIM=1 VLLM_ENGINE_READY_TIMEOUT_S=3600"
   hf_dir: "models--amd--Kimi-K2.5-MXFP4"
 
 MiniMax-M2.5:

benchmarks/multi_node/amd_utils/server_vllm.sh

@@ -341,13 +341,57 @@ if [ "$NODE_RANK" -eq 0 ]; then
 
             source /workspace/benchmarks/benchmark_lib.sh
 
+            # Derive the eval context from the benchmark shape, mirroring the
+            # single-node recipe (which sets MAX_MODEL_LEN=$EVAL_MAX_MODEL_LEN).
+            # Without this, EVAL_MAX_MODEL_LEN is unset here and run_lm_eval
+            # falls back to 16384, asking lm-eval for max_tokens=12288 while the
+            # engine was launched with --max-model-len 9472 from models_vllm.yaml
+            # — every request 400s on context length and the eval fails.
+            export MODEL="${MODEL_PATH}"
+            export ISL="${BENCH_INPUT_LEN}"
+            export OSL="${BENCH_OUTPUT_LEN}"
+            setup_eval_context
+
+            # The engine's context window is fixed at launch by models_vllm.yaml,
+            # so never ask lm-eval for more than what is actually served.
+            SERVED_MAX_LEN=$(echo "$DECODE_SERVER_CONFIG" | grep -oE -- '--max-model-len[[:space:]]+[0-9]+' | awk '{print $2}' | head -1)
+            if [[ -n "$SERVED_MAX_LEN" && "${EVAL_MAX_MODEL_LEN:-0}" -gt "$SERVED_MAX_LEN" ]]; then
+                echo "[EVAL] Clamping EVAL_MAX_MODEL_LEN ${EVAL_MAX_MODEL_LEN} -> ${SERVED_MAX_LEN} (engine --max-model-len)"
+                export EVAL_MAX_MODEL_LEN="$SERVED_MAX_LEN"
+            fi
+            echo "[EVAL] EVAL_MAX_MODEL_LEN=${EVAL_MAX_MODEL_LEN} (ISL=${ISL} OSL=${OSL}, served=${SERVED_MAX_LEN:-unknown})"
+
+            # Fail fast on a token budget the engine will reject. /health only
+            # proves the router is up, not that a generation of this size is
+            # accepted: when EVAL_MAX_MODEL_LEN exceeded the served window, every
+            # one of the 1319 gsm8k requests 400'd and lm-eval burned ~20 min over
+            # 4203 retries before returning rc=1 (run 29522769567). One probe at
+            # the real output budget turns that into a few seconds and a message
+            # that names the mismatch.
+            _probe_max_tokens=$(( EVAL_MAX_MODEL_LEN > 4096 ? EVAL_MAX_MODEL_LEN - 4096 : EVAL_MAX_MODEL_LEN / 2 ))
+            [[ "$_probe_max_tokens" -gt 16384 ]] && _probe_max_tokens=16384
+            _probe_body="{\"model\":\"${SERVED_MODEL}\",\"prompt\":\"hi\",\"max_tokens\":${_probe_max_tokens},\"temperature\":0}"
+            _probe_out=$(curl -sS --max-time 120 "http://0.0.0.0:${ROUTER_PORT}/v1/completions" \
+                -H 'Content-Type: application/json' -d "$_probe_body" 2>&1)
+            # Match only error-shaped bodies. A bare "400" would false-positive on
+            # a healthy response (created/total_tokens can contain those digits)
+            # and silently skip a working eval.
+            if echo "$_probe_out" | grep -qiE '"object"[[:space:]]*:[[:space:]]*"error"|"code"[[:space:]]*:[[:space:]]*"?400|maximum context length|BadRequestError|"error"[[:space:]]*:[[:space:]]*\{'; then
+                echo "ERROR: eval token budget rejected by the engine — not running the full eval." >&2
+                echo "ERROR: probed max_tokens=${_probe_max_tokens} against served --max-model-len=${SERVED_MAX_LEN:-unknown}" >&2
+                echo "ERROR: engine said: $(echo "$_probe_out" | head -c 400)" >&2
+                EVAL_FAILED=1
+            fi
+
             if [[ -n "${EVAL_CONC:-}" ]]; then
                 export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}"
             else
                 export EVAL_CONCURRENT_REQUESTS=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1)
             fi
 
-            if [[ "$DRY_RUN" -eq 1 ]]; then
+            if [[ "${EVAL_FAILED:-0}" -eq 1 ]]; then
+                echo "Skipping lm-eval: pre-flight probe already failed" >&2
+            elif [[ "$DRY_RUN" -eq 1 ]]; then
                 echo "DRY RUN: run_eval --framework lm-eval --port $ROUTER_PORT (conc=${EVAL_CONCURRENT_REQUESTS}, ctx=${EVAL_MAX_MODEL_LEN:-auto})"
             else
                 run_eval --framework lm-eval --port "$ROUTER_PORT"

configs/amd-master.yaml

@@ -1293,7 +1293,7 @@ dsr1-fp8-mi355x-sglang-disagg-mtp:
           - "DECODE_MTP_SIZE=2"
 
 kimik2.5-fp4-mi355x-vllm-disagg:
-  image: vllm/vllm-openai-rocm:v0.24.0
+  image: vllm/vllm-openai-rocm:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9
   model: amd/Kimi-K2.5-MXFP4
   model-prefix: kimik2.5
   runner: mi355x-disagg
@@ -1305,43 +1305,63 @@ kimik2.5-fp4-mi355x-vllm-disagg:
   disagg: true
   scenarios:
     fixed-seq-len:
+    # P/D topology: all workers TP4. A real-weight sweep showed TP8 decode does
+    # not help over TP4 (TP4 wins at conc 1-8, matches elsewhere, using far fewer
+    # GPUs), so both prefill and decode run TP4.
     - isl: 1024
       osl: 1024
       search-space:
-      # 1P2D: 1 prefill node (co-located with proxy) + 2 decode nodes = 3 nodes total 
       - spec-decoding: "none"
-        conc-list: [ 8, 16, 32, 64, 128, 256, 512 ]
+        conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256, 512 ]
         prefill:
           num-worker: 1
-          tp: 8
+          tp: 4
           ep: 1
           dp-attn: false
           additional-settings:
           - "PREFILL_NODES=1"
         decode:
-          num-worker: 2
-          tp: 8
-          ep: 8
+          num-worker: 1
+          tp: 4
+          ep: 1
           dp-attn: false
           additional-settings:
-          - "DECODE_NODES=2"
+          - "DECODE_NODES=1"
 
     - isl: 8192
       osl: 1024
       search-space:
+      # 1P(TP4) 1D(TP4) = 2 nodes.
       - spec-decoding: "none"
-        conc-list: [ 8, 16, 32, 64, 128, 256, 512 ]
+        conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256 ]
         prefill:
           num-worker: 1
-          tp: 8
+          tp: 4
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "PREFILL_NODES=1"
+        decode:
+          num-worker: 1
+          tp: 4
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "DECODE_NODES=1"
+      # 1P(TP4) 2D(TP4) = 3 nodes. 2 decode ~1.8x tok/s vs 1D at conc 256.
+      - spec-decoding: "none"
+        conc-list: [ 256 ]
+        prefill:
+          num-worker: 1
+          tp: 4
           ep: 1
           dp-attn: false
           additional-settings:
           - "PREFILL_NODES=1"
         decode:
           num-worker: 2
-          tp: 8
-          ep: 8
+          tp: 4
+          ep: 1
           dp-attn: false
           additional-settings:
           - "DECODE_NODES=2"

perf-changelog.yaml

@@ -4933,3 +4933,17 @@
   description:
     - "Add MiniMax M3 NVFP4 B300 Dynamo-vLLM disaggregated EAGLE3 recipes"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2182
+
+- config-keys:
+    - kimik2.5-fp4-mi355x-vllm-disagg
+  description:
+    - "Bump image to vllm/vllm-openai-rocm:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9"
+    - "Sync per-worker vLLM serve flags/env with the single-node kimik2.5-fp4-mi355x-vllm recipe (VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4, HSA_NO_SCRATCH_RECLAIM=1, AITER defaults, --max-model-len 9472) in benchmarks/multi_node/amd_utils/models_vllm.yaml"
+    - "Retune P/D search space to all-TP4 prefill/decode (dropped TP8 — a real-weight sweep showed TP8 decode is no better than TP4 on tok/s/GPU): 1P(TP4)/1D(TP4) covers the full curve; 1P(TP4)/2D(TP4) adds a 2-decode arm at conc 256 (~1.8x tok/s vs 1D). All layouts keep prefill+decode nodes <= 3"
+    - "Turn expert parallelism off (ep:1): single-node TP8 real-weight sweep showed EP -14% to -27% slower than dense"
+    - "~2-3x tok/s/GPU vs the prior 1P2D TP8/EP8 baseline (PR #1585) across most concurrencies (8 GPU vs 24 GPU)"
+    - "Drop --compilation-config '{\"cudagraph_mode\":\"PIECEWISE\"}' from prefill/decode flags to finish the single-node sync (vLLM's default is FULL_AND_PIECEWISE). Local TP4 8k/1k real-weight A/B on MI350X: default beats the PIECEWISE pin by +15.9%/+11.1%/+12.6%/+1.7% output tok/s at conc 1/4/16/64. FULL_DECODE_ONLY ties the default within noise (+18.5%/+13.6%/+11.9%/+1.5%) and captures cheaper (0.63 GiB/28s vs 7.28 GiB/69s), but KV cache is 89.33 GiB either way, so the default is kept to stay in sync with single-node"
+    - "Re-pin VLLM_ROUTER_IMAGE to vllm/vllm-router:nightly-20260716-1fbcde7 in benchmarks/multi_node/amd_utils/job.slurm; the previous nightly-20260629-e667ebb pin had been garbage-collected from Docker Hub (only ~16 nightlies are retained), so the router container failed to pull on rank 0"
+    - "Fix the multi-node vLLM eval: server_vllm.sh never set EVAL_MAX_MODEL_LEN, so run_lm_eval fell back to 16384 and requested max_tokens=12288 against an engine served with --max-model-len 9472, 400-ing every gsm8k request. Now derives it via setup_eval_context (ISL+OSL+256, capped at the model's native max) and clamps it to the engine's served --max-model-len"
+    - "Add --kv-cache-dtype fp8, --max-model-len 32768, --max-num-seqs 256 and --max-num-batched-tokens 32768 to the prefill/decode workers. Local TP4 8k/1k real-weight A/B vs the prior flags (total tok/s): +16.2%/+19.1%/+27.4% at conc 16/64/128, with TTFT -18% and TPOT -22% at conc 128. fp8 KV alone accounts for +9.7%/+16.1%/+20.5% — it halves KV read bandwidth and doubles resident capacity (1,358,885 -> 2,730,007 tokens), relieving the KV-bound decode tail. No accuracy cost: GSM8K 0.9697 flexible / 0.9712 strict, identical to the bf16-KV run and clear of the 0.90 gate"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2247