diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 4638e5cad..b15a28f86 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -293,8 +293,10 @@ export DOCKER_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_${MODEL_NAME}_${S # vLLM external router container. # NOTE: vllm/vllm-router only retains ~16 recent nightlies on Docker Hub; older -# dated tags are garbage-collected (manifest unknown) -VLLM_ROUTER_IMAGE="${VLLM_ROUTER_IMAGE:-vllm/vllm-router:nightly-20260629-e667ebb}" +# dated tags are garbage-collected (manifest unknown). The previous pin +# (nightly-20260629-e667ebb) had already been evicted — re-pin to a live tag +# whenever this starts failing with "manifest unknown" on rank 0. +VLLM_ROUTER_IMAGE="${VLLM_ROUTER_IMAGE:-vllm/vllm-router:nightly-20260716-1fbcde7}" ROUTER_CONT_NAME="router_vllm_${SANITIZED_USER}_${SLURM_JOB_ID}" export RUN_FILE_FULL="$WS_PATH/${RUN_FILE}" diff --git a/benchmarks/multi_node/amd_utils/models_vllm.yaml b/benchmarks/multi_node/amd_utils/models_vllm.yaml index 9c046b4cf..6e2f505ed 100644 --- a/benchmarks/multi_node/amd_utils/models_vllm.yaml +++ b/benchmarks/multi_node/amd_utils/models_vllm.yaml @@ -25,9 +25,9 @@ amd-Llama-3.3-70B-Instruct-FP8-KV: env: "VLLM_USE_V1=1 VLLM_V1_USE_PREFILL_DECODE_ATTENTION=1 AMDGCN_USE_BUFFER_OPS=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_USE_AITER_TRITON_ROPE=1 TRITON_HIP_ASYNC_COPY_BYPASS_PERMUTE=1 TRITON_HIP_USE_ASYNC_COPY=1 TRITON_HIP_USE_BLOCK_PINGPONG=1 TRITON_HIP_ASYNC_FAST_SWIZZLE=1" Kimi-K2.5-MXFP4: - prefill_flags: "--tensor-parallel-size 8 --compilation-config '{\"cudagraph_mode\":\"PIECEWISE\"}' --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --mm-encoder-tp-mode data" - decode_flags: "--tensor-parallel-size 8 --enable-expert-parallel --all2all-backend mori_low_latency --compilation-config '{\"cudagraph_mode\":\"PIECEWISE\"}' --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --mm-encoder-tp-mode data" - env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_PAGED_ATTN=0 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_USE_AITER_TRITON_SILU_MUL=0 VLLM_ENGINE_READY_TIMEOUT_S=3600" + prefill_flags: "--tensor-parallel-size 8 --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 32768 --mm-encoder-tp-mode data --kv-cache-dtype fp8 --max-num-seqs 256 --max-num-batched-tokens 32768" + decode_flags: "--tensor-parallel-size 8 --all2all-backend mori_low_latency --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 32768 --mm-encoder-tp-mode data --kv-cache-dtype fp8 --max-num-seqs 256 --max-num-batched-tokens 32768" + env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 HSA_NO_SCRATCH_RECLAIM=1 VLLM_ENGINE_READY_TIMEOUT_S=3600" hf_dir: "models--amd--Kimi-K2.5-MXFP4" MiniMax-M2.5: diff --git a/benchmarks/multi_node/amd_utils/server_vllm.sh b/benchmarks/multi_node/amd_utils/server_vllm.sh index f19ce8560..64d6b8faa 100755 --- a/benchmarks/multi_node/amd_utils/server_vllm.sh +++ b/benchmarks/multi_node/amd_utils/server_vllm.sh @@ -341,13 +341,57 @@ if [ "$NODE_RANK" -eq 0 ]; then source /workspace/benchmarks/benchmark_lib.sh + # Derive the eval context from the benchmark shape, mirroring the + # single-node recipe (which sets MAX_MODEL_LEN=$EVAL_MAX_MODEL_LEN). + # Without this, EVAL_MAX_MODEL_LEN is unset here and run_lm_eval + # falls back to 16384, asking lm-eval for max_tokens=12288 while the + # engine was launched with --max-model-len 9472 from models_vllm.yaml + # — every request 400s on context length and the eval fails. + export MODEL="${MODEL_PATH}" + export ISL="${BENCH_INPUT_LEN}" + export OSL="${BENCH_OUTPUT_LEN}" + setup_eval_context + + # The engine's context window is fixed at launch by models_vllm.yaml, + # so never ask lm-eval for more than what is actually served. + SERVED_MAX_LEN=$(echo "$DECODE_SERVER_CONFIG" | grep -oE -- '--max-model-len[[:space:]]+[0-9]+' | awk '{print $2}' | head -1) + if [[ -n "$SERVED_MAX_LEN" && "${EVAL_MAX_MODEL_LEN:-0}" -gt "$SERVED_MAX_LEN" ]]; then + echo "[EVAL] Clamping EVAL_MAX_MODEL_LEN ${EVAL_MAX_MODEL_LEN} -> ${SERVED_MAX_LEN} (engine --max-model-len)" + export EVAL_MAX_MODEL_LEN="$SERVED_MAX_LEN" + fi + echo "[EVAL] EVAL_MAX_MODEL_LEN=${EVAL_MAX_MODEL_LEN} (ISL=${ISL} OSL=${OSL}, served=${SERVED_MAX_LEN:-unknown})" + + # Fail fast on a token budget the engine will reject. /health only + # proves the router is up, not that a generation of this size is + # accepted: when EVAL_MAX_MODEL_LEN exceeded the served window, every + # one of the 1319 gsm8k requests 400'd and lm-eval burned ~20 min over + # 4203 retries before returning rc=1 (run 29522769567). One probe at + # the real output budget turns that into a few seconds and a message + # that names the mismatch. + _probe_max_tokens=$(( EVAL_MAX_MODEL_LEN > 4096 ? EVAL_MAX_MODEL_LEN - 4096 : EVAL_MAX_MODEL_LEN / 2 )) + [[ "$_probe_max_tokens" -gt 16384 ]] && _probe_max_tokens=16384 + _probe_body="{\"model\":\"${SERVED_MODEL}\",\"prompt\":\"hi\",\"max_tokens\":${_probe_max_tokens},\"temperature\":0}" + _probe_out=$(curl -sS --max-time 120 "http://0.0.0.0:${ROUTER_PORT}/v1/completions" \ + -H 'Content-Type: application/json' -d "$_probe_body" 2>&1) + # Match only error-shaped bodies. A bare "400" would false-positive on + # a healthy response (created/total_tokens can contain those digits) + # and silently skip a working eval. + if echo "$_probe_out" | grep -qiE '"object"[[:space:]]*:[[:space:]]*"error"|"code"[[:space:]]*:[[:space:]]*"?400|maximum context length|BadRequestError|"error"[[:space:]]*:[[:space:]]*\{'; then + echo "ERROR: eval token budget rejected by the engine — not running the full eval." >&2 + echo "ERROR: probed max_tokens=${_probe_max_tokens} against served --max-model-len=${SERVED_MAX_LEN:-unknown}" >&2 + echo "ERROR: engine said: $(echo "$_probe_out" | head -c 400)" >&2 + EVAL_FAILED=1 + fi + if [[ -n "${EVAL_CONC:-}" ]]; then export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" else export EVAL_CONCURRENT_REQUESTS=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) fi - if [[ "$DRY_RUN" -eq 1 ]]; then + if [[ "${EVAL_FAILED:-0}" -eq 1 ]]; then + echo "Skipping lm-eval: pre-flight probe already failed" >&2 + elif [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: run_eval --framework lm-eval --port $ROUTER_PORT (conc=${EVAL_CONCURRENT_REQUESTS}, ctx=${EVAL_MAX_MODEL_LEN:-auto})" else run_eval --framework lm-eval --port "$ROUTER_PORT" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 7d8df0e03..7cc188765 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -911,7 +911,7 @@ dsr1-fp8-mi355x-sglang-disagg-mtp: - "DECODE_MTP_SIZE=2" kimik2.5-fp4-mi355x-vllm-disagg: - image: vllm/vllm-openai-rocm:v0.24.0 + image: vllm/vllm-openai-rocm:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9 model: amd/Kimi-K2.5-MXFP4 model-prefix: kimik2.5 runner: mi355x-disagg @@ -923,22 +923,41 @@ kimik2.5-fp4-mi355x-vllm-disagg: disagg: true scenarios: fixed-seq-len: + # All workers TP4 (real-weight sweep: TP8 decode is no better than TP4). - isl: 8192 osl: 1024 search-space: + # 1P(TP4) 1D(TP4) = 2 nodes. - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] + conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256 ] prefill: num-worker: 1 - tp: 8 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + # 1P(TP4) 2D(TP4) = 3 nodes. 2 decode ~1.8x tok/s vs 1D at conc 256. + - spec-decoding: "none" + conc-list: [ 256 ] + prefill: + num-worker: 1 + tp: 4 ep: 1 dp-attn: false additional-settings: - "PREFILL_NODES=1" decode: num-worker: 2 - tp: 8 - ep: 8 + tp: 4 + ep: 1 dp-attn: false additional-settings: - "DECODE_NODES=2" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b358f10bf..34ca221da 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4933,3 +4933,17 @@ description: - "Add MiniMax M3 NVFP4 B300 Dynamo-vLLM disaggregated EAGLE3 recipes" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2182 + +- config-keys: + - kimik2.5-fp4-mi355x-vllm-disagg + description: + - "Bump image to vllm/vllm-openai-rocm:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9" + - "Sync per-worker vLLM serve flags/env with the single-node kimik2.5-fp4-mi355x-vllm recipe (VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4, HSA_NO_SCRATCH_RECLAIM=1, AITER defaults, --max-model-len 9472) in benchmarks/multi_node/amd_utils/models_vllm.yaml" + - "Retune P/D search space to all-TP4 prefill/decode (dropped TP8 — a real-weight sweep showed TP8 decode is no better than TP4 on tok/s/GPU): 1P(TP4)/1D(TP4) covers the full curve; 1P(TP4)/2D(TP4) adds a 2-decode arm at conc 256 (~1.8x tok/s vs 1D). All layouts keep prefill+decode nodes <= 3" + - "Turn expert parallelism off (ep:1): single-node TP8 real-weight sweep showed EP -14% to -27% slower than dense" + - "~2-3x tok/s/GPU vs the prior 1P2D TP8/EP8 baseline (PR #1585) across most concurrencies (8 GPU vs 24 GPU)" + - "Drop --compilation-config '{\"cudagraph_mode\":\"PIECEWISE\"}' from prefill/decode flags to finish the single-node sync (vLLM's default is FULL_AND_PIECEWISE). Local TP4 8k/1k real-weight A/B on MI350X: default beats the PIECEWISE pin by +15.9%/+11.1%/+12.6%/+1.7% output tok/s at conc 1/4/16/64. FULL_DECODE_ONLY ties the default within noise (+18.5%/+13.6%/+11.9%/+1.5%) and captures cheaper (0.63 GiB/28s vs 7.28 GiB/69s), but KV cache is 89.33 GiB either way, so the default is kept to stay in sync with single-node" + - "Re-pin VLLM_ROUTER_IMAGE to vllm/vllm-router:nightly-20260716-1fbcde7 in benchmarks/multi_node/amd_utils/job.slurm; the previous nightly-20260629-e667ebb pin had been garbage-collected from Docker Hub (only ~16 nightlies are retained), so the router container failed to pull on rank 0" + - "Fix the multi-node vLLM eval: server_vllm.sh never set EVAL_MAX_MODEL_LEN, so run_lm_eval fell back to 16384 and requested max_tokens=12288 against an engine served with --max-model-len 9472, 400-ing every gsm8k request. Now derives it via setup_eval_context (ISL+OSL+256, capped at the model's native max) and clamps it to the engine's served --max-model-len" + - "Add --kv-cache-dtype fp8, --max-model-len 32768, --max-num-seqs 256 and --max-num-batched-tokens 32768 to the prefill/decode workers. Local TP4 8k/1k real-weight A/B vs the prior flags (total tok/s): +16.2%/+19.1%/+27.4% at conc 16/64/128, with TTFT -18% and TPOT -22% at conc 128. fp8 KV alone accounts for +9.7%/+16.1%/+20.5% — it halves KV read bandwidth and doubles resident capacity (1,358,885 -> 2,730,007 tokens), relieving the KV-bound decode tail. No accuracy cost: GSM8K 0.9697 flexible / 0.9712 strict, identical to the bf16-KV run and clear of the 0.90 gate" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2247