From daea8da3b544013d084c6c62fdd258bdad44ebec Mon Sep 17 00:00:00 2001 From: Hongxia Yang Date: Mon, 6 Jul 2026 22:41:22 +0000 Subject: [PATCH 1/3] [AMD] Minimax-M3 Eagle3 Draft attention backend override TRITON_ATTN for perf Signed-off-by: Hongxia Yang --- .../fixed_seq_len/minimaxm3_fp8_mi355x_mtp.sh | 17 ++++++----------- perf-changelog.yaml | 7 +++++++ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x_mtp.sh index 9f0c8e83f1..572da5bbd6 100644 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x_mtp.sh @@ -3,17 +3,12 @@ # MiniMax-M3 MXFP8 MI355X (gfx950) single-node vLLM recipe with EAGLE3 # speculative decoding — the spec-decoding=mtp variant of # minimaxm3_fp8_mi355x.sh. Adds the Inferact/MiniMax-M3-EAGLE3 draft head via -# --speculative-config with 3 speculative tokens. Everything else mirrors the -# non-MTP recipe: MXFP8 from TP=4 on gfx950, mandatory --block-size 128, -# --language-model-only for the text-only benchmark, FP8 KV cache, and -# --attention-backend TRITON_ATTN. Runs with CUDA graphs (no --enforce-eager); -# VLLM_USE_BREAKABLE_CUDAGRAPH=0 avoids the M3-decode breakable-cudagraph path. +# --speculative-config with 3 speculative tokens. # -# Unlike the CUDA recipes, the drafter needs no attention_backend override: -# the FlashInfer "page size 128 requires GQA/MQA" limitation that forced -# FLASH_ATTN for the EAGLE3 MHA head on Blackwell is FlashInfer/CUDA-specific. -# Here the whole server runs on TRITON_ATTN (set globally below), which serves -# the MHA draft fine. +# The EAGLE3 drafter (dense Llama MHA head) is pinned to TRITON_ATTN in the +# speculative-config, otherwise it would fallback to a slow default backend. +# Adding the explicit override left the draft's token acceptance unchanged but +# sped up the draft forward enough to turn into a win across the board. # # [AI generated draft test] The shipped vllm/vllm-openai-rocm:minimax-m3 image # does NOT implement SupportsEagle3 on the AMD MiniMax-M3 model, so EAGLE3 @@ -197,7 +192,7 @@ vllm serve "$MODEL" --port "$PORT" \ --kv-cache-dtype fp8 \ --attention-backend TRITON_ATTN \ --linear-backend emulation \ - --speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" \ + --speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"TRITON_ATTN\"}" \ --tool-call-parser minimax_m3 \ --reasoning-parser minimax_m3 \ --enable-auto-tool-choice > "$SERVER_LOG" 2>&1 & diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b9981ef896..035846c4da 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4574,3 +4574,10 @@ description: - "Add high concurrency configs" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1994 + +- config-keys: + - minimaxm3-fp8-mi355x-vllm-mtp + description: + - "Pin the EAGLE3 drafter to TRITON_ATTN via the speculative-config attention_backend override (the draft does not inherit the server's TRITON_ATTN on ROCm and otherwise falls to a slow default backend)" + - "Speeds up the draft forward without changing acceptance (~2.4-2.5): measured on MI355X TP4 MXFP8 1k1k, +2-19% throughput/GPU and -1 to -16% mean TPOT across conc 1-128 vs no override (largest gains at low concurrency)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From f4d6b31d669923379afedbdfd803a54794327482 Mon Sep 17 00:00:00 2001 From: Hongxia Yang Date: Mon, 6 Jul 2026 22:55:24 +0000 Subject: [PATCH 2/3] update with pr-link Signed-off-by: Hongxia Yang --- perf-changelog.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 035846c4da..f64a0017de 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4578,6 +4578,6 @@ - config-keys: - minimaxm3-fp8-mi355x-vllm-mtp description: - - "Pin the EAGLE3 drafter to TRITON_ATTN via the speculative-config attention_backend override (the draft does not inherit the server's TRITON_ATTN on ROCm and otherwise falls to a slow default backend)" - - "Speeds up the draft forward without changing acceptance (~2.4-2.5): measured on MI355X TP4 MXFP8 1k1k, +2-19% throughput/GPU and -1 to -16% mean TPOT across conc 1-128 vs no override (largest gains at low concurrency)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + - "Pin the EAGLE3 drafter to TRITON_ATTN via the speculative-config attention_backend override (and otherwise falls to a slow default backend)" + - "Speeds up the draft forward measured on MI355X TP4 MXFP8, 8k/1k throughput gains growing with concurrency vs no override." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2107 From b4ff7aac451b8b6d837d7cf5ddb06853dac37c93 Mon Sep 17 00:00:00 2001 From: Hongxia Yang Date: Tue, 7 Jul 2026 01:26:30 +0000 Subject: [PATCH 3/3] update grammer or wording Signed-off-by: Hongxia Yang --- .../single_node/fixed_seq_len/minimaxm3_fp8_mi355x_mtp.sh | 2 +- perf-changelog.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x_mtp.sh index 572da5bbd6..7d51f46118 100644 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x_mtp.sh @@ -6,7 +6,7 @@ # --speculative-config with 3 speculative tokens. # # The EAGLE3 drafter (dense Llama MHA head) is pinned to TRITON_ATTN in the -# speculative-config, otherwise it would fallback to a slow default backend. +# speculative-config, otherwise it would fall back to a slow default backend. # Adding the explicit override left the draft's token acceptance unchanged but # sped up the draft forward enough to turn into a win across the board. # diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f64a0017de..6e1249adfc 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4578,6 +4578,6 @@ - config-keys: - minimaxm3-fp8-mi355x-vllm-mtp description: - - "Pin the EAGLE3 drafter to TRITON_ATTN via the speculative-config attention_backend override (and otherwise falls to a slow default backend)" + - "Pin the EAGLE3 drafter to TRITON_ATTN via the speculative-config attention_backend override; without it the drafter falls back to a slow default backend." - "Speeds up the draft forward measured on MI355X TP4 MXFP8, 8k/1k throughput gains growing with concurrency vs no override." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2107