diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x_mtp.sh index 9f0c8e83f1..7d51f46118 100644 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x_mtp.sh @@ -3,17 +3,12 @@ # MiniMax-M3 MXFP8 MI355X (gfx950) single-node vLLM recipe with EAGLE3 # speculative decoding — the spec-decoding=mtp variant of # minimaxm3_fp8_mi355x.sh. Adds the Inferact/MiniMax-M3-EAGLE3 draft head via -# --speculative-config with 3 speculative tokens. Everything else mirrors the -# non-MTP recipe: MXFP8 from TP=4 on gfx950, mandatory --block-size 128, -# --language-model-only for the text-only benchmark, FP8 KV cache, and -# --attention-backend TRITON_ATTN. Runs with CUDA graphs (no --enforce-eager); -# VLLM_USE_BREAKABLE_CUDAGRAPH=0 avoids the M3-decode breakable-cudagraph path. +# --speculative-config with 3 speculative tokens. # -# Unlike the CUDA recipes, the drafter needs no attention_backend override: -# the FlashInfer "page size 128 requires GQA/MQA" limitation that forced -# FLASH_ATTN for the EAGLE3 MHA head on Blackwell is FlashInfer/CUDA-specific. -# Here the whole server runs on TRITON_ATTN (set globally below), which serves -# the MHA draft fine. +# The EAGLE3 drafter (dense Llama MHA head) is pinned to TRITON_ATTN in the +# speculative-config, otherwise it would fall back to a slow default backend. +# Adding the explicit override left the draft's token acceptance unchanged but +# sped up the draft forward enough to turn into a win across the board. # # [AI generated draft test] The shipped vllm/vllm-openai-rocm:minimax-m3 image # does NOT implement SupportsEagle3 on the AMD MiniMax-M3 model, so EAGLE3 @@ -197,7 +192,7 @@ vllm serve "$MODEL" --port "$PORT" \ --kv-cache-dtype fp8 \ --attention-backend TRITON_ATTN \ --linear-backend emulation \ - --speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" \ + --speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"TRITON_ATTN\"}" \ --tool-call-parser minimax_m3 \ --reasoning-parser minimax_m3 \ --enable-auto-tool-choice > "$SERVER_LOG" 2>&1 & diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b9981ef896..6e1249adfc 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4574,3 +4574,10 @@ description: - "Add high concurrency configs" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1994 + +- config-keys: + - minimaxm3-fp8-mi355x-vllm-mtp + description: + - "Pin the EAGLE3 drafter to TRITON_ATTN via the speculative-config attention_backend override; without it the drafter falls back to a slow default backend." + - "Speeds up the draft forward measured on MI355X TP4 MXFP8, 8k/1k throughput gains growing with concurrency vs no override." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2107