Add Kimi K2.6 NVFP4 B300 EAGLE3 AgentX benchmark / 新增 Kimi K2.6 NVFP4 B300 EAGLE3 AgentX 基准测试

.github/workflows/e2e-tests.yml

@@ -210,7 +210,7 @@ jobs:
             isl: '0'
             osl: '0'
             max-model-len: '0'
-            spec-decoding: 'none'
+            spec-decoding: ${{ matrix.config.spec-decoding }}
             disagg: 'false'
             run-eval: false
             scenario-type: agentic-coding

.github/workflows/run-sweep.yml

@@ -512,7 +512,7 @@ jobs:
             isl: '0'
             osl: '0'
             max-model-len: '0'
-            spec-decoding: 'none'
+            spec-decoding: ${{ matrix.config.spec-decoding }}
             disagg: ${{ 'false' }}
             run-eval: false
             scenario-type: agentic-coding

benchmarks/single_node/agentic/kimik2.5_fp4_b300_mtp.sh

@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+set -euo pipefail
+set -x
+
+source "$(dirname "$0")/../../benchmark_lib.sh"
+
+check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION
+
+
+if [[ -n "${SLURM_JOB_ID:-}" ]]; then
+    echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
+fi
+
+DRAFT_MODEL="lightseekorg/kimi-k2.6-eagle3-mla"
+
+if [[ -n "${MODEL_PATH:-}" ]]; then
+    if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
+        hf download "$MODEL" --local-dir "$MODEL_PATH"
+    fi
+    DRAFT_MODEL_PATH="/data/models/${DRAFT_MODEL##*/}"
+    if [[ ! -d "$DRAFT_MODEL_PATH" || -z "$(ls -A "$DRAFT_MODEL_PATH" 2>/dev/null)" ]]; then
+        hf download "$DRAFT_MODEL" --local-dir "$DRAFT_MODEL_PATH"
+    fi
+else
+    hf download "$MODEL"
+    export MODEL_PATH="$MODEL"
+    hf download "$DRAFT_MODEL"
+    DRAFT_MODEL_PATH="$DRAFT_MODEL"
+fi
+nvidia-smi
+
+resolve_trace_source
+install_agentic_deps
+
+SERVER_LOG="$RESULT_DIR/server.log"
+mkdir -p "$RESULT_DIR"
+
+SERVER_PID=""
+
+cleanup_agentic_services() {
+    local exit_code=$?
+    trap - EXIT INT TERM
+    set +e
+    stop_background_process_tree "$SERVER_PID" "vLLM server" 60
+    exit "$exit_code"
+}
+trap cleanup_agentic_services EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+DCP_SIZE="${DCP_SIZE:-1}"
+DCP_ARGS=()
+if [[ "$DCP_SIZE" -gt 1 ]]; then
+    DCP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE")
+fi
+
+# EAGLE3 speculative decoding and TOKENSPEED_MLA are not yet compatible with
+# decode-context-parallel; disable both for DCP configs until support is added.
+# pending PRs: https://github.com/vllm-project/vllm/pull/48180
+SPEC_ARGS=()
+ATTN_BACKEND_ARGS=()
+if [[ "$DCP_SIZE" -le 1 ]]; then
+    SPEC_ARGS=(--speculative-config "{\"method\":\"eagle3\",\"model\":\"$DRAFT_MODEL_PATH\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"synthetic\",\"synthetic_acceptance_length\":3.24}")
+    ATTN_BACKEND_ARGS=(--attention-backend TOKENSPEED_MLA)
+fi
+
+OFFLOAD_ARGS=()
+
+if agentic_kv_offload_enabled; then
+    case "$KV_OFFLOAD_BACKEND" in
+    native)
+        export VLLM_USE_SIMPLE_KV_OFFLOAD=1
+        CPU_OFFLOAD_BYTES=$((TOTAL_CPU_DRAM_GB * 1024 * 1024 * 1024))
+        OFFLOAD_ARGS=(
+            --disable-hybrid-kv-cache-manager
+            --kv-transfer-config
+            "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use\":$CPU_OFFLOAD_BYTES,\"lazy_offload\":false}}"
+        )
+        ;;
+    *) echo "Error: unsupported KV_OFFLOAD_BACKEND value '$KV_OFFLOAD_BACKEND' with EAGLE3 (expected: native)" >&2; exit 1 ;;
+    esac
+fi
+
+
+echo "Starting vllm server..."
+export PYTHONNOUSERSITE=1
+
+export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm
+
+{ set +x; } 2>/dev/null
+VLLM_CMD=(
+    vllm serve "$MODEL_PATH" --served-model-name "$MODEL"
+    --host 0.0.0.0
+    --port "$PORT"
+    --kv-cache-dtype fp8
+    --trust-remote-code
+    --block-size 64
+    --language-model-only
+    --gpu-memory-utilization 0.90
+    --max-num-seqs "$CONC"
+    "${ATTN_BACKEND_ARGS[@]}"
+    --attention-config '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}'
+    --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'
+    --max-cudagraph-capture-size 2048
+    --max-num-batched-tokens 16384
+    --stream-interval 10
+    --enable-prefix-caching
+    --tensor-parallel-size "$TP"
+    "${SPEC_ARGS[@]}"
+    "${DCP_ARGS[@]}"
+    "${OFFLOAD_ARGS[@]}"
+)
+printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt"
+printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt"
+"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 &
+SERVER_PID=$!
+echo "Server PID: $SERVER_PID"
+
+wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
+
+build_replay_cmd "$RESULT_DIR"
+
+run_agentic_replay_and_write_outputs "$RESULT_DIR"

configs/nvidia-master.yaml

@@ -2978,6 +2978,24 @@ kimik2.5-fp4-b300-vllm-agentic:
       - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32, 40, 48, 56, 64] }
       - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: native, conc-list: [1, 2, 4, 8, 16, 32, 40, 48, 56, 64] }
 
+kimik2.5-fp4-b300-vllm-agentic-mtp:
+  image: vllm/vllm-openai:nightly-09663abde0f50944a8d5ea30120666024b503faa
+  model: Kimi-K2.6-NVFP4
+  model-prefix: kimik2.5
+  runner: cluster:b300-nv
+  precision: fp4
+  framework: vllm
+  multinode: false
+  scenarios:
+    agentic-coding:
+    - dram-utilization: 0.80
+      search-space:
+      - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1] }
+      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [2, 4, 8] }
+      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: native, conc-list: [8, 16, 32] }
+      - { tp: 4, ep: 1, spec-decoding: mtp, dcp-size: 4, kv-offloading: none, conc-list: [32, 64, 128] }
+      - { tp: 4, ep: 1, spec-decoding: mtp, dcp-size: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [64, 128, 256] }
+
 dsr1-fp8-b200-trt:
   image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14
   model: deepseek-ai/DeepSeek-R1-0528

perf-changelog.yaml

@@ -4716,3 +4716,10 @@
     - "Clean the export envs"
     - "Enable two batch overlap"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093
+
+- config-keys:
+    - kimik2.5-fp4-b300-vllm-agentic-mtp
+  description:
+    - "Add EAGLE3 speculative-decoding arm for the Kimi K2.6 NVFP4 B300 AgentX recipe (draft lightseekorg/kimi-k2.6-eagle3-mla, TOKENSPEED_MLA attention backend with TRT-LLM ragged MLA kernel)."
+    - "TP8/TP4 GPU-only KV points plus a TP4 native CPU-offload ladder via SimpleCPUOffloadConnector with lazy_offload off; TP4/DCP4 high-concurrency points (conc 32/64) using num_speculative_tokens=3 and synthetic_acceptance_length=2.88."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2158

runners/launch_b300-nv.sh

@@ -352,6 +352,7 @@ else
         Kimi-K2.5
         Kimi-K2.5-NVFP4
         Kimi-K2.6
+        Kimi-K2.6-NVFP4
         MiniMax-M2.5
         MiniMax-M2.5-NVFP4
         MiniMax-M2.7

utils/matrix_logic/generate_sweep_configs.py

@@ -630,6 +630,7 @@ def generate_full_sweep(args, all_config_data, runner_data):
                     pcp_size = bmk.get(Fields.PCP_SIZE.value, 1)
                     ep = bmk.get(Fields.EP.value)
                     dp_attn = bmk.get(Fields.DP_ATTN.value)
+                    spec_decoding = bmk.get(Fields.SPEC_DECODING.value, "none")
                     kv_offloading = bmk[Fields.KV_OFFLOADING.value]
                     kv_offload_backend = bmk.get(Fields.KV_OFFLOAD_BACKEND.value)
                 total_cpu_dram_gb = (
@@ -717,13 +718,15 @@ def generate_full_sweep(args, all_config_data, runner_data):
                                 Fields.PCP_SIZE.value: pcp_size,
                                 Fields.EP.value: ep if ep is not None else 1,
                                 Fields.DP_ATTN.value: dp_attn if dp_attn is not None else False,
+                                Fields.SPEC_DECODING.value: spec_decoding,
                                 Fields.CONC.value: conc,
                                 Fields.KV_OFFLOADING.value: kv_offloading,
                                 Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb,
                                 Fields.DURATION.value: duration,
                                 Fields.EXP_NAME.value: (
                                     f"{model_code}_tp{tp}_conc{conc}_"
                                     f"{agentic_kv_offload_suffix(kv_offloading, kv_offload_backend)}"
+                                    + (f"_spec-{spec_decoding}" if spec_decoding != "none" else "")
                                 ),
                                 Fields.SCENARIO_TYPE.value: "agentic-coding",
                             }
@@ -929,6 +932,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None):
                     pcp_size = bmk.get(Fields.PCP_SIZE.value, 1)
                     ep = bmk.get(Fields.EP.value)
                     dp_attn = bmk.get(Fields.DP_ATTN.value)
+                    spec_decoding = bmk.get(Fields.SPEC_DECODING.value, "none")
                     kv_offloading = bmk[Fields.KV_OFFLOADING.value]
                     kv_offload_backend = bmk.get(Fields.KV_OFFLOAD_BACKEND.value)
                 total_cpu_dram_gb = (
@@ -1009,13 +1013,15 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None):
                                 Fields.PCP_SIZE.value: pcp_size,
                                 Fields.EP.value: ep if ep is not None else 1,
                                 Fields.DP_ATTN.value: dp_attn if dp_attn is not None else False,
+                                Fields.SPEC_DECODING.value: spec_decoding,
                                 Fields.CONC.value: conc,
                                 Fields.KV_OFFLOADING.value: kv_offloading,
                                 Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb,
                                 Fields.DURATION.value: duration,
                                 Fields.EXP_NAME.value: (
                                     f"{model_code}_tp{tp}_conc{conc}_"
                                     f"{agentic_kv_offload_suffix(kv_offloading, kv_offload_backend)}"
+                                    + (f"_spec-{spec_decoding}" if spec_decoding != "none" else "")
                                 ),
                                 Fields.SCENARIO_TYPE.value: "agentic-coding",
                             }

utils/matrix_logic/validation.py

@@ -215,6 +215,9 @@ class SingleNodeAgenticMatrixEntry(BaseModel):
     pcp_size: int = Field(alias=Fields.PCP_SIZE.value, gt=0, strict=True)
     ep: int
     dp_attn: bool = Field(alias=Fields.DP_ATTN.value)
+    spec_decoding: Literal["mtp", "draft_model", "none"] = Field(
+        default="none", alias=Fields.SPEC_DECODING.value
+    )
     conc: int
     kv_offloading: Literal["none", "dram"] = Field(
         alias=Fields.KV_OFFLOADING.value