From 4f48d462af0a03f7f2a51576552b37ea20e5f544 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 21 Sep 2026 14:00:33 -0500 Subject: [PATCH 01/14] perf(gb200): update DeepSeek V4.1 Flash SGLang nightly --- configs/nvidia-master.yaml | 8 ++++---- docs/configuration-procedures.md | 7 ++++++- docs/configuration-procedures_zh.md | 7 ++++++- perf-changelog.yaml | 8 ++++++++ 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 1215a1a5f7..9776392125 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8114,16 +8114,16 @@ dsv41flash-fp4-gb200-vllm-agentic-dspark: # SGLang arm for DeepSeek-V4.1-Flash AgentX on GB200, from the SGLang cookbook # (https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1). -# DSv4.1 Flash support has not shipped in an SGLang release; this uses the -# preview build lmsysorg/sglang:dev-dsv41, digest -# sha256:4a5d132a06a77c8331e15845f2e925adc788b00105097ad55409afa3f4fa4860 (2026-09-11, amd64 + arm64). +# Official CUDA 13 nightly, verified 2026-09-21 (amd64 + arm64): +# manifest sha256:987c7e4bd26918647211a5dcad72a2bdf2a5f394ac1469eff730e7517fc139be +# arm64 sha256:9e1fb4c395b9c406136e10aa445b8784d06bca3839623b52cbe4a3b231a157a8. # Native five-token DSpark (--speculative-algorithm DSPARK # --speculative-dspark-block-size 5), thinking on, golden AL 3.51 for # throughput, real verification for evals. The KV cache stays GPU-resident. # Follows the cookbook's verified Blackwell TP4/EP4 low-latency cell; backends # resolve automatically (dsv4 / flashinfer_mxfp4 / flashinfer_cutedsl). dsv41flash-fp4-gb200-sglang-agentic-dspark: - image: lmsysorg/sglang:dev-dsv41 + image: lmsysorg/sglang:nightly-dev-cu13-20260921-0f6761b5@sha256:987c7e4bd26918647211a5dcad72a2bdf2a5f394ac1469eff730e7517fc139be model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:gb200-nv diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 06906129e9..1f2b176fcd 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -400,7 +400,12 @@ Source: [upstream recipe](https://github.com/vllm-project/recipes/blob/main/mode `dsv41flash-fp4--sglang-agentic-dspark` are the SGLang counterparts of the vLLM arms, one PR per SKU across h100, h200, b200, b300, gb200, gb300 and mi355x. They follow the [SGLang cookbook](https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1), -which has no released SGLang version for this model yet: every NVIDIA arm uses the +which has no released SGLang version for this model yet. The GB200 arm pins the official +`lmsysorg/sglang:nightly-dev-cu13-20260921-0f6761b5` multi-architecture manifest +`sha256:987c7e4bd26918647211a5dcad72a2bdf2a5f394ac1469eff730e7517fc139be`; +its ARM64 image is `sha256:9e1fb4c395b9c406136e10aa445b8784d06bca3839623b52cbe4a3b231a157a8`. +The GB200 recipe retains TP4/EP4, native five-token DSpark, automatic backends, +and the existing GPU-resident KV and golden-AL policy. Other NVIDIA arms use the multi-arch preview build `lmsysorg/sglang:dev-dsv41` and MI355X uses `lmsysorg/sglang:dev-dsv41-mi35x`. Both tags are mutable, so the master configs and the changelog record the digests they were validated against. diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index f97f1737ce..99de1afd50 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -348,7 +348,12 @@ Maximum concurrency for 1,048,576 tokens per request: 6.70x `dsv41flash-fp4--sglang-agentic-dspark` 是 vLLM 配方在 h100、h200、b200、b300、gb200、gb300 与 mi355x 上的 SGLang 对应版本（每个 SKU 一个 PR），遵循 [SGLang cookbook](https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1)。 -该模型尚无正式发布的 SGLang 版本：所有 NVIDIA 配方使用多架构预览镜像 +该模型尚无正式发布的 SGLang 版本。GB200 配方固定官方 +`lmsysorg/sglang:nightly-dev-cu13-20260921-0f6761b5` 的多架构 manifest +`sha256:987c7e4bd26918647211a5dcad72a2bdf2a5f394ac1469eff730e7517fc139be`； +ARM64 镜像为 `sha256:9e1fb4c395b9c406136e10aa445b8784d06bca3839623b52cbe4a3b231a157a8`。 +GB200 配方保留 TP4/EP4、原生五 token DSpark、自动后端选择、GPU 常驻 KV cache 与既有黄金 AL 策略。 +其他 NVIDIA 配方使用多架构预览镜像 `lmsysorg/sglang:dev-dsv41`，MI355X 使用 `lmsysorg/sglang:dev-dsv41-mi35x`。两个标签均可变， 因此 master 配置与 changelog 记录了验证时的 digest。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 515e02bd84..79263e52f9 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8542,3 +8542,11 @@ - "为 GB300 vLLM DeepSeek-V4.1-Flash AgentX 配方在现有 TP4 臂旁新增 TP2 臂，Engram 表继续通过 --engram-config cpu_offload 放在固定页主机 DRAM；每张 277 GiB GPU 的权重升至约 175 GiB" - "在 dsv41flash_fp4_vllm_mtp.sh 中将 B200 TP2 的上限推广到所有 TP2 臂：--max-num-batched-tokens 4096（上游 16384 时 indexer 的 [batched-tokens, 1M] fp8 缓冲区达 32 GiB）、--max-num-seqs 为并发的两倍（16-256）、CUDA graph 捕获上限 512，为每张 GPU 留出约 36 GiB KV；TP4 与 TP8 臂沿用上游默认值" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3321 + +- config-keys: + - dsv41flash-fp4-gb200-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Move GB200 DeepSeek-V4.1-Flash DSpark from dev-dsv41 to official CUDA 13 nightly-dev-cu13-20260921-0f6761b5, pinned at multi-architecture manifest sha256:987c7e4bd26918647211a5dcad72a2bdf2a5f394ac1469eff730e7517fc139be (ARM64 sha256:9e1fb4c395b9c406136e10aa445b8784d06bca3839623b52cbe4a3b231a157a8); retain TP4/EP4 and native DSpark precision and golden acceptance policy" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From ec024762d7c1b54255ed62679f8130046be75db2 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 21 Sep 2026 14:01:24 -0500 Subject: [PATCH 02/14] docs(gb200): link SGLang performance pull request --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 79263e52f9..d892c000d4 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8549,4 +8549,4 @@ - agentic-coding description: - "Move GB200 DeepSeek-V4.1-Flash DSpark from dev-dsv41 to official CUDA 13 nightly-dev-cu13-20260921-0f6761b5, pinned at multi-architecture manifest sha256:987c7e4bd26918647211a5dcad72a2bdf2a5f394ac1469eff730e7517fc139be (ARM64 sha256:9e1fb4c395b9c406136e10aa445b8784d06bca3839623b52cbe4a3b231a157a8); retain TP4/EP4 and native DSpark precision and golden acceptance policy" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3347 From 3c16c9d8953bf4ac3b729db155924ace398dfd3c Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 21 Sep 2026 14:04:57 -0500 Subject: [PATCH 03/14] perf(gb200): use huge-page-capable Engram host shards --- .../agentic/dsv41flash_fp4_gb200_sglang_mtp.sh | 13 +++++++------ docs/configuration-procedures.md | 7 +++++++ docs/configuration-procedures_zh.md | 6 ++++++ perf-changelog.yaml | 8 ++++++++ 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh index 5c9b2c13b5..9cffa3802e 100755 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh @@ -43,13 +43,14 @@ export SGLANG_TIMEOUT_KEEP_ALIVE=900 export SGLANG_DEFAULT_THINKING=1 export SGLANG_DSV41_REASONING_EFFORT=high -# One shared host copy of the two fp8 Engram tables instead of a row-sharded -# copy per rank: the SGLang analogue of the vLLM arm's Engram CPU offload. It -# frees ~46 GiB of HBM per GPU for the 1M-context prefill working set and the -# KV pool, and output is bitwise unchanged (cookbook). The first sweep ran -# with the tables on GPU and the server died on the first long AgentX prompts -# (run 35304536578: c2 came up, then the server exited on the first warmup prompt). +# Keep the Engram weights in row-sharded host DRAM. GB200's 64 KiB-page +# kernel enables anonymous THP with madvise but disables shmem THP, so the +# shared memfd layout cannot obtain huge-page backing. The upstream per-rank +# layout uses anonymous mappings, MADV_HUGEPAGE and MADV_COLLAPSE for 512 MiB +# pages; row ownership and the original FP8 table weights are preserved. +# This trades two TP all-reduces for fewer host-table translation misses. export SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1 +export SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT=per_rank # AgentX concurrency counts live session trees, not individual requests. # Allow subagent fan-out to exceed CONC without clipping request bursts, but diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 1f2b176fcd..dc8e34d035 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -410,6 +410,13 @@ multi-arch preview build `lmsysorg/sglang:dev-dsv41` and MI355X uses `lmsysorg/sglang:dev-dsv41-mi35x`. Both tags are mutable, so the master configs and the changelog record the digests they were validated against. +The GB200 host-table layout is `per_rank`: its compute-node kernel enables +anonymous huge pages through `madvise`, while `shmem_enabled=never` prevents huge +pages for the shared memfd layout. Upstream allocates row shards in anonymous host +memory and requests 512 MiB huge pages with `MADV_HUGEPAGE`/`MADV_COLLAPSE`. +It preserves the original FP8 table weights and restores the two TP all-reduces; +inspect startup's actual resident/huge-page counts before claiming a benefit. + DSpark is the checkpoint's own bundled draft. SGLang exposes no EAGLE or MTP path and no `--speculative-num-steps` knob for it; the recipes pass `--speculative-algorithm DSPARK --speculative-dspark-block-size 5`. Throughput uses the same diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 99de1afd50..3d2e938481 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -357,6 +357,12 @@ GB200 配方保留 TP4/EP4、原生五 token DSpark、自动后端选择、GPU `lmsysorg/sglang:dev-dsv41`，MI355X 使用 `lmsysorg/sglang:dev-dsv41-mi35x`。两个标签均可变， 因此 master 配置与 changelog 记录了验证时的 digest。 +GB200 的主机表布局为 `per_rank`：计算节点内核通过 `madvise` 启用匿名大页， +而 `shmem_enabled=never` 阻止共享 memfd 布局使用大页。上游在匿名主机内存中 +按行分片，并通过 `MADV_HUGEPAGE`/`MADV_COLLAPSE` 请求 512 MiB 大页。 +该方式保留原始 FP8 表权重，并恢复两次 TP all-reduce；声称性能收益前， +须检查启动日志中的实际驻留内存与大页覆盖量。 + DSpark 是检查点自带的草稿模型。SGLang 对它不提供 EAGLE 或 MTP 路径，也没有 `--speculative-num-steps` 参数；配方传入 `--speculative-algorithm DSPARK --speculative-dspark-block-size 5`。吞吐测试通过 `SGLANG_SIMULATE_ACC_LEN`（`match-expected`、 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d892c000d4..f4fdfe6069 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8550,3 +8550,11 @@ description: - "Move GB200 DeepSeek-V4.1-Flash DSpark from dev-dsv41 to official CUDA 13 nightly-dev-cu13-20260921-0f6761b5, pinned at multi-architecture manifest sha256:987c7e4bd26918647211a5dcad72a2bdf2a5f394ac1469eff730e7517fc139be (ARM64 sha256:9e1fb4c395b9c406136e10aa445b8784d06bca3839623b52cbe4a3b231a157a8); retain TP4/EP4 and native DSpark precision and golden acceptance policy" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3347 + +- config-keys: + - dsv41flash-fp4-gb200-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Use the upstream per_rank Engram host-table layout on GB200: anonymous row-sharded mappings can obtain 512 MiB huge pages on the measured madvise/never anonymous/shared THP policy; preserve original FP8 table weights and restore the two TP all-reduces" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3347 From 83907384faeaa491f34765dbce2335fb81b92c35 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 21 Sep 2026 14:10:14 -0500 Subject: [PATCH 04/14] perf(gb200): add native SGLang STP AgentX arm --- .../agentic/dsv41flash_fp4_gb200_sglang.sh | 1 + .../dsv41flash_fp4_gb200_sglang_mtp.sh | 45 ++++++++++++------- configs/nvidia-master.yaml | 17 +++++++ docs/configuration-procedures.md | 7 +++ docs/configuration-procedures_zh.md | 5 +++ perf-changelog.yaml | 8 ++++ runners/launch_gb200-nv.sh | 13 +++++- 7 files changed, 77 insertions(+), 19 deletions(-) create mode 120000 benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang.sh diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang.sh new file mode 120000 index 0000000000..c7ea8773fa --- /dev/null +++ b/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang.sh @@ -0,0 +1 @@ +dsv41flash_fp4_gb200_sglang_mtp.sh \ No newline at end of file diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh index 9cffa3802e..ff92bbf76d 100755 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash set -eo pipefail -# DeepSeek-V4.1-Flash AgentX on GB200 with SGLang native DSpark, following the -# cookbook's verified Blackwell TP4/EP4 low-latency cell. The KV cache is GPU-resident. +# DeepSeek-V4.1-Flash AgentX on GB200 with native STP or DSpark serving. +# Both use the cookbook's Blackwell TP4/EP4 layout and GPU-resident KV cache. # https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1 source "$(dirname "$0")/../../benchmark_lib.sh" check_env_vars MODEL TP EP_SIZE CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY +check_env_vars EVAL_ONLY SPEC_DECODING require_agentic_kv_offload_none export GPU_COUNT="$TP" @@ -78,18 +78,30 @@ export AIPERF_SERVER_METRICS_URLS="${AIPERF_SERVER_URL}/metrics" export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:" echo "Using SGLang endpoint ${AIPERF_SERVER_URL}" -# DSpark is the checkpoint's own bundled draft: no EAGLE/MTP path and no -# --speculative-num-steps knob; the block size is the only tunable. Golden AL: -# golden_al_distribution/dsv41flash_dspark.yaml, thinking_on, five draft tokens. -# Throughput fixes acceptance to AL 3.51; accuracy evals keep real verification. -DSPARK_BLOCK_SIZE=5 -DSV41_GOLDEN_AL=3.51 -if [[ "${EVAL_ONLY}" != true ]]; then - export SGLANG_SIMULATE_ACC_LEN="$DSV41_GOLDEN_AL" - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi -echo "DSpark block size: $DSPARK_BLOCK_SIZE, golden AL=$DSV41_GOLDEN_AL" +# The caller selects native non-speculative serving or the bundled DSpark +# draft. STP and accuracy evals must never inherit synthetic acceptance. +unset SGLANG_SIMULATE_ACC_LEN SGLANG_SIMULATE_ACC_METHOD SGLANG_SIMULATE_ACC_TOKEN_MODE +SPECULATIVE_ARGS=() +case "$SPEC_DECODING" in + mtp) + DSPARK_BLOCK_SIZE=5 + DSV41_GOLDEN_AL=3.51 + SPECULATIVE_ARGS=(--speculative-algorithm DSPARK --speculative-dspark-block-size "$DSPARK_BLOCK_SIZE") + if [[ "$EVAL_ONLY" != true ]]; then + export SGLANG_SIMULATE_ACC_LEN="$DSV41_GOLDEN_AL" + export SGLANG_SIMULATE_ACC_METHOD=match-expected + export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token + fi + echo "DSpark block size: $DSPARK_BLOCK_SIZE, golden AL=$DSV41_GOLDEN_AL" + ;; + none) + echo "Native non-speculative serving; synthetic acceptance disabled" + ;; + *) + echo "Unsupported SPEC_DECODING=$SPEC_DECODING; expected mtp or none" >&2 + exit 1 + ;; +esac SGLANG_CMD=( python3 -m sglang.launch_server @@ -105,8 +117,7 @@ SGLANG_CMD=( # first 66k-99k-token AgentX prompts. --mem-fraction-static 0.70 --chunked-prefill-size 4096 - --speculative-algorithm DSPARK - --speculative-dspark-block-size "$DSPARK_BLOCK_SIZE" + "${SPECULATIVE_ARGS[@]}" --max-running-requests "$MAX_RUNNING_REQUESTS" --cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS" --reasoning-parser auto diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 9776392125..73aba591c4 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8136,6 +8136,23 @@ dsv41flash-fp4-gb200-sglang-agentic-dspark: search-space: - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } +# Native non-speculative serving follows the cookbook's throughput mode. +# No draft model or synthetic acceptance; host Engram tables use anonymous +# per-rank shards for the GB200 kernel's supported huge-page backing. +dsv41flash-fp4-gb200-sglang-agentic: + image: lmsysorg/sglang:nightly-dev-cu13-20260921-0f6761b5@sha256:987c7e4bd26918647211a5dcad72a2bdf2a5f394ac1469eff730e7517fc139be + model: deepseek-ai/DeepSeek-V4.1-Flash + model-prefix: dsv41flash + runner: cluster:gb200-nv + precision: fp4 + framework: sglang + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: none, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + dsv41flash-fp4-b300-vllm-agentic-dspark: image: vllm/vllm-openai:deepseekv41-flash-0909 model: deepseek-ai/DeepSeek-V4.1-Flash diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index dc8e34d035..30982bf529 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -410,6 +410,13 @@ multi-arch preview build `lmsysorg/sglang:dev-dsv41` and MI355X uses `lmsysorg/sglang:dev-dsv41-mi35x`. Both tags are mutable, so the master configs and the changelog record the digests they were validated against. +`dsv41flash-fp4-gb200-sglang-agentic` adds the cookbook's native non-speculative +throughput mode at TP4/EP4. It shares the GB200 implementation through the STP +script entry point, with caller-owned `SPEC_DECODING=none`: no draft model is +loaded, no speculative flags are sent, and inherited synthetic-acceptance +variables are cleared. The existing DSpark arm selects `SPEC_DECODING=mtp` and +keeps its golden-AL policy. Both arms preserve the complete AgentX trace corpus. + The GB200 host-table layout is `per_rank`: its compute-node kernel enables anonymous huge pages through `madvise`, while `shmem_enabled=never` prevents huge pages for the shared memfd layout. Upstream allocates row shards in anonymous host diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 3d2e938481..528b3f5c6c 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -357,6 +357,11 @@ GB200 配方保留 TP4/EP4、原生五 token DSpark、自动后端选择、GPU `lmsysorg/sglang:dev-dsv41`，MI355X 使用 `lmsysorg/sglang:dev-dsv41-mi35x`。两个标签均可变， 因此 master 配置与 changelog 记录了验证时的 digest。 +`dsv41flash-fp4-gb200-sglang-agentic` 新增 cookbook 的原生非推测吞吐模式，使用 TP4/EP4。 +它通过 STP 脚本入口复用 GB200 实现，由调用方设置 `SPEC_DECODING=none`： +不加载草稿模型、不传入推测参数，并清除继承的合成接受率环境变量。 +既有 DSpark 配方选择 `SPEC_DECODING=mtp`，保留黄金 AL 策略；两种模式均使用完整 AgentX 轨迹语料。 + GB200 的主机表布局为 `per_rank`：计算节点内核通过 `madvise` 启用匿名大页， 而 `shmem_enabled=never` 阻止共享 memfd 布局使用大页。上游在匿名主机内存中 按行分片，并通过 `MADV_HUGEPAGE`/`MADV_COLLAPSE` 请求 512 MiB 大页。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f4fdfe6069..9539e5b1e6 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8558,3 +8558,11 @@ description: - "Use the upstream per_rank Engram host-table layout on GB200: anonymous row-sharded mappings can obtain 512 MiB huge pages on the measured madvise/never anonymous/shared THP policy; preserve original FP8 table weights and restore the two TP all-reduces" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3347 + +- config-keys: + - dsv41flash-fp4-gb200-sglang-agentic + scenario-type: + - agentic-coding + description: + - "Add native non-speculative DeepSeek-V4.1-Flash AgentX serving on GB200 using the latest official pinned SGLang nightly, TP4/EP4 and huge-page-capable per-rank Engram host tables; no draft model, speculative flags, or synthetic acceptance" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3347 diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index cedc7c438d..d782a15399 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -97,10 +97,19 @@ import_squash() { # Direct single-tray AgentX uses the existing shared image and HF caches. if [[ "$MODEL_PREFIX" == "dsv41flash" && ( "$FRAMEWORK" == "vllm" || "$FRAMEWORK" == "sglang" ) && "${IS_MULTINODE}" != "true" ]]; then - BENCH_SCRIPT="benchmarks/single_node/agentic/${MODEL_PREFIX}_${PRECISION}_gb200_${FRAMEWORK}_mtp.sh" + check_env_vars SPEC_DECODING + BENCH_SCRIPT="benchmarks/single_node/agentic/${MODEL_PREFIX}_${PRECISION}_gb200_${FRAMEWORK}" + case "$SPEC_DECODING" in + mtp) BENCH_SCRIPT+="_mtp.sh" ;; + none) + [[ "$FRAMEWORK" == "sglang" ]] || { echo "Native STP requires the SGLang recipe" >&2; exit 1; } + BENCH_SCRIPT+=".sh" + ;; + *) echo "Unsupported SPEC_DECODING=$SPEC_DECODING" >&2; exit 1 ;; + esac # Cover DSpark5 verification for concurrent AgentX subagents at c1/c2/c4. export DSV41_MIN_CUDAGRAPH_CAPTURE_SIZE=64 - [[ "${IS_AGENTIC}" == "1" && "${SPEC_DECODING:-}" == "mtp" && -f "$BENCH_SCRIPT" ]] || { + [[ "${IS_AGENTIC}" == "1" && -f "$BENCH_SCRIPT" ]] || { echo "Unsupported single-node recipe: $BENCH_SCRIPT" >&2 exit 1 } From 3c733272ee2605e63b1d3dbed68f1e7a9755a980 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 21 Sep 2026 14:19:50 -0500 Subject: [PATCH 05/14] docs: consolidate performance changelog into one PR entry --- perf-changelog.yaml | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 9539e5b1e6..571c4e2b01 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8544,25 +8544,12 @@ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3321 - config-keys: - - dsv41flash-fp4-gb200-sglang-agentic-dspark - scenario-type: - - agentic-coding - description: - - "Move GB200 DeepSeek-V4.1-Flash DSpark from dev-dsv41 to official CUDA 13 nightly-dev-cu13-20260921-0f6761b5, pinned at multi-architecture manifest sha256:987c7e4bd26918647211a5dcad72a2bdf2a5f394ac1469eff730e7517fc139be (ARM64 sha256:9e1fb4c395b9c406136e10aa445b8784d06bca3839623b52cbe4a3b231a157a8); retain TP4/EP4 and native DSpark precision and golden acceptance policy" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3347 - -- config-keys: - - dsv41flash-fp4-gb200-sglang-agentic-dspark + - dsv41flash-fp4-gb200-sglang-agentic-dspark + - dsv41flash-fp4-gb200-sglang-agentic scenario-type: - - agentic-coding - description: - - "Use the upstream per_rank Engram host-table layout on GB200: anonymous row-sharded mappings can obtain 512 MiB huge pages on the measured madvise/never anonymous/shared THP policy; preserve original FP8 table weights and restore the two TP all-reduces" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3347 - -- config-keys: - - dsv41flash-fp4-gb200-sglang-agentic - scenario-type: - - agentic-coding + - agentic-coding description: - - "Add native non-speculative DeepSeek-V4.1-Flash AgentX serving on GB200 using the latest official pinned SGLang nightly, TP4/EP4 and huge-page-capable per-rank Engram host tables; no draft model, speculative flags, or synthetic acceptance" + - Move GB200 DeepSeek-V4.1-Flash DSpark from dev-dsv41 to official CUDA 13 nightly-dev-cu13-20260921-0f6761b5, pinned at multi-architecture manifest sha256:987c7e4bd26918647211a5dcad72a2bdf2a5f394ac1469eff730e7517fc139be (ARM64 sha256:9e1fb4c395b9c406136e10aa445b8784d06bca3839623b52cbe4a3b231a157a8); retain TP4/EP4 and native DSpark precision and golden acceptance policy + - 'Use the upstream per_rank Engram host-table layout on GB200: anonymous row-sharded mappings can obtain 512 MiB huge pages on the measured madvise/never anonymous/shared THP policy; preserve original FP8 table weights and restore the two TP all-reduces' + - Add native non-speculative DeepSeek-V4.1-Flash AgentX serving on GB200 using the latest official pinned SGLang nightly, TP4/EP4 and huge-page-capable per-rank Engram host tables; no draft model, speculative flags, or synthetic acceptance pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3347 From b6682f0d06e6c49feaa11c2dfe314c618ce98850 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 21 Sep 2026 14:38:09 -0500 Subject: [PATCH 06/14] fix(gb200): retain native FP8 draft projections --- .../dsv41flash_fp4_gb200_sglang_mtp.sh | 10 ++- .../agentic/patch_sglang_dsv41_native_wo_a.py | 66 ++++++++++++++++ .../patches/sglang_dsv41_native_wo_a.patch | 76 +++++++++++++++++++ docs/configuration-procedures.md | 8 ++ docs/configuration-procedures_zh.md | 6 ++ perf-changelog.yaml | 1 + 6 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 benchmarks/single_node/agentic/patch_sglang_dsv41_native_wo_a.py create mode 100644 benchmarks/single_node/agentic/patches/sglang_dsv41_native_wo_a.patch diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh index ff92bbf76d..2aba8821cc 100755 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh @@ -19,7 +19,8 @@ if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL" ]]; then hf download "$MODEL" --local-dir "$MODEL_PATH" else hf download "$MODEL" - export MODEL_PATH="$MODEL" + MODEL_PATH=$(python3 -c 'from huggingface_hub import snapshot_download; import sys; print(snapshot_download(repo_id=sys.argv[1], local_files_only=True))' "$MODEL") + export MODEL_PATH fi nvidia-smi @@ -30,6 +31,13 @@ SERVER_LOG="$RESULT_DIR/server.log" export PYTHONNOUSERSITE=1 export PYTHONUNBUFFERED=1 +# Preserve the three draft WO_A weights in their checkpoint FP8 precision. +# This transparent patch is hash-guarded against the exact upstream nightly. +if [[ "$SPEC_DECODING" == mtp ]]; then + python3 "$(dirname "$0")/patch_sglang_dsv41_native_wo_a.py" \ + | tee "$RESULT_DIR/native_wo_a_patch.txt" +fi + # Agentic warmup dispatches hundreds of large prompts at once and SGLang's # tokenizer can leave bytes unacknowledged past AIPerf's default 30 s # TCP_USER_TIMEOUT, so Linux aborts live localhost connections. diff --git a/benchmarks/single_node/agentic/patch_sglang_dsv41_native_wo_a.py b/benchmarks/single_node/agentic/patch_sglang_dsv41_native_wo_a.py new file mode 100644 index 0000000000..b462590fcd --- /dev/null +++ b/benchmarks/single_node/agentic/patch_sglang_dsv41_native_wo_a.py @@ -0,0 +1,66 @@ +"""Preserve native V4.1 DSpark FP8 WO_A weights in the official SGLang nightly. + +Applies only to the exact upstream source shipped at 0f6761b54facebb47f2068f87ecccd8f14da3a0e. +The dense FP8 linear path supports the checkpoint's 32x32 scales; the upstream +grouped projection instead dequantizes these draft weights permanently to BF16. +""" + +import hashlib +import importlib.util +import subprocess +from pathlib import Path + +BASE_SHA256 = "61dc79f075c9e1e5a68a466de5eb95a85ff1fa2e5a9f4d66c2fa69e956fd7b61" +PATCHED_SHA256 = "f33b370d9e49f87c5909e23109c1f22968166690765a7df169144d149f0f3d87" +HELPER_SHA256 = "9e0d5e2cb5afe0d729ffffa160f1b44ddcca163e44a675a393154a6148a610de" + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def apply_native_wo_a_patch(package_root: Path) -> None: + model = package_root / "srt/models/deepseek_v4_dspark.py" + helper = package_root / "srt/models/dspark_projection.py" + actual = digest(model) + if ( + actual == PATCHED_SHA256 + and helper.is_file() + and digest(helper) == HELPER_SHA256 + ): + print(f"Native DSpark WO_A patch already applied: sha256:{actual}") + return + if actual != BASE_SHA256 or helper.exists(): + raise RuntimeError( + "Refusing to patch unexpected SGLang DSpark source: " + f"model sha256:{actual}; expected sha256:{BASE_SHA256}. " + "Revalidate the native FP8 fix when changing the serving image." + ) + patch = Path(__file__).parent / "patches/sglang_dsv41_native_wo_a.patch" + command = [ + "patch", + "--batch", + "--forward", + "--fuzz=0", + "-p3", + "-i", + str(patch.resolve()), + ] + subprocess.run([*command, "--dry-run"], cwd=package_root, check=True) + subprocess.run(command, cwd=package_root, check=True) + if digest(model) != PATCHED_SHA256 or digest(helper) != HELPER_SHA256: + raise RuntimeError( + "Patched DSpark source does not match the validated native FP8 fix" + ) + print(f"Applied native DSpark WO_A patch: sha256:{PATCHED_SHA256}") + + +def main() -> None: + spec = importlib.util.find_spec("sglang") + if spec is None or spec.origin is None: + raise RuntimeError("SGLang package is unavailable in the serving container") + apply_native_wo_a_patch(Path(spec.origin).parent) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/single_node/agentic/patches/sglang_dsv41_native_wo_a.patch b/benchmarks/single_node/agentic/patches/sglang_dsv41_native_wo_a.patch new file mode 100644 index 0000000000..21c94e0d58 --- /dev/null +++ b/benchmarks/single_node/agentic/patches/sglang_dsv41_native_wo_a.patch @@ -0,0 +1,76 @@ +diff --git a/python/sglang/srt/models/deepseek_v4_dspark.py b/python/sglang/srt/models/deepseek_v4_dspark.py +index baebc2d..e2cd9f8 100644 +--- a/python/sglang/srt/models/deepseek_v4_dspark.py ++++ b/python/sglang/srt/models/deepseek_v4_dspark.py +@@ -55,0 +56 @@ from sglang.srt.models.dspark import ( ++from sglang.srt.models.dspark_projection import apply_grouped_wo_a +@@ -106,0 +108,3 @@ class DSparkAttention(MqaAttentionBase): ++ native_wo_a = ( ++ getattr(config, "model_type", None) == "deepseek_v41" and not _is_npu ++ ) +@@ -117 +121 @@ class DSparkAttention(MqaAttentionBase): +- wo_a_keeps_quant_config=False, ++ wo_a_keeps_quant_config=native_wo_a, +@@ -123,0 +128 @@ class DSparkAttention(MqaAttentionBase): ++ self.native_wo_a = native_wo_a +@@ -355,2 +360,8 @@ class DSparkAttention(MqaAttentionBase): +- wo_a = self.wo_a.weight.view(self.n_local_groups, self.o_lora_rank, -1) +- if self._use_fast_kernel: ++ if self.native_wo_a: ++ # V4.1 uses 32x32-scaled FP8 weights. The grouped absorb GEMM only ++ # supports 128x128 scales, so use the existing dense quant method ++ # and retain each group's own output. At TP8 there is one local ++ # group and this is a single ordinary FP8 linear projection. ++ o = apply_grouped_wo_a(self.wo_a, o, self.o_lora_rank) ++ elif self._use_fast_kernel: ++ wo_a = self.wo_a.weight.view(self.n_local_groups, self.o_lora_rank, -1) +@@ -364,0 +376 @@ class DSparkAttention(MqaAttentionBase): ++ wo_a = self.wo_a.weight.view(self.n_local_groups, self.o_lora_rank, -1) +@@ -1074 +1086,2 @@ class DeepseekV4ForCausalLMDSpark(nn.Module): +- weights = _dequant_fp8_wo_a_streaming(weights) ++ if getattr(self.config, "model_type", None) != "deepseek_v41" or _is_npu: ++ weights = _dequant_fp8_wo_a_streaming(weights) +@@ -1145,0 +1159,17 @@ class DeepseekV4ForCausalLMDSpark(nn.Module): ++ if getattr(self.config, "model_type", None) == "deepseek_v41" and not _is_npu: ++ for stage_id, stage in enumerate(self.stages): ++ prefix = f"stages.{stage_id}.self_attn.wo_a" ++ required = {f"{prefix}.weight", f"{prefix}.weight_scale_inv"} ++ if not required.issubset(loaded_params): ++ raise ValueError( ++ f"Native FP8 DSpark projection missing {required - loaded_params}" ++ ) ++ if stage.self_attn.wo_a.weight.dtype != torch.float8_e4m3fn: ++ raise ValueError( ++ f"{prefix} must retain native FP8 checkpoint weights" ++ ) ++ logger.info( ++ "DSpark native FP8 WO_A weights and block scales loaded for %d stages", ++ len(self.stages), ++ ) ++ +diff --git a/python/sglang/srt/models/dspark_projection.py b/python/sglang/srt/models/dspark_projection.py +new file mode 100644 +index 0000000..89df415 +--- /dev/null ++++ b/python/sglang/srt/models/dspark_projection.py +@@ -0,0 +1,20 @@ ++from typing import Callable, Optional, Tuple ++ ++import torch ++ ++ ++def apply_grouped_wo_a( ++ projection: Callable[[torch.Tensor], Tuple[torch.Tensor, Optional[torch.Tensor]]], ++ hidden_states: torch.Tensor, ++ output_rank: int, ++) -> torch.Tensor: ++ """Apply each local group's projection through its native linear method. ++ ++ The linear owns all local groups' weight rows. Projecting each input group ++ through it and retaining the diagonal supports its existing quantized ++ kernels without changing weight storage. One local group needs one GEMM. ++ """ ++ tokens, groups, width = hidden_states.shape ++ projected, _ = projection(hidden_states.reshape(tokens * groups, width)) ++ projected = projected.view(tokens, groups, groups, output_rank) ++ return projected.diagonal(dim1=1, dim2=2).transpose(1, 2).contiguous() diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 30982bf529..b462eec351 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -417,6 +417,14 @@ loaded, no speculative flags are sent, and inherited synthetic-acceptance variables are cleared. The existing DSpark arm selects `SPEC_DECODING=mtp` and keeps its golden-AL policy. Both arms preserve the complete AgentX trace corpus. +GB200 resolves the completed HF cache download to its local snapshot so upstream +Engram page-cache advice can find the checkpoint files; explicit local model paths +remain unchanged. DSpark applies the hash-verified +`patch_sglang_dsv41_native_wo_a.py` fix so all three WO_A projections keep native +FP8 weights and block scales. Markov weights remain native BF16. The fix uses the +existing quantized linear path without changing the grouped projection's math; +full model evaluation is still required for qualification. + The GB200 host-table layout is `per_rank`: its compute-node kernel enables anonymous huge pages through `madvise`, while `shmem_enabled=never` prevents huge pages for the shared memfd layout. Upstream allocates row shards in anonymous host diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 528b3f5c6c..d33d8a51dd 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -362,6 +362,12 @@ GB200 配方保留 TP4/EP4、原生五 token DSpark、自动后端选择、GPU 不加载草稿模型、不传入推测参数，并清除继承的合成接受率环境变量。 既有 DSpark 配方选择 `SPEC_DECODING=mtp`，保留黄金 AL 策略；两种模式均使用完整 AgentX 轨迹语料。 +GB200 将完成下载的 HF 缓存解析为本地 snapshot，使上游 Engram 页缓存建议能定位检查点文件； +显式指定的本地模型路径保持不变。DSpark 应用经过哈希校验的 +`patch_sglang_dsv41_native_wo_a.py` 修复，使三个 WO_A 投影保留原生 FP8 权重与分块 scale， +Markov 权重保持原生 BF16。修复使用既有量化线性计算路径，不改变分组投影的数学含义； +仍须完成全模型评估后才能确认合格。 + GB200 的主机表布局为 `per_rank`：计算节点内核通过 `madvise` 启用匿名大页， 而 `shmem_enabled=never` 阻止共享 memfd 布局使用大页。上游在匿名主机内存中 按行分片，并通过 `MADV_HUGEPAGE`/`MADV_COLLAPSE` 请求 512 MiB 大页。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 571c4e2b01..fb036ee7ae 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8552,4 +8552,5 @@ - Move GB200 DeepSeek-V4.1-Flash DSpark from dev-dsv41 to official CUDA 13 nightly-dev-cu13-20260921-0f6761b5, pinned at multi-architecture manifest sha256:987c7e4bd26918647211a5dcad72a2bdf2a5f394ac1469eff730e7517fc139be (ARM64 sha256:9e1fb4c395b9c406136e10aa445b8784d06bca3839623b52cbe4a3b231a157a8); retain TP4/EP4 and native DSpark precision and golden acceptance policy - 'Use the upstream per_rank Engram host-table layout on GB200: anonymous row-sharded mappings can obtain 512 MiB huge pages on the measured madvise/never anonymous/shared THP policy; preserve original FP8 table weights and restore the two TP all-reduces' - Add native non-speculative DeepSeek-V4.1-Flash AgentX serving on GB200 using the latest official pinned SGLang nightly, TP4/EP4 and huge-page-capable per-rank Engram host tables; no draft model, speculative flags, or synthetic acceptance + - Preserve native FP8 DSpark WO_A weights and block scales with the exact hash-guarded upstream-source fix; retain native BF16 Markov weights and resolve downloaded HF snapshots locally for Engram page-cache advice pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3347 From 7ffa6536bd36c8212060b2087a3f710386bdc4bb Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 21 Sep 2026 14:46:12 -0500 Subject: [PATCH 07/14] fix(gb200): support strided native FP8 draft inputs --- .../agentic/patch_sglang_dsv41_native_wo_a.py | 2 +- .../patches/sglang_dsv41_native_wo_a.patch | 2 +- .../validate_sglang_dsv41_native_wo_a.py | 111 ++++++++++++++++++ perf-changelog.yaml | 2 +- 4 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 benchmarks/single_node/agentic/patches/validate_sglang_dsv41_native_wo_a.py diff --git a/benchmarks/single_node/agentic/patch_sglang_dsv41_native_wo_a.py b/benchmarks/single_node/agentic/patch_sglang_dsv41_native_wo_a.py index b462590fcd..75a8f9cd28 100644 --- a/benchmarks/single_node/agentic/patch_sglang_dsv41_native_wo_a.py +++ b/benchmarks/single_node/agentic/patch_sglang_dsv41_native_wo_a.py @@ -12,7 +12,7 @@ BASE_SHA256 = "61dc79f075c9e1e5a68a466de5eb95a85ff1fa2e5a9f4d66c2fa69e956fd7b61" PATCHED_SHA256 = "f33b370d9e49f87c5909e23109c1f22968166690765a7df169144d149f0f3d87" -HELPER_SHA256 = "9e0d5e2cb5afe0d729ffffa160f1b44ddcca163e44a675a393154a6148a610de" +HELPER_SHA256 = "4d1eb84af8cd9813c17417d31bdcf1c01fff96e5eb281e8ea2282aa2bae41a0e" def digest(path: Path) -> str: diff --git a/benchmarks/single_node/agentic/patches/sglang_dsv41_native_wo_a.patch b/benchmarks/single_node/agentic/patches/sglang_dsv41_native_wo_a.patch index 21c94e0d58..2e2265755f 100644 --- a/benchmarks/single_node/agentic/patches/sglang_dsv41_native_wo_a.patch +++ b/benchmarks/single_node/agentic/patches/sglang_dsv41_native_wo_a.patch @@ -71,6 +71,6 @@ index 0000000..89df415 + kernels without changing weight storage. One local group needs one GEMM. + """ + tokens, groups, width = hidden_states.shape -+ projected, _ = projection(hidden_states.reshape(tokens * groups, width)) ++ projected, _ = projection(hidden_states.reshape(tokens * groups, width).contiguous()) + projected = projected.view(tokens, groups, groups, output_rank) + return projected.diagonal(dim1=1, dim2=2).transpose(1, 2).contiguous() diff --git a/benchmarks/single_node/agentic/patches/validate_sglang_dsv41_native_wo_a.py b/benchmarks/single_node/agentic/patches/validate_sglang_dsv41_native_wo_a.py new file mode 100644 index 0000000000..b1b815438f --- /dev/null +++ b/benchmarks/single_node/agentic/patches/validate_sglang_dsv41_native_wo_a.py @@ -0,0 +1,111 @@ +"""GPU regression check for the patched native DSpark FP8 WO_A path. + +Run inside the pinned, patched SGLang container. Exercises native checkpoint +loading and contiguous/strided projections with eager and CUDA graph execution. +""" + +import torch +from types import SimpleNamespace +from sglang.srt.layers.linear import ColumnParallelLinear +from sglang.srt.layers.quantization.fp8 import Fp8Config +from sglang.srt.models.deepseek_v4_dspark import DeepseekV4ForCausalLMDSpark +from sglang.srt.models.dspark_projection import apply_grouped_wo_a + +from sglang.srt.runtime_context import get_parallel + +parallel_override = get_parallel().override( + tp_size=1, tp_rank=0, attn_tp_size=1, attn_tp_rank=0 +) +parallel_override.__enter__() +print("GPU:", torch.cuda.get_device_name(), flush=True) +torch.manual_seed(42) +for groups, rank, width in [(1, 1024, 4096), (2, 64, 128)]: + quant = Fp8Config( + is_checkpoint_fp8_serialized=True, + activation_scheme="dynamic", + weight_block_size=[32, 32], + scale_fmt="ue8m0", + ) + with torch.device("cuda"): + linear = ColumnParallelLinear( + width, + groups * rank, + bias=False, + quant_config=quant, + tp_rank=0, + tp_size=1, + params_dtype=torch.bfloat16, + ) + weight = torch.randint(-2, 3, (groups * rank, width)).to(torch.float8_e4m3fn) + scale = ( + 2.0 ** torch.randint(-1, 2, (groups * rank // 32, width // 32)).float() + ).to(torch.float8_e8m0fnu) + model = DeepseekV4ForCausalLMDSpark.__new__(DeepseekV4ForCausalLMDSpark) + torch.nn.Module.__init__(model) + model.config = SimpleNamespace(model_type="deepseek_v41", n_routed_experts=0) + model.num_fused_shared_experts = 0 + model.confidence_head = None + stage = torch.nn.Module() + stage.self_attn = torch.nn.Module() + stage.self_attn.wo_a = linear + model.stages = torch.nn.ModuleList([stage]) + model.load_weights( + [("mtp.0.attn.wo_a.weight", weight), ("mtp.0.attn.wo_a.scale", scale)] + ) + assert linear.weight.dtype == torch.float8_e4m3fn + torch.testing.assert_close( + linear.weight.view(torch.uint8), weight.view(torch.uint8), rtol=0, atol=0 + ) + torch.testing.assert_close( + linear.weight_scale_inv.float(), scale.float(), rtol=0, atol=0 + ) + linear.quant_method.process_weights_after_loading(linear) + assert linear.weight.dtype == torch.float8_e4m3fn + torch.testing.assert_close( + linear.weight.view(torch.uint8), weight.view(torch.uint8), rtol=0, atol=0 + ) + torch.testing.assert_close( + linear.weight_scale_inv.float(), scale.float(), rtol=0, atol=0 + ) + restored = weight.float() * scale.float().repeat_interleave( + 32, 0 + ).repeat_interleave(32, 1) + for layout in ["contiguous", "feature_stride", "token_stride"]: + for tokens in [1, 8, 64, 0]: + if layout == "feature_stride": + x = torch.randint(-2, 3, (tokens, groups, width * 2), device="cuda").to( + torch.bfloat16 + )[..., ::2] + elif layout == "token_stride": + x = torch.randint(-2, 3, (tokens * 2, groups, width), device="cuda").to( + torch.bfloat16 + )[::2] + else: + x = torch.randint(-2, 3, (tokens, groups, width), device="cuda").to( + torch.bfloat16 + ) + if layout == "feature_stride" and tokens: + assert not x.is_contiguous(), x.stride() + if layout == "token_stride" and tokens > 1: + assert not x.is_contiguous(), x.stride() + out = apply_grouped_wo_a(linear, x, rank) + expected = torch.einsum( + "tgd,grd->tgr", x.float(), restored.view(groups, rank, width) + ).to(torch.bfloat16) + torch.testing.assert_close(out, expected, rtol=0.02, atol=1.0) + stream = torch.cuda.Stream() + with torch.cuda.stream(stream): + for _ in range(3): + apply_grouped_wo_a(linear, x, rank) + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = apply_grouped_wo_a(linear, x, rank) + graph.replay() + torch.cuda.synchronize() + torch.testing.assert_close(captured, expected, rtol=0.02, atol=1.0) + print( + f"PASS groups={groups} T={tokens} layout={layout} stride={x.stride()}: native FP8 numerical and CUDA graph replay", + flush=True, + ) +print("ALL NATIVE WO_A CHECKS PASSED", flush=True) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index fb036ee7ae..8c7ba1dc67 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8552,5 +8552,5 @@ - Move GB200 DeepSeek-V4.1-Flash DSpark from dev-dsv41 to official CUDA 13 nightly-dev-cu13-20260921-0f6761b5, pinned at multi-architecture manifest sha256:987c7e4bd26918647211a5dcad72a2bdf2a5f394ac1469eff730e7517fc139be (ARM64 sha256:9e1fb4c395b9c406136e10aa445b8784d06bca3839623b52cbe4a3b231a157a8); retain TP4/EP4 and native DSpark precision and golden acceptance policy - 'Use the upstream per_rank Engram host-table layout on GB200: anonymous row-sharded mappings can obtain 512 MiB huge pages on the measured madvise/never anonymous/shared THP policy; preserve original FP8 table weights and restore the two TP all-reduces' - Add native non-speculative DeepSeek-V4.1-Flash AgentX serving on GB200 using the latest official pinned SGLang nightly, TP4/EP4 and huge-page-capable per-rank Engram host tables; no draft model, speculative flags, or synthetic acceptance - - Preserve native FP8 DSpark WO_A weights and block scales with the exact hash-guarded upstream-source fix; retain native BF16 Markov weights and resolve downloaded HF snapshots locally for Engram page-cache advice + - Preserve native FP8 DSpark WO_A weights and block scales with the exact hash-guarded upstream-source fix, including contiguous inputs for strided draft projections; retain native BF16 Markov weights and resolve downloaded HF snapshots locally for Engram page-cache advice pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3347 From 17a711108f40d1bdea19485323671b0df58fc7db Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 21 Sep 2026 15:11:48 -0500 Subject: [PATCH 08/14] fix: restore default SGLang DSpark precision --- .../dsv41flash_fp4_gb200_sglang_mtp.sh | 7 +- .../agentic/patch_sglang_dsv41_native_wo_a.py | 66 ----------- .../patches/sglang_dsv41_native_wo_a.patch | 76 ------------ .../validate_sglang_dsv41_native_wo_a.py | 111 ------------------ docs/configuration-procedures.md | 8 +- docs/configuration-procedures_zh.md | 6 +- perf-changelog.yaml | 2 +- 7 files changed, 4 insertions(+), 272 deletions(-) delete mode 100644 benchmarks/single_node/agentic/patch_sglang_dsv41_native_wo_a.py delete mode 100644 benchmarks/single_node/agentic/patches/sglang_dsv41_native_wo_a.patch delete mode 100644 benchmarks/single_node/agentic/patches/validate_sglang_dsv41_native_wo_a.py diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh index 2aba8821cc..2d1062ec3d 100755 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh @@ -31,12 +31,7 @@ SERVER_LOG="$RESULT_DIR/server.log" export PYTHONNOUSERSITE=1 export PYTHONUNBUFFERED=1 -# Preserve the three draft WO_A weights in their checkpoint FP8 precision. -# This transparent patch is hash-guarded against the exact upstream nightly. -if [[ "$SPEC_DECODING" == mtp ]]; then - python3 "$(dirname "$0")/patch_sglang_dsv41_native_wo_a.py" \ - | tee "$RESULT_DIR/native_wo_a_patch.txt" -fi +# Use the default DSpark precision shipped by the pinned SGLang nightly. # Agentic warmup dispatches hundreds of large prompts at once and SGLang's # tokenizer can leave bytes unacknowledged past AIPerf's default 30 s diff --git a/benchmarks/single_node/agentic/patch_sglang_dsv41_native_wo_a.py b/benchmarks/single_node/agentic/patch_sglang_dsv41_native_wo_a.py deleted file mode 100644 index 75a8f9cd28..0000000000 --- a/benchmarks/single_node/agentic/patch_sglang_dsv41_native_wo_a.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Preserve native V4.1 DSpark FP8 WO_A weights in the official SGLang nightly. - -Applies only to the exact upstream source shipped at 0f6761b54facebb47f2068f87ecccd8f14da3a0e. -The dense FP8 linear path supports the checkpoint's 32x32 scales; the upstream -grouped projection instead dequantizes these draft weights permanently to BF16. -""" - -import hashlib -import importlib.util -import subprocess -from pathlib import Path - -BASE_SHA256 = "61dc79f075c9e1e5a68a466de5eb95a85ff1fa2e5a9f4d66c2fa69e956fd7b61" -PATCHED_SHA256 = "f33b370d9e49f87c5909e23109c1f22968166690765a7df169144d149f0f3d87" -HELPER_SHA256 = "4d1eb84af8cd9813c17417d31bdcf1c01fff96e5eb281e8ea2282aa2bae41a0e" - - -def digest(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def apply_native_wo_a_patch(package_root: Path) -> None: - model = package_root / "srt/models/deepseek_v4_dspark.py" - helper = package_root / "srt/models/dspark_projection.py" - actual = digest(model) - if ( - actual == PATCHED_SHA256 - and helper.is_file() - and digest(helper) == HELPER_SHA256 - ): - print(f"Native DSpark WO_A patch already applied: sha256:{actual}") - return - if actual != BASE_SHA256 or helper.exists(): - raise RuntimeError( - "Refusing to patch unexpected SGLang DSpark source: " - f"model sha256:{actual}; expected sha256:{BASE_SHA256}. " - "Revalidate the native FP8 fix when changing the serving image." - ) - patch = Path(__file__).parent / "patches/sglang_dsv41_native_wo_a.patch" - command = [ - "patch", - "--batch", - "--forward", - "--fuzz=0", - "-p3", - "-i", - str(patch.resolve()), - ] - subprocess.run([*command, "--dry-run"], cwd=package_root, check=True) - subprocess.run(command, cwd=package_root, check=True) - if digest(model) != PATCHED_SHA256 or digest(helper) != HELPER_SHA256: - raise RuntimeError( - "Patched DSpark source does not match the validated native FP8 fix" - ) - print(f"Applied native DSpark WO_A patch: sha256:{PATCHED_SHA256}") - - -def main() -> None: - spec = importlib.util.find_spec("sglang") - if spec is None or spec.origin is None: - raise RuntimeError("SGLang package is unavailable in the serving container") - apply_native_wo_a_patch(Path(spec.origin).parent) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/single_node/agentic/patches/sglang_dsv41_native_wo_a.patch b/benchmarks/single_node/agentic/patches/sglang_dsv41_native_wo_a.patch deleted file mode 100644 index 2e2265755f..0000000000 --- a/benchmarks/single_node/agentic/patches/sglang_dsv41_native_wo_a.patch +++ /dev/null @@ -1,76 +0,0 @@ -diff --git a/python/sglang/srt/models/deepseek_v4_dspark.py b/python/sglang/srt/models/deepseek_v4_dspark.py -index baebc2d..e2cd9f8 100644 ---- a/python/sglang/srt/models/deepseek_v4_dspark.py -+++ b/python/sglang/srt/models/deepseek_v4_dspark.py -@@ -55,0 +56 @@ from sglang.srt.models.dspark import ( -+from sglang.srt.models.dspark_projection import apply_grouped_wo_a -@@ -106,0 +108,3 @@ class DSparkAttention(MqaAttentionBase): -+ native_wo_a = ( -+ getattr(config, "model_type", None) == "deepseek_v41" and not _is_npu -+ ) -@@ -117 +121 @@ class DSparkAttention(MqaAttentionBase): -- wo_a_keeps_quant_config=False, -+ wo_a_keeps_quant_config=native_wo_a, -@@ -123,0 +128 @@ class DSparkAttention(MqaAttentionBase): -+ self.native_wo_a = native_wo_a -@@ -355,2 +360,8 @@ class DSparkAttention(MqaAttentionBase): -- wo_a = self.wo_a.weight.view(self.n_local_groups, self.o_lora_rank, -1) -- if self._use_fast_kernel: -+ if self.native_wo_a: -+ # V4.1 uses 32x32-scaled FP8 weights. The grouped absorb GEMM only -+ # supports 128x128 scales, so use the existing dense quant method -+ # and retain each group's own output. At TP8 there is one local -+ # group and this is a single ordinary FP8 linear projection. -+ o = apply_grouped_wo_a(self.wo_a, o, self.o_lora_rank) -+ elif self._use_fast_kernel: -+ wo_a = self.wo_a.weight.view(self.n_local_groups, self.o_lora_rank, -1) -@@ -364,0 +376 @@ class DSparkAttention(MqaAttentionBase): -+ wo_a = self.wo_a.weight.view(self.n_local_groups, self.o_lora_rank, -1) -@@ -1074 +1086,2 @@ class DeepseekV4ForCausalLMDSpark(nn.Module): -- weights = _dequant_fp8_wo_a_streaming(weights) -+ if getattr(self.config, "model_type", None) != "deepseek_v41" or _is_npu: -+ weights = _dequant_fp8_wo_a_streaming(weights) -@@ -1145,0 +1159,17 @@ class DeepseekV4ForCausalLMDSpark(nn.Module): -+ if getattr(self.config, "model_type", None) == "deepseek_v41" and not _is_npu: -+ for stage_id, stage in enumerate(self.stages): -+ prefix = f"stages.{stage_id}.self_attn.wo_a" -+ required = {f"{prefix}.weight", f"{prefix}.weight_scale_inv"} -+ if not required.issubset(loaded_params): -+ raise ValueError( -+ f"Native FP8 DSpark projection missing {required - loaded_params}" -+ ) -+ if stage.self_attn.wo_a.weight.dtype != torch.float8_e4m3fn: -+ raise ValueError( -+ f"{prefix} must retain native FP8 checkpoint weights" -+ ) -+ logger.info( -+ "DSpark native FP8 WO_A weights and block scales loaded for %d stages", -+ len(self.stages), -+ ) -+ -diff --git a/python/sglang/srt/models/dspark_projection.py b/python/sglang/srt/models/dspark_projection.py -new file mode 100644 -index 0000000..89df415 ---- /dev/null -+++ b/python/sglang/srt/models/dspark_projection.py -@@ -0,0 +1,20 @@ -+from typing import Callable, Optional, Tuple -+ -+import torch -+ -+ -+def apply_grouped_wo_a( -+ projection: Callable[[torch.Tensor], Tuple[torch.Tensor, Optional[torch.Tensor]]], -+ hidden_states: torch.Tensor, -+ output_rank: int, -+) -> torch.Tensor: -+ """Apply each local group's projection through its native linear method. -+ -+ The linear owns all local groups' weight rows. Projecting each input group -+ through it and retaining the diagonal supports its existing quantized -+ kernels without changing weight storage. One local group needs one GEMM. -+ """ -+ tokens, groups, width = hidden_states.shape -+ projected, _ = projection(hidden_states.reshape(tokens * groups, width).contiguous()) -+ projected = projected.view(tokens, groups, groups, output_rank) -+ return projected.diagonal(dim1=1, dim2=2).transpose(1, 2).contiguous() diff --git a/benchmarks/single_node/agentic/patches/validate_sglang_dsv41_native_wo_a.py b/benchmarks/single_node/agentic/patches/validate_sglang_dsv41_native_wo_a.py deleted file mode 100644 index b1b815438f..0000000000 --- a/benchmarks/single_node/agentic/patches/validate_sglang_dsv41_native_wo_a.py +++ /dev/null @@ -1,111 +0,0 @@ -"""GPU regression check for the patched native DSpark FP8 WO_A path. - -Run inside the pinned, patched SGLang container. Exercises native checkpoint -loading and contiguous/strided projections with eager and CUDA graph execution. -""" - -import torch -from types import SimpleNamespace -from sglang.srt.layers.linear import ColumnParallelLinear -from sglang.srt.layers.quantization.fp8 import Fp8Config -from sglang.srt.models.deepseek_v4_dspark import DeepseekV4ForCausalLMDSpark -from sglang.srt.models.dspark_projection import apply_grouped_wo_a - -from sglang.srt.runtime_context import get_parallel - -parallel_override = get_parallel().override( - tp_size=1, tp_rank=0, attn_tp_size=1, attn_tp_rank=0 -) -parallel_override.__enter__() -print("GPU:", torch.cuda.get_device_name(), flush=True) -torch.manual_seed(42) -for groups, rank, width in [(1, 1024, 4096), (2, 64, 128)]: - quant = Fp8Config( - is_checkpoint_fp8_serialized=True, - activation_scheme="dynamic", - weight_block_size=[32, 32], - scale_fmt="ue8m0", - ) - with torch.device("cuda"): - linear = ColumnParallelLinear( - width, - groups * rank, - bias=False, - quant_config=quant, - tp_rank=0, - tp_size=1, - params_dtype=torch.bfloat16, - ) - weight = torch.randint(-2, 3, (groups * rank, width)).to(torch.float8_e4m3fn) - scale = ( - 2.0 ** torch.randint(-1, 2, (groups * rank // 32, width // 32)).float() - ).to(torch.float8_e8m0fnu) - model = DeepseekV4ForCausalLMDSpark.__new__(DeepseekV4ForCausalLMDSpark) - torch.nn.Module.__init__(model) - model.config = SimpleNamespace(model_type="deepseek_v41", n_routed_experts=0) - model.num_fused_shared_experts = 0 - model.confidence_head = None - stage = torch.nn.Module() - stage.self_attn = torch.nn.Module() - stage.self_attn.wo_a = linear - model.stages = torch.nn.ModuleList([stage]) - model.load_weights( - [("mtp.0.attn.wo_a.weight", weight), ("mtp.0.attn.wo_a.scale", scale)] - ) - assert linear.weight.dtype == torch.float8_e4m3fn - torch.testing.assert_close( - linear.weight.view(torch.uint8), weight.view(torch.uint8), rtol=0, atol=0 - ) - torch.testing.assert_close( - linear.weight_scale_inv.float(), scale.float(), rtol=0, atol=0 - ) - linear.quant_method.process_weights_after_loading(linear) - assert linear.weight.dtype == torch.float8_e4m3fn - torch.testing.assert_close( - linear.weight.view(torch.uint8), weight.view(torch.uint8), rtol=0, atol=0 - ) - torch.testing.assert_close( - linear.weight_scale_inv.float(), scale.float(), rtol=0, atol=0 - ) - restored = weight.float() * scale.float().repeat_interleave( - 32, 0 - ).repeat_interleave(32, 1) - for layout in ["contiguous", "feature_stride", "token_stride"]: - for tokens in [1, 8, 64, 0]: - if layout == "feature_stride": - x = torch.randint(-2, 3, (tokens, groups, width * 2), device="cuda").to( - torch.bfloat16 - )[..., ::2] - elif layout == "token_stride": - x = torch.randint(-2, 3, (tokens * 2, groups, width), device="cuda").to( - torch.bfloat16 - )[::2] - else: - x = torch.randint(-2, 3, (tokens, groups, width), device="cuda").to( - torch.bfloat16 - ) - if layout == "feature_stride" and tokens: - assert not x.is_contiguous(), x.stride() - if layout == "token_stride" and tokens > 1: - assert not x.is_contiguous(), x.stride() - out = apply_grouped_wo_a(linear, x, rank) - expected = torch.einsum( - "tgd,grd->tgr", x.float(), restored.view(groups, rank, width) - ).to(torch.bfloat16) - torch.testing.assert_close(out, expected, rtol=0.02, atol=1.0) - stream = torch.cuda.Stream() - with torch.cuda.stream(stream): - for _ in range(3): - apply_grouped_wo_a(linear, x, rank) - torch.cuda.current_stream().wait_stream(stream) - graph = torch.cuda.CUDAGraph() - with torch.cuda.graph(graph): - captured = apply_grouped_wo_a(linear, x, rank) - graph.replay() - torch.cuda.synchronize() - torch.testing.assert_close(captured, expected, rtol=0.02, atol=1.0) - print( - f"PASS groups={groups} T={tokens} layout={layout} stride={x.stride()}: native FP8 numerical and CUDA graph replay", - flush=True, - ) -print("ALL NATIVE WO_A CHECKS PASSED", flush=True) diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index b462eec351..62116fd7d0 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -417,13 +417,7 @@ loaded, no speculative flags are sent, and inherited synthetic-acceptance variables are cleared. The existing DSpark arm selects `SPEC_DECODING=mtp` and keeps its golden-AL policy. Both arms preserve the complete AgentX trace corpus. -GB200 resolves the completed HF cache download to its local snapshot so upstream -Engram page-cache advice can find the checkpoint files; explicit local model paths -remain unchanged. DSpark applies the hash-verified -`patch_sglang_dsv41_native_wo_a.py` fix so all three WO_A projections keep native -FP8 weights and block scales. Markov weights remain native BF16. The fix uses the -existing quantized linear path without changing the grouped projection's math; -full model evaluation is still required for qualification. +DSpark uses the default precision shipped by the pinned official nightly, without custom draft quantization or precision patches. STP loads no draft; full accuracy and performance validation are still required. The GB200 host-table layout is `per_rank`: its compute-node kernel enables anonymous huge pages through `madvise`, while `shmem_enabled=never` prevents huge diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index d33d8a51dd..6701689103 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -362,11 +362,7 @@ GB200 配方保留 TP4/EP4、原生五 token DSpark、自动后端选择、GPU 不加载草稿模型、不传入推测参数，并清除继承的合成接受率环境变量。 既有 DSpark 配方选择 `SPEC_DECODING=mtp`，保留黄金 AL 策略；两种模式均使用完整 AgentX 轨迹语料。 -GB200 将完成下载的 HF 缓存解析为本地 snapshot，使上游 Engram 页缓存建议能定位检查点文件； -显式指定的本地模型路径保持不变。DSpark 应用经过哈希校验的 -`patch_sglang_dsv41_native_wo_a.py` 修复，使三个 WO_A 投影保留原生 FP8 权重与分块 scale， -Markov 权重保持原生 BF16。修复使用既有量化线性计算路径，不改变分组投影的数学含义； -仍须完成全模型评估后才能确认合格。 +DSpark 使用固定官方 nightly 默认提供的精度，不应用自定义草稿量化或精度补丁。STP 不加载草稿模型；完整准确率和性能验证仍然必需。 GB200 的主机表布局为 `per_rank`：计算节点内核通过 `madvise` 启用匿名大页， 而 `shmem_enabled=never` 阻止共享 memfd 布局使用大页。上游在匿名主机内存中 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 8c7ba1dc67..cd80e3e08b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8552,5 +8552,5 @@ - Move GB200 DeepSeek-V4.1-Flash DSpark from dev-dsv41 to official CUDA 13 nightly-dev-cu13-20260921-0f6761b5, pinned at multi-architecture manifest sha256:987c7e4bd26918647211a5dcad72a2bdf2a5f394ac1469eff730e7517fc139be (ARM64 sha256:9e1fb4c395b9c406136e10aa445b8784d06bca3839623b52cbe4a3b231a157a8); retain TP4/EP4 and native DSpark precision and golden acceptance policy - 'Use the upstream per_rank Engram host-table layout on GB200: anonymous row-sharded mappings can obtain 512 MiB huge pages on the measured madvise/never anonymous/shared THP policy; preserve original FP8 table weights and restore the two TP all-reduces' - Add native non-speculative DeepSeek-V4.1-Flash AgentX serving on GB200 using the latest official pinned SGLang nightly, TP4/EP4 and huge-page-capable per-rank Engram host tables; no draft model, speculative flags, or synthetic acceptance - - Preserve native FP8 DSpark WO_A weights and block scales with the exact hash-guarded upstream-source fix, including contiguous inputs for strided draft projections; retain native BF16 Markov weights and resolve downloaded HF snapshots locally for Engram page-cache advice + - "Use the default MTP/DSpark precision shipped by the official nightly, without custom draft quantization or precision patches." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3347 From f00cf6bcbea062b6d8e3577b9234adf71b1a4591 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 21 Sep 2026 18:53:47 -0500 Subject: [PATCH 09/14] perf(gb200): keep DeepSeek V4.1 sweep DSpark-only --- .../agentic/dsv41flash_fp4_gb200_sglang.sh | 1 - configs/nvidia-master.yaml | 17 ----------------- docs/configuration-procedures.md | 13 +++++-------- docs/configuration-procedures_zh.md | 10 ++++------ perf-changelog.yaml | 3 +-- 5 files changed, 10 insertions(+), 34 deletions(-) delete mode 120000 benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang.sh diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang.sh deleted file mode 120000 index c7ea8773fa..0000000000 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang.sh +++ /dev/null @@ -1 +0,0 @@ -dsv41flash_fp4_gb200_sglang_mtp.sh \ No newline at end of file diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 8f095a0bc6..8ddebdce0a 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8134,23 +8134,6 @@ dsv41flash-fp4-gb200-sglang-agentic-dspark: search-space: - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } -# Native non-speculative serving follows the cookbook's throughput mode. -# No draft model or synthetic acceptance; host Engram tables use anonymous -# per-rank shards for the GB200 kernel's supported huge-page backing. -dsv41flash-fp4-gb200-sglang-agentic: - image: lmsysorg/sglang:nightly-dev-cu13-20260921-0f6761b5@sha256:987c7e4bd26918647211a5dcad72a2bdf2a5f394ac1469eff730e7517fc139be - model: deepseek-ai/DeepSeek-V4.1-Flash - model-prefix: dsv41flash - runner: cluster:gb200-nv - precision: fp4 - framework: sglang - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: none, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } - dsv41flash-fp4-b300-vllm-agentic-dspark: image: vllm/vllm-openai:deepseekv41-flash-0909 model: deepseek-ai/DeepSeek-V4.1-Flash diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 6025fc3d5d..8f446bd773 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -412,14 +412,11 @@ multi-arch preview build `lmsysorg/sglang:dev-dsv41` and MI355X uses `lmsysorg/sglang:dev-dsv41-mi35x`. Both tags are mutable, so the master configs and the changelog record the digests they were validated against. -`dsv41flash-fp4-gb200-sglang-agentic` adds the cookbook's native non-speculative -throughput mode at TP4/EP4. It shares the GB200 implementation through the STP -script entry point, with caller-owned `SPEC_DECODING=none`: no draft model is -loaded, no speculative flags are sent, and inherited synthetic-acceptance -variables are cleared. The existing DSpark arm selects `SPEC_DECODING=mtp` and -keeps its golden-AL policy. Both arms preserve the complete AgentX trace corpus. - -DSpark uses the default precision shipped by the pinned official nightly, without custom draft quantization or precision patches. STP loads no draft; full accuracy and performance validation are still required. +The GB200 sweep contains only `dsv41flash-fp4-gb200-sglang-agentic-dspark`. +The unmeasured STP entry is excluded; adding it would require matched evidence +of a performance-frontier contribution. DSpark uses the default precision shipped +by the pinned official nightly, without custom draft quantization or precision +patches. Full accuracy and performance validation remain required. The GB200 host-table layout is `per_rank`: its compute-node kernel enables anonymous huge pages through `madvise`, while `shmem_enabled=never` prevents huge diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 75030a26e9..677445c433 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -359,12 +359,10 @@ GB200 配方保留 TP4/EP4、原生五 token DSpark、自动后端选择、GPU `lmsysorg/sglang:dev-dsv41`，MI355X 使用 `lmsysorg/sglang:dev-dsv41-mi35x`。两个标签均可变， 因此 master 配置与 changelog 记录了验证时的 digest。 -`dsv41flash-fp4-gb200-sglang-agentic` 新增 cookbook 的原生非推测吞吐模式，使用 TP4/EP4。 -它通过 STP 脚本入口复用 GB200 实现，由调用方设置 `SPEC_DECODING=none`： -不加载草稿模型、不传入推测参数，并清除继承的合成接受率环境变量。 -既有 DSpark 配方选择 `SPEC_DECODING=mtp`，保留黄金 AL 策略；两种模式均使用完整 AgentX 轨迹语料。 - -DSpark 使用固定官方 nightly 默认提供的精度，不应用自定义草稿量化或精度补丁。STP 不加载草稿模型；完整准确率和性能验证仍然必需。 +GB200 sweep 仅包含 `dsv41flash-fp4-gb200-sglang-agentic-dspark`。 +移除尚无实测依据的 STP 条目；若要加入，须通过匹配测试证明其对性能前沿有贡献。 +DSpark 使用固定官方 nightly 默认提供的精度，不应用自定义草稿量化或精度补丁。 +完整准确率和性能验证仍然必需。 GB200 的主机表布局为 `per_rank`：计算节点内核通过 `madvise` 启用匿名大页， 而 `shmem_enabled=never` 阻止共享 memfd 布局使用大页。上游在匿名主机内存中 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4b4bde607c..907029e102 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8556,12 +8556,11 @@ - config-keys: - dsv41flash-fp4-gb200-sglang-agentic-dspark - - dsv41flash-fp4-gb200-sglang-agentic scenario-type: - agentic-coding description: - Move GB200 DeepSeek-V4.1-Flash DSpark from dev-dsv41 to official CUDA 13 nightly-dev-cu13-20260921-0f6761b5, pinned at multi-architecture manifest sha256:987c7e4bd26918647211a5dcad72a2bdf2a5f394ac1469eff730e7517fc139be (ARM64 sha256:9e1fb4c395b9c406136e10aa445b8784d06bca3839623b52cbe4a3b231a157a8); retain TP4/EP4 and native DSpark precision and golden acceptance policy - 'Use the upstream per_rank Engram host-table layout on GB200: anonymous row-sharded mappings can obtain 512 MiB huge pages on the measured madvise/never anonymous/shared THP policy; preserve original FP8 table weights and restore the two TP all-reduces' - - Add native non-speculative DeepSeek-V4.1-Flash AgentX serving on GB200 using the latest official pinned SGLang nightly, TP4/EP4 and huge-page-capable per-rank Engram host tables; no draft model, speculative flags, or synthetic acceptance - "Use the default MTP/DSpark precision shipped by the official nightly, without custom draft quantization or precision patches." + - Keep the sweep DSpark-only; remove the unmeasured STP entry and alias. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3347 From 79a97afbdd8e4b0ff86448fa270028e4d20d0673 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 21 Sep 2026 20:37:06 -0500 Subject: [PATCH 10/14] perf(gb200): qualify September 22 SGLang nightly --- configs/nvidia-master.yaml | 8 ++++---- docs/configuration-procedures.md | 6 +++--- docs/configuration-procedures_zh.md | 6 +++--- perf-changelog.yaml | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 8ddebdce0a..17fb34b1be 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8112,16 +8112,16 @@ dsv41flash-fp4-gb200-vllm-agentic-dspark: # SGLang arm for DeepSeek-V4.1-Flash AgentX on GB200, from the SGLang cookbook # (https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1). -# Official CUDA 13 nightly, verified 2026-09-21 (amd64 + arm64): -# manifest sha256:987c7e4bd26918647211a5dcad72a2bdf2a5f394ac1469eff730e7517fc139be -# arm64 sha256:9e1fb4c395b9c406136e10aa445b8784d06bca3839623b52cbe4a3b231a157a8. +# Official CUDA 13 nightly, verified 2026-09-22 (amd64 + arm64): +# manifest sha256:0e1b14e302619a42ef5581b87db6804651d4f946301cf543d74a3a1eb1c33b40 +# arm64 sha256:044b18a3a7ce0437e366c60a32d7e7109d1d846aee4e301e3ff1c7932c30695a. # Native five-token DSpark (--speculative-algorithm DSPARK # --speculative-dspark-block-size 5), thinking on, golden AL 3.51 for # throughput, real verification for evals. The KV cache stays GPU-resident. # Follows the cookbook's verified Blackwell TP4/EP4 low-latency cell; backends # resolve automatically (dsv4 / flashinfer_mxfp4 / flashinfer_cutedsl). dsv41flash-fp4-gb200-sglang-agentic-dspark: - image: lmsysorg/sglang:nightly-dev-cu13-20260921-0f6761b5@sha256:987c7e4bd26918647211a5dcad72a2bdf2a5f394ac1469eff730e7517fc139be + image: lmsysorg/sglang:nightly-dev-cu13-20260922-582389ce@sha256:0e1b14e302619a42ef5581b87db6804651d4f946301cf543d74a3a1eb1c33b40 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:gb200-nv diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 8f446bd773..88824f1cb5 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -403,9 +403,9 @@ Source: [upstream recipe](https://github.com/vllm-project/recipes/blob/main/mode arms, one PR per SKU across h100, h200, b200, b300, gb200, gb300 and mi355x. They follow the [SGLang cookbook](https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1), which has no released SGLang version for this model yet. The GB200 arm pins the official -`lmsysorg/sglang:nightly-dev-cu13-20260921-0f6761b5` multi-architecture manifest -`sha256:987c7e4bd26918647211a5dcad72a2bdf2a5f394ac1469eff730e7517fc139be`; -its ARM64 image is `sha256:9e1fb4c395b9c406136e10aa445b8784d06bca3839623b52cbe4a3b231a157a8`. +`lmsysorg/sglang:nightly-dev-cu13-20260922-582389ce` multi-architecture manifest +`sha256:0e1b14e302619a42ef5581b87db6804651d4f946301cf543d74a3a1eb1c33b40`; +its ARM64 image is `sha256:044b18a3a7ce0437e366c60a32d7e7109d1d846aee4e301e3ff1c7932c30695a`. The GB200 recipe retains TP4/EP4, native five-token DSpark, automatic backends, and the existing GPU-resident KV and golden-AL policy. Other NVIDIA arms use the multi-arch preview build `lmsysorg/sglang:dev-dsv41` and MI355X uses diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 677445c433..9edc121ded 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -351,9 +351,9 @@ Maximum concurrency for 1,048,576 tokens per request: 6.70x 与 mi355x 上的 SGLang 对应版本（每个 SKU 一个 PR），遵循 [SGLang cookbook](https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1)。 该模型尚无正式发布的 SGLang 版本。GB200 配方固定官方 -`lmsysorg/sglang:nightly-dev-cu13-20260921-0f6761b5` 的多架构 manifest -`sha256:987c7e4bd26918647211a5dcad72a2bdf2a5f394ac1469eff730e7517fc139be`； -ARM64 镜像为 `sha256:9e1fb4c395b9c406136e10aa445b8784d06bca3839623b52cbe4a3b231a157a8`。 +`lmsysorg/sglang:nightly-dev-cu13-20260922-582389ce` 的多架构 manifest +`sha256:0e1b14e302619a42ef5581b87db6804651d4f946301cf543d74a3a1eb1c33b40`； +ARM64 镜像为 `sha256:044b18a3a7ce0437e366c60a32d7e7109d1d846aee4e301e3ff1c7932c30695a`。 GB200 配方保留 TP4/EP4、原生五 token DSpark、自动后端选择、GPU 常驻 KV cache 与既有黄金 AL 策略。 其他 NVIDIA 配方使用多架构预览镜像 `lmsysorg/sglang:dev-dsv41`，MI355X 使用 `lmsysorg/sglang:dev-dsv41-mi35x`。两个标签均可变， diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 907029e102..dd6b6ffc6d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8559,7 +8559,7 @@ scenario-type: - agentic-coding description: - - Move GB200 DeepSeek-V4.1-Flash DSpark from dev-dsv41 to official CUDA 13 nightly-dev-cu13-20260921-0f6761b5, pinned at multi-architecture manifest sha256:987c7e4bd26918647211a5dcad72a2bdf2a5f394ac1469eff730e7517fc139be (ARM64 sha256:9e1fb4c395b9c406136e10aa445b8784d06bca3839623b52cbe4a3b231a157a8); retain TP4/EP4 and native DSpark precision and golden acceptance policy + - Move GB200 DeepSeek-V4.1-Flash DSpark from dev-dsv41 to official CUDA 13 nightly-dev-cu13-20260922-582389ce, pinned at multi-architecture manifest sha256:0e1b14e302619a42ef5581b87db6804651d4f946301cf543d74a3a1eb1c33b40 (ARM64 sha256:044b18a3a7ce0437e366c60a32d7e7109d1d846aee4e301e3ff1c7932c30695a); retain TP4/EP4 and native DSpark precision and golden acceptance policy - 'Use the upstream per_rank Engram host-table layout on GB200: anonymous row-sharded mappings can obtain 512 MiB huge pages on the measured madvise/never anonymous/shared THP policy; preserve original FP8 table weights and restore the two TP all-reduces' - "Use the default MTP/DSpark precision shipped by the official nightly, without custom draft quantization or precision patches." - Keep the sweep DSpark-only; remove the unmeasured STP entry and alias. From d8398c0da69f2fd5f8300a9b80fa43c6f80ae2aa Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 22 Sep 2026 04:06:37 -0500 Subject: [PATCH 11/14] perf(gb200): retain SWA prefixes and tune C16 scheduling --- .../agentic/dsv41flash_fp4_gb200_sglang_mtp.sh | 17 +++++++++++++++++ docs/configuration-procedures.md | 9 +++++++++ docs/configuration-procedures_zh.md | 7 +++++++ perf-changelog.yaml | 1 + 4 files changed, 34 insertions(+) diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh index 2d1062ec3d..7e9bcd05d2 100755 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh @@ -106,6 +106,21 @@ case "$SPEC_DECODING" in ;; esac +# Cached prefixes need their final SWA window as well as full-attention KV. +# C16 measured 27.0M full tokens with 1,024 retained tails. Cap the reserve: +# uncapped 64*CONC at C128 would exceed this node's measured KV budget. +SWA_PREFIX_TAILS=$((64 * CONC)) +if (( SWA_PREFIX_TAILS > 1024 )); then + SWA_PREFIX_TAILS=1024 +fi + +# C16 canonical comparison: +13.65% p90 interactivity, -0.30% throughput, +# with p90 TTFT increasing from 2.35 s to 3.51 s. Other points keep defaults. +SCHEDULING_ARGS=() +if [[ "$TP" -eq 4 && "$CONC" -eq 16 ]]; then + SCHEDULING_ARGS=(--prefill-decode-interval 16) +fi + SGLANG_CMD=( python3 -m sglang.launch_server --model-path "$MODEL_PATH" --served-model-name "$MODEL" @@ -120,6 +135,8 @@ SGLANG_CMD=( # first 66k-99k-token AgentX prompts. --mem-fraction-static 0.70 --chunked-prefill-size 4096 + --swa-prefix-tails "$SWA_PREFIX_TAILS" + "${SCHEDULING_ARGS[@]}" "${SPECULATIVE_ARGS[@]}" --max-running-requests "$MAX_RUNNING_REQUESTS" --cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS" diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 88824f1cb5..6655b13956 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -418,6 +418,15 @@ of a performance-frontier contribution. DSpark uses the default precision shippe by the pinned official nightly, without custom draft quantization or precision patches. Full accuracy and performance validation remain required. +GB200 reserves `min(64*CONC, 1024)` SWA prefix tails while retaining static memory +0.70 and chunk size 4096. The C16 reserve leaves a measured 27.0M full-context +KV slots and 439,040 SWA slots; the cap avoids exhausting the measured 51.82 GiB +KV budget at high concurrency. Only TP4 C16 uses prefill/decode interval 16: +its canonical comparison improved p90 interactivity 13.65% for 0.30% lower +throughput, with p90 TTFT increasing from 2.35 to 3.51 seconds. Full GSM8K +passed all 1,319 samples. Other concurrency points still require the full sweep; +these C16 results do not establish a benefit at every concurrency. + The GB200 host-table layout is `per_rank`: its compute-node kernel enables anonymous huge pages through `madvise`, while `shmem_enabled=never` prevents huge pages for the shared memfd layout. Upstream allocates row shards in anonymous host diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 9edc121ded..9cb3d810c2 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -364,6 +364,13 @@ GB200 sweep 仅包含 `dsv41flash-fp4-gb200-sglang-agentic-dspark`。 DSpark 使用固定官方 nightly 默认提供的精度，不应用自定义草稿量化或精度补丁。 完整准确率和性能验证仍然必需。 +GB200 保留 `min(64*CONC, 1024)` 个 SWA prefix tails，并维持 static memory 0.70 +和 chunk size 4096。C16 实测保留 2,700 万个 full-context KV slots 与 439,040 个 SWA slots； +该上限避免高并发时耗尽实测 51.82 GiB KV 预算。仅 TP4 C16 使用 prefill/decode interval 16： +canonical 对比中 p90 interactivity 提升 13.65%，吞吐下降 0.30%，p90 TTFT 从 2.35 秒增至 +3.51 秒。完整 GSM8K 的 1,319 个样本通过验证。其他并发点仍需完整 sweep； +C16 结果不能证明该设置在所有并发下均有收益。 + GB200 的主机表布局为 `per_rank`：计算节点内核通过 `madvise` 启用匿名大页， 而 `shmem_enabled=never` 阻止共享 memfd 布局使用大页。上游在匿名主机内存中 按行分片，并通过 `MADV_HUGEPAGE`/`MADV_COLLAPSE` 请求 512 MiB 大页。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7fec8ba4e2..3a2e805896 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8597,4 +8597,5 @@ - 'Use the upstream per_rank Engram host-table layout on GB200: anonymous row-sharded mappings can obtain 512 MiB huge pages on the measured madvise/never anonymous/shared THP policy; preserve original FP8 table weights and restore the two TP all-reduces' - "Use the default MTP/DSpark precision shipped by the official nightly, without custom draft quantization or precision patches." - Keep the sweep DSpark-only; remove the unmeasured STP entry and alias. + - Retain min(64*concurrency, 1024) SWA prefix tails within the measured GB200 KV budget; select prefill/decode interval 16 only at TP4 C16 after canonical performance and full GSM8K qualification. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3347 From a205a985da4bfe600601ef5eea9b59a13f36e8a0 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 23 Sep 2026 09:39:03 -0500 Subject: [PATCH 12/14] perf(gb200): match the full vLLM TP2 and TP4 EP1 grid --- .github/workflows/benchmark-tmpl.yml | 2 +- .../dsv41flash_fp4_gb200_sglang_mtp.sh | 36 +++++++++++++++---- configs/nvidia-master.yaml | 13 +++---- docs/configuration-procedures.md | 8 +++-- docs/configuration-procedures_zh.md | 8 +++-- perf-changelog.yaml | 3 +- runners/runtime_settings.sh | 9 +++++ 7 files changed, 58 insertions(+), 21 deletions(-) diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 11b14aadec..c80f6a17f3 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -203,7 +203,7 @@ jobs: ) || format('[{0}]', toJSON(inputs.runner)) ) }} - timeout-minutes: 500 + timeout-minutes: ${{ inputs.runner == 'cluster:gb200-nv' && fromJSON(inputs.config).model-prefix == 'dsv41flash' && fromJSON(inputs.config).framework == 'sglang' && inputs.scenario-type == 'agentic-coding' && !inputs.eval-only && fromJSON(inputs.config).conc >= 64 && 1470 || 500 }} name: >- ${{ inputs.klaud-run && 'klaud | ' || '' }}p${{ inputs.priority }} | ${{ fromJSON(inputs.config).model-prefix }} ${{ fromJSON(inputs.config).precision }} ${{ inputs.runner }} ${{ fromJSON(inputs.config).framework == 'sglang' && 'sgl' || fromJSON(inputs.config).framework == 'dynamo-sglang' && 'dyn-sgl' || fromJSON(inputs.config).framework == 'sglang-disagg' && 'sgl-disagg' || fromJSON(inputs.config).framework }} TP${{ fromJSON(inputs.config).tp }}${{ format('{0}', fromJSON(inputs.config).pp) != '' && format('{0}', fromJSON(inputs.config).pp) != '1' && format('/PP{0}', fromJSON(inputs.config).pp) || '' }}${{ format('{0}', fromJSON(inputs.config).dcp-size) != '' && format('{0}', fromJSON(inputs.config).dcp-size) != '1' && format('/DCP{0}', fromJSON(inputs.config).dcp-size) || '' }}${{ format('{0}', fromJSON(inputs.config).pcp-size) != '' && format('{0}', fromJSON(inputs.config).pcp-size) != '1' && format('/PCP{0}', fromJSON(inputs.config).pcp-size) || '' }}${{ format('{0}', fromJSON(inputs.config).ep) != '' && format('{0}', fromJSON(inputs.config).ep) != '1' && format('/EP{0}', fromJSON(inputs.config).ep) || '' }}${{ inputs.dp-attn && '/DPA' || '' }} diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh index 7e9bcd05d2..643673967b 100755 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh @@ -1,14 +1,18 @@ #!/usr/bin/env bash set -eo pipefail -# DeepSeek-V4.1-Flash AgentX on GB200 with native STP or DSpark serving. -# Both use the cookbook's Blackwell TP4/EP4 layout and GPU-resident KV cache. +# DeepSeek-V4.1-Flash AgentX on GB200 with shipped-default DSpark serving. +# Match vLLM's TP2/EP1 and TP4/EP1 layouts with GPU-resident KV cache. # https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1 source "$(dirname "$0")/../../benchmark_lib.sh" check_env_vars MODEL TP EP_SIZE CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION check_env_vars EVAL_ONLY SPEC_DECODING require_agentic_kv_offload_none export GPU_COUNT="$TP" +if (( TP == 2 )); then + # Bound fragmentation during stock MXFP4 loading and long-context prefills. + export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +fi if [[ -n "${SLURM_JOB_ID:-}" ]]; then echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" @@ -63,6 +67,20 @@ export SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT=per_rank # (6.4 GiB allocation with 2 GiB free, run 35306704553). Batches within the # graph tier reuse the capture-time workspace instead. CUDA_GRAPH_MAX_BS=64 +MEM_FRACTION_STATIC=0.70 +CHUNKED_PREFILL_SIZE=4096 +case "$TP" in + 2) + # EP1 keeps all 384 experts tensor-sharded per rank. The nearby B200 + # EP1 run passed full GSM8K with these supported memory limits; GB200 + # still requires its own pool, graph and full-curve validation. + MEM_FRACTION_STATIC=0.92 + CHUNKED_PREFILL_SIZE=2048 + CUDA_GRAPH_MAX_BS=16 + ;; + 4) ;; + *) echo "Unsupported TP=$TP; expected 2 or 4" >&2; exit 1 ;; +esac MAX_RUNNING_REQUESTS=$((2 * CONC)) if (( MAX_RUNNING_REQUESTS > CUDA_GRAPH_MAX_BS )); then MAX_RUNNING_REQUESTS=$CUDA_GRAPH_MAX_BS @@ -110,14 +128,18 @@ esac # C16 measured 27.0M full tokens with 1,024 retained tails. Cap the reserve: # uncapped 64*CONC at C128 would exceed this node's measured KV budget. SWA_PREFIX_TAILS=$((64 * CONC)) +if (( TP == 2 )); then + SWA_PREFIX_TAILS=$((128 * CONC)) +fi if (( SWA_PREFIX_TAILS > 1024 )); then SWA_PREFIX_TAILS=1024 fi -# C16 canonical comparison: +13.65% p90 interactivity, -0.30% throughput, -# with p90 TTFT increasing from 2.35 s to 3.51 s. Other points keep defaults. +# Earlier TP4/EP4 C16 canonical comparison: +13.65% p90 interactivity, -0.30% throughput, +# with p90 TTFT increasing from 2.35 s to 3.51 s. Other TP4 points keep defaults. +# TP2 retains the supported interval used by its B200 EP1 memory qualification. SCHEDULING_ARGS=() -if [[ "$TP" -eq 4 && "$CONC" -eq 16 ]]; then +if (( TP == 2 || CONC == 16 )); then SCHEDULING_ARGS=(--prefill-decode-interval 16) fi @@ -133,8 +155,8 @@ SGLANG_CMD=( # sparse-attention indexer and DSpark prefill buffers scale with the chunk # times the 1M context, and the default 16384 chunk exhausted HBM on the # first 66k-99k-token AgentX prompts. - --mem-fraction-static 0.70 - --chunked-prefill-size 4096 + --mem-fraction-static "$MEM_FRACTION_STATIC" + --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" --swa-prefix-tails "$SWA_PREFIX_TAILS" "${SCHEDULING_ARGS[@]}" "${SPECULATIVE_ARGS[@]}" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index eb73d31836..fd2e6d6ace 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8112,16 +8112,16 @@ dsv41flash-fp4-gb200-vllm-agentic-dspark: # SGLang arm for DeepSeek-V4.1-Flash AgentX on GB200, from the SGLang cookbook # (https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1). -# Official CUDA 13 nightly, verified 2026-09-22 (amd64 + arm64): -# manifest sha256:0e1b14e302619a42ef5581b87db6804651d4f946301cf543d74a3a1eb1c33b40 -# arm64 sha256:044b18a3a7ce0437e366c60a32d7e7109d1d846aee4e301e3ff1c7932c30695a. +# Official CUDA 13 nightly, verified 2026-09-23 (amd64 + arm64): +# manifest sha256:5921361fcf358cdde4df1968c941c14157f418613b099ad7f3e5aeed6427ae15 +# arm64 sha256:d49261d2edd82fed2dd6254c33e68871ccf7a399498e059ec91dc4453a5808c3. # Native five-token DSpark (--speculative-algorithm DSPARK # --speculative-dspark-block-size 5), thinking on, golden AL 3.51 for # throughput, real verification for evals. The KV cache stays GPU-resident. -# Follows the cookbook's verified Blackwell TP4/EP4 low-latency cell; backends +# Match published vLLM TP2/EP1 and TP4/EP1 across C1-128; native backends # resolve automatically (dsv4 / flashinfer_mxfp4 / flashinfer_cutedsl). dsv41flash-fp4-gb200-sglang-agentic-dspark: - image: lmsysorg/sglang:nightly-dev-cu13-20260922-582389ce@sha256:0e1b14e302619a42ef5581b87db6804651d4f946301cf543d74a3a1eb1c33b40 + image: lmsysorg/sglang:nightly-dev-cu13-20260923-06008c17@sha256:5921361fcf358cdde4df1968c941c14157f418613b099ad7f3e5aeed6427ae15 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:gb200-nv @@ -8132,7 +8132,8 @@ dsv41flash-fp4-gb200-sglang-agentic-dspark: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 2, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } dsv41flash-fp4-b300-vllm-agentic-dspark: image: vllm/vllm-openai:deepseekv41-flash-0909 diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index d51dc5f785..3dca2287ad 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -437,7 +437,7 @@ manifest-reference syntax and stops immediately on import failure. DSpark uses the default precision shipped by the pinned official nightly, without custom draft quantization or precision patches. STP loads no draft; full accuracy and performance validation are still required. -GB200 pins the official CUDA 13 nightly separately; its topology and memory settings are described below. +GB200 pins official CUDA 13 nightly `20260923-06008c17` to manifest `sha256:5921361fcf358cdde4df1968c941c14157f418613b099ad7f3e5aeed6427ae15` (ARM64 `sha256:d49261d2edd82fed2dd6254c33e68871ccf7a399498e059ec91dc4453a5808c3`). It matches the published vLLM TP2/EP1 and TP4/EP1 grids at C1/2/4/8/16/32/64/128, with no DP attention. The earlier staged TP4/EP4 sweep is historical evidence, not qualification of these topologies. The GB200 sweep contains only `dsv41flash-fp4-gb200-sglang-agentic-dspark`. The unmeasured STP entry is excluded; adding it would require matched evidence @@ -445,8 +445,8 @@ of a performance-frontier contribution. DSpark uses the default precision shippe by the pinned official nightly, without custom draft quantization or precision patches. Full accuracy and performance validation remain required. -GB200 reserves `min(64*CONC, 1024)` SWA prefix tails while retaining static memory -0.70 and chunk size 4096. The C16 reserve leaves a measured 27.0M full-context +GB200 TP4 reserves `min(64*CONC, 1024)` SWA prefix tails while retaining static memory +0.70 and chunk size 4096. The earlier TP4/EP4 C16 reserve left a measured 27.0M full-context KV slots and 439,040 SWA slots; the cap avoids exhausting the measured 51.82 GiB KV budget at high concurrency. Only TP4 C16 uses prefill/decode interval 16: its canonical comparison improved p90 interactivity 13.65% for 0.30% lower @@ -454,6 +454,8 @@ throughput, with p90 TTFT increasing from 2.35 to 3.51 seconds. Full GSM8K passed all 1,319 samples. Other concurrency points still require the full sweep; these C16 results do not establish a benefit at every concurrency. +GB200 TP2 uses static memory fraction 0.92, a 2048-token prefill chunk, `min(128*CONC,1024)` SWA tails, prefill/decode interval 16 and graph/running capacity bounded to 16 requests. These supported limits follow the completed B200 EP1 memory qualification; GB200 must independently pass loading, graph capture, full-context pool checks and every performance/evaluation cell. Expandable CUDA allocator segments reduce fragmentation without changing weights or precision. C64/C128 performance receives a 24-hour allocation plus 30 minutes for workflow packaging; full warmup, the 3600-second scoring window and uncapped 1,319-question GSM8K remain unchanged. + The GB200 host-table layout is `per_rank`: its compute-node kernel enables anonymous huge pages through `madvise`, while `shmem_enabled=never` prevents huge pages for the shared memfd layout. Upstream allocates row shards in anonymous host diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 6f4a8c9ea4..43502eb3a1 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -380,20 +380,22 @@ B200 启动器还将固定 Docker digest DSpark 使用固定官方 nightly 默认提供的精度，不应用自定义草稿量化或精度补丁。STP 不加载草稿模型；完整准确率和性能验证仍然必需。 -GB200 单独固定官方 CUDA 13 nightly；其拓扑和显存设置见下文。 +GB200 固定官方 CUDA 13 nightly `20260923-06008c17`，manifest 为 `sha256:5921361fcf358cdde4df1968c941c14157f418613b099ad7f3e5aeed6427ae15`（ARM64 为 `sha256:d49261d2edd82fed2dd6254c33e68871ccf7a399498e059ec91dc4453a5808c3`）。拓扑与已发布 vLLM 一致：TP2/EP1 和 TP4/EP1，均覆盖 C1/2/4/8/16/32/64/128，不启用 DP attention。此前已暂存的 TP4/EP4 sweep 仅为历史证据，不能替代本次拓扑验证。 GB200 sweep 仅包含 `dsv41flash-fp4-gb200-sglang-agentic-dspark`。 移除尚无实测依据的 STP 条目；若要加入，须通过匹配测试证明其对性能前沿有贡献。 DSpark 使用固定官方 nightly 默认提供的精度，不应用自定义草稿量化或精度补丁。 完整准确率和性能验证仍然必需。 -GB200 保留 `min(64*CONC, 1024)` 个 SWA prefix tails，并维持 static memory 0.70 -和 chunk size 4096。C16 实测保留 2,700 万个 full-context KV slots 与 439,040 个 SWA slots； +GB200 TP4 保留 `min(64*CONC, 1024)` 个 SWA prefix tails，并维持 static memory 0.70 +和 chunk size 4096。此前 TP4/EP4 C16 实测保留 2,700 万个 full-context KV slots 与 439,040 个 SWA slots； 该上限避免高并发时耗尽实测 51.82 GiB KV 预算。仅 TP4 C16 使用 prefill/decode interval 16： canonical 对比中 p90 interactivity 提升 13.65%，吞吐下降 0.30%，p90 TTFT 从 2.35 秒增至 3.51 秒。完整 GSM8K 的 1,319 个样本通过验证。其他并发点仍需完整 sweep； C16 结果不能证明该设置在所有并发下均有收益。 +GB200 TP2 使用 0.92 静态显存比例、2048-token 预填充块、`min(128*CONC,1024)` 个 SWA tails、16 的 prefill/decode interval，graph 与 running capacity 上限为 16 个请求。这些受支持的限制参考已完成的 B200 EP1 显存验证；GB200 仍须独立通过加载、图捕获、完整上下文缓存池以及全部性能和准确率测试。可扩展 CUDA allocator segments 仅减少碎片，不改变权重或精度。C64/C128 性能任务获得 24 小时 allocation，工作流额外预留 30 分钟打包产物；完整预热、3600 秒计分时段及无样本限制的 1,319 题 GSM8K 均保持不变。 + GB200 的主机表布局为 `per_rank`：计算节点内核通过 `madvise` 启用匿名大页， 而 `shmem_enabled=never` 阻止共享 memfd 布局使用大页。上游在匿名主机内存中 按行分片，并通过 `MADV_HUGEPAGE`/`MADV_COLLAPSE` 请求 512 MiB 大页。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e29daa537f..78d835b789 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8764,9 +8764,10 @@ scenario-type: - agentic-coding description: - - Move GB200 DeepSeek-V4.1-Flash DSpark from dev-dsv41 to official CUDA 13 nightly-dev-cu13-20260922-582389ce, pinned at multi-architecture manifest sha256:0e1b14e302619a42ef5581b87db6804651d4f946301cf543d74a3a1eb1c33b40 (ARM64 sha256:044b18a3a7ce0437e366c60a32d7e7109d1d846aee4e301e3ff1c7932c30695a); retain TP4/EP4 and native DSpark precision and golden acceptance policy + - Move GB200 DeepSeek-V4.1-Flash DSpark from dev-dsv41 to official CUDA 13 nightly-dev-cu13-20260923-06008c17, pinned at multi-architecture manifest sha256:5921361fcf358cdde4df1968c941c14157f418613b099ad7f3e5aeed6427ae15 (ARM64 sha256:d49261d2edd82fed2dd6254c33e68871ccf7a399498e059ec91dc4453a5808c3); match TP2/EP1 and TP4/EP1 with native DSpark precision and golden acceptance policy - 'Use the upstream per_rank Engram host-table layout on GB200: anonymous row-sharded mappings can obtain 512 MiB huge pages on the measured madvise/never anonymous/shared THP policy; preserve original FP8 table weights and restore the two TP all-reduces' - "Use the default MTP/DSpark precision shipped by the official nightly, without custom draft quantization or precision patches." - Keep the sweep DSpark-only; remove the unmeasured STP entry and alias. - Retain min(64*concurrency, 1024) SWA prefix tails within the measured GB200 KV budget; select prefill/decode interval 16 only at TP4 C16 after canonical performance and full GSM8K qualification. + - "Match the published vLLM grid exactly: TP2/EP1 and TP4/EP1, each C1/2/4/8/16/32/64/128. Use stock DSpark precision; TP2 uses static memory 0.92, chunk 2048, graph/running cap 16, interval 16, expandable allocator and min(128*CONC,1024) SWA tails. Requalify all 16 performance and full GSM8K points. Avoid deadline-only failures with 24-hour allocations for C64/C128 performance while preserving complete warmup and 3600-second scoring." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3347 diff --git a/runners/runtime_settings.sh b/runners/runtime_settings.sh index 09de8f25e5..c864d06c7a 100644 --- a/runners/runtime_settings.sh +++ b/runners/runtime_settings.sh @@ -27,6 +27,15 @@ case "${RUNNER_NAME%%_*}" in fi ;; gb300-nv) export SLURM_PARTITION=batch_1 ;; + gb200-nv) + if [[ "$MODEL_PREFIX" == dsv41flash && "$FRAMEWORK" == sglang && "$IS_MULTINODE" == false && "$IS_AGENTIC" == 1 ]]; then + check_env_vars CONC EVAL_ONLY + if [[ "$EVAL_ONLY" == false ]] && (( CONC >= 64 )); then + # Preserve the complete high-concurrency warmup and scored hour. + export SALLOC_TIME_LIMIT=1440 + fi + fi + ;; h200-dgxc-slurm) export HF_HUB_CACHE_MOUNT=/models/gharunners/hf-hub-cache export AIPERF_MMAP_CACHE_HOST_PATH=/home/sa-shared/gharunners/ai-perf-cache From 85d8aee8a879b5aa27f827e472673a055fd370f0 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 23 Sep 2026 09:45:57 -0500 Subject: [PATCH 13/14] fix(gb200): respect the Slurm partition allocation limit --- .github/workflows/benchmark-tmpl.yml | 2 +- docs/configuration-procedures.md | 2 +- docs/configuration-procedures_zh.md | 2 +- perf-changelog.yaml | 2 +- runners/runtime_settings.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index c80f6a17f3..56dd4f6547 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -203,7 +203,7 @@ jobs: ) || format('[{0}]', toJSON(inputs.runner)) ) }} - timeout-minutes: ${{ inputs.runner == 'cluster:gb200-nv' && fromJSON(inputs.config).model-prefix == 'dsv41flash' && fromJSON(inputs.config).framework == 'sglang' && inputs.scenario-type == 'agentic-coding' && !inputs.eval-only && fromJSON(inputs.config).conc >= 64 && 1470 || 500 }} + timeout-minutes: ${{ inputs.runner == 'cluster:gb200-nv' && fromJSON(inputs.config).model-prefix == 'dsv41flash' && fromJSON(inputs.config).framework == 'sglang' && inputs.scenario-type == 'agentic-coding' && !inputs.eval-only && fromJSON(inputs.config).conc >= 64 && 750 || 500 }} name: >- ${{ inputs.klaud-run && 'klaud | ' || '' }}p${{ inputs.priority }} | ${{ fromJSON(inputs.config).model-prefix }} ${{ fromJSON(inputs.config).precision }} ${{ inputs.runner }} ${{ fromJSON(inputs.config).framework == 'sglang' && 'sgl' || fromJSON(inputs.config).framework == 'dynamo-sglang' && 'dyn-sgl' || fromJSON(inputs.config).framework == 'sglang-disagg' && 'sgl-disagg' || fromJSON(inputs.config).framework }} TP${{ fromJSON(inputs.config).tp }}${{ format('{0}', fromJSON(inputs.config).pp) != '' && format('{0}', fromJSON(inputs.config).pp) != '1' && format('/PP{0}', fromJSON(inputs.config).pp) || '' }}${{ format('{0}', fromJSON(inputs.config).dcp-size) != '' && format('{0}', fromJSON(inputs.config).dcp-size) != '1' && format('/DCP{0}', fromJSON(inputs.config).dcp-size) || '' }}${{ format('{0}', fromJSON(inputs.config).pcp-size) != '' && format('{0}', fromJSON(inputs.config).pcp-size) != '1' && format('/PCP{0}', fromJSON(inputs.config).pcp-size) || '' }}${{ format('{0}', fromJSON(inputs.config).ep) != '' && format('{0}', fromJSON(inputs.config).ep) != '1' && format('/EP{0}', fromJSON(inputs.config).ep) || '' }}${{ inputs.dp-attn && '/DPA' || '' }} diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 3dca2287ad..eb1e07a541 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -454,7 +454,7 @@ throughput, with p90 TTFT increasing from 2.35 to 3.51 seconds. Full GSM8K passed all 1,319 samples. Other concurrency points still require the full sweep; these C16 results do not establish a benefit at every concurrency. -GB200 TP2 uses static memory fraction 0.92, a 2048-token prefill chunk, `min(128*CONC,1024)` SWA tails, prefill/decode interval 16 and graph/running capacity bounded to 16 requests. These supported limits follow the completed B200 EP1 memory qualification; GB200 must independently pass loading, graph capture, full-context pool checks and every performance/evaluation cell. Expandable CUDA allocator segments reduce fragmentation without changing weights or precision. C64/C128 performance receives a 24-hour allocation plus 30 minutes for workflow packaging; full warmup, the 3600-second scoring window and uncapped 1,319-question GSM8K remain unchanged. +GB200 TP2 uses static memory fraction 0.92, a 2048-token prefill chunk, `min(128*CONC,1024)` SWA tails, prefill/decode interval 16 and graph/running capacity bounded to 16 requests. These supported limits follow the completed B200 EP1 memory qualification; GB200 must independently pass loading, graph capture, full-context pool checks and every performance/evaluation cell. Expandable CUDA allocator segments reduce fragmentation without changing weights or precision. C64/C128 performance receives the partition maximum 12-hour allocation plus 30 minutes for workflow packaging; full warmup, the 3600-second scoring window and uncapped 1,319-question GSM8K remain unchanged. The GB200 host-table layout is `per_rank`: its compute-node kernel enables anonymous huge pages through `madvise`, while `shmem_enabled=never` prevents huge diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 43502eb3a1..78b48d85b5 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -394,7 +394,7 @@ canonical 对比中 p90 interactivity 提升 13.65%，吞吐下降 0.30%，p90 T 3.51 秒。完整 GSM8K 的 1,319 个样本通过验证。其他并发点仍需完整 sweep； C16 结果不能证明该设置在所有并发下均有收益。 -GB200 TP2 使用 0.92 静态显存比例、2048-token 预填充块、`min(128*CONC,1024)` 个 SWA tails、16 的 prefill/decode interval，graph 与 running capacity 上限为 16 个请求。这些受支持的限制参考已完成的 B200 EP1 显存验证；GB200 仍须独立通过加载、图捕获、完整上下文缓存池以及全部性能和准确率测试。可扩展 CUDA allocator segments 仅减少碎片，不改变权重或精度。C64/C128 性能任务获得 24 小时 allocation，工作流额外预留 30 分钟打包产物；完整预热、3600 秒计分时段及无样本限制的 1,319 题 GSM8K 均保持不变。 +GB200 TP2 使用 0.92 静态显存比例、2048-token 预填充块、`min(128*CONC,1024)` 个 SWA tails、16 的 prefill/decode interval，graph 与 running capacity 上限为 16 个请求。这些受支持的限制参考已完成的 B200 EP1 显存验证；GB200 仍须独立通过加载、图捕获、完整上下文缓存池以及全部性能和准确率测试。可扩展 CUDA allocator segments 仅减少碎片，不改变权重或精度。C64/C128 性能任务获得分区允许的最大 12 小时 allocation，工作流额外预留 30 分钟打包产物；完整预热、3600 秒计分时段及无样本限制的 1,319 题 GSM8K 均保持不变。 GB200 的主机表布局为 `per_rank`：计算节点内核通过 `madvise` 启用匿名大页， 而 `shmem_enabled=never` 阻止共享 memfd 布局使用大页。上游在匿名主机内存中 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b66cb48c05..853e0875a0 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8781,5 +8781,5 @@ - "Use the default MTP/DSpark precision shipped by the official nightly, without custom draft quantization or precision patches." - Keep the sweep DSpark-only; remove the unmeasured STP entry and alias. - Retain min(64*concurrency, 1024) SWA prefix tails within the measured GB200 KV budget; select prefill/decode interval 16 only at TP4 C16 after canonical performance and full GSM8K qualification. - - "Match the published vLLM grid exactly: TP2/EP1 and TP4/EP1, each C1/2/4/8/16/32/64/128. Use stock DSpark precision; TP2 uses static memory 0.92, chunk 2048, graph/running cap 16, interval 16, expandable allocator and min(128*CONC,1024) SWA tails. Requalify all 16 performance and full GSM8K points. Avoid deadline-only failures with 24-hour allocations for C64/C128 performance while preserving complete warmup and 3600-second scoring." + - "Match the published vLLM grid exactly: TP2/EP1 and TP4/EP1, each C1/2/4/8/16/32/64/128. Use stock DSpark precision; TP2 uses static memory 0.92, chunk 2048, graph/running cap 16, interval 16, expandable allocator and min(128*CONC,1024) SWA tails. Requalify all 16 performance and full GSM8K points. Avoid deadline-only failures with the supported 12-hour partition maximum for C64/C128 performance while preserving complete warmup and 3600-second scoring." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3347 diff --git a/runners/runtime_settings.sh b/runners/runtime_settings.sh index c864d06c7a..9de89faf6a 100644 --- a/runners/runtime_settings.sh +++ b/runners/runtime_settings.sh @@ -32,7 +32,7 @@ case "${RUNNER_NAME%%_*}" in check_env_vars CONC EVAL_ONLY if [[ "$EVAL_ONLY" == false ]] && (( CONC >= 64 )); then # Preserve the complete high-concurrency warmup and scored hour. - export SALLOC_TIME_LIMIT=1440 + export SALLOC_TIME_LIMIT=720 fi fi ;; From 33808dcce064fabd281af678b4e647c57c94b949 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 24 Sep 2026 01:14:36 -0500 Subject: [PATCH 14/14] refactor: use HF download result for GB200 snapshot path [skip-sweep] --- .../single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh index 643673967b..a41ea220f1 100755 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh @@ -22,8 +22,7 @@ fi if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL" ]]; then hf download "$MODEL" --local-dir "$MODEL_PATH" else - hf download "$MODEL" - MODEL_PATH=$(python3 -c 'from huggingface_hub import snapshot_download; import sys; print(snapshot_download(repo_id=sys.argv[1], local_files_only=True))' "$MODEL") + MODEL_PATH=$(hf download "$MODEL") export MODEL_PATH fi