diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index e0f57452f3..b9a66f22ba 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -203,7 +203,7 @@ jobs: ) || format('[{0}]', toJSON(inputs.runner)) ) }} - timeout-minutes: ${{ inputs.runner == 'cluster:h200-dgxc' && fromJSON(inputs.config).model-prefix == 'dsv41flash' && fromJSON(inputs.config).framework == 'sglang' && inputs.scenario-type == 'agentic-coding' && !inputs.eval-only && fromJSON(inputs.config).conc >= 64 && 1470 || 500 }} + timeout-minutes: ${{ inputs.runner == 'cluster:h200-dgxc' && fromJSON(inputs.config).model-prefix == 'dsv41flash' && fromJSON(inputs.config).framework == 'sglang' && inputs.scenario-type == 'agentic-coding' && !inputs.eval-only && fromJSON(inputs.config).conc >= 64 && 1470 || inputs.runner == 'cluster:gb200-nv' && fromJSON(inputs.config).model-prefix == 'dsv41flash' && fromJSON(inputs.config).framework == 'sglang' && inputs.scenario-type == 'agentic-coding' && !inputs.eval-only && fromJSON(inputs.config).conc >= 64 && 750 || 500 }} name: >- ${{ inputs.klaud-run && 'klaud | ' || '' }}p${{ inputs.priority }} | ${{ fromJSON(inputs.config).model-prefix }} ${{ fromJSON(inputs.config).precision }} ${{ inputs.runner }} ${{ fromJSON(inputs.config).framework == 'sglang' && 'sgl' || fromJSON(inputs.config).framework == 'dynamo-sglang' && 'dyn-sgl' || fromJSON(inputs.config).framework == 'sglang-disagg' && 'sgl-disagg' || fromJSON(inputs.config).framework }} TP${{ fromJSON(inputs.config).tp }}${{ format('{0}', fromJSON(inputs.config).pp) != '' && format('{0}', fromJSON(inputs.config).pp) != '1' && format('/PP{0}', fromJSON(inputs.config).pp) || '' }}${{ format('{0}', fromJSON(inputs.config).dcp-size) != '' && format('{0}', fromJSON(inputs.config).dcp-size) != '1' && format('/DCP{0}', fromJSON(inputs.config).dcp-size) || '' }}${{ format('{0}', fromJSON(inputs.config).pcp-size) != '' && format('{0}', fromJSON(inputs.config).pcp-size) != '1' && format('/PCP{0}', fromJSON(inputs.config).pcp-size) || '' }}${{ format('{0}', fromJSON(inputs.config).ep) != '' && format('{0}', fromJSON(inputs.config).ep) != '1' && format('/EP{0}', fromJSON(inputs.config).ep) || '' }}${{ inputs.dp-attn && '/DPA' || '' }} diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh index 5c9b2c13b5..a41ea220f1 100755 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh @@ -1,14 +1,18 @@ #!/usr/bin/env bash set -eo pipefail -# DeepSeek-V4.1-Flash AgentX on GB200 with SGLang native DSpark, following the -# cookbook's verified Blackwell TP4/EP4 low-latency cell. The KV cache is GPU-resident. +# DeepSeek-V4.1-Flash AgentX on GB200 with shipped-default DSpark serving. +# Match vLLM's TP2/EP1 and TP4/EP1 layouts with GPU-resident KV cache. # https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1 source "$(dirname "$0")/../../benchmark_lib.sh" check_env_vars MODEL TP EP_SIZE CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY +check_env_vars EVAL_ONLY SPEC_DECODING require_agentic_kv_offload_none export GPU_COUNT="$TP" +if (( TP == 2 )); then + # Bound fragmentation during stock MXFP4 loading and long-context prefills. + export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +fi if [[ -n "${SLURM_JOB_ID:-}" ]]; then echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" @@ -18,8 +22,8 @@ fi if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL" ]]; then hf download "$MODEL" --local-dir "$MODEL_PATH" else - hf download "$MODEL" - export MODEL_PATH="$MODEL" + MODEL_PATH=$(hf download "$MODEL") + export MODEL_PATH fi nvidia-smi @@ -30,6 +34,8 @@ SERVER_LOG="$RESULT_DIR/server.log" export PYTHONNOUSERSITE=1 export PYTHONUNBUFFERED=1 +# Use the default DSpark precision shipped by the pinned SGLang nightly. + # Agentic warmup dispatches hundreds of large prompts at once and SGLang's # tokenizer can leave bytes unacknowledged past AIPerf's default 30 s # TCP_USER_TIMEOUT, so Linux aborts live localhost connections. @@ -43,13 +49,14 @@ export SGLANG_TIMEOUT_KEEP_ALIVE=900 export SGLANG_DEFAULT_THINKING=1 export SGLANG_DSV41_REASONING_EFFORT=high -# One shared host copy of the two fp8 Engram tables instead of a row-sharded -# copy per rank: the SGLang analogue of the vLLM arm's Engram CPU offload. It -# frees ~46 GiB of HBM per GPU for the 1M-context prefill working set and the -# KV pool, and output is bitwise unchanged (cookbook). The first sweep ran -# with the tables on GPU and the server died on the first long AgentX prompts -# (run 35304536578: c2 came up, then the server exited on the first warmup prompt). +# Keep the Engram weights in row-sharded host DRAM. GB200's 64 KiB-page +# kernel enables anonymous THP with madvise but disables shmem THP, so the +# shared memfd layout cannot obtain huge-page backing. The upstream per-rank +# layout uses anonymous mappings, MADV_HUGEPAGE and MADV_COLLAPSE for 512 MiB +# pages; row ownership and the original FP8 table weights are preserved. +# This trades two TP all-reduces for fewer host-table translation misses. export SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1 +export SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT=per_rank # AgentX concurrency counts live session trees, not individual requests. # Allow subagent fan-out to exceed CONC without clipping request bursts, but @@ -59,6 +66,20 @@ export SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1 # (6.4 GiB allocation with 2 GiB free, run 35306704553). Batches within the # graph tier reuse the capture-time workspace instead. CUDA_GRAPH_MAX_BS=64 +MEM_FRACTION_STATIC=0.70 +CHUNKED_PREFILL_SIZE=4096 +case "$TP" in + 2) + # EP1 keeps all 384 experts tensor-sharded per rank. The nearby B200 + # EP1 run passed full GSM8K with these supported memory limits; GB200 + # still requires its own pool, graph and full-curve validation. + MEM_FRACTION_STATIC=0.92 + CHUNKED_PREFILL_SIZE=2048 + CUDA_GRAPH_MAX_BS=16 + ;; + 4) ;; + *) echo "Unsupported TP=$TP; expected 2 or 4" >&2; exit 1 ;; +esac MAX_RUNNING_REQUESTS=$((2 * CONC)) if (( MAX_RUNNING_REQUESTS > CUDA_GRAPH_MAX_BS )); then MAX_RUNNING_REQUESTS=$CUDA_GRAPH_MAX_BS @@ -77,18 +98,49 @@ export AIPERF_SERVER_METRICS_URLS="${AIPERF_SERVER_URL}/metrics" export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:" echo "Using SGLang endpoint ${AIPERF_SERVER_URL}" -# DSpark is the checkpoint's own bundled draft: no EAGLE/MTP path and no -# --speculative-num-steps knob; the block size is the only tunable. Golden AL: -# golden_al_distribution/dsv41flash_dspark.yaml, thinking_on, five draft tokens. -# Throughput fixes acceptance to AL 3.51; accuracy evals keep real verification. -DSPARK_BLOCK_SIZE=5 -DSV41_GOLDEN_AL=3.51 -if [[ "${EVAL_ONLY}" != true ]]; then - export SGLANG_SIMULATE_ACC_LEN="$DSV41_GOLDEN_AL" - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token +# The caller selects native non-speculative serving or the bundled DSpark +# draft. STP and accuracy evals must never inherit synthetic acceptance. +unset SGLANG_SIMULATE_ACC_LEN SGLANG_SIMULATE_ACC_METHOD SGLANG_SIMULATE_ACC_TOKEN_MODE +SPECULATIVE_ARGS=() +case "$SPEC_DECODING" in + mtp) + DSPARK_BLOCK_SIZE=5 + DSV41_GOLDEN_AL=3.51 + SPECULATIVE_ARGS=(--speculative-algorithm DSPARK --speculative-dspark-block-size "$DSPARK_BLOCK_SIZE") + if [[ "$EVAL_ONLY" != true ]]; then + export SGLANG_SIMULATE_ACC_LEN="$DSV41_GOLDEN_AL" + export SGLANG_SIMULATE_ACC_METHOD=match-expected + export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token + fi + echo "DSpark block size: $DSPARK_BLOCK_SIZE, golden AL=$DSV41_GOLDEN_AL" + ;; + none) + echo "Native non-speculative serving; synthetic acceptance disabled" + ;; + *) + echo "Unsupported SPEC_DECODING=$SPEC_DECODING; expected mtp or none" >&2 + exit 1 + ;; +esac + +# Cached prefixes need their final SWA window as well as full-attention KV. +# C16 measured 27.0M full tokens with 1,024 retained tails. Cap the reserve: +# uncapped 64*CONC at C128 would exceed this node's measured KV budget. +SWA_PREFIX_TAILS=$((64 * CONC)) +if (( TP == 2 )); then + SWA_PREFIX_TAILS=$((128 * CONC)) +fi +if (( SWA_PREFIX_TAILS > 1024 )); then + SWA_PREFIX_TAILS=1024 +fi + +# Earlier TP4/EP4 C16 canonical comparison: +13.65% p90 interactivity, -0.30% throughput, +# with p90 TTFT increasing from 2.35 s to 3.51 s. Other TP4 points keep defaults. +# TP2 retains the supported interval used by its B200 EP1 memory qualification. +SCHEDULING_ARGS=() +if (( TP == 2 || CONC == 16 )); then + SCHEDULING_ARGS=(--prefill-decode-interval 16) fi -echo "DSpark block size: $DSPARK_BLOCK_SIZE, golden AL=$DSV41_GOLDEN_AL" SGLANG_CMD=( python3 -m sglang.launch_server @@ -102,10 +154,11 @@ SGLANG_CMD=( # sparse-attention indexer and DSpark prefill buffers scale with the chunk # times the 1M context, and the default 16384 chunk exhausted HBM on the # first 66k-99k-token AgentX prompts. - --mem-fraction-static 0.70 - --chunked-prefill-size 4096 - --speculative-algorithm DSPARK - --speculative-dspark-block-size "$DSPARK_BLOCK_SIZE" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" + --swa-prefix-tails "$SWA_PREFIX_TAILS" + "${SCHEDULING_ARGS[@]}" + "${SPECULATIVE_ARGS[@]}" --max-running-requests "$MAX_RUNNING_REQUESTS" --cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS" --reasoning-parser auto diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 7b195bc2fe..ee88e5802a 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8112,16 +8112,16 @@ dsv41flash-fp4-gb200-vllm-agentic-dspark: # SGLang arm for DeepSeek-V4.1-Flash AgentX on GB200, from the SGLang cookbook # (https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1). -# DSv4.1 Flash support has not shipped in an SGLang release; this uses the -# preview build lmsysorg/sglang:dev-dsv41, digest -# sha256:4a5d132a06a77c8331e15845f2e925adc788b00105097ad55409afa3f4fa4860 (2026-09-11, amd64 + arm64). +# Official CUDA 13 nightly, verified 2026-09-23 (amd64 + arm64): +# manifest sha256:5921361fcf358cdde4df1968c941c14157f418613b099ad7f3e5aeed6427ae15 +# arm64 sha256:d49261d2edd82fed2dd6254c33e68871ccf7a399498e059ec91dc4453a5808c3. # Native five-token DSpark (--speculative-algorithm DSPARK # --speculative-dspark-block-size 5), thinking on, golden AL 3.51 for # throughput, real verification for evals. The KV cache stays GPU-resident. -# Follows the cookbook's verified Blackwell TP4/EP4 low-latency cell; backends +# Match published vLLM TP2/EP1 and TP4/EP1 across C1-128; native backends # resolve automatically (dsv4 / flashinfer_mxfp4 / flashinfer_cutedsl). dsv41flash-fp4-gb200-sglang-agentic-dspark: - image: lmsysorg/sglang:dev-dsv41 + image: lmsysorg/sglang:nightly-dev-cu13-20260923-06008c17@sha256:5921361fcf358cdde4df1968c941c14157f418613b099ad7f3e5aeed6427ae15 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:gb200-nv @@ -8132,7 +8132,8 @@ dsv41flash-fp4-gb200-sglang-agentic-dspark: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 2, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } dsv41flash-fp4-b300-vllm-agentic-dspark: image: vllm/vllm-openai:deepseekv41-flash-0909 diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index c6941009a7..eb1e07a541 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -437,6 +437,32 @@ manifest-reference syntax and stops immediately on import failure. DSpark uses the default precision shipped by the pinned official nightly, without custom draft quantization or precision patches. STP loads no draft; full accuracy and performance validation are still required. +GB200 pins official CUDA 13 nightly `20260923-06008c17` to manifest `sha256:5921361fcf358cdde4df1968c941c14157f418613b099ad7f3e5aeed6427ae15` (ARM64 `sha256:d49261d2edd82fed2dd6254c33e68871ccf7a399498e059ec91dc4453a5808c3`). It matches the published vLLM TP2/EP1 and TP4/EP1 grids at C1/2/4/8/16/32/64/128, with no DP attention. The earlier staged TP4/EP4 sweep is historical evidence, not qualification of these topologies. + +The GB200 sweep contains only `dsv41flash-fp4-gb200-sglang-agentic-dspark`. +The unmeasured STP entry is excluded; adding it would require matched evidence +of a performance-frontier contribution. DSpark uses the default precision shipped +by the pinned official nightly, without custom draft quantization or precision +patches. Full accuracy and performance validation remain required. + +GB200 TP4 reserves `min(64*CONC, 1024)` SWA prefix tails while retaining static memory +0.70 and chunk size 4096. The earlier TP4/EP4 C16 reserve left a measured 27.0M full-context +KV slots and 439,040 SWA slots; the cap avoids exhausting the measured 51.82 GiB +KV budget at high concurrency. Only TP4 C16 uses prefill/decode interval 16: +its canonical comparison improved p90 interactivity 13.65% for 0.30% lower +throughput, with p90 TTFT increasing from 2.35 to 3.51 seconds. Full GSM8K +passed all 1,319 samples. Other concurrency points still require the full sweep; +these C16 results do not establish a benefit at every concurrency. + +GB200 TP2 uses static memory fraction 0.92, a 2048-token prefill chunk, `min(128*CONC,1024)` SWA tails, prefill/decode interval 16 and graph/running capacity bounded to 16 requests. These supported limits follow the completed B200 EP1 memory qualification; GB200 must independently pass loading, graph capture, full-context pool checks and every performance/evaluation cell. Expandable CUDA allocator segments reduce fragmentation without changing weights or precision. C64/C128 performance receives the partition maximum 12-hour allocation plus 30 minutes for workflow packaging; full warmup, the 3600-second scoring window and uncapped 1,319-question GSM8K remain unchanged. + +The GB200 host-table layout is `per_rank`: its compute-node kernel enables +anonymous huge pages through `madvise`, while `shmem_enabled=never` prevents huge +pages for the shared memfd layout. Upstream allocates row shards in anonymous host +memory and requests 512 MiB huge pages with `MADV_HUGEPAGE`/`MADV_COLLAPSE`. +It preserves the original FP8 table weights and restores the two TP all-reduces; +inspect startup's actual resident/huge-page counts before claiming a benefit. + DSpark is the checkpoint's own bundled draft. SGLang exposes no EAGLE or MTP path and no `--speculative-num-steps` knob for it; the recipes pass `--speculative-algorithm DSPARK --speculative-dspark-block-size 5`. Throughput uses the same diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 54986db75c..78b48d85b5 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -380,6 +380,28 @@ B200 启动器还将固定 Docker digest DSpark 使用固定官方 nightly 默认提供的精度,不应用自定义草稿量化或精度补丁。STP 不加载草稿模型;完整准确率和性能验证仍然必需。 +GB200 固定官方 CUDA 13 nightly `20260923-06008c17`,manifest 为 `sha256:5921361fcf358cdde4df1968c941c14157f418613b099ad7f3e5aeed6427ae15`(ARM64 为 `sha256:d49261d2edd82fed2dd6254c33e68871ccf7a399498e059ec91dc4453a5808c3`)。拓扑与已发布 vLLM 一致:TP2/EP1 和 TP4/EP1,均覆盖 C1/2/4/8/16/32/64/128,不启用 DP attention。此前已暂存的 TP4/EP4 sweep 仅为历史证据,不能替代本次拓扑验证。 + +GB200 sweep 仅包含 `dsv41flash-fp4-gb200-sglang-agentic-dspark`。 +移除尚无实测依据的 STP 条目;若要加入,须通过匹配测试证明其对性能前沿有贡献。 +DSpark 使用固定官方 nightly 默认提供的精度,不应用自定义草稿量化或精度补丁。 +完整准确率和性能验证仍然必需。 + +GB200 TP4 保留 `min(64*CONC, 1024)` 个 SWA prefix tails,并维持 static memory 0.70 +和 chunk size 4096。此前 TP4/EP4 C16 实测保留 2,700 万个 full-context KV slots 与 439,040 个 SWA slots; +该上限避免高并发时耗尽实测 51.82 GiB KV 预算。仅 TP4 C16 使用 prefill/decode interval 16: +canonical 对比中 p90 interactivity 提升 13.65%,吞吐下降 0.30%,p90 TTFT 从 2.35 秒增至 +3.51 秒。完整 GSM8K 的 1,319 个样本通过验证。其他并发点仍需完整 sweep; +C16 结果不能证明该设置在所有并发下均有收益。 + +GB200 TP2 使用 0.92 静态显存比例、2048-token 预填充块、`min(128*CONC,1024)` 个 SWA tails、16 的 prefill/decode interval,graph 与 running capacity 上限为 16 个请求。这些受支持的限制参考已完成的 B200 EP1 显存验证;GB200 仍须独立通过加载、图捕获、完整上下文缓存池以及全部性能和准确率测试。可扩展 CUDA allocator segments 仅减少碎片,不改变权重或精度。C64/C128 性能任务获得分区允许的最大 12 小时 allocation,工作流额外预留 30 分钟打包产物;完整预热、3600 秒计分时段及无样本限制的 1,319 题 GSM8K 均保持不变。 + +GB200 的主机表布局为 `per_rank`:计算节点内核通过 `madvise` 启用匿名大页, +而 `shmem_enabled=never` 阻止共享 memfd 布局使用大页。上游在匿名主机内存中 +按行分片,并通过 `MADV_HUGEPAGE`/`MADV_COLLAPSE` 请求 512 MiB 大页。 +该方式保留原始 FP8 表权重,并恢复两次 TP all-reduce;声称性能收益前, +须检查启动日志中的实际驻留内存与大页覆盖量。 + DSpark 是检查点自带的草稿模型。SGLang 对它不提供 EAGLE 或 MTP 路径,也没有 `--speculative-num-steps` 参数;配方传入 `--speculative-algorithm DSPARK --speculative-dspark-block-size 5`。吞吐测试通过 `SGLANG_SIMULATE_ACC_LEN`(`match-expected`、 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 079e48e090..5655eb4067 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8802,3 +8802,16 @@ - "Keep radix caching enabled for AgentX and bound prefill chunks to the 4096-token bound after native FP4 prefill scratch exhausted HBM with the 16384-token default in five canonical cells. Reserve additional native scratch headroom with static memory fraction 0.70 after C32 still exhausted HBM at 0.80 with 4096-token chunks. Enable native allocator garbage collection at threshold 0.8 with process fraction 0.99 after physical HBM still filled during long prefill at static fraction 0.70. Cap server admission at 32 after native C32 warmup reached 63 running requests and failed with HIP illegal access; preserve all client fanout as queued work. Disable native prefill graphs after the admission-capped C32 test still failed with HIP illegal access during breakable prefill; keep decode graphs and treat this as a bounded runtime correction pending qualification. Cap the full KV pool at 3145728 tokens after the pinned native compressed-cache store and its compiled gfx950 code showed signed int32 byte offsets that overflow for large full-token locations. This supported capacity limit keeps ratio-1 storage below 2 GiB without patching kernels or reducing the model context limit; validate canonical C32 and full GSM before selection. Scheduler cadence remains at the official default. Preserve full 1M context, shipped-default DSpark precision, canonical thinking-on AL3.51 throughput and real full1319 GSM verification." - "Use image-native attention, dense FP8 and MoE kernels, allocator defaults and EP4 tuning data; remove the downstream nightly compatibility installer, source payload, tuning copy, hash checks and obsolete waiver documentation. The model-preview image explicitly supersedes the earlier latest-nightly selection for this lane." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3343 + +- config-keys: + - dsv41flash-fp4-gb200-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - Move GB200 DeepSeek-V4.1-Flash DSpark from dev-dsv41 to official CUDA 13 nightly-dev-cu13-20260923-06008c17, pinned at multi-architecture manifest sha256:5921361fcf358cdde4df1968c941c14157f418613b099ad7f3e5aeed6427ae15 (ARM64 sha256:d49261d2edd82fed2dd6254c33e68871ccf7a399498e059ec91dc4453a5808c3); match TP2/EP1 and TP4/EP1 with native DSpark precision and golden acceptance policy + - 'Use the upstream per_rank Engram host-table layout on GB200: anonymous row-sharded mappings can obtain 512 MiB huge pages on the measured madvise/never anonymous/shared THP policy; preserve original FP8 table weights and restore the two TP all-reduces' + - "Use the default MTP/DSpark precision shipped by the official nightly, without custom draft quantization or precision patches." + - Keep the sweep DSpark-only; remove the unmeasured STP entry and alias. + - Retain min(64*concurrency, 1024) SWA prefix tails within the measured GB200 KV budget; select prefill/decode interval 16 only at TP4 C16 after canonical performance and full GSM8K qualification. + - "Match the published vLLM grid exactly: TP2/EP1 and TP4/EP1, each C1/2/4/8/16/32/64/128. Use stock DSpark precision; TP2 uses static memory 0.92, chunk 2048, graph/running cap 16, interval 16, expandable allocator and min(128*CONC,1024) SWA tails. Requalify all 16 performance and full GSM8K points. Avoid deadline-only failures with the supported 12-hour partition maximum for C64/C128 performance while preserving complete warmup and 3600-second scoring." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3347 diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 60dd034fcd..0e5c773196 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -97,10 +97,19 @@ import_squash() { # Direct single-tray AgentX uses the existing shared image and HF caches. if [[ "$MODEL_PREFIX" == "dsv41flash" && ( "$FRAMEWORK" == "vllm" || "$FRAMEWORK" == "sglang" ) && "${IS_MULTINODE}" != "true" ]]; then - BENCH_SCRIPT="benchmarks/single_node/agentic/${MODEL_PREFIX}_${PRECISION}_gb200_${FRAMEWORK}_mtp.sh" + check_env_vars SPEC_DECODING + BENCH_SCRIPT="benchmarks/single_node/agentic/${MODEL_PREFIX}_${PRECISION}_gb200_${FRAMEWORK}" + case "$SPEC_DECODING" in + mtp) BENCH_SCRIPT+="_mtp.sh" ;; + none) + [[ "$FRAMEWORK" == "sglang" ]] || { echo "Native STP requires the SGLang recipe" >&2; exit 1; } + BENCH_SCRIPT+=".sh" + ;; + *) echo "Unsupported SPEC_DECODING=$SPEC_DECODING" >&2; exit 1 ;; + esac # Cover DSpark5 verification for concurrent AgentX subagents at c1/c2/c4. export DSV41_MIN_CUDAGRAPH_CAPTURE_SIZE=64 - [[ "${IS_AGENTIC}" == "1" && "${SPEC_DECODING:-}" == "mtp" && -f "$BENCH_SCRIPT" ]] || { + [[ "${IS_AGENTIC}" == "1" && -f "$BENCH_SCRIPT" ]] || { echo "Unsupported single-node recipe: $BENCH_SCRIPT" >&2 exit 1 } diff --git a/runners/runtime_settings.sh b/runners/runtime_settings.sh index 13713d3f8a..ab2ccf616f 100644 --- a/runners/runtime_settings.sh +++ b/runners/runtime_settings.sh @@ -27,6 +27,15 @@ case "${RUNNER_NAME%%_*}" in fi ;; gb300-nv) export SLURM_PARTITION=batch_1 ;; + gb200-nv) + if [[ "$MODEL_PREFIX" == dsv41flash && "$FRAMEWORK" == sglang && "$IS_MULTINODE" == false && "$IS_AGENTIC" == 1 ]]; then + check_env_vars CONC EVAL_ONLY + if [[ "$EVAL_ONLY" == false ]] && (( CONC >= 64 )); then + # Preserve the complete high-concurrency warmup and scored hour. + export SALLOC_TIME_LIMIT=720 + fi + fi + ;; h200-dgxc-slurm) if [[ "$MODEL_PREFIX" == dsv41flash && "$FRAMEWORK" == sglang && "$IS_MULTINODE" == false && "$IS_AGENTIC" == 1 ]]; then check_env_vars CONC EVAL_ONLY