From 9b75330773fc77d10f7fb94d90a26e1a72043a54 Mon Sep 17 00:00:00 2001 From: carlosmolina0615 Date: Fri, 3 Jul 2026 20:22:30 +0200 Subject: [PATCH 1/2] feat: add DeepSeek-V4 Flash DSpark node perf recipe + required mods - fix-dspark-dsv4-d2t: guard speculator.py:86 against DeepSeek-V4 (no draft_id_to_target_id attribute; full-vocab Markov drafting) - add-dsv4-topk256: sparse-MLA sm120 topk=256 decode instantiation (flashinfer#3817); REMOVE once upstream merged + released - deepseek-v4-flash-dspark-node-perf.yaml: perf pass 1 recipe with safe generic flags, memory-conservative for GB10 121GB unified --- mods/add-dsv4-topk256/run.sh | 71 +++++++++++++++++++ .../dspark_speculator_d2t.patch | 17 +++++ mods/fix-dspark-dsv4-d2t/run.sh | 27 +++++++ .../deepseek-v4-flash-dspark-node-perf.yaml | 63 ++++++++++++++++ 4 files changed, 178 insertions(+) create mode 100755 mods/add-dsv4-topk256/run.sh create mode 100644 mods/fix-dspark-dsv4-d2t/dspark_speculator_d2t.patch create mode 100755 mods/fix-dspark-dsv4-d2t/run.sh create mode 100644 recipes/deepseek-v4-flash-dspark-node-perf.yaml diff --git a/mods/add-dsv4-topk256/run.sh b/mods/add-dsv4-topk256/run.sh new file mode 100755 index 00000000..f5c2a73e --- /dev/null +++ b/mods/add-dsv4-topk256/run.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# VALIDATION mod: add the (num_heads=32, topk=256) sparse-MLA decode-dsv4 +# instantiation that the DeepSeek-V4 DSpark draft decode needs. +# +# The DSpark draft decodes with topk=256, which is absent from both the Python +# dispatch table (_DECODE_DSV4_DISPATCH: TOPK in {128,512,1024}) and the CUDA +# instantiation switch (DSV4_DISPATCH macros). The shape therefore falls through +# to the paged kernel, which hard-asserts num_tokens>64 and aborts. +# +# This patches BOTH sides so the JIT module rebuilds with the (32,256) kernel: +# - Python: add (32,256) to _DECODE_DSV4_DISPATCH so the dispatcher routes to +# the standalone decode kernel instead of the paged orchestrator. +# - CUDA: add DSV4_DISPATCH(32,256) so launch_decode_dsv4_impl +# is instantiated; ninja recompiles the object on next launch (mtime bump). +# +# Validation only. The upstream PR adds the full TOPK=256 column. REMOVE after. +set -e +D=/usr/local/lib/python3.12/dist-packages/flashinfer +PY=$D/mla/_sparse_mla_sm120.py +CU=$D/data/csrc/sparse_mla_sm120_decode_dsv4.cu +for f in "$PY" "$CU"; do + [ -f "$f" ] || { echo "--- [dsv4-topk256] missing $f; skipping."; exit 0; } +done +python3 - "$PY" "$CU" <<'PY' +import sys +py, cu = sys.argv[1], sys.argv[2] + +# 1) Python dispatch table: add (32, 256) to _DECODE_DSV4_DISPATCH (DSV4 block +# comes before DSV3_2 in the file, so replace(count=1) targets the DSV4 set). +s = open(py).read() +anchor = "_DECODE_DSV4_DISPATCH = frozenset(\n {\n" +if anchor not in s: + print("--- [dsv4-topk256] python anchor not found; ABORT (layout changed).") + sys.exit(1) +dsv4_region = s.split("_DECODE_DSV3_2_DISPATCH")[0] +if "(32, 256)" in dsv4_region: + print("--- [dsv4-topk256] python already has (32,256); skipping python.") +else: + s = s.replace(anchor, anchor + " (32, 256),\n", 1) + open(py, "w").write(s) + print("--- [dsv4-topk256] python patched: (32,256) added to dispatch table.") + +# 2) CUDA instantiation: add DSV4_DISPATCH(32, 256) before #undef DSV4_DISPATCH. +c = open(cu).read() +if "DSV4_DISPATCH(32, 256)" in c: + print("--- [dsv4-topk256] cuda already has DSV4_DISPATCH(32,256); skipping cuda.") +else: + marker = "#undef DSV4_DISPATCH" + if marker not in c: + print("--- [dsv4-topk256] cuda marker not found; ABORT (layout changed).") + sys.exit(1) + c = c.replace(marker, " DSV4_DISPATCH(32, 256)\n" + marker, 1) + open(cu, "w").write(c) + print("--- [dsv4-topk256] cuda patched: DSV4_DISPATCH(32,256) added.") +PY + +# 3) Force JIT rebuild: the image ships an AOT-prebuilt sparse_mla_sm120.so. +# JitSpec.build_and_load() short-circuits to that .so whenever it exists +# (is_aot == aot_path.exists()), so our data/csrc patch is otherwise ignored. +# Rename it so is_aot becomes False and the module JIT-compiles from source +# (picking up DSV4_DISPATCH(32,256)). Container-layer only; gone on recreate. +AOT=/usr/local/lib/python3.12/dist-packages/flashinfer_jit_cache/jit_cache/sparse_mla_sm120/sparse_mla_sm120.so +if [ -f "$AOT" ]; then + mv "$AOT" "$AOT.disabled-for-topk256" + echo "--- [dsv4-topk256] AOT prebuilt .so disabled; module will JIT-rebuild from patched source." +elif [ -f "$AOT.disabled-for-topk256" ]; then + echo "--- [dsv4-topk256] AOT prebuilt .so already disabled; skipping." +else + echo "--- [dsv4-topk256] AOT prebuilt .so not found (already JIT?); continuing." +fi +echo "=== OK" diff --git a/mods/fix-dspark-dsv4-d2t/dspark_speculator_d2t.patch b/mods/fix-dspark-dsv4-d2t/dspark_speculator_d2t.patch new file mode 100644 index 00000000..b45b4ea9 --- /dev/null +++ b/mods/fix-dspark-dsv4-d2t/dspark_speculator_d2t.patch @@ -0,0 +1,17 @@ +--- a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py ++++ b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py +@@ -83,8 +83,12 @@ + # Reduced draft vocab: probabilistic rejection sampling indexes draft + # logits by target id, so precompute the draft->target column map and a + # scratch buffer to scatter logits into target vocab before sampling. +- if self.draft_logits is not None and model.draft_id_to_target_id is not None: +- d2t = model.draft_id_to_target_id ++ # DSpark models without a reduced draft vocab (e.g. DeepSeek-V4) do not ++ # define draft_id_to_target_id; guard with getattr so the scatter path ++ # is simply skipped (identity / full-vocab drafting), as the None defaults intend. ++ _d2t = getattr(model, "draft_id_to_target_id", None) ++ if self.draft_logits is not None and _d2t is not None: ++ d2t = _d2t + self._d2t_scatter_index = ( + torch.arange(d2t.shape[0], device=d2t.device) + d2t + ) diff --git a/mods/fix-dspark-dsv4-d2t/run.sh b/mods/fix-dspark-dsv4-d2t/run.sh new file mode 100755 index 00000000..a7ebd287 --- /dev/null +++ b/mods/fix-dspark-dsv4-d2t/run.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Mod: guard DSpark speculator against models without draft_id_to_target_id. +# Fixes: AttributeError: 'DSparkDeepseekV4ForCausalLM' object has no attribute +# 'draft_id_to_target_id' (vllm/v1/worker/gpu/spec_decode/dspark/speculator.py:86) +# Upstream speculator assumes every DSpark model exposes draft_id_to_target_id +# (reduced-vocab drafting). DeepSeek-V4 DSpark uses full-vocab Markov drafting +# and does not define it. interfaces.py already uses getattr(..., None) for the +# same attribute; this applies the same guard to the speculator, so the optional +# reduced-vocab scatter path is simply skipped (the None defaults). +set -e + +VLLM_DIR=/usr/local/lib/python3.12/dist-packages +F="$VLLM_DIR/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py" + +if [ ! -f "$F" ]; then + echo "--- [fix-dspark-dsv4-d2t] speculator not found; this image has no DSpark speculator, skipping." + exit 0 +fi + +if grep -q '_d2t = getattr(model, "draft_id_to_target_id", None)' "$F"; then + echo "--- [fix-dspark-dsv4-d2t] patch already applied, skipping." + exit 0 +fi + +echo "--- [fix-dspark-dsv4-d2t] applying DSpark speculator draft_id_to_target_id guard..." +patch -p1 -d "$VLLM_DIR" < dspark_speculator_d2t.patch +echo "=== OK" diff --git a/recipes/deepseek-v4-flash-dspark-node-perf.yaml b/recipes/deepseek-v4-flash-dspark-node-perf.yaml new file mode 100644 index 00000000..504fa1c5 --- /dev/null +++ b/recipes/deepseek-v4-flash-dspark-node-perf.yaml @@ -0,0 +1,63 @@ +# Recipe: DeepSeek V4 Flash DSpark (node, perf pass 1 -- SAFE tanda) +# Base = deepseek-v4-flash-dspark-node.yaml (lean, working) + re-adds only the +# generic, low-risk vLLM perf flags the lean recipe dropped. NO RoCE/NCCL block +# and NO VLLM_DSPARK_*/B12X fork envs yet (those are pass 2 / pass 3). +# Levers added: FULL cudagraphs, async-scheduling, flashinfer autotune, explicit +# sparse-MLA DSV4 attention backend, chunked prefill, larger batches. +# Draft only -- validate flags exist in the vllm-node build before launching. +recipe_version: "1" +name: deepseek-v4-flash-dspark-node-perf +description: DeepSeek V4 Flash DSpark on dual DGX Spark TP=2, vllm-node image. Perf pass 1 (safe generic flags, memory-conservative for GB10 121GB unified). +model: deepseek-ai/DeepSeek-V4-Flash-DSpark +container: vllm-node +cluster_only: true +mods: + - mods/fix-dspark-dsv4-d2t # guards speculator.py:86 against DeepSeek-V4 (no draft_id_to_target_id) + - mods/add-dsv4-topk256 # sparse-MLA sm120 topk=256 decode instantiation (flashinfer#3817); REMOVE once merged + released +defaults: + port: 8000 + host: 0.0.0.0 + tensor_parallel: 2 + gpu_memory_utilization: 0.8 # KV rate is ~63k tok/GiB (MLA fp8, measured: 12.84GiB=810k tok at 0.80). 0.78 -> ~11GiB KV = ~660k token pool, enough for 500k context (1.3x concurrency), with more RAM buffer -> less swap. + max_model_len: 500000 # fits: KV pool is token-capacity ~660k at util 0.78 (>500k). max_model_len only caps per-request length; the KV pool memory is fixed by util, NOT by max_model_len. So 500k = same footprint as 256k. + block_size: 256 + max_num_seqs: 2 + max_num_batched_tokens: 8192 + num_speculative_tokens: 5 # design point: vendor dspark_block_size=5 (head trained for blocks of 5); 5 beat 4 on eff t/s; 6 predicted flat/negative (pos-6 accept ~0.5 for a full extra Markov step). +env: + DG_JIT_USE_NVRTC: "0" + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" # don't reserve FULL-cudagraph memory in the profiler (it ate ~99% of budget -> 0.61GiB KV). KV gets the budget; capture uses on-top headroom. Fork uses this. + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" # reduce unified-memory fragmentation -> a bit more usable for KV. Fork uses this. + VLLM_USE_FLASHINFER_SAMPLER: "1" # spec-decode hygiene (tonyd2wild garble-fix); takes effect on next relaunch. +command: | + vllm serve deepseek-ai/DeepSeek-V4-Flash-DSpark \ + --host {host} \ + --port {port} \ + --trust-remote-code \ + --tensor-parallel-size {tensor_parallel} \ + --kv-cache-dtype fp8 \ + --block-size {block_size} \ + --max-model-len {max_model_len} \ + --max-num-seqs {max_num_seqs} \ + --max-num-batched-tokens {max_num_batched_tokens} \ + --gpu-memory-utilization {gpu_memory_utilization} \ + --enable-prefix-caching \ + --enable-chunked-prefill \ + --async-scheduling \ + --enable-flashinfer-autotune \ + --attention-backend FLASHINFER_MLA_SPARSE_DSV4 \ + --compilation-config '{{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}}' \ + --max-cudagraph-capture-size {max_num_seqs} \ + --speculative-config '{{"method":"dspark","num_speculative_tokens":{num_speculative_tokens},"draft_sample_method":"probabilistic","rejection_sample_method":"block"}}' \ + --override-generation-config '{{"temperature":0.4}}' \ + --tokenizer-mode deepseek_v4 \ + --distributed-executor-backend ray \ + --tool-call-parser deepseek_v4 \ + --enable-auto-tool-choice \ + --reasoning-parser deepseek_v4 \ + --reasoning-config '{{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}}' \ + --default-chat-template-kwargs.thinking=true \ + --load-format instanttensor \ + --default-chat-template-kwargs.reasoning_effort=high From a2424eb1330abceaa512a5649636e5e4d13f2a95 Mon Sep 17 00:00:00 2001 From: carlosmolina0615 Date: Wed, 15 Jul 2026 11:33:36 +0200 Subject: [PATCH 2/2] fix: carry both FlashInfer TOPK=256 paths Co-Authored-By: Claude --- mods/add-dsv4-topk256-prefill/run.sh | 44 ++++++ mods/add-dsv4-topk256/run.sh | 132 ++++++++++-------- .../deepseek-v4-flash-dspark-node-perf.yaml | 4 +- 3 files changed, 123 insertions(+), 57 deletions(-) create mode 100755 mods/add-dsv4-topk256-prefill/run.sh diff --git a/mods/add-dsv4-topk256-prefill/run.sh b/mods/add-dsv4-topk256-prefill/run.sh new file mode 100755 index 00000000..2dc6015b --- /dev/null +++ b/mods/add-dsv4-topk256-prefill/run.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Carry FlashInfer #3834: add the DSV4 TOPK=256 prefill instantiation for SM12x. +# Keep paired with mods/add-dsv4-topk256 (FlashInfer #3817). +set -euo pipefail + +PREFILL_CU=/usr/local/lib/python3.12/dist-packages/flashinfer/data/csrc/sparse_mla_sm120_prefill.cu +[ -f "$PREFILL_CU" ] || { echo "--- [dsv4-topk256-prefill] missing $PREFILL_CU; ABORT."; exit 1; } + +python3 - "$PREFILL_CU" <<'PY' +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +text = path.read_text() +branch = " else if (topk == 256)\n DISPATCH_BY_NH_CM(BF16, 256);" +wrong_branch = " else if (topk == 256)\n DISPATCH_BY_NH_CM(FP8, 256);" +anchor = " else if (topk == 512)\n DISPATCH_BY_NH_CM(FP8, 512);" + +if wrong_branch in text: + print("--- [dsv4-topk256-prefill] found obsolete FP8 TOPK=256 carry; ABORT.") + raise SystemExit(1) +if branch in text: + print("--- [dsv4-topk256-prefill] BF16 TOPK=256 dispatch already present.") +elif anchor not in text: + print("--- [dsv4-topk256-prefill] dispatch anchor not found; FlashInfer layout changed; ABORT.") + raise SystemExit(1) +else: + path.write_text(text.replace(anchor, branch + "\n" + anchor, 1)) + print("--- [dsv4-topk256-prefill] added BF16 TOPK=256 dispatch.") +PY + +# The image's AOT module shadows patched package sources; force one JIT rebuild. +AOT=/usr/local/lib/python3.12/dist-packages/flashinfer_jit_cache/jit_cache/sparse_mla_sm120/sparse_mla_sm120.so +DISABLED=$AOT.disabled-for-topk256 +if [ -f "$AOT" ]; then + mv "$AOT" "$DISABLED" + echo "--- [dsv4-topk256-prefill] disabled stale AOT module." +elif [ -f "$DISABLED" ]; then + echo "--- [dsv4-topk256-prefill] stale AOT module already disabled." +else + echo "--- [dsv4-topk256-prefill] no AOT module found; JIT will use patched sources." +fi + +echo "=== OK" diff --git a/mods/add-dsv4-topk256/run.sh b/mods/add-dsv4-topk256/run.sh index f5c2a73e..e8e250f0 100755 --- a/mods/add-dsv4-topk256/run.sh +++ b/mods/add-dsv4-topk256/run.sh @@ -1,71 +1,91 @@ #!/bin/bash -# VALIDATION mod: add the (num_heads=32, topk=256) sparse-MLA decode-dsv4 -# instantiation that the DeepSeek-V4 DSpark draft decode needs. -# -# The DSpark draft decodes with topk=256, which is absent from both the Python -# dispatch table (_DECODE_DSV4_DISPATCH: TOPK in {128,512,1024}) and the CUDA -# instantiation switch (DSV4_DISPATCH macros). The shape therefore falls through -# to the paged kernel, which hard-asserts num_tokens>64 and aborts. -# -# This patches BOTH sides so the JIT module rebuilds with the (32,256) kernel: -# - Python: add (32,256) to _DECODE_DSV4_DISPATCH so the dispatcher routes to -# the standalone decode kernel instead of the paged orchestrator. -# - CUDA: add DSV4_DISPATCH(32,256) so launch_decode_dsv4_impl -# is instantiated; ninja recompiles the object on next launch (mtime bump). -# -# Validation only. The upstream PR adds the full TOPK=256 column. REMOVE after. -set -e +# Carry FlashInfer #3817: add DSV4 TOPK=256 decode instantiations for SM12x. +# Keep paired with mods/add-dsv4-topk256-prefill (FlashInfer #3834). +set -euo pipefail + D=/usr/local/lib/python3.12/dist-packages/flashinfer -PY=$D/mla/_sparse_mla_sm120.py -CU=$D/data/csrc/sparse_mla_sm120_decode_dsv4.cu -for f in "$PY" "$CU"; do - [ -f "$f" ] || { echo "--- [dsv4-topk256] missing $f; skipping."; exit 0; } +PY_FILE=$D/mla/_sparse_mla_sm120.py +DECODE_CU=$D/data/csrc/sparse_mla_sm120_decode_dsv4.cu + +for f in "$PY_FILE" "$DECODE_CU"; do + [ -f "$f" ] || { echo "--- [dsv4-topk256-decode] missing $f; ABORT."; exit 1; } done -python3 - "$PY" "$CU" <<'PY' + +python3 - "$PY_FILE" "$DECODE_CU" <<'PY' import sys -py, cu = sys.argv[1], sys.argv[2] +from pathlib import Path + +py_path, decode_path = map(Path, sys.argv[1:]) +head_sizes = (8, 16, 32, 64, 128) + + +def fail(message: str) -> None: + print(f"--- [dsv4-topk256-decode] {message}") + raise SystemExit(1) -# 1) Python dispatch table: add (32, 256) to _DECODE_DSV4_DISPATCH (DSV4 block -# comes before DSV3_2 in the file, so replace(count=1) targets the DSV4 set). -s = open(py).read() -anchor = "_DECODE_DSV4_DISPATCH = frozenset(\n {\n" -if anchor not in s: - print("--- [dsv4-topk256] python anchor not found; ABORT (layout changed).") - sys.exit(1) -dsv4_region = s.split("_DECODE_DSV3_2_DISPATCH")[0] -if "(32, 256)" in dsv4_region: - print("--- [dsv4-topk256] python already has (32,256); skipping python.") + +py_text = py_path.read_text() +dispatch_anchor = "_DECODE_DSV4_DISPATCH = frozenset(\n {\n" +if dispatch_anchor not in py_text: + fail("Python dispatch anchor not found; FlashInfer layout changed.") + +dsv4_region = py_text.split("_DECODE_DSV3_2_DISPATCH", 1)[0] +missing_python = [h for h in head_sizes if f"({h}, 256)" not in dsv4_region] +if missing_python: + entries = "".join(f" ({h}, 256),\n" for h in missing_python) + py_text = py_text.replace(dispatch_anchor, dispatch_anchor + entries, 1) + print("--- [dsv4-topk256-decode] added Python dispatch entries: " + ", ".join(map(str, missing_python))) +else: + print("--- [dsv4-topk256-decode] Python dispatch already patched.") + +guard_marker = "SM120 sparse-MLA has no decode kernel for this shape:" +if guard_marker not in py_text: + paged_call = " module.sparse_mla_sm120_paged_attention(\n" + if paged_call not in py_text: + fail("Paged-attention call anchor not found; FlashInfer layout changed.") + guard = ''' # Decode inputs must use a standalone decode instantiation. The paged + # orchestrator only supports prefill and otherwise aborts in C++. + if num_tokens <= _DECODE_MAX_TOKENS: + raise ValueError( + "SM120 sparse-MLA has no decode kernel for this shape: " + f"num_tokens={num_tokens}, num_heads={num_heads}, topk={topk}, " + f"d_qk={d_qk}, page_block_size={kv_pbs}, model_type={model_type}, " + f"extra_topk={extra_topk}. Supported decode shapes are enumerated in " + "_DECODE_DSV4_DISPATCH / _DECODE_DSV3_2_DISPATCH; add the matching " + "(num_heads, topk) instantiation to support it." + ) + +''' + py_text = py_text.replace(paged_call, guard + paged_call, 1) + print("--- [dsv4-topk256-decode] added unsupported-shape guard.") else: - s = s.replace(anchor, anchor + " (32, 256),\n", 1) - open(py, "w").write(s) - print("--- [dsv4-topk256] python patched: (32,256) added to dispatch table.") + print("--- [dsv4-topk256-decode] unsupported-shape guard already present.") +py_path.write_text(py_text) -# 2) CUDA instantiation: add DSV4_DISPATCH(32, 256) before #undef DSV4_DISPATCH. -c = open(cu).read() -if "DSV4_DISPATCH(32, 256)" in c: - print("--- [dsv4-topk256] cuda already has DSV4_DISPATCH(32,256); skipping cuda.") +decode_text = decode_path.read_text() +decode_marker = "#undef DSV4_DISPATCH" +if decode_marker not in decode_text: + fail("CUDA decode marker not found; FlashInfer layout changed.") +missing_cuda = [h for h in head_sizes if f"DSV4_DISPATCH({h}, 256)" not in decode_text] +if missing_cuda: + entries = "".join(f" DSV4_DISPATCH({h}, 256)\n" for h in missing_cuda) + decode_text = decode_text.replace(decode_marker, entries + decode_marker, 1) + decode_path.write_text(decode_text) + print("--- [dsv4-topk256-decode] added CUDA instantiations: " + ", ".join(map(str, missing_cuda))) else: - marker = "#undef DSV4_DISPATCH" - if marker not in c: - print("--- [dsv4-topk256] cuda marker not found; ABORT (layout changed).") - sys.exit(1) - c = c.replace(marker, " DSV4_DISPATCH(32, 256)\n" + marker, 1) - open(cu, "w").write(c) - print("--- [dsv4-topk256] cuda patched: DSV4_DISPATCH(32,256) added.") + print("--- [dsv4-topk256-decode] CUDA decode already patched.") PY -# 3) Force JIT rebuild: the image ships an AOT-prebuilt sparse_mla_sm120.so. -# JitSpec.build_and_load() short-circuits to that .so whenever it exists -# (is_aot == aot_path.exists()), so our data/csrc patch is otherwise ignored. -# Rename it so is_aot becomes False and the module JIT-compiles from source -# (picking up DSV4_DISPATCH(32,256)). Container-layer only; gone on recreate. +# The image's AOT module shadows patched package sources; force one JIT rebuild. AOT=/usr/local/lib/python3.12/dist-packages/flashinfer_jit_cache/jit_cache/sparse_mla_sm120/sparse_mla_sm120.so +DISABLED=$AOT.disabled-for-topk256 if [ -f "$AOT" ]; then - mv "$AOT" "$AOT.disabled-for-topk256" - echo "--- [dsv4-topk256] AOT prebuilt .so disabled; module will JIT-rebuild from patched source." -elif [ -f "$AOT.disabled-for-topk256" ]; then - echo "--- [dsv4-topk256] AOT prebuilt .so already disabled; skipping." + mv "$AOT" "$DISABLED" + echo "--- [dsv4-topk256-decode] disabled stale AOT module." +elif [ -f "$DISABLED" ]; then + echo "--- [dsv4-topk256-decode] stale AOT module already disabled." else - echo "--- [dsv4-topk256] AOT prebuilt .so not found (already JIT?); continuing." + echo "--- [dsv4-topk256-decode] no AOT module found; JIT will use patched sources." fi + echo "=== OK" diff --git a/recipes/deepseek-v4-flash-dspark-node-perf.yaml b/recipes/deepseek-v4-flash-dspark-node-perf.yaml index 504fa1c5..a8b540c2 100644 --- a/recipes/deepseek-v4-flash-dspark-node-perf.yaml +++ b/recipes/deepseek-v4-flash-dspark-node-perf.yaml @@ -13,7 +13,9 @@ container: vllm-node cluster_only: true mods: - mods/fix-dspark-dsv4-d2t # guards speculator.py:86 against DeepSeek-V4 (no draft_id_to_target_id) - - mods/add-dsv4-topk256 # sparse-MLA sm120 topk=256 decode instantiation (flashinfer#3817); REMOVE once merged + released + # SM12x DSpark needs both TOPK=256 halves. Remove this pair only after the FlashInfer version pinned by the image contains #3817 + #3834; #3896 was superseded by #3834. + - mods/add-dsv4-topk256 # FlashInfer #3817: sparse-MLA SM120 TOPK=256 decode instantiations + - mods/add-dsv4-topk256-prefill # FlashInfer #3834: sparse-MLA SM120 TOPK=256 BF16 prefill instantiation defaults: port: 8000 host: 0.0.0.0