feat: add DeepSeek-V4 Flash DSpark node perf recipe + required mods

mods/add-dsv4-topk256-prefill/run.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# Carry FlashInfer #3834: add the DSV4 TOPK=256 prefill instantiation for SM12x.
+# Keep paired with mods/add-dsv4-topk256 (FlashInfer #3817).
+set -euo pipefail
+
+PREFILL_CU=/usr/local/lib/python3.12/dist-packages/flashinfer/data/csrc/sparse_mla_sm120_prefill.cu
+[ -f "$PREFILL_CU" ] || { echo "--- [dsv4-topk256-prefill] missing $PREFILL_CU; ABORT."; exit 1; }
+
+python3 - "$PREFILL_CU" <<'PY'
+import sys
+from pathlib import Path
+
+path = Path(sys.argv[1])
+text = path.read_text()
+branch = "  else if (topk == 256)\n    DISPATCH_BY_NH_CM(BF16, 256);"
+wrong_branch = "  else if (topk == 256)\n    DISPATCH_BY_NH_CM(FP8, 256);"
+anchor = "  else if (topk == 512)\n    DISPATCH_BY_NH_CM(FP8, 512);"
+
+if wrong_branch in text:
+    print("--- [dsv4-topk256-prefill] found obsolete FP8 TOPK=256 carry; ABORT.")
+    raise SystemExit(1)
+if branch in text:
+    print("--- [dsv4-topk256-prefill] BF16 TOPK=256 dispatch already present.")
+elif anchor not in text:
+    print("--- [dsv4-topk256-prefill] dispatch anchor not found; FlashInfer layout changed; ABORT.")
+    raise SystemExit(1)
+else:
+    path.write_text(text.replace(anchor, branch + "\n" + anchor, 1))
+    print("--- [dsv4-topk256-prefill] added BF16 TOPK=256 dispatch.")
+PY
+
+# The image's AOT module shadows patched package sources; force one JIT rebuild.
+AOT=/usr/local/lib/python3.12/dist-packages/flashinfer_jit_cache/jit_cache/sparse_mla_sm120/sparse_mla_sm120.so
+DISABLED=$AOT.disabled-for-topk256
+if [ -f "$AOT" ]; then
+  mv "$AOT" "$DISABLED"
+  echo "--- [dsv4-topk256-prefill] disabled stale AOT module."
+elif [ -f "$DISABLED" ]; then
+  echo "--- [dsv4-topk256-prefill] stale AOT module already disabled."
+else
+  echo "--- [dsv4-topk256-prefill] no AOT module found; JIT will use patched sources."
+fi
+
+echo "=== OK"

mods/add-dsv4-topk256/run.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+# Carry FlashInfer #3817: add DSV4 TOPK=256 decode instantiations for SM12x.
+# Keep paired with mods/add-dsv4-topk256-prefill (FlashInfer #3834).
+set -euo pipefail
+
+D=/usr/local/lib/python3.12/dist-packages/flashinfer
+PY_FILE=$D/mla/_sparse_mla_sm120.py
+DECODE_CU=$D/data/csrc/sparse_mla_sm120_decode_dsv4.cu
+
+for f in "$PY_FILE" "$DECODE_CU"; do
+  [ -f "$f" ] || { echo "--- [dsv4-topk256-decode] missing $f; ABORT."; exit 1; }
+done
+
+python3 - "$PY_FILE" "$DECODE_CU" <<'PY'
+import sys
+from pathlib import Path
+
+py_path, decode_path = map(Path, sys.argv[1:])
+head_sizes = (8, 16, 32, 64, 128)
+
+
+def fail(message: str) -> None:
+    print(f"--- [dsv4-topk256-decode] {message}")
+    raise SystemExit(1)
+
+
+py_text = py_path.read_text()
+dispatch_anchor = "_DECODE_DSV4_DISPATCH = frozenset(\n    {\n"
+if dispatch_anchor not in py_text:
+    fail("Python dispatch anchor not found; FlashInfer layout changed.")
+
+dsv4_region = py_text.split("_DECODE_DSV3_2_DISPATCH", 1)[0]
+missing_python = [h for h in head_sizes if f"({h}, 256)" not in dsv4_region]
+if missing_python:
+    entries = "".join(f"        ({h}, 256),\n" for h in missing_python)
+    py_text = py_text.replace(dispatch_anchor, dispatch_anchor + entries, 1)
+    print("--- [dsv4-topk256-decode] added Python dispatch entries: " + ", ".join(map(str, missing_python)))
+else:
+    print("--- [dsv4-topk256-decode] Python dispatch already patched.")
+
+guard_marker = "SM120 sparse-MLA has no decode kernel for this shape:"
+if guard_marker not in py_text:
+    paged_call = "        module.sparse_mla_sm120_paged_attention(\n"
+    if paged_call not in py_text:
+        fail("Paged-attention call anchor not found; FlashInfer layout changed.")
+    guard = '''        # Decode inputs must use a standalone decode instantiation. The paged
+        # orchestrator only supports prefill and otherwise aborts in C++.
+        if num_tokens <= _DECODE_MAX_TOKENS:
+            raise ValueError(
+                "SM120 sparse-MLA has no decode kernel for this shape: "
+                f"num_tokens={num_tokens}, num_heads={num_heads}, topk={topk}, "
+                f"d_qk={d_qk}, page_block_size={kv_pbs}, model_type={model_type}, "
+                f"extra_topk={extra_topk}. Supported decode shapes are enumerated in "
+                "_DECODE_DSV4_DISPATCH / _DECODE_DSV3_2_DISPATCH; add the matching "
+                "(num_heads, topk) instantiation to support it."
+            )
+
+'''
+    py_text = py_text.replace(paged_call, guard + paged_call, 1)
+    print("--- [dsv4-topk256-decode] added unsupported-shape guard.")
+else:
+    print("--- [dsv4-topk256-decode] unsupported-shape guard already present.")
+py_path.write_text(py_text)
+
+decode_text = decode_path.read_text()
+decode_marker = "#undef DSV4_DISPATCH"
+if decode_marker not in decode_text:
+    fail("CUDA decode marker not found; FlashInfer layout changed.")
+missing_cuda = [h for h in head_sizes if f"DSV4_DISPATCH({h}, 256)" not in decode_text]
+if missing_cuda:
+    entries = "".join(f"  DSV4_DISPATCH({h}, 256)\n" for h in missing_cuda)
+    decode_text = decode_text.replace(decode_marker, entries + decode_marker, 1)
+    decode_path.write_text(decode_text)
+    print("--- [dsv4-topk256-decode] added CUDA instantiations: " + ", ".join(map(str, missing_cuda)))
+else:
+    print("--- [dsv4-topk256-decode] CUDA decode already patched.")
+PY
+
+# The image's AOT module shadows patched package sources; force one JIT rebuild.
+AOT=/usr/local/lib/python3.12/dist-packages/flashinfer_jit_cache/jit_cache/sparse_mla_sm120/sparse_mla_sm120.so
+DISABLED=$AOT.disabled-for-topk256
+if [ -f "$AOT" ]; then
+  mv "$AOT" "$DISABLED"
+  echo "--- [dsv4-topk256-decode] disabled stale AOT module."
+elif [ -f "$DISABLED" ]; then
+  echo "--- [dsv4-topk256-decode] stale AOT module already disabled."
+else
+  echo "--- [dsv4-topk256-decode] no AOT module found; JIT will use patched sources."
+fi
+
+echo "=== OK"

mods/fix-dspark-dsv4-d2t/dspark_speculator_d2t.patch

@@ -0,0 +1,17 @@
+--- a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py
++++ b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py
+@@ -83,8 +83,12 @@
+         # Reduced draft vocab: probabilistic rejection sampling indexes draft
+         # logits by target id, so precompute the draft->target column map and a
+         # scratch buffer to scatter logits into target vocab before sampling.
+-        if self.draft_logits is not None and model.draft_id_to_target_id is not None:
+-            d2t = model.draft_id_to_target_id
++        # DSpark models without a reduced draft vocab (e.g. DeepSeek-V4) do not
++        # define draft_id_to_target_id; guard with getattr so the scatter path
++        # is simply skipped (identity / full-vocab drafting), as the None defaults intend.
++        _d2t = getattr(model, "draft_id_to_target_id", None)
++        if self.draft_logits is not None and _d2t is not None:
++            d2t = _d2t
+             self._d2t_scatter_index = (
+                 torch.arange(d2t.shape[0], device=d2t.device) + d2t
+             )

mods/fix-dspark-dsv4-d2t/run.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Mod: guard DSpark speculator against models without draft_id_to_target_id.
+# Fixes: AttributeError: 'DSparkDeepseekV4ForCausalLM' object has no attribute
+#        'draft_id_to_target_id'  (vllm/v1/worker/gpu/spec_decode/dspark/speculator.py:86)
+# Upstream speculator assumes every DSpark model exposes draft_id_to_target_id
+# (reduced-vocab drafting). DeepSeek-V4 DSpark uses full-vocab Markov drafting
+# and does not define it. interfaces.py already uses getattr(..., None) for the
+# same attribute; this applies the same guard to the speculator, so the optional
+# reduced-vocab scatter path is simply skipped (the None defaults).
+set -e
+
+VLLM_DIR=/usr/local/lib/python3.12/dist-packages
+F="$VLLM_DIR/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py"
+
+if [ ! -f "$F" ]; then
+  echo "--- [fix-dspark-dsv4-d2t] speculator not found; this image has no DSpark speculator, skipping."
+  exit 0
+fi
+
+if grep -q '_d2t = getattr(model, "draft_id_to_target_id", None)' "$F"; then
+  echo "--- [fix-dspark-dsv4-d2t] patch already applied, skipping."
+  exit 0
+fi
+
+echo "--- [fix-dspark-dsv4-d2t] applying DSpark speculator draft_id_to_target_id guard..."
+patch -p1 -d "$VLLM_DIR" < dspark_speculator_d2t.patch
+echo "=== OK"

recipes/deepseek-v4-flash-dspark-node-perf.yaml

@@ -0,0 +1,65 @@
+# Recipe: DeepSeek V4 Flash DSpark (node, perf pass 1 -- SAFE tanda)
+# Base = deepseek-v4-flash-dspark-node.yaml (lean, working) + re-adds only the
+# generic, low-risk vLLM perf flags the lean recipe dropped. NO RoCE/NCCL block
+# and NO VLLM_DSPARK_*/B12X fork envs yet (those are pass 2 / pass 3).
+# Levers added: FULL cudagraphs, async-scheduling, flashinfer autotune, explicit
+# sparse-MLA DSV4 attention backend, chunked prefill, larger batches.
+# Draft only -- validate flags exist in the vllm-node build before launching.
+recipe_version: "1"
+name: deepseek-v4-flash-dspark-node-perf
+description: DeepSeek V4 Flash DSpark on dual DGX Spark TP=2, vllm-node image. Perf pass 1 (safe generic flags, memory-conservative for GB10 121GB unified).
+model: deepseek-ai/DeepSeek-V4-Flash-DSpark
+container: vllm-node
+cluster_only: true
+mods:
+  - mods/fix-dspark-dsv4-d2t   # guards speculator.py:86 against DeepSeek-V4 (no draft_id_to_target_id)
+  # SM12x DSpark needs both TOPK=256 halves. Remove this pair only after the FlashInfer version pinned by the image contains #3817 + #3834; #3896 was superseded by #3834.
+  - mods/add-dsv4-topk256          # FlashInfer #3817: sparse-MLA SM120 TOPK=256 decode instantiations
+  - mods/add-dsv4-topk256-prefill  # FlashInfer #3834: sparse-MLA SM120 TOPK=256 BF16 prefill instantiation
+defaults:
+  port: 8000
+  host: 0.0.0.0
+  tensor_parallel: 2
+  gpu_memory_utilization: 0.8   # KV rate is ~63k tok/GiB (MLA fp8, measured: 12.84GiB=810k tok at 0.80). 0.78 -> ~11GiB KV = ~660k token pool, enough for 500k context (1.3x concurrency), with more RAM buffer -> less swap.
+  max_model_len: 500000   # fits: KV pool is token-capacity ~660k at util 0.78 (>500k). max_model_len only caps per-request length; the KV pool memory is fixed by util, NOT by max_model_len. So 500k = same footprint as 256k.
+  block_size: 256
+  max_num_seqs: 2
+  max_num_batched_tokens: 8192
+  num_speculative_tokens: 5   # design point: vendor dspark_block_size=5 (head trained for blocks of 5); 5 beat 4 on eff t/s; 6 predicted flat/negative (pos-6 accept ~0.5 for a full extra Markov step).
+env:
+  DG_JIT_USE_NVRTC: "0"
+  VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+  VLLM_USE_BREAKABLE_CUDAGRAPH: "0"
+  VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0"   # don't reserve FULL-cudagraph memory in the profiler (it ate ~99% of budget -> 0.61GiB KV). KV gets the budget; capture uses on-top headroom. Fork uses this.
+  PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True"   # reduce unified-memory fragmentation -> a bit more usable for KV. Fork uses this.
+  VLLM_USE_FLASHINFER_SAMPLER: "1"   # spec-decode hygiene (tonyd2wild garble-fix); takes effect on next relaunch.
+command: |
+  vllm serve deepseek-ai/DeepSeek-V4-Flash-DSpark \
+      --host {host} \
+      --port {port} \
+      --trust-remote-code \
+      --tensor-parallel-size {tensor_parallel} \
+      --kv-cache-dtype fp8 \
+      --block-size {block_size} \
+      --max-model-len {max_model_len} \
+      --max-num-seqs {max_num_seqs} \
+      --max-num-batched-tokens {max_num_batched_tokens} \
+      --gpu-memory-utilization {gpu_memory_utilization} \
+      --enable-prefix-caching \
+      --enable-chunked-prefill \
+      --async-scheduling \
+      --enable-flashinfer-autotune \
+      --attention-backend FLASHINFER_MLA_SPARSE_DSV4 \
+      --compilation-config '{{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}}' \
+      --max-cudagraph-capture-size {max_num_seqs} \
+      --speculative-config '{{"method":"dspark","num_speculative_tokens":{num_speculative_tokens},"draft_sample_method":"probabilistic","rejection_sample_method":"block"}}' \
+      --override-generation-config '{{"temperature":0.4}}' \
+      --tokenizer-mode deepseek_v4 \
+      --distributed-executor-backend ray \
+      --tool-call-parser deepseek_v4 \
+      --enable-auto-tool-choice \
+      --reasoning-parser deepseek_v4 \
+      --reasoning-config '{{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}}' \
+      --default-chat-template-kwargs.thinking=true \
+      --load-format instanttensor \
+      --default-chat-template-kwargs.reasoning_effort=high