diff --git a/docs_new/src/snippets/_deployment.jsx b/docs_new/src/snippets/_deployment.jsx index 32ad7606ca36..39a7f3795662 100644 --- a/docs_new/src/snippets/_deployment.jsx +++ b/docs_new/src/snippets/_deployment.jsx @@ -13,7 +13,15 @@ // (nodesOptions id is `single` or `multi-N` → --nnodes N) // cells {match, verified?, env, flags}[] — one per // (hw × variant × quant × strategy × nodes); env/flags are -// flat literals, only {{PLACEHOLDER}} subst applied +// flat literals, only {{PLACEHOLDER}} subst applied. +// PD-disaggregated cells carry `roles` instead of env/flags: +// [{title, notes?, env, flags} | {title, notes?, raw}] — +// one serve command per role (prefill / decode / router), +// rendered in a single blurb separated by `#` headings. +// Role flags include their own --nnodes/--node-rank/ +// --dist-init-addr literals (each worker group has its own +// head node), so the auto-injected multi-node trio and the +// generic multi-node header are skipped. // modelNames HF slug lookup, `hw|variant|quant` then `variant|quant` // placeholders {{KEY}} → {target: 'command'|'curl', label, default?} // curl cURL template (uses {{MODEL_NAME}} + placeholders) @@ -484,30 +492,17 @@ export const Deployment = ({ config, benchmarks }) => { const modelName = resolveModelName(sel); const nnodes = parseNnodes(sel.nodes); const multinode = nnodes > 1; - const cellEnv = cell.env || []; - const flags = [...(cell.flags || [])]; - if (multinode) { - // Insert the multi-node trio after the last parallelism flag, - // falling back to right after --model-path. - const PARALLELISM_ANCHORS = ["--enable-dp-attention", "--dp", "--tp"]; - let i = -1; - for (const anchor of PARALLELISM_ANCHORS) { - i = flags.findIndex((f) => f.split(/[\s=]/)[0] === anchor); - if (i !== -1) break; - } - if (i === -1) i = flags.findIndex((f) => f.startsWith("--model-path")); - flags.splice(i + 1, 0, - `--nnodes ${nnodes}`, - `--node-rank {{NODE_RANK}}`, - `--dist-init-addr {{NODE0_IP}}:20000`); - } - let cmd; - if (mode === "docker") { + const buildServe = (envList, flagList) => { + if (mode !== "docker") { + const flagBlock = flagList.map((f) => " " + f).join(" \\\n"); + const envBlock = envList.length ? envList.join(" \\\n") + " \\\n" : ""; + return `${envBlock}sglang serve \\\n${flagBlock}`; + } // Image keyed by `hw|quant` (most specific) then `hw`; `:dev` if unmapped. const di = config.dockerImages || {}; const image = di[`${sel.hw}|${sel.quant}`] || di[sel.hw] || "lmsysorg/sglang:dev"; - const portFlag = flags.find((x) => x.split(/[\s=]/)[0] === "--port"); + const portFlag = flagList.find((x) => x.split(/[\s=]/)[0] === "--port"); const servePort = portFlag ? portFlag.slice("--port".length).trim() : "{{PORT}}"; const vendorOf = (hwId) => { for (const [vendor, list] of Object.entries(HARDWARE_CATALOG)) { @@ -538,19 +533,49 @@ export const Deployment = ({ config, benchmarks }) => { // HF token only for gated checkpoints — configs that declare an HF_TOKEN placeholder. ...(config.placeholders && config.placeholders.HF_TOKEN ? [` --env "HF_TOKEN={{HF_TOKEN}}"`] : []), - ...cellEnv.map((e) => ` --env ${e}`), + ...envList.map((e) => ` --env ${e}`), " --ipc=host", ` ${image}`, " sglang serve", - ...flags.map((f) => " " + f), + ...flagList.map((f) => " " + f), ]; - cmd = dockerLines.join(" \\\n"); - } else { - const flagBlock = flags.map((f) => " " + f).join(" \\\n"); - const envBlock = cellEnv.length ? cellEnv.join(" \\\n") + " \\\n" : ""; - cmd = `${envBlock}sglang serve \\\n${flagBlock}`; + return dockerLines.join(" \\\n"); + }; + + if (cell.roles) { + // PD-disaggregated cell: one command block per role (prefill / decode / + // router), separated by `#` heading comments. Role flags carry their own + // --nnodes/--node-rank/--dist-init-addr literals (each worker group has + // its own head node), so the auto-injected trio and generic multi-node + // header don't apply. `raw` roles (e.g. the router) render verbatim. + const blocks = cell.roles.map((role) => { + const heading = [role.title, ...(role.notes || [])] + .map((line) => (line.length ? "# " + line : "#")).join("\n"); + const body = role.raw != null ? role.raw : buildServe(role.env || [], role.flags || []); + return `${heading}\n${body}`; + }); + return interpolate(blocks.join("\n\n"), envValues, modelName); } + const flags = [...(cell.flags || [])]; + if (multinode) { + // Insert the multi-node trio after the last parallelism flag, + // falling back to right after --model-path. + const PARALLELISM_ANCHORS = ["--enable-dp-attention", "--dp", "--tp"]; + let i = -1; + for (const anchor of PARALLELISM_ANCHORS) { + i = flags.findIndex((f) => f.split(/[\s=]/)[0] === anchor); + if (i !== -1) break; + } + if (i === -1) i = flags.findIndex((f) => f.startsWith("--model-path")); + flags.splice(i + 1, 0, + `--nnodes ${nnodes}`, + `--node-rank {{NODE_RANK}}`, + `--dist-init-addr {{NODE0_IP}}:20000`); + } + + let cmd = buildServe(cell.env || [], flags); + if (multinode && config.multiNodeHints && config.multiNodeHints[sel.hw]) { const hint = config.multiNodeHints[sel.hw] .map((line) => (line.length ? "# " + line : "#")).join("\n"); diff --git a/docs_new/src/snippets/_playground.jsx b/docs_new/src/snippets/_playground.jsx index 8d4fb4ea7732..de0e055d4dfc 100644 --- a/docs_new/src/snippets/_playground.jsx +++ b/docs_new/src/snippets/_playground.jsx @@ -1620,7 +1620,10 @@ export const Playground = ({ config }) => { // 10. Derived values // ========================================================================== const s = makeStyles(isDark); - const baseCell = findCell(config.cells, base); + // PD-disaggregated cells (`roles`) have no single flat flag list to overlay + // playground deltas on — treat them as no-base (playground-only view). + const cellAtSel = findCell(config.cells, base); + const baseCell = cellAtSel && cellAtSel.roles ? null : cellAtSel; const modelName = resolveModelName(base); // Per-axis state recovered from the base cell's flags (deriveFromBase). diff --git a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx index 431a473053dd..35333aa20d59 100644 --- a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx +++ b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx @@ -1,5 +1,6 @@ // Single `export const config` literal — no spreads/calls/IIFE (Mintlify re-evals at hydration). // Cells are denormalized: no `--nnodes`/`--node-rank`/`--dist-init-addr`/`--host`/`--port` literals — engine injects them. +// Exception: PD-disaggregated cells (`roles`) DO carry per-worker `--nnodes` literals — each worker group has its own head node. export const config = { modelName: "DeepSeek-V4", @@ -34,7 +35,9 @@ export const config = { { id: "balanced", label: "Balanced" }, { id: "high-throughput", label: "High-Throughput" }, ], - // `multi-N` id carries the node count for `--nnodes N`. + // `multi-N` id carries the node count for `--nnodes N`. PD-disaggregated + // cells (`roles`) also live under `multi-2`; their roles carry the + // per-worker `--nnodes` literals instead of the injected trio. nodesOptions: [ { id: "single", label: "Single Node" }, { id: "multi-2", label: "Multi-Nodes" }, @@ -991,6 +994,325 @@ sgl-eval run aime25 \\ "--port {{PORT}}", ], }, + + // ==== GB300 + FP4 multi-node: PD-disaggregated (MTP + HiCache KV offload). + // Adapted from SemiAnalysisAI/InferenceX#2157 (GB300 DeepSeek-V4 agentic + // Pareto recipes). All prefill workers are 2-node DEP8; low-latency was + // tuned at concurrency 8 (2P1D), balanced at 256 (4P4D), high-throughput + // at 1024 (12P4D). + { + match: { hw: "gb300", variant: "pro", quant: "fp4", strategy: "low-latency", nodes: "multi-2" }, + verified: true, + roles: [ + { + title: "Prefill worker — 1 worker × 2 nodes (8 GPUs, DEP8).", + notes: ["Run the same command on both prefill nodes ( 0/1)."], + env: [ + "NCCL_MNNVL_ENABLE=1", + "NCCL_CUMEM_ENABLE=1", + "SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True", + "MC_FORCE_MNNVL=1", + "SGLANG_DSV4_MHC_PREWARM=1", + "SGLANG_ENABLE_UNIFIED_RADIX_TREE=1", + "SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS=1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND=1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=9216", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--disaggregation-mode prefill", + "--disaggregation-transfer-backend mooncake", + "--tp 8", + "--dp 8", + "--ep-size 8", + "--enable-dp-attention", + "--enable-dp-lm-head", + "--nnodes 2", + "--node-rank ", + "--dist-init-addr :20000", + "--moe-dense-tp-size 1", + "--moe-a2a-backend megamoe", + "--load-balance-method total_tokens", + "--mem-fraction-static 0.9", + "--swa-full-tokens-ratio 0.02", + "--chunked-prefill-size 32768", + "--max-running-requests 128", + "--cuda-graph-max-bs-decode 128", + "--disable-flashinfer-autotune", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--enable-hierarchical-cache", + "--hicache-ratio 1", + "--hicache-write-policy write_back", + "--hicache-io-backend direct", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + title: "Decode worker — 1 node (4 GPUs, TP4).", + env: [ + "NCCL_MNNVL_ENABLE=1", + "NCCL_CUMEM_ENABLE=1", + "SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True", + "MC_FORCE_MNNVL=1", + "SGLANG_ENABLE_UNIFIED_RADIX_TREE=1", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--disaggregation-mode decode", + "--disaggregation-transfer-backend mooncake", + "--tp 4", + "--moe-runner-backend flashinfer_mxfp4", + "--mem-fraction-static 0.9", + "--swa-full-tokens-ratio 0.02", + "--max-running-requests 128", + "--cuda-graph-max-bs-decode 128", + "--disable-flashinfer-autotune", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + title: "Router — fronts the prefill and decode workers (run on any node).", + raw: +`python3 -m sglang_router.launch_router \\ + --pd-disaggregation \\ + --prefill http://:{{PORT}} \\ + --decode http://:{{PORT}} \\ + --host 0.0.0.0 --port 8000`, + }, + ], + }, + { + match: { hw: "gb300", variant: "pro", quant: "fp4", strategy: "balanced", nodes: "multi-2" }, + verified: true, + roles: [ + { + title: "Prefill workers — 2 workers × 2 nodes each (8 GPUs, DEP8).", + notes: ["Launch one instance per worker; run its command on both of the worker's nodes ( 0/1)."], + env: [ + "NCCL_MNNVL_ENABLE=1", + "NCCL_CUMEM_ENABLE=1", + "SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True", + "MC_FORCE_MNNVL=1", + "SGLANG_DSV4_MHC_PREWARM=1", + "SGLANG_ENABLE_UNIFIED_RADIX_TREE=1", + "SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS=1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND=1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=9216", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--disaggregation-mode prefill", + "--disaggregation-transfer-backend mooncake", + "--tp 8", + "--dp 8", + "--ep-size 8", + "--enable-dp-attention", + "--enable-dp-lm-head", + "--nnodes 2", + "--node-rank ", + "--dist-init-addr :20000", + "--moe-dense-tp-size 1", + "--moe-a2a-backend megamoe", + "--load-balance-method total_tokens", + "--mem-fraction-static 0.9", + "--swa-full-tokens-ratio 0.02", + "--chunked-prefill-size 32768", + "--max-running-requests 256", + "--cuda-graph-max-bs-decode 256", + "--disable-flashinfer-autotune", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--enable-hierarchical-cache", + "--hicache-ratio 1", + "--hicache-write-policy write_back", + "--hicache-io-backend direct", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + title: "Decode worker — 1 worker × 4 nodes (16 GPUs, DEP16).", + notes: ["Run the same command on all 4 decode nodes ( 0-3)."], + env: [ + "NCCL_MNNVL_ENABLE=1", + "NCCL_CUMEM_ENABLE=1", + "SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True", + "MC_FORCE_MNNVL=1", + "SGLANG_ENABLE_UNIFIED_RADIX_TREE=1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE=1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS=1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND=1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=4096", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--disaggregation-mode decode", + "--disaggregation-transfer-backend mooncake", + "--tp 16", + "--dp 16", + "--ep-size 16", + "--enable-dp-attention", + "--enable-dp-lm-head", + "--nnodes 4", + "--node-rank ", + "--dist-init-addr :20000", + "--moe-dense-tp-size 1", + "--moe-a2a-backend megamoe", + "--load-balance-method total_tokens", + "--mem-fraction-static 0.9", + "--swa-full-tokens-ratio 0.02", + "--max-running-requests 3072", + "--cuda-graph-max-bs-decode 256", + "--disable-flashinfer-autotune", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + title: "Router — fronts the prefill and decode workers (run on any node).", + raw: +`python3 -m sglang_router.launch_router \\ + --pd-disaggregation \\ + --prefill http://:{{PORT}} \\ + --prefill http://:{{PORT}} \\ + --decode http://:{{PORT}} \\ + --host 0.0.0.0 --port 8000`, + }, + ], + }, + { + match: { hw: "gb300", variant: "pro", quant: "fp4", strategy: "high-throughput", nodes: "multi-2" }, + verified: true, + roles: [ + { + title: "Prefill workers — 6 workers × 2 nodes each (8 GPUs, DEP8).", + notes: ["Launch one instance per worker; run its command on both of the worker's nodes ( 0/1)."], + env: [ + "NCCL_MNNVL_ENABLE=1", + "NCCL_CUMEM_ENABLE=1", + "SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True", + "MC_FORCE_MNNVL=1", + "SGLANG_DSV4_MHC_PREWARM=1", + "SGLANG_ENABLE_UNIFIED_RADIX_TREE=1", + "SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS=1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND=1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=9216", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--disaggregation-mode prefill", + "--disaggregation-transfer-backend mooncake", + "--tp 8", + "--dp 8", + "--ep-size 8", + "--enable-dp-attention", + "--enable-dp-lm-head", + "--nnodes 2", + "--node-rank ", + "--dist-init-addr :20000", + "--moe-dense-tp-size 1", + "--moe-a2a-backend megamoe", + "--load-balance-method total_tokens", + "--mem-fraction-static 0.9", + "--swa-full-tokens-ratio 0.02", + "--chunked-prefill-size 32768", + "--max-running-requests 1024", + "--cuda-graph-max-bs-decode 1024", + "--disable-flashinfer-autotune", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--enable-hierarchical-cache", + "--hicache-ratio 1", + "--hicache-write-policy write_back", + "--hicache-io-backend direct", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + title: "Decode worker — 1 worker × 4 nodes (16 GPUs, DEP16).", + notes: ["Run the same command on all 4 decode nodes ( 0-3)."], + env: [ + "NCCL_MNNVL_ENABLE=1", + "NCCL_CUMEM_ENABLE=1", + "SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True", + "MC_FORCE_MNNVL=1", + "SGLANG_ENABLE_UNIFIED_RADIX_TREE=1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE=1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS=1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND=1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=4096", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--disaggregation-mode decode", + "--disaggregation-transfer-backend mooncake", + "--tp 16", + "--dp 16", + "--ep-size 16", + "--enable-dp-attention", + "--enable-dp-lm-head", + "--nnodes 4", + "--node-rank ", + "--dist-init-addr :20000", + "--moe-dense-tp-size 1", + "--moe-a2a-backend megamoe", + "--load-balance-method total_tokens", + "--mem-fraction-static 0.9", + "--swa-full-tokens-ratio 0.02", + "--max-running-requests 3072", + "--cuda-graph-max-bs-decode 1024", + "--disable-flashinfer-autotune", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + title: "Router — fronts the prefill and decode workers (run on any node).", + raw: +`python3 -m sglang_router.launch_router \\ + --pd-disaggregation \\ + --prefill http://:{{PORT}} \\ + --prefill http://:{{PORT}} \\ + --prefill http://:{{PORT}} \\ + --prefill http://:{{PORT}} \\ + --prefill http://:{{PORT}} \\ + --prefill http://:{{PORT}} \\ + --decode http://:{{PORT}} \\ + --host 0.0.0.0 --port 8000`, + }, + ], + }, // ==================================================================== // GB200 + NVFP4 // ====================================================================