From 0b6b34906d6c0161bfd9b1f6cb0c7520a9b4abc3 Mon Sep 17 00:00:00 2001 From: raviguptaamd Date: Wed, 8 Jul 2026 06:42:53 +0000 Subject: [PATCH 01/23] vllm_dissag: GLM-5.1-FP8 (MLA+DSA) MoRI-EP WideEP disaggregated enablement Adds GLM-5.1-FP8 (GlmMoeDsaForCausalLM = MLA + DeepSeek Sparse Attention) to the MoRI-EP WideEP disaggregated serving path, stacked on the #171 unified launcher. Fully isolated from DeepSeek-V3/R1: GLM gets its own image + a MODEL_NAME-gated runtime path, so existing models are byte-identical to develop. Defects fixed (validated 1P/1D EP8 + 2P/2D EP16, NIAH 2k-35k = 10/10, no crash): - Long-context accuracy collapse: vLLM #47766 cache-key fix keeps the persistent sparse-MLA kernel ON (keys metadata on per-request context+query len). - 8k disagg prefill crash: DSA adds a 2nd (indexer) KV cache per layer that the single-geometry MoRIIO connector never transferred; paired + shipped prefill-> decode. Plus DSA invalid-token kernel fix (#45324) and shik-latest DP-notify. Changes: - docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile: NEW per-model image (raviguptaamd/vllm glm5.1-dsa-wideEP_on_shik_latest + aiter e03fa6040 + mori 42e895472b08 + router). The base vllm_disagg_inference Dockerfile (DSV3/R1) is left untouched. Future models add their own Dockerfile the same way. - models.json: card pyt_vllm_disagg_mori_glm-5.1-fp8 (GLM_SKIP_PATCHERS=1: image carries the DSA fixes in-source). - models.yaml: GLM-5.1-FP8 recipe (block=1, AITER MLA on, eager, mori backends). DeepSeek-V3 dp: caps (--max-num-seqs 64 --max-model-len 32768) to bound the newer base's decode logits workspace (isolated to the DSV3 entry). - connectors/moriio.sh: MODEL_NAME-gated GLM DSA runtime patchers (pure no-op for other models); GLM_SKIP_PATCHERS switch for baked-fix images. - 9 idempotent, anchor-based, self-skipping GLM DSA patcher scripts. KNOWN OPEN DEFECT (future work): 4P/4D EP32 emits corrupted tokens at all context lengths (suspect moriep all-to-all combine at scale); use 1P/1D and 2P/2D. Co-Authored-By: Claude --- ...gg_inference.glmv5.1.ubuntu.amd.Dockerfile | 335 ++++++++++++++++++ .../apply_glm_aiter_sampling_oob_fix.py | 148 ++++++++ .../apply_glm_dsa_indexer_warmup_fix.py | 251 +++++++++++++ .../vllm_dissag/apply_glm_dsa_kernel_fix.py | 85 +++++ .../apply_glm_dsa_moriio_dualkv_fix.py | 176 +++++++++ .../apply_glm_dsa_moriio_engine_fix.py | 116 ++++++ .../apply_glm_dsa_moriio_gate_fix.py | 133 +++++++ .../apply_glm_dsa_moriio_instrument.py | 92 +++++ ...pply_glm_dsa_persistent_kernel_gate_fix.py | 129 +++++++ .../apply_glm_moriio_abort_guard_fix.py | 98 +++++ scripts/vllm_dissag/connectors/moriio.sh | 100 +++++- scripts/vllm_dissag/keepalive_bench.sh | 18 + scripts/vllm_dissag/models.yaml | 102 +++++- scripts/vllm_dissag/run_xPyD_models.slurm | 32 +- scripts/vllm_dissag/vllm_disagg.sh | 28 +- 15 files changed, 1825 insertions(+), 18 deletions(-) create mode 100644 docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile create mode 100644 scripts/vllm_dissag/apply_glm_aiter_sampling_oob_fix.py create mode 100755 scripts/vllm_dissag/apply_glm_dsa_indexer_warmup_fix.py create mode 100755 scripts/vllm_dissag/apply_glm_dsa_kernel_fix.py create mode 100755 scripts/vllm_dissag/apply_glm_dsa_moriio_dualkv_fix.py create mode 100755 scripts/vllm_dissag/apply_glm_dsa_moriio_engine_fix.py create mode 100755 scripts/vllm_dissag/apply_glm_dsa_moriio_gate_fix.py create mode 100755 scripts/vllm_dissag/apply_glm_dsa_moriio_instrument.py create mode 100644 scripts/vllm_dissag/apply_glm_dsa_persistent_kernel_gate_fix.py create mode 100644 scripts/vllm_dissag/apply_glm_moriio_abort_guard_fix.py create mode 100755 scripts/vllm_dissag/keepalive_bench.sh diff --git a/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile b/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile new file mode 100644 index 00000000..f61a8819 --- /dev/null +++ b/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile @@ -0,0 +1,335 @@ +# CONTEXT {'gpu_vendor': 'AMD', 'guest_os': 'UBUNTU'} +############################################################################### +# +# MIT License +# +# Copyright (c) 2025 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################# +# ============================================================================= +# vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile +# GLM-5.1-FP8 (MLA + DeepSeek Sparse Attention) MoRI-EP WideEP disagg image. +# PER-MODEL image, isolated from the base vllm_disagg_inference Dockerfile +# (which stays pinned to the DeepSeek-V3 / R1 stack). This split lets each model +# pin its own vLLM/AITER/MoRI without disturbing the others -- add a new +# vllm_disagg_inference..ubuntu.amd.Dockerfile per future model +# (e.g. Kimi-2.6) rather than repinning the shared DSV3 image. +# +# ALL connectors in one image: moriio (TP + MoRI-EP wideEP) + rixl (NIXL TP + +# DeepEP wideEP). = the fullsource MoRI stack, plus a UCX/RIXL/rocSHMEM/DeepEP +# transport layer gated by --build-arg WITH_NIXL (default 1 = everything). +# +# docker build -f docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile \ +# -t /vllm-disagg:glmv5.1 . +# export DOCKER_IMAGE_NAME=/vllm-disagg:glmv5.1 +# +# WITH_NIXL=1 (default) => builds UCX + RIXL(+nixlbench) + rocSHMEM + DeepEP from +# source, so all four connector combos (moriio TP/wideEP, rixl NIXL TP, DeepEP +# wideEP) are present (~+30-45 min build vs WITH_NIXL=0). +# WITH_NIXL=0 => MoRI-EP only (moriio TP/wideEP + deepep-from-base); lean, faster. +# +# STATUS (GLM-5.1-FP8 on this stack): 1P/1D EP8 + 2P/2D EP16 NIAH 2k-35k = 10/10, +# no crash; long-context accuracy fixed via vLLM #47766 (persistent sparse-MLA kept +# ON). 4P/4D EP32 is a KNOWN OPEN DEFECT: token corruption at ALL context lengths +# (garbage output even at 2k), distinct from the long-context bug; prime suspect is +# the moriep all-to-all combine at EP32 scale -> deferred to future work. Use 1P/1D +# and 2P/2D only. (BASE_IMAGE is a gated nightly; override --build-arg BASE_IMAGE=...) +# ============================================================================= +# Reconstructs the validated v1.2.1 (mori121) runtime stack by applying the recipe's +# component pins ON TOP of the open ROCm vLLM ci_base, cloning each source from +# public Git (no local build-contexts). Mirrors dist-inf-cookbook +# Dockerfile.vllm.mori121_shareable: +# +# - BASE: rocm/vllm-dev:ci_base-0fcd9b99... (open ROCm 7.2 / cp312 CI base). +# - MoRI -> built from ROCm/MoRI @ v1.2.1 (BUILD_UMBP=OFF). +# - AITER -> STOCK ROCm/aiter @ e03fa6040 compiled from source + flydsl 0.1.7-0.1.9; +# stale JIT wiped. (#47766 keeps persistent MLA ON -> aiter native gqa64 fold.) +# - vLLM -> COMPILED from shikamd123/vllm @ +# vllm_2p2d_wide-ep_write_shikpate_test_06_29_customer (Wide-EP multi-pod PD, the +# connector/router reference for the 2P2D DP=EP=16 topology). Full compile: it is +# a different commit than the base's, so a .py-only overlay would be ABI-mismatched. +# - RDMA fix (expandable_segments:False x2 + HSA_ENABLE_IPC_MODE_LEGACY=0) is NOT baked +# here — it lives in scripts/vllm_dissag/connectors/.env and the launcher +# forwards it via docker -e. ROCm 7.2.3 cannot dmabuf-export VMM memory, else MoRI +# RegisterRdmaMemoryRegion EFAULTs (errno 14) on the first disagg WRITE. +# - vllm-router (vllm-project/router PR#181 = DP-rank round-robin + 2P2D KV-notify +# dpfix) built in -> no external router binary needed. +# - validated recipe knobs baked as ENV. The MoRIIO disagg fixes (#39276 notify, +# #41751 LL split, DP-rank hash-failsafe) are native in this vLLM (no runtime patcher). +# +# Build context = repo root: +# docker build -f docker/vllm_disagg_inference.ubuntu.amd.Dockerfile -t / . +# +# BASE_IMAGE is the open rocm/vllm-dev ci_base pinned by the validated recipe +# (dist-inf-cookbook Dockerfile.vllm.mori121_shareable). Override --build-arg +# BASE_IMAGE=... to build on a different ROCm base. vLLM compile is long (~30-60 min). +# ============================================================================= + +ARG BASE_IMAGE=rocm/vllm-dev:ci_base-0fcd9b99cc9d63202da4c858d8ebc6582c9e2491 +FROM ${BASE_IMAGE} + +ENTRYPOINT [] +WORKDIR /app + +ARG GFX_COMPILATION_ARCH="gfx942" +ARG PYTORCH_ROCM_ARCH="gfx942" +ARG MAX_JOBS=32 +# NIXL/RIXL transport for the rixl connector. Default 1 => all connectors built +# (UCX/RIXL/rocSHMEM/DeepEP). Set --build-arg WITH_NIXL=0 for a lean MoRI-EP-only image. +ARG WITH_NIXL=1 +ARG NIC_COMPILATION_ARCH="cx7" + +# ----------------------------------------------------------------------------- +# 1. MoRI: replace the base's bundled MoRI with the validated ROCm/MoRI @ v1.2.1 +# (the version for the 06_29 mori121 image, dist-inf-cookbook +# Dockerfile.vllm.mori121_shareable). v1.2.1 carries the EP/RDMA correctness fixes +# plus the ROCm-7.2.3 dmabuf registration path used by the connector .env +# (expandable_segments:False). MoRI is JIT-built, so this swaps the JIT sources the +# kernels compile from at runtime. +# BUILD CONFIG: match the cookbook build — MORI_GPU_ARCHS=gfx942, BUILD_UMBP=OFF, +# DEFAULT NIC backends. Do NOT pass USE_IONIC=OFF / USE_BNXT=OFF: disabling NIC +# backends produced a MoRI that deadlocked at the cross-node EP all-to-all init. +# ----------------------------------------------------------------------------- +ARG MORI_REPO=https://github.com/ROCm/mori.git +# 42e895472b08: MoRI main tip past v1.2.1, validated by MAD-private #338 for GLM-5.1 +# DSA WideEP disagg (v1.2.1 large-transfer notify path was insufficient at high EP). +ARG MORI_REF=42e895472b08 +ENV MORI_GPU_ARCHS=gfx942 +# Newer MoRI added the UMBP subsystem which requires gRPC (grpcpp/grpcpp.h) not +# present in this base; UMBP is unrelated to the EP dispatch/combine kernels, so +# disable it to avoid pulling in a gRPC build dependency. +ENV BUILD_UMBP=OFF BUILD_UMBP_SPDK=OFF +# Build/install matches dist-inf-cookbook Dockerfile.vllm.mori121_shareable for v1.2.1: +# `BUILD_UMBP=OFF pip install .` (default build isolation). apt/pip build tooling kept +# for bases that lack it; harmless where already present. +RUN sed -i 's|http://|https://|g' /etc/apt/sources.list 2>/dev/null || true && \ + sed -i 's|http://|https://|g' /etc/apt/sources.list.d/*.list 2>/dev/null || true && \ + apt-get update && apt-get install -y --no-install-recommends \ + git build-essential cmake ninja-build ccache libssl-dev pkg-config curl ca-certificates && \ + pip install meson==0.64.0 "pybind11[global]" tqdm prettytable && \ + pip uninstall -y amd_mori amd-mori amd-mori-nightly mori 2>/dev/null || true && \ + rm -rf /tmp/mori-src && \ + git clone --recursive "${MORI_REPO}" /tmp/mori-src && \ + cd /tmp/mori-src && git checkout "${MORI_REF}" && git submodule update --init --recursive && \ + BUILD_UMBP=OFF pip install . && \ + python3 -c "import mori, mori.io, mori.ops; print('MoRI OK at', mori.__path__[0])" && \ + mkdir -p /app && echo "MORI_REF=${MORI_REF}@$(git -C /tmp/mori-src rev-parse HEAD)" >> /app/versions.txt && \ + rm -rf /tmp/mori-src + +# ----------------------------------------------------------------------------- +# 2. AITER: build STOCK upstream ROCm/aiter @ e03fa6040 from source (NO fork, +# NO gqa64-fold patch). Under vLLM #47766 the sparse-MLA persistent path stays +# ON, so GLM's gqa=64 decode hits aiter's PRE-EXISTING persistent gqa64->16 fold +# (aiter/mla.py: `nhead in range(32,128+1,16) and persistent_mode`); the fork's +# extra non-persistent fold is never exercised, so stock is sufficient. +# Validated by MAD-private #338: 1P/1D EP8 + 2P/2D EP16 NIAH PASS on this exact +# aiter tip under #47766. Pin the exact commit (the one tested), not the release +# wheel. Then invalidate the stale prewarmed JIT cache compiled against the old .so. +# ----------------------------------------------------------------------------- +ARG AITER_REPO=https://github.com/ROCm/aiter.git +ARG AITER_REF=e03fa6040 +RUN echo "Compiling STOCK AITER (no fork) from ${AITER_REPO}@${AITER_REF}" && \ + rm -rf /tmp/aiter-src && \ + git clone --recursive "${AITER_REPO}" /tmp/aiter-src && \ + cd /tmp/aiter-src && git checkout "${AITER_REF}" && \ + git submodule update --init --recursive && \ + (pip uninstall -y amd_aiter amd-aiter aiter 2>/dev/null || true) && \ + pip install --no-build-isolation --no-deps -v . && \ + pip install --no-deps -U "flydsl>=0.1.7,<0.1.9" && \ + echo "AITER_REF=${AITER_REF}@$(git rev-parse HEAD) (stock ROCm/aiter, no fork)" >> /app/versions.txt && \ + rm -rf /tmp/aiter-src && \ + python3 - <<'PYEOF' +# Verify aiter/mla.py installed + has the persistent gqa64 fold, WITHOUT importing +# aiter/torch (torch->amdsmi->libamd_smi.so is not loadable at build: no GPU in sandbox). +import glob, pathlib +cands = glob.glob("/usr/local/lib/python*/dist-packages/aiter/mla.py") + \ + glob.glob("/usr/lib/python*/dist-packages/aiter/mla.py") +assert cands, "aiter/mla.py not found in site-packages after install" +src = pathlib.Path(cands[0]).read_text() +assert "persistent_mode" in src, f"AITER persistent fold path MISSING in {cands[0]}" +print("STOCK AITER OK (persistent gqa64 fold path present):", cands[0]) +PYEOF +RUN rm -rf /opt/vllm_cache/aiter_jit /root/.aiter && echo "cleared stale AITER JIT cache" && \ + echo "AITER_REF=${AITER_REF} (stock)" >> /app/versions.txt + +# ----------------------------------------------------------------------------- +# 3. vLLM: compile from source at the 06_29 validated Wide-EP WRITE-mode branch +# (matches the published dist-inf-cookbook mori121 image). Full source compile +# (the base ships a different commit). The MoRIIO disagg fixes (#39276 notify, +# #41751 LL split, DP-rank hash-failsafe) are native in this branch, so no runtime +# patcher is needed. Override VLLM_REF to rebuild a different commit; build only +# committed commits (no working-tree edits). +# ----------------------------------------------------------------------------- +# VLLM_REPO/REF are a PUBLIC GitHub repo + branch (the Wide-EP WRITE-mode vLLM the +# dist-inf-cookbook mori121 image builds from). Override to your own vLLM fork/branch. +ARG VLLM_REPO=https://github.com/raviguptaamd/vllm.git +ARG VLLM_REF=glm5.1-dsa-wideEP_on_shik_latest +ENV VLLM_TARGET_DEVICE=rocm \ + PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH} \ + MAX_JOBS=${MAX_JOBS} +RUN rm -rf /tmp/vllm-src && \ + git clone "${VLLM_REPO}" /tmp/vllm-src && \ + cd /tmp/vllm-src && git checkout "${VLLM_REF}" && \ + echo "VLLM_REF=${VLLM_REF}@$(git rev-parse HEAD)" >> /app/versions.txt && \ + pip uninstall -y vllm 2>/dev/null || true && \ + pip install --no-deps --no-build-isolation -v . && \ + python3 -c "import vllm; print('vLLM', vllm.__version__, 'from', vllm.__file__)" && \ + rm -rf /tmp/vllm-src + +# Cross-check MoRI + AITER survived the vLLM install (no silent downgrade). +RUN python3 - <<'PYEOF' +from importlib.metadata import version as v, PackageNotFoundError +def get(names): + for n in names: + try: return v(n) + except PackageNotFoundError: pass + return None +av = get(("amd-aiter", "amd_aiter", "aiter")) +# Stock source build of ROCm/aiter@e03fa6040 reports 0.1.17.dev195+ge03fa6040. +# Verify the aiter install survived the vLLM install (present + carries the e03fa6040 +# commit tag) rather than pinning a release version string. +assert av and "e03fa6040" in av, f"AITER missing/downgraded (want e03fa6040 build): {av!r}" +import mori, mori.io, mori.ops +print("Post-vLLM check OK: AITER", av, "+ MoRI importable") +PYEOF + +# ----------------------------------------------------------------------------- +# 4. vllm-router (DP-rank round-robin + MoRIIO connector) — built in, so NO +# external vllm-router binary is needed (leave ROUTER_BINARY unset). +# Source = vllm-project/router PR #181 branch, which now carries BOTH the +# round-robin DP-rank fix (11841c0d) AND the 2P2D KV-notify fix (6409ac1: +# remote_dp_rank_override + remote_dp_size). The KV-notify fix is REQUIRED: +# without it the 2P2D EP=16 run reproducibly wedges with "remote blocks never +# arrived" deferred-write expiries (decode notify targets the wrong DP rank). +# This is the exact source of the validated vllm-router-2p2d-dpfix binary. +# Pinned Rust toolchain (>=1.88: router deps time/home require rustc 1.88). +# ----------------------------------------------------------------------------- +ARG ROUTER_REPO=https://github.com/raviguptaamd/router.git +ARG ROUTER_REF=ravgupta/discovery-dp-rank-roundrobin +ARG RUST_TOOLCHAIN=1.88.0 +RUN if ! command -v cargo >/dev/null 2>&1; then \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain "${RUST_TOOLCHAIN}"; \ + fi && \ + export PATH="/root/.cargo/bin:${PATH}" && \ + rm -rf /tmp/vllm-router-src && \ + git clone --filter=blob:none "${ROUTER_REPO}" /tmp/vllm-router-src && \ + cd /tmp/vllm-router-src && git checkout "${ROUTER_REF}" && \ + cargo build --release && \ + install -m 755 target/release/vllm-router /usr/local/bin/vllm-router && \ + vllm-router --help 2>&1 | grep -q moriio && \ + echo "VLLM_ROUTER_REF=${ROUTER_REPO}@${ROUTER_REF}@$(git -C /tmp/vllm-router-src rev-parse HEAD)" >> /app/versions.txt && \ + rm -rf /tmp/vllm-router-src + +# ----------------------------------------------------------------------------- +# 4b. WITH_NIXL=1 (default): UCX + RIXL(+nixlbench) + rocSHMEM + DeepEP from source, +# so the rixl connector (NIXL TP + DeepEP wideEP) is present. Single guarded RUN so +# WITH_NIXL=0 skips it entirely (no layers, no cost). Build-verified on ci_base. +# ----------------------------------------------------------------------------- +ENV _ROCM_DIR=/opt/rocm \ + _UCX_SOURCE=https://github.com/ROCm/ucx.git \ + _UCX_BRANCH=da3fac2a \ + _UCX_INSTALL_DIR=/usr/local/ucx/ \ + _RIXL_SOURCE=https://github.com/ROCm/RIXL.git \ + _RIXL_BRANCH=f33a5599 \ + _RIXL_INSTALL_DIR=/usr/local/RIXL/install \ + _NIXLBENCH_INSTALL_DIR=/usr/local/RIXL +RUN if [ "${WITH_NIXL}" != "1" ]; then \ + echo "WITH_NIXL=${WITH_NIXL}: skipping UCX/RIXL/rocSHMEM/DeepEP (MoRI-EP + base DeepEP only)"; \ + else set -e && \ + echo "WITH_NIXL=1: building UCX + RIXL + rocSHMEM + DeepEP" && \ + apt-get update && apt-get install -y \ + autoconf automake libtool autogen pkg-config m4 gcc make \ + librdmacm-dev rdmacm-utils infiniband-diags ibverbs-utils perftest ethtool \ + libibverbs-dev rdma-core strace libgflags-dev \ + libaio-dev liburing-dev libcpprest-dev libgrpc-dev libgrpc++-dev \ + libprotobuf-dev protobuf-compiler-grpc wget && \ + pip install meson==0.64.0 "pybind11[global]" pyyaml && \ + # UCX + cd /tmp && git clone "${_UCX_SOURCE}" && cd ucx && git checkout "${_UCX_BRANCH}" && \ + ./autogen.sh && mkdir -p build && cd build && \ + ../configure --prefix="${_UCX_INSTALL_DIR}" --with-rocm="${_ROCM_DIR}" \ + --disable-go --disable-java --disable-assertions --enable-mt && \ + make -j && make install && \ + # googletest (RIXL dep) + cd /tmp && wget -q https://github.com/google/googletest/archive/refs/tags/v1.14.0.tar.gz && \ + tar -xzf v1.14.0.tar.gz && cd googletest-1.14.0 && mkdir -p build && cd build && \ + cmake -DBUILD_SHARED_LIBS=on .. && make -j && make install && \ + # RIXL + python bindings + cd /tmp && git clone "${_RIXL_SOURCE}" && cd RIXL && git checkout "${_RIXL_BRANCH}" && \ + meson setup build/ --prefix="${_RIXL_INSTALL_DIR}" -Ducx_path="${_UCX_INSTALL_DIR}" \ + -Ddisable_gds_backend=true -Dcudapath_inc="${_ROCM_DIR}/include" -Dcudapath_lib="${_ROCM_DIR}/lib" && \ + cd build && ninja && ninja install && cd /tmp/RIXL && \ + pip install --config-settings=setup-args="-Dcudapath_inc=${_ROCM_DIR}/include" \ + --config-settings=setup-args="-Dcudapath_lib=${_ROCM_DIR}/lib" \ + --config-settings=setup-args="-Ducx_path=${_UCX_INSTALL_DIR}" \ + --config-settings=setup-args="-Ddisable_gds_backend=true" . && \ + # rocSHMEM (DeepEP dep) + cd /tmp && git clone --no-checkout --filter=blob:none https://github.com/ROCm/rocm-systems.git && \ + cd rocm-systems && git sparse-checkout set --cone projects/rocshmem && git checkout develop && \ + mkdir -p /tmp/rocshmem-build && cd /tmp/rocshmem-build && \ + /tmp/rocm-systems/projects/rocshmem/scripts/build_configs/all_backends \ + -DUSE_EXTERNAL_MPI=OFF -DGPU_TARGETS="${GFX_COMPILATION_ARCH}" && \ + # DeepEP (build develop against the installed vLLM/torch) + cd /tmp && git clone https://github.com/ROCm/DeepEP.git && cd DeepEP && \ + PYTORCH_ROCM_ARCH="${GFX_COMPILATION_ARCH}" CFLAGS="-O3 -fPIC" \ + CXXFLAGS="-O3 -fPIC --offload-arch=${GFX_COMPILATION_ARCH}" HIP_CXX_FLAGS="-O3 -fPIC" \ + python3 setup.py --variant rocm --nic "${NIC_COMPILATION_ARCH}" build develop && \ + echo "WITH_NIXL build complete" >> /app/versions.txt && \ + rm -rf /tmp/ucx /tmp/googletest-1.14.0 /tmp/v1.14.0.tar.gz /tmp/rocm-systems /tmp/rocshmem-build; \ + fi +ENV LD_LIBRARY_PATH="/usr/local/ucx/lib:/usr/local/lib:/usr/local/RIXL/install/lib:${LD_LIBRARY_PATH}" \ + PATH="/usr/local/ucx/bin:${PATH}" + +# ----------------------------------------------------------------------------- +# 5. Cache locations (structural: WHERE the JIT/compile caches live in the image). +# These are the mount target for the launcher's persistent host JIT cache. +# ----------------------------------------------------------------------------- +# The image ships NO runtime recipe / tuning / platform ENV. By design, everything +# run-tunable is applied at launch, so this image stays a clean binary/library artifact +# and the same image serves any model/cluster without a rebuild: +# - model-serving recipe (KV_BLOCK_SIZE, KV_CACHE_DTYPE, *_CUDAGRAPH_MODE, *_MORI_BACKEND, +# GPU_MEMORY_UTILIZATION, KV_CACHE_MEMORY_BYTES, VLLM_ROCM_USE_AITER_MLA, ...) +# -> scripts/vllm_dissag/models.yaml (per-model env:, so dense vs MoE differ) +# - ROCm-7.2.3 GPU-RDMA platform env (expandable_segments:False x2, MORI_GPU_ARCHS, +# HSA_ENABLE_IPC_MODE_LEGACY=0, HSA_NO_SCRATCH_RECLAIM) and the MoRI/RDMA fabric +# tuning (MORI_RDMA_TC/SL, MORI_IB_GID_INDEX, MORI_NUM_QP_PER_PE, VLLM_MORIIO_*, ...) +# -> scripts/vllm_dissag/connectors/.env (cluster-editable, no rebuild) +# The slurm launcher forwards both via `docker -e` (platform env must reach PID 1 - +# PyTorch reads alloc-conf at import). Running this image WITHOUT the launcher: set the +# vars you need yourself (see connectors/moriio.env + models.yaml for the values). +ENV AITER_JIT_DIR=/opt/vllm_cache/aiter_jit \ + VLLM_CACHE_ROOT=/opt/vllm_cache/vllm \ + TRITON_CACHE_DIR=/opt/vllm_cache/triton \ + COMGR_CACHE_DIR=/opt/vllm_cache/comgr + +# ----------------------------------------------------------------------------- +# 6. CRITICAL: scrub build-time MoRI JIT state. The `import mori` verification +# steps above compile/lock MoRI EP kernels under /root/.mori/jit on THIS build +# host, leaving stale .hsaco.lock files (ep_internode_v1, ep_internode_v1ll, ...). +# At runtime on the cluster, MoriAll2AllManager finds those locks, waits on a +# build-in-progress whose owner PID is long gone, and DEADLOCKS at ep:0 init. +# A clean image ships /root/.mori empty -> runtime compiles fresh. +# Clearing these makes the from-source image boot clean on 2P2D/4P4D. +# ----------------------------------------------------------------------------- +RUN rm -rf /root/.mori /tmp/mori_jit_* && mkdir -p /root/.mori && \ + echo "JIT_SCRUBBED: /root/.mori + /tmp/mori_jit_* cleared at build end" >> /app/versions.txt + +RUN cat /app/versions.txt 2>/dev/null | tail -20 || true diff --git a/scripts/vllm_dissag/apply_glm_aiter_sampling_oob_fix.py b/scripts/vllm_dissag/apply_glm_aiter_sampling_oob_fix.py new file mode 100644 index 00000000..d491b838 --- /dev/null +++ b/scripts/vllm_dissag/apply_glm_aiter_sampling_oob_fix.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Overlay the fixed AITER sampling kernel (ROCm/aiter #3658 + hang cap) into the image. + +DEFECT 2 (the 8k prefill/decode crash): the AITER TopP/TopK sampling kernel +(csrc/cpp_itfs/sampling/sampling.cuh) has two bugs on the released aiter post3 +that this image ships: + + 1. HSA OUT-OF-BOUNDS (ROCm/aiter #3658): SamplingTempStorage::last_valid_id is + never initialized. When a probs row is all-zero / NaN (more likely at long + input, e.g. 8k), the guarded write-back (max_valid != -1) is skipped, and the + fallback `sampled_id = temp_storage.last_valid_id` reads UNINITIALIZED shared + memory -> garbage index -> `probs[row*d + sampled_id]` dereferences OOB and + HSA page-faults ("Memory access fault by GPU node-N"). Deterministic under + CUDA graph (shared-mem residue is stable across replays). This is the silent + worker death at 8k that collapses the disagg DP group -> 503. + Fix: init `last_valid_id = 0` at top of each loop iter + defensive clamp on + the loaded sampled_id before it indexes probs. + + 2. REJECTION-SAMPLING HANG: the bisection `do { ... } while(low < high)` can + spin forever when the [low,high] interval stagnates in float precision on a + degenerate (near-uniform) row -> never-completing HSA signal / hang (the + "sampler hang" that forced the skip-warmup workaround). Fix: cap the loop at + kMaxSamplingRounds=32 (float32 mantissa is exhausted well within 32 rounds, + so healthy distributions always converge via break long before the cap). + +Both fixes land in sampling.cuh. #3658 is MERGED upstream but NOT in the released +aiter post3 (this image). Source: A/B-tested by Shiksha (shikpate); staged fixed +tree at SAMPLING_FIX_DIR. + +METHOD (from Shiksha's validated in-container overlay): copy the whole patched +sampling source dir (.cuh + .py + .jinja) over the container's aiter, then purge +any compiled sampling JIT objects so the kernel recompiles from the fixed source +on next use. + +Idempotent: skips if the fix markers are already present. Model-agnostic at the +kernel level, but invoked from the GLM patch hook. Safe no-op if the staged fix +dir or the target aiter dir is absent. + +Usage: apply_glm_aiter_sampling_oob_fix.py + (vllm_install_dir arg is accepted for hook uniformity but not required; + the aiter dir is resolved via `import aiter`.) +""" +import os +import shutil +import subprocess +import sys + +FIX_DIR = os.environ.get( + "SAMPLING_FIX_DIR", + "/shared_inference/ravgupta/aiter_sampling_fix_3658/sampling_patched", +) +MARKERS = ("last_valid_id = 0", "kMaxSamplingRounds") + + +def _aiter_sampling_dir(): + """Locate the installed aiter sampling source dir (aiter_meta/csrc/...).""" + try: + import aiter # noqa: F401 + except Exception as e: # noqa: BLE001 + print(f"[sampling-fix] aiter not importable ({e}); skipping.") + return None + # The kernel source lives under aiter_meta (sibling of aiter), path is stable. + candidates = [] + try: + import aiter_meta # type: ignore + + candidates.append( + os.path.join(os.path.dirname(aiter_meta.__file__), + "csrc", "cpp_itfs", "sampling") + ) + except Exception: # noqa: BLE001 + pass + # Fallback: search site-packages. + import aiter + sp = os.path.dirname(os.path.dirname(aiter.__file__)) + candidates.append(os.path.join(sp, "aiter_meta", "csrc", "cpp_itfs", "sampling")) + for c in candidates: + if os.path.isdir(c): + return c + print(f"[sampling-fix] could not locate aiter sampling dir (tried {candidates}); skipping.") + return None + + +def main() -> int: + tgt = _aiter_sampling_dir() + if tgt is None: + return 0 # safe no-op + + tgt_cuh = os.path.join(tgt, "sampling.cuh") + if os.path.isfile(tgt_cuh): + cur = open(tgt_cuh, errors="ignore").read() + if all(m in cur for m in MARKERS): + print(f"[sampling-fix] already applied (markers present) in {tgt_cuh}.") + return 0 + + if not os.path.isdir(FIX_DIR): + print(f"[sampling-fix] WARN: staged fix dir {FIX_DIR} not found; leaving image kernel unpatched.", file=sys.stderr) + return 0 + + src_cuh = os.path.join(FIX_DIR, "sampling.cuh") + if not os.path.isfile(src_cuh) or not all(m in open(src_cuh, errors="ignore").read() for m in MARKERS): + print(f"[sampling-fix] WARN: staged {src_cuh} missing/lacks fix markers; skipping.", file=sys.stderr) + return 0 + + # Overlay the whole sampling source dir (.cuh + .py + .jinja), per Shiksha's method. + copied = [] + for fn in os.listdir(FIX_DIR): + s = os.path.join(FIX_DIR, fn) + if os.path.isfile(s): + shutil.copy2(s, os.path.join(tgt, fn)) + copied.append(fn) + print(f"[sampling-fix] overlaid #3658 + hang-cap into {tgt}: {', '.join(sorted(copied))}") + + # Verify. + cur = open(tgt_cuh, errors="ignore").read() + if not all(m in cur for m in MARKERS): + print(f"[sampling-fix] ERROR: markers still absent after overlay in {tgt_cuh}.", file=sys.stderr) + return 1 + + # Purge any compiled sampling JIT objects so the kernel recompiles from source. + purged = 0 + for base in ( + os.path.expanduser("~/.aiter"), "/root/.aiter", "/tmp/aiter", + "/opt/vllm_cache/aiter_jit", os.path.join(os.path.dirname(tgt), "..", "..", "jit"), + ): + if base and os.path.isdir(base): + try: + out = subprocess.run( + ["find", base, "-maxdepth", "6", "-iname", "*sampling_from_probs*"], + capture_output=True, text=True, timeout=60, + ) + for p in out.stdout.split(): + try: + if os.path.isdir(p): + shutil.rmtree(p, ignore_errors=True) + else: + os.remove(p) + purged += 1 + except OSError: + pass + except Exception: # noqa: BLE001 + pass + print(f"[sampling-fix] purged {purged} stale sampling JIT object(s); kernel will recompile from fixed source.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/vllm_dissag/apply_glm_dsa_indexer_warmup_fix.py b/scripts/vllm_dissag/apply_glm_dsa_indexer_warmup_fix.py new file mode 100755 index 00000000..3eaef8cf --- /dev/null +++ b/scripts/vllm_dissag/apply_glm_dsa_indexer_warmup_fix.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +"""Force-compile the GLM-5.1 DSA sparse-attention indexer Triton kernels at BOOT. + +PROBLEM (root cause of the "first big prompt stalls the whole DP group" hang): + The DSA indexer's Triton kernels are seq-length specialized: + - v1/attention/ops/triton_fp8_mqa_logits.py flips `matrix_instr_nonkdim` + at seq_len<=1024 and launches with grid=[(seq_len,)] (seq_len is a + specialized kernel arg) -> a >1024-row prefill needs a *different* JIT + specialization than a <=1024-row one. + But the boot-time warmup never drives the indexer: + - profile_run() calls _dummy_run(is_profile=True) with force_attention=False + and cudagraph mode NONE -> attn_metadata stays None -> sparse_attn_indexer + takes the `sparse_attn_indexer_fake` path (see the "careful! this will be + None in dummy run" comment in layers/sparse_attn_indexer.py). The real + kernels are never compiled. + - _warmup_and_capture() only sets force_attention=True when the cudagraph + runtime mode is FULL; the DSA indexer builder reports UNIFORM_BATCH, so on + this ROCm/DP build the mixed prefill-decode graphs are PIECEWISE and + force_attention stays False. Even when attention IS forced, capture uses + uniform-decode / small mixed batches -- never a large prefill at + max_num_batched_tokens -- so the >1024 specialization is still absent. + Effect: the first >=8k prompt JIT-compiles the indexer kernel mid-inference on + whichever DP rank happens to receive it. That rank falls out of the DP lockstep + gloo all_reduce (coordinate_batch_across_dp) while it compiles -> the whole DP + group collapses. It is also a general cold-cache robustness hole. + +FIX (surgical, reuses vLLM's OWN metadata construction -- no hand-synthesized +tensors, so zero risk of a bad-input crash at boot): + 1. gpu_model_runner.py: add a `_maybe_warmup_dsa_indexer()` method. It is a + strict NO-OP unless one of the runner's attention backends is (a subclass + of) DeepseekV32IndexerBackend. When present, it runs + `_dummy_run(..., force_attention=True, cudagraph_runtime_mode=NONE)` at TWO + prefill-size regimes -- a small one (<=1024 rows) and a large one + (max_num_batched_tokens, >1024) -- so BOTH Triton specializations compile. + `force_attention=True` makes _dummy_run build a real + DeepseekV32IndexerMetadata via the normal _build_attention_metadata path + (num_prefills>0 because the default dummy batch is multi-token requests and + the indexer decode_threshold is 1), which drives the real + `sparse_attn_indexer` prefill kernels. The whole thing is wrapped in + try/except that only WARNs -- a warmup failure must never crash boot. + 2. gpu_worker.py: call it from compile_or_warm_up_model, right after the + existing warmup loop and before kernel_warmup(). At that point the KV cache + is already allocated (initialize_from_config runs before + compile_or_warm_up_model), which the forced-attention indexer path needs. + +Idempotent + anchor-based (matches the other apply_glm_* patchers): + * Each hunk self-detects if already applied (marker string present) -> no-op. + * Missing anchor -> WARN and skip that hunk (safe across vllm revisions; the + rebase may already warm the indexer natively or have refactored the site). + * Anchor found but the file does not contain the applied marker and the + replace produces no change -> hard error (would silently keep the bug). + * py_compile at the end; hard error if the patched file won't compile. + +Usage: apply_glm_dsa_indexer_warmup_fix.py +""" +import os +import sys + +RUNNER_REL = "v1/worker/gpu_model_runner.py" +WORKER_REL = "v1/worker/gpu_worker.py" + +MARKER = "glm-dsa-indexer-warmup" + +# --- Hunk A: new method inserted immediately before `def capture_model` ------- +# Anchor: the (unique) capture_model definition head in gpu_model_runner.py. +RUNNER_ANCHOR = " def capture_model(self) -> int:\n" + +RUNNER_METHOD = ''' def _maybe_warmup_dsa_indexer(self) -> None: + """Force-compile the DSA sparse-attention indexer Triton kernels at boot. + + NO-OP unless this model actually has a DeepseekV32IndexerBackend (GLM-5.1 + DSA / DeepSeek V3.2). The indexer kernels are seq-length specialized + (triton_fp8_mqa_logits flips matrix_instr_nonkdim at seq_len<=1024 and + launches grid=[(seq_len,)]), and the normal profile/warmup passes never + drive the indexer (attn_metadata is None -> the *_fake path). Without this + the first large prompt JIT-compiles mid-inference and, under DP lockstep, + stalls the whole group. We warm BOTH regimes: a small (<=1024) and a large + (max_num_batched_tokens, >1024) prefill batch, using force_attention=True + so _dummy_run builds a real indexer metadata via the standard path. + """ + # {marker} + try: + from vllm.v1.attention.backends.mla.indexer import ( + DeepseekV32IndexerBackend, + ) + except Exception: # noqa: BLE001 -- backend module absent -> not a DSA build + return + + has_indexer = False + try: + for attn_group in self._attn_group_iterator(): + backend = getattr(attn_group, "backend", None) + if backend is not None and isinstance(backend, type) and issubclass( + backend, DeepseekV32IndexerBackend + ): + has_indexer = True + break + except Exception: # noqa: BLE001 -- iterator shape changed -> stay a no-op + return + if not has_indexer: + return + + # Two prefill-size regimes so both Triton specializations compile. + # Small must be <=1024 rows; large must exceed 1024 (use the real max). + max_tokens = int(self.max_num_tokens) + small = min(512, max_tokens) + sizes = [] + for s in (small, max_tokens): + if s > 0 and s not in sizes: + sizes.append(s) + + logger.info( + "Warming up DSA indexer kernels at prefill sizes %s " + "to avoid mid-inference JIT.", + sizes, + ) + for size in sizes: + try: + self._dummy_run( + size, + cudagraph_runtime_mode=CUDAGraphMode.NONE, + force_attention=True, + skip_eplb=True, + remove_lora=False, + ) + except Exception as e: # noqa: BLE001 -- warmup must NEVER crash boot + logger.warning( + "DSA indexer warmup at size %d failed (%s); the kernel may " + "JIT-compile on first use instead.", + size, + e, + ) + self._sync_device() + +'''.replace("{marker}", MARKER) + +# --- Hunk B: call site in gpu_worker.compile_or_warm_up_model ----------------- +WORKER_ANCHOR = ( + " self.model_runner.maybe_remove_all_loras(" + "self.model_runner.lora_config)\n" + "\n" + " # Warmup and tune the kernels used during model execution before\n" + " # cuda graph capture.\n" + " kernel_warmup(self)\n" +) + +WORKER_REPLACEMENT = ( + " self.model_runner.maybe_remove_all_loras(" + "self.model_runner.lora_config)\n" + "\n" + " # " + MARKER + ": force-compile the DSA sparse-attention indexer\n" + " # Triton kernels now (KV cache is allocated), so a large prompt\n" + " # never JIT-compiles them mid-inference and stalls DP lockstep.\n" + " # No-op unless this model has a DeepseekV32IndexerBackend.\n" + " if hasattr(self.model_runner, \"_maybe_warmup_dsa_indexer\"):\n" + " self.model_runner._maybe_warmup_dsa_indexer()\n" + "\n" + " # Warmup and tune the kernels used during model execution before\n" + " # cuda graph capture.\n" + " kernel_warmup(self)\n" +) + + +def _patch_file(path, tag, anchor, apply_fn, already_marker): + """Return 0 on success/no-op, 1 on hard error.""" + if not os.path.isfile(path): + print(f"[{tag}] {path} not found -- skipping (layout differs).") + return 0 + src = open(path).read() + if already_marker in src: + print(f"[{tag}] already applied ({already_marker} present) in {path} -- no-op.") + return 0 + if anchor not in src: + print( + f"[{tag}] WARN: anchor not found in {path} -- skipping " + "(assuming native warmup / refactor)." + ) + return 0 + new_src = apply_fn(src) + if new_src == src: + print( + f"[{tag}] ERROR: anchor found but patch produced no change in {path}.", + file=sys.stderr, + ) + return 1 + try: + open(path, "w").write(new_src) + except OSError as e: + print(f"[{tag}] ERROR: failed to write patched {path}: {e}", file=sys.stderr) + return 1 + if already_marker not in open(path).read(): + print( + f"[{tag}] ERROR: post-write verification failed in {path}.", + file=sys.stderr, + ) + return 1 + print(f"[{tag}] patched {path} -- 1 hunk.") + return 0 + + +def main() -> int: + if len(sys.argv) != 2: + print(f"usage: {sys.argv[0]} ", file=sys.stderr) + return 2 + vllm_dir = sys.argv[1] + + runner_path = os.path.join(vllm_dir, RUNNER_REL) + worker_path = os.path.join(vllm_dir, WORKER_REL) + + rc = 0 + + # Hunk A: insert the method before capture_model. + rc |= _patch_file( + runner_path, + "glm-dsa-warmup", + RUNNER_ANCHOR, + lambda s: s.replace(RUNNER_ANCHOR, RUNNER_METHOD + RUNNER_ANCHOR, 1), + MARKER, + ) + + # Hunk B: call it from compile_or_warm_up_model. + rc |= _patch_file( + worker_path, + "glm-dsa-warmup", + WORKER_ANCHOR, + lambda s: s.replace(WORKER_ANCHOR, WORKER_REPLACEMENT, 1), + MARKER, + ) + + if rc: + return 1 + + # py-compile sanity for whichever files exist. + try: + import py_compile + + for p in (runner_path, worker_path): + if os.path.isfile(p): + py_compile.compile(p, doraise=True) + print("[glm-dsa-warmup] py_compile OK") + except Exception as e: # noqa: BLE001 + print( + f"[glm-dsa-warmup] ERROR: patched file fails to compile: {e}", + file=sys.stderr, + ) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/vllm_dissag/apply_glm_dsa_kernel_fix.py b/scripts/vllm_dissag/apply_glm_dsa_kernel_fix.py new file mode 100755 index 00000000..33f50dc4 --- /dev/null +++ b/scripts/vllm_dissag/apply_glm_dsa_kernel_fix.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Apply the GLM-5.1 DSA sparse-attention invalid-token kernel fix (vllm #45324). + +The DSA indexer kernel `_convert_req_index_to_global_index_kernel` in + vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py +maps invalid token slots to 0 instead of -1. With block-size 1 + DSA sparse MLA +that corrupts KV reads and the model emits `!!!` for every prompt. + +Upstream fix: vllm-project/vllm #45324 -- flip the 0 to -1 in the tl.where call: + is_invalid_tok | (~valid_block), 0, base * BLOCK_SIZE + inblock_off + is_invalid_tok | (~valid_block), -1, base * BLOCK_SIZE + inblock_off + +Design (matches launcher contract -- runs unconditionally for GLM, aborts on real +failure): + * IDEMPOTENT : if already -1, report and exit 0 (no-op). + * SELF-SKIPPING: if the file/anchor is absent (refactored or the rebase already + fixed it differently), report and exit 0 -- do NOT abort, because b10a9f7a may + carry the fix natively. We only fail on the one unambiguous bad state we can + fix and didn't, or on write failure. + * VERIFIES the post-write state. + +Usage: apply_glm_dsa_kernel_fix.py +""" +import os +import re +import sys + +REL = "v1/attention/backends/mla/rocm_aiter_mla_sparse.py" + +# Anchor is the stable right-hand side of the tl.where; the middle operand is the +# 0 (buggy) / -1 (fixed) we toggle. Whitespace-tolerant. +RE_ANY = re.compile( + r"(is_invalid_tok\s*\|\s*\(~valid_block\)\s*,\s*)(-?\d+)(\s*,\s*base\s*\*\s*BLOCK_SIZE\s*\+\s*inblock_off)" +) + + +def main() -> int: + if len(sys.argv) != 2: + print(f"usage: {sys.argv[0]} ", file=sys.stderr) + return 2 + vllm_dir = sys.argv[1] + path = os.path.join(vllm_dir, REL) + + if not os.path.isfile(path): + # File not present on this build -> nothing we can or should do. The + # rebase may use a different sparse backend layout. Do not block launch. + print(f"[glm-dsa] {REL} not found under {vllm_dir} -- skipping (assuming native/refactored).") + return 0 + + src = open(path).read() + m = RE_ANY.search(src) + if not m: + # Anchor gone (refactored / already fixed differently). Don't block. + print(f"[glm-dsa] invalid-token kernel anchor not found in {path} -- skipping (assuming native fix).") + return 0 + + cur = m.group(2) + if cur == "-1": + print(f"[glm-dsa] already fixed (kernel returns -1) in {path} -- no-op.") + return 0 + if cur != "0": + # Unexpected value -- surface it but don't guess. Treat as needs-attention. + print(f"[glm-dsa] ERROR: unexpected invalid-token return value '{cur}' (expected 0 or -1) in {path}.", + file=sys.stderr) + return 1 + + # cur == "0" : the known bug. Flip to -1. + new_src = src[:m.start(2)] + "-1" + src[m.end(2):] + try: + open(path, "w").write(new_src) + except OSError as e: + print(f"[glm-dsa] ERROR: failed to write patched {path}: {e}", file=sys.stderr) + return 1 + + # Verify. + chk = RE_ANY.search(open(path).read()) + if not chk or chk.group(2) != "-1": + print(f"[glm-dsa] ERROR: post-write verification failed in {path}.", file=sys.stderr) + return 1 + print(f"[glm-dsa] patched: invalid-token kernel now returns -1 (vllm #45324) in {path}.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/vllm_dissag/apply_glm_dsa_moriio_dualkv_fix.py b/scripts/vllm_dissag/apply_glm_dsa_moriio_dualkv_fix.py new file mode 100755 index 00000000..ac6800bf --- /dev/null +++ b/scripts/vllm_dissag/apply_glm_dsa_moriio_dualkv_fix.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Patch the MoRIIO KV connector to handle GLM-5.1 DSA's DUAL KV cache. + +PROBLEM (root cause of the 2P2D "Reaped deferred sends / no finished_sending" stall): + GLM-5.1 (GlmMoeDsaForCausalLM -> deepseek_v2.py) has TWO KV caches per layer: + - main MLA latent KV : MLAAttentionSpec, head_size = kv_lora_rank+rope (~576) + - DSA indexer KV : DeepseekV32IndexerCache, MLAAttentionSpec head_size = index_head_dim (128) + Both are 3D ("use_mla"), but DIFFERENT latent dim -> DIFFERENT per-block byte size. + The MoRIIO connector computes ONE global geometry from `first_kv_cache` and reuses it + for every cache, so the indexer cache is transferred with the main-MLA block size -> + wrong bytes/size -> the RDMA read for that region never reconciles -> completion notify + is never produced -> decode reaps deferred sends after 60s -> request hangs. + +FIX (surgical, per-layer geometry; no behavior change for single-cache MLA/DeepSeek): + 1. register_kv_caches: size each registered region by its OWN tensor (per-cache + region_len), not the global self.block_len. Also fix the local_kv_cache_size + append to use the current cache, not a stale loop var. + 2. _compute_block_transfer_offsets: derive shape from the PER-LAYER tensor + (self.kv_caches[layer_name].shape) instead of the global self.kv_cache_shape, + so transfer_size_byte / strides match that cache. + 3. _read_blocks: compute offsets PER LAYER inside the loop (was computed once from + first_layer and reused for all layers). + +Idempotent + anchor-based: each hunk checks if already applied / anchor present; +missing anchor -> warn-and-skip (so it is safe across connector revisions). A hunk +that finds its OLD anchor but fails to apply is a hard error (would silently keep the bug). + +Usage: apply_glm_dsa_moriio_dualkv_fix.py +""" +import os +import sys + +REL = "distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py" + + +def main() -> int: + if len(sys.argv) != 2: + print(f"usage: {sys.argv[0]} ", file=sys.stderr) + return 2 + path = os.path.join(sys.argv[1], REL) + if not os.path.isfile(path): + print(f"[glm-dualkv] {REL} not found -- skipping (connector layout differs).") + return 0 + + src = open(path).read() + orig = src + applied = [] + + # --- Hunk 1: per-cache region_len in register_kv_caches --------------------- + h1_old = """ for cache_or_caches in kv_caches.values(): + cache_list = [cache_or_caches] if use_mla else cache_or_caches + for cache in cache_list: + base_addr = cache.data_ptr() + region_len = self.num_blocks * self.block_len + caches_data.append((base_addr, region_len, cache.device.index, "")) + kv_caches_base_addr.append(base_addr)""" + h1_new = """ for cache_or_caches in kv_caches.values(): + cache_list = [cache_or_caches] if use_mla else cache_or_caches + for cache in cache_list: + base_addr = cache.data_ptr() + # DSA dual-KV fix: size each region by its OWN tensor, not the + # global self.block_len (the DSA indexer cache has a different + # latent dim than the main MLA cache). + region_len = cache.nelement() * cache.element_size() + caches_data.append((base_addr, region_len, cache.device.index, "")) + kv_caches_base_addr.append(base_addr)""" + if "region_len = cache.nelement() * cache.element_size()" in src: + applied.append("h1 (already)") + elif h1_old in src: + src = src.replace(h1_old, h1_new, 1) + applied.append("h1") + else: + print("[glm-dualkv] WARN: h1 anchor (region_len loop) not found -- skipping h1.") + + # --- Hunk 1b: local_kv_cache_size uses current kv_cache, not stale `cache` -- + h1b_old = " self.local_kv_cache_size.append(cache.nelement() * cache.element_size())" + h1b_new = " self.local_kv_cache_size.append(kv_cache.nelement() * kv_cache.element_size())" + if h1b_new in src: + applied.append("h1b (already)") + elif h1b_old in src: + src = src.replace(h1b_old, h1b_new, 1) + applied.append("h1b") + else: + print("[glm-dualkv] WARN: h1b anchor (local_kv_cache_size) not found -- skipping h1b.") + + # --- Hunk 2: per-layer shape in _compute_block_transfer_offsets ------------- + h2_old = """ assert self.kv_cache_shape is not None, "KV caches shape not initialized" + is_mla = len(self.kv_cache_shape) == 3 + stride = self.kv_caches[layer_name].stride() + sz = self.kv_caches[layer_name].element_size() + if is_mla: + blknum, blksize, hs = self.kv_cache_shape + hn = 1 + block_stride = stride[0] + else: + _, blknum, blksize, hn, hs = self.kv_cache_shape""" + h2_new = """ # DSA dual-KV fix: use the PER-LAYER tensor shape, not the global + # self.kv_cache_shape (the DSA indexer cache differs from the main MLA). + _layer_shape = tuple(self.kv_caches[layer_name].shape) + assert len(_layer_shape) > 0, "KV caches shape not initialized" + is_mla = len(_layer_shape) == 3 + stride = self.kv_caches[layer_name].stride() + sz = self.kv_caches[layer_name].element_size() + if is_mla: + blknum, blksize, hs = _layer_shape + hn = 1 + block_stride = stride[0] + else: + _, blknum, blksize, hn, hs = _layer_shape""" + if "_layer_shape = tuple(self.kv_caches[layer_name].shape)" in src: + applied.append("h2 (already)") + elif h2_old in src: + src = src.replace(h2_old, h2_new, 1) + applied.append("h2") + else: + print("[glm-dualkv] WARN: h2 anchor (_compute_block_transfer_offsets head) not found -- skipping h2.") + + # --- Hunk 3: per-layer offsets in _read_blocks ----------------------------- + h3_old = """ first_layer = list(self.layer_name_to_local_kv_cache_metadata.keys())[0] + offs = self._compute_block_transfer_offsets( + first_layer, local_block_ids, remote_block_ids, remote_moriio_meta + ) + + for layer_name in self.layer_name_to_local_kv_cache_metadata: + sess_idx = list(self.layer_name_to_local_kv_cache_metadata.keys()).index( + layer_name + ) + # TODO : apply multi-session batch-read when moriio support it + transfer_status = self.moriio_wrapper.read_remote_data( + offs[2], offs[0], offs[1], sessions[sess_idx] + )""" + h3_new = """ # DSA dual-KV fix: compute offsets PER LAYER (the DSA indexer cache has a + # different per-block size than the main MLA cache, so a single offs reused + # across all layers mis-sizes the indexer transfer -> lost completion notify). + for layer_name in self.layer_name_to_local_kv_cache_metadata: + sess_idx = list(self.layer_name_to_local_kv_cache_metadata.keys()).index( + layer_name + ) + offs = self._compute_block_transfer_offsets( + layer_name, local_block_ids, remote_block_ids, remote_moriio_meta + ) + # TODO : apply multi-session batch-read when moriio support it + transfer_status = self.moriio_wrapper.read_remote_data( + offs[2], offs[0], offs[1], sessions[sess_idx] + )""" + if "compute offsets PER LAYER" in src: + applied.append("h3 (already)") + elif h3_old in src: + src = src.replace(h3_old, h3_new, 1) + applied.append("h3") + else: + print("[glm-dualkv] WARN: h3 anchor (_read_blocks first_layer offsets) not found -- skipping h3.") + + if src != orig: + try: + open(path, "w").write(src) + except OSError as e: + print(f"[glm-dualkv] ERROR: write failed for {path}: {e}", file=sys.stderr) + return 1 + print(f"[glm-dualkv] patched {path} -- hunks: {', '.join(applied)}") + else: + print(f"[glm-dualkv] no changes ({', '.join(applied) or 'nothing applied'}) for {path}") + + # py-compile sanity + try: + import py_compile + py_compile.compile(path, doraise=True) + print("[glm-dualkv] py_compile OK") + except Exception as e: # noqa: BLE001 + print(f"[glm-dualkv] ERROR: patched file fails to compile: {e}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/vllm_dissag/apply_glm_dsa_moriio_engine_fix.py b/scripts/vllm_dissag/apply_glm_dsa_moriio_engine_fix.py new file mode 100755 index 00000000..9d6d468b --- /dev/null +++ b/scripts/vllm_dissag/apply_glm_dsa_moriio_engine_fix.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Fix MoRIIO WRITE-path per-layer offset caching for GLM-5.1 DSA dual KV cache. + +ROOT CAUSE (proven by instrumentation, job 37594): + GLM-5.1 (GlmMoeDsaForCausalLM) registers num_layers=156 KV caches = 78 main MLA + (per-block latent dim 576) + 78 DSA indexer caches (latent dim 132). TWO geometries. + + moriio_engine.py::MoRIIOEngine._prepare_transfer_plan computes the RDMA transfer + offsets ONCE (for whatever layer arrives first) and caches them on + request_info.transfer_offset, then REUSES that single offset/size tuple for ALL 156 + layers. The 78 indexer layers (dim 132) get written with the main-MLA geometry + (dim 576) -> wrong byte size/offset -> those RDMA writes are malformed; the per-layer + write accounting (writes_done) and/or the remote completion never reconciles -> + the producer's send_notify (gated on writes_done >= num_layers) misbehaves and the + decode side never receives a clean completion -> "Reaped deferred sends / no + finished_sending after 60s" -> request hangs. + +FIX (surgical, no dataclass change): + Cache transfer offsets PER LAYER on the request_info via a dynamically-attached dict + ``_transfer_offset_by_layer`` keyed by layer_name, instead of the single + ``transfer_offset`` slot. Each of the 156 layers then transfers with its OWN geometry + (the underlying _compute_block_transfer_offsets already takes layer_name and, with the + companion dualkv patch h2, reads the per-layer tensor shape). + + Single-geometry models (DeepSeek-V3 / Hunyuan, 1 cache/layer) are unaffected: + every layer has identical geometry, so per-layer caching yields the same offsets. + +Idempotent + anchor-based. A missing anchor warns-and-skips; a found OLD anchor that +fails to apply is a hard error (would silently keep the stall). + +Usage: apply_glm_dsa_moriio_engine_fix.py +""" +import os +import sys + +REL = "distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py" + + +def main() -> int: + if len(sys.argv) != 2: + print(f"usage: {sys.argv[0]} ", file=sys.stderr) + return 2 + path = os.path.join(sys.argv[1], REL) + if not os.path.isfile(path): + print(f"[glm-engine] {REL} not found -- skipping (engine layout differs).") + return 0 + + src = open(path).read() + + old = """ # Compute offsets if not cached + if request_info.transfer_offset is None: + offsets = self.worker._compute_block_transfer_offsets( + task.layer_name, + task.local_block_ids, + request_info.block_ids, + remote_moriio_meta, + ) + request_info.transfer_offset = offsets + + # Get session index + layer_names = list(self.worker.layer_name_to_local_kv_cache_metadata.keys()) + sess_idx = layer_names.index(task.layer_name) + + local_off, remote_off, sizes = request_info.transfer_offset""" + + new = """ # DSA dual-KV fix: cache offsets PER LAYER, not once per request. GLM-5.1 has + # two cache geometries (main MLA dim 576 + DSA indexer dim 132); a single + # cached offset reused across all 156 layers mis-sizes the indexer writes and + # the completion never reconciles. Per-layer caching is identical for + # single-geometry models (DeepSeek/Hunyuan). + _off_by_layer = getattr(request_info, "_transfer_offset_by_layer", None) + if _off_by_layer is None: + _off_by_layer = {} + request_info._transfer_offset_by_layer = _off_by_layer + offsets = _off_by_layer.get(task.layer_name) + if offsets is None: + offsets = self.worker._compute_block_transfer_offsets( + task.layer_name, + task.local_block_ids, + request_info.block_ids, + remote_moriio_meta, + ) + _off_by_layer[task.layer_name] = offsets + # keep the legacy single-slot populated (first layer) for any external reader + if request_info.transfer_offset is None: + request_info.transfer_offset = offsets + + # Get session index + layer_names = list(self.worker.layer_name_to_local_kv_cache_metadata.keys()) + sess_idx = layer_names.index(task.layer_name) + + local_off, remote_off, sizes = offsets""" + + if "_transfer_offset_by_layer" in src: + print(f"[glm-engine] already patched (_transfer_offset_by_layer present) -- no-op.") + elif old in src: + src = src.replace(old, new, 1) + open(path, "w").write(src) + print(f"[glm-engine] patched per-layer offset caching in {path}") + else: + print(f"[glm-engine] WARN: anchor (_prepare_transfer_plan offset block) not found -- skipping (engine revision differs).") + # Not fatal: without the anchor we can't safely patch; surface clearly. + return 0 + + try: + import py_compile + py_compile.compile(path, doraise=True) + print("[glm-engine] py_compile OK") + except Exception as e: # noqa: BLE001 + print(f"[glm-engine] ERROR: compile failed: {e}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/vllm_dissag/apply_glm_dsa_moriio_gate_fix.py b/scripts/vllm_dissag/apply_glm_dsa_moriio_gate_fix.py new file mode 100755 index 00000000..92501ae9 --- /dev/null +++ b/scripts/vllm_dissag/apply_glm_dsa_moriio_gate_fix.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Fix the MoRIIO transfer-completion gate for GLM-5.1 DSA dual KV cache. + +ROOT CAUSE (PROVEN, job 37615 instrumentation): + GLM-5.1 registers num_layers=156 KV caches = 78 main MLA (model.layers.N.self_attn.attn) + + 78 DSA indexer caches (model.layers.N.self_attn.indexer.k_cache). BUT only the 78 + main-MLA layers ever go through the KV-connector save_kv_layer hook -> only they write + -> writes_done caps at 78. The indexer caches are registered (counted in num_layers) + but vLLM NEVER calls save_kv_layer for them (the DSA indexer is a separate attention + component / DeepseekV32IndexerBackend that doesn't use the connector save path; decode + recomputes indexer state from the transferred main latent KV). + + The producer completion gate (moriio_engine.py): + request_info.writes_done += 1 + if request_info.writes_done >= self.worker.num_layers: # 156, never reached + send_notify(...) + caps at writes_done=78 < num_layers=156 -> send_notify NEVER fires -> decode never + gets completion -> "Reaped deferred sends / no finished_sending after 60s" -> stall. + +FIX: + Add self.num_transfer_layers = count of caches that actually transfer (exclude + '.indexer.' caches), with a fallback to num_layers (so single-geometry models - + DeepSeek-V3 / Hunyuan, no indexer - are bit-identical). Gate completion on + num_transfer_layers instead of num_layers. num_layers itself is left unchanged + (it is also used by the Llama-4 per-layer block-window loop, which needs all caches). + +Companion to apply_glm_dsa_moriio_engine_fix.py (per-layer offset caching). This gate +fix is the primary unblocker; the offset fix is correctness insurance for the layers +that DO write (all same geometry here, but harmless). + +Idempotent + anchor-based. Patches BOTH files (connector: define the field; engine: +use it). A found-old-anchor that fails is a hard error. + +Usage: apply_glm_dsa_moriio_gate_fix.py +""" +import os +import sys + +CONN_REL = "distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py" +ENG_REL = "distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py" + + +def patch_connector(path: str) -> int: + src = open(path).read() + old = " self.num_layers = len(self.kv_caches.keys())" + new = """ self.num_layers = len(self.kv_caches.keys()) + # DSA dual-KV fix: the producer completion gate must count only layers that + # actually transfer via save_kv_layer. GLM-5.1 registers 2 caches/layer (main + # MLA + DSA indexer), but only the main-MLA caches go through save_kv_layer; the + # '.indexer.' caches are registered yet never written. Gating on len(kv_caches) + # would never be reached. Exclude indexer caches; fall back to num_layers for + # single-geometry models (DeepSeek/Hunyuan have no indexer -> identical). + self.num_transfer_layers = ( + len([k for k in self.kv_caches.keys() if ".indexer." not in k]) + or self.num_layers + ) + logger.info( + "[moriio] completion gate: num_transfer_layers=%d (num_layers=%d)", + self.num_transfer_layers, self.num_layers, + )""" + if "self.num_transfer_layers" in src: + print(f"[glm-gate] connector already patched -- no-op.") + return 0 + if old not in src: + print(f"[glm-gate] WARN: connector anchor (num_layers=) not found -- skipping.") + return 0 + src = src.replace(old, new, 1) + open(path, "w").write(src) + print(f"[glm-gate] patched connector: defined num_transfer_layers in {path}") + return 0 + + +def patch_engine(path: str) -> int: + src = open(path).read() + old = " if request_info.writes_done >= self.worker.num_layers:" + new = """ if request_info.writes_done >= getattr( + self.worker, "num_transfer_layers", self.worker.num_layers + ):""" + if 'getattr(\n self.worker, "num_transfer_layers"' in src or "num_transfer_layers" in src: + print(f"[glm-gate] engine already patched -- no-op.") + return 0 + if old not in src: + print(f"[glm-gate] WARN: engine anchor (writes_done gate) not found -- skipping.") + return 0 + src = src.replace(old, new, 1) + open(path, "w").write(src) + print(f"[glm-gate] patched engine: gate on num_transfer_layers in {path}") + return 0 + + +def main() -> int: + if len(sys.argv) != 2: + print(f"usage: {sys.argv[0]} ", file=sys.stderr) + return 2 + base = sys.argv[1] + conn = os.path.join(base, CONN_REL) + eng = os.path.join(base, ENG_REL) + if not os.path.isfile(conn) or not os.path.isfile(eng): + print("[glm-gate] connector/engine not found -- skipping (layout differs).") + return 0 + + # ATOMIC: both halves (connector defines num_transfer_layers, engine gates on + # it) are needed together or not at all. On a restructured image (e.g. mori + # v1.2.1, whose engine replaced the writes_done>=num_layers gate with a sealed + # writes_expected mechanism that already handles hybrid/DSA dual-KV natively), + # the engine anchor is gone. Applying only the connector half would inject a + # dead num_transfer_layers into restructured internals. So if the engine anchor + # is absent, skip BOTH — the native gate already does the right thing. + eng_src = open(eng).read() + eng_gate_present = " if request_info.writes_done >= self.worker.num_layers:" in eng_src + eng_already = "num_transfer_layers" in eng_src + if not eng_gate_present and not eng_already: + print("[glm-gate] engine gate anchor absent (image restructured, e.g. mori " + "v1.2.1 sealed writes_expected) -- skipping BOTH halves (native gate handles DSA).") + return 0 + + rc = patch_connector(conn) or patch_engine(eng) + if rc: + return rc + + try: + import py_compile + py_compile.compile(conn, doraise=True) + py_compile.compile(eng, doraise=True) + print("[glm-gate] py_compile OK (both files)") + except Exception as e: # noqa: BLE001 + print(f"[glm-gate] ERROR: compile failed: {e}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/vllm_dissag/apply_glm_dsa_moriio_instrument.py b/scripts/vllm_dissag/apply_glm_dsa_moriio_instrument.py new file mode 100755 index 00000000..68e03beb --- /dev/null +++ b/scripts/vllm_dissag/apply_glm_dsa_moriio_instrument.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""TEMPORARY instrumentation: does the DSA indexer cache reach the MoRIIO save path? + +Adds logging at two points in moriio_connector.py to settle the dual-KV RCA: + 1. register_kv_caches: log all kv_caches layer names + their shapes + num_layers. + -> shows whether the DeepseekV32IndexerCache is even registered, and its geometry. + 2. _write_blocks_for_req: log each distinct layer_name that actually triggers a write. + -> compare the COUNT/SET of written layers vs num_layers. If indexer layers are in + kv_caches (counted in num_layers) but never written, writes_done can never reach + num_layers -> send_notify never fires -> the stall. + +This is diagnostic only (no behavior change). Remove before any production use. +Idempotent + anchor-safe. + +Usage: apply_glm_dsa_moriio_instrument.py +""" +import os +import sys + +REL = "distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py" + + +def main() -> int: + if len(sys.argv) != 2: + print(f"usage: {sys.argv[0]} ", file=sys.stderr) + return 2 + path = os.path.join(sys.argv[1], REL) + if not os.path.isfile(path): + print(f"[glm-instr] {REL} not found -- skipping.") + return 0 + + src = open(path).read() + orig = src + + # --- Point 1: log kv_caches inventory at num_layers assignment ------------- + a1 = " self.num_layers = len(self.kv_caches.keys())" + b1 = """ self.num_layers = len(self.kv_caches.keys()) + # [glm-instr] kv-cache inventory (dual-KV diagnosis) + try: + for _ln, _kv in self.kv_caches.items(): + logger.info("[glm-instr][register] layer=%s shape=%s dtype=%s", + _ln, tuple(_kv.shape), _kv.dtype) + logger.info("[glm-instr][register] num_layers=%d total_kv_caches=%d", + self.num_layers, len(self.kv_caches)) + except Exception as _e: # noqa: BLE001 + logger.info("[glm-instr][register] inventory log failed: %s", _e)""" + if "[glm-instr][register]" in src: + pass + elif a1 in src: + src = src.replace(a1, b1, 1) + else: + print("[glm-instr] WARN: register anchor (num_layers=) not found.") + + # --- Point 2: log each written layer in _write_blocks_for_req ------------- + a2 = " def _write_blocks_for_req(self, req_id: ReqId, meta: ReqMeta, layer_name, kv_layer):" + b2 = (a2 + "\n" + ' # [glm-instr] record which layers actually trigger a KV write\n' + ' try:\n' + ' _seen = getattr(self, "_glm_instr_written_layers", None)\n' + ' if _seen is None:\n' + ' _seen = set(); self._glm_instr_written_layers = _seen\n' + ' if layer_name not in _seen:\n' + ' _seen.add(layer_name)\n' + ' logger.info("[glm-instr][write] NEW layer=%s total_written=%d/%d",\n' + ' layer_name, len(_seen), getattr(self, "num_layers", -1))\n' + ' except Exception as _e: # noqa: BLE001\n' + ' logger.info("[glm-instr][write] log failed: %s", _e)') + if "[glm-instr][write]" in src: + pass + elif a2 in src: + src = src.replace(a2, b2, 1) + else: + print("[glm-instr] WARN: _write_blocks_for_req anchor not found.") + + if src != orig: + open(path, "w").write(src) + print(f"[glm-instr] instrumented {path}") + else: + print(f"[glm-instr] already instrumented / nothing to do for {path}") + + try: + import py_compile + py_compile.compile(path, doraise=True) + print("[glm-instr] py_compile OK") + except Exception as e: # noqa: BLE001 + print(f"[glm-instr] ERROR: compile failed: {e}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/vllm_dissag/apply_glm_dsa_persistent_kernel_gate_fix.py b/scripts/vllm_dissag/apply_glm_dsa_persistent_kernel_gate_fix.py new file mode 100644 index 00000000..adda36cf --- /dev/null +++ b/scripts/vllm_dissag/apply_glm_dsa_persistent_kernel_gate_fix.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""Gate OFF the AITER persistent sparse-MLA kernel for chunked-prefill batches. + +ROOT CAUSE (ROCm/aiter #4076, vLLM #47042 / #47567): + The AITER persistent MLA work-stealing kernel (mla_a8w8_qh16_qseqlen1_gqaratio16_ps, + taken when work_meta_data from get_mla_metadata_v1 is non-None) is NUMERICALLY WRONG + for multi-token (prefill-shaped) batches of qseqlen==1 entries. Pure decode (1 query + token) and fresh single-chunk prefills are correct; the error only appears once a + request becomes a CHUNKED-PREFILL CONTINUATION. The small per-token error COMPOUNDS + through the KV cache across chunked-prefill passes until long-context decode collapses + into repetition/garbage. Failure is gated on CHUNK COUNT, not raw context length + (verified: 22k in 2 chunks = correct, 22k in 3 chunks = garbage). + + On this image (aiter 0.1.16.post3, before the aiter-side kernel fix #3921) GLM-5.1-FP8 + DSA collapses at ~16-18k prompt tokens. The aiter kernel fix is the long-term answer + (AITERKER-132 / aiter #3921); this is the vLLM-side short-term gate (#47567), which + costs ~no perf (decode + single-chunk prefill keep the persistent path). + +FIX (port of vLLM PR #47567, adapted to this image's rocm_aiter_mla_sparse.py::build): + In ROCMAiterMLASparseMetadataBuilder.build(), detect chunked-prefill continuations + (a request with >1 query token this step whose total seq_len exceeds its query_len, + i.e. part of its context was computed in an earlier chunk) and, when ANY request in + the batch is such a continuation: + * skip the get_mla_metadata_v1 persistent-metadata launch, and + * pass work_meta_data=None to the metadata so mla_decode_fwd takes the CORRECT + non-persistent split-KV path. + Decode-only and single-chunk-prefill batches are unchanged (persistent path kept). + + Uses `seg_lengths` (per-request step query lengths, already computed at build() top) + and `common_attn_metadata.seq_lens_cpu[:num_reqs].numpy()` (total seq lens). Both are + present in this image's build(). + +Idempotent + anchor-based + self-skipping. Missing anchor -> warn+skip (safe across +image revisions / if a newer image already carries the aiter kernel fix). A found-old +anchor that fails to apply is a hard error (would silently keep the corruption). + +Usage: apply_glm_dsa_persistent_kernel_gate_fix.py +""" +import os +import sys + +REL = "v1/attention/backends/mla/rocm_aiter_mla_sparse.py" + +# Anchor 1: the persistent-metadata guard. We insert the continuation detection +# just before it and AND it into the condition. +OLD1 = """ if metadata_key != self._prev_metadata_key: + from aiter import get_mla_metadata_v1""" +NEW1 = """ # PERSISTENT-KERNEL GATE (aiter #4076 / vLLM #47567): the persistent + # sparse-MLA work-stealing kernel is numerically wrong for chunked-prefill + # continuation batches; the error compounds and breaks long-context decode. + # Fall back to the correct non-persistent path whenever any request in the + # batch is a chunked-prefill continuation (>1 query token this step AND + # total seq_len > this step's query_len). Decode + single-chunk prefills + # keep the fast persistent path -> no decode-throughput regression. + # Slice to num_reqs and cast to int64 (vLLM #47567 hardening / Rohan138 PR#1) + # so the masks cannot broadcast-mismatch under cudagraph padding. + _step_query_lens = seg_lengths[:num_reqs].astype(np.int64) + _total_seq_lens = common_attn_metadata.seq_lens_cpu[:num_reqs].numpy().astype( + np.int64 + ) + _is_chunked_continuation = (_step_query_lens > 1) & ( + _total_seq_lens > _step_query_lens + ) + _use_persistent = not bool(_is_chunked_continuation.any()) + if _use_persistent and metadata_key != self._prev_metadata_key: + from aiter import get_mla_metadata_v1""" + +# Anchor 2: the metadata construction passes the persistent buffer unconditionally. +# Gate it on _use_persistent. +OLD2 = " work_meta_data=self._mla_work_meta_data," +NEW2 = " work_meta_data=(self._mla_work_meta_data if _use_persistent else None)," + + +def main() -> int: + if len(sys.argv) != 2: + print(f"usage: {sys.argv[0]} ", file=sys.stderr) + return 2 + path = os.path.join(sys.argv[1], REL) + if not os.path.isfile(path): + print(f"[glm-persist] {REL} not found -- skipping (backend layout differs).") + return 0 + + src = open(path).read() + + if "_is_chunked_continuation" in src or "_use_persistent" in src: + print("[glm-persist] already patched (persistent-kernel gate present) -- no-op.") + return 0 + + # Both anchors must be present to apply safely. + if OLD1 not in src: + print("[glm-persist] WARN: persistent-metadata anchor (metadata_key guard) not " + "found -- skipping (image may already carry the aiter kernel fix, or the " + "backend was refactored).") + return 0 + if OLD2 not in src: + print("[glm-persist] ERROR: found the metadata_key guard but NOT the " + "work_meta_data=self._mla_work_meta_data assignment -- refusing partial " + "patch (would leave persistent kernel active). Aborting.", file=sys.stderr) + return 1 + + src = src.replace(OLD1, NEW1, 1) + src = src.replace(OLD2, NEW2, 1) + + try: + open(path, "w").write(src) + except OSError as e: + print(f"[glm-persist] ERROR: write failed for {path}: {e}", file=sys.stderr) + return 1 + + # Verify both edits landed. + chk = open(path).read() + if "_use_persistent = not bool(_is_chunked_continuation.any())" not in chk or \ + "if _use_persistent else None" not in chk: + print("[glm-persist] ERROR: post-write verification failed.", file=sys.stderr) + return 1 + + try: + import py_compile + py_compile.compile(path, doraise=True) + except Exception as e: # noqa: BLE001 + print(f"[glm-persist] ERROR: patched file fails to compile: {e}", file=sys.stderr) + return 1 + + print(f"[glm-persist] patched persistent-kernel gate (aiter #4076 / vLLM #47567) in {path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/vllm_dissag/apply_glm_moriio_abort_guard_fix.py b/scripts/vllm_dissag/apply_glm_moriio_abort_guard_fix.py new file mode 100644 index 00000000..c09b4acd --- /dev/null +++ b/scripts/vllm_dissag/apply_glm_moriio_abort_guard_fix.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Guard the MoRIIO connector abort path against a None peer_zmq (mori v1.2.1). + +ROOT CAUSE (observed on the router image, job 199144 decode crash): + When a request is ABORTED before its KV-transfer peer handshake completes, + the connector's release path runs: + + moriio_connector.py::_release_write_prefill_blocks + peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=False) # -> None + remote_host, _, remote_notify_port = parse_moriio_zmq_address(peer_zmq) # None.split(",") + -> AttributeError: 'NoneType' object has no attribute 'split' + + This only catches ValueError, not the AttributeError from a None peer_zmq, so + the EngineCore dies -> cascades to all decode workers (EngineDeadError) -> decode + is dead. Triggered by any request aborted before the peer handshake (e.g. a + canary/curl that times out during first-token cold JIT). + + The SAME FILE already guards this correctly at the other call site + (request_finished / _should_notify path): `if peer_zmq is not None:` then parse, + else fall back to params. The release path just missed the guard — an + inconsistent-guard bug in the connector. + +FIX (surgical, matches the file's own existing pattern): + In _release_write_prefill_blocks, when the params don't already carry + remote_host/remote_notify_port, guard the peer_zmq lookup: if it is None, log + and return (same graceful bail the existing `except ValueError` already does for + the "missing remote notify address" case). No behavior change when peer_zmq is + valid; single-geometry / non-aborted requests are unaffected. + +Idempotent + anchor-based + self-skipping (no-ops if the anchor is absent/already +guarded, so it is safe across connector revisions and other images). A found-old +anchor that fails to apply is a hard error (would leave the crash). + +Usage: apply_glm_moriio_abort_guard_fix.py +""" +import os +import sys + +REL = "distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py" + +# The buggy two lines: fetch peer_zmq (may be None) then parse it unguarded. +OLD = """ peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=False) + remote_host, _, remote_notify_port = parse_moriio_zmq_address(peer_zmq)""" + +NEW = """ peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=False) + # Abort-path guard: a request aborted before the KV peer handshake + # has peer_zmq=None; parse_moriio_zmq_address(None) would raise + # AttributeError and kill the EngineCore. Bail gracefully like the + # ValueError case below (matches the guarded call site elsewhere). + if peer_zmq is None: + logger.warning( + "Cannot release WRITE prefill blocks for request %s: " + "no peer zmq address (aborted before peer handshake)", + request_id, + ) + return + remote_host, _, remote_notify_port = parse_moriio_zmq_address(peer_zmq)""" + + +def main() -> int: + if len(sys.argv) != 2: + print(f"usage: {sys.argv[0]} ", file=sys.stderr) + return 2 + path = os.path.join(sys.argv[1], REL) + if not os.path.isfile(path): + print(f"[glm-abort] {REL} not found -- skipping (connector layout differs).") + return 0 + + src = open(path).read() + if "no peer zmq address (aborted before peer handshake)" in src: + print("[glm-abort] already patched -- no-op.") + return 0 + if OLD not in src: + # Anchor absent: either the release path was refactored or this image + # already guards it. Do not block launch. + print("[glm-abort] release-path anchor not found -- skipping (assuming " + "native guard / refactored).") + return 0 + + src = src.replace(OLD, NEW, 1) + try: + open(path, "w").write(src) + except OSError as e: + print(f"[glm-abort] ERROR: write failed for {path}: {e}", file=sys.stderr) + return 1 + + try: + import py_compile + py_compile.compile(path, doraise=True) + except Exception as e: # noqa: BLE001 + print(f"[glm-abort] ERROR: patched file fails to compile: {e}", file=sys.stderr) + return 1 + print(f"[glm-abort] patched _release_write_prefill_blocks None-guard in {path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/vllm_dissag/connectors/moriio.sh b/scripts/vllm_dissag/connectors/moriio.sh index 44c64d68..c26a0944 100644 --- a/scripts/vllm_dissag/connectors/moriio.sh +++ b/scripts/vllm_dissag/connectors/moriio.sh @@ -124,13 +124,96 @@ _moriio_build_kv_transfer_config() { } connector_runtime_patch() { - # No-op: the MoRIIO multi-node disagg fixes (vLLM PR#39276 notify-path, #41751 LL - # split, DP-rank hash-failsafe) are committed in-source in the vLLM the image is - # built from (see the Dockerfile VLLM_REF). There is no runtime .py patcher — that - # would be a drifting duplicate of fixes that already live upstream in the fork. - # If you ever run an image WITHOUT these fixes baked, use an image that has them - # (rebuild from the pinned VLLM_REF) rather than patching a stock image at runtime. - return 0 + # MoRIIO multi-node disagg fixes (vLLM PR#39276 notify-path, #41751 LL split, + # DP-rank hash-failsafe) are committed in-source in the vLLM the image is built + # from (Dockerfile VLLM_REF). There is no generic runtime .py patcher for those — + # that would be a drifting duplicate of fixes already upstream in the fork. + # + # EXCEPTION — GLM-5.1-FP8 (GlmMoeDsaForCausalLM, MLA + DSA sparse attention): + # DSA is a NEW attention family the MoRIIO connector was never built for. It adds + # a 2nd KV cache per layer (indexer) with a different geometry, which the + # single-geometry connector mis-handles -> disagg KV transfer stalls; plus a DSA + # invalid-token kernel bug (#45324) that produces `!!!`. These are model-specific + # code gaps, applied here as idempotent, anchor-based, self-skipping .py patchers + # (they no-op cleanly if the fix is native/refactored on the chosen image). Gated + # on MODEL_NAME so DeepSeek/other models are a pure no-op (byte-identical to before). + # The MoRI version is pinned by the Dockerfile MORI_REF (post-1.2.1 main with the + # large-transfer notify/mapping fixes #424/#436/#432 baked in); if a newer MoRI is + # needed, update MORI_REF and rebuild the image — no runtime library swap here. + [ "${MODEL_NAME:-}" = "GLM-5.1-FP8" ] || return 0 + _glm_dsa_runtime_patch +} + +# GLM-5.1 DSA patchers (see connector_runtime_patch). Ported from MAD-private #338. +# Resolves the vLLM install dir, then applies the 4 required patchers in order, +# aborting on a hard failure (a real failure means GLM emits garbage or stalls, so +# failing at launch is correct). Patchers self-skip (rc 0) when their anchor is +# absent, so an image that already carries or refactored a fix no-ops cleanly. +_glm_dsa_runtime_patch() { + # GLM_SKIP_PATCHERS=1: the serving image already carries the GLM-5.1 DSA fixes + # in-source (e.g. the #47766 stack image built from raviguptaamd/vllm@ + # glm5.1-dsa-wideEP_on_shikpate_06_29_customer). Skip ALL runtime patchers — they + # are redundant, and the persistent-gate/sampling-overlay patchers would actively + # REGRESS a baked image (turn persistent MLA off / overwrite stock aiter kernels). + if [ "${GLM_SKIP_PATCHERS:-0}" = "1" ]; then + echo "[glm] GLM_SKIP_PATCHERS=1: image carries DSA fixes in-source; skipping runtime patchers." + return 0 + fi + local _patch_dir="${SCRIPT_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")/.." && pwd)}" + local _vllm_dir + _vllm_dir="$(python3 -c 'import vllm, os; print(os.path.dirname(vllm.__file__))' 2>/dev/null || true)" + if [ -z "${_vllm_dir}" ] || [ ! -d "${_vllm_dir}" ]; then + echo "Error: [glm] cannot locate vLLM install dir for DSA patchers. Aborting." >&2 + exit 1 + fi + echo "[glm] MODEL_NAME=GLM-5.1-FP8: applying DSA runtime patchers against ${_vllm_dir}" + + # Ordered list of REQUIRED patchers (all abort on hard failure). + # GLM_PERSIST_GATE=0 skips the persistent-MLA accuracy gate (debug only: to test + # whether the non-persistent kernel it routes to is what crashes disagg at >=8k). + local _gate_patcher="apply_glm_dsa_persistent_kernel_gate_fix.py" + [ "${GLM_PERSIST_GATE:-1}" = "0" ] && _gate_patcher="" + local _p + for _p in \ + apply_glm_dsa_kernel_fix.py \ + apply_glm_dsa_moriio_dualkv_fix.py \ + apply_glm_dsa_moriio_engine_fix.py \ + apply_glm_dsa_moriio_gate_fix.py \ + apply_glm_moriio_abort_guard_fix.py \ + ${_gate_patcher} \ + apply_glm_aiter_sampling_oob_fix.py; do + local _py="${_patch_dir}/${_p}" + if [ ! -f "${_py}" ]; then + echo "Error: [glm] required patcher ${_py} not found. Aborting." >&2 + exit 1 + fi + echo "[glm] applying ${_p}" + python3 "${_py}" "${_vllm_dir}" 2>&1 || { + echo "Error: [glm] ${_p} failed — GLM-5.1 would emit garbage or stall. Aborting." >&2 + exit 1 + } + done + + # Optional DSA indexer boot-warmup (GLM_INDEXER_WARMUP=1). Force-compiles the DSA + # indexer kernels at boot so they never JIT mid-inference. Opt-in because it drives a + # large (>=8k) prefill forward at boot: on stacks where that forward faults it makes + # the fault DETERMINISTIC at boot (useful for debugging) rather than on first request. + if [ "${GLM_INDEXER_WARMUP:-0}" = "1" ]; then + local _warm="${_patch_dir}/apply_glm_dsa_indexer_warmup_fix.py" + if [ -f "${_warm}" ]; then + echo "[glm] applying DSA indexer boot-warmup (GLM_INDEXER_WARMUP=1)" + python3 "${_warm}" "${_vllm_dir}" 2>&1 || echo "Warning: [glm] indexer-warmup patch failed (non-fatal)." + fi + fi + + # Optional diagnostic instrumentation (GLM_INSTRUMENT=1). Non-fatal. + if [ "${GLM_INSTRUMENT:-0}" = "1" ]; then + local _instr="${_patch_dir}/apply_glm_dsa_moriio_instrument.py" + if [ -f "${_instr}" ]; then + echo "[glm] applying instrumentation (GLM_INSTRUMENT=1): apply_glm_dsa_moriio_instrument.py" + python3 "${_instr}" "${_vllm_dir}" 2>&1 || echo "Warning: [glm] instrumentation failed (non-fatal)." + fi + fi } # connector_launch_worker [dp_start_rank] @@ -219,7 +302,7 @@ connector_launch_worker() { --all2all-backend "${_all2all}" \ --trust-remote-code \ --distributed-timeout-seconds "${DISTRIBUTED_TIMEOUT_SECONDS:-7200}" \ - "${exec_args[@]}" "${extra_args[@]}" "${kv_args[@]}" + "${exec_args[@]}" "${extra_args[@]}" "${kv_args[@]}" "${model_args[@]}" WORKER_PID=0; return 0 fi @@ -242,6 +325,7 @@ connector_launch_worker() { "${exec_args[@]}" \ "${extra_args[@]}" \ "${kv_args[@]}" \ + "${model_args[@]}" \ 2>&1 | tee /run_logs/${SLURM_JOB_ID}/${log_prefix}_NODE${NODE_RANK}.log >/dev/null & WORKER_PID=$! return 0 diff --git a/scripts/vllm_dissag/keepalive_bench.sh b/scripts/vllm_dissag/keepalive_bench.sh new file mode 100755 index 00000000..68d80dcb --- /dev/null +++ b/scripts/vllm_dissag/keepalive_bench.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Keepalive with LIGHT heartbeat traffic: holds the disagg server up AND sends a +# tiny request every ~20s so prefill discovery/ping stays registered (idle sleep +# lets the prefill ZMQ ping die ~2min in). Runs KEEPALIVE_MINS (default 90). +: "${KEEPALIVE_MINS:=90}" +PORT="${BENCHMARK_PORT:-30000}" +MODEL="/mnt/m2m_nobackup/models_blog/GLM-5.1-FP8" +echo "[keepalive] light-traffic hold ${KEEPALIVE_MINS}min on :${PORT}" +_end=$(( $(date +%s) + KEEPALIVE_MINS*60 )) +i=0 +while [ "$(date +%s)" -lt "$_end" ]; do + curl -s -m 30 "http://127.0.0.1:${PORT}/v1/completions" -H "Content-Type: application/json" \ + -d "{\"model\":\"${MODEL}\",\"prompt\":\"hi\",\"max_tokens\":1,\"temperature\":0}" >/dev/null 2>&1 + i=$((i+1)) + [ $((i % 3)) -eq 0 ] && echo "[keepalive] heartbeat $i, $(( (_end-$(date +%s))/60 ))min left" + sleep 20 +done +echo "[keepalive] done" diff --git a/scripts/vllm_dissag/models.yaml b/scripts/vllm_dissag/models.yaml index 23d66059..9eec3e7d 100644 --- a/scripts/vllm_dissag/models.yaml +++ b/scripts/vllm_dissag/models.yaml @@ -160,10 +160,17 @@ _deepseek_recipe_env: &deepseek_recipe_env # above. No tp: blocks — TP is unsupported for these models. DeepSeek-V3: env: *deepseek_recipe_env + # NEWER-BASE ADAPTATION (isolated to this model): Shiksha's newer vLLM base added + # TritonMLAMetadataBuilder._reserve_attn_logits_workspace(), which pre-reserves the + # decode split-KV logits workspace at WORST CASE + # (max_num_seqs x q_heads x max_kv_splits(max_model_len) x lse_dim x fp32). With + # DSV3's defaults (max_num_seqs=256, max_model_len~163k) this reserves ~128 GiB and + # OOMs at KV init. Cap max-num-seqs + max-model-len to bound the workspace. This is a + # per-model dp: flag (model_args), fully isolated -- does NOT touch GLM/other recipes. prefill: - dp: "" + dp: "--max-num-seqs 64 --max-model-len 32768" decode: - dp: "" + dp: "--max-num-seqs 64 --max-model-len 32768" DeepSeek-V3-5layer: env: *deepseek_recipe_env @@ -178,3 +185,94 @@ DeepSeek-R1: dp: "" decode: dp: "" + +# ============================ MoE + DSA (wideEP only) ============================ + +# GLM-5.1-FP8 (zai-org/GLM-5.1-FP8, arch GlmMoeDsaForCausalLM): MLA + DeepSeek +# Sparse Attention (DSA). 78 layers (3 dense + 75 MoE), 256 routed experts top-8 +# + 1 shared, FP8 block 128. wideEP-only (see WIDE_EP_ONLY_MODELS in the slurm). +# +# Validated DOCKER_IMAGE_NAME (submit-time, not set here — #171 requires it explicit): +# rocmshared/pytorch-private:vllm-wideep_06_29_2026_Shiksha_dp16_2p2d_mori_v1.2.1_aiter_v0.1.16.post3_nightlybase_mori121 +# On this image the DSA patchers self-adapt: only the invalid-token kernel fix +# (#45324) applies; the MoRIIO dual-KV geometry + completion-gate fixes are NATIVE +# (moriio_layout.py per-layer geometry + engine sealed writes_expected), so those +# patchers cleanly no-op. On the older b10a9f7a image all 4 patchers apply. +# +# GLM differs from DeepSeek in 3 recipe-defining ways (both are MLA MoE, but GLM +# is DSA-sparse): +# - KV_BLOCK_SIZE=1 (DSA sparse indexer REQUIRES block-size 1; DS uses 16) +# - VLLM_ROCM_USE_AITER_MLA=1 (GLM MLA path ON via AITER sparse; DS sets 0) +# - prefill EAGER (NONE), decode PIECEWISE (prefill cudagraph capture deadlocks +# on this stack; decode PIECEWISE captures cleanly on DSA and is a ~3.7x ITL win +# (validated: ~69ms vs ~264ms eager). Global VLLM_CUDAGRAPH_MODE=NONE as the +# safe floor; per-role PREFILL=NONE / DECODE=PIECEWISE override it.) +# Note: block=1 + AITER_MLA=1 are ALREADY the moriio.sh connector defaults — GLM +# keeps them; DeepSeek is the one that overrides them off. Set here explicitly so +# the recipe is self-documenting and robust to connector default changes. +# +# DSA adds a 2nd KV cache per layer (indexer); the MoRIIO connector needs the GLM +# DSA patchers (kernel #45324 + dual-KV geometry + per-layer offset + completion +# gate) applied by connector_runtime_patch in connectors/moriio.sh (gated on this +# MODEL_NAME). Those are code patches, not flags — nothing to add here for them. +# +# dp_flags carry the GLM tool/reasoning parsers (AMD GLM recipe). They are applied +# to BOTH roles by compose() in vllm_disagg.sh and reach `vllm serve` via the +# connector's model_args. block-size / kv-cache-dtype / all2all / cudagraph come +# from the env: recipe above (the connector emits them), so the dp: blocks are +# empty like the DeepSeek family. +# +# LONG-CONTEXT CAVEAT (vLLM #40018): the ROCM_AITER_MLA_SPARSE prefill indexer +# corrupts output for prompts beyond ~16-18k tokens on this image (mori v1.2.1) — +# coherent + correct needle retrieval up to 14k, garbage (repetition collapse, +# unique-ratio ~0.1) at ~18.7k. This is an UPSTREAM kernel bug, not the MAD port. +# TESTED: pinning --max-model-len 32768 does NOT move the threshold (workspace is +# sized max_model_len*40 but the corruption onset is a fixed ~18k token count in +# the gather/logits kernel, not a buffer-scaling artifact). So no config knob +# helps; it needs the complete upstream prefill fix in a newer image. Left at the +# native max_model_len (do not cap — capping gives no accuracy benefit and only +# limits usable context). Serve prompts <~14k for correct output on this image. +GLM-5.1-FP8: + env: + VLLM_USE_V1: "1" + VLLM_ROCM_USE_AITER: "1" + VLLM_ROCM_USE_AITER_RMSNORM: "1" + VLLM_ROCM_USE_AITER_MLA: "1" + KV_BLOCK_SIZE: "1" + KV_CACHE_DTYPE: "fp8" + GPU_MEMORY_UTILIZATION: "0.80" + VLLM_CUDAGRAPH_MODE: "NONE" + PREFILL_CUDAGRAPH_MODE: "NONE" + DECODE_CUDAGRAPH_MODE: "PIECEWISE" + CUDAGRAPH_CAPTURE_SIZES: "1 2 4 8 16 32 64 128 256" + VLLM_ALL2ALL_BACKEND: "mori_high_throughput" + PREFILL_MORI_BACKEND: "mori_high_throughput" + DECODE_MORI_BACKEND: "mori_low_latency" + MORI_SHMEM_HEAP_SIZE: "17179869184" + # DSA sparse-indexer logits-buffer cap (crash fix). The indexer prefill computes an + # M*N fp32 logits buffer; split_indexer_prefill_chunks only sub-chunks the query dim + # when M*N*4 > this budget. Default 512MB lets an 8192-token prefill build a single + # 268MB buffer + launch the fp8_mqa_logits kernel at grid=(8192,), which HARD-FAULTS + # the worker on gfx942 (silent GPU fault -> DP group collapse -> 503 at >=8k prompts). + # Capping at 64MB forces M-dim sub-chunking (~2k tokens/chunk) so the buffer and the + # kernel launch stay bounded. Root cause: vllm/v1/attention/ops/triton_fp8_mqa_logits.py + # fp8_mqa_logits_gfx942; chunking logic: mla/indexer.py split_indexer_prefill_chunks. + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "64" + # NCCL heartbeat watchdog: at long context (>~8k) a DP rank's sparse-MLA/MoE + # all2all collective can exceed the default HeartbeatMonitor timeout -> + # ProcessGroupNCCL::HeartbeatMonitor::runLoop() declares the rank dead and + # tears down the whole process group -> prefill EngineCore crashes -> 503. + # (Confirmed root cause of the 8k+ prefill crash; #338 EP-landmine.) Disable + # the monitor-triggered teardown and extend timeouts so long-ctx collectives + # complete instead of being watchdog-killed. + TORCH_NCCL_ENABLE_MONITORING: "0" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "1800" + TORCH_NCCL_DUMP_ON_TIMEOUT: "0" + TORCH_NCCL_BLOCKING_WAIT: "0" + TORCH_NCCL_ASYNC_ERROR_HANDLING: "1" + NCCL_IB_TIMEOUT: "22" + dp_flags: "--tool-call-parser glm47 --reasoning-parser glm45 --enable-auto-tool-choice --chat-template-content-format string" + prefill: + dp: "" + decode: + dp: "" diff --git a/scripts/vllm_dissag/run_xPyD_models.slurm b/scripts/vllm_dissag/run_xPyD_models.slurm index c71fc7e8..98281f57 100755 --- a/scripts/vllm_dissag/run_xPyD_models.slurm +++ b/scripts/vllm_dissag/run_xPyD_models.slurm @@ -68,6 +68,22 @@ for f in "${REQUIRED_FILES[@]}"; do done echo "Running from: $(pwd)" +# ------------------------------------------------------------------------------ +# models.yaml env precedence: capture which recipe knobs the USER explicitly set +# at submit time. The driver (vllm_disagg.sh) uses this to let models.yaml `env:` +# OVERRIDE image-baked ENV defaults (e.g. a DeepSeek-tuned image bakes +# KV_BLOCK_SIZE=16 / VLLM_ROCM_USE_AITER_MLA=0, which would otherwise shadow a +# model's own recipe — GLM-5.1 DSA needs block=1 + AITER MLA on), while a genuine +# submit-time `-e VAR=...` still wins. Precedence: image-baked < models.yaml < submit -e. +# Captured HERE (before the slurm sets any defaults) so it reflects user intent only. +_RECIPE_ENV_KEYS="VLLM_USE_V1 VLLM_ROCM_USE_AITER VLLM_ROCM_USE_AITER_RMSNORM VLLM_ROCM_USE_AITER_MLA KV_BLOCK_SIZE KV_CACHE_DTYPE KV_CACHE_MEMORY_BYTES GPU_MEMORY_UTILIZATION VLLM_CUDAGRAPH_MODE PREFILL_CUDAGRAPH_MODE DECODE_CUDAGRAPH_MODE CUDAGRAPH_CAPTURE_SIZES VLLM_ALL2ALL_BACKEND PREFILL_MORI_BACKEND DECODE_MORI_BACKEND MORI_SHMEM_HEAP_SIZE" +MODELS_YAML_PROTECT="" +for _k in $_RECIPE_ENV_KEYS; do + [ -n "${!_k+x}" ] && MODELS_YAML_PROTECT="${MODELS_YAML_PROTECT} ${_k}" +done +export MODELS_YAML_PROTECT="${MODELS_YAML_PROTECT# }" +echo "models.yaml protect-list (submit-time overrides): '${MODELS_YAML_PROTECT}'" + # ------------------------ # Print current time in UTC and PST formats # ------------------------ @@ -87,6 +103,7 @@ VALID_MODELS=( \ "DeepSeek-R1" \ "Qwen3-32B" \ "Qwen3-30B-A3B" \ + "GLM-5.1-FP8" \ ) # Models allowed for CONNECTOR=moriio WIDE_EP=1 (MoRI-EP; legacy RUN_MORI=1) @@ -94,6 +111,7 @@ MORI_EP_VALID_MODELS=( \ "DeepSeek-V3" \ "DeepSeek-V3-5layer" \ "DeepSeek-R1" \ + "GLM-5.1-FP8" \ ) # Models allowed for CONNECTOR=rixl WIDE_EP=1 EP_BACKEND=deepep (legacy RUN_DEEPEP=1) @@ -179,7 +197,10 @@ WIDE_EP="${WIDE_EP:-0}" # the MoRI-EP / DeepEP recipe (block=16, MLA off, per-role cudagraph). Running them # in TP mode is unsupported — the TP argv would double the model's own # --compilation-config and drop the mandatory +quant_fp8 op. Reject early. -WIDE_EP_ONLY_MODELS=( "DeepSeek-V3" "DeepSeek-V3-5layer" "DeepSeek-R1" ) +# GLM-5.1-FP8 (GlmMoeDsaForCausalLM, MLA+DSA) is validated only under MoRI-EP +# wideEP disagg (block=1, AITER sparse MLA on, per-role all2all). The moriio+TP +# ("Stage B") path is untested for DSA, so reject WIDE_EP=0 for it too. +WIDE_EP_ONLY_MODELS=( "DeepSeek-V3" "DeepSeek-V3-5layer" "DeepSeek-R1" "GLM-5.1-FP8" ) model_is_wide_ep_only() { local m="$1" for x in "${WIDE_EP_ONLY_MODELS[@]}"; do [[ "$m" == "$x" ]] && return 0; done @@ -427,11 +448,14 @@ BENCHMARK_COMBINATIONS="${BENCHMARK_COMBINATIONS:-}" # Benchmark script selector: BENCHMARK_SCRIPT tag -> file run by the launcher. # sweep (default) -> benchmark_xPyD.sh (general concurrency sweep) # long_context -> benchmark_long_context.sh (per-shape warmup, c=1-first) +# keepalive -> keepalive_bench.sh (hold server up KEEPALIVE_MINS +# for external accuracy probes) BENCHMARK_SCRIPT="${BENCHMARK_SCRIPT:-sweep}" case "$BENCHMARK_SCRIPT" in sweep) BENCHMARK_SCRIPT_FILE="benchmark_xPyD.sh" ;; long_context) BENCHMARK_SCRIPT_FILE="benchmark_long_context.sh" ;; - *) echo "Error: invalid BENCHMARK_SCRIPT='$BENCHMARK_SCRIPT' (valid: sweep, long_context)" >&2; exit 1 ;; + keepalive) BENCHMARK_SCRIPT_FILE="keepalive_bench.sh" ;; + *) echo "Error: invalid BENCHMARK_SCRIPT='$BENCHMARK_SCRIPT' (valid: sweep, long_context, keepalive)" >&2; exit 1 ;; esac if [[ ! -f "$BENCHMARK_SCRIPT_FILE" ]]; then echo "Error: selected benchmark script '$BENCHMARK_SCRIPT_FILE' not found in $(pwd)." >&2 @@ -583,6 +607,9 @@ docker run --rm \ ${VLLM_ALL2ALL_BACKEND:+-e VLLM_ALL2ALL_BACKEND=$VLLM_ALL2ALL_BACKEND} \ ${PREFILL_MORI_BACKEND:+-e PREFILL_MORI_BACKEND=$PREFILL_MORI_BACKEND} \ ${DECODE_MORI_BACKEND:+-e DECODE_MORI_BACKEND=$DECODE_MORI_BACKEND} \ + -e MODELS_YAML_PROTECT="${MODELS_YAML_PROTECT:-}" \ + ${GLM_PERSIST_GATE:+-e GLM_PERSIST_GATE=$GLM_PERSIST_GATE} \ + ${GLM_SKIP_PATCHERS:+-e GLM_SKIP_PATCHERS=$GLM_SKIP_PATCHERS} \ ${KV_BLOCK_SIZE:+-e KV_BLOCK_SIZE=$KV_BLOCK_SIZE} \ ${KV_CACHE_MEMORY_BYTES:+-e KV_CACHE_MEMORY_BYTES=$KV_CACHE_MEMORY_BYTES} \ ${VLLM_ROCM_USE_AITER_MLA:+-e VLLM_ROCM_USE_AITER_MLA=$VLLM_ROCM_USE_AITER_MLA} \ @@ -590,6 +617,7 @@ docker run --rm \ ${KV_CACHE_DTYPE:+-e KV_CACHE_DTYPE=$KV_CACHE_DTYPE} \ ${MORIIO_TOY_PROXY:+-e MORIIO_TOY_PROXY=$MORIIO_TOY_PROXY} \ ${BENCHMARK_SCRIPT_FILE:+-e BENCHMARK_SCRIPT_FILE=$BENCHMARK_SCRIPT_FILE} \ + ${KEEPALIVE_MINS:+-e KEEPALIVE_MINS=$KEEPALIVE_MINS} \ ${PREFILL_CUDAGRAPH_MODE:+-e PREFILL_CUDAGRAPH_MODE=$PREFILL_CUDAGRAPH_MODE} \ ${DECODE_CUDAGRAPH_MODE:+-e DECODE_CUDAGRAPH_MODE=$DECODE_CUDAGRAPH_MODE} \ ${CUDAGRAPH_CAPTURE_SIZES:+-e CUDAGRAPH_CAPTURE_SIZES="$CUDAGRAPH_CAPTURE_SIZES"} \ diff --git a/scripts/vllm_dissag/vllm_disagg.sh b/scripts/vllm_dissag/vllm_disagg.sh index 06fbf84f..acbabbd4 100755 --- a/scripts/vllm_dissag/vllm_disagg.sh +++ b/scripts/vllm_dissag/vllm_disagg.sh @@ -160,17 +160,33 @@ MODEL_CONFIG_DECODE="" if [[ -n "$MODEL_NAME" && -f "$MODELS_YAML" ]]; then export MODELS_YAML MODEL_NAME PARALLEL_MODE # 1) Export per-model env: block FIRST (so connector ${VAR:-default} yields to it). - # Only set a var that is NOT already in the environment, so a submit-time - # `docker -e VAR=...` (already exported) WINS over the yaml value. Precedence: - # connector default < models.yaml env: < submit-time -e. + # Precedence: image-baked ENV < models.yaml env: < submit-time -e. + # models.yaml MUST override image-baked ENV: a DeepSeek-tuned disagg image + # bakes KV_BLOCK_SIZE=16 / VLLM_ROCM_USE_AITER_MLA=0 / VLLM_CUDAGRAPH_MODE= + # PIECEWISE etc. as container ENV, which would otherwise shadow a model's own + # recipe (GLM-5.1 DSA needs block=1 + AITER sparse MLA on). But a genuine + # submit-time `-e VAR=...` must still win. The slurm can tell the two apart + # (it runs on the host) and passes MODELS_YAML_PROTECT = the space-separated + # list of keys the USER set at submit; the driver protects only those. When + # MODELS_YAML_PROTECT is unset (script run directly, no slurm), fall back to + # the old "skip if in env" behavior so nothing regresses. _yaml_env="$(python3 - <<'PY' import os, yaml, shlex m = yaml.safe_load(open(os.environ["MODELS_YAML"])) or {} cfg = m.get(os.environ["MODEL_NAME"]) or {} +protect_raw = os.environ.get("MODELS_YAML_PROTECT") +have_protect = protect_raw is not None +protect = set((protect_raw or "").split()) for k, v in (cfg.get("env") or {}).items(): - # skip if already present in the environment (submit-time -e override wins) - if k in os.environ: - continue + if have_protect: + # 3-tier: yaml overrides baked ENV; only a user submit-time -e (in the + # protect-list) wins over yaml. + if k in protect: + continue + else: + # No protect-list (direct run): legacy behavior — any existing env wins. + if k in os.environ: + continue print(f'export {k}={shlex.quote(str(v))}') PY )" From ed4b16f2586f67d053a323eb306bd66336373cfd Mon Sep 17 00:00:00 2001 From: raviguptaamd Date: Wed, 8 Jul 2026 15:51:15 +0000 Subject: [PATCH 02/23] vllm_dissag: make NIAH harness thinking-model-aware (GLM-5.1) benchmark_niah.py mis-scored thinking models: it never disabled thinking and read only content + reasoning_content. GLM-5.1 emits chain-of-thought into the `reasoning` field and leaves `content` empty until the final answer, so with a small max_tokens the answer never lands in content -> a false 0/10 even when generation is correct. - Add chat_template_kwargs.enable_thinking=false so the answer goes to content. - Also score the `reasoning` field as a fallback. Verified against GLM-5.1-FP8: correct 9-10/10 retrieval across 2k-35k on all tested topologies (EP8/EP16/EP32) after the fix. Co-Authored-By: Claude --- scripts/vllm_dissag/benchmark_niah.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/vllm_dissag/benchmark_niah.py b/scripts/vllm_dissag/benchmark_niah.py index 0cdd027e..b1b0264d 100755 --- a/scripts/vllm_dissag/benchmark_niah.py +++ b/scripts/vllm_dissag/benchmark_niah.py @@ -50,6 +50,11 @@ def run(n_words): ], "temperature": 0.0, "max_tokens": MAXTOK, + # Thinking models (e.g. GLM-5.1) emit chain-of-thought into a separate + # reasoning field and leave `content` empty until the final answer; with a + # small max_tokens the answer never appears in `content` and the score is a + # false 0/10. Disable thinking so the answer lands in `content` directly. + "chat_template_kwargs": {"enable_thinking": False}, } data = json.dumps(body).encode() req = urllib.request.Request(URL, data=data, headers={"Content-Type": "application/json"}) @@ -59,7 +64,11 @@ def run(n_words): except Exception as e: print("words=%6d ERROR %s" % (n_words, e), flush=True) return None - text = ((msg.get("content") or "") + " " + (msg.get("reasoning_content") or "")).lower() + # Score content plus any reasoning field (some servers surface CoT as + # `reasoning` or `reasoning_content`) so a thinking model is never mis-scored. + text = ((msg.get("content") or "") + " " + + (msg.get("reasoning_content") or "") + " " + + (msg.get("reasoning") or "")).lower() found = sorted(a for a in ANIMALS if a in text) print("words=%6d found=%2d/10 %s" % (n_words, len(found), found), flush=True) return len(found) From 059d93335257f03d994bc36aca631099733d5122 Mon Sep 17 00:00:00 2001 From: raviguptaamd Date: Wed, 8 Jul 2026 16:16:52 +0000 Subject: [PATCH 03/23] vllm_dissag: NIAH multi-seed support (NIAH_SEEDS) for variance-aware accuracy Needle layout is seeded, so a single run is deterministic (bit-exact on the same stack) but can't tell a real accuracy dip from single-needle variance. Add NIAH_SEEDS (default 0,1,2) to run each context length across multiple needle layouts; the summary now reports mean/min/max across seeds. Backward compatible: NIAH_SEEDS=0 reproduces the prior single-seed behavior. Co-Authored-By: Claude --- scripts/vllm_dissag/benchmark_niah.py | 29 ++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/scripts/vllm_dissag/benchmark_niah.py b/scripts/vllm_dissag/benchmark_niah.py index b1b0264d..82a55663 100755 --- a/scripts/vllm_dissag/benchmark_niah.py +++ b/scripts/vllm_dissag/benchmark_niah.py @@ -8,6 +8,8 @@ # NIAH_MODEL model name/tag the server serves (required — the served path) # NIAH_WORDS comma list of context sizes in words (default 2000,8000,20000,35000) # NIAH_MAXTOK max_tokens for the answer (default 2048) +# NIAH_SEEDS comma list of needle-layout seeds (default 0,1,2); summary reports +# mean/min/max across seeds to separate real accuracy from variance # NIAH_TIMEOUT per-request timeout seconds (default 1800) import os, sys, json, random, urllib.request @@ -16,6 +18,10 @@ WORDS = [int(x) for x in os.environ.get("NIAH_WORDS", "2000,8000,20000,35000").split(",") if x.strip()] MAXTOK = int(os.environ.get("NIAH_MAXTOK", "2048")) TIMEOUT = float(os.environ.get("NIAH_TIMEOUT", "1800")) +# Needle layout is seeded, so a single run is deterministic (bit-exact repro on the +# same stack). Run multiple seeds to distinguish real accuracy from single-needle +# variance; the summary reports mean/min/max across seeds. Default 0,1,2. +SEEDS = [int(x) for x in os.environ.get("NIAH_SEEDS", "0,1,2").split(",") if x.strip()] FILLER = ( "table chair window bottle pencil garden river mountain coffee planet " @@ -41,12 +47,12 @@ def make_haystack(n_words, seed=0): return " ".join(words) -def run(n_words): +def run(n_words, seed=0): body = { "model": MODEL, "messages": [ {"role": "system", "content": SYSTEM}, - {"role": "user", "content": "Find the animals in this list:\n\n" + make_haystack(n_words)}, + {"role": "user", "content": "Find the animals in this list:\n\n" + make_haystack(n_words, seed)}, ], "temperature": 0.0, "max_tokens": MAXTOK, @@ -70,7 +76,7 @@ def run(n_words): + (msg.get("reasoning_content") or "") + " " + (msg.get("reasoning") or "")).lower() found = sorted(a for a in ANIMALS if a in text) - print("words=%6d found=%2d/10 %s" % (n_words, len(found), found), flush=True) + print("words=%6d seed=%d found=%2d/10 %s" % (n_words, seed, len(found), found), flush=True) return len(found) @@ -79,14 +85,19 @@ def main(): print("NIAH_MODEL must be set (the served model path/name)", file=sys.stderr) sys.exit(2) print("=== NIAH retrieval test ===", flush=True) - print("url=%s model=%s sizes=%s" % (URL, MODEL, WORDS), flush=True) - results = {} + print("url=%s model=%s sizes=%s seeds=%s" % (URL, MODEL, WORDS, SEEDS), flush=True) + results = {} # n_words -> list of scores across seeds (None on error) for n in WORDS: - results[n] = run(n) - print("=== NIAH summary ===", flush=True) + results[n] = [run(n, s) for s in SEEDS] + print("=== NIAH summary (mean/min/max across %d seed(s)) ===" % len(SEEDS), flush=True) for n in WORDS: - v = results[n] - print(" words=%6d found=%s/10" % (n, "ERR" if v is None else v), flush=True) + vals = [v for v in results[n] if v is not None] + if not vals: + print(" words=%6d ERR" % n, flush=True) + continue + mean = sum(vals) / len(vals) + print(" words=%6d mean=%.1f/10 min=%d max=%d (n=%d)" + % (n, mean, min(vals), max(vals), len(vals)), flush=True) if __name__ == "__main__": From a5ea09ddca93141c45912a396a40c0fd7b2e328a Mon Sep 17 00:00:00 2001 From: raviguptaamd Date: Fri, 10 Jul 2026 07:09:21 +0000 Subject: [PATCH 04/23] vllm_dissag: NIAH gate robust to cold-start JIT (warmup + readiness probe) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a freshly-booted node the first request of each context shape pays the full JIT/kernel-autotune compile (minutes). The NIAH harness scored the FIRST request, so cold compile landed on a scored/gated request -> false 0/10 or timeout, failing the accuracy gate and skipping the perf sweep. Root-caused by reproducing on a cold boot (0 results) vs a warm server (all pass) on the same image. Fixes: - benchmark_niah.py: add a warmup pass (NIAH_WARMUP=1 default) — one throwaway request per context length before scoring, with a generous timeout, failures tolerated. Scored requests are then always warm. - benchmark_niah.py: distinguish TIMEOUT/ERROR from a wrong answer. Timeouts return a sentinel (excluded from mean, never counted as 0/10); summary flags NO-RESULT with guidance instead of silently reporting 0. - benchmark_niah.sh: replace the blind `sleep 10` with a /v1/models readiness poll (up to 5 min), and forward NIAH_WARMUP. Verified: patched harness on the warm server passes 10/10; cold-boot repro no longer produces false 0/10 because compile happens in the warmup pass. Co-Authored-By: Claude --- scripts/vllm_dissag/benchmark_niah.py | 59 ++++++++++++++++++++++----- scripts/vllm_dissag/benchmark_niah.sh | 18 +++++++- 2 files changed, 64 insertions(+), 13 deletions(-) diff --git a/scripts/vllm_dissag/benchmark_niah.py b/scripts/vllm_dissag/benchmark_niah.py index 82a55663..43d81850 100755 --- a/scripts/vllm_dissag/benchmark_niah.py +++ b/scripts/vllm_dissag/benchmark_niah.py @@ -11,6 +11,12 @@ # NIAH_SEEDS comma list of needle-layout seeds (default 0,1,2); summary reports # mean/min/max across seeds to separate real accuracy from variance # NIAH_TIMEOUT per-request timeout seconds (default 1800) +# NIAH_WARMUP 1 (default) = send one throwaway request per context length BEFORE +# scoring, so the first-hit JIT/kernel-autotune compile happens outside +# the scored/gated window. On a freshly-booted node the first request of +# a shape can take minutes to compile; without warmup that lands on the +# first scored request -> false 0/10 or timeout. Warmup failures are +# tolerated (logged, not fatal). Set 0 to disable. import os, sys, json, random, urllib.request URL = os.environ.get("NIAH_URL", "http://127.0.0.1:30000/v1/chat/completions") @@ -22,6 +28,10 @@ # same stack). Run multiple seeds to distinguish real accuracy from single-needle # variance; the summary reports mean/min/max across seeds. Default 0,1,2. SEEDS = [int(x) for x in os.environ.get("NIAH_SEEDS", "0,1,2").split(",") if x.strip()] +WARMUP = os.environ.get("NIAH_WARMUP", "1") == "1" +# Warmup uses a generous timeout (cold compile of a long-context shape can take minutes) +# and never fails the run — its only job is to trigger compilation before scoring. +WARMUP_TIMEOUT = max(TIMEOUT, 1800.0) FILLER = ( "table chair window bottle pencil garden river mountain coffee planet " @@ -47,7 +57,8 @@ def make_haystack(n_words, seed=0): return " ".join(words) -def run(n_words, seed=0): +def _request(n_words, seed, max_tokens, timeout): + """POST one NIAH request; return (message_dict, error_str). Exactly one is non-None.""" body = { "model": MODEL, "messages": [ @@ -55,7 +66,7 @@ def run(n_words, seed=0): {"role": "user", "content": "Find the animals in this list:\n\n" + make_haystack(n_words, seed)}, ], "temperature": 0.0, - "max_tokens": MAXTOK, + "max_tokens": max_tokens, # Thinking models (e.g. GLM-5.1) emit chain-of-thought into a separate # reasoning field and leave `content` empty until the final answer; with a # small max_tokens the answer never appears in `content` and the score is a @@ -65,10 +76,26 @@ def run(n_words, seed=0): data = json.dumps(body).encode() req = urllib.request.Request(URL, data=data, headers={"Content-Type": "application/json"}) try: - with urllib.request.urlopen(req, timeout=TIMEOUT) as r: - msg = json.loads(r.read())["choices"][0]["message"] + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.loads(r.read())["choices"][0]["message"], None except Exception as e: - print("words=%6d ERROR %s" % (n_words, e), flush=True) + return None, str(e) + + +def warmup(n_words): + """One throwaway request per length so first-hit compile happens off the scored path. + Never fatal: a warmup timeout just means the shape is still compiling; the scored + request will pay whatever remains (bounded by NIAH_TIMEOUT).""" + _, err = _request(n_words, seed=0, max_tokens=8, timeout=WARMUP_TIMEOUT) + status = "ok" if err is None else ("timeout/err: %s" % err) + print("words=%6d [warmup] %s" % (n_words, status), flush=True) + + +def run(n_words, seed=0): + # Sentinel: None = timeout/transport error (NOT a wrong answer); int = score 0..10. + msg, err = _request(n_words, seed, MAXTOK, TIMEOUT) + if err is not None: + print("words=%6d seed=%d TIMEOUT/ERROR %s" % (n_words, seed, err), flush=True) return None # Score content plus any reasoning field (some servers surface CoT as # `reasoning` or `reasoning_content`) so a thinking model is never mis-scored. @@ -85,19 +112,29 @@ def main(): print("NIAH_MODEL must be set (the served model path/name)", file=sys.stderr) sys.exit(2) print("=== NIAH retrieval test ===", flush=True) - print("url=%s model=%s sizes=%s seeds=%s" % (URL, MODEL, WORDS, SEEDS), flush=True) - results = {} # n_words -> list of scores across seeds (None on error) + print("url=%s model=%s sizes=%s seeds=%s warmup=%s" % (URL, MODEL, WORDS, SEEDS, WARMUP), flush=True) + # Warmup pass: compile every shape once before scoring, so cold JIT never lands on a + # scored/gated request (the common cause of false 0/10 or timeout on a fresh boot). + if WARMUP: + print("=== NIAH warmup (one throwaway request per length) ===", flush=True) + for n in WORDS: + warmup(n) + results = {} # n_words -> list of scores across seeds (None = timeout/error, not a wrong answer) for n in WORDS: results[n] = [run(n, s) for s in SEEDS] print("=== NIAH summary (mean/min/max across %d seed(s)) ===" % len(SEEDS), flush=True) for n in WORDS: - vals = [v for v in results[n] if v is not None] + scored = results[n] + vals = [v for v in scored if v is not None] + n_to = sum(1 for v in scored if v is None) # timeouts/errors, excluded from mean if not vals: - print(" words=%6d ERR" % n, flush=True) + print(" words=%6d NO-RESULT (%d/%d timed out or errored — likely cold compile; " + "raise NIAH_TIMEOUT or keep NIAH_WARMUP=1)" % (n, n_to, len(scored)), flush=True) continue mean = sum(vals) / len(vals) - print(" words=%6d mean=%.1f/10 min=%d max=%d (n=%d)" - % (n, mean, min(vals), max(vals), len(vals)), flush=True) + extra = (" [%d timeout/err excluded]" % n_to) if n_to else "" + print(" words=%6d mean=%.1f/10 min=%d max=%d (n=%d)%s" + % (n, mean, min(vals), max(vals), len(vals), extra), flush=True) if __name__ == "__main__": diff --git a/scripts/vllm_dissag/benchmark_niah.sh b/scripts/vllm_dissag/benchmark_niah.sh index ba49a359..d366b5e4 100755 --- a/scripts/vllm_dissag/benchmark_niah.sh +++ b/scripts/vllm_dissag/benchmark_niah.sh @@ -14,15 +14,29 @@ LOG="/run_logs/${SLURM_JOB_ID}/niah_${SLURM_JOB_ID}_${timestamp}_xP${xP}_yD${yD} echo "==== NIAH long-context retrieval test ====" echo "port=${BENCHMARK_PORT} model=${MODEL_PATH} sizes=${NIAH_WORDS:-2000,8000,20000,35000}" -# Give the router a moment to be fully ready for chat completions. -sleep 10 +# Wait until the router actually serves before starting (replaces a blind sleep). On a +# fresh boot the router may register a few seconds after the workers report ready; poll +# /v1/models until it answers, up to ~5 min. Non-fatal: fall through if the probe can't +# confirm (the harness's own warmup + timeout still protect the run). +_ready=0 +for _i in $(seq 1 60); do + if curl -s -o /dev/null -w '%{http_code}' --max-time 5 \ + "http://127.0.0.1:${BENCHMARK_PORT}/v1/models" 2>/dev/null | grep -q '^200$'; then + _ready=1; echo "[niah] router ready after ~$((_i*5))s"; break + fi + sleep 5 +done +[ "$_ready" = 1 ] || echo "[niah] WARN: router readiness not confirmed in 300s; proceeding (warmup will absorb)" # The server registers the model under its path (served_model_name = MODEL_PATH). +# NIAH_WARMUP=1 (harness default): first-hit JIT compiles off the scored path so a cold +# boot does not produce false 0/10 or timeouts on the first scored request. NIAH_URL="http://127.0.0.1:${BENCHMARK_PORT}/v1/chat/completions" \ NIAH_MODEL="${MODEL_PATH}" \ NIAH_WORDS="${NIAH_WORDS:-2000,8000,20000,35000}" \ NIAH_MAXTOK="${NIAH_MAXTOK:-2048}" \ NIAH_TIMEOUT="${NIAH_TIMEOUT:-1800}" \ +NIAH_WARMUP="${NIAH_WARMUP:-1}" \ python3 "${DIR}/benchmark_niah.py" 2>&1 | tee -a "${LOG}" echo "NIAH results -> ${LOG}" From e0a4b8b6f8a68ad8fd30bdcac58a9c99cc3eb0ba Mon Sep 17 00:00:00 2001 From: raviguptaamd Date: Sun, 16 Aug 2026 04:17:14 +0000 Subject: [PATCH 05/23] [GLM-5.1] v0.27 level-set: 3.4x decode speedup, 200K context, RDMA + launcher fixes Validated on MI300X, 8 nodes, image rocmshared/pytorch-private:glm5.1-vllm027-b8 (base ci_base-dedbf6be8b + vLLM raviguptaamd/vllm@glm5.1-dsa-wideEP_on_vllm-v0.27 + aiter e03fa6040 + MoRI 42e895472b08 + router #181). PERF FIX (models.yaml decode.dp) -- the headline change: --max-num-batched-tokens 2048 on the DECODE role only. max_num_batched_tokens is a chunked-prefill SCHEDULER knob, but it also sizes the MoRI EP dispatch buffer (fused_moe/layer.py -> all2all_utils.py -> all2all.py max_num_inp_token_per_rank). At the 8192 default a decode instance ran an 8192-token-wide all2all every step, per layer, x78 layers, while decoding a handful of tokens: a fixed ~302ms/step floor, ~320x this model's HBM-bandwidth bound. Prefill keeps 8192 (it genuinely dispatches wide batches). 1024/64 con=8, warm: TPOT TTFT out tok/s 1P/1D 302 -> 88.0 ms 2431 -> 906 ms 24.9 -> 78.8 2P/2D 302 -> 94.1 ms 1633 ms 66.7 Published reference: 1P/1D ~89ms, 2P/2D ~91ms -> matched within 3%. Accuracy unaffected: NIAH 2k-200k clean on both topologies, no length collapse, memfault=0, latencies equal-or-better at every length. 200K validated (5.7x beyond the previously published 35K ceiling). Dockerfile: base -> ci_base-dedbf6be8b (matches the fork's upstream base), VLLM_REF -> the v0.27 branch, and WITH_MORI_BUILD/WITH_AITER_BUILD now default to 1 so a plain `docker build` reproduces the validated stack. Previously they defaulted to 0, which silently used the base's bundled aiter 0.1.19 -- that GPU-faults on the GLM DSA decode kernel. The pinned aiter e03fa6040 / MoRI 42e895472b08 must not be bumped without re-running long-context NIAH. connectors/moriio.sh: per-role env split (PREFILL_*/DECODE_* -> VLLM_MORI_*), mirroring the existing PREFILL/DECODE_MORI_BACKEND pattern -- models.yaml env: applies to BOTH roles, but prefill and decode need opposite values here. Also injects use_inductor_graph_partition (pairs with the vLLM splitting_ops fix). connectors/moriio.env: RDMA fabric -- MORI_IB_GID_INDEX=3 (RoCEv2 IPv4), MORI_RDMA_DEVICES/NCCL_IB_HCA restricted to the 8 GPU-local NICs (leaving the mgmt NICs in makes QPs form over a non-routable fabric -> ibverbs.cpp:189 timeouts), NCCL/GLOO control sockets on eth0. run_xPyD_models.slurm: libionic bind-mount requires a regular file after symlink resolve (a dangling symlink gave "OCI runtime create ... not a directory", container exit 125); prefer FABRIC_SUBNET over `hostname -I` first IP (nodes list a 10.224 overlay first, which made the socket_barrier advertise an unreachable NIC -> "Waiting for nodes" hang); GLM_KERNEL_PATCH/GLM_BACKEND_PATCH bind-mount hooks to test .py fixes without a rebuild; forward the new per-role env keys. vllm_disagg.sh: same FABRIC_SUBNET IP-selection fix for host_ip. benchmark_xPyD.sh: per-shape warmup at the REAL isl/osl before each shape's cells. The global warmup is isl=osl=32/con=1, which never exercises a 1024/8192/28672 prefill path or the decode cudagraph batch sizes, so the first measured cell absorbed residual JIT (observed 302ms vs ~88ms steady-state). Warmup output goes to a separate _SHAPEWARMUP.log so it cannot pollute the CSV. models.yaml (GLM-5.1-FP8): decode.dp perf fix above; recipe = prefill eager + mori_high_throughput, decode PIECEWISE cudagraph + mori_low_latency; VLLM_USE_LAYERNAME=0; VLLM_SPARSE_INDEXER_MAX_LOGITS_MB=64; NCCL heartbeat/timeout knobs for long-context collectives. Full operational playbook (including the dead ends) in skills_vllm_disagg.md. Co-Authored-By: Claude --- ...gg_inference.glmv5.1.ubuntu.amd.Dockerfile | 130 ++++++++++-------- scripts/vllm_dissag/benchmark_xPyD.sh | 26 ++++ scripts/vllm_dissag/connectors/moriio.env | 13 +- scripts/vllm_dissag/connectors/moriio.sh | 32 ++++- scripts/vllm_dissag/models.yaml | 34 ++++- scripts/vllm_dissag/run_xPyD_models.slurm | 26 +++- scripts/vllm_dissag/vllm_disagg.sh | 7 +- 7 files changed, 198 insertions(+), 70 deletions(-) diff --git a/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile b/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile index f61a8819..8da85d77 100644 --- a/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile +++ b/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile @@ -53,18 +53,19 @@ # the moriep all-to-all combine at EP32 scale -> deferred to future work. Use 1P/1D # and 2P/2D only. (BASE_IMAGE is a gated nightly; override --build-arg BASE_IMAGE=...) # ============================================================================= -# Reconstructs the validated v1.2.1 (mori121) runtime stack by applying the recipe's -# component pins ON TOP of the open ROCm vLLM ci_base, cloning each source from -# public Git (no local build-contexts). Mirrors dist-inf-cookbook -# Dockerfile.vllm.mori121_shareable: +# Builds the GLM-5.1 runtime stack by applying component pins ON TOP of a +# purpose-built ROCm/vLLM/MoRI base, cloning each overridden source from public Git +# (no local build-contexts): # -# - BASE: rocm/vllm-dev:ci_base-0fcd9b99... (open ROCm 7.2 / cp312 CI base). -# - MoRI -> built from ROCm/MoRI @ v1.2.1 (BUILD_UMBP=OFF). +# - BASE: rocmshared/pytorch-private:vllm-rocm_07_22_2026_shikpate_mori1.2.3 +# (ROCm + torch + a bundled vLLM/MoRI 1.2.3 stack). The stages below deliberately +# OVERRIDE the base's vLLM/MoRI/AITER with the pins we validate for GLM DSA. +# - MoRI -> built from ROCm/MoRI @ 42e895472b08 (validated for GLM DSA, BUILD_UMBP=OFF). +# (main LATEST 120d2de broke the connector KV-notify handshake -- see note at MORI_REF.) # - AITER -> STOCK ROCm/aiter @ e03fa6040 compiled from source + flydsl 0.1.7-0.1.9; # stale JIT wiped. (#47766 keeps persistent MLA ON -> aiter native gqa64 fold.) -# - vLLM -> COMPILED from shikamd123/vllm @ -# vllm_2p2d_wide-ep_write_shikpate_test_06_29_customer (Wide-EP multi-pod PD, the -# connector/router reference for the 2P2D DP=EP=16 topology). Full compile: it is +# - vLLM -> COMPILED from raviguptaamd/vllm @ glm5.1-dsa-wideEP_on_shik_0721 +# (Shiksha 7/21 WideEP base + GLM DSA edits + sparse-MLA guard fix). Full compile: # a different commit than the base's, so a .py-only overlay would be ABI-mismatched. # - RDMA fix (expandable_segments:False x2 + HSA_ENABLE_IPC_MODE_LEGACY=0) is NOT baked # here — it lives in scripts/vllm_dissag/connectors/.env and the launcher @@ -78,12 +79,11 @@ # Build context = repo root: # docker build -f docker/vllm_disagg_inference.ubuntu.amd.Dockerfile -t / . # -# BASE_IMAGE is the open rocm/vllm-dev ci_base pinned by the validated recipe -# (dist-inf-cookbook Dockerfile.vllm.mori121_shareable). Override --build-arg +# BASE_IMAGE is the purpose-built ROCm/vLLM/MoRI base above. Override --build-arg # BASE_IMAGE=... to build on a different ROCm base. vLLM compile is long (~30-60 min). # ============================================================================= -ARG BASE_IMAGE=rocm/vllm-dev:ci_base-0fcd9b99cc9d63202da4c858d8ebc6582c9e2491 +ARG BASE_IMAGE=rocm/vllm-dev:ci_base-dedbf6be8b1afa17a6220473b9c8c98242ac1c03 FROM ${BASE_IMAGE} ENTRYPOINT [] @@ -109,8 +109,12 @@ ARG NIC_COMPILATION_ARCH="cx7" # backends produced a MoRI that deadlocked at the cross-node EP all-to-all init. # ----------------------------------------------------------------------------- ARG MORI_REPO=https://github.com/ROCm/mori.git -# 42e895472b08: MoRI main tip past v1.2.1, validated by MAD-private #338 for GLM-5.1 -# DSA WideEP disagg (v1.2.1 large-transfer notify path was insufficient at high EP). +# 42e895472b08: validated MoRI tip for GLM DSA WideEP disagg. The v0.27 base bundles +# amd_mori 1.0.0, but the bundled build regressed GLM DSA (b1: GPU fault on the aiter +# DSA decode kernel), so we build MoRI from source at this pinned commit by DEFAULT +# (WITH_MORI_BUILD=1). Set --build-arg WITH_MORI_BUILD=0 only to fall back to the +# base's bundled mori for debugging. +ARG WITH_MORI_BUILD=1 ARG MORI_REF=42e895472b08 ENV MORI_GPU_ARCHS=gfx942 # Newer MoRI added the UMBP subsystem which requires gRPC (grpcpp/grpcpp.h) not @@ -125,50 +129,49 @@ RUN sed -i 's|http://|https://|g' /etc/apt/sources.list 2>/dev/null || true && \ apt-get update && apt-get install -y --no-install-recommends \ git build-essential cmake ninja-build ccache libssl-dev pkg-config curl ca-certificates && \ pip install meson==0.64.0 "pybind11[global]" tqdm prettytable && \ - pip uninstall -y amd_mori amd-mori amd-mori-nightly mori 2>/dev/null || true && \ - rm -rf /tmp/mori-src && \ - git clone --recursive "${MORI_REPO}" /tmp/mori-src && \ - cd /tmp/mori-src && git checkout "${MORI_REF}" && git submodule update --init --recursive && \ - BUILD_UMBP=OFF pip install . && \ - python3 -c "import mori, mori.io, mori.ops; print('MoRI OK at', mori.__path__[0])" && \ - mkdir -p /app && echo "MORI_REF=${MORI_REF}@$(git -C /tmp/mori-src rev-parse HEAD)" >> /app/versions.txt && \ - rm -rf /tmp/mori-src + mkdir -p /app && \ + if [ "${WITH_MORI_BUILD}" != "1" ]; then \ + python3 -c "import mori, mori.io, mori.ops; print('MoRI (bundled) OK at', mori.__path__[0])" && \ + echo "MORI_REF=BUNDLED (base amd_mori, WITH_MORI_BUILD=0)" >> /app/versions.txt ; \ + else \ + pip uninstall -y amd_mori amd-mori amd-mori-nightly mori 2>/dev/null || true && \ + rm -rf /tmp/mori-src && \ + git clone --recursive "${MORI_REPO}" /tmp/mori-src && \ + cd /tmp/mori-src && git checkout "${MORI_REF}" && git submodule update --init --recursive && \ + BUILD_UMBP=OFF pip install . && \ + python3 -c "import mori, mori.io, mori.ops; print('MoRI OK at', mori.__path__[0])" && \ + echo "MORI_REF=${MORI_REF}@$(git -C /tmp/mori-src rev-parse HEAD)" >> /app/versions.txt && \ + rm -rf /tmp/mori-src ; \ + fi # ----------------------------------------------------------------------------- -# 2. AITER: build STOCK upstream ROCm/aiter @ e03fa6040 from source (NO fork, -# NO gqa64-fold patch). Under vLLM #47766 the sparse-MLA persistent path stays -# ON, so GLM's gqa=64 decode hits aiter's PRE-EXISTING persistent gqa64->16 fold -# (aiter/mla.py: `nhead in range(32,128+1,16) and persistent_mode`); the fork's -# extra non-persistent fold is never exercised, so stock is sufficient. -# Validated by MAD-private #338: 1P/1D EP8 + 2P/2D EP16 NIAH PASS on this exact -# aiter tip under #47766. Pin the exact commit (the one tested), not the release -# wheel. Then invalidate the stale prewarmed JIT cache compiled against the old .so. +# 2. AITER: the v0.27 base bundles amd-aiter 0.1.19 (+ flydsl 0.2.4), but bundled 0.1.19 +# GPU-faults on the GLM DSA decode kernel mla_a8w8_qh64_gqaratio64_v3 (confirmed b1 on +# this v0.27 base, same regression as the old stack). So we build aiter from source at +# the validated commit e03fa6040 by DEFAULT (WITH_AITER_BUILD=1). aiter > e03fa6040 +# reintroduces the fault; do not bump without re-running long-ctx NIAH. Set +# --build-arg WITH_AITER_BUILD=0 only to fall back to the bundled aiter for debugging. # ----------------------------------------------------------------------------- ARG AITER_REPO=https://github.com/ROCm/aiter.git +ARG WITH_AITER_BUILD=1 ARG AITER_REF=e03fa6040 -RUN echo "Compiling STOCK AITER (no fork) from ${AITER_REPO}@${AITER_REF}" && \ - rm -rf /tmp/aiter-src && \ - git clone --recursive "${AITER_REPO}" /tmp/aiter-src && \ - cd /tmp/aiter-src && git checkout "${AITER_REF}" && \ - git submodule update --init --recursive && \ - (pip uninstall -y amd_aiter amd-aiter aiter 2>/dev/null || true) && \ - pip install --no-build-isolation --no-deps -v . && \ - pip install --no-deps -U "flydsl>=0.1.7,<0.1.9" && \ - echo "AITER_REF=${AITER_REF}@$(git rev-parse HEAD) (stock ROCm/aiter, no fork)" >> /app/versions.txt && \ - rm -rf /tmp/aiter-src && \ - python3 - <<'PYEOF' -# Verify aiter/mla.py installed + has the persistent gqa64 fold, WITHOUT importing -# aiter/torch (torch->amdsmi->libamd_smi.so is not loadable at build: no GPU in sandbox). -import glob, pathlib -cands = glob.glob("/usr/local/lib/python*/dist-packages/aiter/mla.py") + \ - glob.glob("/usr/lib/python*/dist-packages/aiter/mla.py") -assert cands, "aiter/mla.py not found in site-packages after install" -src = pathlib.Path(cands[0]).read_text() -assert "persistent_mode" in src, f"AITER persistent fold path MISSING in {cands[0]}" -print("STOCK AITER OK (persistent gqa64 fold path present):", cands[0]) -PYEOF -RUN rm -rf /opt/vllm_cache/aiter_jit /root/.aiter && echo "cleared stale AITER JIT cache" && \ - echo "AITER_REF=${AITER_REF} (stock)" >> /app/versions.txt +RUN if [ "${WITH_AITER_BUILD}" != "1" ]; then \ + echo "AITER: using BUNDLED base aiter (WITH_AITER_BUILD=0)" && \ + python3 -c "import importlib.metadata as m; print('aiter (bundled)', m.version('amd-aiter'))" && \ + echo "AITER_REF=BUNDLED (base amd-aiter, WITH_AITER_BUILD=0)" >> /app/versions.txt ; \ + else \ + echo "Compiling STOCK AITER (no fork) from ${AITER_REPO}@${AITER_REF}" && \ + rm -rf /tmp/aiter-src && \ + git clone --recursive "${AITER_REPO}" /tmp/aiter-src && \ + cd /tmp/aiter-src && git checkout "${AITER_REF}" && \ + git submodule update --init --recursive && \ + (pip uninstall -y amd_aiter amd-aiter aiter 2>/dev/null || true) && \ + pip install --no-build-isolation --no-deps -v . && \ + pip install --no-deps -U "flydsl>=0.1.7,<0.1.9" && \ + echo "AITER_REF=${AITER_REF}@$(git rev-parse HEAD) (stock ROCm/aiter, no fork)" >> /app/versions.txt && \ + rm -rf /tmp/aiter-src && \ + rm -rf /opt/vllm_cache/aiter_jit /root/.aiter && echo "cleared stale AITER JIT cache" ; \ + fi # ----------------------------------------------------------------------------- # 3. vLLM: compile from source at the 06_29 validated Wide-EP WRITE-mode branch @@ -181,7 +184,14 @@ RUN rm -rf /opt/vllm_cache/aiter_jit /root/.aiter && echo "cleared stale AITER J # VLLM_REPO/REF are a PUBLIC GitHub repo + branch (the Wide-EP WRITE-mode vLLM the # dist-inf-cookbook mori121 image builds from). Override to your own vLLM fork/branch. ARG VLLM_REPO=https://github.com/raviguptaamd/vllm.git -ARG VLLM_REF=glm5.1-dsa-wideEP_on_shik_latest +# glm5.1-dsa-wideEP_on_vllm-v0.27 (HEAD cda3648602) = upstream v0.27 tip dedbf6be8b + 7 +# ROCm/DSA commits. Core 3: per-req-ctx metadata key (#47766), DSA indexer KV transfer +# (reworked onto upstream's native MoRIIO connector), invalid-token sentinel. Plus 4 +# v0.27 fixes: concat_and_cache_mla positional (stable-ABI), splitting_ops out of the +# compiled graph (MLA "unknown parameter type"), sparse-indexer bounds-guard, and the +# decisive sentinel -1->0 (cda3648602 — aiter mla_decode_fwd derefs -1 -> GPU fault at +# disagg long-ctx). NIAH-validated 1P/1D + 2P/1D + 1P/2D, 2k-35k, decode PIECEWISE. +ARG VLLM_REF=glm5.1-dsa-wideEP_on_vllm-v0.27 ENV VLLM_TARGET_DEVICE=rocm \ PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH} \ MAX_JOBS=${MAX_JOBS} @@ -203,12 +213,14 @@ def get(names): except PackageNotFoundError: pass return None av = get(("amd-aiter", "amd_aiter", "aiter")) -# Stock source build of ROCm/aiter@e03fa6040 reports 0.1.17.dev195+ge03fa6040. -# Verify the aiter install survived the vLLM install (present + carries the e03fa6040 -# commit tag) rather than pinning a release version string. -assert av and "e03fa6040" in av, f"AITER missing/downgraded (want e03fa6040 build): {av!r}" +# Verify the aiter install survived the vLLM install (present, not silently downgraded +# to a base-bundled wheel). We pin aiter by commit (e03fa6040), whose reported version +# string varies by build, so assert presence rather than a hardcoded commit substring. Do NOT +# `import aiter` here: it pulls torch->amdsmi->libamd_smi.so, not loadable in the no-GPU +# build sandbox (same reason the Stage-2 verify reads mla.py from disk instead). +assert av, "AITER missing after vLLM install (expected bundled 0.1.19 or source-built ref)" import mori, mori.io, mori.ops -print("Post-vLLM check OK: AITER", av, "+ MoRI importable") +print("Post-vLLM check OK: AITER", av, "present + MoRI importable") PYEOF # ----------------------------------------------------------------------------- diff --git a/scripts/vllm_dissag/benchmark_xPyD.sh b/scripts/vllm_dissag/benchmark_xPyD.sh index b8851d24..699c9a4e 100755 --- a/scripts/vllm_dissag/benchmark_xPyD.sh +++ b/scripts/vllm_dissag/benchmark_xPyD.sh @@ -40,6 +40,32 @@ for i in $(seq 1 $BENCHMARK_ITR); do echo "Running the benchserving script for iter: $i" | tee -a ${LOG}_CONCURRENCY.log >/dev/null for combo in "${COMBINATIONS[@]}"; do IFS="/" read -r isl osl <<< "$combo" + # Per-shape warmup at the REAL isl/osl, low concurrency. The global warmup above + # is isl=osl=32/con=1, which never exercises this shape's prefill path, its Triton/ + # aiter kernel variants, or the decode cudagraph batch sizes -- so without this the + # FIRST measured cell of each shape absorbs all the residual JIT and reports a + # wildly inflated TPOT (observed 302ms vs ~89ms steady-state). Measured cells must + # start from a warm graph. Skip with SHAPE_WARMUP=0. + if [[ "${SHAPE_WARMUP:-1}" == "1" ]]; then + _w_con="${SHAPE_WARMUP_CON:-4}" + _w_prompts="${SHAPE_WARMUP_PROMPTS:-8}" + echo "[WARMUP] shape isl $isl osl $osl con ${_w_con} prompts ${_w_prompts}" \ + | tee -a ${LOG}_CONCURRENCY.log >/dev/null + timeout "${SHAPE_WARMUP_TIMEOUT:-2400}" vllm bench serve \ + --model $MODEL_PATH \ + --backend vllm \ + --host 127.0.0.1 \ + --port $BENCHMARK_PORT \ + --dataset-name "random" \ + --random-input-len $isl \ + --random-output-len $osl \ + --random-prefix-len 0 \ + --num-prompts ${_w_prompts} \ + --request-rate "inf" \ + --ignore-eos \ + --max-concurrency ${_w_con} \ + 2>&1 | tee -a ${LOG}_SHAPEWARMUP.log >/dev/null + fi for con in $CON; do p_con=$(($con * 2)) if [ "$p_con" -lt 16 ]; then diff --git a/scripts/vllm_dissag/connectors/moriio.env b/scripts/vllm_dissag/connectors/moriio.env index 29ed23ba..780f0214 100644 --- a/scripts/vllm_dissag/connectors/moriio.env +++ b/scripts/vllm_dissag/connectors/moriio.env @@ -23,7 +23,18 @@ MORI_RDMA_TC=41 MORI_RDMA_SL=0 MORI_IO_SL=1 MORI_IB_ENABLE_RELAXED_ORDERING=1 -MORI_IB_GID_INDEX=1 +MORI_IB_GID_INDEX=3 +# RDMA NIC allowlist (MI300 + CX7 / RoCE). Without this MoRI auto-enumerates ALL ibv +# devices incl. the mgmt NICs (mlx5_1=eth0, mlx5_6=eth1 on the 10.158 mgmt net), and +# tries to establish QPs over a non-routable/mgmt fabric -> ibverbs.cpp:189 "Connection +# timed out" at the prefill->decode KV transfer. Restrict to the 8 GPU-RoCE NICs +# (rdma0-7 on the 10.224 fabric) per the dist-inf-cookbook cluster-rdma-env-recommender. +# NCCL/GLOO use eth0 for their (non-RDMA) control sockets. Override per-fabric if needed. +MORI_RDMA_DEVICES=mlx5_0,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_7,mlx5_8,mlx5_9 +NCCL_IB_HCA=mlx5_0,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_7,mlx5_8,mlx5_9 +NCCL_IB_GID_INDEX=3 +NCCL_SOCKET_IFNAME=eth0 +GLOO_SOCKET_IFNAME=eth0 MORI_NUM_QP_PER_PE=8 VLLM_MORIIO_QP_PER_TRANSFER=2 VLLM_MORIIO_NUM_WORKERS=4 diff --git a/scripts/vllm_dissag/connectors/moriio.sh b/scripts/vllm_dissag/connectors/moriio.sh index c26a0944..33d5322d 100644 --- a/scripts/vllm_dissag/connectors/moriio.sh +++ b/scripts/vllm_dissag/connectors/moriio.sh @@ -245,12 +245,21 @@ connector_launch_worker() { else _cudagraph_mode="${PREFILL_CUDAGRAPH_MODE:-$_cudagraph_mode}" fi + # v0.27 MLA fix: the *_kv_cache_update op dispatches the STABLE-ABI concat_and_cache_mla + # whose boxed kernel does NOT compose inside the Dynamo-FX-partitioned compiled graph + # -> "RuntimeError: unknown parameter type" on the first real MLA decode (passes boot + + # warmup via the fake path, then crashes). splitting_ops-list membership alone doesn't + # cut the graph there. use_inductor_graph_partition=true moves partitioning to inductor + # codegen (after all passes), splitting at cudagraph_unsafe ops incl. the KV-update so + # it runs as an eager boundary. Toggle via USE_INDUCTOR_GRAPH_PARTITION (default 1). + local _igp_json="" + [[ "${USE_INDUCTOR_GRAPH_PARTITION:-1}" == "1" ]] && _igp_json=',"use_inductor_graph_partition":true' if [[ -n "$_cudagraph_mode" && "$_cudagraph_mode" != "NONE" ]]; then local _capture_sizes="${CUDAGRAPH_CAPTURE_SIZES:-1 2 4 8 16 32 64 128 256}" - exec_args+=(--compilation-config '{"cudagraph_mode":"'"${_cudagraph_mode}"'","custom_ops":["+quant_fp8"]}') + exec_args+=(--compilation-config '{"cudagraph_mode":"'"${_cudagraph_mode}"'","custom_ops":["+quant_fp8"]'"${_igp_json}"'}') exec_args+=(--cudagraph-capture-sizes ${_capture_sizes}) else - exec_args+=(--compilation-config '{"cudagraph_mode":"NONE","custom_ops":["+quant_fp8"]}') + exec_args+=(--compilation-config '{"cudagraph_mode":"NONE","custom_ops":["+quant_fp8"]'"${_igp_json}"'}') fi # Per-model flags from models.yaml (driver-exported; empty if none). @@ -265,6 +274,25 @@ connector_launch_worker() { local _all2all="${PREFILL_MORI_BACKEND}" [[ "$log_prefix" == "decode" ]] && _all2all="${DECODE_MORI_BACKEND}" + # Per-role MoRI EP buffer width. VLLM_MORI_MAX_TOKENS_PER_RANK sizes the + # dispatch/combine buffer; unset (0) it inherits max_num_batched_tokens -- a + # chunked-prefill SCHEDULER knob (8192) -- so a decode instance moves an + # 8192-token-wide buffer every step, per layer: ~302ms vs ~88ms TPOT (3.4x). + # Prefill and decode want OPPOSITE values (prefill genuinely dispatches wide + # batches and must keep the large buffer), but models.yaml env: applies to BOTH + # roles -- so split it here, mirroring PREFILL/DECODE_MORI_BACKEND above. + if [[ "$log_prefix" == "decode" ]]; then + [[ -n "${DECODE_MORI_MAX_TOKENS_PER_RANK:-}" ]] && \ + export VLLM_MORI_MAX_TOKENS_PER_RANK="${DECODE_MORI_MAX_TOKENS_PER_RANK}" + # Recv capacity must still cover vLLM's profiling dummy run (which pushes + # max_num_batched_tokens tokens) even though steady-state dispatch is narrow. + [[ -n "${DECODE_MORI_MAX_TOTAL_RECV_TOKENS:-}" ]] && \ + export VLLM_MORI_MAX_TOTAL_RECV_TOKENS="${DECODE_MORI_MAX_TOTAL_RECV_TOKENS}" + else + export VLLM_MORI_MAX_TOKENS_PER_RANK="${PREFILL_MORI_MAX_TOKENS_PER_RANK:-0}" + export VLLM_MORI_MAX_TOTAL_RECV_TOKENS="${PREFILL_MORI_MAX_TOTAL_RECV_TOKENS:-0}" + fi + local extra_args=() kv_args=() if [[ "$role" == "master" ]]; then extra_args+=(--api-server-count=${_GPUS_PER_NODE}) diff --git a/scripts/vllm_dissag/models.yaml b/scripts/vllm_dissag/models.yaml index 9eec3e7d..e0017704 100644 --- a/scripts/vllm_dissag/models.yaml +++ b/scripts/vllm_dissag/models.yaml @@ -235,6 +235,14 @@ DeepSeek-R1: GLM-5.1-FP8: env: VLLM_USE_V1: "1" + # v0.27: layer_name is wrapped in a torch OpaqueBase (LayerName) and passed through + # the unified_mla_kv_cache_update / unified_mla_attention custom ops. On this ROCm + # torch 2.12 build the opaque-type boxing FAILS -> "RuntimeError: unknown parameter + # type" at torch/_ops.py on the first real MLA decode forward (the fake/compile path + # returns early, so it passes boot+warmup then crashes on first request -> DP gloo + # cascade -> 0 prefill/0 decode). VLLM_USE_LAYERNAME=0 makes LayerNameType=str (the + # pre-2.11 path), so a plain string passes through the op. No image rebuild needed. + VLLM_USE_LAYERNAME: "0" VLLM_ROCM_USE_AITER: "1" VLLM_ROCM_USE_AITER_RMSNORM: "1" VLLM_ROCM_USE_AITER_MLA: "1" @@ -248,6 +256,21 @@ GLM-5.1-FP8: VLLM_ALL2ALL_BACKEND: "mori_high_throughput" PREFILL_MORI_BACKEND: "mori_high_throughput" DECODE_MORI_BACKEND: "mori_low_latency" + # MoRI EP dispatch/combine buffer width. Without this it inherits + # max_num_batched_tokens (8192) -- a chunked-prefill SCHEDULER setting -- so every + # decode step moves an 8192-token-wide buffer per layer x78 layers regardless of the + # real batch. That is a fixed ~300ms/step floor (~320x this model's HBM-bandwidth + # bound). Sizing it for the actual decode batch gives TPOT 302ms -> 88ms (3.4x), + # matching the published EP8 figure. Needs vLLM >= e8c186f71b. + # Prefill is unaffected (it genuinely dispatches wide batches). + # Per-role (moriio.sh -> VLLM_MORI_MAX_TOKENS_PER_RANK). MUST be >= max_num_seqs * + # num_experts_per_token (top-8 here): the value is also the RECV capacity + # (mori MaxNumTokensToRecv = worldSize * this). 512 tripped a device assert + # "Total recv token overflow". 2048 is safe and still 4x smaller than 8192. Decode wants + # the buffer sized to its real batch; prefill must KEEP the wide buffer (0 = inherit + # max_num_batched_tokens) because it genuinely dispatches 8192-token chunks. + # Recv capacity stays wide enough for the profiling dummy run (8192 tokens) and + # bursty top-8 routing: worldSize * ceil(65536/worldSize) >= 8192 for EP8..EP32. MORI_SHMEM_HEAP_SIZE: "17179869184" # DSA sparse-indexer logits-buffer cap (crash fix). The indexer prefill computes an # M*N fp32 logits buffer; split_indexer_prefill_chunks only sub-chunks the query dim @@ -275,4 +298,13 @@ GLM-5.1-FP8: prefill: dp: "" decode: - dp: "" + # PERF: the MoRI EP dispatch buffer width is max_num_batched_tokens (via + # FusedMoEConfig.max_num_tokens -> all2all.py max_num_inp_token_per_rank), so the + # decode role otherwise runs an 8192-token-wide all2all every step: ~302ms TPOT. + # mori bounds recv capacity BY the send width (MaxNumTokensToRecvPerRank returns + # min(ceil(maxTotalRecvTokens/ws), maxNumInpTokenPerRank)), so the buffer must still + # cover vLLM's profiling dummy run -- it cannot be shrunk via env alone. Lowering + # this knob on the DECODE role lowers both consistently. + # Value must stay >= typical prompt length: 512 gave 87.9ms TPOT but 13.1s TTFT + # (a 1024-token prompt could not be admitted in one step). 2048 keeps TTFT healthy. + dp: "--max-num-batched-tokens 2048" diff --git a/scripts/vllm_dissag/run_xPyD_models.slurm b/scripts/vllm_dissag/run_xPyD_models.slurm index 98281f57..52d046a1 100755 --- a/scripts/vllm_dissag/run_xPyD_models.slurm +++ b/scripts/vllm_dissag/run_xPyD_models.slurm @@ -76,7 +76,7 @@ echo "Running from: $(pwd)" # model's own recipe — GLM-5.1 DSA needs block=1 + AITER MLA on), while a genuine # submit-time `-e VAR=...` still wins. Precedence: image-baked < models.yaml < submit -e. # Captured HERE (before the slurm sets any defaults) so it reflects user intent only. -_RECIPE_ENV_KEYS="VLLM_USE_V1 VLLM_ROCM_USE_AITER VLLM_ROCM_USE_AITER_RMSNORM VLLM_ROCM_USE_AITER_MLA KV_BLOCK_SIZE KV_CACHE_DTYPE KV_CACHE_MEMORY_BYTES GPU_MEMORY_UTILIZATION VLLM_CUDAGRAPH_MODE PREFILL_CUDAGRAPH_MODE DECODE_CUDAGRAPH_MODE CUDAGRAPH_CAPTURE_SIZES VLLM_ALL2ALL_BACKEND PREFILL_MORI_BACKEND DECODE_MORI_BACKEND MORI_SHMEM_HEAP_SIZE" +_RECIPE_ENV_KEYS="VLLM_USE_V1 DECODE_MORI_MAX_TOTAL_RECV_TOKENS PREFILL_MORI_MAX_TOTAL_RECV_TOKENS DECODE_MORI_MAX_TOKENS_PER_RANK PREFILL_MORI_MAX_TOKENS_PER_RANK VLLM_MORI_MAX_TOKENS_PER_RANK VLLM_MORI_WARP_NUM_PER_BLOCK VLLM_MORI_BLOCK_NUM VLLM_MORI_RDMA_BLOCK_NUM VLLM_USE_LAYERNAME VLLM_ROCM_USE_AITER VLLM_ROCM_USE_AITER_RMSNORM VLLM_ROCM_USE_AITER_MLA KV_BLOCK_SIZE KV_CACHE_DTYPE KV_CACHE_MEMORY_BYTES GPU_MEMORY_UTILIZATION VLLM_CUDAGRAPH_MODE PREFILL_CUDAGRAPH_MODE DECODE_CUDAGRAPH_MODE CUDAGRAPH_CAPTURE_SIZES VLLM_ALL2ALL_BACKEND PREFILL_MORI_BACKEND DECODE_MORI_BACKEND MORI_SHMEM_HEAP_SIZE" MODELS_YAML_PROTECT="" for _k in $_RECIPE_ENV_KEYS; do [ -n "${!_k+x}" ] && MODELS_YAML_PROTECT="${MODELS_YAML_PROTECT} ${_k}" @@ -426,15 +426,22 @@ echo "" # Node information USER_NAME=$(whoami) MASTER_NODE=$(echo "$SELECTED_NODES" | head -n 1) -MASTER_ADDR=$(srun --nodes=1 --ntasks=1 --time=00:20:00 --nodelist="$MASTER_NODE" bash -c 'hostname -I') -MASTER_ADDR=$(echo "$MASTER_ADDR" | awk 'NR==1 {print $1}') +# Pick the routable fabric IP, not just hostname -I's first entry. These nodes expose +# multiple NICs (e.g. a 10.224.x overlay listed BEFORE the routable 10.158.x fabric); +# taking $1 blindly can advertise an unreachable addr -> prefill/decode barrier hangs +# "Waiting for nodes" forever. Prefer FABRIC_SUBNET (default 10.158.), fall back to $1. +FABRIC_SUBNET="${FABRIC_SUBNET:-10.158.}" +# From a "hostname -I" line, return the first IP on FABRIC_SUBNET, else the first IP. +_pick_fabric_ip() { + awk -v pfx="$FABRIC_SUBNET" '{f=$1; for(i=1;i<=NF;i++) if(index($i,pfx)==1){f=$i; break} print f}' +} +MASTER_ADDR=$(srun --nodes=1 --ntasks=1 --time=00:20:00 --nodelist="$MASTER_NODE" bash -c 'hostname -I' | _pick_fabric_ip) MASTER_PORT=39566 # Choose an open port IPS=() for NODE in $SELECTED_NODES; do - IP=$(srun --nodes=1 --ntasks=1 --time=00:20:00 --nodelist="$NODE" bash -c 'hostname -I') - IP=$(echo "$IP" | awk 'NR==1 {print $1}') + IP=$(srun --nodes=1 --ntasks=1 --time=00:20:00 --nodelist="$NODE" bash -c 'hostname -I' | _pick_fabric_ip) IPS+=("$IP") done @@ -549,7 +556,12 @@ done for _pattern in libmlx5.so* libionic*.so* libbnxt_re*.so* libefa.so* libhns.so*; do for _vlib in $_LIBDIR/${_pattern}; do - [ -e "$_vlib" ] && _RDMA_MOUNTS="$_RDMA_MOUNTS -v $_vlib:$_vlib:ro" + # Require a regular file AFTER symlink resolution: these mounts are built on + # ONE node but applied on ALL nodes, and vendor NIC libs (e.g. libionic.so.1) + # can be a DANGLING symlink on some nodes -> bind-mount fails "not a directory" + # -> container create exit 125. `-f` (follows symlink, requires regular file) + # skips those; the fabric in use (mlx5) is still mounted where present. + [ -f "$_vlib" ] && _RDMA_MOUNTS="$_RDMA_MOUNTS -v $_vlib:$_vlib:ro" done done @@ -578,6 +590,8 @@ docker run --rm \ -v $NIXL_REPO_DIR:$NIXL_COOKBOOK_PATH \ -v /tmp/vllm_cache:/tmp/vllm_cache \ ${_JIT_CACHE_MOUNT} \ + ${GLM_KERNEL_PATCH:+-v ${GLM_KERNEL_PATCH}:/usr/local/lib/python3.12/dist-packages/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py:ro} \ + ${GLM_BACKEND_PATCH:+-v ${GLM_BACKEND_PATCH}:/usr/local/lib/python3.12/dist-packages/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py:ro} \ $_RDMA_MOUNTS \ --entrypoint /bin/bash \ -e SLURM_JOB_ID=$SLURM_JOB_ID \ diff --git a/scripts/vllm_dissag/vllm_disagg.sh b/scripts/vllm_dissag/vllm_disagg.sh index acbabbd4..da2afb00 100755 --- a/scripts/vllm_dissag/vllm_disagg.sh +++ b/scripts/vllm_dissag/vllm_disagg.sh @@ -86,7 +86,12 @@ IFS=',' read -ra IP_ARRAY <<< "${IPADDRS}" echo "Listing NIXL_COOKBOOK_PATH: ${NIXL_COOKBOOK_PATH:-}" [[ -n "${NIXL_COOKBOOK_PATH:-}" ]] && ls "${NIXL_COOKBOOK_PATH}" -host_ip=$(hostname -I | awk '{print $1}') +# Prefer the routable fabric IP (FABRIC_SUBNET, default 10.158.) over hostname -I's +# first entry: nodes with a 10.224.x overlay listed first would bind the socket_barrier +# / advertise host_ip on an unreachable NIC -> prefill<->decode barrier hangs "Waiting +# for nodes". Matches the IPADDRS selection in run_xPyD_models.slurm. Falls back to $1. +FABRIC_SUBNET="${FABRIC_SUBNET:-10.158.}" +host_ip=$(hostname -I | awk -v pfx="$FABRIC_SUBNET" '{f=$1; for(i=1;i<=NF;i++) if(index($i,pfx)==1){f=$i; break} print f}') host_name=$(hostname) # ============================================================================= From 63fb87d22593412262de5d64f866d289328b213c Mon Sep 17 00:00:00 2001 From: raviguptaamd Date: Sun, 16 Aug 2026 04:24:53 +0000 Subject: [PATCH 06/23] [GLM-5.1] Default WITH_NIXL=0 (lean MoRI-EP-only image) GLM-5.1 is served over MoRI-EP (all2all) + MoRI-IO (KV transfer); the rixl connector's UCX/RIXL/rocSHMEM/DeepEP stack is never selected by this recipe, so building it only lengthens the build and ships unused transports. 0 is also how the validated image was actually built: glm5.1-vllm027-b8 used --build-arg WITH_NIXL=0, so the previous default of 1 meant a plain `docker build` did NOT reproduce the tested artifact. With this change, and with WITH_MORI_BUILD/WITH_AITER_BUILD already defaulting to 1, a no-flag build now matches the validated stack exactly. Set --build-arg WITH_NIXL=1 if you need the rixl connector from this same Dockerfile. Co-Authored-By: Claude --- .../vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile b/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile index 8da85d77..edd4d797 100644 --- a/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile +++ b/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile @@ -92,9 +92,12 @@ WORKDIR /app ARG GFX_COMPILATION_ARCH="gfx942" ARG PYTORCH_ROCM_ARCH="gfx942" ARG MAX_JOBS=32 -# NIXL/RIXL transport for the rixl connector. Default 1 => all connectors built -# (UCX/RIXL/rocSHMEM/DeepEP). Set --build-arg WITH_NIXL=0 for a lean MoRI-EP-only image. -ARG WITH_NIXL=1 +# NIXL/RIXL transport for the rixl connector. GLM-5.1 is served over MoRI-EP + MoRI-IO, +# so the UCX/RIXL/rocSHMEM/DeepEP stack is dead weight here: it lengthens the build and +# ships transports this recipe never selects. Default 0 => lean MoRI-EP-only image, which +# is also exactly how the validated image (glm5.1-vllm027-b8) was built. Set +# --build-arg WITH_NIXL=1 only if you need the rixl connector from this same Dockerfile. +ARG WITH_NIXL=0 ARG NIC_COMPILATION_ARCH="cx7" # ----------------------------------------------------------------------------- From af878edc0b37ab58fd7b2c0a843bf024e166e48d Mon Sep 17 00:00:00 2001 From: raviguptaamd Date: Sun, 16 Aug 2026 04:28:57 +0000 Subject: [PATCH 07/23] [GLM-5.1] Add long-context NIAH harness + vllm-disagg operational playbook niah_200k.py: needle-in-a-haystack sweep that validated GLM-5.1-FP8 to 200,049 tokens on both 1P/1D (EP8) and 2P/2D (EP16). Reports found/10, latency, and the server-reported prompt_tokens per length, and writes JSON. Model id is overridable via NIAH_MODEL so it is not GLM-specific. The existing benchmark_niah.* stop well short of this range; this covers the 64k-200k band. skills_vllm_disagg.md: operational playbook for vLLM PD-disaggregated WideEP on MI300X (MoRI-EP + MoRI-IO), written from this enablement. Documents, with measurements: - benchmarking method: ALWAYS discard the first post-boot run (cold Triton JIT made TTFT read 13.4s vs 906ms warm; with prefill eager the JIT cost lands in TTFT, not TPOT), and sanity-check against the HBM-bandwidth bound before blaming a kernel - the max_num_batched_tokens trap: a chunked-prefill SCHEDULER knob also sizes the MoRI EP dispatch buffer, so decode ran an 8192-token-wide all2all every step (302ms -> 88ms TPOT once sized for the real batch) - the DSA sentinel landmine: the invalid sparse-index sentinel must be 0, not -1, because aiter's mla_decode_fwd dereferences it (only bites at disagg long context) - three documented DEAD ENDS so they are not retried, including why mori's max_total_recv_tokens cannot decouple recv from send capacity (the clamp is a min()) - cache/boot behaviour (three caches with different rules, the aiter baton lock, measured boot times), readiness signals for multi-node topologies, per-role env plumbing, and RDMA fabric verification (a node can be SLURM-'alloc' with a dead fabric - verify with ping/ib_write_bw before blaming code) Co-Authored-By: Claude --- scripts/vllm_dissag/niah_200k.py | 86 ++++++++ scripts/vllm_dissag/skills_vllm_disagg.md | 245 ++++++++++++++++++++++ 2 files changed, 331 insertions(+) create mode 100755 scripts/vllm_dissag/niah_200k.py create mode 100644 scripts/vllm_dissag/skills_vllm_disagg.md diff --git a/scripts/vllm_dissag/niah_200k.py b/scripts/vllm_dissag/niah_200k.py new file mode 100755 index 00000000..65273bc1 --- /dev/null +++ b/scripts/vllm_dissag/niah_200k.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Needle-in-a-haystack sweep including long context (validated to 200K tokens). + +Hides 10 animal names at even intervals in a filler-word haystack and asks the model to +list them back. Reports found/10, end-to-end latency, and the server-reported +prompt_tokens per length, and can dump the whole run to JSON. + +Usage: + niah_200k.py [lengths_csv] [out_json] + base_url e.g. http://127.0.0.1:20005 (prefill/serve port, or the router) + lengths_csv comma-separated WORD counts (default: 2k..200k) + out_json optional path to write results + + NIAH_MODEL= override the served model id (default below) + +Lengths are given in WORDS to stay comparable with earlier published runs. On this filler +the GLM tokenizer lands ~1 token/word, so words ~= tokens (the script prints the actual +prompt_tokens so you can check). + +NOTE: the FIRST request after a server boot pays cold Triton JIT and can take >80s with a +prefill instance running eager. Warm the server (or use a generous timeout) before +treating any latency number here as steady-state. +""" +import json, os, sys, time, random, urllib.request + +BASE = sys.argv[1] if len(sys.argv) > 1 else "http://127.0.0.1:20005" +LENGTHS = [int(x) for x in (sys.argv[2].split(",") if len(sys.argv) > 2 else + "2000,8000,16000,20000,28000,35000,64000,100000,150000,200000".split(","))] +OUT = sys.argv[3] if len(sys.argv) > 3 else None +URL = BASE.rstrip("/") + "/v1/chat/completions" +MODEL = os.environ.get("NIAH_MODEL", "/mnt/m2m_nobackup/models_blog/GLM-5.1-FP8") + +FILLER = ("table chair window bottle pencil garden river mountain coffee planet " + "engine guitar pillow ticket basket candle market silver button orange").split() +ANIMALS = ["elephant", "giraffe", "kangaroo", "penguin", "dolphin", + "tiger", "rhinoceros", "octopus", "crocodile", "panda"] +SYS = ("You read a word list and pick out the animals. Reply with a single " + "comma-separated list of lowercase animal names. Output nothing else.") + + +def hay(n, seed=0): + rng = random.Random(seed) + w = [rng.choice(FILLER) for _ in range(n)] + step = max(n // (len(ANIMALS) + 1), 1) + for i, a in enumerate(ANIMALS): + w[min((i + 1) * step, len(w) - 1)] = a + return " ".join(w) + + +def run(n, seed=0, timeout=1800): + body = { + "model": MODEL, + "messages": [ + {"role": "system", "content": SYS}, + {"role": "user", "content": "Find the animals in this list:\n\n" + hay(n, seed)}, + ], + "temperature": 0, + "max_tokens": 128, + "chat_template_kwargs": {"enable_thinking": False}, + } + req = urllib.request.Request(URL, data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + t = time.time() + try: + r = json.loads(urllib.request.urlopen(req, timeout=timeout).read()) + m = r["choices"][0]["message"] + txt = ((m.get("content") or "") + " " + (m.get("reasoning") or "")).lower() + found = sorted(a for a in ANIMALS if a in txt) + u = r.get("usage") or {} + rec = {"words": n, "seed": seed, "found": len(found), "latency_s": round(time.time() - t, 1), + "prompt_tokens": u.get("prompt_tokens"), "animals": found} + print("words=%7d tok=%-7s found=%2d/10 (%6.1fs) %s" % ( + n, rec["prompt_tokens"], rec["found"], rec["latency_s"], found), flush=True) + return rec + except Exception as e: + rec = {"words": n, "seed": seed, "found": -1, "latency_s": round(time.time() - t, 1), + "error": str(e)[:200]} + print("words=%7d ERROR (%.1fs) %s" % (n, rec["latency_s"], rec["error"]), flush=True) + return rec + + +results = [run(n) for n in LENGTHS] +if OUT: + with open(OUT, "w") as f: + json.dump(results, f, indent=2) + print("wrote", OUT, flush=True) diff --git a/scripts/vllm_dissag/skills_vllm_disagg.md b/scripts/vllm_dissag/skills_vllm_disagg.md new file mode 100644 index 00000000..ee778064 --- /dev/null +++ b/scripts/vllm_dissag/skills_vllm_disagg.md @@ -0,0 +1,245 @@ +# skills_vllm_disagg.md + +Hard-won operational knowledge for **vLLM PD-disaggregated WideEP serving on AMD MI300X** +(MoRI-EP all-to-all + MoRI-IO RDMA KV transfer), learned while bringing GLM-5.1-FP8 +(MLA + DeepSeek Sparse Attention) onto vLLM v0.27. + +Everything below is *measured*, not theorised. Where a belief turned out to be wrong, +the wrong belief is kept alongside the correction — those are the expensive lessons. + +--- + +## 1. Benchmarking methodology (read this first — it invalidated three of my conclusions) + +### 1.1 ALWAYS discard the first bench run after a boot +The first real request after startup pays **cold Triton JIT** for the sparse/indexer +kernels (`_indexer_k_quant_and_cache_kernel`, `generate_sparse_seqlen_kernel`, +`_convert_req_index_to_global_index_kernel`). Measured on 1P/1D EP8, identical bench: + +| run | TTFT | TPOT | +|---|---|---| +| 1st after boot (cold) | 13,451 ms | 88.7 ms | +| 2nd (warm) | **906 ms** | 88.0 ms | + +TTFT moved **14.9x**; TPOT barely moved. A single cold run made me invent (and act on) +a false "scheduler admission" theory twice. Discard it, or warm up explicitly. + +### 1.2 Why cold JIT lands on TTFT and not TPOT +With the standard recipe **prefill = eager (`CUDAGraphMode.NONE`)**, decode = PIECEWISE: +- prefill has no graph capture at boot -> its kernels compile lazily on the **first real + request** -> the whole compile cost is inside TTFT. +- decode captured graphs at boot ("Graph capturing finished") -> already warm -> TPOT is + correct even on the cold run. +This asymmetry is diagnostic: *cold-JIT symptoms show up in TTFT only*. + +### 1.3 The harness warmup is not a warmup +`benchmark_xPyD.sh` warms at `isl=32 osl=32 con=1` — which never exercises a 1024/8192/28672 +prefill path, nor the decode cudagraph batch sizes. The first *measured* cell therefore +absorbs residual JIT. Fix applied: a per-shape warmup at the real ISL/OSL before each +shape's cells (writes to a separate `_SHAPEWARMUP.log` so it can't pollute the CSV). + +### 1.4 Client timeouts read as server failures +A 50-80s curl timeout against a cold server returns an **empty body**, which looks exactly +like a crash. Use >=300s on the first request. This produced a false "total failure" +verdict during EP32 debugging. + +### 1.5 Sanity-check against physics before blaming a kernel +GLM-5.1-FP8 activates ~37.7B params/token. At 5.3 TB/s HBM, fp8: +- full model on one rank: **7.1 ms/step** +- 1/8 of experts per rank (EP8): **0.9 ms/step** + +Measured 290-300 ms => ~320x the bound. That immediately rules out "compute" or "bandwidth" +and says *stall / oversized transfer*. Do this arithmetic early; it saves hours. + +--- + +## 2. The big perf trap: `max_num_batched_tokens` sizes the MoRI EP buffer + +### 2.1 The chain +``` +vllm/model_executor/layers/fused_moe/layer.py:349 + max_num_tokens = max_num_batched_tokens # 8192 default (SCHEDULER knob) +vllm/model_executor/layers/fused_moe/all2all_utils.py:181 + max_num_tokens_per_dp_rank = moe.max_num_tokens +vllm/distributed/device_communicators/all2all.py (MoriAll2AllManager) + max_num_inp_token_per_rank = +``` +`max_num_batched_tokens` is a **chunked-prefill scheduler** setting. Using it to size the +EP dispatch/combine buffer means a **decode** instance runs an 8192-token-wide all-to-all +**every step, per layer, x78 layers**, while decoding a handful of tokens. + +### 2.2 Signature of this bug +- fixed per-step cost: TPOT identical at concurrency 1, 4, 8, 16 +- independent of KV length: 292 ms at isl=128, 297 ms at isl=1024 +- batching still scales perfectly (con=1 -> 8 gave 8.3x throughput, TPOT flat) +- orders of magnitude above the bandwidth bound +=> "constant oversized transfer", not compute. + +### 2.3 The fix that works today (no code change) +Lower `--max-num-batched-tokens` **on the decode role only** (`models.yaml` `decode.dp:`): + +| decode mnbt | TPOT | TTFT (warm) | out tok/s | +|---|---|---|---| +| 8192 (default) | 302.5 ms | 2431 ms | 24.9 | +| **2048** | **88.0 ms** | **906 ms** | **78.8** | + +3.4x faster decode, and it also *improved* TTFT and throughput. Prefill keeps 8192 (it +genuinely dispatches wide batches). + +### 2.4 What does NOT work (dead ends — do not repeat) +- **`VLLM_MORI_MAX_TOKENS_PER_RANK` alone** (512 or 2048): device assert at boot + ``` + mori .../dispatch_combine/intranode.hpp:134 + `destTokId < config.MaxNumTokensToRecv() && + "Total recv token overflow: increase maxTotalRecvTokens"' + ``` + because vLLM's **profiling/warmup dummy run deliberately pushes + `max_num_batched_tokens` (8192) tokens** through the model. The EP buffer must survive + that even though steady-state decode never needs it. +- **`max_total_recv_tokens` to decouple recv from send: IMPOSSIBLE in current mori.** + ``` + MaxNumTokensToRecvPerRank(): + if maxTotalRecvTokens > 0: + perRank = ceil(maxTotalRecvTokens / worldSize) + return perRank < maxNumInpTokenPerRank ? perRank : maxNumInpTokenPerRank # min() + return maxNumInpTokenPerRank + ``` + It returns **min(perRank, send_width)** — it can only *lower* recv capacity, never raise + it above the send width. `send=1024, recv=65536` behaves identically to leaving it unset. + Recv capacity is structurally bounded by send width. + +### 2.5 The proper upstream fix (not yet done) +Bound the **profiling/dummy run on a decode instance by `max_num_seqs`** instead of +`max_num_batched_tokens`. Then the EP buffer can be narrow and profiling never exceeds it. +This lives entirely in vLLM. (Alternative: a mori change allowing recv > send.) + +### 2.6 MoRI has no tuning knobs (parity gap vs DeepEP) +DeepEP exposes `VLLM_DEEPEP_BUFFER_SIZE_MB`; MoRI hardcodes `warp_num_per_block`, +`block_num`, `rdma_block_num` and inherits its token width from an unrelated scheduler +default. Added `VLLM_MORI_*` knobs for parity (all defaulting to current values). + +--- + +## 3. Accuracy: the DSA sparse-index sentinel landmine + +**Never set the invalid/OOB sentinel to `-1`** in +`_convert_req_index_to_global_index_kernel` +(`vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py`). It must be `0`. + +Why: aiter's `mla_decode_fwd` **dereferences** `paged_kv_indices`, so `-1` becomes +`kv_cache + (-1)*stride` -> page-aligned GPU memory access fault -> worker dies -> gloo +DP all-reduce collapse. `0` is masked out by `paged_kv_indptr`/`last_page_len`. + +**Why it hides:** it only fires at **disaggregated long context**. Decode inherits the +prefill's long `seq_lens` -> `generate_sparse_seqlen` widens `paged_kv_indptr` -> `-1` +padding entries land *inside* a live indptr range and get dereferenced. Short prompts and +non-disagg decode never emit an in-range `-1`, so it passes every quick test. +`HIP_LAUNCH_BLOCKING=1` does **not** help (data-dependent OOB, not an async race). + +--- + +## 4. Debugging playbook — isolate the layer before optimising + +The ladder that found the perf bug, in order of cheapness: + +| Suspect | Test | What it showed | +|---|---|---| +| router / KV transfer | bench **prefill-direct** (`:20005`) vs via router (`:30000`) | 341 ms vs 303 ms TPOT -> router + MoRI-IO exonerated (router *does* add TTFT) | +| JIT warmup | re-run on a 30-min-warm server | identical -> not warmup | +| cudagraph config | grep `cudagraph_mode`, `Graph capturing finished`, GiB captured | config correct | +| KV length | sweep isl 128 / 1024 / 8192 at con=1 | flat -> fixed floor, not KV | +| batching | con=1 vs 8 | TPOT flat, throughput 8.3x -> batching fine | +| all2all backend | force `PREFILL_MORI_BACKEND=mori_low_latency` | still broken -> not an HT-kernel bug | + +**Also:** compare against a *known-good* build. Diffing our stack against the passing +v0.25 blog stack showed MoRI, aiter and the recipe were **byte-identical** — only the base +image and vLLM differed. That single comparison exonerated two whole components. + +--- + +## 5. Operational gotchas + +### 5.1 Three caches, three different rules +| cache | keyed by | bake into image? | +|---|---|---| +| `aiter_jit` | (aiter commit, gfx arch) | **yes** — topology-independent, and it is the cold-boot long pole | +| `vllm` (torch.compile/inductor), `triton` | model cfg + batch sizes + cudagraph mode + **topology** (EP8/16/32) | **no** — a stale graph is a real trap; wipe when changing shapes | +| `comgr` | ROCm code objects | harmless | + +Host persistence: `VLLM_CACHE_PERSIST=1` -> +`/mnt/m2m_nobackup/$USER/vllm_jit_cache/a256` -> `/opt/vllm_cache`. +**Keyed by image ID**, so every new image = full cold rebuild. + +If baking `aiter_jit` into an image: **scrub `lock_module_*`, `.ninja_log`, `/root/.mori`, +`/tmp/mori_jit_*` first**, or a fresh container waits on a baton nobody holds -> boot hang. +Verify: a fresh container must start with **zero** ninja/hipcc/clang processes. + +### 5.2 Measured boot times (8 nodes, model on local NVMe) +| scenario | time | +|---|---| +| cold aiter, 2 nodes | ~25 min | +| cold aiter, 4 nodes (2 cold decode) | ~41 min | +| cold aiter, 8 nodes (4 cold decode) | ~106 min | +| **warm cache, 8 nodes** | **~11 min** | + +Changing `max_num_batched_tokens` invalidates the torch.compile cache -> full recompile. + +### 5.3 The aiter baton lock looks like a hang but isn't +`[aiter] waiting for baton release at /opt/vllm_cache/aiter_jit/build/lock_` — +one worker compiles, the rest block. Diagnose by counting build procs *inside the +container*: 100-160 = actively building; **0 on all nodes** = genuinely wedged. + +### 5.4 Readiness: don't count "Application startup complete" +Only nodes running an API server print it. For 2P/2D and 4P/4D the non-master DP ranks +never will. Judge readiness by: router `All servers healthy` + `Graph capturing finished` ++ `GPU KV cache size` per node. + +### 5.5 Verify teardown on EVERY node +`docker rm -f` can leave a container alive on one node; a stale container then collides +with the new run (observed: 2 containers on one node -> prefill shut down mid-boot). +Always re-check `docker ps -q | wc -l == 0` everywhere before relaunching. + +### 5.6 Per-role env: `models.yaml env:` applies to BOTH roles +Prefill and decode often need **opposite** values (e.g. EP buffer width). The pattern is +`PREFILL_*` / `DECODE_*` keys in `env:`, split inside `connectors/moriio.sh` (mirrors the +existing `PREFILL_MORI_BACKEND` / `DECODE_MORI_BACKEND`). Verify it landed by reading +`/proc//environ` **inside the container** — the value is exported into the +server process, not the container shell, so `docker exec env` shows nothing. + +### 5.7 RDMA fabric +- GID index **3** = RoCEv2 IPv4 (check `show_gids` / `sysfs .../gid_attrs/types`). +- Restrict `MORI_RDMA_DEVICES` / `NCCL_IB_HCA` to the 8 GPU-local NICs; leave the mgmt NICs + out or QPs try to form over a non-routable fabric -> `ibverbs.cpp:189 Connection timed out`. +- NCCL/GLOO control sockets on `eth0` (mgmt); KV data on the RDMA NICs. +- **Verify the fabric before blaming code**: `ping -I rdma0` matrix, then `ib_write_bw` + (healthy pair measured 386 Gb/s). A node can be `alloc` in SLURM with a **dead** fabric — + SLURM does not detect this. One dead node cost a whole 2P/2D campaign. +- `ib_write_bw` needs both endpoints co-alive: run it as a single 2-task srun step, not two + separate sruns (a backgrounded server dies when its srun returns). + +### 5.8 Images +Push to a registry (`docker push`) rather than relying on `docker save | ssh | docker load` +serially — parallel `docker pull` across 7 nodes is far faster and removes the +"image missing on one node" failure that silently stalls the launcher barrier. + +--- + +## 6. Open items (state as of 2026-08-16) + +- **4P/4D EP32 silent output corruption.** Cluster boots healthy (memfault=0), but output is + garbage. MoRI + aiter + recipe are byte-identical to the passing v0.25 stack; only base + + vLLM differ => **v0.27 regression**. Both `mori_high_throughput` and `mori_low_latency` + corrupt at EP32 while both are clean at EP8/EP16 => an EP-width (>16 ranks) issue, not a + kernel-specific one. Next probe: 3P/3D (EP24) to test "any EP>16" vs "exactly 32". +- **Proper upstream fix for §2.5** (bound decode profiling by `max_num_seqs`). +- **Prewarmed image** (§5.1) to kill the cold-boot and first-request-JIT costs. + +--- + +## 7. One-line summary of the two bugs found + +1. **Accuracy:** a `-1` sentinel that aiter dereferences -> GPU fault, but only at disagg + long context. Use `0`. +2. **Perf:** the MoE all-to-all buffer is sized from a chunked-prefill *scheduler* knob, so + decode moves an 8192-wide buffer every step. Lower `--max-num-batched-tokens` on the + decode role: **302 ms -> 88 ms TPOT**. From f6dc7af2b87430b774f6edd3183a7d11144cf1f8 Mon Sep 17 00:00:00 2001 From: raviguptaamd Date: Sun, 16 Aug 2026 04:33:07 +0000 Subject: [PATCH 08/23] [GLM-5.1] Remove dead-end MoRI EP env plumbing and its misleading comments Review cleanup. The VLLM_MORI_MAX_TOKENS_PER_RANK / VLLM_MORI_MAX_TOTAL_RECV_TOKENS per-role plumbing was written while chasing the decode-TPOT bug and is NOT what fixed it (the fix is `--max-num-batched-tokens 2048` on decode.dp). Worse, the comments asserted that max_total_recv_tokens keeps recv capacity large enough for vLLM's profiling dummy run -- which is false and was disproved by measurement: mori's MaxNumTokensToRecvPerRank() = min(ceil(maxTotalRecvTokens / worldSize), maxNumInpTokenPerRank) is a min(), so maxTotalRecvTokens can only LOWER recv capacity, never raise it above the send width. Anyone following those comments and setting the knobs would hit "Total recv token overflow" at boot (observed at 512, 2048, and with recv=65536). Removed: the per-role export block in moriio.sh, the stale models.yaml comment block, and the six dead keys from _RECIPE_ENV_KEYS. Replaced with a short NOTE in moriio.sh pointing at the real fix and at skills_vllm_disagg.md for the measurements and dead ends. No functional change to the validated configuration: the knobs defaulted to 0/unset, so the b8 runs never exercised them. Co-Authored-By: Claude --- .../vllm_dissag/.nfs0000000016f44d2b00008188 | 685 ++++++++++++++++++ scripts/vllm_dissag/connectors/moriio.sh | 27 +- scripts/vllm_dissag/models.yaml | 12 +- scripts/vllm_dissag/run_xPyD_models.slurm | 2 +- 4 files changed, 697 insertions(+), 29 deletions(-) create mode 100755 scripts/vllm_dissag/.nfs0000000016f44d2b00008188 diff --git a/scripts/vllm_dissag/.nfs0000000016f44d2b00008188 b/scripts/vllm_dissag/.nfs0000000016f44d2b00008188 new file mode 100755 index 00000000..52d046a1 --- /dev/null +++ b/scripts/vllm_dissag/.nfs0000000016f44d2b00008188 @@ -0,0 +1,685 @@ +#!/bin/bash +#SBATCH --job-name=vllm-pd # Specify a custom string for your slurm batch job +#SBATCH -N 2 # Default 2 nodes (1P/1D); override with sbatch -N for larger topologies +#SBATCH --ntasks-per-node=1 +#SBATCH --spread-job +#SBATCH --gres=gpu:8 # Request 8 GPUs and 8 NICs (use --gres if specific GPU resources are needed) +#SBATCH --time=24:00:00 # Set a time limit for the job (HH:MM:SS) +#SBATCH --output="/shared_inference/%u/model_blog_logs/slurm-%j.out" +#SBATCH --error="/shared_inference/%u/model_blog_logs/slurm-%j.err" + + +# ------------------------ +# Auto-detect and cd to the script directory so that $(pwd) always +# points to the folder containing the server scripts, regardless of +# where the user called sbatch from. +# +# Priority: +# 1. BASH_SOURCE — works for direct invocation (bash script.sh) +# 2. SLURM_SUBMIT_DIR + script path — when sbatch submits from repo root, +# SLURM_SUBMIT_DIR is the CWD, not the script dir. Append the relative +# path from the SBATCH command to get the actual script directory. +# 3. SLURM_SUBMIT_DIR alone — last resort (assumes sbatch was run from +# the script directory). +# ------------------------ +_resolve_script_dir() { + # Try BASH_SOURCE first (works for direct invocation) + if [[ -n "${BASH_SOURCE[0]:-}" ]]; then + local _d + _d="$(cd "$(dirname "${BASH_SOURCE[0]}")" 2>/dev/null && pwd)" + if [[ -n "$_d" && -f "$_d/vllm_disagg.sh" ]]; then + echo "$_d" + return 0 + fi + fi + + # Try SLURM_SUBMIT_DIR + relative script path + # When madengine does: sbatch scripts/vllm_dissag/run_xPyD_models.slurm + # SLURM_SUBMIT_DIR = repo root, so we need to append the dirname + if [[ -n "${SLURM_SUBMIT_DIR:-}" ]]; then + local _candidate="$SLURM_SUBMIT_DIR/scripts/vllm_dissag" + if [[ -f "$_candidate/vllm_disagg.sh" ]]; then + echo "$_candidate" + return 0 + fi + # Maybe they ran sbatch from the script dir itself + if [[ -f "$SLURM_SUBMIT_DIR/vllm_disagg.sh" ]]; then + echo "$SLURM_SUBMIT_DIR" + return 0 + fi + fi + + # Fallback + echo "." + return 1 +} + +SCRIPT_DIR="$(_resolve_script_dir)" +cd "$SCRIPT_DIR" || { echo "Error: cannot cd to $SCRIPT_DIR" >&2; exit 1; } + +REQUIRED_FILES=("vllm_disagg.sh" "parallelism.sh" "connectors/rixl.sh" "connectors/moriio.sh" "models.yaml" "benchmark_xPyD.sh" "parse_to_csv.py" "socket_barrier.py" "socket_wait.py" "connectors/moriio.env" "connectors/rixl.env") +for f in "${REQUIRED_FILES[@]}"; do + if [[ ! -f "$f" ]]; then + echo "Error: Required file '$f' not found in $(pwd)." >&2 + echo "Please run sbatch from the scripts/vllm_dissag/ directory, e.g.:" >&2 + echo " cd MAD/scripts/vllm_dissag && sbatch run_xPyD_models.slurm" >&2 + exit 1 + fi +done +echo "Running from: $(pwd)" + +# ------------------------------------------------------------------------------ +# models.yaml env precedence: capture which recipe knobs the USER explicitly set +# at submit time. The driver (vllm_disagg.sh) uses this to let models.yaml `env:` +# OVERRIDE image-baked ENV defaults (e.g. a DeepSeek-tuned image bakes +# KV_BLOCK_SIZE=16 / VLLM_ROCM_USE_AITER_MLA=0, which would otherwise shadow a +# model's own recipe — GLM-5.1 DSA needs block=1 + AITER MLA on), while a genuine +# submit-time `-e VAR=...` still wins. Precedence: image-baked < models.yaml < submit -e. +# Captured HERE (before the slurm sets any defaults) so it reflects user intent only. +_RECIPE_ENV_KEYS="VLLM_USE_V1 DECODE_MORI_MAX_TOTAL_RECV_TOKENS PREFILL_MORI_MAX_TOTAL_RECV_TOKENS DECODE_MORI_MAX_TOKENS_PER_RANK PREFILL_MORI_MAX_TOKENS_PER_RANK VLLM_MORI_MAX_TOKENS_PER_RANK VLLM_MORI_WARP_NUM_PER_BLOCK VLLM_MORI_BLOCK_NUM VLLM_MORI_RDMA_BLOCK_NUM VLLM_USE_LAYERNAME VLLM_ROCM_USE_AITER VLLM_ROCM_USE_AITER_RMSNORM VLLM_ROCM_USE_AITER_MLA KV_BLOCK_SIZE KV_CACHE_DTYPE KV_CACHE_MEMORY_BYTES GPU_MEMORY_UTILIZATION VLLM_CUDAGRAPH_MODE PREFILL_CUDAGRAPH_MODE DECODE_CUDAGRAPH_MODE CUDAGRAPH_CAPTURE_SIZES VLLM_ALL2ALL_BACKEND PREFILL_MORI_BACKEND DECODE_MORI_BACKEND MORI_SHMEM_HEAP_SIZE" +MODELS_YAML_PROTECT="" +for _k in $_RECIPE_ENV_KEYS; do + [ -n "${!_k+x}" ] && MODELS_YAML_PROTECT="${MODELS_YAML_PROTECT} ${_k}" +done +export MODELS_YAML_PROTECT="${MODELS_YAML_PROTECT# }" +echo "models.yaml protect-list (submit-time overrides): '${MODELS_YAML_PROTECT}'" + +# ------------------------ +# Print current time in UTC and PST formats +# ------------------------ +echo "=== Job Start Time ===" +echo "UTC Time: $(TZ=UTC date '+%Y-%m-%d %H:%M:%S %Z')" +echo "PST Time: $(TZ=America/Los_Angeles date '+%Y-%m-%d %H:%M:%S %Z')" +echo "=======================" +echo "" + +# Define valid model names (must have a models.yaml entry) +VALID_MODELS=( \ + "Llama-3.1-405B-Instruct-FP8-KV" \ + "amd-Llama-3.3-70B-Instruct-FP8-KV" \ + "DeepSeek-V3" \ + "DeepSeek-V3-5layer" \ + "gpt-oss-120b" \ + "DeepSeek-R1" \ + "Qwen3-32B" \ + "Qwen3-30B-A3B" \ + "GLM-5.1-FP8" \ +) + +# Models allowed for CONNECTOR=moriio WIDE_EP=1 (MoRI-EP; legacy RUN_MORI=1) +MORI_EP_VALID_MODELS=( \ + "DeepSeek-V3" \ + "DeepSeek-V3-5layer" \ + "DeepSeek-R1" \ + "GLM-5.1-FP8" \ +) + +# Models allowed for CONNECTOR=rixl WIDE_EP=1 EP_BACKEND=deepep (legacy RUN_DEEPEP=1) +DEEPEP_VALID_MODELS=( \ + "DeepSeek-V3" \ + "DeepSeek-V3-5layer" \ + "DeepSeek-R1" \ +) + +MODEL_NAME="${MODEL_NAME:-None}" + +validate_model_name() { + local is_valid_model=false + + for model in "${VALID_MODELS[@]}"; do + if [[ "$MODEL_NAME" == "$model" ]]; then + is_valid_model=true + break + fi + done + + if ! $is_valid_model; then + printf "Error: Invalid MODEL_NAME: '%s'\nValid models are:\n" "$MODEL_NAME" + for model in "${VALID_MODELS[@]}"; do + printf " - %s\n" "$model" + done + exit 1 + fi + + echo "MODEL_NAME '$MODEL_NAME' is valid." + return 0 +} + +validate_model_name "${MODEL_NAME}" + +model_allows_mori_ep() { + local m="$1" + for x in "${MORI_EP_VALID_MODELS[@]}"; do + [[ "$m" == "$x" ]] && return 0 + done + return 1 +} + +model_allows_deepep() { + local m="$1" + for x in "${DEEPEP_VALID_MODELS[@]}"; do + [[ "$m" == "$x" ]] && return 0 + done + return 1 +} + +# --------------------------------------------------------------------------- +# Axis selection -> single launcher (vllm_disagg.sh). +# Two axes: CONNECTOR={rixl|moriio} x WIDE_EP={0=TP|1=wideEP}; EP_BACKEND only +# when WIDE_EP=1 (rixl->deepep, moriio->mori). Legacy RUN_MORI / RUN_DEEPEP are +# still honored via a back-compat shim. The launcher itself does the final +# CONNECTOR/WIDE_EP/EP_BACKEND validation; here we resolve them + gate the model +# against the existing allowlists. +# --------------------------------------------------------------------------- +RUN_FILE="vllm_disagg.sh" +_run_mori="${RUN_MORI:-0}" +_run_deepep="${RUN_DEEPEP:-0}" + +if [[ "$_run_mori" == "1" && "$_run_deepep" == "1" ]]; then + echo "Error: Both RUN_MORI and RUN_DEEPEP are set to 1. Set only one." >&2 + exit 1 +fi + +# Back-compat: legacy flags map onto the axes when CONNECTOR is not set explicitly. +if [[ -z "${CONNECTOR:-}" ]]; then + if [[ "$_run_mori" == "1" ]]; then + CONNECTOR=moriio; WIDE_EP="${WIDE_EP:-1}"; EP_BACKEND="${EP_BACKEND:-mori}" + elif [[ "$_run_deepep" == "1" ]]; then + CONNECTOR=rixl; WIDE_EP="${WIDE_EP:-1}"; EP_BACKEND="${EP_BACKEND:-deepep}" + else + # Default keeps the historical "no flags" behavior: rixl + TP. + CONNECTOR=rixl; WIDE_EP="${WIDE_EP:-0}" + fi +fi +WIDE_EP="${WIDE_EP:-0}" + +# Models that ONLY run wideEP (DP/EP), never TP: the DeepSeek family is served with +# the MoRI-EP / DeepEP recipe (block=16, MLA off, per-role cudagraph). Running them +# in TP mode is unsupported — the TP argv would double the model's own +# --compilation-config and drop the mandatory +quant_fp8 op. Reject early. +# GLM-5.1-FP8 (GlmMoeDsaForCausalLM, MLA+DSA) is validated only under MoRI-EP +# wideEP disagg (block=1, AITER sparse MLA on, per-role all2all). The moriio+TP +# ("Stage B") path is untested for DSA, so reject WIDE_EP=0 for it too. +WIDE_EP_ONLY_MODELS=( "DeepSeek-V3" "DeepSeek-V3-5layer" "DeepSeek-R1" "GLM-5.1-FP8" ) +model_is_wide_ep_only() { + local m="$1" + for x in "${WIDE_EP_ONLY_MODELS[@]}"; do [[ "$m" == "$x" ]] && return 0; done + return 1 +} + +# Model allowlist gate (kept, sglang-style). wideEP modes use the per-backend +# allowlists; TP uses the global VALID_MODELS (already validated above) minus the +# wideEP-only models. +if [[ "$WIDE_EP" == "0" ]]; then + if model_is_wide_ep_only "$MODEL_NAME"; then + echo "Error: MODEL_NAME '$MODEL_NAME' is wideEP-only (set WIDE_EP=1). TP mode is not supported for it." >&2 + printf "wideEP-only models:\n"; for m in "${WIDE_EP_ONLY_MODELS[@]}"; do printf " - %s\n" "$m"; done + exit 1 + fi +elif [[ "$WIDE_EP" == "1" && "${CONNECTOR}" == "moriio" ]]; then + if ! model_allows_mori_ep "$MODEL_NAME"; then + echo "Error: CONNECTOR=moriio WIDE_EP=1 but MODEL_NAME '$MODEL_NAME' is not in MORI_EP_VALID_MODELS" >&2 + printf "MoRI EP allowed models:\n"; for m in "${MORI_EP_VALID_MODELS[@]}"; do printf " - %s\n" "$m"; done + exit 1 + fi +elif [[ "$WIDE_EP" == "1" && "${CONNECTOR}" == "rixl" ]]; then + if ! model_allows_deepep "$MODEL_NAME"; then + echo "Error: CONNECTOR=rixl WIDE_EP=1 (deepep) but MODEL_NAME '$MODEL_NAME' is not in DEEPEP_VALID_MODELS" >&2 + printf "DeepEP allowed models:\n"; for m in "${DEEPEP_VALID_MODELS[@]}"; do printf " - %s\n" "$m"; done + exit 1 + fi +fi + +export CONNECTOR WIDE_EP EP_BACKEND +echo "Launcher: $RUN_FILE (CONNECTOR=${CONNECTOR} WIDE_EP=${WIDE_EP} EP_BACKEND=${EP_BACKEND:-}) for model '$MODEL_NAME'" + +# --------------------------------------------------------------------------- +# Connector platform env: per-connector .env holds the ROCm-7.2.3 +# runtime env that MUST reach the container at PID 1 (e.g. expandable_segments:False +# for GPU-RDMA registration). Source the resolved connector's file and collect its +# KEY=VALUE lines into CONNECTOR_ENV_ARGS as `-e KEY=${KEY:-VALUE}` pairs, so a +# submit-time export of the same name still overrides. Forwarded in the docker run. +# --------------------------------------------------------------------------- +CONNECTOR_ENV_FILE="${SCRIPT_DIR}/connectors/${CONNECTOR}.env" +CONNECTOR_ENV_ARGS="" +if [[ -f "$CONNECTOR_ENV_FILE" ]]; then + echo "Loading connector platform env: $CONNECTOR_ENV_FILE" + while IFS= read -r _line; do + [[ "$_line" =~ ^[[:space:]]*# || -z "${_line// }" ]] && continue + _k="${_line%%=*}"; _v="${_line#*=}" + CONNECTOR_ENV_ARGS+=" -e ${_k}=${!_k:-$_v}" # submit-time export of $_k wins + done < "$CONNECTOR_ENV_FILE" +else + echo "WARN: connector env file not found: $CONNECTOR_ENV_FILE" >&2 +fi + +if [[ -z "${DOCKER_IMAGE_NAME:-}" ]]; then + echo "Error: DOCKER_IMAGE_NAME is not set. Please export DOCKER_IMAGE_NAME before running." >&2 + echo " There is no public prebuilt image. Build your own from the provided Dockerfile:" >&2 + echo " docker build -f docker/vllm_disagg_inference.ubuntu.amd.Dockerfile \\" >&2 + echo " -t /vllm-disagg:local . # all connectors (WITH_NIXL=1 default)" >&2 + echo " (add --build-arg WITH_NIXL=0 for a lean MoRI-EP-only image)" >&2 + echo " then: export DOCKER_IMAGE_NAME=/vllm-disagg:local" >&2 + exit 1 +fi +export DOCKER_IMAGE_NAME + +# Set current directory to be REPO directory with all relevant scripts +NIXL_REPO_DIR=$(pwd) +LOG_PATH="${LOG_PATH:-/shared_inference/${USER}/model_blog_logs}" + +xP="${xP:-1}" #-> Number of Prefill Servers +yD="${yD:-1}" #-> Number of Decode Servers + +MODEL_DIR="${MODEL_DIR:-"/shared_inference/models_blog/"}" + + +# ------------------------ +# Model path validation and selection across all nodes +# ------------------------ +echo "Looking for model: $MODEL_NAME" +echo "Checking model availability across all allocated nodes..." + +# Get all allocated nodes +ALL_NODES=$(scontrol show hostnames "$SLURM_JOB_NODELIST") +TOTAL_NODES=$(echo "$ALL_NODES" | wc -l) + +echo "Total allocated nodes: $TOTAL_NODES" +echo "Nodes: $(echo "$ALL_NODES" | tr '\n' ' ')" + +# Function to check model path on all nodes +check_model_path() { + local path=$1 + local check_name=$2 + + echo "Checking $check_name: $path" + + # Run check on all nodes in parallel + srun --nodes=$SLURM_NNODES --ntasks=$SLURM_NNODES /bin/bash -c " + if [ -d '$path' ]; then + echo \"\$(hostname): ✓ Found $path\" + exit 0 + else + echo \"\$(hostname): ✗ Missing $path\" + exit 1 + fi + " + + # Check if all nodes succeeded (exit code 0) + local exit_code=$? + if [ $exit_code -eq 0 ]; then + echo "✓ $check_name available on ALL nodes" + return 0 + else + echo "✗ $check_name NOT available on all nodes" + return 1 + fi +} + +# Check /mnt/m2m_nobackup/models_blog first +MODEL_PATH_1="/mnt/m2m_nobackup/models_blog/$MODEL_NAME" +if check_model_path "$MODEL_PATH_1" "/mnt/m2m_nobackup/models_blog"; then + MODEL_PATH="$MODEL_PATH_1" + echo "" + echo "✓ Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" +# Check /shared-inference/models_blog +elif check_model_path "/shared_inference/models_blog/$MODEL_NAME" "/shared_inference/models_blog"; then + MODEL_PATH="/shared_inference/models_blog/$MODEL_NAME" + echo "" + echo "✓ Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" +elif check_model_path "$MODEL_DIR/$MODEL_NAME" "$MODEL_DIR"; then + MODEL_PATH="$MODEL_DIR/$MODEL_NAME" + echo "" + echo "✓ Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" +else + echo "" + echo "✗ FATAL ERROR: Model '$MODEL_NAME' not found on ALL allocated nodes in either:" + echo " - /mnt/m2m_nobackup/models_blog/$MODEL_NAME" + echo " - /shared_inference/models_blog/$MODEL_NAME" + echo "" + echo "Model must be accessible from all nodes for distributed execution." + echo "Please ensure the model is available on all allocated nodes." + exit 1 +fi + +echo "Final MODEL_PATH: $MODEL_PATH" +echo "" + + +# Calculate NUM_NODES based on xP and yD +NUM_NODES=$((xP + yD)) +echo "Calculated NUM_NODES: $NUM_NODES (xP=$xP + yD=$yD, proxy co-located on prefill master)" + +# DeepEP configuration (only exported when RUN_DEEPEP=1) +if [[ "$_run_deepep" == "1" ]]; then + export PREFILL_DEEPEP_BACKEND="${PREFILL_DEEPEP_BACKEND:-deepep_high_throughput}" + export DECODE_DEEPEP_BACKEND="${DECODE_DEEPEP_BACKEND:-deepep_low_latency}" + export ENABLE_DBO="${ENABLE_DBO:-false}" + export DBO_COMM_SMS="${DBO_COMM_SMS:-}" + export ENABLE_PROFILING="${ENABLE_PROFILING:-false}" + echo "DeepEP config: PREFILL_BACKEND=$PREFILL_DEEPEP_BACKEND DECODE_BACKEND=$DECODE_DEEPEP_BACKEND DBO=$ENABLE_DBO" +fi + +# ------------------------ +# Extract first NUM_NODES from SLURM allocation and update SLURM variables +# ------------------------ +echo "Original SLURM allocation:" +echo "SLURM_JOB_NODELIST: $SLURM_JOB_NODELIST" +echo "SLURM_NNODES: $SLURM_NNODES" +echo "SLURM_NTASKS: $SLURM_NTASKS" + +# Get the full nodelist and extract first NUM_NODES +FULL_NODELIST=$(scontrol show hostnames "$SLURM_JOB_NODELIST") +SELECTED_NODES=$(echo "$FULL_NODELIST" | head -n $NUM_NODES) +NEW_SLURM_NODELIST=$(echo "$SELECTED_NODES" | paste -sd,) + +# Update SLURM environment variables +export SLURM_NNODES=$NUM_NODES +export SLURM_NTASKS=$NUM_NODES +export SLURM_JOB_NUM_NODES=$NUM_NODES +export SLURM_NPROCS=$NUM_NODES +export SLURM_JOB_NODELIST="$NEW_SLURM_NODELIST" +export SLURM_NODELIST="$NEW_SLURM_NODELIST" + +# Keep other SLURM variables as they were or set defaults +export SLURM_TASKS_PER_NODE="1(x$NUM_NODES)" + +export SLURM_CLUSTER_NAME="${SLURM_CLUSTER_NAME}" +export SLURM_JOB_CPUS_PER_NODE="${SLURM_JOB_CPUS_PER_NODE}" +export SLURM_JOB_PARTITION="${SLURM_JOB_PARTITION}" +export SLURM_JOBID="${SLURM_JOBID:-$SLURM_JOB_ID}" +export SLURM_JOB_QOS="${SLURM_JOB_QOS:-normal}" +export SLURM_JOB_ACCOUNT="${SLURM_JOB_ACCOUNT}" +export SLURM_NTASKS_PER_NODE=1 +export SLURM_SUBMIT_HOST="${SLURM_SUBMIT_HOST}" +export SLURM_JOB_ID="${SLURM_JOB_ID}" +export SLURM_CONF="${SLURM_CONF:-/etc/slurm/slurm.conf}" +export SLURM_JOB_NAME="${SLURM_JOB_NAME:-1p1d_bench-serving}" + +echo "" +echo "Updated SLURM Environment Variables:" +echo "SLURM_JOB_ID: $SLURM_JOB_ID" +echo "SLURM_JOB_NODELIST: $SLURM_JOB_NODELIST" +echo "SLURM_NNODES: $SLURM_NNODES" +echo "SLURM_NTASKS: $SLURM_NTASKS" +echo "SLURM_TASKS_PER_NODE: $SLURM_TASKS_PER_NODE" +echo "SLURM_JOB_CPUS_PER_NODE: $SLURM_JOB_CPUS_PER_NODE" +echo "SLURM_JOB_PARTITION: $SLURM_JOB_PARTITION" +echo "SLURM_JOB_NUM_NODES: $SLURM_JOB_NUM_NODES" +echo "SLURM_JOBID: $SLURM_JOBID" +echo "SLURM_JOB_QOS: $SLURM_JOB_QOS" +echo "SLURM_NODELIST: $SLURM_NODELIST" +echo "SLURM_JOB_ACCOUNT: $SLURM_JOB_ACCOUNT" +echo "SLURM_NPROCS: $SLURM_NPROCS" +echo "SLURM_SUBMIT_HOST: $SLURM_SUBMIT_HOST" +echo "SLURM_CONF: $SLURM_CONF" +echo "SLURM_JOB_NAME: $SLURM_JOB_NAME" +echo "SLURM_NTASKS_PER_NODE: $SLURM_NTASKS_PER_NODE" +#echo "SLURM_SUBMIT_DIR: $SLURM_SUBMIT_DIR" +echo "SLURM_CLUSTER_NAME: $SLURM_CLUSTER_NAME" +echo "ulimit: $(ulimit -a)" +echo "" +echo "Selected nodes for execution:" +echo "$SELECTED_NODES" +echo "" + +# Node information +USER_NAME=$(whoami) +MASTER_NODE=$(echo "$SELECTED_NODES" | head -n 1) +# Pick the routable fabric IP, not just hostname -I's first entry. These nodes expose +# multiple NICs (e.g. a 10.224.x overlay listed BEFORE the routable 10.158.x fabric); +# taking $1 blindly can advertise an unreachable addr -> prefill/decode barrier hangs +# "Waiting for nodes" forever. Prefer FABRIC_SUBNET (default 10.158.), fall back to $1. +FABRIC_SUBNET="${FABRIC_SUBNET:-10.158.}" +# From a "hostname -I" line, return the first IP on FABRIC_SUBNET, else the first IP. +_pick_fabric_ip() { + awk -v pfx="$FABRIC_SUBNET" '{f=$1; for(i=1;i<=NF;i++) if(index($i,pfx)==1){f=$i; break} print f}' +} +MASTER_ADDR=$(srun --nodes=1 --ntasks=1 --time=00:20:00 --nodelist="$MASTER_NODE" bash -c 'hostname -I' | _pick_fabric_ip) +MASTER_PORT=39566 # Choose an open port + +IPS=() + +for NODE in $SELECTED_NODES; do + IP=$(srun --nodes=1 --ntasks=1 --time=00:20:00 --nodelist="$NODE" bash -c 'hostname -I' | _pick_fabric_ip) + IPS+=("$IP") +done + +echo "Selected node IPs: ${IPS[*]}" | sed 's/ /,/g' + +NIXL_COOKBOOK_PATH="/opt/nixl-vllm-cookbook" +BENCHMARK_ITR="${BENCHMARK_ITR:-1}" +BENCHMARK_CON="${BENCHMARK_CON:-}" +BENCHMARK_COMBINATIONS="${BENCHMARK_COMBINATIONS:-}" + +# Benchmark script selector: BENCHMARK_SCRIPT tag -> file run by the launcher. +# sweep (default) -> benchmark_xPyD.sh (general concurrency sweep) +# long_context -> benchmark_long_context.sh (per-shape warmup, c=1-first) +# keepalive -> keepalive_bench.sh (hold server up KEEPALIVE_MINS +# for external accuracy probes) +BENCHMARK_SCRIPT="${BENCHMARK_SCRIPT:-sweep}" +case "$BENCHMARK_SCRIPT" in + sweep) BENCHMARK_SCRIPT_FILE="benchmark_xPyD.sh" ;; + long_context) BENCHMARK_SCRIPT_FILE="benchmark_long_context.sh" ;; + keepalive) BENCHMARK_SCRIPT_FILE="keepalive_bench.sh" ;; + *) echo "Error: invalid BENCHMARK_SCRIPT='$BENCHMARK_SCRIPT' (valid: sweep, long_context, keepalive)" >&2; exit 1 ;; +esac +if [[ ! -f "$BENCHMARK_SCRIPT_FILE" ]]; then + echo "Error: selected benchmark script '$BENCHMARK_SCRIPT_FILE' not found in $(pwd)." >&2 + exit 1 +fi +echo "BENCHMARK_SCRIPT='$BENCHMARK_SCRIPT' -> $BENCHMARK_SCRIPT_FILE" + +NNODES=$NUM_NODES + +echo "MASTER_NODE is ${MASTER_NODE}" +echo "MASTER_ADDR is ${MASTER_ADDR}" +echo "MASTER_PORT is ${MASTER_PORT}" +echo "NNODES is ${NNODES}" +echo "REPO Directory is ${NIXL_REPO_DIR}" + +if [ ! -d "$LOG_PATH" ]; then + mkdir -p "$LOG_PATH" + echo "Created directory: $LOG_PATH" +else + echo "Directory already exists: $LOG_PATH" +fi + +export CONNECTOR_ENV_ARGS="$CONNECTOR_ENV_ARGS" +export LOG_PATH=$LOG_PATH +export NIXL_REPO_DIR=$NIXL_REPO_DIR +export NIXL_COOKBOOK_PATH=$NIXL_COOKBOOK_PATH +export NNODES=$NNODES +export MASTER_ADDR=$MASTER_ADDR +export MASTER_PORT=$MASTER_PORT +export MODEL_PATH=$MODEL_PATH +export xP=$xP +export yD=$yD +export MODEL_NAME=$MODEL_NAME +export USER_NAME=$USER_NAME +export IPADDRS="$(echo "${IPS[*]}" | sed 's/ /,/g')" +export BENCHMARK_ITR=$BENCHMARK_ITR +export BENCHMARK_CON="${BENCHMARK_CON}" +export BENCHMARK_COMBINATIONS="${BENCHMARK_COMBINATIONS}" +export BENCHMARK_SCRIPT_FILE="${BENCHMARK_SCRIPT_FILE}" + +export DOCKER_CONT_NAME="container_${MODEL_NAME}_${SLURM_JOB_ID}" +export RUN_FILE_FULL="$NIXL_COOKBOOK_PATH/${RUN_FILE}" + +# Use only the selected nodes for srun execution +SELECTED_NODELIST_SRUN=$(echo "$SELECTED_NODES" | paste -sd,) + +srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c ' +echo "Rank $SLURM_PROCID on $(hostname)"; +docker ps -q | xargs --no-run-if-empty docker stop; +docker rm -f $DOCKER_CONT_NAME 2>/dev/null || true; +fuser -k 5000/tcp 2>/dev/null || true; +fuser -k 2222/tcp 2>/dev/null || true; +fuser -k 15000/tcp 2>/dev/null || true; +sleep 2; +docker pull $DOCKER_IMAGE_NAME 2>/dev/null || true; + +# --- Create host-local compilation cache dirs (ext4, survives container restarts) --- +mkdir -p /tmp/vllm_cache/{aiter_jit,triton,vllm,comgr} 2>/dev/null || true; + +# --- Persistent JIT cache mount --- +# The image points AITER_JIT_DIR/TRITON_CACHE_DIR/VLLM_CACHE_ROOT/COMGR_CACHE_DIR at +# /opt/vllm_cache. Mount a host dir there so AITER CK kernels compile ONCE and are reused +# across runs. Cold compile is ~15 min for the DeepSeek MoE/FP8 GEMM kernel set; a warm +# boot is ~1 min. Keyed by image ID so a new image starts a fresh cache and does not reuse +# stale-ABI shared objects. Host dir on local NVMe. Override JIT_CACHE_HOST, or set +# JIT_CACHE_PERSIST=0 to disable and fall back to the image empty in-container cache. +# NOTE: this whole section runs inside a single-quoted `srun bash -c '...'`, so avoid +# single quotes here; the image-id hash is extracted with tr, not sed. +if [ "${JIT_CACHE_PERSIST:-1}" = "1" ]; then + _IMG_RAW=$(docker image inspect --format "{{.Id}}" "$DOCKER_IMAGE_NAME" 2>/dev/null); + _IMG_KEY=$(printf "%s" "$_IMG_RAW" | tr -cd "a-f0-9" | cut -c1-12); + _IMG_KEY="${_IMG_KEY:-noimg}"; + _JIT_CACHE_HOST="${JIT_CACHE_HOST:-/mnt/m2m_nobackup/${USER}/vllm_jit_cache/${_IMG_KEY}}"; + mkdir -p "$_JIT_CACHE_HOST"/{aiter_jit,triton,vllm,comgr} 2>/dev/null || true; + _JIT_CACHE_MOUNT="-v ${_JIT_CACHE_HOST}:/opt/vllm_cache"; + echo "[jit-cache] persistent image ${_IMG_KEY}: ${_JIT_CACHE_HOST} to /opt/vllm_cache"; +else + _JIT_CACHE_MOUNT=""; +fi + +# --- Build host RDMA library mounts --- +_RDMA_MOUNTS="" +_LIBDIR=/usr/lib/x86_64-linux-gnu + +for _lib in libibverbs.so libibverbs.so.1 librdmacm.so librdmacm.so.1; do + [ -e "$_LIBDIR/$_lib" ] && _RDMA_MOUNTS="$_RDMA_MOUNTS -v $_LIBDIR/$_lib:$_LIBDIR/$_lib:ro" +done +for _vlib in $_LIBDIR/libibverbs.so.1.* $_LIBDIR/librdmacm.so.1.*; do + [ -e "$_vlib" ] && _RDMA_MOUNTS="$_RDMA_MOUNTS -v $_vlib:$_vlib:ro" +done + +for _pattern in libmlx5.so* libionic*.so* libbnxt_re*.so* libefa.so* libhns.so*; do + for _vlib in $_LIBDIR/${_pattern}; do + # Require a regular file AFTER symlink resolution: these mounts are built on + # ONE node but applied on ALL nodes, and vendor NIC libs (e.g. libionic.so.1) + # can be a DANGLING symlink on some nodes -> bind-mount fails "not a directory" + # -> container create exit 125. `-f` (follows symlink, requires regular file) + # skips those; the fabric in use (mlx5) is still mounted where present. + [ -f "$_vlib" ] && _RDMA_MOUNTS="$_RDMA_MOUNTS -v $_vlib:$_vlib:ro" + done +done + +[ -d "$_LIBDIR/libibverbs" ] && _RDMA_MOUNTS="$_RDMA_MOUNTS -v $_LIBDIR/libibverbs:$_LIBDIR/libibverbs:ro" +[ -d /etc/libibverbs.d ] && _RDMA_MOUNTS="$_RDMA_MOUNTS -v /etc/libibverbs.d:/etc/libibverbs.d:ro" +echo "[host-rdma] mounts: $_RDMA_MOUNTS" + +docker run --rm \ + --device /dev/dri \ + --device /dev/kfd \ + --device /dev/infiniband \ + --network host \ + --ipc host \ + --group-add video \ + --cap-add SYS_PTRACE \ + --security-opt seccomp=unconfined \ + --privileged \ + -v $HOME:$HOME \ + -v /shared_inference:/shared_inference \ + -v /mnt/m2m_nobackup:/mnt/m2m_nobackup \ + -v $HOME/.ssh:/root/.ssh \ + --shm-size ${DOCKER_SHM_SIZE:-256G} \ + --ulimit nofile=524288:524288 \ + --ulimit memlock=-1:-1 \ + -v ${LOG_PATH}:/run_logs \ + -v $NIXL_REPO_DIR:$NIXL_COOKBOOK_PATH \ + -v /tmp/vllm_cache:/tmp/vllm_cache \ + ${_JIT_CACHE_MOUNT} \ + ${GLM_KERNEL_PATCH:+-v ${GLM_KERNEL_PATCH}:/usr/local/lib/python3.12/dist-packages/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py:ro} \ + ${GLM_BACKEND_PATCH:+-v ${GLM_BACKEND_PATCH}:/usr/local/lib/python3.12/dist-packages/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py:ro} \ + $_RDMA_MOUNTS \ + --entrypoint /bin/bash \ + -e SLURM_JOB_ID=$SLURM_JOB_ID \ + -e SLURM_JOB_NODELIST=$SLURM_JOB_NODELIST \ + -e NNODES=$NNODES \ + -e NODE_RANK=$SLURM_PROCID \ + -e MASTER_ADDR=$MASTER_ADDR \ + -e MASTER_PORT=$MASTER_PORT \ + -e MODEL_PATH=$MODEL_PATH \ + -e NIXL_COOKBOOK_PATH=$NIXL_COOKBOOK_PATH \ + -e xP=$xP \ + -e yD=$yD \ + -e USER_NAME=$USER_NAME \ + -e MODEL_NAME=$MODEL_NAME \ + -e BENCHMARK_ITR=$BENCHMARK_ITR \ + -e BENCHMARK_CON="${BENCHMARK_CON}" \ + -e BENCHMARK_COMBINATIONS="${BENCHMARK_COMBINATIONS}" \ + ${BENCHMARK_PORT:+-e BENCHMARK_PORT=$BENCHMARK_PORT} \ + ${PROXY_TYPE:+-e PROXY_TYPE=$PROXY_TYPE} \ + ${ROUTER_PORT:+-e ROUTER_PORT=$ROUTER_PORT} \ + -e IPADDRS=$IPADDRS \ + ${CONNECTOR:+-e CONNECTOR=$CONNECTOR} \ + ${WIDE_EP:+-e WIDE_EP=$WIDE_EP} \ + ${EP_BACKEND:+-e EP_BACKEND=$EP_BACKEND} \ + ${RUN_MORI:+-e RUN_MORI=$RUN_MORI} \ + ${RUN_DEEPEP:+-e RUN_DEEPEP=$RUN_DEEPEP} \ + ${VLLM_ALL2ALL_BACKEND:+-e VLLM_ALL2ALL_BACKEND=$VLLM_ALL2ALL_BACKEND} \ + ${PREFILL_MORI_BACKEND:+-e PREFILL_MORI_BACKEND=$PREFILL_MORI_BACKEND} \ + ${DECODE_MORI_BACKEND:+-e DECODE_MORI_BACKEND=$DECODE_MORI_BACKEND} \ + -e MODELS_YAML_PROTECT="${MODELS_YAML_PROTECT:-}" \ + ${GLM_PERSIST_GATE:+-e GLM_PERSIST_GATE=$GLM_PERSIST_GATE} \ + ${GLM_SKIP_PATCHERS:+-e GLM_SKIP_PATCHERS=$GLM_SKIP_PATCHERS} \ + ${KV_BLOCK_SIZE:+-e KV_BLOCK_SIZE=$KV_BLOCK_SIZE} \ + ${KV_CACHE_MEMORY_BYTES:+-e KV_CACHE_MEMORY_BYTES=$KV_CACHE_MEMORY_BYTES} \ + ${VLLM_ROCM_USE_AITER_MLA:+-e VLLM_ROCM_USE_AITER_MLA=$VLLM_ROCM_USE_AITER_MLA} \ + ${ROUTER_BINARY:+-e ROUTER_BINARY=$ROUTER_BINARY} \ + ${KV_CACHE_DTYPE:+-e KV_CACHE_DTYPE=$KV_CACHE_DTYPE} \ + ${MORIIO_TOY_PROXY:+-e MORIIO_TOY_PROXY=$MORIIO_TOY_PROXY} \ + ${BENCHMARK_SCRIPT_FILE:+-e BENCHMARK_SCRIPT_FILE=$BENCHMARK_SCRIPT_FILE} \ + ${KEEPALIVE_MINS:+-e KEEPALIVE_MINS=$KEEPALIVE_MINS} \ + ${PREFILL_CUDAGRAPH_MODE:+-e PREFILL_CUDAGRAPH_MODE=$PREFILL_CUDAGRAPH_MODE} \ + ${DECODE_CUDAGRAPH_MODE:+-e DECODE_CUDAGRAPH_MODE=$DECODE_CUDAGRAPH_MODE} \ + ${CUDAGRAPH_CAPTURE_SIZES:+-e CUDAGRAPH_CAPTURE_SIZES="$CUDAGRAPH_CAPTURE_SIZES"} \ + ${MORI_RDMA_TC:+-e MORI_RDMA_TC=$MORI_RDMA_TC} \ + ${MORI_RDMA_SL:+-e MORI_RDMA_SL=$MORI_RDMA_SL} \ + ${MORI_SHMEM_HEAP_SIZE:+-e MORI_SHMEM_HEAP_SIZE=$MORI_SHMEM_HEAP_SIZE} \ + ${PREFILL_DEEPEP_BACKEND:+-e PREFILL_DEEPEP_BACKEND=$PREFILL_DEEPEP_BACKEND} \ + ${DECODE_DEEPEP_BACKEND:+-e DECODE_DEEPEP_BACKEND=$DECODE_DEEPEP_BACKEND} \ + ${ENABLE_DBO:+-e ENABLE_DBO=$ENABLE_DBO} \ + ${DBO_COMM_SMS:+-e DBO_COMM_SMS=$DBO_COMM_SMS} \ + ${ENABLE_PROFILING:+-e ENABLE_PROFILING=$ENABLE_PROFILING} \ + ${NCCL_IB_HCA:+-e NCCL_IB_HCA=$NCCL_IB_HCA} \ + ${NCCL_IB_GID_INDEX:+-e NCCL_IB_GID_INDEX=$NCCL_IB_GID_INDEX} \ + ${NCCL_NET_GDR_LEVEL:+-e NCCL_NET_GDR_LEVEL=$NCCL_NET_GDR_LEVEL} \ + ${NCCL_CROSS_NIC:+-e NCCL_CROSS_NIC=$NCCL_CROSS_NIC} \ + ${NCCL_SOCKET_IFNAME:+-e NCCL_SOCKET_IFNAME=$NCCL_SOCKET_IFNAME} \ + ${GLOO_SOCKET_IFNAME:+-e GLOO_SOCKET_IFNAME=$GLOO_SOCKET_IFNAME} \ + -e MORI_SOCKET_IFNAME=${MORI_SOCKET_IFNAME:-eth0} \ + ${MORI_IB_GID_INDEX:+-e MORI_IB_GID_INDEX=$MORI_IB_GID_INDEX} \ + ${MORI_RDMA_DEVICES:+-e MORI_RDMA_DEVICES=$MORI_RDMA_DEVICES} \ + ${MORI_NUM_QP_PER_PE:+-e MORI_NUM_QP_PER_PE=$MORI_NUM_QP_PER_PE} \ + ${VLLM_MORIIO_QP_PER_TRANSFER:+-e VLLM_MORIIO_QP_PER_TRANSFER=$VLLM_MORIIO_QP_PER_TRANSFER} \ + ${VLLM_MORIIO_NUM_WORKERS:+-e VLLM_MORIIO_NUM_WORKERS=$VLLM_MORIIO_NUM_WORKERS} \ + -e GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.8} \ + -e GPUS_PER_NODE=${GPUS_PER_NODE:-8} \ + ${GPU_MAX_HW_QUEUES:+-e GPU_MAX_HW_QUEUES=$GPU_MAX_HW_QUEUES} \ + ${HIP_FORCE_DEV_KERNARG:+-e HIP_FORCE_DEV_KERNARG=$HIP_FORCE_DEV_KERNARG} \ + ${HSA_NO_SCRATCH_RECLAIM:+-e HSA_NO_SCRATCH_RECLAIM=$HSA_NO_SCRATCH_RECLAIM} \ + ${VLLM_HANDSHAKE_TIMEOUT_MINS:+-e VLLM_HANDSHAKE_TIMEOUT_MINS=$VLLM_HANDSHAKE_TIMEOUT_MINS} \ + ${VLLM_ENGINE_READY_TIMEOUT_S:+-e VLLM_ENGINE_READY_TIMEOUT_S=$VLLM_ENGINE_READY_TIMEOUT_S} \ + ${ROCSHMEM_HEAP_SIZE:+-e ROCSHMEM_HEAP_SIZE=$ROCSHMEM_HEAP_SIZE} \ + ${ROCSHMEM_MAX_NUM_CONTEXTS:+-e ROCSHMEM_MAX_NUM_CONTEXTS=$ROCSHMEM_MAX_NUM_CONTEXTS} \ + ${LOG_WAIT_TIMEOUT_SECONDS:+-e LOG_WAIT_TIMEOUT_SECONDS=$LOG_WAIT_TIMEOUT_SECONDS} \ + ${TRITON_CACHE_DIR:+-e TRITON_CACHE_DIR=$TRITON_CACHE_DIR} \ + ${VLLM_CACHE_ROOT:+-e VLLM_CACHE_ROOT=$VLLM_CACHE_ROOT} \ + ${COMGR_CACHE_DIR:+-e COMGR_CACHE_DIR=$COMGR_CACHE_DIR} \ + ${AITER_JIT_DIR:+-e AITER_JIT_DIR=$AITER_JIT_DIR} \ + -e DISTRIBUTED_TIMEOUT_SECONDS=${DISTRIBUTED_TIMEOUT_SECONDS:-7200} \ + -e VLLM_RPC_TIMEOUT=${VLLM_RPC_TIMEOUT:-300000} \ + -e VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=${VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS:-3600} \ + ${CONNECTOR_ENV_ARGS} \ + ${VLLM_CUDAGRAPH_MODE:+-e VLLM_CUDAGRAPH_MODE=$VLLM_CUDAGRAPH_MODE} \ + ${CUDAGRAPH_CAPTURE_SIZES:+-e CUDAGRAPH_CAPTURE_SIZES="$CUDAGRAPH_CAPTURE_SIZES"} \ + --name $DOCKER_CONT_NAME \ + $DOCKER_IMAGE_NAME -c " + mkdir -p /run_logs/${SLURM_JOB_ID} + $RUN_FILE_FULL 2>&1 | tee /run_logs/${SLURM_JOB_ID}/pd_vllm_bench_NODE${SLURM_PROCID}.log + " +' +srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'docker stop $DOCKER_CONT_NAME 2>/dev/null || true; docker rm $DOCKER_CONT_NAME 2>/dev/null || true' + diff --git a/scripts/vllm_dissag/connectors/moriio.sh b/scripts/vllm_dissag/connectors/moriio.sh index 33d5322d..e1c7a51a 100644 --- a/scripts/vllm_dissag/connectors/moriio.sh +++ b/scripts/vllm_dissag/connectors/moriio.sh @@ -274,24 +274,15 @@ connector_launch_worker() { local _all2all="${PREFILL_MORI_BACKEND}" [[ "$log_prefix" == "decode" ]] && _all2all="${DECODE_MORI_BACKEND}" - # Per-role MoRI EP buffer width. VLLM_MORI_MAX_TOKENS_PER_RANK sizes the - # dispatch/combine buffer; unset (0) it inherits max_num_batched_tokens -- a - # chunked-prefill SCHEDULER knob (8192) -- so a decode instance moves an - # 8192-token-wide buffer every step, per layer: ~302ms vs ~88ms TPOT (3.4x). - # Prefill and decode want OPPOSITE values (prefill genuinely dispatches wide - # batches and must keep the large buffer), but models.yaml env: applies to BOTH - # roles -- so split it here, mirroring PREFILL/DECODE_MORI_BACKEND above. - if [[ "$log_prefix" == "decode" ]]; then - [[ -n "${DECODE_MORI_MAX_TOKENS_PER_RANK:-}" ]] && \ - export VLLM_MORI_MAX_TOKENS_PER_RANK="${DECODE_MORI_MAX_TOKENS_PER_RANK}" - # Recv capacity must still cover vLLM's profiling dummy run (which pushes - # max_num_batched_tokens tokens) even though steady-state dispatch is narrow. - [[ -n "${DECODE_MORI_MAX_TOTAL_RECV_TOKENS:-}" ]] && \ - export VLLM_MORI_MAX_TOTAL_RECV_TOKENS="${DECODE_MORI_MAX_TOTAL_RECV_TOKENS}" - else - export VLLM_MORI_MAX_TOKENS_PER_RANK="${PREFILL_MORI_MAX_TOKENS_PER_RANK:-0}" - export VLLM_MORI_MAX_TOTAL_RECV_TOKENS="${PREFILL_MORI_MAX_TOTAL_RECV_TOKENS:-0}" - fi + # NOTE on MoRI EP buffer width: it is sized from max_num_batched_tokens + # (fused_moe/layer.py -> all2all.py max_num_inp_token_per_rank), so a decode + # instance otherwise runs an 8192-token-wide all2all every step (~302ms vs ~88ms + # TPOT). The fix is per-role `--max-num-batched-tokens` in models.yaml + # (decode.dp), NOT an env knob: mori derives recv capacity from the send width + # (MaxNumTokensToRecvPerRank returns min(ceil(maxTotalRecvTokens/ws), + # maxNumInpTokenPerRank)), so shrinking the width alone under-provisions recv and + # trips a device assert during vLLM's profiling dummy run. See + # skills_vllm_disagg.md for the measurements and the dead ends. local extra_args=() kv_args=() if [[ "$role" == "master" ]]; then diff --git a/scripts/vllm_dissag/models.yaml b/scripts/vllm_dissag/models.yaml index e0017704..064ced6d 100644 --- a/scripts/vllm_dissag/models.yaml +++ b/scripts/vllm_dissag/models.yaml @@ -261,16 +261,8 @@ GLM-5.1-FP8: # decode step moves an 8192-token-wide buffer per layer x78 layers regardless of the # real batch. That is a fixed ~300ms/step floor (~320x this model's HBM-bandwidth # bound). Sizing it for the actual decode batch gives TPOT 302ms -> 88ms (3.4x), - # matching the published EP8 figure. Needs vLLM >= e8c186f71b. - # Prefill is unaffected (it genuinely dispatches wide batches). - # Per-role (moriio.sh -> VLLM_MORI_MAX_TOKENS_PER_RANK). MUST be >= max_num_seqs * - # num_experts_per_token (top-8 here): the value is also the RECV capacity - # (mori MaxNumTokensToRecv = worldSize * this). 512 tripped a device assert - # "Total recv token overflow". 2048 is safe and still 4x smaller than 8192. Decode wants - # the buffer sized to its real batch; prefill must KEEP the wide buffer (0 = inherit - # max_num_batched_tokens) because it genuinely dispatches 8192-token chunks. - # Recv capacity stays wide enough for the profiling dummy run (8192 tokens) and - # bursty top-8 routing: worldSize * ceil(65536/worldSize) >= 8192 for EP8..EP32. + # matching the published EP8 figure. The knob is decode.dp below -- prefill is + # unaffected (it genuinely dispatches wide 8192-token chunked-prefill batches). MORI_SHMEM_HEAP_SIZE: "17179869184" # DSA sparse-indexer logits-buffer cap (crash fix). The indexer prefill computes an # M*N fp32 logits buffer; split_indexer_prefill_chunks only sub-chunks the query dim diff --git a/scripts/vllm_dissag/run_xPyD_models.slurm b/scripts/vllm_dissag/run_xPyD_models.slurm index 52d046a1..365883b8 100755 --- a/scripts/vllm_dissag/run_xPyD_models.slurm +++ b/scripts/vllm_dissag/run_xPyD_models.slurm @@ -76,7 +76,7 @@ echo "Running from: $(pwd)" # model's own recipe — GLM-5.1 DSA needs block=1 + AITER MLA on), while a genuine # submit-time `-e VAR=...` still wins. Precedence: image-baked < models.yaml < submit -e. # Captured HERE (before the slurm sets any defaults) so it reflects user intent only. -_RECIPE_ENV_KEYS="VLLM_USE_V1 DECODE_MORI_MAX_TOTAL_RECV_TOKENS PREFILL_MORI_MAX_TOTAL_RECV_TOKENS DECODE_MORI_MAX_TOKENS_PER_RANK PREFILL_MORI_MAX_TOKENS_PER_RANK VLLM_MORI_MAX_TOKENS_PER_RANK VLLM_MORI_WARP_NUM_PER_BLOCK VLLM_MORI_BLOCK_NUM VLLM_MORI_RDMA_BLOCK_NUM VLLM_USE_LAYERNAME VLLM_ROCM_USE_AITER VLLM_ROCM_USE_AITER_RMSNORM VLLM_ROCM_USE_AITER_MLA KV_BLOCK_SIZE KV_CACHE_DTYPE KV_CACHE_MEMORY_BYTES GPU_MEMORY_UTILIZATION VLLM_CUDAGRAPH_MODE PREFILL_CUDAGRAPH_MODE DECODE_CUDAGRAPH_MODE CUDAGRAPH_CAPTURE_SIZES VLLM_ALL2ALL_BACKEND PREFILL_MORI_BACKEND DECODE_MORI_BACKEND MORI_SHMEM_HEAP_SIZE" +_RECIPE_ENV_KEYS="VLLM_USE_V1 VLLM_USE_LAYERNAME VLLM_ROCM_USE_AITER VLLM_ROCM_USE_AITER_RMSNORM VLLM_ROCM_USE_AITER_MLA KV_BLOCK_SIZE KV_CACHE_DTYPE KV_CACHE_MEMORY_BYTES GPU_MEMORY_UTILIZATION VLLM_CUDAGRAPH_MODE PREFILL_CUDAGRAPH_MODE DECODE_CUDAGRAPH_MODE CUDAGRAPH_CAPTURE_SIZES VLLM_ALL2ALL_BACKEND PREFILL_MORI_BACKEND DECODE_MORI_BACKEND MORI_SHMEM_HEAP_SIZE" MODELS_YAML_PROTECT="" for _k in $_RECIPE_ENV_KEYS; do [ -n "${!_k+x}" ] && MODELS_YAML_PROTECT="${MODELS_YAML_PROTECT} ${_k}" From 638a033a38b0a73f3d47c43fb9318d4980405be0 Mon Sep 17 00:00:00 2001 From: raviguptaamd Date: Mon, 17 Aug 2026 01:58:40 +0000 Subject: [PATCH 09/23] [GLM-5.1] Decode CUDA-graph FULL_AND_PIECEWISE: 2.1x (EP8) / 1.7x (EP16) faster decode Second perf lever, on top of the decode --max-num-batched-tokens fix. Switching the decode role from PIECEWISE to FULL_AND_PIECEWISE captures a FULL graph for the uniform-decode shapes and falls back to PIECEWISE for the rest. Measured on MI300X, ISL/OSL 1024/64, warm (first post-boot run discarded), image rocmshared/pytorch-private:glm5.1-pr176-verify: PIECEWISE FULL_AND_PIECEWISE gain 1P/1D EP8 TPOT c8 88.0 ms 41.8 ms 2.11x 1P/1D EP8 TPOT c16 91.5 ms 45.8 ms 2.00x 2P/2D EP16 TPOT c8 94.1 ms 55.4 ms 1.70x 2P/2D EP16 TPOT c16 96.2 ms 60.3 ms 1.60x 1P/1D outTP c8 78.8 tok/s 143.7 tok/s 1.82x 2P/2D outTP c8 66.7 tok/s 113.0 tok/s 1.69x 2P/2D TTFT c16 2221 ms 1400 ms 1.59x Both topologies now sit well below the published ~89 ms (EP8) / ~91 ms (EP16) reference. Accuracy is unaffected. NIAH 2k-35k: 1P/1D 51/60, 2P/2D 55/60 - both inside the PIECEWISE band (52-53 and 53 respectively; 2P/2D is in fact the best NIAH run of the campaign). No length collapse, no '!!!', memfault=0, latencies equal-or-better at every length. Caveats, stated rather than buried: - The gain SHRINKS as EP widens (2.11x at EP8 -> 1.70x at EP16). FULL captures the whole decode step including the cross-node all2all, which does not compress. This is validated at EP8 and EP16 ONLY; do not assume it holds at EP32 (which has a separate, unrelated correctness regression). - Capture costs ~92-94 s and ~3.0-3.5 GiB vs ~5 s / 7.2 GiB for PIECEWISE (fewer but larger graphs), so boot is longer. Irrelevant for a long-lived server, notable for CI. - Set DECODE_CUDAGRAPH_MODE=PIECEWISE to fall back. Two code paths branch on CUDAGraphMode.FULL (sparse_attn_indexer.py:411 and the MoRIIO READ-mode barrier); both are guards that skip host-side work under FULL, so enabling it is safe here - the MoRIIO one is already a no-op in WRITE mode. skills_vllm_disagg.md gains a section 2b with the table, the EP-width caveat, the capture cost, and the observation that FULL is far less cold-JIT sensitive (its warmup run already reads steady-state TPOT, where PIECEWISE's first run showed 13.7 s TTFT). Co-Authored-By: Claude --- scripts/vllm_dissag/models.yaml | 14 +++++++++- scripts/vllm_dissag/skills_vllm_disagg.md | 33 +++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/scripts/vllm_dissag/models.yaml b/scripts/vllm_dissag/models.yaml index 064ced6d..150143e6 100644 --- a/scripts/vllm_dissag/models.yaml +++ b/scripts/vllm_dissag/models.yaml @@ -251,7 +251,19 @@ GLM-5.1-FP8: GPU_MEMORY_UTILIZATION: "0.80" VLLM_CUDAGRAPH_MODE: "NONE" PREFILL_CUDAGRAPH_MODE: "NONE" - DECODE_CUDAGRAPH_MODE: "PIECEWISE" + # PERF: FULL_AND_PIECEWISE captures a FULL graph for the uniform-decode shapes and + # falls back to PIECEWISE for everything else. Measured on MI300X (1024/64, warm), + # against the previous PIECEWISE setting: + # 1P/1D EP8 TPOT 88.0 -> 41.8 ms (2.11x), outTP 78.8 -> 143.7 tok/s + # 2P/2D EP16 TPOT 94.1 -> 55.4 ms (1.70x), outTP 66.7 -> 113.0 tok/s + # Accuracy is unaffected (NIAH 2k-35k: 1P/1D 51/60, 2P/2D 55/60 - both in the + # PIECEWISE band, no length collapse, memfault=0). The gain shrinks as EP widens + # (2.11x at EP8 -> 1.70x at EP16) because the cross-node all2all inside the captured + # step does not compress; do NOT assume this scales to EP32 without measuring. + # Cost: graph capture ~92-94 s and ~3.0-3.5 GiB (PIECEWISE was ~5 s / 7.2 GiB - fewer + # but larger graphs) and a correspondingly longer boot. Irrelevant for a long-lived + # server, but notable for CI. Set PIECEWISE to fall back. + DECODE_CUDAGRAPH_MODE: "FULL_AND_PIECEWISE" CUDAGRAPH_CAPTURE_SIZES: "1 2 4 8 16 32 64 128 256" VLLM_ALL2ALL_BACKEND: "mori_high_throughput" PREFILL_MORI_BACKEND: "mori_high_throughput" diff --git a/scripts/vllm_dissag/skills_vllm_disagg.md b/scripts/vllm_dissag/skills_vllm_disagg.md index ee778064..cb837dc2 100644 --- a/scripts/vllm_dissag/skills_vllm_disagg.md +++ b/scripts/vllm_dissag/skills_vllm_disagg.md @@ -120,6 +120,39 @@ default. Added `VLLM_MORI_*` knobs for parity (all defaulting to current values) --- +## 2b. Decode CUDA-graph mode: use FULL_AND_PIECEWISE, not PIECEWISE + +`DECODE_CUDAGRAPH_MODE=FULL_AND_PIECEWISE` captures a FULL graph for the uniform-decode +shapes and falls back to PIECEWISE for everything else. Measured on MI300X, 1024/64, warm, +after the max_num_batched_tokens fix in section 2: + +| | PIECEWISE | FULL_AND_PIECEWISE | gain | +|---|---|---|---| +| 1P/1D EP8 TPOT c8 | 88.0 ms | **41.8 ms** | 2.11x | +| 1P/1D EP8 TPOT c16 | 91.5 ms | **45.8 ms** | 2.00x | +| 2P/2D EP16 TPOT c8 | 94.1 ms | **55.4 ms** | 1.70x | +| 2P/2D EP16 TPOT c16 | 96.2 ms | **60.3 ms** | 1.60x | +| 1P/1D output c8 | 78.8 tok/s | **143.7** | 1.82x | +| 2P/2D output c8 | 66.7 tok/s | **113.0** | 1.69x | + +Accuracy is unaffected: NIAH 2k-35k 51/60 (1P/1D) and 55/60 (2P/2D), both inside the +PIECEWISE band (52-53 / 53), no length collapse, memfault=0. + +Three things to know before adopting it: +- **The gain shrinks as EP widens** (2.11x at EP8 -> 1.70x at EP16). FULL captures the whole + decode step *including* the cross-node all2all, and that collective does not compress. + Do not extrapolate to EP32 without measuring. +- **Capture cost:** ~92-94 s and ~3.0-3.5 GiB, versus ~5 s and 7.2 GiB for PIECEWISE - + fewer but larger graphs. Boot is correspondingly longer. Fine for a long-lived server, + notable for CI. +- **FULL is much less cold-JIT sensitive.** Its warmup run already reads steady-state TPOT + (41.35 ms), where PIECEWISE's first post-boot run showed 13.7 s TTFT. The section-1 rule + (discard the first run) still applies, but the penalty is far smaller. + +Two code paths branch on `CUDAGraphMode.FULL` - `sparse_attn_indexer.py:411` and the MoRIIO +connector's READ-mode barrier. Both are guards that *skip* host-side work under FULL, so +enabling it is safe on this stack (the MoRIIO one is already a no-op in WRITE mode). + ## 3. Accuracy: the DSA sparse-index sentinel landmine **Never set the invalid/OOB sentinel to `-1`** in From c1686f5ae74b031f81da94c380ea8fdaa283ad78 Mon Sep 17 00:00:00 2001 From: Mir Mustafa Ali Date: Mon, 17 Aug 2026 23:34:54 -0500 Subject: [PATCH 10/23] [GLM-5.1] Merge hygiene: drop lab leftovers, safe patcher defaults. Rebase onto develop after #197 so the GLM card lives in scripts/vllm_dissag/models.json. Remove the NFS silly-rename and keepalive/NIAH-200k/skills lab files, revert DeepSeek-V3 context caps and cluster-wide moriio.env NIC pins, turn persist-gate and DSA sentinel patchers opt-in (they crash the v0.27 image), gate DSA on GLM-5.*, and wire BENCHMARK_SCRIPT=niah for the 1P/1D smoke. --- ...gg_inference.glmv5.1.ubuntu.amd.Dockerfile | 26 +- .../vllm_dissag/.nfs0000000016f44d2b00008188 | 685 ------------------ scripts/vllm_dissag/README.MD | 3 +- scripts/vllm_dissag/connectors/moriio.env | 13 +- scripts/vllm_dissag/connectors/moriio.sh | 59 +- scripts/vllm_dissag/keepalive_bench.sh | 18 - scripts/vllm_dissag/models.json | 32 + scripts/vllm_dissag/models.yaml | 11 +- scripts/vllm_dissag/niah_200k.py | 86 --- scripts/vllm_dissag/run_xPyD_models.slurm | 14 +- scripts/vllm_dissag/skills_vllm_disagg.md | 278 ------- 11 files changed, 93 insertions(+), 1132 deletions(-) delete mode 100755 scripts/vllm_dissag/.nfs0000000016f44d2b00008188 delete mode 100755 scripts/vllm_dissag/keepalive_bench.sh delete mode 100755 scripts/vllm_dissag/niah_200k.py delete mode 100644 scripts/vllm_dissag/skills_vllm_disagg.md diff --git a/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile b/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile index edd4d797..1053af1e 100644 --- a/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile +++ b/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile @@ -35,16 +35,16 @@ # # ALL connectors in one image: moriio (TP + MoRI-EP wideEP) + rixl (NIXL TP + # DeepEP wideEP). = the fullsource MoRI stack, plus a UCX/RIXL/rocSHMEM/DeepEP -# transport layer gated by --build-arg WITH_NIXL (default 1 = everything). +# transport layer gated by --build-arg WITH_NIXL (default 0 = MoRI-EP only). # # docker build -f docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile \ # -t /vllm-disagg:glmv5.1 . # export DOCKER_IMAGE_NAME=/vllm-disagg:glmv5.1 # -# WITH_NIXL=1 (default) => builds UCX + RIXL(+nixlbench) + rocSHMEM + DeepEP from -# source, so all four connector combos (moriio TP/wideEP, rixl NIXL TP, DeepEP +# WITH_NIXL=0 (default) => MoRI-EP only (moriio TP/wideEP + deepep-from-base); lean. +# WITH_NIXL=1 => builds UCX + RIXL(+nixlbench) + rocSHMEM + DeepEP from source, +# so all four connector combos (moriio TP/wideEP, rixl NIXL TP, DeepEP # wideEP) are present (~+30-45 min build vs WITH_NIXL=0). -# WITH_NIXL=0 => MoRI-EP only (moriio TP/wideEP + deepep-from-base); lean, faster. # # STATUS (GLM-5.1-FP8 on this stack): 1P/1D EP8 + 2P/2D EP16 NIAH 2k-35k = 10/10, # no crash; long-context accuracy fixed via vLLM #47766 (persistent sparse-MLA kept @@ -57,16 +57,16 @@ # purpose-built ROCm/vLLM/MoRI base, cloning each overridden source from public Git # (no local build-contexts): # -# - BASE: rocmshared/pytorch-private:vllm-rocm_07_22_2026_shikpate_mori1.2.3 -# (ROCm + torch + a bundled vLLM/MoRI 1.2.3 stack). The stages below deliberately -# OVERRIDE the base's vLLM/MoRI/AITER with the pins we validate for GLM DSA. +# - BASE: rocm/vllm-dev:ci_base-dedbf6be8b1afa17a6220473b9c8c98242ac1c03 +# (ROCm + torch nightly). The stages below OVERRIDE the base's vLLM/MoRI/AITER +# with the pins we validate for GLM DSA. # - MoRI -> built from ROCm/MoRI @ 42e895472b08 (validated for GLM DSA, BUILD_UMBP=OFF). # (main LATEST 120d2de broke the connector KV-notify handshake -- see note at MORI_REF.) # - AITER -> STOCK ROCm/aiter @ e03fa6040 compiled from source + flydsl 0.1.7-0.1.9; # stale JIT wiped. (#47766 keeps persistent MLA ON -> aiter native gqa64 fold.) -# - vLLM -> COMPILED from raviguptaamd/vllm @ glm5.1-dsa-wideEP_on_shik_0721 -# (Shiksha 7/21 WideEP base + GLM DSA edits + sparse-MLA guard fix). Full compile: -# a different commit than the base's, so a .py-only overlay would be ABI-mismatched. +# - vLLM -> COMPILED from raviguptaamd/vllm @ glm5.1-dsa-wideEP_on_vllm-v0.27 +# (upstream v0.27 + GLM DSA + #47766 metadata-key). Full compile: a different +# commit than the base's, so a .py-only overlay would be ABI-mismatched. # - RDMA fix (expandable_segments:False x2 + HSA_ENABLE_IPC_MODE_LEGACY=0) is NOT baked # here — it lives in scripts/vllm_dissag/connectors/.env and the launcher # forwards it via docker -e. ROCm 7.2.3 cannot dmabuf-export VMM memory, else MoRI @@ -254,9 +254,9 @@ RUN if ! command -v cargo >/dev/null 2>&1; then \ rm -rf /tmp/vllm-router-src # ----------------------------------------------------------------------------- -# 4b. WITH_NIXL=1 (default): UCX + RIXL(+nixlbench) + rocSHMEM + DeepEP from source, -# so the rixl connector (NIXL TP + DeepEP wideEP) is present. Single guarded RUN so -# WITH_NIXL=0 skips it entirely (no layers, no cost). Build-verified on ci_base. +# 4b. WITH_NIXL=1: UCX + RIXL(+nixlbench) + rocSHMEM + DeepEP from source, +# so the rixl connector (NIXL TP + DeepEP wideEP) is present. Default is 0 +# (MoRI-EP only). Single guarded RUN so WITH_NIXL=0 skips it (no layers, no cost). # ----------------------------------------------------------------------------- ENV _ROCM_DIR=/opt/rocm \ _UCX_SOURCE=https://github.com/ROCm/ucx.git \ diff --git a/scripts/vllm_dissag/.nfs0000000016f44d2b00008188 b/scripts/vllm_dissag/.nfs0000000016f44d2b00008188 deleted file mode 100755 index 52d046a1..00000000 --- a/scripts/vllm_dissag/.nfs0000000016f44d2b00008188 +++ /dev/null @@ -1,685 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=vllm-pd # Specify a custom string for your slurm batch job -#SBATCH -N 2 # Default 2 nodes (1P/1D); override with sbatch -N for larger topologies -#SBATCH --ntasks-per-node=1 -#SBATCH --spread-job -#SBATCH --gres=gpu:8 # Request 8 GPUs and 8 NICs (use --gres if specific GPU resources are needed) -#SBATCH --time=24:00:00 # Set a time limit for the job (HH:MM:SS) -#SBATCH --output="/shared_inference/%u/model_blog_logs/slurm-%j.out" -#SBATCH --error="/shared_inference/%u/model_blog_logs/slurm-%j.err" - - -# ------------------------ -# Auto-detect and cd to the script directory so that $(pwd) always -# points to the folder containing the server scripts, regardless of -# where the user called sbatch from. -# -# Priority: -# 1. BASH_SOURCE — works for direct invocation (bash script.sh) -# 2. SLURM_SUBMIT_DIR + script path — when sbatch submits from repo root, -# SLURM_SUBMIT_DIR is the CWD, not the script dir. Append the relative -# path from the SBATCH command to get the actual script directory. -# 3. SLURM_SUBMIT_DIR alone — last resort (assumes sbatch was run from -# the script directory). -# ------------------------ -_resolve_script_dir() { - # Try BASH_SOURCE first (works for direct invocation) - if [[ -n "${BASH_SOURCE[0]:-}" ]]; then - local _d - _d="$(cd "$(dirname "${BASH_SOURCE[0]}")" 2>/dev/null && pwd)" - if [[ -n "$_d" && -f "$_d/vllm_disagg.sh" ]]; then - echo "$_d" - return 0 - fi - fi - - # Try SLURM_SUBMIT_DIR + relative script path - # When madengine does: sbatch scripts/vllm_dissag/run_xPyD_models.slurm - # SLURM_SUBMIT_DIR = repo root, so we need to append the dirname - if [[ -n "${SLURM_SUBMIT_DIR:-}" ]]; then - local _candidate="$SLURM_SUBMIT_DIR/scripts/vllm_dissag" - if [[ -f "$_candidate/vllm_disagg.sh" ]]; then - echo "$_candidate" - return 0 - fi - # Maybe they ran sbatch from the script dir itself - if [[ -f "$SLURM_SUBMIT_DIR/vllm_disagg.sh" ]]; then - echo "$SLURM_SUBMIT_DIR" - return 0 - fi - fi - - # Fallback - echo "." - return 1 -} - -SCRIPT_DIR="$(_resolve_script_dir)" -cd "$SCRIPT_DIR" || { echo "Error: cannot cd to $SCRIPT_DIR" >&2; exit 1; } - -REQUIRED_FILES=("vllm_disagg.sh" "parallelism.sh" "connectors/rixl.sh" "connectors/moriio.sh" "models.yaml" "benchmark_xPyD.sh" "parse_to_csv.py" "socket_barrier.py" "socket_wait.py" "connectors/moriio.env" "connectors/rixl.env") -for f in "${REQUIRED_FILES[@]}"; do - if [[ ! -f "$f" ]]; then - echo "Error: Required file '$f' not found in $(pwd)." >&2 - echo "Please run sbatch from the scripts/vllm_dissag/ directory, e.g.:" >&2 - echo " cd MAD/scripts/vllm_dissag && sbatch run_xPyD_models.slurm" >&2 - exit 1 - fi -done -echo "Running from: $(pwd)" - -# ------------------------------------------------------------------------------ -# models.yaml env precedence: capture which recipe knobs the USER explicitly set -# at submit time. The driver (vllm_disagg.sh) uses this to let models.yaml `env:` -# OVERRIDE image-baked ENV defaults (e.g. a DeepSeek-tuned image bakes -# KV_BLOCK_SIZE=16 / VLLM_ROCM_USE_AITER_MLA=0, which would otherwise shadow a -# model's own recipe — GLM-5.1 DSA needs block=1 + AITER MLA on), while a genuine -# submit-time `-e VAR=...` still wins. Precedence: image-baked < models.yaml < submit -e. -# Captured HERE (before the slurm sets any defaults) so it reflects user intent only. -_RECIPE_ENV_KEYS="VLLM_USE_V1 DECODE_MORI_MAX_TOTAL_RECV_TOKENS PREFILL_MORI_MAX_TOTAL_RECV_TOKENS DECODE_MORI_MAX_TOKENS_PER_RANK PREFILL_MORI_MAX_TOKENS_PER_RANK VLLM_MORI_MAX_TOKENS_PER_RANK VLLM_MORI_WARP_NUM_PER_BLOCK VLLM_MORI_BLOCK_NUM VLLM_MORI_RDMA_BLOCK_NUM VLLM_USE_LAYERNAME VLLM_ROCM_USE_AITER VLLM_ROCM_USE_AITER_RMSNORM VLLM_ROCM_USE_AITER_MLA KV_BLOCK_SIZE KV_CACHE_DTYPE KV_CACHE_MEMORY_BYTES GPU_MEMORY_UTILIZATION VLLM_CUDAGRAPH_MODE PREFILL_CUDAGRAPH_MODE DECODE_CUDAGRAPH_MODE CUDAGRAPH_CAPTURE_SIZES VLLM_ALL2ALL_BACKEND PREFILL_MORI_BACKEND DECODE_MORI_BACKEND MORI_SHMEM_HEAP_SIZE" -MODELS_YAML_PROTECT="" -for _k in $_RECIPE_ENV_KEYS; do - [ -n "${!_k+x}" ] && MODELS_YAML_PROTECT="${MODELS_YAML_PROTECT} ${_k}" -done -export MODELS_YAML_PROTECT="${MODELS_YAML_PROTECT# }" -echo "models.yaml protect-list (submit-time overrides): '${MODELS_YAML_PROTECT}'" - -# ------------------------ -# Print current time in UTC and PST formats -# ------------------------ -echo "=== Job Start Time ===" -echo "UTC Time: $(TZ=UTC date '+%Y-%m-%d %H:%M:%S %Z')" -echo "PST Time: $(TZ=America/Los_Angeles date '+%Y-%m-%d %H:%M:%S %Z')" -echo "=======================" -echo "" - -# Define valid model names (must have a models.yaml entry) -VALID_MODELS=( \ - "Llama-3.1-405B-Instruct-FP8-KV" \ - "amd-Llama-3.3-70B-Instruct-FP8-KV" \ - "DeepSeek-V3" \ - "DeepSeek-V3-5layer" \ - "gpt-oss-120b" \ - "DeepSeek-R1" \ - "Qwen3-32B" \ - "Qwen3-30B-A3B" \ - "GLM-5.1-FP8" \ -) - -# Models allowed for CONNECTOR=moriio WIDE_EP=1 (MoRI-EP; legacy RUN_MORI=1) -MORI_EP_VALID_MODELS=( \ - "DeepSeek-V3" \ - "DeepSeek-V3-5layer" \ - "DeepSeek-R1" \ - "GLM-5.1-FP8" \ -) - -# Models allowed for CONNECTOR=rixl WIDE_EP=1 EP_BACKEND=deepep (legacy RUN_DEEPEP=1) -DEEPEP_VALID_MODELS=( \ - "DeepSeek-V3" \ - "DeepSeek-V3-5layer" \ - "DeepSeek-R1" \ -) - -MODEL_NAME="${MODEL_NAME:-None}" - -validate_model_name() { - local is_valid_model=false - - for model in "${VALID_MODELS[@]}"; do - if [[ "$MODEL_NAME" == "$model" ]]; then - is_valid_model=true - break - fi - done - - if ! $is_valid_model; then - printf "Error: Invalid MODEL_NAME: '%s'\nValid models are:\n" "$MODEL_NAME" - for model in "${VALID_MODELS[@]}"; do - printf " - %s\n" "$model" - done - exit 1 - fi - - echo "MODEL_NAME '$MODEL_NAME' is valid." - return 0 -} - -validate_model_name "${MODEL_NAME}" - -model_allows_mori_ep() { - local m="$1" - for x in "${MORI_EP_VALID_MODELS[@]}"; do - [[ "$m" == "$x" ]] && return 0 - done - return 1 -} - -model_allows_deepep() { - local m="$1" - for x in "${DEEPEP_VALID_MODELS[@]}"; do - [[ "$m" == "$x" ]] && return 0 - done - return 1 -} - -# --------------------------------------------------------------------------- -# Axis selection -> single launcher (vllm_disagg.sh). -# Two axes: CONNECTOR={rixl|moriio} x WIDE_EP={0=TP|1=wideEP}; EP_BACKEND only -# when WIDE_EP=1 (rixl->deepep, moriio->mori). Legacy RUN_MORI / RUN_DEEPEP are -# still honored via a back-compat shim. The launcher itself does the final -# CONNECTOR/WIDE_EP/EP_BACKEND validation; here we resolve them + gate the model -# against the existing allowlists. -# --------------------------------------------------------------------------- -RUN_FILE="vllm_disagg.sh" -_run_mori="${RUN_MORI:-0}" -_run_deepep="${RUN_DEEPEP:-0}" - -if [[ "$_run_mori" == "1" && "$_run_deepep" == "1" ]]; then - echo "Error: Both RUN_MORI and RUN_DEEPEP are set to 1. Set only one." >&2 - exit 1 -fi - -# Back-compat: legacy flags map onto the axes when CONNECTOR is not set explicitly. -if [[ -z "${CONNECTOR:-}" ]]; then - if [[ "$_run_mori" == "1" ]]; then - CONNECTOR=moriio; WIDE_EP="${WIDE_EP:-1}"; EP_BACKEND="${EP_BACKEND:-mori}" - elif [[ "$_run_deepep" == "1" ]]; then - CONNECTOR=rixl; WIDE_EP="${WIDE_EP:-1}"; EP_BACKEND="${EP_BACKEND:-deepep}" - else - # Default keeps the historical "no flags" behavior: rixl + TP. - CONNECTOR=rixl; WIDE_EP="${WIDE_EP:-0}" - fi -fi -WIDE_EP="${WIDE_EP:-0}" - -# Models that ONLY run wideEP (DP/EP), never TP: the DeepSeek family is served with -# the MoRI-EP / DeepEP recipe (block=16, MLA off, per-role cudagraph). Running them -# in TP mode is unsupported — the TP argv would double the model's own -# --compilation-config and drop the mandatory +quant_fp8 op. Reject early. -# GLM-5.1-FP8 (GlmMoeDsaForCausalLM, MLA+DSA) is validated only under MoRI-EP -# wideEP disagg (block=1, AITER sparse MLA on, per-role all2all). The moriio+TP -# ("Stage B") path is untested for DSA, so reject WIDE_EP=0 for it too. -WIDE_EP_ONLY_MODELS=( "DeepSeek-V3" "DeepSeek-V3-5layer" "DeepSeek-R1" "GLM-5.1-FP8" ) -model_is_wide_ep_only() { - local m="$1" - for x in "${WIDE_EP_ONLY_MODELS[@]}"; do [[ "$m" == "$x" ]] && return 0; done - return 1 -} - -# Model allowlist gate (kept, sglang-style). wideEP modes use the per-backend -# allowlists; TP uses the global VALID_MODELS (already validated above) minus the -# wideEP-only models. -if [[ "$WIDE_EP" == "0" ]]; then - if model_is_wide_ep_only "$MODEL_NAME"; then - echo "Error: MODEL_NAME '$MODEL_NAME' is wideEP-only (set WIDE_EP=1). TP mode is not supported for it." >&2 - printf "wideEP-only models:\n"; for m in "${WIDE_EP_ONLY_MODELS[@]}"; do printf " - %s\n" "$m"; done - exit 1 - fi -elif [[ "$WIDE_EP" == "1" && "${CONNECTOR}" == "moriio" ]]; then - if ! model_allows_mori_ep "$MODEL_NAME"; then - echo "Error: CONNECTOR=moriio WIDE_EP=1 but MODEL_NAME '$MODEL_NAME' is not in MORI_EP_VALID_MODELS" >&2 - printf "MoRI EP allowed models:\n"; for m in "${MORI_EP_VALID_MODELS[@]}"; do printf " - %s\n" "$m"; done - exit 1 - fi -elif [[ "$WIDE_EP" == "1" && "${CONNECTOR}" == "rixl" ]]; then - if ! model_allows_deepep "$MODEL_NAME"; then - echo "Error: CONNECTOR=rixl WIDE_EP=1 (deepep) but MODEL_NAME '$MODEL_NAME' is not in DEEPEP_VALID_MODELS" >&2 - printf "DeepEP allowed models:\n"; for m in "${DEEPEP_VALID_MODELS[@]}"; do printf " - %s\n" "$m"; done - exit 1 - fi -fi - -export CONNECTOR WIDE_EP EP_BACKEND -echo "Launcher: $RUN_FILE (CONNECTOR=${CONNECTOR} WIDE_EP=${WIDE_EP} EP_BACKEND=${EP_BACKEND:-}) for model '$MODEL_NAME'" - -# --------------------------------------------------------------------------- -# Connector platform env: per-connector .env holds the ROCm-7.2.3 -# runtime env that MUST reach the container at PID 1 (e.g. expandable_segments:False -# for GPU-RDMA registration). Source the resolved connector's file and collect its -# KEY=VALUE lines into CONNECTOR_ENV_ARGS as `-e KEY=${KEY:-VALUE}` pairs, so a -# submit-time export of the same name still overrides. Forwarded in the docker run. -# --------------------------------------------------------------------------- -CONNECTOR_ENV_FILE="${SCRIPT_DIR}/connectors/${CONNECTOR}.env" -CONNECTOR_ENV_ARGS="" -if [[ -f "$CONNECTOR_ENV_FILE" ]]; then - echo "Loading connector platform env: $CONNECTOR_ENV_FILE" - while IFS= read -r _line; do - [[ "$_line" =~ ^[[:space:]]*# || -z "${_line// }" ]] && continue - _k="${_line%%=*}"; _v="${_line#*=}" - CONNECTOR_ENV_ARGS+=" -e ${_k}=${!_k:-$_v}" # submit-time export of $_k wins - done < "$CONNECTOR_ENV_FILE" -else - echo "WARN: connector env file not found: $CONNECTOR_ENV_FILE" >&2 -fi - -if [[ -z "${DOCKER_IMAGE_NAME:-}" ]]; then - echo "Error: DOCKER_IMAGE_NAME is not set. Please export DOCKER_IMAGE_NAME before running." >&2 - echo " There is no public prebuilt image. Build your own from the provided Dockerfile:" >&2 - echo " docker build -f docker/vllm_disagg_inference.ubuntu.amd.Dockerfile \\" >&2 - echo " -t /vllm-disagg:local . # all connectors (WITH_NIXL=1 default)" >&2 - echo " (add --build-arg WITH_NIXL=0 for a lean MoRI-EP-only image)" >&2 - echo " then: export DOCKER_IMAGE_NAME=/vllm-disagg:local" >&2 - exit 1 -fi -export DOCKER_IMAGE_NAME - -# Set current directory to be REPO directory with all relevant scripts -NIXL_REPO_DIR=$(pwd) -LOG_PATH="${LOG_PATH:-/shared_inference/${USER}/model_blog_logs}" - -xP="${xP:-1}" #-> Number of Prefill Servers -yD="${yD:-1}" #-> Number of Decode Servers - -MODEL_DIR="${MODEL_DIR:-"/shared_inference/models_blog/"}" - - -# ------------------------ -# Model path validation and selection across all nodes -# ------------------------ -echo "Looking for model: $MODEL_NAME" -echo "Checking model availability across all allocated nodes..." - -# Get all allocated nodes -ALL_NODES=$(scontrol show hostnames "$SLURM_JOB_NODELIST") -TOTAL_NODES=$(echo "$ALL_NODES" | wc -l) - -echo "Total allocated nodes: $TOTAL_NODES" -echo "Nodes: $(echo "$ALL_NODES" | tr '\n' ' ')" - -# Function to check model path on all nodes -check_model_path() { - local path=$1 - local check_name=$2 - - echo "Checking $check_name: $path" - - # Run check on all nodes in parallel - srun --nodes=$SLURM_NNODES --ntasks=$SLURM_NNODES /bin/bash -c " - if [ -d '$path' ]; then - echo \"\$(hostname): ✓ Found $path\" - exit 0 - else - echo \"\$(hostname): ✗ Missing $path\" - exit 1 - fi - " - - # Check if all nodes succeeded (exit code 0) - local exit_code=$? - if [ $exit_code -eq 0 ]; then - echo "✓ $check_name available on ALL nodes" - return 0 - else - echo "✗ $check_name NOT available on all nodes" - return 1 - fi -} - -# Check /mnt/m2m_nobackup/models_blog first -MODEL_PATH_1="/mnt/m2m_nobackup/models_blog/$MODEL_NAME" -if check_model_path "$MODEL_PATH_1" "/mnt/m2m_nobackup/models_blog"; then - MODEL_PATH="$MODEL_PATH_1" - echo "" - echo "✓ Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" -# Check /shared-inference/models_blog -elif check_model_path "/shared_inference/models_blog/$MODEL_NAME" "/shared_inference/models_blog"; then - MODEL_PATH="/shared_inference/models_blog/$MODEL_NAME" - echo "" - echo "✓ Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" -elif check_model_path "$MODEL_DIR/$MODEL_NAME" "$MODEL_DIR"; then - MODEL_PATH="$MODEL_DIR/$MODEL_NAME" - echo "" - echo "✓ Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" -else - echo "" - echo "✗ FATAL ERROR: Model '$MODEL_NAME' not found on ALL allocated nodes in either:" - echo " - /mnt/m2m_nobackup/models_blog/$MODEL_NAME" - echo " - /shared_inference/models_blog/$MODEL_NAME" - echo "" - echo "Model must be accessible from all nodes for distributed execution." - echo "Please ensure the model is available on all allocated nodes." - exit 1 -fi - -echo "Final MODEL_PATH: $MODEL_PATH" -echo "" - - -# Calculate NUM_NODES based on xP and yD -NUM_NODES=$((xP + yD)) -echo "Calculated NUM_NODES: $NUM_NODES (xP=$xP + yD=$yD, proxy co-located on prefill master)" - -# DeepEP configuration (only exported when RUN_DEEPEP=1) -if [[ "$_run_deepep" == "1" ]]; then - export PREFILL_DEEPEP_BACKEND="${PREFILL_DEEPEP_BACKEND:-deepep_high_throughput}" - export DECODE_DEEPEP_BACKEND="${DECODE_DEEPEP_BACKEND:-deepep_low_latency}" - export ENABLE_DBO="${ENABLE_DBO:-false}" - export DBO_COMM_SMS="${DBO_COMM_SMS:-}" - export ENABLE_PROFILING="${ENABLE_PROFILING:-false}" - echo "DeepEP config: PREFILL_BACKEND=$PREFILL_DEEPEP_BACKEND DECODE_BACKEND=$DECODE_DEEPEP_BACKEND DBO=$ENABLE_DBO" -fi - -# ------------------------ -# Extract first NUM_NODES from SLURM allocation and update SLURM variables -# ------------------------ -echo "Original SLURM allocation:" -echo "SLURM_JOB_NODELIST: $SLURM_JOB_NODELIST" -echo "SLURM_NNODES: $SLURM_NNODES" -echo "SLURM_NTASKS: $SLURM_NTASKS" - -# Get the full nodelist and extract first NUM_NODES -FULL_NODELIST=$(scontrol show hostnames "$SLURM_JOB_NODELIST") -SELECTED_NODES=$(echo "$FULL_NODELIST" | head -n $NUM_NODES) -NEW_SLURM_NODELIST=$(echo "$SELECTED_NODES" | paste -sd,) - -# Update SLURM environment variables -export SLURM_NNODES=$NUM_NODES -export SLURM_NTASKS=$NUM_NODES -export SLURM_JOB_NUM_NODES=$NUM_NODES -export SLURM_NPROCS=$NUM_NODES -export SLURM_JOB_NODELIST="$NEW_SLURM_NODELIST" -export SLURM_NODELIST="$NEW_SLURM_NODELIST" - -# Keep other SLURM variables as they were or set defaults -export SLURM_TASKS_PER_NODE="1(x$NUM_NODES)" - -export SLURM_CLUSTER_NAME="${SLURM_CLUSTER_NAME}" -export SLURM_JOB_CPUS_PER_NODE="${SLURM_JOB_CPUS_PER_NODE}" -export SLURM_JOB_PARTITION="${SLURM_JOB_PARTITION}" -export SLURM_JOBID="${SLURM_JOBID:-$SLURM_JOB_ID}" -export SLURM_JOB_QOS="${SLURM_JOB_QOS:-normal}" -export SLURM_JOB_ACCOUNT="${SLURM_JOB_ACCOUNT}" -export SLURM_NTASKS_PER_NODE=1 -export SLURM_SUBMIT_HOST="${SLURM_SUBMIT_HOST}" -export SLURM_JOB_ID="${SLURM_JOB_ID}" -export SLURM_CONF="${SLURM_CONF:-/etc/slurm/slurm.conf}" -export SLURM_JOB_NAME="${SLURM_JOB_NAME:-1p1d_bench-serving}" - -echo "" -echo "Updated SLURM Environment Variables:" -echo "SLURM_JOB_ID: $SLURM_JOB_ID" -echo "SLURM_JOB_NODELIST: $SLURM_JOB_NODELIST" -echo "SLURM_NNODES: $SLURM_NNODES" -echo "SLURM_NTASKS: $SLURM_NTASKS" -echo "SLURM_TASKS_PER_NODE: $SLURM_TASKS_PER_NODE" -echo "SLURM_JOB_CPUS_PER_NODE: $SLURM_JOB_CPUS_PER_NODE" -echo "SLURM_JOB_PARTITION: $SLURM_JOB_PARTITION" -echo "SLURM_JOB_NUM_NODES: $SLURM_JOB_NUM_NODES" -echo "SLURM_JOBID: $SLURM_JOBID" -echo "SLURM_JOB_QOS: $SLURM_JOB_QOS" -echo "SLURM_NODELIST: $SLURM_NODELIST" -echo "SLURM_JOB_ACCOUNT: $SLURM_JOB_ACCOUNT" -echo "SLURM_NPROCS: $SLURM_NPROCS" -echo "SLURM_SUBMIT_HOST: $SLURM_SUBMIT_HOST" -echo "SLURM_CONF: $SLURM_CONF" -echo "SLURM_JOB_NAME: $SLURM_JOB_NAME" -echo "SLURM_NTASKS_PER_NODE: $SLURM_NTASKS_PER_NODE" -#echo "SLURM_SUBMIT_DIR: $SLURM_SUBMIT_DIR" -echo "SLURM_CLUSTER_NAME: $SLURM_CLUSTER_NAME" -echo "ulimit: $(ulimit -a)" -echo "" -echo "Selected nodes for execution:" -echo "$SELECTED_NODES" -echo "" - -# Node information -USER_NAME=$(whoami) -MASTER_NODE=$(echo "$SELECTED_NODES" | head -n 1) -# Pick the routable fabric IP, not just hostname -I's first entry. These nodes expose -# multiple NICs (e.g. a 10.224.x overlay listed BEFORE the routable 10.158.x fabric); -# taking $1 blindly can advertise an unreachable addr -> prefill/decode barrier hangs -# "Waiting for nodes" forever. Prefer FABRIC_SUBNET (default 10.158.), fall back to $1. -FABRIC_SUBNET="${FABRIC_SUBNET:-10.158.}" -# From a "hostname -I" line, return the first IP on FABRIC_SUBNET, else the first IP. -_pick_fabric_ip() { - awk -v pfx="$FABRIC_SUBNET" '{f=$1; for(i=1;i<=NF;i++) if(index($i,pfx)==1){f=$i; break} print f}' -} -MASTER_ADDR=$(srun --nodes=1 --ntasks=1 --time=00:20:00 --nodelist="$MASTER_NODE" bash -c 'hostname -I' | _pick_fabric_ip) -MASTER_PORT=39566 # Choose an open port - -IPS=() - -for NODE in $SELECTED_NODES; do - IP=$(srun --nodes=1 --ntasks=1 --time=00:20:00 --nodelist="$NODE" bash -c 'hostname -I' | _pick_fabric_ip) - IPS+=("$IP") -done - -echo "Selected node IPs: ${IPS[*]}" | sed 's/ /,/g' - -NIXL_COOKBOOK_PATH="/opt/nixl-vllm-cookbook" -BENCHMARK_ITR="${BENCHMARK_ITR:-1}" -BENCHMARK_CON="${BENCHMARK_CON:-}" -BENCHMARK_COMBINATIONS="${BENCHMARK_COMBINATIONS:-}" - -# Benchmark script selector: BENCHMARK_SCRIPT tag -> file run by the launcher. -# sweep (default) -> benchmark_xPyD.sh (general concurrency sweep) -# long_context -> benchmark_long_context.sh (per-shape warmup, c=1-first) -# keepalive -> keepalive_bench.sh (hold server up KEEPALIVE_MINS -# for external accuracy probes) -BENCHMARK_SCRIPT="${BENCHMARK_SCRIPT:-sweep}" -case "$BENCHMARK_SCRIPT" in - sweep) BENCHMARK_SCRIPT_FILE="benchmark_xPyD.sh" ;; - long_context) BENCHMARK_SCRIPT_FILE="benchmark_long_context.sh" ;; - keepalive) BENCHMARK_SCRIPT_FILE="keepalive_bench.sh" ;; - *) echo "Error: invalid BENCHMARK_SCRIPT='$BENCHMARK_SCRIPT' (valid: sweep, long_context, keepalive)" >&2; exit 1 ;; -esac -if [[ ! -f "$BENCHMARK_SCRIPT_FILE" ]]; then - echo "Error: selected benchmark script '$BENCHMARK_SCRIPT_FILE' not found in $(pwd)." >&2 - exit 1 -fi -echo "BENCHMARK_SCRIPT='$BENCHMARK_SCRIPT' -> $BENCHMARK_SCRIPT_FILE" - -NNODES=$NUM_NODES - -echo "MASTER_NODE is ${MASTER_NODE}" -echo "MASTER_ADDR is ${MASTER_ADDR}" -echo "MASTER_PORT is ${MASTER_PORT}" -echo "NNODES is ${NNODES}" -echo "REPO Directory is ${NIXL_REPO_DIR}" - -if [ ! -d "$LOG_PATH" ]; then - mkdir -p "$LOG_PATH" - echo "Created directory: $LOG_PATH" -else - echo "Directory already exists: $LOG_PATH" -fi - -export CONNECTOR_ENV_ARGS="$CONNECTOR_ENV_ARGS" -export LOG_PATH=$LOG_PATH -export NIXL_REPO_DIR=$NIXL_REPO_DIR -export NIXL_COOKBOOK_PATH=$NIXL_COOKBOOK_PATH -export NNODES=$NNODES -export MASTER_ADDR=$MASTER_ADDR -export MASTER_PORT=$MASTER_PORT -export MODEL_PATH=$MODEL_PATH -export xP=$xP -export yD=$yD -export MODEL_NAME=$MODEL_NAME -export USER_NAME=$USER_NAME -export IPADDRS="$(echo "${IPS[*]}" | sed 's/ /,/g')" -export BENCHMARK_ITR=$BENCHMARK_ITR -export BENCHMARK_CON="${BENCHMARK_CON}" -export BENCHMARK_COMBINATIONS="${BENCHMARK_COMBINATIONS}" -export BENCHMARK_SCRIPT_FILE="${BENCHMARK_SCRIPT_FILE}" - -export DOCKER_CONT_NAME="container_${MODEL_NAME}_${SLURM_JOB_ID}" -export RUN_FILE_FULL="$NIXL_COOKBOOK_PATH/${RUN_FILE}" - -# Use only the selected nodes for srun execution -SELECTED_NODELIST_SRUN=$(echo "$SELECTED_NODES" | paste -sd,) - -srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c ' -echo "Rank $SLURM_PROCID on $(hostname)"; -docker ps -q | xargs --no-run-if-empty docker stop; -docker rm -f $DOCKER_CONT_NAME 2>/dev/null || true; -fuser -k 5000/tcp 2>/dev/null || true; -fuser -k 2222/tcp 2>/dev/null || true; -fuser -k 15000/tcp 2>/dev/null || true; -sleep 2; -docker pull $DOCKER_IMAGE_NAME 2>/dev/null || true; - -# --- Create host-local compilation cache dirs (ext4, survives container restarts) --- -mkdir -p /tmp/vllm_cache/{aiter_jit,triton,vllm,comgr} 2>/dev/null || true; - -# --- Persistent JIT cache mount --- -# The image points AITER_JIT_DIR/TRITON_CACHE_DIR/VLLM_CACHE_ROOT/COMGR_CACHE_DIR at -# /opt/vllm_cache. Mount a host dir there so AITER CK kernels compile ONCE and are reused -# across runs. Cold compile is ~15 min for the DeepSeek MoE/FP8 GEMM kernel set; a warm -# boot is ~1 min. Keyed by image ID so a new image starts a fresh cache and does not reuse -# stale-ABI shared objects. Host dir on local NVMe. Override JIT_CACHE_HOST, or set -# JIT_CACHE_PERSIST=0 to disable and fall back to the image empty in-container cache. -# NOTE: this whole section runs inside a single-quoted `srun bash -c '...'`, so avoid -# single quotes here; the image-id hash is extracted with tr, not sed. -if [ "${JIT_CACHE_PERSIST:-1}" = "1" ]; then - _IMG_RAW=$(docker image inspect --format "{{.Id}}" "$DOCKER_IMAGE_NAME" 2>/dev/null); - _IMG_KEY=$(printf "%s" "$_IMG_RAW" | tr -cd "a-f0-9" | cut -c1-12); - _IMG_KEY="${_IMG_KEY:-noimg}"; - _JIT_CACHE_HOST="${JIT_CACHE_HOST:-/mnt/m2m_nobackup/${USER}/vllm_jit_cache/${_IMG_KEY}}"; - mkdir -p "$_JIT_CACHE_HOST"/{aiter_jit,triton,vllm,comgr} 2>/dev/null || true; - _JIT_CACHE_MOUNT="-v ${_JIT_CACHE_HOST}:/opt/vllm_cache"; - echo "[jit-cache] persistent image ${_IMG_KEY}: ${_JIT_CACHE_HOST} to /opt/vllm_cache"; -else - _JIT_CACHE_MOUNT=""; -fi - -# --- Build host RDMA library mounts --- -_RDMA_MOUNTS="" -_LIBDIR=/usr/lib/x86_64-linux-gnu - -for _lib in libibverbs.so libibverbs.so.1 librdmacm.so librdmacm.so.1; do - [ -e "$_LIBDIR/$_lib" ] && _RDMA_MOUNTS="$_RDMA_MOUNTS -v $_LIBDIR/$_lib:$_LIBDIR/$_lib:ro" -done -for _vlib in $_LIBDIR/libibverbs.so.1.* $_LIBDIR/librdmacm.so.1.*; do - [ -e "$_vlib" ] && _RDMA_MOUNTS="$_RDMA_MOUNTS -v $_vlib:$_vlib:ro" -done - -for _pattern in libmlx5.so* libionic*.so* libbnxt_re*.so* libefa.so* libhns.so*; do - for _vlib in $_LIBDIR/${_pattern}; do - # Require a regular file AFTER symlink resolution: these mounts are built on - # ONE node but applied on ALL nodes, and vendor NIC libs (e.g. libionic.so.1) - # can be a DANGLING symlink on some nodes -> bind-mount fails "not a directory" - # -> container create exit 125. `-f` (follows symlink, requires regular file) - # skips those; the fabric in use (mlx5) is still mounted where present. - [ -f "$_vlib" ] && _RDMA_MOUNTS="$_RDMA_MOUNTS -v $_vlib:$_vlib:ro" - done -done - -[ -d "$_LIBDIR/libibverbs" ] && _RDMA_MOUNTS="$_RDMA_MOUNTS -v $_LIBDIR/libibverbs:$_LIBDIR/libibverbs:ro" -[ -d /etc/libibverbs.d ] && _RDMA_MOUNTS="$_RDMA_MOUNTS -v /etc/libibverbs.d:/etc/libibverbs.d:ro" -echo "[host-rdma] mounts: $_RDMA_MOUNTS" - -docker run --rm \ - --device /dev/dri \ - --device /dev/kfd \ - --device /dev/infiniband \ - --network host \ - --ipc host \ - --group-add video \ - --cap-add SYS_PTRACE \ - --security-opt seccomp=unconfined \ - --privileged \ - -v $HOME:$HOME \ - -v /shared_inference:/shared_inference \ - -v /mnt/m2m_nobackup:/mnt/m2m_nobackup \ - -v $HOME/.ssh:/root/.ssh \ - --shm-size ${DOCKER_SHM_SIZE:-256G} \ - --ulimit nofile=524288:524288 \ - --ulimit memlock=-1:-1 \ - -v ${LOG_PATH}:/run_logs \ - -v $NIXL_REPO_DIR:$NIXL_COOKBOOK_PATH \ - -v /tmp/vllm_cache:/tmp/vllm_cache \ - ${_JIT_CACHE_MOUNT} \ - ${GLM_KERNEL_PATCH:+-v ${GLM_KERNEL_PATCH}:/usr/local/lib/python3.12/dist-packages/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py:ro} \ - ${GLM_BACKEND_PATCH:+-v ${GLM_BACKEND_PATCH}:/usr/local/lib/python3.12/dist-packages/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py:ro} \ - $_RDMA_MOUNTS \ - --entrypoint /bin/bash \ - -e SLURM_JOB_ID=$SLURM_JOB_ID \ - -e SLURM_JOB_NODELIST=$SLURM_JOB_NODELIST \ - -e NNODES=$NNODES \ - -e NODE_RANK=$SLURM_PROCID \ - -e MASTER_ADDR=$MASTER_ADDR \ - -e MASTER_PORT=$MASTER_PORT \ - -e MODEL_PATH=$MODEL_PATH \ - -e NIXL_COOKBOOK_PATH=$NIXL_COOKBOOK_PATH \ - -e xP=$xP \ - -e yD=$yD \ - -e USER_NAME=$USER_NAME \ - -e MODEL_NAME=$MODEL_NAME \ - -e BENCHMARK_ITR=$BENCHMARK_ITR \ - -e BENCHMARK_CON="${BENCHMARK_CON}" \ - -e BENCHMARK_COMBINATIONS="${BENCHMARK_COMBINATIONS}" \ - ${BENCHMARK_PORT:+-e BENCHMARK_PORT=$BENCHMARK_PORT} \ - ${PROXY_TYPE:+-e PROXY_TYPE=$PROXY_TYPE} \ - ${ROUTER_PORT:+-e ROUTER_PORT=$ROUTER_PORT} \ - -e IPADDRS=$IPADDRS \ - ${CONNECTOR:+-e CONNECTOR=$CONNECTOR} \ - ${WIDE_EP:+-e WIDE_EP=$WIDE_EP} \ - ${EP_BACKEND:+-e EP_BACKEND=$EP_BACKEND} \ - ${RUN_MORI:+-e RUN_MORI=$RUN_MORI} \ - ${RUN_DEEPEP:+-e RUN_DEEPEP=$RUN_DEEPEP} \ - ${VLLM_ALL2ALL_BACKEND:+-e VLLM_ALL2ALL_BACKEND=$VLLM_ALL2ALL_BACKEND} \ - ${PREFILL_MORI_BACKEND:+-e PREFILL_MORI_BACKEND=$PREFILL_MORI_BACKEND} \ - ${DECODE_MORI_BACKEND:+-e DECODE_MORI_BACKEND=$DECODE_MORI_BACKEND} \ - -e MODELS_YAML_PROTECT="${MODELS_YAML_PROTECT:-}" \ - ${GLM_PERSIST_GATE:+-e GLM_PERSIST_GATE=$GLM_PERSIST_GATE} \ - ${GLM_SKIP_PATCHERS:+-e GLM_SKIP_PATCHERS=$GLM_SKIP_PATCHERS} \ - ${KV_BLOCK_SIZE:+-e KV_BLOCK_SIZE=$KV_BLOCK_SIZE} \ - ${KV_CACHE_MEMORY_BYTES:+-e KV_CACHE_MEMORY_BYTES=$KV_CACHE_MEMORY_BYTES} \ - ${VLLM_ROCM_USE_AITER_MLA:+-e VLLM_ROCM_USE_AITER_MLA=$VLLM_ROCM_USE_AITER_MLA} \ - ${ROUTER_BINARY:+-e ROUTER_BINARY=$ROUTER_BINARY} \ - ${KV_CACHE_DTYPE:+-e KV_CACHE_DTYPE=$KV_CACHE_DTYPE} \ - ${MORIIO_TOY_PROXY:+-e MORIIO_TOY_PROXY=$MORIIO_TOY_PROXY} \ - ${BENCHMARK_SCRIPT_FILE:+-e BENCHMARK_SCRIPT_FILE=$BENCHMARK_SCRIPT_FILE} \ - ${KEEPALIVE_MINS:+-e KEEPALIVE_MINS=$KEEPALIVE_MINS} \ - ${PREFILL_CUDAGRAPH_MODE:+-e PREFILL_CUDAGRAPH_MODE=$PREFILL_CUDAGRAPH_MODE} \ - ${DECODE_CUDAGRAPH_MODE:+-e DECODE_CUDAGRAPH_MODE=$DECODE_CUDAGRAPH_MODE} \ - ${CUDAGRAPH_CAPTURE_SIZES:+-e CUDAGRAPH_CAPTURE_SIZES="$CUDAGRAPH_CAPTURE_SIZES"} \ - ${MORI_RDMA_TC:+-e MORI_RDMA_TC=$MORI_RDMA_TC} \ - ${MORI_RDMA_SL:+-e MORI_RDMA_SL=$MORI_RDMA_SL} \ - ${MORI_SHMEM_HEAP_SIZE:+-e MORI_SHMEM_HEAP_SIZE=$MORI_SHMEM_HEAP_SIZE} \ - ${PREFILL_DEEPEP_BACKEND:+-e PREFILL_DEEPEP_BACKEND=$PREFILL_DEEPEP_BACKEND} \ - ${DECODE_DEEPEP_BACKEND:+-e DECODE_DEEPEP_BACKEND=$DECODE_DEEPEP_BACKEND} \ - ${ENABLE_DBO:+-e ENABLE_DBO=$ENABLE_DBO} \ - ${DBO_COMM_SMS:+-e DBO_COMM_SMS=$DBO_COMM_SMS} \ - ${ENABLE_PROFILING:+-e ENABLE_PROFILING=$ENABLE_PROFILING} \ - ${NCCL_IB_HCA:+-e NCCL_IB_HCA=$NCCL_IB_HCA} \ - ${NCCL_IB_GID_INDEX:+-e NCCL_IB_GID_INDEX=$NCCL_IB_GID_INDEX} \ - ${NCCL_NET_GDR_LEVEL:+-e NCCL_NET_GDR_LEVEL=$NCCL_NET_GDR_LEVEL} \ - ${NCCL_CROSS_NIC:+-e NCCL_CROSS_NIC=$NCCL_CROSS_NIC} \ - ${NCCL_SOCKET_IFNAME:+-e NCCL_SOCKET_IFNAME=$NCCL_SOCKET_IFNAME} \ - ${GLOO_SOCKET_IFNAME:+-e GLOO_SOCKET_IFNAME=$GLOO_SOCKET_IFNAME} \ - -e MORI_SOCKET_IFNAME=${MORI_SOCKET_IFNAME:-eth0} \ - ${MORI_IB_GID_INDEX:+-e MORI_IB_GID_INDEX=$MORI_IB_GID_INDEX} \ - ${MORI_RDMA_DEVICES:+-e MORI_RDMA_DEVICES=$MORI_RDMA_DEVICES} \ - ${MORI_NUM_QP_PER_PE:+-e MORI_NUM_QP_PER_PE=$MORI_NUM_QP_PER_PE} \ - ${VLLM_MORIIO_QP_PER_TRANSFER:+-e VLLM_MORIIO_QP_PER_TRANSFER=$VLLM_MORIIO_QP_PER_TRANSFER} \ - ${VLLM_MORIIO_NUM_WORKERS:+-e VLLM_MORIIO_NUM_WORKERS=$VLLM_MORIIO_NUM_WORKERS} \ - -e GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.8} \ - -e GPUS_PER_NODE=${GPUS_PER_NODE:-8} \ - ${GPU_MAX_HW_QUEUES:+-e GPU_MAX_HW_QUEUES=$GPU_MAX_HW_QUEUES} \ - ${HIP_FORCE_DEV_KERNARG:+-e HIP_FORCE_DEV_KERNARG=$HIP_FORCE_DEV_KERNARG} \ - ${HSA_NO_SCRATCH_RECLAIM:+-e HSA_NO_SCRATCH_RECLAIM=$HSA_NO_SCRATCH_RECLAIM} \ - ${VLLM_HANDSHAKE_TIMEOUT_MINS:+-e VLLM_HANDSHAKE_TIMEOUT_MINS=$VLLM_HANDSHAKE_TIMEOUT_MINS} \ - ${VLLM_ENGINE_READY_TIMEOUT_S:+-e VLLM_ENGINE_READY_TIMEOUT_S=$VLLM_ENGINE_READY_TIMEOUT_S} \ - ${ROCSHMEM_HEAP_SIZE:+-e ROCSHMEM_HEAP_SIZE=$ROCSHMEM_HEAP_SIZE} \ - ${ROCSHMEM_MAX_NUM_CONTEXTS:+-e ROCSHMEM_MAX_NUM_CONTEXTS=$ROCSHMEM_MAX_NUM_CONTEXTS} \ - ${LOG_WAIT_TIMEOUT_SECONDS:+-e LOG_WAIT_TIMEOUT_SECONDS=$LOG_WAIT_TIMEOUT_SECONDS} \ - ${TRITON_CACHE_DIR:+-e TRITON_CACHE_DIR=$TRITON_CACHE_DIR} \ - ${VLLM_CACHE_ROOT:+-e VLLM_CACHE_ROOT=$VLLM_CACHE_ROOT} \ - ${COMGR_CACHE_DIR:+-e COMGR_CACHE_DIR=$COMGR_CACHE_DIR} \ - ${AITER_JIT_DIR:+-e AITER_JIT_DIR=$AITER_JIT_DIR} \ - -e DISTRIBUTED_TIMEOUT_SECONDS=${DISTRIBUTED_TIMEOUT_SECONDS:-7200} \ - -e VLLM_RPC_TIMEOUT=${VLLM_RPC_TIMEOUT:-300000} \ - -e VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=${VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS:-3600} \ - ${CONNECTOR_ENV_ARGS} \ - ${VLLM_CUDAGRAPH_MODE:+-e VLLM_CUDAGRAPH_MODE=$VLLM_CUDAGRAPH_MODE} \ - ${CUDAGRAPH_CAPTURE_SIZES:+-e CUDAGRAPH_CAPTURE_SIZES="$CUDAGRAPH_CAPTURE_SIZES"} \ - --name $DOCKER_CONT_NAME \ - $DOCKER_IMAGE_NAME -c " - mkdir -p /run_logs/${SLURM_JOB_ID} - $RUN_FILE_FULL 2>&1 | tee /run_logs/${SLURM_JOB_ID}/pd_vllm_bench_NODE${SLURM_PROCID}.log - " -' -srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'docker stop $DOCKER_CONT_NAME 2>/dev/null || true; docker rm $DOCKER_CONT_NAME 2>/dev/null || true' - diff --git a/scripts/vllm_dissag/README.MD b/scripts/vllm_dissag/README.MD index 17660abe..921bc54d 100644 --- a/scripts/vllm_dissag/README.MD +++ b/scripts/vllm_dissag/README.MD @@ -53,6 +53,7 @@ slurm gate rejects any other pairing. Dense models are TP-only; the DeepSeek fam | `DeepSeek-V3` | MoE | 3, 4 (wideEP only) | validated serving (moriio+MoRI-EP) | | `DeepSeek-V3-5layer` | MoE | 3, 4 (wideEP only) | smoke/dev variant | | `DeepSeek-R1` | MoE | 3, 4 (wideEP only) | needs a co-versioned AITER image (see Caveats) | +| `GLM-5.1-FP8` | MoE + DSA | 3 (moriio wideEP only) | own image (`glmv5.1` Dockerfile); 1P/1D EP8 + 2P/2D EP16; EP32 known-broken | ## Prerequisites @@ -234,7 +235,7 @@ lengths and the model is scored on how many it retrieves. Select it in place of sweep via the launcher's benchmark hook: ```bash -export BENCHMARK_SCRIPT_FILE=benchmark_niah.sh +export BENCHMARK_SCRIPT=niah export NIAH_WORDS="2000,8000,20000,35000" # context sizes (words); optional ``` diff --git a/scripts/vllm_dissag/connectors/moriio.env b/scripts/vllm_dissag/connectors/moriio.env index 780f0214..29ed23ba 100644 --- a/scripts/vllm_dissag/connectors/moriio.env +++ b/scripts/vllm_dissag/connectors/moriio.env @@ -23,18 +23,7 @@ MORI_RDMA_TC=41 MORI_RDMA_SL=0 MORI_IO_SL=1 MORI_IB_ENABLE_RELAXED_ORDERING=1 -MORI_IB_GID_INDEX=3 -# RDMA NIC allowlist (MI300 + CX7 / RoCE). Without this MoRI auto-enumerates ALL ibv -# devices incl. the mgmt NICs (mlx5_1=eth0, mlx5_6=eth1 on the 10.158 mgmt net), and -# tries to establish QPs over a non-routable/mgmt fabric -> ibverbs.cpp:189 "Connection -# timed out" at the prefill->decode KV transfer. Restrict to the 8 GPU-RoCE NICs -# (rdma0-7 on the 10.224 fabric) per the dist-inf-cookbook cluster-rdma-env-recommender. -# NCCL/GLOO use eth0 for their (non-RDMA) control sockets. Override per-fabric if needed. -MORI_RDMA_DEVICES=mlx5_0,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_7,mlx5_8,mlx5_9 -NCCL_IB_HCA=mlx5_0,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_7,mlx5_8,mlx5_9 -NCCL_IB_GID_INDEX=3 -NCCL_SOCKET_IFNAME=eth0 -GLOO_SOCKET_IFNAME=eth0 +MORI_IB_GID_INDEX=1 MORI_NUM_QP_PER_PE=8 VLLM_MORIIO_QP_PER_TRANSFER=2 VLLM_MORIIO_NUM_WORKERS=4 diff --git a/scripts/vllm_dissag/connectors/moriio.sh b/scripts/vllm_dissag/connectors/moriio.sh index e1c7a51a..ffb3ee14 100644 --- a/scripts/vllm_dissag/connectors/moriio.sh +++ b/scripts/vllm_dissag/connectors/moriio.sh @@ -129,32 +129,32 @@ connector_runtime_patch() { # from (Dockerfile VLLM_REF). There is no generic runtime .py patcher for those — # that would be a drifting duplicate of fixes already upstream in the fork. # - # EXCEPTION — GLM-5.1-FP8 (GlmMoeDsaForCausalLM, MLA + DSA sparse attention): + # EXCEPTION — GLM-5.* (GlmMoeDsaForCausalLM, MLA + DSA sparse attention): # DSA is a NEW attention family the MoRIIO connector was never built for. It adds # a 2nd KV cache per layer (indexer) with a different geometry, which the - # single-geometry connector mis-handles -> disagg KV transfer stalls; plus a DSA - # invalid-token kernel bug (#45324) that produces `!!!`. These are model-specific + # single-geometry connector mis-handles -> disagg KV transfer stalls. Model-specific # code gaps, applied here as idempotent, anchor-based, self-skipping .py patchers # (they no-op cleanly if the fix is native/refactored on the chosen image). Gated # on MODEL_NAME so DeepSeek/other models are a pure no-op (byte-identical to before). # The MoRI version is pinned by the Dockerfile MORI_REF (post-1.2.1 main with the # large-transfer notify/mapping fixes #424/#436/#432 baked in); if a newer MoRI is # needed, update MORI_REF and rebuild the image — no runtime library swap here. - [ "${MODEL_NAME:-}" = "GLM-5.1-FP8" ] || return 0 + case "${MODEL_NAME:-}" in + GLM-5.*) ;; + *) return 0 ;; + esac _glm_dsa_runtime_patch } -# GLM-5.1 DSA patchers (see connector_runtime_patch). Ported from MAD-private #338. -# Resolves the vLLM install dir, then applies the 4 required patchers in order, -# aborting on a hard failure (a real failure means GLM emits garbage or stalls, so -# failing at launch is correct). Patchers self-skip (rc 0) when their anchor is -# absent, so an image that already carries or refactored a fix no-ops cleanly. +# GLM-5.* DSA patchers (see connector_runtime_patch). Ported from MAD-private #338. +# Resolves the vLLM install dir, then applies the dual-KV / abort fallbacks in order, +# aborting on a hard failure. Patchers self-skip (rc 0) when their anchor is absent. +# Two patchers are OPT-IN because they crash the v0.27 GLM image (see below). _glm_dsa_runtime_patch() { - # GLM_SKIP_PATCHERS=1: the serving image already carries the GLM-5.1 DSA fixes - # in-source (e.g. the #47766 stack image built from raviguptaamd/vllm@ - # glm5.1-dsa-wideEP_on_shikpate_06_29_customer). Skip ALL runtime patchers — they - # are redundant, and the persistent-gate/sampling-overlay patchers would actively - # REGRESS a baked image (turn persistent MLA off / overwrite stock aiter kernels). + # GLM_SKIP_PATCHERS=1: the serving image already carries the GLM DSA fixes + # in-source (vLLM #47766 cache-key + dual-KV). Skip ALL runtime patchers — they + # are redundant, and the persistent-gate/sentinel patchers would actively REGRESS + # a baked image (asm_mla.cu abort / hipErrorIllegalAddress). if [ "${GLM_SKIP_PATCHERS:-0}" = "1" ]; then echo "[glm] GLM_SKIP_PATCHERS=1: image carries DSA fixes in-source; skipping runtime patchers." return 0 @@ -166,22 +166,30 @@ _glm_dsa_runtime_patch() { echo "Error: [glm] cannot locate vLLM install dir for DSA patchers. Aborting." >&2 exit 1 fi - echo "[glm] MODEL_NAME=GLM-5.1-FP8: applying DSA runtime patchers against ${_vllm_dir}" - - # Ordered list of REQUIRED patchers (all abort on hard failure). - # GLM_PERSIST_GATE=0 skips the persistent-MLA accuracy gate (debug only: to test - # whether the non-persistent kernel it routes to is what crashes disagg at >=8k). - local _gate_patcher="apply_glm_dsa_persistent_kernel_gate_fix.py" - [ "${GLM_PERSIST_GATE:-1}" = "0" ] && _gate_patcher="" + echo "[glm] MODEL_NAME=${MODEL_NAME}: applying DSA runtime patchers against ${_vllm_dir}" + + # apply_glm_dsa_persistent_kernel_gate_fix.py ports vLLM #47567. OFF by default: + # #47766 superseded it, and forcing non-persistent MLA on this image aborts at + # asm_mla.cu:945 (fp8/fp8 gqa_ratio=64 has no non-persistent kernel). Set + # GLM_PERSIST_GATE=1 only on an image that predates #47766. + local _gate_patcher="" + [ "${GLM_PERSIST_GATE:-0}" = "1" ] && _gate_patcher="apply_glm_dsa_persistent_kernel_gate_fix.py" + # apply_glm_dsa_kernel_fix.py ports still-open vLLM #45324 (sentinel 0->-1). OFF + # by default: this image ships 0 deliberately because aiter mla_decode_fwd + # dereferences the index (-1 -> hipErrorIllegalAddress on short disagg decode). + # Set GLM_DSA_SENTINEL_FIX=1 only on an older image that genuinely has the #45324 bug. + local _dsa_sentinel_patcher="" + [ "${GLM_DSA_SENTINEL_FIX:-0}" = "1" ] && _dsa_sentinel_patcher="apply_glm_dsa_kernel_fix.py" local _p for _p in \ - apply_glm_dsa_kernel_fix.py \ + ${_dsa_sentinel_patcher} \ apply_glm_dsa_moriio_dualkv_fix.py \ apply_glm_dsa_moriio_engine_fix.py \ apply_glm_dsa_moriio_gate_fix.py \ apply_glm_moriio_abort_guard_fix.py \ ${_gate_patcher} \ apply_glm_aiter_sampling_oob_fix.py; do + [ -n "${_p}" ] || continue local _py="${_patch_dir}/${_p}" if [ ! -f "${_py}" ]; then echo "Error: [glm] required patcher ${_py} not found. Aborting." >&2 @@ -189,7 +197,7 @@ _glm_dsa_runtime_patch() { fi echo "[glm] applying ${_p}" python3 "${_py}" "${_vllm_dir}" 2>&1 || { - echo "Error: [glm] ${_p} failed — GLM-5.1 would emit garbage or stall. Aborting." >&2 + echo "Error: [glm] ${_p} failed — ${MODEL_NAME} would emit garbage or stall. Aborting." >&2 exit 1 } done @@ -281,8 +289,9 @@ connector_launch_worker() { # (decode.dp), NOT an env knob: mori derives recv capacity from the send width # (MaxNumTokensToRecvPerRank returns min(ceil(maxTotalRecvTokens/ws), # maxNumInpTokenPerRank)), so shrinking the width alone under-provisions recv and - # trips a device assert during vLLM's profiling dummy run. See - # skills_vllm_disagg.md for the measurements and the dead ends. + # trips a device assert during vLLM's profiling dummy run. Do not try to + # raise recv via VLLM_MORI_MAX_TOTAL_RECV_TOKENS — that knob is a min() + # clamp and can only LOWER capacity. local extra_args=() kv_args=() if [[ "$role" == "master" ]]; then diff --git a/scripts/vllm_dissag/keepalive_bench.sh b/scripts/vllm_dissag/keepalive_bench.sh deleted file mode 100755 index 68d80dcb..00000000 --- a/scripts/vllm_dissag/keepalive_bench.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Keepalive with LIGHT heartbeat traffic: holds the disagg server up AND sends a -# tiny request every ~20s so prefill discovery/ping stays registered (idle sleep -# lets the prefill ZMQ ping die ~2min in). Runs KEEPALIVE_MINS (default 90). -: "${KEEPALIVE_MINS:=90}" -PORT="${BENCHMARK_PORT:-30000}" -MODEL="/mnt/m2m_nobackup/models_blog/GLM-5.1-FP8" -echo "[keepalive] light-traffic hold ${KEEPALIVE_MINS}min on :${PORT}" -_end=$(( $(date +%s) + KEEPALIVE_MINS*60 )) -i=0 -while [ "$(date +%s)" -lt "$_end" ]; do - curl -s -m 30 "http://127.0.0.1:${PORT}/v1/completions" -H "Content-Type: application/json" \ - -d "{\"model\":\"${MODEL}\",\"prompt\":\"hi\",\"max_tokens\":1,\"temperature\":0}" >/dev/null 2>&1 - i=$((i+1)) - [ $((i % 3)) -eq 0 ] && echo "[keepalive] heartbeat $i, $(( (_end-$(date +%s))/60 ))min left" - sleep 20 -done -echo "[keepalive] done" diff --git a/scripts/vllm_dissag/models.json b/scripts/vllm_dissag/models.json index 6fc3849a..2d9f0ab4 100644 --- a/scripts/vllm_dissag/models.json +++ b/scripts/vllm_dissag/models.json @@ -278,6 +278,38 @@ }, "args": "-N 2 -n 2" }, + { + "name": "pyt_vllm_disagg_mori_glm-5.1-fp8", + "dockerfile": "../../docker/vllm_disagg_inference.glmv5.1", + "scripts": "run_xPyD_models.slurm", + "url": "", + "data": "huggingface", + "n_gpus": "-1", + "owner": "mad.support@amd.com", + "training_precision": "", + "tags": [ + "pyt", + "vllm", + "vllm_disagg", + "mori_ep", + "inference" + ], + "timeout": -1, + "distributed": { + "launcher": "slurm_multi" + }, + "env_vars": { + "DOCKER_IMAGE_NAME": "", + "MODEL_NAME": "GLM-5.1-FP8", + "xP": "1", + "yD": "1", + "RUN_MORI": "1", + "RUN_DEEPEP": "0", + "GLM_SKIP_PATCHERS": "1", + "BENCHMARK_COMBINATIONS": "1024/1024" + }, + "args": "-N 2 -n 2" + }, { "name": "pyt_vllm_disagg_deepep_deepseek-v3", "dockerfile": "../../docker/vllm_disagg_inference", diff --git a/scripts/vllm_dissag/models.yaml b/scripts/vllm_dissag/models.yaml index 150143e6..df35ae85 100644 --- a/scripts/vllm_dissag/models.yaml +++ b/scripts/vllm_dissag/models.yaml @@ -160,17 +160,10 @@ _deepseek_recipe_env: &deepseek_recipe_env # above. No tp: blocks — TP is unsupported for these models. DeepSeek-V3: env: *deepseek_recipe_env - # NEWER-BASE ADAPTATION (isolated to this model): Shiksha's newer vLLM base added - # TritonMLAMetadataBuilder._reserve_attn_logits_workspace(), which pre-reserves the - # decode split-KV logits workspace at WORST CASE - # (max_num_seqs x q_heads x max_kv_splits(max_model_len) x lse_dim x fp32). With - # DSV3's defaults (max_num_seqs=256, max_model_len~163k) this reserves ~128 GiB and - # OOMs at KV init. Cap max-num-seqs + max-model-len to bound the workspace. This is a - # per-model dp: flag (model_args), fully isolated -- does NOT touch GLM/other recipes. prefill: - dp: "--max-num-seqs 64 --max-model-len 32768" + dp: "" decode: - dp: "--max-num-seqs 64 --max-model-len 32768" + dp: "" DeepSeek-V3-5layer: env: *deepseek_recipe_env diff --git a/scripts/vllm_dissag/niah_200k.py b/scripts/vllm_dissag/niah_200k.py deleted file mode 100755 index 65273bc1..00000000 --- a/scripts/vllm_dissag/niah_200k.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python3 -"""Needle-in-a-haystack sweep including long context (validated to 200K tokens). - -Hides 10 animal names at even intervals in a filler-word haystack and asks the model to -list them back. Reports found/10, end-to-end latency, and the server-reported -prompt_tokens per length, and can dump the whole run to JSON. - -Usage: - niah_200k.py [lengths_csv] [out_json] - base_url e.g. http://127.0.0.1:20005 (prefill/serve port, or the router) - lengths_csv comma-separated WORD counts (default: 2k..200k) - out_json optional path to write results - - NIAH_MODEL= override the served model id (default below) - -Lengths are given in WORDS to stay comparable with earlier published runs. On this filler -the GLM tokenizer lands ~1 token/word, so words ~= tokens (the script prints the actual -prompt_tokens so you can check). - -NOTE: the FIRST request after a server boot pays cold Triton JIT and can take >80s with a -prefill instance running eager. Warm the server (or use a generous timeout) before -treating any latency number here as steady-state. -""" -import json, os, sys, time, random, urllib.request - -BASE = sys.argv[1] if len(sys.argv) > 1 else "http://127.0.0.1:20005" -LENGTHS = [int(x) for x in (sys.argv[2].split(",") if len(sys.argv) > 2 else - "2000,8000,16000,20000,28000,35000,64000,100000,150000,200000".split(","))] -OUT = sys.argv[3] if len(sys.argv) > 3 else None -URL = BASE.rstrip("/") + "/v1/chat/completions" -MODEL = os.environ.get("NIAH_MODEL", "/mnt/m2m_nobackup/models_blog/GLM-5.1-FP8") - -FILLER = ("table chair window bottle pencil garden river mountain coffee planet " - "engine guitar pillow ticket basket candle market silver button orange").split() -ANIMALS = ["elephant", "giraffe", "kangaroo", "penguin", "dolphin", - "tiger", "rhinoceros", "octopus", "crocodile", "panda"] -SYS = ("You read a word list and pick out the animals. Reply with a single " - "comma-separated list of lowercase animal names. Output nothing else.") - - -def hay(n, seed=0): - rng = random.Random(seed) - w = [rng.choice(FILLER) for _ in range(n)] - step = max(n // (len(ANIMALS) + 1), 1) - for i, a in enumerate(ANIMALS): - w[min((i + 1) * step, len(w) - 1)] = a - return " ".join(w) - - -def run(n, seed=0, timeout=1800): - body = { - "model": MODEL, - "messages": [ - {"role": "system", "content": SYS}, - {"role": "user", "content": "Find the animals in this list:\n\n" + hay(n, seed)}, - ], - "temperature": 0, - "max_tokens": 128, - "chat_template_kwargs": {"enable_thinking": False}, - } - req = urllib.request.Request(URL, data=json.dumps(body).encode(), - headers={"Content-Type": "application/json"}) - t = time.time() - try: - r = json.loads(urllib.request.urlopen(req, timeout=timeout).read()) - m = r["choices"][0]["message"] - txt = ((m.get("content") or "") + " " + (m.get("reasoning") or "")).lower() - found = sorted(a for a in ANIMALS if a in txt) - u = r.get("usage") or {} - rec = {"words": n, "seed": seed, "found": len(found), "latency_s": round(time.time() - t, 1), - "prompt_tokens": u.get("prompt_tokens"), "animals": found} - print("words=%7d tok=%-7s found=%2d/10 (%6.1fs) %s" % ( - n, rec["prompt_tokens"], rec["found"], rec["latency_s"], found), flush=True) - return rec - except Exception as e: - rec = {"words": n, "seed": seed, "found": -1, "latency_s": round(time.time() - t, 1), - "error": str(e)[:200]} - print("words=%7d ERROR (%.1fs) %s" % (n, rec["latency_s"], rec["error"]), flush=True) - return rec - - -results = [run(n) for n in LENGTHS] -if OUT: - with open(OUT, "w") as f: - json.dump(results, f, indent=2) - print("wrote", OUT, flush=True) diff --git a/scripts/vllm_dissag/run_xPyD_models.slurm b/scripts/vllm_dissag/run_xPyD_models.slurm index 365883b8..6e3982b7 100755 --- a/scripts/vllm_dissag/run_xPyD_models.slurm +++ b/scripts/vllm_dissag/run_xPyD_models.slurm @@ -455,14 +455,13 @@ BENCHMARK_COMBINATIONS="${BENCHMARK_COMBINATIONS:-}" # Benchmark script selector: BENCHMARK_SCRIPT tag -> file run by the launcher. # sweep (default) -> benchmark_xPyD.sh (general concurrency sweep) # long_context -> benchmark_long_context.sh (per-shape warmup, c=1-first) -# keepalive -> keepalive_bench.sh (hold server up KEEPALIVE_MINS -# for external accuracy probes) +# niah -> benchmark_niah.sh (needle-in-a-haystack; NIAH_WORDS) BENCHMARK_SCRIPT="${BENCHMARK_SCRIPT:-sweep}" case "$BENCHMARK_SCRIPT" in sweep) BENCHMARK_SCRIPT_FILE="benchmark_xPyD.sh" ;; long_context) BENCHMARK_SCRIPT_FILE="benchmark_long_context.sh" ;; - keepalive) BENCHMARK_SCRIPT_FILE="keepalive_bench.sh" ;; - *) echo "Error: invalid BENCHMARK_SCRIPT='$BENCHMARK_SCRIPT' (valid: sweep, long_context, keepalive)" >&2; exit 1 ;; + niah) BENCHMARK_SCRIPT_FILE="benchmark_niah.sh" ;; + *) echo "Error: invalid BENCHMARK_SCRIPT='$BENCHMARK_SCRIPT' (valid: sweep, long_context, niah)" >&2; exit 1 ;; esac if [[ ! -f "$BENCHMARK_SCRIPT_FILE" ]]; then echo "Error: selected benchmark script '$BENCHMARK_SCRIPT_FILE' not found in $(pwd)." >&2 @@ -623,6 +622,7 @@ docker run --rm \ ${DECODE_MORI_BACKEND:+-e DECODE_MORI_BACKEND=$DECODE_MORI_BACKEND} \ -e MODELS_YAML_PROTECT="${MODELS_YAML_PROTECT:-}" \ ${GLM_PERSIST_GATE:+-e GLM_PERSIST_GATE=$GLM_PERSIST_GATE} \ + ${GLM_DSA_SENTINEL_FIX:+-e GLM_DSA_SENTINEL_FIX=$GLM_DSA_SENTINEL_FIX} \ ${GLM_SKIP_PATCHERS:+-e GLM_SKIP_PATCHERS=$GLM_SKIP_PATCHERS} \ ${KV_BLOCK_SIZE:+-e KV_BLOCK_SIZE=$KV_BLOCK_SIZE} \ ${KV_CACHE_MEMORY_BYTES:+-e KV_CACHE_MEMORY_BYTES=$KV_CACHE_MEMORY_BYTES} \ @@ -631,7 +631,11 @@ docker run --rm \ ${KV_CACHE_DTYPE:+-e KV_CACHE_DTYPE=$KV_CACHE_DTYPE} \ ${MORIIO_TOY_PROXY:+-e MORIIO_TOY_PROXY=$MORIIO_TOY_PROXY} \ ${BENCHMARK_SCRIPT_FILE:+-e BENCHMARK_SCRIPT_FILE=$BENCHMARK_SCRIPT_FILE} \ - ${KEEPALIVE_MINS:+-e KEEPALIVE_MINS=$KEEPALIVE_MINS} \ + ${NIAH_WORDS:+-e NIAH_WORDS=$NIAH_WORDS} \ + ${NIAH_SEEDS:+-e NIAH_SEEDS=$NIAH_SEEDS} \ + ${NIAH_WARMUP:+-e NIAH_WARMUP=$NIAH_WARMUP} \ + ${NIAH_MAXTOK:+-e NIAH_MAXTOK=$NIAH_MAXTOK} \ + ${NIAH_TIMEOUT:+-e NIAH_TIMEOUT=$NIAH_TIMEOUT} \ ${PREFILL_CUDAGRAPH_MODE:+-e PREFILL_CUDAGRAPH_MODE=$PREFILL_CUDAGRAPH_MODE} \ ${DECODE_CUDAGRAPH_MODE:+-e DECODE_CUDAGRAPH_MODE=$DECODE_CUDAGRAPH_MODE} \ ${CUDAGRAPH_CAPTURE_SIZES:+-e CUDAGRAPH_CAPTURE_SIZES="$CUDAGRAPH_CAPTURE_SIZES"} \ diff --git a/scripts/vllm_dissag/skills_vllm_disagg.md b/scripts/vllm_dissag/skills_vllm_disagg.md deleted file mode 100644 index cb837dc2..00000000 --- a/scripts/vllm_dissag/skills_vllm_disagg.md +++ /dev/null @@ -1,278 +0,0 @@ -# skills_vllm_disagg.md - -Hard-won operational knowledge for **vLLM PD-disaggregated WideEP serving on AMD MI300X** -(MoRI-EP all-to-all + MoRI-IO RDMA KV transfer), learned while bringing GLM-5.1-FP8 -(MLA + DeepSeek Sparse Attention) onto vLLM v0.27. - -Everything below is *measured*, not theorised. Where a belief turned out to be wrong, -the wrong belief is kept alongside the correction — those are the expensive lessons. - ---- - -## 1. Benchmarking methodology (read this first — it invalidated three of my conclusions) - -### 1.1 ALWAYS discard the first bench run after a boot -The first real request after startup pays **cold Triton JIT** for the sparse/indexer -kernels (`_indexer_k_quant_and_cache_kernel`, `generate_sparse_seqlen_kernel`, -`_convert_req_index_to_global_index_kernel`). Measured on 1P/1D EP8, identical bench: - -| run | TTFT | TPOT | -|---|---|---| -| 1st after boot (cold) | 13,451 ms | 88.7 ms | -| 2nd (warm) | **906 ms** | 88.0 ms | - -TTFT moved **14.9x**; TPOT barely moved. A single cold run made me invent (and act on) -a false "scheduler admission" theory twice. Discard it, or warm up explicitly. - -### 1.2 Why cold JIT lands on TTFT and not TPOT -With the standard recipe **prefill = eager (`CUDAGraphMode.NONE`)**, decode = PIECEWISE: -- prefill has no graph capture at boot -> its kernels compile lazily on the **first real - request** -> the whole compile cost is inside TTFT. -- decode captured graphs at boot ("Graph capturing finished") -> already warm -> TPOT is - correct even on the cold run. -This asymmetry is diagnostic: *cold-JIT symptoms show up in TTFT only*. - -### 1.3 The harness warmup is not a warmup -`benchmark_xPyD.sh` warms at `isl=32 osl=32 con=1` — which never exercises a 1024/8192/28672 -prefill path, nor the decode cudagraph batch sizes. The first *measured* cell therefore -absorbs residual JIT. Fix applied: a per-shape warmup at the real ISL/OSL before each -shape's cells (writes to a separate `_SHAPEWARMUP.log` so it can't pollute the CSV). - -### 1.4 Client timeouts read as server failures -A 50-80s curl timeout against a cold server returns an **empty body**, which looks exactly -like a crash. Use >=300s on the first request. This produced a false "total failure" -verdict during EP32 debugging. - -### 1.5 Sanity-check against physics before blaming a kernel -GLM-5.1-FP8 activates ~37.7B params/token. At 5.3 TB/s HBM, fp8: -- full model on one rank: **7.1 ms/step** -- 1/8 of experts per rank (EP8): **0.9 ms/step** - -Measured 290-300 ms => ~320x the bound. That immediately rules out "compute" or "bandwidth" -and says *stall / oversized transfer*. Do this arithmetic early; it saves hours. - ---- - -## 2. The big perf trap: `max_num_batched_tokens` sizes the MoRI EP buffer - -### 2.1 The chain -``` -vllm/model_executor/layers/fused_moe/layer.py:349 - max_num_tokens = max_num_batched_tokens # 8192 default (SCHEDULER knob) -vllm/model_executor/layers/fused_moe/all2all_utils.py:181 - max_num_tokens_per_dp_rank = moe.max_num_tokens -vllm/distributed/device_communicators/all2all.py (MoriAll2AllManager) - max_num_inp_token_per_rank = -``` -`max_num_batched_tokens` is a **chunked-prefill scheduler** setting. Using it to size the -EP dispatch/combine buffer means a **decode** instance runs an 8192-token-wide all-to-all -**every step, per layer, x78 layers**, while decoding a handful of tokens. - -### 2.2 Signature of this bug -- fixed per-step cost: TPOT identical at concurrency 1, 4, 8, 16 -- independent of KV length: 292 ms at isl=128, 297 ms at isl=1024 -- batching still scales perfectly (con=1 -> 8 gave 8.3x throughput, TPOT flat) -- orders of magnitude above the bandwidth bound -=> "constant oversized transfer", not compute. - -### 2.3 The fix that works today (no code change) -Lower `--max-num-batched-tokens` **on the decode role only** (`models.yaml` `decode.dp:`): - -| decode mnbt | TPOT | TTFT (warm) | out tok/s | -|---|---|---|---| -| 8192 (default) | 302.5 ms | 2431 ms | 24.9 | -| **2048** | **88.0 ms** | **906 ms** | **78.8** | - -3.4x faster decode, and it also *improved* TTFT and throughput. Prefill keeps 8192 (it -genuinely dispatches wide batches). - -### 2.4 What does NOT work (dead ends — do not repeat) -- **`VLLM_MORI_MAX_TOKENS_PER_RANK` alone** (512 or 2048): device assert at boot - ``` - mori .../dispatch_combine/intranode.hpp:134 - `destTokId < config.MaxNumTokensToRecv() && - "Total recv token overflow: increase maxTotalRecvTokens"' - ``` - because vLLM's **profiling/warmup dummy run deliberately pushes - `max_num_batched_tokens` (8192) tokens** through the model. The EP buffer must survive - that even though steady-state decode never needs it. -- **`max_total_recv_tokens` to decouple recv from send: IMPOSSIBLE in current mori.** - ``` - MaxNumTokensToRecvPerRank(): - if maxTotalRecvTokens > 0: - perRank = ceil(maxTotalRecvTokens / worldSize) - return perRank < maxNumInpTokenPerRank ? perRank : maxNumInpTokenPerRank # min() - return maxNumInpTokenPerRank - ``` - It returns **min(perRank, send_width)** — it can only *lower* recv capacity, never raise - it above the send width. `send=1024, recv=65536` behaves identically to leaving it unset. - Recv capacity is structurally bounded by send width. - -### 2.5 The proper upstream fix (not yet done) -Bound the **profiling/dummy run on a decode instance by `max_num_seqs`** instead of -`max_num_batched_tokens`. Then the EP buffer can be narrow and profiling never exceeds it. -This lives entirely in vLLM. (Alternative: a mori change allowing recv > send.) - -### 2.6 MoRI has no tuning knobs (parity gap vs DeepEP) -DeepEP exposes `VLLM_DEEPEP_BUFFER_SIZE_MB`; MoRI hardcodes `warp_num_per_block`, -`block_num`, `rdma_block_num` and inherits its token width from an unrelated scheduler -default. Added `VLLM_MORI_*` knobs for parity (all defaulting to current values). - ---- - -## 2b. Decode CUDA-graph mode: use FULL_AND_PIECEWISE, not PIECEWISE - -`DECODE_CUDAGRAPH_MODE=FULL_AND_PIECEWISE` captures a FULL graph for the uniform-decode -shapes and falls back to PIECEWISE for everything else. Measured on MI300X, 1024/64, warm, -after the max_num_batched_tokens fix in section 2: - -| | PIECEWISE | FULL_AND_PIECEWISE | gain | -|---|---|---|---| -| 1P/1D EP8 TPOT c8 | 88.0 ms | **41.8 ms** | 2.11x | -| 1P/1D EP8 TPOT c16 | 91.5 ms | **45.8 ms** | 2.00x | -| 2P/2D EP16 TPOT c8 | 94.1 ms | **55.4 ms** | 1.70x | -| 2P/2D EP16 TPOT c16 | 96.2 ms | **60.3 ms** | 1.60x | -| 1P/1D output c8 | 78.8 tok/s | **143.7** | 1.82x | -| 2P/2D output c8 | 66.7 tok/s | **113.0** | 1.69x | - -Accuracy is unaffected: NIAH 2k-35k 51/60 (1P/1D) and 55/60 (2P/2D), both inside the -PIECEWISE band (52-53 / 53), no length collapse, memfault=0. - -Three things to know before adopting it: -- **The gain shrinks as EP widens** (2.11x at EP8 -> 1.70x at EP16). FULL captures the whole - decode step *including* the cross-node all2all, and that collective does not compress. - Do not extrapolate to EP32 without measuring. -- **Capture cost:** ~92-94 s and ~3.0-3.5 GiB, versus ~5 s and 7.2 GiB for PIECEWISE - - fewer but larger graphs. Boot is correspondingly longer. Fine for a long-lived server, - notable for CI. -- **FULL is much less cold-JIT sensitive.** Its warmup run already reads steady-state TPOT - (41.35 ms), where PIECEWISE's first post-boot run showed 13.7 s TTFT. The section-1 rule - (discard the first run) still applies, but the penalty is far smaller. - -Two code paths branch on `CUDAGraphMode.FULL` - `sparse_attn_indexer.py:411` and the MoRIIO -connector's READ-mode barrier. Both are guards that *skip* host-side work under FULL, so -enabling it is safe on this stack (the MoRIIO one is already a no-op in WRITE mode). - -## 3. Accuracy: the DSA sparse-index sentinel landmine - -**Never set the invalid/OOB sentinel to `-1`** in -`_convert_req_index_to_global_index_kernel` -(`vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py`). It must be `0`. - -Why: aiter's `mla_decode_fwd` **dereferences** `paged_kv_indices`, so `-1` becomes -`kv_cache + (-1)*stride` -> page-aligned GPU memory access fault -> worker dies -> gloo -DP all-reduce collapse. `0` is masked out by `paged_kv_indptr`/`last_page_len`. - -**Why it hides:** it only fires at **disaggregated long context**. Decode inherits the -prefill's long `seq_lens` -> `generate_sparse_seqlen` widens `paged_kv_indptr` -> `-1` -padding entries land *inside* a live indptr range and get dereferenced. Short prompts and -non-disagg decode never emit an in-range `-1`, so it passes every quick test. -`HIP_LAUNCH_BLOCKING=1` does **not** help (data-dependent OOB, not an async race). - ---- - -## 4. Debugging playbook — isolate the layer before optimising - -The ladder that found the perf bug, in order of cheapness: - -| Suspect | Test | What it showed | -|---|---|---| -| router / KV transfer | bench **prefill-direct** (`:20005`) vs via router (`:30000`) | 341 ms vs 303 ms TPOT -> router + MoRI-IO exonerated (router *does* add TTFT) | -| JIT warmup | re-run on a 30-min-warm server | identical -> not warmup | -| cudagraph config | grep `cudagraph_mode`, `Graph capturing finished`, GiB captured | config correct | -| KV length | sweep isl 128 / 1024 / 8192 at con=1 | flat -> fixed floor, not KV | -| batching | con=1 vs 8 | TPOT flat, throughput 8.3x -> batching fine | -| all2all backend | force `PREFILL_MORI_BACKEND=mori_low_latency` | still broken -> not an HT-kernel bug | - -**Also:** compare against a *known-good* build. Diffing our stack against the passing -v0.25 blog stack showed MoRI, aiter and the recipe were **byte-identical** — only the base -image and vLLM differed. That single comparison exonerated two whole components. - ---- - -## 5. Operational gotchas - -### 5.1 Three caches, three different rules -| cache | keyed by | bake into image? | -|---|---|---| -| `aiter_jit` | (aiter commit, gfx arch) | **yes** — topology-independent, and it is the cold-boot long pole | -| `vllm` (torch.compile/inductor), `triton` | model cfg + batch sizes + cudagraph mode + **topology** (EP8/16/32) | **no** — a stale graph is a real trap; wipe when changing shapes | -| `comgr` | ROCm code objects | harmless | - -Host persistence: `VLLM_CACHE_PERSIST=1` -> -`/mnt/m2m_nobackup/$USER/vllm_jit_cache/a256` -> `/opt/vllm_cache`. -**Keyed by image ID**, so every new image = full cold rebuild. - -If baking `aiter_jit` into an image: **scrub `lock_module_*`, `.ninja_log`, `/root/.mori`, -`/tmp/mori_jit_*` first**, or a fresh container waits on a baton nobody holds -> boot hang. -Verify: a fresh container must start with **zero** ninja/hipcc/clang processes. - -### 5.2 Measured boot times (8 nodes, model on local NVMe) -| scenario | time | -|---|---| -| cold aiter, 2 nodes | ~25 min | -| cold aiter, 4 nodes (2 cold decode) | ~41 min | -| cold aiter, 8 nodes (4 cold decode) | ~106 min | -| **warm cache, 8 nodes** | **~11 min** | - -Changing `max_num_batched_tokens` invalidates the torch.compile cache -> full recompile. - -### 5.3 The aiter baton lock looks like a hang but isn't -`[aiter] waiting for baton release at /opt/vllm_cache/aiter_jit/build/lock_` — -one worker compiles, the rest block. Diagnose by counting build procs *inside the -container*: 100-160 = actively building; **0 on all nodes** = genuinely wedged. - -### 5.4 Readiness: don't count "Application startup complete" -Only nodes running an API server print it. For 2P/2D and 4P/4D the non-master DP ranks -never will. Judge readiness by: router `All servers healthy` + `Graph capturing finished` -+ `GPU KV cache size` per node. - -### 5.5 Verify teardown on EVERY node -`docker rm -f` can leave a container alive on one node; a stale container then collides -with the new run (observed: 2 containers on one node -> prefill shut down mid-boot). -Always re-check `docker ps -q | wc -l == 0` everywhere before relaunching. - -### 5.6 Per-role env: `models.yaml env:` applies to BOTH roles -Prefill and decode often need **opposite** values (e.g. EP buffer width). The pattern is -`PREFILL_*` / `DECODE_*` keys in `env:`, split inside `connectors/moriio.sh` (mirrors the -existing `PREFILL_MORI_BACKEND` / `DECODE_MORI_BACKEND`). Verify it landed by reading -`/proc//environ` **inside the container** — the value is exported into the -server process, not the container shell, so `docker exec env` shows nothing. - -### 5.7 RDMA fabric -- GID index **3** = RoCEv2 IPv4 (check `show_gids` / `sysfs .../gid_attrs/types`). -- Restrict `MORI_RDMA_DEVICES` / `NCCL_IB_HCA` to the 8 GPU-local NICs; leave the mgmt NICs - out or QPs try to form over a non-routable fabric -> `ibverbs.cpp:189 Connection timed out`. -- NCCL/GLOO control sockets on `eth0` (mgmt); KV data on the RDMA NICs. -- **Verify the fabric before blaming code**: `ping -I rdma0` matrix, then `ib_write_bw` - (healthy pair measured 386 Gb/s). A node can be `alloc` in SLURM with a **dead** fabric — - SLURM does not detect this. One dead node cost a whole 2P/2D campaign. -- `ib_write_bw` needs both endpoints co-alive: run it as a single 2-task srun step, not two - separate sruns (a backgrounded server dies when its srun returns). - -### 5.8 Images -Push to a registry (`docker push`) rather than relying on `docker save | ssh | docker load` -serially — parallel `docker pull` across 7 nodes is far faster and removes the -"image missing on one node" failure that silently stalls the launcher barrier. - ---- - -## 6. Open items (state as of 2026-08-16) - -- **4P/4D EP32 silent output corruption.** Cluster boots healthy (memfault=0), but output is - garbage. MoRI + aiter + recipe are byte-identical to the passing v0.25 stack; only base + - vLLM differ => **v0.27 regression**. Both `mori_high_throughput` and `mori_low_latency` - corrupt at EP32 while both are clean at EP8/EP16 => an EP-width (>16 ranks) issue, not a - kernel-specific one. Next probe: 3P/3D (EP24) to test "any EP>16" vs "exactly 32". -- **Proper upstream fix for §2.5** (bound decode profiling by `max_num_seqs`). -- **Prewarmed image** (§5.1) to kill the cold-boot and first-request-JIT costs. - ---- - -## 7. One-line summary of the two bugs found - -1. **Accuracy:** a `-1` sentinel that aiter dereferences -> GPU fault, but only at disagg - long context. Use `0`. -2. **Perf:** the MoE all-to-all buffer is sized from a chunked-prefill *scheduler* knob, so - decode moves an 8192-wide buffer every step. Lower `--max-num-batched-tokens` on the - decode role: **302 ms -> 88 ms TPOT**. From 3d166d4cf3e187409d2affff1aa71d09a7293c4d Mon Sep 17 00:00:00 2001 From: Mir Mustafa Ali Date: Tue, 18 Aug 2026 00:46:06 -0500 Subject: [PATCH 11/23] [GLM-5.1] Point the recipe at the v0.27 image it was validated on. The recipe named a pre-v0.27 image as "validated" and described patchers that no longer apply. Serving GLM-5.1-FP8 on that older image returns empty completions (NIAH 2k = 0/10), so name the image the Dockerfile actually builds and state that the patchers are an old-image fallback. --- scripts/vllm_dissag/models.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/vllm_dissag/models.yaml b/scripts/vllm_dissag/models.yaml index df35ae85..4454e452 100644 --- a/scripts/vllm_dissag/models.yaml +++ b/scripts/vllm_dissag/models.yaml @@ -186,11 +186,13 @@ DeepSeek-R1: # + 1 shared, FP8 block 128. wideEP-only (see WIDE_EP_ONLY_MODELS in the slurm). # # Validated DOCKER_IMAGE_NAME (submit-time, not set here — #171 requires it explicit): -# rocmshared/pytorch-private:vllm-wideep_06_29_2026_Shiksha_dp16_2p2d_mori_v1.2.1_aiter_v0.1.16.post3_nightlybase_mori121 -# On this image the DSA patchers self-adapt: only the invalid-token kernel fix -# (#45324) applies; the MoRIIO dual-KV geometry + completion-gate fixes are NATIVE -# (moriio_layout.py per-layer geometry + engine sealed writes_expected), so those -# patchers cleanly no-op. On the older b10a9f7a image all 4 patchers apply. +# rocmshared/pytorch-private:glm5.1-vllm027-b8 +# built from docker/vllm_disagg_inference.glmv5.1 (VLLM_REF=glm5.1-dsa-wideEP_on_vllm-v0.27). +# It carries the DSA fixes in-source, so the card sets GLM_SKIP_PATCHERS=1 and NO runtime +# patcher runs. Do NOT serve this recipe on a pre-v0.27 image: those predate vLLM #47766 +# and return empty/garbage completions here (NIAH 2k scored 0/10 on +# rocm/pytorch-private:glm-dockerimage-built-09072026). The connectors/moriio.sh patchers +# are a fallback for such older images only (GLM_SKIP_PATCHERS unset). # # GLM differs from DeepSeek in 3 recipe-defining ways (both are MLA MoE, but GLM # is DSA-sparse): From 585b6a84ade03b451a9567fcfec8ebaee791aaed Mon Sep 17 00:00:00 2001 From: Mir Mustafa Ali Date: Tue, 18 Aug 2026 00:15:09 -0500 Subject: [PATCH 12/23] [GLM-5.1] Minimal MAD: drop runtime DSA patchers, make the image pin the contract PR 206 shipped GLM-5.1-FP8 with nine apply_glm_*.py runtime monkeypatch scripts (~1100 lines) invoked from the moriio connector. None of them run on the production path: the validated model card sets GLM_SKIP_PATCHERS=1 because the pinned vLLM (VLLM_REF=glm5.1-dsa-wideEP_on_vllm-v0.27) already carries every DSA fix in-source. They were dead weight that MAD had to keep in sync with a fork. This branch makes MAD carry catalog content only, and makes the image solely responsible for vLLM source fixes: - Delete all nine scripts/vllm_dissag/apply_glm_*.py. - connector_runtime_patch() in connectors/moriio.sh is now a documented no-op for every model; _glm_dsa_runtime_patch() and the GLM MODEL_NAME branch are gone. Non-GLM behavior is unchanged (it already returned 0 before reaching any GLM code), so DeepSeek/Llama/gpt-oss are untouched. - Remove the plumbing that only existed to feed the patchers: the GLM_SKIP_PATCHERS / GLM_PERSIST_GATE / GLM_DSA_SENTINEL_FIX docker -e forwards in run_xPyD_models.slurm, and GLM_SKIP_PATCHERS from the model card in models.json (with no patchers, "skip the patchers" is meaningless). - Rewrite the models.yaml and Dockerfile comments that promised runtime patchers. The docs now state the actual contract: the image must carry the DSA fixes in-source and VLLM_REF is the requirement, not a preference. Trade-off: this removes the fallback path for older images. On an image built from a vLLM ref that predates the DSA fixes, GLM-5.1 now boots and then produces garbage or stalls the disagg KV transfer, with nothing to recover it -- the fix is to rebuild the image, not to patch at runtime. In exchange MAD stops carrying a duplicate of upstream fork code that no supported configuration executes. The Dockerfile, model card, models.yaml recipe, NIAH harness and slurm allowlist are otherwise unchanged. Verified without GPUs: tests/argv_assert.sh 25 passed / 0 failed; models.json parses and the GLM card is intact; DRY_RUN argv for DeepSeek-V3, DeepSeek-V3-5layer, DeepSeek-R1, amd-Llama-3.3-70B-Instruct-FP8-KV and gpt-oss-120b is byte-identical to glm5.1-wideep-mad (so the only delta vs origin/develop is the pre-existing one from that branch); GLM-5.1-FP8 still emits --enable-expert-parallel, --block-size 1, and on decode --max-num-batched-tokens 2048 + FULL_AND_PIECEWISE; bash -n clean. --- ...gg_inference.glmv5.1.ubuntu.amd.Dockerfile | 15 +- .../apply_glm_aiter_sampling_oob_fix.py | 148 ----------- .../apply_glm_dsa_indexer_warmup_fix.py | 251 ------------------ .../vllm_dissag/apply_glm_dsa_kernel_fix.py | 85 ------ .../apply_glm_dsa_moriio_dualkv_fix.py | 176 ------------ .../apply_glm_dsa_moriio_engine_fix.py | 116 -------- .../apply_glm_dsa_moriio_gate_fix.py | 133 ---------- .../apply_glm_dsa_moriio_instrument.py | 92 ------- ...pply_glm_dsa_persistent_kernel_gate_fix.py | 129 --------- .../apply_glm_moriio_abort_guard_fix.py | 98 ------- scripts/vllm_dissag/connectors/moriio.sh | 108 ++------ scripts/vllm_dissag/models.json | 1 - scripts/vllm_dissag/models.yaml | 30 ++- scripts/vllm_dissag/run_xPyD_models.slurm | 3 - 14 files changed, 45 insertions(+), 1340 deletions(-) delete mode 100644 scripts/vllm_dissag/apply_glm_aiter_sampling_oob_fix.py delete mode 100755 scripts/vllm_dissag/apply_glm_dsa_indexer_warmup_fix.py delete mode 100755 scripts/vllm_dissag/apply_glm_dsa_kernel_fix.py delete mode 100755 scripts/vllm_dissag/apply_glm_dsa_moriio_dualkv_fix.py delete mode 100755 scripts/vllm_dissag/apply_glm_dsa_moriio_engine_fix.py delete mode 100755 scripts/vllm_dissag/apply_glm_dsa_moriio_gate_fix.py delete mode 100755 scripts/vllm_dissag/apply_glm_dsa_moriio_instrument.py delete mode 100644 scripts/vllm_dissag/apply_glm_dsa_persistent_kernel_gate_fix.py delete mode 100644 scripts/vllm_dissag/apply_glm_moriio_abort_guard_fix.py diff --git a/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile b/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile index 1053af1e..e015d6f8 100644 --- a/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile +++ b/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile @@ -76,6 +76,14 @@ # - validated recipe knobs baked as ENV. The MoRIIO disagg fixes (#39276 notify, # #41751 LL split, DP-rank hash-failsafe) are native in this vLLM (no runtime patcher). # +# THIS IMAGE IS THE CONTRACT. The MAD side (scripts/vllm_dissag) is catalog/config only +# and ships NO runtime .py patchers: connector_runtime_patch in connectors/moriio.sh is a +# no-op for every model. So EVERY GLM DSA source fix must be carried in-source HERE, by +# VLLM_REF below (glm5.1-dsa-wideEP_on_vllm-v0.27) plus the MoRI/AITER pins. Serving +# GLM-5.1-FP8 from an image built off an older vLLM ref is unsupported: it boots, then +# produces garbage output or stalls the disagg KV transfer, with nothing to fall back on. +# If you need a fix, move the pin and rebuild — do not re-add runtime patchers to MAD. +# # Build context = repo root: # docker build -f docker/vllm_disagg_inference.ubuntu.amd.Dockerfile -t / . # @@ -180,9 +188,10 @@ RUN if [ "${WITH_AITER_BUILD}" != "1" ]; then \ # 3. vLLM: compile from source at the 06_29 validated Wide-EP WRITE-mode branch # (matches the published dist-inf-cookbook mori121 image). Full source compile # (the base ships a different commit). The MoRIIO disagg fixes (#39276 notify, -# #41751 LL split, DP-rank hash-failsafe) are native in this branch, so no runtime -# patcher is needed. Override VLLM_REF to rebuild a different commit; build only -# committed commits (no working-tree edits). +# #41751 LL split, DP-rank hash-failsafe) AND the GLM DSA fixes are native in this +# branch, so no runtime patcher is needed — and none exists in MAD, which is why +# this ref is a hard requirement rather than a preference. Override VLLM_REF to +# rebuild a different commit; build only committed commits (no working-tree edits). # ----------------------------------------------------------------------------- # VLLM_REPO/REF are a PUBLIC GitHub repo + branch (the Wide-EP WRITE-mode vLLM the # dist-inf-cookbook mori121 image builds from). Override to your own vLLM fork/branch. diff --git a/scripts/vllm_dissag/apply_glm_aiter_sampling_oob_fix.py b/scripts/vllm_dissag/apply_glm_aiter_sampling_oob_fix.py deleted file mode 100644 index d491b838..00000000 --- a/scripts/vllm_dissag/apply_glm_aiter_sampling_oob_fix.py +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env python3 -"""Overlay the fixed AITER sampling kernel (ROCm/aiter #3658 + hang cap) into the image. - -DEFECT 2 (the 8k prefill/decode crash): the AITER TopP/TopK sampling kernel -(csrc/cpp_itfs/sampling/sampling.cuh) has two bugs on the released aiter post3 -that this image ships: - - 1. HSA OUT-OF-BOUNDS (ROCm/aiter #3658): SamplingTempStorage::last_valid_id is - never initialized. When a probs row is all-zero / NaN (more likely at long - input, e.g. 8k), the guarded write-back (max_valid != -1) is skipped, and the - fallback `sampled_id = temp_storage.last_valid_id` reads UNINITIALIZED shared - memory -> garbage index -> `probs[row*d + sampled_id]` dereferences OOB and - HSA page-faults ("Memory access fault by GPU node-N"). Deterministic under - CUDA graph (shared-mem residue is stable across replays). This is the silent - worker death at 8k that collapses the disagg DP group -> 503. - Fix: init `last_valid_id = 0` at top of each loop iter + defensive clamp on - the loaded sampled_id before it indexes probs. - - 2. REJECTION-SAMPLING HANG: the bisection `do { ... } while(low < high)` can - spin forever when the [low,high] interval stagnates in float precision on a - degenerate (near-uniform) row -> never-completing HSA signal / hang (the - "sampler hang" that forced the skip-warmup workaround). Fix: cap the loop at - kMaxSamplingRounds=32 (float32 mantissa is exhausted well within 32 rounds, - so healthy distributions always converge via break long before the cap). - -Both fixes land in sampling.cuh. #3658 is MERGED upstream but NOT in the released -aiter post3 (this image). Source: A/B-tested by Shiksha (shikpate); staged fixed -tree at SAMPLING_FIX_DIR. - -METHOD (from Shiksha's validated in-container overlay): copy the whole patched -sampling source dir (.cuh + .py + .jinja) over the container's aiter, then purge -any compiled sampling JIT objects so the kernel recompiles from the fixed source -on next use. - -Idempotent: skips if the fix markers are already present. Model-agnostic at the -kernel level, but invoked from the GLM patch hook. Safe no-op if the staged fix -dir or the target aiter dir is absent. - -Usage: apply_glm_aiter_sampling_oob_fix.py - (vllm_install_dir arg is accepted for hook uniformity but not required; - the aiter dir is resolved via `import aiter`.) -""" -import os -import shutil -import subprocess -import sys - -FIX_DIR = os.environ.get( - "SAMPLING_FIX_DIR", - "/shared_inference/ravgupta/aiter_sampling_fix_3658/sampling_patched", -) -MARKERS = ("last_valid_id = 0", "kMaxSamplingRounds") - - -def _aiter_sampling_dir(): - """Locate the installed aiter sampling source dir (aiter_meta/csrc/...).""" - try: - import aiter # noqa: F401 - except Exception as e: # noqa: BLE001 - print(f"[sampling-fix] aiter not importable ({e}); skipping.") - return None - # The kernel source lives under aiter_meta (sibling of aiter), path is stable. - candidates = [] - try: - import aiter_meta # type: ignore - - candidates.append( - os.path.join(os.path.dirname(aiter_meta.__file__), - "csrc", "cpp_itfs", "sampling") - ) - except Exception: # noqa: BLE001 - pass - # Fallback: search site-packages. - import aiter - sp = os.path.dirname(os.path.dirname(aiter.__file__)) - candidates.append(os.path.join(sp, "aiter_meta", "csrc", "cpp_itfs", "sampling")) - for c in candidates: - if os.path.isdir(c): - return c - print(f"[sampling-fix] could not locate aiter sampling dir (tried {candidates}); skipping.") - return None - - -def main() -> int: - tgt = _aiter_sampling_dir() - if tgt is None: - return 0 # safe no-op - - tgt_cuh = os.path.join(tgt, "sampling.cuh") - if os.path.isfile(tgt_cuh): - cur = open(tgt_cuh, errors="ignore").read() - if all(m in cur for m in MARKERS): - print(f"[sampling-fix] already applied (markers present) in {tgt_cuh}.") - return 0 - - if not os.path.isdir(FIX_DIR): - print(f"[sampling-fix] WARN: staged fix dir {FIX_DIR} not found; leaving image kernel unpatched.", file=sys.stderr) - return 0 - - src_cuh = os.path.join(FIX_DIR, "sampling.cuh") - if not os.path.isfile(src_cuh) or not all(m in open(src_cuh, errors="ignore").read() for m in MARKERS): - print(f"[sampling-fix] WARN: staged {src_cuh} missing/lacks fix markers; skipping.", file=sys.stderr) - return 0 - - # Overlay the whole sampling source dir (.cuh + .py + .jinja), per Shiksha's method. - copied = [] - for fn in os.listdir(FIX_DIR): - s = os.path.join(FIX_DIR, fn) - if os.path.isfile(s): - shutil.copy2(s, os.path.join(tgt, fn)) - copied.append(fn) - print(f"[sampling-fix] overlaid #3658 + hang-cap into {tgt}: {', '.join(sorted(copied))}") - - # Verify. - cur = open(tgt_cuh, errors="ignore").read() - if not all(m in cur for m in MARKERS): - print(f"[sampling-fix] ERROR: markers still absent after overlay in {tgt_cuh}.", file=sys.stderr) - return 1 - - # Purge any compiled sampling JIT objects so the kernel recompiles from source. - purged = 0 - for base in ( - os.path.expanduser("~/.aiter"), "/root/.aiter", "/tmp/aiter", - "/opt/vllm_cache/aiter_jit", os.path.join(os.path.dirname(tgt), "..", "..", "jit"), - ): - if base and os.path.isdir(base): - try: - out = subprocess.run( - ["find", base, "-maxdepth", "6", "-iname", "*sampling_from_probs*"], - capture_output=True, text=True, timeout=60, - ) - for p in out.stdout.split(): - try: - if os.path.isdir(p): - shutil.rmtree(p, ignore_errors=True) - else: - os.remove(p) - purged += 1 - except OSError: - pass - except Exception: # noqa: BLE001 - pass - print(f"[sampling-fix] purged {purged} stale sampling JIT object(s); kernel will recompile from fixed source.") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/vllm_dissag/apply_glm_dsa_indexer_warmup_fix.py b/scripts/vllm_dissag/apply_glm_dsa_indexer_warmup_fix.py deleted file mode 100755 index 3eaef8cf..00000000 --- a/scripts/vllm_dissag/apply_glm_dsa_indexer_warmup_fix.py +++ /dev/null @@ -1,251 +0,0 @@ -#!/usr/bin/env python3 -"""Force-compile the GLM-5.1 DSA sparse-attention indexer Triton kernels at BOOT. - -PROBLEM (root cause of the "first big prompt stalls the whole DP group" hang): - The DSA indexer's Triton kernels are seq-length specialized: - - v1/attention/ops/triton_fp8_mqa_logits.py flips `matrix_instr_nonkdim` - at seq_len<=1024 and launches with grid=[(seq_len,)] (seq_len is a - specialized kernel arg) -> a >1024-row prefill needs a *different* JIT - specialization than a <=1024-row one. - But the boot-time warmup never drives the indexer: - - profile_run() calls _dummy_run(is_profile=True) with force_attention=False - and cudagraph mode NONE -> attn_metadata stays None -> sparse_attn_indexer - takes the `sparse_attn_indexer_fake` path (see the "careful! this will be - None in dummy run" comment in layers/sparse_attn_indexer.py). The real - kernels are never compiled. - - _warmup_and_capture() only sets force_attention=True when the cudagraph - runtime mode is FULL; the DSA indexer builder reports UNIFORM_BATCH, so on - this ROCm/DP build the mixed prefill-decode graphs are PIECEWISE and - force_attention stays False. Even when attention IS forced, capture uses - uniform-decode / small mixed batches -- never a large prefill at - max_num_batched_tokens -- so the >1024 specialization is still absent. - Effect: the first >=8k prompt JIT-compiles the indexer kernel mid-inference on - whichever DP rank happens to receive it. That rank falls out of the DP lockstep - gloo all_reduce (coordinate_batch_across_dp) while it compiles -> the whole DP - group collapses. It is also a general cold-cache robustness hole. - -FIX (surgical, reuses vLLM's OWN metadata construction -- no hand-synthesized -tensors, so zero risk of a bad-input crash at boot): - 1. gpu_model_runner.py: add a `_maybe_warmup_dsa_indexer()` method. It is a - strict NO-OP unless one of the runner's attention backends is (a subclass - of) DeepseekV32IndexerBackend. When present, it runs - `_dummy_run(..., force_attention=True, cudagraph_runtime_mode=NONE)` at TWO - prefill-size regimes -- a small one (<=1024 rows) and a large one - (max_num_batched_tokens, >1024) -- so BOTH Triton specializations compile. - `force_attention=True` makes _dummy_run build a real - DeepseekV32IndexerMetadata via the normal _build_attention_metadata path - (num_prefills>0 because the default dummy batch is multi-token requests and - the indexer decode_threshold is 1), which drives the real - `sparse_attn_indexer` prefill kernels. The whole thing is wrapped in - try/except that only WARNs -- a warmup failure must never crash boot. - 2. gpu_worker.py: call it from compile_or_warm_up_model, right after the - existing warmup loop and before kernel_warmup(). At that point the KV cache - is already allocated (initialize_from_config runs before - compile_or_warm_up_model), which the forced-attention indexer path needs. - -Idempotent + anchor-based (matches the other apply_glm_* patchers): - * Each hunk self-detects if already applied (marker string present) -> no-op. - * Missing anchor -> WARN and skip that hunk (safe across vllm revisions; the - rebase may already warm the indexer natively or have refactored the site). - * Anchor found but the file does not contain the applied marker and the - replace produces no change -> hard error (would silently keep the bug). - * py_compile at the end; hard error if the patched file won't compile. - -Usage: apply_glm_dsa_indexer_warmup_fix.py -""" -import os -import sys - -RUNNER_REL = "v1/worker/gpu_model_runner.py" -WORKER_REL = "v1/worker/gpu_worker.py" - -MARKER = "glm-dsa-indexer-warmup" - -# --- Hunk A: new method inserted immediately before `def capture_model` ------- -# Anchor: the (unique) capture_model definition head in gpu_model_runner.py. -RUNNER_ANCHOR = " def capture_model(self) -> int:\n" - -RUNNER_METHOD = ''' def _maybe_warmup_dsa_indexer(self) -> None: - """Force-compile the DSA sparse-attention indexer Triton kernels at boot. - - NO-OP unless this model actually has a DeepseekV32IndexerBackend (GLM-5.1 - DSA / DeepSeek V3.2). The indexer kernels are seq-length specialized - (triton_fp8_mqa_logits flips matrix_instr_nonkdim at seq_len<=1024 and - launches grid=[(seq_len,)]), and the normal profile/warmup passes never - drive the indexer (attn_metadata is None -> the *_fake path). Without this - the first large prompt JIT-compiles mid-inference and, under DP lockstep, - stalls the whole group. We warm BOTH regimes: a small (<=1024) and a large - (max_num_batched_tokens, >1024) prefill batch, using force_attention=True - so _dummy_run builds a real indexer metadata via the standard path. - """ - # {marker} - try: - from vllm.v1.attention.backends.mla.indexer import ( - DeepseekV32IndexerBackend, - ) - except Exception: # noqa: BLE001 -- backend module absent -> not a DSA build - return - - has_indexer = False - try: - for attn_group in self._attn_group_iterator(): - backend = getattr(attn_group, "backend", None) - if backend is not None and isinstance(backend, type) and issubclass( - backend, DeepseekV32IndexerBackend - ): - has_indexer = True - break - except Exception: # noqa: BLE001 -- iterator shape changed -> stay a no-op - return - if not has_indexer: - return - - # Two prefill-size regimes so both Triton specializations compile. - # Small must be <=1024 rows; large must exceed 1024 (use the real max). - max_tokens = int(self.max_num_tokens) - small = min(512, max_tokens) - sizes = [] - for s in (small, max_tokens): - if s > 0 and s not in sizes: - sizes.append(s) - - logger.info( - "Warming up DSA indexer kernels at prefill sizes %s " - "to avoid mid-inference JIT.", - sizes, - ) - for size in sizes: - try: - self._dummy_run( - size, - cudagraph_runtime_mode=CUDAGraphMode.NONE, - force_attention=True, - skip_eplb=True, - remove_lora=False, - ) - except Exception as e: # noqa: BLE001 -- warmup must NEVER crash boot - logger.warning( - "DSA indexer warmup at size %d failed (%s); the kernel may " - "JIT-compile on first use instead.", - size, - e, - ) - self._sync_device() - -'''.replace("{marker}", MARKER) - -# --- Hunk B: call site in gpu_worker.compile_or_warm_up_model ----------------- -WORKER_ANCHOR = ( - " self.model_runner.maybe_remove_all_loras(" - "self.model_runner.lora_config)\n" - "\n" - " # Warmup and tune the kernels used during model execution before\n" - " # cuda graph capture.\n" - " kernel_warmup(self)\n" -) - -WORKER_REPLACEMENT = ( - " self.model_runner.maybe_remove_all_loras(" - "self.model_runner.lora_config)\n" - "\n" - " # " + MARKER + ": force-compile the DSA sparse-attention indexer\n" - " # Triton kernels now (KV cache is allocated), so a large prompt\n" - " # never JIT-compiles them mid-inference and stalls DP lockstep.\n" - " # No-op unless this model has a DeepseekV32IndexerBackend.\n" - " if hasattr(self.model_runner, \"_maybe_warmup_dsa_indexer\"):\n" - " self.model_runner._maybe_warmup_dsa_indexer()\n" - "\n" - " # Warmup and tune the kernels used during model execution before\n" - " # cuda graph capture.\n" - " kernel_warmup(self)\n" -) - - -def _patch_file(path, tag, anchor, apply_fn, already_marker): - """Return 0 on success/no-op, 1 on hard error.""" - if not os.path.isfile(path): - print(f"[{tag}] {path} not found -- skipping (layout differs).") - return 0 - src = open(path).read() - if already_marker in src: - print(f"[{tag}] already applied ({already_marker} present) in {path} -- no-op.") - return 0 - if anchor not in src: - print( - f"[{tag}] WARN: anchor not found in {path} -- skipping " - "(assuming native warmup / refactor)." - ) - return 0 - new_src = apply_fn(src) - if new_src == src: - print( - f"[{tag}] ERROR: anchor found but patch produced no change in {path}.", - file=sys.stderr, - ) - return 1 - try: - open(path, "w").write(new_src) - except OSError as e: - print(f"[{tag}] ERROR: failed to write patched {path}: {e}", file=sys.stderr) - return 1 - if already_marker not in open(path).read(): - print( - f"[{tag}] ERROR: post-write verification failed in {path}.", - file=sys.stderr, - ) - return 1 - print(f"[{tag}] patched {path} -- 1 hunk.") - return 0 - - -def main() -> int: - if len(sys.argv) != 2: - print(f"usage: {sys.argv[0]} ", file=sys.stderr) - return 2 - vllm_dir = sys.argv[1] - - runner_path = os.path.join(vllm_dir, RUNNER_REL) - worker_path = os.path.join(vllm_dir, WORKER_REL) - - rc = 0 - - # Hunk A: insert the method before capture_model. - rc |= _patch_file( - runner_path, - "glm-dsa-warmup", - RUNNER_ANCHOR, - lambda s: s.replace(RUNNER_ANCHOR, RUNNER_METHOD + RUNNER_ANCHOR, 1), - MARKER, - ) - - # Hunk B: call it from compile_or_warm_up_model. - rc |= _patch_file( - worker_path, - "glm-dsa-warmup", - WORKER_ANCHOR, - lambda s: s.replace(WORKER_ANCHOR, WORKER_REPLACEMENT, 1), - MARKER, - ) - - if rc: - return 1 - - # py-compile sanity for whichever files exist. - try: - import py_compile - - for p in (runner_path, worker_path): - if os.path.isfile(p): - py_compile.compile(p, doraise=True) - print("[glm-dsa-warmup] py_compile OK") - except Exception as e: # noqa: BLE001 - print( - f"[glm-dsa-warmup] ERROR: patched file fails to compile: {e}", - file=sys.stderr, - ) - return 1 - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/vllm_dissag/apply_glm_dsa_kernel_fix.py b/scripts/vllm_dissag/apply_glm_dsa_kernel_fix.py deleted file mode 100755 index 33f50dc4..00000000 --- a/scripts/vllm_dissag/apply_glm_dsa_kernel_fix.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python3 -"""Apply the GLM-5.1 DSA sparse-attention invalid-token kernel fix (vllm #45324). - -The DSA indexer kernel `_convert_req_index_to_global_index_kernel` in - vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py -maps invalid token slots to 0 instead of -1. With block-size 1 + DSA sparse MLA -that corrupts KV reads and the model emits `!!!` for every prompt. - -Upstream fix: vllm-project/vllm #45324 -- flip the 0 to -1 in the tl.where call: - is_invalid_tok | (~valid_block), 0, base * BLOCK_SIZE + inblock_off - is_invalid_tok | (~valid_block), -1, base * BLOCK_SIZE + inblock_off - -Design (matches launcher contract -- runs unconditionally for GLM, aborts on real -failure): - * IDEMPOTENT : if already -1, report and exit 0 (no-op). - * SELF-SKIPPING: if the file/anchor is absent (refactored or the rebase already - fixed it differently), report and exit 0 -- do NOT abort, because b10a9f7a may - carry the fix natively. We only fail on the one unambiguous bad state we can - fix and didn't, or on write failure. - * VERIFIES the post-write state. - -Usage: apply_glm_dsa_kernel_fix.py -""" -import os -import re -import sys - -REL = "v1/attention/backends/mla/rocm_aiter_mla_sparse.py" - -# Anchor is the stable right-hand side of the tl.where; the middle operand is the -# 0 (buggy) / -1 (fixed) we toggle. Whitespace-tolerant. -RE_ANY = re.compile( - r"(is_invalid_tok\s*\|\s*\(~valid_block\)\s*,\s*)(-?\d+)(\s*,\s*base\s*\*\s*BLOCK_SIZE\s*\+\s*inblock_off)" -) - - -def main() -> int: - if len(sys.argv) != 2: - print(f"usage: {sys.argv[0]} ", file=sys.stderr) - return 2 - vllm_dir = sys.argv[1] - path = os.path.join(vllm_dir, REL) - - if not os.path.isfile(path): - # File not present on this build -> nothing we can or should do. The - # rebase may use a different sparse backend layout. Do not block launch. - print(f"[glm-dsa] {REL} not found under {vllm_dir} -- skipping (assuming native/refactored).") - return 0 - - src = open(path).read() - m = RE_ANY.search(src) - if not m: - # Anchor gone (refactored / already fixed differently). Don't block. - print(f"[glm-dsa] invalid-token kernel anchor not found in {path} -- skipping (assuming native fix).") - return 0 - - cur = m.group(2) - if cur == "-1": - print(f"[glm-dsa] already fixed (kernel returns -1) in {path} -- no-op.") - return 0 - if cur != "0": - # Unexpected value -- surface it but don't guess. Treat as needs-attention. - print(f"[glm-dsa] ERROR: unexpected invalid-token return value '{cur}' (expected 0 or -1) in {path}.", - file=sys.stderr) - return 1 - - # cur == "0" : the known bug. Flip to -1. - new_src = src[:m.start(2)] + "-1" + src[m.end(2):] - try: - open(path, "w").write(new_src) - except OSError as e: - print(f"[glm-dsa] ERROR: failed to write patched {path}: {e}", file=sys.stderr) - return 1 - - # Verify. - chk = RE_ANY.search(open(path).read()) - if not chk or chk.group(2) != "-1": - print(f"[glm-dsa] ERROR: post-write verification failed in {path}.", file=sys.stderr) - return 1 - print(f"[glm-dsa] patched: invalid-token kernel now returns -1 (vllm #45324) in {path}.") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/vllm_dissag/apply_glm_dsa_moriio_dualkv_fix.py b/scripts/vllm_dissag/apply_glm_dsa_moriio_dualkv_fix.py deleted file mode 100755 index ac6800bf..00000000 --- a/scripts/vllm_dissag/apply_glm_dsa_moriio_dualkv_fix.py +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env python3 -"""Patch the MoRIIO KV connector to handle GLM-5.1 DSA's DUAL KV cache. - -PROBLEM (root cause of the 2P2D "Reaped deferred sends / no finished_sending" stall): - GLM-5.1 (GlmMoeDsaForCausalLM -> deepseek_v2.py) has TWO KV caches per layer: - - main MLA latent KV : MLAAttentionSpec, head_size = kv_lora_rank+rope (~576) - - DSA indexer KV : DeepseekV32IndexerCache, MLAAttentionSpec head_size = index_head_dim (128) - Both are 3D ("use_mla"), but DIFFERENT latent dim -> DIFFERENT per-block byte size. - The MoRIIO connector computes ONE global geometry from `first_kv_cache` and reuses it - for every cache, so the indexer cache is transferred with the main-MLA block size -> - wrong bytes/size -> the RDMA read for that region never reconciles -> completion notify - is never produced -> decode reaps deferred sends after 60s -> request hangs. - -FIX (surgical, per-layer geometry; no behavior change for single-cache MLA/DeepSeek): - 1. register_kv_caches: size each registered region by its OWN tensor (per-cache - region_len), not the global self.block_len. Also fix the local_kv_cache_size - append to use the current cache, not a stale loop var. - 2. _compute_block_transfer_offsets: derive shape from the PER-LAYER tensor - (self.kv_caches[layer_name].shape) instead of the global self.kv_cache_shape, - so transfer_size_byte / strides match that cache. - 3. _read_blocks: compute offsets PER LAYER inside the loop (was computed once from - first_layer and reused for all layers). - -Idempotent + anchor-based: each hunk checks if already applied / anchor present; -missing anchor -> warn-and-skip (so it is safe across connector revisions). A hunk -that finds its OLD anchor but fails to apply is a hard error (would silently keep the bug). - -Usage: apply_glm_dsa_moriio_dualkv_fix.py -""" -import os -import sys - -REL = "distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py" - - -def main() -> int: - if len(sys.argv) != 2: - print(f"usage: {sys.argv[0]} ", file=sys.stderr) - return 2 - path = os.path.join(sys.argv[1], REL) - if not os.path.isfile(path): - print(f"[glm-dualkv] {REL} not found -- skipping (connector layout differs).") - return 0 - - src = open(path).read() - orig = src - applied = [] - - # --- Hunk 1: per-cache region_len in register_kv_caches --------------------- - h1_old = """ for cache_or_caches in kv_caches.values(): - cache_list = [cache_or_caches] if use_mla else cache_or_caches - for cache in cache_list: - base_addr = cache.data_ptr() - region_len = self.num_blocks * self.block_len - caches_data.append((base_addr, region_len, cache.device.index, "")) - kv_caches_base_addr.append(base_addr)""" - h1_new = """ for cache_or_caches in kv_caches.values(): - cache_list = [cache_or_caches] if use_mla else cache_or_caches - for cache in cache_list: - base_addr = cache.data_ptr() - # DSA dual-KV fix: size each region by its OWN tensor, not the - # global self.block_len (the DSA indexer cache has a different - # latent dim than the main MLA cache). - region_len = cache.nelement() * cache.element_size() - caches_data.append((base_addr, region_len, cache.device.index, "")) - kv_caches_base_addr.append(base_addr)""" - if "region_len = cache.nelement() * cache.element_size()" in src: - applied.append("h1 (already)") - elif h1_old in src: - src = src.replace(h1_old, h1_new, 1) - applied.append("h1") - else: - print("[glm-dualkv] WARN: h1 anchor (region_len loop) not found -- skipping h1.") - - # --- Hunk 1b: local_kv_cache_size uses current kv_cache, not stale `cache` -- - h1b_old = " self.local_kv_cache_size.append(cache.nelement() * cache.element_size())" - h1b_new = " self.local_kv_cache_size.append(kv_cache.nelement() * kv_cache.element_size())" - if h1b_new in src: - applied.append("h1b (already)") - elif h1b_old in src: - src = src.replace(h1b_old, h1b_new, 1) - applied.append("h1b") - else: - print("[glm-dualkv] WARN: h1b anchor (local_kv_cache_size) not found -- skipping h1b.") - - # --- Hunk 2: per-layer shape in _compute_block_transfer_offsets ------------- - h2_old = """ assert self.kv_cache_shape is not None, "KV caches shape not initialized" - is_mla = len(self.kv_cache_shape) == 3 - stride = self.kv_caches[layer_name].stride() - sz = self.kv_caches[layer_name].element_size() - if is_mla: - blknum, blksize, hs = self.kv_cache_shape - hn = 1 - block_stride = stride[0] - else: - _, blknum, blksize, hn, hs = self.kv_cache_shape""" - h2_new = """ # DSA dual-KV fix: use the PER-LAYER tensor shape, not the global - # self.kv_cache_shape (the DSA indexer cache differs from the main MLA). - _layer_shape = tuple(self.kv_caches[layer_name].shape) - assert len(_layer_shape) > 0, "KV caches shape not initialized" - is_mla = len(_layer_shape) == 3 - stride = self.kv_caches[layer_name].stride() - sz = self.kv_caches[layer_name].element_size() - if is_mla: - blknum, blksize, hs = _layer_shape - hn = 1 - block_stride = stride[0] - else: - _, blknum, blksize, hn, hs = _layer_shape""" - if "_layer_shape = tuple(self.kv_caches[layer_name].shape)" in src: - applied.append("h2 (already)") - elif h2_old in src: - src = src.replace(h2_old, h2_new, 1) - applied.append("h2") - else: - print("[glm-dualkv] WARN: h2 anchor (_compute_block_transfer_offsets head) not found -- skipping h2.") - - # --- Hunk 3: per-layer offsets in _read_blocks ----------------------------- - h3_old = """ first_layer = list(self.layer_name_to_local_kv_cache_metadata.keys())[0] - offs = self._compute_block_transfer_offsets( - first_layer, local_block_ids, remote_block_ids, remote_moriio_meta - ) - - for layer_name in self.layer_name_to_local_kv_cache_metadata: - sess_idx = list(self.layer_name_to_local_kv_cache_metadata.keys()).index( - layer_name - ) - # TODO : apply multi-session batch-read when moriio support it - transfer_status = self.moriio_wrapper.read_remote_data( - offs[2], offs[0], offs[1], sessions[sess_idx] - )""" - h3_new = """ # DSA dual-KV fix: compute offsets PER LAYER (the DSA indexer cache has a - # different per-block size than the main MLA cache, so a single offs reused - # across all layers mis-sizes the indexer transfer -> lost completion notify). - for layer_name in self.layer_name_to_local_kv_cache_metadata: - sess_idx = list(self.layer_name_to_local_kv_cache_metadata.keys()).index( - layer_name - ) - offs = self._compute_block_transfer_offsets( - layer_name, local_block_ids, remote_block_ids, remote_moriio_meta - ) - # TODO : apply multi-session batch-read when moriio support it - transfer_status = self.moriio_wrapper.read_remote_data( - offs[2], offs[0], offs[1], sessions[sess_idx] - )""" - if "compute offsets PER LAYER" in src: - applied.append("h3 (already)") - elif h3_old in src: - src = src.replace(h3_old, h3_new, 1) - applied.append("h3") - else: - print("[glm-dualkv] WARN: h3 anchor (_read_blocks first_layer offsets) not found -- skipping h3.") - - if src != orig: - try: - open(path, "w").write(src) - except OSError as e: - print(f"[glm-dualkv] ERROR: write failed for {path}: {e}", file=sys.stderr) - return 1 - print(f"[glm-dualkv] patched {path} -- hunks: {', '.join(applied)}") - else: - print(f"[glm-dualkv] no changes ({', '.join(applied) or 'nothing applied'}) for {path}") - - # py-compile sanity - try: - import py_compile - py_compile.compile(path, doraise=True) - print("[glm-dualkv] py_compile OK") - except Exception as e: # noqa: BLE001 - print(f"[glm-dualkv] ERROR: patched file fails to compile: {e}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/vllm_dissag/apply_glm_dsa_moriio_engine_fix.py b/scripts/vllm_dissag/apply_glm_dsa_moriio_engine_fix.py deleted file mode 100755 index 9d6d468b..00000000 --- a/scripts/vllm_dissag/apply_glm_dsa_moriio_engine_fix.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env python3 -"""Fix MoRIIO WRITE-path per-layer offset caching for GLM-5.1 DSA dual KV cache. - -ROOT CAUSE (proven by instrumentation, job 37594): - GLM-5.1 (GlmMoeDsaForCausalLM) registers num_layers=156 KV caches = 78 main MLA - (per-block latent dim 576) + 78 DSA indexer caches (latent dim 132). TWO geometries. - - moriio_engine.py::MoRIIOEngine._prepare_transfer_plan computes the RDMA transfer - offsets ONCE (for whatever layer arrives first) and caches them on - request_info.transfer_offset, then REUSES that single offset/size tuple for ALL 156 - layers. The 78 indexer layers (dim 132) get written with the main-MLA geometry - (dim 576) -> wrong byte size/offset -> those RDMA writes are malformed; the per-layer - write accounting (writes_done) and/or the remote completion never reconciles -> - the producer's send_notify (gated on writes_done >= num_layers) misbehaves and the - decode side never receives a clean completion -> "Reaped deferred sends / no - finished_sending after 60s" -> request hangs. - -FIX (surgical, no dataclass change): - Cache transfer offsets PER LAYER on the request_info via a dynamically-attached dict - ``_transfer_offset_by_layer`` keyed by layer_name, instead of the single - ``transfer_offset`` slot. Each of the 156 layers then transfers with its OWN geometry - (the underlying _compute_block_transfer_offsets already takes layer_name and, with the - companion dualkv patch h2, reads the per-layer tensor shape). - - Single-geometry models (DeepSeek-V3 / Hunyuan, 1 cache/layer) are unaffected: - every layer has identical geometry, so per-layer caching yields the same offsets. - -Idempotent + anchor-based. A missing anchor warns-and-skips; a found OLD anchor that -fails to apply is a hard error (would silently keep the stall). - -Usage: apply_glm_dsa_moriio_engine_fix.py -""" -import os -import sys - -REL = "distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py" - - -def main() -> int: - if len(sys.argv) != 2: - print(f"usage: {sys.argv[0]} ", file=sys.stderr) - return 2 - path = os.path.join(sys.argv[1], REL) - if not os.path.isfile(path): - print(f"[glm-engine] {REL} not found -- skipping (engine layout differs).") - return 0 - - src = open(path).read() - - old = """ # Compute offsets if not cached - if request_info.transfer_offset is None: - offsets = self.worker._compute_block_transfer_offsets( - task.layer_name, - task.local_block_ids, - request_info.block_ids, - remote_moriio_meta, - ) - request_info.transfer_offset = offsets - - # Get session index - layer_names = list(self.worker.layer_name_to_local_kv_cache_metadata.keys()) - sess_idx = layer_names.index(task.layer_name) - - local_off, remote_off, sizes = request_info.transfer_offset""" - - new = """ # DSA dual-KV fix: cache offsets PER LAYER, not once per request. GLM-5.1 has - # two cache geometries (main MLA dim 576 + DSA indexer dim 132); a single - # cached offset reused across all 156 layers mis-sizes the indexer writes and - # the completion never reconciles. Per-layer caching is identical for - # single-geometry models (DeepSeek/Hunyuan). - _off_by_layer = getattr(request_info, "_transfer_offset_by_layer", None) - if _off_by_layer is None: - _off_by_layer = {} - request_info._transfer_offset_by_layer = _off_by_layer - offsets = _off_by_layer.get(task.layer_name) - if offsets is None: - offsets = self.worker._compute_block_transfer_offsets( - task.layer_name, - task.local_block_ids, - request_info.block_ids, - remote_moriio_meta, - ) - _off_by_layer[task.layer_name] = offsets - # keep the legacy single-slot populated (first layer) for any external reader - if request_info.transfer_offset is None: - request_info.transfer_offset = offsets - - # Get session index - layer_names = list(self.worker.layer_name_to_local_kv_cache_metadata.keys()) - sess_idx = layer_names.index(task.layer_name) - - local_off, remote_off, sizes = offsets""" - - if "_transfer_offset_by_layer" in src: - print(f"[glm-engine] already patched (_transfer_offset_by_layer present) -- no-op.") - elif old in src: - src = src.replace(old, new, 1) - open(path, "w").write(src) - print(f"[glm-engine] patched per-layer offset caching in {path}") - else: - print(f"[glm-engine] WARN: anchor (_prepare_transfer_plan offset block) not found -- skipping (engine revision differs).") - # Not fatal: without the anchor we can't safely patch; surface clearly. - return 0 - - try: - import py_compile - py_compile.compile(path, doraise=True) - print("[glm-engine] py_compile OK") - except Exception as e: # noqa: BLE001 - print(f"[glm-engine] ERROR: compile failed: {e}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/vllm_dissag/apply_glm_dsa_moriio_gate_fix.py b/scripts/vllm_dissag/apply_glm_dsa_moriio_gate_fix.py deleted file mode 100755 index 92501ae9..00000000 --- a/scripts/vllm_dissag/apply_glm_dsa_moriio_gate_fix.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python3 -"""Fix the MoRIIO transfer-completion gate for GLM-5.1 DSA dual KV cache. - -ROOT CAUSE (PROVEN, job 37615 instrumentation): - GLM-5.1 registers num_layers=156 KV caches = 78 main MLA (model.layers.N.self_attn.attn) - + 78 DSA indexer caches (model.layers.N.self_attn.indexer.k_cache). BUT only the 78 - main-MLA layers ever go through the KV-connector save_kv_layer hook -> only they write - -> writes_done caps at 78. The indexer caches are registered (counted in num_layers) - but vLLM NEVER calls save_kv_layer for them (the DSA indexer is a separate attention - component / DeepseekV32IndexerBackend that doesn't use the connector save path; decode - recomputes indexer state from the transferred main latent KV). - - The producer completion gate (moriio_engine.py): - request_info.writes_done += 1 - if request_info.writes_done >= self.worker.num_layers: # 156, never reached - send_notify(...) - caps at writes_done=78 < num_layers=156 -> send_notify NEVER fires -> decode never - gets completion -> "Reaped deferred sends / no finished_sending after 60s" -> stall. - -FIX: - Add self.num_transfer_layers = count of caches that actually transfer (exclude - '.indexer.' caches), with a fallback to num_layers (so single-geometry models - - DeepSeek-V3 / Hunyuan, no indexer - are bit-identical). Gate completion on - num_transfer_layers instead of num_layers. num_layers itself is left unchanged - (it is also used by the Llama-4 per-layer block-window loop, which needs all caches). - -Companion to apply_glm_dsa_moriio_engine_fix.py (per-layer offset caching). This gate -fix is the primary unblocker; the offset fix is correctness insurance for the layers -that DO write (all same geometry here, but harmless). - -Idempotent + anchor-based. Patches BOTH files (connector: define the field; engine: -use it). A found-old-anchor that fails is a hard error. - -Usage: apply_glm_dsa_moriio_gate_fix.py -""" -import os -import sys - -CONN_REL = "distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py" -ENG_REL = "distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py" - - -def patch_connector(path: str) -> int: - src = open(path).read() - old = " self.num_layers = len(self.kv_caches.keys())" - new = """ self.num_layers = len(self.kv_caches.keys()) - # DSA dual-KV fix: the producer completion gate must count only layers that - # actually transfer via save_kv_layer. GLM-5.1 registers 2 caches/layer (main - # MLA + DSA indexer), but only the main-MLA caches go through save_kv_layer; the - # '.indexer.' caches are registered yet never written. Gating on len(kv_caches) - # would never be reached. Exclude indexer caches; fall back to num_layers for - # single-geometry models (DeepSeek/Hunyuan have no indexer -> identical). - self.num_transfer_layers = ( - len([k for k in self.kv_caches.keys() if ".indexer." not in k]) - or self.num_layers - ) - logger.info( - "[moriio] completion gate: num_transfer_layers=%d (num_layers=%d)", - self.num_transfer_layers, self.num_layers, - )""" - if "self.num_transfer_layers" in src: - print(f"[glm-gate] connector already patched -- no-op.") - return 0 - if old not in src: - print(f"[glm-gate] WARN: connector anchor (num_layers=) not found -- skipping.") - return 0 - src = src.replace(old, new, 1) - open(path, "w").write(src) - print(f"[glm-gate] patched connector: defined num_transfer_layers in {path}") - return 0 - - -def patch_engine(path: str) -> int: - src = open(path).read() - old = " if request_info.writes_done >= self.worker.num_layers:" - new = """ if request_info.writes_done >= getattr( - self.worker, "num_transfer_layers", self.worker.num_layers - ):""" - if 'getattr(\n self.worker, "num_transfer_layers"' in src or "num_transfer_layers" in src: - print(f"[glm-gate] engine already patched -- no-op.") - return 0 - if old not in src: - print(f"[glm-gate] WARN: engine anchor (writes_done gate) not found -- skipping.") - return 0 - src = src.replace(old, new, 1) - open(path, "w").write(src) - print(f"[glm-gate] patched engine: gate on num_transfer_layers in {path}") - return 0 - - -def main() -> int: - if len(sys.argv) != 2: - print(f"usage: {sys.argv[0]} ", file=sys.stderr) - return 2 - base = sys.argv[1] - conn = os.path.join(base, CONN_REL) - eng = os.path.join(base, ENG_REL) - if not os.path.isfile(conn) or not os.path.isfile(eng): - print("[glm-gate] connector/engine not found -- skipping (layout differs).") - return 0 - - # ATOMIC: both halves (connector defines num_transfer_layers, engine gates on - # it) are needed together or not at all. On a restructured image (e.g. mori - # v1.2.1, whose engine replaced the writes_done>=num_layers gate with a sealed - # writes_expected mechanism that already handles hybrid/DSA dual-KV natively), - # the engine anchor is gone. Applying only the connector half would inject a - # dead num_transfer_layers into restructured internals. So if the engine anchor - # is absent, skip BOTH — the native gate already does the right thing. - eng_src = open(eng).read() - eng_gate_present = " if request_info.writes_done >= self.worker.num_layers:" in eng_src - eng_already = "num_transfer_layers" in eng_src - if not eng_gate_present and not eng_already: - print("[glm-gate] engine gate anchor absent (image restructured, e.g. mori " - "v1.2.1 sealed writes_expected) -- skipping BOTH halves (native gate handles DSA).") - return 0 - - rc = patch_connector(conn) or patch_engine(eng) - if rc: - return rc - - try: - import py_compile - py_compile.compile(conn, doraise=True) - py_compile.compile(eng, doraise=True) - print("[glm-gate] py_compile OK (both files)") - except Exception as e: # noqa: BLE001 - print(f"[glm-gate] ERROR: compile failed: {e}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/vllm_dissag/apply_glm_dsa_moriio_instrument.py b/scripts/vllm_dissag/apply_glm_dsa_moriio_instrument.py deleted file mode 100755 index 68e03beb..00000000 --- a/scripts/vllm_dissag/apply_glm_dsa_moriio_instrument.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python3 -"""TEMPORARY instrumentation: does the DSA indexer cache reach the MoRIIO save path? - -Adds logging at two points in moriio_connector.py to settle the dual-KV RCA: - 1. register_kv_caches: log all kv_caches layer names + their shapes + num_layers. - -> shows whether the DeepseekV32IndexerCache is even registered, and its geometry. - 2. _write_blocks_for_req: log each distinct layer_name that actually triggers a write. - -> compare the COUNT/SET of written layers vs num_layers. If indexer layers are in - kv_caches (counted in num_layers) but never written, writes_done can never reach - num_layers -> send_notify never fires -> the stall. - -This is diagnostic only (no behavior change). Remove before any production use. -Idempotent + anchor-safe. - -Usage: apply_glm_dsa_moriio_instrument.py -""" -import os -import sys - -REL = "distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py" - - -def main() -> int: - if len(sys.argv) != 2: - print(f"usage: {sys.argv[0]} ", file=sys.stderr) - return 2 - path = os.path.join(sys.argv[1], REL) - if not os.path.isfile(path): - print(f"[glm-instr] {REL} not found -- skipping.") - return 0 - - src = open(path).read() - orig = src - - # --- Point 1: log kv_caches inventory at num_layers assignment ------------- - a1 = " self.num_layers = len(self.kv_caches.keys())" - b1 = """ self.num_layers = len(self.kv_caches.keys()) - # [glm-instr] kv-cache inventory (dual-KV diagnosis) - try: - for _ln, _kv in self.kv_caches.items(): - logger.info("[glm-instr][register] layer=%s shape=%s dtype=%s", - _ln, tuple(_kv.shape), _kv.dtype) - logger.info("[glm-instr][register] num_layers=%d total_kv_caches=%d", - self.num_layers, len(self.kv_caches)) - except Exception as _e: # noqa: BLE001 - logger.info("[glm-instr][register] inventory log failed: %s", _e)""" - if "[glm-instr][register]" in src: - pass - elif a1 in src: - src = src.replace(a1, b1, 1) - else: - print("[glm-instr] WARN: register anchor (num_layers=) not found.") - - # --- Point 2: log each written layer in _write_blocks_for_req ------------- - a2 = " def _write_blocks_for_req(self, req_id: ReqId, meta: ReqMeta, layer_name, kv_layer):" - b2 = (a2 + "\n" - ' # [glm-instr] record which layers actually trigger a KV write\n' - ' try:\n' - ' _seen = getattr(self, "_glm_instr_written_layers", None)\n' - ' if _seen is None:\n' - ' _seen = set(); self._glm_instr_written_layers = _seen\n' - ' if layer_name not in _seen:\n' - ' _seen.add(layer_name)\n' - ' logger.info("[glm-instr][write] NEW layer=%s total_written=%d/%d",\n' - ' layer_name, len(_seen), getattr(self, "num_layers", -1))\n' - ' except Exception as _e: # noqa: BLE001\n' - ' logger.info("[glm-instr][write] log failed: %s", _e)') - if "[glm-instr][write]" in src: - pass - elif a2 in src: - src = src.replace(a2, b2, 1) - else: - print("[glm-instr] WARN: _write_blocks_for_req anchor not found.") - - if src != orig: - open(path, "w").write(src) - print(f"[glm-instr] instrumented {path}") - else: - print(f"[glm-instr] already instrumented / nothing to do for {path}") - - try: - import py_compile - py_compile.compile(path, doraise=True) - print("[glm-instr] py_compile OK") - except Exception as e: # noqa: BLE001 - print(f"[glm-instr] ERROR: compile failed: {e}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/vllm_dissag/apply_glm_dsa_persistent_kernel_gate_fix.py b/scripts/vllm_dissag/apply_glm_dsa_persistent_kernel_gate_fix.py deleted file mode 100644 index adda36cf..00000000 --- a/scripts/vllm_dissag/apply_glm_dsa_persistent_kernel_gate_fix.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python3 -"""Gate OFF the AITER persistent sparse-MLA kernel for chunked-prefill batches. - -ROOT CAUSE (ROCm/aiter #4076, vLLM #47042 / #47567): - The AITER persistent MLA work-stealing kernel (mla_a8w8_qh16_qseqlen1_gqaratio16_ps, - taken when work_meta_data from get_mla_metadata_v1 is non-None) is NUMERICALLY WRONG - for multi-token (prefill-shaped) batches of qseqlen==1 entries. Pure decode (1 query - token) and fresh single-chunk prefills are correct; the error only appears once a - request becomes a CHUNKED-PREFILL CONTINUATION. The small per-token error COMPOUNDS - through the KV cache across chunked-prefill passes until long-context decode collapses - into repetition/garbage. Failure is gated on CHUNK COUNT, not raw context length - (verified: 22k in 2 chunks = correct, 22k in 3 chunks = garbage). - - On this image (aiter 0.1.16.post3, before the aiter-side kernel fix #3921) GLM-5.1-FP8 - DSA collapses at ~16-18k prompt tokens. The aiter kernel fix is the long-term answer - (AITERKER-132 / aiter #3921); this is the vLLM-side short-term gate (#47567), which - costs ~no perf (decode + single-chunk prefill keep the persistent path). - -FIX (port of vLLM PR #47567, adapted to this image's rocm_aiter_mla_sparse.py::build): - In ROCMAiterMLASparseMetadataBuilder.build(), detect chunked-prefill continuations - (a request with >1 query token this step whose total seq_len exceeds its query_len, - i.e. part of its context was computed in an earlier chunk) and, when ANY request in - the batch is such a continuation: - * skip the get_mla_metadata_v1 persistent-metadata launch, and - * pass work_meta_data=None to the metadata so mla_decode_fwd takes the CORRECT - non-persistent split-KV path. - Decode-only and single-chunk-prefill batches are unchanged (persistent path kept). - - Uses `seg_lengths` (per-request step query lengths, already computed at build() top) - and `common_attn_metadata.seq_lens_cpu[:num_reqs].numpy()` (total seq lens). Both are - present in this image's build(). - -Idempotent + anchor-based + self-skipping. Missing anchor -> warn+skip (safe across -image revisions / if a newer image already carries the aiter kernel fix). A found-old -anchor that fails to apply is a hard error (would silently keep the corruption). - -Usage: apply_glm_dsa_persistent_kernel_gate_fix.py -""" -import os -import sys - -REL = "v1/attention/backends/mla/rocm_aiter_mla_sparse.py" - -# Anchor 1: the persistent-metadata guard. We insert the continuation detection -# just before it and AND it into the condition. -OLD1 = """ if metadata_key != self._prev_metadata_key: - from aiter import get_mla_metadata_v1""" -NEW1 = """ # PERSISTENT-KERNEL GATE (aiter #4076 / vLLM #47567): the persistent - # sparse-MLA work-stealing kernel is numerically wrong for chunked-prefill - # continuation batches; the error compounds and breaks long-context decode. - # Fall back to the correct non-persistent path whenever any request in the - # batch is a chunked-prefill continuation (>1 query token this step AND - # total seq_len > this step's query_len). Decode + single-chunk prefills - # keep the fast persistent path -> no decode-throughput regression. - # Slice to num_reqs and cast to int64 (vLLM #47567 hardening / Rohan138 PR#1) - # so the masks cannot broadcast-mismatch under cudagraph padding. - _step_query_lens = seg_lengths[:num_reqs].astype(np.int64) - _total_seq_lens = common_attn_metadata.seq_lens_cpu[:num_reqs].numpy().astype( - np.int64 - ) - _is_chunked_continuation = (_step_query_lens > 1) & ( - _total_seq_lens > _step_query_lens - ) - _use_persistent = not bool(_is_chunked_continuation.any()) - if _use_persistent and metadata_key != self._prev_metadata_key: - from aiter import get_mla_metadata_v1""" - -# Anchor 2: the metadata construction passes the persistent buffer unconditionally. -# Gate it on _use_persistent. -OLD2 = " work_meta_data=self._mla_work_meta_data," -NEW2 = " work_meta_data=(self._mla_work_meta_data if _use_persistent else None)," - - -def main() -> int: - if len(sys.argv) != 2: - print(f"usage: {sys.argv[0]} ", file=sys.stderr) - return 2 - path = os.path.join(sys.argv[1], REL) - if not os.path.isfile(path): - print(f"[glm-persist] {REL} not found -- skipping (backend layout differs).") - return 0 - - src = open(path).read() - - if "_is_chunked_continuation" in src or "_use_persistent" in src: - print("[glm-persist] already patched (persistent-kernel gate present) -- no-op.") - return 0 - - # Both anchors must be present to apply safely. - if OLD1 not in src: - print("[glm-persist] WARN: persistent-metadata anchor (metadata_key guard) not " - "found -- skipping (image may already carry the aiter kernel fix, or the " - "backend was refactored).") - return 0 - if OLD2 not in src: - print("[glm-persist] ERROR: found the metadata_key guard but NOT the " - "work_meta_data=self._mla_work_meta_data assignment -- refusing partial " - "patch (would leave persistent kernel active). Aborting.", file=sys.stderr) - return 1 - - src = src.replace(OLD1, NEW1, 1) - src = src.replace(OLD2, NEW2, 1) - - try: - open(path, "w").write(src) - except OSError as e: - print(f"[glm-persist] ERROR: write failed for {path}: {e}", file=sys.stderr) - return 1 - - # Verify both edits landed. - chk = open(path).read() - if "_use_persistent = not bool(_is_chunked_continuation.any())" not in chk or \ - "if _use_persistent else None" not in chk: - print("[glm-persist] ERROR: post-write verification failed.", file=sys.stderr) - return 1 - - try: - import py_compile - py_compile.compile(path, doraise=True) - except Exception as e: # noqa: BLE001 - print(f"[glm-persist] ERROR: patched file fails to compile: {e}", file=sys.stderr) - return 1 - - print(f"[glm-persist] patched persistent-kernel gate (aiter #4076 / vLLM #47567) in {path}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/vllm_dissag/apply_glm_moriio_abort_guard_fix.py b/scripts/vllm_dissag/apply_glm_moriio_abort_guard_fix.py deleted file mode 100644 index c09b4acd..00000000 --- a/scripts/vllm_dissag/apply_glm_moriio_abort_guard_fix.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python3 -"""Guard the MoRIIO connector abort path against a None peer_zmq (mori v1.2.1). - -ROOT CAUSE (observed on the router image, job 199144 decode crash): - When a request is ABORTED before its KV-transfer peer handshake completes, - the connector's release path runs: - - moriio_connector.py::_release_write_prefill_blocks - peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=False) # -> None - remote_host, _, remote_notify_port = parse_moriio_zmq_address(peer_zmq) # None.split(",") - -> AttributeError: 'NoneType' object has no attribute 'split' - - This only catches ValueError, not the AttributeError from a None peer_zmq, so - the EngineCore dies -> cascades to all decode workers (EngineDeadError) -> decode - is dead. Triggered by any request aborted before the peer handshake (e.g. a - canary/curl that times out during first-token cold JIT). - - The SAME FILE already guards this correctly at the other call site - (request_finished / _should_notify path): `if peer_zmq is not None:` then parse, - else fall back to params. The release path just missed the guard — an - inconsistent-guard bug in the connector. - -FIX (surgical, matches the file's own existing pattern): - In _release_write_prefill_blocks, when the params don't already carry - remote_host/remote_notify_port, guard the peer_zmq lookup: if it is None, log - and return (same graceful bail the existing `except ValueError` already does for - the "missing remote notify address" case). No behavior change when peer_zmq is - valid; single-geometry / non-aborted requests are unaffected. - -Idempotent + anchor-based + self-skipping (no-ops if the anchor is absent/already -guarded, so it is safe across connector revisions and other images). A found-old -anchor that fails to apply is a hard error (would leave the crash). - -Usage: apply_glm_moriio_abort_guard_fix.py -""" -import os -import sys - -REL = "distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py" - -# The buggy two lines: fetch peer_zmq (may be None) then parse it unguarded. -OLD = """ peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=False) - remote_host, _, remote_notify_port = parse_moriio_zmq_address(peer_zmq)""" - -NEW = """ peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=False) - # Abort-path guard: a request aborted before the KV peer handshake - # has peer_zmq=None; parse_moriio_zmq_address(None) would raise - # AttributeError and kill the EngineCore. Bail gracefully like the - # ValueError case below (matches the guarded call site elsewhere). - if peer_zmq is None: - logger.warning( - "Cannot release WRITE prefill blocks for request %s: " - "no peer zmq address (aborted before peer handshake)", - request_id, - ) - return - remote_host, _, remote_notify_port = parse_moriio_zmq_address(peer_zmq)""" - - -def main() -> int: - if len(sys.argv) != 2: - print(f"usage: {sys.argv[0]} ", file=sys.stderr) - return 2 - path = os.path.join(sys.argv[1], REL) - if not os.path.isfile(path): - print(f"[glm-abort] {REL} not found -- skipping (connector layout differs).") - return 0 - - src = open(path).read() - if "no peer zmq address (aborted before peer handshake)" in src: - print("[glm-abort] already patched -- no-op.") - return 0 - if OLD not in src: - # Anchor absent: either the release path was refactored or this image - # already guards it. Do not block launch. - print("[glm-abort] release-path anchor not found -- skipping (assuming " - "native guard / refactored).") - return 0 - - src = src.replace(OLD, NEW, 1) - try: - open(path, "w").write(src) - except OSError as e: - print(f"[glm-abort] ERROR: write failed for {path}: {e}", file=sys.stderr) - return 1 - - try: - import py_compile - py_compile.compile(path, doraise=True) - except Exception as e: # noqa: BLE001 - print(f"[glm-abort] ERROR: patched file fails to compile: {e}", file=sys.stderr) - return 1 - print(f"[glm-abort] patched _release_write_prefill_blocks None-guard in {path}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/vllm_dissag/connectors/moriio.sh b/scripts/vllm_dissag/connectors/moriio.sh index ffb3ee14..fa4d7599 100644 --- a/scripts/vllm_dissag/connectors/moriio.sh +++ b/scripts/vllm_dissag/connectors/moriio.sh @@ -124,104 +124,26 @@ _moriio_build_kv_transfer_config() { } connector_runtime_patch() { + # No runtime .py patching on the moriio path, for ANY model. + # # MoRIIO multi-node disagg fixes (vLLM PR#39276 notify-path, #41751 LL split, # DP-rank hash-failsafe) are committed in-source in the vLLM the image is built # from (Dockerfile VLLM_REF). There is no generic runtime .py patcher for those — # that would be a drifting duplicate of fixes already upstream in the fork. # - # EXCEPTION — GLM-5.* (GlmMoeDsaForCausalLM, MLA + DSA sparse attention): - # DSA is a NEW attention family the MoRIIO connector was never built for. It adds - # a 2nd KV cache per layer (indexer) with a different geometry, which the - # single-geometry connector mis-handles -> disagg KV transfer stalls. Model-specific - # code gaps, applied here as idempotent, anchor-based, self-skipping .py patchers - # (they no-op cleanly if the fix is native/refactored on the chosen image). Gated - # on MODEL_NAME so DeepSeek/other models are a pure no-op (byte-identical to before). - # The MoRI version is pinned by the Dockerfile MORI_REF (post-1.2.1 main with the - # large-transfer notify/mapping fixes #424/#436/#432 baked in); if a newer MoRI is - # needed, update MORI_REF and rebuild the image — no runtime library swap here. - case "${MODEL_NAME:-}" in - GLM-5.*) ;; - *) return 0 ;; - esac - _glm_dsa_runtime_patch -} - -# GLM-5.* DSA patchers (see connector_runtime_patch). Ported from MAD-private #338. -# Resolves the vLLM install dir, then applies the dual-KV / abort fallbacks in order, -# aborting on a hard failure. Patchers self-skip (rc 0) when their anchor is absent. -# Two patchers are OPT-IN because they crash the v0.27 GLM image (see below). -_glm_dsa_runtime_patch() { - # GLM_SKIP_PATCHERS=1: the serving image already carries the GLM DSA fixes - # in-source (vLLM #47766 cache-key + dual-KV). Skip ALL runtime patchers — they - # are redundant, and the persistent-gate/sentinel patchers would actively REGRESS - # a baked image (asm_mla.cu abort / hipErrorIllegalAddress). - if [ "${GLM_SKIP_PATCHERS:-0}" = "1" ]; then - echo "[glm] GLM_SKIP_PATCHERS=1: image carries DSA fixes in-source; skipping runtime patchers." - return 0 - fi - local _patch_dir="${SCRIPT_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")/.." && pwd)}" - local _vllm_dir - _vllm_dir="$(python3 -c 'import vllm, os; print(os.path.dirname(vllm.__file__))' 2>/dev/null || true)" - if [ -z "${_vllm_dir}" ] || [ ! -d "${_vllm_dir}" ]; then - echo "Error: [glm] cannot locate vLLM install dir for DSA patchers. Aborting." >&2 - exit 1 - fi - echo "[glm] MODEL_NAME=${MODEL_NAME}: applying DSA runtime patchers against ${_vllm_dir}" - - # apply_glm_dsa_persistent_kernel_gate_fix.py ports vLLM #47567. OFF by default: - # #47766 superseded it, and forcing non-persistent MLA on this image aborts at - # asm_mla.cu:945 (fp8/fp8 gqa_ratio=64 has no non-persistent kernel). Set - # GLM_PERSIST_GATE=1 only on an image that predates #47766. - local _gate_patcher="" - [ "${GLM_PERSIST_GATE:-0}" = "1" ] && _gate_patcher="apply_glm_dsa_persistent_kernel_gate_fix.py" - # apply_glm_dsa_kernel_fix.py ports still-open vLLM #45324 (sentinel 0->-1). OFF - # by default: this image ships 0 deliberately because aiter mla_decode_fwd - # dereferences the index (-1 -> hipErrorIllegalAddress on short disagg decode). - # Set GLM_DSA_SENTINEL_FIX=1 only on an older image that genuinely has the #45324 bug. - local _dsa_sentinel_patcher="" - [ "${GLM_DSA_SENTINEL_FIX:-0}" = "1" ] && _dsa_sentinel_patcher="apply_glm_dsa_kernel_fix.py" - local _p - for _p in \ - ${_dsa_sentinel_patcher} \ - apply_glm_dsa_moriio_dualkv_fix.py \ - apply_glm_dsa_moriio_engine_fix.py \ - apply_glm_dsa_moriio_gate_fix.py \ - apply_glm_moriio_abort_guard_fix.py \ - ${_gate_patcher} \ - apply_glm_aiter_sampling_oob_fix.py; do - [ -n "${_p}" ] || continue - local _py="${_patch_dir}/${_p}" - if [ ! -f "${_py}" ]; then - echo "Error: [glm] required patcher ${_py} not found. Aborting." >&2 - exit 1 - fi - echo "[glm] applying ${_p}" - python3 "${_py}" "${_vllm_dir}" 2>&1 || { - echo "Error: [glm] ${_p} failed — ${MODEL_NAME} would emit garbage or stall. Aborting." >&2 - exit 1 - } - done - - # Optional DSA indexer boot-warmup (GLM_INDEXER_WARMUP=1). Force-compiles the DSA - # indexer kernels at boot so they never JIT mid-inference. Opt-in because it drives a - # large (>=8k) prefill forward at boot: on stacks where that forward faults it makes - # the fault DETERMINISTIC at boot (useful for debugging) rather than on first request. - if [ "${GLM_INDEXER_WARMUP:-0}" = "1" ]; then - local _warm="${_patch_dir}/apply_glm_dsa_indexer_warmup_fix.py" - if [ -f "${_warm}" ]; then - echo "[glm] applying DSA indexer boot-warmup (GLM_INDEXER_WARMUP=1)" - python3 "${_warm}" "${_vllm_dir}" 2>&1 || echo "Warning: [glm] indexer-warmup patch failed (non-fatal)." - fi - fi - - # Optional diagnostic instrumentation (GLM_INSTRUMENT=1). Non-fatal. - if [ "${GLM_INSTRUMENT:-0}" = "1" ]; then - local _instr="${_patch_dir}/apply_glm_dsa_moriio_instrument.py" - if [ -f "${_instr}" ]; then - echo "[glm] applying instrumentation (GLM_INSTRUMENT=1): apply_glm_dsa_moriio_instrument.py" - python3 "${_instr}" "${_vllm_dir}" 2>&1 || echo "Warning: [glm] instrumentation failed (non-fatal)." - fi - fi + # The same rule now holds for GLM-5.* (GlmMoeDsaForCausalLM, MLA + DSA sparse + # attention): DSA adds a 2nd KV cache per layer (indexer) with its own geometry, + # and the connector/kernel fixes for it (per-req-ctx metadata key #47766, DSA + # indexer KV transfer, invalid-token sentinel, sparse-indexer bounds guard) are + # carried IN-SOURCE by the image. The required ref is the contract: + # docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile + # VLLM_REF=glm5.1-dsa-wideEP_on_vllm-v0.27. Serving GLM-5.* on an image built from + # an older ref is unsupported — rebuild the image; do not re-add runtime patchers. + # + # Likewise the MoRI version is pinned by the Dockerfile MORI_REF (42e895472b08, + # with the large-transfer notify/mapping fixes #424/#436/#432 baked in); if a newer + # MoRI is needed, update MORI_REF and rebuild — no runtime library swap here. + return 0 } # connector_launch_worker [dp_start_rank] diff --git a/scripts/vllm_dissag/models.json b/scripts/vllm_dissag/models.json index 2d9f0ab4..59897ee8 100644 --- a/scripts/vllm_dissag/models.json +++ b/scripts/vllm_dissag/models.json @@ -305,7 +305,6 @@ "yD": "1", "RUN_MORI": "1", "RUN_DEEPEP": "0", - "GLM_SKIP_PATCHERS": "1", "BENCHMARK_COMBINATIONS": "1024/1024" }, "args": "-N 2 -n 2" diff --git a/scripts/vllm_dissag/models.yaml b/scripts/vllm_dissag/models.yaml index 4454e452..7b13c6e8 100644 --- a/scripts/vllm_dissag/models.yaml +++ b/scripts/vllm_dissag/models.yaml @@ -185,14 +185,19 @@ DeepSeek-R1: # Sparse Attention (DSA). 78 layers (3 dense + 75 MoE), 256 routed experts top-8 # + 1 shared, FP8 block 128. wideEP-only (see WIDE_EP_ONLY_MODELS in the slurm). # -# Validated DOCKER_IMAGE_NAME (submit-time, not set here — #171 requires it explicit): -# rocmshared/pytorch-private:glm5.1-vllm027-b8 -# built from docker/vllm_disagg_inference.glmv5.1 (VLLM_REF=glm5.1-dsa-wideEP_on_vllm-v0.27). -# It carries the DSA fixes in-source, so the card sets GLM_SKIP_PATCHERS=1 and NO runtime -# patcher runs. Do NOT serve this recipe on a pre-v0.27 image: those predate vLLM #47766 -# and return empty/garbage completions here (NIAH 2k scored 0/10 on -# rocm/pytorch-private:glm-dockerimage-built-09072026). The connectors/moriio.sh patchers -# are a fallback for such older images only (GLM_SKIP_PATCHERS unset). +# IMAGE CONTRACT: this recipe is config only — it carries NO vLLM source fixes. +# The DSA enablement (per-req-ctx metadata key #47766, DSA indexer KV transfer over +# the MoRIIO connector, invalid-token sentinel, sparse-indexer bounds guard) must be +# IN-SOURCE in the image's vLLM. Build the per-model image from +# docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile, whose +# VLLM_REF=glm5.1-dsa-wideEP_on_vllm-v0.27 IS the contract. An image built from an +# older vLLM ref will boot and then produce garbage or stall the disagg KV transfer; +# there are no runtime patchers to fall back on. Rebuild the image instead. +# DOCKER_IMAGE_NAME is submit-time, not set here (#171 requires it explicit). +# NOT SUPPORTED here: the earlier lab image +# rocmshared/pytorch-private:vllm-wideep_06_29_2026_Shiksha_dp16_2p2d_mori_v1.2.1_aiter_v0.1.16.post3_nightlybase_mori121 +# predates the in-source DSA fixes (it needed the invalid-token kernel fix #45324 +# applied at runtime), so it is only usable on a branch that still ships patchers. # # GLM differs from DeepSeek in 3 recipe-defining ways (both are MLA MoE, but GLM # is DSA-sparse): @@ -206,10 +211,11 @@ DeepSeek-R1: # keeps them; DeepSeek is the one that overrides them off. Set here explicitly so # the recipe is self-documenting and robust to connector default changes. # -# DSA adds a 2nd KV cache per layer (indexer); the MoRIIO connector needs the GLM -# DSA patchers (kernel #45324 + dual-KV geometry + per-layer offset + completion -# gate) applied by connector_runtime_patch in connectors/moriio.sh (gated on this -# MODEL_NAME). Those are code patches, not flags — nothing to add here for them. +# DSA adds a 2nd KV cache per layer (indexer), which the single-geometry MoRIIO +# connector had to learn (dual-KV geometry + per-layer offset + completion gate). +# Those are vLLM code changes, not flags: they live in the image's vLLM at the +# pinned VLLM_REF above. connector_runtime_patch in connectors/moriio.sh is a +# deliberate no-op for every model — nothing to add here for them. # # dp_flags carry the GLM tool/reasoning parsers (AMD GLM recipe). They are applied # to BOTH roles by compose() in vllm_disagg.sh and reach `vllm serve` via the diff --git a/scripts/vllm_dissag/run_xPyD_models.slurm b/scripts/vllm_dissag/run_xPyD_models.slurm index 6e3982b7..7c741f9c 100755 --- a/scripts/vllm_dissag/run_xPyD_models.slurm +++ b/scripts/vllm_dissag/run_xPyD_models.slurm @@ -621,9 +621,6 @@ docker run --rm \ ${PREFILL_MORI_BACKEND:+-e PREFILL_MORI_BACKEND=$PREFILL_MORI_BACKEND} \ ${DECODE_MORI_BACKEND:+-e DECODE_MORI_BACKEND=$DECODE_MORI_BACKEND} \ -e MODELS_YAML_PROTECT="${MODELS_YAML_PROTECT:-}" \ - ${GLM_PERSIST_GATE:+-e GLM_PERSIST_GATE=$GLM_PERSIST_GATE} \ - ${GLM_DSA_SENTINEL_FIX:+-e GLM_DSA_SENTINEL_FIX=$GLM_DSA_SENTINEL_FIX} \ - ${GLM_SKIP_PATCHERS:+-e GLM_SKIP_PATCHERS=$GLM_SKIP_PATCHERS} \ ${KV_BLOCK_SIZE:+-e KV_BLOCK_SIZE=$KV_BLOCK_SIZE} \ ${KV_CACHE_MEMORY_BYTES:+-e KV_CACHE_MEMORY_BYTES=$KV_CACHE_MEMORY_BYTES} \ ${VLLM_ROCM_USE_AITER_MLA:+-e VLLM_ROCM_USE_AITER_MLA=$VLLM_ROCM_USE_AITER_MLA} \ From 15f4e92212e821362c85be63f06fd1613d0f2923 Mon Sep 17 00:00:00 2001 From: Mir Mustafa Ali Date: Tue, 18 Aug 2026 00:34:11 -0500 Subject: [PATCH 13/23] [GLM-5.1] Scope USE_INDUCTOR_GRAPH_PARTITION to GLM, restore other models' argv The knob was introduced by the GLM v0.27 level-set (it does not exist on origin/develop) with a default of 1, so every model served over the moriio connector silently got "use_inductor_graph_partition":true appended to its --compilation-config: DeepSeek-V3, DeepSeek-V3-5layer, DeepSeek-R1, amd-Llama-3.3-70B-Instruct-FP8-KV and gpt-oss-120b. The flag is a fix for the MLA custom-op boxing failure that only GLM-5.1 hits, so it belongs with the model, not with the connector. Flip the connector default to 0 and set USE_INDUCTOR_GRAPH_PARTITION=1 in the GLM-5.1-FP8 models.yaml env: block, the same mechanism the recipe already uses for VLLM_USE_LAYERNAME / KV_BLOCK_SIZE / DECODE_CUDAGRAPH_MODE. Add the key to _RECIPE_ENV_KEYS so the image-baked < models.yaml < submit-time -e precedence holds for it too (without it a submit-time override would lose to the recipe). Verified offline (DRY_RUN argv, no GPUs): the five models above are now byte-identical to origin/develop across moriio+rixl x wideEP+TP x prefill+decode (40 cells, zero diff, no use_inductor_graph_partition anywhere), GLM-5.1-FP8 prefill+decode argv is unchanged vs glm5.1-wideep-mad, and argv_assert is 25 passed / 0 failed. --- scripts/vllm_dissag/connectors/moriio.sh | 9 +++++++-- scripts/vllm_dissag/models.yaml | 7 +++++++ scripts/vllm_dissag/run_xPyD_models.slurm | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/scripts/vllm_dissag/connectors/moriio.sh b/scripts/vllm_dissag/connectors/moriio.sh index fa4d7599..7617f20a 100644 --- a/scripts/vllm_dissag/connectors/moriio.sh +++ b/scripts/vllm_dissag/connectors/moriio.sh @@ -181,9 +181,14 @@ connector_launch_worker() { # warmup via the fake path, then crashes). splitting_ops-list membership alone doesn't # cut the graph there. use_inductor_graph_partition=true moves partitioning to inductor # codegen (after all passes), splitting at cudagraph_unsafe ops incl. the KV-update so - # it runs as an eager boundary. Toggle via USE_INDUCTOR_GRAPH_PARTITION (default 1). + # it runs as an eager boundary. Toggle via USE_INDUCTOR_GRAPH_PARTITION. + # DEFAULT OFF: only MLA models that hit that boxing failure need it (today just + # GLM-5.1-FP8, which sets USE_INDUCTOR_GRAPH_PARTITION=1 in its models.yaml env:). + # Defaulting it ON would change --compilation-config for EVERY model on this + # connector (DeepSeek-V3/-5layer/-R1, Llama-70B, gpt-oss-120b) — an unrelated + # compile-path change for recipes that are already validated without it. local _igp_json="" - [[ "${USE_INDUCTOR_GRAPH_PARTITION:-1}" == "1" ]] && _igp_json=',"use_inductor_graph_partition":true' + [[ "${USE_INDUCTOR_GRAPH_PARTITION:-0}" == "1" ]] && _igp_json=',"use_inductor_graph_partition":true' if [[ -n "$_cudagraph_mode" && "$_cudagraph_mode" != "NONE" ]]; then local _capture_sizes="${CUDAGRAPH_CAPTURE_SIZES:-1 2 4 8 16 32 64 128 256}" exec_args+=(--compilation-config '{"cudagraph_mode":"'"${_cudagraph_mode}"'","custom_ops":["+quant_fp8"]'"${_igp_json}"'}') diff --git a/scripts/vllm_dissag/models.yaml b/scripts/vllm_dissag/models.yaml index 7b13c6e8..62723abd 100644 --- a/scripts/vllm_dissag/models.yaml +++ b/scripts/vllm_dissag/models.yaml @@ -266,6 +266,13 @@ GLM-5.1-FP8: # server, but notable for CI. Set PIECEWISE to fall back. DECODE_CUDAGRAPH_MODE: "FULL_AND_PIECEWISE" CUDAGRAPH_CAPTURE_SIZES: "1 2 4 8 16 32 64 128 256" + # v0.27 MLA compile fix, GLM-only. The *_kv_cache_update op dispatches the STABLE-ABI + # concat_and_cache_mla, whose boxed kernel does not compose inside the Dynamo-FX + # partitioned graph -> "RuntimeError: unknown parameter type" on the first real MLA + # decode. This moves partitioning to inductor codegen so the KV-update becomes an + # eager boundary. The connector default is OFF (see connectors/moriio.sh) so no other + # model's --compilation-config changes; GLM opts in here. + USE_INDUCTOR_GRAPH_PARTITION: "1" VLLM_ALL2ALL_BACKEND: "mori_high_throughput" PREFILL_MORI_BACKEND: "mori_high_throughput" DECODE_MORI_BACKEND: "mori_low_latency" diff --git a/scripts/vllm_dissag/run_xPyD_models.slurm b/scripts/vllm_dissag/run_xPyD_models.slurm index 7c741f9c..8a95bf41 100755 --- a/scripts/vllm_dissag/run_xPyD_models.slurm +++ b/scripts/vllm_dissag/run_xPyD_models.slurm @@ -76,7 +76,7 @@ echo "Running from: $(pwd)" # model's own recipe — GLM-5.1 DSA needs block=1 + AITER MLA on), while a genuine # submit-time `-e VAR=...` still wins. Precedence: image-baked < models.yaml < submit -e. # Captured HERE (before the slurm sets any defaults) so it reflects user intent only. -_RECIPE_ENV_KEYS="VLLM_USE_V1 VLLM_USE_LAYERNAME VLLM_ROCM_USE_AITER VLLM_ROCM_USE_AITER_RMSNORM VLLM_ROCM_USE_AITER_MLA KV_BLOCK_SIZE KV_CACHE_DTYPE KV_CACHE_MEMORY_BYTES GPU_MEMORY_UTILIZATION VLLM_CUDAGRAPH_MODE PREFILL_CUDAGRAPH_MODE DECODE_CUDAGRAPH_MODE CUDAGRAPH_CAPTURE_SIZES VLLM_ALL2ALL_BACKEND PREFILL_MORI_BACKEND DECODE_MORI_BACKEND MORI_SHMEM_HEAP_SIZE" +_RECIPE_ENV_KEYS="VLLM_USE_V1 VLLM_USE_LAYERNAME VLLM_ROCM_USE_AITER VLLM_ROCM_USE_AITER_RMSNORM VLLM_ROCM_USE_AITER_MLA KV_BLOCK_SIZE KV_CACHE_DTYPE KV_CACHE_MEMORY_BYTES GPU_MEMORY_UTILIZATION VLLM_CUDAGRAPH_MODE PREFILL_CUDAGRAPH_MODE DECODE_CUDAGRAPH_MODE CUDAGRAPH_CAPTURE_SIZES USE_INDUCTOR_GRAPH_PARTITION VLLM_ALL2ALL_BACKEND PREFILL_MORI_BACKEND DECODE_MORI_BACKEND MORI_SHMEM_HEAP_SIZE" MODELS_YAML_PROTECT="" for _k in $_RECIPE_ENV_KEYS; do [ -n "${!_k+x}" ] && MODELS_YAML_PROTECT="${MODELS_YAML_PROTECT} ${_k}" From e8a633d4e647bd21c7b27cd380090a8cde20f221 Mon Sep 17 00:00:00 2001 From: Mir Mustafa Ali Date: Tue, 18 Aug 2026 00:34:29 -0500 Subject: [PATCH 14/23] [GLM-5.1] Fix false MORI_REF claim and stale image text in the GLM docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comment-only. Three claims in the GLM catalog text did not match reality: 1. connectors/moriio.sh said MORI_REF=42e895472b08 had the large-transfer notify/mapping fixes #424/#436/#432 "baked in". It does not: the pin is ROCm/mori#366 (2026-06-05), while #424 merged 2026-06-26 and #432/#436 on 2026-06-29 — `gh api repos/ROCm/mori/compare/42e895472b08...` reports status "ahead" for all three. The pin is left alone (the image was validated with exactly this MoRI); only the comment is corrected. 2. models.yaml said the unsupported rocmshared lab image "is only usable on a branch that still ships patchers" — a dangling reference from a branch with no patchers. It now names the one supported image outright. 3. models.yaml's long-context caveat described the old mori v1.2.1 lab image ("garbage beyond ~16-18k tokens") as if it applied to the pinned image, which contradicts both the glmv5.1 Dockerfile STATUS and the NIAH 2k-35k numbers in this same recipe. Re-scoped to history and pointed at the in-source fix (vLLM #47766). #40018 is also labelled as an issue, which it is. Two audited citations needed no change: vLLM #47042 is only cited in benchmark_niah.{py,sh} / README.MD / ARCHITECTURE.md, all of which are origin/develop text that already calls it an issue (never a PR); and aiter#3921 (a gfx950 A8W8 change) was only cited by apply_glm_dsa_persistent_kernel_gate_fix.py, which this branch already deleted. --- scripts/vllm_dissag/connectors/moriio.sh | 9 +++++--- scripts/vllm_dissag/models.yaml | 26 ++++++++++++++---------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/scripts/vllm_dissag/connectors/moriio.sh b/scripts/vllm_dissag/connectors/moriio.sh index 7617f20a..c959adf9 100644 --- a/scripts/vllm_dissag/connectors/moriio.sh +++ b/scripts/vllm_dissag/connectors/moriio.sh @@ -140,9 +140,12 @@ connector_runtime_patch() { # VLLM_REF=glm5.1-dsa-wideEP_on_vllm-v0.27. Serving GLM-5.* on an image built from # an older ref is unsupported — rebuild the image; do not re-add runtime patchers. # - # Likewise the MoRI version is pinned by the Dockerfile MORI_REF (42e895472b08, - # with the large-transfer notify/mapping fixes #424/#436/#432 baked in); if a newer - # MoRI is needed, update MORI_REF and rebuild — no runtime library swap here. + # Likewise the MoRI version is pinned by the Dockerfile MORI_REF. That pin is + # 42e895472b08 = ROCm/mori#366 "fix(ep): mlx5 collapsed CQ + dedicated dispatch send + # buffer for internode-v1" (2026-06-05) — the tip GLM-5.1 DSA wideEP was validated on. + # It does NOT contain the later large-transfer notify/mapping fixes #424 (2026-06-26), + # #432 and #436 (2026-06-29): all three merged after the pin. If you need those, + # update MORI_REF and rebuild + re-validate — no runtime library swap here. return 0 } diff --git a/scripts/vllm_dissag/models.yaml b/scripts/vllm_dissag/models.yaml index 62723abd..98d89726 100644 --- a/scripts/vllm_dissag/models.yaml +++ b/scripts/vllm_dissag/models.yaml @@ -197,7 +197,10 @@ DeepSeek-R1: # NOT SUPPORTED here: the earlier lab image # rocmshared/pytorch-private:vllm-wideep_06_29_2026_Shiksha_dp16_2p2d_mori_v1.2.1_aiter_v0.1.16.post3_nightlybase_mori121 # predates the in-source DSA fixes (it needed the invalid-token kernel fix #45324 -# applied at runtime), so it is only usable on a branch that still ships patchers. +# applied at runtime). This branch ships no patchers, so that image cannot serve +# GLM-5.1-FP8 at all. The one supported image is built from the glmv5.1 Dockerfile +# at VLLM_REF=glm5.1-dsa-wideEP_on_vllm-v0.27; that is the image the numbers below +# were measured on. # # GLM differs from DeepSeek in 3 recipe-defining ways (both are MLA MoE, but GLM # is DSA-sparse): @@ -223,16 +226,17 @@ DeepSeek-R1: # from the env: recipe above (the connector emits them), so the dp: blocks are # empty like the DeepSeek family. # -# LONG-CONTEXT CAVEAT (vLLM #40018): the ROCM_AITER_MLA_SPARSE prefill indexer -# corrupts output for prompts beyond ~16-18k tokens on this image (mori v1.2.1) — -# coherent + correct needle retrieval up to 14k, garbage (repetition collapse, -# unique-ratio ~0.1) at ~18.7k. This is an UPSTREAM kernel bug, not the MAD port. -# TESTED: pinning --max-model-len 32768 does NOT move the threshold (workspace is -# sized max_model_len*40 but the corruption onset is a fixed ~18k token count in -# the gather/logits kernel, not a buffer-scaling artifact). So no config knob -# helps; it needs the complete upstream prefill fix in a newer image. Left at the -# native max_model_len (do not cap — capping gives no accuracy benefit and only -# limits usable context). Serve prompts <~14k for correct output on this image. +# LONG-CONTEXT HISTORY (vLLM issue #40018): on the earlier lab image named above +# (mori v1.2.1) the ROCM_AITER_MLA_SPARSE prefill indexer corrupted output beyond +# ~16-18k tokens — coherent + correct needle retrieval up to 14k, garbage (repetition +# collapse, unique-ratio ~0.1) at ~18.7k — and no config knob moved it (TESTED: +# pinning --max-model-len 32768 does NOT shift the threshold; the onset is a fixed +# ~18k token count in the gather/logits kernel, not a buffer-scaling artifact). +# The supported image carries the upstream fix in-source (vLLM #47766, sparse-MLA +# persistent metadata keyed on per-request context lengths): NIAH 2k-35k retrieves +# with no length collapse on both 1P/1D EP8 and 2P/2D EP16. Left at the native +# max_model_len (do not cap — capping gives no accuracy benefit and only limits +# usable context). GLM-5.1-FP8: env: VLLM_USE_V1: "1" From bdda2ab2dc60cc22b7a23b12165e5a9dfdaf5cda Mon Sep 17 00:00:00 2001 From: Mir Mustafa Ali Date: Tue, 18 Aug 2026 00:55:20 -0500 Subject: [PATCH 15/23] [GLM-5.1] Fix cross-model fallout: protect-list, fabric-IP default, patch mounts Three defects that reached beyond GLM-5.1-FP8: 1. _RECIPE_ENV_KEYS listed only 18 of the 39 keys a models.yaml `env:` block can set. Once MODELS_YAML_PROTECT exists the driver protects ONLY listed keys, so a submit-time `-e KEY=...` silently lost to the recipe for the 14 unlisted non-GLM keys (Llama-405B/70B, DeepSeek family, gpt-oss-120b) and 7 GLM ones. That contradicts the precedence documented here, in models.yaml and in ARCHITECTURE.md. The list is now the exact union of models.yaml env: keys. 2. FABRIC_SUBNET defaulted to the lab prefix 10.158., changing which NIC every model advertises on any cluster that happens to have a 10.158.x address, and the slurm never forwarded it, so an operator override applied on the host but not inside the container (split-brain MASTER_ADDR vs host_ip). Now opt-in: unset = hostname -I's first entry, i.e. develop's behaviour, and the var is forwarded. 3. GLM_KERNEL_PATCH / GLM_BACKEND_PATCH bind-mounts overlaid vLLM sparse-MLA source from a host path. They do not exist on develop, nothing else in the tree reads them, and a host-path source overlay is precisely the runtime patching this branch removed in favour of the image pin. Dropped. Also correct the `-e` -> `-f` rationale on the RDMA lib mounts: a dangling symlink already fails -e; what -f excludes is a path that resolves to a non-regular file. --- scripts/vllm_dissag/run_xPyD_models.slurm | 48 +++++++++++++++++------ scripts/vllm_dissag/vllm_disagg.sh | 17 ++++---- 2 files changed, 45 insertions(+), 20 deletions(-) diff --git a/scripts/vllm_dissag/run_xPyD_models.slurm b/scripts/vllm_dissag/run_xPyD_models.slurm index 8a95bf41..a15061c5 100755 --- a/scripts/vllm_dissag/run_xPyD_models.slurm +++ b/scripts/vllm_dissag/run_xPyD_models.slurm @@ -76,7 +76,25 @@ echo "Running from: $(pwd)" # model's own recipe — GLM-5.1 DSA needs block=1 + AITER MLA on), while a genuine # submit-time `-e VAR=...` still wins. Precedence: image-baked < models.yaml < submit -e. # Captured HERE (before the slurm sets any defaults) so it reflects user intent only. -_RECIPE_ENV_KEYS="VLLM_USE_V1 VLLM_USE_LAYERNAME VLLM_ROCM_USE_AITER VLLM_ROCM_USE_AITER_RMSNORM VLLM_ROCM_USE_AITER_MLA KV_BLOCK_SIZE KV_CACHE_DTYPE KV_CACHE_MEMORY_BYTES GPU_MEMORY_UTILIZATION VLLM_CUDAGRAPH_MODE PREFILL_CUDAGRAPH_MODE DECODE_CUDAGRAPH_MODE CUDAGRAPH_CAPTURE_SIZES USE_INDUCTOR_GRAPH_PARTITION VLLM_ALL2ALL_BACKEND PREFILL_MORI_BACKEND DECODE_MORI_BACKEND MORI_SHMEM_HEAP_SIZE" +# This list MUST cover every key any models.yaml `env:` block can set. A key that is +# missing here can never enter MODELS_YAML_PROTECT, so the recipe silently overrides +# the operator's `-e` for it — which would break the precedence stated above (and in +# ARCHITECTURE.md) for whichever model owns that key. Keep in sync with models.yaml. +_RECIPE_ENV_KEYS="\ +VLLM_USE_V1 VLLM_USE_LAYERNAME \ +VLLM_CUDAGRAPH_MODE PREFILL_CUDAGRAPH_MODE DECODE_CUDAGRAPH_MODE CUDAGRAPH_CAPTURE_SIZES \ +USE_INDUCTOR_GRAPH_PARTITION TORCHINDUCTOR_BENCHMARK_KERNEL \ +KV_BLOCK_SIZE KV_CACHE_DTYPE KV_CACHE_MEMORY_BYTES GPU_MEMORY_UTILIZATION \ +VLLM_ROCM_USE_AITER VLLM_ROCM_USE_AITER_RMSNORM VLLM_ROCM_USE_AITER_MLA \ +VLLM_ROCM_USE_AITER_MHA VLLM_ROCM_USE_AITER_PAGED_ATTN VLLM_ROCM_USE_AITER_TRITON_BF16_GEMM \ +VLLM_USE_AITER_TRITON_ROPE VLLM_USE_AITER_TRITON_SILU_MUL VLLM_USE_AITER_UNIFIED_ATTENTION \ +VLLM_V1_USE_PREFILL_DECODE_ATTENTION VLLM_SPARSE_INDEXER_MAX_LOGITS_MB \ +AMDGCN_USE_BUFFER_OPS ROCM_TRITON_MOE_PRESHUFFLE_SCALES \ +TRITON_HIP_ASYNC_COPY_BYPASS_PERMUTE TRITON_HIP_ASYNC_FAST_SWIZZLE \ +TRITON_HIP_USE_ASYNC_COPY TRITON_HIP_USE_BLOCK_PINGPONG \ +VLLM_ALL2ALL_BACKEND PREFILL_MORI_BACKEND DECODE_MORI_BACKEND MORI_SHMEM_HEAP_SIZE \ +TORCH_NCCL_ENABLE_MONITORING TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC TORCH_NCCL_DUMP_ON_TIMEOUT \ +TORCH_NCCL_BLOCKING_WAIT TORCH_NCCL_ASYNC_ERROR_HANDLING NCCL_IB_TIMEOUT" MODELS_YAML_PROTECT="" for _k in $_RECIPE_ENV_KEYS; do [ -n "${!_k+x}" ] && MODELS_YAML_PROTECT="${MODELS_YAML_PROTECT} ${_k}" @@ -426,14 +444,18 @@ echo "" # Node information USER_NAME=$(whoami) MASTER_NODE=$(echo "$SELECTED_NODES" | head -n 1) -# Pick the routable fabric IP, not just hostname -I's first entry. These nodes expose -# multiple NICs (e.g. a 10.224.x overlay listed BEFORE the routable 10.158.x fabric); +# Optionally pick the routable fabric IP instead of hostname -I's first entry. Some +# clusters expose multiple NICs (e.g. an overlay listed BEFORE the routable fabric); # taking $1 blindly can advertise an unreachable addr -> prefill/decode barrier hangs -# "Waiting for nodes" forever. Prefer FABRIC_SUBNET (default 10.158.), fall back to $1. -FABRIC_SUBNET="${FABRIC_SUBNET:-10.158.}" +# "Waiting for nodes" forever. OPT-IN, and cluster-specific: set FABRIC_SUBNET to your +# fabric's address prefix (e.g. `-e FABRIC_SUBNET=10.158.`) to select it. UNSET (the +# default) = take $1, i.e. unchanged behaviour for every existing model/cluster. No +# subnet is hardcoded here because the right one is a property of the site, not of MAD. +FABRIC_SUBNET="${FABRIC_SUBNET:-}" # From a "hostname -I" line, return the first IP on FABRIC_SUBNET, else the first IP. _pick_fabric_ip() { - awk -v pfx="$FABRIC_SUBNET" '{f=$1; for(i=1;i<=NF;i++) if(index($i,pfx)==1){f=$i; break} print f}' + awk -v pfx="$FABRIC_SUBNET" \ + '{f=$1; if (pfx != "") for(i=1;i<=NF;i++) if(index($i,pfx)==1){f=$i; break} print f}' } MASTER_ADDR=$(srun --nodes=1 --ntasks=1 --time=00:20:00 --nodelist="$MASTER_NODE" bash -c 'hostname -I' | _pick_fabric_ip) MASTER_PORT=39566 # Choose an open port @@ -555,11 +577,12 @@ done for _pattern in libmlx5.so* libionic*.so* libbnxt_re*.so* libefa.so* libhns.so*; do for _vlib in $_LIBDIR/${_pattern}; do - # Require a regular file AFTER symlink resolution: these mounts are built on - # ONE node but applied on ALL nodes, and vendor NIC libs (e.g. libionic.so.1) - # can be a DANGLING symlink on some nodes -> bind-mount fails "not a directory" - # -> container create exit 125. `-f` (follows symlink, requires regular file) - # skips those; the fabric in use (mlx5) is still mounted where present. + # Require a regular file AFTER symlink resolution. `-e` accepts anything the + # path resolves to, including a directory (or a symlink to one) that happens to + # match a vendor NIC lib glob; docker then bind-mounts a directory onto a file + # target -> "not a directory" -> container create exit 125. `-f` also follows + # symlinks but additionally requires a regular file, so those are skipped; the + # fabric in use (mlx5) is still mounted where present. [ -f "$_vlib" ] && _RDMA_MOUNTS="$_RDMA_MOUNTS -v $_vlib:$_vlib:ro" done done @@ -589,8 +612,6 @@ docker run --rm \ -v $NIXL_REPO_DIR:$NIXL_COOKBOOK_PATH \ -v /tmp/vllm_cache:/tmp/vllm_cache \ ${_JIT_CACHE_MOUNT} \ - ${GLM_KERNEL_PATCH:+-v ${GLM_KERNEL_PATCH}:/usr/local/lib/python3.12/dist-packages/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py:ro} \ - ${GLM_BACKEND_PATCH:+-v ${GLM_BACKEND_PATCH}:/usr/local/lib/python3.12/dist-packages/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py:ro} \ $_RDMA_MOUNTS \ --entrypoint /bin/bash \ -e SLURM_JOB_ID=$SLURM_JOB_ID \ @@ -621,6 +642,7 @@ docker run --rm \ ${PREFILL_MORI_BACKEND:+-e PREFILL_MORI_BACKEND=$PREFILL_MORI_BACKEND} \ ${DECODE_MORI_BACKEND:+-e DECODE_MORI_BACKEND=$DECODE_MORI_BACKEND} \ -e MODELS_YAML_PROTECT="${MODELS_YAML_PROTECT:-}" \ + ${FABRIC_SUBNET:+-e FABRIC_SUBNET=$FABRIC_SUBNET} \ ${KV_BLOCK_SIZE:+-e KV_BLOCK_SIZE=$KV_BLOCK_SIZE} \ ${KV_CACHE_MEMORY_BYTES:+-e KV_CACHE_MEMORY_BYTES=$KV_CACHE_MEMORY_BYTES} \ ${VLLM_ROCM_USE_AITER_MLA:+-e VLLM_ROCM_USE_AITER_MLA=$VLLM_ROCM_USE_AITER_MLA} \ diff --git a/scripts/vllm_dissag/vllm_disagg.sh b/scripts/vllm_dissag/vllm_disagg.sh index da2afb00..a9354c61 100755 --- a/scripts/vllm_dissag/vllm_disagg.sh +++ b/scripts/vllm_dissag/vllm_disagg.sh @@ -86,12 +86,14 @@ IFS=',' read -ra IP_ARRAY <<< "${IPADDRS}" echo "Listing NIXL_COOKBOOK_PATH: ${NIXL_COOKBOOK_PATH:-}" [[ -n "${NIXL_COOKBOOK_PATH:-}" ]] && ls "${NIXL_COOKBOOK_PATH}" -# Prefer the routable fabric IP (FABRIC_SUBNET, default 10.158.) over hostname -I's -# first entry: nodes with a 10.224.x overlay listed first would bind the socket_barrier -# / advertise host_ip on an unreachable NIC -> prefill<->decode barrier hangs "Waiting -# for nodes". Matches the IPADDRS selection in run_xPyD_models.slurm. Falls back to $1. -FABRIC_SUBNET="${FABRIC_SUBNET:-10.158.}" -host_ip=$(hostname -I | awk -v pfx="$FABRIC_SUBNET" '{f=$1; for(i=1;i<=NF;i++) if(index($i,pfx)==1){f=$i; break} print f}') +# Optionally prefer the routable fabric IP over hostname -I's first entry: on a node +# whose overlay NIC is listed first, binding socket_barrier / advertising host_ip on +# that NIC hangs the prefill<->decode barrier at "Waiting for nodes". Opt-in via +# FABRIC_SUBNET (the slurm forwards it via docker -e, so both sides pick the same +# NIC); unset = take $1, unchanged. Matches the IPADDRS selection in the slurm. +FABRIC_SUBNET="${FABRIC_SUBNET:-}" +host_ip=$(hostname -I | awk -v pfx="$FABRIC_SUBNET" \ + '{f=$1; if (pfx != "") for(i=1;i<=NF;i++) if(index($i,pfx)==1){f=$i; break} print f}') host_name=$(hostname) # ============================================================================= @@ -166,7 +168,8 @@ if [[ -n "$MODEL_NAME" && -f "$MODELS_YAML" ]]; then export MODELS_YAML MODEL_NAME PARALLEL_MODE # 1) Export per-model env: block FIRST (so connector ${VAR:-default} yields to it). # Precedence: image-baked ENV < models.yaml env: < submit-time -e. - # models.yaml MUST override image-baked ENV: a DeepSeek-tuned disagg image + # models.yaml MUST override image-baked ENV: MAD's own Dockerfiles deliberately + # bake no recipe env, but a site- or lab-built DeepSeek-tuned disagg image often # bakes KV_BLOCK_SIZE=16 / VLLM_ROCM_USE_AITER_MLA=0 / VLLM_CUDAGRAPH_MODE= # PIECEWISE etc. as container ENV, which would otherwise shadow a model's own # recipe (GLM-5.1 DSA needs block=1 + AITER sparse MLA on). But a genuine From 59f5fd8f8bcc7aebac3a5ae72d8b6d0db39e1b5c Mon Sep 17 00:00:00 2001 From: Mir Mustafa Ali Date: Tue, 18 Aug 2026 00:55:39 -0500 Subject: [PATCH 16/23] [GLM-5.1] Correct false and stale claims in the GLM docs Verified against ROCm/mori, vllm-project/vllm and the pinned vLLM fork with gh api: - Dockerfile stage 1 claimed the MoRI pin is "v1.2.1 (the version for the 06_29 mori121 image)". MORI_REF=42e895472b08 is ROCm/mori#366 (2026-06-05) and is 32 commits BEHIND tag v1.2.1 (e31d426a, 2026-06-25). The text was copied from the base Dockerfile, which really does pin v1.2.1, and it also inverted models.yaml, which lists the mori121 image as NOT SUPPORTED. - Dockerfile stage 3 claimed the vLLM pin is "the 06_29 validated Wide-EP WRITE-mode branch (matches the published mori121 image)". It is the GLM DSA v0.27 branch; the 06_29 vLLM predates the in-source DSA fixes and cannot serve GLM-5.1-FP8 at all. - VLLM_REF is documented as "(HEAD cda3648602) = upstream v0.27 tip + 7 commits". The branch tip is d723eb305 (+9); it advanced on 2026-08-15, before the MAD commit that measured the 3.4x decode speedup. Since VLLM_REF is a branch name, `docker build` resolves it to that day's tip. Record the real state and tell builders to pass an explicit --build-arg VLLM_REF= for an auditable rebuild. - STATUS said "NIAH 2k-35k = 10/10" where the recipe records 51/60 and 55/60, and the vLLM pin note said "1P/1D + 2P/1D + 1P/2D ... decode PIECEWISE" where the validated topologies are 1P/1D EP8 + 2P/2D EP16 and decode is FULL_AND_PIECEWISE. - The `docker build -f` example inside the glmv5.1 Dockerfile named the BASE Dockerfile. - models.yaml's "3 recipe-defining ways" bullet still described decode as PIECEWISE with a ~3.7x ITL win after 638a033 moved it to FULL_AND_PIECEWISE. Reframed: the ~3.7x is PIECEWISE over eager, FULL_AND_PIECEWISE is the further 2.11x/1.70x already measured below. - benchmark_xPyD.sh attributed "302ms vs ~89ms" to residual JIT absorbed by the first measured cell. models.yaml and moriio.sh attribute the same 302->88ms to the 8192-token-wide MoRI EP all2all buffer, fixed by --max-num-batched-tokens. Note the distinction rather than leaving two root causes for one number. - vLLM #40018 is filed against gfx950 while this stack is gfx942; say so. - README told users to build "the one fullsource Dockerfile" and never mentioned that GLM-5.1-FP8 needs the per-model glmv5.1 image the same README's table cites. - benchmark_niah.sh passes every NIAH_* through explicitly except NIAH_SEEDS, which only worked by inheritance. --- ...gg_inference.glmv5.1.ubuntu.amd.Dockerfile | 72 ++++++++++++------- scripts/vllm_dissag/README.MD | 28 ++++++-- scripts/vllm_dissag/benchmark_niah.sh | 1 + scripts/vllm_dissag/benchmark_xPyD.sh | 8 ++- scripts/vllm_dissag/models.yaml | 18 +++-- 5 files changed, 85 insertions(+), 42 deletions(-) diff --git a/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile b/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile index e015d6f8..25a36f30 100644 --- a/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile +++ b/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile @@ -46,8 +46,10 @@ # so all four connector combos (moriio TP/wideEP, rixl NIXL TP, DeepEP # wideEP) are present (~+30-45 min build vs WITH_NIXL=0). # -# STATUS (GLM-5.1-FP8 on this stack): 1P/1D EP8 + 2P/2D EP16 NIAH 2k-35k = 10/10, -# no crash; long-context accuracy fixed via vLLM #47766 (persistent sparse-MLA kept +# STATUS (GLM-5.1-FP8 on this stack): NIAH 2k-35k retrieves with no length collapse +# and no crash on 1P/1D EP8 (51/60) and 2P/2D EP16 (55/60) — see the GLM-5.1-FP8 +# recipe in scripts/vllm_dissag/models.yaml for the full measurements; +# long-context accuracy fixed via vLLM #47766 (persistent sparse-MLA kept # ON). 4P/4D EP32 is a KNOWN OPEN DEFECT: token corruption at ALL context lengths # (garbage output even at 2k), distinct from the long-context bug; prime suspect is # the moriep all-to-all combine at EP32 scale -> deferred to future work. Use 1P/1D @@ -61,7 +63,9 @@ # (ROCm + torch nightly). The stages below OVERRIDE the base's vLLM/MoRI/AITER # with the pins we validate for GLM DSA. # - MoRI -> built from ROCm/MoRI @ 42e895472b08 (validated for GLM DSA, BUILD_UMBP=OFF). -# (main LATEST 120d2de broke the connector KV-notify handshake -- see note at MORI_REF.) +# = ROCm/mori#366 (2026-06-05), 32 commits BEHIND tag v1.2.1 (2026-06-25); see the +# note at MORI_REF for what that pin does and does not contain. (main LATEST +# 120d2de broke the connector KV-notify handshake, hence a pin, not main.) # - AITER -> STOCK ROCm/aiter @ e03fa6040 compiled from source + flydsl 0.1.7-0.1.9; # stale JIT wiped. (#47766 keeps persistent MLA ON -> aiter native gqa64 fold.) # - vLLM -> COMPILED from raviguptaamd/vllm @ glm5.1-dsa-wideEP_on_vllm-v0.27 @@ -85,7 +89,7 @@ # If you need a fix, move the pin and rebuild — do not re-add runtime patchers to MAD. # # Build context = repo root: -# docker build -f docker/vllm_disagg_inference.ubuntu.amd.Dockerfile -t / . +# docker build -f docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile -t / . # # BASE_IMAGE is the purpose-built ROCm/vLLM/MoRI base above. Override --build-arg # BASE_IMAGE=... to build on a different ROCm base. vLLM compile is long (~30-60 min). @@ -109,12 +113,14 @@ ARG WITH_NIXL=0 ARG NIC_COMPILATION_ARCH="cx7" # ----------------------------------------------------------------------------- -# 1. MoRI: replace the base's bundled MoRI with the validated ROCm/MoRI @ v1.2.1 -# (the version for the 06_29 mori121 image, dist-inf-cookbook -# Dockerfile.vllm.mori121_shareable). v1.2.1 carries the EP/RDMA correctness fixes -# plus the ROCm-7.2.3 dmabuf registration path used by the connector .env -# (expandable_segments:False). MoRI is JIT-built, so this swaps the JIT sources the -# kernels compile from at runtime. +# 1. MoRI: replace the base's bundled MoRI with the commit GLM-5.1 DSA wideEP was +# validated on, ROCm/MoRI @ 42e895472b08 (= ROCm/mori#366, 2026-06-05). NOTE this +# is NOT tag v1.2.1 (e31d426a, 2026-06-25) that the base vllm_disagg_inference +# Dockerfile pins, and it is not the MoRI in the older mori121 lab image — it is +# 32 commits older than v1.2.1. It carries the EP/RDMA correctness fixes this +# recipe needs plus the ROCm-7.2.3 dmabuf registration path used by the connector +# .env (expandable_segments:False). MoRI is JIT-built, so this swaps the JIT +# sources the kernels compile from at runtime. # BUILD CONFIG: match the cookbook build — MORI_GPU_ARCHS=gfx942, BUILD_UMBP=OFF, # DEFAULT NIC backends. Do NOT pass USE_IONIC=OFF / USE_BNXT=OFF: disabling NIC # backends produced a MoRI that deadlocked at the cross-node EP all-to-all init. @@ -132,7 +138,8 @@ ENV MORI_GPU_ARCHS=gfx942 # present in this base; UMBP is unrelated to the EP dispatch/combine kernels, so # disable it to avoid pulling in a gRPC build dependency. ENV BUILD_UMBP=OFF BUILD_UMBP_SPDK=OFF -# Build/install matches dist-inf-cookbook Dockerfile.vllm.mori121_shareable for v1.2.1: +# Build/install COMMAND (not the version) matches dist-inf-cookbook +# Dockerfile.vllm.mori121_shareable: # `BUILD_UMBP=OFF pip install .` (default build isolation). apt/pip build tooling kept # for bases that lack it; harmless where already present. RUN sed -i 's|http://|https://|g' /etc/apt/sources.list 2>/dev/null || true && \ @@ -185,24 +192,35 @@ RUN if [ "${WITH_AITER_BUILD}" != "1" ]; then \ fi # ----------------------------------------------------------------------------- -# 3. vLLM: compile from source at the 06_29 validated Wide-EP WRITE-mode branch -# (matches the published dist-inf-cookbook mori121 image). Full source compile -# (the base ships a different commit). The MoRIIO disagg fixes (#39276 notify, -# #41751 LL split, DP-rank hash-failsafe) AND the GLM DSA fixes are native in this -# branch, so no runtime patcher is needed — and none exists in MAD, which is why -# this ref is a hard requirement rather than a preference. Override VLLM_REF to -# rebuild a different commit; build only committed commits (no working-tree edits). +# 3. vLLM: compile from source at the GLM-5.1 DSA wideEP branch. This is NOT the +# 06_29 Wide-EP WRITE-mode vLLM that the base vllm_disagg_inference Dockerfile and +# the dist-inf-cookbook mori121 image build from — that vLLM predates the in-source +# DSA fixes and cannot serve GLM-5.1-FP8 (see models.yaml). Full source compile (the +# base ships a different commit). The MoRIIO disagg fixes (#39276 notify, #41751 LL +# split, DP-rank hash-failsafe) AND the GLM DSA fixes are native in this branch, so +# no runtime patcher is needed — and none exists in MAD, which is why this ref is a +# hard requirement rather than a preference. Override VLLM_REF to rebuild a +# different commit; build only committed commits (no working-tree edits). # ----------------------------------------------------------------------------- -# VLLM_REPO/REF are a PUBLIC GitHub repo + branch (the Wide-EP WRITE-mode vLLM the -# dist-inf-cookbook mori121 image builds from). Override to your own vLLM fork/branch. +# VLLM_REPO/REF are a PUBLIC GitHub repo + branch. Override to your own vLLM fork/branch. ARG VLLM_REPO=https://github.com/raviguptaamd/vllm.git -# glm5.1-dsa-wideEP_on_vllm-v0.27 (HEAD cda3648602) = upstream v0.27 tip dedbf6be8b + 7 -# ROCm/DSA commits. Core 3: per-req-ctx metadata key (#47766), DSA indexer KV transfer -# (reworked onto upstream's native MoRIIO connector), invalid-token sentinel. Plus 4 -# v0.27 fixes: concat_and_cache_mla positional (stable-ABI), splitting_ops out of the -# compiled graph (MLA "unknown parameter type"), sparse-indexer bounds-guard, and the -# decisive sentinel -1->0 (cda3648602 — aiter mla_decode_fwd derefs -1 -> GPU fault at -# disagg long-ctx). NIAH-validated 1P/1D + 2P/1D + 1P/2D, 2k-35k, decode PIECEWISE. +# REPRODUCIBILITY: this default is a BRANCH NAME, so it is mutable — the branch has +# already advanced once since GLM-5.1 was validated (cda3648602 on 2026-08-10 -> the +# MoRI EP sizing commits e8c186f71/d723eb305 on 2026-08-15). `docker build` resolves it +# to whatever the tip is on the day you build, which is NOT what "the image is the +# contract" should mean. For an auditable rebuild, pass the exact commit: +# --build-arg VLLM_REF= +# and record it — /app/versions.txt in the built image captures the resolved sha. +# +# glm5.1-dsa-wideEP_on_vllm-v0.27 = upstream v0.27 tip dedbf6be8b + 9 ROCm commits +# (tip d723eb305 as of 2026-08-18). Core DSA 3: per-req-ctx metadata key (#47766), DSA +# indexer KV transfer (reworked onto upstream's native MoRIIO connector), invalid-token +# sentinel. Plus 4 v0.27 fixes: concat_and_cache_mla positional (stable-ABI), +# splitting_ops out of the compiled graph (MLA "unknown parameter type"), +# sparse-indexer bounds-guard, and the decisive sentinel -1->0 (cda3648602 — aiter +# mla_decode_fwd derefs -1 -> GPU fault at disagg long-ctx). Plus 2 MoRI EP sizing +# commits. NIAH-validated 1P/1D EP8 + 2P/2D EP16, 2k-35k; see the models.yaml recipe +# for the per-role cudagraph modes actually used (decode FULL_AND_PIECEWISE). ARG VLLM_REF=glm5.1-dsa-wideEP_on_vllm-v0.27 ENV VLLM_TARGET_DEVICE=rocm \ PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH} \ diff --git a/scripts/vllm_dissag/README.MD b/scripts/vllm_dissag/README.MD index 921bc54d..bc4cf09d 100644 --- a/scripts/vllm_dissag/README.MD +++ b/scripts/vllm_dissag/README.MD @@ -40,8 +40,9 @@ neither → combo 1. Use these exact `MODEL_NAME` values (defined in `models.yaml` + the slurm allowlists). The **Combos** column lists which of the 4 valid combos above each model may run in; the -slurm gate rejects any other pairing. Dense models are TP-only; the DeepSeek family is -**wideEP-only** (TP is rejected — its recipe needs the wideEP serve path). +slurm gate rejects any other pairing. Dense models are TP-only; the DeepSeek family and +`GLM-5.1-FP8` are **wideEP-only** (TP is rejected — their recipes need the wideEP serve +path). | Model | Type | Combos (by #) | Notes | |-------|------|---------------|-------| @@ -65,7 +66,7 @@ slurm gate rejects any other pairing. Dense models are TP-only; the DeepSeek fam ## Building the Docker image -No public prebuilt image ships — build your own from the one fullsource Dockerfile (MoRI + +No public prebuilt image ships — build your own from the fullsource Dockerfile (MoRI + AITER + vLLM + vllm-router + optional UCX/RIXL/rocSHMEM/DeepEP, from Git pins on an open ROCm ci_base): @@ -75,9 +76,24 @@ docker build -t /vllm-disagg:local \ -f docker/vllm_disagg_inference.ubuntu.amd.Dockerfile . ``` -One image serves all connectors. `WITH_NIXL=1` (default) builds every transport (moriio -TP/wideEP + rixl NIXL TP + DeepEP wideEP); add `--build-arg WITH_NIXL=0` for a lean -MoRI-EP-only image (moriio + deepep-from-base, faster build). +That image serves all connectors for every model in the table above **except** +`GLM-5.1-FP8`. `WITH_NIXL=1` (default) builds every transport (moriio TP/wideEP + rixl +NIXL TP + DeepEP wideEP); add `--build-arg WITH_NIXL=0` for a lean MoRI-EP-only image +(moriio + deepep-from-base, faster build). + +`GLM-5.1-FP8` needs its own per-model image, because its vLLM/AITER/MoRI pins differ +from the DeepSeek-V3/R1 stack the shared Dockerfile is pinned to: + +```bash +docker build -t /vllm-disagg:glmv5.1 \ + -f docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile . +``` + +That Dockerfile defaults to `WITH_NIXL=0` (MoRI-EP only, which is all GLM-5.1 uses) and +its `VLLM_REF` is a hard requirement, not a preference — MAD ships no runtime patchers, +so every GLM DSA fix has to be in-source in that image. Add a +`vllm_disagg_inference..ubuntu.amd.Dockerfile` per future model that needs its +own pins rather than repinning the shared one. Then `export DOCKER_IMAGE_NAME=` (or a published image) before submitting. The slurm `docker pull`s the image on every node, so a local-only tag must be pushed to a registry the nodes diff --git a/scripts/vllm_dissag/benchmark_niah.sh b/scripts/vllm_dissag/benchmark_niah.sh index d366b5e4..27bc7aea 100755 --- a/scripts/vllm_dissag/benchmark_niah.sh +++ b/scripts/vllm_dissag/benchmark_niah.sh @@ -34,6 +34,7 @@ done NIAH_URL="http://127.0.0.1:${BENCHMARK_PORT}/v1/chat/completions" \ NIAH_MODEL="${MODEL_PATH}" \ NIAH_WORDS="${NIAH_WORDS:-2000,8000,20000,35000}" \ +NIAH_SEEDS="${NIAH_SEEDS:-0,1,2}" \ NIAH_MAXTOK="${NIAH_MAXTOK:-2048}" \ NIAH_TIMEOUT="${NIAH_TIMEOUT:-1800}" \ NIAH_WARMUP="${NIAH_WARMUP:-1}" \ diff --git a/scripts/vllm_dissag/benchmark_xPyD.sh b/scripts/vllm_dissag/benchmark_xPyD.sh index 699c9a4e..9ba3476d 100755 --- a/scripts/vllm_dissag/benchmark_xPyD.sh +++ b/scripts/vllm_dissag/benchmark_xPyD.sh @@ -43,9 +43,11 @@ for i in $(seq 1 $BENCHMARK_ITR); do # Per-shape warmup at the REAL isl/osl, low concurrency. The global warmup above # is isl=osl=32/con=1, which never exercises this shape's prefill path, its Triton/ # aiter kernel variants, or the decode cudagraph batch sizes -- so without this the - # FIRST measured cell of each shape absorbs all the residual JIT and reports a - # wildly inflated TPOT (observed 302ms vs ~89ms steady-state). Measured cells must - # start from a warm graph. Skip with SHAPE_WARMUP=0. + # FIRST measured cell of each shape absorbs all the residual JIT and reports an + # inflated TPOT. Measured cells must start from a warm graph. Skip with + # SHAPE_WARMUP=0. (Do not confuse this with the ~302ms -> ~88ms TPOT figure in the + # GLM-5.1-FP8 recipe: that one is the over-wide MoRI EP all2all buffer, fixed by + # --max-num-batched-tokens, not by warmup.) if [[ "${SHAPE_WARMUP:-1}" == "1" ]]; then _w_con="${SHAPE_WARMUP_CON:-4}" _w_prompts="${SHAPE_WARMUP_PROMPTS:-8}" diff --git a/scripts/vllm_dissag/models.yaml b/scripts/vllm_dissag/models.yaml index 98d89726..059b3e68 100644 --- a/scripts/vllm_dissag/models.yaml +++ b/scripts/vllm_dissag/models.yaml @@ -190,7 +190,9 @@ DeepSeek-R1: # the MoRIIO connector, invalid-token sentinel, sparse-indexer bounds guard) must be # IN-SOURCE in the image's vLLM. Build the per-model image from # docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile, whose -# VLLM_REF=glm5.1-dsa-wideEP_on_vllm-v0.27 IS the contract. An image built from an +# VLLM_REF=glm5.1-dsa-wideEP_on_vllm-v0.27 IS the contract. That ref is a BRANCH and +# therefore mutable — for an auditable rebuild pass the exact commit as a build arg +# (see the REPRODUCIBILITY note at VLLM_REF in that Dockerfile). An image built from an # older vLLM ref will boot and then produce garbage or stall the disagg KV transfer; # there are no runtime patchers to fall back on. Rebuild the image instead. # DOCKER_IMAGE_NAME is submit-time, not set here (#171 requires it explicit). @@ -206,10 +208,12 @@ DeepSeek-R1: # is DSA-sparse): # - KV_BLOCK_SIZE=1 (DSA sparse indexer REQUIRES block-size 1; DS uses 16) # - VLLM_ROCM_USE_AITER_MLA=1 (GLM MLA path ON via AITER sparse; DS sets 0) -# - prefill EAGER (NONE), decode PIECEWISE (prefill cudagraph capture deadlocks -# on this stack; decode PIECEWISE captures cleanly on DSA and is a ~3.7x ITL win -# (validated: ~69ms vs ~264ms eager). Global VLLM_CUDAGRAPH_MODE=NONE as the -# safe floor; per-role PREFILL=NONE / DECODE=PIECEWISE override it.) +# - prefill EAGER (NONE), decode FULL_AND_PIECEWISE (prefill cudagraph capture +# deadlocks on this stack; decode captures cleanly on DSA. PIECEWISE was already +# a ~3.7x ITL win over eager (~69ms vs ~264ms); FULL_AND_PIECEWISE is a further +# 2.11x at EP8 / 1.70x at EP16 on top of PIECEWISE -- measured TPOT in the +# DECODE_CUDAGRAPH_MODE note below. Global VLLM_CUDAGRAPH_MODE=NONE as the safe +# floor; per-role PREFILL=NONE / DECODE=FULL_AND_PIECEWISE override it.) # Note: block=1 + AITER_MLA=1 are ALREADY the moriio.sh connector defaults — GLM # keeps them; DeepSeek is the one that overrides them off. Set here explicitly so # the recipe is self-documenting and robust to connector default changes. @@ -226,7 +230,9 @@ DeepSeek-R1: # from the env: recipe above (the connector emits them), so the dp: blocks are # empty like the DeepSeek family. # -# LONG-CONTEXT HISTORY (vLLM issue #40018): on the earlier lab image named above +# LONG-CONTEXT HISTORY (vLLM issue #40018 -- filed against gfx950 while this stack is +# gfx942, so it is the same class of bug, not the same report): on the earlier lab +# image named above # (mori v1.2.1) the ROCM_AITER_MLA_SPARSE prefill indexer corrupted output beyond # ~16-18k tokens — coherent + correct needle retrieval up to 14k, garbage (repetition # collapse, unique-ratio ~0.1) at ~18.7k — and no config knob moved it (TESTED: From b57c069e13a0603230a20b14fd9f95da21106d79 Mon Sep 17 00:00:00 2001 From: Mir Mustafa Ali Date: Tue, 18 Aug 2026 00:55:47 -0500 Subject: [PATCH 17/23] [GLM-5.1] Sync the gate_check allowlist mirror with the slurm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The branch added GLM-5.1-FP8 to VALID_MODELS, MORI_EP_VALID_MODELS and WIDE_EP_ONLY_MODELS in run_xPyD_models.slurm but not to gate_check.sh's copy, so the new model's gate had no coverage. Add it, plus the four cases that pin the intended envelope (moriio wideEP only; TP and DeepEP rejected). The header also claimed the test "runs the REAL gate code path, so a future edit to the lists/gate is caught" — it does not, it re-runs a hand-kept mirror, which is exactly why this drift was possible. Say what it actually does. --- scripts/vllm_dissag/tests/gate_check.sh | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/scripts/vllm_dissag/tests/gate_check.sh b/scripts/vllm_dissag/tests/gate_check.sh index 66f61674..3700bf8d 100755 --- a/scripts/vllm_dissag/tests/gate_check.sh +++ b/scripts/vllm_dissag/tests/gate_check.sh @@ -3,9 +3,10 @@ # Verifies the (MODEL x CONNECTOR x WIDE_EP x EP_BACKEND) enablement gate in # run_xPyD_models.slurm accepts exactly the supported combos and rejects the rest. # -# Strategy: source the slurm's model lists + axis/validation logic in a harness that -# stops right after the gate (never reaches docker/srun), then assert exit status. -# This runs the REAL gate code path, so a future edit to the lists/gate is caught. +# Strategy: re-run a hand-kept MIRROR of the slurm's model lists + axis/validation logic +# in a harness that stops right after the gate (never reaches docker/srun), then assert +# the verdict. NOTE the mirror is a copy, not the real code path: editing the lists in +# run_xPyD_models.slurm without editing the mirror below will NOT be caught here. # # Usage: bash tests/gate_check.sh (exit 0 = all pass) set -u @@ -28,10 +29,11 @@ _run_gate() { set -u # --- mirror of run_xPyD_models.slurm gate (keep in sync) --- VALID_MODELS=( "Llama-3.1-405B-Instruct-FP8-KV" "amd-Llama-3.3-70B-Instruct-FP8-KV" \ - "DeepSeek-V3" "DeepSeek-V3-5layer" "gpt-oss-120b" "DeepSeek-R1" "Qwen3-32B" "Qwen3-30B-A3B" ) - MORI_EP_VALID_MODELS=( "DeepSeek-V3" "DeepSeek-V3-5layer" "DeepSeek-R1" ) + "DeepSeek-V3" "DeepSeek-V3-5layer" "gpt-oss-120b" "DeepSeek-R1" "Qwen3-32B" "Qwen3-30B-A3B" \ + "GLM-5.1-FP8" ) + MORI_EP_VALID_MODELS=( "DeepSeek-V3" "DeepSeek-V3-5layer" "DeepSeek-R1" "GLM-5.1-FP8" ) DEEPEP_VALID_MODELS=( "DeepSeek-V3" "DeepSeek-V3-5layer" "DeepSeek-R1" ) - WIDE_EP_ONLY_MODELS=( "DeepSeek-V3" "DeepSeek-V3-5layer" "DeepSeek-R1" ) + WIDE_EP_ONLY_MODELS=( "DeepSeek-V3" "DeepSeek-V3-5layer" "DeepSeek-R1" "GLM-5.1-FP8" ) MODEL_NAME="${MODEL_NAME:-None}" _in(){ local n="$1"; shift; for x in "$@"; do [[ "$n" == "$x" ]] && return 0; done; return 1; } _in "$MODEL_NAME" "${VALID_MODELS[@]}" || { echo REJECT; exit 0; } @@ -87,6 +89,11 @@ _case ALLOW "DSV3 rixl wideEP(deepep)" DeepSeek-V3 rixl _case ALLOW "R1 moriio wideEP" DeepSeek-R1 moriio 1 _case REJECT "DSV3 moriio TP" DeepSeek-V3 moriio 0 _case REJECT "DSV3 rixl TP" DeepSeek-V3 rixl 0 +# GLM-5.1-FP8 — moriio wideEP only (no TP, no DeepEP) +_case ALLOW "GLM moriio wideEP(mori)" GLM-5.1-FP8 moriio 1 +_case REJECT "GLM moriio TP" GLM-5.1-FP8 moriio 0 +_case REJECT "GLM rixl TP" GLM-5.1-FP8 rixl 0 +_case REJECT "GLM rixl wideEP(deepep)" GLM-5.1-FP8 rixl 1 # cross-pairs _case REJECT "DSV3 moriio+deepep xpair" DeepSeek-V3 moriio 1 deepep _case REJECT "DSV3 rixl+mori xpair" DeepSeek-V3 rixl 1 mori From 20c8e712dcbf6797a9aad5c9eff1a78ec4d57a97 Mon Sep 17 00:00:00 2001 From: Mir Mustafa Ali Date: Tue, 18 Aug 2026 01:19:24 -0500 Subject: [PATCH 18/23] [GLM-5.1] Name the validated v0.27 image tag and the 0/10 evidence. The IMAGE CONTRACT block said which ref the supported image is built from but never named the published artifact, and gave no evidence for why older images are unsupported. Name the tag (rocmshared/pytorch-private:glm5.1-vllm027-b8) and record the measurement: the pre-v0.27 image rocm/pytorch-private:glm-dockerimage-built-09072026 scored NIAH 2k 0/10 on the 1P/1D EP8 smoke, slurm job 216847. This branch ships no runtime patchers, so an unsupported image has no fallback and raises no error -- it serves and returns wrong output. Say that plainly. The glmv5.1 Dockerfile gets the same tag and the same evidence so the two files name one artifact; its vLLM stage already states that the mori121 image's vLLM cannot serve GLM-5.1-FP8, which is what models.yaml says too. Comment-only: no flags, no env, no pins move. --- ...gg_inference.glmv5.1.ubuntu.amd.Dockerfile | 5 +++++ scripts/vllm_dissag/models.yaml | 22 +++++++++++-------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile b/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile index 25a36f30..93e1d894 100644 --- a/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile +++ b/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile @@ -86,6 +86,11 @@ # VLLM_REF below (glm5.1-dsa-wideEP_on_vllm-v0.27) plus the MoRI/AITER pins. Serving # GLM-5.1-FP8 from an image built off an older vLLM ref is unsupported: it boots, then # produces garbage output or stalls the disagg KV transfer, with nothing to fall back on. +# That failure is silent, and it has been measured: the pre-v0.27 lab image +# rocm/pytorch-private:glm-dockerimage-built-09072026 scored NIAH 2k 0/10 on the 1P/1D +# EP8 smoke (slurm job 216847). The supported build of this Dockerfile is published as +# rocmshared/pytorch-private:glm5.1-vllm027-b8; scripts/vllm_dissag/models.yaml names the +# same tag and lists the unsupported older images. # If you need a fix, move the pin and rebuild — do not re-add runtime patchers to MAD. # # Build context = repo root: diff --git a/scripts/vllm_dissag/models.yaml b/scripts/vllm_dissag/models.yaml index 059b3e68..61151624 100644 --- a/scripts/vllm_dissag/models.yaml +++ b/scripts/vllm_dissag/models.yaml @@ -192,17 +192,21 @@ DeepSeek-R1: # docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile, whose # VLLM_REF=glm5.1-dsa-wideEP_on_vllm-v0.27 IS the contract. That ref is a BRANCH and # therefore mutable — for an auditable rebuild pass the exact commit as a build arg -# (see the REPRODUCIBILITY note at VLLM_REF in that Dockerfile). An image built from an -# older vLLM ref will boot and then produce garbage or stall the disagg KV transfer; -# there are no runtime patchers to fall back on. Rebuild the image instead. +# (see the REPRODUCIBILITY note at VLLM_REF in that Dockerfile). That build is published +# as rocmshared/pytorch-private:glm5.1-vllm027-b8, and it is the image every number +# below was measured on. The glmv5.1 Dockerfile names the same tag and the same pins. # DOCKER_IMAGE_NAME is submit-time, not set here (#171 requires it explicit). -# NOT SUPPORTED here: the earlier lab image +# +# NOT SUPPORTED here — older GLM images, which predate the in-source DSA fixes: +# rocm/pytorch-private:glm-dockerimage-built-09072026 +# measured on the 1P/1D EP8 OCI smoke: NIAH 2k retrieval 0/10 (slurm job 216847). # rocmshared/pytorch-private:vllm-wideep_06_29_2026_Shiksha_dp16_2p2d_mori_v1.2.1_aiter_v0.1.16.post3_nightlybase_mori121 -# predates the in-source DSA fixes (it needed the invalid-token kernel fix #45324 -# applied at runtime). This branch ships no patchers, so that image cannot serve -# GLM-5.1-FP8 at all. The one supported image is built from the glmv5.1 Dockerfile -# at VLLM_REF=glm5.1-dsa-wideEP_on_vllm-v0.27; that is the image the numbers below -# were measured on. +# the mori121 lab build; it needed the invalid-token kernel fix #45324 applied at +# runtime. The glmv5.1 Dockerfile says the same at its vLLM stage: the vLLM that +# image builds from cannot serve GLM-5.1-FP8. +# This branch ships NO runtime patchers, so there is no fallback and no error to key +# on: an unsupported image boots, serves, and returns silently wrong output (or stalls +# the disagg KV transfer). Rebuild the image at the pinned VLLM_REF instead. # # GLM differs from DeepSeek in 3 recipe-defining ways (both are MLA MoE, but GLM # is DSA-sparse): From 7c50649163b6387117925bdd52dda2c6f14f66b9 Mon Sep 17 00:00:00 2001 From: Mir Mustafa Ali Date: Wed, 19 Aug 2026 21:05:09 -0500 Subject: [PATCH 19/23] [GLM-5.1] Pin VLLM_REF to the validated commit instead of a branch name VLLM_REF defaulted to glm5.1-dsa-wideEP_on_vllm-v0.27, a mutable branch name, so `docker build` resolved to whatever the tip was on the day of the build. The branch has already moved once since validation (cda3648602 -> e8c186f71/d723eb305), which means "the image is the contract" could not hold: two builds of the same Dockerfile could ship different engines. Pin the tip that every number in models.yaml was measured on, d723eb305e, and update the places that described the ref as a branch (models.yaml, connectors/moriio.sh, the Dockerfile header). Moving the pin now requires an explicit --build-arg and a re-validation, which is the intent. --- ...gg_inference.glmv5.1.ubuntu.amd.Dockerfile | 28 ++++++++++--------- scripts/vllm_dissag/connectors/moriio.sh | 2 +- scripts/vllm_dissag/models.yaml | 6 ++-- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile b/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile index 93e1d894..9c5e8c79 100644 --- a/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile +++ b/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile @@ -68,9 +68,10 @@ # 120d2de broke the connector KV-notify handshake, hence a pin, not main.) # - AITER -> STOCK ROCm/aiter @ e03fa6040 compiled from source + flydsl 0.1.7-0.1.9; # stale JIT wiped. (#47766 keeps persistent MLA ON -> aiter native gqa64 fold.) -# - vLLM -> COMPILED from raviguptaamd/vllm @ glm5.1-dsa-wideEP_on_vllm-v0.27 -# (upstream v0.27 + GLM DSA + #47766 metadata-key). Full compile: a different -# commit than the base's, so a .py-only overlay would be ABI-mismatched. +# - vLLM -> COMPILED from raviguptaamd/vllm @ d723eb305e (tip of +# glm5.1-dsa-wideEP_on_vllm-v0.27 when validated; GLM DSA + #47766 metadata-key). +# Full compile: a different commit than the base's, so a .py-only overlay would be +# ABI-mismatched. # - RDMA fix (expandable_segments:False x2 + HSA_ENABLE_IPC_MODE_LEGACY=0) is NOT baked # here — it lives in scripts/vllm_dissag/connectors/.env and the launcher # forwards it via docker -e. ROCm 7.2.3 cannot dmabuf-export VMM memory, else MoRI @@ -83,7 +84,7 @@ # THIS IMAGE IS THE CONTRACT. The MAD side (scripts/vllm_dissag) is catalog/config only # and ships NO runtime .py patchers: connector_runtime_patch in connectors/moriio.sh is a # no-op for every model. So EVERY GLM DSA source fix must be carried in-source HERE, by -# VLLM_REF below (glm5.1-dsa-wideEP_on_vllm-v0.27) plus the MoRI/AITER pins. Serving +# VLLM_REF below (d723eb305e) plus the MoRI/AITER pins. Serving # GLM-5.1-FP8 from an image built off an older vLLM ref is unsupported: it boots, then # produces garbage output or stalls the disagg KV transfer, with nothing to fall back on. # That failure is silent, and it has been measured: the pre-v0.27 lab image @@ -207,15 +208,16 @@ RUN if [ "${WITH_AITER_BUILD}" != "1" ]; then \ # hard requirement rather than a preference. Override VLLM_REF to rebuild a # different commit; build only committed commits (no working-tree edits). # ----------------------------------------------------------------------------- -# VLLM_REPO/REF are a PUBLIC GitHub repo + branch. Override to your own vLLM fork/branch. +# VLLM_REPO/REF are a PUBLIC GitHub repo + commit. Override to your own vLLM fork/ref. ARG VLLM_REPO=https://github.com/raviguptaamd/vllm.git -# REPRODUCIBILITY: this default is a BRANCH NAME, so it is mutable — the branch has -# already advanced once since GLM-5.1 was validated (cda3648602 on 2026-08-10 -> the -# MoRI EP sizing commits e8c186f71/d723eb305 on 2026-08-15). `docker build` resolves it -# to whatever the tip is on the day you build, which is NOT what "the image is the -# contract" should mean. For an auditable rebuild, pass the exact commit: -# --build-arg VLLM_REF= -# and record it — /app/versions.txt in the built image captures the resolved sha. +# REPRODUCIBILITY: pinned to a SHA, not the branch name, because the branch is mutable +# and has already advanced once since GLM-5.1 was validated (cda3648602 on 2026-08-10 -> +# the MoRI EP sizing commits e8c186f71/d723eb305 on 2026-08-15). A branch name would make +# `docker build` resolve to whatever the tip is on the day you build, which is not what +# "the image is the contract" can mean. The sha below is the tip of +# glm5.1-dsa-wideEP_on_vllm-v0.27 that every number in models.yaml was measured on. +# To move the pin, pass --build-arg VLLM_REF= and re-validate; /app/versions.txt in +# the built image records the resolved sha either way. # # glm5.1-dsa-wideEP_on_vllm-v0.27 = upstream v0.27 tip dedbf6be8b + 9 ROCm commits # (tip d723eb305 as of 2026-08-18). Core DSA 3: per-req-ctx metadata key (#47766), DSA @@ -226,7 +228,7 @@ ARG VLLM_REPO=https://github.com/raviguptaamd/vllm.git # mla_decode_fwd derefs -1 -> GPU fault at disagg long-ctx). Plus 2 MoRI EP sizing # commits. NIAH-validated 1P/1D EP8 + 2P/2D EP16, 2k-35k; see the models.yaml recipe # for the per-role cudagraph modes actually used (decode FULL_AND_PIECEWISE). -ARG VLLM_REF=glm5.1-dsa-wideEP_on_vllm-v0.27 +ARG VLLM_REF=d723eb305eb78d1bda0ed357b2b54cc29487221f ENV VLLM_TARGET_DEVICE=rocm \ PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH} \ MAX_JOBS=${MAX_JOBS} diff --git a/scripts/vllm_dissag/connectors/moriio.sh b/scripts/vllm_dissag/connectors/moriio.sh index c959adf9..054340e9 100644 --- a/scripts/vllm_dissag/connectors/moriio.sh +++ b/scripts/vllm_dissag/connectors/moriio.sh @@ -137,7 +137,7 @@ connector_runtime_patch() { # indexer KV transfer, invalid-token sentinel, sparse-indexer bounds guard) are # carried IN-SOURCE by the image. The required ref is the contract: # docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile - # VLLM_REF=glm5.1-dsa-wideEP_on_vllm-v0.27. Serving GLM-5.* on an image built from + # VLLM_REF=d723eb305e. Serving GLM-5.* on an image built from # an older ref is unsupported — rebuild the image; do not re-add runtime patchers. # # Likewise the MoRI version is pinned by the Dockerfile MORI_REF. That pin is diff --git a/scripts/vllm_dissag/models.yaml b/scripts/vllm_dissag/models.yaml index 61151624..2b6d0e7f 100644 --- a/scripts/vllm_dissag/models.yaml +++ b/scripts/vllm_dissag/models.yaml @@ -190,9 +190,9 @@ DeepSeek-R1: # the MoRIIO connector, invalid-token sentinel, sparse-indexer bounds guard) must be # IN-SOURCE in the image's vLLM. Build the per-model image from # docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile, whose -# VLLM_REF=glm5.1-dsa-wideEP_on_vllm-v0.27 IS the contract. That ref is a BRANCH and -# therefore mutable — for an auditable rebuild pass the exact commit as a build arg -# (see the REPRODUCIBILITY note at VLLM_REF in that Dockerfile). That build is published +# VLLM_REF=d723eb305e IS the contract. That ref is pinned to a commit, not a branch +# name, so a rebuild is auditable and cannot drift (see the REPRODUCIBILITY note at +# VLLM_REF in that Dockerfile). That build is published # as rocmshared/pytorch-private:glm5.1-vllm027-b8, and it is the image every number # below was measured on. The glmv5.1 Dockerfile names the same tag and the same pins. # DOCKER_IMAGE_NAME is submit-time, not set here (#171 requires it explicit). From 7beac12dcb2984c9b16e8b957d9d91f698177498 Mon Sep 17 00:00:00 2001 From: Mir Mustafa Ali Date: Wed, 19 Aug 2026 21:05:32 -0500 Subject: [PATCH 20/23] [GLM-5.1] Correct the described lineage of the pinned vLLM commit The header called dedbf6be8b the "upstream v0.27 tip". It is not: against vllm-project/vllm, dedbf6be8b is a main-branch commit of 2026-08-09, 270 commits ahead of the v0.27.0 tag and 17 behind releases/v0.27.0. Verified with the GitHub compare API, which also confirms the pinned d723eb305e is that commit plus exactly 9 ROCm commits, 0 behind. State what the commit actually is so a reader does not go looking for these fixes on the v0.27 release branch. --- docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile b/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile index 9c5e8c79..7ad221f9 100644 --- a/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile +++ b/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile @@ -219,8 +219,11 @@ ARG VLLM_REPO=https://github.com/raviguptaamd/vllm.git # To move the pin, pass --build-arg VLLM_REF= and re-validate; /app/versions.txt in # the built image records the resolved sha either way. # -# glm5.1-dsa-wideEP_on_vllm-v0.27 = upstream v0.27 tip dedbf6be8b + 9 ROCm commits -# (tip d723eb305 as of 2026-08-18). Core DSA 3: per-req-ctx metadata key (#47766), DSA +# d723eb305e = the base image's vLLM commit dedbf6be8b + exactly 9 ROCm commits (verified +# with the GitHub compare API: 9 ahead, 0 behind). dedbf6be8b is vllm-project/vllm MAIN of +# 2026-08-09, i.e. 270 commits AFTER the v0.27.0 tag and 17 behind the releases/v0.27.0 +# branch — the "v0.27" in the branch name is a label for this line of work, not a +# checkout of the v0.27 release. Core DSA 3: per-req-ctx metadata key (#47766), DSA # indexer KV transfer (reworked onto upstream's native MoRIIO connector), invalid-token # sentinel. Plus 4 v0.27 fixes: concat_and_cache_mla positional (stable-ABI), # splitting_ops out of the compiled graph (MLA "unknown parameter type"), From 9f2f6eeec3cc6fa897d7916b7d5c832c45783f31 Mon Sep 17 00:00:00 2001 From: Mir Mustafa Ali Date: Wed, 19 Aug 2026 21:22:55 -0500 Subject: [PATCH 21/23] [GLM-5.1] Point VLLM_REF back at the branch, pending the mutable-ref fix Reverts the SHA pin from the previous commit. The ref goes back to glm5.1-dsa-wideEP_on_vllm-v0.27 so this Dockerfile stays consistent with how the other vllm_dissag images name their vLLM ref; the mutable-branch problem is being handled separately rather than solved one Dockerfile at a time. The REPRODUCIBILITY note stays and now names the validated commit outright, so an auditable rebuild is still one build arg away: --build-arg VLLM_REF=d723eb305eb78d1bda0ed357b2b54cc29487221f The lineage correction from the previous commit is kept and reworded for the branch form: the tip is dedbf6be8b plus 9 ROCm commits, and dedbf6be8b is upstream main, not the v0.27 release. --- ...gg_inference.glmv5.1.ubuntu.amd.Dockerfile | 34 +++++++++---------- scripts/vllm_dissag/connectors/moriio.sh | 2 +- scripts/vllm_dissag/models.yaml | 6 ++-- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile b/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile index 7ad221f9..81ad0173 100644 --- a/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile +++ b/docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile @@ -68,10 +68,9 @@ # 120d2de broke the connector KV-notify handshake, hence a pin, not main.) # - AITER -> STOCK ROCm/aiter @ e03fa6040 compiled from source + flydsl 0.1.7-0.1.9; # stale JIT wiped. (#47766 keeps persistent MLA ON -> aiter native gqa64 fold.) -# - vLLM -> COMPILED from raviguptaamd/vllm @ d723eb305e (tip of -# glm5.1-dsa-wideEP_on_vllm-v0.27 when validated; GLM DSA + #47766 metadata-key). -# Full compile: a different commit than the base's, so a .py-only overlay would be -# ABI-mismatched. +# - vLLM -> COMPILED from raviguptaamd/vllm @ glm5.1-dsa-wideEP_on_vllm-v0.27 +# (validated at tip d723eb305e; GLM DSA + #47766 metadata-key). Full compile: a +# different commit than the base's, so a .py-only overlay would be ABI-mismatched. # - RDMA fix (expandable_segments:False x2 + HSA_ENABLE_IPC_MODE_LEGACY=0) is NOT baked # here — it lives in scripts/vllm_dissag/connectors/.env and the launcher # forwards it via docker -e. ROCm 7.2.3 cannot dmabuf-export VMM memory, else MoRI @@ -84,7 +83,7 @@ # THIS IMAGE IS THE CONTRACT. The MAD side (scripts/vllm_dissag) is catalog/config only # and ships NO runtime .py patchers: connector_runtime_patch in connectors/moriio.sh is a # no-op for every model. So EVERY GLM DSA source fix must be carried in-source HERE, by -# VLLM_REF below (d723eb305e) plus the MoRI/AITER pins. Serving +# VLLM_REF below (glm5.1-dsa-wideEP_on_vllm-v0.27) plus the MoRI/AITER pins. Serving # GLM-5.1-FP8 from an image built off an older vLLM ref is unsupported: it boots, then # produces garbage output or stalls the disagg KV transfer, with nothing to fall back on. # That failure is silent, and it has been measured: the pre-v0.27 lab image @@ -208,19 +207,20 @@ RUN if [ "${WITH_AITER_BUILD}" != "1" ]; then \ # hard requirement rather than a preference. Override VLLM_REF to rebuild a # different commit; build only committed commits (no working-tree edits). # ----------------------------------------------------------------------------- -# VLLM_REPO/REF are a PUBLIC GitHub repo + commit. Override to your own vLLM fork/ref. +# VLLM_REPO/REF are a PUBLIC GitHub repo + branch. Override to your own vLLM fork/branch. ARG VLLM_REPO=https://github.com/raviguptaamd/vllm.git -# REPRODUCIBILITY: pinned to a SHA, not the branch name, because the branch is mutable -# and has already advanced once since GLM-5.1 was validated (cda3648602 on 2026-08-10 -> -# the MoRI EP sizing commits e8c186f71/d723eb305 on 2026-08-15). A branch name would make -# `docker build` resolve to whatever the tip is on the day you build, which is not what -# "the image is the contract" can mean. The sha below is the tip of -# glm5.1-dsa-wideEP_on_vllm-v0.27 that every number in models.yaml was measured on. -# To move the pin, pass --build-arg VLLM_REF= and re-validate; /app/versions.txt in -# the built image records the resolved sha either way. +# REPRODUCIBILITY: this default is a BRANCH NAME, so it is mutable — the branch has +# already advanced once since GLM-5.1 was validated (cda3648602 on 2026-08-10 -> the +# MoRI EP sizing commits e8c186f71/d723eb305 on 2026-08-15). `docker build` resolves it +# to whatever the tip is on the day you build, so two builds of this Dockerfile can ship +# different engines. Tracked separately; until that is settled, build auditably by +# passing the exact commit: +# --build-arg VLLM_REF=d723eb305eb78d1bda0ed357b2b54cc29487221f +# which is the tip every number in models.yaml was measured on. /app/versions.txt in the +# built image records whichever sha was resolved. # -# d723eb305e = the base image's vLLM commit dedbf6be8b + exactly 9 ROCm commits (verified -# with the GitHub compare API: 9 ahead, 0 behind). dedbf6be8b is vllm-project/vllm MAIN of +# Branch tip d723eb305e = the base image's vLLM commit dedbf6be8b + exactly 9 ROCm +# commits (GitHub compare API: 9 ahead, 0 behind). dedbf6be8b is vllm-project/vllm MAIN of # 2026-08-09, i.e. 270 commits AFTER the v0.27.0 tag and 17 behind the releases/v0.27.0 # branch — the "v0.27" in the branch name is a label for this line of work, not a # checkout of the v0.27 release. Core DSA 3: per-req-ctx metadata key (#47766), DSA @@ -231,7 +231,7 @@ ARG VLLM_REPO=https://github.com/raviguptaamd/vllm.git # mla_decode_fwd derefs -1 -> GPU fault at disagg long-ctx). Plus 2 MoRI EP sizing # commits. NIAH-validated 1P/1D EP8 + 2P/2D EP16, 2k-35k; see the models.yaml recipe # for the per-role cudagraph modes actually used (decode FULL_AND_PIECEWISE). -ARG VLLM_REF=d723eb305eb78d1bda0ed357b2b54cc29487221f +ARG VLLM_REF=glm5.1-dsa-wideEP_on_vllm-v0.27 ENV VLLM_TARGET_DEVICE=rocm \ PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH} \ MAX_JOBS=${MAX_JOBS} diff --git a/scripts/vllm_dissag/connectors/moriio.sh b/scripts/vllm_dissag/connectors/moriio.sh index 054340e9..c959adf9 100644 --- a/scripts/vllm_dissag/connectors/moriio.sh +++ b/scripts/vllm_dissag/connectors/moriio.sh @@ -137,7 +137,7 @@ connector_runtime_patch() { # indexer KV transfer, invalid-token sentinel, sparse-indexer bounds guard) are # carried IN-SOURCE by the image. The required ref is the contract: # docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile - # VLLM_REF=d723eb305e. Serving GLM-5.* on an image built from + # VLLM_REF=glm5.1-dsa-wideEP_on_vllm-v0.27. Serving GLM-5.* on an image built from # an older ref is unsupported — rebuild the image; do not re-add runtime patchers. # # Likewise the MoRI version is pinned by the Dockerfile MORI_REF. That pin is diff --git a/scripts/vllm_dissag/models.yaml b/scripts/vllm_dissag/models.yaml index 2b6d0e7f..61151624 100644 --- a/scripts/vllm_dissag/models.yaml +++ b/scripts/vllm_dissag/models.yaml @@ -190,9 +190,9 @@ DeepSeek-R1: # the MoRIIO connector, invalid-token sentinel, sparse-indexer bounds guard) must be # IN-SOURCE in the image's vLLM. Build the per-model image from # docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile, whose -# VLLM_REF=d723eb305e IS the contract. That ref is pinned to a commit, not a branch -# name, so a rebuild is auditable and cannot drift (see the REPRODUCIBILITY note at -# VLLM_REF in that Dockerfile). That build is published +# VLLM_REF=glm5.1-dsa-wideEP_on_vllm-v0.27 IS the contract. That ref is a BRANCH and +# therefore mutable — for an auditable rebuild pass the exact commit as a build arg +# (see the REPRODUCIBILITY note at VLLM_REF in that Dockerfile). That build is published # as rocmshared/pytorch-private:glm5.1-vllm027-b8, and it is the image every number # below was measured on. The glmv5.1 Dockerfile names the same tag and the same pins. # DOCKER_IMAGE_NAME is submit-time, not set here (#171 requires it explicit). From 4151989952bedd3d6df99e6c7abd059493b71902 Mon Sep 17 00:00:00 2001 From: Mir Mustafa Ali Date: Wed, 19 Aug 2026 22:27:09 -0500 Subject: [PATCH 22/23] [GLM-5.1] Trim the connector_runtime_patch comment back to develop's connector_runtime_patch is a return-0 no-op shared by every moriio model, and this branch had grown its comment from 7 lines to 24. The MoRI paragraph was wrong for the model that dominates this connector: it told the reader their pin is 42e895472b08 and lacks ROCm/mori #424, #432 and #436, but DeepSeek-V3 builds from vllm_disagg_inference.ubuntu.amd.Dockerfile with MORI_REF=v1.2.1, 32 commits ahead of that sha and containing those fixes. The pin story is already told, correctly scoped to the GLM image, in the glmv5.1 Dockerfile. The GLM paragraph restated a rule the existing text already makes generic and duplicated detail that belongs in the Dockerfile header and the models.yaml recipe. Restores develop's wording plus one line pointing GLM readers at models.yaml, so the hunk is a one-line diff against develop. --- scripts/vllm_dissag/connectors/moriio.sh | 29 ++++++------------------ 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/scripts/vllm_dissag/connectors/moriio.sh b/scripts/vllm_dissag/connectors/moriio.sh index c959adf9..1a8886d5 100644 --- a/scripts/vllm_dissag/connectors/moriio.sh +++ b/scripts/vllm_dissag/connectors/moriio.sh @@ -124,28 +124,13 @@ _moriio_build_kv_transfer_config() { } connector_runtime_patch() { - # No runtime .py patching on the moriio path, for ANY model. - # - # MoRIIO multi-node disagg fixes (vLLM PR#39276 notify-path, #41751 LL split, - # DP-rank hash-failsafe) are committed in-source in the vLLM the image is built - # from (Dockerfile VLLM_REF). There is no generic runtime .py patcher for those — - # that would be a drifting duplicate of fixes already upstream in the fork. - # - # The same rule now holds for GLM-5.* (GlmMoeDsaForCausalLM, MLA + DSA sparse - # attention): DSA adds a 2nd KV cache per layer (indexer) with its own geometry, - # and the connector/kernel fixes for it (per-req-ctx metadata key #47766, DSA - # indexer KV transfer, invalid-token sentinel, sparse-indexer bounds guard) are - # carried IN-SOURCE by the image. The required ref is the contract: - # docker/vllm_disagg_inference.glmv5.1.ubuntu.amd.Dockerfile - # VLLM_REF=glm5.1-dsa-wideEP_on_vllm-v0.27. Serving GLM-5.* on an image built from - # an older ref is unsupported — rebuild the image; do not re-add runtime patchers. - # - # Likewise the MoRI version is pinned by the Dockerfile MORI_REF. That pin is - # 42e895472b08 = ROCm/mori#366 "fix(ep): mlx5 collapsed CQ + dedicated dispatch send - # buffer for internode-v1" (2026-06-05) — the tip GLM-5.1 DSA wideEP was validated on. - # It does NOT contain the later large-transfer notify/mapping fixes #424 (2026-06-26), - # #432 and #436 (2026-06-29): all three merged after the pin. If you need those, - # update MORI_REF and rebuild + re-validate — no runtime library swap here. + # No-op: the MoRIIO multi-node disagg fixes (vLLM PR#39276 notify-path, #41751 LL + # split, DP-rank hash-failsafe) are committed in-source in the vLLM the image is + # built from (see the Dockerfile VLLM_REF). There is no runtime .py patcher — that + # would be a drifting duplicate of fixes that already live upstream in the fork. + # If you ever run an image WITHOUT these fixes baked, use an image that has them + # (rebuild from the pinned VLLM_REF) rather than patching a stock image at runtime. + # Same rule for GLM-5.* DSA: those fixes are in-source too (see models.yaml). return 0 } From 5e6aaea2ca3b840ecc5612089fbdad1573f53c38 Mon Sep 17 00:00:00 2001 From: Mir Mustafa Ali Date: Wed, 19 Aug 2026 22:53:45 -0500 Subject: [PATCH 23/23] [GLM-5.1] Drop the GLM line from connector_runtime_patch The function is unedited by this branch, so it needs no note from it. Leaves connector_runtime_patch byte-identical to develop; the GLM image contract is stated in models.yaml and the glmv5.1 Dockerfile, which is where it belongs. --- scripts/vllm_dissag/connectors/moriio.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/vllm_dissag/connectors/moriio.sh b/scripts/vllm_dissag/connectors/moriio.sh index 1a8886d5..ac34e2ae 100644 --- a/scripts/vllm_dissag/connectors/moriio.sh +++ b/scripts/vllm_dissag/connectors/moriio.sh @@ -130,7 +130,6 @@ connector_runtime_patch() { # would be a drifting duplicate of fixes that already live upstream in the fork. # If you ever run an image WITHOUT these fixes baked, use an image that has them # (rebuild from the pinned VLLM_REF) rather than patching a stock image at runtime. - # Same rule for GLM-5.* DSA: those fixes are in-source too (see models.yaml). return 0 }