SemiAnalysisAI · ApostaC · Jul 10, 2026 · Jul 10, 2026 · Jul 10, 2026 · Jul 10, 2026
diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh
@@ -21,7 +21,7 @@ set -x
 # Required env vars:
 #   MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
 #
-# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake.
+# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake or lmcache.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
@@ -99,15 +99,31 @@ export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768
 SERVER_LOG="$RESULT_DIR/server.log"
 ROUTER_LOG="$RESULT_DIR/router.log"
 MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log"
+LMCACHE_SERVER_LOG="$RESULT_DIR/lmcache_server.log"
 mkdir -p "$RESULT_DIR"
 
 SERVER_PID=""
 ROUTER_PID=""
 MOONCAKE_MASTER_PID=""
+LMCACHE_SERVER_PID=""
+
+# AgentX concurrency counts live session trees, not individual requests.
+# Subagent fan-out can push instantaneous request concurrency above CONC, so
+# leave 2x headroom rather than clipping those bursts at the scheduler.
+MAX_NUM_SEQS=$((2 * CONC))
+
+# Serving-flag overlays: the lmcache arm replaces these with its tuned
+# recipe; Mooncake and GPU-cache baselines keep the stock flags.
+VLLM_COMPILATION_CONFIG='{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'
+VLLM_TUNING_ARGS=()
+EP_TUNING_ARGS=()
 
 OFFLOAD_ARGS=()
 
-if require_agentic_kv_offload_backend mooncake; then
+if agentic_kv_offload_enabled; then
+    case "$KV_OFFLOAD_BACKEND" in
+    mooncake)
+        require_agentic_kv_offload_backend mooncake
         # Embedded mode contributes one segment per GPU rank to a shared
         # distributed store, so pre-divide the aggregate host-memory budget.
         PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / GPU_COUNT))
@@ -167,6 +183,133 @@ EOF
             --kv-transfer-config
             '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}'
         )
+        ;;
+    lmcache)
+        require_agentic_kv_offload_backend lmcache
+        # The LMCache MP server owns the host-DRAM KV pool as one shared
+        # tier; vLLM ranks attach via LMCacheMPConnector, so the aggregate
+        # host budget is passed through undivided (unlike Mooncake's
+        # per-rank segments). Follows the LMCache DeepSeek-V4 recipe
+        # (docs.lmcache.ai/recipes/deepseek_v4_flash.html); LMCache handles
+        # DSV4's Sparse-MLA hybrid KV geometries automatically.
+        LMCACHE_VERSION=0.5.1
+        agentic_pip_install --quiet --no-cache-dir "lmcache==$LMCACHE_VERSION"
+        python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null
+
+        LMCACHE_HOST=127.0.0.1
+        LMCACHE_PORT=$((PORT + 12000))
+        LMCACHE_HTTP_PORT=$((PORT + 13000))
+        # LMCacheMPConnector concatenates lmcache.mp.host and port into the
+        # ZMQ endpoint. Bind the server to a raw host, but pass the connector
+        # a ZMQ-style host string.
+        LMCACHE_CONNECT_HOST="tcp://$LMCACHE_HOST"
+        # Pool target derated to 75% of the aggregate budget: pinned host
+        # memory is unswappable and also consumes GPU-side mapping
+        # resources, so leave headroom for vLLM host buffers and the OS.
+        # Full-budget targets OOM-killed the node (host OOM-killer or
+        # cudaErrorMemoryAllocation) as the cache filled past ~2 TB during
+        # PR #2153 bring-up.
+        LMCACHE_L1_SIZE_GB=$((TOTAL_CPU_DRAM_GB * 3 / 4))
+        # The pool grows lazily from the initial allocation, so the full
+        # --l1-size-gb target is not pinned at startup.
+        LMCACHE_L1_INIT_SIZE_GB=20
+        LMCACHE_MQ_TIMEOUT=300
+        # Identical prefixes must hash to identical cache keys across DP ranks.
+        export PYTHONHASHSEED=0
+
+        # Tuned DeepSeek-V4 serving recipe for this section's image:
+        # decode-only CUDA graphs, sparse MLA with prefill query
+        # quantization, AMXF4 Mega-MoE, and NUMA binding.
+        # --enable-cumem-allocator is deliberately absent: LMCacheMPConnector
+        # exports the KV cache to the LMCache server through legacy CUDA IPC
+        # handles, and cuMem/VMM allocations cannot be exported that way
+        # (register_kv_caches fails with cudaErrorInvalidValue).
+        VLLM_COMPILATION_CONFIG='{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
+        VLLM_TUNING_ARGS=(
+            --numa-bind
+            --prefill-schedule-interval 8
+            --max-cudagraph-capture-size "$MAX_NUM_SEQS"
+            --no-enable-flashinfer-autotune
+            --attention-config '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}'
+            --disable-uvicorn-access-log
+        )
+        # fastsafetensors stages checkpoint shards on-GPU while loading.
+        # Without --enable-ep-weight-filter every rank stages the full
+        # expert weights and OOMs during load, so it is EP-only here.
+        # GPU memory utilization stays at the vLLM default: the Mega-MoE
+        # path JIT-loads DeepGEMM/TileLang kernels at runtime and those
+        # driver allocations live outside the vLLM pool, so
+        # VLLM_DEEP_GEMM_WARMUP=skip (exported below) keeps DeepGEMM from
+        # front-loading them all during startup.
+        EP_TUNING_ARGS=(
+            --moe-backend deep_gemm_amxf4_mega_moe
+            --enable-ep-weight-filter
+            --load-format fastsafetensors
+        )
+        export VLLM_USE_V2_MODEL_RUNNER=1
+        export VLLM_USE_RUST_FRONTEND=1
+        export VLLM_DEEPSEEK_V4_USE_MEGA_MOE=1
+        export VLLM_USE_FLASHINFER_MOE_FP8=0
+        export VLLM_DEEP_GEMM_WARMUP=skip
+        export VLLM_DSV4_MEGA_FP8_COMBINE=1
+        export VLLM_RPC_TIMEOUT=600000
+        export TILELANG_CLEANUP_TEMP_FILES=1
+        # Per-engine scheduler stats every 5s, to diagnose per-DP-rank KV
+        # cache imbalance under the session-sticky router.
+        export VLLM_LOG_STATS_INTERVAL=5
+
+        echo "Starting LMCache MP server on port $LMCACHE_PORT..."
+        # One GPU-side transfer worker avoids concurrent-GPU-transfer stalls
+        # under heavy async-load pressure; CPU-side workers stay at 8.
+        lmcache server \
+            --host "$LMCACHE_HOST" \
+            --port "$LMCACHE_PORT" \
+            --http-host "$LMCACHE_HOST" \
+            --http-port "$LMCACHE_HTTP_PORT" \
+            --l1-size-gb "$LMCACHE_L1_SIZE_GB" \
+            --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" \
+            --max-gpu-workers 1 \
+            --max-cpu-workers 8 \
+            --chunk-size 1024 \
+            --l1-align-bytes 16384 \
+            --eviction-trigger-watermark 0.85 \
+            --eviction-ratio 0.10 \
+            --eviction-policy LRU \
+            --supported-transfer-mode lmcache_driven \
+            --no-separate-object-groups \
+            > "$LMCACHE_SERVER_LOG" 2>&1 &
+        LMCACHE_SERVER_PID=$!
+        LMCACHE_READY=0
+        for _ in $(seq 1 60); do
+            if ! kill -0 "$LMCACHE_SERVER_PID" 2>/dev/null; then
+                echo "LMCache server died during startup." >&2
+                cat "$LMCACHE_SERVER_LOG" >&2
+                exit 1
+            fi
+            if curl --output /dev/null --silent --fail \
+                "http://127.0.0.1:$LMCACHE_HTTP_PORT/healthcheck"; then
+                LMCACHE_READY=1
+                break
+            fi
+            sleep 2
+        done
+        if [ "$LMCACHE_READY" -ne 1 ]; then
+            echo "LMCache server did not become healthy in time." >&2
+            cat "$LMCACHE_SERVER_LOG" >&2
+            exit 1
+        fi
+
+        unset VLLM_USE_SIMPLE_KV_OFFLOAD
+        OFFLOAD_ARGS=(
+            --kv-transfer-config
+            "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":$LMCACHE_MQ_TIMEOUT}}"
+        )
+        ;;
+    *)
+        echo "Error: unsupported KV_OFFLOAD_BACKEND '$KV_OFFLOAD_BACKEND' (expected one of: mooncake, lmcache)" >&2
+        exit 1
+        ;;
+    esac
 fi
 
 PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1)
@@ -176,14 +319,9 @@ fi
 
 EP_ARGS=()
 if [ "$EP_SIZE" -gt 1 ]; then
-    EP_ARGS=(--enable-expert-parallel)
+    EP_ARGS=(--enable-expert-parallel "${EP_TUNING_ARGS[@]}")
 fi
 
-# AgentX concurrency counts live session trees, not individual requests.
-# Subagent fan-out can push instantaneous request concurrency above CONC, so
-# leave 2x headroom rather than clipping those bursts at the scheduler.
-MAX_NUM_SEQS=$((2 * CONC))
-
 echo "Starting vllm server..."
 export TORCH_CUDA_ARCH_LIST="10.0"
 export PYTHONNOUSERSITE=1
@@ -200,7 +338,8 @@ VLLM_CMD=(
     "${PARALLEL_ARGS[@]}"
     "${VLLM_CP_ARGS[@]}"
     "${EP_ARGS[@]}"
-    --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'
+    --compilation-config "$VLLM_COMPILATION_CONFIG"
+    "${VLLM_TUNING_ARGS[@]}"
     --attention_config.use_fp4_indexer_cache=True
     --tokenizer-mode deepseek_v4
     --tool-call-parser deepseek_v4

diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
@@ -20,7 +20,7 @@ set -x
 # Required env vars:
 #   MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
 #
-# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake.
+# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake or lmcache.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
@@ -103,14 +103,19 @@ export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768
 SERVER_LOG="$RESULT_DIR/server.log"
 ROUTER_LOG="$RESULT_DIR/router.log"
 MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log"
+LMCACHE_SERVER_LOG="$RESULT_DIR/lmcache_server.log"
 mkdir -p "$RESULT_DIR"
 
 SERVER_PID=""
 ROUTER_PID=""
 MOONCAKE_MASTER_PID=""
+LMCACHE_SERVER_PID=""
 
 OFFLOAD_ARGS=()
-if require_agentic_kv_offload_backend mooncake; then
+if agentic_kv_offload_enabled; then
+    case "$KV_OFFLOAD_BACKEND" in
+    mooncake)
+        require_agentic_kv_offload_backend mooncake
         # Mooncake embedded mode contributes one global segment per GPU rank to
         # a shared distributed store. Pre-divide the aggregate host budget
         # across those rank-contributed segments.
@@ -169,6 +174,91 @@ EOF
             --kv-transfer-config
             '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}'
         )
+        ;;
+    lmcache)
+        require_agentic_kv_offload_backend lmcache
+        # The LMCache MP server owns the host-DRAM KV pool as one shared
+        # tier; vLLM ranks attach via LMCacheMPConnector, so the aggregate
+        # host budget is passed through undivided (unlike Mooncake's
+        # per-rank segments). Follows the LMCache DeepSeek-V4 recipe
+        # (docs.lmcache.ai/recipes/deepseek_v4_flash.html); LMCache handles
+        # DSV4's Sparse-MLA hybrid KV geometries automatically.
+        LMCACHE_VERSION=0.5.1
+        agentic_pip_install --quiet --no-cache-dir "lmcache==$LMCACHE_VERSION"
+        python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null
+
+        LMCACHE_HOST=127.0.0.1
+        LMCACHE_PORT=$((PORT + 12000))
+        LMCACHE_HTTP_PORT=$((PORT + 13000))
+        # LMCacheMPConnector concatenates lmcache.mp.host and port into the
+        # ZMQ endpoint. Bind the server to a raw host, but pass the connector
+        # a ZMQ-style host string.
+        LMCACHE_CONNECT_HOST="tcp://$LMCACHE_HOST"
+        # Pool target derated to 75% of the aggregate budget: pinned host
+        # memory is unswappable and also consumes GPU-side mapping
+        # resources, so leave headroom for vLLM host buffers and the OS.
+        # Full-budget targets OOM-killed the node (host OOM-killer or
+        # cudaErrorMemoryAllocation) as the cache filled past ~2 TB during
+        # PR #2153 bring-up.
+        LMCACHE_L1_SIZE_GB=$((TOTAL_CPU_DRAM_GB * 3 / 4))
+        # The pool grows lazily from the initial allocation, so the full
+        # --l1-size-gb target is not pinned at startup.
+        LMCACHE_L1_INIT_SIZE_GB=20
+        LMCACHE_MQ_TIMEOUT=300
+        # Identical prefixes must hash to identical cache keys across DP ranks.
+        export PYTHONHASHSEED=0
+
+        echo "Starting LMCache MP server on port $LMCACHE_PORT..."
+        # One GPU-side transfer worker avoids concurrent-GPU-transfer stalls
+        # under heavy async-load pressure; CPU-side workers stay at 8.
+        lmcache server \
+            --host "$LMCACHE_HOST" \
+            --port "$LMCACHE_PORT" \
+            --http-host "$LMCACHE_HOST" \
+            --http-port "$LMCACHE_HTTP_PORT" \
+            --l1-size-gb "$LMCACHE_L1_SIZE_GB" \
+            --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" \
+            --max-gpu-workers 1 \
+            --max-cpu-workers 8 \
+            --chunk-size 1024 \
+            --l1-align-bytes 16384 \
+            --eviction-trigger-watermark 0.85 \
+            --eviction-ratio 0.10 \
+            --eviction-policy LRU \
+            --supported-transfer-mode lmcache_driven \
+            > "$LMCACHE_SERVER_LOG" 2>&1 &
+        LMCACHE_SERVER_PID=$!
+        LMCACHE_READY=0
+        for _ in $(seq 1 60); do
+            if ! kill -0 "$LMCACHE_SERVER_PID" 2>/dev/null; then
+                echo "LMCache server died during startup." >&2
+                cat "$LMCACHE_SERVER_LOG" >&2
+                exit 1
+            fi
+            if curl --output /dev/null --silent --fail \
+                "http://127.0.0.1:$LMCACHE_HTTP_PORT/healthcheck"; then
+                LMCACHE_READY=1
+                break
+            fi
+            sleep 2
+        done
+        if [ "$LMCACHE_READY" -ne 1 ]; then
+            echo "LMCache server did not become healthy in time." >&2
+            cat "$LMCACHE_SERVER_LOG" >&2
+            exit 1
+        fi
+
+        unset VLLM_USE_SIMPLE_KV_OFFLOAD
+        OFFLOAD_ARGS=(
+            --kv-transfer-config
+            "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":$LMCACHE_MQ_TIMEOUT}}"
+        )
+        ;;
+    *)
+        echo "Error: unsupported KV_OFFLOAD_BACKEND '$KV_OFFLOAD_BACKEND' (expected one of: mooncake, lmcache)" >&2
+        exit 1
+        ;;
+    esac
 fi
 
 PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1)

@@ -1791,6 +1791,26 @@ dsv4-fp4-b200-vllm-agentic:
       # Retain the external-cache transition and peak-throughput region.
       - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [16, 38, 44, 56, 64, 66, 68], router: { name: vllm-router, version: "0.1.14" } }
 
+# LMCache CPU-offload arm of dsv4-fp4-b200-vllm-agentic, split into its own
+# config so it can be triggered/tested independently of the Mooncake curve.
+# Points mirror the Mooncake offload points; the GPU-cache (kv-offloading:
+# none) baselines live in the parent config only. Runs a tuned vLLM build
+# while the parent stays on the stock v0.23.0 image.
+dsv4-fp4-b200-vllm-agentic-lmcache:
+  image: inferactinc/public:sprint-agentx-fast-x86_64-cu13.0.1-6a1e7bf
+  model: deepseek-ai/DeepSeek-V4-Pro
+  model-prefix: dsv4
+  runner: cluster:b200-dgxc
+  precision: fp4
+  framework: vllm
+  multinode: false
+  scenarios:
+    agentic-coding:
+    - dram-utilization: 0.80
+      search-space:
+      - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.1" }, conc-list: [8, 10, 16] }
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.1" }, conc-list: [16, 38, 44, 56, 64, 66, 68] }
+
 dsv4-fp4-b200-trt:
   image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066
   model: deepseek-ai/DeepSeek-V4-Pro

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
@@ -4781,3 +4781,16 @@
     - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260708"
     - "Clean the export envs"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2198
+
+- config-keys:
+    - dsv4-fp4-b200-vllm-agentic-lmcache
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Add LMCache 0.5.1 DRAM KV-offload config (kv-offload-backend: lmcache) as a standalone section alongside the Mooncake parent (B300 section deferred)"
+    - "Image inferactinc/public:sprint-agentx-fast-x86_64-cu13.0.1-6a1e7bf for the LMCache section (parent stays on v0.23.0)"
+    - "LMCache MP server (lmcache_driven transfer mode) + LMCacheMPConnector; L1 pool derated to 75% of TOTAL_CPU_DRAM_GB after full-budget pinned pools OOM-killed nodes in bring-up"
+    - "Adopt a tuned DeepSeek-V4 serving recipe for the LMCache arm (AMXF4 Mega-MoE, FULL_DECODE_ONLY cudagraphs, sparse MLA with prefill query quantization, NUMA binding, v2 model runner + rust frontend; fastsafetensors on EP points only — non-EP ranks OOM staging full expert weights; no cumem allocator — cuMem/VMM memory cannot be CUDA-IPC-shared with the LMCache server)"
+    - "The image ships the vLLM PR #45406 scheduler fix for async KV loads, so no engine patch is applied"
+    - "LMCache points mirror the Mooncake offload points: B200 TP8 conc 8-16, TP8-DEP8 conc 16-68"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2153