diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh index 241f042ff..e74ddaed3 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh @@ -235,7 +235,7 @@ EOF wait_for_lmcache_ready() { { set +x; } 2>/dev/null - local attempts="${LMCACHE_READY_ATTEMPTS:-120}" + local attempts="${LMCACHE_READY_ATTEMPTS:-1800}" local tail_pid="" while [ ! -f "$LMCACHE_LOG" ]; do @@ -276,8 +276,7 @@ EOF git clone https://github.com/LMCache/LMCache.git cd LMCache - # https://github.com/LMCache/LMCache/pull/3853 - git checkout 9229067cec0b3a63bb8a39368d101db7ac0bc3c1 + git checkout v0.5.1 pip install -r requirements/build.txt pip install grpcio==1.78.0 CXX=hipcc BUILD_WITH_HIP=1 pip install -e . --no-build-isolation @@ -317,6 +316,9 @@ EOF LMCACHE_TX_MODE="lmcache_driven" echo "Starting LMCache MP server..." + # TEMP DEBUG: remove before merging + df -h /dev/shm + free -h LMCACHE_CMD=( lmcache server --host "$LMCACHE_HOST" @@ -330,6 +332,14 @@ EOF --max-workers "$LMCACHE_MAX_WORKERS" --eviction-policy LRU --supported-transfer-mode "$LMCACHE_TX_MODE" + --no-separate-object-group + --no-l1-use-lazy + # lmcache_driven transfer mode uses CPU SHM by default for the L1 + # pool (MixedMemoryAllocator's default shm-name); that pool is + # sized to the full --l1-size-gb and blows past the container's + # /dev/shm cap, SIGBUSing. --shm-name "" forces the pickle + # (non-SHM) path instead. + --shm-name "" ) printf '%q ' "${LMCACHE_CMD[@]}" > "$RESULT_DIR/lmcache_command.txt" printf '\n' >> "$RESULT_DIR/lmcache_command.txt" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 4260b43bf..4a999754c 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -2199,7 +2199,7 @@ qwen3.5-fp8-mi355x-sglang-agentic-hicache: # DP-attention on/off and EP=8. dsv4-fp4-mi355x-vllm-agentic: - image: vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa + image: vllm/vllm-openai-rocm:v0.25.1 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6fdff162d..f22d77438 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4927,3 +4927,13 @@ - "Expand mi355x-disagg runner pool from 3 to 9 nodes." - "Concurrency sweep: 1K1K and 8K1K both [ 8, 16, 32, 64, 128 ]." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2094 + +- config-keys: + - dsv4-fp4-mi355x-vllm-agentic + description: + - "Bump LMCache source build from pinned commit 9229067 to the v0.5.1 release tag (CXX=hipcc BUILD_WITH_HIP=1)" + - "Add --no-separate-object-group and --no-l1-use-lazy to the LMCache MP server launch command" + - "Bump default LMCACHE_READY_ATTEMPTS from 120 to 1800 (eager L1 pool allocation takes longer to become healthy)" + - "Bump image from vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa (rotated off Docker Hub retention, 404s) to the stable vllm/vllm-openai-rocm:v0.25.1 release tag" + - "Add --shm-name '' to the LMCache MP server launch command: lmcache_driven transfer mode's L1 pool defaults to CPU-SHM backing (sized to the full --l1-size-gb with --no-l1-use-lazy), exceeding the CI container's /dev/shm cap and SIGBUSing on startup. --shm-name '' forces the pickle (non-SHM) path" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2216 diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index e14c50c66..14c5d9124 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -733,6 +733,7 @@ def generate_full_sweep(args, all_config_data, runner_data): ), Fields.DISAGG.value: disagg, Fields.SCENARIO_TYPE.value: "agentic-coding", + Fields.RUN_EVAL.value: False, } if kv_offload_backend is not None: entry[Fields.KV_OFFLOAD_BACKEND.value] = kv_offload_backend @@ -766,6 +767,7 @@ def generate_full_sweep(args, all_config_data, runner_data): + (f"_spec-{spec_decoding}" if spec_decoding != "none" else "") ), Fields.SCENARIO_TYPE.value: "agentic-coding", + Fields.RUN_EVAL.value: False, } if kv_offload_backend is not None: entry[Fields.KV_OFFLOAD_BACKEND.value] = kv_offload_backend @@ -1033,6 +1035,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): ), Fields.DISAGG.value: disagg, Fields.SCENARIO_TYPE.value: "agentic-coding", + Fields.RUN_EVAL.value: False, } if kv_offload_backend is not None: entry[Fields.KV_OFFLOAD_BACKEND.value] = kv_offload_backend @@ -1065,6 +1068,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): + (f"_spec-{spec_decoding}" if spec_decoding != "none" else "") ), Fields.SCENARIO_TYPE.value: "agentic-coding", + Fields.RUN_EVAL.value: False, } if kv_offload_backend is not None: entry[Fields.KV_OFFLOAD_BACKEND.value] = kv_offload_backend diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 6b7c67128..103a20b3c 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -291,6 +291,8 @@ class SingleNodeAgenticMatrixEntry(BaseModel): duration: int = Field(alias=Fields.DURATION.value) exp_name: str = Field(alias=Fields.EXP_NAME.value) scenario_type: str = Field(alias=Fields.SCENARIO_TYPE.value) + run_eval: bool = Field(alias=Fields.RUN_EVAL.value) + eval_only: bool = Field(alias=Fields.EVAL_ONLY.value, default=False) @model_validator(mode='after') def validate_kv_offload_fields(self): @@ -329,6 +331,8 @@ class MultiNodeAgenticMatrixEntry(BaseModel): exp_name: str = Field(alias=Fields.EXP_NAME.value) disagg: bool scenario_type: str = Field(alias=Fields.SCENARIO_TYPE.value) + run_eval: bool = Field(alias=Fields.RUN_EVAL.value) + eval_only: bool = Field(alias=Fields.EVAL_ONLY.value, default=False) @model_validator(mode='after') def validate_worker_hardware_pair(self): @@ -889,8 +893,10 @@ class ChangelogMatrixEntry(BaseModel): ] = Field(default_factory=dict) multi_node: dict[str, list[Union[MultiNodeMatrixEntry, MultiNodeAgenticMatrixEntry]] ] = Field(default_factory=dict) - evals: list[SingleNodeMatrixEntry] = Field(default_factory=list) - multinode_evals: list[MultiNodeMatrixEntry] = Field(default_factory=list) + evals: list[Union[SingleNodeMatrixEntry, SingleNodeAgenticMatrixEntry] + ] = Field(default_factory=list) + multinode_evals: list[Union[MultiNodeMatrixEntry, MultiNodeAgenticMatrixEntry] + ] = Field(default_factory=list) changelog_metadata: ChangelogMetadata