diff --git a/benchmarks/multi_node/dsv4_fp4_mi355x_atom-disagg.sh b/benchmarks/multi_node/dsv4_fp4_mi355x_atom-disagg.sh deleted file mode 100644 index d17d1a323..000000000 --- a/benchmarks/multi_node/dsv4_fp4_mi355x_atom-disagg.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml new file mode 100644 index 000000000..6d100edc2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml @@ -0,0 +1,124 @@ +# DeepSeek-V4-Pro 1P1D ATOM P/D serving on MI355X. Infera provides +# discovery and KV-aware routing; ATOM transfers the KV state through +# Mooncake. The custom benchmark preserves the legacy 8k/1k search space. + +name: "mi355x-atom-dsv4-disagg-1p1d-tp8-8k1k" + +model: + path: "hf:deepseek-ai/DeepSeek-V4-Pro" + container: "infera-atom-v0.1.1" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "rocm/infera:atom-v0.1.1" + frameworks: + atom: "0.1.4.dev113+g5837907f3" + infera: "0.0.0" + +slurm: + time_limit: "08:00:00" + +resources: + gpu_type: "mi355x" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +frontend: + type: infera + enable_multiple_frontends: false + env: + PYTHONPATH: "/atom-source:/infera-source" + args: + router-policy: kv-aware + +backend: + type: atom + connector: mooncake + # The production-green legacy sweep used Mooncake RDMA on rdma0-rdma7. + # The pinned ATOM source contains the ROCm local-device filtering fix and + # selects the per-GPU RDMA device; srt-slurm supplies the cluster GID index. + mooncake_protocol: rdma + enable_kv_events: true + prefill_environment: &worker_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONPATH: "/atom-source:/infera-source" + PYTHONUNBUFFERED: "1" + OMP_NUM_THREADS: "1" + ATOM_MOE_GU_ITLV: "1" + AITER_BF16_FP8_MOE_BOUND: "0" + decode_environment: *worker_environment + atom_config: + prefill: + kv_cache_dtype: fp8 + # Leave registration/runtime headroom after the initial 0.85 run reached + # server registration with less than 6 MiB free and failed a 6 MiB calloc. + gpu-memory-utilization: 0.84 + max-num-seqs: 256 + block-size: 16 + no-enable_prefix_caching: true + trust-remote-code: true + decode: + kv_cache_dtype: fp8 + gpu-memory-utilization: 0.84 + max-num-seqs: 128 + block-size: 16 + no-enable_prefix_caching: true + trust-remote-code: true + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 720 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + archive_runtime_logs() { + tar -C /logs -czf "${result_root}/runtime-logs.tar.gz" . 2>/dev/null || true + } + trap archive_runtime_logs EXIT + for concurrency in 4 8 16 32 64 128; do + num_prompts=$((concurrency * 10)) + if (( num_prompts < 16 )); then num_prompts=16; fi + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/completions \ + --model deepseek-ai/DeepSeek-V4-Pro \ + --tokenizer deepseek-ai/DeepSeek-V4-Pro \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --num-warmups "$((concurrency * 2))" \ + --num-prompts "${num_prompts}" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --trust-remote-code \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "dsv4-atom-mi355x-disagg-1p1d-tp8-isl8192-osl1024-c${concurrency}.json" + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml new file mode 100644 index 000000000..676dae777 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml @@ -0,0 +1,127 @@ +# DeepSeek-V4-Pro 2P1D ATOM P/D serving on MI355X. Each logical endpoint +# occupies one TP8 node; the two prefill endpoints and decode endpoint use +# data-parallel attention/TBO as in the legacy production configuration. + +name: "mi355x-atom-dsv4-disagg-2p1d-dpa-tp8-8k1k" + +model: + path: "hf:deepseek-ai/DeepSeek-V4-Pro" + container: "infera-atom-v0.1.1" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "rocm/infera:atom-v0.1.1" + frameworks: + atom: "0.1.4.dev113+g5837907f3" + infera: "0.0.0" + +slurm: + time_limit: "08:00:00" + +resources: + gpu_type: "mi355x" + gpus_per_node: 8 + prefill_nodes: 2 + decode_nodes: 1 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +frontend: + type: infera + enable_multiple_frontends: false + env: + PYTHONPATH: "/atom-source:/infera-source" + args: + router-policy: kv-aware + +backend: + type: atom + connector: mooncake + # Match the production-green legacy sweep: this high-throughput search uses + # Mooncake RDMA across rdma0-rdma7. TCP saturates during c256 and causes the + # decode endpoint to miss transfer deadlines even with connection pooling. + mooncake_protocol: rdma + enable_kv_events: true + prefill_environment: &worker_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONPATH: "/atom-source:/infera-source" + PYTHONUNBUFFERED: "1" + OMP_NUM_THREADS: "1" + ATOM_MOE_GU_ITLV: "1" + AITER_BF16_FP8_MOE_BOUND: "0" + GPU_MAX_HW_QUEUES: "5" + ATOM_CPU_AFFINITY: "1" + decode_environment: *worker_environment + atom_config: + prefill: + kv_cache_dtype: fp8 + gpu-memory-utilization: 0.85 + max-num-seqs: 256 + block-size: 16 + no-enable_prefix_caching: true + enable-dp-attention: true + enable-tbo: true + trust-remote-code: true + decode: + kv_cache_dtype: fp8 + gpu-memory-utilization: 0.85 + max-num-seqs: 2048 + block-size: 16 + no-enable_prefix_caching: true + enable-dp-attention: true + enable-tbo: true + trust-remote-code: true + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 720 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + archive_runtime_logs() { + tar -C /logs -czf "${result_root}/runtime-logs.tar.gz" . 2>/dev/null || true + } + trap archive_runtime_logs EXIT + for concurrency in 256 512 768 1024 2048; do + num_prompts=$((concurrency * 10)) + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/completions \ + --model deepseek-ai/DeepSeek-V4-Pro \ + --tokenizer deepseek-ai/DeepSeek-V4-Pro \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --num-warmups "$((concurrency * 2))" \ + --num-prompts "${num_prompts}" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --trust-remote-code \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "dsv4-atom-mi355x-disagg-2p1d-dpa-tp8-isl8192-osl1024-c${concurrency}.json" + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 15ec613e4..a9ea0162a 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1381,17 +1381,14 @@ dsr1-fp4-mi355x-sglang-disagg-mtp: - "DECODE_NODES=1" - "DECODE_MTP_SIZE=1" -# MiniMax-M3 MXFP8 MI355X recipe: -# https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 -# MXFP8 runs from TP=4 on gfx950; block size 128 is mandatory for MSA. dsv4-fp4-mi355x-atom-disagg: - image: rocm/atom-dev:nightly_202606101403 + image: rocm/infera:atom-v0.1.1 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 - runner: mi355x + runner: cluster:mi355x-amds precision: fp4 framework: atom-disagg - router: { name: atomesh, version: "087b82d9c1f630e79149ba37e6213257ec9a76f8" } + router: { name: infera, version: "0.1.1" } kv-p2p-transfer: mooncake multinode: true disagg: true @@ -1400,8 +1397,11 @@ dsv4-fp4-mi355x-atom-disagg: - isl: 8192 osl: 1024 search-space: - # 2P1D DPA+TP8 - - conc-list: [ 256, 512, 768, 1024, 2048 ] + # The custom srt-slurm recipes own the exact concurrency loops. Keeping + # one matrix row per topology avoids launching the complete loop once per + # concurrency while preserving every original measurement point. + # 2P1D DPA+TP8 + - conc-list: [ 1 ] prefill: num-worker: 2 tp: 8 @@ -1409,6 +1409,7 @@ dsv4-fp4-mi355x-atom-disagg: dp-attn: true additional-settings: - "PREFILL_NODES=2" + - "CONFIG_FILE=recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml" decode: num-worker: 1 tp: 8 @@ -1416,8 +1417,8 @@ dsv4-fp4-mi355x-atom-disagg: dp-attn: true additional-settings: - "DECODE_NODES=1" - # 1P1D TP8 - - conc-list: [ 4, 8, 16, 32, 64, 128 ] + # 1P1D TP8 + - conc-list: [ 1 ] prefill: num-worker: 1 tp: 8 @@ -1425,6 +1426,7 @@ dsv4-fp4-mi355x-atom-disagg: dp-attn: false additional-settings: - "PREFILL_NODES=1" + - "CONFIG_FILE=recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml" decode: num-worker: 1 tp: 8 @@ -1432,7 +1434,9 @@ dsv4-fp4-mi355x-atom-disagg: dp-attn: false additional-settings: - "DECODE_NODES=1" - # 1P1D TP8 +# MiniMax-M3 MXFP8 MI300X recipe: +# https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 +# MXFP8 runs from TP=4 on gfx942; block size 128 is mandatory for MSA. minimaxm3-fp8-mi300x-vllm-agentic: image: vllm/vllm-openai-rocm:nightly-04c2a8deac44fdb1ca3e2b5ec3e6bf16f3f6a914 model: MiniMaxAI/MiniMax-M3-MXFP8 diff --git a/configs/deprecated/amd-1k1k-master.yaml b/configs/deprecated/amd-1k1k-master.yaml index 3457698c3..d573e91be 100644 --- a/configs/deprecated/amd-1k1k-master.yaml +++ b/configs/deprecated/amd-1k1k-master.yaml @@ -1331,38 +1331,6 @@ dsr1-fp4-mi355x-sglang-disagg-mtp: - "DECODE_NODES=1" - "DECODE_MTP_SIZE=1" -dsv4-fp4-mi355x-atom-disagg: - image: rocm/atom-dev:nightly_202606101403 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: mi355x - precision: fp4 - framework: atom-disagg - router: { name: atomesh, version: "087b82d9c1f630e79149ba37e6213257ec9a76f8" } - kv-p2p-transfer: mooncake - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - conc-list: [ 4, 8, 16, 32, 64, 128, 256, 512, 1024 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # MiniMax-M3 MXFP8 MI355X recipe: # https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 # MXFP8 runs from TP=4 on gfx950; block size 128 is mandatory for MSA. diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 635670c08..cb72db136 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5807,3 +5807,14 @@ - "Overlay the exact upstream Infera KV-event decoder fix from AMD-AGI/Infera#114 so ATOM's tagged-map events populate the router cache view during pre-release hardware validation." - "Serialize shared Infera and ATOM source staging per checkout and publish completed clones atomically so concurrent aggregate and disaggregated validation jobs cannot corrupt one another." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2558 + +- config-keys: + - dsv4-fp4-mi355x-atom-disagg + description: + - "Port the DeepSeek-V4-Pro FP4 MI355X ATOM disaggregated 8k/1k submission from the configuration-specific AMD launcher to srt-slurm ATOM orchestration and Infera KV-aware routing." + - "Preserve all original search points: 1P1D TP8 at concurrency 4, 8, 16, 32, 64, and 128; 2P1D DPA+TP8 at concurrency 256, 512, 768, 1024, and 2048." + - "Use the unchanged InferenceX benchmark_serving.py through the custom benchmark contract and remove the exact legacy wrapper plus its obsolete deprecated 1k/1k entry." + - "Restore the production-proven Mooncake RDMA transport used by the all-green legacy 8k/1k sweep; the TCP bring-up path saturated at c256 and lost the decode worker despite connection pooling." + - "Lower only the 1P1D prefill/decode gpu-memory-utilization from 0.85 to 0.84 after both roles registered their Mooncake chunks with less than 6 MiB free and failed a 6 MiB runtime allocation; keep the already-healthy 2P1D DPA path at 0.85." + - "Relay KV events from every ATOM data-parallel rank through one worker-local Infera endpoint, close the custom benchmark concurrency loops, and use a shell function for reliable runtime-log archival." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2627 diff --git a/runners/launch_mi355x-amds-srt.sh b/runners/launch_mi355x-amds-srt.sh index 72ab623ae..2d541eddc 100755 --- a/runners/launch_mi355x-amds-srt.sh +++ b/runners/launch_mi355x-amds-srt.sh @@ -7,7 +7,7 @@ set -euo pipefail SRT_SLURM_REPOSITORY="https://github.com/SemiAnalysisAI/srt-slurm.git" SRT_SLURM_COMMIT="5ecfb13d1ba0960045482f1ef006312d8729d37a" INFERA_REPOSITORY="https://github.com/cquil11/Infera.git" -INFERA_COMMIT="8ed8f1728c745d4e91ba9eaa09ed81159aa57e41" +INFERA_COMMIT="503dcbd1b60229a5a508fd3da45994c56a2b40ec" ATOM_REPOSITORY="https://github.com/cquil11/ATOM.git" ATOM_COMMIT="2ab42bc2c64d1ad04f698c396da48473e71a6dbb" SLURM_PARTITION="compute" diff --git a/utils/test_mi355x_atom_srt_contract.py b/utils/test_mi355x_atom_srt_contract.py index 50a9e7f44..b4495f4af 100644 --- a/utils/test_mi355x_atom_srt_contract.py +++ b/utils/test_mi355x_atom_srt_contract.py @@ -75,7 +75,7 @@ def test_matrix_rows_route_only_through_the_atom_srt_launcher(): "CONFIG_FILE=recipes/atom/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml" ] assert "5ecfb13d1ba0960045482f1ef006312d8729d37a" in launcher - assert "8ed8f1728c745d4e91ba9eaa09ed81159aa57e41" in launcher + assert "503dcbd1b60229a5a508fd3da45994c56a2b40ec" in launcher assert "2ab42bc2c64d1ad04f698c396da48473e71a6dbb" in launcher assert "ensure_git_checkout()" in launcher assert 'exec {lock_fd}>"\\${target}.lock"' in launcher