Skip to content

[None][test] Remove GLM-5, Qwen3 and DSV4-Pro gen_only perf-sanity cases; add MiniMax-M3, Kimi-K3 and GLM-5.3-Flash cases #21832

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
chenfeiz0326 wants to merge 1 commit into
base: main
Choose a base branch
from
+528 −276
Open

[None][test] Remove GLM-5, Qwen3 and DSV4-Pro gen_only perf-sanity cases; add MiniMax-M3, Kimi-K3 and GLM-5.3-Flash cases #21832

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 0 additions & 11 deletions jenkins/L0_MergeRequest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2082,8 +2082,6 @@ def getMultiGpuFileChanged(pipeline, testFilter, globalVars)
"tensorrt_llm/serve/serving_extensions.py",
"tests/integration/defs/cpp/test_multi_gpu.py",
"tests/integration/defs/model_express/",
"tests/integration/test_lists/test-db/l0_b200_multi_gpus_perf_sanity.yml",
"tests/integration/test_lists/test-db/l0_b200_visual_gen_perf_sanity.yml",
"tests/integration/test_lists/test-db/l0_dgx_b200.yml",
"tests/integration/test_lists/test-db/l0_dgx_b300.yml",
"tests/integration/test_lists/test-db/l0_dgx_b300_efa_multi_nodes.yml",
Expand All @@ -2092,21 +2090,12 @@ def getMultiGpuFileChanged(pipeline, testFilter, globalVars)
"tests/scripts/perf-sanity/disaggregated/b300_qwen3_0.6b_efa_functional.yaml",
"tests/integration/test_lists/test-db/l0_dgx_h100.yml",
"tests/integration/test_lists/test-db/l0_dgx_h200.yml",
"tests/integration/test_lists/test-db/l0_dgx_h200_perf_sanity.yml",
"tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml",
"tests/integration/test_lists/test-db/l0_gb200_multi_gpus_perf_sanity.yml",
"tests/integration/test_lists/test-db/l0_gb200_multi_nodes.yml",
"tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml",
"tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node2_gpu8.yml",
"tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node2_gpu8_gen1_node8_gpu32.yml",
"tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_node2_gpu8.yml",
"tests/integration/test_lists/test-db/l0_gb300.yml",
"tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml",
"tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml",
"tests/integration/test_lists/test-db/l0_gb300_multi_nodes_node2_gpu8.yml",
"tests/integration/test_lists/test-db/l0_gb300_multi_nodes_node4_gpu16.yml",
"tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml",
"tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node8_gpu32.yml",
"tests/integration/test_lists/test-db/l0_model_express.yml",
"tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml",
"tests/unittest/_torch/multi_gpu/",
Expand Down
96 changes: 46 additions & 50 deletions jenkins/L0_Test.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7555,7 +7555,6 @@ def launchTestJobs(pipeline, testFilter, globalVars)
"H100_PCIe-FMHA-Post-Merge-1": ["h100-cr", "l0_h100", 1, 1],
"DGX_H200-8_GPUs-PyTorch-Post-Merge-1": ["dgx-h200-x8", "l0_dgx_h200", 1, 1, 8],
"DGX_H200-4_GPUs-PyTorch-Post-Merge-1": ["dgx-h200-x4", "l0_dgx_h200", 1, 1, 4],
"DGX_H200-8_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["dgx-h200-x8", "l0_dgx_h200_perf_sanity", 1, 1, 8],
// Disable RTXPro6000 stages due to nodes will be offline temporarily.
Comment thread
chenfeiz0326 marked this conversation as resolved.
Resolved
// [TODO] Split tests between RTXPro6000 and RTXPro6000D and move reasonable mount of tests to pre-merge.
// "RTXPro6000-PyTorch-Post-Merge-1": ["rtx-pro-6000", "l0_rtx_pro_6000", 1, 1],
Expand Down Expand Up @@ -7701,14 +7700,12 @@ def launchTestJobs(pipeline, testFilter, globalVars)
"GB300-4_GPUs-PyTorch-Post-Merge-2": ["auto:gb300-x4", "l0_gb300_multi_gpus", 2, 3, 4, 1, true, false],
"GB300-4_GPUs-PyTorch-Post-Merge-3": ["auto:gb300-x4", "l0_gb300_multi_gpus", 3, 3, 4, 1, true, false],
// PerfSanity post-merge tests
"GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 1, 1, 4],
"GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 1, 7, 4, 1, true, false],
"GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 2, 7, 4, 1, true, false],
"GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 3, 7, 4, 1, true, false],
"GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-4": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 4, 7, 4, 1, true, false],
"GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-5": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 5, 7, 4, 1, true, false],
"GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-6": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 6, 7, 4, 1, true, false],
"GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-7": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 7, 7, 4, 1, true, false],
"GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 1, 6, 4, 1, true, false],
"GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 2, 6, 4, 1, true, false],
"GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 3, 6, 4, 1, true, false],
"GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-4": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 4, 6, 4, 1, true, false],
"GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-5": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 5, 6, 4, 1, true, false],
"GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-6": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 6, 6, 4, 1, true, false],
]
SBSASlurmTestConfigs = cbtsResizeSplits(SBSASlurmTestConfigs)
fullSet += SBSASlurmTestConfigs.keySet()
Expand All @@ -7728,68 +7725,58 @@ def launchTestJobs(pipeline, testFilter, globalVars)
"GB300-16_GPUs-4_Nodes-PyTorch-SingleNvlinkDomain-Post-Merge-1": ["auto:gb300-flex", "l0_gb300_multi_nodes_node4_gpu16", 1, 1, 16, 4],
]
// PerfSanity post-merge aggregated
// 2 Nodes
// 4 Nodes: Kimi K3 MXFP4 modeling_v2, TP16 attention on 16 GPUs in one NVLink domain
multiNodesSBSAConfigs += buildStageConfigs(
"GB200-8_GPUs-2_Nodes-PyTorch-PerfSanity-Node2-GPU8-Post-Merge",
"GB200-16_GPUs-4_Nodes-PyTorch-PerfSanity-Node4-GPU16-Post-Merge",
Comment thread
chenfeiz0326 marked this conversation as resolved.
Resolved
"auto:gb200-flex",
"l0_gb200_multi_nodes_perf_sanity_node2_gpu8",
"l0_gb200_multi_nodes_perf_sanity_node4_gpu16",
2,
8,
2
16,
4
)
// PerfSanity post-merge disaggregated
// 2 Nodes
// GB300 PerfSanity post-merge disaggregated
// MiniMax-M3 150k1k con1 and con64: ctx1 (1 node, 2 GPUs) + gen1 tp4 (1 node, 4 GPUs) = 8 GPUs
multiNodesSBSAConfigs += buildStageConfigs(
"GB200-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU1-GEN1-NODE1-GPU4-Post-Merge",
"auto:gb200-flex",
"l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4",
"GB300-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU2-GEN1-NODE1-GPU4-Post-Merge",
"auto:gb300-flex",
"l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node1_gpu4",
2,
8,
2
)
// 3 Nodes
// MiniMax-M3 150k1k con128 (gen dep8) and con1 (gen tp8): ctx1 (1 node, 2 GPUs) + gen1 (2 nodes, 8 GPUs) = 12 GPUs
multiNodesSBSAConfigs += buildStageConfigs(
"GB200-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU1-GEN1-NODE2-GPU8-Post-Merge",
"auto:gb200-flex",
"l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node2_gpu8",
"GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU2-GEN1-NODE2-GPU8-Post-Merge",
"auto:gb300-flex",
"l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8",
2,
12,
3
)
// gen_only_no_context: gen1 (2 nodes, 8 GPUs), no ctx fleet = 8 GPUs
// Kimi-K3 160k1k con16: ctx16 (16 single-GPU DWDP ctx servers, 4 nodes) + gen1 helix8 (2 nodes, 8 GPUs) = 24 GPUs
multiNodesSBSAConfigs += buildStageConfigs(
"GB200-8_GPUs-2_Nodes-PyTorch-PerfSanity-GEN1-NODE2-GPU8-Post-Merge",
"auto:gb200-flex",
"l0_gb200_multi_nodes_perf_sanity_gen1_node2_gpu8",
1,
8,
2
)
// GB300 PerfSanity post-merge disaggregated
// GLM-5 (ctx DEP2): 3 Nodes
multiNodesSBSAConfigs += buildStageConfigs(
"GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU2-GEN1-NODE2-GPU8-Post-Merge",
"GB300-24_GPUs-6_Nodes-PyTorch-Disagg-PerfSanity-CTX16-NODE4-GPU16-GEN1-NODE2-GPU8-Post-Merge",
"auto:gb300-flex",
"l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8",
3,
12,
3
"l0_gb300_multi_nodes_perf_sanity_ctx16_node4_gpu16_gen1_node2_gpu8",
1,
24,
6
)
// 9 Nodes
// Kimi-K3 160k1k con32: ctx16 (16 single-GPU DWDP ctx servers, 4 nodes) + gen1 helix16 (4 nodes, 16 GPUs) = 32 GPUs
multiNodesSBSAConfigs += buildStageConfigs(
"GB300-36_GPUs-9_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU2-GEN1-NODE8-GPU32-Post-Merge",
"GB300-32_GPUs-8_Nodes-PyTorch-Disagg-PerfSanity-CTX16-NODE4-GPU16-GEN1-NODE4-GPU16-Post-Merge",
"auto:gb300-flex",
"l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node8_gpu32",
"l0_gb300_multi_nodes_perf_sanity_ctx16_node4_gpu16_gen1_node4_gpu16",
1,
36,
9
32,
8
)
// 9 Nodes: ctx1 (1 node, 4 GPUs) + gen4 (2 nodes, 8 GPUs each) = 36 GPUs
multiNodesSBSAConfigs += buildStageConfigs(
"GB300-36_GPUs-9_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN4-NODE2-GPU8-Post-Merge",
"auto:gb300-flex",
"l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8",
2,
1,
36,
9
)
Expand All @@ -7798,7 +7785,7 @@ def launchTestJobs(pipeline, testFilter, globalVars)
"GB300-40_GPUs-10_Nodes-PyTorch-Disagg-PerfSanity-CTX6-NODE1-GPU4-GEN1-NODE4-GPU16-Post-Merge",
"auto:gb300-flex",
"l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16",
2,
1,
40,
10
)
Expand All @@ -7807,7 +7794,7 @@ def launchTestJobs(pipeline, testFilter, globalVars)
"GB300-44_GPUs-11_Nodes-PyTorch-Disagg-PerfSanity-CTX3-NODE1-GPU4-GEN1-NODE8-GPU32-Post-Merge",
"auto:gb300-flex",
"l0_gb300_multi_nodes_perf_sanity_ctx3_node1_gpu4_gen1_node8_gpu32",
2,
1,
44,
11
)
Expand All @@ -7816,7 +7803,7 @@ def launchTestJobs(pipeline, testFilter, globalVars)
"GB300-56_GPUs-14_Nodes-PyTorch-Disagg-PerfSanity-CTX12-NODE1-GPU4-GEN1-NODE2-GPU8-Post-Merge",
"auto:gb300-flex",
"l0_gb300_multi_nodes_perf_sanity_ctx12_node1_gpu4_gen1_node2_gpu8",
2,
1,
56,
14
)
Expand All @@ -7825,7 +7812,7 @@ def launchTestJobs(pipeline, testFilter, globalVars)
"GB300-8_GPUs-2_Nodes-PyTorch-PerfSanity-GEN1-NODE2-GPU8-Post-Merge",
"auto:gb300-flex",
"l0_gb300_multi_nodes_perf_sanity_gen1_node2_gpu8",
6,
5,
8,
2
)
Expand All @@ -7838,12 +7825,21 @@ def launchTestJobs(pipeline, testFilter, globalVars)
16,
4
)
// Kimi-K3 160k1k ctx_only: the DWDP16 ctx side as one 16-GPU attention-DP server on 4 nodes
multiNodesSBSAConfigs += buildStageConfigs(
"GB300-16_GPUs-4_Nodes-PyTorch-PerfSanity-Node4-GPU16-Post-Merge",
"auto:gb300-flex",
"l0_gb300_multi_nodes_perf_sanity_node4_gpu16",
1,
16,
4
)
// gen_only_no_context: gen1 (8 nodes, 32 GPUs), no ctx fleet = 32 GPUs
multiNodesSBSAConfigs += buildStageConfigs(
"GB300-32_GPUs-8_Nodes-PyTorch-PerfSanity-GEN1-NODE8-GPU32-Post-Merge",
"auto:gb300-flex",
"l0_gb300_multi_nodes_perf_sanity_gen1_node8_gpu32",
2,
1,
32,
8
)
Expand Down
10 changes: 10 additions & 0 deletions jenkins/scripts/perf/disaggregated/dwdp_ctx_rank.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,20 @@
# hides from its rank-0 pytest) and kept as TLLM_DWDP_SAVED_<name>. pytest
# restores them for the child only.
#
# The ranks of a node share one container, and the step starts their pytest
# together. Only the node's first rank lets pytest pip-install the media deps
# (TRTLLM_AUTO_INSTALL_MEDIA_DEPS in tests/integration/defs/conftest.py):
# concurrent installs into one site-packages fail with OSError, and the failed
# rank takes the whole step down.
#
# Usage: bash dwdp_ctx_rank.sh pytest <args>...

set -euo pipefail

if [[ "${SLURM_LOCALID:-0}" != "0" ]]; then
export TRTLLM_AUTO_INSTALL_MEDIA_DEPS=0
fi

mpi_prefixes=(OMPI_ PMIX_ PMI_ SLURM_ MPI_ I_MPI_ HYDRA_ MPICH_ MV2_ CRAY_)
for var in $(compgen -e); do
for prefix in "${mpi_prefixes[@]}"; do
Expand Down
Loading
Loading