Add GLM-5.1-FP8 GB300 multinode dynamo-sglang MTP benchmark / 新增 GLM-5.1-FP8 GB300 多节点 Dynamo SGLang MTP 基准测试

benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_3.yaml

@@ -0,0 +1,152 @@
+name: gb300-fp8-glm5-mtp_1k1k_hightpt_3
+
+model:
+  path: glm-5.1-fp8
+  container: "lmsysorg/sglang:v0.5.11-cu130"
+  precision: fp8
+
+resources:
+  gpu_type: gb300
+  gpus_per_node: 4
+  prefill_nodes: 6
+  prefill_workers: 6
+  decode_nodes: 12
+  decode_workers: 1
+frontend:
+  type: dynamo
+  enable_multiple_frontends: true
+  num_additional_frontends: 9
+dynamo:
+  version: 1.2.1
+
+backend:
+  prefill_environment:
+    TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
+    PYTHONUNBUFFERED: '1'
+    DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
+    SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
+    SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
+    SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
+    MC_TE_METRIC: 'true'
+    MC_FORCE_MNNVL: '1'
+    NCCL_MNNVL_ENABLE: '1'
+    NCCL_CUMEM_ENABLE: '1'
+    SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
+    SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
+    SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
+    DYN_REQUEST_PLANE: nats
+
+  decode_environment:
+    TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
+    PYTHONUNBUFFERED: '1'
+    DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
+    SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
+    SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
+    SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
+    MC_TE_METRIC: 'true'
+    MC_FORCE_MNNVL: '1'
+    NCCL_MNNVL_ENABLE: '1'
+    NCCL_CUMEM_ENABLE: '1'
+    SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
+    SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
+    SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
+    DYN_REQUEST_PLANE: nats
+      # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size).
+      # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024.
+    SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024'
+
+  sglang_config:
+    prefill:
+        # Model configuration
+      served-model-name: GLM-5.1-FP8
+      trust-remote-code: true
+      quantization: fp8
+      kv-cache-dtype: fp8_e4m3
+
+        # Disaggregation mode
+      disaggregation-mode: prefill
+      disaggregation-transfer-backend: nixl
+
+        # Size limits
+      max-running-requests: 256
+      cuda-graph-max-bs: 256
+      mem-fraction-static: 0.7
+      context-length: 9600
+      chunked-prefill-size: 32768
+      max-prefill-tokens: 8192
+
+        # Parallelism
+      tensor-parallel-size: 4
+      data-parallel-size: 4
+      expert-parallel-size: 1
+      enable-dp-attention: true
+      enable-dp-lm-head: true
+      load-balance-method: total_tokens
+
+        # Backend
+      nsa-decode-backend: trtllm
+      nsa-prefill-backend: trtllm
+      moe-runner-backend: flashinfer_trtllm
+
+        # Other flags
+      enable-flashinfer-allreduce-fusion: true
+      disable-radix-cache: true
+      weight-loader-prefetch-checkpoints: true
+      model-loader-extra-config: '{"enable_multithread_load": true}'
+
+    decode:
+        # Model configuration
+      served-model-name: GLM-5.1-FP8
+      trust-remote-code: true
+
+      quantization: fp8
+      kv-cache-dtype: fp8_e4m3
+
+        # Disaggregation mode
+      disaggregation-mode: decode
+      disaggregation-transfer-backend: nixl
+
+        # Memory and token limits
+      mem-fraction-static: 0.8
+      context-length: 9600
+
+        # Backend
+      nsa-decode-backend: trtllm
+      nsa-prefill-backend: trtllm
+        # moe-runner-backend: "cutedsl"
+
+        # Detokenizer
+      skip-tokenizer-init: true
+      stream-interval: 30
+
+        # Other flags
+      disable-radix-cache: true
+      weight-loader-prefetch-checkpoints: true
+      model-loader-extra-config: '{"enable_multithread_load": true}'
+      tensor-parallel-size: 48
+      expert-parallel-size: 48
+      data-parallel-size: 48
+      enable-dp-lm-head: true
+      enable-dp-attention: true
+      moe-dense-tp-size: 1
+      ep-num-redundant-experts: 32
+      ep-dispatch-algorithm: static
+      moe-a2a-backend: deepep
+      deepep-mode: low_latency
+      deepep-config: /configs/deepep_config.json
+      max-running-requests: 6144
+      cuda-graph-max-bs: 128
+      speculative-algorithm: "EAGLE"
+      speculative-num-steps: 2
+      speculative-eagle-topk: 1
+      speculative-num-draft-tokens: 3
+health_check:
+  max_attempts: 360
+  interval_seconds: 10
+
+benchmark:
+  type: sa-bench
+  req_rate: inf
+  isl: 1024
+  osl: 1024
+  concurrencies: '6500'

benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_4.yaml

@@ -0,0 +1,152 @@
+name: gb300-fp8-glm5-mtp_1k1k_hightpt_4
+
+model:
+  path: glm-5.1-fp8
+  container: "lmsysorg/sglang:v0.5.11-cu130"
+  precision: fp8
+
+resources:
+  gpu_type: gb300
+  gpus_per_node: 4
+  prefill_nodes: 4
+  prefill_workers: 4
+  decode_nodes: 14
+  decode_workers: 1
+frontend:
+  type: dynamo
+  enable_multiple_frontends: true
+  num_additional_frontends: 9
+dynamo:
+  version: 1.2.1
+
+backend:
+  prefill_environment:
+    TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
+    PYTHONUNBUFFERED: '1'
+    DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
+    SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
+    SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
+    SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
+    MC_TE_METRIC: 'true'
+    MC_FORCE_MNNVL: '1'
+    NCCL_MNNVL_ENABLE: '1'
+    NCCL_CUMEM_ENABLE: '1'
+    SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
+    SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
+    SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
+    DYN_REQUEST_PLANE: nats
+
+  decode_environment:
+    TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
+    PYTHONUNBUFFERED: '1'
+    DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
+    SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
+    SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
+    SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
+    MC_TE_METRIC: 'true'
+    MC_FORCE_MNNVL: '1'
+    NCCL_MNNVL_ENABLE: '1'
+    NCCL_CUMEM_ENABLE: '1'
+    SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
+    SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
+    SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
+    DYN_REQUEST_PLANE: nats
+      # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size).
+      # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024.
+    SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024'
+
+  sglang_config:
+    prefill:
+        # Model configuration
+      served-model-name: GLM-5.1-FP8
+      trust-remote-code: true
+      quantization: fp8
+      kv-cache-dtype: fp8_e4m3
+
+        # Disaggregation mode
+      disaggregation-mode: prefill
+      disaggregation-transfer-backend: nixl
+
+        # Size limits
+      max-running-requests: 256
+      cuda-graph-max-bs: 256
+      mem-fraction-static: 0.7
+      context-length: 9600
+      chunked-prefill-size: 32768
+      max-prefill-tokens: 8192
+
+        # Parallelism
+      tensor-parallel-size: 4
+      data-parallel-size: 4
+      expert-parallel-size: 1
+      enable-dp-attention: true
+      enable-dp-lm-head: true
+      load-balance-method: total_tokens
+
+        # Backend
+      nsa-decode-backend: trtllm
+      nsa-prefill-backend: trtllm
+      moe-runner-backend: flashinfer_trtllm
+
+        # Other flags
+      enable-flashinfer-allreduce-fusion: true
+      disable-radix-cache: true
+      weight-loader-prefetch-checkpoints: true
+      model-loader-extra-config: '{"enable_multithread_load": true}'
+
+    decode:
+        # Model configuration
+      served-model-name: GLM-5.1-FP8
+      trust-remote-code: true
+
+      quantization: fp8
+      kv-cache-dtype: fp8_e4m3
+
+        # Disaggregation mode
+      disaggregation-mode: decode
+      disaggregation-transfer-backend: nixl
+
+        # Memory and token limits
+      mem-fraction-static: 0.8
+      context-length: 9600
+
+        # Backend
+      nsa-decode-backend: trtllm
+      nsa-prefill-backend: trtllm
+        # moe-runner-backend: "cutedsl"
+
+        # Detokenizer
+      skip-tokenizer-init: true
+      stream-interval: 30
+
+        # Other flags
+      disable-radix-cache: true
+      weight-loader-prefetch-checkpoints: true
+      model-loader-extra-config: '{"enable_multithread_load": true}'
+      tensor-parallel-size: 56
+      expert-parallel-size: 56
+      data-parallel-size: 56
+      enable-dp-lm-head: true
+      enable-dp-attention: true
+      moe-dense-tp-size: 1
+      ep-num-redundant-experts: 24
+      ep-dispatch-algorithm: static
+      moe-a2a-backend: deepep
+      deepep-mode: low_latency
+      deepep-config: /configs/deepep_config.json
+      max-running-requests: 5600
+      cuda-graph-max-bs: 100
+      speculative-algorithm: "EAGLE"
+      speculative-num-steps: 2
+      speculative-eagle-topk: 1
+      speculative-num-draft-tokens: 3
+health_check:
+  max_attempts: 360
+  interval_seconds: 10
+
+benchmark:
+  type: sa-bench
+  req_rate: inf
+  isl: 1024
+  osl: 1024
+  concurrencies: '5700'