From b8856c1ecaae0a59c595d0a72a011dc81b05589f Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 9 Aug 2026 19:42:11 -0500 Subject: [PATCH 01/46] feat: add native static router frontends --- docs/config-reference.md | 19 ++- docs/sglang-router.md | 20 ++- src/srtctl/README.md | 2 +- src/srtctl/backends/sglang.py | 7 +- src/srtctl/backends/vllm.py | 27 ++- src/srtctl/benchmarks/router.py | 6 +- src/srtctl/cli/do_sweep.py | 2 +- src/srtctl/cli/mixins/benchmark_stage.py | 11 +- src/srtctl/core/config.py | 7 + src/srtctl/core/health.py | 16 +- src/srtctl/core/schema.py | 34 +++- src/srtctl/core/telemetry.py | 7 +- src/srtctl/frontends/__init__.py | 9 +- src/srtctl/frontends/base.py | 50 ++++-- src/srtctl/frontends/dynamo.py | 2 + src/srtctl/frontends/sglang.py | 172 +++---------------- src/srtctl/frontends/static_router.py | 175 +++++++++++++++++++ src/srtctl/frontends/trtllm_serve.py | 2 + src/srtctl/frontends/vllm.py | 2 + src/srtctl/frontends/vllm_router.py | 20 +++ tests/test_configs.py | 84 +++++++++ tests/test_frontend_topology.py | 29 +++- tests/test_static_router_frontends.py | 207 +++++++++++++++++++++++ 23 files changed, 694 insertions(+), 216 deletions(-) create mode 100644 src/srtctl/frontends/static_router.py create mode 100644 src/srtctl/frontends/vllm_router.py create mode 100644 tests/test_static_router_frontends.py diff --git a/docs/config-reference.md b/docs/config-reference.md index 2d751cf4c..e56dcbca6 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -270,7 +270,8 @@ Frontend/router configuration. ```yaml frontend: - # Frontend type: "dynamo" (default), "sglang", or "trtllm_serve" + # Frontend type: "dynamo" (default), "sgl-router", "vllm-router", + # "trtllm_serve", or direct "vllm". "sglang" is a compatibility alias. type: dynamo # Scaling @@ -290,20 +291,34 @@ frontend: # Environment variables for frontend processes env: MY_VAR: "value" + + # Optional router-specific image; defaults to model.container + container_image: "router-image" ``` | Field | Type | Default | Description | | --------------------------- | ---- | ------------- | ----------------------------------- | -| `type` | str | dynamo | Frontend type: "dynamo", "sglang", or "trtllm_serve" | +| `type` | str | dynamo | Frontend type: `dynamo`, `sgl-router`, `vllm-router`, `trtllm_serve`, or direct `vllm`; `sglang` is a compatibility alias | | `enable_multiple_frontends` | bool | true | Scale with nginx + multiple routers | | `num_additional_frontends` | int | 9 | Additional routers beyond master | | `nginx_container` | str | nginx:1.27.4 | Custom nginx container image | | `nginx_raise_ulimit` | bool | false | When true with nginx in use, run `ulimit -n 1048576` before nginx and emit `worker_rlimit_nofile 1048576` in generated `nginx.conf`. Off by default so restrictive clusters do not fail. Cluster `srtslurm.yaml` may set `nginx_raise_ulimit` for jobs that omit this field. | | `args` | dict | null | CLI args for the frontend | | `env` | dict | null | Env vars for frontend processes | +| `container_image` | str | null | Router process image; defaults to `model.container` | See [SGLang Router](sglang-router.md) for detailed architecture. +### vllm-router frontend + +`type: vllm-router` pairs with `backend.type: vllm` and launches the official +`vllm-router` process against direct private `vllm serve` endpoints. Aggregate +layouts use `--worker-urls`; disaggregated layouts use +`--vllm-pd-disaggregation` with the allocated prefill and decode leader URLs. +Each logical vLLM endpoint must currently fit on one node, but a job may scale +across many single-node aggregate, prefill, or decode endpoints. No NATS or etcd +infrastructure is started for this frontend. + ### trtllm_serve frontend `type: trtllm_serve` runs the `trtllm-serve disaggregated` orchestrator as the diff --git a/docs/sglang-router.md b/docs/sglang-router.md index b4a48b74b..2d2553fcb 100644 --- a/docs/sglang-router.md +++ b/docs/sglang-router.md @@ -1,6 +1,7 @@ # SGLang Router Mode -This page explains the sglang router mode for prefill-decode (PD) disaggregation, an alternative to the default Dynamo frontend architecture. +This page explains the first-class SGLang Model Gateway router mode for aggregate +and prefill-decode (PD) topologies, an alternative to the default Dynamo frontend. ## Table of Contents @@ -39,10 +40,13 @@ Enable sglang router in your recipe's `frontend` section: ```yaml frontend: - type: sglang + type: sgl-router ``` -That's it. The workers will launch with `sglang.launch_server` instead of `dynamo.sglang`, and the router will handle request distribution. +The legacy `type: sglang` spelling remains an exact compatibility alias. New +recipes should use `sgl-router`. Workers launch with `sglang.launch_server` +instead of `dynamo.sglang`, and the router receives only logical worker-leader +URLs from srtctl's allocated topology. ### Router Arguments @@ -50,7 +54,7 @@ Pass extra CLI args to the router: ```yaml frontend: - type: sglang + type: sgl-router args: kv-overlap-score-weight: 1 router-temperature: 0 @@ -74,7 +78,7 @@ Pass environment variables to frontend processes: ```yaml frontend: - type: sglang + type: sgl-router env: MY_CUSTOM_VAR: "value" ``` @@ -87,7 +91,7 @@ The simplest mode - one router on node 0, no nginx: ```yaml frontend: - type: sglang + type: sgl-router enable_multiple_frontends: false ``` @@ -111,7 +115,7 @@ Nginx load balances across multiple router instances: ```yaml frontend: - type: sglang + type: sgl-router enable_multiple_frontends: true # default num_additional_frontends: 9 # default, total = 1 + 9 = 10 routers ``` @@ -200,7 +204,7 @@ resources: decode_workers: 2 frontend: - type: sglang + type: sgl-router enable_multiple_frontends: true num_additional_frontends: 3 # 4 total routers diff --git a/src/srtctl/README.md b/src/srtctl/README.md index bad1da34c..f48a1941c 100644 --- a/src/srtctl/README.md +++ b/src/srtctl/README.md @@ -106,7 +106,7 @@ wait_for_model( port=8000, n_prefill=2, n_decode=4, - frontend_type="sglang", # or "dynamo" + frontend_type="sgl-router", # or "dynamo"; "sglang" remains an alias timeout=300, ) ``` diff --git a/src/srtctl/backends/sglang.py b/src/srtctl/backends/sglang.py index 22510ae98..a06057dc7 100644 --- a/src/srtctl/backends/sglang.py +++ b/src/srtctl/backends/sglang.py @@ -296,7 +296,8 @@ def build_worker_command( process: The process to start endpoint_processes: All processes for this endpoint (for multi-node) runtime: Runtime context with paths and settings - frontend_type: Frontend type - "sglang" uses sglang.launch_server, "dynamo" uses dynamo.sglang + frontend_type: Frontend type - "sglang"/"sgl-router" use + sglang.launch_server, while "dynamo" uses dynamo.sglang nsys_prefix: Optional nsys profiling command prefix dump_config_path: Path to dump config JSON """ @@ -320,7 +321,7 @@ def build_worker_command( dist_init_port = SGLANG_DIST_INIT_PORT_BASE # Choose Python module based on frontend type - use_sglang = frontend_type == "sglang" + use_sglang = frontend_type in {"sglang", "sgl-router"} python_module = "sglang.launch_server" if use_sglang else "dynamo.sglang" # Get served model name from config @@ -377,7 +378,7 @@ def build_worker_command( ) # Add config dump path (not when using sglang frontend) - if dump_config_path and frontend_type != "sglang": + if dump_config_path and frontend_type not in {"sglang", "sgl-router"}: cmd.extend(["--dump-config-to", str(dump_config_path)]) # Add kv-events-config if enabled for this mode and we have an allocated port diff --git a/src/srtctl/backends/vllm.py b/src/srtctl/backends/vllm.py index 0588df212..6dfda324b 100644 --- a/src/srtctl/backends/vllm.py +++ b/src/srtctl/backends/vllm.py @@ -498,13 +498,13 @@ def endpoints_to_processes( """Convert endpoints to processes. Dynamo DP+EP mode uses the configured per-GPU or per-node process layout. - For direct vLLM aggregate jobs, `vllm serve` manages local DP ranks from - one process, so keep the standard one-process-per-node topology. + For direct vLLM and vLLM Router jobs, `vllm serve` manages local DP ranks + from one process, so keep the standard one-process-per-node topology. For standard TP mode, creates one process per node. """ from srtctl.core.topology import NodePortAllocator, Process, endpoints_to_processes - if frontend_type == "vllm": + if frontend_type in {"vllm", "vllm-router"}: return endpoints_to_processes(endpoints, base_sys_port=base_sys_port, port_allocator=port_allocator) # Check if any endpoint uses DP mode @@ -675,7 +675,7 @@ def build_worker_command( process: The process to start endpoint_processes: All processes for this endpoint (for multi-node) runtime: Runtime context with paths and settings - frontend_type: Frontend type ("dynamo" or direct "vllm") + frontend_type: Frontend type ("dynamo", direct "vllm", or "vllm-router") nsys_prefix: Optional nsys profiling command prefix dump_config_path: Path to dump config JSON profiling: Profiling config; drives --profiler-config for iteration-based nsys @@ -714,17 +714,26 @@ def build_worker_command( } ) - if frontend_type == "vllm": - if mode != "agg": + if frontend_type in {"vllm", "vllm-router"}: + if frontend_type == "vllm" and mode != "agg": raise ValueError("frontend.type: vllm supports aggregate vLLM jobs only") if is_multi_node: - raise ValueError("frontend.type: vllm currently supports single-node aggregate jobs only") + raise ValueError(f"frontend.type: {frontend_type} requires each vLLM endpoint to fit on one node") config.pop("host", None) config.pop("port", None) - config.pop("connector", None) config.setdefault("served-model-name", served_model_name) + if frontend_type == "vllm": + config.pop("connector", None) + worker_port = runtime.frontend_port + else: + worker_port = process.http_port + mode_connector = config.pop("connector", None) + connector = mode_connector if mode_connector is not None else self.connector + if connector and connector not in ("null", "none", None): + config.setdefault("kv-transfer-config", _connector_to_kv_transfer_config(connector)) + cmd.extend( [ "vllm", @@ -733,7 +742,7 @@ def build_worker_command( "--host", "0.0.0.0", "--port", - str(runtime.frontend_port), + str(worker_port), ] ) if not self.set_cuda_visible_devices: diff --git a/src/srtctl/benchmarks/router.py b/src/srtctl/benchmarks/router.py index 5d2cfd930..27618af03 100644 --- a/src/srtctl/benchmarks/router.py +++ b/src/srtctl/benchmarks/router.py @@ -43,9 +43,9 @@ def local_script_dir(self) -> str: def validate_config(self, config: SrtConfig) -> list[str]: errors = [] - # Router benchmark requires sglang frontend - if config.frontend.type != "sglang": - errors.append("router benchmark requires frontend.type: sglang") + # Router benchmark exercises the SGLang router's prefix-aware policies. + if config.frontend.type not in {"sglang", "sgl-router"}: + errors.append("router benchmark requires frontend.type: sgl-router") return errors diff --git a/src/srtctl/cli/do_sweep.py b/src/srtctl/cli/do_sweep.py index 1f846e481..09c1d0fbd 100644 --- a/src/srtctl/cli/do_sweep.py +++ b/src/srtctl/cli/do_sweep.py @@ -677,7 +677,7 @@ def run(self) -> int: try: # Stage 1: Head infrastructure (NATS, etcd). Only the dynamo request # plane uses it; static/direct frontends skip it. - if self.config.frontend.type in {"trtllm_serve", "vllm"}: + if self.config.frontend.type in {"sglang", "sgl-router", "trtllm_serve", "vllm", "vllm-router"}: logger.info("Skipping NATS/etcd infrastructure (frontend.type=%s)", self.config.frontend.type) else: reporter.report(JobStatus.STARTING, JobStage.HEAD_INFRASTRUCTURE, "Starting head infrastructure") diff --git a/src/srtctl/cli/mixins/benchmark_stage.py b/src/srtctl/cli/mixins/benchmark_stage.py index 24710765c..244dab270 100644 --- a/src/srtctl/cli/mixins/benchmark_stage.py +++ b/src/srtctl/cli/mixins/benchmark_stage.py @@ -430,8 +430,8 @@ def _get_sa_bench_slow_down_env(self) -> dict[str, str]: "benchmark slow_down: slow_down_sleep_time and slow_down_wait_time must be positive; skipping" ) return {} - if self.config.frontend.type != "sglang": - logger.warning("benchmark.slow_down_* ignored: frontend.type is not sglang") + if self.config.frontend.type not in {"sglang", "sgl-router"}: + logger.warning("benchmark.slow_down_* ignored: frontend.type is not sgl-router") return {} decode_urls: list[str] = [] @@ -472,11 +472,12 @@ def _get_aiperf_server_metrics_env( logical_endpoints = self._logical_worker_endpoints() urls = [f"http://{host}:{port}/metrics" for _, host, port in logical_endpoints] else: - if self.config.frontend.type == "vllm": + if self.config.frontend.type in {"vllm", "vllm-router"}: for process in self.backend_processes: - if process.endpoint_mode == "agg" and process.is_leader: + if process.is_leader: host = get_hostname_ip(process.node, self.runtime.network_interface) - urls.append(f"http://{host}:{FRONTEND_PUBLIC_PORT}/metrics") + port = FRONTEND_PUBLIC_PORT if self.config.frontend.type == "vllm" else process.http_port + urls.append(f"http://{host}:{port}/metrics") if urls: return {"AIPERF_SERVER_METRICS_URLS": ",".join(sorted(set(urls)))} diff --git a/src/srtctl/core/config.py b/src/srtctl/core/config.py index 70db894e6..d6e404bea 100755 --- a/src/srtctl/core/config.py +++ b/src/srtctl/core/config.py @@ -171,6 +171,13 @@ def resolve_config_with_defaults(user_config: dict[str, Any], cluster_config: di config["frontend"] = frontend logger.debug(f"Resolved nginx_container alias '{nginx_container}' -> '{resolved_nginx}'") + router_container = frontend.get("container_image", "") + if containers and router_container in containers: + resolved_router = containers[router_container] + frontend["container_image"] = resolved_router + config["frontend"] = frontend + logger.debug(f"Resolved frontend.container_image alias '{router_container}' -> '{resolved_router}'") + # Cluster-level default for nginx nofile ulimit (job yaml wins if present). if "nginx_raise_ulimit" not in frontend and cluster_config.get("nginx_raise_ulimit") is not None: frontend["nginx_raise_ulimit"] = cluster_config["nginx_raise_ulimit"] diff --git a/src/srtctl/core/health.py b/src/srtctl/core/health.py index 35ab7498e..e0993a7cf 100644 --- a/src/srtctl/core/health.py +++ b/src/srtctl/core/health.py @@ -422,14 +422,18 @@ def wait_for_model( poll_interval: Seconds between health checks timeout: Maximum wait time in seconds report_every: Log progress every N seconds - frontend_type: Frontend type - "sglang" uses /workers, "dynamo" uses /health + frontend_type: Registered frontend type; its adapter selects and parses + the appropriate health endpoint. stop_event: Optional threading.Event to abort waiting Returns: True if model is ready with expected workers, False if timeout/aborted """ - if frontend_type == "sglang": - health_url = f"http://{host}:{port}/workers" + from srtctl.frontends import get_frontend + + frontend = get_frontend(frontend_type) + health_url = f"http://{host}:{port}{frontend.health_endpoint}" + if frontend.health_endpoint == "/workers": logger.info( "Polling %s every %.1fs for %d prefills and %d decodes (sglang frontend)", health_url, @@ -484,11 +488,7 @@ def wait_for_model( response_json = response.json() - # Check worker counts based on frontend type - if frontend_type == "sglang": - result = check_sglang_router_health(response_json, n_prefill, n_decode) - else: - result = check_dynamo_health(response_json, n_prefill, n_decode) + result = frontend.parse_health(response_json, n_prefill, n_decode) if result.ready: logger.info(result.message) diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py index f966ba3f6..f480223eb 100755 --- a/src/srtctl/core/schema.py +++ b/src/srtctl/core/schema.py @@ -1437,7 +1437,8 @@ class FrontendConfig: """Frontend/router configuration. Attributes: - type: Frontend type - "dynamo" (default), "sglang", "trtllm_serve", or "vllm" + type: Frontend type - "dynamo" (default), "sgl-router", "vllm-router", + "trtllm_serve", or direct "vllm". "sglang" remains a compatibility alias. enable_multiple_frontends: Scale with nginx + multiple routers. When ``True`` (default), srtctl stands up nginx and fans out to ``num_additional_frontends + 1`` router replicas. When @@ -1463,6 +1464,8 @@ class FrontendConfig: carry the session id in that header instead. args: CLI arguments passed to the frontend/router process env: Environment variables for frontend processes + container_image: Optional router-specific container image. Defaults to + the model/backend container when omitted. """ type: str = "dynamo" @@ -1475,6 +1478,7 @@ class FrontendConfig: nginx_keepalive_timeout: str = "600s" args: dict[str, Any] | None = None env: dict[str, str] | None = None + container_image: str | None = None # trtllm_serve orchestrator (ser.yaml) options; ignored by other frontends. ctx_router: dict[str, Any] | None = None # context_servers.router, e.g. {type: conversation} gen_router: dict[str, Any] | None = None # generation_servers.router @@ -1588,6 +1592,7 @@ def __post_init__(self): self._validate_het_jobs() self._validate_trtllm_serve() self._validate_vllm_frontend() + self._validate_static_router_frontend() def _validate_trtllm_serve(self): """Catch trtllm_serve misconfigurations at load time (dry-run) instead of @@ -1635,6 +1640,33 @@ def _validate_vllm_frontend(self): if (self.resources.agg_nodes or 1) != 1: raise ValidationError("frontend.type: vllm currently supports single-node aggregate jobs only") + def _validate_static_router_frontend(self): + """Validate native static-router/backend pairings and endpoint shape.""" + required_backend = { + "sglang": "sglang", + "sgl-router": "sglang", + "vllm-router": "vllm", + }.get(self.frontend.type) + if required_backend is None: + return + if self.backend_type != required_backend: + raise ValidationError( + f"frontend.type: {self.frontend.type} requires backend.type: {required_backend}; " + f"got {self.backend_type!r}" + ) + + if self.frontend.type == "vllm-router": + endpoint_gpu_counts = ( + self.resources.gpus_per_prefill if self.resources.num_prefill else 0, + self.resources.gpus_per_decode if self.resources.num_decode else 0, + self.resources.gpus_per_agg if self.resources.num_agg else 0, + ) + if any(count > self.resources.gpus_per_node for count in endpoint_gpu_counts): + raise ValidationError( + "frontend.type: vllm-router currently requires each logical vLLM endpoint " + "to fit on one node; scale with multiple aggregate/prefill/decode workers" + ) + def _validate_het_jobs(self): """When ``resources.het_jobs`` is set to True, enforce supported shape. diff --git a/src/srtctl/core/telemetry.py b/src/srtctl/core/telemetry.py index 12d75391f..79a8974d1 100644 --- a/src/srtctl/core/telemetry.py +++ b/src/srtctl/core/telemetry.py @@ -86,7 +86,12 @@ def generate_telemetry_config( for process in sorted(processes, key=lambda p: (p.endpoint_mode, p.endpoint_index, p.node_rank, p.node)): node_ip = get_hostname_ip(process.node, runtime.network_interface) - port = FRONTEND_PUBLIC_PORT if frontend_type == "vllm" and process.endpoint_mode == "agg" else process.sys_port + if frontend_type == "vllm" and process.endpoint_mode == "agg": + port = FRONTEND_PUBLIC_PORT + elif frontend_type == "vllm-router": + port = process.http_port + else: + port = process.sys_port node_metadata = { "hostname": process.node, "worker_index": str(process.endpoint_index), diff --git a/src/srtctl/frontends/__init__.py b/src/srtctl/frontends/__init__.py index f840b3249..06912a331 100644 --- a/src/srtctl/frontends/__init__.py +++ b/src/srtctl/frontends/__init__.py @@ -6,8 +6,10 @@ Supported frontend types: - dynamo: Dynamo frontend with NATS/etcd communication -- sglang: SGLang native router with direct worker connections +- sgl-router: SGLang Model Gateway with direct worker connections +- sglang: Backward-compatible alias for sgl-router - vllm: Direct vLLM OpenAI server for aggregate jobs +- vllm-router: vLLM Router with direct worker connections """ from srtctl.frontends.base import ( @@ -16,16 +18,19 @@ get_frontend, ) from srtctl.frontends.dynamo import DynamoFrontend -from srtctl.frontends.sglang import SGLangFrontend +from srtctl.frontends.sglang import SGLangFrontend, SGLRouterFrontend from srtctl.frontends.trtllm_serve import TRTLLMServeFrontend from srtctl.frontends.vllm import VLLMFrontend +from srtctl.frontends.vllm_router import VLLMRouterFrontend __all__ = [ "DynamoFrontend", "FrontendProtocol", "FrontendType", + "SGLRouterFrontend", "SGLangFrontend", "TRTLLMServeFrontend", "VLLMFrontend", + "VLLMRouterFrontend", "get_frontend", ] diff --git a/src/srtctl/frontends/base.py b/src/srtctl/frontends/base.py index b7540c940..9e9ad8706 100644 --- a/src/srtctl/frontends/base.py +++ b/src/srtctl/frontends/base.py @@ -11,7 +11,8 @@ """ import threading -from typing import TYPE_CHECKING, Any, Literal, Protocol +from collections.abc import Callable +from typing import TYPE_CHECKING, Any, Literal, Protocol, TypeVar if TYPE_CHECKING: from srtctl.core.health import WorkerHealthResult @@ -20,7 +21,29 @@ from srtctl.core.topology import Process # Supported frontend types - extensible by adding new literals -FrontendType = Literal["dynamo", "sglang", "trtllm_serve", "vllm"] +FrontendType = Literal["dynamo", "sglang", "sgl-router", "trtllm_serve", "vllm", "vllm-router"] + +FrontendFactory = Callable[[], "FrontendProtocol"] +_FRONTEND_REGISTRY: dict[str, FrontendFactory] = {} +_FrontendClass = TypeVar("_FrontendClass", bound=type) + + +def register_frontend(*names: str) -> Callable[[_FrontendClass], _FrontendClass]: + """Register a frontend implementation under one or more config names.""" + + def decorator(frontend_class: _FrontendClass) -> _FrontendClass: + for name in names: + if name in _FRONTEND_REGISTRY: + raise ValueError(f"Frontend type {name!r} is already registered") + _FRONTEND_REGISTRY[name] = frontend_class + return frontend_class + + return decorator + + +def _load_builtin_frontends() -> None: + """Import built-ins once so their registration decorators run.""" + from srtctl.frontends import dynamo, sglang, trtllm_serve, vllm, vllm_router # noqa: F401 class FrontendProtocol(Protocol): @@ -93,19 +116,10 @@ def get_frontend(frontend_type: str) -> FrontendProtocol: Raises: ValueError: If frontend type is unknown """ - # Import here to avoid circular imports - from srtctl.frontends.dynamo import DynamoFrontend - from srtctl.frontends.sglang import SGLangFrontend - from srtctl.frontends.trtllm_serve import TRTLLMServeFrontend - from srtctl.frontends.vllm import VLLMFrontend - - if frontend_type == "dynamo": - return DynamoFrontend() - elif frontend_type == "sglang": - return SGLangFrontend() - elif frontend_type == "trtllm_serve": - return TRTLLMServeFrontend() - elif frontend_type == "vllm": - return VLLMFrontend() - else: - raise ValueError(f"Unknown frontend type: {frontend_type!r}. Supported: dynamo, sglang, trtllm_serve, vllm") + _load_builtin_frontends() + try: + factory = _FRONTEND_REGISTRY[frontend_type] + except KeyError as exc: + supported = ", ".join(sorted(_FRONTEND_REGISTRY)) + raise ValueError(f"Unknown frontend type: {frontend_type!r}. Supported: {supported}") from exc + return factory() diff --git a/src/srtctl/frontends/dynamo.py b/src/srtctl/frontends/dynamo.py index 20821a7fe..ea2694b6e 100644 --- a/src/srtctl/frontends/dynamo.py +++ b/src/srtctl/frontends/dynamo.py @@ -15,6 +15,7 @@ from srtctl.core.health import WorkerHealthResult, check_dynamo_health from srtctl.core.schema import build_otel_env from srtctl.core.slurm import CONTAINER_REMAP_ROOT_EXPORT, start_srun_process +from srtctl.frontends.base import register_frontend from srtctl.ports import ETCD_CLIENT_PORT, NATS_PORT if TYPE_CHECKING: @@ -25,6 +26,7 @@ logger = logging.getLogger(__name__) +@register_frontend("dynamo") class DynamoFrontend: """Dynamo frontend implementation. diff --git a/src/srtctl/frontends/sglang.py b/src/srtctl/frontends/sglang.py index a6ec3d4ca..c210e0ffb 100644 --- a/src/srtctl/frontends/sglang.py +++ b/src/srtctl/frontends/sglang.py @@ -1,165 +1,37 @@ # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -""" -SGLang router frontend implementation. +"""SGLang Model Gateway router frontend.""" -Uses sglang_router for direct communication with backend workers. -""" +from typing import Any, ClassVar -import logging -import shlex -import threading -from typing import TYPE_CHECKING, Any - -from srtctl.core.health import WorkerHealthResult, check_sglang_router_health from srtctl.core.slurm import get_hostname_ip, start_srun_process +from srtctl.frontends.base import register_frontend +from srtctl.frontends.static_router import StaticRouterFrontend -if TYPE_CHECKING: - from srtctl.core.processes import ManagedProcess - from srtctl.core.runtime import RuntimeContext - from srtctl.core.topology import Process - -logger = logging.getLogger(__name__) - - -class SGLangFrontend: - """SGLang router frontend implementation. - - Uses sglang_router.launch_router for direct worker connections. - Health checks via /workers endpoint. - """ - - @property - def type(self) -> str: - return "sglang" - - @property - def health_endpoint(self) -> str: - return "/workers" - - def parse_health( - self, - response_json: dict, - expected_prefill: int, - expected_decode: int, - ) -> WorkerHealthResult: - """Parse sglang /workers endpoint response.""" - return check_sglang_router_health(response_json, expected_prefill, expected_decode) - - def get_frontend_args_list(self, args: dict[str, Any] | None) -> list[str]: - """Convert frontend args dict to CLI arguments.""" - if not args: - return [] - result = [] - for key, value in args.items(): - if value is True: - result.append(f"--{key}") - elif value is not False and value is not None: - result.extend([f"--{key}", str(value)]) - return result - - def start_frontends( - self, - topology: Any, # FrontendTopology - runtime: "RuntimeContext", - config: Any, # SrtConfig - backend: Any, # BackendProtocol - backend_processes: list["Process"], - stop_event: "threading.Event | None" = None, # unused: returns immediately - ) -> list["ManagedProcess"]: - """Start sglang routers on designated nodes. - - Supports two modes: - - Aggregated: --worker-urls http://w1:port1 http://w2:port2 ... - - Disaggregated: --pd-disaggregation --prefill url bootstrap_port --decode url - """ - from srtctl.backends.sglang import SGLangProtocol - from srtctl.core.processes import ManagedProcess - - r = config.resources - is_disaggregated = r.num_prefill > 0 or r.num_decode > 0 - - # Collect worker info by mode - agg_workers: list[tuple[str, int]] = [] # (ip, http_port) - prefill_leaders: list[tuple[str, int, int | None]] = [] # (ip, http_port, bootstrap_port) - decode_leaders: list[tuple[str, int]] = [] # (ip, http_port) - - # Determine URL schemes based on gRPC mode - prefill_scheme = "http://" - decode_scheme = "http://" - agg_scheme = "http://" - if isinstance(backend, SGLangProtocol): - if backend.is_grpc_mode("prefill"): - prefill_scheme = "grpc://" - if backend.is_grpc_mode("decode"): - decode_scheme = "grpc://" - if backend.is_grpc_mode("agg"): - agg_scheme = "grpc://" - - for process in backend_processes: - if not process.is_leader: - continue - leader_ip = get_hostname_ip(process.node) - if process.endpoint_mode == "agg": - agg_workers.append((leader_ip, process.http_port)) - elif process.endpoint_mode == "prefill": - prefill_leaders.append((leader_ip, process.http_port, process.bootstrap_port)) - elif process.endpoint_mode == "decode": - decode_leaders.append((leader_ip, process.http_port)) - - processes: list[ManagedProcess] = [] - - for idx, node in enumerate(topology.frontend_nodes): - logger.info("Starting sglang-router %d on %s", idx, node) - - router_log = runtime.log_dir / f"{node}_router_{idx}.out" - cmd = ["python", "-m", "sglang_router.launch_router"] +@register_frontend("sgl-router") +class SGLRouterFrontend(StaticRouterFrontend): + """First-class SGLang Model Gateway static router.""" - if is_disaggregated: - # Disaggregated mode: --pd-disaggregation with --prefill and --decode - cmd.append("--pd-disaggregation") - for ip, http_port, bootstrap_port in prefill_leaders: - cmd.extend(["--prefill", f"{prefill_scheme}{ip}:{http_port}"]) - # Add bootstrap port if available - if bootstrap_port is not None: - cmd.append(str(bootstrap_port)) - for ip, http_port in decode_leaders: - cmd.extend(["--decode", f"{decode_scheme}{ip}:{http_port}"]) - else: - # Aggregated mode: --worker-urls with space-separated URLs - worker_urls = [f"{agg_scheme}{ip}:{port}" for ip, port in agg_workers] - cmd.extend(["--worker-urls"] + worker_urls) + type: ClassVar[str] = "sgl-router" + backend_type: ClassVar[str] = "sglang" + executable: ClassVar[tuple[str, ...]] = ("python", "-m", "sglang_router.launch_router") + pd_flag: ClassVar[str] = "--pd-disaggregation" + process_name: ClassVar[str] = "sglang_router" - cmd.extend(["--host", "0.0.0.0", "--port", str(topology.frontend_port)]) - cmd.extend(self.get_frontend_args_list(config.frontend.args)) + def worker_scheme(self, backend: Any, mode: str) -> str: + return "grpc" if backend.is_grpc_mode(mode) else "http" - logger.info("Router command: %s", shlex.join(cmd)) + def get_hostname_ip(self, node: str) -> str: + return get_hostname_ip(node) - # Build env vars - env_to_set: dict[str, str] = {} - if config.frontend.env: - env_to_set.update(config.frontend.env) + def start_process(self, **kwargs: Any) -> Any: + return start_srun_process(**kwargs) - proc = start_srun_process( - command=cmd, - nodelist=[node], - output=str(router_log), - container_image=str(runtime.container_image), - container_mounts=runtime.container_mounts, - env_to_set=env_to_set if env_to_set else None, - het_group=runtime.nodes.het_group_for(node), - ) - processes.append( - ManagedProcess( - name=f"sglang_router_{idx}", - popen=proc, - log_file=router_log, - node=node, - critical=True, - ) - ) +@register_frontend("sglang") +class SGLangFrontend(SGLRouterFrontend): + """Backward-compatible alias for the historical ``sglang`` frontend type.""" - return processes + type: ClassVar[str] = "sglang" diff --git a/src/srtctl/frontends/static_router.py b/src/srtctl/frontends/static_router.py new file mode 100644 index 000000000..8c7192838 --- /dev/null +++ b/src/srtctl/frontends/static_router.py @@ -0,0 +1,175 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared implementation for routers configured with static worker URLs.""" + +from __future__ import annotations + +import logging +import shlex +import threading +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, ClassVar + +from srtctl.core.health import WorkerHealthResult, check_sglang_router_health +from srtctl.core.slurm import get_hostname_ip, start_srun_process + +if TYPE_CHECKING: + from srtctl.core.processes import ManagedProcess + from srtctl.core.runtime import RuntimeContext + from srtctl.core.topology import Process + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class RouterWorker: + """A logical backend endpoint exposed to a static router.""" + + mode: str + url: str + bootstrap_port: int | None = None + + +class StaticRouterFrontend: + """Base class for routers whose worker topology is supplied on the CLI.""" + + type: ClassVar[str] + backend_type: ClassVar[str] + executable: ClassVar[tuple[str, ...]] + pd_flag: ClassVar[str] + process_name: ClassVar[str] + + @property + def health_endpoint(self) -> str: + return "/workers" + + def parse_health( + self, + response_json: dict, + expected_prefill: int, + expected_decode: int, + ) -> WorkerHealthResult: + return check_sglang_router_health(response_json, expected_prefill, expected_decode) + + def get_frontend_args_list(self, args: dict[str, Any] | None) -> list[str]: + """Convert config values to CLI arguments, preserving repeated values.""" + if not args: + return [] + result: list[str] = [] + for key, value in args.items(): + flag = f"--{key}" + if value is True: + result.append(flag) + elif value is False or value is None: + continue + elif isinstance(value, list): + for item in value: + result.extend([flag, str(item)]) + else: + result.extend([flag, str(value)]) + return result + + def worker_scheme(self, backend: Any, mode: str) -> str: + """Return the protocol used to reach a worker endpoint.""" + return "http" + + def get_hostname_ip(self, node: str) -> str: + """Resolve a worker node to the address advertised to the router.""" + return get_hostname_ip(node) + + def start_process(self, **kwargs: Any) -> Any: + """Launch one router process. Split out for adapter-specific testing.""" + return start_srun_process(**kwargs) + + def collect_workers(self, backend: Any, backend_processes: list[Process]) -> list[RouterWorker]: + workers: list[RouterWorker] = [] + for process in backend_processes: + if not process.is_leader: + continue + scheme = self.worker_scheme(backend, process.endpoint_mode) + workers.append( + RouterWorker( + mode=process.endpoint_mode, + url=f"{scheme}://{self.get_hostname_ip(process.node)}:{process.http_port}", + bootstrap_port=process.bootstrap_port, + ) + ) + return workers + + def build_router_command(self, workers: list[RouterWorker], host: str, port: int) -> list[str]: + """Build the router CLI for aggregate or prefill/decode topologies.""" + aggregate = [worker for worker in workers if worker.mode == "agg"] + prefills = [worker for worker in workers if worker.mode == "prefill"] + decodes = [worker for worker in workers if worker.mode == "decode"] + + cmd = list(self.executable) + if prefills or decodes: + if aggregate: + raise ValueError("Static router topology cannot mix aggregate and disaggregated workers") + if not prefills or not decodes: + raise ValueError("Disaggregated static router topology requires prefill and decode workers") + cmd.append(self.pd_flag) + for worker in prefills: + cmd.extend(["--prefill", worker.url]) + if worker.bootstrap_port is not None: + cmd.append(str(worker.bootstrap_port)) + for worker in decodes: + cmd.extend(["--decode", worker.url]) + else: + if not aggregate: + raise ValueError("Static router topology has no logical workers") + cmd.extend(["--worker-urls", *(worker.url for worker in aggregate)]) + + cmd.extend(["--host", host, "--port", str(port)]) + return cmd + + def start_frontends( + self, + topology: Any, + runtime: RuntimeContext, + config: Any, + backend: Any, + backend_processes: list[Process], + stop_event: threading.Event | None = None, + ) -> list[ManagedProcess]: + del stop_event # static routers return immediately after launch + from srtctl.core.processes import ManagedProcess + + configured_backend = getattr(getattr(config, "backend", None), "type", self.backend_type) + if configured_backend != self.backend_type: + raise ValueError( + f"frontend.type: {self.type} requires backend.type: {self.backend_type} (got {configured_backend!r})" + ) + + workers = self.collect_workers(backend, backend_processes) + processes: list[ManagedProcess] = [] + for idx, node in enumerate(topology.frontend_nodes): + router_log = runtime.log_dir / f"{node}_{self.type}_{idx}.out" + cmd = self.build_router_command(workers, "0.0.0.0", topology.frontend_port) + cmd.extend(self.get_frontend_args_list(config.frontend.args)) + logger.info("Starting %s %d on %s: %s", self.type, idx, node, shlex.join(cmd)) + + container_image = getattr(config.frontend, "container_image", None) or str(runtime.container_image) + het_group_for = getattr(runtime.nodes, "het_group_for", lambda _node: None) + router_env = dict(getattr(runtime, "environment", {})) + router_env.update(config.frontend.env or {}) + proc = self.start_process( + command=cmd, + nodelist=[node], + output=str(router_log), + container_image=container_image, + container_mounts=runtime.container_mounts, + env_to_set=router_env or None, + het_group=het_group_for(node), + ) + processes.append( + ManagedProcess( + name=f"{self.process_name}_{idx}", + popen=proc, + log_file=router_log, + node=node, + critical=True, + ) + ) + return processes diff --git a/src/srtctl/frontends/trtllm_serve.py b/src/srtctl/frontends/trtllm_serve.py index f2e24e4c3..589cc0240 100644 --- a/src/srtctl/frontends/trtllm_serve.py +++ b/src/srtctl/frontends/trtllm_serve.py @@ -19,6 +19,7 @@ from srtctl.core.health import WorkerHealthResult, check_trtllm_serve_health, wait_for_health from srtctl.core.slurm import get_hostname_ip, start_srun_process +from srtctl.frontends.base import register_frontend if TYPE_CHECKING: from srtctl.core.processes import ManagedProcess @@ -28,6 +29,7 @@ logger = logging.getLogger(__name__) +@register_frontend("trtllm_serve") class TRTLLMServeFrontend: """trtllm-serve disaggregated frontend. diff --git a/src/srtctl/frontends/vllm.py b/src/srtctl/frontends/vllm.py index b3f817d75..c1af62669 100644 --- a/src/srtctl/frontends/vllm.py +++ b/src/srtctl/frontends/vllm.py @@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Any from srtctl.core.health import WorkerHealthResult +from srtctl.frontends.base import register_frontend if TYPE_CHECKING: from srtctl.core.processes import ManagedProcess @@ -24,6 +25,7 @@ logger = logging.getLogger(__name__) +@register_frontend("vllm") class VLLMFrontend: """Direct vLLM OpenAI server frontend. diff --git a/src/srtctl/frontends/vllm_router.py b/src/srtctl/frontends/vllm_router.py new file mode 100644 index 000000000..b46b2c0ef --- /dev/null +++ b/src/srtctl/frontends/vllm_router.py @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""vLLM Router frontend.""" + +from typing import ClassVar + +from srtctl.frontends.base import register_frontend +from srtctl.frontends.static_router import StaticRouterFrontend + + +@register_frontend("vllm-router") +class VLLMRouterFrontend(StaticRouterFrontend): + """Route requests to direct vLLM OpenAI-compatible worker endpoints.""" + + type: ClassVar[str] = "vllm-router" + backend_type: ClassVar[str] = "vllm" + executable: ClassVar[tuple[str, ...]] = ("vllm-router",) + pd_flag: ClassVar[str] = "--vllm-pd-disaggregation" + process_name: ClassVar[str] = "vllm_router" diff --git a/tests/test_configs.py b/tests/test_configs.py index 1443a1ae8..ef60849f5 100644 --- a/tests/test_configs.py +++ b/tests/test_configs.py @@ -815,6 +815,26 @@ def test_telemetry_container_aliases_resolve(self): assert resolved["telemetry"]["dcgm_exporter"]["container_image"] == "/path/to/dcgm.sqsh" assert resolved["telemetry"]["node_exporter"]["container_image"] == "/path/to/node.sqsh" + def test_router_container_alias_resolves(self): + from srtctl.core.config import resolve_config_with_defaults + + user_config = { + "name": "test", + "model": {"path": "/model", "container": "worker", "precision": "fp8"}, + "resources": {"gpu_type": "h100", "gpus_per_node": 8, "agg_nodes": 1}, + "frontend": {"type": "vllm-router", "container_image": "router"}, + } + cluster_config = { + "containers": { + "worker": "/path/to/worker.sqsh", + "router": "/path/to/router.sqsh", + } + } + + resolved = resolve_config_with_defaults(user_config, cluster_config) + + assert resolved["frontend"]["container_image"] == "/path/to/router.sqsh" + def test_telemetry_literal_paths_pass_through(self): from srtctl.core.config import resolve_config_with_defaults @@ -2118,6 +2138,70 @@ def test_direct_vllm_command_preserves_current_main_device_binding(self): assert "--request-plane" not in cmd assert "dynamo.vllm" not in cmd + def test_vllm_router_keeps_one_direct_server_per_logical_endpoint(self): + """vLLM Router uses direct private servers rather than Dynamo runtimes.""" + from srtctl.backends import VLLMProtocol + from srtctl.core.topology import Endpoint + + backend = VLLMProtocol() + endpoints = [ + Endpoint( + mode="agg", + index=index, + nodes=(node,), + gpu_indices=frozenset(range(8)), + gpus_per_node=8, + ) + for index, node in enumerate(("node0", "node1")) + ] + + processes = backend.endpoints_to_processes(endpoints, frontend_type="vllm-router") + + assert len(processes) == 2 + assert all(process.is_leader for process in processes) + assert len({process.http_port for process in processes}) == 1 # ports may repeat on distinct nodes + + def test_vllm_router_worker_uses_private_port_and_pd_connector(self): + """Disaggregated vLLM Router workers are direct servers with KV transfer.""" + from pathlib import Path + from unittest.mock import MagicMock, patch + + from srtctl.backends import VLLMProtocol, VLLMServerConfig + from srtctl.core.topology import Process + + backend = VLLMProtocol( + connector="nixl", + vllm_config=VLLMServerConfig(prefill={"tensor-parallel-size": 8}), + ) + process = Process( + node="node0", + gpu_indices=frozenset(range(8)), + sys_port=8081, + http_port=30123, + endpoint_mode="prefill", + endpoint_index=0, + node_rank=0, + bootstrap_port=30001, + ) + runtime = MagicMock() + runtime.model_path = Path("/model") + runtime.is_hf_model = False + runtime.frontend_port = 8000 + + with patch("srtctl.core.slurm.get_hostname_ip", return_value="10.0.0.1"): + cmd = backend.build_worker_command( + process=process, + endpoint_processes=[process], + runtime=runtime, + frontend_type="vllm-router", + ) + + assert cmd[:3] == ["vllm", "serve", "/model"] + assert cmd[cmd.index("--port") + 1] == "30123" + assert "dynamo.vllm" not in cmd + kv_config = json.loads(cmd[cmd.index("--kv-transfer-config") + 1]) + assert kv_config == {"kv_connector": "NixlConnector", "kv_role": "kv_both"} + def test_direct_vllm_command_keeps_iteration_profiler_config(self): """Direct vllm serve retains main's profiling-derived server option.""" from pathlib import Path diff --git a/tests/test_frontend_topology.py b/tests/test_frontend_topology.py index 73cc95047..ede820a83 100644 --- a/tests/test_frontend_topology.py +++ b/tests/test_frontend_topology.py @@ -4,6 +4,7 @@ """Tests for frontend topology logic (nginx + multiple frontends).""" from pathlib import Path +from types import SimpleNamespace from unittest.mock import MagicMock, patch from srtctl.cli.do_sweep import SweepOrchestrator @@ -285,8 +286,9 @@ def test_single_node_starts_one_dynamo_frontend(self, mock_mixin_srun, mock_dyna assert processes[0].node == "node0" @patch("srtctl.frontends.sglang.start_srun_process") + @patch("srtctl.frontends.sglang.get_hostname_ip", return_value="10.0.0.1") @patch("srtctl.cli.mixins.frontend_stage.start_srun_process") - def test_single_node_starts_one_sglang_router(self, mock_mixin_srun, mock_sglang_srun): + def test_single_node_starts_one_sglang_router(self, mock_mixin_srun, _mock_ip, mock_sglang_srun): """Single node starts one sglang router, no nginx.""" mock_mixin_srun.return_value = MagicMock() mock_sglang_srun.return_value = MagicMock() @@ -294,7 +296,16 @@ def test_single_node_starts_one_sglang_router(self, mock_mixin_srun, mock_sglang config = make_config(enable_multiple_frontends=True, frontend_type="sglang") runtime = make_runtime(["node0"]) orchestrator = SweepOrchestrator(config=config, runtime=runtime) - orchestrator._backend_processes = [] # No workers for this test + orchestrator.__dict__["backend_processes"] = [ + SimpleNamespace( + is_leader=True, + endpoint_mode=mode, + node="node0", + http_port=30000 + index, + bootstrap_port=30010 if mode == "prefill" else None, + ) + for index, mode in enumerate(("prefill", "decode")) + ] registry = MagicMock() processes = orchestrator.start_frontend(registry) @@ -389,8 +400,9 @@ def test_multi_node_nginx_ulimit_when_opt_in(self, mock_mixin_srun, mock_dynamo_ assert "ulimit -n 1048576" in nginx_cmd[2] @patch("srtctl.frontends.sglang.start_srun_process") + @patch("srtctl.frontends.sglang.get_hostname_ip", return_value="10.0.0.1") @patch("srtctl.cli.mixins.frontend_stage.start_srun_process") - def test_multi_node_sglang_with_nginx(self, mock_mixin_srun, mock_sglang_srun, tmp_path): + def test_multi_node_sglang_with_nginx(self, mock_mixin_srun, _mock_ip, mock_sglang_srun, tmp_path): """Multi-node with sglang router starts nginx + routers.""" mock_mixin_srun.return_value = MagicMock() mock_sglang_srun.return_value = MagicMock() @@ -413,7 +425,16 @@ def test_multi_node_sglang_with_nginx(self, mock_mixin_srun, mock_sglang_srun, t environment=runtime.environment, ) orchestrator = SweepOrchestrator(config=config, runtime=runtime) - orchestrator._backend_processes = [] + orchestrator.__dict__["backend_processes"] = [ + SimpleNamespace( + is_leader=True, + endpoint_mode=mode, + node="node0", + http_port=30000 + index, + bootstrap_port=30010 if mode == "prefill" else None, + ) + for index, mode in enumerate(("prefill", "decode")) + ] registry = MagicMock() processes = orchestrator.start_frontend(registry) diff --git a/tests/test_static_router_frontends.py b/tests/test_static_router_frontends.py new file mode 100644 index 000000000..42914832e --- /dev/null +++ b/tests/test_static_router_frontends.py @@ -0,0 +1,207 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for native static-router frontend adapters.""" + +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +from srtctl.frontends import SGLRouterFrontend, VLLMRouterFrontend, get_frontend +from srtctl.frontends.static_router import RouterWorker + + +def test_registry_exposes_explicit_router_names_and_legacy_alias() -> None: + assert isinstance(get_frontend("sgl-router"), SGLRouterFrontend) + assert get_frontend("sglang").type == "sglang" + assert isinstance(get_frontend("vllm-router"), VLLMRouterFrontend) + + +@pytest.mark.parametrize("frontend", [SGLRouterFrontend(), VLLMRouterFrontend()]) +def test_aggregate_command_advertises_all_logical_workers(frontend) -> None: + command = frontend.build_router_command( + [ + RouterWorker("agg", "http://10.0.0.1:30000"), + RouterWorker("agg", "http://10.0.0.2:30000"), + ], + "0.0.0.0", + 8000, + ) + + assert command[-4:] == ["--host", "0.0.0.0", "--port", "8000"] + worker_urls = command[command.index("--worker-urls") + 1 : -4] + assert worker_urls == ["http://10.0.0.1:30000", "http://10.0.0.2:30000"] + + +@pytest.mark.parametrize( + ("frontend", "pd_flag"), + [ + (SGLRouterFrontend(), "--pd-disaggregation"), + (VLLMRouterFrontend(), "--vllm-pd-disaggregation"), + ], +) +def test_disaggregated_command_preserves_modes_and_bootstrap(frontend, pd_flag: str) -> None: + command = frontend.build_router_command( + [ + RouterWorker("prefill", "http://10.0.0.1:30000", 30001), + RouterWorker("decode", "http://10.0.0.2:30000"), + ], + "0.0.0.0", + 8000, + ) + + assert pd_flag in command + assert command[command.index("--prefill") + 1 : command.index("--decode")] == [ + "http://10.0.0.1:30000", + "30001", + ] + assert command[command.index("--decode") + 1] == "http://10.0.0.2:30000" + + +def test_router_command_rejects_incomplete_or_mixed_topology() -> None: + frontend = VLLMRouterFrontend() + with pytest.raises(ValueError, match="requires prefill and decode"): + frontend.build_router_command([RouterWorker("prefill", "http://p:1")], "0.0.0.0", 8000) + with pytest.raises(ValueError, match="cannot mix"): + frontend.build_router_command( + [ + RouterWorker("agg", "http://a:1"), + RouterWorker("prefill", "http://p:1"), + RouterWorker("decode", "http://d:1"), + ], + "0.0.0.0", + 8000, + ) + + +def test_frontend_args_repeat_list_values() -> None: + frontend = VLLMRouterFrontend() + assert frontend.get_frontend_args_list({"routing-logic": ["round_robin", "session"]}) == [ + "--routing-logic", + "round_robin", + "--routing-logic", + "session", + ] + + +def test_vllm_router_launch_uses_router_container_env_and_only_leaders() -> None: + frontend = VLLMRouterFrontend() + runtime = SimpleNamespace( + log_dir=Path("/logs"), + container_image=Path("/worker.sqsh"), + container_mounts={"/host": "/container"}, + environment={"GLOBAL": "value", "ROUTER_LOG": "info"}, + nodes=SimpleNamespace(het_group_for=lambda node: 1), + ) + config = SimpleNamespace( + backend=SimpleNamespace(type="vllm"), + frontend=SimpleNamespace( + args={"routing-logic": "session"}, + env={"ROUTER_LOG": "debug"}, + container_image="docker://router:test", + ), + ) + topology = SimpleNamespace(frontend_nodes=["node0"], frontend_port=8180) + workers = [ + SimpleNamespace( + is_leader=True, + endpoint_mode="agg", + node="node1", + http_port=30000, + bootstrap_port=None, + ), + SimpleNamespace( + is_leader=False, + endpoint_mode="agg", + node="node2", + http_port=0, + bootstrap_port=None, + ), + ] + + with ( + patch.object(frontend, "get_hostname_ip", return_value="10.0.0.1"), + patch.object(frontend, "start_process", return_value=MagicMock()) as start, + ): + frontend.start_frontends(topology, runtime, config, MagicMock(), workers) + + kwargs = start.call_args.kwargs + assert kwargs["container_image"] == "docker://router:test" + assert kwargs["env_to_set"] == {"GLOBAL": "value", "ROUTER_LOG": "debug"} + assert kwargs["het_group"] == 1 + assert kwargs["command"].count("http://10.0.0.1:30000") == 1 + assert "--routing-logic" in kwargs["command"] + + +def test_router_rejects_backend_mismatch_before_launch() -> None: + frontend = VLLMRouterFrontend() + config = SimpleNamespace( + backend=SimpleNamespace(type="sglang"), + frontend=SimpleNamespace(args=None, env=None, container_image=None), + ) + topology = SimpleNamespace(frontend_nodes=["node0"], frontend_port=8180) + runtime = SimpleNamespace(log_dir=Path("/logs"), container_image=Path("/worker.sqsh")) + + with pytest.raises(ValueError, match="requires backend.type: vllm"): + frontend.start_frontends(topology, runtime, config, MagicMock(), []) + + +def test_schema_rejects_router_backend_mismatch() -> None: + from marshmallow import ValidationError + + from srtctl.backends import SGLangProtocol + from srtctl.core.schema import FrontendConfig, ResourceConfig, SrtConfig + + with pytest.raises(ValidationError, match="vllm-router requires backend.type: vllm"): + SrtConfig( + name="bad-router-pair", + model={"path": "model", "container": "image", "precision": "fp8"}, + resources=ResourceConfig(gpu_type="h100", gpus_per_node=8, agg_nodes=1, agg_workers=1), + frontend=FrontendConfig(type="vllm-router", enable_multiple_frontends=False), + backend=SGLangProtocol(), + ) + + +def test_vllm_router_accepts_many_single_node_endpoints() -> None: + from srtctl.backends import VLLMProtocol + from srtctl.core.schema import FrontendConfig, ResourceConfig, SrtConfig + + config = SrtConfig( + name="multi-endpoint-router", + model={"path": "model", "container": "image", "precision": "fp8"}, + resources=ResourceConfig( + gpu_type="h100", + gpus_per_node=8, + agg_nodes=4, + agg_workers=4, + ), + frontend=FrontendConfig(type="vllm-router", enable_multiple_frontends=False), + backend=VLLMProtocol(), + ) + + assert config.resources.gpus_per_agg == 8 + + +def test_vllm_router_rejects_endpoint_spanning_nodes() -> None: + from marshmallow import ValidationError + + from srtctl.backends import VLLMProtocol + from srtctl.core.schema import FrontendConfig, ResourceConfig, SrtConfig + + with pytest.raises(ValidationError, match="each logical vLLM endpoint"): + SrtConfig( + name="multi-node-endpoint", + model={"path": "model", "container": "image", "precision": "fp8"}, + resources=ResourceConfig( + gpu_type="h100", + gpus_per_node=8, + prefill_nodes=2, + prefill_workers=1, + decode_nodes=1, + decode_workers=1, + ), + frontend=FrontendConfig(type="vllm-router", enable_multiple_frontends=False), + backend=VLLMProtocol(), + ) From a888beaf13c55f2b4336bb269c9ca2a3e1e622e7 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 9 Aug 2026 19:45:52 -0500 Subject: [PATCH 02/46] fix: run setup scripts inside router containers --- src/srtctl/frontends/base.py | 17 +++++++++++++++++ src/srtctl/frontends/dynamo.py | 18 ++++-------------- src/srtctl/frontends/static_router.py | 2 ++ tests/test_static_router_frontends.py | 2 ++ 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/srtctl/frontends/base.py b/src/srtctl/frontends/base.py index 9e9ad8706..c808f4918 100644 --- a/src/srtctl/frontends/base.py +++ b/src/srtctl/frontends/base.py @@ -10,6 +10,7 @@ - Building CLI arguments from config """ +import shlex import threading from collections.abc import Callable from typing import TYPE_CHECKING, Any, Literal, Protocol, TypeVar @@ -46,6 +47,22 @@ def _load_builtin_frontends() -> None: from srtctl.frontends import dynamo, sglang, trtllm_serve, vllm, vllm_router # noqa: F401 +def build_setup_script_preamble(setup_script: str | None) -> str | None: + """Build the standard in-container recipe setup-script invocation.""" + if not setup_script: + return None + script_name = shlex.quote(setup_script) + return ( + f"setup_script={script_name} && " + 'script_path="/configs/${setup_script}" && ' + 'patch_script_path="/configs/patches/${setup_script}" && ' + 'echo "Running setup script: ${script_path} (fallback ${patch_script_path})" && ' + 'if [ -f "${script_path}" ]; then bash "${script_path}"; ' + 'elif [ -f "${patch_script_path}" ]; then bash "${patch_script_path}"; ' + 'else echo "WARNING: ${script_path} or ${patch_script_path} not found"; fi' + ) + + class FrontendProtocol(Protocol): """Protocol that all frontend implementations must implement. diff --git a/src/srtctl/frontends/dynamo.py b/src/srtctl/frontends/dynamo.py index ea2694b6e..d6cfeacc9 100644 --- a/src/srtctl/frontends/dynamo.py +++ b/src/srtctl/frontends/dynamo.py @@ -8,14 +8,13 @@ """ import logging -import shlex import threading from typing import TYPE_CHECKING, Any from srtctl.core.health import WorkerHealthResult, check_dynamo_health from srtctl.core.schema import build_otel_env from srtctl.core.slurm import CONTAINER_REMAP_ROOT_EXPORT, start_srun_process -from srtctl.frontends.base import register_frontend +from srtctl.frontends.base import build_setup_script_preamble, register_frontend from srtctl.ports import ETCD_CLIENT_PORT, NATS_PORT if TYPE_CHECKING: @@ -141,18 +140,9 @@ def _build_preamble(self, config: Any) -> str | None: parts = [] # Custom setup script - setup_script = getattr(config, "setup_script", None) - if isinstance(setup_script, str) and setup_script: - script_name = shlex.quote(setup_script) - parts.append( - f"setup_script={script_name} && " - 'script_path="/configs/${setup_script}" && ' - 'patch_script_path="/configs/patches/${setup_script}" && ' - 'echo "Running setup script: ${script_path} (fallback ${patch_script_path})" && ' - 'if [ -f "${script_path}" ]; then bash "${script_path}"; ' - 'elif [ -f "${patch_script_path}" ]; then bash "${patch_script_path}"; ' - 'else echo "WARNING: ${script_path} or ${patch_script_path} not found"; fi' - ) + setup_preamble = build_setup_script_preamble(getattr(config, "setup_script", None)) + if setup_preamble: + parts.append(setup_preamble) # Dynamo installation (required for dynamo frontend) # Skip if dynamo.install is False (container already has dynamo installed) diff --git a/src/srtctl/frontends/static_router.py b/src/srtctl/frontends/static_router.py index 8c7192838..de3abf93c 100644 --- a/src/srtctl/frontends/static_router.py +++ b/src/srtctl/frontends/static_router.py @@ -13,6 +13,7 @@ from srtctl.core.health import WorkerHealthResult, check_sglang_router_health from srtctl.core.slurm import get_hostname_ip, start_srun_process +from srtctl.frontends.base import build_setup_script_preamble if TYPE_CHECKING: from srtctl.core.processes import ManagedProcess @@ -161,6 +162,7 @@ def start_frontends( container_image=container_image, container_mounts=runtime.container_mounts, env_to_set=router_env or None, + bash_preamble=build_setup_script_preamble(getattr(config, "setup_script", None)), het_group=het_group_for(node), ) processes.append( diff --git a/tests/test_static_router_frontends.py b/tests/test_static_router_frontends.py index 42914832e..87f4fb3fc 100644 --- a/tests/test_static_router_frontends.py +++ b/tests/test_static_router_frontends.py @@ -102,6 +102,7 @@ def test_vllm_router_launch_uses_router_container_env_and_only_leaders() -> None env={"ROUTER_LOG": "debug"}, container_image="docker://router:test", ), + setup_script="router-deps.sh", ) topology = SimpleNamespace(frontend_nodes=["node0"], frontend_port=8180) workers = [ @@ -131,6 +132,7 @@ def test_vllm_router_launch_uses_router_container_env_and_only_leaders() -> None assert kwargs["container_image"] == "docker://router:test" assert kwargs["env_to_set"] == {"GLOBAL": "value", "ROUTER_LOG": "debug"} assert kwargs["het_group"] == 1 + assert "/configs/${setup_script}" in kwargs["bash_preamble"] assert kwargs["command"].count("http://10.0.0.1:30000") == 1 assert "--routing-logic" in kwargs["command"] From fedc84413c1515191e5adb127fdf34f11440f239 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 9 Aug 2026 19:51:52 -0500 Subject: [PATCH 03/46] fix: advertise vLLM NIXL bootstrap ports --- src/srtctl/frontends/static_router.py | 6 +++++- src/srtctl/frontends/vllm_router.py | 11 ++++++++++- tests/test_static_router_frontends.py | 19 +++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/srtctl/frontends/static_router.py b/src/srtctl/frontends/static_router.py index de3abf93c..34c1209a0 100644 --- a/src/srtctl/frontends/static_router.py +++ b/src/srtctl/frontends/static_router.py @@ -79,6 +79,10 @@ def get_hostname_ip(self, node: str) -> str: """Resolve a worker node to the address advertised to the router.""" return get_hostname_ip(node) + def worker_bootstrap_port(self, backend: Any, process: Process) -> int | None: + """Return the optional P/D bootstrap port advertised for a worker.""" + return process.bootstrap_port + def start_process(self, **kwargs: Any) -> Any: """Launch one router process. Split out for adapter-specific testing.""" return start_srun_process(**kwargs) @@ -93,7 +97,7 @@ def collect_workers(self, backend: Any, backend_processes: list[Process]) -> lis RouterWorker( mode=process.endpoint_mode, url=f"{scheme}://{self.get_hostname_ip(process.node)}:{process.http_port}", - bootstrap_port=process.bootstrap_port, + bootstrap_port=self.worker_bootstrap_port(backend, process), ) ) return workers diff --git a/src/srtctl/frontends/vllm_router.py b/src/srtctl/frontends/vllm_router.py index b46b2c0ef..36f689c5e 100644 --- a/src/srtctl/frontends/vllm_router.py +++ b/src/srtctl/frontends/vllm_router.py @@ -3,11 +3,16 @@ """vLLM Router frontend.""" -from typing import ClassVar +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, ClassVar from srtctl.frontends.base import register_frontend from srtctl.frontends.static_router import StaticRouterFrontend +if TYPE_CHECKING: + from srtctl.core.topology import Process + @register_frontend("vllm-router") class VLLMRouterFrontend(StaticRouterFrontend): @@ -18,3 +23,7 @@ class VLLMRouterFrontend(StaticRouterFrontend): executable: ClassVar[tuple[str, ...]] = ("vllm-router",) pd_flag: ClassVar[str] = "--vllm-pd-disaggregation" process_name: ClassVar[str] = "vllm_router" + + def worker_bootstrap_port(self, backend: Any, process: Process) -> int | None: + """Advertise vLLM's NIXL side-channel port to the P/D router.""" + return process.nixl_port diff --git a/tests/test_static_router_frontends.py b/tests/test_static_router_frontends.py index 87f4fb3fc..abee10407 100644 --- a/tests/test_static_router_frontends.py +++ b/tests/test_static_router_frontends.py @@ -86,6 +86,23 @@ def test_frontend_args_repeat_list_values() -> None: ] +def test_vllm_router_advertises_nixl_side_channel_port() -> None: + frontend = VLLMRouterFrontend() + process = SimpleNamespace( + is_leader=True, + endpoint_mode="prefill", + node="node1", + http_port=30000, + bootstrap_port=12000, + nixl_port=13000, + ) + + with patch.object(frontend, "get_hostname_ip", return_value="10.0.0.1"): + workers = frontend.collect_workers(MagicMock(), [process]) + + assert workers == [RouterWorker("prefill", "http://10.0.0.1:30000", 13000)] + + def test_vllm_router_launch_uses_router_container_env_and_only_leaders() -> None: frontend = VLLMRouterFrontend() runtime = SimpleNamespace( @@ -112,6 +129,7 @@ def test_vllm_router_launch_uses_router_container_env_and_only_leaders() -> None node="node1", http_port=30000, bootstrap_port=None, + nixl_port=None, ), SimpleNamespace( is_leader=False, @@ -119,6 +137,7 @@ def test_vllm_router_launch_uses_router_container_env_and_only_leaders() -> None node="node2", http_port=0, bootstrap_port=None, + nixl_port=None, ), ] From c328cbc58d11b8e76d2b0bf45e63a1325f0adcec Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 9 Aug 2026 20:09:44 -0500 Subject: [PATCH 04/46] validate SGLang tensor and data parallelism --- src/srtctl/core/schema.py | 31 +++++++++++++++++++++ tests/test_static_router_frontends.py | 39 +++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py index f480223eb..6fe7ac0d1 100755 --- a/src/srtctl/core/schema.py +++ b/src/srtctl/core/schema.py @@ -1593,6 +1593,7 @@ def __post_init__(self): self._validate_trtllm_serve() self._validate_vllm_frontend() self._validate_static_router_frontend() + self._validate_sglang_data_parallelism() def _validate_trtllm_serve(self): """Catch trtllm_serve misconfigurations at load time (dry-run) instead of @@ -1667,6 +1668,36 @@ def _validate_static_router_frontend(self): "to fit on one node; scale with multiple aggregate/prefill/decode workers" ) + def _validate_sglang_data_parallelism(self): + """Reject SGLang TP/DP combinations that the server cannot initialize. + + SGLang partitions each tensor-parallel group across its data-parallel + attention ranks, so ``tp_size`` must be divisible by ``dp_size``. Its + CLI otherwise accepts the flags and fails later in ``ServerArgs`` after + the Slurm allocation and container have already started. + """ + if self.backend_type != "sglang": + return + + sglang_cfg = getattr(self.backend, "sglang_config", None) + if sglang_cfg is None: + return + + for mode, mode_cfg in ( + ("prefill", sglang_cfg.prefill), + ("decode", sglang_cfg.decode), + ("aggregated", sglang_cfg.aggregated), + ): + if not mode_cfg: + continue + tp_size = int(mode_cfg.get("tp-size", mode_cfg.get("tp_size", 1))) + dp_size = int(mode_cfg.get("dp-size", mode_cfg.get("dp_size", 1))) + if tp_size % dp_size != 0: + raise ValidationError( + f"sglang_config.{mode}: tp-size={tp_size} must be divisible by " + f"dp-size={dp_size}; SGLang rejects this data-parallel layout" + ) + def _validate_het_jobs(self): """When ``resources.het_jobs`` is set to True, enforce supported shape. diff --git a/tests/test_static_router_frontends.py b/tests/test_static_router_frontends.py index abee10407..1b4f04cd8 100644 --- a/tests/test_static_router_frontends.py +++ b/tests/test_static_router_frontends.py @@ -226,3 +226,42 @@ def test_vllm_router_rejects_endpoint_spanning_nodes() -> None: frontend=FrontendConfig(type="vllm-router", enable_multiple_frontends=False), backend=VLLMProtocol(), ) + + +def test_sgl_router_rejects_non_divisible_tp_dp_layout() -> None: + from marshmallow import ValidationError + + from srtctl.backends import SGLangProtocol, SGLangServerConfig + from srtctl.core.schema import FrontendConfig, ResourceConfig, SrtConfig + + with pytest.raises(ValidationError, match="tp-size=1 must be divisible by dp-size=8"): + SrtConfig( + name="invalid-sglang-dpa", + model={"path": "model", "container": "image", "precision": "fp8"}, + resources=ResourceConfig(gpu_type="h100", gpus_per_node=8, agg_nodes=1, agg_workers=1), + frontend=FrontendConfig(type="sgl-router", enable_multiple_frontends=False), + backend=SGLangProtocol( + sglang_config=SGLangServerConfig( + aggregated={"tp-size": 1, "dp-size": 8, "enable-dp-attention": True} + ) + ), + ) + + +def test_sgl_router_accepts_divisible_tp_dp_layout() -> None: + from srtctl.backends import SGLangProtocol, SGLangServerConfig + from srtctl.core.schema import FrontendConfig, ResourceConfig, SrtConfig + + config = SrtConfig( + name="valid-sglang-dpa", + model={"path": "model", "container": "image", "precision": "fp8"}, + resources=ResourceConfig(gpu_type="h100", gpus_per_node=8, agg_nodes=1, agg_workers=1), + frontend=FrontendConfig(type="sgl-router", enable_multiple_frontends=False), + backend=SGLangProtocol( + sglang_config=SGLangServerConfig( + aggregated={"tp-size": 8, "dp-size": 8, "enable-dp-attention": True} + ) + ), + ) + + assert config.backend.sglang_config.aggregated["tp-size"] == 8 From 8605639c8d79ee5821ea7db327237f746e3bde7f Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 9 Aug 2026 20:25:00 -0500 Subject: [PATCH 05/46] fix: label static router health checks accurately --- src/srtctl/core/health.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/srtctl/core/health.py b/src/srtctl/core/health.py index e0993a7cf..d379b59b2 100644 --- a/src/srtctl/core/health.py +++ b/src/srtctl/core/health.py @@ -435,11 +435,12 @@ def wait_for_model( health_url = f"http://{host}:{port}{frontend.health_endpoint}" if frontend.health_endpoint == "/workers": logger.info( - "Polling %s every %.1fs for %d prefills and %d decodes (sglang frontend)", + "Polling %s every %.1fs for %d prefills and %d decodes (%s frontend)", health_url, poll_interval, n_prefill, n_decode, + frontend_type, ) else: health_url = f"http://{host}:{port}/health" From 1630a9ac9ea964c4b3375f1652fbe7ccf16a705b Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 9 Aug 2026 20:54:26 -0500 Subject: [PATCH 06/46] fix: align vLLM Router startup timeout --- docs/config-reference.md | 4 ++++ src/srtctl/frontends/static_router.py | 5 +++++ src/srtctl/frontends/vllm_router.py | 10 ++++++++++ tests/test_static_router_frontends.py | 26 ++++++++++++++++++++------ 4 files changed, 39 insertions(+), 6 deletions(-) diff --git a/docs/config-reference.md b/docs/config-reference.md index e56dcbca6..0f52bd266 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -307,6 +307,10 @@ frontend: | `env` | dict | null | Env vars for frontend processes | | `container_image` | str | null | Router process image; defaults to `model.container` | +For `vllm-router`, srtctl sets Router's `--worker-startup-timeout-secs` to the +total `health_check` window so large-model compilation cannot outlive the router. +Set `frontend.args.worker-startup-timeout-secs` to override it explicitly. + See [SGLang Router](sglang-router.md) for detailed architecture. ### vllm-router frontend diff --git a/src/srtctl/frontends/static_router.py b/src/srtctl/frontends/static_router.py index 34c1209a0..6dbb77e24 100644 --- a/src/srtctl/frontends/static_router.py +++ b/src/srtctl/frontends/static_router.py @@ -71,6 +71,10 @@ def get_frontend_args_list(self, args: dict[str, Any] | None) -> list[str]: result.extend([flag, str(value)]) return result + def get_managed_frontend_args(self, config: Any) -> list[str]: + """Return adapter-managed CLI arguments derived from srtctl config.""" + return [] + def worker_scheme(self, backend: Any, mode: str) -> str: """Return the protocol used to reach a worker endpoint.""" return "http" @@ -152,6 +156,7 @@ def start_frontends( for idx, node in enumerate(topology.frontend_nodes): router_log = runtime.log_dir / f"{node}_{self.type}_{idx}.out" cmd = self.build_router_command(workers, "0.0.0.0", topology.frontend_port) + cmd.extend(self.get_managed_frontend_args(config)) cmd.extend(self.get_frontend_args_list(config.frontend.args)) logger.info("Starting %s %d on %s: %s", self.type, idx, node, shlex.join(cmd)) diff --git a/src/srtctl/frontends/vllm_router.py b/src/srtctl/frontends/vllm_router.py index 36f689c5e..cdfb3e3c6 100644 --- a/src/srtctl/frontends/vllm_router.py +++ b/src/srtctl/frontends/vllm_router.py @@ -24,6 +24,16 @@ class VLLMRouterFrontend(StaticRouterFrontend): pd_flag: ClassVar[str] = "--vllm-pd-disaggregation" process_name: ClassVar[str] = "vllm_router" + def get_managed_frontend_args(self, config: Any) -> list[str]: + """Keep Router's worker wait alive for srtctl's model-readiness window.""" + frontend_args = config.frontend.args or {} + if "worker-startup-timeout-secs" in frontend_args: + return [] + + health_check = config.health_check + timeout_seconds = health_check.max_attempts * health_check.interval_seconds + return ["--worker-startup-timeout-secs", str(timeout_seconds)] + def worker_bootstrap_port(self, backend: Any, process: Process) -> int | None: """Advertise vLLM's NIXL side-channel port to the P/D router.""" return process.nixl_port diff --git a/tests/test_static_router_frontends.py b/tests/test_static_router_frontends.py index 1b4f04cd8..154785e14 100644 --- a/tests/test_static_router_frontends.py +++ b/tests/test_static_router_frontends.py @@ -114,6 +114,7 @@ def test_vllm_router_launch_uses_router_container_env_and_only_leaders() -> None ) config = SimpleNamespace( backend=SimpleNamespace(type="vllm"), + health_check=SimpleNamespace(max_attempts=360, interval_seconds=10), frontend=SimpleNamespace( args={"routing-logic": "session"}, env={"ROUTER_LOG": "debug"}, @@ -154,6 +155,23 @@ def test_vllm_router_launch_uses_router_container_env_and_only_leaders() -> None assert "/configs/${setup_script}" in kwargs["bash_preamble"] assert kwargs["command"].count("http://10.0.0.1:30000") == 1 assert "--routing-logic" in kwargs["command"] + timeout_index = kwargs["command"].index("--worker-startup-timeout-secs") + assert kwargs["command"][timeout_index + 1] == "3600" + + +def test_vllm_router_explicit_worker_startup_timeout_overrides_managed_value() -> None: + frontend = VLLMRouterFrontend() + config = SimpleNamespace( + health_check=SimpleNamespace(max_attempts=360, interval_seconds=10), + frontend=SimpleNamespace(args={"worker-startup-timeout-secs": 7200}), + ) + + command = [ + *frontend.get_managed_frontend_args(config), + *frontend.get_frontend_args_list(config.frontend.args), + ] + + assert command == ["--worker-startup-timeout-secs", "7200"] def test_router_rejects_backend_mismatch_before_launch() -> None: @@ -241,9 +259,7 @@ def test_sgl_router_rejects_non_divisible_tp_dp_layout() -> None: resources=ResourceConfig(gpu_type="h100", gpus_per_node=8, agg_nodes=1, agg_workers=1), frontend=FrontendConfig(type="sgl-router", enable_multiple_frontends=False), backend=SGLangProtocol( - sglang_config=SGLangServerConfig( - aggregated={"tp-size": 1, "dp-size": 8, "enable-dp-attention": True} - ) + sglang_config=SGLangServerConfig(aggregated={"tp-size": 1, "dp-size": 8, "enable-dp-attention": True}) ), ) @@ -258,9 +274,7 @@ def test_sgl_router_accepts_divisible_tp_dp_layout() -> None: resources=ResourceConfig(gpu_type="h100", gpus_per_node=8, agg_nodes=1, agg_workers=1), frontend=FrontendConfig(type="sgl-router", enable_multiple_frontends=False), backend=SGLangProtocol( - sglang_config=SGLangServerConfig( - aggregated={"tp-size": 8, "dp-size": 8, "enable-dp-attention": True} - ) + sglang_config=SGLangServerConfig(aggregated={"tp-size": 8, "dp-size": 8, "enable-dp-attention": True}) ), ) From 8affae442835aa23ea6de65d61ffdf6431db91a9 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 9 Aug 2026 22:10:23 -0500 Subject: [PATCH 07/46] fix: skip local SGLang P/D warmup behind router --- src/srtctl/backends/sglang.py | 8 +++++++ tests/test_configs.py | 42 +++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/src/srtctl/backends/sglang.py b/src/srtctl/backends/sglang.py index a06057dc7..1c2d30e6d 100644 --- a/src/srtctl/backends/sglang.py +++ b/src/srtctl/backends/sglang.py @@ -355,6 +355,14 @@ def build_worker_command( # Add disaggregation mode for prefill/decode workers (both dynamo and sglang frontend) if mode != "agg": cmd.extend(["--disaggregation-mode", mode]) + if use_sglang: + # Direct P/D workers are started together and validated through the + # static router. SGLang's built-in disaggregation warmup instead + # posts a synthetic request to the local worker with a fake + # bootstrap host, which cannot exercise this topology and blocks + # server readiness until its 30-minute timeout. The benchmark + # warmup exercises the real router-mediated P/D transfer path. + cmd.append("--skip-server-warmup") # Always pass bootstrap port for prefill workers regardless of frontend type. # Dynamo does NOT handle this internally — SGLang's CommonKVBootstrapServer # still runs on every prefill node for KV transfer coordination, and workers diff --git a/tests/test_configs.py b/tests/test_configs.py index ef60849f5..a26e804c2 100644 --- a/tests/test_configs.py +++ b/tests/test_configs.py @@ -541,6 +541,48 @@ def test_grpc_mode_enabled_per_mode(self): assert config.is_grpc_mode("decode") is True assert config.is_grpc_mode("agg") is False + @pytest.mark.parametrize( + ("frontend_type", "mode", "expected"), + [ + ("sgl-router", "prefill", True), + ("sgl-router", "decode", True), + ("sgl-router", "agg", False), + ("dynamo", "decode", False), + ], + ) + def test_static_router_pd_workers_skip_local_fake_bootstrap_warmup( + self, frontend_type: str, mode: str, expected: bool + ) -> None: + """Only native P/D routing relies on the router-level warmup.""" + from unittest.mock import MagicMock, patch + + from srtctl.core.topology import Process + + process = Process( + node="node0", + gpu_indices=frozenset(range(8)), + sys_port=8081, + http_port=6100, + endpoint_mode=mode, + endpoint_index=0, + node_rank=0, + bootstrap_port=7200, + ) + runtime = MagicMock() + runtime.model_path = Path("/model") + runtime.is_hf_model = False + runtime.request_plane = "tcp" + + with patch("srtctl.core.slurm.get_hostname_ip", return_value="10.0.0.1"): + command = SGLangProtocol().build_worker_command( + process=process, + endpoint_processes=[process], + runtime=runtime, + frontend_type=frontend_type, + ) + + assert ("--skip-server-warmup" in command) is expected + class TestServedModelName: """Tests for served_model_name property extraction from backend configs.""" From 0afc6c206cff4b3f8f1379dba913a55b1659b891 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 10 Aug 2026 08:07:36 -0500 Subject: [PATCH 08/46] docs: cover stable vllm device binding --- docs/config-reference.md | 18 ++++++++++++++++++ tests/test_configs.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/docs/config-reference.md b/docs/config-reference.md index 0f52bd266..4345ceb9c 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -479,6 +479,24 @@ the configured value. `headless` is incompatible with `per_node` DP because a headless process does not register with Dynamo, so srtslurm rejects that combination during configuration loading. +### vLLM device binding compatibility + +Direct `vllm` and `vllm-router` frontends use vLLM's `--device-ids` option by +default. For stable vLLM releases from before +[vllm-project/vllm#45026](https://github.com/vllm-project/vllm/pull/45026), +select the existing CUDA namespace binding instead: + +```yaml +backend: + type: vllm + set_cuda_visible_devices: true +``` + +srtslurm then omits `--device-ids` and scopes sub-node workers with +`CUDA_VISIBLE_DEVICES`. A worker that owns every GPU on its node needs no +explicit CUDA mask. This setting changes only device binding; worker topology, +private router ports, and P/D KV-transfer arguments remain unchanged. + ### TRTLLM Backend When using `type: trtllm`, the backend uses TRTLLM with MPI-style launching: diff --git a/tests/test_configs.py b/tests/test_configs.py index a26e804c2..86b1605e3 100644 --- a/tests/test_configs.py +++ b/tests/test_configs.py @@ -2244,6 +2244,44 @@ def test_vllm_router_worker_uses_private_port_and_pd_connector(self): kv_config = json.loads(cmd[cmd.index("--kv-transfer-config") + 1]) assert kv_config == {"kv_connector": "NixlConnector", "kv_role": "kv_both"} + def test_vllm_router_stable_release_uses_legacy_cuda_binding(self): + """Stable vLLM builds can avoid the newer --device-ids CLI.""" + from pathlib import Path + from unittest.mock import MagicMock, patch + + from srtctl.backends import VLLMProtocol, VLLMServerConfig + from srtctl.core.topology import Process + + backend = VLLMProtocol( + set_cuda_visible_devices=True, + vllm_config=VLLMServerConfig(decode={"tensor-parallel-size": 4}), + ) + process = Process( + node="node0", + gpu_indices=frozenset(range(4)), + sys_port=8081, + http_port=30123, + endpoint_mode="decode", + endpoint_index=0, + node_rank=0, + ) + runtime = MagicMock() + runtime.model_path = Path("/model") + runtime.is_hf_model = False + runtime.frontend_port = 8000 + + with patch("srtctl.core.slurm.get_hostname_ip", return_value="10.0.0.1"): + cmd = backend.build_worker_command( + process=process, + endpoint_processes=[process], + runtime=runtime, + frontend_type="vllm-router", + ) + + assert cmd[:3] == ["vllm", "serve", "/model"] + assert "--device-ids" not in cmd + assert backend.should_set_cuda_visible_devices(process) + def test_direct_vllm_command_keeps_iteration_profiler_config(self): """Direct vllm serve retains main's profiling-derived server option.""" from pathlib import Path From 5d1389927e963da075c431a3231602ac08691291 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 10 Aug 2026 12:36:23 -0500 Subject: [PATCH 09/46] make router frontend names semantic --- docs/config-reference.md | 21 ++++-- docs/sglang-router.md | 19 +++-- src/srtctl/README.md | 2 +- src/srtctl/backends/sglang.py | 6 +- src/srtctl/backends/vllm.py | 14 ++-- src/srtctl/benchmarks/router.py | 4 +- src/srtctl/cli/do_sweep.py | 2 +- src/srtctl/cli/mixins/benchmark_stage.py | 12 ++- src/srtctl/core/schema.py | 32 ++++---- src/srtctl/core/telemetry.py | 4 +- src/srtctl/frontends/__init__.py | 14 ++-- src/srtctl/frontends/base.py | 4 +- src/srtctl/frontends/sglang.py | 15 +--- src/srtctl/frontends/vllm.py | 96 ++++++------------------ src/srtctl/frontends/vllm_direct.py | 84 +++++++++++++++++++++ src/srtctl/frontends/vllm_router.py | 39 ---------- tests/test_configs.py | 20 ++--- tests/test_frontends.py | 24 +++++- tests/test_static_router_frontends.py | 40 +++++----- 19 files changed, 233 insertions(+), 219 deletions(-) create mode 100644 src/srtctl/frontends/vllm_direct.py delete mode 100644 src/srtctl/frontends/vllm_router.py diff --git a/docs/config-reference.md b/docs/config-reference.md index 4345ceb9c..1db324d51 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -270,8 +270,8 @@ Frontend/router configuration. ```yaml frontend: - # Frontend type: "dynamo" (default), "sgl-router", "vllm-router", - # "trtllm_serve", or direct "vllm". "sglang" is a compatibility alias. + # Frontend type: "dynamo" (default), SGLang Router "sglang", + # vLLM Router "vllm", direct "vllm-direct", or "trtllm_serve". type: dynamo # Scaling @@ -298,7 +298,7 @@ frontend: | Field | Type | Default | Description | | --------------------------- | ---- | ------------- | ----------------------------------- | -| `type` | str | dynamo | Frontend type: `dynamo`, `sgl-router`, `vllm-router`, `trtllm_serve`, or direct `vllm`; `sglang` is a compatibility alias | +| `type` | str | dynamo | Frontend type: `dynamo`, SGLang Router `sglang`, vLLM Router `vllm`, direct `vllm-direct`, or `trtllm_serve` | | `enable_multiple_frontends` | bool | true | Scale with nginx + multiple routers | | `num_additional_frontends` | int | 9 | Additional routers beyond master | | `nginx_container` | str | nginx:1.27.4 | Custom nginx container image | @@ -307,15 +307,15 @@ frontend: | `env` | dict | null | Env vars for frontend processes | | `container_image` | str | null | Router process image; defaults to `model.container` | -For `vllm-router`, srtctl sets Router's `--worker-startup-timeout-secs` to the +For `vllm`, srtctl sets Router's `--worker-startup-timeout-secs` to the total `health_check` window so large-model compilation cannot outlive the router. Set `frontend.args.worker-startup-timeout-secs` to override it explicitly. See [SGLang Router](sglang-router.md) for detailed architecture. -### vllm-router frontend +### vLLM Router frontend -`type: vllm-router` pairs with `backend.type: vllm` and launches the official +`frontend.type: vllm` pairs with `backend.type: vllm` and launches the official `vllm-router` process against direct private `vllm serve` endpoints. Aggregate layouts use `--worker-urls`; disaggregated layouts use `--vllm-pd-disaggregation` with the allocated prefill and decode leader URLs. @@ -323,6 +323,13 @@ Each logical vLLM endpoint must currently fit on one node, but a job may scale across many single-node aggregate, prefill, or decode endpoints. No NATS or etcd infrastructure is started for this frontend. +### Direct vLLM frontend + +`frontend.type: vllm-direct` preserves the narrow router-free serving path: +one single-node aggregate `vllm serve` endpoint binds the public port directly. +Set `frontend.enable_multiple_frontends: false`. This mode does not support P/D +or multiple logical endpoints; use `frontend.type: vllm` for those topologies. + ### trtllm_serve frontend `type: trtllm_serve` runs the `trtllm-serve disaggregated` orchestrator as the @@ -481,7 +488,7 @@ combination during configuration loading. ### vLLM device binding compatibility -Direct `vllm` and `vllm-router` frontends use vLLM's `--device-ids` option by +The `vllm` Router and `vllm-direct` frontends use vLLM's `--device-ids` option by default. For stable vLLM releases from before [vllm-project/vllm#45026](https://github.com/vllm-project/vllm/pull/45026), select the existing CUDA namespace binding instead: diff --git a/docs/sglang-router.md b/docs/sglang-router.md index 2d2553fcb..1f8b55aa9 100644 --- a/docs/sglang-router.md +++ b/docs/sglang-router.md @@ -40,13 +40,12 @@ Enable sglang router in your recipe's `frontend` section: ```yaml frontend: - type: sgl-router + type: sglang ``` -The legacy `type: sglang` spelling remains an exact compatibility alias. New -recipes should use `sgl-router`. Workers launch with `sglang.launch_server` -instead of `dynamo.sglang`, and the router receives only logical worker-leader -URLs from srtctl's allocated topology. +Workers launch with `sglang.launch_server` instead of `dynamo.sglang`, and the +native SGLang router receives only logical worker-leader URLs from srtctl's +allocated topology. ### Router Arguments @@ -54,7 +53,7 @@ Pass extra CLI args to the router: ```yaml frontend: - type: sgl-router + type: sglang args: kv-overlap-score-weight: 1 router-temperature: 0 @@ -78,7 +77,7 @@ Pass environment variables to frontend processes: ```yaml frontend: - type: sgl-router + type: sglang env: MY_CUSTOM_VAR: "value" ``` @@ -91,7 +90,7 @@ The simplest mode - one router on node 0, no nginx: ```yaml frontend: - type: sgl-router + type: sglang enable_multiple_frontends: false ``` @@ -115,7 +114,7 @@ Nginx load balances across multiple router instances: ```yaml frontend: - type: sgl-router + type: sglang enable_multiple_frontends: true # default num_additional_frontends: 9 # default, total = 1 + 9 = 10 routers ``` @@ -204,7 +203,7 @@ resources: decode_workers: 2 frontend: - type: sgl-router + type: sglang enable_multiple_frontends: true num_additional_frontends: 3 # 4 total routers diff --git a/src/srtctl/README.md b/src/srtctl/README.md index f48a1941c..f003404b5 100644 --- a/src/srtctl/README.md +++ b/src/srtctl/README.md @@ -106,7 +106,7 @@ wait_for_model( port=8000, n_prefill=2, n_decode=4, - frontend_type="sgl-router", # or "dynamo"; "sglang" remains an alias + frontend_type="sglang", # native SGLang Router; or "dynamo" timeout=300, ) ``` diff --git a/src/srtctl/backends/sglang.py b/src/srtctl/backends/sglang.py index 1c2d30e6d..66cc6d4e1 100644 --- a/src/srtctl/backends/sglang.py +++ b/src/srtctl/backends/sglang.py @@ -296,7 +296,7 @@ def build_worker_command( process: The process to start endpoint_processes: All processes for this endpoint (for multi-node) runtime: Runtime context with paths and settings - frontend_type: Frontend type - "sglang"/"sgl-router" use + frontend_type: Frontend type - "sglang" uses sglang.launch_server, while "dynamo" uses dynamo.sglang nsys_prefix: Optional nsys profiling command prefix dump_config_path: Path to dump config JSON @@ -321,7 +321,7 @@ def build_worker_command( dist_init_port = SGLANG_DIST_INIT_PORT_BASE # Choose Python module based on frontend type - use_sglang = frontend_type in {"sglang", "sgl-router"} + use_sglang = frontend_type == "sglang" python_module = "sglang.launch_server" if use_sglang else "dynamo.sglang" # Get served model name from config @@ -386,7 +386,7 @@ def build_worker_command( ) # Add config dump path (not when using sglang frontend) - if dump_config_path and frontend_type not in {"sglang", "sgl-router"}: + if dump_config_path and frontend_type != "sglang": cmd.extend(["--dump-config-to", str(dump_config_path)]) # Add kv-events-config if enabled for this mode and we have an allocated port diff --git a/src/srtctl/backends/vllm.py b/src/srtctl/backends/vllm.py index 6dfda324b..6e78c962a 100644 --- a/src/srtctl/backends/vllm.py +++ b/src/srtctl/backends/vllm.py @@ -498,13 +498,13 @@ def endpoints_to_processes( """Convert endpoints to processes. Dynamo DP+EP mode uses the configured per-GPU or per-node process layout. - For direct vLLM and vLLM Router jobs, `vllm serve` manages local DP ranks + For vLLM Router and direct vLLM jobs, `vllm serve` manages local DP ranks from one process, so keep the standard one-process-per-node topology. For standard TP mode, creates one process per node. """ from srtctl.core.topology import NodePortAllocator, Process, endpoints_to_processes - if frontend_type in {"vllm", "vllm-router"}: + if frontend_type in {"vllm", "vllm-direct"}: return endpoints_to_processes(endpoints, base_sys_port=base_sys_port, port_allocator=port_allocator) # Check if any endpoint uses DP mode @@ -675,7 +675,7 @@ def build_worker_command( process: The process to start endpoint_processes: All processes for this endpoint (for multi-node) runtime: Runtime context with paths and settings - frontend_type: Frontend type ("dynamo", direct "vllm", or "vllm-router") + frontend_type: Frontend type ("dynamo", router "vllm", or "vllm-direct") nsys_prefix: Optional nsys profiling command prefix dump_config_path: Path to dump config JSON profiling: Profiling config; drives --profiler-config for iteration-based nsys @@ -714,9 +714,9 @@ def build_worker_command( } ) - if frontend_type in {"vllm", "vllm-router"}: - if frontend_type == "vllm" and mode != "agg": - raise ValueError("frontend.type: vllm supports aggregate vLLM jobs only") + if frontend_type in {"vllm", "vllm-direct"}: + if frontend_type == "vllm-direct" and mode != "agg": + raise ValueError("frontend.type: vllm-direct supports aggregate vLLM jobs only") if is_multi_node: raise ValueError(f"frontend.type: {frontend_type} requires each vLLM endpoint to fit on one node") @@ -724,7 +724,7 @@ def build_worker_command( config.pop("port", None) config.setdefault("served-model-name", served_model_name) - if frontend_type == "vllm": + if frontend_type == "vllm-direct": config.pop("connector", None) worker_port = runtime.frontend_port else: diff --git a/src/srtctl/benchmarks/router.py b/src/srtctl/benchmarks/router.py index 27618af03..8e15b0a16 100644 --- a/src/srtctl/benchmarks/router.py +++ b/src/srtctl/benchmarks/router.py @@ -44,8 +44,8 @@ def validate_config(self, config: SrtConfig) -> list[str]: errors = [] # Router benchmark exercises the SGLang router's prefix-aware policies. - if config.frontend.type not in {"sglang", "sgl-router"}: - errors.append("router benchmark requires frontend.type: sgl-router") + if config.frontend.type != "sglang": + errors.append("router benchmark requires frontend.type: sglang") return errors diff --git a/src/srtctl/cli/do_sweep.py b/src/srtctl/cli/do_sweep.py index 09c1d0fbd..7c5cb5886 100644 --- a/src/srtctl/cli/do_sweep.py +++ b/src/srtctl/cli/do_sweep.py @@ -677,7 +677,7 @@ def run(self) -> int: try: # Stage 1: Head infrastructure (NATS, etcd). Only the dynamo request # plane uses it; static/direct frontends skip it. - if self.config.frontend.type in {"sglang", "sgl-router", "trtllm_serve", "vllm", "vllm-router"}: + if self.config.frontend.type in {"sglang", "trtllm_serve", "vllm", "vllm-direct"}: logger.info("Skipping NATS/etcd infrastructure (frontend.type=%s)", self.config.frontend.type) else: reporter.report(JobStatus.STARTING, JobStage.HEAD_INFRASTRUCTURE, "Starting head infrastructure") diff --git a/src/srtctl/cli/mixins/benchmark_stage.py b/src/srtctl/cli/mixins/benchmark_stage.py index 244dab270..087841ff2 100644 --- a/src/srtctl/cli/mixins/benchmark_stage.py +++ b/src/srtctl/cli/mixins/benchmark_stage.py @@ -430,8 +430,8 @@ def _get_sa_bench_slow_down_env(self) -> dict[str, str]: "benchmark slow_down: slow_down_sleep_time and slow_down_wait_time must be positive; skipping" ) return {} - if self.config.frontend.type not in {"sglang", "sgl-router"}: - logger.warning("benchmark.slow_down_* ignored: frontend.type is not sgl-router") + if self.config.frontend.type != "sglang": + logger.warning("benchmark.slow_down_* ignored: frontend.type is not sglang") return {} decode_urls: list[str] = [] @@ -472,11 +472,15 @@ def _get_aiperf_server_metrics_env( logical_endpoints = self._logical_worker_endpoints() urls = [f"http://{host}:{port}/metrics" for _, host, port in logical_endpoints] else: - if self.config.frontend.type in {"vllm", "vllm-router"}: + if self.config.frontend.type in {"vllm", "vllm-direct"}: for process in self.backend_processes: if process.is_leader: host = get_hostname_ip(process.node, self.runtime.network_interface) - port = FRONTEND_PUBLIC_PORT if self.config.frontend.type == "vllm" else process.http_port + port = ( + FRONTEND_PUBLIC_PORT + if self.config.frontend.type == "vllm-direct" + else process.http_port + ) urls.append(f"http://{host}:{port}/metrics") if urls: return {"AIPERF_SERVER_METRICS_URLS": ",".join(sorted(set(urls)))} diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py index 6fe7ac0d1..9d036ef95 100755 --- a/src/srtctl/core/schema.py +++ b/src/srtctl/core/schema.py @@ -1437,8 +1437,8 @@ class FrontendConfig: """Frontend/router configuration. Attributes: - type: Frontend type - "dynamo" (default), "sgl-router", "vllm-router", - "trtllm_serve", or direct "vllm". "sglang" remains a compatibility alias. + type: Frontend type - "dynamo" (default), SGLang Router "sglang", + vLLM Router "vllm", direct "vllm-direct", or "trtllm_serve". enable_multiple_frontends: Scale with nginx + multiple routers. When ``True`` (default), srtctl stands up nginx and fans out to ``num_additional_frontends + 1`` router replicas. When @@ -1591,7 +1591,7 @@ def __post_init__(self): self._validate_mooncake_kv_store() self._validate_het_jobs() self._validate_trtllm_serve() - self._validate_vllm_frontend() + self._validate_vllm_direct_frontend() self._validate_static_router_frontend() self._validate_sglang_data_parallelism() @@ -1619,34 +1619,38 @@ def _validate_trtllm_serve(self): "(set resources.prefill_nodes/prefill_workers and decode_nodes/decode_workers)" ) - def _validate_vllm_frontend(self): + def _validate_vllm_direct_frontend(self): """Catch direct-vLLM frontend misconfigurations at load time. Direct vLLM means the aggregate `vllm serve` worker owns the OpenAI port itself. It is not a disaggregated router and does not support the nginx multi-frontend path. """ - if self.frontend.type != "vllm": + if self.frontend.type != "vllm-direct": return if self.backend_type != "vllm": - raise ValidationError(f"frontend.type: vllm requires backend.type: vllm; got {self.backend_type!r}") + raise ValidationError( + f"frontend.type: vllm-direct requires backend.type: vllm; got {self.backend_type!r}" + ) if self.frontend.enable_multiple_frontends: raise ValidationError( - "frontend.type: vllm binds vllm serve directly; set frontend.enable_multiple_frontends: false" + "frontend.type: vllm-direct binds vllm serve directly; " + "set frontend.enable_multiple_frontends: false" ) if self.resources.is_disaggregated: - raise ValidationError("frontend.type: vllm supports aggregate jobs only, not disaggregated layouts") + raise ValidationError( + "frontend.type: vllm-direct supports aggregate jobs only, not disaggregated layouts" + ) if self.resources.num_agg < 1: - raise ValidationError("frontend.type: vllm requires resources.agg_workers >= 1") + raise ValidationError("frontend.type: vllm-direct requires resources.agg_workers >= 1") if (self.resources.agg_nodes or 1) != 1: - raise ValidationError("frontend.type: vllm currently supports single-node aggregate jobs only") + raise ValidationError("frontend.type: vllm-direct currently supports single-node aggregate jobs only") def _validate_static_router_frontend(self): """Validate native static-router/backend pairings and endpoint shape.""" required_backend = { "sglang": "sglang", - "sgl-router": "sglang", - "vllm-router": "vllm", + "vllm": "vllm", }.get(self.frontend.type) if required_backend is None: return @@ -1656,7 +1660,7 @@ def _validate_static_router_frontend(self): f"got {self.backend_type!r}" ) - if self.frontend.type == "vllm-router": + if self.frontend.type == "vllm": endpoint_gpu_counts = ( self.resources.gpus_per_prefill if self.resources.num_prefill else 0, self.resources.gpus_per_decode if self.resources.num_decode else 0, @@ -1664,7 +1668,7 @@ def _validate_static_router_frontend(self): ) if any(count > self.resources.gpus_per_node for count in endpoint_gpu_counts): raise ValidationError( - "frontend.type: vllm-router currently requires each logical vLLM endpoint " + "frontend.type: vllm currently requires each logical vLLM endpoint " "to fit on one node; scale with multiple aggregate/prefill/decode workers" ) diff --git a/src/srtctl/core/telemetry.py b/src/srtctl/core/telemetry.py index 79a8974d1..ea3810717 100644 --- a/src/srtctl/core/telemetry.py +++ b/src/srtctl/core/telemetry.py @@ -86,9 +86,9 @@ def generate_telemetry_config( for process in sorted(processes, key=lambda p: (p.endpoint_mode, p.endpoint_index, p.node_rank, p.node)): node_ip = get_hostname_ip(process.node, runtime.network_interface) - if frontend_type == "vllm" and process.endpoint_mode == "agg": + if frontend_type == "vllm-direct" and process.endpoint_mode == "agg": port = FRONTEND_PUBLIC_PORT - elif frontend_type == "vllm-router": + elif frontend_type == "vllm": port = process.http_port else: port = process.sys_port diff --git a/src/srtctl/frontends/__init__.py b/src/srtctl/frontends/__init__.py index 06912a331..846c42345 100644 --- a/src/srtctl/frontends/__init__.py +++ b/src/srtctl/frontends/__init__.py @@ -6,10 +6,9 @@ Supported frontend types: - dynamo: Dynamo frontend with NATS/etcd communication -- sgl-router: SGLang Model Gateway with direct worker connections -- sglang: Backward-compatible alias for sgl-router -- vllm: Direct vLLM OpenAI server for aggregate jobs -- vllm-router: vLLM Router with direct worker connections +- sglang: SGLang Model Gateway with direct worker connections +- vllm: Official vLLM Router with direct worker connections +- vllm-direct: Direct vLLM OpenAI server for aggregate jobs """ from srtctl.frontends.base import ( @@ -18,19 +17,18 @@ get_frontend, ) from srtctl.frontends.dynamo import DynamoFrontend -from srtctl.frontends.sglang import SGLangFrontend, SGLRouterFrontend +from srtctl.frontends.sglang import SGLangFrontend from srtctl.frontends.trtllm_serve import TRTLLMServeFrontend from srtctl.frontends.vllm import VLLMFrontend -from srtctl.frontends.vllm_router import VLLMRouterFrontend +from srtctl.frontends.vllm_direct import VLLMDirectFrontend __all__ = [ "DynamoFrontend", "FrontendProtocol", "FrontendType", - "SGLRouterFrontend", "SGLangFrontend", "TRTLLMServeFrontend", + "VLLMDirectFrontend", "VLLMFrontend", - "VLLMRouterFrontend", "get_frontend", ] diff --git a/src/srtctl/frontends/base.py b/src/srtctl/frontends/base.py index c808f4918..1a6cf0bc2 100644 --- a/src/srtctl/frontends/base.py +++ b/src/srtctl/frontends/base.py @@ -22,7 +22,7 @@ from srtctl.core.topology import Process # Supported frontend types - extensible by adding new literals -FrontendType = Literal["dynamo", "sglang", "sgl-router", "trtllm_serve", "vllm", "vllm-router"] +FrontendType = Literal["dynamo", "sglang", "trtllm_serve", "vllm", "vllm-direct"] FrontendFactory = Callable[[], "FrontendProtocol"] _FRONTEND_REGISTRY: dict[str, FrontendFactory] = {} @@ -44,7 +44,7 @@ def decorator(frontend_class: _FrontendClass) -> _FrontendClass: def _load_builtin_frontends() -> None: """Import built-ins once so their registration decorators run.""" - from srtctl.frontends import dynamo, sglang, trtllm_serve, vllm, vllm_router # noqa: F401 + from srtctl.frontends import dynamo, sglang, trtllm_serve, vllm, vllm_direct # noqa: F401 def build_setup_script_preamble(setup_script: str | None) -> str | None: diff --git a/src/srtctl/frontends/sglang.py b/src/srtctl/frontends/sglang.py index c210e0ffb..3a6d779cf 100644 --- a/src/srtctl/frontends/sglang.py +++ b/src/srtctl/frontends/sglang.py @@ -10,11 +10,11 @@ from srtctl.frontends.static_router import StaticRouterFrontend -@register_frontend("sgl-router") -class SGLRouterFrontend(StaticRouterFrontend): - """First-class SGLang Model Gateway static router.""" +@register_frontend("sglang") +class SGLangFrontend(StaticRouterFrontend): + """SGLang Model Gateway static router.""" - type: ClassVar[str] = "sgl-router" + type: ClassVar[str] = "sglang" backend_type: ClassVar[str] = "sglang" executable: ClassVar[tuple[str, ...]] = ("python", "-m", "sglang_router.launch_router") pd_flag: ClassVar[str] = "--pd-disaggregation" @@ -28,10 +28,3 @@ def get_hostname_ip(self, node: str) -> str: def start_process(self, **kwargs: Any) -> Any: return start_srun_process(**kwargs) - - -@register_frontend("sglang") -class SGLangFrontend(SGLRouterFrontend): - """Backward-compatible alias for the historical ``sglang`` frontend type.""" - - type: ClassVar[str] = "sglang" diff --git a/src/srtctl/frontends/vllm.py b/src/srtctl/frontends/vllm.py index c1af62669..08107c2cd 100644 --- a/src/srtctl/frontends/vllm.py +++ b/src/srtctl/frontends/vllm.py @@ -1,91 +1,39 @@ # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -""" -Direct vLLM frontend implementation. - -For aggregate vLLM jobs the OpenAI-compatible HTTP server is the worker -process itself (`vllm serve`). There is no separate router/frontend process. -""" +"""Official vLLM Router frontend.""" from __future__ import annotations -import logging -import threading -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar -from srtctl.core.health import WorkerHealthResult from srtctl.frontends.base import register_frontend +from srtctl.frontends.static_router import StaticRouterFrontend if TYPE_CHECKING: - from srtctl.core.processes import ManagedProcess - from srtctl.core.runtime import RuntimeContext from srtctl.core.topology import Process -logger = logging.getLogger(__name__) - @register_frontend("vllm") -class VLLMFrontend: - """Direct vLLM OpenAI server frontend. - - This frontend is intentionally narrow: aggregate vLLM jobs only, with the - backend worker binding the public OpenAI port directly. Disaggregated vLLM - still needs a real router/orchestrator such as Dynamo. - """ - - @property - def type(self) -> str: - return "vllm" - - @property - def health_endpoint(self) -> str: - return "/health" - - def parse_health( - self, - response_json: dict, - expected_prefill: int, - expected_decode: int, - ) -> WorkerHealthResult: - return WorkerHealthResult( - ready=True, - message="vLLM OpenAI server healthy", - prefill_ready=expected_prefill, - prefill_expected=expected_prefill, - decode_ready=expected_decode, - decode_expected=expected_decode, - ) - - def get_frontend_args_list(self, args: dict[str, Any] | None) -> list[str]: - if not args: +class VLLMFrontend(StaticRouterFrontend): + """Route requests through the official vLLM Router.""" + + type: ClassVar[str] = "vllm" + backend_type: ClassVar[str] = "vllm" + executable: ClassVar[tuple[str, ...]] = ("vllm-router",) + pd_flag: ClassVar[str] = "--vllm-pd-disaggregation" + process_name: ClassVar[str] = "vllm_router" + + def get_managed_frontend_args(self, config: Any) -> list[str]: + """Keep Router's worker wait alive for srtctl's model-readiness window.""" + frontend_args = config.frontend.args or {} + if "worker-startup-timeout-secs" in frontend_args: return [] - result = [] - for key, value in args.items(): - if value is True: - result.append(f"--{key}") - elif value is not False and value is not None: - result.extend([f"--{key}", str(value)]) - return result - def start_frontends( - self, - topology: Any, - runtime: RuntimeContext, - config: Any, - backend: Any, - backend_processes: list[Process], - stop_event: threading.Event | None = None, - ) -> list[ManagedProcess]: - if config.backend.type != "vllm": - raise ValueError(f"frontend.type: vllm requires backend.type: vllm (got {config.backend.type!r})") - if topology.uses_nginx or len(topology.frontend_nodes) != 1: - raise ValueError( - "frontend.type: vllm binds vllm serve directly to the public port; " - "set frontend.enable_multiple_frontends: false" - ) - if config.resources.is_disaggregated or config.resources.num_agg < 1: - raise ValueError("frontend.type: vllm supports aggregate vLLM jobs only") + health_check = config.health_check + timeout_seconds = health_check.max_attempts * health_check.interval_seconds + return ["--worker-startup-timeout-secs", str(timeout_seconds)] - logger.info("frontend.type=vllm: no separate frontend process; vllm serve owns port %d", topology.public_port) - return [] + def worker_bootstrap_port(self, backend: Any, process: Process) -> int | None: + """Advertise vLLM's NIXL side-channel port to the P/D router.""" + return process.nixl_port diff --git a/src/srtctl/frontends/vllm_direct.py b/src/srtctl/frontends/vllm_direct.py new file mode 100644 index 000000000..60a23f33d --- /dev/null +++ b/src/srtctl/frontends/vllm_direct.py @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Direct vLLM serving without a separate frontend process.""" + +from __future__ import annotations + +import logging +import threading +from typing import TYPE_CHECKING, Any + +from srtctl.core.health import WorkerHealthResult +from srtctl.frontends.base import register_frontend + +if TYPE_CHECKING: + from srtctl.core.processes import ManagedProcess + from srtctl.core.runtime import RuntimeContext + from srtctl.core.topology import Process + +logger = logging.getLogger(__name__) + + +@register_frontend("vllm-direct") +class VLLMDirectFrontend: + """Expose one aggregate ``vllm serve`` endpoint without a router.""" + + @property + def type(self) -> str: + return "vllm-direct" + + @property + def health_endpoint(self) -> str: + return "/health" + + def parse_health( + self, + response_json: dict, + expected_prefill: int, + expected_decode: int, + ) -> WorkerHealthResult: + return WorkerHealthResult( + ready=True, + message="vLLM OpenAI server healthy", + prefill_ready=expected_prefill, + prefill_expected=expected_prefill, + decode_ready=expected_decode, + decode_expected=expected_decode, + ) + + def get_frontend_args_list(self, args: dict[str, Any] | None) -> list[str]: + if not args: + return [] + result = [] + for key, value in args.items(): + if value is True: + result.append(f"--{key}") + elif value is not False and value is not None: + result.extend([f"--{key}", str(value)]) + return result + + def start_frontends( + self, + topology: Any, + runtime: RuntimeContext, + config: Any, + backend: Any, + backend_processes: list[Process], + stop_event: threading.Event | None = None, + ) -> list[ManagedProcess]: + if config.backend.type != "vllm": + raise ValueError(f"frontend.type: vllm-direct requires backend.type: vllm (got {config.backend.type!r})") + if topology.uses_nginx or len(topology.frontend_nodes) != 1: + raise ValueError( + "frontend.type: vllm-direct binds vllm serve directly to the public port; " + "set frontend.enable_multiple_frontends: false" + ) + if config.resources.is_disaggregated or config.resources.num_agg < 1: + raise ValueError("frontend.type: vllm-direct supports aggregate vLLM jobs only") + + logger.info( + "frontend.type=vllm-direct: no separate frontend process; vllm serve owns port %d", + topology.public_port, + ) + return [] diff --git a/src/srtctl/frontends/vllm_router.py b/src/srtctl/frontends/vllm_router.py deleted file mode 100644 index cdfb3e3c6..000000000 --- a/src/srtctl/frontends/vllm_router.py +++ /dev/null @@ -1,39 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""vLLM Router frontend.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any, ClassVar - -from srtctl.frontends.base import register_frontend -from srtctl.frontends.static_router import StaticRouterFrontend - -if TYPE_CHECKING: - from srtctl.core.topology import Process - - -@register_frontend("vllm-router") -class VLLMRouterFrontend(StaticRouterFrontend): - """Route requests to direct vLLM OpenAI-compatible worker endpoints.""" - - type: ClassVar[str] = "vllm-router" - backend_type: ClassVar[str] = "vllm" - executable: ClassVar[tuple[str, ...]] = ("vllm-router",) - pd_flag: ClassVar[str] = "--vllm-pd-disaggregation" - process_name: ClassVar[str] = "vllm_router" - - def get_managed_frontend_args(self, config: Any) -> list[str]: - """Keep Router's worker wait alive for srtctl's model-readiness window.""" - frontend_args = config.frontend.args or {} - if "worker-startup-timeout-secs" in frontend_args: - return [] - - health_check = config.health_check - timeout_seconds = health_check.max_attempts * health_check.interval_seconds - return ["--worker-startup-timeout-secs", str(timeout_seconds)] - - def worker_bootstrap_port(self, backend: Any, process: Process) -> int | None: - """Advertise vLLM's NIXL side-channel port to the P/D router.""" - return process.nixl_port diff --git a/tests/test_configs.py b/tests/test_configs.py index 86b1605e3..d4a9fa16a 100644 --- a/tests/test_configs.py +++ b/tests/test_configs.py @@ -544,9 +544,9 @@ def test_grpc_mode_enabled_per_mode(self): @pytest.mark.parametrize( ("frontend_type", "mode", "expected"), [ - ("sgl-router", "prefill", True), - ("sgl-router", "decode", True), - ("sgl-router", "agg", False), + ("sglang", "prefill", True), + ("sglang", "decode", True), + ("sglang", "agg", False), ("dynamo", "decode", False), ], ) @@ -864,7 +864,7 @@ def test_router_container_alias_resolves(self): "name": "test", "model": {"path": "/model", "container": "worker", "precision": "fp8"}, "resources": {"gpu_type": "h100", "gpus_per_node": 8, "agg_nodes": 1}, - "frontend": {"type": "vllm-router", "container_image": "router"}, + "frontend": {"type": "vllm", "container_image": "router"}, } cluster_config = { "containers": { @@ -2130,7 +2130,7 @@ def test_direct_vllm_dp_mode_keeps_single_process(self): gpus_per_node=8, ) - processes = backend.endpoints_to_processes([endpoint], frontend_type="vllm") + processes = backend.endpoints_to_processes([endpoint], frontend_type="vllm-direct") assert len(processes) == 1 assert processes[0].node == "node0" @@ -2171,7 +2171,7 @@ def test_direct_vllm_command_preserves_current_main_device_binding(self): process=process, endpoint_processes=[process], runtime=runtime, - frontend_type="vllm", + frontend_type="vllm-direct", ) assert cmd[:3] == ["vllm", "serve", "/model"] @@ -2197,7 +2197,7 @@ def test_vllm_router_keeps_one_direct_server_per_logical_endpoint(self): for index, node in enumerate(("node0", "node1")) ] - processes = backend.endpoints_to_processes(endpoints, frontend_type="vllm-router") + processes = backend.endpoints_to_processes(endpoints, frontend_type="vllm") assert len(processes) == 2 assert all(process.is_leader for process in processes) @@ -2235,7 +2235,7 @@ def test_vllm_router_worker_uses_private_port_and_pd_connector(self): process=process, endpoint_processes=[process], runtime=runtime, - frontend_type="vllm-router", + frontend_type="vllm", ) assert cmd[:3] == ["vllm", "serve", "/model"] @@ -2275,7 +2275,7 @@ def test_vllm_router_stable_release_uses_legacy_cuda_binding(self): process=process, endpoint_processes=[process], runtime=runtime, - frontend_type="vllm-router", + frontend_type="vllm", ) assert cmd[:3] == ["vllm", "serve", "/model"] @@ -2318,7 +2318,7 @@ def test_direct_vllm_command_keeps_iteration_profiler_config(self): process=process, endpoint_processes=[process], runtime=runtime, - frontend_type="vllm", + frontend_type="vllm-direct", profiling=profiling, ) diff --git a/tests/test_frontends.py b/tests/test_frontends.py index f00507c8d..7a3240429 100644 --- a/tests/test_frontends.py +++ b/tests/test_frontends.py @@ -11,7 +11,7 @@ import pytest from srtctl.core.schema import ObservabilityConfig -from srtctl.frontends import DynamoFrontend, SGLangFrontend, VLLMFrontend, get_frontend +from srtctl.frontends import DynamoFrontend, SGLangFrontend, VLLMDirectFrontend, VLLMFrontend, get_frontend # ============================================================================ # get_frontend() Tests @@ -34,11 +34,17 @@ def test_get_sglang_frontend(self): assert frontend.type == "sglang" def test_get_vllm_frontend(self): - """get_frontend('vllm') returns VLLMFrontend.""" + """get_frontend('vllm') returns the vLLM Router frontend.""" frontend = get_frontend("vllm") assert isinstance(frontend, VLLMFrontend) assert frontend.type == "vllm" + def test_get_vllm_direct_frontend(self): + """get_frontend('vllm-direct') returns the router-free adapter.""" + frontend = get_frontend("vllm-direct") + assert isinstance(frontend, VLLMDirectFrontend) + assert frontend.type == "vllm-direct" + def test_get_unknown_frontend_raises(self): """get_frontend() with unknown type raises ValueError.""" with pytest.raises(ValueError, match="Unknown frontend type"): @@ -67,10 +73,15 @@ def test_sglang_type(self): assert frontend.type == "sglang" def test_vllm_type(self): - """VLLMFrontend.type is 'vllm'.""" + """The vLLM Router frontend type is 'vllm'.""" frontend = VLLMFrontend() assert frontend.type == "vllm" + def test_vllm_direct_type(self): + """The direct adapter is explicitly named 'vllm-direct'.""" + frontend = VLLMDirectFrontend() + assert frontend.type == "vllm-direct" + def test_dynamo_health_endpoint(self): """DynamoFrontend uses /health endpoint.""" frontend = DynamoFrontend() @@ -82,8 +93,13 @@ def test_sglang_health_endpoint(self): assert frontend.health_endpoint == "/workers" def test_vllm_health_endpoint(self): - """VLLMFrontend uses /health endpoint.""" + """vLLM Router uses its worker-registration endpoint.""" frontend = VLLMFrontend() + assert frontend.health_endpoint == "/workers" + + def test_vllm_direct_health_endpoint(self): + """Direct vLLM uses the server health endpoint.""" + frontend = VLLMDirectFrontend() assert frontend.health_endpoint == "/health" diff --git a/tests/test_static_router_frontends.py b/tests/test_static_router_frontends.py index 154785e14..99f689c23 100644 --- a/tests/test_static_router_frontends.py +++ b/tests/test_static_router_frontends.py @@ -9,17 +9,17 @@ import pytest -from srtctl.frontends import SGLRouterFrontend, VLLMRouterFrontend, get_frontend +from srtctl.frontends import SGLangFrontend, VLLMDirectFrontend, VLLMFrontend, get_frontend from srtctl.frontends.static_router import RouterWorker -def test_registry_exposes_explicit_router_names_and_legacy_alias() -> None: - assert isinstance(get_frontend("sgl-router"), SGLRouterFrontend) - assert get_frontend("sglang").type == "sglang" - assert isinstance(get_frontend("vllm-router"), VLLMRouterFrontend) +def test_registry_uses_engine_names_for_routers_and_explicit_direct_name() -> None: + assert isinstance(get_frontend("sglang"), SGLangFrontend) + assert isinstance(get_frontend("vllm"), VLLMFrontend) + assert isinstance(get_frontend("vllm-direct"), VLLMDirectFrontend) -@pytest.mark.parametrize("frontend", [SGLRouterFrontend(), VLLMRouterFrontend()]) +@pytest.mark.parametrize("frontend", [SGLangFrontend(), VLLMFrontend()]) def test_aggregate_command_advertises_all_logical_workers(frontend) -> None: command = frontend.build_router_command( [ @@ -38,8 +38,8 @@ def test_aggregate_command_advertises_all_logical_workers(frontend) -> None: @pytest.mark.parametrize( ("frontend", "pd_flag"), [ - (SGLRouterFrontend(), "--pd-disaggregation"), - (VLLMRouterFrontend(), "--vllm-pd-disaggregation"), + (SGLangFrontend(), "--pd-disaggregation"), + (VLLMFrontend(), "--vllm-pd-disaggregation"), ], ) def test_disaggregated_command_preserves_modes_and_bootstrap(frontend, pd_flag: str) -> None: @@ -61,7 +61,7 @@ def test_disaggregated_command_preserves_modes_and_bootstrap(frontend, pd_flag: def test_router_command_rejects_incomplete_or_mixed_topology() -> None: - frontend = VLLMRouterFrontend() + frontend = VLLMFrontend() with pytest.raises(ValueError, match="requires prefill and decode"): frontend.build_router_command([RouterWorker("prefill", "http://p:1")], "0.0.0.0", 8000) with pytest.raises(ValueError, match="cannot mix"): @@ -77,7 +77,7 @@ def test_router_command_rejects_incomplete_or_mixed_topology() -> None: def test_frontend_args_repeat_list_values() -> None: - frontend = VLLMRouterFrontend() + frontend = VLLMFrontend() assert frontend.get_frontend_args_list({"routing-logic": ["round_robin", "session"]}) == [ "--routing-logic", "round_robin", @@ -87,7 +87,7 @@ def test_frontend_args_repeat_list_values() -> None: def test_vllm_router_advertises_nixl_side_channel_port() -> None: - frontend = VLLMRouterFrontend() + frontend = VLLMFrontend() process = SimpleNamespace( is_leader=True, endpoint_mode="prefill", @@ -104,7 +104,7 @@ def test_vllm_router_advertises_nixl_side_channel_port() -> None: def test_vllm_router_launch_uses_router_container_env_and_only_leaders() -> None: - frontend = VLLMRouterFrontend() + frontend = VLLMFrontend() runtime = SimpleNamespace( log_dir=Path("/logs"), container_image=Path("/worker.sqsh"), @@ -160,7 +160,7 @@ def test_vllm_router_launch_uses_router_container_env_and_only_leaders() -> None def test_vllm_router_explicit_worker_startup_timeout_overrides_managed_value() -> None: - frontend = VLLMRouterFrontend() + frontend = VLLMFrontend() config = SimpleNamespace( health_check=SimpleNamespace(max_attempts=360, interval_seconds=10), frontend=SimpleNamespace(args={"worker-startup-timeout-secs": 7200}), @@ -175,7 +175,7 @@ def test_vllm_router_explicit_worker_startup_timeout_overrides_managed_value() - def test_router_rejects_backend_mismatch_before_launch() -> None: - frontend = VLLMRouterFrontend() + frontend = VLLMFrontend() config = SimpleNamespace( backend=SimpleNamespace(type="sglang"), frontend=SimpleNamespace(args=None, env=None, container_image=None), @@ -193,12 +193,12 @@ def test_schema_rejects_router_backend_mismatch() -> None: from srtctl.backends import SGLangProtocol from srtctl.core.schema import FrontendConfig, ResourceConfig, SrtConfig - with pytest.raises(ValidationError, match="vllm-router requires backend.type: vllm"): + with pytest.raises(ValidationError, match="vllm requires backend.type: vllm"): SrtConfig( name="bad-router-pair", model={"path": "model", "container": "image", "precision": "fp8"}, resources=ResourceConfig(gpu_type="h100", gpus_per_node=8, agg_nodes=1, agg_workers=1), - frontend=FrontendConfig(type="vllm-router", enable_multiple_frontends=False), + frontend=FrontendConfig(type="vllm", enable_multiple_frontends=False), backend=SGLangProtocol(), ) @@ -216,7 +216,7 @@ def test_vllm_router_accepts_many_single_node_endpoints() -> None: agg_nodes=4, agg_workers=4, ), - frontend=FrontendConfig(type="vllm-router", enable_multiple_frontends=False), + frontend=FrontendConfig(type="vllm", enable_multiple_frontends=False), backend=VLLMProtocol(), ) @@ -241,7 +241,7 @@ def test_vllm_router_rejects_endpoint_spanning_nodes() -> None: decode_nodes=1, decode_workers=1, ), - frontend=FrontendConfig(type="vllm-router", enable_multiple_frontends=False), + frontend=FrontendConfig(type="vllm", enable_multiple_frontends=False), backend=VLLMProtocol(), ) @@ -257,7 +257,7 @@ def test_sgl_router_rejects_non_divisible_tp_dp_layout() -> None: name="invalid-sglang-dpa", model={"path": "model", "container": "image", "precision": "fp8"}, resources=ResourceConfig(gpu_type="h100", gpus_per_node=8, agg_nodes=1, agg_workers=1), - frontend=FrontendConfig(type="sgl-router", enable_multiple_frontends=False), + frontend=FrontendConfig(type="sglang", enable_multiple_frontends=False), backend=SGLangProtocol( sglang_config=SGLangServerConfig(aggregated={"tp-size": 1, "dp-size": 8, "enable-dp-attention": True}) ), @@ -272,7 +272,7 @@ def test_sgl_router_accepts_divisible_tp_dp_layout() -> None: name="valid-sglang-dpa", model={"path": "model", "container": "image", "precision": "fp8"}, resources=ResourceConfig(gpu_type="h100", gpus_per_node=8, agg_nodes=1, agg_workers=1), - frontend=FrontendConfig(type="sgl-router", enable_multiple_frontends=False), + frontend=FrontendConfig(type="sglang", enable_multiple_frontends=False), backend=SGLangProtocol( sglang_config=SGLangServerConfig(aggregated={"tp-size": 8, "dp-size": 8, "enable-dp-attention": True}) ), From d573ba6a2c5ac920170f73d27ae358dd397cae49 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 10 Aug 2026 12:39:06 -0500 Subject: [PATCH 10/46] Revert "make router frontend names semantic" This reverts commit 5d1389927e963da075c431a3231602ac08691291. --- docs/config-reference.md | 21 ++---- docs/sglang-router.md | 19 ++--- src/srtctl/README.md | 2 +- src/srtctl/backends/sglang.py | 6 +- src/srtctl/backends/vllm.py | 14 ++-- src/srtctl/benchmarks/router.py | 4 +- src/srtctl/cli/do_sweep.py | 2 +- src/srtctl/cli/mixins/benchmark_stage.py | 12 +-- src/srtctl/core/schema.py | 32 ++++---- src/srtctl/core/telemetry.py | 4 +- src/srtctl/frontends/__init__.py | 14 ++-- src/srtctl/frontends/base.py | 4 +- src/srtctl/frontends/sglang.py | 15 +++- src/srtctl/frontends/vllm.py | 96 ++++++++++++++++++------ src/srtctl/frontends/vllm_direct.py | 84 --------------------- src/srtctl/frontends/vllm_router.py | 39 ++++++++++ tests/test_configs.py | 20 ++--- tests/test_frontends.py | 24 +----- tests/test_static_router_frontends.py | 40 +++++----- 19 files changed, 219 insertions(+), 233 deletions(-) delete mode 100644 src/srtctl/frontends/vllm_direct.py create mode 100644 src/srtctl/frontends/vllm_router.py diff --git a/docs/config-reference.md b/docs/config-reference.md index 1db324d51..4345ceb9c 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -270,8 +270,8 @@ Frontend/router configuration. ```yaml frontend: - # Frontend type: "dynamo" (default), SGLang Router "sglang", - # vLLM Router "vllm", direct "vllm-direct", or "trtllm_serve". + # Frontend type: "dynamo" (default), "sgl-router", "vllm-router", + # "trtllm_serve", or direct "vllm". "sglang" is a compatibility alias. type: dynamo # Scaling @@ -298,7 +298,7 @@ frontend: | Field | Type | Default | Description | | --------------------------- | ---- | ------------- | ----------------------------------- | -| `type` | str | dynamo | Frontend type: `dynamo`, SGLang Router `sglang`, vLLM Router `vllm`, direct `vllm-direct`, or `trtllm_serve` | +| `type` | str | dynamo | Frontend type: `dynamo`, `sgl-router`, `vllm-router`, `trtllm_serve`, or direct `vllm`; `sglang` is a compatibility alias | | `enable_multiple_frontends` | bool | true | Scale with nginx + multiple routers | | `num_additional_frontends` | int | 9 | Additional routers beyond master | | `nginx_container` | str | nginx:1.27.4 | Custom nginx container image | @@ -307,15 +307,15 @@ frontend: | `env` | dict | null | Env vars for frontend processes | | `container_image` | str | null | Router process image; defaults to `model.container` | -For `vllm`, srtctl sets Router's `--worker-startup-timeout-secs` to the +For `vllm-router`, srtctl sets Router's `--worker-startup-timeout-secs` to the total `health_check` window so large-model compilation cannot outlive the router. Set `frontend.args.worker-startup-timeout-secs` to override it explicitly. See [SGLang Router](sglang-router.md) for detailed architecture. -### vLLM Router frontend +### vllm-router frontend -`frontend.type: vllm` pairs with `backend.type: vllm` and launches the official +`type: vllm-router` pairs with `backend.type: vllm` and launches the official `vllm-router` process against direct private `vllm serve` endpoints. Aggregate layouts use `--worker-urls`; disaggregated layouts use `--vllm-pd-disaggregation` with the allocated prefill and decode leader URLs. @@ -323,13 +323,6 @@ Each logical vLLM endpoint must currently fit on one node, but a job may scale across many single-node aggregate, prefill, or decode endpoints. No NATS or etcd infrastructure is started for this frontend. -### Direct vLLM frontend - -`frontend.type: vllm-direct` preserves the narrow router-free serving path: -one single-node aggregate `vllm serve` endpoint binds the public port directly. -Set `frontend.enable_multiple_frontends: false`. This mode does not support P/D -or multiple logical endpoints; use `frontend.type: vllm` for those topologies. - ### trtllm_serve frontend `type: trtllm_serve` runs the `trtllm-serve disaggregated` orchestrator as the @@ -488,7 +481,7 @@ combination during configuration loading. ### vLLM device binding compatibility -The `vllm` Router and `vllm-direct` frontends use vLLM's `--device-ids` option by +Direct `vllm` and `vllm-router` frontends use vLLM's `--device-ids` option by default. For stable vLLM releases from before [vllm-project/vllm#45026](https://github.com/vllm-project/vllm/pull/45026), select the existing CUDA namespace binding instead: diff --git a/docs/sglang-router.md b/docs/sglang-router.md index 1f8b55aa9..2d2553fcb 100644 --- a/docs/sglang-router.md +++ b/docs/sglang-router.md @@ -40,12 +40,13 @@ Enable sglang router in your recipe's `frontend` section: ```yaml frontend: - type: sglang + type: sgl-router ``` -Workers launch with `sglang.launch_server` instead of `dynamo.sglang`, and the -native SGLang router receives only logical worker-leader URLs from srtctl's -allocated topology. +The legacy `type: sglang` spelling remains an exact compatibility alias. New +recipes should use `sgl-router`. Workers launch with `sglang.launch_server` +instead of `dynamo.sglang`, and the router receives only logical worker-leader +URLs from srtctl's allocated topology. ### Router Arguments @@ -53,7 +54,7 @@ Pass extra CLI args to the router: ```yaml frontend: - type: sglang + type: sgl-router args: kv-overlap-score-weight: 1 router-temperature: 0 @@ -77,7 +78,7 @@ Pass environment variables to frontend processes: ```yaml frontend: - type: sglang + type: sgl-router env: MY_CUSTOM_VAR: "value" ``` @@ -90,7 +91,7 @@ The simplest mode - one router on node 0, no nginx: ```yaml frontend: - type: sglang + type: sgl-router enable_multiple_frontends: false ``` @@ -114,7 +115,7 @@ Nginx load balances across multiple router instances: ```yaml frontend: - type: sglang + type: sgl-router enable_multiple_frontends: true # default num_additional_frontends: 9 # default, total = 1 + 9 = 10 routers ``` @@ -203,7 +204,7 @@ resources: decode_workers: 2 frontend: - type: sglang + type: sgl-router enable_multiple_frontends: true num_additional_frontends: 3 # 4 total routers diff --git a/src/srtctl/README.md b/src/srtctl/README.md index f003404b5..f48a1941c 100644 --- a/src/srtctl/README.md +++ b/src/srtctl/README.md @@ -106,7 +106,7 @@ wait_for_model( port=8000, n_prefill=2, n_decode=4, - frontend_type="sglang", # native SGLang Router; or "dynamo" + frontend_type="sgl-router", # or "dynamo"; "sglang" remains an alias timeout=300, ) ``` diff --git a/src/srtctl/backends/sglang.py b/src/srtctl/backends/sglang.py index 66cc6d4e1..1c2d30e6d 100644 --- a/src/srtctl/backends/sglang.py +++ b/src/srtctl/backends/sglang.py @@ -296,7 +296,7 @@ def build_worker_command( process: The process to start endpoint_processes: All processes for this endpoint (for multi-node) runtime: Runtime context with paths and settings - frontend_type: Frontend type - "sglang" uses + frontend_type: Frontend type - "sglang"/"sgl-router" use sglang.launch_server, while "dynamo" uses dynamo.sglang nsys_prefix: Optional nsys profiling command prefix dump_config_path: Path to dump config JSON @@ -321,7 +321,7 @@ def build_worker_command( dist_init_port = SGLANG_DIST_INIT_PORT_BASE # Choose Python module based on frontend type - use_sglang = frontend_type == "sglang" + use_sglang = frontend_type in {"sglang", "sgl-router"} python_module = "sglang.launch_server" if use_sglang else "dynamo.sglang" # Get served model name from config @@ -386,7 +386,7 @@ def build_worker_command( ) # Add config dump path (not when using sglang frontend) - if dump_config_path and frontend_type != "sglang": + if dump_config_path and frontend_type not in {"sglang", "sgl-router"}: cmd.extend(["--dump-config-to", str(dump_config_path)]) # Add kv-events-config if enabled for this mode and we have an allocated port diff --git a/src/srtctl/backends/vllm.py b/src/srtctl/backends/vllm.py index 6e78c962a..6dfda324b 100644 --- a/src/srtctl/backends/vllm.py +++ b/src/srtctl/backends/vllm.py @@ -498,13 +498,13 @@ def endpoints_to_processes( """Convert endpoints to processes. Dynamo DP+EP mode uses the configured per-GPU or per-node process layout. - For vLLM Router and direct vLLM jobs, `vllm serve` manages local DP ranks + For direct vLLM and vLLM Router jobs, `vllm serve` manages local DP ranks from one process, so keep the standard one-process-per-node topology. For standard TP mode, creates one process per node. """ from srtctl.core.topology import NodePortAllocator, Process, endpoints_to_processes - if frontend_type in {"vllm", "vllm-direct"}: + if frontend_type in {"vllm", "vllm-router"}: return endpoints_to_processes(endpoints, base_sys_port=base_sys_port, port_allocator=port_allocator) # Check if any endpoint uses DP mode @@ -675,7 +675,7 @@ def build_worker_command( process: The process to start endpoint_processes: All processes for this endpoint (for multi-node) runtime: Runtime context with paths and settings - frontend_type: Frontend type ("dynamo", router "vllm", or "vllm-direct") + frontend_type: Frontend type ("dynamo", direct "vllm", or "vllm-router") nsys_prefix: Optional nsys profiling command prefix dump_config_path: Path to dump config JSON profiling: Profiling config; drives --profiler-config for iteration-based nsys @@ -714,9 +714,9 @@ def build_worker_command( } ) - if frontend_type in {"vllm", "vllm-direct"}: - if frontend_type == "vllm-direct" and mode != "agg": - raise ValueError("frontend.type: vllm-direct supports aggregate vLLM jobs only") + if frontend_type in {"vllm", "vllm-router"}: + if frontend_type == "vllm" and mode != "agg": + raise ValueError("frontend.type: vllm supports aggregate vLLM jobs only") if is_multi_node: raise ValueError(f"frontend.type: {frontend_type} requires each vLLM endpoint to fit on one node") @@ -724,7 +724,7 @@ def build_worker_command( config.pop("port", None) config.setdefault("served-model-name", served_model_name) - if frontend_type == "vllm-direct": + if frontend_type == "vllm": config.pop("connector", None) worker_port = runtime.frontend_port else: diff --git a/src/srtctl/benchmarks/router.py b/src/srtctl/benchmarks/router.py index 8e15b0a16..27618af03 100644 --- a/src/srtctl/benchmarks/router.py +++ b/src/srtctl/benchmarks/router.py @@ -44,8 +44,8 @@ def validate_config(self, config: SrtConfig) -> list[str]: errors = [] # Router benchmark exercises the SGLang router's prefix-aware policies. - if config.frontend.type != "sglang": - errors.append("router benchmark requires frontend.type: sglang") + if config.frontend.type not in {"sglang", "sgl-router"}: + errors.append("router benchmark requires frontend.type: sgl-router") return errors diff --git a/src/srtctl/cli/do_sweep.py b/src/srtctl/cli/do_sweep.py index 7c5cb5886..09c1d0fbd 100644 --- a/src/srtctl/cli/do_sweep.py +++ b/src/srtctl/cli/do_sweep.py @@ -677,7 +677,7 @@ def run(self) -> int: try: # Stage 1: Head infrastructure (NATS, etcd). Only the dynamo request # plane uses it; static/direct frontends skip it. - if self.config.frontend.type in {"sglang", "trtllm_serve", "vllm", "vllm-direct"}: + if self.config.frontend.type in {"sglang", "sgl-router", "trtllm_serve", "vllm", "vllm-router"}: logger.info("Skipping NATS/etcd infrastructure (frontend.type=%s)", self.config.frontend.type) else: reporter.report(JobStatus.STARTING, JobStage.HEAD_INFRASTRUCTURE, "Starting head infrastructure") diff --git a/src/srtctl/cli/mixins/benchmark_stage.py b/src/srtctl/cli/mixins/benchmark_stage.py index 087841ff2..244dab270 100644 --- a/src/srtctl/cli/mixins/benchmark_stage.py +++ b/src/srtctl/cli/mixins/benchmark_stage.py @@ -430,8 +430,8 @@ def _get_sa_bench_slow_down_env(self) -> dict[str, str]: "benchmark slow_down: slow_down_sleep_time and slow_down_wait_time must be positive; skipping" ) return {} - if self.config.frontend.type != "sglang": - logger.warning("benchmark.slow_down_* ignored: frontend.type is not sglang") + if self.config.frontend.type not in {"sglang", "sgl-router"}: + logger.warning("benchmark.slow_down_* ignored: frontend.type is not sgl-router") return {} decode_urls: list[str] = [] @@ -472,15 +472,11 @@ def _get_aiperf_server_metrics_env( logical_endpoints = self._logical_worker_endpoints() urls = [f"http://{host}:{port}/metrics" for _, host, port in logical_endpoints] else: - if self.config.frontend.type in {"vllm", "vllm-direct"}: + if self.config.frontend.type in {"vllm", "vllm-router"}: for process in self.backend_processes: if process.is_leader: host = get_hostname_ip(process.node, self.runtime.network_interface) - port = ( - FRONTEND_PUBLIC_PORT - if self.config.frontend.type == "vllm-direct" - else process.http_port - ) + port = FRONTEND_PUBLIC_PORT if self.config.frontend.type == "vllm" else process.http_port urls.append(f"http://{host}:{port}/metrics") if urls: return {"AIPERF_SERVER_METRICS_URLS": ",".join(sorted(set(urls)))} diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py index 9d036ef95..6fe7ac0d1 100755 --- a/src/srtctl/core/schema.py +++ b/src/srtctl/core/schema.py @@ -1437,8 +1437,8 @@ class FrontendConfig: """Frontend/router configuration. Attributes: - type: Frontend type - "dynamo" (default), SGLang Router "sglang", - vLLM Router "vllm", direct "vllm-direct", or "trtllm_serve". + type: Frontend type - "dynamo" (default), "sgl-router", "vllm-router", + "trtllm_serve", or direct "vllm". "sglang" remains a compatibility alias. enable_multiple_frontends: Scale with nginx + multiple routers. When ``True`` (default), srtctl stands up nginx and fans out to ``num_additional_frontends + 1`` router replicas. When @@ -1591,7 +1591,7 @@ def __post_init__(self): self._validate_mooncake_kv_store() self._validate_het_jobs() self._validate_trtllm_serve() - self._validate_vllm_direct_frontend() + self._validate_vllm_frontend() self._validate_static_router_frontend() self._validate_sglang_data_parallelism() @@ -1619,38 +1619,34 @@ def _validate_trtllm_serve(self): "(set resources.prefill_nodes/prefill_workers and decode_nodes/decode_workers)" ) - def _validate_vllm_direct_frontend(self): + def _validate_vllm_frontend(self): """Catch direct-vLLM frontend misconfigurations at load time. Direct vLLM means the aggregate `vllm serve` worker owns the OpenAI port itself. It is not a disaggregated router and does not support the nginx multi-frontend path. """ - if self.frontend.type != "vllm-direct": + if self.frontend.type != "vllm": return if self.backend_type != "vllm": - raise ValidationError( - f"frontend.type: vllm-direct requires backend.type: vllm; got {self.backend_type!r}" - ) + raise ValidationError(f"frontend.type: vllm requires backend.type: vllm; got {self.backend_type!r}") if self.frontend.enable_multiple_frontends: raise ValidationError( - "frontend.type: vllm-direct binds vllm serve directly; " - "set frontend.enable_multiple_frontends: false" + "frontend.type: vllm binds vllm serve directly; set frontend.enable_multiple_frontends: false" ) if self.resources.is_disaggregated: - raise ValidationError( - "frontend.type: vllm-direct supports aggregate jobs only, not disaggregated layouts" - ) + raise ValidationError("frontend.type: vllm supports aggregate jobs only, not disaggregated layouts") if self.resources.num_agg < 1: - raise ValidationError("frontend.type: vllm-direct requires resources.agg_workers >= 1") + raise ValidationError("frontend.type: vllm requires resources.agg_workers >= 1") if (self.resources.agg_nodes or 1) != 1: - raise ValidationError("frontend.type: vllm-direct currently supports single-node aggregate jobs only") + raise ValidationError("frontend.type: vllm currently supports single-node aggregate jobs only") def _validate_static_router_frontend(self): """Validate native static-router/backend pairings and endpoint shape.""" required_backend = { "sglang": "sglang", - "vllm": "vllm", + "sgl-router": "sglang", + "vllm-router": "vllm", }.get(self.frontend.type) if required_backend is None: return @@ -1660,7 +1656,7 @@ def _validate_static_router_frontend(self): f"got {self.backend_type!r}" ) - if self.frontend.type == "vllm": + if self.frontend.type == "vllm-router": endpoint_gpu_counts = ( self.resources.gpus_per_prefill if self.resources.num_prefill else 0, self.resources.gpus_per_decode if self.resources.num_decode else 0, @@ -1668,7 +1664,7 @@ def _validate_static_router_frontend(self): ) if any(count > self.resources.gpus_per_node for count in endpoint_gpu_counts): raise ValidationError( - "frontend.type: vllm currently requires each logical vLLM endpoint " + "frontend.type: vllm-router currently requires each logical vLLM endpoint " "to fit on one node; scale with multiple aggregate/prefill/decode workers" ) diff --git a/src/srtctl/core/telemetry.py b/src/srtctl/core/telemetry.py index ea3810717..79a8974d1 100644 --- a/src/srtctl/core/telemetry.py +++ b/src/srtctl/core/telemetry.py @@ -86,9 +86,9 @@ def generate_telemetry_config( for process in sorted(processes, key=lambda p: (p.endpoint_mode, p.endpoint_index, p.node_rank, p.node)): node_ip = get_hostname_ip(process.node, runtime.network_interface) - if frontend_type == "vllm-direct" and process.endpoint_mode == "agg": + if frontend_type == "vllm" and process.endpoint_mode == "agg": port = FRONTEND_PUBLIC_PORT - elif frontend_type == "vllm": + elif frontend_type == "vllm-router": port = process.http_port else: port = process.sys_port diff --git a/src/srtctl/frontends/__init__.py b/src/srtctl/frontends/__init__.py index 846c42345..06912a331 100644 --- a/src/srtctl/frontends/__init__.py +++ b/src/srtctl/frontends/__init__.py @@ -6,9 +6,10 @@ Supported frontend types: - dynamo: Dynamo frontend with NATS/etcd communication -- sglang: SGLang Model Gateway with direct worker connections -- vllm: Official vLLM Router with direct worker connections -- vllm-direct: Direct vLLM OpenAI server for aggregate jobs +- sgl-router: SGLang Model Gateway with direct worker connections +- sglang: Backward-compatible alias for sgl-router +- vllm: Direct vLLM OpenAI server for aggregate jobs +- vllm-router: vLLM Router with direct worker connections """ from srtctl.frontends.base import ( @@ -17,18 +18,19 @@ get_frontend, ) from srtctl.frontends.dynamo import DynamoFrontend -from srtctl.frontends.sglang import SGLangFrontend +from srtctl.frontends.sglang import SGLangFrontend, SGLRouterFrontend from srtctl.frontends.trtllm_serve import TRTLLMServeFrontend from srtctl.frontends.vllm import VLLMFrontend -from srtctl.frontends.vllm_direct import VLLMDirectFrontend +from srtctl.frontends.vllm_router import VLLMRouterFrontend __all__ = [ "DynamoFrontend", "FrontendProtocol", "FrontendType", + "SGLRouterFrontend", "SGLangFrontend", "TRTLLMServeFrontend", - "VLLMDirectFrontend", "VLLMFrontend", + "VLLMRouterFrontend", "get_frontend", ] diff --git a/src/srtctl/frontends/base.py b/src/srtctl/frontends/base.py index 1a6cf0bc2..c808f4918 100644 --- a/src/srtctl/frontends/base.py +++ b/src/srtctl/frontends/base.py @@ -22,7 +22,7 @@ from srtctl.core.topology import Process # Supported frontend types - extensible by adding new literals -FrontendType = Literal["dynamo", "sglang", "trtllm_serve", "vllm", "vllm-direct"] +FrontendType = Literal["dynamo", "sglang", "sgl-router", "trtllm_serve", "vllm", "vllm-router"] FrontendFactory = Callable[[], "FrontendProtocol"] _FRONTEND_REGISTRY: dict[str, FrontendFactory] = {} @@ -44,7 +44,7 @@ def decorator(frontend_class: _FrontendClass) -> _FrontendClass: def _load_builtin_frontends() -> None: """Import built-ins once so their registration decorators run.""" - from srtctl.frontends import dynamo, sglang, trtllm_serve, vllm, vllm_direct # noqa: F401 + from srtctl.frontends import dynamo, sglang, trtllm_serve, vllm, vllm_router # noqa: F401 def build_setup_script_preamble(setup_script: str | None) -> str | None: diff --git a/src/srtctl/frontends/sglang.py b/src/srtctl/frontends/sglang.py index 3a6d779cf..c210e0ffb 100644 --- a/src/srtctl/frontends/sglang.py +++ b/src/srtctl/frontends/sglang.py @@ -10,11 +10,11 @@ from srtctl.frontends.static_router import StaticRouterFrontend -@register_frontend("sglang") -class SGLangFrontend(StaticRouterFrontend): - """SGLang Model Gateway static router.""" +@register_frontend("sgl-router") +class SGLRouterFrontend(StaticRouterFrontend): + """First-class SGLang Model Gateway static router.""" - type: ClassVar[str] = "sglang" + type: ClassVar[str] = "sgl-router" backend_type: ClassVar[str] = "sglang" executable: ClassVar[tuple[str, ...]] = ("python", "-m", "sglang_router.launch_router") pd_flag: ClassVar[str] = "--pd-disaggregation" @@ -28,3 +28,10 @@ def get_hostname_ip(self, node: str) -> str: def start_process(self, **kwargs: Any) -> Any: return start_srun_process(**kwargs) + + +@register_frontend("sglang") +class SGLangFrontend(SGLRouterFrontend): + """Backward-compatible alias for the historical ``sglang`` frontend type.""" + + type: ClassVar[str] = "sglang" diff --git a/src/srtctl/frontends/vllm.py b/src/srtctl/frontends/vllm.py index 08107c2cd..c1af62669 100644 --- a/src/srtctl/frontends/vllm.py +++ b/src/srtctl/frontends/vllm.py @@ -1,39 +1,91 @@ # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Official vLLM Router frontend.""" +""" +Direct vLLM frontend implementation. + +For aggregate vLLM jobs the OpenAI-compatible HTTP server is the worker +process itself (`vllm serve`). There is no separate router/frontend process. +""" from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar +import logging +import threading +from typing import TYPE_CHECKING, Any +from srtctl.core.health import WorkerHealthResult from srtctl.frontends.base import register_frontend -from srtctl.frontends.static_router import StaticRouterFrontend if TYPE_CHECKING: + from srtctl.core.processes import ManagedProcess + from srtctl.core.runtime import RuntimeContext from srtctl.core.topology import Process +logger = logging.getLogger(__name__) + @register_frontend("vllm") -class VLLMFrontend(StaticRouterFrontend): - """Route requests through the official vLLM Router.""" - - type: ClassVar[str] = "vllm" - backend_type: ClassVar[str] = "vllm" - executable: ClassVar[tuple[str, ...]] = ("vllm-router",) - pd_flag: ClassVar[str] = "--vllm-pd-disaggregation" - process_name: ClassVar[str] = "vllm_router" - - def get_managed_frontend_args(self, config: Any) -> list[str]: - """Keep Router's worker wait alive for srtctl's model-readiness window.""" - frontend_args = config.frontend.args or {} - if "worker-startup-timeout-secs" in frontend_args: +class VLLMFrontend: + """Direct vLLM OpenAI server frontend. + + This frontend is intentionally narrow: aggregate vLLM jobs only, with the + backend worker binding the public OpenAI port directly. Disaggregated vLLM + still needs a real router/orchestrator such as Dynamo. + """ + + @property + def type(self) -> str: + return "vllm" + + @property + def health_endpoint(self) -> str: + return "/health" + + def parse_health( + self, + response_json: dict, + expected_prefill: int, + expected_decode: int, + ) -> WorkerHealthResult: + return WorkerHealthResult( + ready=True, + message="vLLM OpenAI server healthy", + prefill_ready=expected_prefill, + prefill_expected=expected_prefill, + decode_ready=expected_decode, + decode_expected=expected_decode, + ) + + def get_frontend_args_list(self, args: dict[str, Any] | None) -> list[str]: + if not args: return [] + result = [] + for key, value in args.items(): + if value is True: + result.append(f"--{key}") + elif value is not False and value is not None: + result.extend([f"--{key}", str(value)]) + return result - health_check = config.health_check - timeout_seconds = health_check.max_attempts * health_check.interval_seconds - return ["--worker-startup-timeout-secs", str(timeout_seconds)] + def start_frontends( + self, + topology: Any, + runtime: RuntimeContext, + config: Any, + backend: Any, + backend_processes: list[Process], + stop_event: threading.Event | None = None, + ) -> list[ManagedProcess]: + if config.backend.type != "vllm": + raise ValueError(f"frontend.type: vllm requires backend.type: vllm (got {config.backend.type!r})") + if topology.uses_nginx or len(topology.frontend_nodes) != 1: + raise ValueError( + "frontend.type: vllm binds vllm serve directly to the public port; " + "set frontend.enable_multiple_frontends: false" + ) + if config.resources.is_disaggregated or config.resources.num_agg < 1: + raise ValueError("frontend.type: vllm supports aggregate vLLM jobs only") - def worker_bootstrap_port(self, backend: Any, process: Process) -> int | None: - """Advertise vLLM's NIXL side-channel port to the P/D router.""" - return process.nixl_port + logger.info("frontend.type=vllm: no separate frontend process; vllm serve owns port %d", topology.public_port) + return [] diff --git a/src/srtctl/frontends/vllm_direct.py b/src/srtctl/frontends/vllm_direct.py deleted file mode 100644 index 60a23f33d..000000000 --- a/src/srtctl/frontends/vllm_direct.py +++ /dev/null @@ -1,84 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Direct vLLM serving without a separate frontend process.""" - -from __future__ import annotations - -import logging -import threading -from typing import TYPE_CHECKING, Any - -from srtctl.core.health import WorkerHealthResult -from srtctl.frontends.base import register_frontend - -if TYPE_CHECKING: - from srtctl.core.processes import ManagedProcess - from srtctl.core.runtime import RuntimeContext - from srtctl.core.topology import Process - -logger = logging.getLogger(__name__) - - -@register_frontend("vllm-direct") -class VLLMDirectFrontend: - """Expose one aggregate ``vllm serve`` endpoint without a router.""" - - @property - def type(self) -> str: - return "vllm-direct" - - @property - def health_endpoint(self) -> str: - return "/health" - - def parse_health( - self, - response_json: dict, - expected_prefill: int, - expected_decode: int, - ) -> WorkerHealthResult: - return WorkerHealthResult( - ready=True, - message="vLLM OpenAI server healthy", - prefill_ready=expected_prefill, - prefill_expected=expected_prefill, - decode_ready=expected_decode, - decode_expected=expected_decode, - ) - - def get_frontend_args_list(self, args: dict[str, Any] | None) -> list[str]: - if not args: - return [] - result = [] - for key, value in args.items(): - if value is True: - result.append(f"--{key}") - elif value is not False and value is not None: - result.extend([f"--{key}", str(value)]) - return result - - def start_frontends( - self, - topology: Any, - runtime: RuntimeContext, - config: Any, - backend: Any, - backend_processes: list[Process], - stop_event: threading.Event | None = None, - ) -> list[ManagedProcess]: - if config.backend.type != "vllm": - raise ValueError(f"frontend.type: vllm-direct requires backend.type: vllm (got {config.backend.type!r})") - if topology.uses_nginx or len(topology.frontend_nodes) != 1: - raise ValueError( - "frontend.type: vllm-direct binds vllm serve directly to the public port; " - "set frontend.enable_multiple_frontends: false" - ) - if config.resources.is_disaggregated or config.resources.num_agg < 1: - raise ValueError("frontend.type: vllm-direct supports aggregate vLLM jobs only") - - logger.info( - "frontend.type=vllm-direct: no separate frontend process; vllm serve owns port %d", - topology.public_port, - ) - return [] diff --git a/src/srtctl/frontends/vllm_router.py b/src/srtctl/frontends/vllm_router.py new file mode 100644 index 000000000..cdfb3e3c6 --- /dev/null +++ b/src/srtctl/frontends/vllm_router.py @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""vLLM Router frontend.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, ClassVar + +from srtctl.frontends.base import register_frontend +from srtctl.frontends.static_router import StaticRouterFrontend + +if TYPE_CHECKING: + from srtctl.core.topology import Process + + +@register_frontend("vllm-router") +class VLLMRouterFrontend(StaticRouterFrontend): + """Route requests to direct vLLM OpenAI-compatible worker endpoints.""" + + type: ClassVar[str] = "vllm-router" + backend_type: ClassVar[str] = "vllm" + executable: ClassVar[tuple[str, ...]] = ("vllm-router",) + pd_flag: ClassVar[str] = "--vllm-pd-disaggregation" + process_name: ClassVar[str] = "vllm_router" + + def get_managed_frontend_args(self, config: Any) -> list[str]: + """Keep Router's worker wait alive for srtctl's model-readiness window.""" + frontend_args = config.frontend.args or {} + if "worker-startup-timeout-secs" in frontend_args: + return [] + + health_check = config.health_check + timeout_seconds = health_check.max_attempts * health_check.interval_seconds + return ["--worker-startup-timeout-secs", str(timeout_seconds)] + + def worker_bootstrap_port(self, backend: Any, process: Process) -> int | None: + """Advertise vLLM's NIXL side-channel port to the P/D router.""" + return process.nixl_port diff --git a/tests/test_configs.py b/tests/test_configs.py index d4a9fa16a..86b1605e3 100644 --- a/tests/test_configs.py +++ b/tests/test_configs.py @@ -544,9 +544,9 @@ def test_grpc_mode_enabled_per_mode(self): @pytest.mark.parametrize( ("frontend_type", "mode", "expected"), [ - ("sglang", "prefill", True), - ("sglang", "decode", True), - ("sglang", "agg", False), + ("sgl-router", "prefill", True), + ("sgl-router", "decode", True), + ("sgl-router", "agg", False), ("dynamo", "decode", False), ], ) @@ -864,7 +864,7 @@ def test_router_container_alias_resolves(self): "name": "test", "model": {"path": "/model", "container": "worker", "precision": "fp8"}, "resources": {"gpu_type": "h100", "gpus_per_node": 8, "agg_nodes": 1}, - "frontend": {"type": "vllm", "container_image": "router"}, + "frontend": {"type": "vllm-router", "container_image": "router"}, } cluster_config = { "containers": { @@ -2130,7 +2130,7 @@ def test_direct_vllm_dp_mode_keeps_single_process(self): gpus_per_node=8, ) - processes = backend.endpoints_to_processes([endpoint], frontend_type="vllm-direct") + processes = backend.endpoints_to_processes([endpoint], frontend_type="vllm") assert len(processes) == 1 assert processes[0].node == "node0" @@ -2171,7 +2171,7 @@ def test_direct_vllm_command_preserves_current_main_device_binding(self): process=process, endpoint_processes=[process], runtime=runtime, - frontend_type="vllm-direct", + frontend_type="vllm", ) assert cmd[:3] == ["vllm", "serve", "/model"] @@ -2197,7 +2197,7 @@ def test_vllm_router_keeps_one_direct_server_per_logical_endpoint(self): for index, node in enumerate(("node0", "node1")) ] - processes = backend.endpoints_to_processes(endpoints, frontend_type="vllm") + processes = backend.endpoints_to_processes(endpoints, frontend_type="vllm-router") assert len(processes) == 2 assert all(process.is_leader for process in processes) @@ -2235,7 +2235,7 @@ def test_vllm_router_worker_uses_private_port_and_pd_connector(self): process=process, endpoint_processes=[process], runtime=runtime, - frontend_type="vllm", + frontend_type="vllm-router", ) assert cmd[:3] == ["vllm", "serve", "/model"] @@ -2275,7 +2275,7 @@ def test_vllm_router_stable_release_uses_legacy_cuda_binding(self): process=process, endpoint_processes=[process], runtime=runtime, - frontend_type="vllm", + frontend_type="vllm-router", ) assert cmd[:3] == ["vllm", "serve", "/model"] @@ -2318,7 +2318,7 @@ def test_direct_vllm_command_keeps_iteration_profiler_config(self): process=process, endpoint_processes=[process], runtime=runtime, - frontend_type="vllm-direct", + frontend_type="vllm", profiling=profiling, ) diff --git a/tests/test_frontends.py b/tests/test_frontends.py index 7a3240429..f00507c8d 100644 --- a/tests/test_frontends.py +++ b/tests/test_frontends.py @@ -11,7 +11,7 @@ import pytest from srtctl.core.schema import ObservabilityConfig -from srtctl.frontends import DynamoFrontend, SGLangFrontend, VLLMDirectFrontend, VLLMFrontend, get_frontend +from srtctl.frontends import DynamoFrontend, SGLangFrontend, VLLMFrontend, get_frontend # ============================================================================ # get_frontend() Tests @@ -34,17 +34,11 @@ def test_get_sglang_frontend(self): assert frontend.type == "sglang" def test_get_vllm_frontend(self): - """get_frontend('vllm') returns the vLLM Router frontend.""" + """get_frontend('vllm') returns VLLMFrontend.""" frontend = get_frontend("vllm") assert isinstance(frontend, VLLMFrontend) assert frontend.type == "vllm" - def test_get_vllm_direct_frontend(self): - """get_frontend('vllm-direct') returns the router-free adapter.""" - frontend = get_frontend("vllm-direct") - assert isinstance(frontend, VLLMDirectFrontend) - assert frontend.type == "vllm-direct" - def test_get_unknown_frontend_raises(self): """get_frontend() with unknown type raises ValueError.""" with pytest.raises(ValueError, match="Unknown frontend type"): @@ -73,15 +67,10 @@ def test_sglang_type(self): assert frontend.type == "sglang" def test_vllm_type(self): - """The vLLM Router frontend type is 'vllm'.""" + """VLLMFrontend.type is 'vllm'.""" frontend = VLLMFrontend() assert frontend.type == "vllm" - def test_vllm_direct_type(self): - """The direct adapter is explicitly named 'vllm-direct'.""" - frontend = VLLMDirectFrontend() - assert frontend.type == "vllm-direct" - def test_dynamo_health_endpoint(self): """DynamoFrontend uses /health endpoint.""" frontend = DynamoFrontend() @@ -93,13 +82,8 @@ def test_sglang_health_endpoint(self): assert frontend.health_endpoint == "/workers" def test_vllm_health_endpoint(self): - """vLLM Router uses its worker-registration endpoint.""" + """VLLMFrontend uses /health endpoint.""" frontend = VLLMFrontend() - assert frontend.health_endpoint == "/workers" - - def test_vllm_direct_health_endpoint(self): - """Direct vLLM uses the server health endpoint.""" - frontend = VLLMDirectFrontend() assert frontend.health_endpoint == "/health" diff --git a/tests/test_static_router_frontends.py b/tests/test_static_router_frontends.py index 99f689c23..154785e14 100644 --- a/tests/test_static_router_frontends.py +++ b/tests/test_static_router_frontends.py @@ -9,17 +9,17 @@ import pytest -from srtctl.frontends import SGLangFrontend, VLLMDirectFrontend, VLLMFrontend, get_frontend +from srtctl.frontends import SGLRouterFrontend, VLLMRouterFrontend, get_frontend from srtctl.frontends.static_router import RouterWorker -def test_registry_uses_engine_names_for_routers_and_explicit_direct_name() -> None: - assert isinstance(get_frontend("sglang"), SGLangFrontend) - assert isinstance(get_frontend("vllm"), VLLMFrontend) - assert isinstance(get_frontend("vllm-direct"), VLLMDirectFrontend) +def test_registry_exposes_explicit_router_names_and_legacy_alias() -> None: + assert isinstance(get_frontend("sgl-router"), SGLRouterFrontend) + assert get_frontend("sglang").type == "sglang" + assert isinstance(get_frontend("vllm-router"), VLLMRouterFrontend) -@pytest.mark.parametrize("frontend", [SGLangFrontend(), VLLMFrontend()]) +@pytest.mark.parametrize("frontend", [SGLRouterFrontend(), VLLMRouterFrontend()]) def test_aggregate_command_advertises_all_logical_workers(frontend) -> None: command = frontend.build_router_command( [ @@ -38,8 +38,8 @@ def test_aggregate_command_advertises_all_logical_workers(frontend) -> None: @pytest.mark.parametrize( ("frontend", "pd_flag"), [ - (SGLangFrontend(), "--pd-disaggregation"), - (VLLMFrontend(), "--vllm-pd-disaggregation"), + (SGLRouterFrontend(), "--pd-disaggregation"), + (VLLMRouterFrontend(), "--vllm-pd-disaggregation"), ], ) def test_disaggregated_command_preserves_modes_and_bootstrap(frontend, pd_flag: str) -> None: @@ -61,7 +61,7 @@ def test_disaggregated_command_preserves_modes_and_bootstrap(frontend, pd_flag: def test_router_command_rejects_incomplete_or_mixed_topology() -> None: - frontend = VLLMFrontend() + frontend = VLLMRouterFrontend() with pytest.raises(ValueError, match="requires prefill and decode"): frontend.build_router_command([RouterWorker("prefill", "http://p:1")], "0.0.0.0", 8000) with pytest.raises(ValueError, match="cannot mix"): @@ -77,7 +77,7 @@ def test_router_command_rejects_incomplete_or_mixed_topology() -> None: def test_frontend_args_repeat_list_values() -> None: - frontend = VLLMFrontend() + frontend = VLLMRouterFrontend() assert frontend.get_frontend_args_list({"routing-logic": ["round_robin", "session"]}) == [ "--routing-logic", "round_robin", @@ -87,7 +87,7 @@ def test_frontend_args_repeat_list_values() -> None: def test_vllm_router_advertises_nixl_side_channel_port() -> None: - frontend = VLLMFrontend() + frontend = VLLMRouterFrontend() process = SimpleNamespace( is_leader=True, endpoint_mode="prefill", @@ -104,7 +104,7 @@ def test_vllm_router_advertises_nixl_side_channel_port() -> None: def test_vllm_router_launch_uses_router_container_env_and_only_leaders() -> None: - frontend = VLLMFrontend() + frontend = VLLMRouterFrontend() runtime = SimpleNamespace( log_dir=Path("/logs"), container_image=Path("/worker.sqsh"), @@ -160,7 +160,7 @@ def test_vllm_router_launch_uses_router_container_env_and_only_leaders() -> None def test_vllm_router_explicit_worker_startup_timeout_overrides_managed_value() -> None: - frontend = VLLMFrontend() + frontend = VLLMRouterFrontend() config = SimpleNamespace( health_check=SimpleNamespace(max_attempts=360, interval_seconds=10), frontend=SimpleNamespace(args={"worker-startup-timeout-secs": 7200}), @@ -175,7 +175,7 @@ def test_vllm_router_explicit_worker_startup_timeout_overrides_managed_value() - def test_router_rejects_backend_mismatch_before_launch() -> None: - frontend = VLLMFrontend() + frontend = VLLMRouterFrontend() config = SimpleNamespace( backend=SimpleNamespace(type="sglang"), frontend=SimpleNamespace(args=None, env=None, container_image=None), @@ -193,12 +193,12 @@ def test_schema_rejects_router_backend_mismatch() -> None: from srtctl.backends import SGLangProtocol from srtctl.core.schema import FrontendConfig, ResourceConfig, SrtConfig - with pytest.raises(ValidationError, match="vllm requires backend.type: vllm"): + with pytest.raises(ValidationError, match="vllm-router requires backend.type: vllm"): SrtConfig( name="bad-router-pair", model={"path": "model", "container": "image", "precision": "fp8"}, resources=ResourceConfig(gpu_type="h100", gpus_per_node=8, agg_nodes=1, agg_workers=1), - frontend=FrontendConfig(type="vllm", enable_multiple_frontends=False), + frontend=FrontendConfig(type="vllm-router", enable_multiple_frontends=False), backend=SGLangProtocol(), ) @@ -216,7 +216,7 @@ def test_vllm_router_accepts_many_single_node_endpoints() -> None: agg_nodes=4, agg_workers=4, ), - frontend=FrontendConfig(type="vllm", enable_multiple_frontends=False), + frontend=FrontendConfig(type="vllm-router", enable_multiple_frontends=False), backend=VLLMProtocol(), ) @@ -241,7 +241,7 @@ def test_vllm_router_rejects_endpoint_spanning_nodes() -> None: decode_nodes=1, decode_workers=1, ), - frontend=FrontendConfig(type="vllm", enable_multiple_frontends=False), + frontend=FrontendConfig(type="vllm-router", enable_multiple_frontends=False), backend=VLLMProtocol(), ) @@ -257,7 +257,7 @@ def test_sgl_router_rejects_non_divisible_tp_dp_layout() -> None: name="invalid-sglang-dpa", model={"path": "model", "container": "image", "precision": "fp8"}, resources=ResourceConfig(gpu_type="h100", gpus_per_node=8, agg_nodes=1, agg_workers=1), - frontend=FrontendConfig(type="sglang", enable_multiple_frontends=False), + frontend=FrontendConfig(type="sgl-router", enable_multiple_frontends=False), backend=SGLangProtocol( sglang_config=SGLangServerConfig(aggregated={"tp-size": 1, "dp-size": 8, "enable-dp-attention": True}) ), @@ -272,7 +272,7 @@ def test_sgl_router_accepts_divisible_tp_dp_layout() -> None: name="valid-sglang-dpa", model={"path": "model", "container": "image", "precision": "fp8"}, resources=ResourceConfig(gpu_type="h100", gpus_per_node=8, agg_nodes=1, agg_workers=1), - frontend=FrontendConfig(type="sglang", enable_multiple_frontends=False), + frontend=FrontendConfig(type="sgl-router", enable_multiple_frontends=False), backend=SGLangProtocol( sglang_config=SGLangServerConfig(aggregated={"tp-size": 8, "dp-size": 8, "enable-dp-attention": True}) ), From be02143642ec10b5dacd82cd2e2803c6897abe96 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 10 Aug 2026 12:45:15 -0500 Subject: [PATCH 11/46] preserve native frontend compatibility --- docs/config-reference.md | 6 +++--- docs/sglang-router.md | 19 +++++++++---------- src/srtctl/README.md | 2 +- src/srtctl/backends/sglang.py | 8 ++++---- src/srtctl/benchmarks/router.py | 4 ++-- src/srtctl/cli/do_sweep.py | 2 +- src/srtctl/cli/mixins/benchmark_stage.py | 4 ++-- src/srtctl/core/schema.py | 5 ++--- src/srtctl/frontends/__init__.py | 6 ++---- src/srtctl/frontends/base.py | 2 +- src/srtctl/frontends/sglang.py | 15 ++++----------- tests/test_configs.py | 6 +++--- tests/test_static_router_frontends.py | 15 +++++++-------- 13 files changed, 41 insertions(+), 53 deletions(-) diff --git a/docs/config-reference.md b/docs/config-reference.md index 4345ceb9c..1a614503b 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -270,8 +270,8 @@ Frontend/router configuration. ```yaml frontend: - # Frontend type: "dynamo" (default), "sgl-router", "vllm-router", - # "trtllm_serve", or direct "vllm". "sglang" is a compatibility alias. + # Frontend type: "dynamo" (default), "sglang", "vllm-router", + # "trtllm_serve", or direct "vllm". type: dynamo # Scaling @@ -298,7 +298,7 @@ frontend: | Field | Type | Default | Description | | --------------------------- | ---- | ------------- | ----------------------------------- | -| `type` | str | dynamo | Frontend type: `dynamo`, `sgl-router`, `vllm-router`, `trtllm_serve`, or direct `vllm`; `sglang` is a compatibility alias | +| `type` | str | dynamo | Frontend type: `dynamo`, `sglang`, `vllm-router`, `trtllm_serve`, or direct `vllm` | | `enable_multiple_frontends` | bool | true | Scale with nginx + multiple routers | | `num_additional_frontends` | int | 9 | Additional routers beyond master | | `nginx_container` | str | nginx:1.27.4 | Custom nginx container image | diff --git a/docs/sglang-router.md b/docs/sglang-router.md index 2d2553fcb..68f3de15f 100644 --- a/docs/sglang-router.md +++ b/docs/sglang-router.md @@ -40,13 +40,12 @@ Enable sglang router in your recipe's `frontend` section: ```yaml frontend: - type: sgl-router + type: sglang ``` -The legacy `type: sglang` spelling remains an exact compatibility alias. New -recipes should use `sgl-router`. Workers launch with `sglang.launch_server` -instead of `dynamo.sglang`, and the router receives only logical worker-leader -URLs from srtctl's allocated topology. +Workers launch with `sglang.launch_server` instead of `dynamo.sglang`, and the +router receives only logical worker-leader URLs from srtctl's allocated +topology. ### Router Arguments @@ -54,7 +53,7 @@ Pass extra CLI args to the router: ```yaml frontend: - type: sgl-router + type: sglang args: kv-overlap-score-weight: 1 router-temperature: 0 @@ -78,7 +77,7 @@ Pass environment variables to frontend processes: ```yaml frontend: - type: sgl-router + type: sglang env: MY_CUSTOM_VAR: "value" ``` @@ -91,7 +90,7 @@ The simplest mode - one router on node 0, no nginx: ```yaml frontend: - type: sgl-router + type: sglang enable_multiple_frontends: false ``` @@ -115,7 +114,7 @@ Nginx load balances across multiple router instances: ```yaml frontend: - type: sgl-router + type: sglang enable_multiple_frontends: true # default num_additional_frontends: 9 # default, total = 1 + 9 = 10 routers ``` @@ -204,7 +203,7 @@ resources: decode_workers: 2 frontend: - type: sgl-router + type: sglang enable_multiple_frontends: true num_additional_frontends: 3 # 4 total routers diff --git a/src/srtctl/README.md b/src/srtctl/README.md index f48a1941c..bad1da34c 100644 --- a/src/srtctl/README.md +++ b/src/srtctl/README.md @@ -106,7 +106,7 @@ wait_for_model( port=8000, n_prefill=2, n_decode=4, - frontend_type="sgl-router", # or "dynamo"; "sglang" remains an alias + frontend_type="sglang", # or "dynamo" timeout=300, ) ``` diff --git a/src/srtctl/backends/sglang.py b/src/srtctl/backends/sglang.py index 1c2d30e6d..833442451 100644 --- a/src/srtctl/backends/sglang.py +++ b/src/srtctl/backends/sglang.py @@ -296,8 +296,8 @@ def build_worker_command( process: The process to start endpoint_processes: All processes for this endpoint (for multi-node) runtime: Runtime context with paths and settings - frontend_type: Frontend type - "sglang"/"sgl-router" use - sglang.launch_server, while "dynamo" uses dynamo.sglang + frontend_type: Frontend type - "sglang" uses sglang.launch_server, + while "dynamo" uses dynamo.sglang nsys_prefix: Optional nsys profiling command prefix dump_config_path: Path to dump config JSON """ @@ -321,7 +321,7 @@ def build_worker_command( dist_init_port = SGLANG_DIST_INIT_PORT_BASE # Choose Python module based on frontend type - use_sglang = frontend_type in {"sglang", "sgl-router"} + use_sglang = frontend_type == "sglang" python_module = "sglang.launch_server" if use_sglang else "dynamo.sglang" # Get served model name from config @@ -386,7 +386,7 @@ def build_worker_command( ) # Add config dump path (not when using sglang frontend) - if dump_config_path and frontend_type not in {"sglang", "sgl-router"}: + if dump_config_path and frontend_type != "sglang": cmd.extend(["--dump-config-to", str(dump_config_path)]) # Add kv-events-config if enabled for this mode and we have an allocated port diff --git a/src/srtctl/benchmarks/router.py b/src/srtctl/benchmarks/router.py index 27618af03..8e15b0a16 100644 --- a/src/srtctl/benchmarks/router.py +++ b/src/srtctl/benchmarks/router.py @@ -44,8 +44,8 @@ def validate_config(self, config: SrtConfig) -> list[str]: errors = [] # Router benchmark exercises the SGLang router's prefix-aware policies. - if config.frontend.type not in {"sglang", "sgl-router"}: - errors.append("router benchmark requires frontend.type: sgl-router") + if config.frontend.type != "sglang": + errors.append("router benchmark requires frontend.type: sglang") return errors diff --git a/src/srtctl/cli/do_sweep.py b/src/srtctl/cli/do_sweep.py index 09c1d0fbd..8f385b6f6 100644 --- a/src/srtctl/cli/do_sweep.py +++ b/src/srtctl/cli/do_sweep.py @@ -677,7 +677,7 @@ def run(self) -> int: try: # Stage 1: Head infrastructure (NATS, etcd). Only the dynamo request # plane uses it; static/direct frontends skip it. - if self.config.frontend.type in {"sglang", "sgl-router", "trtllm_serve", "vllm", "vllm-router"}: + if self.config.frontend.type in {"sglang", "trtllm_serve", "vllm", "vllm-router"}: logger.info("Skipping NATS/etcd infrastructure (frontend.type=%s)", self.config.frontend.type) else: reporter.report(JobStatus.STARTING, JobStage.HEAD_INFRASTRUCTURE, "Starting head infrastructure") diff --git a/src/srtctl/cli/mixins/benchmark_stage.py b/src/srtctl/cli/mixins/benchmark_stage.py index 244dab270..edda440bd 100644 --- a/src/srtctl/cli/mixins/benchmark_stage.py +++ b/src/srtctl/cli/mixins/benchmark_stage.py @@ -430,8 +430,8 @@ def _get_sa_bench_slow_down_env(self) -> dict[str, str]: "benchmark slow_down: slow_down_sleep_time and slow_down_wait_time must be positive; skipping" ) return {} - if self.config.frontend.type not in {"sglang", "sgl-router"}: - logger.warning("benchmark.slow_down_* ignored: frontend.type is not sgl-router") + if self.config.frontend.type != "sglang": + logger.warning("benchmark.slow_down_* ignored: frontend.type is not sglang") return {} decode_urls: list[str] = [] diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py index 6fe7ac0d1..8cbed761c 100755 --- a/src/srtctl/core/schema.py +++ b/src/srtctl/core/schema.py @@ -1437,8 +1437,8 @@ class FrontendConfig: """Frontend/router configuration. Attributes: - type: Frontend type - "dynamo" (default), "sgl-router", "vllm-router", - "trtllm_serve", or direct "vllm". "sglang" remains a compatibility alias. + type: Frontend type - "dynamo" (default), "sglang", "vllm-router", + "trtllm_serve", or direct "vllm". enable_multiple_frontends: Scale with nginx + multiple routers. When ``True`` (default), srtctl stands up nginx and fans out to ``num_additional_frontends + 1`` router replicas. When @@ -1645,7 +1645,6 @@ def _validate_static_router_frontend(self): """Validate native static-router/backend pairings and endpoint shape.""" required_backend = { "sglang": "sglang", - "sgl-router": "sglang", "vllm-router": "vllm", }.get(self.frontend.type) if required_backend is None: diff --git a/src/srtctl/frontends/__init__.py b/src/srtctl/frontends/__init__.py index 06912a331..c5b352b26 100644 --- a/src/srtctl/frontends/__init__.py +++ b/src/srtctl/frontends/__init__.py @@ -6,8 +6,7 @@ Supported frontend types: - dynamo: Dynamo frontend with NATS/etcd communication -- sgl-router: SGLang Model Gateway with direct worker connections -- sglang: Backward-compatible alias for sgl-router +- sglang: SGLang Model Gateway with direct worker connections - vllm: Direct vLLM OpenAI server for aggregate jobs - vllm-router: vLLM Router with direct worker connections """ @@ -18,7 +17,7 @@ get_frontend, ) from srtctl.frontends.dynamo import DynamoFrontend -from srtctl.frontends.sglang import SGLangFrontend, SGLRouterFrontend +from srtctl.frontends.sglang import SGLangFrontend from srtctl.frontends.trtllm_serve import TRTLLMServeFrontend from srtctl.frontends.vllm import VLLMFrontend from srtctl.frontends.vllm_router import VLLMRouterFrontend @@ -27,7 +26,6 @@ "DynamoFrontend", "FrontendProtocol", "FrontendType", - "SGLRouterFrontend", "SGLangFrontend", "TRTLLMServeFrontend", "VLLMFrontend", diff --git a/src/srtctl/frontends/base.py b/src/srtctl/frontends/base.py index c808f4918..9d4fd8543 100644 --- a/src/srtctl/frontends/base.py +++ b/src/srtctl/frontends/base.py @@ -22,7 +22,7 @@ from srtctl.core.topology import Process # Supported frontend types - extensible by adding new literals -FrontendType = Literal["dynamo", "sglang", "sgl-router", "trtllm_serve", "vllm", "vllm-router"] +FrontendType = Literal["dynamo", "sglang", "trtllm_serve", "vllm", "vllm-router"] FrontendFactory = Callable[[], "FrontendProtocol"] _FRONTEND_REGISTRY: dict[str, FrontendFactory] = {} diff --git a/src/srtctl/frontends/sglang.py b/src/srtctl/frontends/sglang.py index c210e0ffb..3a6d779cf 100644 --- a/src/srtctl/frontends/sglang.py +++ b/src/srtctl/frontends/sglang.py @@ -10,11 +10,11 @@ from srtctl.frontends.static_router import StaticRouterFrontend -@register_frontend("sgl-router") -class SGLRouterFrontend(StaticRouterFrontend): - """First-class SGLang Model Gateway static router.""" +@register_frontend("sglang") +class SGLangFrontend(StaticRouterFrontend): + """SGLang Model Gateway static router.""" - type: ClassVar[str] = "sgl-router" + type: ClassVar[str] = "sglang" backend_type: ClassVar[str] = "sglang" executable: ClassVar[tuple[str, ...]] = ("python", "-m", "sglang_router.launch_router") pd_flag: ClassVar[str] = "--pd-disaggregation" @@ -28,10 +28,3 @@ def get_hostname_ip(self, node: str) -> str: def start_process(self, **kwargs: Any) -> Any: return start_srun_process(**kwargs) - - -@register_frontend("sglang") -class SGLangFrontend(SGLRouterFrontend): - """Backward-compatible alias for the historical ``sglang`` frontend type.""" - - type: ClassVar[str] = "sglang" diff --git a/tests/test_configs.py b/tests/test_configs.py index 86b1605e3..706e36b30 100644 --- a/tests/test_configs.py +++ b/tests/test_configs.py @@ -544,9 +544,9 @@ def test_grpc_mode_enabled_per_mode(self): @pytest.mark.parametrize( ("frontend_type", "mode", "expected"), [ - ("sgl-router", "prefill", True), - ("sgl-router", "decode", True), - ("sgl-router", "agg", False), + ("sglang", "prefill", True), + ("sglang", "decode", True), + ("sglang", "agg", False), ("dynamo", "decode", False), ], ) diff --git a/tests/test_static_router_frontends.py b/tests/test_static_router_frontends.py index 154785e14..ba9175a12 100644 --- a/tests/test_static_router_frontends.py +++ b/tests/test_static_router_frontends.py @@ -9,17 +9,16 @@ import pytest -from srtctl.frontends import SGLRouterFrontend, VLLMRouterFrontend, get_frontend +from srtctl.frontends import SGLangFrontend, VLLMRouterFrontend, get_frontend from srtctl.frontends.static_router import RouterWorker -def test_registry_exposes_explicit_router_names_and_legacy_alias() -> None: - assert isinstance(get_frontend("sgl-router"), SGLRouterFrontend) - assert get_frontend("sglang").type == "sglang" +def test_registry_exposes_native_router_names() -> None: + assert isinstance(get_frontend("sglang"), SGLangFrontend) assert isinstance(get_frontend("vllm-router"), VLLMRouterFrontend) -@pytest.mark.parametrize("frontend", [SGLRouterFrontend(), VLLMRouterFrontend()]) +@pytest.mark.parametrize("frontend", [SGLangFrontend(), VLLMRouterFrontend()]) def test_aggregate_command_advertises_all_logical_workers(frontend) -> None: command = frontend.build_router_command( [ @@ -38,7 +37,7 @@ def test_aggregate_command_advertises_all_logical_workers(frontend) -> None: @pytest.mark.parametrize( ("frontend", "pd_flag"), [ - (SGLRouterFrontend(), "--pd-disaggregation"), + (SGLangFrontend(), "--pd-disaggregation"), (VLLMRouterFrontend(), "--vllm-pd-disaggregation"), ], ) @@ -257,7 +256,7 @@ def test_sgl_router_rejects_non_divisible_tp_dp_layout() -> None: name="invalid-sglang-dpa", model={"path": "model", "container": "image", "precision": "fp8"}, resources=ResourceConfig(gpu_type="h100", gpus_per_node=8, agg_nodes=1, agg_workers=1), - frontend=FrontendConfig(type="sgl-router", enable_multiple_frontends=False), + frontend=FrontendConfig(type="sglang", enable_multiple_frontends=False), backend=SGLangProtocol( sglang_config=SGLangServerConfig(aggregated={"tp-size": 1, "dp-size": 8, "enable-dp-attention": True}) ), @@ -272,7 +271,7 @@ def test_sgl_router_accepts_divisible_tp_dp_layout() -> None: name="valid-sglang-dpa", model={"path": "model", "container": "image", "precision": "fp8"}, resources=ResourceConfig(gpu_type="h100", gpus_per_node=8, agg_nodes=1, agg_workers=1), - frontend=FrontendConfig(type="sgl-router", enable_multiple_frontends=False), + frontend=FrontendConfig(type="sglang", enable_multiple_frontends=False), backend=SGLangProtocol( sglang_config=SGLangServerConfig(aggregated={"tp-size": 8, "dp-size": 8, "enable-dp-attention": True}) ), From ab98030f0993bdb1595acc25e883b59d76ad7791 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 10 Aug 2026 12:51:08 -0500 Subject: [PATCH 12/46] test router log capture path --- tests/test_static_router_frontends.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_static_router_frontends.py b/tests/test_static_router_frontends.py index ba9175a12..eb977f191 100644 --- a/tests/test_static_router_frontends.py +++ b/tests/test_static_router_frontends.py @@ -145,9 +145,10 @@ def test_vllm_router_launch_uses_router_container_env_and_only_leaders() -> None patch.object(frontend, "get_hostname_ip", return_value="10.0.0.1"), patch.object(frontend, "start_process", return_value=MagicMock()) as start, ): - frontend.start_frontends(topology, runtime, config, MagicMock(), workers) + processes = frontend.start_frontends(topology, runtime, config, MagicMock(), workers) kwargs = start.call_args.kwargs + assert kwargs["output"] == "/logs/node0_vllm-router_0.out" assert kwargs["container_image"] == "docker://router:test" assert kwargs["env_to_set"] == {"GLOBAL": "value", "ROUTER_LOG": "debug"} assert kwargs["het_group"] == 1 @@ -156,6 +157,7 @@ def test_vllm_router_launch_uses_router_container_env_and_only_leaders() -> None assert "--routing-logic" in kwargs["command"] timeout_index = kwargs["command"].index("--worker-startup-timeout-secs") assert kwargs["command"][timeout_index + 1] == "3600" + assert processes[0].log_file == Path("/logs/node0_vllm-router_0.out") def test_vllm_router_explicit_worker_startup_timeout_overrides_managed_value() -> None: From cebe610b92eae59fa4220bd9d455e520ef805c3c Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 10 Aug 2026 14:36:05 -0500 Subject: [PATCH 13/46] support router-native vLLM data parallelism --- docs/config-reference.md | 27 +++++-- src/srtctl/backends/vllm.py | 37 ++++++++- src/srtctl/cli/mixins/benchmark_stage.py | 40 +++++++--- src/srtctl/core/schema.py | 27 ++++--- src/srtctl/frontends/static_router.py | 15 +++- src/srtctl/frontends/vllm_router.py | 56 ++++++++++++-- tests/test_benchmarks.py | 22 ++++++ tests/test_configs.py | 76 +++++++++++++++++++ tests/test_health_expectations.py | 71 +++++++++++++++-- tests/test_static_router_frontends.py | 97 +++++++++++++++++++++++- 10 files changed, 418 insertions(+), 50 deletions(-) diff --git a/docs/config-reference.md b/docs/config-reference.md index 1a614503b..37562b862 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -318,10 +318,15 @@ See [SGLang Router](sglang-router.md) for detailed architecture. `type: vllm-router` pairs with `backend.type: vllm` and launches the official `vllm-router` process against direct private `vllm serve` endpoints. Aggregate layouts use `--worker-urls`; disaggregated layouts use -`--vllm-pd-disaggregation` with the allocated prefill and decode leader URLs. -Each logical vLLM endpoint must currently fit on one node, but a job may scale -across many single-node aggregate, prefill, or decode endpoints. No NATS or etcd -infrastructure is started for this frontend. +`--vllm-pd-disaggregation` with the allocated prefill and decode URLs. For +data-parallel endpoints, srtctl derives Router's +`--intra-node-data-parallel-size`. Router expands each node-local backend URL +into DP-aware targets and injects `X-Data-Parallel-Rank`; vLLM continues to own +the engine processes behind that HTTP server. Multi-node DP endpoints use one +hybrid-LB `vllm serve` process per node and require +`backend.dp_launch_mode: per_node`. Direct `frontend.type: vllm` retains its +existing single-server behavior. No NATS or etcd infrastructure is started for +this frontend. ### trtllm_serve frontend @@ -463,6 +468,14 @@ backend: | `per_gpu` | One process per DP rank/GPU (default) | | `per_node` | One process manages all DP ranks allocated per node | +For `frontend.type: vllm-router`, Router-native DP expansion keeps one backend +URL per node and sends `X-Data-Parallel-Rank` to select a node-local engine. A +single-node endpoint needs no special launch mode. A multi-node DP endpoint must +use `per_node`; srtctl then derives the global/local DP topology and advertises +each node-local HTTP server to Router. All routed backends must have the same +node-local DP size because Router exposes one global +`--intra-node-data-parallel-size` setting. + `per_gpu` remains the compatibility default for now, but srtslurm will switch the default to `per_node` in a future release. Existing vLLM DP configurations should set `backend.dp_launch_mode: per_node` now; srtslurm emits a @@ -471,9 +484,9 @@ configuration-time migration warning while they still use `per_gpu`. In `per_node` mode, srtslurm derives `--data-parallel-size-local` and `--data-parallel-start-rank` from the allocated topology. Do not set those two flags manually. srtslurm also always enables `--data-parallel-hybrid-lb` -so every node-local process registers with the Dynamo frontend. This is the -recommended vLLM topology for Dynamo and ensures the frontend can route to -each node-local DP engine. Do not set `data-parallel-hybrid-lb` manually; +so every node-local process registers with the Dynamo frontend or exposes its +local ranks to vLLM Router. This is the recommended multi-node vLLM topology. +Do not set `data-parallel-hybrid-lb` manually; srtslurm enables it automatically, warns when it is configured, and ignores the configured value. `headless` is incompatible with `per_node` DP because a headless process does not register with Dynamo, so srtslurm rejects that diff --git a/src/srtctl/backends/vllm.py b/src/srtctl/backends/vllm.py index 6dfda324b..d24b41d04 100644 --- a/src/srtctl/backends/vllm.py +++ b/src/srtctl/backends/vllm.py @@ -498,13 +498,14 @@ def endpoints_to_processes( """Convert endpoints to processes. Dynamo DP+EP mode uses the configured per-GPU or per-node process layout. - For direct vLLM and vLLM Router jobs, `vllm serve` manages local DP ranks - from one process, so keep the standard one-process-per-node topology. + For direct vLLM and single-node vLLM Router jobs, `vllm serve` manages + local DP ranks from one process. Multi-node vLLM Router DP jobs use one + hybrid-LB process per node so Router can address each node-local DP pool. For standard TP mode, creates one process per node. """ from srtctl.core.topology import NodePortAllocator, Process, endpoints_to_processes - if frontend_type in {"vllm", "vllm-router"}: + if frontend_type == "vllm": return endpoints_to_processes(endpoints, base_sys_port=base_sys_port, port_allocator=port_allocator) # Check if any endpoint uses DP mode @@ -514,6 +515,13 @@ def endpoints_to_processes( # Standard TP mode: one process per node return endpoints_to_processes(endpoints, base_sys_port=base_sys_port, port_allocator=port_allocator) + if frontend_type == "vllm-router" and all(not endpoint.is_multi_node for endpoint in endpoints): + # Router expands each single-node backend URL into node-local DP ranks. + return endpoints_to_processes(endpoints, base_sys_port=base_sys_port, port_allocator=port_allocator) + + if frontend_type == "vllm-router" and self.dp_launch_mode != "per_node": + raise ValueError("multi-node vLLM Router DP endpoints require backend.dp_launch_mode: per_node") + if self.dp_launch_mode == "per_node": return self._dp_per_node_endpoints_to_processes( endpoints, @@ -717,7 +725,10 @@ def build_worker_command( if frontend_type in {"vllm", "vllm-router"}: if frontend_type == "vllm" and mode != "agg": raise ValueError("frontend.type: vllm supports aggregate vLLM jobs only") - if is_multi_node: + is_router_hybrid_dp = ( + frontend_type == "vllm-router" and self._is_dp_mode(mode) and self.dp_launch_mode == "per_node" + ) + if is_multi_node and not is_router_hybrid_dp: raise ValueError(f"frontend.type: {frontend_type} requires each vLLM endpoint to fit on one node") config.pop("host", None) @@ -734,6 +745,24 @@ def build_worker_command( if connector and connector not in ("null", "none", None): config.setdefault("kv-transfer-config", _connector_to_kv_transfer_config(connector)) + if is_router_hybrid_dp: + rpc_port_kebab = config.pop("data-parallel-rpc-port", None) + rpc_port_snake = config.pop("data_parallel_rpc_port", None) + dp_rpc_port = process.dp_rpc_port or rpc_port_kebab or rpc_port_snake or VLLM_DATA_PARALLEL_RPC_PORT + + config.pop("data-parallel-size-local", None) + config.pop("data_parallel_size_local", None) + config.pop("data-parallel-start-rank", None) + config.pop("data_parallel_start_rank", None) + config.pop("data-parallel-hybrid-lb", None) + config.pop("data_parallel_hybrid_lb", None) + config.pop("headless", None) + config["data-parallel-size-local"] = len(process.gpu_indices) + config["data-parallel-start-rank"] = process.node_rank + config["data-parallel-address"] = leader_ip + config["data-parallel-rpc-port"] = dp_rpc_port + config["data-parallel-hybrid-lb"] = True + cmd.extend( [ "vllm", diff --git a/src/srtctl/cli/mixins/benchmark_stage.py b/src/srtctl/cli/mixins/benchmark_stage.py index edda440bd..884d9e529 100644 --- a/src/srtctl/cli/mixins/benchmark_stage.py +++ b/src/srtctl/cli/mixins/benchmark_stage.py @@ -70,8 +70,9 @@ def _get_health_expectations( Dynamo's /health endpoint reports registered generate instances. For vLLM DP workers, per-GPU launch registers one entry per DP rank, while per-node - launch registers one entry per node-local process. Other frontends keep - using logical worker counts. + launch registers one entry per node-local process. vLLM Router expands each + routed backend URL into its node-local DP ranks. Other frontends keep using + logical worker counts. """ r = config.resources @@ -95,6 +96,24 @@ def _get_health_expectations( count_desc = f"{n_prefill}P + {n_decode}D Dynamo generate instances; logical workers: {worker_desc}" return n_prefill, n_decode, count_desc, n_prefill + n_decode + if config.frontend.type == "vllm-router" and backend_processes is not None: + from srtctl.frontends.vllm_router import node_local_data_parallel_size + + local_dp_size = node_local_data_parallel_size(config.backend, backend_processes) + + n_prefill = sum( + local_dp_size + for process in backend_processes + if process.endpoint_mode == "prefill" and process.http_port > 0 + ) + n_decode = sum( + local_dp_size + for process in backend_processes + if process.endpoint_mode in {"decode", "agg"} and process.http_port > 0 + ) + count_desc = f"{n_prefill}P + {n_decode}D Router DP workers; logical workers: {worker_desc}" + return n_prefill, n_decode, count_desc, n_prefill + n_decode + count_desc = worker_desc return logical_prefill, logical_decode, count_desc, logical_prefill + logical_decode @@ -146,11 +165,10 @@ def _benchmark_node(self) -> str: ) def _logical_worker_endpoints(self) -> list[tuple[str, str, int]]: - """Return ``(mode, IP, port)`` for every logical worker leader. + """Return ``(mode, IP, port)`` for every routable worker endpoint. - ``backend_processes`` contains one process per physical node for - multi-node workers. Only rank zero owns the logical worker endpoint, - so follower ranks must not be advertised to benchmark clients. + ``backend_processes`` may contain non-routable TP followers (HTTP port + zero) or multiple node-local vLLM DP pools (one positive port per pool). Dynamo exposes worker metrics on each leader's system port. Other frontends expose them on the worker HTTP port, matching the endpoint @@ -159,7 +177,7 @@ def _logical_worker_endpoints(self) -> list[tuple[str, str, int]]: use_sys_port = self.config.frontend.type == "dynamo" endpoints: list[tuple[str, str, int]] = [] for process in self.backend_processes: - if not process.is_leader: + if use_sys_port and not process.is_leader: continue port = process.sys_port if use_sys_port else process.http_port if port <= 0: @@ -474,10 +492,12 @@ def _get_aiperf_server_metrics_env( else: if self.config.frontend.type in {"vllm", "vllm-router"}: for process in self.backend_processes: - if process.is_leader: + if self.config.frontend.type == "vllm" and process.is_leader: + host = get_hostname_ip(process.node, self.runtime.network_interface) + urls.append(f"http://{host}:{FRONTEND_PUBLIC_PORT}/metrics") + elif self.config.frontend.type == "vllm-router" and process.http_port > 0: host = get_hostname_ip(process.node, self.runtime.network_interface) - port = FRONTEND_PUBLIC_PORT if self.config.frontend.type == "vllm" else process.http_port - urls.append(f"http://{host}:{port}/metrics") + urls.append(f"http://{host}:{process.http_port}/metrics") if urls: return {"AIPERF_SERVER_METRICS_URLS": ",".join(sorted(set(urls)))} diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py index 8cbed761c..42b360bff 100755 --- a/src/srtctl/core/schema.py +++ b/src/srtctl/core/schema.py @@ -1656,16 +1656,23 @@ def _validate_static_router_frontend(self): ) if self.frontend.type == "vllm-router": - endpoint_gpu_counts = ( - self.resources.gpus_per_prefill if self.resources.num_prefill else 0, - self.resources.gpus_per_decode if self.resources.num_decode else 0, - self.resources.gpus_per_agg if self.resources.num_agg else 0, - ) - if any(count > self.resources.gpus_per_node for count in endpoint_gpu_counts): - raise ValidationError( - "frontend.type: vllm-router currently requires each logical vLLM endpoint " - "to fit on one node; scale with multiple aggregate/prefill/decode workers" - ) + endpoint_gpu_counts = { + "prefill": self.resources.gpus_per_prefill if self.resources.num_prefill else 0, + "decode": self.resources.gpus_per_decode if self.resources.num_decode else 0, + "agg": self.resources.gpus_per_agg if self.resources.num_agg else 0, + } + multi_node_modes = [ + mode for mode, count in endpoint_gpu_counts.items() if count > self.resources.gpus_per_node + ] + if multi_node_modes and self.backend.dp_launch_mode != "per_node": + raise ValidationError("multi-node vLLM Router DP endpoints require backend.dp_launch_mode: per_node") + for mode in multi_node_modes: + gpu_count = endpoint_gpu_counts[mode] + if not self.backend._is_dp_mode(mode) or self.backend._get_dp_size(mode) != gpu_count: + raise ValidationError( + f"multi-node vLLM Router {mode} endpoints require data-parallel-size={gpu_count}; " + "multi-node TP-only direct serving is not supported" + ) def _validate_sglang_data_parallelism(self): """Reject SGLang TP/DP combinations that the server cannot initialize. diff --git a/src/srtctl/frontends/static_router.py b/src/srtctl/frontends/static_router.py index 6dbb77e24..c2568f51a 100644 --- a/src/srtctl/frontends/static_router.py +++ b/src/srtctl/frontends/static_router.py @@ -71,8 +71,14 @@ def get_frontend_args_list(self, args: dict[str, Any] | None) -> list[str]: result.extend([flag, str(value)]) return result - def get_managed_frontend_args(self, config: Any) -> list[str]: + def get_managed_frontend_args( + self, + config: Any, + backend: Any | None = None, + backend_processes: list[Process] | None = None, + ) -> list[str]: """Return adapter-managed CLI arguments derived from srtctl config.""" + del backend, backend_processes return [] def worker_scheme(self, backend: Any, mode: str) -> str: @@ -94,7 +100,10 @@ def start_process(self, **kwargs: Any) -> Any: def collect_workers(self, backend: Any, backend_processes: list[Process]) -> list[RouterWorker]: workers: list[RouterWorker] = [] for process in backend_processes: - if not process.is_leader: + # An allocated HTTP port is the source of truth for whether a backend + # process is independently routable. Multi-node vLLM DP exposes one + # node-local pool per URL, while TP follower processes retain port 0. + if process.http_port <= 0: continue scheme = self.worker_scheme(backend, process.endpoint_mode) workers.append( @@ -156,7 +165,7 @@ def start_frontends( for idx, node in enumerate(topology.frontend_nodes): router_log = runtime.log_dir / f"{node}_{self.type}_{idx}.out" cmd = self.build_router_command(workers, "0.0.0.0", topology.frontend_port) - cmd.extend(self.get_managed_frontend_args(config)) + cmd.extend(self.get_managed_frontend_args(config, backend, backend_processes)) cmd.extend(self.get_frontend_args_list(config.frontend.args)) logger.info("Starting %s %d on %s: %s", self.type, idx, node, shlex.join(cmd)) diff --git a/src/srtctl/frontends/vllm_router.py b/src/srtctl/frontends/vllm_router.py index cdfb3e3c6..8a8b1b99b 100644 --- a/src/srtctl/frontends/vllm_router.py +++ b/src/srtctl/frontends/vllm_router.py @@ -14,6 +14,28 @@ from srtctl.core.topology import Process +def node_local_data_parallel_size(backend: Any, backend_processes: list[Process]) -> int: + """Return Router's single node-local DP expansion factor.""" + grouped_processes: dict[tuple[str, int], list[Process]] = {} + for process in backend_processes: + if process.http_port > 0: + grouped_processes.setdefault((process.endpoint_mode, process.endpoint_index), []).append(process) + + local_dp_sizes: set[int] = set() + for (mode, _endpoint_index), processes in grouped_processes.items(): + global_dp_size = backend._get_dp_size(mode) or 1 + if global_dp_size % len(processes) != 0: + raise ValueError( + f"vLLM Router {mode} data-parallel-size={global_dp_size} cannot be evenly split " + f"across {len(processes)} routed node-local servers" + ) + local_dp_sizes.add(global_dp_size // len(processes)) + + if len(local_dp_sizes) > 1: + raise ValueError("vLLM Router requires the same node-local data-parallel size for every routed backend") + return next(iter(local_dp_sizes), 1) + + @register_frontend("vllm-router") class VLLMRouterFrontend(StaticRouterFrontend): """Route requests to direct vLLM OpenAI-compatible worker endpoints.""" @@ -24,15 +46,35 @@ class VLLMRouterFrontend(StaticRouterFrontend): pd_flag: ClassVar[str] = "--vllm-pd-disaggregation" process_name: ClassVar[str] = "vllm_router" - def get_managed_frontend_args(self, config: Any) -> list[str]: - """Keep Router's worker wait alive for srtctl's model-readiness window.""" + def get_managed_frontend_args( + self, + config: Any, + backend: Any | None = None, + backend_processes: list[Process] | None = None, + ) -> list[str]: + """Derive Router DP expansion and worker-readiness arguments.""" frontend_args = config.frontend.args or {} - if "worker-startup-timeout-secs" in frontend_args: - return [] + managed_args: list[str] = [] + + if backend is not None and backend_processes is not None: + local_dp_size = node_local_data_parallel_size(backend, backend_processes) + configured_dp_size = frontend_args.get( + "intra-node-data-parallel-size", + frontend_args.get("intra_node_data_parallel_size"), + ) + if configured_dp_size is not None and int(configured_dp_size) != local_dp_size: + raise ValueError( + "frontend.args.intra-node-data-parallel-size conflicts with the allocated vLLM topology: " + f"configured {configured_dp_size}, derived {local_dp_size}" + ) + if local_dp_size > 1 and configured_dp_size is None: + managed_args.extend(["--intra-node-data-parallel-size", str(local_dp_size)]) - health_check = config.health_check - timeout_seconds = health_check.max_attempts * health_check.interval_seconds - return ["--worker-startup-timeout-secs", str(timeout_seconds)] + if "worker-startup-timeout-secs" not in frontend_args: + health_check = config.health_check + timeout_seconds = health_check.max_attempts * health_check.interval_seconds + managed_args.extend(["--worker-startup-timeout-secs", str(timeout_seconds)]) + return managed_args def worker_bootstrap_port(self, backend: Any, process: Process) -> int | None: """Advertise vLLM's NIXL side-channel port to the P/D router.""" diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py index 386168331..5ea6d12f7 100644 --- a/tests/test_benchmarks.py +++ b/tests/test_benchmarks.py @@ -358,6 +358,28 @@ def test_aggregated_worker_endpoint_uses_http_port_without_dynamo(self): assert "SRT_DECODE_ENDPOINTS" not in env assert env["AIPERF_SERVER_METRICS_URLS"] == "http://ip-node-a:6100/metrics" + def test_vllm_router_exports_each_node_local_dp_backend(self): + """Multi-node DEP8 metrics use two node-local server URLs, not eight rank URLs.""" + from unittest.mock import patch + + from srtctl.benchmarks.custom import CustomBenchmarkRunner + from srtctl.core.topology import Process + + processes = [ + Process("node-a", frozenset(range(4)), 7500, 6100, "agg", 0, node_rank=0), + Process("node-b", frozenset(range(4)), 7501, 6100, "agg", 0, node_rank=4), + ] + stage = self._benchmark_stage("vllm-router", processes) + + with patch( + "srtctl.cli.mixins.benchmark_stage.get_hostname_ip", + side_effect=lambda node, interface: f"ip-{node}", + ): + env = stage._get_benchmark_env(CustomBenchmarkRunner()) + + assert env["SRT_AGG_ENDPOINTS"] == "ip-node-a:6100,ip-node-b:6100" + assert env["AIPERF_SERVER_METRICS_URLS"] == ("http://ip-node-a:6100/metrics,http://ip-node-b:6100/metrics") + def test_worker_endpoint_order_keeps_colocated_logical_workers_aligned(self): from unittest.mock import patch diff --git a/tests/test_configs.py b/tests/test_configs.py index 706e36b30..c913aaceb 100644 --- a/tests/test_configs.py +++ b/tests/test_configs.py @@ -2203,6 +2203,82 @@ def test_vllm_router_keeps_one_direct_server_per_logical_endpoint(self): assert all(process.is_leader for process in processes) assert len({process.http_port for process in processes}) == 1 # ports may repeat on distinct nodes + def test_vllm_router_uses_one_backend_url_for_single_node_dep4(self): + """Router expands one direct backend URL into four node-local DP ranks.""" + from srtctl.backends import VLLMProtocol, VLLMServerConfig + from srtctl.core.topology import Endpoint + + backend = VLLMProtocol( + vllm_config=VLLMServerConfig( + aggregated={"data-parallel-size": 4, "enable-expert-parallel": True}, + ), + ) + endpoint = Endpoint( + mode="agg", + index=0, + nodes=("node0",), + gpu_indices=frozenset(range(4)), + gpus_per_node=4, + ) + + processes = backend.endpoints_to_processes([endpoint], frontend_type="vllm-router") + + assert len(processes) == 1 + assert processes[0].gpu_indices == frozenset(range(4)) + assert processes[0].http_port > 0 + + def test_vllm_router_multinode_dep8_uses_hybrid_node_local_pools(self): + """Two DEP8 nodes expose two DP4 HTTP pools sharing one global coordinator.""" + from pathlib import Path + from unittest.mock import MagicMock, patch + + from srtctl.backends import VLLMProtocol, VLLMServerConfig + from srtctl.core.topology import Endpoint + + backend = VLLMProtocol( + dp_launch_mode="per_node", + vllm_config=VLLMServerConfig( + aggregated={ + "data-parallel-size": 8, + "enable-expert-parallel": True, + }, + ), + ) + endpoint = Endpoint( + mode="agg", + index=0, + nodes=("node0", "node1"), + gpu_indices=frozenset(range(4)), + gpus_per_node=4, + ) + processes = backend.endpoints_to_processes([endpoint], frontend_type="vllm-router") + runtime = MagicMock() + runtime.model_path = Path("/model") + runtime.is_hf_model = False + runtime.frontend_port = 8000 + + with patch("srtctl.core.slurm.get_hostname_ip", return_value="10.0.0.1"): + commands = [ + backend.build_worker_command( + process=process, + endpoint_processes=processes, + runtime=runtime, + frontend_type="vllm-router", + ) + for process in processes + ] + + assert len(processes) == 2 + assert [process.node_rank for process in processes] == [0, 4] + assert all(process.http_port > 0 for process in processes) + assert len({process.dp_rpc_port for process in processes}) == 1 + for start_rank, command in zip((0, 4), commands, strict=True): + assert command[command.index("--data-parallel-size") + 1] == "8" + assert command[command.index("--data-parallel-size-local") + 1] == "4" + assert command[command.index("--data-parallel-start-rank") + 1] == str(start_rank) + assert command[command.index("--data-parallel-address") + 1] == "10.0.0.1" + assert "--data-parallel-hybrid-lb" in command + def test_vllm_router_worker_uses_private_port_and_pd_connector(self): """Disaggregated vLLM Router workers are direct servers with KV transfer.""" from pathlib import Path diff --git a/tests/test_health_expectations.py b/tests/test_health_expectations.py index 11fcfb23b..3f836daf1 100644 --- a/tests/test_health_expectations.py +++ b/tests/test_health_expectations.py @@ -19,7 +19,26 @@ def _config( dp_launch_mode="per_gpu", ): """Build a duck-typed stand-in for SrtConfig with only the fields the helpers read.""" - backend = SimpleNamespace(type=backend_type, vllm_config=vllm_config, dp_launch_mode=dp_launch_mode) + + def is_dp_mode(mode): + mode_name = "aggregated" if mode == "agg" else mode + mode_config = getattr(vllm_config, mode_name, None) if vllm_config else None + return bool(mode_config and (mode_config.get("data-parallel-size") or mode_config.get("data_parallel_size"))) + + def get_dp_size(mode): + mode_name = "aggregated" if mode == "agg" else mode + mode_config = getattr(vllm_config, mode_name, None) if vllm_config else None + if not mode_config: + return None + return mode_config.get("data-parallel-size") or mode_config.get("data_parallel_size") + + backend = SimpleNamespace( + type=backend_type, + vllm_config=vllm_config, + dp_launch_mode=dp_launch_mode, + _is_dp_mode=is_dp_mode, + _get_dp_size=get_dp_size, + ) return SimpleNamespace( frontend=SimpleNamespace(type=frontend_type), backend=backend, @@ -27,12 +46,36 @@ def _config( ) -def _processes(*, prefill=0, decode=0, agg=0): +def _processes(*, prefill=0, decode=0, agg=0, gpus_per_process=1): """Build backend-process stand-ins grouped by endpoint mode.""" return [ - *(SimpleNamespace(endpoint_mode="prefill") for _ in range(prefill)), - *(SimpleNamespace(endpoint_mode="decode") for _ in range(decode)), - *(SimpleNamespace(endpoint_mode="agg") for _ in range(agg)), + *( + SimpleNamespace( + endpoint_mode="prefill", + endpoint_index=index, + http_port=6100, + gpu_indices=frozenset(range(gpus_per_process)), + ) + for index in range(prefill) + ), + *( + SimpleNamespace( + endpoint_mode="decode", + endpoint_index=index, + http_port=6100, + gpu_indices=frozenset(range(gpus_per_process)), + ) + for index in range(decode) + ), + *( + SimpleNamespace( + endpoint_mode="agg", + endpoint_index=index, + http_port=6100, + gpu_indices=frozenset(range(gpus_per_process)), + ) + for index in range(agg) + ), ] @@ -122,6 +165,24 @@ def test_non_dynamo_frontend_uses_logical_worker_counts(): assert count_desc == "6P + 1D" +def test_vllm_router_counts_dp_workers_expanded_from_backend_urls(): + """Router health waits for four ranks behind each of one P and two D URLs.""" + vllm_config = SimpleNamespace( + prefill={"data-parallel-size": 4}, + decode={"data-parallel-size": 4}, + aggregated=None, + ) + config = _config("vllm-router", "vllm", num_prefill=1, num_decode=2, vllm_config=vllm_config) + + n_prefill, n_decode, count_desc, num_workers = _get_health_expectations( + config, + _processes(prefill=1, decode=2, gpus_per_process=4), + ) + + assert (n_prefill, n_decode, num_workers) == (4, 8, 12) + assert count_desc == "4P + 8D Router DP workers; logical workers: 1P + 2D" + + def test_dynamo_non_vllm_backend_uses_logical_worker_counts(): """Dynamo + sglang has no DP-rank fan-out in these units; stay logical.""" config = _config("dynamo", "sglang", num_prefill=6, num_decode=1) diff --git a/tests/test_static_router_frontends.py b/tests/test_static_router_frontends.py index eb977f191..27591ec24 100644 --- a/tests/test_static_router_frontends.py +++ b/tests/test_static_router_frontends.py @@ -102,6 +102,62 @@ def test_vllm_router_advertises_nixl_side_channel_port() -> None: assert workers == [RouterWorker("prefill", "http://10.0.0.1:30000", 13000)] +def test_vllm_router_derives_dep4_expansion_for_1p2d() -> None: + """One P URL and two D URLs are each expanded to four ranks by Router.""" + frontend = VLLMRouterFrontend() + backend = MagicMock() + backend._is_dp_mode.return_value = True + backend._get_dp_size.return_value = 4 + processes = [ + SimpleNamespace( + endpoint_mode="prefill", + endpoint_index=0, + node="prefill", + gpu_indices=frozenset(range(4)), + http_port=6100, + nixl_port=5400, + node_rank=0, + ), + SimpleNamespace( + endpoint_mode="decode", + endpoint_index=0, + node="decode0", + gpu_indices=frozenset(range(4)), + http_port=6100, + nixl_port=5500, + node_rank=0, + ), + SimpleNamespace( + endpoint_mode="decode", + endpoint_index=1, + node="decode1", + gpu_indices=frozenset(range(4)), + http_port=6100, + nixl_port=5504, + node_rank=0, + ), + ] + config = SimpleNamespace( + frontend=SimpleNamespace(args={}), + health_check=SimpleNamespace(max_attempts=360, interval_seconds=10), + ) + + with patch.object(frontend, "get_hostname_ip", side_effect=lambda node: f"ip-{node}"): + workers = frontend.collect_workers(backend, processes) + command = frontend.build_router_command(workers, "0.0.0.0", 8000) + + assert len([worker for worker in workers if worker.mode == "prefill"]) == 1 + assert len([worker for worker in workers if worker.mode == "decode"]) == 2 + assert command.count("--prefill") == 1 + assert command.count("--decode") == 2 + assert frontend.get_managed_frontend_args(config, backend, processes) == [ + "--intra-node-data-parallel-size", + "4", + "--worker-startup-timeout-secs", + "3600", + ] + + def test_vllm_router_launch_uses_router_container_env_and_only_leaders() -> None: frontend = VLLMRouterFrontend() runtime = SimpleNamespace( @@ -126,7 +182,9 @@ def test_vllm_router_launch_uses_router_container_env_and_only_leaders() -> None SimpleNamespace( is_leader=True, endpoint_mode="agg", + endpoint_index=0, node="node1", + gpu_indices=frozenset(range(8)), http_port=30000, bootstrap_port=None, nixl_port=None, @@ -134,18 +192,24 @@ def test_vllm_router_launch_uses_router_container_env_and_only_leaders() -> None SimpleNamespace( is_leader=False, endpoint_mode="agg", + endpoint_index=0, node="node2", + gpu_indices=frozenset(range(8)), http_port=0, bootstrap_port=None, nixl_port=None, ), ] + backend = MagicMock() + backend._is_dp_mode.return_value = False + backend._get_dp_size.return_value = None + with ( patch.object(frontend, "get_hostname_ip", return_value="10.0.0.1"), patch.object(frontend, "start_process", return_value=MagicMock()) as start, ): - processes = frontend.start_frontends(topology, runtime, config, MagicMock(), workers) + processes = frontend.start_frontends(topology, runtime, config, backend, workers) kwargs = start.call_args.kwargs assert kwargs["output"] == "/logs/node0_vllm-router_0.out" @@ -224,13 +288,13 @@ def test_vllm_router_accepts_many_single_node_endpoints() -> None: assert config.resources.gpus_per_agg == 8 -def test_vllm_router_rejects_endpoint_spanning_nodes() -> None: +def test_vllm_router_rejects_multinode_tp_only_endpoint() -> None: from marshmallow import ValidationError from srtctl.backends import VLLMProtocol from srtctl.core.schema import FrontendConfig, ResourceConfig, SrtConfig - with pytest.raises(ValidationError, match="each logical vLLM endpoint"): + with pytest.raises(ValidationError, match="multi-node TP-only"): SrtConfig( name="multi-node-endpoint", model={"path": "model", "container": "image", "precision": "fp8"}, @@ -243,10 +307,35 @@ def test_vllm_router_rejects_endpoint_spanning_nodes() -> None: decode_workers=1, ), frontend=FrontendConfig(type="vllm-router", enable_multiple_frontends=False), - backend=VLLMProtocol(), + backend=VLLMProtocol(dp_launch_mode="per_node"), ) +def test_vllm_router_accepts_multinode_dep8_endpoint() -> None: + from srtctl.backends import VLLMProtocol, VLLMServerConfig + from srtctl.core.schema import FrontendConfig, ResourceConfig, SrtConfig + + config = SrtConfig( + name="multi-node-dep8", + model={"path": "model", "container": "image", "precision": "fp8"}, + resources=ResourceConfig( + gpu_type="gb200", + gpus_per_node=4, + agg_nodes=2, + agg_workers=1, + ), + frontend=FrontendConfig(type="vllm-router", enable_multiple_frontends=False), + backend=VLLMProtocol( + dp_launch_mode="per_node", + vllm_config=VLLMServerConfig( + aggregated={"data-parallel-size": 8, "enable-expert-parallel": True}, + ), + ), + ) + + assert config.resources.gpus_per_agg == 8 + + def test_sgl_router_rejects_non_divisible_tp_dp_layout() -> None: from marshmallow import ValidationError From f8b8e36a5768dc0c5cf4b6506da95e9a66b949a5 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 9 Aug 2026 19:00:16 -0500 Subject: [PATCH 14/46] feat(runtime): add AMD accelerator device masking --- docs/config-reference.md | 1 + docs/installation.md | 1 + src/srtctl/backends/vllm.py | 10 +++++--- src/srtctl/cli/mixins/worker_stage.py | 21 ++++++++++++----- src/srtctl/core/accelerator.py | 23 ++++++++++++++++++ src/srtctl/core/runtime.py | 5 +++- src/srtctl/core/schema.py | 1 + tests/test_accelerator.py | 34 +++++++++++++++++++++++++++ 8 files changed, 86 insertions(+), 10 deletions(-) create mode 100644 src/srtctl/core/accelerator.py create mode 100644 tests/test_accelerator.py diff --git a/docs/config-reference.md b/docs/config-reference.md index 37562b862..371df2d8e 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -111,6 +111,7 @@ The `srtslurm.yaml` file can contain the following fields: | `default_time_limit` | string | Default job time limit | | `gpus_per_node` | int | Default GPUs per node | | `network_interface` | string | Network interface for NCCL | +| `accelerator_vendor` | string | Accelerator runtime: `nvidia` (default) or `amd` | | `srtctl_root` | string | Root directory for srtctl | | `output_dir` | string | Custom output directory (overrides srtctl_root/outputs) | | `model_paths` | dict | Model path aliases | diff --git a/docs/installation.md b/docs/installation.md index fded7680f..8ca0e0181 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -109,6 +109,7 @@ gpus_per_node: 4 # SLURM directive compatibility use_gpus_per_node_directive: true # Set false if cluster doesn't support --gpus-per-node +accelerator_vendor: nvidia # Use amd for ROCm clusters use_segment_sbatch_directive: true # Set false if cluster doesn't support --segment use_exclusive_sbatch_directive: false # Set true if cluster requires --exclusive diff --git a/src/srtctl/backends/vllm.py b/src/srtctl/backends/vllm.py index d24b41d04..14c8285b3 100644 --- a/src/srtctl/backends/vllm.py +++ b/src/srtctl/backends/vllm.py @@ -479,15 +479,19 @@ def _get_dp_size(self, mode: WorkerMode) -> int | None: config = self.get_config_for_mode(mode) return config.get("data-parallel-size") or config.get("data_parallel_size") - def should_set_cuda_visible_devices(self, process: Process) -> bool: - """Whether worker_stage should set CUDA_VISIBLE_DEVICES. + def should_set_visible_devices(self, process: Process) -> bool: + """Whether worker_stage should set a vendor-native device mask. Newer vLLM builds should use ``--device-ids`` instead. Older builds before https://github.com/vllm-project/vllm/pull/45026 should set - CUDA_VISIBLE_DEVICES. + the platform's visible-device environment. """ return self.set_cuda_visible_devices + def should_set_cuda_visible_devices(self, process: Process) -> bool: + """Deprecated compatibility wrapper for third-party callers.""" + return self.should_set_visible_devices(process) + def endpoints_to_processes( self, endpoints: list[Endpoint], diff --git a/src/srtctl/cli/mixins/worker_stage.py b/src/srtctl/cli/mixins/worker_stage.py index ec7b54d72..17c15357e 100644 --- a/src/srtctl/cli/mixins/worker_stage.py +++ b/src/srtctl/cli/mixins/worker_stage.py @@ -12,6 +12,7 @@ from collections import defaultdict from typing import TYPE_CHECKING, Any +from srtctl.core.accelerator import visible_device_environment from srtctl.core.fingerprint import generate_capture_script from srtctl.core.processes import ManagedProcess, NamedProcesses from srtctl.core.schema import build_otel_env, installs_dynamo @@ -92,6 +93,18 @@ def _build_worker_preamble(self) -> str | None: return " && ".join(parts) + def _visible_device_environment(self, process: "Process") -> dict[str, str]: + """Build a vendor-native device mask when the backend needs one.""" + should_set_devices = getattr(self.backend, "should_set_visible_devices", None) + if should_set_devices is None: + # Backward compatibility for third-party backends implementing the + # original CUDA-named hook. + should_set_devices = getattr(self.backend, "should_set_cuda_visible_devices", lambda _process: True) + + if not should_set_devices(process) or len(process.gpu_indices) >= self.runtime.gpus_per_node: + return {} + return visible_device_environment(self.runtime.accelerator_vendor, process.cuda_visible_devices) + def _apply_kvbm_endpoint_env(self, env_to_set: dict[str, str], endpoint_processes: list["Process"]) -> None: """Fill KVBM leader ZMQ settings for an endpoint. @@ -196,9 +209,7 @@ def __missing__(self, key: str) -> str: profile_dir = str(self.runtime.log_dir / "profiles") env_to_set.update(profiling.get_env_vars(mode, profile_dir)) - should_set_cvd = getattr(self.backend, "should_set_cuda_visible_devices", lambda _process: True) - if should_set_cvd(process) and len(process.gpu_indices) < self.runtime.gpus_per_node: - env_to_set["CUDA_VISIBLE_DEVICES"] = process.cuda_visible_devices + env_to_set.update(self._visible_device_environment(process)) # Add backend-specific process environment variables (e.g., unique ports) env_to_set.update(self.backend.get_process_environment(process)) @@ -335,9 +346,7 @@ def start_endpoint_worker(self, endpoint_processes: list["Process"]) -> ManagedP profile_dir = str(self.runtime.log_dir / "profiles") env_to_set.update(profiling.get_env_vars(mode, profile_dir)) - should_set_cvd = getattr(self.backend, "should_set_cuda_visible_devices", lambda _process: True) - if should_set_cvd(leader) and len(leader.gpu_indices) < self.runtime.gpus_per_node: - env_to_set["CUDA_VISIBLE_DEVICES"] = leader.cuda_visible_devices + env_to_set.update(self._visible_device_environment(leader)) # Add mooncake worker env vars if configured (SGLang only). For MPI-style # endpoint launching we use the leader node's IP — mooncake's per-worker diff --git a/src/srtctl/core/accelerator.py b/src/srtctl/core/accelerator.py new file mode 100644 index 000000000..d7080987e --- /dev/null +++ b/src/srtctl/core/accelerator.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Accelerator-specific runtime environment helpers.""" + +from typing import Literal + +AcceleratorVendor = Literal["nvidia", "amd"] + + +def visible_device_environment(vendor: AcceleratorVendor, device_ids: str) -> dict[str, str]: + """Return the vendor-native environment used to restrict visible GPUs. + + ROCm recommends ``ROCR_VISIBLE_DEVICES`` for GPU isolation on Linux. Do + not also set ``HIP_VISIBLE_DEVICES`` or ``CUDA_VISIBLE_DEVICES`` here: + those variables are interpreted after ROCr device masking by some stacks, + so repeating physical indices can accidentally hide the selected devices. + """ + if vendor == "nvidia": + return {"CUDA_VISIBLE_DEVICES": device_ids} + if vendor == "amd": + return {"ROCR_VISIBLE_DEVICES": device_ids} + raise ValueError(f"Unsupported accelerator vendor: {vendor}") diff --git a/src/srtctl/core/runtime.py b/src/srtctl/core/runtime.py index 143c9955e..0b3c8703c 100644 --- a/src/srtctl/core/runtime.py +++ b/src/srtctl/core/runtime.py @@ -11,7 +11,7 @@ import os from dataclasses import dataclass, field from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal from srtctl.ports import FRONTEND_PUBLIC_PORT @@ -184,6 +184,7 @@ class RuntimeContext: # HuggingFace model support - True if model.path was "hf:model/name" is_hf_model: bool = False gpu_type: str | None = None + accelerator_vendor: Literal["nvidia", "amd"] = "nvidia" # Container mounts: host_path -> container_path container_mounts: dict[Path, Path] = field(default_factory=dict) @@ -360,6 +361,7 @@ def from_config( gpus_per_node=config.resources.gpus_per_node, gpu_type=config.resources.gpu_type, network_interface=get_srtslurm_setting("network_interface", "eth0"), + accelerator_vendor=get_srtslurm_setting("accelerator_vendor", "nvidia"), container_mounts={}, srun_options=dict(config.srun_options), environment=environment, @@ -385,6 +387,7 @@ def from_config( gpus_per_node=config.resources.gpus_per_node, gpu_type=config.resources.gpu_type, network_interface=get_srtslurm_setting("network_interface", "eth0"), + accelerator_vendor=get_srtslurm_setting("accelerator_vendor", "nvidia"), container_mounts=container_mounts, srun_options=dict(config.srun_options), environment=environment, diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py index 42b360bff..f4edd3092 100755 --- a/src/srtctl/core/schema.py +++ b/src/srtctl/core/schema.py @@ -184,6 +184,7 @@ class ClusterConfig: default_time_limit: str | None = None gpus_per_node: int | None = None network_interface: str | None = None + accelerator_vendor: Literal["nvidia", "amd"] = "nvidia" use_gpus_per_node_directive: bool = True use_segment_sbatch_directive: bool = True use_exclusive_sbatch_directive: bool = False diff --git a/tests/test_accelerator.py b/tests/test_accelerator.py new file mode 100644 index 000000000..5150c310e --- /dev/null +++ b/tests/test_accelerator.py @@ -0,0 +1,34 @@ +"""Tests for accelerator-specific runtime behavior.""" + +import pytest + +from srtctl.core.accelerator import visible_device_environment +from srtctl.core.schema import ClusterConfig + + +def test_nvidia_visible_device_environment() -> None: + assert visible_device_environment("nvidia", "2,3") == {"CUDA_VISIBLE_DEVICES": "2,3"} + + +def test_amd_visible_device_environment_uses_rocr_linux_contract() -> None: + assert visible_device_environment("amd", "2,3") == {"ROCR_VISIBLE_DEVICES": "2,3"} + + +def test_unknown_accelerator_vendor_is_rejected() -> None: + with pytest.raises(ValueError, match="Unsupported accelerator vendor"): + visible_device_environment("intel", "0") # type: ignore[arg-type] + + +def test_cluster_config_defaults_to_nvidia() -> None: + config = ClusterConfig.Schema().load({}) + assert config.accelerator_vendor == "nvidia" + + +def test_cluster_config_accepts_amd() -> None: + config = ClusterConfig.Schema().load({"accelerator_vendor": "amd"}) + assert config.accelerator_vendor == "amd" + + +def test_cluster_config_rejects_unknown_accelerator() -> None: + with pytest.raises(Exception, match="accelerator_vendor"): + ClusterConfig.Schema().load({"accelerator_vendor": "intel"}) From c99c0a40fbac54293eddc456805d23ff15058c55 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 9 Aug 2026 19:12:18 -0500 Subject: [PATCH 15/46] feat(slurm): support configurable GPU directives --- docs/config-reference.md | 1 + docs/installation.md | 3 +- src/srtctl/cli/submit.py | 9 ++- src/srtctl/core/schema.py | 1 + src/srtctl/templates/job_script_minimal.j2 | 8 ++- tests/test_accelerator.py | 65 +++++++++++++++++++++- 6 files changed, 82 insertions(+), 5 deletions(-) diff --git a/docs/config-reference.md b/docs/config-reference.md index 371df2d8e..37c759915 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -112,6 +112,7 @@ The `srtslurm.yaml` file can contain the following fields: | `gpus_per_node` | int | Default GPUs per node | | `network_interface` | string | Network interface for NCCL | | `accelerator_vendor` | string | Accelerator runtime: `nvidia` (default) or `amd` | +| `gpu_sbatch_directive` | string | GPU allocation directive: `gpus-per-node`, `gres`, or `none` | | `srtctl_root` | string | Root directory for srtctl | | `output_dir` | string | Custom output directory (overrides srtctl_root/outputs) | | `model_paths` | dict | Model path aliases | diff --git a/docs/installation.md b/docs/installation.md index 8ca0e0181..006e8b0c2 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -108,7 +108,8 @@ default_time_limit: "4:00:00" gpus_per_node: 4 # SLURM directive compatibility -use_gpus_per_node_directive: true # Set false if cluster doesn't support --gpus-per-node +gpu_sbatch_directive: gpus-per-node # Use gres for --gres=gpu:N clusters, or none +# Legacy compatibility: use_gpus_per_node_directive: true accelerator_vendor: nvidia # Use amd for ROCm clusters use_segment_sbatch_directive: true # Set false if cluster doesn't support --segment use_exclusive_sbatch_directive: false # Set true if cluster requires --exclusive diff --git a/src/srtctl/cli/submit.py b/src/srtctl/cli/submit.py index 3d0652719..4f8e457b3 100755 --- a/src/srtctl/cli/submit.py +++ b/src/srtctl/cli/submit.py @@ -463,6 +463,13 @@ def generate_minimal_sbatch_script( config_environment = config.dynamo.get_wheel_environment() config_environment.update(config.environment) + gpu_sbatch_directive = get_srtslurm_setting("gpu_sbatch_directive") + if gpu_sbatch_directive is None: + # Backward compatibility for existing cluster configurations. + gpu_sbatch_directive = ( + "gpus-per-node" if get_srtslurm_setting("use_gpus_per_node_directive", True) else "none" + ) + rendered = template.render( job_name=job_name, total_nodes=total_nodes, @@ -475,7 +482,7 @@ def generate_minimal_sbatch_script( config_path=str(config_path.resolve()), runtime_config_filename=runtime_config_filename, timestamp=timestamp, - use_gpus_per_node_directive=get_srtslurm_setting("use_gpus_per_node_directive", True), + gpu_sbatch_directive=gpu_sbatch_directive, use_segment_sbatch_directive=get_srtslurm_setting("use_segment_sbatch_directive", True), use_exclusive_sbatch_directive=get_srtslurm_setting("use_exclusive_sbatch_directive", False), sbatch_directives=config.sbatch_directives, diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py index f4edd3092..da87450de 100755 --- a/src/srtctl/core/schema.py +++ b/src/srtctl/core/schema.py @@ -185,6 +185,7 @@ class ClusterConfig: gpus_per_node: int | None = None network_interface: str | None = None accelerator_vendor: Literal["nvidia", "amd"] = "nvidia" + gpu_sbatch_directive: Literal["gpus-per-node", "gres", "none"] | None = None use_gpus_per_node_directive: bool = True use_segment_sbatch_directive: bool = True use_exclusive_sbatch_directive: bool = False diff --git a/src/srtctl/templates/job_script_minimal.j2 b/src/srtctl/templates/job_script_minimal.j2 index 990499a47..889a469b5 100644 --- a/src/srtctl/templates/job_script_minimal.j2 +++ b/src/srtctl/templates/job_script_minimal.j2 @@ -20,8 +20,10 @@ #SBATCH --ntasks={{ c.nodes }} #SBATCH --ntasks-per-node=1 {% endif %} -{% if use_gpus_per_node_directive %} +{% if gpu_sbatch_directive == "gpus-per-node" %} #SBATCH --gpus-per-node={{ c.gpus_per_node }} +{% elif gpu_sbatch_directive == "gres" %} +#SBATCH --gres=gpu:{{ c.gpus_per_node }} {% endif %} {% if use_segment_sbatch_directive %} #SBATCH --segment={{ c.segment }} @@ -49,8 +51,10 @@ #SBATCH --ntasks={{ total_nodes }} #SBATCH --ntasks-per-node=1 {% endif %} -{% if use_gpus_per_node_directive %} +{% if gpu_sbatch_directive == "gpus-per-node" %} #SBATCH --gpus-per-node={{ gpus_per_node }} +{% elif gpu_sbatch_directive == "gres" %} +#SBATCH --gres=gpu:{{ gpus_per_node }} {% endif %} {% if use_segment_sbatch_directive %} #SBATCH --segment={{ total_nodes }} diff --git a/tests/test_accelerator.py b/tests/test_accelerator.py index 5150c310e..0e620a5ca 100644 --- a/tests/test_accelerator.py +++ b/tests/test_accelerator.py @@ -1,5 +1,7 @@ """Tests for accelerator-specific runtime behavior.""" +from pathlib import Path + import pytest from srtctl.core.accelerator import visible_device_environment @@ -25,10 +27,71 @@ def test_cluster_config_defaults_to_nvidia() -> None: def test_cluster_config_accepts_amd() -> None: - config = ClusterConfig.Schema().load({"accelerator_vendor": "amd"}) + config = ClusterConfig.Schema().load({"accelerator_vendor": "amd", "gpu_sbatch_directive": "gres"}) assert config.accelerator_vendor == "amd" + assert config.gpu_sbatch_directive == "gres" def test_cluster_config_rejects_unknown_accelerator() -> None: with pytest.raises(Exception, match="accelerator_vendor"): ClusterConfig.Schema().load({"accelerator_vendor": "intel"}) + + +def test_cluster_config_rejects_unknown_gpu_directive() -> None: + with pytest.raises(Exception, match="gpu_sbatch_directive"): + ClusterConfig.Schema().load({"gpu_sbatch_directive": "rocm"}) + + +@pytest.mark.parametrize( + ("directive", "expected", "unexpected"), + [ + ("gpus-per-node", "#SBATCH --gpus-per-node=8", "#SBATCH --gres=gpu:8"), + ("gres", "#SBATCH --gres=gpu:8", "#SBATCH --gpus-per-node=8"), + ("none", None, "#SBATCH --gpus-per-node=8"), + ], +) +def test_gpu_sbatch_directive_rendering(monkeypatch, directive, expected, unexpected) -> None: + from srtctl.cli import submit + from srtctl.core.schema import ModelConfig, ResourceConfig, SrtConfig + + settings = { + "gpu_sbatch_directive": directive, + "use_segment_sbatch_directive": False, + } + monkeypatch.setattr(submit, "get_srtslurm_setting", lambda key, default=None: settings.get(key, default)) + config = SrtConfig( + name="amd-render-test", + model=ModelConfig(path="/model", container="/container.sqsh", precision="fp16"), + resources=ResourceConfig(gpu_type="mi300x", gpus_per_node=8, agg_nodes=1), + ) + + script = submit.generate_minimal_sbatch_script(config, Path("/tmp/amd-render-test.yaml")) + + if expected is not None: + assert expected in script + else: + assert "#SBATCH --gpus-per-node=" not in script + assert "#SBATCH --gres=gpu:" not in script + assert unexpected not in script + + +def test_legacy_gpu_directive_boolean_is_preserved(monkeypatch) -> None: + from srtctl.cli import submit + from srtctl.core.schema import ModelConfig, ResourceConfig, SrtConfig + + settings = { + "gpu_sbatch_directive": None, + "use_gpus_per_node_directive": False, + "use_segment_sbatch_directive": False, + } + monkeypatch.setattr(submit, "get_srtslurm_setting", lambda key, default=None: settings.get(key, default)) + config = SrtConfig( + name="legacy-render-test", + model=ModelConfig(path="/model", container="/container.sqsh", precision="fp16"), + resources=ResourceConfig(gpu_type="h100", gpus_per_node=8, agg_nodes=1), + ) + + script = submit.generate_minimal_sbatch_script(config, Path("/tmp/legacy-render-test.yaml")) + + assert "#SBATCH --gpus-per-node=" not in script + assert "#SBATCH --gres=gpu:" not in script From 0d61ad75d0620144a3d5616a8e9db598107698b2 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 9 Aug 2026 19:54:33 -0500 Subject: [PATCH 16/46] feat(vllm): add vendor-neutral device binding --- docs/config-reference.md | 16 ++++++++++++++++ src/srtctl/backends/vllm.py | 12 +++++++++--- tests/test_accelerator.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/docs/config-reference.md b/docs/config-reference.md index 37c759915..e8f29f05d 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -448,6 +448,22 @@ Each worker leader gets a globally unique port starting at 5550: | decode_0 | 5552 | | decode_1 | 5553 | +### vLLM device binding + +Recent vLLM builds accept `--device-ids`. For builds that do not, set +`backend.set_visible_devices: true` and srtslurm will bind each worker with the +accelerator vendor's native environment variable: `CUDA_VISIBLE_DEVICES` for +NVIDIA and `ROCR_VISIBLE_DEVICES` for AMD. + +```yaml +backend: + type: vllm + set_visible_devices: true +``` + +The legacy `set_cuda_visible_devices` field remains supported for existing +recipes, but new cross-platform recipes should use `set_visible_devices`. + ### vLLM DP launch mode vLLM data-parallel endpoints use one process per GPU by default. Set diff --git a/src/srtctl/backends/vllm.py b/src/srtctl/backends/vllm.py index 14c8285b3..cb2bad92e 100644 --- a/src/srtctl/backends/vllm.py +++ b/src/srtctl/backends/vllm.py @@ -168,7 +168,11 @@ class VLLMProtocol: # vLLM server CLI config per mode vllm_config: VLLMServerConfig | None = None - # Legacy device binding for vLLM builds without --device-ids. + # Vendor-neutral device binding for vLLM builds without --device-ids. + # When unset, preserve the legacy CUDA-named option below. + set_visible_devices: bool | None = None + + # Legacy compatibility alias. New recipes should use set_visible_devices. set_cuda_visible_devices: bool = False # Default KV connector: "nixl", "lmcache", or a raw JSON string for --kv-transfer-config. @@ -486,6 +490,8 @@ def should_set_visible_devices(self, process: Process) -> bool: before https://github.com/vllm-project/vllm/pull/45026 should set the platform's visible-device environment. """ + if self.set_visible_devices is not None: + return self.set_visible_devices return self.set_cuda_visible_devices def should_set_cuda_visible_devices(self, process: Process) -> bool: @@ -778,7 +784,7 @@ def build_worker_command( str(worker_port), ] ) - if not self.set_cuda_visible_devices: + if not self.should_set_visible_devices(process): device_ids = ",".join(str(i) for i in sorted(process.gpu_indices)) if device_ids: cmd.extend(["--device-ids", device_ids]) @@ -812,7 +818,7 @@ def build_worker_command( kv_transfer_cfg = _connector_to_kv_transfer_config(connector) cmd.extend(["--kv-transfer-config", kv_transfer_cfg]) - if not self.set_cuda_visible_devices: + if not self.should_set_visible_devices(process): device_ids = ",".join(str(i) for i in sorted(process.gpu_indices)) if device_ids: cmd.extend(["--device-ids", device_ids]) diff --git a/tests/test_accelerator.py b/tests/test_accelerator.py index 0e620a5ca..24c47dfd0 100644 --- a/tests/test_accelerator.py +++ b/tests/test_accelerator.py @@ -6,6 +6,7 @@ from srtctl.core.accelerator import visible_device_environment from srtctl.core.schema import ClusterConfig +from srtctl.core.topology import Process def test_nvidia_visible_device_environment() -> None: @@ -42,6 +43,39 @@ def test_cluster_config_rejects_unknown_gpu_directive() -> None: ClusterConfig.Schema().load({"gpu_sbatch_directive": "rocm"}) +def test_vllm_vendor_neutral_visible_device_setting() -> None: + from srtctl.backends import VLLMProtocol + + process = Process( + node="node0", + gpu_indices=frozenset({0}), + sys_port=7500, + http_port=8000, + endpoint_mode="agg", + endpoint_index=0, + ) + assert VLLMProtocol(set_visible_devices=True).should_set_visible_devices(process) + assert not VLLMProtocol(set_visible_devices=False).should_set_visible_devices(process) + + +def test_vllm_legacy_cuda_named_setting_remains_compatible() -> None: + from srtctl.backends import VLLMProtocol + + process = Process( + node="node0", + gpu_indices=frozenset({0}), + sys_port=7500, + http_port=8000, + endpoint_mode="agg", + endpoint_index=0, + ) + assert VLLMProtocol(set_cuda_visible_devices=True).should_set_visible_devices(process) + assert not VLLMProtocol( + set_visible_devices=False, + set_cuda_visible_devices=True, + ).should_set_visible_devices(process) + + @pytest.mark.parametrize( ("directive", "expected", "unexpected"), [ From 1a63cb935411a103b09c6f1299160d0f311bd897 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 9 Aug 2026 19:56:44 -0500 Subject: [PATCH 17/46] fix(ci): format GPU directive fallback --- src/srtctl/cli/submit.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/srtctl/cli/submit.py b/src/srtctl/cli/submit.py index 4f8e457b3..3beec5839 100755 --- a/src/srtctl/cli/submit.py +++ b/src/srtctl/cli/submit.py @@ -466,9 +466,7 @@ def generate_minimal_sbatch_script( gpu_sbatch_directive = get_srtslurm_setting("gpu_sbatch_directive") if gpu_sbatch_directive is None: # Backward compatibility for existing cluster configurations. - gpu_sbatch_directive = ( - "gpus-per-node" if get_srtslurm_setting("use_gpus_per_node_directive", True) else "none" - ) + gpu_sbatch_directive = "gpus-per-node" if get_srtslurm_setting("use_gpus_per_node_directive", True) else "none" rendered = template.render( job_name=job_name, From 750dc88382bf0fdbbc79681d345a86a3d8268e48 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 9 Aug 2026 20:55:25 -0500 Subject: [PATCH 18/46] Support node-local runtime config transport --- docs/config-reference.md | 3 + docs/installation.md | 1 + src/srtctl/cli/submit.py | 44 ++++++++++++-- src/srtctl/core/schema.py | 1 + src/srtctl/templates/job_script_minimal.j2 | 23 ++++++++ tests/test_accelerator.py | 68 +++++++++++++++++++++- 6 files changed, 134 insertions(+), 6 deletions(-) diff --git a/docs/config-reference.md b/docs/config-reference.md index e8f29f05d..7d9cd9f06 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -113,6 +113,7 @@ The `srtslurm.yaml` file can contain the following fields: | `network_interface` | string | Network interface for NCCL | | `accelerator_vendor` | string | Accelerator runtime: `nvidia` (default) or `amd` | | `gpu_sbatch_directive` | string | GPU allocation directive: `gpus-per-node`, `gres`, or `none` | +| `runtime_config_transport` | string | Runtime config transport: `shared-filesystem` (default) or `embedded` | | `srtctl_root` | string | Root directory for srtctl | | `output_dir` | string | Custom output directory (overrides srtctl_root/outputs) | | `model_paths` | dict | Model path aliases | @@ -123,6 +124,8 @@ The `srtslurm.yaml` file can contain the following fields: **output_dir**: When set, job logs are written to `output_dir/{job_id}/logs` instead of `srtctl_root/outputs/{job_id}/logs`. Useful for CI/CD and ephemeral environments. +**runtime_config_transport**: Leave this as `shared-filesystem` when the submitter and compute nodes see the same output directory. Use `embedded` when the output path is node-local: srtctl safely embeds the exact resolved YAML in the Slurm script, materializes it with owner-only permissions on the allocated head node, and bootstraps the Slurm log from the existing output base into the normal per-job log directory. Embedded payloads are data-safe but not a secrets store: users who can inspect Slurm batch scripts can decode them. + **default_bash_preamble**: A shell snippet (e.g. `"ulimit -n 1048576 -s unlimited -u 1048576"`) prepended to every container srun launched by srtctl — workers, frontends, telemetry, benchmark, postprocess. Runs before per-call `bash_preamble` and the main command, so cluster-wide ulimits apply to everything downstream. Silently dropped for distroless containers (e.g. `prom/node-exporter`) that bypass the bash wrapper; a WARNING log is emitted in that case. **nginx_raise_ulimit**: When set to `true` or `false`, this value is applied to jobs that omit `frontend.nginx_raise_ulimit` in the recipe. Use `true` on clusters where raising the nginx container’s open-file limit is allowed; leave unset if each job should rely on the frontend default (`false`). A recipe that sets `frontend.nginx_raise_ulimit` always wins. diff --git a/docs/installation.md b/docs/installation.md index 006e8b0c2..a5abd93bb 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -111,6 +111,7 @@ gpus_per_node: 4 gpu_sbatch_directive: gpus-per-node # Use gres for --gres=gpu:N clusters, or none # Legacy compatibility: use_gpus_per_node_directive: true accelerator_vendor: nvidia # Use amd for ROCm clusters +runtime_config_transport: shared-filesystem # Use embedded for node-local output paths use_segment_sbatch_directive: true # Set false if cluster doesn't support --segment use_exclusive_sbatch_directive: false # Set true if cluster requires --exclusive diff --git a/src/srtctl/cli/submit.py b/src/srtctl/cli/submit.py index 3beec5839..24aeb7c50 100755 --- a/src/srtctl/cli/submit.py +++ b/src/srtctl/cli/submit.py @@ -14,6 +14,7 @@ """ import argparse +import base64 import contextlib import json import logging @@ -401,6 +402,8 @@ def generate_minimal_sbatch_script( setup_script: str | None = None, output_dir: Path | None = None, runtime_config_filename: str = "config.yaml", + runtime_config_text: str | None = None, + source_config_text: str | None = None, ) -> str: """Generate minimal sbatch script that calls the Python orchestrator. @@ -413,6 +416,11 @@ def generate_minimal_sbatch_script( setup_script: Optional setup script override (passed via env var) output_dir: Custom output directory (CLI flag, highest priority) runtime_config_filename: Config file name under OUTPUT_DIR used by do_sweep + runtime_config_text: Exact runtime YAML to embed when the cluster uses + the ``embedded`` runtime config transport. Read from config_path + when omitted. + source_config_text: Original source YAML to preserve as config.yaml for + resolved override jobs using the embedded transport. Returns: Rendered sbatch script as string @@ -468,6 +476,27 @@ def generate_minimal_sbatch_script( # Backward compatibility for existing cluster configurations. gpu_sbatch_directive = "gpus-per-node" if get_srtslurm_setting("use_gpus_per_node_directive", True) else "none" + runtime_config_transport = get_srtslurm_setting("runtime_config_transport", "shared-filesystem") + embedded_config_files: list[dict[str, str]] = [] + if runtime_config_transport == "embedded": + if runtime_config_text is None: + runtime_config_text = config_path.read_text() + if source_config_text is not None and runtime_config_filename != "config.yaml": + embedded_config_files.append( + { + "filename": "config.yaml", + "payload": base64.b64encode(source_config_text.encode()).decode("ascii"), + } + ) + embedded_config_files.append( + { + "filename": runtime_config_filename, + "payload": base64.b64encode(runtime_config_text.encode()).decode("ascii"), + } + ) + elif runtime_config_transport != "shared-filesystem": + raise ValueError(f"Unsupported runtime config transport: {runtime_config_transport}") + rendered = template.render( job_name=job_name, total_nodes=total_nodes, @@ -479,6 +508,7 @@ def generate_minimal_sbatch_script( time_limit=config.slurm.time_limit or "01:00:00", config_path=str(config_path.resolve()), runtime_config_filename=runtime_config_filename, + embedded_config_files=embedded_config_files, timestamp=timestamp, gpu_sbatch_directive=gpu_sbatch_directive, use_segment_sbatch_directive=get_srtslurm_setting("use_segment_sbatch_directive", True), @@ -597,6 +627,8 @@ def submit_with_orchestrator( setup_script=setup_script, output_dir=output_dir, runtime_config_filename=runtime_config_filename, + runtime_config_text=resolved_runtime_config_text, + source_config_text=source_config_path.read_text() if source_config_path else None, ) # Identity validation (inline, <1s) — runs for both dry-run and submit @@ -667,11 +699,13 @@ def submit_with_orchestrator( job_output_dir = srtctl_source / "outputs" / job_id job_output_dir.mkdir(parents=True, exist_ok=True) - shutil.copy(source_config_path or config_path, job_output_dir / "config.yaml") - if source_config_path: - assert resolved_runtime_config_text is not None - runtime_config_path = job_output_dir / runtime_config_filename - runtime_config_path.write_text(resolved_runtime_config_text) + runtime_config_transport = get_srtslurm_setting("runtime_config_transport", "shared-filesystem") + if runtime_config_transport != "embedded": + shutil.copy(source_config_path or config_path, job_output_dir / "config.yaml") + if source_config_path: + assert resolved_runtime_config_text is not None + runtime_config_path = job_output_dir / runtime_config_filename + runtime_config_path.write_text(resolved_runtime_config_text) shutil.copy(script_path, job_output_dir / "sbatch_script.sh") git_sources = git_snapshot_sources_from_extra_mounts(config) if git_sources: diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py index da87450de..3f4cd29fa 100755 --- a/src/srtctl/core/schema.py +++ b/src/srtctl/core/schema.py @@ -186,6 +186,7 @@ class ClusterConfig: network_interface: str | None = None accelerator_vendor: Literal["nvidia", "amd"] = "nvidia" gpu_sbatch_directive: Literal["gpus-per-node", "gres", "none"] | None = None + runtime_config_transport: Literal["shared-filesystem", "embedded"] = "shared-filesystem" use_gpus_per_node_directive: bool = True use_segment_sbatch_directive: bool = True use_exclusive_sbatch_directive: bool = False diff --git a/src/srtctl/templates/job_script_minimal.j2 b/src/srtctl/templates/job_script_minimal.j2 index 889a469b5..bddbc6286 100644 --- a/src/srtctl/templates/job_script_minimal.j2 +++ b/src/srtctl/templates/job_script_minimal.j2 @@ -2,7 +2,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 #SBATCH --job-name={{ job_name }} +{% if embedded_config_files %} +# Node-local output bases cannot have their per-job directory created by the +# submitter. SLURM opens stdout before the batch script runs, so open it in the +# existing base and move the live file after creating OUTPUT_DIR below. +#SBATCH --output={{ output_base }}/.srtctl-sweep-%j.log +{% else %} #SBATCH --output={{ output_base }}/%j/logs/sweep_%j.log +{% endif %} {% if het_components %} {# SLURM applies each `#SBATCH ...` to the component immediately preceding it. Per-component required scheduling directives (--account/--partition/--time @@ -90,6 +97,22 @@ OUTPUT_DIR="${OUTPUT_BASE}/${SLURM_JOB_ID}" LOG_DIR="${OUTPUT_DIR}/logs" mkdir -p "${LOG_DIR}" +{% if embedded_config_files %} +BOOTSTRAP_LOG="${OUTPUT_BASE}/.srtctl-sweep-${SLURM_JOB_ID}.log" +if [ -e "${BOOTSTRAP_LOG}" ]; then + mv "${BOOTSTRAP_LOG}" "${LOG_DIR}/sweep_${SLURM_JOB_ID}.log" +fi + +# These payloads contain data only. Base64 keeps arbitrary YAML from becoming +# shell syntax; the temporary file and atomic rename prevent partial reads. +{% for config_file in embedded_config_files %} +RUNTIME_CONFIG="${OUTPUT_DIR}/{{ config_file.filename }}" +RUNTIME_CONFIG_TMP="${RUNTIME_CONFIG}.tmp.$$" +(umask 077; printf '%s' '{{ config_file.payload }}' | base64 --decode > "${RUNTIME_CONFIG_TMP}") +mv "${RUNTIME_CONFIG_TMP}" "${RUNTIME_CONFIG}" +{% endfor %} +{% endif %} + # Export for Python orchestrator to use the same paths export SRTCTL_OUTPUT_DIR="${OUTPUT_DIR}" diff --git a/tests/test_accelerator.py b/tests/test_accelerator.py index 24c47dfd0..4d7d69571 100644 --- a/tests/test_accelerator.py +++ b/tests/test_accelerator.py @@ -1,5 +1,7 @@ """Tests for accelerator-specific runtime behavior.""" +import base64 +import re from pathlib import Path import pytest @@ -28,9 +30,16 @@ def test_cluster_config_defaults_to_nvidia() -> None: def test_cluster_config_accepts_amd() -> None: - config = ClusterConfig.Schema().load({"accelerator_vendor": "amd", "gpu_sbatch_directive": "gres"}) + config = ClusterConfig.Schema().load( + { + "accelerator_vendor": "amd", + "gpu_sbatch_directive": "gres", + "runtime_config_transport": "embedded", + } + ) assert config.accelerator_vendor == "amd" assert config.gpu_sbatch_directive == "gres" + assert config.runtime_config_transport == "embedded" def test_cluster_config_rejects_unknown_accelerator() -> None: @@ -129,3 +138,60 @@ def test_legacy_gpu_directive_boolean_is_preserved(monkeypatch) -> None: assert "#SBATCH --gpus-per-node=" not in script assert "#SBATCH --gres=gpu:" not in script + + +def test_shared_filesystem_runtime_config_transport_is_unchanged(monkeypatch) -> None: + from srtctl.cli import submit + from srtctl.core.schema import ModelConfig, ResourceConfig, SrtConfig + + monkeypatch.setattr(submit, "get_srtslurm_setting", lambda key, default=None: default) + config = SrtConfig( + name="shared-output-test", + model=ModelConfig(path="/model", container="/container.sqsh", precision="fp16"), + resources=ResourceConfig(gpu_type="h100", gpus_per_node=8, agg_nodes=1), + ) + + script = submit.generate_minimal_sbatch_script(config, Path("/tmp/not-required-for-shared.yaml")) + + assert "#SBATCH --output=" in script + assert "/%j/logs/sweep_%j.log" in script + assert ".srtctl-sweep-%j.log" not in script + assert "base64 --decode" not in script + + +def test_embedded_transport_preserves_resolved_yaml_as_inert_data(monkeypatch, tmp_path) -> None: + from srtctl.cli import submit + from srtctl.core.schema import ModelConfig, ResourceConfig, SrtConfig + + settings = {"runtime_config_transport": "embedded"} + monkeypatch.setattr(submit, "get_srtslurm_setting", lambda key, default=None: settings.get(key, default)) + config = SrtConfig( + name="node-local-output-test", + model=ModelConfig(path="/model", container="/container.sqsh", precision="fp16"), + resources=ResourceConfig(gpu_type="mi300x", gpus_per_node=1, agg_nodes=1), + ) + source_text = "name: source\nnote: original\n" + runtime_text = 'name: resolved\nnote: "\'; touch /tmp/must-not-run; #"\n' + config_path = tmp_path / "resolved.yaml" + config_path.write_text(runtime_text) + + script = submit.generate_minimal_sbatch_script( + config, + config_path, + runtime_config_filename="config_variant.yaml", + runtime_config_text=runtime_text, + source_config_text=source_text, + ) + + assert "#SBATCH --output=" in script + assert "/.srtctl-sweep-%j.log" in script + assert 'mv "${BOOTSTRAP_LOG}" "${LOG_DIR}/sweep_${SLURM_JOB_ID}.log"' in script + assert "touch /tmp/must-not-run" not in script + + embedded = re.findall( + r'RUNTIME_CONFIG="\$\{OUTPUT_DIR\}/([^\"]+)".*?printf \'%s\' \'([^\']+)\' \| base64 --decode', + script, + flags=re.DOTALL, + ) + decoded = {filename: base64.b64decode(payload).decode() for filename, payload in embedded} + assert decoded == {"config.yaml": source_text, "config_variant.yaml": runtime_text} From 4e2355a9646f6adf5e92647ce87e4c3ec0e75617 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 9 Aug 2026 21:30:52 -0500 Subject: [PATCH 19/46] fix: translate node-local log mounts in containers --- src/srtctl/cli/do_sweep.py | 5 +++- src/srtctl/templates/job_script_minimal.j2 | 8 ++++++ tests/test_accelerator.py | 1 + tests/test_slurm.py | 31 ++++++++++++++++++++++ 4 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/srtctl/cli/do_sweep.py b/src/srtctl/cli/do_sweep.py index 8f385b6f6..1b167c4f1 100644 --- a/src/srtctl/cli/do_sweep.py +++ b/src/srtctl/cli/do_sweep.py @@ -165,6 +165,9 @@ def start_head_infrastructure(self, registry: ProcessRegistry) -> ManagedProcess setup_script_container = Path("/tmp/setup_head.py") infra_log = self.runtime.log_dir / "infra.out" + container_log_dir = self.runtime.container_mounts.get(self.runtime.log_dir) + if container_log_dir is None: + raise RuntimeError(f"Runtime log directory is not mounted in the container: {self.runtime.log_dir}") cmd = [ "python3", @@ -172,7 +175,7 @@ def start_head_infrastructure(self, registry: ProcessRegistry) -> ManagedProcess "--name", self.config.name, "--log-dir", - str(self.runtime.log_dir), + str(container_log_dir), ] if self.config.infra.nats_max_payload_mb is not None: cmd += ["--nats-max-payload-mb", str(self.config.infra.nats_max_payload_mb)] diff --git a/src/srtctl/templates/job_script_minimal.j2 b/src/srtctl/templates/job_script_minimal.j2 index bddbc6286..97005e18c 100644 --- a/src/srtctl/templates/job_script_minimal.j2 +++ b/src/srtctl/templates/job_script_minimal.j2 @@ -98,6 +98,14 @@ LOG_DIR="${OUTPUT_DIR}/logs" mkdir -p "${LOG_DIR}" {% if embedded_config_files %} +# The output base is node-local in embedded mode. Create the identical host +# mount source on every allocated node before any containerized worker starts. +# Worker stdout remains node-local; result collection is handled separately. +if [ "${SLURM_JOB_NUM_NODES:-1}" -gt 1 ]; then + srun --overlap --nodes="${SLURM_JOB_NUM_NODES}" --ntasks="${SLURM_JOB_NUM_NODES}" \ + --ntasks-per-node=1 mkdir -p "${LOG_DIR}" +fi + BOOTSTRAP_LOG="${OUTPUT_BASE}/.srtctl-sweep-${SLURM_JOB_ID}.log" if [ -e "${BOOTSTRAP_LOG}" ]; then mv "${BOOTSTRAP_LOG}" "${LOG_DIR}/sweep_${SLURM_JOB_ID}.log" diff --git a/tests/test_accelerator.py b/tests/test_accelerator.py index 4d7d69571..cb62ade45 100644 --- a/tests/test_accelerator.py +++ b/tests/test_accelerator.py @@ -186,6 +186,7 @@ def test_embedded_transport_preserves_resolved_yaml_as_inert_data(monkeypatch, t assert "#SBATCH --output=" in script assert "/.srtctl-sweep-%j.log" in script assert 'mv "${BOOTSTRAP_LOG}" "${LOG_DIR}/sweep_${SLURM_JOB_ID}.log"' in script + assert '--ntasks-per-node=1 mkdir -p "${LOG_DIR}"' in script assert "touch /tmp/must-not-run" not in script embedded = re.findall( diff --git a/tests/test_slurm.py b/tests/test_slurm.py index 94a427ec2..d88b37959 100644 --- a/tests/test_slurm.py +++ b/tests/test_slurm.py @@ -10,6 +10,7 @@ import pytest +from srtctl.cli.do_sweep import SweepOrchestrator from srtctl.cli.mixins.worker_stage import WorkerStageMixin from srtctl.core.schema import ObservabilityConfig from srtctl.core.slurm import get_slurm_het_nodelists, start_srun_process @@ -21,6 +22,36 @@ def _built_bash_command(mock_popen: MagicMock) -> str: return srun_cmd[-1] +def test_head_infrastructure_uses_container_log_mount(tmp_path: Path) -> None: + """Infrastructure must receive its in-container log path, not the host path.""" + host_log_dir = tmp_path / "outputs" / "123" / "logs" + host_log_dir.mkdir(parents=True) + nodes = MagicMock() + nodes.infra = "node-a" + nodes.het_group_for.return_value = None + orchestrator = SweepOrchestrator( + config=SimpleNamespace(name="test-run", infra=SimpleNamespace(nats_max_payload_mb=None)), + runtime=SimpleNamespace( + nodes=nodes, + log_dir=host_log_dir, + container_image=Path("/container.sqsh"), + container_mounts={host_log_dir: Path("/logs")}, + ), + ) + + with ( + patch("srtctl.cli.do_sweep.start_srun_process", return_value=MagicMock()) as mock_srun, + patch("srtctl.cli.do_sweep.wait_for_port", return_value=True), + ): + orchestrator.start_head_infrastructure(MagicMock()) + + call = mock_srun.call_args.kwargs + log_dir_index = call["command"].index("--log-dir") + 1 + assert call["command"][log_dir_index] == "/logs" + assert call["output"] == str(host_log_dir / "infra.out") + assert call["container_mounts"][host_log_dir] == Path("/logs") + + def test_start_srun_exports_env_before_preamble() -> None: with ( patch("srtctl.core.slurm.get_slurm_job_id", return_value="12345"), From 96f10cb35034c4561879cd1d4b6ea00d87348c98 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 9 Aug 2026 21:50:54 -0500 Subject: [PATCH 20/46] fix: support immutable worker containers --- src/srtctl/cli/mixins/worker_stage.py | 12 ++++++++++-- src/srtctl/core/schema.py | 19 +++++++++---------- tests/test_configs.py | 14 +++++++------- tests/test_slurm.py | 22 +++++++++++++++++++--- 4 files changed, 45 insertions(+), 22 deletions(-) diff --git a/src/srtctl/cli/mixins/worker_stage.py b/src/srtctl/cli/mixins/worker_stage.py index 17c15357e..66431bbbe 100644 --- a/src/srtctl/cli/mixins/worker_stage.py +++ b/src/srtctl/cli/mixins/worker_stage.py @@ -10,6 +10,7 @@ import logging import shlex from collections import defaultdict +from pathlib import Path from typing import TYPE_CHECKING, Any from srtctl.core.accelerator import visible_device_environment @@ -105,6 +106,13 @@ def _visible_device_environment(self, process: "Process") -> dict[str, str]: return {} return visible_device_environment(self.runtime.accelerator_vendor, process.cuda_visible_devices) + def _container_log_path(self, filename: str) -> Path: + """Return a worker-visible path under the runtime log mount.""" + container_log_dir = self.runtime.container_mounts.get(self.runtime.log_dir) + if container_log_dir is None: + raise RuntimeError(f"Runtime log directory is not mounted in the container: {self.runtime.log_dir}") + return container_log_dir / filename + def _apply_kvbm_endpoint_env(self, env_to_set: dict[str, str], endpoint_processes: list["Process"]) -> None: """Fill KVBM leader ZMQ settings for an endpoint. @@ -143,7 +151,7 @@ def start_worker(self, process: "Process", endpoint_processes: list["Process"]) # Log and config files worker_log = self.runtime.log_dir / f"{process.node}_{mode}_w{index}.out" - config_dump = self.runtime.log_dir / f"{process.node}_config.json" + config_dump = self._container_log_path(f"{process.node}_config.json") # Profiling setup profiling = self.config.profiling @@ -295,7 +303,7 @@ def start_endpoint_worker(self, endpoint_processes: list["Process"]) -> ManagedP # Log and config files (use leader node in name) worker_log = self.runtime.log_dir / f"{leader.node}_{mode}_w{index}.out" - config_dump = self.runtime.log_dir / f"{leader.node}_config.json" + config_dump = self._container_log_path(f"{leader.node}_config.json") # Profiling setup profiling = self.config.profiling diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py index 3f4cd29fa..3ae37c754 100755 --- a/src/srtctl/core/schema.py +++ b/src/srtctl/core/schema.py @@ -1254,22 +1254,21 @@ def _serialize_node_install(install_cmd: str) -> str: serializes them; a sentinel lets every task after the first short-circuit the (idempotent) install entirely. - The lock/sentinel are anchored in the active Python environment - (``sys.prefix``) — the exact resource being protected. That location is - part of the container root filesystem, so it is: - * shared by every task sharing that site-packages (correct serialization), - * private to each container instance, so co-located containers with a - bind-mounted /tmp neither over-serialize nor wrongly skip each other's - install, and distinct across jobs (no cross-job/version staleness). + The lock/sentinel live in a writable runtime directory keyed by Slurm job + and step. Every task in one srun step shares that key, while separate + frontend/worker steps and separate jobs cannot reuse a stale sentinel. The + writable runtime path also supports immutable container roots, where + ``sys.prefix`` is readable even when Pyxis remaps the container user. FD 200 (node-local) is kept distinct from the ``flock -x 201`` that the hash-pinned source install nests on the /configs cache lock inside a subshell. Distinct FDs keep the two node-local and cross-node locks independent and refactor-proof even if that inner subshell is removed. """ - # Resolve the env dir at runtime; fall back to $HOME (also container-private) - # if python3 is somehow unavailable before the install runs. - resolve_dir = 'DYN_LOCK_DIR="$(python3 -c \'import sys; print(sys.prefix)\' 2>/dev/null || echo "${HOME:-/root}")"' + resolve_dir = ( + 'DYN_LOCK_DIR="${XDG_RUNTIME_DIR:-/tmp}/srtctl-dynamo-' + '${SLURM_JOB_ID:-job}-${SLURM_STEP_ID:-step}" && mkdir -p "$DYN_LOCK_DIR"' + ) lock = '"$DYN_LOCK_DIR/.srtctl_dynamo_install.lock"' sentinel = '"$DYN_LOCK_DIR/.srtctl_dynamo_install.complete"' return ( diff --git a/tests/test_configs.py b/tests/test_configs.py index c913aaceb..94f0dbd50 100644 --- a/tests/test_configs.py +++ b/tests/test_configs.py @@ -194,13 +194,12 @@ def test_wheel_install_command(self): assert "git clone" not in cmd def test_install_command_serialized_with_flock(self): - """Install command is wrapped in a per-environment flock + sentinel. + """Install command is wrapped in a writable per-step flock + sentinel. With --ntasks-per-node > 1 (e.g. TRTLLM), co-located tasks race concurrent pip installs into the shared container site-packages. The - wrapper serializes them and lets tasks after the first skip. The lock - is anchored in the Python env (sys.prefix), NOT /tmp, so co-located - containers with a bind-mounted /tmp don't collide. + wrapper serializes them and lets tasks after the first skip. Slurm's + job/step identity prevents distinct containers from sharing a sentinel. """ from srtctl.core.schema import DynamoConfig @@ -209,9 +208,10 @@ def test_install_command_serialized_with_flock(self): DynamoConfig(wheel="1.2.0.dev20260426"), ): cmd = config.get_install_commands() - # Lock dir resolved from the active Python env, not /tmp. - assert "sys.prefix" in cmd - assert "/tmp/srtctl_dynamo_install" not in cmd + assert "${XDG_RUNTIME_DIR:-/tmp}" in cmd + assert "${SLURM_JOB_ID:-job}-${SLURM_STEP_ID:-step}" in cmd + assert 'mkdir -p "$DYN_LOCK_DIR"' in cmd + assert "sys.prefix" not in cmd # FD 200 node-local; the hash source install nests flock -x 201 on # the /configs cache lock; distinct FDs keep the locks independent. assert "flock -x 200" in cmd diff --git a/tests/test_slurm.py b/tests/test_slurm.py index d88b37959..7139523d9 100644 --- a/tests/test_slurm.py +++ b/tests/test_slurm.py @@ -229,7 +229,7 @@ def test_worker_stage_wraps_nonfatal_fingerprint_hook(tmp_path: Path) -> None: gpus_per_node=8, environment={}, container_image=Path("/container.sqsh"), - container_mounts={}, + container_mounts={tmp_path: Path("/logs")}, srun_options=[], ) process = SimpleNamespace( @@ -286,7 +286,7 @@ def _remap_worker_mixin(tmp_path: Path, *, frontend_type: str, dynamo_install: b gpus_per_node=8, environment={}, container_image=Path("/container.sqsh"), - container_mounts={}, + container_mounts={tmp_path: Path("/logs")}, srun_options=[], ) process = SimpleNamespace( @@ -301,6 +301,22 @@ def _remap_worker_mixin(tmp_path: Path, *, frontend_type: str, dynamo_install: b return mixin, process +@pytest.mark.parametrize("launch_method", ["start_worker", "start_endpoint_worker"]) +def test_worker_config_dump_uses_container_log_mount(tmp_path: Path, launch_method: str) -> None: + """Backend config dumps must use a path visible inside the worker container.""" + mixin, process = _remap_worker_mixin(tmp_path, frontend_type="sglang", dynamo_install=False) + with ( + patch("srtctl.cli.mixins.worker_stage.generate_capture_script", return_value="fingerprint || true"), + patch("srtctl.cli.mixins.worker_stage.start_srun_process", return_value=MagicMock()), + ): + if launch_method == "start_worker": + mixin.start_worker(process, [process]) + else: + mixin.start_endpoint_worker([process]) + + assert mixin.backend.build_worker_command.call_args.kwargs["dump_config_path"] == Path("/logs/node-a_config.json") + + def test_worker_stage_injects_remap_root_for_dynamo_install(tmp_path: Path) -> None: mixin, process = _remap_worker_mixin(tmp_path, frontend_type="dynamo", dynamo_install=True) with ( @@ -481,7 +497,7 @@ def test_worker_stage_unsets_vllm_port_for_multinode_endpoint(tmp_path: Path) -> gpus_per_node=8, environment={}, container_image=Path("/container.sqsh"), - container_mounts={}, + container_mounts={tmp_path: Path("/logs")}, srun_options=[], ) process = SimpleNamespace( From 98a74482bd4ee1a8c3a26fa8bd151c04f3b28a49 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 9 Aug 2026 21:58:16 -0500 Subject: [PATCH 21/46] feat: separate submit and runtime source paths --- src/srtctl/cli/submit.py | 11 +++++++++-- tests/test_configs.py | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/srtctl/cli/submit.py b/src/srtctl/cli/submit.py index 24aeb7c50..cd5909a8e 100755 --- a/src/srtctl/cli/submit.py +++ b/src/srtctl/cli/submit.py @@ -432,8 +432,15 @@ def generate_minimal_sbatch_script( template_dir = Path(__file__).parent.parent / "templates" srtctl_root = get_srtslurm_setting("srtctl_root") - # srtctl source is the parent of src/srtctl (i.e., the repo root) - srtctl_source = Path(srtctl_root) if srtctl_root else Path(__file__).parent.parent.parent.parent + runtime_source_override = os.environ.get("SRTCTL_RUNTIME_SOURCE_DIR") + # The submitter checkout may be login-node-local while compute nodes use a + # separately staged source tree. Keep that runtime override distinct from + # SRTCTL_SOURCE_DIR, which the generated job exports for its child processes. + if runtime_source_override: + srtctl_source = Path(os.path.expandvars(runtime_source_override)).expanduser() + else: + # srtctl source is the parent of src/srtctl (i.e., the repo root) + srtctl_source = Path(srtctl_root) if srtctl_root else Path(__file__).parent.parent.parent.parent # Determine output base directory # Priority: CLI -o flag > srtslurm.yaml output_dir > srtctl_root/outputs diff --git a/tests/test_configs.py b/tests/test_configs.py index 94f0dbd50..fb5ab1aa0 100644 --- a/tests/test_configs.py +++ b/tests/test_configs.py @@ -975,6 +975,24 @@ def test_sbatch_template_includes_setup_script_env_var(self): ) assert 'export SRTCTL_SETUP_SCRIPT="install-sglang-main.sh"' in script + def test_sbatch_template_accepts_node_local_runtime_source(self, monkeypatch): + """A login-node submission can target a separately staged compute checkout.""" + from pathlib import Path + + from srtctl.cli.submit import generate_minimal_sbatch_script + from srtctl.core.schema import ModelConfig, ResourceConfig, SrtConfig + + config = SrtConfig( + name="test", + model=ModelConfig(path="/model", container="/container.sqsh", precision="fp8"), + resources=ResourceConfig(gpu_type="mi300x", gpus_per_node=1, agg_nodes=1), + ) + monkeypatch.setenv("SRTCTL_RUNTIME_SOURCE_DIR", "/raid/runtime/srt-slurm") + + script = generate_minimal_sbatch_script(config, Path("/tmp/test.yaml")) + + assert 'SRTCTL_SOURCE="/raid/runtime/srt-slurm"' in script + def test_sbatch_template_prefetches_dynamo_wheel(self): """dynamo.wheel is exported and prefetched before orchestrator launch.""" from pathlib import Path From 29b9ddd866f89592b59f11fc62a78f5c8aa1097b Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 9 Aug 2026 22:23:45 -0500 Subject: [PATCH 22/46] fix: support immutable AMD control plane --- src/srtctl/cli/mixins/worker_stage.py | 8 ++-- src/srtctl/core/runtime.py | 13 ++++--- src/srtctl/core/schema.py | 56 ++++++++++++++++++--------- src/srtctl/frontends/dynamo.py | 4 +- tests/test_configs.py | 48 ++++++++++++++++------- tests/test_frontends.py | 7 ++++ tests/test_ip_utils.py | 33 ++++++++++++++++ tests/test_slurm.py | 10 +++++ 8 files changed, 135 insertions(+), 44 deletions(-) diff --git a/src/srtctl/cli/mixins/worker_stage.py b/src/srtctl/cli/mixins/worker_stage.py index 66431bbbe..e35ae54f2 100644 --- a/src/srtctl/cli/mixins/worker_stage.py +++ b/src/srtctl/cli/mixins/worker_stage.py @@ -179,8 +179,8 @@ def start_worker(self, process: "Process", endpoint_processes: list["Process"]) # Environment variables env_to_set = { "HEAD_NODE_IP": self.runtime.head_node_ip, - "ETCD_ENDPOINTS": f"http://{self.runtime.nodes.infra}:{ETCD_CLIENT_PORT}", - "NATS_SERVER": f"nats://{self.runtime.nodes.infra}:{NATS_PORT}", + "ETCD_ENDPOINTS": f"http://{self.runtime.infra_node_ip}:{ETCD_CLIENT_PORT}", + "NATS_SERVER": f"nats://{self.runtime.infra_node_ip}:{NATS_PORT}", "DYN_SYSTEM_PORT": str(process.sys_port), "DYN_REQUEST_PLANE": self.config.dynamo.request_plane, "DYN_SKIP_SGLANG_LOG_FORMATTING": "1", @@ -330,8 +330,8 @@ def start_endpoint_worker(self, endpoint_processes: list["Process"]) -> ManagedP # Environment variables env_to_set = { "HEAD_NODE_IP": self.runtime.head_node_ip, - "ETCD_ENDPOINTS": f"http://{self.runtime.nodes.infra}:{ETCD_CLIENT_PORT}", - "NATS_SERVER": f"nats://{self.runtime.nodes.infra}:{NATS_PORT}", + "ETCD_ENDPOINTS": f"http://{self.runtime.infra_node_ip}:{ETCD_CLIENT_PORT}", + "NATS_SERVER": f"nats://{self.runtime.infra_node_ip}:{NATS_PORT}", "DYN_SYSTEM_PORT": str(leader.sys_port), "DYN_SKIP_SGLANG_LOG_FORMATTING": "1", } diff --git a/src/srtctl/core/runtime.py b/src/srtctl/core/runtime.py index 0b3c8703c..d3c4015e3 100644 --- a/src/srtctl/core/runtime.py +++ b/src/srtctl/core/runtime.py @@ -229,9 +229,12 @@ def from_config( # Compute run_name run_name = f"{config.name}_{job_id}" - # Resolve node IPs - head_node_ip = get_hostname_ip(nodes.head) - infra_node_ip = get_hostname_ip(nodes.infra) + # Resolve node IPs on the cluster-selected fabric. Some systems expose + # a public default route and a separate private control/data plane; the + # latter is what containers on peer Slurm nodes can reliably reach. + network_interface = get_srtslurm_setting("network_interface", "eth0") + head_node_ip = get_hostname_ip(nodes.head, network_interface) + infra_node_ip = get_hostname_ip(nodes.infra, network_interface) # Compute log directory using FormattablePath or default logic # Check for SRTCTL_OUTPUT_DIR from sbatch script first (ensures consistency) @@ -360,7 +363,7 @@ def from_config( container_image=container_image, gpus_per_node=config.resources.gpus_per_node, gpu_type=config.resources.gpu_type, - network_interface=get_srtslurm_setting("network_interface", "eth0"), + network_interface=network_interface, accelerator_vendor=get_srtslurm_setting("accelerator_vendor", "nvidia"), container_mounts={}, srun_options=dict(config.srun_options), @@ -386,7 +389,7 @@ def from_config( container_image=container_image, gpus_per_node=config.resources.gpus_per_node, gpu_type=config.resources.gpu_type, - network_interface=get_srtslurm_setting("network_interface", "eth0"), + network_interface=network_interface, accelerator_vendor=get_srtslurm_setting("accelerator_vendor", "nvidia"), container_mounts=container_mounts, srun_options=dict(config.srun_options), diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py index 3ae37c754..765c6d726 100755 --- a/src/srtctl/core/schema.py +++ b/src/srtctl/core/schema.py @@ -1244,7 +1244,7 @@ def _live_source_install_for_top_of_tree() -> str: ) -def _serialize_node_install(install_cmd: str) -> str: +def _serialize_node_install(install_cmd: str, *, job_local_site_packages: bool = False) -> str: """Serialize a node-shared dynamo install across co-located srun tasks. With ``--ntasks-per-node > 1`` (e.g. TRTLLM's MPI-style launch, one task @@ -1254,30 +1254,40 @@ def _serialize_node_install(install_cmd: str) -> str: serializes them; a sentinel lets every task after the first short-circuit the (idempotent) install entirely. - The lock/sentinel live in a writable runtime directory keyed by Slurm job - and step. Every task in one srun step shares that key, while separate - frontend/worker steps and separate jobs cannot reuse a stale sentinel. The - writable runtime path also supports immutable container roots, where - ``sys.prefix`` is readable even when Pyxis remaps the container user. + Stable release installs use the writable per-job ``/logs`` mount. Separate + frontend/worker steps on one node therefore share both the install and the + adjacent site-packages directory, while separate jobs and nodes remain + isolated. This avoids assuming that the container root or user home is + writable. Source and wheel installs retain a per-step runtime lock because + those legacy paths still install into each container's private root. FD 200 (node-local) is kept distinct from the ``flock -x 201`` that the hash-pinned source install nests on the /configs cache lock inside a subshell. Distinct FDs keep the two node-local and cross-node locks independent and refactor-proof even if that inner subshell is removed. """ - resolve_dir = ( - 'DYN_LOCK_DIR="${XDG_RUNTIME_DIR:-/tmp}/srtctl-dynamo-' - '${SLURM_JOB_ID:-job}-${SLURM_STEP_ID:-step}" && mkdir -p "$DYN_LOCK_DIR"' - ) - lock = '"$DYN_LOCK_DIR/.srtctl_dynamo_install.lock"' - sentinel = '"$DYN_LOCK_DIR/.srtctl_dynamo_install.complete"' + if job_local_site_packages: + resolve_dir = ( + 'DYN_INSTALL_DIR="/logs/.srtctl-dynamo-${SLURM_JOB_ID:-job}" && ' + 'DYN_SITE_PACKAGES="$DYN_INSTALL_DIR/site-packages" && ' + 'mkdir -p "$DYN_SITE_PACKAGES"' + ) + expose_install = ' && export PYTHONPATH="$DYN_SITE_PACKAGES${PYTHONPATH:+:$PYTHONPATH}"' + else: + resolve_dir = ( + 'DYN_INSTALL_DIR="${XDG_RUNTIME_DIR:-/tmp}/srtctl-dynamo-' + '${SLURM_JOB_ID:-job}-${SLURM_STEP_ID:-step}" && mkdir -p "$DYN_INSTALL_DIR"' + ) + expose_install = "" + lock = '"$DYN_INSTALL_DIR/.srtctl_dynamo_install.lock"' + sentinel = '"$DYN_INSTALL_DIR/.srtctl_dynamo_install.complete"' return ( f"{resolve_dir} && " f"( flock -x 200; " f"if [ -f {sentinel} ]; then " f"echo 'dynamo install already completed in this environment, skipping'; " f"else {{ {install_cmd} ; }} && touch {sentinel}; fi " - f") 200>{lock}" + f") 200>{lock}{expose_install}" ) @@ -1386,12 +1396,15 @@ def get_wheel_environment(self) -> dict[str, str]: def get_install_commands(self) -> str: """Get the bash commands to install dynamo. - The returned command is wrapped in a node-local flock + sentinel so - that co-located srun tasks (``--ntasks-per-node > 1``, e.g. TRTLLM) - install once per node instead of racing concurrent pip installs into - the shared container site-packages. See ``_serialize_node_install``. + The returned command is wrapped in a node-local flock + sentinel. + Stable releases install once per node into a job-local directory that + frontend and worker containers share; legacy source/wheel installs are + serialized within each container step. See ``_serialize_node_install``. """ - return _serialize_node_install(self._build_install_commands()) + return _serialize_node_install( + self._build_install_commands(), + job_local_site_packages=self.version is not None, + ) def _build_install_commands(self) -> str: """Build the raw (unserialized) dynamo install command.""" @@ -1416,7 +1429,12 @@ def _build_install_commands(self) -> str: if self.version is not None: return ( f"echo 'Installing dynamo {self.version}...' && " - f"pip install --break-system-packages --quiet --extra-index-url https://pypi.nvidia.com ai-dynamo-runtime=={self.version} ai-dynamo=={self.version} && " + # The backend image already owns its framework dependencies + # (torch, numpy, fsspec, etc.). Pull only Dynamo into the + # overlay so its resolver cannot shadow those validated pins. + 'python3 -m pip install --quiet --upgrade --no-deps --target "$DYN_SITE_PACKAGES" ' + "--extra-index-url https://pypi.nvidia.com " + f"ai-dynamo-runtime=={self.version} ai-dynamo=={self.version} && " f"echo 'Dynamo {self.version} installed'" ) diff --git a/src/srtctl/frontends/dynamo.py b/src/srtctl/frontends/dynamo.py index d6cfeacc9..eac40d224 100644 --- a/src/srtctl/frontends/dynamo.py +++ b/src/srtctl/frontends/dynamo.py @@ -84,8 +84,8 @@ def start_frontends( cmd.extend(self.get_frontend_args_list(config.frontend.args)) env_to_set = { - "ETCD_ENDPOINTS": f"http://{runtime.nodes.infra}:{ETCD_CLIENT_PORT}", - "NATS_SERVER": f"nats://{runtime.nodes.infra}:{NATS_PORT}", + "ETCD_ENDPOINTS": f"http://{runtime.infra_node_ip}:{ETCD_CLIENT_PORT}", + "NATS_SERVER": f"nats://{runtime.infra_node_ip}:{NATS_PORT}", "DYN_REQUEST_PLANE": config.dynamo.request_plane, "DYN_SKIP_SGLANG_LOG_FORMATTING": "1", } diff --git a/tests/test_configs.py b/tests/test_configs.py index fb5ab1aa0..c9ef820ce 100644 --- a/tests/test_configs.py +++ b/tests/test_configs.py @@ -194,32 +194,52 @@ def test_wheel_install_command(self): assert "git clone" not in cmd def test_install_command_serialized_with_flock(self): - """Install command is wrapped in a writable per-step flock + sentinel. + """Install command is wrapped in a writable per-job flock + sentinel. With --ntasks-per-node > 1 (e.g. TRTLLM), co-located tasks race concurrent pip installs into the shared container site-packages. The - wrapper serializes them and lets tasks after the first skip. Slurm's - job/step identity prevents distinct containers from sharing a sentinel. + wrapper serializes them and lets tasks after the first skip. Frontend + and worker steps on the same node share it, while job identity prevents + distinct sweeps from sharing a sentinel. """ from srtctl.core.schema import DynamoConfig - for config in ( - DynamoConfig(version="0.8.0"), - DynamoConfig(wheel="1.2.0.dev20260426"), + for cmd in ( + DynamoConfig(version="0.8.0").get_install_commands(), + DynamoConfig(wheel="1.2.0.dev20260426").get_install_commands(), ): - cmd = config.get_install_commands() - assert "${XDG_RUNTIME_DIR:-/tmp}" in cmd - assert "${SLURM_JOB_ID:-job}-${SLURM_STEP_ID:-step}" in cmd - assert 'mkdir -p "$DYN_LOCK_DIR"' in cmd assert "sys.prefix" not in cmd # FD 200 node-local; the hash source install nests flock -x 201 on # the /configs cache lock; distinct FDs keep the locks independent. assert "flock -x 200" in cmd - assert "$DYN_LOCK_DIR/.srtctl_dynamo_install.lock" in cmd - assert "$DYN_LOCK_DIR/.srtctl_dynamo_install.complete" in cmd + assert "$DYN_INSTALL_DIR/.srtctl_dynamo_install.lock" in cmd + assert "$DYN_INSTALL_DIR/.srtctl_dynamo_install.complete" in cmd # Sentinel short-circuits repeat installs; touched on success. - assert 'touch "$DYN_LOCK_DIR/.srtctl_dynamo_install.complete"' in cmd - assert '200>"$DYN_LOCK_DIR/.srtctl_dynamo_install.lock"' in cmd + assert 'touch "$DYN_INSTALL_DIR/.srtctl_dynamo_install.complete"' in cmd + assert '200>"$DYN_INSTALL_DIR/.srtctl_dynamo_install.lock"' in cmd + + release_cmd = DynamoConfig(version="0.8.0").get_install_commands() + assert 'DYN_INSTALL_DIR="/logs/.srtctl-dynamo-${SLURM_JOB_ID:-job}"' in release_cmd + assert "SLURM_STEP_ID" not in release_cmd + assert 'mkdir -p "$DYN_SITE_PACKAGES"' in release_cmd + assert 'export PYTHONPATH="$DYN_SITE_PACKAGES${PYTHONPATH:+:$PYTHONPATH}"' in release_cmd + + wheel_cmd = DynamoConfig(wheel="1.2.0.dev20260426").get_install_commands() + assert "${XDG_RUNTIME_DIR:-/tmp}" in wheel_cmd + assert "${SLURM_JOB_ID:-job}-${SLURM_STEP_ID:-step}" in wheel_cmd + assert 'mkdir -p "$DYN_INSTALL_DIR"' in wheel_cmd + assert "DYN_SITE_PACKAGES" not in wheel_cmd + assert "PYTHONPATH" not in wheel_cmd + + def test_release_install_targets_job_local_site_packages(self): + """Stable Dynamo releases must not write the immutable container root/home.""" + from srtctl.core.schema import DynamoConfig + + cmd = DynamoConfig(version="1.3.1").get_install_commands() + + assert 'python3 -m pip install --quiet --upgrade --no-deps --target "$DYN_SITE_PACKAGES"' in cmd + assert "--break-system-packages" not in cmd + assert "/root/.local" not in cmd def test_hash_install_command(self): """Hash config generates a cache-aware source-install command. diff --git a/tests/test_frontends.py b/tests/test_frontends.py index f00507c8d..2061a1458 100644 --- a/tests/test_frontends.py +++ b/tests/test_frontends.py @@ -533,6 +533,7 @@ def _dynamo_frontend_call(*, dynamo_install: bool, event_plane: str | None = "zm runtime = SimpleNamespace( log_dir=Path("/logs"), nodes=SimpleNamespace(infra="infra-node", het_group_for=lambda node: None), + infra_node_ip="10.0.0.9", container_image=Path("/container.sqsh"), container_mounts={}, environment={}, @@ -573,6 +574,12 @@ def test_default_not_injected(self): mock_srun = _dynamo_frontend_call(dynamo_install=False, event_plane=None) assert "DYN_EVENT_PLANE" not in mock_srun.call_args.kwargs["env_to_set"] + def test_control_plane_uses_routable_infra_ip(self): + env = _dynamo_frontend_call(dynamo_install=False).call_args.kwargs["env_to_set"] + assert env["NATS_SERVER"] == "nats://10.0.0.9:4222" + assert env["ETCD_ENDPOINTS"] == "http://10.0.0.9:2379" + assert "infra-node" not in env["NATS_SERVER"] + @pytest.mark.parametrize("event_plane", ["zmq", "nats"]) def test_explicit_injected(self, event_plane): mock_srun = _dynamo_frontend_call(dynamo_install=False, event_plane=event_plane) diff --git a/tests/test_ip_utils.py b/tests/test_ip_utils.py index 3ef14ed84..888c6de30 100644 --- a/tests/test_ip_utils.py +++ b/tests/test_ip_utils.py @@ -4,6 +4,8 @@ """Tests for IP address resolution helpers.""" import os +from pathlib import Path +from unittest.mock import patch from srtctl.core.ip_utils import get_node_ip @@ -25,3 +27,34 @@ def test_get_node_ip_ignores_srun_step_created_output(tmp_path, monkeypatch): ip = get_node_ip("nvl72156-T15", slurm_job_id="2279904") assert ip == "10.109.25.246" + + +def test_runtime_resolves_control_plane_ips_on_cluster_interface(tmp_path): + """Runtime endpoints must use the selected private/fabric interface.""" + from srtctl.core.runtime import Nodes, RuntimeContext + from srtctl.core.schema import ModelConfig, ResourceConfig, SrtConfig + + model = tmp_path / "model" + model.mkdir() + container = tmp_path / "image.sqsh" + container.touch() + config = SrtConfig( + name="network-contract", + model=ModelConfig(path=str(model), container=str(container), precision="fp16"), + resources=ResourceConfig(gpu_type="mi300x", gpus_per_node=8), + ) + nodes = Nodes(head="node-a", bench="node-a", infra="node-b", worker=("node-a", "node-b")) + + def setting(name, default=None): + return "fabric0" if name == "network_interface" else default + + with ( + patch("srtctl.core.runtime.Nodes.from_slurm", return_value=nodes), + patch("srtctl.core.runtime.get_srtslurm_setting", side_effect=setting), + patch("srtctl.core.runtime.get_hostname_ip", side_effect=["10.0.0.1", "10.0.0.2"]) as resolve, + ): + runtime = RuntimeContext.from_config(config, job_id="42", log_dir_base=Path(tmp_path)) + + assert resolve.call_args_list[0].args == ("node-a", "fabric0") + assert resolve.call_args_list[1].args == ("node-b", "fabric0") + assert runtime.infra_node_ip == "10.0.0.2" diff --git a/tests/test_slurm.py b/tests/test_slurm.py index 7139523d9..ba60c1587 100644 --- a/tests/test_slurm.py +++ b/tests/test_slurm.py @@ -397,6 +397,16 @@ def test_start_endpoint_worker_event_plane_default_not_injected(tmp_path: Path) assert "DYN_EVENT_PLANE" not in env +def test_worker_control_plane_uses_routable_infra_ip(tmp_path: Path) -> None: + for env in ( + _start_worker_env(tmp_path, event_plane=None), + _start_endpoint_worker_env(tmp_path, event_plane=None), + ): + assert env["NATS_SERVER"] == "nats://10.0.0.1:4222" + assert env["ETCD_ENDPOINTS"] == "http://10.0.0.1:2379" + assert "infra-node" not in env["NATS_SERVER"] + + @pytest.mark.parametrize("event_plane", ["zmq", "nats"]) def test_start_endpoint_worker_event_plane_injected(tmp_path: Path, event_plane: str) -> None: env = _start_endpoint_worker_env(tmp_path, event_plane=event_plane) From f2e95e4b1809f6e34cf78a7e35bb201923496604 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 9 Aug 2026 22:41:33 -0500 Subject: [PATCH 23/46] fix: prefer private fabric addresses --- src/srtctl/core/ip_utils/get_node_ip.sh | 32 ++++++++++++------------- tests/test_ip_utils.py | 29 ++++++++++++++++++++++ 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/src/srtctl/core/ip_utils/get_node_ip.sh b/src/srtctl/core/ip_utils/get_node_ip.sh index 6f513213c..c0f264ec3 100644 --- a/src/srtctl/core/ip_utils/get_node_ip.sh +++ b/src/srtctl/core/ip_utils/get_node_ip.sh @@ -62,14 +62,7 @@ _resolve_ip() { fi fi - # Method 2: Use ip route to find default source IP - ip=$(ip route get 8.8.8.8 2>/dev/null | awk -F'src ' 'NR==1{split($2,a," ");print a[1]}') - if [ -n "$ip" ] && ! _is_bad_ip "$ip"; then - echo "$ip" - return 0 - fi - - # Method 3: Use hostname -I (prefer RFC1918, avoid loopback/link-local) + # Method 2: Use hostname -I (prefer RFC1918, avoid loopback/link-local) ips=$(hostname -I 2>/dev/null) if [ -n "$ips" ]; then ip=$(_select_best_ip $ips) @@ -79,6 +72,13 @@ _resolve_ip() { fi fi + # Method 3: Use ip route to find the default source IP + ip=$(ip route get 8.8.8.8 2>/dev/null | awk -F'src ' 'NR==1{split($2,a," ");print a[1]}') + if [ -n "$ip" ] && ! _is_bad_ip "$ip"; then + echo "$ip" + return 0 + fi + return 1 } @@ -156,14 +156,7 @@ get_node_ip() { fi fi - # Method 2: Use ip route to find default source IP - ip=\$(ip route get 8.8.8.8 2>/dev/null | awk -F'src ' 'NR==1{split(\$2,a,\" \");print a[1]}') - if [ -n \"\$ip\" ] && ! _is_bad_ip \"\$ip\"; then - echo \"\$ip\" - exit 0 - fi - - # Method 3: Use hostname -I (prefer RFC1918, avoid loopback/link-local) + # Method 2: Use hostname -I (prefer RFC1918, avoid loopback/link-local) ips=\$(hostname -I 2>/dev/null) if [ -n \"\$ips\" ]; then ip=\$(_select_best_ip \$ips) @@ -173,6 +166,13 @@ get_node_ip() { fi fi + # Method 3: Use ip route to find the default source IP + ip=\$(ip route get 8.8.8.8 2>/dev/null | awk -F'src ' 'NR==1{split(\$2,a,\" \");print a[1]}') + if [ -n \"\$ip\" ] && ! _is_bad_ip \"\$ip\"; then + echo \"\$ip\" + exit 0 + fi + exit 1 " diff --git a/tests/test_ip_utils.py b/tests/test_ip_utils.py index 888c6de30..b780eb29a 100644 --- a/tests/test_ip_utils.py +++ b/tests/test_ip_utils.py @@ -4,11 +4,14 @@ """Tests for IP address resolution helpers.""" import os +import subprocess from pathlib import Path from unittest.mock import patch from srtctl.core.ip_utils import get_node_ip +IP_SCRIPT = Path(__file__).resolve().parents[1] / "src/srtctl/core/ip_utils/get_node_ip.sh" + def test_get_node_ip_ignores_srun_step_created_output(tmp_path, monkeypatch): """get_node_ip() should ignore SLURM informational lines mixed into output.""" @@ -29,6 +32,32 @@ def test_get_node_ip_ignores_srun_step_created_output(tmp_path, monkeypatch): assert ip == "10.109.25.246" +def test_shell_resolver_prefers_private_hostname_ip_over_public_default_route(tmp_path): + """Automatic discovery should select the fabric IP when NIC names vary by node.""" + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + (fake_bin / "hostname").write_text( + "#!/bin/bash\n[ \"$1\" = -I ] && echo '203.0.113.8 10.20.30.40'\n", + encoding="ascii", + ) + (fake_bin / "ip").write_text( + "#!/bin/bash\necho '8.8.8.8 via 203.0.113.1 dev eth0 src 203.0.113.8'\n", + encoding="ascii", + ) + (fake_bin / "hostname").chmod(0o755) + (fake_bin / "ip").chmod(0o755) + + result = subprocess.run( + ["bash", "-c", 'source "$1"; _resolve_ip ""', "bash", str(IP_SCRIPT)], + check=True, + capture_output=True, + text=True, + env={**os.environ, "PATH": f"{fake_bin}:{os.environ['PATH']}"}, + ) + + assert result.stdout.strip() == "10.20.30.40" + + def test_runtime_resolves_control_plane_ips_on_cluster_interface(tmp_path): """Runtime endpoints must use the selected private/fabric interface.""" from srtctl.core.runtime import Nodes, RuntimeContext From 1d1ce612a1993d261fb16fd55af629f80e173237 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 9 Aug 2026 23:01:38 -0500 Subject: [PATCH 24/46] feat: orchestrate MoRI-IO vLLM workers --- docs/config-reference.md | 7 ++++ src/srtctl/backends/vllm.py | 48 +++++++++++++++++++++++-- src/srtctl/core/schema.py | 14 ++++++++ src/srtctl/frontends/static_router.py | 34 +++++++++++++----- src/srtctl/frontends/vllm_router.py | 36 ++++++++++++++++++- src/srtctl/ports.py | 4 +++ tests/test_configs.py | 52 +++++++++++++++++++++++++++ tests/test_static_router_frontends.py | 26 +++++++++++++- 8 files changed, 208 insertions(+), 13 deletions(-) diff --git a/docs/config-reference.md b/docs/config-reference.md index 7d9cd9f06..69115ec2d 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -333,6 +333,13 @@ hybrid-LB `vllm serve` process per node and require existing single-server behavior. No NATS or etcd infrastructure is started for this frontend. +For ROCm P/D deployments, `backend.connector: moriio` switches the same +frontend to vLLM Router's ZMQ discovery mode. srtctl supplies +`--kv-connector moriio`, owns the discovery port, and generates each direct +`vllm serve` worker's role-aware `MoRIIOConnector` JSON from the realized Slurm +node address and HTTP port. This mode requires one router on the head node, so +set `frontend.enable_multiple_frontends: false`. + ### trtllm_serve frontend `type: trtllm_serve` runs the `trtllm-serve disaggregated` orchestrator as the diff --git a/src/srtctl/backends/vllm.py b/src/srtctl/backends/vllm.py index cb2bad92e..fe0a5676e 100644 --- a/src/srtctl/backends/vllm.py +++ b/src/srtctl/backends/vllm.py @@ -31,6 +31,7 @@ MOONCAKE_HTTP_METADATA_PORT, MOONCAKE_MASTER_PORT, VLLM_DATA_PARALLEL_RPC_PORT, + VLLM_DISCOVERY_PORT, VLLM_PORT_BASE, VLLM_PORT_STRIDE, ) @@ -175,7 +176,8 @@ class VLLMProtocol: # Legacy compatibility alias. New recipes should use set_visible_devices. set_cuda_visible_devices: bool = False - # Default KV connector: "nixl", "lmcache", or a raw JSON string for --kv-transfer-config. + # Default KV connector: "nixl", "lmcache", "moriio", or a raw JSON + # string for --kv-transfer-config. # Can be overridden per mode by setting "connector" in vllm_config.prefill/decode/aggregated. # dynamo 1.0.0+: translated to --kv-transfer-config (--connector was removed). connector: str | None = "nixl" @@ -330,6 +332,16 @@ def get_environment_for_mode(self, mode: WorkerMode) -> dict[str, str]: return dict(self.aggregated_environment) return {} + def get_connector_for_mode(self, mode: WorkerMode) -> str | None: + """Return the effective connector after applying a mode override.""" + config = self.get_config_for_mode(mode) + mode_connector = config.get("connector") + return mode_connector if mode_connector is not None else self.connector + + def uses_moriio(self) -> bool: + """Whether this backend uses srtctl's managed MoRI-IO integration.""" + return isinstance(self.connector, str) and self.connector.lower() == "moriio" + def get_process_environment(self, process: Process) -> dict[str, str]: """Get process-specific environment variables for vLLM workers. @@ -348,7 +360,8 @@ def get_process_environment(self, process: Process) -> dict[str, str]: env: dict[str, str] = {} if process.kv_events_port is not None: env["DYN_VLLM_KV_EVENT_PORT"] = str(process.kv_events_port) - if process.nixl_port is not None: + connector = self.get_connector_for_mode(process.endpoint_mode) + if process.nixl_port is not None and not (isinstance(connector, str) and connector.lower() == "moriio"): env["VLLM_NIXL_SIDE_CHANNEL_PORT"] = str(process.nixl_port) env["VLLM_NIXL_SIDE_CHANNEL_HOST"] = get_hostname_ip(process.node) # Unique per-process VLLM_PORT base to avoid EADDRINUSE rendezvous races @@ -753,7 +766,10 @@ def build_worker_command( mode_connector = config.pop("connector", None) connector = mode_connector if mode_connector is not None else self.connector if connector and connector not in ("null", "none", None): - config.setdefault("kv-transfer-config", _connector_to_kv_transfer_config(connector)) + config.setdefault( + "kv-transfer-config", + _build_direct_kv_transfer_config(connector, mode, process, runtime), + ) if is_router_hybrid_dp: rpc_port_kebab = config.pop("data-parallel-rpc-port", None) @@ -940,6 +956,32 @@ def _connector_to_kv_transfer_config(connector: str) -> str: return connector +def _build_direct_kv_transfer_config( + connector: str, + mode: WorkerMode, + process: Process, + runtime: RuntimeContext, +) -> str: + """Build connector JSON that depends on the realized Slurm topology.""" + if connector.lower() != "moriio": + return _connector_to_kv_transfer_config(connector) + if mode not in {"prefill", "decode"}: + raise ValueError("MoRI-IO requires a prefill/decode topology") + + return json.dumps( + { + "kv_connector": "MoRIIOConnector", + "kv_role": "kv_producer" if mode == "prefill" else "kv_consumer", + "kv_connector_extra_config": { + "proxy_ip": runtime.head_node_ip, + "proxy_ping_port": str(VLLM_DISCOVERY_PORT), + "http_port": str(process.http_port), + "read_mode": True, + }, + } + ) + + def _config_to_cli_args(config: dict[str, Any]) -> list[str]: """Convert config dict to CLI arguments.""" args: list[str] = [] diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py index 765c6d726..1dbf62ea0 100755 --- a/src/srtctl/core/schema.py +++ b/src/srtctl/core/schema.py @@ -1676,6 +1676,20 @@ def _validate_static_router_frontend(self): ) if self.frontend.type == "vllm-router": + connector = getattr(self.backend, "connector", None) + if isinstance(connector, str) and connector.lower() == "moriio": + if self.frontend.enable_multiple_frontends: + raise ValidationError( + "vLLM Router MoRI-IO discovery uses one registration endpoint; " + "set frontend.enable_multiple_frontends: false" + ) + if self.frontend.orchestrator_placement != "head": + raise ValidationError( + "vLLM Router MoRI-IO discovery requires frontend.orchestrator_placement: head" + ) + if self.resources.num_agg: + raise ValidationError("vLLM Router MoRI-IO requires a prefill/decode topology") + endpoint_gpu_counts = { "prefill": self.resources.gpus_per_prefill if self.resources.num_prefill else 0, "decode": self.resources.gpus_per_decode if self.resources.num_decode else 0, diff --git a/src/srtctl/frontends/static_router.py b/src/srtctl/frontends/static_router.py index c2568f51a..4e5b478f0 100644 --- a/src/srtctl/frontends/static_router.py +++ b/src/srtctl/frontends/static_router.py @@ -81,6 +81,10 @@ def get_managed_frontend_args( del backend, backend_processes return [] + def uses_dynamic_worker_discovery(self, backend: Any) -> bool: + """Whether workers register with the router instead of using static URLs.""" + return False + def worker_scheme(self, backend: Any, mode: str) -> str: """Return the protocol used to reach a worker endpoint.""" return "http" @@ -115,7 +119,14 @@ def collect_workers(self, backend: Any, backend_processes: list[Process]) -> lis ) return workers - def build_router_command(self, workers: list[RouterWorker], host: str, port: int) -> list[str]: + def build_router_command( + self, + workers: list[RouterWorker], + host: str, + port: int, + *, + dynamic_discovery: bool = False, + ) -> list[str]: """Build the router CLI for aggregate or prefill/decode topologies.""" aggregate = [worker for worker in workers if worker.mode == "agg"] prefills = [worker for worker in workers if worker.mode == "prefill"] @@ -128,12 +139,13 @@ def build_router_command(self, workers: list[RouterWorker], host: str, port: int if not prefills or not decodes: raise ValueError("Disaggregated static router topology requires prefill and decode workers") cmd.append(self.pd_flag) - for worker in prefills: - cmd.extend(["--prefill", worker.url]) - if worker.bootstrap_port is not None: - cmd.append(str(worker.bootstrap_port)) - for worker in decodes: - cmd.extend(["--decode", worker.url]) + if not dynamic_discovery: + for worker in prefills: + cmd.extend(["--prefill", worker.url]) + if worker.bootstrap_port is not None: + cmd.append(str(worker.bootstrap_port)) + for worker in decodes: + cmd.extend(["--decode", worker.url]) else: if not aggregate: raise ValueError("Static router topology has no logical workers") @@ -161,10 +173,16 @@ def start_frontends( ) workers = self.collect_workers(backend, backend_processes) + dynamic_discovery = self.uses_dynamic_worker_discovery(backend) processes: list[ManagedProcess] = [] for idx, node in enumerate(topology.frontend_nodes): router_log = runtime.log_dir / f"{node}_{self.type}_{idx}.out" - cmd = self.build_router_command(workers, "0.0.0.0", topology.frontend_port) + cmd = self.build_router_command( + workers, + "0.0.0.0", + topology.frontend_port, + dynamic_discovery=dynamic_discovery, + ) cmd.extend(self.get_managed_frontend_args(config, backend, backend_processes)) cmd.extend(self.get_frontend_args_list(config.frontend.args)) logger.info("Starting %s %d on %s: %s", self.type, idx, node, shlex.join(cmd)) diff --git a/src/srtctl/frontends/vllm_router.py b/src/srtctl/frontends/vllm_router.py index 8a8b1b99b..38d4746cc 100644 --- a/src/srtctl/frontends/vllm_router.py +++ b/src/srtctl/frontends/vllm_router.py @@ -9,6 +9,7 @@ from srtctl.frontends.base import register_frontend from srtctl.frontends.static_router import StaticRouterFrontend +from srtctl.ports import VLLM_DISCOVERY_PORT if TYPE_CHECKING: from srtctl.core.topology import Process @@ -76,6 +77,39 @@ def get_managed_frontend_args( managed_args.extend(["--worker-startup-timeout-secs", str(timeout_seconds)]) return managed_args + def uses_dynamic_worker_discovery(self, backend: Any) -> bool: + """Use ZMQ registration for connector-managed P/D workers.""" + return backend.uses_moriio() + + def build_router_command( + self, + workers: list[Any], + host: str, + port: int, + *, + dynamic_discovery: bool = False, + ) -> list[str]: + """Add vLLM Router's MoRI-IO discovery contract when requested.""" + command = super().build_router_command( + workers, + host, + port, + dynamic_discovery=dynamic_discovery, + ) + if not dynamic_discovery: + return command + + insertion = command.index("--host") + command[insertion:insertion] = [ + "--kv-connector", + "moriio", + "--vllm-discovery-address", + f"0.0.0.0:{VLLM_DISCOVERY_PORT}", + ] + return command + def worker_bootstrap_port(self, backend: Any, process: Process) -> int | None: - """Advertise vLLM's NIXL side-channel port to the P/D router.""" + """Advertise vLLM's NIXL side-channel port for static P/D routing.""" + if backend.uses_moriio(): + return None return process.nixl_port diff --git a/src/srtctl/ports.py b/src/srtctl/ports.py index 93ac44013..304cf401f 100644 --- a/src/srtctl/ports.py +++ b/src/srtctl/ports.py @@ -33,6 +33,10 @@ # vLLM backend ports. VLLM_NIXL_PORT_BASE = 5400 +# ZMQ registration endpoint used by discovery-based vLLM P/D connectors such +# as MoRI-IO. Workers register their HTTP and transfer addresses with the +# router at this port. +VLLM_DISCOVERY_PORT = 36367 VLLM_DATA_PARALLEL_RPC_PORT = 8400 VLLM_PORT_BASE = 20000 VLLM_PORT_STRIDE = 50 diff --git a/tests/test_configs.py b/tests/test_configs.py index c9ef820ce..ed4530d90 100644 --- a/tests/test_configs.py +++ b/tests/test_configs.py @@ -2396,6 +2396,58 @@ def test_vllm_router_stable_release_uses_legacy_cuda_binding(self): assert "--device-ids" not in cmd assert backend.should_set_cuda_visible_devices(process) + @pytest.mark.parametrize( + ("mode", "role"), + [("prefill", "kv_producer"), ("decode", "kv_consumer")], + ) + def test_vllm_router_moriio_worker_uses_realized_slurm_topology(self, mode, role): + """MoRI workers self-register their realized private HTTP endpoint.""" + from pathlib import Path + from types import SimpleNamespace + + from srtctl.backends import VLLMProtocol, VLLMServerConfig + from srtctl.core.topology import Process + + backend = VLLMProtocol( + connector="moriio", + vllm_config=VLLMServerConfig(**{mode: {"tensor-parallel-size": 1}}), + ) + process = Process( + node=f"{mode}-node", + gpu_indices=frozenset({0}), + sys_port=8081, + http_port=6100, + endpoint_mode=mode, + endpoint_index=0, + nixl_port=5400, + ) + runtime = SimpleNamespace( + model_path=Path("Qwen/Qwen3-0.6B"), + is_hf_model=True, + frontend_port=8000, + head_node_ip="10.20.30.40", + ) + + cmd = backend.build_worker_command( + process=process, + endpoint_processes=[process], + runtime=runtime, + frontend_type="vllm-router", + ) + + kv_config = json.loads(cmd[cmd.index("--kv-transfer-config") + 1]) + assert kv_config == { + "kv_connector": "MoRIIOConnector", + "kv_role": role, + "kv_connector_extra_config": { + "proxy_ip": "10.20.30.40", + "proxy_ping_port": "36367", + "http_port": "6100", + "read_mode": True, + }, + } + assert "VLLM_NIXL_SIDE_CHANNEL_PORT" not in backend.get_process_environment(process) + def test_direct_vllm_command_keeps_iteration_profiler_config(self): """Direct vllm serve retains main's profiling-derived server option.""" from pathlib import Path diff --git a/tests/test_static_router_frontends.py b/tests/test_static_router_frontends.py index 27591ec24..e3c8fd859 100644 --- a/tests/test_static_router_frontends.py +++ b/tests/test_static_router_frontends.py @@ -97,7 +97,7 @@ def test_vllm_router_advertises_nixl_side_channel_port() -> None: ) with patch.object(frontend, "get_hostname_ip", return_value="10.0.0.1"): - workers = frontend.collect_workers(MagicMock(), [process]) + workers = frontend.collect_workers(SimpleNamespace(uses_moriio=lambda: False), [process]) assert workers == [RouterWorker("prefill", "http://10.0.0.1:30000", 13000)] @@ -158,6 +158,30 @@ def test_vllm_router_derives_dep4_expansion_for_1p2d() -> None: ] +def test_vllm_router_moriio_uses_discovery_instead_of_static_workers() -> None: + frontend = VLLMRouterFrontend() + command = frontend.build_router_command( + [ + RouterWorker("prefill", "http://10.0.0.1:30000"), + RouterWorker("decode", "http://10.0.0.2:30000"), + ], + "0.0.0.0", + 8000, + dynamic_discovery=True, + ) + + assert command[:5] == [ + "vllm-router", + "--vllm-pd-disaggregation", + "--kv-connector", + "moriio", + "--vllm-discovery-address", + ] + assert command[5] == "0.0.0.0:36367" + assert "--prefill" not in command + assert "--decode" not in command + + def test_vllm_router_launch_uses_router_container_env_and_only_leaders() -> None: frontend = VLLMRouterFrontend() runtime = SimpleNamespace( From 10d2bc17ecb24a775686a52373abc4bef2fd818b Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 9 Aug 2026 23:17:50 -0500 Subject: [PATCH 25/46] fix: embed cluster profile for node-local jobs --- docs/config-reference.md | 2 +- src/srtctl/cli/submit.py | 12 ++++++++++++ src/srtctl/templates/job_script_minimal.j2 | 6 ++++++ tests/test_accelerator.py | 12 +++++++++++- 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/docs/config-reference.md b/docs/config-reference.md index 69115ec2d..5ca3462ec 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -124,7 +124,7 @@ The `srtslurm.yaml` file can contain the following fields: **output_dir**: When set, job logs are written to `output_dir/{job_id}/logs` instead of `srtctl_root/outputs/{job_id}/logs`. Useful for CI/CD and ephemeral environments. -**runtime_config_transport**: Leave this as `shared-filesystem` when the submitter and compute nodes see the same output directory. Use `embedded` when the output path is node-local: srtctl safely embeds the exact resolved YAML in the Slurm script, materializes it with owner-only permissions on the allocated head node, and bootstraps the Slurm log from the existing output base into the normal per-job log directory. Embedded payloads are data-safe but not a secrets store: users who can inspect Slurm batch scripts can decode them. +**runtime_config_transport**: Leave this as `shared-filesystem` when the submitter and compute nodes see the same output directory. Use `embedded` when the output path is node-local: srtctl safely embeds the exact resolved recipe and active `srtslurm.yaml` in the Slurm script, materializes both with owner-only permissions on the allocated head node, points compute-side cluster lookups at that embedded profile, and bootstraps the Slurm log from the existing output base into the normal per-job log directory. This preserves container aliases, mounts, accelerator settings, and network selection even when the login and compute nodes do not share the original cluster-profile path. Embedded payloads are data-safe but not a secrets store: users who can inspect Slurm batch scripts can decode them. **default_bash_preamble**: A shell snippet (e.g. `"ulimit -n 1048576 -s unlimited -u 1048576"`) prepended to every container srun launched by srtctl — workers, frontends, telemetry, benchmark, postprocess. Runs before per-call `bash_preamble` and the main command, so cluster-wide ulimits apply to everything downstream. Silently dropped for distroless containers (e.g. `prom/node-exporter`) that bypass the bash wrapper; a WARNING log is emitted in that case. diff --git a/src/srtctl/cli/submit.py b/src/srtctl/cli/submit.py index cd5909a8e..a8b916474 100755 --- a/src/srtctl/cli/submit.py +++ b/src/srtctl/cli/submit.py @@ -37,6 +37,7 @@ from rich.table import Table from srtctl.core.config import ( + find_cluster_config_path, generate_override_configs, get_srtslurm_setting, load_cluster_config, @@ -485,6 +486,7 @@ def generate_minimal_sbatch_script( runtime_config_transport = get_srtslurm_setting("runtime_config_transport", "shared-filesystem") embedded_config_files: list[dict[str, str]] = [] + embedded_cluster_config_filename: str | None = None if runtime_config_transport == "embedded": if runtime_config_text is None: runtime_config_text = config_path.read_text() @@ -501,6 +503,15 @@ def generate_minimal_sbatch_script( "payload": base64.b64encode(runtime_config_text.encode()).decode("ascii"), } ) + cluster_config_path = find_cluster_config_path() + if cluster_config_path is not None: + embedded_cluster_config_filename = "srtslurm.yaml" + embedded_config_files.append( + { + "filename": embedded_cluster_config_filename, + "payload": base64.b64encode(cluster_config_path.read_bytes()).decode("ascii"), + } + ) elif runtime_config_transport != "shared-filesystem": raise ValueError(f"Unsupported runtime config transport: {runtime_config_transport}") @@ -516,6 +527,7 @@ def generate_minimal_sbatch_script( config_path=str(config_path.resolve()), runtime_config_filename=runtime_config_filename, embedded_config_files=embedded_config_files, + embedded_cluster_config_filename=embedded_cluster_config_filename, timestamp=timestamp, gpu_sbatch_directive=gpu_sbatch_directive, use_segment_sbatch_directive=get_srtslurm_setting("use_segment_sbatch_directive", True), diff --git a/src/srtctl/templates/job_script_minimal.j2 b/src/srtctl/templates/job_script_minimal.j2 index 97005e18c..a5f55bfe2 100644 --- a/src/srtctl/templates/job_script_minimal.j2 +++ b/src/srtctl/templates/job_script_minimal.j2 @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 #SBATCH --job-name={{ job_name }} +#SBATCH --chdir={{ srtctl_source }} {% if embedded_config_files %} # Node-local output bases cannot have their per-job directory created by the # submitter. SLURM opens stdout before the batch script runs, so open it in the @@ -119,6 +120,11 @@ RUNTIME_CONFIG_TMP="${RUNTIME_CONFIG}.tmp.$$" (umask 077; printf '%s' '{{ config_file.payload }}' | base64 --decode > "${RUNTIME_CONFIG_TMP}") mv "${RUNTIME_CONFIG_TMP}" "${RUNTIME_CONFIG}" {% endfor %} +{% if embedded_cluster_config_filename %} +# The submitter and compute nodes may not share the cluster-profile path. +# Point every compute-side config lookup at the exact embedded profile. +export SRTSLURM_CONFIG="${OUTPUT_DIR}/{{ embedded_cluster_config_filename }}" +{% endif %} {% endif %} # Export for Python orchestrator to use the same paths diff --git a/tests/test_accelerator.py b/tests/test_accelerator.py index cb62ade45..23f97299d 100644 --- a/tests/test_accelerator.py +++ b/tests/test_accelerator.py @@ -165,6 +165,10 @@ def test_embedded_transport_preserves_resolved_yaml_as_inert_data(monkeypatch, t settings = {"runtime_config_transport": "embedded"} monkeypatch.setattr(submit, "get_srtslurm_setting", lambda key, default=None: settings.get(key, default)) + cluster_config_path = tmp_path / "srtslurm-source.yaml" + cluster_config_text = "cluster: mi300x-amds\ncontainers:\n rocm: /images/rocm.sqsh\n" + cluster_config_path.write_text(cluster_config_text) + monkeypatch.setattr(submit, "find_cluster_config_path", lambda: cluster_config_path) config = SrtConfig( name="node-local-output-test", model=ModelConfig(path="/model", container="/container.sqsh", precision="fp16"), @@ -187,6 +191,8 @@ def test_embedded_transport_preserves_resolved_yaml_as_inert_data(monkeypatch, t assert "/.srtctl-sweep-%j.log" in script assert 'mv "${BOOTSTRAP_LOG}" "${LOG_DIR}/sweep_${SLURM_JOB_ID}.log"' in script assert '--ntasks-per-node=1 mkdir -p "${LOG_DIR}"' in script + assert '#SBATCH --chdir=' in script + assert 'export SRTSLURM_CONFIG="${OUTPUT_DIR}/srtslurm.yaml"' in script assert "touch /tmp/must-not-run" not in script embedded = re.findall( @@ -195,4 +201,8 @@ def test_embedded_transport_preserves_resolved_yaml_as_inert_data(monkeypatch, t flags=re.DOTALL, ) decoded = {filename: base64.b64decode(payload).decode() for filename, payload in embedded} - assert decoded == {"config.yaml": source_text, "config_variant.yaml": runtime_text} + assert decoded == { + "config.yaml": source_text, + "config_variant.yaml": runtime_text, + "srtslurm.yaml": cluster_config_text, + } From 6e3e9b9e44ffadb7280ed5ea0192ed08a5eb7da2 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 9 Aug 2026 23:45:31 -0500 Subject: [PATCH 26/46] fix(router): probe dynamic MoRI readiness end to end --- docs/config-reference.md | 6 ++++ src/srtctl/cli/mixins/benchmark_stage.py | 5 +++ src/srtctl/core/health.py | 33 +++++++++++++++-- tests/test_health.py | 45 ++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 2 deletions(-) diff --git a/docs/config-reference.md b/docs/config-reference.md index 5ca3462ec..4d365416e 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -340,6 +340,12 @@ frontend to vLLM Router's ZMQ discovery mode. srtctl supplies node address and HTTP port. This mode requires one router on the head node, so set `frontend.enable_multiple_frontends: false`. +Router's `/workers` response currently covers its static worker registry, not +the MoRI ZMQ discovery registry. For dynamic MoRI discovery, srtctl therefore +waits on a one-token `/v1/completions` probe instead. This validates that both +roles have registered and that the complete Router-to-prefill-to-decode path is +usable before the configured benchmark begins. + ### trtllm_serve frontend `type: trtllm_serve` runs the `trtllm-serve disaggregated` orchestrator as the diff --git a/src/srtctl/cli/mixins/benchmark_stage.py b/src/srtctl/cli/mixins/benchmark_stage.py index 884d9e529..61cddc3b4 100644 --- a/src/srtctl/cli/mixins/benchmark_stage.py +++ b/src/srtctl/cli/mixins/benchmark_stage.py @@ -211,6 +211,9 @@ def run_benchmark( logger.info("Waiting for server health (expecting %d health entries: %s)...", num_workers, count_desc) hc = self.config.health_check + uses_dynamic_worker_discovery = bool( + getattr(self.frontend, "uses_dynamic_worker_discovery", lambda _backend: False)(self.backend) + ) if not wait_for_model( host=self._orchestrator_node(), port=FRONTEND_PUBLIC_PORT, @@ -220,6 +223,8 @@ def run_benchmark( timeout=float(hc.max_attempts * hc.interval_seconds), report_every=60.0, frontend_type=self.config.frontend.type, + model_name=self.config.served_model_name, + dynamic_worker_discovery=uses_dynamic_worker_discovery, stop_event=stop_event, ): logger.error("Server did not become healthy") diff --git a/src/srtctl/core/health.py b/src/srtctl/core/health.py index d379b59b2..87bfadc50 100644 --- a/src/srtctl/core/health.py +++ b/src/srtctl/core/health.py @@ -407,6 +407,8 @@ def wait_for_model( timeout: float = 600.0, report_every: float = 60.0, frontend_type: str = "dynamo", + model_name: str | None = None, + dynamic_worker_discovery: bool = False, stop_event: threading.Event | None = None, ) -> bool: """Wait for model to be ready with expected worker counts. @@ -432,8 +434,20 @@ def wait_for_model( from srtctl.frontends import get_frontend frontend = get_frontend(frontend_type) + use_vllm_router_generation_probe = frontend_type == "vllm-router" and dynamic_worker_discovery + if use_vllm_router_generation_probe and not model_name: + raise ValueError("model_name is required for vLLM Router dynamic-discovery health checks") + health_url = f"http://{host}:{port}{frontend.health_endpoint}" - if frontend.health_endpoint == "/workers": + if use_vllm_router_generation_probe: + health_url = f"http://{host}:{port}/v1/completions" + logger.info( + "Polling %s every %.1fs with an end-to-end generation probe (%s frontend)", + health_url, + poll_interval, + frontend_type, + ) + elif frontend.health_endpoint == "/workers": logger.info( "Polling %s every %.1fs for %d prefills and %d decodes (%s frontend)", health_url, @@ -469,7 +483,22 @@ def wait_for_model( # Try to fetch health try: - response = requests.get(health_url, timeout=5.0) + if use_vllm_router_generation_probe: + response = requests.post( + health_url, + json={ + "model": model_name, + "prompt": "srtctl readiness probe", + "max_tokens": 1, + "temperature": 0, + }, + timeout=30.0, + ) + if response.status_code == 200: + logger.info("vLLM Router dynamic-discovery generation probe succeeded") + return True + else: + response = requests.get(health_url, timeout=5.0) if response.status_code == 200: # trtllm-serve /health may return an empty body; a 200 is sufficient # (workers were gated by the frontend before the orchestrator started). diff --git a/tests/test_health.py b/tests/test_health.py index b894104ef..1d5dc4982 100644 --- a/tests/test_health.py +++ b/tests/test_health.py @@ -3,10 +3,15 @@ """Tests for health check parsing (Dynamo and SGLang router).""" +from unittest.mock import Mock + +import pytest + from srtctl.core.health import ( WorkerHealthResult, check_dynamo_health, check_sglang_router_health, + wait_for_model, ) # ============================================================================ @@ -427,3 +432,43 @@ def test_with_counts(self): assert result.prefill_ready == 2 assert result.decode_ready == 4 + + +def test_vllm_router_dynamic_discovery_health_uses_generation_probe(monkeypatch): + """MoRI discovery readiness must validate the live P/D request path.""" + response = Mock(status_code=200) + post = Mock(return_value=response) + get = Mock() + monkeypatch.setattr("srtctl.core.health.requests.post", post) + monkeypatch.setattr("srtctl.core.health.requests.get", get) + + assert wait_for_model( + "router-host", + 8000, + frontend_type="vllm-router", + model_name="Qwen/Qwen3-0.6B", + dynamic_worker_discovery=True, + timeout=1, + ) + get.assert_not_called() + post.assert_called_once_with( + "http://router-host:8000/v1/completions", + json={ + "model": "Qwen/Qwen3-0.6B", + "prompt": "srtctl readiness probe", + "max_tokens": 1, + "temperature": 0, + }, + timeout=30.0, + ) + + +def test_vllm_router_dynamic_discovery_health_requires_model(): + with pytest.raises(ValueError, match="model_name is required"): + wait_for_model( + "router-host", + 8000, + frontend_type="vllm-router", + dynamic_worker_discovery=True, + timeout=1, + ) From b0c60c34f95b656451fc61dae1843775fd509f76 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 9 Aug 2026 23:53:03 -0500 Subject: [PATCH 27/46] fix(runtime): preserve default and mock compatibility --- src/srtctl/cli/mixins/benchmark_stage.py | 4 +++- src/srtctl/cli/submit.py | 2 +- src/srtctl/mock.py | 9 ++++++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/srtctl/cli/mixins/benchmark_stage.py b/src/srtctl/cli/mixins/benchmark_stage.py index 61cddc3b4..4968e8f05 100644 --- a/src/srtctl/cli/mixins/benchmark_stage.py +++ b/src/srtctl/cli/mixins/benchmark_stage.py @@ -19,6 +19,7 @@ from srtctl.core.lockfile import collect_worker_fingerprints from srtctl.core.slurm import get_hostname_ip, start_srun_process from srtctl.core.status import JobStage, JobStatus, StatusReporter +from srtctl.frontends import get_frontend from srtctl.ports import FRONTEND_PUBLIC_PORT, SGLANG_HTTP_PORT_BASE if TYPE_CHECKING: @@ -211,8 +212,9 @@ def run_benchmark( logger.info("Waiting for server health (expecting %d health entries: %s)...", num_workers, count_desc) hc = self.config.health_check + frontend = get_frontend(self.config.frontend.type) uses_dynamic_worker_discovery = bool( - getattr(self.frontend, "uses_dynamic_worker_discovery", lambda _backend: False)(self.backend) + getattr(frontend, "uses_dynamic_worker_discovery", lambda _backend: False)(self.backend) ) if not wait_for_model( host=self._orchestrator_node(), diff --git a/src/srtctl/cli/submit.py b/src/srtctl/cli/submit.py index a8b916474..48f0c6b5e 100755 --- a/src/srtctl/cli/submit.py +++ b/src/srtctl/cli/submit.py @@ -484,7 +484,7 @@ def generate_minimal_sbatch_script( # Backward compatibility for existing cluster configurations. gpu_sbatch_directive = "gpus-per-node" if get_srtslurm_setting("use_gpus_per_node_directive", True) else "none" - runtime_config_transport = get_srtslurm_setting("runtime_config_transport", "shared-filesystem") + runtime_config_transport = get_srtslurm_setting("runtime_config_transport", "shared-filesystem") or "shared-filesystem" embedded_config_files: list[dict[str, str]] = [] embedded_cluster_config_filename: str | None = None if runtime_config_transport == "embedded": diff --git a/src/srtctl/mock.py b/src/srtctl/mock.py index a3ae32f5f..bab72a214 100644 --- a/src/srtctl/mock.py +++ b/src/srtctl/mock.py @@ -143,6 +143,8 @@ class MockOptions: nodelist: tuple[str, ...] = ("mock-node-01",) # Fake IP returned for every hostname lookup. hostname_ip: str = "127.0.0.1" + # Deterministic per-node CPU allocation exposed through the mock Slurm env. + cpus_per_node: int = 64 # --------------------------------------------------------------------------- @@ -352,18 +354,19 @@ def _write_final_result(output_dir: Path, *, job_id: str, exit_code: int) -> Non (output_dir / "result.json").write_text(json.dumps(payload, indent=2) + "\n") -def _set_slurm_env(job_id: str, nodelist: Iterable[str]) -> dict[str, str | None]: +def _set_slurm_env(job_id: str, nodelist: Iterable[str], cpus_per_node: int) -> dict[str, str | None]: """Set SLURM_* env vars for the duration of the mock run. Returns a snapshot so callers can restore the prior values. """ - keys = ["SLURM_JOB_ID", "SLURM_JOBID", "SLURM_NODELIST", "SLURM_NNODES"] + keys = ["SLURM_JOB_ID", "SLURM_JOBID", "SLURM_NODELIST", "SLURM_NNODES", "SLURM_JOB_CPUS_PER_NODE"] prior = {k: os.environ.get(k) for k in keys} os.environ["SLURM_JOB_ID"] = job_id os.environ["SLURM_JOBID"] = job_id nodes_csv = ",".join(nodelist) os.environ["SLURM_NODELIST"] = nodes_csv os.environ["SLURM_NNODES"] = str(len(nodes_csv.split(","))) + os.environ["SLURM_JOB_CPUS_PER_NODE"] = f"{cpus_per_node}(x{len(nodes_csv.split(','))})" return prior @@ -413,7 +416,7 @@ def run_mock_sweep( # the real output_dir so artifacts land where the upstream harness expects. prior_output = os.environ.get("SRTCTL_OUTPUT_DIR") os.environ["SRTCTL_OUTPUT_DIR"] = str(output_dir) - prior_slurm = _set_slurm_env(job_id, opts.nodelist) + prior_slurm = _set_slurm_env(job_id, opts.nodelist, opts.cpus_per_node) try: with mock_infrastructure(options=opts, output_dir=output_dir) as sink: From cfa04c3f50b4c86b7fde63ad3cf5864c68897b0b Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 9 Aug 2026 23:54:31 -0500 Subject: [PATCH 28/46] style: format transport fallback --- src/srtctl/cli/submit.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/srtctl/cli/submit.py b/src/srtctl/cli/submit.py index 48f0c6b5e..c774a5769 100755 --- a/src/srtctl/cli/submit.py +++ b/src/srtctl/cli/submit.py @@ -484,7 +484,9 @@ def generate_minimal_sbatch_script( # Backward compatibility for existing cluster configurations. gpu_sbatch_directive = "gpus-per-node" if get_srtslurm_setting("use_gpus_per_node_directive", True) else "none" - runtime_config_transport = get_srtslurm_setting("runtime_config_transport", "shared-filesystem") or "shared-filesystem" + runtime_config_transport = ( + get_srtslurm_setting("runtime_config_transport", "shared-filesystem") or "shared-filesystem" + ) embedded_config_files: list[dict[str, str]] = [] embedded_cluster_config_filename: str | None = None if runtime_config_transport == "embedded": From 24de885672cdaf437f8546d12779b08f4431ebda Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sun, 9 Aug 2026 23:59:04 -0500 Subject: [PATCH 29/46] fix(setup): skip unused infra for native routers --- src/srtctl/cli/submit.py | 22 +++++++++++++++------- tests/test_validate_setup.py | 7 +++++++ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/srtctl/cli/submit.py b/src/srtctl/cli/submit.py index c774a5769..4b2d067b1 100755 --- a/src/srtctl/cli/submit.py +++ b/src/srtctl/cli/submit.py @@ -370,7 +370,7 @@ def show_config_details(config: SrtConfig) -> None: console.print(Panel(details, border_style="blue")) -def validate_setup(srtctl_source: Path) -> None: +def validate_setup(srtctl_source: Path, *, requires_head_infrastructure: bool = True) -> None: """Validate that make setup has been run and required binaries exist. Checks for NATS, etcd, and compute-arch uv binaries. Raises SystemExit @@ -378,11 +378,12 @@ def validate_setup(srtctl_source: Path) -> None: """ missing = [] - configs_dir = srtctl_source / "configs" - if not (configs_dir / "nats-server").exists(): - missing.append("configs/nats-server") - if not (configs_dir / "etcd").exists(): - missing.append("configs/etcd") + if requires_head_infrastructure: + configs_dir = srtctl_source / "configs" + if not (configs_dir / "nats-server").exists(): + missing.append("configs/nats-server") + if not (configs_dir / "etcd").exists(): + missing.append("configs/etcd") if not (srtctl_source / "bin" / "uv").exists(): missing.append("bin/uv (compute-arch uv)") @@ -684,7 +685,14 @@ def submit_with_orchestrator( # Validate setup before submitting (not during dry-run) srtctl_root = get_srtslurm_setting("srtctl_root") srtctl_source = Path(srtctl_root) if srtctl_root else Path(__file__).parent.parent.parent.parent - validate_setup(srtctl_source) + requires_head_infrastructure = config.frontend.type not in { + "sglang", + "sgl-router", + "trtllm_serve", + "vllm", + "vllm-router", + } + validate_setup(srtctl_source, requires_head_infrastructure=requires_head_infrastructure) # Write script to temp file fd, script_path = tempfile.mkstemp(suffix=".slurm", prefix="srtctl_", text=True) diff --git a/tests/test_validate_setup.py b/tests/test_validate_setup.py index 14562bb7a..8b1474736 100644 --- a/tests/test_validate_setup.py +++ b/tests/test_validate_setup.py @@ -59,6 +59,13 @@ def test_fails_when_all_missing(self, tmp_path: Path): with pytest.raises(SystemExit): validate_setup(tmp_path) + def test_static_frontend_setup_only_requires_compute_uv(self, tmp_path: Path): + """Native routers must not require unused Dynamo infrastructure binaries.""" + (tmp_path / "bin").mkdir() + (tmp_path / "bin" / "uv").touch() + + validate_setup(tmp_path, requires_head_infrastructure=False) + class TestMakefileArchDetection: """Test that the file | grep pattern used in Makefile matches correctly. From 0e18f98b06d716e483086a33c68216d053aff758 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 10 Aug 2026 00:32:52 -0500 Subject: [PATCH 30/46] build: add compute-only setup target --- Makefile | 34 +++++++++++++--------- docs/installation.md | 11 +++++++ src/srtctl/cli/submit.py | 7 +++-- src/srtctl/templates/job_script_minimal.j2 | 6 ++-- tests/test_validate_setup.py | 7 ++++- 5 files changed, 45 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 9b6c52982..5e55ed6ac 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: lint test test-cov ci check setup cleanup gb200-fp8 gb200-fp4 +.PHONY: lint test test-cov ci check setup setup-compute cleanup gb200-fp8 gb200-fp4 NATS_VERSION ?= v2.10.28 ETCD_VERSION ?= v3.5.21 @@ -40,6 +40,24 @@ gb200-fp4: srtctl apply -f recipes/gb200-fp4/8k1k/max-tpt.yaml srtctl apply -f recipes/gb200-fp4/8k1k/mid-curve.yaml +setup-compute: + @case "$(ARCH)" in \ + x86_64) ARCH_FILE_PATTERN="x86-64" ;; \ + aarch64) ARCH_FILE_PATTERN="aarch64" ;; \ + *) echo "❌ Unsupported architecture: $(ARCH)"; exit 1 ;; \ + esac; \ + echo "--- uv (compute node arch: $(ARCH)) ---"; \ + if [ -f bin/uv ] && file bin/uv | grep -q "$$ARCH_FILE_PATTERN"; then \ + echo "✅ uv already installed at bin/uv ($(ARCH))"; \ + else \ + echo "⬇️ Downloading uv for $(ARCH)..."; \ + mkdir -p bin; \ + UV_URL="https://github.com/astral-sh/uv/releases/latest/download/uv-$(ARCH)-unknown-linux-gnu.tar.gz"; \ + curl -LsSf "$$UV_URL" | tar -xz --strip-components=1 -C bin; \ + chmod +x bin/uv bin/uvx 2>/dev/null; \ + echo "✅ uv installed to bin/uv ($$(file bin/uv | grep -o 'ARM aarch64\|x86-64'))"; \ + fi + setup: @echo "📦 Setting up configs and logs directories..." @mkdir -p logs @@ -90,17 +108,7 @@ setup: echo "✅ ETCD installed to configs/etcd"; \ fi; \ echo ""; \ - echo "--- uv (compute node arch: $(ARCH)) ---"; \ - if [ -f bin/uv ] && file bin/uv | grep -q "$$ARCH_FILE_PATTERN"; then \ - echo "✅ uv already installed at bin/uv ($(ARCH))"; \ - else \ - echo "⬇️ Downloading uv for $(ARCH)..."; \ - mkdir -p bin; \ - UV_URL="https://github.com/astral-sh/uv/releases/latest/download/uv-$(ARCH)-unknown-linux-gnu.tar.gz"; \ - curl -LsSf "$$UV_URL" | tar -xz --strip-components=1 -C bin; \ - chmod +x bin/uv bin/uvx 2>/dev/null; \ - echo "✅ uv installed to bin/uv ($$(file bin/uv | grep -o 'ARM aarch64\|x86-64'))"; \ - fi; \ + $(MAKE) --no-print-directory setup-compute ARCH=$(ARCH); \ echo ""; \ echo "--- srtslurm.yaml ---"; \ if [ -f srtslurm.yaml ]; then \ @@ -175,4 +183,4 @@ cleanup: echo "✅ Cleanup complete!"; \ else \ echo "❌ Cleanup cancelled."; \ - fi \ No newline at end of file + fi diff --git a/docs/installation.md b/docs/installation.md index a5abd93bb..0ef57fb58 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -51,6 +51,17 @@ If you are trying to deploy onto Grace (GH200, GB200, etc.), you need to use the make setup ARCH=aarch64 # or ARCH=x86_64 ``` +Native SGLang Router, vLLM Router, and direct-backend deployments do not run +the Dynamo control plane. CI or cluster launchers for those paths can install +only the compute-architecture `uv` binary: + +```bash +make setup-compute ARCH=aarch64 # or ARCH=x86_64 +``` + +This target does not download NATS or etcd and does not create +`srtslurm.yaml`; provide the cluster profile separately. + The setup will: 1. Download NATS/ETCD binaries for your architecture diff --git a/src/srtctl/cli/submit.py b/src/srtctl/cli/submit.py index 4b2d067b1..28da9027b 100755 --- a/src/srtctl/cli/submit.py +++ b/src/srtctl/cli/submit.py @@ -391,10 +391,11 @@ def validate_setup(srtctl_source: Path, *, requires_head_infrastructure: bool = console.print(f"\n[red bold]ERROR:[/] Required binaries not found in {srtctl_source}:") for m in missing: console.print(f" [red]✗[/] {m}") - console.print("\nRun [bold]make setup ARCH=[/] first:") + target = "setup" if requires_head_infrastructure else "setup-compute" + console.print(f"\nRun [bold]make {target} ARCH=[/] first:") console.print(f" cd {srtctl_source}") - console.print(" make setup ARCH=aarch64 [dim]# for GB200/Grace compute nodes[/]") - console.print(" make setup ARCH=x86_64 [dim]# for x86_64 compute nodes[/]\n") + console.print(f" make {target} ARCH=aarch64 [dim]# for ARM compute nodes[/]") + console.print(f" make {target} ARCH=x86_64 [dim]# for x86_64 compute nodes[/]\n") raise SystemExit(1) diff --git a/src/srtctl/templates/job_script_minimal.j2 b/src/srtctl/templates/job_script_minimal.j2 index a5f55bfe2..eb219c998 100644 --- a/src/srtctl/templates/job_script_minimal.j2 +++ b/src/srtctl/templates/job_script_minimal.j2 @@ -165,7 +165,7 @@ export {{ key }}={{ value }} echo "" echo "Preparing srtctl environment..." -# Use compute-arch uv from srt-slurm/bin (installed by make setup ARCH=) +# Use compute-arch uv from srt-slurm/bin (installed by make setup-compute ARCH=) # This avoids arch mismatch when login node (x86_64) != compute node (aarch64) export PATH="${SRTCTL_SOURCE}/bin:$HOME/.local/bin:$PATH" @@ -174,8 +174,8 @@ export PATH="${SRTCTL_SOURCE}/bin:$HOME/.local/bin:$PATH" export UV_PROJECT_ENVIRONMENT="${SRTCTL_SOURCE}/.venv-compute" if ! command -v uv &> /dev/null; then - echo "ERROR: uv not found. Run 'make setup ARCH=' first." - echo " e.g., make setup ARCH=aarch64 (for GB200/Grace compute nodes)" + echo "ERROR: uv not found. Run 'make setup-compute ARCH=' first." + echo " e.g., make setup-compute ARCH=x86_64 (for AMD compute nodes)" exit 1 fi diff --git a/tests/test_validate_setup.py b/tests/test_validate_setup.py index 8b1474736..c4dd514af 100644 --- a/tests/test_validate_setup.py +++ b/tests/test_validate_setup.py @@ -59,13 +59,18 @@ def test_fails_when_all_missing(self, tmp_path: Path): with pytest.raises(SystemExit): validate_setup(tmp_path) - def test_static_frontend_setup_only_requires_compute_uv(self, tmp_path: Path): + def test_static_frontend_setup_only_requires_compute_uv(self, tmp_path: Path, capsys): """Native routers must not require unused Dynamo infrastructure binaries.""" (tmp_path / "bin").mkdir() (tmp_path / "bin" / "uv").touch() validate_setup(tmp_path, requires_head_infrastructure=False) + (tmp_path / "bin" / "uv").unlink() + with pytest.raises(SystemExit): + validate_setup(tmp_path, requires_head_infrastructure=False) + assert "make setup-compute ARCH=" in capsys.readouterr().out + class TestMakefileArchDetection: """Test that the file | grep pattern used in Makefile matches correctly. From 9d44a0e984867aae6e31c1d9a31a7305107bb83c Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 10 Aug 2026 02:22:18 -0500 Subject: [PATCH 31/46] fix(identity): detect native router and AMD MoRI --- src/srtctl/core/fingerprint.py | 2 ++ tests/test_fingerprint.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/src/srtctl/core/fingerprint.py b/src/srtctl/core/fingerprint.py index 15a50fa4a..2fa0feecc 100644 --- a/src/srtctl/core/fingerprint.py +++ b/src/srtctl/core/fingerprint.py @@ -47,6 +47,8 @@ FRAMEWORK_PACKAGES: dict[str, str] = { "vllm": "vllm", "sglang": "sglang", + "sglang-router": "sglang-router", + "amd-mori": "amd_mori", "tensorrt_llm": "tensorrt-llm", "dynamo": "ai-dynamo", } diff --git a/tests/test_fingerprint.py b/tests/test_fingerprint.py index 4d7160e4c..198c37375 100644 --- a/tests/test_fingerprint.py +++ b/tests/test_fingerprint.py @@ -33,6 +33,7 @@ generate_capture_script, load_fingerprint, probe_cpu, + probe_frameworks, write_fingerprint, ) @@ -165,6 +166,20 @@ def test_probe_result_failure(self): assert r.value == UNAVAILABLE assert r.error == "broken" + def test_probe_frameworks_includes_native_router_and_amd_transport(self, monkeypatch): + versions = {"sglang-router": "0.3.2", "amd_mori": "0.5.16.dev0"} + + def fake_run(command: str): + return next((version for package, version in versions.items() if f"'{package}'" in command), None) + + monkeypatch.setattr("srtctl.core.fingerprint._run_cmd", fake_run) + + result = probe_frameworks() + + assert result.ok is True + assert result.value["sglang-router"] == "0.3.2" + assert result.value["amd-mori"] == "0.5.16.dev0" + def test_cpu_model_prefers_x86_model_name(self): assert cpu_model_from_cpuinfo("processor: 0\nmodel name: Intel(R) Xeon(R) Platinum\n") == ( "Intel(R) Xeon(R) Platinum" From e1ac409cf26ae40db53de010cb1b1c5da9111746 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 10 Aug 2026 02:44:19 -0500 Subject: [PATCH 32/46] fix(hf): propagate cache environment during prefetch --- src/srtctl/cli/do_sweep.py | 7 ++++++- tests/test_hf_cache.py | 3 +++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/srtctl/cli/do_sweep.py b/src/srtctl/cli/do_sweep.py index 1b167c4f1..866daf156 100644 --- a/src/srtctl/cli/do_sweep.py +++ b/src/srtctl/cli/do_sweep.py @@ -508,7 +508,12 @@ def _ensure_model_cached(self) -> None: container_image=str(self.runtime.container_image), container_mounts=self.runtime.container_mounts, env_to_set=hf_env, - use_bash_wrapper=False, # command is already bash -c + # The wrapper exports HF_HUB_CACHE, HF_TOKEN, and the other + # backend-provided HF variables before invoking this bash -c. + # Disabling it silently drops env_to_set and can populate a + # second cache under $HF_HOME/hub instead of reusing the + # configured shared cache. + use_bash_wrapper=True, het_group=self.runtime.nodes.het_group_for(download_node), ) diff --git a/tests/test_hf_cache.py b/tests/test_hf_cache.py index 269a2894c..a4d7bf616 100644 --- a/tests/test_hf_cache.py +++ b/tests/test_hf_cache.py @@ -394,6 +394,7 @@ def test_runs_srun_on_single_node(self, tmp_path: Path): type: vllm prefill_environment: HF_HOME: "{tmp_path / "cache"}" + HF_HUB_CACHE: "{tmp_path / "shared-hub-cache"}" vllm_config: prefill: tensor-parallel-size: 1 @@ -433,6 +434,8 @@ def test_runs_srun_on_single_node(self, tmp_path: Path): # Should pass HF env vars to srun assert "HF_HOME" in kwargs["env_to_set"] + assert kwargs["env_to_set"]["HF_HUB_CACHE"] == str(tmp_path / "shared-hub-cache") + assert kwargs["use_bash_wrapper"] is True # Should wait with a timeout mock_proc.wait.assert_called_once() From 3b6b13c691ef0003efc0a63c999e9b51d3873b46 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 10 Aug 2026 03:22:48 -0500 Subject: [PATCH 33/46] fix(hf): resolve the worker hub cache exactly --- src/srtctl/cli/do_sweep.py | 65 ++++++++++++++++++++++++++++---------- tests/test_hf_cache.py | 12 +++++-- 2 files changed, 57 insertions(+), 20 deletions(-) diff --git a/src/srtctl/cli/do_sweep.py b/src/srtctl/cli/do_sweep.py index 866daf156..475674020 100644 --- a/src/srtctl/cli/do_sweep.py +++ b/src/srtctl/cli/do_sweep.py @@ -344,6 +344,21 @@ def _get_hf_home(self) -> str | None: return env["HF_HOME"] return None + def _get_hf_cache_dir(self) -> str | None: + """Resolve the exact Hugging Face Hub cache used by backend workers. + + ``HF_HUB_CACHE`` takes precedence over the legacy + ``HUGGINGFACE_HUB_CACHE`` variable. When neither is explicit, + Hugging Face defaults to ``$HF_HOME/hub``. + """ + for mode in ("prefill", "decode", "agg"): + env = self.config.backend.get_environment_for_mode(mode) + for key in ("HF_HUB_CACHE", "HUGGINGFACE_HUB_CACHE"): + if key in env: + return env[key] + hf_home = self._get_hf_home() + return str(Path(hf_home) / "hub") if hf_home else None + def _get_hf_env(self) -> dict[str, str]: """Collect HF-related environment variables from backend config. @@ -354,7 +369,7 @@ def _get_hf_env(self) -> dict[str, str]: hf_env: dict[str, str] = {} for mode in ("prefill", "decode", "agg"): for key, val in self.config.backend.get_environment_for_mode(mode).items(): - if key.startswith(("HF_", "HUGGING_FACE_")): + if key.startswith(("HF_", "HUGGING_FACE_", "HUGGINGFACE_")): hf_env[key] = val return hf_env @@ -366,11 +381,11 @@ def _clean_stale_hf_locks(self) -> None: "Lock acquisition failed". This removes locks older than 30 minutes (no legitimate download takes that long). """ - hf_home = self._get_hf_home() - if not hf_home: + hf_cache_dir = self._get_hf_cache_dir() + if not hf_cache_dir: return - cache_dir = Path(hf_home) + cache_dir = Path(hf_cache_dir) if not cache_dir.is_dir(): return @@ -387,7 +402,11 @@ def _clean_stale_hf_locks(self) -> None: pass # Permission denied or already deleted if removed > 0: - logger.info("Cleaned %d stale .lock files from HF cache: %s", removed, hf_home) + logger.info( + "Cleaned %d stale .lock files from HF cache: %s", + removed, + hf_cache_dir, + ) def _stage_model(self) -> None: """Copy the model from shared storage to node-local storage on every @@ -436,12 +455,13 @@ def _ensure_model_cached(self) -> None: return hf_home = self._get_hf_home() - if not hf_home: + hf_cache_dir = self._get_hf_cache_dir() + if not hf_cache_dir: logger.warning( - "HF model '%s' specified but HF_HOME is not set in backend environment config. " + "HF model '%s' specified but no Hugging Face cache is set in backend environment config. " "Workers will use the default HuggingFace cache (~/.cache/huggingface) which may not " - "be shared across nodes. Set HF_HOME in prefill_environment/decode_environment to use " - "a shared cache directory (e.g., HF_HOME: /lustre/fsw/.../common/cache).", + "be shared across nodes. Set HF_HUB_CACHE or HF_HOME in " + "prefill_environment/decode_environment to use a shared cache directory.", self.runtime.model_path, ) return @@ -451,13 +471,17 @@ def _ensure_model_cached(self) -> None: # Check if model is already fully cached using huggingface_hub API. # snapshot_download with local_files_only=True succeeds only if every # file in the model repo is already present in the local cache. - # Note: HF_HOME stores models in $HF_HOME/hub/, so we pass cache_dir=$HF_HOME/hub - # to match the actual storage location used by workers. + # Use the exact resolved worker cache. Explicit HF_HUB_CACHE may point + # somewhere other than $HF_HOME/hub. try: from huggingface_hub import snapshot_download # type: ignore[import-untyped] - snapshot_download(model_id, cache_dir=str(Path(hf_home) / "hub"), local_files_only=True) - logger.info("Model '%s' already cached at %s, skipping pre-download", model_id, hf_home) + snapshot_download(model_id, cache_dir=hf_cache_dir, local_files_only=True) + logger.info( + "Model '%s' already cached at %s, skipping pre-download", + model_id, + hf_cache_dir, + ) return except ImportError: logger.debug("huggingface_hub not installed on host, will use container to check/download") @@ -466,7 +490,12 @@ def _ensure_model_cached(self) -> None: download_node = self.runtime.nodes.worker[0] - logger.info("Ensuring model '%s' is cached on %s (cache: %s)", model_id, download_node, hf_home) + logger.info( + "Ensuring model '%s' is cached on %s (cache: %s)", + model_id, + download_node, + hf_cache_dir, + ) # The srun command uses HF_HOME (not --cache-dir) to match the exact # cache path workers use ($HF_HOME/hub/models--*/). @@ -475,14 +504,16 @@ def _ensure_model_cached(self) -> None: # Uses 'hf download' (new CLI) with 'huggingface-cli download' as fallback. import shlex - q_hf_home = shlex.quote(hf_home) + q_hf_home = shlex.quote(hf_home) if hf_home else None + q_hf_cache_dir = shlex.quote(hf_cache_dir) q_model_id = shlex.quote(model_id) + hf_home_export = f"export HF_HOME={q_hf_home}; " if q_hf_home else "" download_cmd = [ "bash", "-c", ( - f"export HF_HOME={q_hf_home}; " - f"find {q_hf_home} -name '*.lock' -mmin +30 -delete 2>/dev/null; " + f"{hf_home_export}" + f"find {q_hf_cache_dir} -name '*.lock' -mmin +30 -delete 2>/dev/null; " f"DL_CMD='hf download'; " f"command -v hf >/dev/null 2>&1 || DL_CMD='huggingface-cli download'; " f"if HF_HUB_OFFLINE=1 $DL_CMD {q_model_id} --quiet 2>/dev/null; then " diff --git a/tests/test_hf_cache.py b/tests/test_hf_cache.py index a4d7bf616..252ed1855 100644 --- a/tests/test_hf_cache.py +++ b/tests/test_hf_cache.py @@ -3,7 +3,7 @@ """Tests for HuggingFace cache management in SweepOrchestrator. -Tests _get_hf_home(), _clean_stale_hf_locks(), and _ensure_model_cached(). +Tests cache path resolution, stale-lock cleanup, and model prefetch. """ import os @@ -83,6 +83,7 @@ def test_returns_hf_home_from_prefill_env(self, tmp_path: Path): type: vllm prefill_environment: HF_HOME: "/cache/hub" + HF_HUB_CACHE: "/cache/canonical-hub" vllm_config: prefill: tensor-parallel-size: 1 @@ -99,6 +100,7 @@ def test_returns_hf_home_from_prefill_env(self, tmp_path: Path): ) orch = _make_orchestrator(str(config_file)) assert orch._get_hf_home() == "/cache/hub" + assert orch._get_hf_cache_dir() == "/cache/canonical-hub" def test_returns_none_when_not_set(self, tmp_path: Path): """No HF_HOME in any environment config should return None.""" @@ -128,6 +130,7 @@ def test_returns_none_when_not_set(self, tmp_path: Path): ) orch = _make_orchestrator(str(config_file)) assert orch._get_hf_home() is None + assert orch._get_hf_cache_dir() is None # Tests for _clean_stale_hf_locks @@ -330,7 +333,7 @@ def test_skips_and_warns_when_no_hf_home(self, tmp_path: Path, caplog): with caplog.at_level(logging.WARNING): orch._ensure_model_cached() mock_srun.assert_not_called() - assert "HF_HOME is not set" in caplog.text + assert "no Hugging Face cache is set" in caplog.text def test_skips_when_model_already_cached(self, tmp_path: Path): """Pre-download should skip if huggingface_hub reports model is cached.""" @@ -379,6 +382,9 @@ def test_skips_when_model_already_cached(self, tmp_path: Path): orch._ensure_model_cached() mock_srun.assert_not_called() fake_hf_hub.snapshot_download.assert_called_once() + assert fake_hf_hub.snapshot_download.call_args.kwargs["cache_dir"] == str( + tmp_path / "cache" / "hub" + ) def test_runs_srun_on_single_node(self, tmp_path: Path): """Pre-download should run huggingface-cli on exactly one node.""" @@ -427,7 +433,7 @@ def test_runs_srun_on_single_node(self, tmp_path: Path): cmd_str = " ".join(kwargs["command"]) assert "huggingface-cli download" in cmd_str assert "nvidia/Kimi-K2.5-NVFP4" in cmd_str - assert str(tmp_path / "cache") in cmd_str + assert str(tmp_path / "shared-hub-cache") in cmd_str # Should use the container image assert kwargs["container_image"] == "test-image:latest" From ab0ba1485a2ac4e31f7139396b62cd7a6c3c649d Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 10 Aug 2026 03:54:31 -0500 Subject: [PATCH 34/46] fix(router): propagate runtime srun options --- src/srtctl/frontends/static_router.py | 1 + tests/test_frontends.py | 35 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/srtctl/frontends/static_router.py b/src/srtctl/frontends/static_router.py index 4e5b478f0..7e4529357 100644 --- a/src/srtctl/frontends/static_router.py +++ b/src/srtctl/frontends/static_router.py @@ -200,6 +200,7 @@ def start_frontends( env_to_set=router_env or None, bash_preamble=build_setup_script_preamble(getattr(config, "setup_script", None)), het_group=het_group_for(node), + srun_options=runtime.srun_options, ) processes.append( ManagedProcess( diff --git a/tests/test_frontends.py b/tests/test_frontends.py index 2061a1458..326becf73 100644 --- a/tests/test_frontends.py +++ b/tests/test_frontends.py @@ -449,6 +449,41 @@ def test_sglang_env_passed_to_process(self, mock_get_ip, mock_srun): assert env_to_set["MY_VAR"] == "my_value" assert env_to_set["ANOTHER"] == "123" + @patch("srtctl.frontends.sglang.start_srun_process") + @patch("srtctl.frontends.sglang.get_hostname_ip") + def test_sglang_runtime_srun_options_passed_to_process(self, mock_get_ip, mock_srun): + """Static routers inherit runtime-level Slurm launch options.""" + mock_get_ip.return_value = "10.0.0.1" + mock_srun.return_value = MagicMock() + + frontend = SGLangFrontend() + topology = MockTopology(frontend_nodes=["node0"]) + config = MockConfig( + frontend=MockFrontendConfig(), + resources=MockResourceConfig(num_agg=1), + ) + backend = MagicMock() + backend.is_grpc_mode.return_value = False + runtime = MagicMock() + runtime.log_dir = MagicMock() + runtime.log_dir.__truediv__ = lambda self, x: f"/logs/{x}" + runtime.container_image = "/container.sqsh" + runtime.container_mounts = {} + runtime.srun_options = { + "container-writable": "", + "container-remap-root": "", + } + + frontend.start_frontends( + topology, + runtime, + config, + backend, + [MockProcess(node="node1", endpoint_mode="agg", http_port=30000)], + ) + + assert mock_srun.call_args.kwargs["srun_options"] == runtime.srun_options + @patch("srtctl.frontends.sglang.start_srun_process") @patch("srtctl.frontends.sglang.get_hostname_ip") def test_sglang_no_env_when_empty(self, mock_get_ip, mock_srun): From f94026bd08612ed9f7677fdf0a42f8d8391fd736 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 10 Aug 2026 04:25:02 -0500 Subject: [PATCH 35/46] test(router): cover runtime srun options --- tests/test_static_router_frontends.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_static_router_frontends.py b/tests/test_static_router_frontends.py index e3c8fd859..330810f49 100644 --- a/tests/test_static_router_frontends.py +++ b/tests/test_static_router_frontends.py @@ -189,6 +189,7 @@ def test_vllm_router_launch_uses_router_container_env_and_only_leaders() -> None container_image=Path("/worker.sqsh"), container_mounts={"/host": "/container"}, environment={"GLOBAL": "value", "ROUTER_LOG": "info"}, + srun_options={"container-writable": True, "container-remap-root": True}, nodes=SimpleNamespace(het_group_for=lambda node: 1), ) config = SimpleNamespace( @@ -240,6 +241,7 @@ def test_vllm_router_launch_uses_router_container_env_and_only_leaders() -> None assert kwargs["container_image"] == "docker://router:test" assert kwargs["env_to_set"] == {"GLOBAL": "value", "ROUTER_LOG": "debug"} assert kwargs["het_group"] == 1 + assert kwargs["srun_options"] == runtime.srun_options assert "/configs/${setup_script}" in kwargs["bash_preamble"] assert kwargs["command"].count("http://10.0.0.1:30000") == 1 assert "--routing-logic" in kwargs["command"] From dd0109d4043141072ad37c043f1100332008b77f Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 10 Aug 2026 20:26:00 -0500 Subject: [PATCH 36/46] fix(router): skip Dynamo infra for SGLang Router --- src/srtctl/cli/do_sweep.py | 2 +- tests/test_benchmarks.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/srtctl/cli/do_sweep.py b/src/srtctl/cli/do_sweep.py index 475674020..a2be81ee3 100644 --- a/src/srtctl/cli/do_sweep.py +++ b/src/srtctl/cli/do_sweep.py @@ -716,7 +716,7 @@ def run(self) -> int: try: # Stage 1: Head infrastructure (NATS, etcd). Only the dynamo request # plane uses it; static/direct frontends skip it. - if self.config.frontend.type in {"sglang", "trtllm_serve", "vllm", "vllm-router"}: + if self.config.frontend.type in {"sglang", "sgl-router", "trtllm_serve", "vllm", "vllm-router"}: logger.info("Skipping NATS/etcd infrastructure (frontend.type=%s)", self.config.frontend.type) else: reporter.report(JobStatus.STARTING, JobStage.HEAD_INFRASTRUCTURE, "Starting head infrastructure") diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py index 5ea6d12f7..c11af7875 100644 --- a/tests/test_benchmarks.py +++ b/tests/test_benchmarks.py @@ -1426,6 +1426,30 @@ class TestSweepRunEvalIntegration: def _make_orchestrator(): return TestRunPostEval._make_orchestrator() + def test_sgl_router_skips_dynamo_infrastructure(self): + """The native SGLang Router must not launch NATS or etcd.""" + import os + from dataclasses import replace + from unittest.mock import MagicMock, patch + + orch = self._make_orchestrator() + orch.config = replace(orch.config, frontend=replace(orch.config.frontend, type="sgl-router")) + + with ( + patch.dict(os.environ, {"EVAL_ONLY": "false", "RUN_EVAL": "false"}, clear=False), + patch.object(orch, "start_head_infrastructure") as mock_head, + patch.object(orch, "start_all_workers", return_value={}), + patch.object(orch, "start_frontend", return_value=[]), + patch.object(orch, "run_benchmark", return_value=0), + patch.object(orch, "run_postprocess"), + patch("srtctl.cli.do_sweep.StatusReporter") as mock_reporter_cls, + ): + mock_reporter_cls.from_config.return_value = MagicMock() + exit_code = orch.run() + + mock_head.assert_not_called() + assert exit_code == 0 + def test_run_eval_only_mode(self): """EVAL_ONLY=true skips benchmark and runs _run_post_eval.""" import os From 31e72da43ed21fe941c039be51b2cad1a3cf428a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 10 Aug 2026 20:34:36 -0500 Subject: [PATCH 37/46] fix(router): use the native SGLang frontend key --- src/srtctl/cli/do_sweep.py | 2 +- src/srtctl/cli/submit.py | 1 - tests/test_benchmarks.py | 24 ------------------------ 3 files changed, 1 insertion(+), 26 deletions(-) diff --git a/src/srtctl/cli/do_sweep.py b/src/srtctl/cli/do_sweep.py index a2be81ee3..475674020 100644 --- a/src/srtctl/cli/do_sweep.py +++ b/src/srtctl/cli/do_sweep.py @@ -716,7 +716,7 @@ def run(self) -> int: try: # Stage 1: Head infrastructure (NATS, etcd). Only the dynamo request # plane uses it; static/direct frontends skip it. - if self.config.frontend.type in {"sglang", "sgl-router", "trtllm_serve", "vllm", "vllm-router"}: + if self.config.frontend.type in {"sglang", "trtllm_serve", "vllm", "vllm-router"}: logger.info("Skipping NATS/etcd infrastructure (frontend.type=%s)", self.config.frontend.type) else: reporter.report(JobStatus.STARTING, JobStage.HEAD_INFRASTRUCTURE, "Starting head infrastructure") diff --git a/src/srtctl/cli/submit.py b/src/srtctl/cli/submit.py index 28da9027b..c18b1cfd1 100755 --- a/src/srtctl/cli/submit.py +++ b/src/srtctl/cli/submit.py @@ -688,7 +688,6 @@ def submit_with_orchestrator( srtctl_source = Path(srtctl_root) if srtctl_root else Path(__file__).parent.parent.parent.parent requires_head_infrastructure = config.frontend.type not in { "sglang", - "sgl-router", "trtllm_serve", "vllm", "vllm-router", diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py index c11af7875..5ea6d12f7 100644 --- a/tests/test_benchmarks.py +++ b/tests/test_benchmarks.py @@ -1426,30 +1426,6 @@ class TestSweepRunEvalIntegration: def _make_orchestrator(): return TestRunPostEval._make_orchestrator() - def test_sgl_router_skips_dynamo_infrastructure(self): - """The native SGLang Router must not launch NATS or etcd.""" - import os - from dataclasses import replace - from unittest.mock import MagicMock, patch - - orch = self._make_orchestrator() - orch.config = replace(orch.config, frontend=replace(orch.config.frontend, type="sgl-router")) - - with ( - patch.dict(os.environ, {"EVAL_ONLY": "false", "RUN_EVAL": "false"}, clear=False), - patch.object(orch, "start_head_infrastructure") as mock_head, - patch.object(orch, "start_all_workers", return_value={}), - patch.object(orch, "start_frontend", return_value=[]), - patch.object(orch, "run_benchmark", return_value=0), - patch.object(orch, "run_postprocess"), - patch("srtctl.cli.do_sweep.StatusReporter") as mock_reporter_cls, - ): - mock_reporter_cls.from_config.return_value = MagicMock() - exit_code = orch.run() - - mock_head.assert_not_called() - assert exit_code == 0 - def test_run_eval_only_mode(self): """EVAL_ONLY=true skips benchmark and runs _run_post_eval.""" import os From 8bd8aef4089174d9e5acdab6e99184b63615255e Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 10 Aug 2026 21:45:06 -0500 Subject: [PATCH 38/46] fix(logs): tolerate non-UTF-8 backend output --- src/srtctl/core/processes.py | 6 +++++- tests/test_process_registry.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/srtctl/core/processes.py b/src/srtctl/core/processes.py index a853bb72b..3bf838864 100644 --- a/src/srtctl/core/processes.py +++ b/src/srtctl/core/processes.py @@ -199,7 +199,11 @@ def print_failure_details(self, tail_lines: int = 50) -> None: # Tail the log file if available if proc.log_file and proc.log_file.exists(): try: - lines = proc.log_file.read_text().splitlines() + # Backend logs can contain terminal progress output or + # compiler diagnostics with bytes that are not valid + # UTF-8. Failure reporting must remain available even + # when the worker log is not clean text. + lines = proc.log_file.read_text(errors="replace").splitlines() if lines: logger.error("\nLast %d lines of log:", tail_lines) for line in lines[-tail_lines:]: diff --git a/tests/test_process_registry.py b/tests/test_process_registry.py index 885dd2501..999b6d5eb 100644 --- a/tests/test_process_registry.py +++ b/tests/test_process_registry.py @@ -186,3 +186,32 @@ def test_cleanup(self): registry.cleanup() mock_popen.terminate.assert_called_once() + + def test_failure_details_tolerate_non_utf8_worker_logs(self, tmp_path, caplog): + """ROCm/compiler output must not hide the useful failure log tail.""" + log_file = tmp_path / "worker.out" + log_file.write_bytes(b"valid line\ninvalid: \xff\xfe\nlast line\n") + + mock_popen = MagicMock(spec=Popen) + mock_popen.poll.return_value = 137 + mock_popen.returncode = 137 + mock_popen.pid = 12345 + + registry = ProcessRegistry(job_id="test_job") + registry.add_process( + ManagedProcess( + name="decode_0", + popen=mock_popen, + log_file=log_file, + node="amd-worker", + critical=True, + ) + ) + assert registry.check_failures() + + with caplog.at_level("ERROR"): + registry.print_failure_details() + + assert "Could not read log file" not in caplog.text + assert "invalid: \ufffd\ufffd" in caplog.text + assert "last line" in caplog.text From b4bf6eb0abefa16c5108b339dabb903214a4c806 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 10 Aug 2026 23:55:24 -0500 Subject: [PATCH 39/46] Gate benchmarks on advertised vLLM workers --- src/srtctl/cli/mixins/benchmark_stage.py | 23 +++++++++- src/srtctl/core/__init__.py | 2 + src/srtctl/core/health.py | 55 ++++++++++++++++++++++++ src/srtctl/frontends/base.py | 13 ++++++ src/srtctl/frontends/dynamo.py | 5 +++ src/srtctl/frontends/static_router.py | 5 +++ src/srtctl/frontends/trtllm_serve.py | 5 +++ src/srtctl/frontends/vllm.py | 5 +++ src/srtctl/frontends/vllm_router.py | 10 +++++ tests/test_health.py | 34 +++++++++++++++ tests/test_static_router_frontends.py | 22 ++++++++++ 11 files changed, 178 insertions(+), 1 deletion(-) diff --git a/src/srtctl/cli/mixins/benchmark_stage.py b/src/srtctl/cli/mixins/benchmark_stage.py index 884d9e529..a9c5e8b47 100644 --- a/src/srtctl/cli/mixins/benchmark_stage.py +++ b/src/srtctl/cli/mixins/benchmark_stage.py @@ -15,7 +15,7 @@ from typing import TYPE_CHECKING from srtctl.core.fingerprint import format_identity_verification, verify_identity -from srtctl.core.health import wait_for_model +from srtctl.core.health import wait_for_http_endpoints, wait_for_model from srtctl.core.lockfile import collect_worker_fingerprints from srtctl.core.slurm import get_hostname_ip, start_srun_process from srtctl.core.status import JobStage, JobStatus, StatusReporter @@ -227,6 +227,27 @@ def run_benchmark( reporter.report(JobStatus.FAILED, JobStage.BENCHMARK, "Workers failed health check") return 1 + from srtctl.frontends import get_frontend + + frontend = get_frontend(self.config.frontend.type) + backend_health_urls = frontend.get_backend_health_urls(self.config.backend, self.backend_processes) + if backend_health_urls: + logger.info( + "Frontend requires direct readiness from %d advertised backend URLs", + len(backend_health_urls), + ) + if not wait_for_http_endpoints( + backend_health_urls, + poll_interval=float(hc.interval_seconds), + timeout=float(hc.max_attempts * hc.interval_seconds), + report_every=60.0, + stop_event=stop_event, + ): + logger.error("Advertised backend URLs did not become healthy") + if reporter: + reporter.report(JobStatus.FAILED, JobStage.BENCHMARK, "Backends failed direct health check") + return 1 + logger.info("Server is healthy - starting benchmark") # Identity verification: compare recipe identity against runtime fingerprints diff --git a/src/srtctl/core/__init__.py b/src/srtctl/core/__init__.py index b1276e694..1fc13ed11 100644 --- a/src/srtctl/core/__init__.py +++ b/src/srtctl/core/__init__.py @@ -37,6 +37,7 @@ check_sglang_router_health, wait_for_etcd, wait_for_health, + wait_for_http_endpoints, wait_for_model, wait_for_port, ) @@ -140,6 +141,7 @@ "start_srun_process", "wait_for_etcd", "wait_for_health", + "wait_for_http_endpoints", "wait_for_model", # Health checks "wait_for_port", diff --git a/src/srtctl/core/health.py b/src/srtctl/core/health.py index d379b59b2..7bb679b4e 100644 --- a/src/srtctl/core/health.py +++ b/src/srtctl/core/health.py @@ -9,6 +9,7 @@ - wait_for_health(): HTTP health check with worker count validation - wait_for_etcd(): Wait for etcd to be ready - wait_for_model(): Wait for model with worker count validation (replaces bash version) +- wait_for_http_endpoints(): Wait until every adapter-provided HTTP endpoint is ready - check_dynamo_health(): Parse dynamo /health response for worker counts - check_sglang_router_health(): Parse sglang /workers response for worker counts """ @@ -398,6 +399,60 @@ def wait_for_etcd( # ============================================================================ +def wait_for_http_endpoints( + urls: list[str], + poll_interval: float = 1.0, + timeout: float = 600.0, + report_every: float = 60.0, + stop_event: threading.Event | None = None, +) -> bool: + """Wait until every URL returns HTTP 200 in the same polling pass. + + Frontend adapters use this for direct backend readiness requirements that + are additional to the frontend's own health response. + """ + targets = list(dict.fromkeys(urls)) + if not targets: + return True + + logger.info("Polling %d backend health endpoints every %.1fs", len(targets), poll_interval) + start_time = time.time() + last_report_time = start_time + + while True: + if stop_event and stop_event.is_set(): + logger.warning("Wait for backend health endpoints aborted by stop event") + return False + + if time.time() - start_time >= timeout: + logger.error("Backend health endpoints did not all become ready in %.0f seconds", timeout) + return False + + pending: list[str] = [] + for url in targets: + try: + response = requests.get(url, timeout=5.0) + if response.status_code != 200: + pending.append(url) + except requests.exceptions.RequestException: + pending.append(url) + + if not pending: + logger.info("All %d backend health endpoints are ready", len(targets)) + return True + + if time.time() - last_report_time >= report_every: + logger.info( + "Waiting for %d/%d backend health endpoints: %s", + len(pending), + len(targets), + ", ".join(pending), + ) + last_report_time = time.time() + + time.sleep(poll_interval) + + def wait_for_model( host: str, port: int, diff --git a/src/srtctl/frontends/base.py b/src/srtctl/frontends/base.py index 9d4fd8543..f2cb6b405 100644 --- a/src/srtctl/frontends/base.py +++ b/src/srtctl/frontends/base.py @@ -91,6 +91,19 @@ def parse_health( """Parse health check response and return worker status.""" ... + def get_backend_health_urls( + self, + backend: Any, + backend_processes: list["Process"], + ) -> list[str]: + """Return backend URLs that must be directly healthy before benchmarking. + + Frontends that discover or gate their own backends return an empty list. + Static adapters may use this hook to require readiness at the exact URLs + they advertise to their router. + """ + ... + def start_frontends( self, topology: Any, # FrontendTopology diff --git a/src/srtctl/frontends/dynamo.py b/src/srtctl/frontends/dynamo.py index d6cfeacc9..e1f37c120 100644 --- a/src/srtctl/frontends/dynamo.py +++ b/src/srtctl/frontends/dynamo.py @@ -50,6 +50,11 @@ def parse_health( """Parse dynamo /health endpoint response.""" return check_dynamo_health(response_json, expected_prefill, expected_decode) + def get_backend_health_urls(self, backend: Any, backend_processes: list["Process"]) -> list[str]: + """Dynamo owns backend discovery and exposes readiness through its frontend.""" + del backend, backend_processes + return [] + def get_frontend_args_list(self, args: dict[str, Any] | None) -> list[str]: """Convert frontend args dict to CLI arguments.""" if not args: diff --git a/src/srtctl/frontends/static_router.py b/src/srtctl/frontends/static_router.py index c2568f51a..007b6d20e 100644 --- a/src/srtctl/frontends/static_router.py +++ b/src/srtctl/frontends/static_router.py @@ -81,6 +81,11 @@ def get_managed_frontend_args( del backend, backend_processes return [] + def get_backend_health_urls(self, backend: Any, backend_processes: list[Process]) -> list[str]: + """Keep existing static-router readiness semantics unless an adapter opts in.""" + del backend, backend_processes + return [] + def worker_scheme(self, backend: Any, mode: str) -> str: """Return the protocol used to reach a worker endpoint.""" return "http" diff --git a/src/srtctl/frontends/trtllm_serve.py b/src/srtctl/frontends/trtllm_serve.py index 589cc0240..6c965a673 100644 --- a/src/srtctl/frontends/trtllm_serve.py +++ b/src/srtctl/frontends/trtllm_serve.py @@ -55,6 +55,11 @@ def parse_health( """Parse trtllm-serve /health response (200 => ready).""" return check_trtllm_serve_health(response_json, expected_prefill, expected_decode) + def get_backend_health_urls(self, backend: Any, backend_processes: list["Process"]) -> list[str]: + """trtllm-serve gates its configured workers before its own health succeeds.""" + del backend, backend_processes + return [] + def get_frontend_args_list(self, args: dict[str, Any] | None) -> list[str]: """Convert frontend args dict to CLI arguments.""" if not args: diff --git a/src/srtctl/frontends/vllm.py b/src/srtctl/frontends/vllm.py index c1af62669..725b974fc 100644 --- a/src/srtctl/frontends/vllm.py +++ b/src/srtctl/frontends/vllm.py @@ -57,6 +57,11 @@ def parse_health( decode_expected=expected_decode, ) + def get_backend_health_urls(self, backend: Any, backend_processes: list[Process]) -> list[str]: + """The direct frontend health endpoint is the vLLM backend itself.""" + del backend, backend_processes + return [] + def get_frontend_args_list(self, args: dict[str, Any] | None) -> list[str]: if not args: return [] diff --git a/src/srtctl/frontends/vllm_router.py b/src/srtctl/frontends/vllm_router.py index 8a8b1b99b..7d1b93036 100644 --- a/src/srtctl/frontends/vllm_router.py +++ b/src/srtctl/frontends/vllm_router.py @@ -46,6 +46,16 @@ class VLLMRouterFrontend(StaticRouterFrontend): pd_flag: ClassVar[str] = "--vllm-pd-disaggregation" process_name: ClassVar[str] = "vllm_router" + def get_backend_health_urls(self, backend: Any, backend_processes: list[Process]) -> list[str]: + """Return the exact logical vLLM endpoints advertised to Router. + + Router expands node-local DP pools internally, but its worker-count view + can become complete before every advertised HTTP server is accepting + requests. Polling each logical server closes that readiness race without + changing the semantics of other frontend adapters. + """ + return [f"{worker.url.rstrip('/')}/health" for worker in self.collect_workers(backend, backend_processes)] + def get_managed_frontend_args( self, config: Any, diff --git a/tests/test_health.py b/tests/test_health.py index b894104ef..137158282 100644 --- a/tests/test_health.py +++ b/tests/test_health.py @@ -3,12 +3,46 @@ """Tests for health check parsing (Dynamo and SGLang router).""" +import threading +from unittest.mock import MagicMock, patch + from srtctl.core.health import ( WorkerHealthResult, check_dynamo_health, check_sglang_router_health, + wait_for_http_endpoints, ) + +def test_wait_for_http_endpoints_keeps_2p2d_blocked_while_one_base_is_unavailable() -> None: + urls = [f"http://{mode}{index}/health" for mode in ("p", "d") for index in range(2)] + responses = [ + MagicMock(status_code=503), + MagicMock(status_code=200), + MagicMock(status_code=200), + MagicMock(status_code=200), + *[MagicMock(status_code=200) for _ in urls], + ] + + with ( + patch("srtctl.core.health.requests.get", side_effect=responses) as get, + patch("srtctl.core.health.time.sleep"), + ): + assert wait_for_http_endpoints(urls, timeout=10.0) + + assert [call.args[0] for call in get.call_args_list] == [*urls, *urls] + + +def test_wait_for_http_endpoints_honors_stop_event() -> None: + stop_event = threading.Event() + stop_event.set() + + with patch("srtctl.core.health.requests.get") as get: + assert not wait_for_http_endpoints(["http://p/health"], stop_event=stop_event) + + get.assert_not_called() + + # ============================================================================ # Dynamo Health Check Tests # ============================================================================ diff --git a/tests/test_static_router_frontends.py b/tests/test_static_router_frontends.py index 27591ec24..8ea01abfe 100644 --- a/tests/test_static_router_frontends.py +++ b/tests/test_static_router_frontends.py @@ -102,6 +102,28 @@ def test_vllm_router_advertises_nixl_side_channel_port() -> None: assert workers == [RouterWorker("prefill", "http://10.0.0.1:30000", 13000)] +def test_vllm_router_health_gates_every_advertised_2p2d_logical_worker() -> None: + frontend = VLLMRouterFrontend() + backend = MagicMock() + processes = [ + SimpleNamespace(endpoint_mode="prefill", node="p0", http_port=6100, nixl_port=5400), + SimpleNamespace(endpoint_mode="prefill", node="p1", http_port=6100, nixl_port=5401), + SimpleNamespace(endpoint_mode="decode", node="d0", http_port=6100, nixl_port=5500), + SimpleNamespace(endpoint_mode="decode", node="d1", http_port=6100, nixl_port=5501), + SimpleNamespace(endpoint_mode="decode", node="tp-follower", http_port=0, nixl_port=5502), + ] + + with patch.object(frontend, "get_hostname_ip", side_effect=lambda node: f"ip-{node}"): + urls = frontend.get_backend_health_urls(backend, processes) + + assert urls == [ + "http://ip-p0:6100/health", + "http://ip-p1:6100/health", + "http://ip-d0:6100/health", + "http://ip-d1:6100/health", + ] + + def test_vllm_router_derives_dep4_expansion_for_1p2d() -> None: """One P URL and two D URLs are each expanded to four ranks by Router.""" frontend = VLLMRouterFrontend() From 9bded70f69ccbf96b97525df3e064086b5a6cda6 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 11 Aug 2026 10:02:09 -0500 Subject: [PATCH 40/46] feat: add ATOM backend and Infera frontend --- src/srtctl/__init__.py | 4 + src/srtctl/backends/__init__.py | 5 +- src/srtctl/backends/atom.py | 265 +++++++++++++++++++++++ src/srtctl/backends/base.py | 5 + src/srtctl/backends/mocker.py | 3 + src/srtctl/backends/sglang.py | 3 + src/srtctl/backends/trtllm.py | 3 + src/srtctl/backends/vllm.py | 3 + src/srtctl/cli/do_sweep.py | 4 +- src/srtctl/cli/mixins/benchmark_stage.py | 15 +- src/srtctl/cli/mixins/telemetry_stage.py | 1 + src/srtctl/core/__init__.py | 4 + src/srtctl/core/schema.py | 60 ++++- src/srtctl/core/telemetry.py | 48 ++-- src/srtctl/frontends/__init__.py | 2 + src/srtctl/frontends/base.py | 4 +- src/srtctl/frontends/infera.py | 181 ++++++++++++++++ tests/test_atom_infera.py | 201 +++++++++++++++++ 18 files changed, 772 insertions(+), 39 deletions(-) create mode 100644 src/srtctl/backends/atom.py create mode 100644 src/srtctl/frontends/infera.py create mode 100644 tests/test_atom_infera.py diff --git a/src/srtctl/__init__.py b/src/srtctl/__init__.py index c9a991848..aaff5a3d9 100644 --- a/src/srtctl/__init__.py +++ b/src/srtctl/__init__.py @@ -33,6 +33,8 @@ # Logging utilities (should be first) # Backend configs from .backends import ( + AtomProtocol, + AtomServerConfig, BackendConfig, BackendProtocol, BackendType, @@ -54,6 +56,8 @@ from .logging_utils import setup_logging __all__ = [ + "AtomProtocol", + "AtomServerConfig", "BackendConfig", # Backends "BackendProtocol", diff --git a/src/srtctl/backends/__init__.py b/src/srtctl/backends/__init__.py index 99618d70a..dffdde942 100644 --- a/src/srtctl/backends/__init__.py +++ b/src/srtctl/backends/__init__.py @@ -9,6 +9,7 @@ - TRTLLM: TensorRT-LLM backend with prefill/decode disaggregation """ +from .atom import AtomProtocol, AtomServerConfig from .base import BackendProtocol, BackendType, SrunConfig from .mocker import MockerProtocol, MockerServerConfig from .sglang import SGLangProtocol, SGLangServerConfig @@ -16,9 +17,11 @@ from .vllm import VLLMProtocol, VLLMServerConfig # Union type for all backend configs -BackendConfig = SGLangProtocol | TRTLLMProtocol | VLLMProtocol | MockerProtocol +BackendConfig = AtomProtocol | SGLangProtocol | TRTLLMProtocol | VLLMProtocol | MockerProtocol __all__ = [ + "AtomProtocol", + "AtomServerConfig", "BackendConfig", # Base types "BackendProtocol", diff --git a/src/srtctl/backends/atom.py b/src/srtctl/backends/atom.py new file mode 100644 index 000000000..b43bd58dc --- /dev/null +++ b/src/srtctl/backends/atom.py @@ -0,0 +1,265 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""ROCm ATOM backend launched through the Infera worker adapter.""" + +from __future__ import annotations + +import builtins +import json +from collections.abc import Sequence +from dataclasses import field +from pathlib import Path +from typing import TYPE_CHECKING, Any, ClassVar, Literal + +from marshmallow import Schema +from marshmallow_dataclass import dataclass + +from srtctl.ports import DYN_SYSTEM_PORT_BASE, ETCD_CLIENT_PORT + +if TYPE_CHECKING: + from srtctl.backends.base import SrunConfig + from srtctl.core.runtime import RuntimeContext + from srtctl.core.schema import ProfilingConfig + from srtctl.core.topology import Endpoint, NodePortAllocator, Process + +WorkerMode = Literal["prefill", "decode", "agg"] + + +@dataclass(frozen=True) +class AtomServerConfig: + """Native ATOM CLI configuration for each serving role. + + Keys are emitted exactly as written because ATOM intentionally exposes a + mixture of underscore and kebab-case flags. srtctl owns model, ports, + tensor parallelism, discovery, and P/D transfer arguments. + """ + + prefill: dict[str, Any] | None = None + decode: dict[str, Any] | None = None + aggregated: dict[str, Any] | None = None + + Schema: ClassVar[type[Schema]] = Schema + + +@dataclass(frozen=True) +class AtomProtocol: + """ATOM engine configuration using Infera for discovery and routing.""" + + type: Literal["atom"] = "atom" + + prefill_environment: dict[str, str] = field(default_factory=dict) + decode_environment: dict[str, str] = field(default_factory=dict) + aggregated_environment: dict[str, str] = field(default_factory=dict) + atom_config: AtomServerConfig | None = None + + # Infera's ATOM adapter publishes prefix-cache events only when explicitly + # enabled. Keep the engine default conservative; recipes proving KV-aware + # routing opt in intentionally. + enable_kv_events: bool = False + + # Infera's released ATOM P/D path supports Mooncake. Each realized worker + # gets a topology-derived, collision-free handshake port. + connector: str = "mooncake" + mooncake_ib_device: str | None = None + + Schema: ClassVar[builtins.type[Schema]] = Schema + + def get_srun_config(self) -> SrunConfig: + from srtctl.backends.base import SrunConfig + + return SrunConfig(mpi=None, oversubscribe=False, launch_per_endpoint=False) + + def get_config_for_mode(self, mode: WorkerMode) -> dict[str, Any]: + if self.atom_config is None: + return {} + if mode == "prefill": + return dict(self.atom_config.prefill or {}) + if mode == "decode": + return dict(self.atom_config.decode or {}) + if mode == "agg": + return dict(self.atom_config.aggregated or {}) + return {} + + def get_environment_for_mode(self, mode: WorkerMode) -> dict[str, str]: + configured = { + "prefill": self.prefill_environment, + "decode": self.decode_environment, + "agg": self.aggregated_environment, + }.get(mode, {}) + env = dict(configured) + if mode in {"prefill", "decode"}: + # Cross-node Mooncake must use the ROCm RDMA path, not host-local + # hipIPC. Recipes may override the GID index for their fabric. + env.setdefault("MC_DISABLE_HIP_TRANSPORT", "1") + env.setdefault("MC_GID_INDEX", "1") + env.setdefault("RDMAV_FORK_SAFE", "1") + return env + + def get_process_environment(self, process: Process) -> dict[str, str]: + del process + return {} + + def get_served_model_name(self, default: str) -> str: + # ATOM registers its load path as the OpenAI model identity. SrtConfig + # supplies the exact runtime path (/model, staged path, or HF id). + return default + + def exposes_worker_metrics(self) -> bool: + """ATOM's native OpenAI server does not expose Prometheus metrics.""" + return False + + def should_set_visible_devices(self, process: Process) -> bool: + del process + return True + + def should_set_cuda_visible_devices(self, process: Process) -> bool: + """Compatibility hook for callers predating vendor-neutral binding.""" + return self.should_set_visible_devices(process) + + def allocate_endpoints( + self, + num_prefill: int, + num_decode: int, + num_agg: int, + gpus_per_prefill: int, + gpus_per_decode: int, + gpus_per_agg: int, + gpus_per_node: int, + available_nodes: Sequence[str], + spread_workers: bool = False, + ) -> list[Endpoint]: + from srtctl.core.topology import allocate_endpoints + + return allocate_endpoints( + num_prefill=num_prefill, + num_decode=num_decode, + num_agg=num_agg, + gpus_per_prefill=gpus_per_prefill, + gpus_per_decode=gpus_per_decode, + gpus_per_agg=gpus_per_agg, + gpus_per_node=gpus_per_node, + available_nodes=available_nodes, + spread_workers=spread_workers, + ) + + def endpoints_to_processes( + self, + endpoints: list[Endpoint], + base_sys_port: int = DYN_SYSTEM_PORT_BASE, + port_allocator: NodePortAllocator | None = None, + frontend_type: str = "infera", + ) -> list[Process]: + del frontend_type + from srtctl.core.topology import endpoints_to_processes + + return endpoints_to_processes( + endpoints, + base_sys_port=base_sys_port, + port_allocator=port_allocator, + ) + + def _build_kv_transfer_config(self, process: Process, worker_ip: str) -> str: + mode = process.endpoint_mode + if mode not in {"prefill", "decode"}: + raise ValueError("ATOM KV transfer config is only valid for prefill/decode workers") + if self.connector.lower() != "mooncake": + raise ValueError("Infera ATOM P/D currently supports connector: mooncake") + if process.nixl_port is None: + raise ValueError("ATOM P/D worker is missing its allocated Mooncake handshake port") + + payload: dict[str, Any] = { + "kv_role": "kv_producer" if mode == "prefill" else "kv_consumer", + "kv_connector": "mooncake", + "proxy_ip": worker_ip, + "http_port": process.http_port, + "handshake_port": process.nixl_port, + } + if self.mooncake_ib_device: + payload["ib_device"] = self.mooncake_ib_device + return json.dumps(payload, separators=(",", ":")) + + def build_worker_command( + self, + process: Process, + endpoint_processes: list[Process], + runtime: RuntimeContext, + frontend_type: str = "infera", + nsys_prefix: list[str] | None = None, + dump_config_path: Path | None = None, + profiling: ProfilingConfig | None = None, + ) -> list[str]: + del dump_config_path, profiling + if frontend_type not in {"infera", "atomesh"}: + raise ValueError(f"backend.type: atom requires frontend.type: infera or atomesh (got {frontend_type!r})") + if len({p.node for p in endpoint_processes}) != 1: + raise ValueError("ATOM workers currently support one Slurm node per logical endpoint") + + from srtctl.core.slurm import get_hostname_ip + + worker_ip = get_hostname_ip(process.node, runtime.network_interface) + model_arg = runtime.worker_model_arg + config = self.get_config_for_mode(process.endpoint_mode) + reserved = { + "model", + "host", + "server-port", + "port", + "tp", + "tensor-parallel-size", + "etcd-endpoint", + "advertise-host", + "kv-transfer-config", + } + overlap = reserved.intersection(_canonical_arg_key(key) for key in config) + if overlap: + raise ValueError(f"ATOM config cannot override srtctl-managed argument(s): {sorted(overlap)}") + + cmd: list[str] = list(nsys_prefix or []) + cmd.extend( + [ + "python3", + "-m", + "infera.engine.atom", + "--model", + model_arg, + "--host", + "0.0.0.0", + "--server-port", + str(process.http_port), + "--port", + str(process.sys_port), + "-tp", + str(len(process.gpu_indices)), + "--advertise-host", + worker_ip, + "--etcd-endpoint", + f"{runtime.infra_node_ip}:{ETCD_CLIENT_PORT}", + ] + ) + if self.enable_kv_events: + cmd.append("--enable-kv-events") + if process.endpoint_mode in {"prefill", "decode"}: + cmd.extend(["--kv-transfer-config", self._build_kv_transfer_config(process, worker_ip)]) + cmd.extend(_config_to_cli_args(config)) + return cmd + + +def _config_to_cli_args(config: dict[str, Any]) -> list[str]: + """Convert ATOM-native config to CLI arguments without renaming flags.""" + args: list[str] = [] + for key, value in sorted(config.items()): + flag = key if key.startswith("-") else f"--{key}" + if isinstance(value, bool): + if value: + args.append(flag) + elif isinstance(value, list): + args.append(flag) + args.extend(str(item) for item in value) + elif value is not None: + args.extend([flag, str(value)]) + return args + + +def _canonical_arg_key(key: str) -> str: + return key.lstrip("-").replace("_", "-") diff --git a/src/srtctl/backends/base.py b/src/srtctl/backends/base.py index 0383d328c..b16cf6ef3 100644 --- a/src/srtctl/backends/base.py +++ b/src/srtctl/backends/base.py @@ -26,6 +26,7 @@ class BackendType(str, Enum): SGLANG = "sglang" TRTLLM = "trtllm" VLLM = "vllm" + ATOM = "atom" MOCKER = "mocker" @@ -133,3 +134,7 @@ def get_process_environment(self, process: "Process") -> dict[str, str]: def get_served_model_name(self, default: str) -> str: """Get served model name from backend config, or return default.""" ... + + def exposes_worker_metrics(self) -> bool: + """Whether worker HTTP/system endpoints expose Prometheus metrics.""" + ... diff --git a/src/srtctl/backends/mocker.py b/src/srtctl/backends/mocker.py index 6298d1dbe..c3cc1a3b1 100644 --- a/src/srtctl/backends/mocker.py +++ b/src/srtctl/backends/mocker.py @@ -158,6 +158,9 @@ def get_served_model_name(self, default: str) -> str: """Get served model name — mocker uses default (model path basename).""" return default + def exposes_worker_metrics(self) -> bool: + return True + def allocate_endpoints( self, num_prefill: int, diff --git a/src/srtctl/backends/sglang.py b/src/srtctl/backends/sglang.py index 833442451..08e170800 100644 --- a/src/srtctl/backends/sglang.py +++ b/src/srtctl/backends/sglang.py @@ -212,6 +212,9 @@ def get_served_model_name(self, default: str) -> str: return name return default + def exposes_worker_metrics(self) -> bool: + return True + def get_kv_events_config_for_mode(self, mode: WorkerMode) -> dict[str, str] | None: """Get kv-events config for a worker mode. diff --git a/src/srtctl/backends/trtllm.py b/src/srtctl/backends/trtllm.py index f06c06cca..52103b16a 100644 --- a/src/srtctl/backends/trtllm.py +++ b/src/srtctl/backends/trtllm.py @@ -127,6 +127,9 @@ def get_served_model_name(self, default: str) -> str: # TRTLLM doesn't have served-model-name in config, just use default return default + def exposes_worker_metrics(self) -> bool: + return True + def allocate_endpoints( self, num_prefill: int, diff --git a/src/srtctl/backends/vllm.py b/src/srtctl/backends/vllm.py index fe0a5676e..3e3f4b4ba 100644 --- a/src/srtctl/backends/vllm.py +++ b/src/srtctl/backends/vllm.py @@ -426,6 +426,9 @@ def get_served_model_name(self, default: str) -> str: return name return default + def exposes_worker_metrics(self) -> bool: + return True + def should_colocate_prefill_decode( self, *, diff --git a/src/srtctl/cli/do_sweep.py b/src/srtctl/cli/do_sweep.py index 475674020..8e838112c 100644 --- a/src/srtctl/cli/do_sweep.py +++ b/src/srtctl/cli/do_sweep.py @@ -714,8 +714,8 @@ def run(self) -> int: exit_code = 1 try: - # Stage 1: Head infrastructure (NATS, etcd). Only the dynamo request - # plane uses it; static/direct frontends skip it. + # Stage 1: Head infrastructure (NATS, etcd). Discovery frontends + # (Dynamo and Infera/ATOMesh) use it; static/direct frontends skip it. if self.config.frontend.type in {"sglang", "trtllm_serve", "vllm", "vllm-router"}: logger.info("Skipping NATS/etcd infrastructure (frontend.type=%s)", self.config.frontend.type) else: diff --git a/src/srtctl/cli/mixins/benchmark_stage.py b/src/srtctl/cli/mixins/benchmark_stage.py index 2db264392..e02046fe0 100644 --- a/src/srtctl/cli/mixins/benchmark_stage.py +++ b/src/srtctl/cli/mixins/benchmark_stage.py @@ -214,7 +214,7 @@ def run_benchmark( hc = self.config.health_check frontend = get_frontend(self.config.frontend.type) uses_dynamic_worker_discovery = bool( - getattr(frontend, "uses_dynamic_worker_discovery", lambda _backend: False)(self.backend) + getattr(frontend, "uses_dynamic_worker_discovery", lambda _backend: False)(self.config.backend) ) if not wait_for_model( host=self._orchestrator_node(), @@ -234,9 +234,6 @@ def run_benchmark( reporter.report(JobStatus.FAILED, JobStage.BENCHMARK, "Workers failed health check") return 1 - from srtctl.frontends import get_frontend - - frontend = get_frontend(self.config.frontend.type) backend_health_urls = frontend.get_backend_health_urls(self.config.backend, self.backend_processes) if backend_health_urls: logger.info( @@ -513,11 +510,21 @@ def _get_aiperf_server_metrics_env( ranks are not advertised as separate engines. """ urls: list[str] = [] + exposes_worker_metrics = getattr(self.config.backend, "exposes_worker_metrics", lambda: True)() + + if self.config.frontend.type in {"infera", "atomesh"}: + frontend_host = get_hostname_ip(self._orchestrator_node(), self.runtime.network_interface) + return {"AIPERF_SERVER_METRICS_URLS": f"http://{frontend_host}:{FRONTEND_PUBLIC_PORT}/metrics"} + if logical_workers_only: + if not exposes_worker_metrics: + return {} if logical_endpoints is None: logical_endpoints = self._logical_worker_endpoints() urls = [f"http://{host}:{port}/metrics" for _, host, port in logical_endpoints] else: + if not exposes_worker_metrics: + return {} if self.config.frontend.type in {"vllm", "vllm-router"}: for process in self.backend_processes: if self.config.frontend.type == "vllm" and process.is_leader: diff --git a/src/srtctl/cli/mixins/telemetry_stage.py b/src/srtctl/cli/mixins/telemetry_stage.py index 05afe6053..adaaa9d67 100644 --- a/src/srtctl/cli/mixins/telemetry_stage.py +++ b/src/srtctl/cli/mixins/telemetry_stage.py @@ -116,6 +116,7 @@ def start_telemetry(self) -> list[ManagedProcess]: runtime=self.runtime, telemetry=telemetry, frontend_type=self.config.frontend.type, + backend_exposes_metrics=getattr(self.config.backend, "exposes_worker_metrics", lambda: True)(), ) ) diff --git a/src/srtctl/core/__init__.py b/src/srtctl/core/__init__.py index 1fc13ed11..72aaca1d0 100644 --- a/src/srtctl/core/__init__.py +++ b/src/srtctl/core/__init__.py @@ -18,6 +18,8 @@ # Re-export backend configs from srtctl.backends import ( + AtomProtocol, + AtomServerConfig, BackendConfig, BackendProtocol, BackendType, @@ -85,6 +87,8 @@ __all__ = [ "DEFAULT_AI_ANALYSIS_PROMPT", "AIAnalysisConfig", + "AtomProtocol", + "AtomServerConfig", "BackendConfig", "BackendProtocol", "BackendType", diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py index 1dbf62ea0..75ad24710 100755 --- a/src/srtctl/core/schema.py +++ b/src/srtctl/core/schema.py @@ -33,6 +33,7 @@ from marshmallow_dataclass import dataclass from srtctl.backends import ( + AtomProtocol, BackendConfig, MockerProtocol, SGLangProtocol, @@ -281,7 +282,7 @@ def _deserialize( # Default to SGLang return SGLangProtocol() - if isinstance(value, SGLangProtocol | TRTLLMProtocol | VLLMProtocol | MockerProtocol): + if isinstance(value, AtomProtocol | SGLangProtocol | TRTLLMProtocol | VLLMProtocol | MockerProtocol): return value if not isinstance(value, dict): @@ -290,7 +291,10 @@ def _deserialize( # Get backend type from the value dict backend_type = value.get("type", "sglang") - if backend_type == "sglang": + if backend_type == "atom": + schema = AtomProtocol.Schema() + return schema.load(value) + elif backend_type == "sglang": schema = SGLangProtocol.Schema() return schema.load(value) elif backend_type == "trtllm": @@ -304,13 +308,15 @@ def _deserialize( return schema.load(value) else: raise ValidationError( - f"Unknown backend type: {backend_type!r}. Supported types: sglang, trtllm, vllm, mocker" + f"Unknown backend type: {backend_type!r}. Supported types: atom, sglang, trtllm, vllm, mocker" ) def _serialize(self, value: Any | None, attr: str | None, obj: Any, **kwargs) -> Any: """Serialize backend config to dict.""" if value is None: return None + if isinstance(value, AtomProtocol): + return AtomProtocol.Schema().dump(value) if isinstance(value, SGLangProtocol): return SGLangProtocol.Schema().dump(value) if isinstance(value, TRTLLMProtocol): @@ -1613,6 +1619,7 @@ def __post_init__(self): self._validate_trtllm_serve() self._validate_vllm_frontend() self._validate_static_router_frontend() + self._validate_infera_frontend() self._validate_sglang_data_parallelism() def _validate_trtllm_serve(self): @@ -1676,6 +1683,7 @@ def _validate_static_router_frontend(self): ) if self.frontend.type == "vllm-router": + assert isinstance(self.backend, VLLMProtocol) connector = getattr(self.backend, "connector", None) if isinstance(connector, str) and connector.lower() == "moriio": if self.frontend.enable_multiple_frontends: @@ -1695,9 +1703,10 @@ def _validate_static_router_frontend(self): "decode": self.resources.gpus_per_decode if self.resources.num_decode else 0, "agg": self.resources.gpus_per_agg if self.resources.num_agg else 0, } - multi_node_modes = [ - mode for mode, count in endpoint_gpu_counts.items() if count > self.resources.gpus_per_node - ] + multi_node_modes: list[Literal["prefill", "decode", "agg"]] = [] + for mode in ("prefill", "decode", "agg"): + if endpoint_gpu_counts[mode] > self.resources.gpus_per_node: + multi_node_modes.append(mode) if multi_node_modes and self.backend.dp_launch_mode != "per_node": raise ValidationError("multi-node vLLM Router DP endpoints require backend.dp_launch_mode: per_node") for mode in multi_node_modes: @@ -1708,6 +1717,34 @@ def _validate_static_router_frontend(self): "multi-node TP-only direct serving is not supported" ) + def _validate_infera_frontend(self): + """Validate ROCm ATOM workers behind Infera dynamic discovery.""" + if self.frontend.type not in {"infera", "atomesh"}: + return + if self.backend_type != "atom": + raise ValidationError( + f"frontend.type: {self.frontend.type} requires backend.type: atom; got {self.backend_type!r}" + ) + assert isinstance(self.backend, AtomProtocol) + + endpoint_gpu_counts = { + "prefill": self.resources.gpus_per_prefill if self.resources.num_prefill else 0, + "decode": self.resources.gpus_per_decode if self.resources.num_decode else 0, + "agg": self.resources.gpus_per_agg if self.resources.num_agg else 0, + } + multi_node_modes = [mode for mode, count in endpoint_gpu_counts.items() if count > self.resources.gpus_per_node] + if multi_node_modes: + raise ValidationError( + "ATOM currently requires each logical worker to fit on one Slurm node; " + f"multi-node endpoint(s): {', '.join(multi_node_modes)}" + ) + + if self.resources.is_disaggregated: + if self.backend.connector.lower() != "mooncake": + raise ValidationError("Infera ATOM disaggregation currently requires backend.connector: mooncake") + if self.resources.num_prefill < 1 or self.resources.num_decode < 1: + raise ValidationError("Infera ATOM disaggregation requires at least one prefill and one decode worker") + def _validate_sglang_data_parallelism(self): """Reject SGLang TP/DP combinations that the server cannot initialize. @@ -1949,7 +1986,16 @@ def from_yaml(cls, yaml_path: Path) -> "SrtConfig": @property def served_model_name(self) -> str: """Get the served model name from backend config or model path.""" - default = Path(self.model.path).name + model_path = os.path.expandvars(self.model.path) + if isinstance(self.backend, AtomProtocol): + if model_path.startswith("hf:"): + default = model_path[3:] + elif self.model.stage_dir: + default = str(Path(os.path.expandvars(self.model.stage_dir)) / Path(model_path).name) + else: + default = "/model" + else: + default = Path(model_path).name return self.backend.get_served_model_name(default) @property diff --git a/src/srtctl/core/telemetry.py b/src/srtctl/core/telemetry.py index 79a8974d1..ac9ea2870 100644 --- a/src/srtctl/core/telemetry.py +++ b/src/srtctl/core/telemetry.py @@ -38,6 +38,7 @@ def generate_telemetry_config( runtime: RuntimeContext, telemetry: TelemetryConfig, frontend_type: str = "dynamo", + backend_exposes_metrics: bool = True, ) -> str: """Generate telemetry TOML from backend and frontend topology.""" dcgm_exporter = telemetry.dcgm_exporter @@ -84,30 +85,31 @@ def generate_telemetry_config( ) ) - for process in sorted(processes, key=lambda p: (p.endpoint_mode, p.endpoint_index, p.node_rank, p.node)): - node_ip = get_hostname_ip(process.node, runtime.network_interface) - if frontend_type == "vllm" and process.endpoint_mode == "agg": - port = FRONTEND_PUBLIC_PORT - elif frontend_type == "vllm-router": - port = process.http_port - else: - port = process.sys_port - node_metadata = { - "hostname": process.node, - "worker_index": str(process.endpoint_index), - "worker_process": str(process.node_rank), - "worker_role": process.endpoint_mode, - } - node_metadata.update(telemetry.extra_metadata) - endpoints.append( - TelemetryEndpoint( - name=f"backend_{process.endpoint_mode}{process.endpoint_index}_rank{process.node_rank}", - url=f"http://{node_ip}:{port}/metrics", - frequency=telemetry.default_frequency, - filter="backend", - node_metadata=node_metadata, + if backend_exposes_metrics: + for process in sorted(processes, key=lambda p: (p.endpoint_mode, p.endpoint_index, p.node_rank, p.node)): + node_ip = get_hostname_ip(process.node, runtime.network_interface) + if frontend_type == "vllm" and process.endpoint_mode == "agg": + port = FRONTEND_PUBLIC_PORT + elif frontend_type == "vllm-router": + port = process.http_port + else: + port = process.sys_port + node_metadata = { + "hostname": process.node, + "worker_index": str(process.endpoint_index), + "worker_process": str(process.node_rank), + "worker_role": process.endpoint_mode, + } + node_metadata.update(telemetry.extra_metadata) + endpoints.append( + TelemetryEndpoint( + name=f"backend_{process.endpoint_mode}{process.endpoint_index}_rank{process.node_rank}", + url=f"http://{node_ip}:{port}/metrics", + frequency=telemetry.default_frequency, + filter="backend", + node_metadata=node_metadata, + ) ) - ) for frontend_index, node in enumerate(frontend_topology.frontend_nodes): node_ip = get_hostname_ip(node, runtime.network_interface) diff --git a/src/srtctl/frontends/__init__.py b/src/srtctl/frontends/__init__.py index c5b352b26..e0f182389 100644 --- a/src/srtctl/frontends/__init__.py +++ b/src/srtctl/frontends/__init__.py @@ -17,6 +17,7 @@ get_frontend, ) from srtctl.frontends.dynamo import DynamoFrontend +from srtctl.frontends.infera import InferaFrontend from srtctl.frontends.sglang import SGLangFrontend from srtctl.frontends.trtllm_serve import TRTLLMServeFrontend from srtctl.frontends.vllm import VLLMFrontend @@ -26,6 +27,7 @@ "DynamoFrontend", "FrontendProtocol", "FrontendType", + "InferaFrontend", "SGLangFrontend", "TRTLLMServeFrontend", "VLLMFrontend", diff --git a/src/srtctl/frontends/base.py b/src/srtctl/frontends/base.py index f2cb6b405..a62c4ab18 100644 --- a/src/srtctl/frontends/base.py +++ b/src/srtctl/frontends/base.py @@ -22,7 +22,7 @@ from srtctl.core.topology import Process # Supported frontend types - extensible by adding new literals -FrontendType = Literal["dynamo", "sglang", "trtllm_serve", "vllm", "vllm-router"] +FrontendType = Literal["atomesh", "dynamo", "infera", "sglang", "trtllm_serve", "vllm", "vllm-router"] FrontendFactory = Callable[[], "FrontendProtocol"] _FRONTEND_REGISTRY: dict[str, FrontendFactory] = {} @@ -44,7 +44,7 @@ def decorator(frontend_class: _FrontendClass) -> _FrontendClass: def _load_builtin_frontends() -> None: """Import built-ins once so their registration decorators run.""" - from srtctl.frontends import dynamo, sglang, trtllm_serve, vllm, vllm_router # noqa: F401 + from srtctl.frontends import dynamo, infera, sglang, trtllm_serve, vllm, vllm_router # noqa: F401 def build_setup_script_preamble(setup_script: str | None) -> str | None: diff --git a/src/srtctl/frontends/infera.py b/src/srtctl/frontends/infera.py new file mode 100644 index 000000000..b5cd697af --- /dev/null +++ b/src/srtctl/frontends/infera.py @@ -0,0 +1,181 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""ROCm Infera frontend, the open-source successor to ATOMesh.""" + +from __future__ import annotations + +import logging +import shlex +import threading +from typing import TYPE_CHECKING, Any + +from srtctl.core.health import WorkerHealthResult +from srtctl.core.slurm import start_srun_process +from srtctl.frontends.base import build_setup_script_preamble, register_frontend +from srtctl.ports import ETCD_CLIENT_PORT + +if TYPE_CHECKING: + from srtctl.core.processes import ManagedProcess + from srtctl.core.runtime import RuntimeContext + from srtctl.core.topology import Process + +logger = logging.getLogger(__name__) + + +def check_infera_workers( + response_json: dict[str, Any], + expected_prefill: int, + expected_decode: int, +) -> WorkerHealthResult: + """Validate Infera's live worker registry by serving role.""" + workers = response_json.get("workers") + if not isinstance(workers, list): + return WorkerHealthResult(ready=False, message=f"Key 'workers' not found in response: {response_json}") + + active = [worker for worker in workers if worker.get("status") == "active"] + prefill = sum(worker.get("disagg_mode") == "prefill" for worker in active) + decode = sum(worker.get("disagg_mode") == "decode" for worker in active) + mixed = sum(worker.get("disagg_mode") == "mixed" for worker in active) + effective_decode = decode + mixed + ready = prefill >= expected_prefill and effective_decode >= expected_decode + message = ( + f"Infera {'is ready' if ready else 'is not ready'}: " + f"{prefill}/{expected_prefill} prefill and {effective_decode}/{expected_decode} decode-equivalent " + f"workers ({mixed} mixed)." + ) + return WorkerHealthResult( + ready=ready, + message=message, + prefill_ready=prefill, + prefill_expected=expected_prefill, + decode_ready=effective_decode, + decode_expected=expected_decode, + ) + + +@register_frontend("infera", "atomesh") +class InferaFrontend: + """Infera dynamic-discovery router for ROCm ATOM workers. + + ``atomesh`` remains an accepted compatibility spelling for the earlier + preview name; both values launch the current ``infera.server`` module. + """ + + @property + def type(self) -> str: + return "infera" + + @property + def health_endpoint(self) -> str: + return "/v1/workers" + + def parse_health( + self, + response_json: dict[str, Any], + expected_prefill: int, + expected_decode: int, + ) -> WorkerHealthResult: + return check_infera_workers(response_json, expected_prefill, expected_decode) + + def get_backend_health_urls(self, backend: Any, backend_processes: list[Process]) -> list[str]: + """Workers register only after their native ATOM /health probe succeeds.""" + del backend, backend_processes + return [] + + def get_frontend_args_list(self, args: dict[str, Any] | None) -> list[str]: + if not args: + return [] + managed = { + "host", + "port", + "router-tokenizer-path", + "discovery-backend", + "etcd-endpoint", + "request-transport", + "kv-event-transport", + } + overlap = managed.intersection(key.lstrip("-").replace("_", "-") for key in args) + if overlap: + raise ValueError(f"Infera frontend args cannot override srtctl-managed value(s): {sorted(overlap)}") + result: list[str] = [] + for key, value in args.items(): + flag = key if key.startswith("-") else f"--{key}" + if value is True: + result.append(flag) + elif value is False or value is None: + continue + elif isinstance(value, list): + for item in value: + result.extend([flag, str(item)]) + else: + result.extend([flag, str(value)]) + return result + + def start_frontends( + self, + topology: Any, + runtime: RuntimeContext, + config: Any, + backend: Any, + backend_processes: list[Process], + stop_event: threading.Event | None = None, + ) -> list[ManagedProcess]: + del backend_processes, stop_event + from srtctl.core.processes import ManagedProcess + + if backend.type != "atom": + raise ValueError( + f"frontend.type: {config.frontend.type} currently requires backend.type: atom (got {backend.type!r})" + ) + + model_arg = str(runtime.model_path) if runtime.is_hf_model else "/model" + processes: list[ManagedProcess] = [] + for index, node in enumerate(topology.frontend_nodes): + log_file = runtime.log_dir / f"{node}_infera_{index}.out" + cmd = [ + "python3", + "-m", + "infera.server", + "--host", + "0.0.0.0", + "--port", + str(topology.frontend_port), + "--router-tokenizer-path", + model_arg, + "--discovery-backend", + "etcd", + "--etcd-endpoint", + f"{runtime.infra_node_ip}:{ETCD_CLIENT_PORT}", + "--request-transport", + "http", + "--kv-event-transport", + "zmq", + ] + cmd.extend(self.get_frontend_args_list(config.frontend.args)) + + env = dict(runtime.environment) + env.update(config.frontend.env or {}) + container_image = getattr(config.frontend, "container_image", None) or str(runtime.container_image) + logger.info("Starting Infera frontend %d on %s: %s", index, node, shlex.join(cmd)) + proc = start_srun_process( + command=cmd, + nodelist=[node], + output=str(log_file), + container_image=container_image, + container_mounts=runtime.container_mounts, + env_to_set=env or None, + bash_preamble=build_setup_script_preamble(getattr(config, "setup_script", None)), + het_group=runtime.nodes.het_group_for(node), + srun_options=runtime.srun_options, + ) + processes.append( + ManagedProcess( + name=f"infera_{index}", + popen=proc, + log_file=log_file, + node=node, + critical=True, + ) + ) + return processes diff --git a/tests/test_atom_infera.py b/tests/test_atom_infera.py new file mode 100644 index 000000000..0d9ee0d21 --- /dev/null +++ b/tests/test_atom_infera.py @@ -0,0 +1,201 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 SemiAnalysis LLC. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""High-signal contract tests for the ATOM backend and Infera frontend.""" + +import json +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest +from marshmallow import ValidationError + +from srtctl.backends import AtomProtocol, AtomServerConfig +from srtctl.cli.mixins.benchmark_stage import BenchmarkStageMixin +from srtctl.core.schema import SrtConfig +from srtctl.core.topology import Process +from srtctl.frontends import InferaFrontend, get_frontend + + +def _config(*, disaggregated: bool = False, frontend: str = "infera") -> dict: + resources = { + "gpu_type": "mi300x", + "gpus_per_node": 8, + "prefill_nodes": 1, + "decode_nodes": 1, + "prefill_workers": 1, + "decode_workers": 1, + } + if not disaggregated: + resources = { + "gpu_type": "mi300x", + "gpus_per_node": 8, + "agg_nodes": 1, + "agg_workers": 1, + } + return { + "name": "atom-infera", + "model": { + "path": "hf:Qwen/Qwen3-0.6B", + "container": "rocm/infera:atom-v0.1.1", + "precision": "bf16", + }, + "resources": resources, + "backend": { + "type": "atom", + "enable_kv_events": True, + "atom_config": {"aggregated": {"max-model-len": 4096}}, + }, + "frontend": {"type": frontend, "enable_multiple_frontends": False}, + } + + +def test_atom_infera_schema_roundtrip_and_legacy_atomesh_alias() -> None: + config = SrtConfig.Schema().load(_config(frontend="atomesh")) + + assert isinstance(config.backend, AtomProtocol) + assert isinstance(get_frontend("atomesh"), InferaFrontend) + assert config.served_model_name == "Qwen/Qwen3-0.6B" + assert SrtConfig.Schema().load(SrtConfig.Schema().dump(config)) == config + + +def test_atom_schema_rejects_cross_node_logical_worker() -> None: + raw = _config() + raw["resources"].update({"agg_nodes": 2, "agg_workers": 1}) + + with pytest.raises(ValidationError, match="fit on one Slurm node"): + SrtConfig.Schema().load(raw) + + +def test_atom_builds_managed_aggregate_worker_command() -> None: + backend = AtomProtocol( + enable_kv_events=True, + atom_config=AtomServerConfig(aggregated={"max-model-len": 4096, "trust_remote_code": True}), + ) + process = Process("node0", frozenset(range(8)), 7500, 6100, "agg", 0, nixl_port=5400) + runtime = SimpleNamespace( + worker_model_arg="/model", + infra_node_ip="10.0.0.10", + network_interface="hsn0", + ) + + with patch("srtctl.core.slurm.get_hostname_ip", return_value="10.0.0.20"): + command = backend.build_worker_command(process, [process], runtime) + + assert command[:3] == ["python3", "-m", "infera.engine.atom"] + assert command[command.index("--server-port") + 1] == "6100" + assert command[command.index("--port") + 1] == "7500" + assert command[command.index("-tp") + 1] == "8" + assert command[command.index("--etcd-endpoint") + 1] == "10.0.0.10:2379" + assert "--enable-kv-events" in command + assert "--kv-transfer-config" not in command + assert command[-3:] == ["--max-model-len", "4096", "--trust_remote_code"] + + +@pytest.mark.parametrize( + ("mode", "role"), + [("prefill", "kv_producer"), ("decode", "kv_consumer")], +) +def test_atom_builds_mooncake_pd_contract(mode: str, role: str) -> None: + backend = AtomProtocol(connector="mooncake", mooncake_ib_device="mlx5_0") + process = Process("node0", frozenset(range(4)), 7501, 6132, mode, 0, nixl_port=5407) + runtime = SimpleNamespace(worker_model_arg="/model", infra_node_ip="10.0.0.10", network_interface=None) + + with patch("srtctl.core.slurm.get_hostname_ip", return_value="10.0.0.20"): + command = backend.build_worker_command(process, [process], runtime) + + payload = json.loads(command[command.index("--kv-transfer-config") + 1]) + assert payload == { + "kv_role": role, + "kv_connector": "mooncake", + "proxy_ip": "10.0.0.20", + "http_port": 6132, + "handshake_port": 5407, + "ib_device": "mlx5_0", + } + assert backend.get_environment_for_mode(mode) == { + "MC_DISABLE_HIP_TRANSPORT": "1", + "MC_GID_INDEX": "1", + "RDMAV_FORK_SAFE": "1", + } + + +def test_infera_health_counts_active_roles_and_mixed_workers() -> None: + frontend = InferaFrontend() + result = frontend.parse_health( + { + "workers": [ + {"status": "active", "disagg_mode": "prefill"}, + {"status": "active", "disagg_mode": "decode"}, + {"status": "active", "disagg_mode": "mixed"}, + {"status": "inactive", "disagg_mode": "decode"}, + ] + }, + expected_prefill=1, + expected_decode=2, + ) + + assert result.ready + assert result.prefill_ready == 1 + assert result.decode_ready == 2 + + +def test_infera_launch_owns_discovery_and_transport_contract() -> None: + frontend = InferaFrontend() + runtime = SimpleNamespace( + log_dir=Path("/logs"), + model_path=Path("Qwen/Qwen3-0.6B"), + is_hf_model=True, + infra_node_ip="10.0.0.10", + environment={"GLOBAL": "value"}, + container_image=Path("rocm/infera:atom-v0.1.1"), + container_mounts={Path("/host"): Path("/container")}, + srun_options={"container-remap-root": True}, + nodes=SimpleNamespace(het_group_for=lambda _node: 1), + ) + config = SimpleNamespace( + frontend=SimpleNamespace( + type="infera", + args={"router-policy": "kv-aware"}, + env={"ROUTER_LOG": "debug"}, + container_image=None, + ), + setup_script="infera-setup.sh", + ) + topology = SimpleNamespace(frontend_nodes=["node0"], frontend_port=8000) + + with patch("srtctl.frontends.infera.start_srun_process", return_value=MagicMock()) as start: + processes = frontend.start_frontends(topology, runtime, config, AtomProtocol(), []) + + command = start.call_args.kwargs["command"] + assert command[:3] == ["python3", "-m", "infera.server"] + assert command[command.index("--discovery-backend") + 1] == "etcd" + assert command[command.index("--request-transport") + 1] == "http" + assert command[command.index("--kv-event-transport") + 1] == "zmq" + assert command[command.index("--router-policy") + 1] == "kv-aware" + assert start.call_args.kwargs["env_to_set"] == {"GLOBAL": "value", "ROUTER_LOG": "debug"} + assert processes[0].log_file == Path("/logs/node0_infera_0.out") + + +def test_infera_aiperf_metrics_use_router_not_atom_worker_ports() -> None: + class Stage(BenchmarkStageMixin): + @property + def backend_processes(self) -> list[Process]: + return self._processes + + def _orchestrator_node(self) -> str: + return "head" + + stage = Stage() + stage.config = SimpleNamespace( + frontend=SimpleNamespace(type="infera"), + backend=AtomProtocol(), + ) + stage.runtime = SimpleNamespace(network_interface="hsn0") + stage._processes = [Process("worker", frozenset(range(8)), 7500, 6100, "agg", 0)] + + with patch("srtctl.cli.mixins.benchmark_stage.get_hostname_ip", return_value="10.0.0.10"): + env = stage._get_aiperf_server_metrics_env() + + assert env == {"AIPERF_SERVER_METRICS_URLS": "http://10.0.0.10:8000/metrics"} From d93b48165ff60c6441feb5dd04504337f0bd7bc5 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 11 Aug 2026 10:13:41 -0500 Subject: [PATCH 41/46] fix(atom): pin Mooncake to advertised worker IP --- src/srtctl/backends/atom.py | 7 ++++++- tests/test_atom_infera.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/srtctl/backends/atom.py b/src/srtctl/backends/atom.py index b43bd58dc..6bfc5c8ed 100644 --- a/src/srtctl/backends/atom.py +++ b/src/srtctl/backends/atom.py @@ -215,7 +215,12 @@ def build_worker_command( if overlap: raise ValueError(f"ATOM config cannot override srtctl-managed argument(s): {sorted(overlap)}") - cmd: list[str] = list(nsys_prefix or []) + # Keep Mooncake's own interface selection aligned with the exact + # private-fabric address advertised through Infera and the transfer + # config. Without this override Mooncake may select a management NIC + # on multi-homed ROCm nodes even though proxy_ip is correct. + cmd: list[str] = ["env", f"ATOM_HOST_IP={worker_ip}"] + cmd.extend(nsys_prefix or []) cmd.extend( [ "python3", diff --git a/tests/test_atom_infera.py b/tests/test_atom_infera.py index 0d9ee0d21..627d18047 100644 --- a/tests/test_atom_infera.py +++ b/tests/test_atom_infera.py @@ -83,7 +83,7 @@ def test_atom_builds_managed_aggregate_worker_command() -> None: with patch("srtctl.core.slurm.get_hostname_ip", return_value="10.0.0.20"): command = backend.build_worker_command(process, [process], runtime) - assert command[:3] == ["python3", "-m", "infera.engine.atom"] + assert command[:5] == ["env", "ATOM_HOST_IP=10.0.0.20", "python3", "-m", "infera.engine.atom"] assert command[command.index("--server-port") + 1] == "6100" assert command[command.index("--port") + 1] == "7500" assert command[command.index("-tp") + 1] == "8" From 297da661ad058bb1ea4bad06be528ce4a0bbe9e2 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 11 Aug 2026 11:09:18 -0500 Subject: [PATCH 42/46] fix(infera): honor worker registry health endpoint --- src/srtctl/core/health.py | 4 ++-- tests/test_atom_infera.py | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/srtctl/core/health.py b/src/srtctl/core/health.py index 037711f4e..81d71afb6 100644 --- a/src/srtctl/core/health.py +++ b/src/srtctl/core/health.py @@ -512,13 +512,13 @@ def wait_for_model( frontend_type, ) else: - health_url = f"http://{host}:{port}/health" logger.info( - "Polling %s every %.1fs for %d prefills and %d decodes", + "Polling %s every %.1fs for %d prefills and %d decodes (%s frontend)", health_url, poll_interval, n_prefill, n_decode, + frontend_type, ) start_time = time.time() diff --git a/tests/test_atom_infera.py b/tests/test_atom_infera.py index 627d18047..35499ad18 100644 --- a/tests/test_atom_infera.py +++ b/tests/test_atom_infera.py @@ -13,6 +13,7 @@ from srtctl.backends import AtomProtocol, AtomServerConfig from srtctl.cli.mixins.benchmark_stage import BenchmarkStageMixin +from srtctl.core.health import wait_for_model from srtctl.core.schema import SrtConfig from srtctl.core.topology import Process from srtctl.frontends import InferaFrontend, get_frontend @@ -141,6 +142,30 @@ def test_infera_health_counts_active_roles_and_mixed_workers() -> None: assert result.decode_ready == 2 +def test_infera_wait_for_model_polls_worker_registry_endpoint() -> None: + response = MagicMock( + status_code=200, + json=lambda: { + "workers": [ + {"status": "active", "disagg_mode": "mixed"}, + {"status": "active", "disagg_mode": "mixed"}, + ] + }, + ) + + with patch("srtctl.core.health.requests.get", return_value=response) as get: + assert wait_for_model( + "router-host", + 8000, + n_prefill=0, + n_decode=2, + frontend_type="infera", + timeout=1, + ) + + get.assert_called_once_with("http://router-host:8000/v1/workers", timeout=5.0) + + def test_infera_launch_owns_discovery_and_transport_contract() -> None: frontend = InferaFrontend() runtime = SimpleNamespace( From f71cbb1cd7a4247ce50d54d84254c1842258b6e3 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 11 Aug 2026 12:21:32 -0500 Subject: [PATCH 43/46] fix(atom): pool Mooncake TCP fallback connections --- src/srtctl/backends/atom.py | 6 +++++- tests/test_atom_infera.py | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/srtctl/backends/atom.py b/src/srtctl/backends/atom.py index 6bfc5c8ed..52d2eb51f 100644 --- a/src/srtctl/backends/atom.py +++ b/src/srtctl/backends/atom.py @@ -90,9 +90,13 @@ def get_environment_for_mode(self, mode: WorkerMode) -> dict[str, str]: env = dict(configured) if mode in {"prefill", "decode"}: # Cross-node Mooncake must use the ROCm RDMA path, not host-local - # hipIPC. Recipes may override the GID index for their fabric. + # hipIPC. Keep TCP fallback usable as well: Mooncake otherwise + # opens one short-lived socket per transfer slice and can exhaust + # the initiator's ephemeral-port range under modest concurrency. + # Recipes may override either setting for their fabric/build. env.setdefault("MC_DISABLE_HIP_TRANSPORT", "1") env.setdefault("MC_GID_INDEX", "1") + env.setdefault("MC_TCP_ENABLE_CONNECTION_POOL", "1") env.setdefault("RDMAV_FORK_SAFE", "1") return env diff --git a/tests/test_atom_infera.py b/tests/test_atom_infera.py index 35499ad18..63e998012 100644 --- a/tests/test_atom_infera.py +++ b/tests/test_atom_infera.py @@ -118,6 +118,7 @@ def test_atom_builds_mooncake_pd_contract(mode: str, role: str) -> None: assert backend.get_environment_for_mode(mode) == { "MC_DISABLE_HIP_TRANSPORT": "1", "MC_GID_INDEX": "1", + "MC_TCP_ENABLE_CONNECTION_POOL": "1", "RDMAV_FORK_SAFE": "1", } From 83ef50ba6bf0bf4a91341d31fdcded141fdaa1b4 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 11 Aug 2026 12:36:48 -0500 Subject: [PATCH 44/46] feat(atom): make Mooncake transport explicit --- src/srtctl/backends/atom.py | 19 ++++++++++--------- tests/test_atom_infera.py | 28 +++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/srtctl/backends/atom.py b/src/srtctl/backends/atom.py index 52d2eb51f..deeddaef6 100644 --- a/src/srtctl/backends/atom.py +++ b/src/srtctl/backends/atom.py @@ -61,6 +61,7 @@ class AtomProtocol: # Infera's released ATOM P/D path supports Mooncake. Each realized worker # gets a topology-derived, collision-free handshake port. connector: str = "mooncake" + mooncake_protocol: Literal["rdma", "tcp"] = "rdma" mooncake_ib_device: str | None = None Schema: ClassVar[builtins.type[Schema]] = Schema @@ -89,15 +90,14 @@ def get_environment_for_mode(self, mode: WorkerMode) -> dict[str, str]: }.get(mode, {}) env = dict(configured) if mode in {"prefill", "decode"}: - # Cross-node Mooncake must use the ROCm RDMA path, not host-local - # hipIPC. Keep TCP fallback usable as well: Mooncake otherwise - # opens one short-lived socket per transfer slice and can exhaust - # the initiator's ephemeral-port range under modest concurrency. - # Recipes may override either setting for their fabric/build. - env.setdefault("MC_DISABLE_HIP_TRANSPORT", "1") - env.setdefault("MC_GID_INDEX", "1") + # Mooncake otherwise opens one short-lived socket per transfer + # slice in TCP mode and can exhaust the initiator's ephemeral-port + # range under modest concurrency. Recipes may override this for + # builds that do not support connection pooling. env.setdefault("MC_TCP_ENABLE_CONNECTION_POOL", "1") - env.setdefault("RDMAV_FORK_SAFE", "1") + if self.mooncake_protocol == "rdma": + env.setdefault("MC_GID_INDEX", "1") + env.setdefault("RDMAV_FORK_SAFE", "1") return env def get_process_environment(self, process: Process) -> dict[str, str]: @@ -175,11 +175,12 @@ def _build_kv_transfer_config(self, process: Process, worker_ip: str) -> str: payload: dict[str, Any] = { "kv_role": "kv_producer" if mode == "prefill" else "kv_consumer", "kv_connector": "mooncake", + "protocol": self.mooncake_protocol, "proxy_ip": worker_ip, "http_port": process.http_port, "handshake_port": process.nixl_port, } - if self.mooncake_ib_device: + if self.mooncake_protocol == "rdma" and self.mooncake_ib_device: payload["ib_device"] = self.mooncake_ib_device return json.dumps(payload, separators=(",", ":")) diff --git a/tests/test_atom_infera.py b/tests/test_atom_infera.py index 63e998012..5a5c8ff3b 100644 --- a/tests/test_atom_infera.py +++ b/tests/test_atom_infera.py @@ -110,19 +110,45 @@ def test_atom_builds_mooncake_pd_contract(mode: str, role: str) -> None: assert payload == { "kv_role": role, "kv_connector": "mooncake", + "protocol": "rdma", "proxy_ip": "10.0.0.20", "http_port": 6132, "handshake_port": 5407, "ib_device": "mlx5_0", } assert backend.get_environment_for_mode(mode) == { - "MC_DISABLE_HIP_TRANSPORT": "1", "MC_GID_INDEX": "1", "MC_TCP_ENABLE_CONNECTION_POOL": "1", "RDMAV_FORK_SAFE": "1", } +def test_atom_builds_explicit_mooncake_tcp_contract() -> None: + backend = AtomProtocol( + connector="mooncake", + mooncake_protocol="tcp", + mooncake_ib_device="mlx5_0", + ) + process = Process("node0", frozenset({0}), 7501, 6132, "prefill", 0, nixl_port=5407) + runtime = SimpleNamespace(worker_model_arg="/model", infra_node_ip="10.0.0.10", network_interface=None) + + with patch("srtctl.core.slurm.get_hostname_ip", return_value="10.0.0.20"): + command = backend.build_worker_command(process, [process], runtime) + + payload = json.loads(command[command.index("--kv-transfer-config") + 1]) + assert payload == { + "kv_role": "kv_producer", + "kv_connector": "mooncake", + "protocol": "tcp", + "proxy_ip": "10.0.0.20", + "http_port": 6132, + "handshake_port": 5407, + } + assert backend.get_environment_for_mode("prefill") == { + "MC_TCP_ENABLE_CONNECTION_POOL": "1", + } + + def test_infera_health_counts_active_roles_and_mixed_workers() -> None: frontend = InferaFrontend() result = frontend.parse_health( From 141f035b5539fa8bbc1b4018ae4817283093092d Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 11 Aug 2026 12:56:22 -0500 Subject: [PATCH 45/46] fix(fingerprint): detect ATOM and Infera versions --- src/srtctl/core/fingerprint.py | 2 ++ tests/test_fingerprint.py | 11 +++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/srtctl/core/fingerprint.py b/src/srtctl/core/fingerprint.py index 2fa0feecc..96c15b4a1 100644 --- a/src/srtctl/core/fingerprint.py +++ b/src/srtctl/core/fingerprint.py @@ -45,6 +45,8 @@ # Framework name -> pip package name mapping. # Used in both native Python probes and the bash capture script. FRAMEWORK_PACKAGES: dict[str, str] = { + "atom": "atom", + "infera": "amd-infera", "vllm": "vllm", "sglang": "sglang", "sglang-router": "sglang-router", diff --git a/tests/test_fingerprint.py b/tests/test_fingerprint.py index 198c37375..f9f16bf9e 100644 --- a/tests/test_fingerprint.py +++ b/tests/test_fingerprint.py @@ -166,8 +166,13 @@ def test_probe_result_failure(self): assert r.value == UNAVAILABLE assert r.error == "broken" - def test_probe_frameworks_includes_native_router_and_amd_transport(self, monkeypatch): - versions = {"sglang-router": "0.3.2", "amd_mori": "0.5.16.dev0"} + def test_probe_frameworks_includes_router_and_amd_frameworks(self, monkeypatch): + versions = { + "atom": "0.1.4.dev113+g5837907f3", + "amd-infera": "0.0.0", + "sglang-router": "0.3.2", + "amd_mori": "0.5.16.dev0", + } def fake_run(command: str): return next((version for package, version in versions.items() if f"'{package}'" in command), None) @@ -177,6 +182,8 @@ def fake_run(command: str): result = probe_frameworks() assert result.ok is True + assert result.value["atom"] == "0.1.4.dev113+g5837907f3" + assert result.value["infera"] == "0.0.0" assert result.value["sglang-router"] == "0.3.2" assert result.value["amd-mori"] == "0.5.16.dev0" From 5ecfb13d1ba0960045482f1ef006312d8729d37a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 11 Aug 2026 13:36:58 -0500 Subject: [PATCH 46/46] fix: force Mooncake TCP transport for ATOM --- src/srtctl/backends/atom.py | 6 ++++++ tests/test_atom_infera.py | 1 + 2 files changed, 7 insertions(+) diff --git a/src/srtctl/backends/atom.py b/src/srtctl/backends/atom.py index deeddaef6..db9aea7de 100644 --- a/src/srtctl/backends/atom.py +++ b/src/srtctl/backends/atom.py @@ -98,6 +98,12 @@ def get_environment_for_mode(self, mode: WorkerMode) -> dict[str, str]: if self.mooncake_protocol == "rdma": env.setdefault("MC_GID_INDEX", "1") env.setdefault("RDMAV_FORK_SAFE", "1") + else: + # Mooncake's legacy TransferEngine auto-discovers installed + # HCAs even when ATOM's transfer config says ``tcp``. Force + # the selected transport so TCP recipes cannot silently take + # the RDMA path and emit write-done after a failed KV copy. + env.setdefault("MC_FORCE_TCP", "true") return env def get_process_environment(self, process: Process) -> dict[str, str]: diff --git a/tests/test_atom_infera.py b/tests/test_atom_infera.py index 5a5c8ff3b..545334e76 100644 --- a/tests/test_atom_infera.py +++ b/tests/test_atom_infera.py @@ -145,6 +145,7 @@ def test_atom_builds_explicit_mooncake_tcp_contract() -> None: "handshake_port": 5407, } assert backend.get_environment_for_mode("prefill") == { + "MC_FORCE_TCP": "true", "MC_TCP_ENABLE_CONNECTION_POOL": "1", }