[TRTLLM-17269][refactor] Add per-dialect error renderers in serve/exception_handling

.pre-commit-config.yaml

@@ -417,11 +417,15 @@ common-files: &common_files |
         tensorrt_llm/scaffolding/worker.py |
         tensorrt_llm/scheduling_params.py |
         tensorrt_llm/serve/__init__.py |
+        tensorrt_llm/serve/_base/msgspec_route.py |
+        tensorrt_llm/serve/_base/sse.py |
         tensorrt_llm/serve/chat_utils.py |
         tensorrt_llm/serve/cluster_storage.py |
         tensorrt_llm/serve/disagg_auto_scaling.py |
         tensorrt_llm/serve/harmony_adapter.py |
         tensorrt_llm/serve/metadata_server.py |
+        tensorrt_llm/serve/middleware/arrival_time.py |
+        tensorrt_llm/serve/middleware/user_middleware.py |
         tensorrt_llm/serve/openai_disagg_server.py |
         tensorrt_llm/serve/openai_protocol.py |
         tensorrt_llm/serve/openai_server.py |
@@ -1144,11 +1148,15 @@ legacy-files: &legacy_files |
         tensorrt_llm/scaffolding/worker.py |
         tensorrt_llm/scheduling_params.py |
         tensorrt_llm/serve/__init__.py |
+        tensorrt_llm/serve/_base/msgspec_route.py |
+        tensorrt_llm/serve/_base/sse.py |
         tensorrt_llm/serve/chat_utils.py |
         tensorrt_llm/serve/cluster_storage.py |
         tensorrt_llm/serve/disagg_auto_scaling.py |
         tensorrt_llm/serve/harmony_adapter.py |
         tensorrt_llm/serve/metadata_server.py |
+        tensorrt_llm/serve/middleware/arrival_time.py |
+        tensorrt_llm/serve/middleware/user_middleware.py |
         tensorrt_llm/serve/openai_disagg_server.py |
         tensorrt_llm/serve/openai_protocol.py |
         tensorrt_llm/serve/openai_server.py |
@@ -1620,6 +1628,7 @@ repos:
         # Run mypy with no explicit paths so it type-checks the set declared in
         # pyproject.toml [tool.mypy] `files` (single source of truth). This
         # `files:` pattern only decides when the hook triggers — any change under
-        # the sampler source/test dirs.
+        # the sampler source/test dirs or the serving frontends. A PR that adds
+        # a typed module elsewhere extends this pattern in the same PR.
         pass_filenames: false
-        files: ^(tensorrt_llm/_torch/pyexecutor/sampler/|tests/unittest/_torch/sampler/)
+        files: ^(tensorrt_llm/_torch/pyexecutor/sampler/|tests/unittest/_torch/sampler/|tensorrt_llm/serve/|tensorrt_llm/commands/serve_cli/)

jenkins/L0_MergeRequest.groovy

@@ -1948,10 +1948,17 @@ def getMultiGpuFileChanged(pipeline, testFilter, globalVars)
         "tensorrt_llm/models/llama/",
         "tensorrt_llm/parameter.py",
         "tensorrt_llm/serve/__init__.py",
+        // _MsgspecRoute decodes the disaggregated frontend's msgpack bodies.
+        "tensorrt_llm/serve/_base/",
+        // /health is the disaggregated readiness probe.
+        "tensorrt_llm/serve/api/ops/",
         "tensorrt_llm/serve/cluster_storage.py",
         "tensorrt_llm/serve/disagg_auto_scaling.py",
+        // The error renderers both apps install; the disaggregated frontend runs them.
+        "tensorrt_llm/serve/exception_handling/",
         "tensorrt_llm/serve/extensions/",
         "tensorrt_llm/serve/metadata_server.py",
+        "tensorrt_llm/serve/middleware/",
         "tensorrt_llm/serve/openai_client.py",
         "tensorrt_llm/serve/openai_disagg_server.py",
         "tensorrt_llm/serve/openai_disagg_service.py",

legacy-files.txt

@@ -409,11 +409,15 @@ tensorrt_llm/scaffolding/task_collection.py
 tensorrt_llm/scaffolding/worker.py
 tensorrt_llm/scheduling_params.py
 tensorrt_llm/serve/__init__.py
+tensorrt_llm/serve/_base/msgspec_route.py
+tensorrt_llm/serve/_base/sse.py
 tensorrt_llm/serve/chat_utils.py
 tensorrt_llm/serve/cluster_storage.py
 tensorrt_llm/serve/disagg_auto_scaling.py
 tensorrt_llm/serve/harmony_adapter.py
 tensorrt_llm/serve/metadata_server.py
+tensorrt_llm/serve/middleware/arrival_time.py
+tensorrt_llm/serve/middleware/user_middleware.py
 tensorrt_llm/serve/openai_disagg_server.py
 tensorrt_llm/serve/openai_protocol.py
 tensorrt_llm/serve/openai_server.py

pyproject.toml

@@ -474,11 +474,15 @@ exclude = [
     "tensorrt_llm/scaffolding/worker.py",
     "tensorrt_llm/scheduling_params.py",
     "tensorrt_llm/serve/__init__.py",
+    "tensorrt_llm/serve/_base/msgspec_route.py",
+    "tensorrt_llm/serve/_base/sse.py",
     "tensorrt_llm/serve/chat_utils.py",
     "tensorrt_llm/serve/cluster_storage.py",
     "tensorrt_llm/serve/disagg_auto_scaling.py",
     "tensorrt_llm/serve/harmony_adapter.py",
     "tensorrt_llm/serve/metadata_server.py",
+    "tensorrt_llm/serve/middleware/arrival_time.py",
+    "tensorrt_llm/serve/middleware/user_middleware.py",
     "tensorrt_llm/serve/openai_disagg_server.py",
     "tensorrt_llm/serve/openai_protocol.py",
     "tensorrt_llm/serve/openai_server.py",
@@ -862,6 +866,13 @@ files = [
     "tensorrt_llm/_torch/pyexecutor/sampler/token_ban.py",
     "tensorrt_llm/_torch/pyexecutor/sampler/ops/flashinfer.py",
     "tensorrt_llm/_torch/pyexecutor/sampler/ops/vanilla.py",
+    "tensorrt_llm/serve/_base/app_builder.py",
+    "tensorrt_llm/serve/_base/errors.py",
+    "tensorrt_llm/serve/_base/routing.py",
+    "tensorrt_llm/serve/api/ops/system/api_router.py",
+    "tensorrt_llm/serve/api/ops/system/serving.py",
+    "tensorrt_llm/serve/exception_handling/register.py",
+    "tensorrt_llm/serve/middleware/register.py",
     "tests/unittest/_torch/sampler/test_torch_sampler.py",
     "tests/unittest/_torch/sampler/test_token_ban.py",
     "tests/unittest/_torch/sampler/test_beam_search.py",

ruff-legacy-baseline.json

@@ -528,6 +528,9 @@
   "tensorrt_llm/scaffolding/task.py": {
     "F821": 1
   },
+  "tensorrt_llm/serve/_base/sse.py": {
+    "F821": 1
+  },
   "tensorrt_llm/serve/disagg_auto_scaling.py": {
     "D205": 2,
     "D212": 2
@@ -539,6 +542,9 @@
     "D212": 19,
     "D415": 1
   },
+  "tensorrt_llm/serve/middleware/arrival_time.py": {
+    "D212": 1
+  },
   "tensorrt_llm/serve/openai_disagg_server.py": {
     "D210": 1,
     "F811": 1

ruff-legacy.toml

@@ -426,11 +426,15 @@ include = [
     "tensorrt_llm/scaffolding/worker.py",
     "tensorrt_llm/scheduling_params.py",
     "tensorrt_llm/serve/__init__.py",
+    "tensorrt_llm/serve/_base/msgspec_route.py",
+    "tensorrt_llm/serve/_base/sse.py",
     "tensorrt_llm/serve/chat_utils.py",
     "tensorrt_llm/serve/cluster_storage.py",
     "tensorrt_llm/serve/disagg_auto_scaling.py",
     "tensorrt_llm/serve/harmony_adapter.py",
     "tensorrt_llm/serve/metadata_server.py",
+    "tensorrt_llm/serve/middleware/arrival_time.py",
+    "tensorrt_llm/serve/middleware/user_middleware.py",
     "tensorrt_llm/serve/openai_disagg_server.py",
     "tensorrt_llm/serve/openai_protocol.py",
     "tensorrt_llm/serve/openai_server.py",

tensorrt_llm/commands/AGENTS.md

@@ -0,0 +1,6 @@
+<!--
+SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+-->
+
+Rules for the trtllm-serve CLI (`commands/serve.py`) are in `tensorrt_llm/serve/AGENTS.md`.

tensorrt_llm/commands/CLAUDE.md

@@ -0,0 +1,2 @@
+# In ./CLAUDE.md
+@AGENTS.md

tensorrt_llm/commands/serve.py

@@ -16,8 +16,6 @@
 import atexit
 import contextlib
 import gc
-import importlib
-import inspect
 import json
 import os
 import secrets
@@ -74,6 +72,8 @@
     _frontend_affinity_sockets, _init_multi_frontend_mode,
     _signal_frontend_ready, _spawn_attached_frontends,
     _terminate_attached_frontends, _watch_attached_frontends)
+from tensorrt_llm.serve.middleware.user_middleware import \
+    _apply_fastapi_middlewares
 from tensorrt_llm.serve.multi_frontend import (LAUNCHER_UDS_ENV,
                                                LAUNCHER_UDS_NAME,
                                                MultiFrontendServing)
@@ -158,25 +158,6 @@ def _pop_bool_config_option(config: dict[str, Any], key: str) -> bool:
     return validate_config_bool(config.pop(key, False), key)
 
 
-def _apply_fastapi_middlewares(app, middlewares: Sequence[str]) -> None:
-    """Import and register middleware objects on a FastAPI app."""
-    for middleware in middlewares:
-        try:
-            module_path, object_name = middleware.rsplit(".", 1)
-        except ValueError as e:
-            raise ValueError(f"Invalid middleware import path '{middleware}'. "
-                             "Expected format: <module>.<object>.") from e
-
-        imported = getattr(importlib.import_module(module_path), object_name)
-        if inspect.isclass(imported):
-            app.add_middleware(imported)
-        elif inspect.iscoroutinefunction(imported):
-            app.middleware("http")(imported)
-        else:
-            raise ValueError(f"Invalid middleware {middleware}. "
-                             "Must be a class or an async function.")
-
-
 def is_non_default_or_required(param_name, value, explicit_cli_keys):
     """
     Check if a parameter should be explicitly included in llm_args.

tensorrt_llm/serve/AGENTS.md

@@ -0,0 +1,98 @@
+<!--
+SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+-->
+
+# AGENTS.md - tensorrt_llm/serve
+
+Scope: `tensorrt_llm/serve/` and everything under it. Supplements the repo-root `AGENTS.md`
+and `tests/AGENTS.md`; `tensorrt_llm/commands/AGENTS.md` points here for the serve CLI.
+Where a check enforces a rule, the rule names it; a failing check says what to change.
+Name files and symbols here, never line numbers.
+
+## Where code goes
+
+- New endpoint: a router and a serving class under `serve/api/<dialect>/<endpoint>/`
+  (`api_router.py` + `serving.py`). Create the router with `make_router()`
+  (`serve/_base/routing.py`) and add a `RouterSpec` to `ROUTERS`. Never add a method or a
+  route to `OpenAIServer`. Check: `tests/unittest/api_stability/test_serve_routes.py`.
+- Attach a router with `app.include_router(router)` and no `prefix=`; give the prefix to
+  `make_router(prefix=...)` instead. The composition root sets each serving object on
+  `app.state.<bucket>_serving_<endpoint>` before attaching; routers read it from
+  `request.app.state`, never from a closure.
+- Model-specific behavior: a hook on `ServingExtension` (`serve/serving_extensions.py`,
+  `serve/extensions/`). Do not branch on a model name in a handler.
+- Prompt rendering: `serve/render/` only. Handlers do not call `apply_chat_template`.
+- Request and response models: a new field needs a row in
+  `tests/unittest/api_stability/references/trtllm_serve_api.yaml`.
+  Check: `tests/unittest/api_stability/test_serve_api.py`.
+- Routes: every server configuration's routes (path, method, route class, and the order of
+  routes that can match the same request) are pinned in
+  `tests/unittest/api_stability/references/trtllm_serve_routes.yaml`. A route change
+  regenerates it and needs the API review committee.
+  Check: `tests/unittest/api_stability/test_serve_routes.py`.
+- Middleware: register built-in middleware in `serve/middleware/register.py`, which both apps
+  call. Order matters (`add_middleware` inserts at the front, so the last one added is
+  outermost); do not reorder. Check: `tests/unittest/serve/test_middleware_order.py`.
+- Import directions between layers are tests; a PR that creates a package adds its contract.
+  Check: `tests/unittest/serve/test_layering.py`.
+- VisualGen serve tests live under a path that contains `visual_gen/`
+  (`tests/unittest/serve/api/visual_gen/`), or CI selection misses them.
+
+## Moving code
+
+- A move PR only moves code: no reformatting, no logic change.
+- A move PR is redone on the latest `main` right before merge instead of being rebased,
+  and its PR body lists every moved definition with its old and new location, and every
+  definition the PR changes on purpose.
+- In the same PR update the multi-GPU file list (`getMultiGpuFileChanged` in
+  `jenkins/L0_MergeRequest.groovy`), `.github/CODEOWNERS` and `legacy-files.txt`. A new file
+  takes the formatting group of the file its code comes from.
+- Old import paths: a move deletes the old module (or the moved names) without a shim and
+  rewrites every import of it in the repository, tests included; module paths written as
+  strings (patch targets, subprocess command lines, `_LAZY_ATTRS` in `serve/__init__.py`) are
+  fixed by hand. The PR body lists "old path → new path" for the release notes. Only five old
+  paths keep a shim, because code outside the repository imports them:
+  `tensorrt_llm/llmapi/reasoning_parser.py`, `serve/tool_parser/`,
+  `serve/openai_disagg_server.py`, `serve/openai_server.py` and `commands/serve.py`. A shim
+  re-exports public names only, lists them in `__all__` (or `_LAZY_NAMES` in
+  `commands/serve.py`), warns once with `FutureWarning`, and has its rows in
+  `tests/unittest/serve/test_shim_identity.py`.
+- `mock.patch("old.module.name")` does not follow moved code. Grep the tests for every moved
+  name, including `monkeypatch.setattr(module, "name", ...)` and dotted paths split across lines.
+
+## Tests
+
+- Register every new test file in a CI list in the same PR (`tests/AGENTS.md`). Serve tests
+  that need no GPU go in `tests/integration/test_lists/test-db/l0_cpu.yml` and must contain
+  `pytest.mark.cpu_only`, or the CPU stage collects nothing from them.
+- New serve tests mirror the package: `tests/unittest/serve/` for `tensorrt_llm/serve/`.
+- Build servers with the factory and fake generators in `tests/test_common/serve/`
+  (`isolated_serve_env`), never `object.__new__(OpenAIServer)` or
+  `object.__new__(OpenAIDisaggServer)`. Check: `tests/unittest/serve/test_no_object_new.py`.
+- List an app's routes with `effective_routes()` from `tests/test_common/serve/routes.py`, not
+  `app.routes`: FastAPI 0.139 keeps included routers as objects without `.path`.
+- The harmony tests guard whole files with try-imports. After renaming anything they import,
+  run them with `pytest -rs` and check that nothing is skipped as "not importable": a green run
+  can mean every test was skipped.
+
+## Checklists
+
+- Endpoint: package under `serve/api/`; `RouterSpec` in `ROUTERS`; regenerate the route
+  snapshot (API review); multi-GPU list entry if it runs on the disaggregated path; CPU test
+  registered in `tests/integration/test_lists/test-db/l0_cpu.yml`.
+
+## Commands
+
+- All serve CPU tests (the runner exits when `LLM_MODELS_ROOT` is unset):
+  `LLM_MODELS_ROOT=/path/to/models PYTHONPATH=tests python3 -m test_common.serve.cpu_tests`
+- Route snapshot: `pytest -m cpu_only tests/unittest/api_stability/test_serve_routes.py tests/unittest/serve/api/visual_gen/test_routes_visual_gen.py`
+- Regenerate the route snapshot: `PYTHONPATH=tests python3 -m test_common.serve.routes --write`
+- API gates: `pytest tests/unittest/api_stability/test_serve_api.py tests/unittest/api_stability/test_serve_cli.py`
+- Extensions and parsers: `pytest tests/unittest/llmapi/apps/test_serving_extensions.py tests/unittest/llmapi/apps/test_tool_parsers.py`
+- Disaggregated router and coordinator (CPU): `pytest tests/unittest/disaggregated -m cpu_only -q`
+- Which CI stage runs a test: `python scripts/test_to_stage_mapping.py --tests "<test>"`. It
+  matches the text of list entries and does not expand directories: a file listed through its
+  directory maps only when you pass the directory entry, e.g. `--tests "unittest/serve"` for
+  anything under `tests/unittest/serve/`.
+- Tests that load model files need `LLM_MODELS_ROOT` (the harmony tests' autouse fixture does).

tensorrt_llm/serve/CLAUDE.md

@@ -0,0 +1,2 @@
+# In ./CLAUDE.md
+@AGENTS.md

tensorrt_llm/serve/_base/__init__.py

@@ -0,0 +1,15 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Shared serving infrastructure with no dependency on endpoint packages."""