diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ae107c9c1..c346f6232 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -413,7 +413,6 @@ jobs: nemo-relay nemo-relay-adaptive nemo-relay-pii-redaction - nemo-relay-switchyard nemo-relay-ffi nemo-relay-cli ) diff --git a/ATTRIBUTIONS-Rust.md b/ATTRIBUTIONS-Rust.md index 5a85548e5..277c9b179 100644 --- a/ATTRIBUTIONS-Rust.md +++ b/ATTRIBUTIONS-Rust.md @@ -45181,428 +45181,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ``` -## switchyard-protocol - 0.1.0 -**Repository URL**: https://github.com/NVIDIA-NeMo/Switchyard -**License Type(s)**: Apache-2.0 -### License: https://spdx.org/licenses/Apache-2.0.html -``` -Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -``` - -## switchyard-translation - 0.1.0 -**Repository URL**: https://github.com/NVIDIA-NeMo/Switchyard -**License Type(s)**: Apache-2.0 -### License: https://spdx.org/licenses/Apache-2.0.html -``` -Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -``` - ## syn - 2.0.117 **Repository URL**: https://github.com/dtolnay/syn **License Type(s)**: Apache-2.0 diff --git a/Cargo.lock b/Cargo.lock index 11598441b..11bfa17b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1789,7 +1789,6 @@ dependencies = [ "nemo-relay", "nemo-relay-adaptive", "nemo-relay-pii-redaction", - "nemo-relay-switchyard", "opentelemetry", "opentelemetry_sdk", "percent-encoding", @@ -1903,23 +1902,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "nemo-relay-switchyard" -version = "0.8.0" -dependencies = [ - "async-stream", - "axum", - "futures-util", - "nemo-relay", - "reqwest 0.12.28", - "schemars", - "serde", - "serde_json", - "switchyard-translation", - "tokio", - "uuid", -] - [[package]] name = "nemo-relay-types" version = "0.8.0" @@ -3498,28 +3480,6 @@ dependencies = [ "sval_nested", ] -[[package]] -name = "switchyard-protocol" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c80192a3665425e92b4e10df3bff748cc60180cd028ba207bd70fb95df6869d5" -dependencies = [ - "serde", - "serde_json", -] - -[[package]] -name = "switchyard-translation" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c977a683d8921e1ee7f01373162e14cbbfac28ef2a87dde8441c1691b2fed9" -dependencies = [ - "serde", - "serde_json", - "switchyard-protocol", - "thiserror 2.0.18", -] - [[package]] name = "syn" version = "2.0.117" diff --git a/Cargo.toml b/Cargo.toml index fa4fa8ada..edd9d06bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,6 @@ members = [ "crates/worker", "crates/adaptive", "crates/pii-redaction", - "crates/switchyard", "crates/cli", # Language Bindings "crates/python", @@ -34,8 +33,6 @@ nemo-relay-worker-proto = { version = "0.8.0", path = "crates/worker-proto" } nemo-relay-worker = { version = "0.8.0", path = "crates/worker" } nemo-relay-adaptive = { version = "0.8.0", path = "crates/adaptive" } nemo-relay-pii-redaction = { version = "0.8.0", path = "crates/pii-redaction" } -nemo-relay-switchyard = { version = "0.8.0", path = "crates/switchyard" } -switchyard-translation = "0.1.0" nemo-relay-ffi = { version = "0.8.0", path = "crates/ffi" } nemo-relay-cli = { version = "0.8.0", path = "crates/cli" } opentelemetry = { version = "0.32", default-features = false } diff --git a/RELEASING.md b/RELEASING.md index cf6d058cb..a53e5bdb7 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -30,7 +30,7 @@ The release pipeline publishes these package surfaces from a tag push: | Ecosystem | Published Surface | |---|---| -| crates.io | `nemo-relay-types`, `nemo-relay-plugin`, `nemo-relay-worker-proto`, `nemo-relay-worker`, `nemo-relay`, `nemo-relay-adaptive`, `nemo-relay-pii-redaction`, `nemo-relay-switchyard`, `nemo-relay-ffi`, `nemo-relay-cli` | +| crates.io | `nemo-relay-types`, `nemo-relay-plugin`, `nemo-relay-worker-proto`, `nemo-relay-worker`, `nemo-relay`, `nemo-relay-adaptive`, `nemo-relay-pii-redaction`, `nemo-relay-ffi`, `nemo-relay-cli` | | PyPI | `nemo-relay` wheels and source distribution, `nemo-relay-plugin` and `nemo-relay-cli-bin` wheels | | npm | `nemo-relay-node` and its seven platform packages, and `nemo-relay-openclaw` | | GitHub Releases | CLI binaries, `nemo-relay` and `nemo-relay-cli-bin` wheels, Node npm tarballs, and checksums | @@ -56,8 +56,7 @@ NeMo Relay versions are anchored on the workspace SemVer in the repository root - The root `Cargo.toml` `workspace.dependencies` entries for `nemo-relay-types`, `nemo-relay-plugin`, `nemo-relay-worker-proto`, `nemo-relay-worker`, `nemo-relay`, `nemo-relay-adaptive`, - `nemo-relay-pii-redaction`, `nemo-relay-switchyard`, `nemo-relay-ffi`, and - `nemo-relay-cli` must + `nemo-relay-pii-redaction`, `nemo-relay-ffi`, and `nemo-relay-cli` must stay aligned with that same version. - `crates/node/package.json` carries the base npm version for the Node.js package. The repository-root `package-lock.json` carries the npm workspace @@ -141,7 +140,7 @@ Before you create a release tag, confirm the following: - crates.io trusted publishers for `nemo-relay-types`, `nemo-relay-plugin`, `nemo-relay-worker-proto`, `nemo-relay-worker`, `nemo-relay`, `nemo-relay-adaptive`, `nemo-relay-pii-redaction`, - `nemo-relay-switchyard`, `nemo-relay-ffi`, and `nemo-relay-cli` are + `nemo-relay-ffi`, and `nemo-relay-cli` are configured for the top-level [`.github/workflows/ci.yaml`](.github/workflows/ci.yaml) workflow - GitHub Actions `id-token: write` access is available for the top-level npm publish job @@ -171,8 +170,7 @@ The helper updates: 2. The root [`Cargo.toml`](Cargo.toml) `workspace.dependencies` versions for `nemo-relay-types`, `nemo-relay-plugin`, `nemo-relay-worker-proto`, `nemo-relay-worker`, `nemo-relay`, `nemo-relay-adaptive`, - `nemo-relay-pii-redaction`, `nemo-relay-switchyard`, `nemo-relay-ffi`, and - `nemo-relay-cli`. + `nemo-relay-pii-redaction`, `nemo-relay-ffi`, and `nemo-relay-cli`. 3. [`crates/node/package.json`](crates/node/package.json) and the `crates/node` entry in the root [`package-lock.json`](package-lock.json) to the same release version. @@ -280,7 +278,7 @@ The release pipeline then: runs `cargo publish --package` for `nemo-relay-types`, `nemo-relay-plugin`, `nemo-relay-worker-proto`, `nemo-relay-worker`, `nemo-relay`, `nemo-relay-adaptive`, `nemo-relay-pii-redaction`, - `nemo-relay-switchyard`, `nemo-relay-ffi`, and `nemo-relay-cli` through + `nemo-relay-ffi`, and `nemo-relay-cli` through trusted publishing from the top-level workflow - `publish-python` downloads the `nemo-relay` wheel and source distribution @@ -351,8 +349,7 @@ After the release is live, verify: 1. The `nemo-relay-types`, `nemo-relay-plugin`, `nemo-relay-worker-proto`, `nemo-relay-worker`, `nemo-relay`, `nemo-relay-adaptive`, - `nemo-relay-pii-redaction`, `nemo-relay-switchyard`, `nemo-relay-ffi`, and - `nemo-relay-cli` crates + `nemo-relay-pii-redaction`, `nemo-relay-ffi`, and `nemo-relay-cli` crates are visible on crates.io. 2. The `nemo-relay` and `nemo-relay-cli-bin` wheels are visible on PyPI, and `pip install "nemo-relay[cli]"` exposes `nemo-relay`. diff --git a/crates/adaptive/src/response_cache/key.rs b/crates/adaptive/src/response_cache/key.rs index 4b6ee6a5a..595992a79 100644 --- a/crates/adaptive/src/response_cache/key.rs +++ b/crates/adaptive/src/response_cache/key.rs @@ -29,7 +29,7 @@ use crate::response_cache::store::CACHE_SCHEMA_VERSION; /// dropped before fingerprinting (IDs, routing, bookkeeping, streaming flag). pub const DEFAULT_SKIP_KEYS: &[&str] = &["stream", "user", "metadata", "store"]; -/// Relay-owned Switchyard backend partition. It is always keyed and never +/// Relay-owned routing backend partition. It is always keyed and never /// depends on the user-configured header allowlist. const INTERNAL_DISPATCH_BACKEND_HEADER: &str = "x-nemo-relay-internal-dispatch-backend"; @@ -562,7 +562,7 @@ fn allowlisted_headers(headers: &Map, allowlist: &[String]) -> Map } /// Builds the key's header partition from configured headers plus the -/// Relay-owned Switchyard backend ID. +/// Relay-owned routing backend ID. fn cache_key_headers(headers: &Map, allowlist: &[String]) -> Map { let mut kept = allowlisted_headers(headers, allowlist); for (header_name, value) in headers { diff --git a/crates/adaptive/tests/integration/response_cache_tests.rs b/crates/adaptive/tests/integration/response_cache_tests.rs index d16e8bf3f..e0ac96bd1 100644 --- a/crates/adaptive/tests/integration/response_cache_tests.rs +++ b/crates/adaptive/tests/integration/response_cache_tests.rs @@ -39,7 +39,7 @@ mod response_cache_common; use response_cache_common::{activate_cache, call, chat_request}; static TEST_MUTEX: Mutex<()> = Mutex::const_new(()); -const SWITCHYARD_BACKEND_HEADER: &str = "x-nemo-relay-internal-dispatch-backend"; +const ROUTING_BACKEND_HEADER: &str = "x-nemo-relay-internal-dispatch-backend"; fn reset_global() { let _ = clear_plugin_configuration(); @@ -79,7 +79,7 @@ fn chat_request_for_backend(prompt: &str, backend: &str) -> LlmRequest { let mut request = chat_request(prompt); request .headers - .insert(SWITCHYARD_BACKEND_HEADER.to_string(), json!(backend)); + .insert(ROUTING_BACKEND_HEADER.to_string(), json!(backend)); request } @@ -269,7 +269,7 @@ async fn a_different_request_is_a_miss() { } #[tokio::test] -async fn switchyard_backends_use_independent_buffered_entries() { +async fn routing_backends_use_independent_buffered_entries() { let _guard = TEST_MUTEX.lock().await; reset_global(); activate_cache(scoped_cache_config()).await; @@ -933,7 +933,7 @@ async fn streaming_repeat_is_a_hit_that_skips_the_provider_and_replays_the_aggre } #[tokio::test] -async fn switchyard_backends_use_independent_streaming_entries() { +async fn routing_backends_use_independent_streaming_entries() { let _guard = TEST_MUTEX.lock().await; reset_global(); activate_cache(scoped_cache_config()).await; diff --git a/crates/adaptive/tests/unit/response_cache/key_tests.rs b/crates/adaptive/tests/unit/response_cache/key_tests.rs index 54912deb6..689fedf5f 100644 --- a/crates/adaptive/tests/unit/response_cache/key_tests.rs +++ b/crates/adaptive/tests/unit/response_cache/key_tests.rs @@ -264,7 +264,7 @@ fn namespace_and_provider_separate_keys() { } #[test] -fn switchyard_backend_partition_is_keyed_without_allowlisting() { +fn routing_backend_partition_is_keyed_without_allowlisting() { let config = cache_all_config(); assert!(config.header_allowlist.is_empty()); let make = |backend: &str| { diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 3dba3df83..6cd7e1e4d 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -23,13 +23,11 @@ workspace = true [features] default = ["atof-streaming"] atof-streaming = ["nemo-relay/atof-streaming"] -switchyard = ["dep:nemo-relay-switchyard"] [dependencies] nemo-relay = { workspace = true, features = ["guardrails-remote", "object-store", "worker-grpc"] } nemo-relay-adaptive = { workspace = true, features = ["redis-backend"] } nemo-relay-pii-redaction.workspace = true -nemo-relay-switchyard = { workspace = true, optional = true, features = ["schema"] } async-stream = "0.3" axum = "0.8" base64 = "0.22" @@ -82,8 +80,3 @@ syn = { version = "2", features = ["full", "visit"] } tempfile = "3" tokio = { version = "1", features = ["test-util"] } tower = { version = "0.5", features = ["util"] } - -[[test]] -name = "switchyard_process_e2e" -path = "tests/switchyard_process_e2e.rs" -required-features = ["switchyard"] diff --git a/crates/cli/src/gateway/mod.rs b/crates/cli/src/gateway/mod.rs index 76feccd4c..474d05baa 100644 --- a/crates/cli/src/gateway/mod.rs +++ b/crates/cli/src/gateway/mod.rs @@ -84,7 +84,7 @@ pub(crate) async fn passthrough( /// Exact failure material from one ordinary upstream attempt. /// -/// Retry-aware Switchyard attempts use [`FlowError::Upstream`] for routing. +/// Retry-aware routing attempts use [`FlowError::Upstream`]. /// Ordinary attempts keep their original HTTP bytes and multi-value headers so /// the selected attempt can be relayed without a shared last-write-wins slot. enum CapturedUpstreamFailure { diff --git a/crates/cli/src/plugins/editor_model.rs b/crates/cli/src/plugins/editor_model.rs index ec4735e2a..1d82b244b 100644 --- a/crates/cli/src/plugins/editor_model.rs +++ b/crates/cli/src/plugins/editor_model.rs @@ -11,8 +11,6 @@ use nemo_relay::plugin::{PluginComponentSpec, PluginConfig}; use nemo_relay_adaptive::AdaptiveConfig; use nemo_relay_adaptive::plugin_component::ADAPTIVE_PLUGIN_KIND; use nemo_relay_pii_redaction::component::{PII_REDACTION_PLUGIN_KIND, PiiRedactionConfig}; -#[cfg(feature = "switchyard")] -use nemo_relay_switchyard::{SWITCHYARD_PLUGIN_KIND, SwitchyardConfig}; use serde::Serialize; use serde::de::DeserializeOwned; use serde_json::{Map, Value, json}; @@ -51,8 +49,6 @@ pub(super) enum EditableComponent { Adaptive(Box>), NemoGuardrails(Box>), PiiRedaction(Box>), - #[cfg(feature = "switchyard")] - Switchyard(Box>), } impl EditableComponent { @@ -62,8 +58,6 @@ impl EditableComponent { Self::Adaptive(_) => "Adaptive", Self::NemoGuardrails(_) => "NeMo Guardrails (Deprecated)", Self::PiiRedaction(_) => "PII Redaction", - #[cfg(feature = "switchyard")] - Self::Switchyard(_) => "Switchyard Decision API", } } @@ -73,8 +67,6 @@ impl EditableComponent { Self::Adaptive(_) => AdaptiveConfig::editor_schema().fields, Self::NemoGuardrails(_) => NeMoGuardrailsConfig::editor_schema().fields, Self::PiiRedaction(_) => PiiRedactionConfig::editor_schema().fields, - #[cfg(feature = "switchyard")] - Self::Switchyard(_) => SwitchyardConfig::editor_schema().fields, } } @@ -84,8 +76,6 @@ impl EditableComponent { Self::Adaptive(state) => state.enabled, Self::NemoGuardrails(state) => state.enabled, Self::PiiRedaction(state) => state.enabled, - #[cfg(feature = "switchyard")] - Self::Switchyard(state) => state.enabled, } } @@ -95,8 +85,6 @@ impl EditableComponent { Self::Adaptive(state) => state.toggle_enabled(), Self::NemoGuardrails(state) => state.toggle_enabled(), Self::PiiRedaction(state) => state.toggle_enabled(), - #[cfg(feature = "switchyard")] - Self::Switchyard(state) => state.toggle_enabled(), } } @@ -106,8 +94,6 @@ impl EditableComponent { Self::Adaptive(state) => state.set_enabled(enabled), Self::NemoGuardrails(state) => state.set_enabled(enabled), Self::PiiRedaction(state) => state.set_enabled(enabled), - #[cfg(feature = "switchyard")] - Self::Switchyard(state) => state.set_enabled(enabled), } } @@ -117,8 +103,6 @@ impl EditableComponent { Self::Adaptive(state) => state.reset_enabled(), Self::NemoGuardrails(state) => state.reset_enabled(), Self::PiiRedaction(state) => state.reset_enabled(), - #[cfg(feature = "switchyard")] - Self::Switchyard(state) => state.reset_enabled(), } } @@ -128,8 +112,6 @@ impl EditableComponent { Self::Adaptive(state) => adaptive_summary(state), Self::NemoGuardrails(state) => nemo_guardrails_summary(state), Self::PiiRedaction(state) => pii_redaction_summary(state), - #[cfg(feature = "switchyard")] - Self::Switchyard(state) => switchyard_summary(state), } } @@ -143,10 +125,6 @@ impl EditableComponent { Self::PiiRedaction(state) => { config_field_configured(&state.config, field).unwrap_or(false) } - #[cfg(feature = "switchyard")] - Self::Switchyard(state) => { - config_field_configured(&state.config, field).unwrap_or(false) - } } } @@ -168,11 +146,6 @@ impl EditableComponent { reset_config_field(&mut state.config, field)?; state.mark_config_touched(); } - #[cfg(feature = "switchyard")] - Self::Switchyard(state) => { - reset_config_field(&mut state.config, field)?; - state.mark_config_touched(); - } } Ok(()) } @@ -198,11 +171,6 @@ impl EditableComponent { remove_struct_field(&mut state.config, field.name)?; state.mark_config_touched(); } - #[cfg(feature = "switchyard")] - Self::Switchyard(state) => { - remove_struct_field(&mut state.config, field.name)?; - state.mark_config_touched(); - } } Ok(true) } @@ -213,8 +181,6 @@ impl EditableComponent { Self::Adaptive(state) => store_adaptive_state(config, state), Self::NemoGuardrails(state) => store_nemo_guardrails_state(config, state), Self::PiiRedaction(state) => store_pii_redaction_state(config, state), - #[cfg(feature = "switchyard")] - Self::Switchyard(state) => store_switchyard_state(config, state), } } } @@ -244,14 +210,6 @@ pub(super) fn editable_components( EditableComponent::NemoGuardrails(Box::new(component_nemo_guardrails_state(config)?)), EditableComponent::PiiRedaction(Box::new(component_pii_redaction_state(config)?)), ]; - #[cfg(feature = "switchyard")] - let components = { - let mut components = components; - components.push(EditableComponent::Switchyard(Box::new( - component_switchyard_state(config)?, - ))); - components - }; Ok(components) } @@ -457,13 +415,6 @@ pub(super) fn component_pii_redaction_state( component_editor_state(config, PII_REDACTION_PLUGIN_KIND, false) } -#[cfg(feature = "switchyard")] -pub(super) fn component_switchyard_state( - config: &PluginConfig, -) -> Result, CliError> { - component_editor_state(config, SWITCHYARD_PLUGIN_KIND, false) -} - pub(super) fn store_observability_state( config: &mut PluginConfig, state: &ComponentEditorState, @@ -528,23 +479,6 @@ pub(super) fn store_pii_redaction_state( Ok(()) } -#[cfg(feature = "switchyard")] -pub(super) fn store_switchyard_state( - config: &mut PluginConfig, - state: &ComponentEditorState, -) -> Result<(), CliError> { - if state.should_store(state.config_touched || switchyard_configured(&state.config)) { - store_component_editor_config( - config, - SWITCHYARD_PLUGIN_KIND, - state.enabled, - switchyard_config_map(&state.config)?, - merge_switchyard_editor_config, - ); - } - Ok(()) -} - fn store_component_editor_config( config: &mut PluginConfig, kind: &str, @@ -893,24 +827,6 @@ pub(super) fn pii_redaction_config_map( } } -#[cfg(feature = "switchyard")] -pub(super) fn switchyard_config_map( - config: &SwitchyardConfig, -) -> Result, CliError> { - let value = serde_json::to_value(config).map_err(serde_error)?; - match value { - Value::Object(mut map) => { - if is_version_one(map.get("version")) { - map.remove("version"); - } - Ok(map) - } - _ => Err(CliError::Config( - "switchyard config must serialize to an object".into(), - )), - } -} - pub(super) fn merge_observability_editor_config( existing: &mut Map, edited: Map, @@ -968,22 +884,6 @@ pub(super) fn merge_pii_redaction_editor_config( ); } -#[cfg(feature = "switchyard")] -pub(super) fn merge_switchyard_editor_config( - existing: &mut Map, - edited: Map, -) { - if is_version_one(existing.get("version")) { - existing.remove("version"); - } - merge_known_editor_object( - existing, - edited, - &nested_editor_keys(SwitchyardConfig::editor_schema()), - SwitchyardConfig::editor_schema(), - ); -} - fn is_version_one(value: Option<&Value>) -> bool { value.and_then(Value::as_u64) == Some(1) } @@ -1175,23 +1075,3 @@ pub(super) fn pii_redaction_summary(state: &ComponentEditorState bool { - !config.decision_profile_id.is_empty() || !config.targets.is_empty() -} - -#[cfg(feature = "switchyard")] -pub(super) fn switchyard_summary(state: &ComponentEditorState) -> String { - let profile = if state.config.decision_profile_id.is_empty() { - "unconfigured" - } else { - state.config.decision_profile_id.as_str() - }; - format!( - "component {}, profile {}, targets {}", - if state.enabled { "enabled" } else { "disabled" }, - profile, - state.config.targets.len() - ) -} diff --git a/crates/cli/src/plugins/prompt.rs b/crates/cli/src/plugins/prompt.rs index a055f6a44..3fd48e60c 100644 --- a/crates/cli/src/plugins/prompt.rs +++ b/crates/cli/src/plugins/prompt.rs @@ -179,11 +179,6 @@ fn edit_component_field( edit_config_field(theme, &mut state.config, field)?; state.mark_config_touched(); } - #[cfg(feature = "switchyard")] - EditableComponent::Switchyard(state) => { - edit_config_field(theme, &mut state.config, field)?; - state.mark_config_touched(); - } } Ok(()) } diff --git a/crates/cli/src/server/mod.rs b/crates/cli/src/server/mod.rs index 157a0352d..5a9695114 100644 --- a/crates/cli/src/server/mod.rs +++ b/crates/cli/src/server/mod.rs @@ -27,14 +27,7 @@ use nemo_relay::plugin::{ PluginComponentSpec, PluginConfig, clear_plugin_configuration, initialize_plugins_exact, }; use nemo_relay_adaptive::plugin_component::register_adaptive_component; -#[cfg(feature = "switchyard")] -use nemo_relay_adaptive::{AdaptiveConfig, plugin_component::ADAPTIVE_PLUGIN_KIND}; use nemo_relay_pii_redaction::component::register_pii_redaction_component; -#[cfg(feature = "switchyard")] -use nemo_relay_switchyard::{ - SWITCHYARD_PLUGIN_KIND, SwitchyardConfig, register_switchyard_component, - validate_switchyard_atof_configuration, -}; use reqwest::Client; use serde_json::Value; use subtle::ConstantTimeEq; @@ -847,6 +840,8 @@ enum ServerPluginActivation { Dynamic(PluginActivation), } +const REMOVED_SWITCHYARD_MESSAGE: &str = "the built-in Switchyard service integration was removed in NeMo Relay >=0.8.0; remove this `[[components]]` entry and follow the NeMo Relay Switchyard migration guide for the Switchyard-owned dynamic plugin: https://docs.nvidia.com/nemo/relay/reference/migration-guides#migrate-to-the-switchyard-owned-dynamic-plugin"; + impl ServerPluginActivation { fn clear(self) -> Result<(), CliError> { match self { @@ -861,12 +856,7 @@ impl ServerPluginActivation { pub(crate) enum PluginComponentSetupError { Adaptive(String), PiiRedaction(String), - #[cfg(feature = "switchyard")] - Switchyard(String), - #[cfg(feature = "switchyard")] - SwitchyardAtof(String), - #[cfg(feature = "switchyard")] - SwitchyardResponseCache(String), + RemovedSwitchyard, } impl PluginComponentSetupError { @@ -874,12 +864,7 @@ impl PluginComponentSetupError { match self { Self::Adaptive(_) => "Adaptive plugin", Self::PiiRedaction(_) => "PII redaction plugin", - #[cfg(feature = "switchyard")] - Self::Switchyard(_) => "Switchyard plugin", - #[cfg(feature = "switchyard")] - Self::SwitchyardAtof(_) => "Switchyard ATOF", - #[cfg(feature = "switchyard")] - Self::SwitchyardResponseCache(_) => "Switchyard response cache", + Self::RemovedSwitchyard => "Switchyard migration", } } @@ -888,12 +873,7 @@ impl PluginComponentSetupError { Self::Adaptive(error) | Self::PiiRedaction(error) => { format!("registration failed: {error}") } - #[cfg(feature = "switchyard")] - Self::Switchyard(error) => format!("registration failed: {error}"), - #[cfg(feature = "switchyard")] - Self::SwitchyardAtof(error) => error.clone(), - #[cfg(feature = "switchyard")] - Self::SwitchyardResponseCache(error) => error.clone(), + Self::RemovedSwitchyard => REMOVED_SWITCHYARD_MESSAGE.into(), } } } @@ -910,27 +890,13 @@ impl std::fmt::Display for PluginComponentSetupError { "PII redaction plugin registration failed: {error}" ) } - #[cfg(feature = "switchyard")] - Self::Switchyard(error) => { - write!(formatter, "Switchyard plugin registration failed: {error}") - } - #[cfg(feature = "switchyard")] - Self::SwitchyardAtof(error) => { - write!(formatter, "Switchyard ATOF validation failed: {error}") - } - #[cfg(feature = "switchyard")] - Self::SwitchyardResponseCache(error) => { - write!( - formatter, - "Switchyard response-cache validation failed: {error}" - ) - } + Self::RemovedSwitchyard => formatter.write_str(REMOVED_SWITCHYARD_MESSAGE), } } } pub(crate) fn register_and_validate_plugin_components( - _plugin_config: &PluginConfig, + plugin_config: &PluginConfig, ) -> Vec { let mut errors = Vec::new(); if let Err(error) = register_adaptive_component() { @@ -939,55 +905,14 @@ pub(crate) fn register_and_validate_plugin_components( if let Err(error) = register_pii_redaction_component() { errors.push(PluginComponentSetupError::PiiRedaction(error.to_string())); } - #[cfg(feature = "switchyard")] - if let Err(error) = register_switchyard_component() { - errors.push(PluginComponentSetupError::Switchyard(error.to_string())); - } - #[cfg(feature = "switchyard")] - if let Err(error) = validate_switchyard_atof_configuration(_plugin_config) { - errors.push(PluginComponentSetupError::SwitchyardAtof(error)); - } - #[cfg(feature = "switchyard")] - if let Err(error) = validate_switchyard_response_cache_order(_plugin_config) { - errors.push(PluginComponentSetupError::SwitchyardResponseCache(error)); - } - errors -} - -#[cfg(feature = "switchyard")] -fn validate_switchyard_response_cache_order(config: &PluginConfig) -> Result<(), String> { - let Some(switchyard_component) = config + if plugin_config .components .iter() - .find(|component| component.enabled && component.kind == SWITCHYARD_PLUGIN_KIND) - else { - return Ok(()); - }; - let Some(adaptive_component) = config - .components - .iter() - .find(|component| component.enabled && component.kind == ADAPTIVE_PLUGIN_KIND) - else { - return Ok(()); - }; - - let switchyard: SwitchyardConfig = - serde_json::from_value(Value::Object(switchyard_component.config.clone())) - .map_err(|error| format!("invalid Switchyard plugin config: {error}"))?; - let adaptive: AdaptiveConfig = - serde_json::from_value(Value::Object(adaptive_component.config.clone())) - .map_err(|error| format!("invalid adaptive plugin config: {error}"))?; - let Some(response_cache) = adaptive.response_cache else { - return Ok(()); - }; - if switchyard.priority >= response_cache.priority { - return Err(format!( - "Switchyard priority {} must be lower than response_cache priority {} so backend \ - selection happens before cache key derivation", - switchyard.priority, response_cache.priority - )); + .any(|component| component.kind == "switchyard") + { + errors.push(PluginComponentSetupError::RemovedSwitchyard); } - Ok(()) + errors } async fn initialize_plugin_host( diff --git a/crates/cli/tests/coverage/shared/server_tests.rs b/crates/cli/tests/coverage/shared/server_tests.rs index ff058afe0..d5ed1e88e 100644 --- a/crates/cli/tests/coverage/shared/server_tests.rs +++ b/crates/cli/tests/coverage/shared/server_tests.rs @@ -1858,26 +1858,15 @@ fn plugin_component_setup_errors_render_every_diagnostic_variant() { "PII redaction plugin registration failed: pii failure" ); - #[cfg(feature = "switchyard")] - { - let switchyard = PluginComponentSetupError::Switchyard("registration".into()); - assert_eq!(switchyard.check_name(), "Switchyard plugin"); - assert!(switchyard.to_string().contains("registration failed")); - - let atof = PluginComponentSetupError::SwitchyardAtof("atof ordering".into()); - assert_eq!(atof.check_name(), "Switchyard ATOF"); - assert_eq!(atof.diagnostic_details(), "atof ordering"); - assert!(atof.to_string().contains("ATOF validation failed")); - - let cache = PluginComponentSetupError::SwitchyardResponseCache("cache ordering".into()); - assert_eq!(cache.check_name(), "Switchyard response cache"); - assert_eq!(cache.diagnostic_details(), "cache ordering"); - assert!( - cache - .to_string() - .contains("response-cache validation failed") - ); - } + let switchyard = PluginComponentSetupError::RemovedSwitchyard; + assert_eq!(switchyard.check_name(), "Switchyard migration"); + assert_eq!(switchyard.diagnostic_details(), switchyard.to_string()); + assert!( + switchyard + .to_string() + .contains("removed in NeMo Relay >=0.8.0") + ); + assert!(switchyard.to_string().contains("migration guide")); } fn dynamic_component_without_manifest( @@ -1957,50 +1946,6 @@ async fn shutdown_future_helpers_cover_receiver_combinations() { combine_shutdown_futures(Some(ready), None).unwrap().await; } -#[cfg(feature = "switchyard")] -#[test] -fn switchyard_must_run_before_response_cache() { - let build = |switchyard_priority, cache_priority| { - let switchyard = nemo_relay_switchyard::SwitchyardConfig { - priority: switchyard_priority, - ..nemo_relay_switchyard::SwitchyardConfig::default() - }; - let adaptive = nemo_relay_adaptive::AdaptiveConfig { - response_cache: Some(nemo_relay_adaptive::ResponseCacheConfig { - namespace: "switchyard-order-test".into(), - priority: cache_priority, - ..nemo_relay_adaptive::ResponseCacheConfig::default() - }), - ..nemo_relay_adaptive::AdaptiveConfig::default() - }; - PluginConfig { - components: vec![ - switchyard.into(), - nemo_relay_adaptive::plugin_component::ComponentSpec::new(adaptive).into(), - ], - ..PluginConfig::default() - } - }; - - assert!(validate_switchyard_response_cache_order(&build(0, 50)).is_ok()); - for (switchyard_priority, cache_priority) in [(50, 50), (51, 50)] { - let error = - validate_switchyard_response_cache_order(&build(switchyard_priority, cache_priority)) - .unwrap_err(); - assert!( - error.contains("must be lower"), - "unexpected ordering error: {error}" - ); - } - - let mut disabled = build(50, 50); - disabled.components[0].enabled = false; - assert!( - validate_switchyard_response_cache_order(&disabled).is_ok(), - "disabled Switchyard components do not participate in ordering" - ); -} - #[tokio::test] async fn serve_listener_activates_adaptive_plugin_config() { let _guard = PLUGIN_CONFIG_TEST_LOCK.lock().await; diff --git a/crates/cli/tests/switchyard_process_e2e.rs b/crates/cli/tests/switchyard_process_e2e.rs deleted file mode 100644 index a9ba3397d..000000000 --- a/crates/cli/tests/switchyard_process_e2e.rs +++ /dev/null @@ -1,367 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! CI-safe process-boundary coverage for the Switchyard plugin. - -use std::process::{Child, Command, Stdio}; -use std::sync::{Arc, Mutex}; -use std::time::Duration; - -use axum::body::Body; -use axum::extract::State; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::routing::{get, post}; -use axum::{Json, Router}; -use serde_json::{Value, json}; - -fn gateway_bin() -> &'static str { - env!("CARGO_BIN_EXE_nemo-relay") -} - -struct ChildGuard(Child); - -impl Drop for ChildGuard { - fn drop(&mut self) { - let _ = self.0.kill(); - let _ = self.0.wait(); - } -} - -#[derive(Clone, Default)] -struct DecisionState { - requests: Arc>>, -} - -async fn decide( - State(state): State, - headers: HeaderMap, - Json(request): Json, -) -> Response { - let call = { - let mut requests = state.requests.lock().unwrap(); - requests.push((headers, request)); - requests.len() - }; - if call == 4 { - return Response::builder() - .status(StatusCode::SERVICE_UNAVAILABLE) - .body(Body::from("decision API unavailable")) - .unwrap(); - } - let body = json!({ - "schema_version": "switchyard.routing_decision.v1", - "decision_id": format!("decision-{call}"), - "router": {"name": "fake-ci-router", "version": "1"}, - "route": { - "tier": "strong", - "target_model": "provider/selected", - "backend_id": "selected-chat", - "target_protocol_profile": "openai_chat", - "target_endpoint": "/v1/chat/completions" - }, - "confidence": 0.99, - "reason_code": "ci_fixture", - "reason_summary": "deterministic process E2E decision" - }); - Response::builder() - .status(StatusCode::OK) - .header("content-type", "application/json") - .body(Body::from(body.to_string())) - .unwrap() -} - -async fn switchyard_health() -> Json { - Json(json!({"status": "ok"})) -} - -#[derive(Clone, Default)] -struct ProviderState { - requests: Arc>>, -} - -async fn provide( - State(state): State, - headers: HeaderMap, - Json(request): Json, -) -> Response { - let stream = request["stream"].as_bool().unwrap_or(false); - let model = request["model"].as_str().unwrap_or("unknown").to_string(); - let malformed_response = !stream - && model == "provider/selected" - && headers - .get("x-nemo-relay-request-id") - .is_some_and(|value| value == "malformed-response"); - state.requests.lock().unwrap().push((headers, request)); - if malformed_response { - return Response::builder() - .status(StatusCode::OK) - .header("content-type", "application/json") - .body(Body::from("{invalid-provider-json")) - .unwrap(); - } - if stream { - let first = json!({ - "id": "chat-ci", "object": "chat.completion.chunk", "model": model, - "choices": [{"index": 0, "delta": {"role": "assistant", "content": "streamed"}, "finish_reason": null}] - }); - let last = json!({ - "id": "chat-ci", "object": "chat.completion.chunk", "model": model, - "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 4, "completion_tokens": 1, "total_tokens": 5} - }); - let body = format!("data: {first}\n\ndata: {last}\n\ndata: [DONE]\n\n"); - return Response::builder() - .status(StatusCode::OK) - .header("content-type", "text/event-stream") - .body(Body::from(body)) - .unwrap(); - } - let body = json!({ - "id": "chat-ci", "object": "chat.completion", "model": model, - "choices": [{"index": 0, "message": {"role": "assistant", "content": format!("served by {model}")}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7} - }); - Response::builder() - .status(StatusCode::OK) - .header("content-type", "application/json") - .body(Body::from(body.to_string())) - .unwrap() -} - -async fn start_server(router: Router) -> (String, tokio::task::JoinHandle<()>) { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - let task = tokio::spawn(async move { - axum::serve(listener, router).await.unwrap(); - }); - (format!("http://{address}"), task) -} - -fn unused_address() -> std::net::SocketAddr { - let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); - listener.local_addr().unwrap() -} - -async fn wait_for_gateway(client: &reqwest::Client, url: &str, child: &mut Child) { - for _ in 0..120 { - if let Some(status) = child.try_wait().unwrap() { - panic!("gateway exited before readiness with {status}"); - } - if client - .get(format!("{url}/healthz")) - .send() - .await - .is_ok_and(|response| response.status().is_success()) - { - return; - } - tokio::time::sleep(Duration::from_millis(50)).await; - } - panic!("gateway did not become ready at {url}"); -} - -#[tokio::test(flavor = "multi_thread")] -async fn switchyard_plugin_routes_buffered_and_streaming_then_fails_open() { - let decision_state = DecisionState::default(); - let decision_requests = Arc::clone(&decision_state.requests); - let (decision_url, decision_task) = start_server( - Router::new() - .route("/v1/routing/decision", post(decide)) - .route("/health", get(switchyard_health)) - .with_state(decision_state), - ) - .await; - - let provider_state = ProviderState::default(); - let provider_requests = Arc::clone(&provider_state.requests); - let (provider_url, provider_task) = start_server( - Router::new() - .route("/v1/chat/completions", post(provide)) - .with_state(provider_state), - ) - .await; - - let temp = tempfile::tempdir().unwrap(); - let config_path = temp.path().join("plugins.toml"); - let config = format!( - r#"version = 1 - -[[components]] -kind = "switchyard" -enabled = true - -[components.config] -mode = "enforce" -decision_api_url = "{decision_url}/v1/routing/decision" -decision_profile_id = "ci-process-e2e" -request_materialization = "full_body" -context_mode = "payload_only" -decision_timeout_millis = 1000 -max_retries = 0 - -[components.config.default_targets] -openai_chat = "fallback-chat" -openai_responses = "fallback-responses" -anthropic_messages = "fallback-anthropic" - -[components.config.targets.selected-chat] -model = "provider/selected" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "{provider_url}" - -[components.config.targets.fallback-chat] -model = "provider/fallback" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "{provider_url}" - -[components.config.targets.fallback-responses] -model = "provider/fallback" -protocol = "openai_responses" -endpoint = "/v1/responses" -base_url = "{provider_url}" - -[components.config.targets.fallback-anthropic] -model = "provider/fallback" -protocol = "anthropic_messages" -endpoint = "/v1/messages" -base_url = "{provider_url}" -"# - ); - std::fs::write(&config_path, config).unwrap(); - - let address = unused_address(); - let gateway_url = format!("http://{address}"); - let stderr = std::fs::File::create(temp.path().join("gateway.log")).unwrap(); - let child = Command::new(gateway_bin()) - .arg("--plugin-config-path") - .arg(&config_path) - .arg("--bind") - .arg(address.to_string()) - .stdout(Stdio::null()) - .stderr(Stdio::from(stderr)) - .spawn() - .unwrap(); - let mut gateway = ChildGuard(child); - let client = reqwest::Client::new(); - wait_for_gateway(&client, &gateway_url, &mut gateway.0).await; - - let send_chat = |request_id: &'static str, stream: bool| { - client - .post(format!("{gateway_url}/v1/chat/completions")) - .header("x-nemo-relay-session-id", "ci-process-session") - .header("x-nemo-relay-request-id", request_id) - .header( - "x-nemo-relay-internal-dispatch-url", - "http://attacker.invalid", - ) - .header("x-nemo-relay-internal-dispatch-route", "attacker-route") - .json(&json!({ - "model": "client/model", - "stream": stream, - "messages": [{"role": "user", "content": "process boundary test"}] - })) - .send() - }; - - let buffered = send_chat("buffered-request", false).await.unwrap(); - assert!(buffered.status().is_success()); - let buffered: Value = buffered.json().await.unwrap(); - assert_eq!(buffered["model"], "provider/selected"); - - let translated = client - .post(format!("{gateway_url}/v1/responses")) - .header("x-nemo-relay-session-id", "ci-process-session") - .header("x-nemo-relay-request-id", "translated-request") - .json(&json!({ - "model": "client/model", - "stream": false, - "input": "process boundary response translation" - })) - .send() - .await - .unwrap(); - assert!(translated.status().is_success()); - let translated: Value = translated.json().await.unwrap(); - assert_eq!(translated["object"], "response"); - - let streaming = send_chat("stream-request", true).await.unwrap(); - assert!(streaming.status().is_success()); - let streaming = streaming.text().await.unwrap(); - assert!(streaming.contains("streamed")); - assert!(streaming.contains("[DONE]")); - - let fallback = send_chat("fallback-request", false).await.unwrap(); - assert!(fallback.status().is_success()); - let fallback: Value = fallback.json().await.unwrap(); - assert_eq!(fallback["model"], "provider/fallback"); - - let malformed = send_chat("malformed-response", false).await.unwrap(); - assert!(malformed.status().is_success()); - let malformed: Value = malformed.json().await.unwrap(); - assert_eq!(malformed["model"], "provider/fallback"); - - let decisions = decision_requests.lock().unwrap(); - assert_eq!(decisions.len(), 5); - for (headers, body) in decisions.iter() { - assert!(!headers.contains_key("x-nemo-relay-internal-dispatch-url")); - assert!(!headers.contains_key("x-nemo-relay-internal-dispatch-route")); - assert_eq!( - headers - .get("x-nemo-relay-session-id") - .unwrap() - .to_str() - .unwrap(), - "ci-process-session" - ); - assert_eq!(body["schema_version"], "switchyard.routing_request.v1"); - assert_eq!(body["decision_profile"]["profile_id"], "ci-process-e2e"); - } - drop(decisions); - - let providers = provider_requests.lock().unwrap(); - let models = providers - .iter() - .map(|(_, body)| body["model"].as_str().unwrap()) - .collect::>(); - assert_eq!( - models, - vec![ - "provider/selected", - "provider/selected", - "provider/selected", - "provider/fallback", - "provider/selected", - "provider/fallback" - ] - ); - assert!(providers[1].1["messages"].is_array()); - assert!(providers[1].1.get("input").is_none()); - assert_eq!(providers[1].1["messages"][0]["role"], "user"); - assert_eq!( - providers[1].1["messages"][0]["content"], - "process boundary response translation" - ); - let malformed_models = providers - .iter() - .filter(|(headers, _)| { - headers - .get("x-nemo-relay-request-id") - .is_some_and(|value| value == "malformed-response") - }) - .map(|(_, body)| body["model"].as_str().unwrap()) - .collect::>(); - assert_eq!( - malformed_models, - vec!["provider/selected", "provider/fallback"] - ); - for (headers, _) in providers.iter() { - assert!(!headers.contains_key("x-nemo-relay-internal-dispatch-url")); - assert!(!headers.contains_key("x-nemo-relay-internal-dispatch-route")); - } - - decision_task.abort(); - provider_task.abort(); -} diff --git a/crates/core/src/error.rs b/crates/core/src/error.rs index 23ed1c01a..5ab64f6da 100644 --- a/crates/core/src/error.rs +++ b/crates/core/src/error.rs @@ -48,7 +48,7 @@ pub struct UpstreamFailure { } impl UpstreamFailure { - /// Whether Switchyard may be consulted for another bounded provider attempt. + /// Whether another bounded provider attempt may be made. pub fn is_retryable(&self) -> bool { matches!( self.class, diff --git a/crates/switchyard/Cargo.toml b/crates/switchyard/Cargo.toml deleted file mode 100644 index 7e7466884..000000000 --- a/crates/switchyard/Cargo.toml +++ /dev/null @@ -1,34 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -[package] -name = "nemo-relay-switchyard" -version.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -description = "First-party Switchyard Decision API routing plugin for NeMo Relay." -readme = "README.md" - -[lints] -workspace = true - -[features] -default = [] -schema = ["dep:schemars", "nemo-relay/schema"] - -[dependencies] -nemo-relay.workspace = true -serde = { version = "1", features = ["derive"] } -serde_json = "1" -switchyard-translation.workspace = true -schemars = { version = "0.8", optional = true } -reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots"] } -tokio = { version = "1", default-features = false, features = ["time"] } -uuid = { workspace = true, features = ["v7"] } -futures-util = "0.3" -async-stream = "0.3" - -[dev-dependencies] -axum = "0.8" -tokio = { version = "1", features = ["net", "rt", "macros", "sync", "test-util", "rt-multi-thread", "time"] } diff --git a/crates/switchyard/README.md b/crates/switchyard/README.md deleted file mode 100644 index 0d8dece3b..000000000 --- a/crates/switchyard/README.md +++ /dev/null @@ -1,130 +0,0 @@ - - -[![License](https://img.shields.io/github/license/NVIDIA/NeMo-Relay)](https://github.com/NVIDIA/NeMo-Relay/blob/main/LICENSE) -[![GitHub](https://img.shields.io/badge/github-repo-blue?logo=github)](https://github.com/NVIDIA/NeMo-Relay/) -[![Release](https://img.shields.io/github/v/release/NVIDIA/NeMo-Relay?color=green)](https://github.com/NVIDIA/NeMo-Relay/releases) - -# NeMo Relay Switchyard Plugin - -> **Deprecated:** The experimental `nemo-relay-switchyard` plugin will be -> removed in NeMo Relay 0.8 and replaced by a Switchyard-owned native plugin. -> The NeMo Relay 0.8 documentation will include an updated configuration guide -> and migration plan when the replacement is available. - -`nemo-relay-switchyard` is NeMo Relay's experimental integration -with the [NVIDIA NeMo Switchyard](https://github.com/NVIDIA-NeMo/Switchyard) -Decision API. It adds routing-aware LLM execution intercepts to the Relay -runtime while preserving Relay ownership of provider credentials, target -bindings, dispatch, retries, fallbacks, and observability. - -NeMo Relay 0.6.0 and 0.7.0 use a separately running Switchyard Decision API -from the `topic/nemo-relay-integration` branch. - -Install it from crates.io, or build it from the NeMo Relay source checkout with -the optional CLI feature while the Switchyard Decision API contract and -service/library boundary are still evolving. - -Use the plugin to: - -- **Route through Switchyard decisions**: Select an exact Relay-owned target - using a versioned Decision API contract. -- **Keep provider protocols stable**: Use Switchyard's translation library for - OpenAI Chat, OpenAI Responses, and Anthropic Messages request and response - translation. -- **Preserve Relay execution semantics**: Keep retries, trusted fallbacks, - credentials, streaming behavior, and optimization accounting in Relay. -- **Support staged rollout**: Run in enforce or observe-only mode with - explicit target bindings and protocol defaults. - -## Implementation and Runtime Behavior - -The plugin includes the following implementation and runtime behavior: - -- `SwitchyardConfig`: The typed plugin configuration contract. -- `SwitchyardRuntime`: Buffered and streaming routing intercepts. -- Decision and target validation for exact backend, model, protocol, and - endpoint bindings. -- ATOF-backed or payload-only routing context modes. -- Routing marks and model-routing optimization contributions for Relay's - cumulative accounting pipeline. -- Switchyard-owned protocol translation through the - `switchyard-translation` dependency. - -## Installation and Source Build - -Add the crate from crates.io: - -```bash -cargo add nemo-relay-switchyard -``` - -To build the optional CLI integration from a NeMo Relay source checkout: - -```bash -cargo build -p nemo-relay-cli --features switchyard -cargo test -p nemo-relay-switchyard -``` - -The resulting CLI includes the Switchyard component only when the `switchyard` -feature is enabled. A default Relay build does not include this experimental -integration. - -## Runtime Boundary - -The current integration calls Switchyard's HTTP Decision API at runtime. Relay -does not start or supervise the Switchyard service. For ATOF-backed profiles, -Switchyard also provides the `/v1/atof/events` ingestion and accumulator -runtime. The service must therefore be running before Relay activates the -plugin. Activation performs a bounded request to the service's `/health` -endpoint and fails if it does not return `{"status":"ok"}`. This requirement -applies to enforce and observe-only rollout modes. - -The current service setup is documented in -[`examples/switchyard/README.md`](../../examples/switchyard/README.md), including -the pinned topic-branch commit, local configuration, compatibility smoke test, -and trajectory workflow. - -Translation runs in-process through Switchyard's Rust translation library. - -## Configuration and Registration - -The CLI registers the component when built with `--features switchyard` and -accepts a `[[components]]` entry with `kind = "switchyard"`. A minimal -configuration selects the Decision API and trusted protocol defaults: - -```toml -[[components]] -kind = "switchyard" -enabled = true - -[components.config] -mode = "enforce" -decision_api_url = "http://127.0.0.1:4000/v1/routing/decision" -decision_profile_id = "my-profile" -context_mode = "payload_only" -request_materialization = "summary_only" - -[components.config.default_targets] -openai_chat = "my-openai-target" -openai_responses = "my-responses-target" -anthropic_messages = "my-anthropic-target" -``` - -For ATOF-backed profiles, configure an enabled Relay ATOF HTTP stream sink that -has a unique `name`, targets the Switchyard ingestion URL, and uses -environment-referenced authentication headers. Set `atof_endpoint_name` in the -Switchyard component to that name. Local ATOF JSONL output alone does not -populate the Switchyard accumulator. Keep provider and Decision API credentials -outside tracked configuration files. - -## Documentation - -For more information, refer to the following resources: - -- [Switchyard 0.6.0 setup and validation guide](https://docs.nvidia.com/nemo/relay/v0.6.0/configure-plugins/switchyard/about) -- [Switchyard configuration reference](https://docs.nvidia.com/nemo/relay/v0.6.0/configure-plugins/switchyard/configuration) -- [Switchyard integration examples](../../examples/switchyard/README.md) -- [Switchyard repository](https://github.com/NVIDIA-NeMo/Switchyard) diff --git a/crates/switchyard/src/component.rs b/crates/switchyard/src/component.rs deleted file mode 100644 index fa8efce53..000000000 --- a/crates/switchyard/src/component.rs +++ /dev/null @@ -1,1829 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Switchyard plugin configuration and Relay execution integration. - -use std::collections::{BTreeMap, BTreeSet, VecDeque}; -use std::future::Future; -use std::pin::Pin; -use std::sync::Arc; -use std::task::{Context, Poll}; -use std::time::{Duration, Instant}; - -use futures_util::StreamExt; -use nemo_relay::api::event::{CategoryProfile, DataSchema, EventCategory}; -use nemo_relay::api::llm::LlmRequest; -use nemo_relay::api::optimization::record_llm_optimization_contribution; -use nemo_relay::api::runtime::{ - LlmExecutionFn, LlmJsonStream, LlmStreamExecutionFn, LlmStreamInner, -}; -use nemo_relay::api::scope::{EmitMarkEventParams, event}; -use nemo_relay::codec::optimization::{ - LlmOptimizationContribution, LlmOptimizationKind, LlmOptimizationModel, - LlmOptimizationModelTransition, -}; -use nemo_relay::error::{FlowError, Result as FlowResult}; -use nemo_relay::observability::atof::{AtofEndpointFieldNamePolicy, AtofEndpointTransport}; -use nemo_relay::plugin::{ - ConfigDiagnostic, DiagnosticLevel, Plugin, PluginComponentSpec, PluginConfig, PluginError, - PluginRegistrationContext, Result as PluginResult, deregister_plugin, register_plugin, -}; -use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; -use serde::{Deserialize, Serialize}; -use serde_json::{Map, Value as Json, json}; -use uuid::Uuid; - -use crate::contract::{ - DecisionAttempt, DecisionProfile, ROUTING_DECISION_SCHEMA_VERSION, - ROUTING_REQUEST_SCHEMA_VERSION, RequestIdentity, RequestMaterialization, RequestProtocol, - RequestSummary, RoutingDecision, RoutingRequest, RoutingTarget, -}; -use crate::stream_translation::StreamTranscoder; -use crate::translation::{ - decode_request, encode_request, latest_user_prompt, recent_message_window, translate_response, - translation_engine, validate_portable_request, -}; - -/// Plugin kind used in Relay plugin configuration. -pub const SWITCHYARD_PLUGIN_KIND: &str = "switchyard"; - -const SWITCHYARD_HEALTH_PATH: &str = "/health"; -const SWITCHYARD_HEALTH_TIMEOUT: Duration = Duration::from_secs(2); -const SWITCHYARD_HEALTH_MAX_ATTEMPTS: usize = 3; -const SWITCHYARD_HEALTH_INITIAL_BACKOFF: Duration = Duration::from_millis(100); -const INTERNAL_DISPATCH_BACKEND_HEADER: &str = "x-nemo-relay-internal-dispatch-backend"; -const INTERNAL_DISPATCH_URL_HEADER: &str = "x-nemo-relay-internal-dispatch-url"; -const INTERNAL_DISPATCH_ROUTE_HEADER: &str = "x-nemo-relay-internal-dispatch-route"; -const INTERNAL_RETRY_AWARE_HEADER: &str = "x-nemo-relay-internal-retry-aware"; -const ROUTING_MARK_SCHEMA: &str = "switchyard.routing_mark"; -const ROUTING_CONTRIBUTION_SCHEMA: &str = "nvidia.switchyard.routing_optimization"; - -/// Supported provider wire protocols. -#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "snake_case")] -pub enum WireProtocol { - /// OpenAI Chat Completions. - OpenaiChat, - /// OpenAI Responses. - OpenaiResponses, - /// Anthropic Messages. - AnthropicMessages, -} - -impl WireProtocol { - fn label(self) -> &'static str { - match self { - Self::OpenaiChat => "openai_chat", - Self::OpenaiResponses => "openai_responses", - Self::AnthropicMessages => "anthropic_messages", - } - } - - fn endpoint(self) -> &'static str { - match self { - Self::OpenaiChat => "/v1/chat/completions", - Self::OpenaiResponses => "/v1/responses", - Self::AnthropicMessages => "/v1/messages", - } - } - - fn from_call(name: &str, request: &LlmRequest) -> Option { - match name { - "openai.chat_completions" | "openai_chat" | "openai_chat_completions" => { - Some(Self::OpenaiChat) - } - "openai.responses" | "openai_responses" => Some(Self::OpenaiResponses), - "anthropic.messages" | "anthropic" | "anthropic_messages" => { - Some(Self::AnthropicMessages) - } - _ if request.content.get("input").is_some() => Some(Self::OpenaiResponses), - _ if request.content.get("system").is_some() => Some(Self::AnthropicMessages), - _ if request.content.get("messages").is_some() => Some(Self::OpenaiChat), - _ => None, - } - } -} - -/// Routing rollout mode. -#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "snake_case")] -pub enum RoutingMode { - /// Apply Switchyard decisions. - #[default] - Enforce, - /// Record decisions but dispatch trusted defaults. - ObserveOnly, -} - -impl RoutingMode { - fn label(self) -> &'static str { - match self { - Self::Enforce => "enforce", - Self::ObserveOnly => "observe_only", - } - } -} - -/// Whether the selected Switchyard profile depends on ATOF-derived history. -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "snake_case")] -pub enum ContextMode { - /// The router uses only current request material. - PayloadOnly, - /// Stable identity and a configured ATOF endpoint are required. - AtofRequired, -} - -/// Exact Relay-owned backend binding for one Switchyard backend ID. -#[derive(Clone, Debug, Deserialize, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -pub struct TargetBinding { - /// Exact model expected in the Switchyard decision. - pub model: String, - /// Exact protocol expected in the Switchyard decision. - pub protocol: WireProtocol, - /// Exact endpoint expected in the Switchyard decision. - pub endpoint: String, - /// Relay-owned backend base URL. - pub base_url: String, - /// Static non-sensitive backend headers. - #[serde(default)] - pub headers: BTreeMap, - /// Backend headers resolved from environment variables. - #[serde(default)] - pub header_env: BTreeMap, -} - -/// Trusted fallback target IDs for each inbound protocol. -#[derive(Clone, Debug, Deserialize, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -pub struct ProtocolDefaults { - /// OpenAI Chat fallback target. - #[serde(default)] - pub openai_chat: String, - /// OpenAI Responses fallback target. - #[serde(default)] - pub openai_responses: String, - /// Anthropic Messages fallback target. - #[serde(default)] - pub anthropic_messages: String, -} - -impl ProtocolDefaults { - fn target(&self, protocol: WireProtocol) -> &str { - match protocol { - WireProtocol::OpenaiChat => &self.openai_chat, - WireProtocol::OpenaiResponses => &self.openai_responses, - WireProtocol::AnthropicMessages => &self.anthropic_messages, - } - } -} - -/// Versioned Switchyard plugin configuration. -#[derive(Clone, Debug, Deserialize, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -pub struct SwitchyardConfig { - /// Config schema version. - #[serde(default = "default_version")] - pub version: u32, - /// Enforce or observe-only rollout mode. - #[serde(default)] - pub mode: RoutingMode, - /// Execution-intercept priority. - #[serde(default)] - pub priority: i32, - /// Switchyard Decision API URL. - pub decision_api_url: String, - /// Switchyard profile ID. - pub decision_profile_id: String, - /// Current-request materialization. - pub request_materialization: RequestMaterialization, - /// Profile context requirement. - pub context_mode: ContextMode, - /// Decision call timeout. - #[serde(default = "default_decision_timeout_millis")] - pub decision_timeout_millis: u64, - /// Provider retries after the initial attempt. - #[serde(default = "default_max_retries")] - pub max_retries: u32, - /// Number of messages in recent-message materialization. - #[serde(default = "default_recent_message_count")] - pub recent_message_count: usize, - /// Static non-sensitive Decision API headers. - #[serde(default)] - pub decision_headers: BTreeMap, - /// Decision API headers resolved from environment variables. - #[serde(default)] - pub decision_header_env: BTreeMap, - /// Enabled inbound protocols. - #[serde(default = "default_enabled_protocols")] - pub enabled_inbound_profiles: BTreeSet, - /// Exact backend bindings keyed by Switchyard backend ID. - pub targets: BTreeMap, - /// Trusted per-protocol fallbacks. - pub default_targets: ProtocolDefaults, - /// Named observability ATOF endpoint used by history-backed profiles. - #[serde(default)] - pub atof_endpoint_name: Option, -} - -impl Default for SwitchyardConfig { - fn default() -> Self { - Self { - version: default_version(), - mode: RoutingMode::default(), - priority: 0, - decision_api_url: "http://127.0.0.1:8080/v1/routing/decision".into(), - decision_profile_id: String::new(), - request_materialization: RequestMaterialization::SummaryOnly, - context_mode: ContextMode::PayloadOnly, - decision_timeout_millis: default_decision_timeout_millis(), - max_retries: default_max_retries(), - recent_message_count: default_recent_message_count(), - decision_headers: BTreeMap::new(), - decision_header_env: BTreeMap::new(), - enabled_inbound_profiles: default_enabled_protocols(), - targets: BTreeMap::new(), - default_targets: ProtocolDefaults { - openai_chat: String::new(), - openai_responses: String::new(), - anthropic_messages: String::new(), - }, - atof_endpoint_name: None, - } - } -} - -nemo_relay::editor_config! { - impl SwitchyardConfig { - mode => { label: "Rollout mode", kind: Enum, values: ["enforce", "observe_only"] }, - priority => { label: "Intercept priority", kind: Integer }, - decision_api_url => { label: "Decision API URL", kind: String }, - decision_profile_id => { label: "Decision profile ID", kind: String }, - request_materialization => { - label: "Request materialization", - kind: Enum, - values: ["none", "summary_only", "latest_user_prompt", "recent_message_window", "annotated_request", "full_body"] - }, - context_mode => { label: "Context mode", kind: Enum, values: ["payload_only", "atof_required"] }, - decision_timeout_millis => { label: "Decision timeout (ms)", kind: Integer }, - max_retries => { label: "Maximum provider retries", kind: Integer }, - recent_message_count => { label: "Recent message count", kind: Integer }, - decision_headers => { label: "Decision API static headers", kind: StringMap }, - decision_header_env => { label: "Decision API environment headers", kind: StringMap }, - enabled_inbound_profiles => { label: "Enabled inbound profiles", kind: Json }, - targets => { label: "Backend target bindings", kind: Json }, - default_targets => { label: "Trusted protocol defaults", kind: Json }, - atof_endpoint_name => { label: "ATOF endpoint name", kind: String, optional: true } - } -} - -impl From for PluginComponentSpec { - fn from(value: SwitchyardConfig) -> Self { - let Json::Object(config) = - serde_json::to_value(value).expect("Switchyard config should serialize to an object") - else { - unreachable!("Switchyard config must serialize to an object") - }; - Self { - kind: SWITCHYARD_PLUGIN_KIND.into(), - enabled: true, - config, - } - } -} - -fn default_version() -> u32 { - 1 -} -fn default_decision_timeout_millis() -> u64 { - 25 -} -fn default_max_retries() -> u32 { - 3 -} -fn default_recent_message_count() -> usize { - 8 -} -fn default_enabled_protocols() -> BTreeSet { - BTreeSet::from([ - WireProtocol::OpenaiChat, - WireProtocol::OpenaiResponses, - WireProtocol::AnthropicMessages, - ]) -} - -struct SwitchyardPlugin; - -impl Plugin for SwitchyardPlugin { - fn plugin_kind(&self) -> &str { - SWITCHYARD_PLUGIN_KIND - } - - fn allows_multiple_components(&self) -> bool { - false - } - - fn validate(&self, plugin_config: &Map) -> Vec { - match parse_config(plugin_config).and_then(SwitchyardRuntime::new) { - Ok(_) => Vec::new(), - Err(error) => vec![ConfigDiagnostic { - level: DiagnosticLevel::Error, - code: "switchyard.invalid_config".into(), - component: Some(SWITCHYARD_PLUGIN_KIND.into()), - field: None, - message: error, - }], - } - } - - fn register<'a>( - &'a self, - plugin_config: &Map, - ctx: &'a mut PluginRegistrationContext, - ) -> Pin> + Send + 'a>> { - let parsed = parse_config(plugin_config); - Box::pin(async move { - let runtime = Arc::new( - parsed - .and_then(SwitchyardRuntime::new) - .map_err(PluginError::InvalidConfig)?, - ); - runtime - .require_healthy_sidecar() - .await - .map_err(PluginError::RegistrationFailed)?; - let buffered = Arc::clone(&runtime); - let buffered_intercept: LlmExecutionFn = Arc::new(move |name, request, next| { - let runtime = Arc::clone(&buffered); - let name = name.to_string(); - Box::pin(async move { runtime.execute_buffered(&name, request, next).await }) - }); - ctx.register_llm_execution_intercept( - "decision", - runtime.config.priority, - buffered_intercept, - )?; - - let streaming = Arc::clone(&runtime); - let stream_intercept: LlmStreamExecutionFn = Arc::new(move |name, request, next| { - let runtime = Arc::clone(&streaming); - let name = name.to_string(); - Box::pin(async move { runtime.execute_stream(&name, request, next).await }) - }); - ctx.register_llm_stream_execution_intercept( - "decision_stream", - runtime.config.priority, - stream_intercept, - )?; - Ok(()) - }) - } -} - -/// Register the first-party Switchyard component kind. -pub fn register_switchyard_component() -> PluginResult<()> { - match register_plugin(Arc::new(SwitchyardPlugin)) { - Ok(()) => Ok(()), - Err(PluginError::RegistrationFailed(message)) if message.contains("already registered") => { - Ok(()) - } - Err(error) => Err(error), - } -} - -/// Deregister the first-party Switchyard component kind. -pub fn deregister_switchyard_component() -> bool { - deregister_plugin(SWITCHYARD_PLUGIN_KIND) -} - -/// Validate the cross-component ATOF requirement for enabled history-backed profiles. -pub fn validate_switchyard_atof_configuration(config: &PluginConfig) -> Result<(), String> { - let Some(component) = config - .components - .iter() - .find(|component| component.enabled && component.kind == SWITCHYARD_PLUGIN_KIND) - else { - return Ok(()); - }; - let switchyard = parse_config(&component.config)?; - if switchyard.context_mode != ContextMode::AtofRequired { - return Ok(()); - } - let required_name = validate_atof_endpoint_name(switchyard.atof_endpoint_name.as_deref())? - .ok_or_else(|| { - "atof_required Switchyard profiles require atof_endpoint_name".to_string() - })?; - let observability = config - .components - .iter() - .find(|component| component.enabled && component.kind == "observability") - .ok_or_else(|| "atof_required Switchyard profiles require observability".to_string())?; - let sinks = observability - .config - .get("atof") - .filter(|atof| atof.get("enabled").and_then(Json::as_bool) == Some(true)) - .and_then(|atof| atof.get("sinks")) - .and_then(Json::as_array) - .ok_or_else(|| { - "atof_required Switchyard profiles require an enabled ATOF endpoint".to_string() - })?; - let matching_sinks = sinks - .iter() - .filter(|sink| { - sink.get("type").and_then(Json::as_str) == Some("stream") - && sink.get("name").and_then(Json::as_str) == Some(required_name) - }) - .collect::>(); - let endpoint = match matching_sinks.as_slice() { - [sink] => *sink, - [] => { - return Err(format!( - "atof_required Switchyard profile requires named ATOF endpoint {required_name:?}" - )); - } - _ => { - return Err(format!( - "ATOF endpoint name {required_name:?} must resolve to exactly one endpoint" - )); - } - }; - let transport = endpoint.get("transport").map_or_else( - || Some(AtofEndpointTransport::default()), - |value| value.as_str().and_then(AtofEndpointTransport::parse), - ); - if transport != Some(AtofEndpointTransport::HttpPost) { - return Err(format!( - "Switchyard ATOF endpoint {required_name:?} must use transport = http_post" - )); - } - let field_name_policy = endpoint.get("field_name_policy").map_or_else( - || Some(AtofEndpointFieldNamePolicy::default()), - |value| value.as_str().and_then(AtofEndpointFieldNamePolicy::parse), - ); - if field_name_policy != Some(AtofEndpointFieldNamePolicy::Preserve) { - return Err(format!( - "Switchyard ATOF endpoint {required_name:?} must use field_name_policy = preserve" - )); - } - if endpoint - .get("header_env") - .and_then(Json::as_object) - .is_none_or(Map::is_empty) - { - return Err(format!( - "Switchyard ATOF endpoint {required_name:?} authentication must use at least one environment-referenced header" - )); - } - Ok(()) -} - -fn parse_config(config: &Map) -> Result { - serde_json::from_value(Json::Object(config.clone())) - .map_err(|error| format!("invalid Switchyard plugin config: {error}")) -} - -struct SwitchyardRuntime { - config: SwitchyardConfig, - client: reqwest::Client, - target_headers: BTreeMap>, - translation: switchyard_translation::TranslationEngine, -} - -enum BufferedAttempt { - Complete(Json), - Retry((String, String)), - Fallback(&'static str), -} - -enum StreamAttempt { - Committed(LlmJsonStream), - Retry((String, String)), - Fallback(&'static str), -} - -struct StreamAttemptContext { - routing_request: RoutingRequest, - decision: RoutingDecision, - attempt: u32, - max_attempts: u32, -} - -fn provider_fallback_reason(error: &FlowError) -> &'static str { - if error_is_retryable(error) { - "retry_exhausted" - } else { - "non_retryable_provider_error" - } -} - -impl SwitchyardRuntime { - fn new(config: SwitchyardConfig) -> Result { - validate_config(&config)?; - let headers = resolve_headers(&config.decision_headers, &config.decision_header_env)?; - let client = reqwest::Client::builder() - .default_headers(headers) - .timeout(Duration::from_millis(config.decision_timeout_millis)) - .build() - .map_err(|error| format!("failed to build Decision API client: {error}"))?; - let target_headers = config - .targets - .iter() - .map(|(id, target)| { - let headers = resolve_json_headers(&target.headers, &target.header_env)?; - Ok((id.clone(), headers)) - }) - .collect::>()?; - Ok(Self { - config, - client, - target_headers, - translation: translation_engine(), - }) - } - - // Skip the portability guard when no configured target uses a different protocol: with no - // possible cross-protocol translation, provider-specific fields never need to be portable. - fn may_translate_protocol(&self, inbound: WireProtocol) -> bool { - self.config - .targets - .values() - .any(|target| target.protocol != inbound) - } - - async fn require_healthy_sidecar(&self) -> Result<(), String> { - let health_url = switchyard_health_url(&self.config.decision_api_url)?; - let client = reqwest::Client::builder() - .timeout(SWITCHYARD_HEALTH_TIMEOUT) - .build() - .map_err(|error| format!("failed to build Switchyard health client: {error}"))?; - let mut backoff = SWITCHYARD_HEALTH_INITIAL_BACKOFF; - let mut final_error = None; - for attempt in 1..=SWITCHYARD_HEALTH_MAX_ATTEMPTS { - match check_switchyard_health(&client, &health_url).await { - Ok(()) => return Ok(()), - Err(error) => final_error = Some(error), - } - if attempt < SWITCHYARD_HEALTH_MAX_ATTEMPTS { - tokio::time::sleep(backoff).await; - backoff *= 2; - } - } - Err(final_error.expect("at least one Switchyard health attempt is configured")) - } - - async fn execute_buffered( - &self, - name: &str, - original: LlmRequest, - next: nemo_relay::api::runtime::LlmExecutionNextFn, - ) -> FlowResult { - let Some(inbound) = WireProtocol::from_call(name, &original) else { - return next(original).await; - }; - if !self.config.enabled_inbound_profiles.contains(&inbound) { - return next(original).await; - } - if self.may_translate_protocol(inbound) - && let Err(error) = validate_portable_request(&self.translation, inbound, &original) - { - self.emit_error( - None, - 0, - "unsupported_provider_extension", - &error.to_string(), - ); - return self - .dispatch_fallback_buffered( - inbound, - original, - next, - "unsupported_provider_extension", - ) - .await; - } - - if self.config.mode == RoutingMode::ObserveOnly { - match self.decided_request(inbound, &original, 1, None).await { - Ok((_, decision, _)) => { - self.record_routing_contribution(&decision, 1, false); - } - Err(error) => self.emit_error(None, 1, "decision_api", &error), - } - return self - .dispatch_fallback_buffered(inbound, original, next, "observe_only") - .await; - } - - let max_attempts = self.config.max_retries.saturating_add(1); - let mut previous = None; - for attempt in 1..=max_attempts { - match self - .buffered_attempt(inbound, &original, &next, attempt, previous, max_attempts) - .await? - { - BufferedAttempt::Complete(response) => return Ok(response), - BufferedAttempt::Retry(retry) => previous = Some(retry), - BufferedAttempt::Fallback(reason) => { - return self - .dispatch_fallback_buffered(inbound, original, next, reason) - .await; - } - } - } - unreachable!("routing attempt loop always returns") - } - - async fn buffered_attempt( - &self, - inbound: WireProtocol, - original: &LlmRequest, - next: &nemo_relay::api::runtime::LlmExecutionNextFn, - attempt: u32, - previous: Option<(String, String)>, - max_attempts: u32, - ) -> FlowResult { - let (routing_request, decision, routed) = match self - .decided_request(inbound, original, attempt, previous) - .await - { - Ok(value) => value, - Err(error) => { - self.emit_error(None, attempt, "decision_api", &error); - return Ok(BufferedAttempt::Fallback("decision_error")); - } - }; - let target_protocol = protocol_from_label(&decision.route.target_protocol_profile)?; - match next(routed).await { - Ok(response) => { - match translate_response(&self.translation, target_protocol, inbound, &response) { - Ok(response) => { - self.record_routing_contribution(&decision, attempt, true); - Ok(BufferedAttempt::Complete(response)) - } - Err(error) => { - self.emit_error( - Some(&routing_request), - attempt, - "response_translation", - &error.to_string(), - ); - Ok(BufferedAttempt::Fallback("translation_error")) - } - } - } - Err(error) if error_is_retryable(&error) && attempt < max_attempts => { - let retry_reason = provider_error_summary(&error); - self.emit_error(Some(&routing_request), attempt, "provider", &retry_reason); - self.emit_retry(&routing_request, &decision, attempt, &retry_reason); - Ok(BufferedAttempt::Retry(( - decision.route.backend_id, - retry_reason, - ))) - } - Err(error) => { - let summary = provider_error_summary(&error); - self.emit_error(Some(&routing_request), attempt, "provider", &summary); - Ok(BufferedAttempt::Fallback(provider_fallback_reason(&error))) - } - } - } - - async fn execute_stream( - &self, - name: &str, - original: LlmRequest, - next: nemo_relay::api::runtime::LlmStreamExecutionNextFn, - ) -> FlowResult { - let Some(inbound) = WireProtocol::from_call(name, &original) else { - return next(original).await; - }; - if !self.config.enabled_inbound_profiles.contains(&inbound) { - return next(original).await; - } - if self.may_translate_protocol(inbound) - && let Err(error) = validate_portable_request(&self.translation, inbound, &original) - { - self.emit_error( - None, - 0, - "unsupported_provider_extension", - &error.to_string(), - ); - return self - .dispatch_fallback_stream(inbound, original, next, "unsupported_provider_extension") - .await; - } - if self.config.mode == RoutingMode::ObserveOnly { - match self.decided_request(inbound, &original, 1, None).await { - Ok((_, decision, _)) => { - self.record_routing_contribution(&decision, 1, false); - } - Err(error) => self.emit_error(None, 1, "decision_api", &error), - } - return self - .dispatch_fallback_stream(inbound, original, next, "observe_only") - .await; - } - - let max_attempts = self.config.max_retries.saturating_add(1); - let mut previous = None; - for attempt in 1..=max_attempts { - match self - .stream_attempt(inbound, &original, &next, attempt, previous, max_attempts) - .await? - { - StreamAttempt::Committed(stream) => return Ok(stream), - StreamAttempt::Retry(retry) => previous = Some(retry), - StreamAttempt::Fallback(reason) => { - return self - .dispatch_fallback_stream(inbound, original, next, reason) - .await; - } - } - } - unreachable!("stream routing attempt loop always returns") - } - - async fn stream_attempt( - &self, - inbound: WireProtocol, - original: &LlmRequest, - next: &nemo_relay::api::runtime::LlmStreamExecutionNextFn, - attempt: u32, - previous: Option<(String, String)>, - max_attempts: u32, - ) -> FlowResult { - let (routing_request, decision, routed) = match self - .decided_request(inbound, original, attempt, previous) - .await - { - Ok(value) => value, - Err(error) => { - self.emit_error(None, attempt, "decision_api", &error); - return Ok(StreamAttempt::Fallback("decision_error")); - } - }; - let target_protocol = protocol_from_label(&decision.route.target_protocol_profile)?; - let context = StreamAttemptContext { - routing_request, - decision, - attempt, - max_attempts, - }; - match next(routed).await { - Ok(mut upstream) => { - let first = upstream.next().await; - Ok(self.classify_open_stream(inbound, target_protocol, context, upstream, first)) - } - Err(error) => Ok(self.classify_stream_setup_error(context, error)), - } - } - - fn classify_open_stream( - &self, - inbound: WireProtocol, - target_protocol: WireProtocol, - context: StreamAttemptContext, - upstream: LlmJsonStream, - first: Option>, - ) -> StreamAttempt { - let StreamAttemptContext { - routing_request, - decision, - attempt, - max_attempts, - } = context; - match first { - Some(Ok(first)) => { - self.record_routing_contribution(&decision, attempt, true); - let committed = LlmJsonStream::from_closeable(PrefixedStream { - first: Some(Ok(first)), - upstream, - }); - let output = if target_protocol == inbound { - committed - } else { - translated_stream( - target_protocol, - inbound, - decision.route.target_model.clone(), - committed, - ) - }; - StreamAttempt::Committed(mark_terminal_stream( - output, - "provider_stream_committed", - self.config.mode.label(), - identity_metadata(&routing_request), - )) - } - Some(Err(error)) if error_is_retryable(&error) && attempt < max_attempts => self - .retry_stream_attempt( - &routing_request, - decision, - attempt, - "provider_stream_open", - provider_error_summary(&error), - ), - None if attempt < max_attempts => self.retry_stream_attempt( - &routing_request, - decision, - attempt, - "provider_stream_open", - "empty_stream".into(), - ), - Some(Err(error)) => { - let summary = provider_error_summary(&error); - self.emit_error( - Some(&routing_request), - attempt, - "provider_stream_open", - &summary, - ); - StreamAttempt::Fallback(provider_fallback_reason(&error)) - } - None => StreamAttempt::Fallback("empty_stream"), - } - } - - fn classify_stream_setup_error( - &self, - context: StreamAttemptContext, - error: FlowError, - ) -> StreamAttempt { - let StreamAttemptContext { - routing_request, - decision, - attempt, - max_attempts, - } = context; - let summary = provider_error_summary(&error); - if error_is_retryable(&error) && attempt < max_attempts { - return self.retry_stream_attempt( - &routing_request, - decision, - attempt, - "provider_stream_setup", - summary, - ); - } - self.emit_error( - Some(&routing_request), - attempt, - "provider_stream_setup", - &summary, - ); - StreamAttempt::Fallback(provider_fallback_reason(&error)) - } - - fn retry_stream_attempt( - &self, - routing_request: &RoutingRequest, - decision: RoutingDecision, - attempt: u32, - error_class: &str, - reason: String, - ) -> StreamAttempt { - if reason != "empty_stream" { - self.emit_error(Some(routing_request), attempt, error_class, &reason); - } - self.emit_retry(routing_request, &decision, attempt, &reason); - StreamAttempt::Retry((decision.route.backend_id, reason)) - } - - async fn decided_request( - &self, - inbound: WireProtocol, - original: &LlmRequest, - attempt: u32, - previous: Option<(String, String)>, - ) -> Result<(RoutingRequest, RoutingDecision, LlmRequest), String> { - let request = self.routing_request(inbound, original, attempt, previous)?; - self.emit_requested(&request); - let started = Instant::now(); - let response = self - .client - .post(&self.config.decision_api_url) - .header("x-nemo-relay-session-id", &request.identity.session_id) - .json(&request) - .send() - .await - .map_err(|error| format!("Decision API request failed: {error}"))?; - let status = response.status(); - if !status.is_success() { - let body = response.text().await.unwrap_or_default(); - return Err(format!("Decision API returned HTTP {status}: {body}")); - } - let decision = response - .json::() - .await - .map_err(|error| format!("Decision API returned invalid JSON: {error}"))?; - self.validate_decision(&decision)?; - if let Some(baseline) = decision.baseline_route.as_ref() - && let Err(error) = self.validate_target(baseline) - { - self.emit_error(Some(&request), attempt, "baseline_binding", &error); - } - let routed = self.apply_target(inbound, original.clone(), &decision)?; - let latency = started.elapsed().as_millis() as u64; - self.emit_decision( - &request, - &decision, - attempt, - self.config.mode == RoutingMode::ObserveOnly, - latency, - ); - Ok((request, decision, routed)) - } - - fn routing_request( - &self, - inbound: WireProtocol, - request: &LlmRequest, - attempt: u32, - previous: Option<(String, String)>, - ) -> Result { - let session = header(request, "x-nemo-relay-session-id"); - let stable_request_id = header(request, "x-nemo-relay-request-id"); - if self.config.context_mode == ContextMode::AtofRequired - && (session.is_none() || stable_request_id.is_none()) - { - return Err("stable session and request identity are required for this profile".into()); - } - let identity_is_stable = session.is_some() && stable_request_id.is_some(); - let synthetic_session = format!("request-{}", Uuid::now_v7()); - let session_id = session.unwrap_or_else(|| synthetic_session.clone()); - let request_id = stable_request_id.unwrap_or_else(|| format!("request-{}", Uuid::now_v7())); - let annotated = decode_request(&self.translation, inbound, request) - .map_err(|error| format!("request translation decode failed: {error}"))?; - let current_request = self.materialize(inbound, request, &annotated)?; - let (previous_route, retry_reason) = previous.unzip(); - Ok(RoutingRequest { - schema_version: ROUTING_REQUEST_SCHEMA_VERSION.into(), - decision_profile: DecisionProfile { - profile_id: self.config.decision_profile_id.clone(), - request_materialization: self.config.request_materialization, - }, - identity: RequestIdentity { - session_id, - request_id, - turn_id: header(request, "x-nemo-relay-turn-id"), - parent_scope_id: header(request, "x-nemo-relay-parent-scope-id"), - root_scope_id: header(request, "x-nemo-relay-root-scope-id"), - harness: header(request, "x-nemo-relay-agent-kind") - .unwrap_or_else(|| "unknown".into()), - source: header(request, "x-nemo-relay-source") - .unwrap_or_else(|| "nemo-relay".into()), - owner_id: header(request, "x-nemo-relay-owner-id"), - quality: header(request, "x-nemo-relay-identity-quality").unwrap_or_else(|| { - if identity_is_stable { - "explicit".into() - } else { - "synthetic".into() - } - }), - }, - protocol: RequestProtocol { - inbound_profile: inbound.label().into(), - inbound_endpoint: inbound.endpoint().into(), - desired_response_profile: inbound.label().into(), - }, - request_summary: RequestSummary { - client_requested_model: request - .content - .get("model") - .and_then(Json::as_str) - .map(ToOwned::to_owned), - prompt_token_estimate: None, - tool_count_in_payload: request - .content - .get("tools") - .and_then(Json::as_array) - .map(|tools| tools.len() as u64), - has_system_prompt: Some( - annotated.instructions.iter().any(|instruction| { - instruction.role == switchyard_translation::Role::System - }) || annotated - .messages - .iter() - .any(|message| message.role == switchyard_translation::Role::System), - ), - }, - current_request, - attempt: DecisionAttempt { - routing_attempt: attempt, - max_routing_attempts: self.config.max_retries.saturating_add(1), - previous_route, - retry_reason, - }, - }) - } - - fn materialize( - &self, - inbound: WireProtocol, - request: &LlmRequest, - annotated: &switchyard_translation::LlmRequest, - ) -> Result, String> { - match self.config.request_materialization { - RequestMaterialization::None | RequestMaterialization::SummaryOnly => Ok(None), - RequestMaterialization::FullBody => Ok(Some(json!({"body": request.content}))), - RequestMaterialization::AnnotatedRequest => Ok(Some(json!({ - "body": request.content, - "annotated_request": annotated, - }))), - RequestMaterialization::LatestUserPrompt => { - let prompt = latest_user_prompt(annotated) - .ok_or_else(|| "latest_user_prompt requires a user message".to_string())?; - let latest = recent_message_window(annotated, 1); - let body = encode_request(&self.translation, inbound, &latest, Map::new()) - .map_err(|error| format!("latest user prompt encode failed: {error}"))? - .content; - Ok(Some(json!({"body": body, "latest_user_prompt": prompt}))) - } - RequestMaterialization::RecentMessageWindow => { - let window = recent_message_window(annotated, self.config.recent_message_count); - let body = encode_request(&self.translation, inbound, &window, Map::new()) - .map_err(|error| format!("recent window encode failed: {error}"))? - .content; - Ok(Some(json!({"body": body, "annotated_request": window}))) - } - } - } - - fn validate_decision(&self, decision: &RoutingDecision) -> Result<(), String> { - if decision.schema_version != ROUTING_DECISION_SCHEMA_VERSION { - return Err(format!( - "unsupported decision schema {:?}", - decision.schema_version - )); - } - self.validate_target(&decision.route).map(|_| ()) - } - - fn validate_target(&self, target: &RoutingTarget) -> Result<&TargetBinding, String> { - let binding = self - .config - .targets - .get(&target.backend_id) - .ok_or_else(|| format!("unknown backend_id {:?}", target.backend_id))?; - if binding.model != target.target_model - || binding.protocol.label() != target.target_protocol_profile - || binding.endpoint != target.target_endpoint - { - return Err(format!( - "decision target {:?} does not match its exact Relay binding", - target.backend_id - )); - } - Ok(binding) - } - - fn record_routing_contribution(&self, decision: &RoutingDecision, attempt: u32, applied: bool) { - let Some(contribution) = self.routing_contribution(decision, attempt, applied) else { - return; - }; - let _ = record_llm_optimization_contribution(contribution); - } - - fn routing_contribution( - &self, - decision: &RoutingDecision, - attempt: u32, - applied: bool, - ) -> Option { - let baseline = decision - .baseline_route - .as_ref() - .filter(|baseline| self.validate_target(baseline).is_ok())?; - let mut contribution = LlmOptimizationContribution::new( - SWITCHYARD_PLUGIN_KIND, - LlmOptimizationKind::model_routing(), - ); - contribution.applied = applied; - contribution.model_transition = Some(LlmOptimizationModelTransition { - baseline: Some(LlmOptimizationModel::new(&baseline.target_model)), - effective: Some(LlmOptimizationModel::new(&decision.route.target_model)), - }); - contribution.payload_schema = Some(DataSchema { - name: ROUTING_CONTRIBUTION_SCHEMA.to_string(), - version: "1".to_string(), - }); - contribution.payload = Some(json!({ - "decision_id": decision.decision_id, - "selected_backend_id": decision.route.backend_id, - "selected_tier": decision.route.tier, - "baseline_backend_id": baseline.backend_id, - "baseline_tier": baseline.tier, - "routing_attempt": attempt, - "rollout_mode": self.config.mode.label(), - "reason_code": decision.reason_code, - "reason_summary": decision.reason_summary, - "router_metadata": decision.metadata, - })); - Some(contribution) - } - - fn apply_target( - &self, - inbound: WireProtocol, - request: LlmRequest, - decision: &RoutingDecision, - ) -> Result { - let binding = self - .config - .targets - .get(&decision.route.backend_id) - .ok_or_else(|| format!("unknown backend_id {:?}", decision.route.backend_id))?; - let annotated = decode_request(&self.translation, inbound, &request) - .map_err(|error| format!("request decode failed: {error}"))?; - let mut routed = if inbound == binding.protocol { - request - } else { - encode_request( - &self.translation, - binding.protocol, - &annotated, - request.headers, - ) - .map_err(|error| format!("request translation failed: {error}"))? - }; - let object = routed - .content - .as_object_mut() - .ok_or_else(|| "translated request body is not an object".to_string())?; - object.insert("model".into(), Json::String(binding.model.clone())); - if let Some(headers) = self.target_headers.get(&decision.route.backend_id) { - routed.headers.extend(headers.clone()); - } - // The backend ID is Relay-owned cache-partition metadata. Remove source or - // target-header case variants before publishing the canonical value. - routed - .headers - .retain(|name, _| !name.eq_ignore_ascii_case(INTERNAL_DISPATCH_BACKEND_HEADER)); - routed.headers.insert( - INTERNAL_DISPATCH_BACKEND_HEADER.into(), - Json::String(decision.route.backend_id.clone()), - ); - routed.headers.insert( - INTERNAL_DISPATCH_ROUTE_HEADER.into(), - Json::String(binding.protocol.label().into()), - ); - routed.headers.insert( - INTERNAL_DISPATCH_URL_HEADER.into(), - Json::String(dispatch_url(&binding.base_url, &binding.endpoint)), - ); - routed.headers.insert( - INTERNAL_RETRY_AWARE_HEADER.into(), - Json::String("true".into()), - ); - Ok(routed) - } - - fn fallback_request( - &self, - inbound: WireProtocol, - request: LlmRequest, - ) -> Result { - let id = self.config.default_targets.target(inbound); - let binding = self - .config - .targets - .get(id) - .ok_or_else(|| format!("unknown fallback target {id:?}"))?; - let decision = RoutingDecision { - schema_version: ROUTING_DECISION_SCHEMA_VERSION.into(), - decision_id: "relay-fallback".into(), - router: crate::contract::DecisionProvider { - name: "relay-fallback".into(), - version: "1".into(), - }, - route: crate::contract::RoutingTarget { - tier: "fallback".into(), - target_model: binding.model.clone(), - backend_id: id.to_string(), - target_protocol_profile: binding.protocol.label().into(), - target_endpoint: binding.endpoint.clone(), - }, - baseline_route: None, - confidence: None, - reason_code: Some("relay_trusted_fallback".into()), - reason_summary: None, - metadata: BTreeMap::new(), - extra: BTreeMap::new(), - }; - self.apply_target(inbound, request, &decision) - } - - async fn dispatch_fallback_buffered( - &self, - inbound: WireProtocol, - original: LlmRequest, - next: nemo_relay::api::runtime::LlmExecutionNextFn, - reason: &str, - ) -> FlowResult { - self.emit_fallback(inbound, reason, &original); - let metadata = identity_metadata_from_request(&original); - let request = self - .fallback_request(inbound, original) - .map_err(FlowError::Internal)?; - match next(request).await { - Ok(response) => Ok(response), - Err(error) => { - emit_terminal_error( - &error, - "fallback_buffered", - self.config.mode.label(), - metadata, - ); - Err(error) - } - } - } - - async fn dispatch_fallback_stream( - &self, - inbound: WireProtocol, - original: LlmRequest, - next: nemo_relay::api::runtime::LlmStreamExecutionNextFn, - reason: &str, - ) -> FlowResult { - self.emit_fallback(inbound, reason, &original); - let metadata = identity_metadata_from_request(&original); - let request = self - .fallback_request(inbound, original) - .map_err(FlowError::Internal)?; - match next(request).await { - Ok(stream) => Ok(mark_terminal_stream( - stream, - "fallback_stream", - self.config.mode.label(), - metadata.clone(), - )), - Err(error) => { - emit_terminal_error( - &error, - "fallback_stream_setup", - self.config.mode.label(), - metadata, - ); - Err(error) - } - } - } - - fn emit_requested(&self, request: &RoutingRequest) { - emit_mark( - "switchyard.routing.requested", - json!({ - "session_id": request.identity.session_id, - "request_id": request.identity.request_id, - "routing_attempt": request.attempt.routing_attempt, - "profile_id": request.decision_profile.profile_id, - "rollout_mode": self.config.mode.label(), - }), - identity_metadata(request), - ); - } - - fn emit_decision( - &self, - request: &RoutingRequest, - decision: &RoutingDecision, - attempt: u32, - observe_only: bool, - latency_ms: u64, - ) { - emit_mark( - "switchyard.routing.decision", - json!({ - "decision_id": decision.decision_id, - "profile_id": request.decision_profile.profile_id, - "router": decision.router.name, - "router_version": decision.router.version, - "routing_attempt": attempt, - "backend_id": decision.route.backend_id, - "selected_tier": decision.route.tier, - "selected_model": decision.route.target_model, - "target_protocol_profile": decision.route.target_protocol_profile, - "target_endpoint": decision.route.target_endpoint, - "confidence": decision.confidence, - "reason_code": decision.reason_code, - "reason_summary": decision.reason_summary, - "router_metadata": decision.metadata, - "latency_ms": latency_ms, - "observe_only": observe_only, - "rollout_mode": self.config.mode.label(), - }), - identity_metadata(request), - ); - } - - fn emit_retry( - &self, - request: &RoutingRequest, - decision: &RoutingDecision, - attempt: u32, - reason: &str, - ) { - emit_mark( - "switchyard.routing.retry", - json!({"routing_attempt": attempt, "previous_route": decision.route.backend_id, "retry_reason": reason, "rollout_mode": self.config.mode.label()}), - identity_metadata(request), - ); - } - - fn emit_error(&self, request: Option<&RoutingRequest>, attempt: u32, class: &str, error: &str) { - emit_mark( - "switchyard.routing.error", - json!({"routing_attempt": attempt, "error_class": class, "error": error, "rollout_mode": self.config.mode.label()}), - request.map(identity_metadata).unwrap_or_else(|| json!({})), - ); - } - - fn emit_fallback(&self, inbound: WireProtocol, reason: &str, request: &LlmRequest) { - emit_mark( - "switchyard.routing.fallback", - json!({ - "fallback_reason": reason, - "fallback_route": self.config.default_targets.target(inbound), - "inbound_profile": inbound.label(), - "rollout_mode": self.config.mode.label(), - }), - identity_metadata_from_request(request), - ); - } -} - -async fn check_switchyard_health( - client: &reqwest::Client, - health_url: &reqwest::Url, -) -> Result<(), String> { - let response = client - .get(health_url.clone()) - .send() - .await - .map_err(|error| { - format!("Switchyard service is required but health check {health_url} failed: {error}") - })?; - let status = response.status(); - if !status.is_success() { - return Err(format!( - "Switchyard service is required but health check {health_url} returned HTTP {status}" - )); - } - let body = response.json::().await.map_err(|error| { - format!("Switchyard health check {health_url} returned invalid JSON: {error}") - })?; - if body.get("status").and_then(Json::as_str) != Some("ok") { - return Err(format!( - "Switchyard health check {health_url} did not report status=ok" - )); - } - Ok(()) -} - -fn switchyard_health_url(decision_api_url: &str) -> Result { - let mut url = reqwest::Url::parse(decision_api_url) - .map_err(|error| format!("decision_api_url is invalid: {error}"))?; - url.set_path(SWITCHYARD_HEALTH_PATH); - url.set_query(None); - url.set_fragment(None); - Ok(url) -} - -fn validate_atof_endpoint_name(name: Option<&str>) -> Result, String> { - if let Some(name) = name { - if name.trim().is_empty() { - return Err("atof_endpoint_name must be non-empty when configured".into()); - } - if name != name.trim() { - return Err("atof_endpoint_name must not have leading or trailing whitespace".into()); - } - } - Ok(name) -} - -fn validate_config(config: &SwitchyardConfig) -> Result<(), String> { - validate_scalar_config(config)?; - validate_decision_api_url(&config.decision_api_url)?; - validate_target_bindings(config)?; - validate_default_targets(config) -} - -fn validate_scalar_config(config: &SwitchyardConfig) -> Result<(), String> { - if config.version != 1 { - return Err(format!( - "unsupported Switchyard config version {}", - config.version - )); - } - if config.decision_profile_id.trim().is_empty() { - return Err("decision_profile_id must be non-empty".into()); - } - if config.decision_timeout_millis == 0 { - return Err("decision_timeout_millis must be greater than zero".into()); - } - if config.max_retries > 10 { - return Err("max_retries must not exceed 10".into()); - } - if config.recent_message_count == 0 { - return Err("recent_message_count must be greater than zero".into()); - } - let atof_endpoint_name = validate_atof_endpoint_name(config.atof_endpoint_name.as_deref())?; - if config.context_mode == ContextMode::AtofRequired && atof_endpoint_name.is_none() { - return Err("atof_required Switchyard profiles require atof_endpoint_name".into()); - } - Ok(()) -} - -fn validate_decision_api_url(decision_api_url: &str) -> Result<(), String> { - let url = reqwest::Url::parse(decision_api_url) - .map_err(|error| format!("decision_api_url is invalid: {error}"))?; - if !matches!(url.scheme(), "http" | "https") { - return Err("decision_api_url must use http or https".into()); - } - Ok(()) -} - -fn validate_target_bindings(config: &SwitchyardConfig) -> Result<(), String> { - if config.targets.is_empty() { - return Err("targets must not be empty".into()); - } - if config.enabled_inbound_profiles.is_empty() { - return Err("enabled_inbound_profiles must not be empty".into()); - } - let mut exact_bindings = BTreeSet::new(); - for (id, target) in &config.targets { - if id.trim().is_empty() - || target.model.trim().is_empty() - || target.endpoint.trim().is_empty() - { - return Err("target IDs, models, and endpoints must be non-empty".into()); - } - let base_url = reqwest::Url::parse(&target.base_url) - .map_err(|error| format!("target {id:?} base_url is invalid: {error}"))?; - if !matches!(base_url.scheme(), "http" | "https") { - return Err(format!("target {id:?} base_url must use http or https")); - } - if target.endpoint != target.protocol.endpoint() { - return Err(format!( - "target {id:?} endpoint must be {:?} for {}", - target.protocol.endpoint(), - target.protocol.label() - )); - } - if !exact_bindings.insert(( - target.model.clone(), - target.protocol, - target.endpoint.clone(), - target.base_url.trim_end_matches('/').to_string(), - )) { - return Err(format!( - "target {id:?} conflicts with another exact backend binding" - )); - } - } - Ok(()) -} - -fn validate_default_targets(config: &SwitchyardConfig) -> Result<(), String> { - for &protocol in &config.enabled_inbound_profiles { - let id = config.default_targets.target(protocol); - let target = config - .targets - .get(id) - .ok_or_else(|| format!("default target {id:?} is not configured"))?; - if target.protocol != protocol { - return Err(format!( - "default target {id:?} must use protocol {}", - protocol.label() - )); - } - } - Ok(()) -} - -fn resolve_headers( - static_headers: &BTreeMap, - environment_headers: &BTreeMap, -) -> Result { - let mut headers = HeaderMap::new(); - for (name, value) in static_headers { - insert_http_header(&mut headers, name, value)?; - } - for (name, variable) in environment_headers { - if static_headers - .keys() - .any(|configured| configured.eq_ignore_ascii_case(name)) - { - return Err(format!( - "header {name:?} cannot appear in both headers and header_env" - )); - } - let value = std::env::var(variable) - .map_err(|_| format!("environment variable {variable:?} is not set"))?; - if value.trim().is_empty() { - return Err(format!("environment variable {variable:?} is blank")); - } - insert_http_header(&mut headers, name, &value)?; - } - Ok(headers) -} - -fn resolve_json_headers( - static_headers: &BTreeMap, - environment_headers: &BTreeMap, -) -> Result, String> { - let mut headers = Map::new(); - for (name, value) in static_headers { - headers.insert(name.clone(), Json::String(value.clone())); - } - for (name, variable) in environment_headers { - if static_headers - .keys() - .any(|configured| configured.eq_ignore_ascii_case(name)) - { - return Err(format!( - "target header {name:?} cannot appear in both headers and header_env" - )); - } - let value = std::env::var(variable) - .map_err(|_| format!("environment variable {variable:?} is not set"))?; - if value.trim().is_empty() { - return Err(format!("environment variable {variable:?} is blank")); - } - headers.insert(name.clone(), Json::String(value)); - } - Ok(headers) -} - -fn insert_http_header(headers: &mut HeaderMap, name: &str, value: &str) -> Result<(), String> { - let name = HeaderName::from_bytes(name.as_bytes()) - .map_err(|error| format!("invalid header name: {error}"))?; - let value = - HeaderValue::from_str(value).map_err(|error| format!("invalid header value: {error}"))?; - headers.insert(name, value); - Ok(()) -} - -fn protocol_from_label(label: &str) -> FlowResult { - match label { - "openai_chat" | "openai_chat_completions" | "openai_chat_completions.v1" => { - Ok(WireProtocol::OpenaiChat) - } - "openai_responses" | "openai_responses.v1" => Ok(WireProtocol::OpenaiResponses), - "anthropic_messages" | "anthropic_messages.v1" => Ok(WireProtocol::AnthropicMessages), - value => Err(FlowError::InvalidArgument(format!( - "unsupported Switchyard target protocol {value:?}" - ))), - } -} - -fn header(request: &LlmRequest, name: &str) -> Option { - request - .headers - .get(name) - .and_then(Json::as_str) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(ToOwned::to_owned) -} - -fn dispatch_url(base_url: &str, endpoint: &str) -> String { - let base = base_url.trim_end_matches('/'); - let endpoint = if base.ends_with("/v1") && endpoint.starts_with("/v1/") { - &endpoint[3..] - } else { - endpoint - }; - format!("{base}{endpoint}") -} - -fn identity_metadata(request: &RoutingRequest) -> Json { - json!({ - "session_id": request.identity.session_id, - "request_id": request.identity.request_id, - "turn_id": request.identity.turn_id, - "owner_id": request.identity.owner_id, - }) -} - -fn identity_metadata_from_request(request: &LlmRequest) -> Json { - json!({ - "session_id": header(request, "x-nemo-relay-session-id"), - "request_id": header(request, "x-nemo-relay-request-id"), - "turn_id": header(request, "x-nemo-relay-turn-id"), - "owner_id": header(request, "x-nemo-relay-owner-id"), - }) -} - -fn error_is_retryable(error: &FlowError) -> bool { - matches!(error, FlowError::Upstream(failure) if failure.is_retryable()) -} - -fn emit_mark(name: &str, data: Json, metadata: Json) { - if let Err(error) = event( - EmitMarkEventParams::builder() - .name(name) - .data(data) - .data_schema( - DataSchema::builder() - .name(ROUTING_MARK_SCHEMA) - .version("1") - .build(), - ) - .metadata(metadata) - .category(EventCategory::custom()) - .category_profile(CategoryProfile::builder().subtype(name).build()) - .build(), - ) { - eprintln!("nemo-relay switchyard: failed to emit {name}: {error}"); - } -} - -fn emit_terminal_error(error: &FlowError, phase: &str, rollout_mode: &str, metadata: Json) { - emit_mark( - "switchyard.routing.terminal_error", - json!({"error_class": provider_error_class(error), "error": provider_error_summary(error), "phase": phase, "rollout_mode": rollout_mode}), - metadata, - ); -} - -fn provider_error_class(error: &FlowError) -> &'static str { - match error { - FlowError::Upstream(failure) => match failure.class { - nemo_relay::error::UpstreamFailureClass::Connection => "connection", - nemo_relay::error::UpstreamFailureClass::Timeout => "timeout", - nemo_relay::error::UpstreamFailureClass::RetryableStatus => "retryable_status", - nemo_relay::error::UpstreamFailureClass::ContextWindow => "context_window", - nemo_relay::error::UpstreamFailureClass::ModelUnavailable => "model_unavailable", - nemo_relay::error::UpstreamFailureClass::Authentication => "authentication", - nemo_relay::error::UpstreamFailureClass::InvalidRequest => "invalid_request", - nemo_relay::error::UpstreamFailureClass::Other => "other", - }, - _ => "relay", - } -} - -fn provider_error_summary(error: &FlowError) -> String { - match error { - FlowError::Upstream(failure) => match failure.status { - Some(status) => format!("{}:http_{status}", provider_error_class(error)), - None => provider_error_class(error).to_string(), - }, - _ => error.to_string(), - } -} - -struct PrefixedStream { - first: Option>, - upstream: LlmJsonStream, -} - -impl futures_util::Stream for PrefixedStream { - type Item = FlowResult; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - if let Some(first) = self.first.take() { - Poll::Ready(Some(first)) - } else { - Pin::new(&mut self.upstream).poll_next(cx) - } - } -} - -impl LlmStreamInner for PrefixedStream { - fn close(self: Pin<&mut Self>) -> Pin> + Send + '_>> { - let this = self.get_mut(); - this.first = None; - Box::pin(async move { this.upstream.close().await }) - } -} - -struct TerminalMarkedStream { - upstream: LlmJsonStream, - phase: &'static str, - rollout_mode: &'static str, - metadata: Json, - finished: bool, -} - -impl futures_util::Stream for TerminalMarkedStream { - type Item = FlowResult; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - if self.finished { - return Poll::Ready(None); - } - match Pin::new(&mut self.upstream).poll_next(cx) { - Poll::Ready(Some(Ok(chunk))) => Poll::Ready(Some(Ok(chunk))), - Poll::Ready(Some(Err(error))) => { - self.finished = true; - emit_terminal_error(&error, self.phase, self.rollout_mode, self.metadata.clone()); - Poll::Ready(Some(Err(error))) - } - Poll::Ready(None) => { - self.finished = true; - Poll::Ready(None) - } - Poll::Pending => Poll::Pending, - } - } -} - -impl LlmStreamInner for TerminalMarkedStream { - fn close(self: Pin<&mut Self>) -> Pin> + Send + '_>> { - Box::pin(async move { self.get_mut().upstream.close().await }) - } -} - -struct TranslatedStream { - upstream: LlmJsonStream, - transcoder: StreamTranscoder, - buffered: VecDeque>, - upstream_finished: bool, -} - -impl futures_util::Stream for TranslatedStream { - type Item = FlowResult; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - loop { - if let Some(chunk) = self.buffered.pop_front() { - return Poll::Ready(Some(chunk)); - } - if self.upstream_finished { - return Poll::Ready(None); - } - - match Pin::new(&mut self.upstream).poll_next(cx) { - Poll::Ready(Some(Ok(chunk))) => match self.transcoder.transcode(&chunk) { - Ok(chunks) => self.buffered.extend(chunks.into_iter().map(Ok)), - Err(error) => { - self.upstream_finished = true; - return Poll::Ready(Some(Err(error))); - } - }, - Poll::Ready(Some(Err(error))) => { - self.upstream_finished = true; - return Poll::Ready(Some(Err(error))); - } - Poll::Ready(None) => { - self.upstream_finished = true; - match self.transcoder.finish() { - Ok(chunks) => self.buffered.extend(chunks.into_iter().map(Ok)), - Err(error) => return Poll::Ready(Some(Err(error))), - } - } - Poll::Pending => return Poll::Pending, - } - } - } -} - -impl LlmStreamInner for TranslatedStream { - fn close(self: Pin<&mut Self>) -> Pin> + Send + '_>> { - let this = self.get_mut(); - this.buffered.clear(); - this.upstream_finished = true; - Box::pin(async move { this.upstream.close().await }) - } -} - -fn mark_terminal_stream( - upstream: LlmJsonStream, - phase: &'static str, - rollout_mode: &'static str, - metadata: Json, -) -> LlmJsonStream { - LlmJsonStream::from_closeable(TerminalMarkedStream { - upstream, - phase, - rollout_mode, - metadata, - finished: false, - }) -} - -fn translated_stream( - source: WireProtocol, - target: WireProtocol, - effective_model: String, - upstream: LlmJsonStream, -) -> LlmJsonStream { - LlmJsonStream::from_closeable(TranslatedStream { - upstream, - transcoder: StreamTranscoder::new(source, target, effective_model), - buffered: VecDeque::new(), - upstream_finished: false, - }) -} - -#[cfg(test)] -#[path = "../tests/unit/component_tests.rs"] -mod tests; diff --git a/crates/switchyard/src/contract.rs b/crates/switchyard/src/contract.rs deleted file mode 100644 index 675234ca1..000000000 --- a/crates/switchyard/src/contract.rs +++ /dev/null @@ -1,187 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! JSON compatibility types for the Switchyard Decision API. - -use std::collections::BTreeMap; - -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -/// Routing request schema supported by this plugin. -pub const ROUTING_REQUEST_SCHEMA_VERSION: &str = "switchyard.routing_request.v1"; -/// Routing decision schema supported by this plugin. -pub const ROUTING_DECISION_SCHEMA_VERSION: &str = "switchyard.routing_decision.v1"; - -/// Request-time materialization supplied to Switchyard. -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "snake_case")] -pub enum RequestMaterialization { - /// Identity, protocol, summary, and attempt only. - None, - /// Baseline summary without current request material. - SummaryOnly, - /// Latest user prompt. - LatestUserPrompt, - /// Bounded recent message window. - RecentMessageWindow, - /// Relay-normalized request plus its provider body. - AnnotatedRequest, - /// Complete provider request body. - FullBody, -} - -/// Switchyard profile selection. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct DecisionProfile { - /// Profile ID loaded by Switchyard. - pub profile_id: String, - /// Request materialization mode. - pub request_materialization: RequestMaterialization, -} - -/// Normalized Relay identity. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct RequestIdentity { - /// Stable session identifier. - pub session_id: String, - /// Per-request identifier. - pub request_id: String, - /// Optional turn identifier. - #[serde(skip_serializing_if = "Option::is_none")] - pub turn_id: Option, - /// Optional parent scope identifier. - #[serde(skip_serializing_if = "Option::is_none")] - pub parent_scope_id: Option, - /// Optional root scope identifier. - #[serde(skip_serializing_if = "Option::is_none")] - pub root_scope_id: Option, - /// Harness name. - pub harness: String, - /// Request source. - pub source: String, - /// Optional resolved work owner. - #[serde(skip_serializing_if = "Option::is_none")] - pub owner_id: Option, - /// Native, explicit, or synthetic identity quality. - pub quality: String, -} - -/// Inbound protocol context. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct RequestProtocol { - /// Inbound protocol profile. - pub inbound_profile: String, - /// Inbound endpoint. - pub inbound_endpoint: String, - /// Response profile expected by the harness. - pub desired_response_profile: String, -} - -/// Cheap provider-request summary. -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -pub struct RequestSummary { - /// Client-requested model. - #[serde(skip_serializing_if = "Option::is_none")] - pub client_requested_model: Option, - /// Optional prompt-token estimate. - #[serde(skip_serializing_if = "Option::is_none")] - pub prompt_token_estimate: Option, - /// Number of tools in the request. - #[serde(skip_serializing_if = "Option::is_none")] - pub tool_count_in_payload: Option, - /// Whether a system prompt is present. - #[serde(skip_serializing_if = "Option::is_none")] - pub has_system_prompt: Option, -} - -/// Routing-attempt context. Additive fields are ignored by older Switchyard servers. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct DecisionAttempt { - /// One-indexed routing attempt. - pub routing_attempt: u32, - /// Maximum Decision API attempts. - pub max_routing_attempts: u32, - /// Previously selected backend. - #[serde(skip_serializing_if = "Option::is_none")] - pub previous_route: Option, - /// Reason another decision is requested. - #[serde(skip_serializing_if = "Option::is_none")] - pub retry_reason: Option, -} - -/// Canonical routing request sent by Relay. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct RoutingRequest { - /// Schema identifier. - pub schema_version: String, - /// Switchyard profile selection. - pub decision_profile: DecisionProfile, - /// Normalized identity. - pub identity: RequestIdentity, - /// Inbound protocol. - pub protocol: RequestProtocol, - /// Cheap request summary. - pub request_summary: RequestSummary, - /// Optional request materialization. - #[serde(skip_serializing_if = "Option::is_none")] - pub current_request: Option, - /// Attempt metadata. - pub attempt: DecisionAttempt, -} - -/// Router metadata returned by Switchyard. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct DecisionProvider { - /// Router name. - pub name: String, - /// Router version. - pub version: String, -} - -/// Selected target returned by Switchyard. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct RoutingTarget { - /// Tier label. - pub tier: String, - /// Selected model. - pub target_model: String, - /// Backend binding ID. - pub backend_id: String, - /// Target protocol profile. - pub target_protocol_profile: String, - /// Target endpoint. - pub target_endpoint: String, -} - -/// Canonical Switchyard routing decision. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct RoutingDecision { - /// Schema identifier. - pub schema_version: String, - /// Decision identifier. - pub decision_id: String, - /// Router metadata. - pub router: DecisionProvider, - /// Selected target. - pub route: RoutingTarget, - /// Explicit counterfactual route supplied by routers that define one. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub baseline_route: Option, - /// Optional confidence. - #[serde(default)] - pub confidence: Option, - /// Optional reason code. - #[serde(default)] - pub reason_code: Option, - /// Optional reason summary. - #[serde(default)] - pub reason_summary: Option, - /// Additive router metadata. - #[serde(default)] - pub metadata: BTreeMap, - /// Unknown additive response fields. - #[serde(flatten)] - pub extra: BTreeMap, -} diff --git a/crates/switchyard/src/lib.rs b/crates/switchyard/src/lib.rs deleted file mode 100644 index 238d4337f..000000000 --- a/crates/switchyard/src/lib.rs +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![deny(rustdoc::broken_intra_doc_links, rustdoc::private_intra_doc_links)] - -//! First-party Switchyard Decision API routing plugin for NeMo Relay. - -pub mod component; -pub mod contract; -mod stream_translation; -mod translation; - -pub use component::{ - ContextMode, ProtocolDefaults, RoutingMode, SWITCHYARD_PLUGIN_KIND, SwitchyardConfig, - TargetBinding, WireProtocol, deregister_switchyard_component, register_switchyard_component, - validate_switchyard_atof_configuration, -}; diff --git a/crates/switchyard/src/stream_translation.rs b/crates/switchyard/src/stream_translation.rs deleted file mode 100644 index 808834a2b..000000000 --- a/crates/switchyard/src/stream_translation.rs +++ /dev/null @@ -1,94 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Relay stream adapter for Switchyard's provider translation engine. - -use nemo_relay::error::{FlowError, Result}; -use serde_json::Value as Json; -use switchyard_translation::{StreamTranslationState, TranslationEngine}; - -use crate::component::WireProtocol; -use crate::translation::wire_format; - -pub(crate) struct StreamTranscoder { - engine: TranslationEngine, - source: WireProtocol, - target: WireProtocol, - state: StreamTranslationState, -} - -impl StreamTranscoder { - pub(crate) fn new( - source: WireProtocol, - target: WireProtocol, - effective_model: impl Into, - ) -> Self { - let effective_model = effective_model.into(); - let mut state = StreamTranslationState::new(wire_format(source), wire_format(target)); - state.model = Some(effective_model.clone()); - state.target_model = Some(effective_model); - Self { - engine: TranslationEngine::default(), - source, - target, - state, - } - } - - pub(crate) fn transcode(&mut self, chunk: &Json) -> Result> { - if unsupported_stream_chunk(self.source, chunk) { - return Err(FlowError::InvalidArgument( - "provider-specific streaming extension cannot be translated safely".into(), - )); - } - self.engine - .translate_event( - &mut self.state, - wire_format(self.source), - wire_format(self.target), - chunk, - ) - .map_err(|error| { - FlowError::InvalidArgument(format!("Switchyard stream translation failed: {error}")) - }) - } - - pub(crate) fn finish(&mut self) -> Result> { - self.engine - .finish_stream(&mut self.state, wire_format(self.target)) - .map_err(|error| { - FlowError::InvalidArgument(format!( - "Switchyard stream finalization failed: {error}" - )) - }) - } -} - -fn unsupported_stream_chunk(source: WireProtocol, chunk: &Json) -> bool { - match source { - WireProtocol::OpenaiChat => { - chunk["choices"] - .as_array() - .into_iter() - .flatten() - .any(|choice| { - choice["delta"].get("audio").is_some() - || choice["delta"].get("reasoning_content").is_some() - }) - } - WireProtocol::OpenaiResponses => { - chunk.get("type").and_then(Json::as_str) == Some("response.output_item.added") - && matches!( - chunk["item"].get("type").and_then(Json::as_str), - Some("reasoning" | "computer_call" | "web_search_call") - ) - } - WireProtocol::AnthropicMessages => { - chunk.get("type").and_then(Json::as_str) == Some("content_block_start") - && !matches!( - chunk["content_block"].get("type").and_then(Json::as_str), - Some("text" | "tool_use") - ) - } - } -} diff --git a/crates/switchyard/src/translation.rs b/crates/switchyard/src/translation.rs deleted file mode 100644 index 4823bc845..000000000 --- a/crates/switchyard/src/translation.rs +++ /dev/null @@ -1,320 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -use nemo_relay::api::llm::LlmRequest; -use nemo_relay::error::{FlowError, Result}; -use serde_json::{Map, Value as Json}; -use switchyard_translation::{ - ContentBlock, DeterministicIdPolicy, ImageSource, LlmRequest as TranslationRequest, - LossyConversionPolicy, PreservationPolicy, Role, TargetCapabilities, TranslationDiagnostic, - TranslationEngine, TranslationPolicy, UnknownFieldPolicy, WireFormat, -}; - -use crate::component::WireProtocol; - -pub(crate) fn translation_engine() -> TranslationEngine { - TranslationEngine::default() -} - -pub(crate) fn decode_request( - engine: &TranslationEngine, - protocol: WireProtocol, - request: &LlmRequest, -) -> Result { - let output = engine - .decode_request( - wire_format(protocol), - &request.content, - &translation_policy(), - ) - .map_err(translation_error)?; - ensure_no_diagnostics(&output.diagnostics)?; - Ok(output.request) -} - -pub(crate) fn encode_request( - engine: &TranslationEngine, - protocol: WireProtocol, - request: &TranslationRequest, - headers: Map, -) -> Result { - let output = engine - .encode_request(wire_format(protocol), request, &translation_policy()) - .map_err(translation_error)?; - ensure_no_diagnostics(&output.diagnostics)?; - Ok(LlmRequest { - headers, - content: output.body, - }) -} - -pub(crate) fn validate_portable_request( - engine: &TranslationEngine, - protocol: WireProtocol, - request: &LlmRequest, -) -> Result<()> { - const RESTRICTED_KEYS: &[&str] = &[ - "cache_control", - "audio", - "thinking", - "computer_use", - "server_tool_use", - ]; - if protocol == WireProtocol::AnthropicMessages - && contains_invalid_anthropic_image_source(&request.content) - { - return Err(FlowError::InvalidArgument( - "request uses an unsupported or malformed Anthropic image source".into(), - )); - } - if contains_any_key_recursive(&request.content, RESTRICTED_KEYS) { - return Err(FlowError::InvalidArgument( - "request uses a provider-specific extension that requires same-protocol fail-open" - .into(), - )); - } - let decoded = decode_request(engine, protocol, request)?; - if decoded.reasoning.effort.is_some() - || decoded.reasoning.raw.is_some() - || decoded - .extensions - .fields - .iter() - .any(|(key, value)| key != "stream_options" || !portable_stream_options(value)) - || request_contains_unsupported_content(&decoded) - { - return Err(FlowError::InvalidArgument( - "request uses provider-specific fields that cannot be translated safely".into(), - )); - } - Ok(()) -} - -pub(crate) fn latest_user_prompt(request: &TranslationRequest) -> Option { - request - .messages - .iter() - .rev() - .find(|message| message.role == Role::User) - .and_then(|message| message.text_content("\n")) -} - -pub(crate) fn recent_message_window( - request: &TranslationRequest, - count: usize, -) -> TranslationRequest { - let mut window = request.clone(); - let split = window.messages.len().saturating_sub(count); - window.messages = window.messages.split_off(split); - window -} - -pub(crate) fn translate_response( - engine: &TranslationEngine, - source: WireProtocol, - target: WireProtocol, - response: &Json, -) -> Result { - if source == target { - return Ok(response.clone()); - } - ensure_portable_response(source, response)?; - let output = engine - .translate_response( - wire_format(source), - wire_format(target), - response, - &translation_policy(), - ) - .map_err(translation_error)?; - ensure_no_diagnostics(&output.diagnostics)?; - Ok(output.body) -} - -pub(crate) const fn wire_format(protocol: WireProtocol) -> WireFormat { - match protocol { - WireProtocol::OpenaiChat => WireFormat::OpenAiChat, - WireProtocol::OpenaiResponses => WireFormat::OpenAiResponses, - WireProtocol::AnthropicMessages => WireFormat::AnthropicMessages, - } -} - -fn translation_policy() -> TranslationPolicy { - TranslationPolicy { - unknown_field_policy: UnknownFieldPolicy::Reject, - lossy_conversion_policy: LossyConversionPolicy::Reject, - deterministic_ids: DeterministicIdPolicy::GenerateStable { - prefix: "relay".into(), - }, - preservation: PreservationPolicy::Disabled, - target_capabilities: TargetCapabilities::default(), - } -} - -fn translation_error(error: switchyard_translation::TranslationError) -> FlowError { - FlowError::InvalidArgument(format!("Switchyard translation failed: {error}")) -} - -fn ensure_no_diagnostics(diagnostics: &[TranslationDiagnostic]) -> Result<()> { - if diagnostics.is_empty() { - Ok(()) - } else { - Err(FlowError::InvalidArgument(format!( - "Switchyard translation was not lossless: {diagnostics:?}" - ))) - } -} - -fn portable_stream_options(value: &Json) -> bool { - let Some(options) = value.as_object() else { - return false; - }; - options.len() == 1 && options.get("include_usage").is_some_and(Json::is_boolean) -} - -fn request_contains_unsupported_content(request: &TranslationRequest) -> bool { - request - .instructions - .iter() - .flat_map(|instruction| instruction.content.iter()) - .chain( - request - .messages - .iter() - .flat_map(|message| message.content.iter()), - ) - .any(unsupported_content_block) -} - -fn unsupported_content_block(block: &ContentBlock) -> bool { - match block { - ContentBlock::Text { .. } | ContentBlock::Refusal { .. } | ContentBlock::ToolCall(_) => { - false - } - ContentBlock::Image { source } => invalid_image_source(source), - ContentBlock::ToolResult(result) => result.content.iter().any(unsupported_content_block), - ContentBlock::Reasoning { .. } - | ContentBlock::Audio { .. } - | ContentBlock::Video { .. } - | ContentBlock::File { .. } - | ContentBlock::Unknown { .. } => true, - } -} - -fn invalid_image_source(source: &ImageSource) -> bool { - match source { - ImageSource::Url { url, .. } => { - url.starts_with("data:") && base64_data_uri_parts(url).is_none() - } - ImageSource::Base64 { media_type, data } => { - media_type.as_deref().is_none_or(str::is_empty) || data.is_empty() - } - ImageSource::Raw(_) => true, - } -} - -fn base64_data_uri_parts(url: &str) -> Option<(&str, &str)> { - let (metadata, data) = url.strip_prefix("data:")?.split_once(',')?; - let media_type = metadata.strip_suffix(";base64")?; - (!media_type.is_empty() && !data.is_empty()).then_some((media_type, data)) -} - -fn contains_invalid_anthropic_image_source(value: &Json) -> bool { - match value { - Json::Object(object) => { - if object.get("type").and_then(Json::as_str) == Some("image") { - let Some(source) = object.get("source").and_then(Json::as_object) else { - return true; - }; - match source.get("type").and_then(Json::as_str) { - Some("url") => source - .get("url") - .and_then(Json::as_str) - .is_none_or(str::is_empty), - Some("base64") => { - source - .get("media_type") - .and_then(Json::as_str) - .is_none_or(str::is_empty) - || source - .get("data") - .and_then(Json::as_str) - .is_none_or(str::is_empty) - } - _ => true, - } - } else { - object.values().any(contains_invalid_anthropic_image_source) - } - } - Json::Array(items) => items.iter().any(contains_invalid_anthropic_image_source), - _ => false, - } -} - -fn contains_any_key_recursive(value: &Json, keys: &[&str]) -> bool { - match value { - Json::Object(object) => { - object.keys().any(|key| keys.contains(&key.as_str())) - || object - .values() - .any(|value| contains_any_key_recursive(value, keys)) - } - Json::Array(items) => items - .iter() - .any(|value| contains_any_key_recursive(value, keys)), - _ => false, - } -} - -fn ensure_portable_response(protocol: WireProtocol, response: &Json) -> Result<()> { - let unsupported = - match protocol { - WireProtocol::OpenaiChat => { - response["choices"] - .as_array() - .into_iter() - .flatten() - .any(|choice| { - choice["message"].get("audio").is_some() - || choice["message"].get("reasoning_content").is_some() - }) - } - WireProtocol::OpenaiResponses => response["output"] - .as_array() - .into_iter() - .flatten() - .any(|item| { - matches!( - item.get("type").and_then(Json::as_str), - Some( - "reasoning" - | "computer_call" - | "computer_call_output" - | "web_search_call" - ) - ) - }), - WireProtocol::AnthropicMessages => response["content"] - .as_array() - .into_iter() - .flatten() - .any(|block| { - !matches!( - block.get("type").and_then(Json::as_str), - Some("text" | "tool_use") - ) - }), - }; - if unsupported { - Err(FlowError::InvalidArgument( - "provider-specific response extension cannot be translated safely".into(), - )) - } else { - Ok(()) - } -} - -#[cfg(test)] -#[path = "../tests/unit/translation_tests.rs"] -mod tests; diff --git a/crates/switchyard/tests/contract.rs b/crates/switchyard/tests/contract.rs deleted file mode 100644 index 7051f6a56..000000000 --- a/crates/switchyard/tests/contract.rs +++ /dev/null @@ -1,79 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Integration tests for the public Switchyard Decision API contract. - -use nemo_relay_switchyard::contract::{DecisionAttempt, RoutingDecision}; -use serde_json::json; - -#[test] -fn current_switchyard_decision_contract_accepts_additive_fields() { - let decision: RoutingDecision = serde_json::from_value(json!({ - "schema_version": "switchyard.routing_decision.v1", - "decision_id": "decision-1", - "router": {"name": "stage_router", "version": "1"}, - "route": { - "tier": "capable", - "target_model": "model-a", - "backend_id": "backend-a", - "target_protocol_profile": "openai_chat", - "target_endpoint": "/v1/chat/completions" - }, - "confidence": 0.8, - "future_field": {"safe": true} - })) - .unwrap(); - assert_eq!( - decision.extra.get("future_field"), - Some(&json!({"safe": true})) - ); - assert!(decision.baseline_route.is_none()); -} - -#[test] -fn current_switchyard_decision_contract_accepts_an_explicit_baseline() { - let decision: RoutingDecision = serde_json::from_value(json!({ - "schema_version": "switchyard.routing_decision.v1", - "decision_id": "decision-1", - "router": {"name": "stage", "version": "1"}, - "route": { - "tier": "efficient", - "target_model": "model-small", - "backend_id": "backend-small", - "target_protocol_profile": "openai_chat", - "target_endpoint": "/v1/chat/completions" - }, - "baseline_route": { - "tier": "capable", - "target_model": "model-large", - "backend_id": "backend-large", - "target_protocol_profile": "anthropic_messages", - "target_endpoint": "/v1/messages" - } - })) - .unwrap(); - let baseline = decision.baseline_route.unwrap(); - assert_eq!(baseline.backend_id, "backend-large"); - assert_eq!(baseline.target_protocol_profile, "anthropic_messages"); -} - -#[test] -fn malformed_decisions_are_rejected_by_serde() { - let missing_route = json!({ - "schema_version": "switchyard.routing_decision.v1", - "decision_id": "decision-1", - "router": {"name": "stage_router", "version": "1"} - }); - assert!(serde_json::from_value::(missing_route).is_err()); -} - -#[test] -fn additive_retry_metadata_stays_optional_on_the_wire() { - let attempt: DecisionAttempt = serde_json::from_value(json!({ - "routing_attempt": 1, - "max_routing_attempts": 4 - })) - .unwrap(); - assert!(attempt.previous_route.is_none()); - assert!(attempt.retry_reason.is_none()); -} diff --git a/crates/switchyard/tests/unit/component_tests.rs b/crates/switchyard/tests/unit/component_tests.rs deleted file mode 100644 index bc5b58ed6..000000000 --- a/crates/switchyard/tests/unit/component_tests.rs +++ /dev/null @@ -1,1903 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Unit tests for the Switchyard Relay plugin component. - -use std::future::Future; -use std::pin::Pin; -use std::sync::Mutex; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::task::{Context, Poll}; - -use axum::{ - Json as AxumJson, Router, - extract::State, - http::StatusCode, - routing::{get, post}, -}; -use futures_util::{Stream, stream as futures_stream}; -use nemo_relay::api::event::{Event, ScopeCategory}; -use nemo_relay::api::llm::{ - LlmCallExecuteParams, LlmStreamCallExecuteParams, llm_call_execute, llm_stream_call_execute, -}; -use nemo_relay::api::runtime::{LlmExecutionNextFn, LlmStreamExecutionNextFn, LlmStreamInner}; -use nemo_relay::api::scope::get_handle; -use nemo_relay::api::subscriber::{ - deregister_subscriber, flush_subscribers, register_subscriber, scope_deregister_subscriber, - scope_register_subscriber, -}; -use nemo_relay::codec::optimization::LlmOptimizationSummaryStatus; -use nemo_relay::error::{UpstreamFailure, UpstreamFailureClass}; -use nemo_relay::plugin::rollback_registrations; - -use super::*; - -struct CloseTrackingStream { - close_calls: Arc, - closed: bool, -} - -impl Stream for CloseTrackingStream { - type Item = FlowResult; - - fn poll_next(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { - Poll::Ready(None) - } -} - -impl LlmStreamInner for CloseTrackingStream { - fn close( - mut self: Pin<&mut Self>, - ) -> Pin> + Send + '_>> { - self.closed = true; - self.close_calls.fetch_add(1, Ordering::SeqCst); - Box::pin(async { Ok(()) }) - } -} - -fn close_tracking_stream(close_calls: Arc) -> LlmJsonStream { - LlmJsonStream::from_closeable(CloseTrackingStream { - close_calls, - closed: false, - }) -} - -fn prefixed_adapter(upstream: LlmJsonStream) -> LlmJsonStream { - LlmJsonStream::from_closeable(PrefixedStream { - first: Some(Ok(json!({"first": true}))), - upstream, - }) -} - -fn terminal_adapter(upstream: LlmJsonStream) -> LlmJsonStream { - mark_terminal_stream(upstream, "test", "enforce", json!({"route": "test"})) -} - -fn translated_adapter(upstream: LlmJsonStream) -> LlmJsonStream { - translated_stream( - WireProtocol::OpenaiChat, - WireProtocol::AnthropicMessages, - "selected".into(), - upstream, - ) -} - -fn binding(protocol: WireProtocol, model: &str) -> TargetBinding { - TargetBinding { - model: model.into(), - protocol, - endpoint: protocol.endpoint().into(), - base_url: "http://127.0.0.1:9999".into(), - headers: BTreeMap::new(), - header_env: BTreeMap::new(), - } -} - -fn config(decision_api_url: String) -> SwitchyardConfig { - SwitchyardConfig { - decision_api_url, - decision_profile_id: "stage_router".into(), - request_materialization: RequestMaterialization::SummaryOnly, - context_mode: ContextMode::PayloadOnly, - decision_timeout_millis: 1_000, - targets: BTreeMap::from([ - ( - "selected-chat".into(), - binding(WireProtocol::OpenaiChat, "selected"), - ), - ( - "baseline-chat".into(), - binding(WireProtocol::OpenaiChat, "baseline"), - ), - ( - "fallback-chat".into(), - binding(WireProtocol::OpenaiChat, "fallback"), - ), - ( - "fallback-responses".into(), - binding(WireProtocol::OpenaiResponses, "fallback"), - ), - ( - "fallback-anthropic".into(), - binding(WireProtocol::AnthropicMessages, "fallback"), - ), - ]), - default_targets: ProtocolDefaults { - openai_chat: "fallback-chat".into(), - openai_responses: "fallback-responses".into(), - anthropic_messages: "fallback-anthropic".into(), - }, - ..SwitchyardConfig::default() - } -} - -fn decision() -> RoutingDecision { - RoutingDecision { - schema_version: ROUTING_DECISION_SCHEMA_VERSION.into(), - decision_id: "decision-1".into(), - router: crate::contract::DecisionProvider { - name: "stage_router".into(), - version: "1".into(), - }, - route: crate::contract::RoutingTarget { - tier: "efficient".into(), - target_model: "selected".into(), - backend_id: "selected-chat".into(), - target_protocol_profile: "openai_chat".into(), - target_endpoint: "/v1/chat/completions".into(), - }, - baseline_route: Some(crate::contract::RoutingTarget { - tier: "capable".into(), - target_model: "baseline".into(), - backend_id: "baseline-chat".into(), - target_protocol_profile: "openai_chat".into(), - target_endpoint: "/v1/chat/completions".into(), - }), - confidence: Some(0.9), - reason_code: Some("test".into()), - reason_summary: None, - metadata: BTreeMap::from([ - ("feature_state".into(), json!("fresh")), - ("snapshot_age_millis".into(), json!(37)), - ("snapshot_max_age_millis".into(), json!(300_000)), - ]), - extra: BTreeMap::new(), - } -} - -fn chat_request() -> LlmRequest { - LlmRequest { - headers: Map::new(), - content: json!({ - "model": "inbound", - "messages": [ - {"role": "system", "content": "system"}, - {"role": "user", "content": "first"}, - {"role": "assistant", "content": "answer"}, - {"role": "user", "content": "latest"} - ] - }), - } -} - -fn request(protocol: WireProtocol) -> LlmRequest { - let content = match protocol { - WireProtocol::OpenaiChat => chat_request().content, - WireProtocol::OpenaiResponses => { - json!({"model": "inbound", "instructions": "system", "input": "latest"}) - } - WireProtocol::AnthropicMessages => { - json!({"model": "inbound", "system": "system", "max_tokens": 32, "messages": [{"role": "user", "content": "latest"}]}) - } - }; - LlmRequest { - headers: Map::new(), - content, - } -} - -fn chat_response() -> Json { - json!({ - "id": "chat-1", "object": "chat.completion", "model": "selected", - "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2} - }) -} - -fn chat_chunk(text: &str, finish_reason: Json) -> Json { - json!({ - "id": "chat-1", "object": "chat.completion.chunk", "model": "selected", - "choices": [{"index": 0, "delta": {"content": text}, "finish_reason": finish_reason}] - }) -} - -#[test] -fn wire_protocol_and_plugin_lifecycle_contracts_are_stable() { - for (protocol, label, endpoint) in [ - ( - WireProtocol::OpenaiChat, - "openai_chat", - "/v1/chat/completions", - ), - ( - WireProtocol::OpenaiResponses, - "openai_responses", - "/v1/responses", - ), - ( - WireProtocol::AnthropicMessages, - "anthropic_messages", - "/v1/messages", - ), - ] { - assert_eq!(protocol.label(), label); - assert_eq!(protocol.endpoint(), endpoint); - assert_eq!( - WireProtocol::from_call(label, &request(protocol)), - Some(protocol) - ); - assert_eq!( - WireProtocol::from_call("unknown", &request(protocol)), - Some(protocol) - ); - } - assert_eq!( - WireProtocol::from_call( - "unknown", - &LlmRequest { - headers: Map::new(), - content: json!({}) - } - ), - None - ); - assert_eq!(RoutingMode::Enforce.label(), "enforce"); - assert_eq!(RoutingMode::ObserveOnly.label(), "observe_only"); - - let valid = config("http://127.0.0.1:1/v1/routing/decision".into()); - let component: PluginComponentSpec = valid.clone().into(); - assert_eq!(component.kind, SWITCHYARD_PLUGIN_KIND); - assert!(component.enabled); - assert_eq!(component.config["decision_profile_id"], "stage_router"); - - let plugin = SwitchyardPlugin; - assert_eq!(plugin.plugin_kind(), SWITCHYARD_PLUGIN_KIND); - assert!(!plugin.allows_multiple_components()); - assert!(plugin.validate(&component.config).is_empty()); - let diagnostics = plugin.validate(json!({"version": "invalid"}).as_object().unwrap()); - assert_eq!(diagnostics.len(), 1); - assert_eq!(diagnostics[0].level, DiagnosticLevel::Error); - assert_eq!(diagnostics[0].code, "switchyard.invalid_config"); -} - -#[test] -fn switchyard_target_bindings_keep_protocol_specific_endpoints() { - for (protocol, endpoint) in [ - (WireProtocol::OpenaiChat, "/v1/chat/completions"), - (WireProtocol::OpenaiResponses, "/v1/responses"), - (WireProtocol::AnthropicMessages, "/v1/messages"), - ] { - let binding = binding(protocol, "coverage-model"); - assert_eq!(binding.protocol, protocol); - assert_eq!(binding.endpoint, endpoint); - assert_eq!(binding.base_url, "http://127.0.0.1:9999"); - } - let defaults = ProtocolDefaults { - openai_chat: "chat".into(), - openai_responses: "responses".into(), - anthropic_messages: "anthropic".into(), - }; - assert_eq!(defaults.openai_chat, "chat"); - assert_eq!(defaults.openai_responses, "responses"); - assert_eq!(defaults.anthropic_messages, "anthropic"); -} - -#[tokio::test] -async fn plugin_registration_installs_and_rolls_back_both_execution_intercepts() { - let plugin = SwitchyardPlugin; - let decision_api_url = health_server(StatusCode::OK, r#"{"status":"ok"}"#).await; - for (mode, profile_id) in [ - (RoutingMode::Enforce, "random"), - (RoutingMode::Enforce, "llm"), - (RoutingMode::Enforce, "stage_router"), - (RoutingMode::ObserveOnly, "random"), - (RoutingMode::ObserveOnly, "llm"), - (RoutingMode::ObserveOnly, "stage_router"), - ] { - let mut candidate = config(decision_api_url.clone()); - candidate.mode = mode; - candidate.decision_profile_id = profile_id.into(); - let component: PluginComponentSpec = candidate.into(); - let mut context = PluginRegistrationContext::with_namespace(format!( - "switchyard-test-{}-", - Uuid::now_v7() - )); - - plugin - .register(&component.config, &mut context) - .await - .unwrap(); - - let mut registrations = context.into_registrations(); - assert_eq!(registrations.len(), 2); - rollback_registrations(&mut registrations); - assert!(registrations.is_empty()); - } -} - -#[tokio::test] -async fn plugin_registration_retries_transient_sidecar_health_failures() { - let (decision_api_url, attempts) = recovering_health_server(2).await; - let plugin = SwitchyardPlugin; - let component: PluginComponentSpec = config(decision_api_url).into(); - let mut context = - PluginRegistrationContext::with_namespace(format!("switchyard-test-{}-", Uuid::now_v7())); - - plugin - .register(&component.config, &mut context) - .await - .unwrap(); - - assert_eq!(attempts.load(Ordering::SeqCst), 3); - let mut registrations = context.into_registrations(); - assert_eq!(registrations.len(), 2); - rollback_registrations(&mut registrations); - assert!(registrations.is_empty()); -} - -#[tokio::test] -async fn plugin_registration_returns_the_final_health_failure_after_retry_exhaustion() { - let (decision_api_url, attempts) = recovering_health_server(usize::MAX).await; - let plugin = SwitchyardPlugin; - let component: PluginComponentSpec = config(decision_api_url).into(); - let mut context = - PluginRegistrationContext::with_namespace(format!("switchyard-test-{}-", Uuid::now_v7())); - - let error = plugin - .register(&component.config, &mut context) - .await - .unwrap_err(); - - assert_eq!(attempts.load(Ordering::SeqCst), 3); - assert!( - error - .to_string() - .contains("returned HTTP 503 Service Unavailable") - ); - assert!(context.into_registrations().is_empty()); -} - -#[tokio::test] -async fn plugin_registration_rejects_unhealthy_or_invalid_sidecars() { - for (status, body, expected) in [ - ( - StatusCode::SERVICE_UNAVAILABLE, - r#"{"status":"starting"}"#, - "returned HTTP 503 Service Unavailable", - ), - (StatusCode::OK, "not-json", "returned invalid JSON"), - ( - StatusCode::OK, - r#"{"status":"starting"}"#, - "did not report status=ok", - ), - ] { - let plugin = SwitchyardPlugin; - let component: PluginComponentSpec = config(health_server(status, body).await).into(); - let mut context = PluginRegistrationContext::with_namespace(format!( - "switchyard-test-{}-", - Uuid::now_v7() - )); - let error = plugin - .register(&component.config, &mut context) - .await - .unwrap_err(); - assert!( - error.to_string().contains(expected), - "{error:?} did not contain {expected:?}" - ); - assert!(context.into_registrations().is_empty()); - } -} - -#[tokio::test] -async fn plugin_registration_rejects_an_unreachable_sidecar() { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - drop(listener); - - let plugin = SwitchyardPlugin; - let component: PluginComponentSpec = - config(format!("http://{address}/v1/routing/decision")).into(); - let mut context = - PluginRegistrationContext::with_namespace(format!("switchyard-test-{}-", Uuid::now_v7())); - let error = plugin - .register(&component.config, &mut context) - .await - .unwrap_err(); - assert!(error.to_string().contains("Switchyard service is required")); - assert!(error.to_string().contains("/health")); - assert!(context.into_registrations().is_empty()); -} - -#[test] -fn configuration_validation_rejects_unsafe_or_ambiguous_bindings() { - let url = "http://127.0.0.1:1/v1/routing/decision".to_string(); - let assert_invalid = |config: SwitchyardConfig, expected: &str| { - let error = SwitchyardRuntime::new(config) - .err() - .expect("config must fail"); - assert!( - error.contains(expected), - "{error:?} did not contain {expected:?}" - ); - }; - - let mut candidate = config(url.clone()); - candidate.version = 2; - assert_invalid(candidate, "unsupported Switchyard config version"); - let mut candidate = config(url.clone()); - candidate.decision_profile_id.clear(); - assert_invalid(candidate, "decision_profile_id"); - let mut candidate = config(url.clone()); - candidate.decision_timeout_millis = 0; - assert_invalid(candidate, "decision_timeout_millis"); - let mut candidate = config(url.clone()); - candidate.max_retries = 11; - assert_invalid(candidate, "max_retries"); - let mut candidate = config(url.clone()); - candidate.recent_message_count = 0; - assert_invalid(candidate, "recent_message_count"); - let mut candidate = config(url.clone()); - candidate.context_mode = ContextMode::AtofRequired; - assert_invalid(candidate, "atof_endpoint_name"); - let mut candidate = config(url.clone()); - candidate.atof_endpoint_name = Some(" switchyard ".into()); - assert_invalid(candidate, "leading or trailing whitespace"); - let candidate = config("file:///tmp/decision".into()); - assert_invalid(candidate, "must use http or https"); - let mut candidate = config(url.clone()); - candidate.targets.clear(); - assert_invalid(candidate, "targets must not be empty"); - let mut candidate = config(url.clone()); - candidate.enabled_inbound_profiles.clear(); - assert_invalid(candidate, "enabled_inbound_profiles"); - - let mut candidate = config(url.clone()); - candidate.targets.get_mut("selected-chat").unwrap().endpoint = "/wrong".into(); - assert_invalid(candidate, "endpoint must be"); - let mut candidate = config(url.clone()); - candidate.targets.get_mut("selected-chat").unwrap().base_url = "file:///tmp/provider".into(); - assert_invalid(candidate, "base_url must use http or https"); - let mut candidate = config(url.clone()); - candidate.targets.insert( - "duplicate-chat".into(), - candidate.targets["selected-chat"].clone(), - ); - assert_invalid(candidate, "conflicts with another exact backend binding"); - let mut candidate = config(url.clone()); - candidate.default_targets.openai_chat = "missing".into(); - assert_invalid(candidate, "default target \"missing\" is not configured"); - let mut candidate = config(url.clone()); - candidate.default_targets.openai_chat = "fallback-responses".into(); - assert_invalid(candidate, "must use protocol openai_chat"); - - let mut candidate = config(url.clone()); - candidate - .decision_headers - .insert("bad header".into(), "value".into()); - assert_invalid(candidate, "invalid header name"); - let mut candidate = config(url.clone()); - candidate - .decision_headers - .insert("x-test".into(), "bad\nvalue".into()); - assert_invalid(candidate, "invalid header value"); - let mut candidate = config(url.clone()); - candidate - .decision_headers - .insert("Authorization".into(), "Bearer static".into()); - candidate.decision_header_env.insert( - "authorization".into(), - "SWITCHYARD_TEST_MISSING_DECISION_SECRET".into(), - ); - assert_invalid(candidate, "cannot appear in both headers and header_env"); - let mut candidate = config(url.clone()); - candidate.decision_header_env.insert( - "authorization".into(), - "SWITCHYARD_TEST_MISSING_DECISION_SECRET".into(), - ); - assert_invalid(candidate, "is not set"); - let mut candidate = config(url.clone()); - let target = candidate.targets.get_mut("selected-chat").unwrap(); - target - .headers - .insert("Authorization".into(), "Bearer static".into()); - target.header_env.insert( - "authorization".into(), - "SWITCHYARD_TEST_MISSING_TARGET_SECRET".into(), - ); - assert_invalid(candidate, "cannot appear in both headers and header_env"); - let mut candidate = config(url); - candidate - .targets - .get_mut("selected-chat") - .unwrap() - .header_env - .insert( - "authorization".into(), - "SWITCHYARD_TEST_MISSING_TARGET_SECRET".into(), - ); - assert_invalid(candidate, "is not set"); -} - -#[test] -fn disabled_protocol_defaults_are_optional() { - let defaults: ProtocolDefaults = serde_json::from_value(json!({ - "anthropic_messages": "fallback-anthropic" - })) - .expect("omitted disabled protocol defaults should deserialize"); - assert!(defaults.openai_chat.is_empty()); - assert!(defaults.openai_responses.is_empty()); - - let mut candidate = config("http://127.0.0.1:1/v1/routing/decision".into()); - candidate.enabled_inbound_profiles = BTreeSet::from([WireProtocol::AnthropicMessages]); - candidate.default_targets.openai_chat.clear(); - candidate.default_targets.openai_responses.clear(); - - SwitchyardRuntime::new(candidate).expect("Anthropic-only configuration should validate"); -} - -#[test] -fn routed_requests_carry_one_relay_owned_backend_partition() { - let mut candidate = config("http://127.0.0.1:1/v1/routing/decision".into()); - candidate - .targets - .get_mut("selected-chat") - .unwrap() - .headers - .insert( - "X-NeMo-Relay-Internal-Dispatch-Backend".into(), - "spoofed-target".into(), - ); - let runtime = SwitchyardRuntime::new(candidate).unwrap(); - - let mut source = chat_request(); - source.headers.insert( - INTERNAL_DISPATCH_BACKEND_HEADER.into(), - json!("spoofed-source"), - ); - let routed = runtime - .apply_target(WireProtocol::OpenaiChat, source, &decision()) - .unwrap(); - assert_eq!( - routed.headers.get(INTERNAL_DISPATCH_BACKEND_HEADER), - Some(&json!("selected-chat")) - ); - assert_eq!( - routed - .headers - .keys() - .filter(|name| name.eq_ignore_ascii_case(INTERNAL_DISPATCH_BACKEND_HEADER)) - .count(), - 1, - "source and target case variants must not compete with Relay's partition" - ); - - let fallback = runtime - .fallback_request(WireProtocol::OpenaiChat, chat_request()) - .unwrap(); - assert_eq!( - fallback.headers.get(INTERNAL_DISPATCH_BACKEND_HEADER), - Some(&json!("fallback-chat")), - "fallback dispatch must carry its own backend partition" - ); -} - -#[test] -fn atof_cross_component_validation_rejects_invalid_endpoint_names_before_lookup() { - for (name, expected) in [ - (" ", "atof_endpoint_name must be non-empty when configured"), - ( - " switchyard ", - "atof_endpoint_name must not have leading or trailing whitespace", - ), - ] { - let mut switchyard = config("http://switchyard.test/v1/routing/decision".into()); - switchyard.context_mode = ContextMode::AtofRequired; - switchyard.atof_endpoint_name = Some(name.into()); - let plugin_config = PluginConfig { - components: vec![switchyard.into()], - ..PluginConfig::default() - }; - - assert_eq!( - validate_switchyard_atof_configuration(&plugin_config).unwrap_err(), - expected - ); - } -} - -#[test] -fn atof_cross_component_validation_reports_each_activation_mismatch() { - assert!(validate_switchyard_atof_configuration(&PluginConfig::default()).is_ok()); - - let payload = PluginConfig { - components: vec![config("http://switchyard.test/v1/routing/decision".into()).into()], - ..PluginConfig::default() - }; - assert!(validate_switchyard_atof_configuration(&payload).is_ok()); - - let mut switchyard = config("http://switchyard.test/v1/routing/decision".into()); - switchyard.context_mode = ContextMode::AtofRequired; - switchyard.atof_endpoint_name = Some("switchyard".into()); - let mut plugin_config = PluginConfig { - components: vec![switchyard.into()], - ..PluginConfig::default() - }; - plugin_config.components.push(PluginComponentSpec { - kind: "observability".into(), - enabled: true, - config: json!({"atof": {"enabled": true, "sinks": [{ - "type": "stream", - "name": "other", - "url": "http://events.test/v1/atof/events", - "header_env": {"authorization": "TOKEN"} - }]}}) - .as_object() - .unwrap() - .clone(), - }); - assert!( - validate_switchyard_atof_configuration(&plugin_config) - .unwrap_err() - .contains("requires named ATOF endpoint") - ); - - plugin_config.components[1].config["atof"]["sinks"][0]["name"] = json!("switchyard"); - assert!(validate_switchyard_atof_configuration(&plugin_config).is_ok()); - plugin_config.components[1].config["atof"]["sinks"][0]["transport"] = json!("websocket"); - assert!( - validate_switchyard_atof_configuration(&plugin_config) - .unwrap_err() - .contains("transport = http_post") - ); - plugin_config.components[1].config["atof"]["sinks"][0]["transport"] = json!("http_post"); - let duplicate = plugin_config.components[1].config["atof"]["sinks"][0].clone(); - plugin_config.components[1].config["atof"]["sinks"] - .as_array_mut() - .unwrap() - .push(duplicate); - assert!( - validate_switchyard_atof_configuration(&plugin_config) - .unwrap_err() - .contains("exactly one endpoint") - ); - plugin_config.components[1].config["atof"]["sinks"] - .as_array_mut() - .unwrap() - .pop(); - plugin_config.components[1].config["atof"]["sinks"][0]["field_name_policy"] = - json!("snake_case"); - assert!( - validate_switchyard_atof_configuration(&plugin_config) - .unwrap_err() - .contains("field_name_policy = preserve") - ); - let endpoint = &mut plugin_config.components[1].config["atof"]["sinks"][0]; - endpoint["field_name_policy"] = json!("preserve"); - endpoint.as_object_mut().unwrap().remove("header_env"); - assert!( - validate_switchyard_atof_configuration(&plugin_config) - .unwrap_err() - .contains("environment-referenced header") - ); -} - -#[tokio::test] -async fn execution_bypasses_inapplicable_calls_and_fails_open_on_extensions() { - let runtime = - SwitchyardRuntime::new(config("http://127.0.0.1:1/v1/routing/decision".into())).unwrap(); - let passthrough: LlmExecutionNextFn = - Arc::new(|request| Box::pin(async move { Ok(request.content) })); - let unknown = LlmRequest { - headers: Map::new(), - content: json!({"opaque": true}), - }; - assert_eq!( - runtime - .execute_buffered("custom.provider", unknown, Arc::clone(&passthrough)) - .await - .unwrap()["opaque"], - true - ); - - let mut disabled_config = config("http://127.0.0.1:1/v1/routing/decision".into()); - disabled_config - .enabled_inbound_profiles - .remove(&WireProtocol::OpenaiChat); - let disabled = SwitchyardRuntime::new(disabled_config).unwrap(); - assert_eq!( - disabled - .execute_buffered( - "openai.chat_completions", - chat_request(), - Arc::clone(&passthrough), - ) - .await - .unwrap()["model"], - "inbound" - ); - - let mut unsupported = chat_request(); - unsupported.content["thinking"] = json!({"type": "enabled"}); - let response = runtime - .execute_buffered( - "openai.chat_completions", - unsupported, - Arc::new(|request| { - Box::pin(async move { - assert_eq!(request.content["model"], "fallback"); - Ok(chat_response()) - }) - }), - ) - .await - .unwrap(); - assert_eq!(response["choices"][0]["message"]["content"], "ok"); -} - -#[tokio::test] -async fn stream_setup_retries_and_empty_streams_have_one_bounded_fallback() { - let (url, decisions) = decision_server().await; - let runtime = SwitchyardRuntime::new(config(url)).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmStreamExecutionNextFn = Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - if seen.fetch_add(1, Ordering::SeqCst) == 0 { - return Err(FlowError::Upstream(UpstreamFailure { - status: None, - body: "connect".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::Connection, - })); - } - Ok(LlmJsonStream::new(futures_stream::iter(vec![Ok( - chat_chunk("ok", json!("stop")), - )]))) - }) - }); - let output = runtime - .execute_stream("openai.chat_completions", chat_request(), next) - .await - .unwrap() - .collect::>() - .await; - assert_eq!(output.len(), 1); - assert_eq!(dispatches.load(Ordering::SeqCst), 2); - assert_eq!(decisions.lock().unwrap().len(), 2); - - let (url, decisions) = decision_server().await; - let runtime = SwitchyardRuntime::new(config(url)).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmStreamExecutionNextFn = Arc::new(move |request| { - let seen = Arc::clone(&seen); - Box::pin(async move { - let attempt = seen.fetch_add(1, Ordering::SeqCst); - let items = if attempt < 4 { - Vec::new() - } else { - assert_eq!(request.content["model"], "fallback"); - vec![Ok(chat_chunk("fallback", json!("stop")))] - }; - Ok(LlmJsonStream::new(futures_stream::iter(items))) - }) - }); - let output = runtime - .execute_stream("openai.chat_completions", chat_request(), next) - .await - .unwrap() - .collect::>() - .await; - assert_eq!(output.len(), 1); - assert_eq!(dispatches.load(Ordering::SeqCst), 5); - assert_eq!(decisions.lock().unwrap().len(), 4); -} - -#[tokio::test] -async fn fallback_setup_failures_preserve_the_provider_error() { - let runtime = - SwitchyardRuntime::new(config("http://127.0.0.1:1/v1/routing/decision".into())).unwrap(); - let buffered: LlmExecutionNextFn = Arc::new(|_| { - Box::pin(async { Err(FlowError::Internal("buffered fallback failed".into())) }) - }); - let error = runtime - .dispatch_fallback_buffered(WireProtocol::OpenaiChat, chat_request(), buffered, "test") - .await - .unwrap_err(); - assert!(error.to_string().contains("buffered fallback failed")); - - let streaming: LlmStreamExecutionNextFn = - Arc::new(|_| Box::pin(async { Err(FlowError::Internal("stream fallback failed".into())) })); - let error = runtime - .dispatch_fallback_stream(WireProtocol::OpenaiChat, chat_request(), streaming, "test") - .await - .err() - .expect("stream fallback setup must fail"); - assert!(error.to_string().contains("stream fallback failed")); -} - -#[tokio::test] -async fn translated_stream_preserves_success_and_propagates_both_error_sources() { - let source = LlmJsonStream::new(futures_stream::iter(vec![ - Ok(chat_chunk("hello", Json::Null)), - Ok(chat_chunk("", json!("stop"))), - ])); - let output = translated_stream( - WireProtocol::OpenaiChat, - WireProtocol::AnthropicMessages, - "selected".into(), - source, - ) - .collect::>() - .await; - assert!(output.iter().all(Result::is_ok)); - assert!(output.iter().any(|item| { - item.as_ref() - .is_ok_and(|chunk| chunk.to_string().contains("hello")) - })); - - let upstream_error = FlowError::Internal("upstream stream failed".into()); - let source = LlmJsonStream::new(futures_stream::iter(vec![Err(upstream_error)])); - let output = translated_stream( - WireProtocol::OpenaiChat, - WireProtocol::AnthropicMessages, - "selected".into(), - source, - ) - .collect::>() - .await; - assert!(output[0].is_err()); - - let malformed = LlmJsonStream::new(futures_stream::iter(vec![Ok(json!({ - "choices": [{"delta": {"reasoning_content": "private"}}] - }))])); - let output = translated_stream( - WireProtocol::OpenaiChat, - WireProtocol::AnthropicMessages, - "selected".into(), - malformed, - ) - .collect::>() - .await; - assert!(output[0].is_err()); -} - -#[tokio::test] -async fn stream_adapters_forward_explicit_close_to_the_upstream_stream() { - for make_adapter in [ - prefixed_adapter as fn(LlmJsonStream) -> LlmJsonStream, - terminal_adapter, - translated_adapter, - ] { - let close_calls = Arc::new(AtomicUsize::new(0)); - let mut stream = make_adapter(close_tracking_stream(Arc::clone(&close_calls))); - - stream.close().await.unwrap(); - - assert_eq!(close_calls.load(Ordering::SeqCst), 1); - assert!(stream.next().await.is_none()); - } -} - -#[test] -fn provider_failure_reporting_covers_every_retry_class() { - for (class, label, retryable) in [ - (UpstreamFailureClass::Connection, "connection", true), - (UpstreamFailureClass::Timeout, "timeout", true), - ( - UpstreamFailureClass::RetryableStatus, - "retryable_status", - true, - ), - (UpstreamFailureClass::ContextWindow, "context_window", true), - ( - UpstreamFailureClass::ModelUnavailable, - "model_unavailable", - true, - ), - ( - UpstreamFailureClass::Authentication, - "authentication", - false, - ), - ( - UpstreamFailureClass::InvalidRequest, - "invalid_request", - false, - ), - (UpstreamFailureClass::Other, "other", false), - ] { - let error = FlowError::Upstream(UpstreamFailure { - status: Some(503), - body: "failure".into(), - headers: BTreeMap::new(), - class, - }); - assert_eq!(provider_error_class(&error), label); - assert_eq!(error_is_retryable(&error), retryable); - assert_eq!(provider_error_summary(&error), format!("{label}:http_503")); - } - let relay = FlowError::Internal("failure".into()); - assert_eq!(provider_error_class(&relay), "relay"); - assert_eq!(provider_error_summary(&relay), relay.to_string()); -} - -#[test] -fn all_materialization_modes_are_bounded_and_provider_valid() { - for protocol in [ - WireProtocol::OpenaiChat, - WireProtocol::OpenaiResponses, - WireProtocol::AnthropicMessages, - ] { - for mode in [ - RequestMaterialization::None, - RequestMaterialization::SummaryOnly, - RequestMaterialization::LatestUserPrompt, - RequestMaterialization::RecentMessageWindow, - RequestMaterialization::AnnotatedRequest, - RequestMaterialization::FullBody, - ] { - let mut config = config("http://127.0.0.1:1/v1/routing/decision".into()); - config.request_materialization = mode; - config.recent_message_count = 2; - let runtime = SwitchyardRuntime::new(config).unwrap(); - let routing = runtime - .routing_request(protocol, &request(protocol), 1, None) - .unwrap(); - match mode { - RequestMaterialization::None | RequestMaterialization::SummaryOnly => { - assert!(routing.current_request.is_none()) - } - RequestMaterialization::LatestUserPrompt => { - let current = routing.current_request.unwrap(); - assert_eq!(current["latest_user_prompt"], "latest"); - let body = current["body"].clone(); - decode_request( - &runtime.translation, - protocol, - &LlmRequest { - headers: Map::new(), - content: body, - }, - ) - .unwrap(); - } - RequestMaterialization::RecentMessageWindow => { - let current = routing.current_request.unwrap(); - decode_request( - &runtime.translation, - protocol, - &LlmRequest { - headers: Map::new(), - content: current["body"].clone(), - }, - ) - .unwrap(); - } - RequestMaterialization::AnnotatedRequest | RequestMaterialization::FullBody => { - assert!(routing.current_request.is_some()) - } - } - } - } -} - -#[test] -fn identity_policy_requires_stable_request_scope_only_for_atof_profiles() { - let mut config = config("http://127.0.0.1:1/v1/routing/decision".into()); - let payload_runtime = SwitchyardRuntime::new(config.clone()).unwrap(); - let synthetic = payload_runtime - .routing_request(WireProtocol::OpenaiChat, &chat_request(), 1, None) - .unwrap(); - assert_eq!(synthetic.identity.quality, "synthetic"); - - config.context_mode = ContextMode::AtofRequired; - config.atof_endpoint_name = Some("switchyard".into()); - let atof_runtime = SwitchyardRuntime::new(config).unwrap(); - assert!( - atof_runtime - .routing_request(WireProtocol::OpenaiChat, &chat_request(), 1, None) - .is_err() - ); - let mut stable = chat_request(); - stable - .headers - .insert("x-nemo-relay-session-id".into(), json!("session-1")); - stable - .headers - .insert("x-nemo-relay-request-id".into(), json!("request-1")); - let routed = atof_runtime - .routing_request(WireProtocol::OpenaiChat, &stable, 1, None) - .unwrap(); - assert_eq!(routed.identity.quality, "explicit"); -} - -#[test] -fn exact_target_validation_rejects_any_switchyard_drift() { - let runtime = - SwitchyardRuntime::new(config("http://127.0.0.1:1/v1/routing/decision".into())).unwrap(); - assert!(runtime.validate_decision(&decision()).is_ok()); - let mut drifted = decision(); - drifted.route.target_model = "unbound-model".into(); - assert!(runtime.validate_decision(&drifted).is_err()); - drifted = decision(); - drifted.route.backend_id = "unknown".into(); - assert!(runtime.validate_decision(&drifted).is_err()); -} - -#[test] -fn routing_contribution_requires_an_exact_independent_baseline_binding() { - let runtime = - SwitchyardRuntime::new(config("http://127.0.0.1:1/v1/routing/decision".into())).unwrap(); - - let contribution = runtime.routing_contribution(&decision(), 2, true).unwrap(); - assert!(contribution.applied); - assert_eq!( - contribution.kind.as_str(), - LlmOptimizationKind::MODEL_ROUTING - ); - let transition = contribution.model_transition.unwrap(); - assert_eq!(transition.baseline.unwrap().model, "baseline"); - assert_eq!(transition.effective.unwrap().model, "selected"); - assert_eq!(contribution.payload.as_ref().unwrap()["routing_attempt"], 2); - assert_eq!( - contribution.payload.as_ref().unwrap()["router_metadata"]["feature_state"], - "fresh" - ); - assert_eq!( - contribution.payload.as_ref().unwrap()["router_metadata"]["snapshot_age_millis"], - 37 - ); - assert_eq!( - contribution.payload_schema.as_ref().unwrap().name, - ROUTING_CONTRIBUTION_SCHEMA - ); - - let observed = runtime.routing_contribution(&decision(), 1, false).unwrap(); - assert!(!observed.applied); - - let mut missing = decision(); - missing.baseline_route = None; - assert!(runtime.routing_contribution(&missing, 1, true).is_none()); - - let mut drifted = decision(); - drifted.baseline_route.as_mut().unwrap().target_model = "drifted".into(); - assert!(runtime.routing_contribution(&drifted, 1, true).is_none()); - assert!(runtime.validate_decision(&drifted).is_ok()); -} - -#[test] -fn routing_decision_mark_has_canonical_shape_and_mirrored_identity() { - let subscriber_name = format!("switchyard-mark-shape-{}", uuid::Uuid::now_v7()); - let events = Arc::new(Mutex::new(Vec::::new())); - let captured = Arc::clone(&events); - register_subscriber( - &subscriber_name, - Arc::new(move |event| captured.lock().unwrap().push(event.clone())), - ) - .unwrap(); - - let runtime = - SwitchyardRuntime::new(config("http://127.0.0.1:1/v1/routing/decision".into())).unwrap(); - let routing_request = runtime - .routing_request(WireProtocol::OpenaiChat, &chat_request(), 1, None) - .unwrap(); - runtime.emit_decision(&routing_request, &decision(), 1, false, 17); - flush_subscribers().unwrap(); - deregister_subscriber(&subscriber_name).unwrap(); - - let event = events - .lock() - .unwrap() - .iter() - .map(Event::to_json_value) - .find(|event| { - event["name"] == "switchyard.routing.decision" - && event["metadata"]["session_id"] == routing_request.identity.session_id - && event["metadata"]["request_id"] == routing_request.identity.request_id - }) - .expect("decision mark should be captured"); - assert_eq!(event["kind"], "mark"); - assert_eq!(event["category"], "custom"); - assert_eq!( - event["category_profile"]["subtype"], - "switchyard.routing.decision" - ); - assert_eq!(event["data_schema"]["name"], ROUTING_MARK_SCHEMA); - assert_eq!(event["data_schema"]["version"], "1"); - assert_eq!(event["data"]["profile_id"], "stage_router"); - assert_eq!(event["data"]["selected_model"], "selected"); - assert_eq!(event["data"]["latency_ms"], 17); - assert_eq!(event["data"]["router_metadata"]["feature_state"], "fresh"); - assert_eq!(event["data"]["router_metadata"]["snapshot_age_millis"], 37); - assert_eq!( - event["metadata"]["session_id"], - routing_request.identity.session_id - ); - assert_eq!( - event["metadata"]["request_id"], - routing_request.identity.request_id - ); -} - -#[test] -fn atof_required_cross_component_validation_is_context_sensitive() { - let mut switchyard = config("http://switchyard.test:8080/v1/routing/decision".into()); - switchyard.context_mode = ContextMode::AtofRequired; - switchyard.atof_endpoint_name = Some("switchyard".into()); - let mut plugin_config = PluginConfig { - components: vec![switchyard.into()], - ..PluginConfig::default() - }; - assert!(validate_switchyard_atof_configuration(&plugin_config).is_err()); - plugin_config.components.push(PluginComponentSpec { - kind: "observability".into(), - enabled: true, - config: json!({"atof": { - "enabled": true, - "sinks": [{ - "type": "stream", - "name": "switchyard", - "url": "http://switchyard.test:8080/v1/atof/events", - "transport": "http_post", - "field_name_policy": "preserve", - "header_env": {"authorization": "SWITCHYARD_TOKEN"} - }] - }}) - .as_object() - .unwrap() - .clone(), - }); - assert!(validate_switchyard_atof_configuration(&plugin_config).is_ok()); -} - -#[derive(Clone)] -struct DecisionState { - requests: Arc>>, - decision: RoutingDecision, -} - -async fn decision_handler( - State(state): State, - AxumJson(request): AxumJson, -) -> AxumJson { - state.requests.lock().unwrap().push(request); - AxumJson(state.decision) -} - -async fn decision_server() -> (String, Arc>>) { - decision_server_for(decision()).await -} - -async fn decision_server_for( - decision: RoutingDecision, -) -> (String, Arc>>) { - let requests = Arc::new(Mutex::new(Vec::new())); - let state = DecisionState { - requests: Arc::clone(&requests), - decision, - }; - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve( - listener, - Router::new() - .route("/v1/routing/decision", post(decision_handler)) - .with_state(state), - ) - .await - .unwrap(); - }); - (format!("http://{address}/v1/routing/decision"), requests) -} - -async fn health_server(status: StatusCode, body: &'static str) -> String { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve( - listener, - Router::new().route( - "/health", - get(move || async move { (status, [("content-type", "application/json")], body) }), - ), - ) - .await - .unwrap(); - }); - format!("http://{address}/nested/v1/routing/decision?source=test#ignored") -} - -async fn recovering_health_server(failures_before_ready: usize) -> (String, Arc) { - let attempts = Arc::new(AtomicUsize::new(0)); - let captured = Arc::clone(&attempts); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve( - listener, - Router::new().route( - "/health", - get(move || { - let attempts = Arc::clone(&captured); - async move { - if attempts.fetch_add(1, Ordering::SeqCst) < failures_before_ready { - ( - StatusCode::SERVICE_UNAVAILABLE, - AxumJson(json!({"status": "starting"})), - ) - } else { - (StatusCode::OK, AxumJson(json!({"status": "ok"}))) - } - } - }), - ), - ) - .await - .unwrap(); - }); - (format!("http://{address}/v1/routing/decision"), attempts) -} - -async fn managed_buffered_events( - runtime: SwitchyardRuntime, - next: LlmExecutionNextFn, -) -> Vec { - let subscriber_name = format!("switchyard-accounting-{}", uuid::Uuid::now_v7()); - let events = Arc::new(Mutex::new(Vec::::new())); - let captured = Arc::clone(&events); - let scope_uuid = get_handle().unwrap().uuid; - scope_register_subscriber( - &scope_uuid, - &subscriber_name, - Arc::new(move |event| captured.lock().unwrap().push(event.clone())), - ) - .unwrap(); - let runtime = Arc::new(runtime); - let func: LlmExecutionNextFn = Arc::new(move |request| { - let runtime = Arc::clone(&runtime); - let next = Arc::clone(&next); - Box::pin(async move { - runtime - .execute_buffered("openai.chat_completions", request, next) - .await - }) - }); - llm_call_execute( - LlmCallExecuteParams::builder() - .name("openai.chat_completions") - .request(chat_request()) - .func(func) - .build(), - ) - .await - .unwrap(); - flush_subscribers().unwrap(); - scope_deregister_subscriber(&scope_uuid, &subscriber_name).unwrap(); - Arc::try_unwrap(events).unwrap().into_inner().unwrap() -} - -async fn managed_stream_events( - runtime: SwitchyardRuntime, - next: LlmStreamExecutionNextFn, -) -> Vec { - let subscriber_name = format!("switchyard-stream-accounting-{}", uuid::Uuid::now_v7()); - let events = Arc::new(Mutex::new(Vec::::new())); - let captured = Arc::clone(&events); - let scope_uuid = get_handle().unwrap().uuid; - scope_register_subscriber( - &scope_uuid, - &subscriber_name, - Arc::new(move |event| captured.lock().unwrap().push(event.clone())), - ) - .unwrap(); - let runtime = Arc::new(runtime); - let func: LlmStreamExecutionNextFn = Arc::new(move |request| { - let runtime = Arc::clone(&runtime); - let next = Arc::clone(&next); - Box::pin(async move { - runtime - .execute_stream("openai.chat_completions", request, next) - .await - }) - }); - let mut stream = llm_stream_call_execute( - LlmStreamCallExecuteParams::builder() - .name("openai.chat_completions") - .request(chat_request()) - .func(func) - .collector(Box::new(|_| Ok(()))) - .finalizer(Box::new(|| json!({"done": true}))) - .build(), - ) - .await - .unwrap(); - while stream.next().await.is_some() {} - drop(stream); - flush_subscribers().unwrap(); - scope_deregister_subscriber(&scope_uuid, &subscriber_name).unwrap(); - Arc::try_unwrap(events).unwrap().into_inner().unwrap() -} - -#[tokio::test] -async fn buffered_accounting_records_only_the_terminal_committed_route() { - let (url, _) = decision_server().await; - let successful = managed_buffered_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(|_| Box::pin(async { Ok(chat_response()) })), - ) - .await; - let marks = successful - .iter() - .filter(|event| event.name() == "nemo_relay.llm.optimization") - .collect::>(); - assert_eq!(marks.len(), 1); - assert_eq!(marks[0].data().unwrap()["applied"], true); - assert_eq!( - marks[0].data().unwrap()["model_transition"]["baseline"]["model"], - "baseline" - ); - let summary = successful - .iter() - .find_map(|event| { - event - .annotated_response() - .and_then(|response| response.optimization_summary.as_ref()) - }) - .unwrap(); - assert_eq!(summary.contributions.len(), 1); - assert!(summary.contributions[0].applied); - - let (url, _) = decision_server().await; - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let fallback = managed_buffered_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - if seen.fetch_add(1, Ordering::SeqCst) == 0 { - Err(FlowError::Upstream(UpstreamFailure { - status: Some(401), - body: "unauthorized".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::Authentication, - })) - } else { - Ok(chat_response()) - } - }) - }), - ) - .await; - assert_eq!(dispatches.load(Ordering::SeqCst), 2); - assert!( - fallback - .iter() - .all(|event| event.name() != "nemo_relay.llm.optimization") - ); -} - -#[tokio::test] -async fn retry_then_success_records_one_terminal_route_with_matching_mark_identity() { - let (url, decisions) = decision_server().await; - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let events = managed_buffered_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - if seen.fetch_add(1, Ordering::SeqCst) == 0 { - Err(FlowError::Upstream(UpstreamFailure { - status: Some(503), - body: "retry once".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::RetryableStatus, - })) - } else { - Ok(chat_response()) - } - }) - }), - ) - .await; - - assert_eq!(dispatches.load(Ordering::SeqCst), 2); - let requests = decisions.lock().unwrap(); - assert_eq!(requests.len(), 2); - assert_eq!(requests[1].attempt.routing_attempt, 2); - drop(requests); - - let start = events - .iter() - .find(|event| { - event.name() == "openai.chat_completions" - && event.scope_category() == Some(ScopeCategory::Start) - }) - .unwrap(); - let marks = events - .iter() - .filter(|event| event.name() == "nemo_relay.llm.optimization") - .collect::>(); - assert_eq!(marks.len(), 1); - assert_eq!(marks[0].parent_uuid(), Some(start.uuid())); - assert_eq!(marks[0].data().unwrap()["payload"]["routing_attempt"], 2); - - let summary = events - .iter() - .find_map(|event| { - event - .annotated_response() - .and_then(|response| response.optimization_summary.as_ref()) - }) - .unwrap(); - assert_eq!(summary.contributions.len(), 1); - let contribution = &summary.contributions[0]; - assert!(contribution.applied); - assert_eq!(contribution.payload.as_ref().unwrap()["routing_attempt"], 2); - assert_eq!(marks[0].data().unwrap()["id"], json!(contribution.id)); - assert_eq!( - marks[0].data().unwrap()["sequence"], - json!(contribution.sequence) - ); -} - -#[tokio::test] -async fn oversized_decision_metadata_limits_accounting_without_failing_provider_success() { - let mut oversized = decision(); - oversized - .metadata - .insert("oversized_evidence".into(), json!("x".repeat(20_000))); - let (url, _) = decision_server_for(oversized).await; - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let events = managed_buffered_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - seen.fetch_add(1, Ordering::SeqCst); - Ok(chat_response()) - }) - }), - ) - .await; - - assert_eq!(dispatches.load(Ordering::SeqCst), 1); - assert!( - events - .iter() - .all(|event| event.name() != "nemo_relay.llm.optimization") - ); - let summary = events - .iter() - .find_map(|event| { - event - .annotated_response() - .and_then(|response| response.optimization_summary.as_ref()) - }) - .unwrap(); - assert_eq!(summary.status, LlmOptimizationSummaryStatus::Partial); - assert!(summary.contributions.is_empty()); - assert!( - summary - .limitations - .iter() - .any(|limitation| limitation == "contribution_limit_exceeded") - ); -} - -#[tokio::test] -async fn malformed_baseline_is_telemetry_only_and_does_not_change_the_selected_route() { - let mut drifted = decision(); - drifted.baseline_route.as_mut().unwrap().target_model = "drifted".into(); - let (url, _) = decision_server_for(drifted).await; - let events = managed_buffered_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(|request| { - Box::pin(async move { - assert_eq!(request.content["model"], "selected"); - Ok(chat_response()) - }) - }), - ) - .await; - assert!(events.iter().any(|event| { - event.name() == "switchyard.routing.error" - && event.data().is_some_and(|data| { - data["error_class"] == "baseline_binding" - && data["error"] - .as_str() - .is_some_and(|error| error.contains("exact Relay binding")) - }) - })); - assert!( - events - .iter() - .all(|event| event.name() != "nemo_relay.llm.optimization") - ); -} - -#[tokio::test] -async fn observe_only_accounting_is_visible_but_not_applied() { - let (url, _) = decision_server().await; - let mut observe = config(url); - observe.mode = RoutingMode::ObserveOnly; - let events = managed_buffered_events( - SwitchyardRuntime::new(observe).unwrap(), - Arc::new(|request| { - Box::pin(async move { - assert_eq!(request.content["model"], "fallback"); - Ok(chat_response()) - }) - }), - ) - .await; - let contribution = events - .iter() - .find(|event| event.name() == "nemo_relay.llm.optimization") - .and_then(Event::data) - .unwrap(); - assert_eq!(contribution["applied"], false); - let summary = events - .iter() - .find_map(|event| { - event - .annotated_response() - .and_then(|response| response.optimization_summary.as_ref()) - }) - .unwrap(); - assert_eq!(summary.contributions.len(), 1); - assert!(!summary.contributions[0].applied); - assert!(summary.tokens_saved.total_tokens.is_none()); -} - -#[tokio::test] -async fn streaming_accounting_commits_on_the_first_successful_item_only() { - let (url, _) = decision_server().await; - let committed = managed_stream_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(|_| { - Box::pin(async { - Ok(LlmJsonStream::new(futures_stream::iter(vec![Ok( - chat_chunk("ok", json!("stop")), - )]))) - }) - }), - ) - .await; - assert_eq!( - committed - .iter() - .filter(|event| event.name() == "nemo_relay.llm.optimization") - .count(), - 1 - ); - - let (url, _) = decision_server().await; - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let fallback = managed_stream_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - let items = if seen.fetch_add(1, Ordering::SeqCst) == 0 { - vec![Err(FlowError::Upstream(UpstreamFailure { - status: Some(401), - body: "unauthorized".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::Authentication, - }))] - } else { - vec![Ok(chat_chunk("fallback", json!("stop")))] - }; - Ok(LlmJsonStream::new(futures_stream::iter(items))) - }) - }), - ) - .await; - assert_eq!(dispatches.load(Ordering::SeqCst), 2); - assert!( - fallback - .iter() - .all(|event| event.name() != "nemo_relay.llm.optimization") - ); -} - -#[tokio::test] -async fn committed_stream_error_keeps_one_route_and_never_redispatches() { - let (url, decisions) = decision_server().await; - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let events = managed_stream_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - seen.fetch_add(1, Ordering::SeqCst); - Ok(LlmJsonStream::new(futures_stream::iter(vec![ - Ok(chat_chunk("partial", Json::Null)), - Err(FlowError::Upstream(UpstreamFailure { - status: None, - body: "connection closed after commit".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::Connection, - })), - ]))) - }) - }), - ) - .await; - - assert_eq!(dispatches.load(Ordering::SeqCst), 1); - assert_eq!(decisions.lock().unwrap().len(), 1); - assert_eq!( - events - .iter() - .filter(|event| event.name() == "nemo_relay.llm.optimization") - .count(), - 1 - ); - let summary = events - .iter() - .find_map(|event| { - event - .annotated_response() - .and_then(|response| response.optimization_summary.as_ref()) - }) - .unwrap(); - assert_eq!(summary.contributions.len(), 1); - assert!(summary.contributions[0].applied); - assert!( - summary - .limitations - .iter() - .any(|limitation| limitation == "stream_interrupted") - ); -} - -#[tokio::test] -async fn retry_exhaustion_redecides_four_times_then_dispatches_fallback_once() { - let (url, decisions) = decision_server().await; - let runtime = SwitchyardRuntime::new(config(url)).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmExecutionNextFn = Arc::new(move |request| { - let seen = Arc::clone(&seen); - Box::pin(async move { - let attempt = seen.fetch_add(1, Ordering::SeqCst) + 1; - if attempt <= 4 { - return Err(FlowError::Upstream(UpstreamFailure { - status: Some(503), - body: "temporarily unavailable".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::RetryableStatus, - })); - } - assert_eq!(request.content["model"], "fallback"); - Ok(chat_response()) - }) - }); - let response = runtime - .execute_buffered("openai.chat_completions", chat_request(), next) - .await - .unwrap(); - assert_eq!(response["choices"][0]["message"]["content"], "ok"); - assert_eq!(dispatches.load(Ordering::SeqCst), 5); - let requests = decisions.lock().unwrap(); - assert_eq!(requests.len(), 4); - assert_eq!(requests[3].attempt.routing_attempt, 4); - assert_eq!( - requests[3].attempt.previous_route.as_deref(), - Some("selected-chat") - ); -} - -#[tokio::test] -async fn non_retryable_provider_failure_bypasses_retry_loop() { - let (url, decisions) = decision_server().await; - let runtime = SwitchyardRuntime::new(config(url)).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmExecutionNextFn = Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - let attempt = seen.fetch_add(1, Ordering::SeqCst); - if attempt == 0 { - return Err(FlowError::Upstream(UpstreamFailure { - status: Some(401), - body: "unauthorized".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::Authentication, - })); - } - Ok(chat_response()) - }) - }); - runtime - .execute_buffered("openai.chat_completions", chat_request(), next) - .await - .unwrap(); - assert_eq!(dispatches.load(Ordering::SeqCst), 2); - assert_eq!(decisions.lock().unwrap().len(), 1); -} - -#[tokio::test] -async fn observe_only_records_one_decision_and_dispatches_only_the_trusted_default() { - let (url, decisions) = decision_server().await; - let mut config = config(url); - config.mode = RoutingMode::ObserveOnly; - let runtime = SwitchyardRuntime::new(config).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmExecutionNextFn = Arc::new(move |request| { - let seen = Arc::clone(&seen); - Box::pin(async move { - seen.fetch_add(1, Ordering::SeqCst); - assert_eq!(request.content["model"], "fallback"); - Ok(chat_response()) - }) - }); - runtime - .execute_buffered("openai.chat_completions", chat_request(), next) - .await - .unwrap(); - assert_eq!(dispatches.load(Ordering::SeqCst), 1); - assert_eq!(decisions.lock().unwrap().len(), 1); -} - -#[tokio::test] -async fn streaming_retries_before_first_item() { - let (url, decisions) = decision_server().await; - let runtime = SwitchyardRuntime::new(config(url)).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmStreamExecutionNextFn = Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - let attempt = seen.fetch_add(1, Ordering::SeqCst); - let items = if attempt == 0 { - vec![Err(FlowError::Upstream(UpstreamFailure { - status: Some(503), - body: "retry".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::RetryableStatus, - }))] - } else { - vec![Ok(chat_chunk("ok", json!("stop")))] - }; - Ok(LlmJsonStream::new(futures_stream::iter(items))) - }) - }); - let stream = runtime - .execute_stream("openai.chat_completions", chat_request(), next) - .await - .unwrap(); - let output = stream.collect::>().await; - assert_eq!(output.len(), 1); - assert!(output[0].is_ok()); - assert_eq!(dispatches.load(Ordering::SeqCst), 2); - assert_eq!(decisions.lock().unwrap().len(), 2); -} - -#[tokio::test] -async fn streaming_never_retries_after_first_item() { - let (url, decisions) = decision_server().await; - let runtime = SwitchyardRuntime::new(config(url)).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmStreamExecutionNextFn = Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - seen.fetch_add(1, Ordering::SeqCst); - let items = vec![ - Ok(chat_chunk("partial", Json::Null)), - Err(FlowError::Upstream(UpstreamFailure { - status: None, - body: "connection closed".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::Connection, - })), - ]; - Ok(LlmJsonStream::new(futures_stream::iter(items))) - }) - }); - let stream = runtime - .execute_stream("openai.chat_completions", chat_request(), next) - .await - .unwrap(); - let output = stream.collect::>().await; - assert_eq!(output.len(), 2); - assert!(output[0].is_ok()); - assert!(output[1].is_err()); - assert_eq!(dispatches.load(Ordering::SeqCst), 1); - assert_eq!(decisions.lock().unwrap().len(), 1); -} - -fn responses_config(decision_api_url: String) -> SwitchyardConfig { - let targets = BTreeMap::from([ - ( - "resp-a".into(), - binding(WireProtocol::OpenaiResponses, "model-a"), - ), - ( - "resp-b".into(), - binding(WireProtocol::OpenaiResponses, "model-b"), - ), - ]); - SwitchyardConfig { - decision_api_url, - decision_profile_id: "stage_router".into(), - request_materialization: RequestMaterialization::SummaryOnly, - context_mode: ContextMode::PayloadOnly, - decision_timeout_millis: 1_000, - targets, - default_targets: ProtocolDefaults { - openai_chat: String::new(), - openai_responses: "resp-a".into(), - anthropic_messages: String::new(), - }, - enabled_inbound_profiles: BTreeSet::from([WireProtocol::OpenaiResponses]), - ..SwitchyardConfig::default() - } -} - -fn responses_decision() -> RoutingDecision { - RoutingDecision { - route: crate::contract::RoutingTarget { - tier: "efficient".into(), - target_model: "model-b".into(), - backend_id: "resp-b".into(), - target_protocol_profile: "openai_responses".into(), - target_endpoint: "/v1/responses".into(), - }, - baseline_route: None, - ..decision() - } -} - -// When every configured target shares the inbound protocol, no cross-protocol translation can -// occur, so the portability guard is skipped: a non-portable Responses request (real Codex -// extensions) is routed through the Decision API with its provider-specific fields preserved. -#[tokio::test] -async fn same_protocol_targets_route_nonportable_streaming_requests() { - let (url, decisions) = decision_server_for(responses_decision()).await; - let runtime = SwitchyardRuntime::new(responses_config(url)).unwrap(); - let mut nonportable = request(WireProtocol::OpenaiResponses); - nonportable.content["reasoning"] = json!({"effort": "high"}); - nonportable.content["store"] = json!(true); - let next: LlmStreamExecutionNextFn = Arc::new(|request| { - Box::pin(async move { - assert_eq!(request.content["model"], "model-b"); - assert_eq!(request.content["store"], true); - assert_eq!(request.content["reasoning"]["effort"], "high"); - Ok(LlmJsonStream::new(futures_stream::iter(vec![Ok( - json!({"type": "response.output_text.delta", "delta": "ok"}), - )]))) - }) - }); - let output = runtime - .execute_stream("openai.responses", nonportable, next) - .await - .unwrap() - .collect::>() - .await; - assert_eq!(output.len(), 1); - assert_eq!(decisions.lock().unwrap().len(), 1); -} diff --git a/crates/switchyard/tests/unit/translation_tests.rs b/crates/switchyard/tests/unit/translation_tests.rs deleted file mode 100644 index d1cfc8cc7..000000000 --- a/crates/switchyard/tests/unit/translation_tests.rs +++ /dev/null @@ -1,327 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -use super::*; -use serde_json::json; -use switchyard_translation::{ - FileSource, MediaSource, ToolResult, TranslationDiagnostic, TranslationError, -}; - -fn request(content: Json) -> LlmRequest { - LlmRequest { - headers: Map::new(), - content, - } -} - -#[test] -fn wire_formats_and_message_helpers_preserve_portable_requests() { - assert_eq!( - wire_format(WireProtocol::OpenaiChat), - WireFormat::OpenAiChat - ); - assert_eq!( - wire_format(WireProtocol::OpenaiResponses), - WireFormat::OpenAiResponses - ); - assert_eq!( - wire_format(WireProtocol::AnthropicMessages), - WireFormat::AnthropicMessages - ); - - let engine = translation_engine(); - let decoded = decode_request( - &engine, - WireProtocol::OpenaiChat, - &request(json!({ - "model": "test", - "messages": [ - {"role": "system", "content": "instructions"}, - {"role": "user", "content": "first"}, - {"role": "assistant", "content": "answer"}, - {"role": "user", "content": "latest"} - ] - })), - ) - .unwrap(); - assert_eq!(latest_user_prompt(&decoded).as_deref(), Some("latest")); - assert_eq!(recent_message_window(&decoded, 2).messages.len(), 2); - assert_eq!(recent_message_window(&decoded, 20).messages.len(), 3); - assert!( - validate_portable_request( - &engine, - WireProtocol::OpenaiChat, - &request(json!({"model": "test", "messages": [{"role": "user", "content": "ok"}]})), - ) - .is_ok() - ); -} - -#[test] -fn portability_guards_reject_provider_specific_request_fields() { - let engine = translation_engine(); - for key in [ - "cache_control", - "audio", - "thinking", - "computer_use", - "server_tool_use", - ] { - let mut content = serde_json::Map::from_iter([ - ("model".into(), json!("test")), - ( - "messages".into(), - json!([{"role": "user", "content": "ok"}]), - ), - ]); - content.insert(key.into(), json!(true)); - assert!( - validate_portable_request( - &engine, - WireProtocol::OpenaiChat, - &request(Json::Object(content)), - ) - .is_err() - ); - } - assert!( - validate_portable_request( - &engine, - WireProtocol::OpenaiResponses, - &request(json!({ - "model": "test", - "input": "ok", - "stream_options": {"include_usage": "yes"} - })), - ) - .is_err() - ); -} - -#[test] -fn anthropic_image_sources_must_be_complete_and_known() { - let engine = translation_engine(); - assert!( - validate_portable_request( - &engine, - WireProtocol::AnthropicMessages, - &request(json!({ - "model": "test", - "max_tokens": 1, - "messages": [{"role": "user", "content": [{"type": "image"}]}] - })), - ) - .is_err() - ); - for source in [ - json!({}), - json!({"type": "url", "url": ""}), - json!({"type": "base64", "media_type": "image/png", "data": ""}), - json!({"type": "asset", "id": "image-1"}), - ] { - assert!(contains_invalid_anthropic_image_source(&json!({ - "messages": [{"content": [{"type": "image", "source": source}]}] - }))); - } - assert!(!contains_invalid_anthropic_image_source(&json!({ - "messages": [{"content": [{"type": "image", "source": {"type": "url", "url": "https://example.test/image.png"}}]}] - }))); - assert!(!contains_invalid_anthropic_image_source( - &json!({"text": ["plain", "values"]}) - )); -} - -#[test] -fn response_portability_allows_only_shared_content() { - assert!( - ensure_portable_response( - WireProtocol::OpenaiChat, - &json!({"choices": [{"message": {"content": "ok"}}]}), - ) - .is_ok() - ); - for message in [json!({"audio": {}}), json!({"reasoning_content": "hidden"})] { - assert!( - ensure_portable_response( - WireProtocol::OpenaiChat, - &json!({"choices": [{"message": message}]}), - ) - .is_err() - ); - } - - assert!( - ensure_portable_response( - WireProtocol::OpenaiResponses, - &json!({"output": [{"type": "message"}]}), - ) - .is_ok() - ); - for kind in [ - "reasoning", - "computer_call", - "computer_call_output", - "web_search_call", - ] { - assert!( - ensure_portable_response( - WireProtocol::OpenaiResponses, - &json!({"output": [{"type": kind}]}), - ) - .is_err() - ); - } - - assert!( - ensure_portable_response( - WireProtocol::AnthropicMessages, - &json!({"content": [{"type": "text"}, {"type": "tool_use"}]}), - ) - .is_ok() - ); - assert!( - ensure_portable_response( - WireProtocol::AnthropicMessages, - &json!({"content": [{"type": "thinking"}]}), - ) - .is_err() - ); -} - -#[test] -fn data_uri_and_recursive_key_helpers_distinguish_valid_shapes() { - assert_eq!( - base64_data_uri_parts("data:image/png;base64,Zm9v"), - Some(("image/png", "Zm9v")) - ); - for value in [ - "https://example.test/image.png", - "data:image/png,Zm9v", - "data:;base64,Zm9v", - "data:image/png;base64,", - ] { - assert!(base64_data_uri_parts(value).is_none()); - } - assert!(contains_any_key_recursive( - &json!({"outer": [{"forbidden": true}]}), - &["forbidden"], - )); - assert!(!contains_any_key_recursive( - &json!({"outer": [{"allowed": true}]}), - &["forbidden"], - )); -} - -#[test] -fn unsupported_content_and_image_sources_are_rejected_before_translation() { - assert!(!unsupported_content_block(&ContentBlock::Text { - text: "ok".into(), - })); - assert!(unsupported_content_block(&ContentBlock::Reasoning { - text: "hidden".into(), - signature: None, - })); - assert!(unsupported_content_block(&ContentBlock::Audio { - source: MediaSource::Raw(json!({})), - })); - assert!(unsupported_content_block(&ContentBlock::Video { - source: MediaSource::Raw(json!({})), - })); - assert!(unsupported_content_block(&ContentBlock::File { - source: FileSource::Raw(json!({})), - })); - assert!(unsupported_content_block(&ContentBlock::Unknown { - provider: WireFormat::OpenAiChat.into(), - raw: json!({}), - })); - assert!(unsupported_content_block(&ContentBlock::ToolResult( - ToolResult { - tool_call_id: "call-1".into(), - content: vec![ContentBlock::Audio { - source: MediaSource::Raw(json!({})), - }], - is_error: None, - }, - ))); - - assert!(!invalid_image_source(&ImageSource::Url { - url: "https://example.test/image.png".into(), - detail: None, - })); - assert!(unsupported_content_block(&ContentBlock::Image { - source: ImageSource::Raw(json!({})), - })); - assert!(!invalid_image_source(&ImageSource::Url { - url: "data:image/png;base64,Zm9v".into(), - detail: None, - })); - assert!(invalid_image_source(&ImageSource::Url { - url: "data:image/png,Zm9v".into(), - detail: None, - })); - assert!(!invalid_image_source(&ImageSource::Base64 { - media_type: Some("image/png".into()), - data: "Zm9v".into(), - })); - assert!(invalid_image_source(&ImageSource::Base64 { - media_type: None, - data: "Zm9v".into(), - })); - assert!(invalid_image_source(&ImageSource::Base64 { - media_type: Some("image/png".into()), - data: String::new(), - })); - assert!(invalid_image_source(&ImageSource::Raw(json!({})))); -} - -#[test] -fn translation_helpers_cover_diagnostics_and_cross_protocol_responses() { - assert!(ensure_no_diagnostics(&[]).is_ok()); - assert!( - ensure_no_diagnostics(&[TranslationDiagnostic::warning("lossy", "not portable")]).is_err() - ); - assert!(matches!( - translation_error(TranslationError::Other("invalid".into())), - FlowError::InvalidArgument(message) if message.contains("invalid") - )); - assert!(portable_stream_options(&json!({"include_usage": true}))); - assert!(!portable_stream_options( - &json!({"include_usage": true, "extra": false}) - )); - assert!(!portable_stream_options(&json!(true))); - - let response = json!({ - "id": "chat-1", - "object": "chat.completion", - "model": "test", - "choices": [{ - "index": 0, - "message": {"role": "assistant", "content": "ok"}, - "finish_reason": "stop" - }] - }); - let engine = translation_engine(); - assert_eq!( - translate_response( - &engine, - WireProtocol::OpenaiChat, - WireProtocol::OpenaiChat, - &response, - ) - .unwrap(), - response - ); - let translated = translate_response( - &engine, - WireProtocol::OpenaiChat, - WireProtocol::AnthropicMessages, - &response, - ) - .unwrap(); - assert_eq!(translated["type"], "message"); - assert_eq!(translated["role"], "assistant"); - assert_eq!( - translated["content"], - json!([{ "type": "text", "text": "ok" }]) - ); -} diff --git a/docs/about-nemo-relay/concepts/plugins.mdx b/docs/about-nemo-relay/concepts/plugins.mdx index 78dec894b..d22d94123 100644 --- a/docs/about-nemo-relay/concepts/plugins.mdx +++ b/docs/about-nemo-relay/concepts/plugins.mdx @@ -256,16 +256,6 @@ The core crate ships a built-in `pricing` component. It loads catalog sources that response codecs can use to annotate managed LLM responses with cost estimates. Configure catalog sources through [Model Pricing](/configure-plugins/model-pricing). -### Switchyard (Experimental) - -The experimental `switchyard` component asks a separately running Switchyard -Decision API to select a configured model backend. It is excluded from default -CLI builds and is registered only when the CLI is built with the optional -`switchyard` feature. Direct Rust, Python, and Node.js hosts do not register it -automatically. Refer to -[Switchyard (Experimental)](/configure-plugins/switchyard/about) for the pinned -compatibility and deployment requirements. - For `plugins.toml` discovery, precedence, merge, and gateway editing rules, refer to [Plugin Configuration Files](/configure-plugins/plugin-configuration-files). diff --git a/docs/about-nemo-relay/release-notes/index.mdx b/docs/about-nemo-relay/release-notes/index.mdx index 20e73cf9d..0aca1a23e 100644 --- a/docs/about-nemo-relay/release-notes/index.mdx +++ b/docs/about-nemo-relay/release-notes/index.mdx @@ -118,6 +118,11 @@ Migration guidance for upgrading from 0.7 to 0.8 is available in the NeMo Relay is built into Hermes Agent, and Hermes Agent understands NeMo Relay plugin configurations. No separate observability plugin or Relay CLI setup is required. +- Removed the experimental, service-backed Switchyard integration, including + the `nemo-relay-switchyard` crate, CLI `switchyard` feature, built-in + `switchyard` component, and service examples. Switchyard 0.3.0 will distribute + and document the Switchyard-owned dynamic plugin. Refer to the + [Switchyard migration guide](/reference/migration-guides#migrate-to-the-switchyard-owned-dynamic-plugin). Refer to [Migration Guides](/reference/migration-guides#move-project-configuration-to-a-supported-location) for destination paths and explicit-file alternatives. @@ -146,9 +151,6 @@ for destination paths and explicit-file alternatives. - Codex 0.143 does not expose `SessionEnd`, and Codex multi-agent v2 encrypts delegated-task payloads that Relay cannot decrypt or reliably link. - The Node.js binding and package workflows require Node.js 24 or later. -- The deprecated Switchyard plugin is experimental, excluded from default CLI - builds, and requires a separately managed compatible service and a named - authenticated ATOF HTTP stream sink. It is scheduled for removal in 0.8. - OpenClaw has public hook-backed telemetry. Its security and optimization coverage is partial because it does not own a managed execution path. - The built-in `nemo_guardrails` plugin is deprecated and scheduled for removal diff --git a/docs/configure-plugins/about.mdx b/docs/configure-plugins/about.mdx index a14b301eb..eaf5631b2 100644 --- a/docs/configure-plugins/about.mdx +++ b/docs/configure-plugins/about.mdx @@ -34,10 +34,6 @@ entries in `plugins.toml`. for removal in NeMo Relay 0.9 and should not be used for new deployments. - [PII Redaction](/configure-plugins/pii-redaction/about) sanitizes sensitive data in observability payloads. -- [Switchyard (Deprecated)](/configure-plugins/switchyard/about) routes requests - using decisions from a separately running Switchyard Decision API service. - The experimental plugin will be removed in NeMo Relay 0.8 and replaced by a - Switchyard-owned native plugin. - [Model Pricing](/configure-plugins/model-pricing) configures catalog sources for cost estimates on managed LLM responses. diff --git a/docs/configure-plugins/adaptive/response-cache.mdx b/docs/configure-plugins/adaptive/response-cache.mdx index 9a7faa1f6..e3fc9d7c6 100644 --- a/docs/configure-plugins/adaptive/response-cache.mdx +++ b/docs/configure-plugins/adaptive/response-cache.mdx @@ -352,11 +352,11 @@ normalization, under the default `key_strategy = "exact_request"`: header does not partition the key. Known auth headers are rejected, but validation cannot recognize every custom credential name, so never allowlist credentials. -- The provider name and required namespace partition every key. A - Switchyard-selected backend ID is also partitioned automatically. Without - Switchyard, a provider name cannot distinguish upstreams that reuse that - name, so use separate configurations and namespaces for different trusted - upstream domains. +- The provider name and required namespace partition every key. An internal + dispatch backend supplied by a routing plugin is also partitioned + automatically. A provider name alone cannot distinguish upstreams that reuse + that name, so use separate configurations and namespaces for different + trusted upstream domains. - Requests containing integers outside the exactly representable RFC 8785 range (less than `-2^53` or greater than `2^53`) bypass the cache. @@ -399,10 +399,10 @@ a failure when the config is invalid or the backend is unreachable. | `backend.config.url` | — | Redis connection URL. Required for the `redis` backend. | | `backend.config.key_prefix` | `"nemo-relay:llm-cache:"` | Prefix for keys in Redis. | -For a gateway that uses Switchyard, `switchyard.priority` must be lower than -`response_cache.priority`. To derive keys before ACG rewrites requests, set -`response_cache.priority` lower than `acg.priority`. With all three components, -priorities of `0`, `40`, and `50`, respectively, satisfy both orderings. +A routing plugin that sets `x-nemo-relay-internal-dispatch-backend` must use a +priority lower than `response_cache.priority` so backend selection runs before +cache key derivation. To derive keys before ACG rewrites requests, set +`response_cache.priority` lower than `acg.priority`. Cached responses are stored unredacted. PII sanitize guardrails rewrite emitted @@ -424,5 +424,3 @@ each mutually untrusted tenant or upstream domain. - `backend.kind` is unknown; or `redis` has a missing, non-string, or whitespace-only `backend.config.url`, uses a non-string `key_prefix`, or is unavailable because Relay was built without the `redis-backend` feature. -- Gateway Switchyard priority is equal to or greater than - `response_cache.priority`. diff --git a/docs/configure-plugins/switchyard/about.mdx b/docs/configure-plugins/switchyard/about.mdx deleted file mode 100644 index b6806b1a2..000000000 --- a/docs/configure-plugins/switchyard/about.mdx +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: "Switchyard (Deprecated)" -sidebar-title: "Switchyard (Deprecated)" -description: "Set up and validate the deprecated Switchyard Decision API integration for NeMo Relay 0.6.0 and 0.7.0." -position: 5 ---- -import { MermaidStyles } from "@/components/MermaidStyles"; - -{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -SPDX-License-Identifier: Apache-2.0 */} - - -The experimental `nemo-relay-switchyard` plugin will be removed in NeMo Relay -0.8 and replaced by a Switchyard-owned native plugin. Continue using this page -for the existing integration. The NeMo Relay 0.8 documentation will include an -updated configuration guide and migration plan when the replacement is -available. - - -> **Experimental:** The Switchyard integration is an early-access feature. It -> is not enabled in default Relay builds, its configuration and contracts can -> change, and the current deployment requires a separately running Switchyard -> Decision API service. - -> **NeMo Relay 0.6.0 and 0.7.0 architecture:** These releases use the external -> Switchyard Decision API from the -> [`topic/nemo-relay-integration`](https://github.com/NVIDIA-NeMo/Switchyard/tree/topic/nemo-relay-integration) -> branch. - -Switchyard is an LLM routing decision engine. The `nemo-relay-switchyard` -plugin asks Switchyard which configured target should handle an LLM request. -Relay validates the decision and performs the authenticated provider request. -For example, a routing profile can send a simple prompt to a lower-cost model -and a complex prompt to a more capable model. - -## Architecture - -The following diagram shows the NeMo Relay 0.6.0 and 0.7.0 service boundary: - - - -```mermaid -flowchart LR - Agent["Agent"] -->|LLM request| Relay["NeMo Relay"] - Relay -->|routing request| Decision["Switchyard Decision API"] - Decision -->|backend decision| Relay - Relay -->|authenticated provider request| Backend["Selected LLM backend"] - Relay -->|ATOF events| History["Switchyard ATOF accumulator"] - History -. routing history .-> Decision - - class Agent grey-lightest; - class Relay green-lightest; - class Decision yellow-lightest; - class History yellow-lightest; - class Backend teal-lightest; -``` - -The components divide responsibility as follows: - -| Component | Responsibilities | -| --- | --- | -| Switchyard | Selects a backend and accumulates ATOF routing history for history-based profiles. | -| NeMo Relay | Owns provider credentials, target bindings, decision validation, protocol translation, dispatch, retries, trusted fallback, and observability. | - -Relay does not start or supervise the Switchyard service. Start Switchyard -before Relay activates the plugin. Relay derives the service's root `/health` -URL from `decision_api_url` and fails activation unless the endpoint returns -`{"status":"ok"}`. - -## Prerequisites - -Install the following prerequisites before you start: - -| Requirement | Version or Value | -| --- | --- | -| NeMo Relay | Tags `0.6.0` and `0.7.0` | -| Rust | `1.96.1` | -| Switchyard branch | `topic/nemo-relay-integration` | -| Switchyard commit | `8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8` | -| Local commands | `git`, `python3`, and `curl` | - -Keep the repositories in the following sibling layout. The validation script -uses this layout by default: - -```text -/ -├── NeMo-Relay/ -└── Switchyard-topic-nemo-relay-integration/ -``` - -## Build and Run the Compatibility Test - -Use the following procedure to clone and pin both repositories: - -```bash -mkdir relay-switchyard-0.6 -cd relay-switchyard-0.6 - -git clone --branch 0.6.0 \ - https://github.com/NVIDIA/NeMo-Relay.git \ - NeMo-Relay - -git clone --branch topic/nemo-relay-integration \ - https://github.com/NVIDIA-NeMo/Switchyard.git \ - Switchyard-topic-nemo-relay-integration - -git -C Switchyard-topic-nemo-relay-integration checkout --detach \ - 8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8 - -cd NeMo-Relay -``` - -The Switchyard plugin is excluded from default Relay CLI builds. Build the CLI -with the optional feature explicitly: - -```bash -cargo build -p nemo-relay-cli --features switchyard -``` - -Run the real-service compatibility test from the NeMo Relay repository root: - -```bash -examples/switchyard/run-real-e2e.sh -``` - -The script prints the verified Switchyard revision and ends with the following -line when routing succeeds: - -```text -real Switchyard E2E passed: ['provider/weak', 'provider/strong', 'provider/strong'] -``` - -The test starts these local processes: - -| Process | Port | Readiness Check or Purpose | -| --- | --- | --- | -| Switchyard Decision API | `4000` | Serves `/health`, routing decisions, and ATOF ingestion. | -| NeMo Relay | `4041` | Serves `/healthz` and the OpenAI-compatible gateway. | -| Deterministic provider | `4101` | Records the model selected for each provider request. | - -The first request has no accumulated ATOF history and routes to -`provider/weak`. The script then sends session and tool events to Relay. Relay -exports those events to Switchyard's ATOF accumulator, so the next buffered -request and the final streaming request route to `provider/strong`. The test -also validates the streamed response before reporting success. - -## Configuration Walkthrough - -The compatibility test uses -[`real-e2e-plugins.toml`](https://github.com/NVIDIA/NeMo-Relay/blob/release/0.6/examples/switchyard/real-e2e-plugins.toml) -and -[`real-e2e-profiles.yaml`](https://github.com/NVIDIA/NeMo-Relay/blob/release/0.6/examples/switchyard/real-e2e-profiles.yaml). -The following settings establish the routing boundary: - -| Setting | Purpose | -| --- | --- | -| `decision_api_url` | Points Relay to the separately running Switchyard Decision API. | -| `decision_profile_id` | Selects the Switchyard routing profile. The profile maps its semantic targets to backend IDs. | -| `targets.` | Binds each returned backend ID to a Relay-owned model, protocol, endpoint, base URL, and credentials. | -| `mode = "enforce"` | Applies a valid Switchyard decision and dispatches to the selected target. | -| `mode = "observe_only"` | Records the hypothetical decision but dispatches once to the trusted same-protocol default. | -| `default_targets` | Defines trusted same-protocol fallbacks for unavailable or invalid decisions and provider failures. | -| `context_mode = "atof_required"` | Requires stable identity and accumulated ATOF history for the selected profile. | -| `atof_endpoint_name` | Selects exactly one named Relay ATOF HTTP stream sink that sends events to Switchyard. | - -For history-based routing, a local ATOF JSONL file is not sufficient. -Switchyard must receive events through the named HTTP sink at -`/v1/atof/events` so its accumulator can provide the history used by the -routing profile. Relay rejects a missing, duplicate, disabled, or invalid -named sink during startup validation. - -For the complete option reference, refer to -[Switchyard Configuration](./configuration.mdx). - -## Troubleshooting - -Use the following table to diagnose common compatibility-test failures: - -| Symptom | Cause and Resolution | -| --- | --- | -| `Switchyard worktree not found: ...` | Place the Switchyard checkout next to `NeMo-Relay` as `Switchyard-topic-nemo-relay-integration`, or set `SWITCHYARD_ROOT` to its absolute path. | -| `Switchyard checkout mismatch: expected ..., found ...` | Check out commit `8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8`. Override `SWITCHYARD_EXPECTED_COMMIT` only when deliberately testing a different contract. | -| A build uses Rust `1.96.0` or reports an unsupported toolchain | Run `rustc --version`, install Rust `1.96.1`, and allow the repository's `rust-toolchain.toml` to select it. | -| `timed out waiting for http://127.0.0.1:4000/health` | Review `switchyard.log`. Confirm the pinned checkout builds and port `4000` is free. | -| `timed out waiting for http://127.0.0.1:4041/healthz` or `process ... exited before http://127.0.0.1:4041/healthz became ready` | Review `relay.log`. Confirm the CLI was built with `--features switchyard`, the Switchyard health check passed, and port `4041` is free. | -| The provider cannot start or requests fail | Review `upstream.log` and confirm port `4101` is free. | -| `E2E logs preserved in ...` | The script preserves its temporary directory after a runtime failure and prints the exact path. Inspect `switchyard.log`, `relay.log`, `upstream.log`, and the captured request outputs there. Successful runs remove the directory. | - -## Capabilities - -The integration provides the following capabilities: - -- `enforce` and `observe_only` routing modes. -- OpenAI Chat Completions, OpenAI Responses, and Anthropic Messages inbound - profiles. -- Buffered and streaming protocol translation through Switchyard's - `switchyard-translation` library. -- Exact Relay-owned backend bindings and per-protocol trusted fallbacks. -- Bounded retries before the first streaming item. -- Canonical routing marks and shared LLM optimization accounting. - -## Pages - -For more information, refer to the following pages: - -- [Switchyard Configuration](./configuration.mdx) - provides the complete option and deployment reference. Review its - [experimental limitations](./configuration.mdx#experimental-limitations) - before adopting the integration. -- [Switchyard integration examples](https://github.com/NVIDIA/NeMo-Relay/tree/release/0.6/examples/switchyard) - contains the versioned configuration and validation scripts used by this - guide. diff --git a/docs/configure-plugins/switchyard/configuration.mdx b/docs/configure-plugins/switchyard/configuration.mdx deleted file mode 100644 index 0c406af58..000000000 --- a/docs/configure-plugins/switchyard/configuration.mdx +++ /dev/null @@ -1,365 +0,0 @@ ---- -title: "Switchyard Configuration" -sidebar-title: "Configuration" -description: "Configure the deprecated Relay-native Switchyard Decision API plugin." -position: 2 ---- -{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -SPDX-License-Identifier: Apache-2.0 */} - - -The experimental `nemo-relay-switchyard` plugin will be removed in NeMo Relay -0.8 and replaced by a Switchyard-owned native plugin. The NeMo Relay 0.8 -documentation will include an updated configuration guide and migration plan -when the replacement is available. - - -> **Experimental NeMo Relay 0.6.0 and 0.7.0 integration:** These releases call a -> separately running Switchyard Decision API. - -Start with the -[Switchyard setup and validation guide](./about.mdx) -before using this page as the complete configuration reference. - -The Switchyard component connects Relay's CLI gateway to a separately running -[Switchyard](https://github.com/NVIDIA-NeMo/Switchyard) Decision API. Relay -owns provider credentials, backend endpoints, dispatch, retries, and trusted -fallbacks. The Switchyard service selects a backend. Relay validates the -decision, translates requests and responses in process through the -`switchyard-translation` library, and dispatches directly to the selected -Relay-owned target. - -## How the Configuration Fits Together - -Configure the integration in four core steps. Profiles that use ATOF history -add the fifth requirement: - -1. Set `decision_api_url` and authentication so Relay can call the Switchyard - Decision API. -2. Select the Switchyard routing profile with `decision_profile_id`. -3. Add a Relay-owned `targets` binding for every Switchyard `backend_id` that - the profile can return. Relay validates the returned backend ID, model, - protocol, and endpoint exactly before dispatch. -4. Choose a trusted same-protocol target for each inbound protocol under - `default_targets`. Relay uses these defaults for observe-only traffic and - fail-open dispatch. -5. Configure profiles that depend on ATOF history, including StageRouter, with - `context_mode = "atof_required"` and `atof_endpoint_name`, then configure the - matching observability stream sink. Relay fails startup validation if the - named sink is missing or invalid. - -The following is a complete component configuration. The referenced -environment variables must be set before Relay starts. - -```toml -version = 1 - -[[components]] -kind = "switchyard" -enabled = true - -[components.config] -version = 1 -mode = "enforce" -priority = 0 -decision_api_url = "http://127.0.0.1:4000/v1/routing/decision" -decision_profile_id = "stage-router" -request_materialization = "recent_message_window" -context_mode = "atof_required" -atof_endpoint_name = "switchyard" -decision_timeout_millis = 25 -max_retries = 3 -recent_message_count = 8 -enabled_inbound_profiles = ["openai_chat", "openai_responses", "anthropic_messages"] - -[components.config.decision_header_env] -authorization = "SWITCHYARD_AUTHORIZATION" - -[components.config.default_targets] -openai_chat = "openai-default" -openai_responses = "responses-default" -anthropic_messages = "anthropic-default" - -[components.config.targets.openai-default] -model = "provider/efficient-chat" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "https://provider.example.com" - -[components.config.targets.openai-default.header_env] -authorization = "OPENAI_PROVIDER_AUTHORIZATION" - -[components.config.targets.responses-default] -model = "provider/efficient-responses" -protocol = "openai_responses" -endpoint = "/v1/responses" -base_url = "https://provider.example.com" - -[components.config.targets.responses-default.header_env] -authorization = "OPENAI_PROVIDER_AUTHORIZATION" - -[components.config.targets.anthropic-default] -model = "provider/efficient-messages" -protocol = "anthropic_messages" -endpoint = "/v1/messages" -base_url = "https://provider.example.com" - -[components.config.targets.anthropic-default.header_env] -x-api-key = "ANTHROPIC_PROVIDER_API_KEY" - -[components.config.targets.capable-chat] -model = "provider/capable-chat" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "https://provider.example.com" - -[components.config.targets.capable-chat.header_env] -authorization = "OPENAI_PROVIDER_AUTHORIZATION" - -[[components]] -kind = "observability" -enabled = true - -[components.config] -version = 3 - -[components.config.atof] -enabled = true - -[[components.config.atof.sinks]] -type = "stream" -name = "switchyard" -url = "http://127.0.0.1:4000/v1/atof/events" -transport = "http_post" -field_name_policy = "preserve" - -[components.config.atof.sinks.header_env] -authorization = "SWITCHYARD_AUTHORIZATION" -``` - -## Decision API and Rollout Options - -The following table describes the Decision API and rollout settings: - -| Option | Default | Description and constraints | -| --- | --- | --- | -| `version` | `1` | Component configuration version. | -| `mode` | `"enforce"` | `enforce` applies valid decisions. `observe_only` records the hypothetical decision but dispatches the trusted same-protocol default once. | -| `priority` | `0` | Component execution priority. | -| `decision_api_url` | Required | HTTP or HTTPS URL for the Switchyard Decision API. | -| `decision_profile_id` | Required | Non-empty Switchyard profile identifier sent with each routing request. | -| `decision_timeout_millis` | `25` | Decision API timeout in milliseconds; must be greater than zero. | -| `decision_headers` | Empty | Optional non-sensitive static Decision API headers. | -| `decision_header_env` | Empty | Map of header names to environment-variable names for sensitive Decision API headers. Referenced values must be present and non-blank. | - -Do not define the same header, including case variants, in both a static header -map and its environment map. Keep bearer tokens, API keys, cookies, and other -secrets in environment variables rather than tracked configuration. - -## Request Materialization - -Every Decision API request contains routing identity, inbound protocol, a -request summary, and the attempt number. `request_materialization` controls how -much additional request content is included. - -| Mode | Material sent to Switchyard | -| --- | --- | -| `none` | No `current_request`; use identity, protocol, summary, and attempt metadata only. | -| `summary_only` | No `current_request`; explicitly selects summary-based routing. | -| `latest_user_prompt` | A provider-valid body containing the latest user material plus an explicit prompt field. The request fails open if no user message is available. | -| `recent_message_window` | The most recent normalized messages, bounded by `recent_message_count`, re-encoded for the inbound protocol. | -| `annotated_request` | The final intercepted provider body plus Relay's normalized request representation. | -| `full_body` | The final intercepted provider request body. | - -Choose the least materialization needed by the selected profile. In particular, -`none` and `summary_only` intentionally share the same current wire behavior but -express different configuration intent. - -## Context, Retries, and Inbound Protocols - -The following table describes context, retry, and inbound protocol settings: - -| Option | Default | Description and constraints | -| --- | --- | --- | -| `context_mode` | Required | `payload_only` or `atof_required`. See below. | -| `max_retries` | `3` | Provider retries after the initial attempt; maximum `10`. Each retry obtains a new Switchyard decision. | -| `recent_message_count` | `8` | Message limit for `recent_message_window`; must be greater than zero. | -| `enabled_inbound_profiles` | All three supported protocols | Non-empty list containing `openai_chat`, `openai_responses`, and/or `anthropic_messages`. | -| `atof_endpoint_name` | Required for `atof_required` | Name of exactly one stream sink under the observability component's `atof.sinks` list. | - -`payload_only` supports request-aware routing without an ATOF dependency. -Relay can create request-scoped synthetic identity when stable identity is not -available. - -`atof_required` is for profiles that classify from accumulated ATOF context. It -requires stable session and request identity plus an enabled, authenticated -Relay HTTP ATOF stream sink whose `name` matches `atof_endpoint_name`. The sink -specifies the Switchyard ingestion URL, must use `transport = "http_post"`, and -must set `field_name_policy = "preserve"`. Relay rejects missing or duplicate -named sinks during startup validation. A local ATOF file sink -does not satisfy this delivery requirement by itself. - -## Target Bindings and Trusted Defaults - -The following table describes target bindings and trusted defaults: - -| Option | Required fields | Purpose | -| --- | --- | --- | -| `targets.` | `model`, `protocol`, `endpoint`, `base_url` | Binds an exact Switchyard backend ID to a Relay-owned provider destination. | -| `targets..headers` | None | Optional non-sensitive static provider headers. | -| `targets..header_env` | None | Optional provider header names mapped to environment-variable names. Referenced values must be present and non-blank. | -| `default_targets.openai_chat` | Target using `openai_chat` | Trusted fallback for OpenAI Chat Completions requests. | -| `default_targets.openai_responses` | Target using `openai_responses` | Trusted fallback for OpenAI Responses requests. | -| `default_targets.anthropic_messages` | Target using `anthropic_messages` | Trusted fallback for Anthropic Messages requests. | - -Do not define the same target header, including case variants, in both -`headers` and `header_env`. - -Protocol and endpoint pairs are fixed: - -| Protocol | Endpoint | -| --- | --- | -| `openai_chat` | `/v1/chat/completions` | -| `openai_responses` | `/v1/responses` | -| `anthropic_messages` | `/v1/messages` | - -Each protocol in `enabled_inbound_profiles` requires a corresponding -`default_targets` entry. Defaults for disabled protocols can be omitted. Each -configured default must refer to a target with the matching protocol. Duplicate -exact bindings for model, protocol, endpoint, and base URL are rejected. - -## Recommended Recipes - -### Observe-Only Rollout - -Use observe-only mode to verify decisions and routing marks without sending -traffic to the selected target: - -```toml -[components.config] -mode = "observe_only" -context_mode = "payload_only" -request_materialization = "summary_only" -``` - -Relay calls the Decision API and records the hypothetical selection, but sends -the request to the inbound protocol's trusted default once. Routing retries are -not applied. - -### Payload-Only Request-Aware Routing - -Use recent request content for routing without operating the ATOF accumulator: - -```toml -[components.config] -mode = "enforce" -context_mode = "payload_only" -request_materialization = "recent_message_window" -recent_message_count = 8 -``` - -### ATOF-Backed StageRouter Routing - -Use ATOF context when StageRouter should classify from accumulated session -signals: - -```toml -[components.config] -mode = "enforce" -decision_profile_id = "stage-router" -context_mode = "atof_required" -request_materialization = "summary_only" -atof_endpoint_name = "switchyard" -``` - -Configure the matching observability component with a named stream sink: - -```toml -[[components]] -kind = "observability" -enabled = true - -[components.config] -version = 3 - -[components.config.atof] -enabled = true - -[[components.config.atof.sinks]] -type = "stream" -name = "switchyard" -url = "http://127.0.0.1:4000/v1/atof/events" -transport = "http_post" -field_name_policy = "preserve" - -[components.config.atof.sinks.header_env] -authorization = "SWITCHYARD_AUTHORIZATION" -``` - -## Runtime and Fail-Open Behavior - -The integration handles routing and failures as follows: - -- Relay validates the decision's backend ID, model, protocol, and endpoint - against its target binding before dispatch. Unknown backends, malformed - decisions, and target drift fail open. -- Decision API errors are not recursively retried. Relay immediately dispatches - the trusted same-protocol default. -- Retryable provider connection, timeout, status, context-window, and - model-unavailable failures can obtain a new decision, bounded by - `max_retries`. -- Non-retryable provider failures use the trusted fallback immediately. After - retry exhaustion, Relay dispatches that fallback once. -- A streaming request can retry only before receiving its first upstream item. - After the stream commits, Relay propagates later failures without another - dispatch. -- Relay forwards provider-specific extensions unchanged when every configured - target uses the inbound protocol. If any target uses another protocol, - nonportable extensions fail open to the trusted same-protocol default before - the Decision API call. Request translation failures before provider dispatch - and buffered response translation failures also fail open. After a streaming - response emits its first item, later provider or translation failures - propagate without another dispatch. -- Routing-mark delivery is best-effort and does not change provider results. - -## Experimental Limitations - -Consider the following experimental limitations before adopting the plugin: - -- The plugin is published as `nemo-relay-switchyard` but is excluded from - default Relay CLI builds. Build the CLI with - `cargo build -p nemo-relay-cli --features switchyard`. -- The routing boundary is currently service-based. Relay calls Switchyard's - HTTP Decision API but does not start or supervise the service. Relay derives - `/health` from `decision_api_url` and fails plugin activation unless the - service returns `{"status":"ok"}`. Decision API failures after activation fail - open to the trusted same-protocol default. -- ATOF-backed profiles additionally depend on Switchyard's HTTP ingestion and - accumulator runtime, stable request and session identity, and one named, - authenticated Relay ATOF HTTP stream sink. A local file sink does not populate - the Switchyard accumulator. -- Provider-protocol translation runs in Relay's process through the - `switchyard-translation` Rust library. Routing decisions and ATOF accumulation - remain out of process. -- The Rust Switchyard component is currently registered only by Relay's CLI - gateway. Hermes native plugins and the LangChain, LangGraph, and Deep Agents - integrations do not load it in process. They receive Switchyard routing only - when their provider traffic is explicitly sent through the Relay gateway. -- The Switchyard examples are manual compatibility and trajectory workflows, - not production deployment orchestration. -- Compatibility is pinned to Switchyard commit - [`8f9db9a6`](https://github.com/NVIDIA-NeMo/Switchyard/commit/8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8) - on `topic/nemo-relay-integration`. Testing another revision is deliberate and - can expose contract drift. -- The Decision API and service/library boundary are experimental. -- Nonportable provider extensions in configurations that can translate across - protocols, malformed decisions, service failures, pre-dispatch translation - failures, buffered-response translation failures, and target drift fail open - to Relay's trusted same-protocol default. Failures after a streaming response - commits propagate without another dispatch. -- Secrets must remain environment-referenced. The examples generate ephemeral - local credentials and are not credential-management guidance. - -Run `nemo-relay doctor` to validate component configuration before startup. For -the pinned local service and manual compatibility workflow, refer to the -[Switchyard integration examples](https://github.com/NVIDIA/NeMo-Relay/tree/release/0.6/examples/switchyard). diff --git a/docs/index.yml b/docs/index.yml index 26ebe49cc..3187f9573 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -40,9 +40,6 @@ navigation: - folder: ./configure-plugins/pii-redaction title: "PII Redaction" title-source: frontmatter - - folder: ./configure-plugins/switchyard - title: "Switchyard (Deprecated)" - title-source: frontmatter - page: "Model Pricing" path: ./configure-plugins/model-pricing.mdx slug: model-pricing diff --git a/docs/reference/migration-guides.mdx b/docs/reference/migration-guides.mdx index 97adf5407..f82fbaa47 100644 --- a/docs/reference/migration-guides.mdx +++ b/docs/reference/migration-guides.mdx @@ -30,6 +30,34 @@ NeMo Relay is built into Hermes Agent. Do not install Relay separately or enable an observability plugin. Hermes Agent understands NeMo Relay plugin configurations. +### Migrate to the Switchyard-Owned Dynamic Plugin + +Support for the Switchyard integration now shifts to the +[Switchyard project](https://github.com/NVIDIA-NeMo/Switchyard). Switchyard +0.3.0 will provide the dynamic plugin, release artifacts, and configuration +documentation. + +NeMo Relay >=0.8.0 no longer ships the experimental `nemo-relay-switchyard` +crate, the CLI `switchyard` feature, or the service-backed `switchyard` +component. Configurations that contain `[[components]]` entries with +`kind = "switchyard"` are rejected with migration guidance. + +To migrate an existing configuration: + +1. Remove the CLI `switchyard` Cargo feature and any dependency on + `nemo-relay-switchyard`. +2. Remove the legacy `[[components]]` entry with `kind = "switchyard"`. +3. Remove Decision API settings and ATOF HTTP sinks used only by the former + service integration. Keep any ATOF sink that another Relay consumer uses. +4. Install and configure the Switchyard-owned dynamic plugin by following the + documentation published with Switchyard 0.3.0. + +Do not copy service-era component fields into a dynamic-plugin entry. The +Switchyard release documentation is the source of truth for supported +algorithms, target configuration, and migration details. For Relay's manifest, +trust, policy, and activation model, refer to +[Configure Discoverable Plugins](/configure-plugins/discoverable-plugins). + ### Return Canonical Tool Execution Results diff --git a/examples/switchyard/README.md b/examples/switchyard/README.md deleted file mode 100644 index 883671e4f..000000000 --- a/examples/switchyard/README.md +++ /dev/null @@ -1,109 +0,0 @@ - - -# Switchyard Integration Examples - -> **Deprecated:** These examples exercise the experimental -> `nemo-relay-switchyard` plugin, which will be removed in NeMo Relay 0.8 and -> replaced by a Switchyard-owned native plugin. The NeMo Relay 0.8 documentation -> will include updated examples and a migration plan when the replacement is -> available. - -These examples exercise the experimental NeMo Relay 0.6.0 and 0.7.0 integration with a -separately running Switchyard Decision API service and the in-process Switchyard translation -library. They are manual, local validation workflows rather than production startup -orchestration. - -For the canonical architecture, setup, configuration, validation, and troubleshooting workflow, -refer to the -[Switchyard 0.6.0 setup and validation guide](https://docs.nvidia.com/nemo/relay/v0.6.0/configure-plugins/switchyard/about). - -## Required Switchyard Revision - -The scripts for NeMo Relay 0.6.0 and 0.7.0 require the following public topic branch and commit: - -```text -https://github.com/NVIDIA-NeMo/Switchyard/tree/topic/nemo-relay-integration -8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8 -``` - -Clone the Switchyard repository next to the Relay checkout, then pin the -required commit: - -```bash -git clone --branch topic/nemo-relay-integration \ - https://github.com/NVIDIA-NeMo/Switchyard.git \ - ../Switchyard-topic-nemo-relay-integration -git -C ../Switchyard-topic-nemo-relay-integration checkout --detach \ - 8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8 -``` - -Every real-service script verifies this commit before launching `switchyard-server`. To test a -deliberately different checkout, set both variables explicitly: - -```bash -SWITCHYARD_ROOT=/path/to/Switchyard \ -SWITCHYARD_EXPECTED_COMMIT= \ - examples/switchyard/run-real-e2e.sh -``` - -## Examples - -Run these commands from the root of the NeMo Relay checkout. - -The Relay CLI's Switchyard support is compile-time optional. The scripts enable the `switchyard` -feature automatically; custom builds must pass `--features switchyard`. - -### Manual Switchyard Compatibility Smoke Test - -`run-real-e2e.sh` is a manual compatibility smoke test. It starts the pinned Switchyard server, -Relay, and a fake provider, then verifies cold and warm StageRouter decisions, buffered routing, -SSE routing, and the selected model sequence. It is intended to catch cross-repository service -contract regressions; the CI-safe Relay process regression test covers the faster local behavior -checks. The script requires Rust tooling, Python, and `curl`; its temporary logs are removed after -a successful run. - -```bash -examples/switchyard/run-real-e2e.sh -``` - -A successful run ends with: - -```text -real Switchyard E2E passed: ['provider/weak', 'provider/strong', 'provider/strong'] -``` - -## Configuration Files - -The directory includes the following configuration and support files: - -- `plugins.toml`: minimal plugin configuration example. -- `real-e2e-plugins.toml` and `real-e2e-profiles.yaml`: deterministic fake-provider E2E. -- `fake_upstream.py`: deterministic provider used by the service E2E. -- `otel-collector.yaml`: local OTEL artifact export configuration. - -## Runtime Model - -The scripts launch Switchyard as a separate local process on port `4000`. Relay sends routing -requests to `/v1/routing/decision` and, for ATOF-backed profiles, sends events to -`/v1/atof/events`. Relay owns provider credentials, target bindings, dispatch, retries, and -fallback behavior. Relay executes provider-protocol translation in process through Switchyard's -translation library; the Switchyard service owns ATOF accumulation and routing decisions. The -Switchyard component selects its HTTP ingestion destination by the observability stream sink name, -not by duplicating the sink URL. - -The service is not started automatically by Relay outside these examples. A production deployment -must start a compatible Switchyard service before Relay activates the plugin and configure the -Relay plugin with its Decision API URL. Relay derives the service's root `/health` URL from that -configuration and refuses activation unless it reports `{"status":"ok"}`. - -## Artifacts and Troubleshooting - -Trajectory scripts write to `artifacts/` by default. Set `SWITCHYARD_TRAJECTORY_DIR` to choose a -shareable output directory. On failure, logs are preserved and include the verified Switchyard -revision. Do not place API keys or bearer tokens in configuration files; use environment variables -or an untracked secrets file. For service readiness failures, port conflicts, and the exact -temporary log locations used by `run-real-e2e.sh`, refer to the -[canonical Switchyard 0.6.0 guide](https://docs.nvidia.com/nemo/relay/v0.6.0/configure-plugins/switchyard/about#troubleshooting). diff --git a/examples/switchyard/e2e-common.sh b/examples/switchyard/e2e-common.sh deleted file mode 100755 index bf38627d6..000000000 --- a/examples/switchyard/e2e-common.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Shared process and readiness helpers for the manually-invoked Switchyard E2E scripts. - -e2e_pids=() - -e2e_add_pid() { - e2e_pids+=("$1") -} - -e2e_stop_tree() { - local pid="$1" - local child - for child in $(pgrep -P "$pid" 2>/dev/null || true); do - e2e_stop_tree "$child" - done - kill "$pid" 2>/dev/null || true -} - -e2e_stop_processes() { - local pid - for pid in "${e2e_pids[@]}"; do - e2e_stop_tree "$pid" - done - for pid in "${e2e_pids[@]}"; do - wait "$pid" 2>/dev/null || true - done - e2e_pids=() -} - -e2e_wait_for() { - local url="$1" - local attempts="${2:-120}" - local delay="${3:-0.25}" - local process_pid="${4:-}" - local attempt - for attempt in $(seq 1 "$attempts"); do - if [[ -n "$process_pid" ]] && ! kill -0 "$process_pid" 2>/dev/null; then - echo "process $process_pid exited before $url became ready" >&2 - return 1 - fi - if curl --fail --silent "$url" >/dev/null 2>&1; then - return 0 - fi - sleep "$delay" - done - echo "timed out waiting for $url" >&2 - return 1 -} - -e2e_random_token() { - python3 -c 'import secrets; print(secrets.token_hex(24))' -} - -# The experimental Relay integration is tested against a specific public -# Switchyard topic-branch commit. Override this only when intentionally testing -# another revision. -e2e_verify_switchyard_checkout() { - local root="$1" - local expected="${2:-8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8}" - local actual - actual="$(git -C "$root" rev-parse HEAD 2>/dev/null)" || { - echo "Switchyard worktree is not a git checkout: $root" >&2 - return 1 - } - if [[ "$actual" != "$expected" ]]; then - echo "Switchyard checkout mismatch: expected $expected, found $actual" >&2 - echo "Set SWITCHYARD_EXPECTED_COMMIT only when intentionally testing another revision." >&2 - return 1 - fi - printf 'Switchyard revision: %s (%s)\n' "$actual" "$(git -C "$root" show -s --format=%s HEAD)" -} - -e2e_tail_logs() { - local directory="$1" - local log - for log in "$directory"/*.log; do - [[ -f "$log" ]] || continue - echo "--- $log" >&2 - tail -100 "$log" >&2 - done -} diff --git a/examples/switchyard/fake_upstream.py b/examples/switchyard/fake_upstream.py deleted file mode 100644 index f27527d14..000000000 --- a/examples/switchyard/fake_upstream.py +++ /dev/null @@ -1,85 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Deterministic local OpenAI-compatible upstream for Switchyard plugin smoke tests.""" - -import argparse -import json -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path - - -class Handler(BaseHTTPRequestHandler): - log_path: Path - - def do_POST(self) -> None: # noqa: N802 - length = int(self.headers.get("content-length", "0")) - body = json.loads(self.rfile.read(length) or b"{}") - with self.log_path.open("a", encoding="utf-8") as output: - output.write(json.dumps({"path": self.path, "body": body}) + "\n") - model = body.get("model", "fake-model") - if body.get("stream"): - chunks = [ - { - "id": "chatcmpl-fake", - "object": "chat.completion.chunk", - "model": model, - "choices": [ - { - "index": 0, - "delta": {"role": "assistant", "content": "fake"}, - "finish_reason": None, - } - ], - }, - { - "id": "chatcmpl-fake", - "object": "chat.completion.chunk", - "model": model, - "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], - }, - ] - payload = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) - payload += "data: [DONE]\n\n" - encoded = payload.encode() - self.send_response(200) - self.send_header("content-type", "text/event-stream") - self.send_header("content-length", str(len(encoded))) - self.end_headers() - self.wfile.write(encoded) - return - response = { - "id": "chatcmpl-fake", - "object": "chat.completion", - "model": model, - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": "fake"}, - "finish_reason": "stop", - } - ], - "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, - } - encoded = json.dumps(response).encode() - self.send_response(200) - self.send_header("content-type", "application/json") - self.send_header("content-length", str(len(encoded))) - self.end_headers() - self.wfile.write(encoded) - - def log_message(self, _format: str, *_args: object) -> None: - return - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--port", type=int, default=4101) - parser.add_argument("--log", type=Path, required=True) - args = parser.parse_args() - Handler.log_path = args.log - ThreadingHTTPServer(("127.0.0.1", args.port), Handler).serve_forever() - - -if __name__ == "__main__": - main() diff --git a/examples/switchyard/otel-collector.yaml b/examples/switchyard/otel-collector.yaml deleted file mode 100644 index 330a87077..000000000 --- a/examples/switchyard/otel-collector.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -receivers: - otlp: - protocols: - http: - endpoint: 0.0.0.0:4318 - -exporters: - file/trajectory: - path: /artifacts/trajectory.otel.json - otlp/phoenix: - endpoint: phoenix:4317 - tls: - insecure: true - -service: - pipelines: - traces: - receivers: [otlp] - exporters: [file/trajectory, otlp/phoenix] diff --git a/examples/switchyard/plugins.toml b/examples/switchyard/plugins.toml deleted file mode 100644 index d6a1a69ed..000000000 --- a/examples/switchyard/plugins.toml +++ /dev/null @@ -1,75 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -version = 1 - -[[components]] -kind = "switchyard" -enabled = true - -[components.config] -mode = "enforce" -priority = 0 -decision_api_url = "http://127.0.0.1:4000/v1/routing/decision" -decision_profile_id = "smart-stage-router" -request_materialization = "recent_message_window" -context_mode = "atof_required" -atof_endpoint_name = "switchyard" -decision_timeout_millis = 25 -max_retries = 3 -recent_message_count = 8 -enabled_inbound_profiles = ["openai_chat", "openai_responses", "anthropic_messages"] - -[components.config.decision_header_env] -authorization = "SWITCHYARD_AUTHORIZATION" - -[components.config.default_targets] -openai_chat = "openai-default" -openai_responses = "responses-default" -anthropic_messages = "anthropic-default" - -[components.config.targets.openai-default] -model = "weak-model" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "http://127.0.0.1:4101" - -[components.config.targets.responses-default] -model = "weak-model" -protocol = "openai_responses" -endpoint = "/v1/responses" -base_url = "http://127.0.0.1:4101" - -[components.config.targets.anthropic-default] -model = "weak-model" -protocol = "anthropic_messages" -endpoint = "/v1/messages" -base_url = "http://127.0.0.1:4101" - -[components.config.targets.strong] -model = "strong-model" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "http://127.0.0.1:4102" - -[[components]] -kind = "observability" -enabled = true - -[components.config.atof] -enabled = true - -[[components.config.atof.sinks]] -type = "file" -mode = "append" - -[[components.config.atof.sinks]] -type = "stream" -name = "switchyard" -url = "http://127.0.0.1:4000/v1/atof/events" -transport = "http_post" -field_name_policy = "preserve" -timeout_millis = 3000 - -[components.config.atof.sinks.header_env] -authorization = "SWITCHYARD_AUTHORIZATION" diff --git a/examples/switchyard/real-e2e-plugins.toml b/examples/switchyard/real-e2e-plugins.toml deleted file mode 100644 index 541beb057..000000000 --- a/examples/switchyard/real-e2e-plugins.toml +++ /dev/null @@ -1,73 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -version = 1 - -[[components]] -kind = "switchyard" -enabled = true - -[components.config] -mode = "enforce" -decision_api_url = "http://127.0.0.1:4000/v1/routing/decision" -decision_profile_id = "remote-stage-router" -request_materialization = "summary_only" -context_mode = "atof_required" -atof_endpoint_name = "switchyard" -decision_timeout_millis = 1000 -max_retries = 3 -recent_message_count = 8 - -[components.config.decision_header_env] -authorization = "SWITCHYARD_AUTHORIZATION" - -[components.config.default_targets] -openai_chat = "weak" -openai_responses = "responses-default" -anthropic_messages = "anthropic-default" - -[components.config.targets.strong] -model = "provider/strong" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "http://127.0.0.1:4101" - -[components.config.targets.weak] -model = "provider/weak" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "http://127.0.0.1:4101" - -[components.config.targets.responses-default] -model = "provider/weak-responses" -protocol = "openai_responses" -endpoint = "/v1/responses" -base_url = "http://127.0.0.1:4101" - -[components.config.targets.anthropic-default] -model = "provider/weak-anthropic" -protocol = "anthropic_messages" -endpoint = "/v1/messages" -base_url = "http://127.0.0.1:4101" - -[[components]] -kind = "observability" -enabled = true - -[components.config.atof] -enabled = true - -[[components.config.atof.sinks]] -type = "file" -mode = "append" - -[[components.config.atof.sinks]] -type = "stream" -name = "switchyard" -url = "http://127.0.0.1:4000/v1/atof/events" -transport = "http_post" -field_name_policy = "preserve" -timeout_millis = 1000 - -[components.config.atof.sinks.header_env] -authorization = "SWITCHYARD_AUTHORIZATION" diff --git a/examples/switchyard/real-e2e-profiles.yaml b/examples/switchyard/real-e2e-profiles.yaml deleted file mode 100644 index 6127b46ef..000000000 --- a/examples/switchyard/real-e2e-profiles.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -targets: - strong: - model: provider/strong - format: openai - base_url: http://127.0.0.1:4101/v1 - weak: - model: provider/weak - format: openai - base_url: http://127.0.0.1:4101/v1 - -profiles: - remote-stage-router: - type: stage_router - capable: strong - efficient: weak - fallback_target_on_evict: strong - picker: efficient_first - confidence_threshold: 0.7 diff --git a/examples/switchyard/run-real-e2e.sh b/examples/switchyard/run-real-e2e.sh deleted file mode 100755 index 88a28d358..000000000 --- a/examples/switchyard/run-real-e2e.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -relay_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -source "$relay_root/examples/switchyard/e2e-common.sh" -relay_toolchain="" -if command -v rustup >/dev/null 2>&1; then - relay_toolchain="$(cd "$relay_root" && rustup show active-toolchain)" - relay_toolchain="${relay_toolchain%% *}" -fi -switchyard_root="${SWITCHYARD_ROOT:-$(cd "$relay_root/.." && pwd)/Switchyard-topic-nemo-relay-integration}" -switchyard_expected_commit="${SWITCHYARD_EXPECTED_COMMIT:-8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8}" -work_dir="$(mktemp -d)" -upstream_log="$work_dir/upstream.jsonl" -token="$(e2e_random_token)" - -[[ -d "$switchyard_root" ]] || { echo "Switchyard worktree not found: $switchyard_root" >&2; exit 1; } -e2e_verify_switchyard_checkout "$switchyard_root" "$switchyard_expected_commit" - -cleanup() { - local status=$? - e2e_stop_processes - if [[ $status -eq 0 ]]; then - rm -rf "$work_dir" - else - echo "E2E logs preserved in $work_dir" >&2 - e2e_tail_logs "$work_dir" - fi -} -trap cleanup EXIT - -python3 "$relay_root/examples/switchyard/fake_upstream.py" \ - --port 4101 --log "$upstream_log" >"$work_dir/upstream.log" 2>&1 & -e2e_add_pid "$!" - -( - cd "$switchyard_root" - SWITCHYARD_ATOF_BEARER_TOKEN="$token" cargo run -p switchyard-server -- \ - --config "$relay_root/examples/switchyard/real-e2e-profiles.yaml" --port 4000 -) >"$work_dir/switchyard.log" 2>&1 & -e2e_add_pid "$!" - -e2e_wait_for http://127.0.0.1:4000/health - -( - if [[ -n "$relay_toolchain" ]]; then - export RUSTUP_TOOLCHAIN="$relay_toolchain" - fi - cd "$work_dir" - SWITCHYARD_AUTHORIZATION="Bearer $token" cargo run \ - --manifest-path "$relay_root/Cargo.toml" -p nemo-relay-cli --features switchyard -- \ - --plugin-config-path "$relay_root/examples/switchyard/real-e2e-plugins.toml" \ - --bind 127.0.0.1:4041 -) >"$work_dir/relay.log" 2>&1 & -relay_pid="$!" -e2e_add_pid "$relay_pid" - -e2e_wait_for http://127.0.0.1:4041/healthz 120 0.25 "$relay_pid" - -request() { - local request_id="$1" - local stream="$2" - curl --fail --silent --no-buffer http://127.0.0.1:4041/v1/chat/completions \ - -H 'content-type: application/json' \ - -H 'x-nemo-relay-session-id: e2e-session' \ - -H "x-nemo-relay-request-id: $request_id" \ - --data-binary "{\"model\":\"client-model\",\"stream\":$stream,\"messages\":[{\"role\":\"user\",\"content\":\"hello\"}]}" -} - -request cold-request false >"$work_dir/cold.json" - -for payload in \ - '{"hook_event_name":"SessionStart","session_id":"e2e-session"}' \ - '{"hook_event_name":"PreToolUse","session_id":"e2e-session","tool_name":"Bash","tool_input":{"command":"test"},"tool_use_id":"call-1"}' \ - '{"hook_event_name":"PostToolUse","session_id":"e2e-session","tool_name":"Bash","tool_input":{"command":"test"},"tool_response":{"output":"CUDA out of memory"},"tool_use_id":"call-1"}' -do - curl --fail --silent http://127.0.0.1:4041/hooks/codex \ - -H 'content-type: application/json' --data-binary "$payload" >/dev/null -done - -sleep 1 -request warm-request false >"$work_dir/warm.json" -request stream-request true >"$work_dir/stream.sse" - -python3 - "$upstream_log" "$work_dir/stream.sse" <<'PY' -import json -import pathlib -import sys - -records = [json.loads(line) for line in pathlib.Path(sys.argv[1]).read_text().splitlines()] -models = [record["body"]["model"] for record in records] -if models != ["provider/weak", "provider/strong", "provider/strong"]: - raise SystemExit(f"unexpected cold/warm/stream route sequence: {models}") -stream = pathlib.Path(sys.argv[2]).read_text() -if "fake" not in stream or "[DONE]" not in stream: - raise SystemExit(f"unexpected SSE output: {stream}") -print(f"real Switchyard E2E passed: {models}") -PY diff --git a/fern/docs.yml b/fern/docs.yml index feb89ebe5..e7c3d5838 100644 --- a/fern/docs.yml +++ b/fern/docs.yml @@ -22,6 +22,10 @@ redirects: - source: /nemo/relay/about-nemo-relay/concepts/framework-integrations destination: /nemo/relay/integrate-into-frameworks/about +# Migration relocations +- source: /nemo/relay/configure-plugins/switchyard/about + destination: /nemo/relay/reference/migration-guides#migrate-to-the-switchyard-owned-dynamic-plugin + # Release notes - source: /nemo/relay/about-nemo-relay/release-notes/related-topics destination: /nemo/relay/about-nemo-relay/release-notes diff --git a/justfile b/justfile index 1d5c1c948..9de25f006 100644 --- a/justfile +++ b/justfile @@ -485,7 +485,6 @@ local_dependencies = ( "nemo-relay-plugin", "nemo-relay-adaptive", "nemo-relay-pii-redaction", - "nemo-relay-switchyard", "nemo-relay-ffi", "nemo-relay-cli", ) @@ -775,7 +774,6 @@ published_cargo_packages() { nemo-relay \ nemo-relay-adaptive \ nemo-relay-pii-redaction \ - nemo-relay-switchyard \ nemo-relay-ffi \ nemo-relay-cli } @@ -1659,11 +1657,6 @@ package-rust: cargo_package_config+=(--config 'patch.crates-io.nemo-relay.path="crates/core"') cargo_package_config+=(--config 'patch.crates-io.nemo-relay-plugin.path="crates/plugin"') ;; - nemo-relay-switchyard) - cargo_package_config+=(--config 'patch.crates-io.nemo-relay-types.path="crates/types"') - cargo_package_config+=(--config 'patch.crates-io.nemo-relay.path="crates/core"') - cargo_package_config+=(--config 'patch.crates-io.nemo-relay-plugin.path="crates/plugin"') - ;; nemo-relay-ffi|nemo-relay-cli) cargo_package_config+=(--config 'patch.crates-io.nemo-relay-types.path="crates/types"') cargo_package_config+=(--config 'patch.crates-io.nemo-relay.path="crates/core"')