chore: Migrate smoke tests to pytest

.agents/skills/python-tests/SKILL.md

@@ -26,6 +26,15 @@ license: Apache-2.0
   different input types.
 - If a fixture is needed for a test, but either does not return a value or the value is not used in the test, use the `@pytest.mark.usefixtures` decorator.
 - `tests/conftest.py` contains a `restore_environ_fixture` fixture that restores the environment variables to their original state after each test, it is defined with `autouse=True` so it is automatically applied to all tests. If you need to modify the environment variables in a test, do so using `os.environ` and the fixture will restore them after the test completes. There is no need to use `monkeypatch.setenv` to modify environment variables in tests.
+- Avoid defensive programming in tests. If a test fails, it should fail loudly and clearly, rather than silently passing due to defensive checks. For example
+  ```python
+  data = results["data"]
+  ```
+  is preferred over
+  ```python
+  data = results.get("data")
+  ```
+  Simply allow the resulting KeyError to be raised if the "data" key is not present in the results dictionary, as this will provide a clear indication of what went wrong in the test.
 
 ## Common Commands
 

.agents/skills/validate-change/SKILL.md

@@ -29,8 +29,7 @@ surfaces touched by a change.
   `cargo check -p fabric-python --locked`.
 - If public configuration types changed, confirm the schema snapshot tests in
   `just test-rust` pass and review generated schema diffs.
-- If an adapter or integration changed, run its focused tests or smoke path in
-  addition to the language suite.
+- If an adapter or integration changed, run its focused tests.
 - If documentation or examples changed, run `just docs` when practical and
   verify documented commands against the current repository.
 - If code changes alter APIs, commands, paths, packaging behavior, telemetry
@@ -40,19 +39,15 @@ surfaces touched by a change.
 ## Start With the Change Shape
 
 - **Rust core, CLI, or shared runtime semantics changed**
-  Run Rust formatting and tests. Add Python tests when the behavior is exposed
-  through the SDK, and run relevant CLI smoke tests for CLI behavior.
+  Run Rust formatting and tests. Add Python tests when the behavior is exposed through the SDK, and run relevant tests for CLI behavior.
 - **Python SDK or PyO3 binding changed**
   Use `python-tests`, run focused pytest tests first, then run
   `just test-python`. Rebuild with `just build-python` when native code or
   packaging changed.
 - **Adapter behavior changed**
-  Run the focused adapter tests under `tests/`, then `just test-python`. Add the
-  dependency-free smoke path when launch, environment, or lifecycle behavior
-  changed.
+  Run the focused adapter tests under `tests/adapters`, then `just test-python`.
 - **Harbor integration changed**
-  Run `tests/test_harbor_runner.py`, relevant Python integration smokes, and
-  `just test-python`.
+  Run `tests/test_harbor_runner.py`, then `just test-python`.
 - **Schema or public contract changed**
   Run both language suites and review changes under `schemas/` and generated API
   references.
@@ -92,11 +87,6 @@ just --fmt --check
 git diff --check
 ```
 
-Use `.github/workflows/ci_python.yml` as the source of truth for the
-dependency-free smoke list. Gated integration smokes require their documented
-credentials, sibling checkouts, services, or Docker environment; do not report
-them as run unless those prerequisites were present.
-
 ## Hygiene Checks
 
 Before review or handoff:

.github/workflows/ci_python.yml

@@ -51,43 +51,13 @@ jobs:
       - name: Set up uv
         uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8
 
-      # PyYAML is not a project dependency, but smoke_hermes_config_mapping
-      # imports it to read Hermes config fixtures, so install it for the smokes.
+      # PyYAML is not a project dependency, but test_hermes_config_mapping
+      # imports it to read Hermes config fixtures, so install it for the tests.
       - name: Build SDK with native extension
         run: |
           uv venv --python 3.12 .venv
           uv sync --group test --no-group dev --extra harbor --extra hermes --extra relay
 
-      # Dependency-free smokes only: the gated integration smokes
-      # (smoke_hermes_sdk, smoke_relay_integration, and the Docker-backed
-      # smoke_harbor_swebench_task) need an NVIDIA_API_KEY / a running Hermes /
-      # a sibling harbor checkout, so they are excluded.
-      - name: Run dependency-free smokes
-        run: |
-          set -euo pipefail
-          py=.venv/bin/python
-          smokes=(
-            python/tests/smoke_environment_handle.py
-            python/tests/smoke_sdk.py
-            python/tests/smoke_sdk_concurrency.py
-            python/tests/smoke_native_sdk.py
-            python/tests/smoke_typed_config.py
-            python/tests/smoke_consumer_neutral.py
-            python/tests/smoke_harbor_integration.py
-            python/tests/smoke_readme_examples.py
-            python/tests/smoke_sdk_sessions.py
-            tests/smoke_cli.py
-            tests/smoke_hermes_cli.py
-            tests/smoke_hermes_config_mapping.py
-            tests/smoke_swebench_style.py
-            tests/smoke_local_env_e2e.py
-          )
-          for s in "${smokes[@]}"; do
-            echo "::group::$s"
-            "$py" "$s"
-            echo "::endgroup::"
-          done
-
       - name: Run pytest
         run: |
           set -euo pipefail

README.md

@@ -313,8 +313,8 @@ requires `runtime.mode: session`; use `fabric run` for oneshot profiles and
 machine-readable stdout. Because `chat` is an interactive terminal UI, the
 transcript and metadata are written together on stderr.
 
-The opt-in real integration checks are `tests/smoke_hermes_session.py` and
-`tests/smoke_codex_cli.py`.
+The opt-in real integration checks are `tests/e2e/test_hermes_session.py` and
+`tests/e2e/test_codex_cli.py`.
 
 `FabricClient()` uses the native Rust binding. SDK `run(...)` and
 `start_session(...)` drive the core Fabric runtime lifecycle (`start_runtime` /
@@ -367,14 +367,6 @@ fabric run examples/code-review-agent \
   --input "Review the workspace and summarize the highest-risk issue."
 ```
 
-Run the real one-shot and session smoke after installing Fabric with its native
-extension:
-
-```bash
-python3 -m pip install -e ".[codex]"
-RUN_FABRIC_CODEX_INTEGRATION=1 python3 tests/smoke_codex_cli.py
-```
-
 Run the Hermes CLI adapter:
 
 ```bash
@@ -386,15 +378,37 @@ fabric run examples/code-review-agent \
   --input "Reply with exactly: hermes cli ok"
 ```
 
-Run Hermes with NeMo Relay enabled:
+## Tests
+
+To run the full test suite, bootstrap a virtual environment with the optional dependencies.
 
 ```bash
-uv venv .tmp/fabric-hermes-relay-venv --python 3.12
-uv pip install --python .tmp/fabric-hermes-relay-venv/bin/python \
-  -e ../nemo-relay \
-  -e ../hermes-agent
+uv venv --seed .venv --python 3.12'
+source .venv/bin/activate
+uv sync --all-groups --all-extras
+```
 
-export NVIDIA_API_KEY=...
-export HERMES_PYTHON="$PWD/.tmp/fabric-hermes-relay-venv/bin/python"
-RUN_FABRIC_RELAY_INTEGRATION=1 python3 tests/smoke_relay_integration.py
+Build Fabric and the Python extension, since we have already bootstrapped a virtual environment, we will pass the `no_uv` flag to avoid building reinstalling depdnendencies in the virtual environment.
+```bash
+just no_uv=true build-all
 ```
+
+Run both Rust and Python tests:
+```bash
+just no_uv=true test-all
+```
+
+Run just the Rust tests:
+```bash
+just no_uv=true test-rust
+```
+
+Run just the Python tests:
+```bash
+just no_uv=true test-python
+```
+
+Running `pytest` directly:
+```bash
+pytest
+```

examples/README.md

@@ -5,8 +5,7 @@ SPDX-License-Identifier: Apache-2.0
 
 # Examples
 
-This directory holds sample Fabric agent packages and single-file configs used
-by smoke tests and demos.
+This directory holds sample Fabric agent packages and single-file configs used tests and demos.
 
 The first example focuses on the shared Fabric contract:
 
@@ -29,5 +28,5 @@ fabric plan examples/code-review-agent --profile env_local --profile mcp_github
 fabric plan examples/code-review-agent --profile hermes_cli
 ```
 
-The dependency-free Hermes shim used by smoke tests lives under
+The dependency-free Hermes shim used by tests lives under
 `tests/fixtures/hermes-shim-agent`; it is not a maintained adapter.

integrations/harbor/README.md

@@ -85,30 +85,30 @@ credential-free pipeline check plus real Hermes, Hermes-with-Relay, and Codex
 variants. See [`demo/README.md`](demo/README.md) for the commands and recording
 flow.
 
-## Local Smoke
+## Local Test
 
-The lightweight smoke uses a fake Harbor environment and validates command
+The lightweight test uses a fake Harbor environment and validates command
 construction plus metadata propagation:
 
 ```bash
-python3 python/tests/smoke_harbor_integration.py
+pytest tests/python/test_harbor_integration.py
 ```
 
-## SWE-Bench Smoke
+## SWE-Bench Test
 
-The Docker-backed SWE-Bench smoke is opt-in because it requires Docker, a local
+The Docker-backed SWE-Bench test is opt-in because it requires Docker, a local
 SWE-Bench image, and a Harbor-generated task directory. Harbor still owns task
 materialization and verification; Fabric only invokes the configured harness and
 captures artifacts.
 
 ```bash
-RUN_FABRIC_HARBOR_SWEBENCH_DOCKER=1 python3 tests/smoke_harbor_swebench_task.py
+RUN_FABRIC_HARBOR_SWEBENCH_DOCKER=1 pytest tests/e2e/test_harbor_swebench_task.py
 ```
 
 To run the verifier path as well:
 
 ```bash
 RUN_FABRIC_HARBOR_SWEBENCH_DOCKER=1 \
 RUN_FABRIC_HARBOR_SWEBENCH_VERIFY=1 \
-python3 tests/smoke_harbor_swebench_task.py
+pytest tests/e2e/test_harbor_swebench_task.py
 ```

tests/_utils/utils.py

@@ -1,17 +1,37 @@
 # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+import subprocess
 from pathlib import Path
 
 import yaml
 
+REPO_ROOT = Path(__file__).resolve().parents[2]
+FABRIC_COMMAND = ("cargo", "run", "-q", "-p", "fabric-cli", "--")
+
+
+def run_fabric_cli(
+    *args: object,
+    stdin: str | None = None,
+    timeout: float = 60.0,
+) -> subprocess.CompletedProcess[str]:
+    """Run the Fabric CLI from the repository root and capture its output."""
+    return subprocess.run(
+        [*FABRIC_COMMAND, *(str(arg) for arg in args)],
+        cwd=REPO_ROOT,
+        input=stdin,
+        text=True,
+        capture_output=True,
+        check=False,
+        timeout=timeout,
+    )
+
 
 def assert_relay_disabled_native_observability(result: dict):
     """Assert telemetry-off runs still surface native harness evidence."""
 
     artifact_by_name = {
-        artifact["name"]: artifact
-        for artifact in result["artifacts"]["artifacts"]
+        artifact["name"]: artifact for artifact in result["artifacts"]["artifacts"]
     }
     assert "stdout" in artifact_by_name
     assert "relay_config" not in artifact_by_name

tests/test_adapaters_common_hermes.py → tests/adapters/test_adapaters_common_hermes.py

@@ -121,9 +121,8 @@ def test_validate_hermes_telemetry_provider_rejects_native(
 
 def test_build_hermes_config_maps_fabric_config_to_hermes_config(
     hermes_common: types.ModuleType,
-    monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    monkeypatch.setenv("MCP_URL", "http://localhost:9000/mcp")
+    os.environ["MCP_URL"] = "http://localhost:9000/mcp"
     payload = {
         "runtime_context": {"environment": {"workspace": "/workspace/repo"}},
         "capability_plan": {
@@ -196,7 +195,6 @@ def test_build_hermes_config_maps_fabric_config_to_hermes_config(
 
 def test_hermes_config_variation_matrix_surfaces_supported_capabilities(
     hermes_common: types.ModuleType,
-    monkeypatch: pytest.MonkeyPatch,
     tmp_path: Path,
 ) -> None:
     import nemo_fabric_adapters.common.utils as common_utils
@@ -215,7 +213,7 @@ def test_hermes_config_variation_matrix_surfaces_supported_capabilities(
         ),
         encoding="utf-8",
     )
-    monkeypatch.setenv("FABRIC_RELAY_CONFIG_PATH", str(relay_config))
+    os.environ["FABRIC_RELAY_CONFIG_PATH"] = str(relay_config)
     payload = {
         "runtime_context": {
             "runtime_id": "runtime-matrix",
@@ -395,7 +393,6 @@ def test_summarize_hermes_config(hermes_common: types.ModuleType) -> None:
 
 def test_configure_hermes_relay_sets_hermes_plugin_environment(
     hermes_common: types.ModuleType,
-    monkeypatch: pytest.MonkeyPatch,
     tmp_path: Path,
 ) -> None:
     config_path = tmp_path / "relay.json"
@@ -431,8 +428,8 @@ def test_configure_hermes_relay_sets_hermes_plugin_environment(
         ),
         encoding="utf-8",
     )
-    monkeypatch.setenv("FABRIC_RELAY_ENABLED", "true")
-    monkeypatch.setenv("FABRIC_RELAY_CONFIG_PATH", str(config_path))
+    os.environ["FABRIC_RELAY_ENABLED"] = "true"
+    os.environ["FABRIC_RELAY_CONFIG_PATH"] = str(config_path)
     payload = {
         "runtime_context": {"runtime_id": "runtime-relay"},
         "effective_config": {
@@ -466,8 +463,7 @@ def test_configure_hermes_relay_sets_hermes_plugin_environment(
 
 def test_configure_hermes_relay_returns_none_when_disabled(
     hermes_common: types.ModuleType,
-    monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    monkeypatch.delenv("FABRIC_RELAY_ENABLED", raising=False)
+    os.environ.pop("FABRIC_RELAY_ENABLED", None)
 
     assert hermes_common.configure_hermes_relay({}) is None