From 0d0ee5ac702328339d4953d2616cf251eb7e8104 Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Fri, 26 Jun 2026 13:09:28 -0700 Subject: [PATCH 1/4] Add Hermes capability hardening coverage Signed-off-by: Ajay Thorve --- python/tests/smoke_harbor_integration.py | 63 ++++- tests/smoke_hermes_capability_hardening.py | 270 +++++++++++++++++++++ 2 files changed, 328 insertions(+), 5 deletions(-) create mode 100644 tests/smoke_hermes_capability_hardening.py diff --git a/python/tests/smoke_harbor_integration.py b/python/tests/smoke_harbor_integration.py index 58e818762..6aa958ebd 100644 --- a/python/tests/smoke_harbor_integration.py +++ b/python/tests/smoke_harbor_integration.py @@ -9,6 +9,7 @@ import json import sys import tempfile +import types from dataclasses import dataclass from pathlib import Path from typing import Any @@ -16,13 +17,47 @@ ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / "python" / "src")) + +def install_harbor_stubs() -> None: + """Install minimal Harbor stubs for this smoke when Harbor is not present.""" + + class BaseAgent: + def __init__(self, logs_dir: Path, *args: Any, **kwargs: Any) -> None: + self.logs_dir = logs_dir + self.model_name = kwargs.get("model_name") + self.skills_dir = kwargs.get("skills_dir") + self.mcp_servers = kwargs.get("mcp_servers", []) + + class BaseEnvironment: + pass + + class AgentContext: + def __init__(self) -> None: + self.metadata: dict[str, Any] | None = None + + modules = { + "harbor": types.ModuleType("harbor"), + "harbor.agents": types.ModuleType("harbor.agents"), + "harbor.agents.base": types.ModuleType("harbor.agents.base"), + "harbor.environments": types.ModuleType("harbor.environments"), + "harbor.environments.base": types.ModuleType("harbor.environments.base"), + "harbor.models": types.ModuleType("harbor.models"), + "harbor.models.agent": types.ModuleType("harbor.models.agent"), + "harbor.models.agent.context": types.ModuleType("harbor.models.agent.context"), + } + modules["harbor.agents.base"].BaseAgent = BaseAgent + modules["harbor.environments.base"].BaseEnvironment = BaseEnvironment + modules["harbor.models.agent.context"].AgentContext = AgentContext + sys.modules.update(modules) + + try: from nemo_fabric.integrations.harbor import FabricAgent from harbor.models.agent.context import AgentContext -except ImportError as exc: - raise SystemExit( - "Install Harbor before running this smoke, for example: pip install -e ../harbor" - ) from exc +except ImportError: + install_harbor_stubs() + from nemo_fabric.integrations.harbor import FabricAgent + from harbor.models.agent.context import AgentContext @dataclass @@ -61,7 +96,23 @@ async def exec( "profile": "env_local", "harness_type": "hermes", "adapter_id": "nvidia.fabric.hermes.sdk", - "artifacts": {"artifacts": []}, + "artifacts": { + "root": "/workspace/agent/artifacts", + "artifacts": [ + { + "name": "stdout", + "kind": "log", + "path": "/workspace/agent/artifacts/stdout.txt", + "media_type": "text/plain", + }, + { + "name": "workspace_patch", + "kind": "patch", + "path": "/workspace/agent/artifacts/workspace.patch", + "media_type": "text/x-diff", + }, + ], + }, "telemetry": None, } ) @@ -98,6 +149,8 @@ async def main() -> None: assert context.metadata assert context.metadata["fabric"]["status"] == "succeeded" assert context.metadata["fabric"]["adapter_id"] == "nvidia.fabric.hermes.sdk" + artifacts = context.metadata["fabric"]["artifacts"]["artifacts"] + assert {artifact["name"] for artifact in artifacts} == {"stdout", "workspace_patch"} if __name__ == "__main__": diff --git a/tests/smoke_hermes_capability_hardening.py b/tests/smoke_hermes_capability_hardening.py new file mode 100644 index 000000000..97b878dac --- /dev/null +++ b/tests/smoke_hermes_capability_hardening.py @@ -0,0 +1,270 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Smoke coverage for Hermes capability hardening.""" + +from __future__ import annotations + +import asyncio +import json +import subprocess +import sys +import tempfile +from pathlib import Path +from shutil import copytree +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "python" / "src")) + +from nemo_fabric import FabricClient # noqa: E402 + +COMMAND = ("cargo", "run", "-q", "-p", "fabric-cli", "--") + + +async def main() -> None: + assert_hermes_config_variation_matrix() + + cases = [ + { + "name": "python_adapter", + "fixture": ROOT / "tests" / "fixtures" / "hermes-shim-agent", + "profile": "env_local", + "adapter_kind": "python", + "adapter_id": "test.fabric.hermes_shim", + "adapter_runner": "python", + "mode": "shim", + }, + { + "name": "process_adapter", + "fixture": ROOT / "tests" / "fixtures" / "hermes-cli-agent", + "profile": "env_local", + "adapter_kind": "process", + "adapter_id": "nvidia.fabric.hermes.cli", + "adapter_runner": "process", + "mode": "hermes_cli_oneshot", + }, + ] + + with tempfile.TemporaryDirectory(prefix="hermes-capability-hardening-") as tmpdir: + for case in cases: + cli_result = run_case(case, tmpdir, "cli") + sdk_result = await run_sdk_case(case, tmpdir) + + assert_normalized_runresult_parity(cli_result, sdk_result, case) + assert_relay_disabled_native_observability(cli_result, case) + assert_relay_disabled_native_observability(sdk_result, case) + assert_hermes_capability_config_visible(cli_result, case) + assert_hermes_capability_config_visible(sdk_result, case) + + print("smoke_hermes_capability_hardening ok") + + +def assert_hermes_config_variation_matrix() -> None: + matrix = [ + { + "name": "hermes_sdk", + "agent": ROOT / "examples" / "code-review-agent", + "profiles": ["hermes_sdk"], + "adapter_id": "nvidia.fabric.hermes.sdk", + "adapter_kind": "python", + "runtime_mode": "oneshot", + "relay_enabled": False, + }, + { + "name": "hermes_cli", + "agent": ROOT / "examples" / "code-review-agent", + "profiles": ["hermes_cli"], + "adapter_id": "nvidia.fabric.hermes.cli", + "adapter_kind": "process", + "runtime_mode": "oneshot", + "relay_enabled": False, + }, + { + "name": "hermes_relay", + "agent": ROOT / "examples" / "code-review-agent", + "profiles": ["hermes_relay"], + "adapter_id": "nvidia.fabric.hermes.sdk", + "adapter_kind": "python", + "runtime_mode": "oneshot", + "relay_enabled": True, + }, + { + "name": "hermes_cli_relay", + "agent": ROOT / "examples" / "code-review-agent", + "profiles": ["hermes_cli_relay"], + "adapter_id": "nvidia.fabric.hermes.cli", + "adapter_kind": "process", + "runtime_mode": "oneshot", + "relay_enabled": True, + }, + { + "name": "stacked_mcp_profile", + "agent": ROOT / "tests" / "fixtures" / "hermes-shim-agent", + "profiles": ["env_local", "mcp_github"], + "adapter_id": "test.fabric.hermes_shim", + "adapter_kind": "python", + "runtime_mode": "session", + "relay_enabled": True, + }, + ] + + for case in matrix: + args: list[object] = ["plan", case["agent"]] + for profile in case["profiles"]: + args.extend(["--profile", profile]) + plan = call_json(*args) + + assert plan["profiles"] == case["profiles"], case["name"] + descriptor = plan["adapter_descriptor"]["descriptor"] + assert descriptor["adapter_id"] == case["adapter_id"], case["name"] + assert descriptor["adapter_kind"] == case["adapter_kind"], case["name"] + + config = plan["config"] + assert config["runtime"]["mode"] == case["runtime_mode"], case["name"] + assert config["models"]["default"]["model"], case["name"] + + environment_plan = plan["environment_plan"] + assert environment_plan["workspace"], case["name"] + assert environment_plan["artifacts"], case["name"] + + native_capabilities = plan["capability_plan"]["native"] + assert native_capabilities["skill_paths"], case["name"] + assert "github" in native_capabilities["mcp_servers"], case["name"] + + telemetry_plan = plan["telemetry_plan"] + assert telemetry_plan["relay_enabled"] is case["relay_enabled"], case["name"] + if case["relay_enabled"]: + assert telemetry_plan["relay_output_dir"], case["name"] + + +def run_case(case: dict[str, Any], tmpdir: str, surface: str) -> dict[str, Any]: + temp_agent = Path(tmpdir) / f"{case['name']}-{surface}" + copytree(case["fixture"], temp_agent) + return call_json( + "run", + temp_agent, + "--profile", + case["profile"], + "--input", + f"hello {case['name']}", + ) + + +async def run_sdk_case(case: dict[str, Any], tmpdir: str) -> dict[str, Any]: + temp_agent = Path(tmpdir) / f"{case['name']}-sdk" + copytree(case["fixture"], temp_agent) + async with FabricClient(command=COMMAND, cwd=ROOT) as client: + return await client.run( + temp_agent, + profile=case["profile"], + input_text=f"hello {case['name']}", + ) + + +def assert_normalized_runresult_parity( + cli_result: dict[str, Any], + sdk_result: dict[str, Any], + case: dict[str, Any], +) -> None: + comparable_fields = [ + "agent_name", + "profile", + "harness_type", + "adapter_kind", + "adapter_id", + "status", + ] + for field in comparable_fields: + assert cli_result[field] == sdk_result[field], field + + assert cli_result.get("error") == sdk_result.get("error") + + assert cli_result["adapter_kind"] == case["adapter_kind"] + assert cli_result["adapter_id"] == case["adapter_id"] + assert cli_result["metadata"]["adapter_runner"] == case["adapter_runner"] + assert sdk_result["metadata"]["adapter_runner"] == case["adapter_runner"] + assert cli_result["output"]["harness"] == "hermes" + assert sdk_result["output"]["harness"] == "hermes" + assert cli_result["output"]["mode"] == case["mode"] + assert sdk_result["output"]["mode"] == case["mode"] + + for result in (cli_result, sdk_result): + assert result["status"] == "succeeded" + assert result["runtime_id"].startswith("runtime-") + assert result["invocation_id"].startswith("invocation-") + assert result["request_id"].startswith("request-") + assert isinstance(result["artifacts"]["artifacts"], list) + assert isinstance(result["events"], list) + assert result["events"], "RunResult events should not be empty" + + +def assert_relay_disabled_native_observability( + result: dict[str, Any], + case: dict[str, Any], +) -> None: + artifact_by_name = { + artifact["name"]: artifact + for artifact in result["artifacts"]["artifacts"] + } + assert "stdout" in artifact_by_name + assert "relay_config" not in artifact_by_name + assert not any(name.startswith("relay_") for name in artifact_by_name) + + stdout_path = Path(artifact_by_name["stdout"]["path"]) + assert stdout_path.is_file() + assert stdout_path.read_text(encoding="utf-8").strip() + + event_kinds = {event["kind"] for event in result["events"]} + assert {"runtime_start", "invocation_start", "invocation_end"} <= event_kinds + + telemetry = result["telemetry"] + assert telemetry is not None + assert telemetry["relay_enabled"] is False + + if case["adapter_kind"] == "process": + output = result["output"] + assert output["returncode"] == 0 + assert output["stderr"] == "" + assert Path(output["fabric_invocation"]).is_file() + + +def assert_hermes_capability_config_visible( + result: dict[str, Any], + case: dict[str, Any], +) -> None: + output = result["output"] + if case["adapter_kind"] == "python": + assert output["native_mcp_servers"] == ["github"] + assert output["native_skill_paths"] + assert output["managed_mcp_servers"] == [] + assert output["managed_skill_paths"] == [] + return + + hermes_config_path = Path(output["hermes_config_path"]) + assert hermes_config_path.is_file() + native_config = output["hermes_native_config"] + assert native_config["mcp_servers"] == ["github"] + assert native_config["skill_dirs"] + response = json.loads(output["response"]) + assert response["fake_hermes"] is True + assert response["prompt"] == "hello process_adapter" + + +def call_json(*args: object) -> dict[str, Any]: + completed = subprocess.run( + [*COMMAND, *(str(arg) for arg in args)], + cwd=ROOT, + text=True, + capture_output=True, + check=False, + ) + if completed.returncode != 0: + raise AssertionError( + f"command failed: {completed.args}\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + return json.loads(completed.stdout) + + +if __name__ == "__main__": + asyncio.run(main()) From f0b4f9e925278b80a84709c70486c86b1f908412 Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Fri, 26 Jun 2026 13:34:23 -0700 Subject: [PATCH 2/4] Consolidate Hermes hardening smokes Signed-off-by: Ajay Thorve --- python/tests/smoke_sdk.py | 136 ++++++++++- tests/smoke_cli.py | 40 +++ tests/smoke_hermes_capability_hardening.py | 270 --------------------- tests/smoke_hermes_cli.py | 22 ++ 4 files changed, 196 insertions(+), 272 deletions(-) delete mode 100644 tests/smoke_hermes_capability_hardening.py diff --git a/python/tests/smoke_sdk.py b/python/tests/smoke_sdk.py index e52349796..5dbbd518c 100644 --- a/python/tests/smoke_sdk.py +++ b/python/tests/smoke_sdk.py @@ -6,6 +6,8 @@ from __future__ import annotations import asyncio +import json +import subprocess import sys import tempfile from shutil import copytree @@ -16,10 +18,12 @@ from nemo_fabric import FabricClient +COMMAND = ("cargo", "run", "-q", "-p", "fabric-cli", "--") + async def main() -> None: async with FabricClient( - command=("cargo", "run", "-q", "-p", "fabric-cli", "--"), + command=COMMAND, cwd=ROOT, ) as client: await smoke(client) @@ -28,6 +32,7 @@ async def main() -> None: async def smoke(client: FabricClient) -> None: example_agent = ROOT / "examples" / "code-review-agent" fixture_agent = ROOT / "tests" / "fixtures" / "hermes-shim-agent" + process_fixture_agent = ROOT / "tests" / "fixtures" / "hermes-cli-agent" assert client.validate(example_agent).startswith("validated") @@ -47,13 +52,28 @@ async def smoke(client: FabricClient) -> None: assert multi_plan["telemetry_plan"]["relay_enabled"] is True with tempfile.TemporaryDirectory(prefix="fabric-python-sdk-") as tmpdir: - temp_agent = Path(tmpdir) / "hermes-shim-agent" + temp_agent = Path(tmpdir) / "hermes-shim-agent-sdk" + temp_cli_agent = Path(tmpdir) / "hermes-shim-agent-cli" + temp_process_agent = Path(tmpdir) / "hermes-cli-agent-sdk" + temp_process_cli_agent = Path(tmpdir) / "hermes-cli-agent-cli" copytree(fixture_agent, temp_agent) + copytree(fixture_agent, temp_cli_agent) + copytree(process_fixture_agent, temp_process_agent) + copytree(process_fixture_agent, temp_process_cli_agent) + hermes_result = await client.run( temp_agent, profile="env_local", input_text="hello hermes", ) + hermes_cli_result = call_json( + "run", + temp_cli_agent, + "--profile", + "env_local", + "--input", + "hello hermes", + ) structured = await client.run( temp_agent, profile="env_local", @@ -63,6 +83,38 @@ async def smoke(client: FabricClient) -> None: "context": {"task": {"source": "sdk-smoke"}}, }, ) + process_result = await client.run( + temp_process_agent, + profile="env_local", + input_text="hello process adapter", + ) + process_cli_result = call_json( + "run", + temp_process_cli_agent, + "--profile", + "env_local", + "--input", + "hello process adapter", + ) + + assert_sdk_cli_runresult_parity( + hermes_cli_result, + hermes_result, + adapter_kind="python", + adapter_id="test.fabric.hermes_shim", + adapter_runner="python", + mode="shim", + ) + assert_sdk_cli_runresult_parity( + process_cli_result, + process_result, + adapter_kind="process", + adapter_id="nvidia.fabric.hermes.cli", + adapter_runner="process", + mode="hermes_cli_oneshot", + ) + assert_relay_disabled_native_observability(hermes_result) + assert_relay_disabled_native_observability(process_result) assert hermes_result["status"] == "succeeded" assert hermes_result["adapter_kind"] == "python" @@ -76,6 +128,86 @@ async def smoke(client: FabricClient) -> None: assert structured["request_id"] == "sdk-structured-request" assert structured["output"]["received"] == "hello structured sdk" + process_response = json.loads(process_result["output"]["response"]) + assert process_response["fake_hermes"] is True + assert process_response["prompt"] == "hello process adapter" + + +def assert_sdk_cli_runresult_parity( + cli_result: dict, + sdk_result: dict, + *, + adapter_kind: str, + adapter_id: str, + adapter_runner: str, + mode: str, +) -> None: + comparable_fields = [ + "agent_name", + "profile", + "harness_type", + "adapter_kind", + "adapter_id", + "status", + ] + for field in comparable_fields: + assert cli_result[field] == sdk_result[field], field + + assert cli_result.get("error") == sdk_result.get("error") + assert cli_result["adapter_kind"] == adapter_kind + assert cli_result["adapter_id"] == adapter_id + assert cli_result["metadata"]["adapter_runner"] == adapter_runner + assert sdk_result["metadata"]["adapter_runner"] == adapter_runner + assert cli_result["output"]["harness"] == "hermes" + assert sdk_result["output"]["harness"] == "hermes" + assert cli_result["output"]["mode"] == mode + assert sdk_result["output"]["mode"] == mode + + for result in (cli_result, sdk_result): + assert result["status"] == "succeeded" + assert result["runtime_id"].startswith("runtime-") + assert result["invocation_id"].startswith("invocation-") + assert result["request_id"].startswith("request-") + assert isinstance(result["artifacts"]["artifacts"], list) + assert isinstance(result["events"], list) + assert result["events"], "RunResult events should not be empty" + + +def assert_relay_disabled_native_observability(result: dict) -> None: + artifact_by_name = { + artifact["name"]: artifact + for artifact in result["artifacts"]["artifacts"] + } + assert "stdout" in artifact_by_name + assert "relay_config" not in artifact_by_name + assert not any(name.startswith("relay_") for name in artifact_by_name) + + stdout_path = Path(artifact_by_name["stdout"]["path"]) + assert stdout_path.is_file() + assert stdout_path.read_text(encoding="utf-8").strip() + + event_kinds = {event["kind"] for event in result["events"]} + assert {"runtime_start", "invocation_start", "invocation_end"} <= event_kinds + + telemetry = result["telemetry"] + assert telemetry is not None + assert telemetry["relay_enabled"] is False + + +def call_json(*args: object) -> dict: + completed = subprocess.run( + [*COMMAND, *(str(arg) for arg in args)], + cwd=ROOT, + text=True, + capture_output=True, + check=False, + ) + if completed.returncode != 0: + raise AssertionError( + f"command failed: {completed.args}\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + return json.loads(completed.stdout) + if __name__ == "__main__": asyncio.run(main()) diff --git a/tests/smoke_cli.py b/tests/smoke_cli.py index f1cf51455..8c9de793d 100644 --- a/tests/smoke_cli.py +++ b/tests/smoke_cli.py @@ -58,6 +58,26 @@ def main() -> None: assert direct_plan["profile"] == str(direct_profile) assert direct_plan["adapter_descriptor"]["descriptor"]["adapter_id"] == "nvidia.fabric.hermes.sdk" + profile_plans = [ + ("hermes_sdk", "nvidia.fabric.hermes.sdk", "python", False), + ("hermes_cli", "nvidia.fabric.hermes.cli", "process", False), + ("hermes_relay", "nvidia.fabric.hermes.sdk", "python", True), + ("hermes_cli_relay", "nvidia.fabric.hermes.cli", "process", True), + ] + for profile, adapter_id, adapter_kind, relay_enabled in profile_plans: + profile_plan = call_json("plan", temp_example, "--profile", profile) + assert profile_plan["profiles"] == [profile] + descriptor = profile_plan["adapter_descriptor"]["descriptor"] + assert descriptor["adapter_id"] == adapter_id + assert descriptor["adapter_kind"] == adapter_kind + assert profile_plan["config"]["runtime"]["mode"] == "oneshot" + assert profile_plan["capability_plan"]["native"]["skill_paths"] + assert "github" in profile_plan["capability_plan"]["native"]["mcp_servers"] + telemetry_plan = profile_plan["telemetry_plan"] + assert telemetry_plan["relay_enabled"] is relay_enabled + if relay_enabled: + assert telemetry_plan["relay_output_dir"] + multi_plan = call_json( "plan", temp_fixture, @@ -82,6 +102,7 @@ def main() -> None: assert hermes["output"]["native_mcp_servers"] == ["github"] assert hermes["output"]["managed_skill_paths"] == [] assert hermes["output"]["managed_mcp_servers"] == [] + assert_relay_disabled_native_observability(hermes) request = json.dumps( { @@ -187,5 +208,24 @@ def run_raw(stdin: str, *args: object) -> subprocess.CompletedProcess[str]: ) +def assert_relay_disabled_native_observability(result: dict) -> None: + artifact_by_name = { + artifact["name"]: artifact + for artifact in result["artifacts"]["artifacts"] + } + assert "stdout" in artifact_by_name + assert "relay_config" not in artifact_by_name + assert not any(name.startswith("relay_") for name in artifact_by_name) + + stdout_path = Path(artifact_by_name["stdout"]["path"]) + assert stdout_path.is_file() + assert stdout_path.read_text(encoding="utf-8").strip() + + event_kinds = {event["kind"] for event in result["events"]} + assert {"runtime_start", "invocation_start", "invocation_end"} <= event_kinds + + assert result["telemetry"]["relay_enabled"] is False + + if __name__ == "__main__": main() diff --git a/tests/smoke_hermes_capability_hardening.py b/tests/smoke_hermes_capability_hardening.py deleted file mode 100644 index 97b878dac..000000000 --- a/tests/smoke_hermes_capability_hardening.py +++ /dev/null @@ -1,270 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Smoke coverage for Hermes capability hardening.""" - -from __future__ import annotations - -import asyncio -import json -import subprocess -import sys -import tempfile -from pathlib import Path -from shutil import copytree -from typing import Any - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "python" / "src")) - -from nemo_fabric import FabricClient # noqa: E402 - -COMMAND = ("cargo", "run", "-q", "-p", "fabric-cli", "--") - - -async def main() -> None: - assert_hermes_config_variation_matrix() - - cases = [ - { - "name": "python_adapter", - "fixture": ROOT / "tests" / "fixtures" / "hermes-shim-agent", - "profile": "env_local", - "adapter_kind": "python", - "adapter_id": "test.fabric.hermes_shim", - "adapter_runner": "python", - "mode": "shim", - }, - { - "name": "process_adapter", - "fixture": ROOT / "tests" / "fixtures" / "hermes-cli-agent", - "profile": "env_local", - "adapter_kind": "process", - "adapter_id": "nvidia.fabric.hermes.cli", - "adapter_runner": "process", - "mode": "hermes_cli_oneshot", - }, - ] - - with tempfile.TemporaryDirectory(prefix="hermes-capability-hardening-") as tmpdir: - for case in cases: - cli_result = run_case(case, tmpdir, "cli") - sdk_result = await run_sdk_case(case, tmpdir) - - assert_normalized_runresult_parity(cli_result, sdk_result, case) - assert_relay_disabled_native_observability(cli_result, case) - assert_relay_disabled_native_observability(sdk_result, case) - assert_hermes_capability_config_visible(cli_result, case) - assert_hermes_capability_config_visible(sdk_result, case) - - print("smoke_hermes_capability_hardening ok") - - -def assert_hermes_config_variation_matrix() -> None: - matrix = [ - { - "name": "hermes_sdk", - "agent": ROOT / "examples" / "code-review-agent", - "profiles": ["hermes_sdk"], - "adapter_id": "nvidia.fabric.hermes.sdk", - "adapter_kind": "python", - "runtime_mode": "oneshot", - "relay_enabled": False, - }, - { - "name": "hermes_cli", - "agent": ROOT / "examples" / "code-review-agent", - "profiles": ["hermes_cli"], - "adapter_id": "nvidia.fabric.hermes.cli", - "adapter_kind": "process", - "runtime_mode": "oneshot", - "relay_enabled": False, - }, - { - "name": "hermes_relay", - "agent": ROOT / "examples" / "code-review-agent", - "profiles": ["hermes_relay"], - "adapter_id": "nvidia.fabric.hermes.sdk", - "adapter_kind": "python", - "runtime_mode": "oneshot", - "relay_enabled": True, - }, - { - "name": "hermes_cli_relay", - "agent": ROOT / "examples" / "code-review-agent", - "profiles": ["hermes_cli_relay"], - "adapter_id": "nvidia.fabric.hermes.cli", - "adapter_kind": "process", - "runtime_mode": "oneshot", - "relay_enabled": True, - }, - { - "name": "stacked_mcp_profile", - "agent": ROOT / "tests" / "fixtures" / "hermes-shim-agent", - "profiles": ["env_local", "mcp_github"], - "adapter_id": "test.fabric.hermes_shim", - "adapter_kind": "python", - "runtime_mode": "session", - "relay_enabled": True, - }, - ] - - for case in matrix: - args: list[object] = ["plan", case["agent"]] - for profile in case["profiles"]: - args.extend(["--profile", profile]) - plan = call_json(*args) - - assert plan["profiles"] == case["profiles"], case["name"] - descriptor = plan["adapter_descriptor"]["descriptor"] - assert descriptor["adapter_id"] == case["adapter_id"], case["name"] - assert descriptor["adapter_kind"] == case["adapter_kind"], case["name"] - - config = plan["config"] - assert config["runtime"]["mode"] == case["runtime_mode"], case["name"] - assert config["models"]["default"]["model"], case["name"] - - environment_plan = plan["environment_plan"] - assert environment_plan["workspace"], case["name"] - assert environment_plan["artifacts"], case["name"] - - native_capabilities = plan["capability_plan"]["native"] - assert native_capabilities["skill_paths"], case["name"] - assert "github" in native_capabilities["mcp_servers"], case["name"] - - telemetry_plan = plan["telemetry_plan"] - assert telemetry_plan["relay_enabled"] is case["relay_enabled"], case["name"] - if case["relay_enabled"]: - assert telemetry_plan["relay_output_dir"], case["name"] - - -def run_case(case: dict[str, Any], tmpdir: str, surface: str) -> dict[str, Any]: - temp_agent = Path(tmpdir) / f"{case['name']}-{surface}" - copytree(case["fixture"], temp_agent) - return call_json( - "run", - temp_agent, - "--profile", - case["profile"], - "--input", - f"hello {case['name']}", - ) - - -async def run_sdk_case(case: dict[str, Any], tmpdir: str) -> dict[str, Any]: - temp_agent = Path(tmpdir) / f"{case['name']}-sdk" - copytree(case["fixture"], temp_agent) - async with FabricClient(command=COMMAND, cwd=ROOT) as client: - return await client.run( - temp_agent, - profile=case["profile"], - input_text=f"hello {case['name']}", - ) - - -def assert_normalized_runresult_parity( - cli_result: dict[str, Any], - sdk_result: dict[str, Any], - case: dict[str, Any], -) -> None: - comparable_fields = [ - "agent_name", - "profile", - "harness_type", - "adapter_kind", - "adapter_id", - "status", - ] - for field in comparable_fields: - assert cli_result[field] == sdk_result[field], field - - assert cli_result.get("error") == sdk_result.get("error") - - assert cli_result["adapter_kind"] == case["adapter_kind"] - assert cli_result["adapter_id"] == case["adapter_id"] - assert cli_result["metadata"]["adapter_runner"] == case["adapter_runner"] - assert sdk_result["metadata"]["adapter_runner"] == case["adapter_runner"] - assert cli_result["output"]["harness"] == "hermes" - assert sdk_result["output"]["harness"] == "hermes" - assert cli_result["output"]["mode"] == case["mode"] - assert sdk_result["output"]["mode"] == case["mode"] - - for result in (cli_result, sdk_result): - assert result["status"] == "succeeded" - assert result["runtime_id"].startswith("runtime-") - assert result["invocation_id"].startswith("invocation-") - assert result["request_id"].startswith("request-") - assert isinstance(result["artifacts"]["artifacts"], list) - assert isinstance(result["events"], list) - assert result["events"], "RunResult events should not be empty" - - -def assert_relay_disabled_native_observability( - result: dict[str, Any], - case: dict[str, Any], -) -> None: - artifact_by_name = { - artifact["name"]: artifact - for artifact in result["artifacts"]["artifacts"] - } - assert "stdout" in artifact_by_name - assert "relay_config" not in artifact_by_name - assert not any(name.startswith("relay_") for name in artifact_by_name) - - stdout_path = Path(artifact_by_name["stdout"]["path"]) - assert stdout_path.is_file() - assert stdout_path.read_text(encoding="utf-8").strip() - - event_kinds = {event["kind"] for event in result["events"]} - assert {"runtime_start", "invocation_start", "invocation_end"} <= event_kinds - - telemetry = result["telemetry"] - assert telemetry is not None - assert telemetry["relay_enabled"] is False - - if case["adapter_kind"] == "process": - output = result["output"] - assert output["returncode"] == 0 - assert output["stderr"] == "" - assert Path(output["fabric_invocation"]).is_file() - - -def assert_hermes_capability_config_visible( - result: dict[str, Any], - case: dict[str, Any], -) -> None: - output = result["output"] - if case["adapter_kind"] == "python": - assert output["native_mcp_servers"] == ["github"] - assert output["native_skill_paths"] - assert output["managed_mcp_servers"] == [] - assert output["managed_skill_paths"] == [] - return - - hermes_config_path = Path(output["hermes_config_path"]) - assert hermes_config_path.is_file() - native_config = output["hermes_native_config"] - assert native_config["mcp_servers"] == ["github"] - assert native_config["skill_dirs"] - response = json.loads(output["response"]) - assert response["fake_hermes"] is True - assert response["prompt"] == "hello process_adapter" - - -def call_json(*args: object) -> dict[str, Any]: - completed = subprocess.run( - [*COMMAND, *(str(arg) for arg in args)], - cwd=ROOT, - text=True, - capture_output=True, - check=False, - ) - if completed.returncode != 0: - raise AssertionError( - f"command failed: {completed.args}\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" - ) - return json.loads(completed.stdout) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/tests/smoke_hermes_cli.py b/tests/smoke_hermes_cli.py index c6f8925df..1f9bbf918 100644 --- a/tests/smoke_hermes_cli.py +++ b/tests/smoke_hermes_cli.py @@ -46,6 +46,7 @@ def main() -> None: config_path = Path(result["output"]["hermes_config_path"]) assert config_path.is_file() + assert_relay_disabled_native_observability(result) def call_json(*args: object) -> dict: @@ -63,5 +64,26 @@ def call_json(*args: object) -> dict: return json.loads(completed.stdout) +def assert_relay_disabled_native_observability(result: dict) -> None: + artifact_by_name = { + artifact["name"]: artifact + for artifact in result["artifacts"]["artifacts"] + } + assert "stdout" in artifact_by_name + assert "relay_config" not in artifact_by_name + assert not any(name.startswith("relay_") for name in artifact_by_name) + + stdout_path = Path(artifact_by_name["stdout"]["path"]) + assert stdout_path.is_file() + assert stdout_path.read_text(encoding="utf-8").strip() + + event_kinds = {event["kind"] for event in result["events"]} + assert {"runtime_start", "invocation_start", "invocation_end"} <= event_kinds + + assert result["telemetry"]["relay_enabled"] is False + assert result["output"]["returncode"] == 0 + assert result["output"]["stderr"] == "" + + if __name__ == "__main__": main() From 76dd14aa2b8411ff5bc498f5bcf9d9e363feaa53 Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Fri, 26 Jun 2026 15:37:05 -0700 Subject: [PATCH 3/4] Address Hermes smoke review comments Signed-off-by: Ajay Thorve --- python/tests/smoke_sdk.py | 28 ++++++---------------------- tests/_utils/utils.py | 31 +++++++++++++++++++++++++++++++ tests/smoke_cli.py | 23 ++++------------------- tests/smoke_hermes_cli.py | 25 +++---------------------- 4 files changed, 44 insertions(+), 63 deletions(-) diff --git a/python/tests/smoke_sdk.py b/python/tests/smoke_sdk.py index 5dbbd518c..09c110f91 100644 --- a/python/tests/smoke_sdk.py +++ b/python/tests/smoke_sdk.py @@ -15,7 +15,12 @@ ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / "python" / "src")) +sys.path.insert(0, str(ROOT / "tests")) +from _utils.utils import ( # noqa: E402 + assert_process_adapter_native_observability, + assert_relay_disabled_native_observability, +) from nemo_fabric import FabricClient COMMAND = ("cargo", "run", "-q", "-p", "fabric-cli", "--") @@ -114,7 +119,7 @@ async def smoke(client: FabricClient) -> None: mode="hermes_cli_oneshot", ) assert_relay_disabled_native_observability(hermes_result) - assert_relay_disabled_native_observability(process_result) + assert_process_adapter_native_observability(process_result) assert hermes_result["status"] == "succeeded" assert hermes_result["adapter_kind"] == "python" @@ -173,27 +178,6 @@ def assert_sdk_cli_runresult_parity( assert result["events"], "RunResult events should not be empty" -def assert_relay_disabled_native_observability(result: dict) -> None: - artifact_by_name = { - artifact["name"]: artifact - for artifact in result["artifacts"]["artifacts"] - } - assert "stdout" in artifact_by_name - assert "relay_config" not in artifact_by_name - assert not any(name.startswith("relay_") for name in artifact_by_name) - - stdout_path = Path(artifact_by_name["stdout"]["path"]) - assert stdout_path.is_file() - assert stdout_path.read_text(encoding="utf-8").strip() - - event_kinds = {event["kind"] for event in result["events"]} - assert {"runtime_start", "invocation_start", "invocation_end"} <= event_kinds - - telemetry = result["telemetry"] - assert telemetry is not None - assert telemetry["relay_enabled"] is False - - def call_json(*args: object) -> dict: completed = subprocess.run( [*COMMAND, *(str(arg) for arg in args)], diff --git a/tests/_utils/utils.py b/tests/_utils/utils.py index 105541edc..3750eb3a4 100644 --- a/tests/_utils/utils.py +++ b/tests/_utils/utils.py @@ -6,6 +6,37 @@ import yaml +def assert_relay_disabled_native_observability(result: dict): + """Assert telemetry-off runs still surface native harness evidence.""" + + artifact_by_name = { + artifact["name"]: artifact + for artifact in result["artifacts"]["artifacts"] + } + assert "stdout" in artifact_by_name + assert "relay_config" not in artifact_by_name + assert not any(name.startswith("relay_") for name in artifact_by_name) + + stdout_path = Path(artifact_by_name["stdout"]["path"]) + assert stdout_path.is_file() + assert stdout_path.read_text(encoding="utf-8").strip() + + event_kinds = {event["kind"] for event in result["events"]} + assert {"runtime_start", "invocation_start", "invocation_end"} <= event_kinds + + telemetry = result["telemetry"] + assert telemetry is not None + assert telemetry["relay_enabled"] is False + + +def assert_process_adapter_native_observability(result: dict): + """Assert process adapters preserve native evidence and clean process output.""" + + assert_relay_disabled_native_observability(result) + assert result["output"]["returncode"] == 0 + assert result["output"]["stderr"] == "" + + def update_hermes_cli_relay_base_url(code_review_agent_dir: Path, api_server: str): """ Update the base URL in the Hermes CLI relay profile. diff --git a/tests/smoke_cli.py b/tests/smoke_cli.py index 8c9de793d..f7cb922fa 100644 --- a/tests/smoke_cli.py +++ b/tests/smoke_cli.py @@ -11,6 +11,8 @@ from pathlib import Path from shutil import copytree +from _utils.utils import assert_relay_disabled_native_observability + ROOT = Path(__file__).resolve().parents[1] COMMAND = ("cargo", "run", "-q", "-p", "fabric-cli", "--") @@ -77,6 +79,8 @@ def main() -> None: assert telemetry_plan["relay_enabled"] is relay_enabled if relay_enabled: assert telemetry_plan["relay_output_dir"] + else: + assert not telemetry_plan.get("relay_output_dir") multi_plan = call_json( "plan", @@ -208,24 +212,5 @@ def run_raw(stdin: str, *args: object) -> subprocess.CompletedProcess[str]: ) -def assert_relay_disabled_native_observability(result: dict) -> None: - artifact_by_name = { - artifact["name"]: artifact - for artifact in result["artifacts"]["artifacts"] - } - assert "stdout" in artifact_by_name - assert "relay_config" not in artifact_by_name - assert not any(name.startswith("relay_") for name in artifact_by_name) - - stdout_path = Path(artifact_by_name["stdout"]["path"]) - assert stdout_path.is_file() - assert stdout_path.read_text(encoding="utf-8").strip() - - event_kinds = {event["kind"] for event in result["events"]} - assert {"runtime_start", "invocation_start", "invocation_end"} <= event_kinds - - assert result["telemetry"]["relay_enabled"] is False - - if __name__ == "__main__": main() diff --git a/tests/smoke_hermes_cli.py b/tests/smoke_hermes_cli.py index 1f9bbf918..2e6cf140a 100644 --- a/tests/smoke_hermes_cli.py +++ b/tests/smoke_hermes_cli.py @@ -11,6 +11,8 @@ from pathlib import Path from shutil import copytree +from _utils.utils import assert_process_adapter_native_observability + ROOT = Path(__file__).resolve().parents[1] COMMAND = ("cargo", "run", "-q", "-p", "fabric-cli", "--") @@ -46,7 +48,7 @@ def main() -> None: config_path = Path(result["output"]["hermes_config_path"]) assert config_path.is_file() - assert_relay_disabled_native_observability(result) + assert_process_adapter_native_observability(result) def call_json(*args: object) -> dict: @@ -64,26 +66,5 @@ def call_json(*args: object) -> dict: return json.loads(completed.stdout) -def assert_relay_disabled_native_observability(result: dict) -> None: - artifact_by_name = { - artifact["name"]: artifact - for artifact in result["artifacts"]["artifacts"] - } - assert "stdout" in artifact_by_name - assert "relay_config" not in artifact_by_name - assert not any(name.startswith("relay_") for name in artifact_by_name) - - stdout_path = Path(artifact_by_name["stdout"]["path"]) - assert stdout_path.is_file() - assert stdout_path.read_text(encoding="utf-8").strip() - - event_kinds = {event["kind"] for event in result["events"]} - assert {"runtime_start", "invocation_start", "invocation_end"} <= event_kinds - - assert result["telemetry"]["relay_enabled"] is False - assert result["output"]["returncode"] == 0 - assert result["output"]["stderr"] == "" - - if __name__ == "__main__": main() From 58fbf6e620ce8271f778825b30ebe0392de1dca6 Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Fri, 26 Jun 2026 15:57:59 -0700 Subject: [PATCH 4/4] Run Harbor smoke in Python CI Signed-off-by: Ajay Thorve --- .github/workflows/ci_python.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_python.yml b/.github/workflows/ci_python.yml index d1e6b5930..db7b63a97 100644 --- a/.github/workflows/ci_python.yml +++ b/.github/workflows/ci_python.yml @@ -54,9 +54,9 @@ jobs: uv pip install --python .venv/bin/python -e . "pyyaml>=6" # Dependency-free smokes only: the gated integration smokes - # (smoke_hermes_sdk, smoke_relay_integration, smoke_harbor_*) need an - # NVIDIA_API_KEY / a running Hermes / a sibling harbor checkout, so they - # are excluded. + # (smoke_hermes_sdk, smoke_relay_integration, and the Docker-backed + # smoke_harbor_swebench_task) need an NVIDIA_API_KEY / a running Hermes / + # a sibling harbor checkout, so they are excluded. - name: Run dependency-free smokes run: | set -euo pipefail @@ -68,6 +68,7 @@ jobs: python/tests/smoke_native_sdk.py python/tests/smoke_typed_config.py python/tests/smoke_consumer_neutral.py + python/tests/smoke_harbor_integration.py python/tests/smoke_readme_examples.py python/tests/smoke_sdk_sessions.py tests/smoke_cli.py