diff --git a/adapters/claude/README.md b/adapters/claude/README.md index 18c65a4c1..7c6f0ff50 100644 --- a/adapters/claude/README.md +++ b/adapters/claude/README.md @@ -21,6 +21,17 @@ Claude Code authentication can come from an existing cached login or from module-entrypoint tests. Authentication is validated when Claude starts the invocation. +Relay-enabled runs also require the external `nemo-relay` CLI. Install the CLI +separately: + +```bash +cargo install nemo-relay-cli +``` + +The Python `nemo-relay` package does not install this executable. Refer to the +[NeMo Relay installation guide](https://docs.nvidia.com/nemo/relay/getting-started/installation) +for other supported installation methods. + ## Execution Model Each `invoke` starts a fresh adapter process. The adapter persists the terminal @@ -49,6 +60,7 @@ Only Claude-specific controls belong in `harness.settings`: - `max_turns`, `max_budget_usd`, and `timeout_seconds` - `setting_sources` (defaults to `[]` for deterministic isolation) - `cli_path` for testing or an explicitly installed Claude Code executable +- `nemo_relay_command` for an explicitly installed NeMo Relay CLI executable - `env` for variables explicitly forwarded to Claude Code Putting `model_name`, `cwd`, `tools`, `mcp_servers`, or `skills` in @@ -60,6 +72,31 @@ It retains portable OS/config variables, the selected model's `api_key_env`, and explicitly configured `settings.env` values. Raw Claude stderr is consumed by the SDK and is not persisted as a Fabric artifact. +## Relay Observability + +Enable Relay through the normalized Fabric configuration: + +```python +config.enable_relay( + project="fabric-review", + output_dir="./artifacts/relay", +) +``` + +For each Relay-enabled invocation, Fabric starts one `nemo-relay` gateway, +waits for its health endpoint, and stops it after Claude succeeds, fails, times +out, or is canceled. Fabric passes the gateway URL to Claude Code through +`ANTHROPIC_BASE_URL` and `NEMO_RELAY_GATEWAY_URL`. It also stages an +invocation-scoped Claude plugin that forwards lifecycle hooks with +`nemo-relay hook-forward claude`. + +The Fabric result includes `relay_runtime.gateway_config_path`, +`relay_runtime.gateway_log_path`, and the collected `relay_artifacts`. Relay +startup failures return a stable adapter error and retain the gateway log for +diagnosis. The default Claude Agent SDK dependency bundles a compatible Claude +Code executable. An executable supplied with `cli_path` must support the Relay +plugin's complete hook set, including `UserPromptExpansion`. + ## Typed Configuration Build the agent configuration with the typed SDK models before invoking @@ -148,9 +185,14 @@ underlying transcript store is removed. ## Tests -The default suite uses a deterministic mock Claude Code CLI and requires no -credentials. Run the real integration only on an authenticated developer host: +The default suite uses deterministic mock Claude Code and Relay CLIs and +requires no credentials. Test a current `nemo-relay` CLI with the mock Claude +client, or run the live integrations on an authenticated developer host: ```bash +FABRIC_NEMO_RELAY_COMMAND="$(command -v nemo-relay)" uv run --no-sync pytest tests/e2e/test_claude.py -q -k real_relay_gateway RUN_FABRIC_CLAUDE_INTEGRATION=1 uv run --no-sync pytest tests/e2e/test_claude.py -q -k live +RUN_FABRIC_CLAUDE_RELAY_INTEGRATION=1 uv run --no-sync pytest tests/e2e/test_claude.py -q -k live_claude_relay ``` + +The first command uses the mock Claude client and does not require credentials. diff --git a/adapters/claude/fabric-adapter.json b/adapters/claude/fabric-adapter.json index 17803e9c9..03406131e 100644 --- a/adapters/claude/fabric-adapter.json +++ b/adapters/claude/fabric-adapter.json @@ -8,6 +8,14 @@ "callable": "run" }, "config": { - "accepts": ["models", "tools", "mcp", "skills"] + "accepts": ["models", "tools", "mcp", "skills", "telemetry"] + }, + "telemetry": { + "providers": { + "relay": { + "outputs": ["atif", "otel", "openinference"], + "integration_modes": ["hooks", "gateway"] + } + } } } diff --git a/adapters/claude/pyproject.toml b/adapters/claude/pyproject.toml index 4fb4d8e22..5ca2c65ef 100644 --- a/adapters/claude/pyproject.toml +++ b/adapters/claude/pyproject.toml @@ -27,6 +27,7 @@ requires-python = ">=3.11" dependencies = [ "nemo-fabric-adapters-common == 0.1.0", "claude-agent-sdk==0.2.114", + "tomli-w~=1.2", ] [project.urls] diff --git a/adapters/claude/src/nemo_fabric_adapters/claude/adapter.py b/adapters/claude/src/nemo_fabric_adapters/claude/adapter.py index 249b5f830..144639748 100644 --- a/adapters/claude/src/nemo_fabric_adapters/claude/adapter.py +++ b/adapters/claude/src/nemo_fabric_adapters/claude/adapter.py @@ -11,7 +11,7 @@ import os import shlex import shutil -from dataclasses import asdict, is_dataclass +from dataclasses import asdict, dataclass, is_dataclass from hashlib import sha256 from pathlib import Path from typing import Any @@ -29,10 +29,19 @@ ) from claude_agent_sdk._errors import MessageParseError +import nemo_fabric_adapters.common.relay_gateway as relay_gateway +import nemo_fabric_adapters.common.relay_hooks as relay_hooks import nemo_fabric_adapters.common.utils as common_utils -PERMISSION_MODES = {"default", "acceptEdits", "bypassPermissions", "plan", "dontAsk", "auto"} +PERMISSION_MODES = { + "default", + "acceptEdits", + "bypassPermissions", + "plan", + "dontAsk", + "auto", +} SETTING_SOURCES = {"user", "project", "local"} NORMALIZED_SETTING_FIELDS = { "model_name": "FabricConfig.models", @@ -73,13 +82,29 @@ } +@dataclass(frozen=True) +class ClaudeRelaySettings: + """Invocation-scoped Relay gateway and Claude plugin settings.""" + + gateway: relay_gateway.RelayGatewayLaunch + plugin_config: dict[str, Any] + plugin_path: Path + + class ClaudeAdapterError(Exception): """Expected adapter error with a stable public code.""" - def __init__(self, code: str, message: str) -> None: + def __init__( + self, + code: str, + message: str, + *, + metadata: dict[str, Any] | None = None, + ) -> None: super().__init__(message) self.code = code self.message = message + self.metadata = metadata or {} class AdapterInputError(ClaudeAdapterError): @@ -94,37 +119,52 @@ class AdapterStateError(ClaudeAdapterError): """Invalid persisted runtime state.""" +class AdapterRelayError(ClaudeAdapterError): + """NeMo Relay setup or lifecycle failure.""" + + def _mapping(value: Any, *, name: str) -> dict[str, Any]: if value is None: return {} if not isinstance(value, dict): - raise AdapterConfigError("claude_invalid_configuration", f"{name} must be a mapping") + raise AdapterConfigError( + "claude_invalid_configuration", f"{name} must be a mapping" + ) return value def _string_list(value: Any, *, name: str) -> list[str]: if value is None: return [] - if not isinstance(value, list) or any(not isinstance(item, str) or not item for item in value): + if not isinstance(value, list) or any( + not isinstance(item, str) or not item for item in value + ): raise AdapterConfigError( - "claude_invalid_configuration", f"{name} must be a list of non-empty strings" + "claude_invalid_configuration", + f"{name} must be a list of non-empty strings", ) return list(value) def _positive_number(value: Any, *, name: str) -> float: if isinstance(value, bool) or not isinstance(value, (int, float)): - raise AdapterConfigError("claude_invalid_configuration", f"{name} must be positive") + raise AdapterConfigError( + "claude_invalid_configuration", f"{name} must be positive" + ) number = float(value) if number <= 0 or not math.isfinite(number): - raise AdapterConfigError("claude_invalid_configuration", f"{name} must be positive") + raise AdapterConfigError( + "claude_invalid_configuration", f"{name} must be positive" + ) return number def runtime_id(payload: dict[str, Any]) -> str: value = common_utils.runtime_context(payload).get("runtime_id") if not isinstance(value, str) or not value: - raise AdapterInputError("claude_invalid_request", "Fabric runtime ID is required") + raise AdapterInputError( + "claude_invalid_request", "Fabric runtime ID is required" + ) return value @@ -185,14 +225,19 @@ def selected_model(payload: dict[str, Any]) -> str | None: "models.default.provider must be anthropic for the Claude adapter", ) if not isinstance(value, str) or not value: - raise AdapterConfigError("claude_invalid_configuration", "model must be a non-empty string") + raise AdapterConfigError( + "claude_invalid_configuration", "model must be a non-empty string" + ) return value.removeprefix("anthropic/") def _mcp_servers(payload: dict[str, Any]) -> dict[str, Any]: - native = _mapping( - common_utils.capability_plan(payload), name="capability_plan" - ).get("native") or {} + native = ( + _mapping(common_utils.capability_plan(payload), name="capability_plan").get( + "native" + ) + or {} + ) servers = _mapping(native, name="capability_plan.native").get("mcp_servers") or {} result: dict[str, Any] = {} for name, raw in sorted(_mapping(servers, name="native MCP servers").items()): @@ -200,11 +245,15 @@ def _mcp_servers(payload: dict[str, Any]) -> dict[str, Any]: transport = server.get("transport") url = server.get("url") if not isinstance(url, str) or not url: - raise AdapterConfigError("claude_invalid_configuration", "MCP server URL is required") + raise AdapterConfigError( + "claude_invalid_configuration", "MCP server URL is required" + ) if transport == "stdio": command = shlex.split(url) if not command: - raise AdapterConfigError("claude_invalid_configuration", "MCP command is required") + raise AdapterConfigError( + "claude_invalid_configuration", "MCP command is required" + ) result[name] = {"type": "stdio", "command": command[0], "args": command[1:]} elif transport in {"http", "streamable-http"}: result[name] = {"type": "http", "url": url} @@ -212,7 +261,8 @@ def _mcp_servers(payload: dict[str, Any]) -> dict[str, Any]: result[name] = {"type": "sse", "url": url} else: raise AdapterConfigError( - "claude_invalid_configuration", f"unsupported MCP transport: {transport}" + "claude_invalid_configuration", + f"unsupported MCP transport: {transport}", ) return result @@ -220,9 +270,12 @@ def _mcp_servers(payload: dict[str, Any]) -> dict[str, Any]: def _normalized_tools( payload: dict[str, Any], *, include_skills: bool ) -> list[str] | dict[str, Any] | None: - native = _mapping( - common_utils.capability_plan(payload), name="capability_plan" - ).get("native") or {} + native = ( + _mapping(common_utils.capability_plan(payload), name="capability_plan").get( + "native" + ) + or {} + ) if not _mapping(native, name="capability_plan.native").get("tools_configured"): return None tools = common_utils.fabric_config(payload).get("tools") @@ -242,9 +295,12 @@ def _normalized_tools( def _native_skill_paths(payload: dict[str, Any]) -> list[Path]: - native = _mapping( - common_utils.capability_plan(payload), name="capability_plan" - ).get("native") or {} + native = ( + _mapping(common_utils.capability_plan(payload), name="capability_plan").get( + "native" + ) + or {} + ) values = _mapping(native, name="capability_plan.native").get("skill_paths") or [] if not isinstance(values, list) or any( not isinstance(value, (str, Path)) for value in values @@ -271,13 +327,16 @@ def _stage_skill_plugin(payload: dict[str, Any]) -> list[dict[str, str]]: name = skill_path.name if name in names: raise AdapterConfigError( - "claude_invalid_configuration", f"Fabric skill names must be unique: {name}" + "claude_invalid_configuration", + f"Fabric skill names must be unique: {name}", ) names.add(name) skills.append((name, skill_path)) plugin_key = sha256(runtime_id(payload).encode()).hexdigest() - plugin_root = _artifact_root(payload) / ".fabric" / "claude" / "plugins" / plugin_key + plugin_root = ( + _artifact_root(payload) / ".fabric" / "claude" / "plugins" / plugin_key + ) if plugin_root.exists(): shutil.rmtree(plugin_root) (plugin_root / ".claude-plugin").mkdir(parents=True) @@ -300,45 +359,168 @@ def _stage_skill_plugin(payload: dict[str, Any]) -> list[dict[str, str]]: return [{"type": "local", "path": str(plugin_root)}] +def _stage_relay_plugin(plugin_path: Path, executable: Path) -> None: + if plugin_path.exists(): + shutil.rmtree(plugin_path) + (plugin_path / ".claude-plugin").mkdir(parents=True) + (plugin_path / "hooks").mkdir() + (plugin_path / ".claude-plugin" / "plugin.json").write_text( + json.dumps( + { + "name": "nemo-fabric-relay", + "description": "NeMo Relay hooks managed by NeMo Fabric", + "version": "1.0.0", + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + (plugin_path / "hooks" / "hooks.json").write_text( + json.dumps( + relay_hooks.render_relay_hooks("claude", executable), + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + + +def prepare_claude_relay(payload: dict[str, Any]) -> ClaudeRelaySettings | None: + """Generate invocation-scoped Relay and Claude hook configuration.""" + + if not common_utils.relay_enabled(payload): + return None + settings = _settings(payload) + command = settings.get("nemo_relay_command") or "nemo-relay" + if not isinstance(command, (str, Path)): + raise AdapterConfigError( + "claude_invalid_configuration", + "nemo_relay_command must be a path", + ) + try: + executable = relay_gateway.resolve_relay_command( + Path(common_utils.config_root(payload)).resolve(), + command, + ) + except FileNotFoundError as error: + raise AdapterRelayError( + "claude_relay_unavailable", + "NeMo Relay CLI executable was not found", + ) from error + + try: + observability_version = relay_gateway.relay_cli_observability_version( + executable + ) + plugin_config = common_utils.load_relay_plugin_config(payload) + config_path, plugin_config_path = common_utils.write_relay_configs( + relay_config={"agents": {"claude": {"command": "claude"}}}, + plugin_config=plugin_config, + observability_version=observability_version, + ) + except ( + OSError, + RuntimeError, + ValueError, + json.JSONDecodeError, + ) as error: + raise AdapterRelayError( + "claude_relay_configuration_failed", + "NeMo Relay runtime configuration is unavailable", + ) from error + if config_path is None or plugin_config_path is None: + raise AdapterRelayError( + "claude_relay_configuration_failed", + "NeMo Relay runtime configuration is unavailable", + ) + + port = relay_gateway.find_available_tcp_port() + gateway_bind = f"127.0.0.1:{port}" + gateway = relay_gateway.RelayGatewayLaunch( + executable=executable, + config_path=config_path, + bind=gateway_bind, + url=f"http://{gateway_bind}", + log_path=config_path.parent / "gateway.log", + ) + plugin_path = config_path.parent / "claude-plugin" + try: + _stage_relay_plugin(plugin_path, executable) + except OSError as error: + shutil.rmtree(plugin_path, ignore_errors=True) + raise AdapterRelayError( + "claude_relay_configuration_failed", + "Claude Relay hook configuration could not be generated", + ) from error + return ClaudeRelaySettings( + gateway=gateway, + plugin_config=plugin_config, + plugin_path=plugin_path, + ) + + def discard_stderr(_: str) -> None: """Consume Claude Code stderr without exposing it through Fabric artifacts.""" -def build_options(payload: dict[str, Any], *, resume: str | None) -> ClaudeAgentOptions: +def build_options( + payload: dict[str, Any], + *, + resume: str | None, + relay: ClaudeRelaySettings | None = None, +) -> ClaudeAgentOptions: settings = _settings(payload) _validate_settings_boundary(settings) permission_mode = settings.get("permission_mode") if permission_mode is not None and permission_mode not in PERMISSION_MODES: - raise AdapterConfigError("claude_invalid_configuration", "permission_mode is invalid") + raise AdapterConfigError( + "claude_invalid_configuration", "permission_mode is invalid" + ) max_turns = settings.get("max_turns") if max_turns is not None and ( isinstance(max_turns, bool) or not isinstance(max_turns, int) or max_turns <= 0 ): - raise AdapterConfigError("claude_invalid_configuration", "max_turns must be positive") + raise AdapterConfigError( + "claude_invalid_configuration", "max_turns must be positive" + ) max_budget = settings.get("max_budget_usd") if max_budget is not None: max_budget = _positive_number(max_budget, name="max_budget_usd") sources = settings.get("setting_sources", []) sources = _string_list(sources, name="setting_sources") if any(source not in SETTING_SOURCES for source in sources): - raise AdapterConfigError("claude_invalid_configuration", "setting_sources is invalid") + raise AdapterConfigError( + "claude_invalid_configuration", "setting_sources is invalid" + ) cli_path = settings.get("cli_path") if cli_path is not None and not isinstance(cli_path, (str, Path)): - raise AdapterConfigError("claude_invalid_configuration", "cli_path must be a path") + raise AdapterConfigError( + "claude_invalid_configuration", "cli_path must be a path" + ) system_prompt = settings.get("system_prompt") if system_prompt is not None and not isinstance(system_prompt, (str, dict)): - raise AdapterConfigError("claude_invalid_configuration", "system_prompt is invalid") + raise AdapterConfigError( + "claude_invalid_configuration", "system_prompt is invalid" + ) plugins = _stage_skill_plugin(payload) + has_skill_plugin = bool(plugins) + if relay is not None: + plugins.append({"type": "local", "path": str(relay.plugin_path)}) return ClaudeAgentOptions( resume=resume, cwd=resolve_cwd(payload), model=selected_model(payload), system_prompt=system_prompt, - tools=_normalized_tools(payload, include_skills=bool(plugins)), + tools=_normalized_tools(payload, include_skills=has_skill_plugin), allowed_tools=_string_list(settings.get("allowed_tools"), name="allowed_tools"), - disallowed_tools=_string_list(settings.get("disallowed_tools"), name="disallowed_tools"), + disallowed_tools=_string_list( + settings.get("disallowed_tools"), name="disallowed_tools" + ), permission_mode=permission_mode, max_turns=max_turns, max_budget_usd=max_budget, @@ -346,9 +528,12 @@ def build_options(payload: dict[str, Any], *, resume: str | None) -> ClaudeAgent cli_path=_resolve_path(payload, cli_path) if cli_path is not None else None, mcp_servers=_mcp_servers(payload), strict_mcp_config=True, - skills="all" if plugins else None, + skills="all" if has_skill_plugin else None, plugins=plugins, - env=child_environment(payload), + env=child_environment( + payload, + relay_gateway_url=relay.gateway.url if relay is not None else None, + ), stderr=discard_stderr, ) @@ -368,10 +553,14 @@ def _artifact_root(payload: dict[str, Any]) -> Path: def runtime_state_path(payload: dict[str, Any], fabric_runtime_id: str) -> Path: digest = sha256(fabric_runtime_id.encode("utf-8")).hexdigest() - return _artifact_root(payload) / ".fabric" / "claude" / "runtimes" / f"{digest}.json" + return ( + _artifact_root(payload) / ".fabric" / "claude" / "runtimes" / f"{digest}.json" + ) -def load_claude_session_id(payload: dict[str, Any], fabric_runtime_id: str) -> str | None: +def load_claude_session_id( + payload: dict[str, Any], fabric_runtime_id: str +) -> str | None: path = runtime_state_path(payload, fabric_runtime_id) if not path.exists(): return None @@ -395,10 +584,14 @@ def save_claude_session_id( payload: dict[str, Any], fabric_runtime_id: str, claude_session_id: str ) -> None: if not claude_session_id: - raise AdapterStateError("claude_invalid_runtime_state", "Claude session ID is missing") + raise AdapterStateError( + "claude_invalid_runtime_state", "Claude session ID is missing" + ) path = runtime_state_path(payload, fabric_runtime_id) path.parent.mkdir(parents=True, exist_ok=True) - invocation_id = common_utils.runtime_context(payload).get("invocation_id") or "invocation" + invocation_id = ( + common_utils.runtime_context(payload).get("invocation_id") or "invocation" + ) temporary = path.with_suffix(f".{invocation_id}.tmp") temporary.write_text( json.dumps( @@ -421,7 +614,9 @@ def _json_safe(value: Any) -> Any: return str(value) if value is None or isinstance(value, (str, int, float, bool)): return value - raise AdapterConfigError("claude_invalid_configuration", "Claude message is not JSON-safe") + raise AdapterConfigError( + "claude_invalid_configuration", "Claude message is not JSON-safe" + ) def normalize_message(message: Message) -> dict[str, Any]: @@ -479,7 +674,7 @@ def _failure(code: str, message: str, **metadata: Any) -> dict[str, Any]: def adapter_failure(error: ClaudeAdapterError) -> dict[str, Any]: - return _failure(error.code, error.message) + return _failure(error.code, error.message, **error.metadata) def sdk_failure(error: BaseException) -> dict[str, Any]: @@ -502,52 +697,145 @@ def sdk_failure(error: BaseException) -> dict[str, Any]: return _failure("claude_failed", "Claude invocation failed") -def child_environment(payload: dict[str, Any]) -> dict[str, str]: +def child_environment( + payload: dict[str, Any], + *, + relay_gateway_url: str | None = None, +) -> dict[str, str]: values = {name: "" for name in os.environ} values.update( - { - name: value - for name in INHERITED_ENV_NAMES - if (value := os.environ.get(name)) - } + {name: value for name in INHERITED_ENV_NAMES if (value := os.environ.get(name))} ) model = _selected_model_config(payload) api_key_env = model.get("api_key_env") if isinstance(api_key_env, str) and api_key_env in os.environ: values[api_key_env] = os.environ[api_key_env] configured = _mapping(_settings(payload).get("env"), name="harness.settings.env") - if any(not isinstance(key, str) or not isinstance(value, str) for key, value in configured.items()): + if any( + not isinstance(key, str) or not isinstance(value, str) + for key, value in configured.items() + ): raise AdapterConfigError( "claude_invalid_configuration", "harness.settings.env must contain strings" ) values.update(configured) + if relay_gateway_url is not None: + values["NEMO_RELAY_GATEWAY_URL"] = relay_gateway_url + values["ANTHROPIC_BASE_URL"] = relay_gateway_url return values +def _relay_output( + output: dict[str, Any], + relay: ClaudeRelaySettings, +) -> dict[str, Any]: + output["relay_runtime"] = { + "enabled": True, + "emitter": "claude-agent-sdk/nemo-relay", + "config_path": os.environ.get("FABRIC_RELAY_CONFIG_PATH"), + "gateway_config_path": str(relay.gateway.config_path), + "gateway_url": relay.gateway.url, + "gateway_log_path": str(relay.gateway.log_path), + } + output["relay_artifacts"] = common_utils.collect_relay_artifacts( + relay.plugin_config + ) + return output + + async def run_claude(payload: dict[str, Any]) -> dict[str, Any]: fabric_runtime_id = runtime_id(payload) prior_session_id = load_claude_session_id(payload, fabric_runtime_id) - options = build_options(payload, resume=prior_session_id) + relay = prepare_claude_relay(payload) + gateway_process = None + cleanup_error: AdapterRelayError | None = None messages: list[Message] = [] result: ResultMessage | None = None try: - async with asyncio.timeout(timeout_seconds(payload)): - async for message in query(prompt=request_prompt(payload), options=options): - if isinstance(message, ResultMessage): - result = message - else: - messages.append(message) - except (TimeoutError, ClaudeSDKError) as error: - return sdk_failure(error) - - if result is None: - return _failure("claude_missing_result", "Claude returned no terminal result") - output = normalize_result(payload, messages, result) - if output["failed"]: - return output - if prior_session_id is not None and result.session_id != prior_session_id: - return _failure("claude_session_mismatch", "Claude session identity changed during resume") - save_claude_session_id(payload, fabric_runtime_id, result.session_id) + if relay is not None: + try: + gateway_process = relay_gateway.start_relay_gateway( + launch=relay.gateway, + cwd=resolve_cwd(payload), + ) + except relay_gateway.RelayGatewayError as error: + raise AdapterRelayError( + "claude_relay_start_failed", + "NeMo Relay gateway failed to start", + metadata={"gateway_log_path": str(relay.gateway.log_path)}, + ) from error + options = build_options(payload, resume=prior_session_id, relay=relay) + try: + async with asyncio.timeout(timeout_seconds(payload)): + async for message in query( + prompt=request_prompt(payload), options=options + ): + if isinstance(message, ResultMessage): + result = message + else: + messages.append(message) + except (TimeoutError, ClaudeSDKError) as error: + output = sdk_failure(error) + else: + if result is None: + output = _failure( + "claude_missing_result", "Claude returned no terminal result" + ) + else: + output = normalize_result(payload, messages, result) + if not output["failed"]: + if ( + prior_session_id is not None + and result.session_id != prior_session_id + ): + output = _failure( + "claude_session_mismatch", + "Claude session identity changed during resume", + ) + else: + save_claude_session_id( + payload, fabric_runtime_id, result.session_id + ) + finally: + if gateway_process is not None: + try: + relay_gateway.stop_relay_gateway(gateway_process) + except relay_gateway.RelayGatewayError: + cleanup_error = AdapterRelayError( + "claude_relay_stop_failed", + "NeMo Relay gateway failed to stop", + metadata={ + "gateway_log_path": str(relay.gateway.log_path) + if relay is not None + else "" + }, + ) + if relay is not None and relay.plugin_path.exists(): + try: + shutil.rmtree(relay.plugin_path) + except OSError: + if cleanup_error is None: + cleanup_error = AdapterRelayError( + "claude_relay_cleanup_failed", + "Claude Relay hook configuration could not be removed", + ) + + if relay is not None: + output = _relay_output(output, relay) + if cleanup_error is not None: + cleanup: dict[str, Any] = { + "code": cleanup_error.code, + "message": cleanup_error.message, + "retryable": False, + } + if cleanup_error.metadata: + cleanup["metadata"] = cleanup_error.metadata + output["relay_runtime"]["cleanup_error"] = cleanup + if not output["failed"]: + output["completed"] = False + output["failed"] = True + output["error"] = cleanup + return output @@ -567,7 +855,9 @@ def run(payload: dict[str, Any]) -> dict[str, Any]: def main() -> None: try: payload = common_utils.load_payload() - except Exception: # Malformed invocation input must still satisfy the process contract. + except ( + Exception + ): # Malformed invocation input must still satisfy the process contract. output = _failure( "claude_adapter_internal_error", "Claude adapter failed unexpectedly" ) diff --git a/adapters/claude/uv.lock b/adapters/claude/uv.lock index 03114280b..5f9e4b547 100644 --- a/adapters/claude/uv.lock +++ b/adapters/claude/uv.lock @@ -355,12 +355,14 @@ source = { editable = "." } dependencies = [ { name = "claude-agent-sdk" }, { name = "nemo-fabric-adapters-common" }, + { name = "tomli-w" }, ] [package.metadata] requires-dist = [ { name = "claude-agent-sdk", specifier = "==0.2.114" }, { name = "nemo-fabric-adapters-common", editable = "../common" }, + { name = "tomli-w", specifier = "~=1.2" }, ] [[package]] @@ -734,6 +736,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/bb/2799cc2ede3ed41131f8975621e7213dfc7ef4acbbaadfa440f32500c370/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6", size = 73632, upload-time = "2026-06-12T09:23:10.017Z" }, ] +[[package]] +name = "tomli-w" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/75/241269d1da26b624c0d5e110e8149093c759b7a286138f4efd61a60e75fe/tomli_w-1.2.0.tar.gz", hash = "sha256:2dd14fac5a47c27be9cd4c976af5a12d87fb1f0b4512f81d69cce3b35ae25021", size = 7184, upload-time = "2025-01-15T12:07:24.262Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/18/c86eb8e0202e32dd3df50d43d7ff9854f8e0603945ff398974c1d91ac1ef/tomli_w-1.2.0-py3-none-any.whl", hash = "sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90", size = 6675, upload-time = "2025-01-15T12:07:22.074Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0" diff --git a/adapters/codex-cli/src/nemo_fabric_adapters/codex_cli/adapter.py b/adapters/codex-cli/src/nemo_fabric_adapters/codex_cli/adapter.py index 270a0655d..500fca23c 100755 --- a/adapters/codex-cli/src/nemo_fabric_adapters/codex_cli/adapter.py +++ b/adapters/codex-cli/src/nemo_fabric_adapters/codex_cli/adapter.py @@ -10,55 +10,34 @@ import json import math import os -import socket import subprocess -import time import tomllib -import urllib.error -import urllib.request from collections.abc import Mapping from pathlib import Path from typing import Any from typing import NamedTuple +import nemo_fabric_adapters.common.relay_gateway as relay_gateway +import nemo_fabric_adapters.common.relay_hooks as relay_hooks import nemo_fabric_adapters.common.utils as common_utils import tomli_w SANDBOXES = {"read-only", "workspace-write", "danger-full-access"} DEFAULT_TIMEOUT_SECONDS = 1800 -RELAY_HEALTH_TIMEOUT_SECONDS = 30 -RELAY_HOOK_EVENTS = ( - "Notification", - "PermissionRequest", - "PostCompact", - "PostToolUse", - "PostToolUseFailure", - "PreCompact", - "PreToolUse", - "SessionEnd", - "SessionStart", - "Stop", - "SubagentStart", - "SubagentStop", - "UserPromptSubmit", -) -RELAY_HOOK_MATCHER_EVENTS = { - "PermissionRequest", - "PostToolUse", - "PostToolUseFailure", - "PreToolUse", -} + + +class CodexRelaySettings(NamedTuple): + """Invocation-scoped Relay state consumed by the Codex adapter.""" + + gateway: relay_gateway.RelayGatewayLaunch + plugin_config: dict[str, Any] + class CodexSettings(NamedTuple): telemetry_provider: str - relay_enabled: bool codex_profile_name: str | None codex_profile_path: Path | None - relay_gateway_host: str | None - relay_gateway_url: str | None - relay_gateway_port: int | None - relay_config_path: Path | None - relay_plugin_config: dict[str, Any] | None + relay: CodexRelaySettings | None def state_dir(payload: dict[str, Any]) -> Path: @@ -88,7 +67,11 @@ def load_thread_id(payload: dict[str, Any], runtime_id: str) -> str | None: value = json.loads(path.read_text(encoding="utf-8")) except json.JSONDecodeError as error: raise RuntimeError(f"invalid Codex runtime state in {path}") from error - if not isinstance(value, dict) or value.get("runtime_id") != runtime_id or not value.get("thread_id"): + if ( + not isinstance(value, dict) + or value.get("runtime_id") != runtime_id + or not value.get("thread_id") + ): raise RuntimeError(f"invalid Codex runtime state in {path}") return str(value["thread_id"]) @@ -96,7 +79,9 @@ def load_thread_id(payload: dict[str, Any], runtime_id: str) -> str | None: def save_thread_id(payload: dict[str, Any], runtime_id: str, thread_id: str) -> None: path = runtime_state_path(payload, runtime_id) path.parent.mkdir(parents=True, exist_ok=True) - invocation_id = common_utils.runtime_context(payload).get("invocation_id") or "pending" + invocation_id = ( + common_utils.runtime_context(payload).get("invocation_id") or "pending" + ) temporary = path.with_suffix(f".{invocation_id}.tmp") temporary.write_text( json.dumps({"runtime_id": runtime_id, "thread_id": thread_id}, indent=2), @@ -128,7 +113,9 @@ def build_command( command = resolve_command(payload, settings.get("codex_command") or "codex") sandbox = str(settings.get("sandbox") or "read-only") if sandbox not in SANDBOXES: - raise ValueError(f"unsupported Codex sandbox {sandbox!r}; expected one of {sorted(SANDBOXES)}") + raise ValueError( + f"unsupported Codex sandbox {sandbox!r}; expected one of {sorted(SANDBOXES)}" + ) args = [command, "exec", "--json"] @@ -137,8 +124,7 @@ def build_command( if codex_settings.codex_profile_name is not None: args.extend(("--profile", codex_settings.codex_profile_name)) - # relay_enabled will only ever be true when codex_profile_name is not None - if codex_settings.relay_enabled: + if codex_settings.relay is not None: # By default Codex will not enable hooks for profiles that are not trusted until the user explicitly # enables them. This is a problem for Fabric, because we want to be able to use hooks in a non-interactive # way.So we add the --dangerously-bypass-hook-trust flag to bypass this check. @@ -205,7 +191,9 @@ def native_codex_telemetry_config(payload: dict[str, Any]) -> dict[str, Any]: exporter = "otlp-http" protocol = "json" else: - raise ValueError(f"unsupported Codex native OpenTelemetry transport {transport!r}") + raise ValueError( + f"unsupported Codex native OpenTelemetry transport {transport!r}" + ) otel["trace_exporter"] = { exporter: { "endpoint": endpoint, @@ -229,7 +217,9 @@ def apply_config_overrides( for part in parts[:-1]: existing = target.setdefault(part, {}) if not isinstance(existing, dict): - raise ValueError(f"Codex config override {dotted_key!r} conflicts with {part!r}") + raise ValueError( + f"Codex config override {dotted_key!r} conflicts with {part!r}" + ) target = existing target[parts[-1]] = value @@ -271,47 +261,44 @@ def write_config_files(payload: dict[str, Any]) -> CodexSettings: codex_profile_name = None codex_profile_path = None - relay_gateway_host = None - relay_gateway_url = None - relay_gateway_port = None - relay_config_path = None - relay_plugin_config = None + relay = None if relay_enabled or bool(config) or bool(overrides): codex_profile_name, codex_profile_path = get_codex_profile_path(payload) if relay_enabled: - relay_gateway_port = find_available_tcp_port() - relay_gateway_host = f"127.0.0.1:{relay_gateway_port}" - relay_gateway_url = f"http://{relay_gateway_host}" + relay_gateway_port = relay_gateway.find_available_tcp_port() + relay_gateway_bind = f"127.0.0.1:{relay_gateway_port}" + relay_gateway_url = f"http://{relay_gateway_bind}" # nemo-relay infers the plugin config location from the relay config. relay_plugin_config = common_utils.load_relay_plugin_config(payload) + relay_executable = relay_gateway.resolve_relay_command( + Path(common_utils.config_root(payload)).resolve(), + settings.get("nemo_relay_command") or "nemo-relay", + ) relay_config_path, _ = common_utils.write_relay_configs( relay_config={"agents": {"codex": {"command": "codex"}}}, plugin_config=relay_plugin_config, + observability_version=relay_gateway.relay_cli_observability_version( + relay_executable + ), ) if relay_config_path is None: - raise RuntimeError("NeMo Relay configuration did not produce a gateway config") - - relay_command = resolve_command( - payload, - settings.get("nemo_relay_command") or "nemo-relay", + raise RuntimeError( + "NeMo Relay configuration did not produce a gateway config" + ) + + gateway = relay_gateway.RelayGatewayLaunch( + executable=relay_executable, + config_path=relay_config_path, + bind=relay_gateway_bind, + url=relay_gateway_url, + log_path=relay_config_path.parent / "gateway.log", + ) + relay = CodexRelaySettings( + gateway=gateway, + plugin_config=relay_plugin_config, ) - hook_command = f"{relay_command} hook-forward codex" - hooks = {} - for event in RELAY_HOOK_EVENTS: - hook_group: dict[str, Any] = { - "hooks": [ - { - "type": "command", - "command": hook_command, - "timeout": 30, - } - ] - } - if event in RELAY_HOOK_MATCHER_EVENTS: - hook_group["matcher"] = "*" - hooks[event] = [hook_group] merge_config( config, @@ -327,7 +314,9 @@ def write_config_files(payload: dict[str, Any]) -> CodexSettings: } }, "features": {"hooks": True}, - "hooks": hooks, + "hooks": relay_hooks.render_relay_hooks("codex", relay_executable)[ + "hooks" + ], }, ) @@ -340,21 +329,18 @@ def write_config_files(payload: dict[str, Any]) -> CodexSettings: return CodexSettings( telemetry_provider=telemetry_provider, - relay_enabled=relay_enabled, codex_profile_name=codex_profile_name, codex_profile_path=codex_profile_path, - relay_gateway_host=relay_gateway_host, - relay_gateway_url=relay_gateway_url, - relay_gateway_port=relay_gateway_port, - relay_config_path=relay_config_path, - relay_plugin_config=relay_plugin_config, + relay=relay, ) def get_codex_profile_path(payload: dict[str, Any]) -> tuple[str, Path]: runtime_id = common_utils.runtime_context(payload).get("runtime_id") if not runtime_id: - raise RuntimeError("runtime_context.runtime_id is required for generated Codex profiles") + raise RuntimeError( + "runtime_context.runtime_id is required for generated Codex profiles" + ) name = f"fabric-{runtime_id}" return name, codex_home() / f"{name}.config.toml" @@ -381,7 +367,9 @@ def toml_value(value: Any) -> str: try: document = tomli_w.dumps({"value": value}) except TypeError as error: - raise ValueError("Codex config override values must be a TOML scalar or array") from error + raise ValueError( + "Codex config override values must be a TOML scalar or array" + ) from error prefix = "value = " if not document.startswith(prefix): raise ValueError("Codex config override values must be a TOML scalar or array") @@ -415,7 +403,9 @@ def parse_events(contents: str) -> dict[str, Any]: usage = event.get("usage") elif event_type in {"turn.failed", "error"}: failure = event.get("error") or event.get("message") or event - error = failure.get("message") if isinstance(failure, dict) else str(failure) + error = ( + failure.get("message") if isinstance(failure, dict) else str(failure) + ) return { "events": events, "thread_id": str(thread_id) if thread_id else None, @@ -462,80 +452,16 @@ def build_env( return env -def find_available_tcp_port() -> int: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: - listener.bind(("127.0.0.1", 0)) - return int(listener.getsockname()[1]) - - -def wait_for_relay_gateway( - process: subprocess.Popen[Any], - health_url: str, - *, - timeout: float = RELAY_HEALTH_TIMEOUT_SECONDS, -) -> None: - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - returncode = process.poll() - if returncode is not None: - raise RuntimeError(f"NeMo Relay gateway exited with status {returncode} before becoming ready") - try: - with urllib.request.urlopen(health_url, timeout=1) as response: - if 200 <= response.status < 300: - return - except (OSError, urllib.error.URLError): - pass - time.sleep(0.1) - raise RuntimeError(f"NeMo Relay gateway did not become ready at {health_url}") - - -def stop_relay_gateway(process: subprocess.Popen[Any]) -> None: - if process.poll() is not None: - return - process.terminate() - try: - process.wait(timeout=5) - except subprocess.TimeoutExpired: - process.kill() - process.wait(timeout=5) - - -def start_relay_gateway( - payload: dict[str, Any], - cwd: Path, - codex_settings: CodexSettings, -) -> subprocess.Popen: - settings = common_utils.settings_payload(payload) - relay_command = resolve_command( - payload, - settings.get("nemo_relay_command") or "nemo-relay", - ) - - process = subprocess.Popen( - [ - relay_command, - "--config", - str(codex_settings.relay_config_path), - "--bind", - codex_settings.relay_gateway_host, - ], - cwd=cwd, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - start_new_session=True, - ) - try: - wait_for_relay_gateway(process, f"{codex_settings.relay_gateway_url}/healthz") - except Exception as e: - stop_relay_gateway(process) - raise RuntimeError("NeMo Relay gateway failed to start") from e - - return process - - def process_timeout(payload: dict[str, Any]) -> float: - value = common_utils.settings_payload(payload).get("timeout_seconds", DEFAULT_TIMEOUT_SECONDS) - if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value <= 0: + value = common_utils.settings_payload(payload).get( + "timeout_seconds", DEFAULT_TIMEOUT_SECONDS + ) + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + or value <= 0 + ): raise ValueError("timeout_seconds must be a positive finite number") return float(value) @@ -554,14 +480,10 @@ def run_codex(payload: dict[str, Any]) -> dict[str, Any]: relay_gateway_process = None try: - if codex_settings.relay_enabled: - if codex_settings.relay_config_path is None or codex_settings.relay_gateway_port is None: - raise RuntimeError("NeMo Relay configuration files were not generated") - - relay_gateway_process = start_relay_gateway( - payload, - cwd, - codex_settings, + if codex_settings.relay is not None: + relay_gateway_process = relay_gateway.start_relay_gateway( + launch=codex_settings.relay.gateway, + cwd=cwd, ) command = build_command( @@ -578,7 +500,11 @@ def run_codex(payload: dict[str, Any]) -> dict[str, Any]: cwd=cwd, env=build_env( payload, - relay_gateway_url=codex_settings.relay_gateway_url, + relay_gateway_url=( + codex_settings.relay.gateway.url + if codex_settings.relay is not None + else None + ), ), input=request_to_prompt(payload), text=True, @@ -602,19 +528,25 @@ def run_codex(payload: dict[str, Any]) -> dict[str, Any]: codex_settings.codex_profile_path.unlink(missing_ok=True) if relay_gateway_process is not None: - stop_relay_gateway(relay_gateway_process) + relay_gateway.stop_relay_gateway(relay_gateway_process) parsed = parse_events(completed.stdout) thread_id = parsed["thread_id"] or prior_thread_id error = launch_error or parsed["error"] if completed.returncode != 0: - error = error or completed.stderr.strip() or "Codex CLI exited with a non-zero status" + error = ( + error + or completed.stderr.strip() + or "Codex CLI exited with a non-zero status" + ) if parsed["response"] is None: error = error or "Codex invocation did not return a final agent message" if not thread_id: error = error or "Codex runtime invocation did not return a thread identity" if prior_thread_id and thread_id != prior_thread_id: - error = error or (f"Codex resumed thread {thread_id}, expected persisted thread {prior_thread_id}") + error = error or ( + f"Codex resumed thread {thread_id}, expected persisted thread {prior_thread_id}" + ) if thread_id and not error: save_thread_id(payload, runtime_id, thread_id) @@ -634,12 +566,16 @@ def run_codex(payload: dict[str, Any]) -> dict[str, Any]: "state_dir": str(state_dir(payload)), } - if codex_settings.relay_plugin_config is not None: - relay_artifacts = common_utils.collect_relay_artifacts(codex_settings.relay_plugin_config) + if codex_settings.relay is not None: + relay_artifacts = common_utils.collect_relay_artifacts( + codex_settings.relay.plugin_config + ) output["relay_runtime"] = { "enabled": True, "config_path": os.environ.get("FABRIC_RELAY_CONFIG_PATH"), "emitter": "nemo-relay", + "gateway_config_path": str(codex_settings.relay.gateway.config_path), + "gateway_log_path": str(codex_settings.relay.gateway.log_path), } output["relay_artifacts"] = relay_artifacts @@ -650,7 +586,8 @@ def redact_command(command: list[str]) -> list[str]: redacted = list(command) for index, value in enumerate(redacted[:-1]): if value == "--config" and any( - marker in redacted[index + 1].lower() for marker in ("key", "token", "secret", "password") + marker in redacted[index + 1].lower() + for marker in ("key", "token", "secret", "password") ): redacted[index + 1] = "" return redacted diff --git a/adapters/common/src/nemo_fabric_adapters/common/relay_gateway.py b/adapters/common/src/nemo_fabric_adapters/common/relay_gateway.py new file mode 100644 index 000000000..b4360031d --- /dev/null +++ b/adapters/common/src/nemo_fabric_adapters/common/relay_gateway.py @@ -0,0 +1,177 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Supervise the NeMo Relay CLI gateway used by coding-agent adapters.""" + +from __future__ import annotations + +import re +import shutil +import socket +import subprocess +import time +import urllib.error +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +RELAY_HEALTH_TIMEOUT_SECONDS = 10.0 +RELAY_STOP_TIMEOUT_SECONDS = 5.0 +RELAY_VERSION_TIMEOUT_SECONDS = 5.0 + + +class RelayGatewayError(RuntimeError): + """NeMo Relay gateway lifecycle failure.""" + + +@dataclass(frozen=True) +class RelayGatewayLaunch: + """Complete invocation-scoped inputs for launching a Relay gateway.""" + + executable: Path + config_path: Path + bind: str + url: str + log_path: Path + + +def resolve_relay_command(config_root: Path, value: str | Path) -> Path: + """Resolve the configured Relay CLI to one absolute executable path.""" + + command = Path(value) + if len(command.parts) == 1: + resolved = shutil.which(str(command)) + else: + candidate = command if command.is_absolute() else config_root / command + resolved = shutil.which(str(candidate.resolve())) + if resolved is None: + raise FileNotFoundError("NeMo Relay CLI executable was not found") + return Path(resolved).resolve() + + +def find_available_tcp_port(host: str = "127.0.0.1") -> int: + """Return an available loopback TCP port for an imminent gateway launch.""" + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: + listener.bind((host, 0)) + return int(listener.getsockname()[1]) + + +def relay_cli_observability_version(executable: Path) -> int: + """Return the observability config version accepted by a Relay CLI.""" + + try: + completed = subprocess.run( + [str(executable), "--version"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=RELAY_VERSION_TIMEOUT_SECONDS, + check=False, + ) + except (OSError, subprocess.TimeoutExpired) as error: + raise RelayGatewayError( + "NeMo Relay CLI version could not be determined" + ) from error + match = re.search(r"\b(\d+)\.(\d+)\.(\d+)", completed.stdout) + if completed.returncode != 0 or match is None: + raise RelayGatewayError("NeMo Relay CLI version could not be determined") + major, minor, _ = (int(value) for value in match.groups()) + return 2 if (major, minor) >= (0, 6) else 1 + + +def wait_for_relay_gateway( + process: subprocess.Popen[Any], + health_url: str, + *, + timeout: float = RELAY_HEALTH_TIMEOUT_SECONDS, +) -> None: + """Wait until the Relay health endpoint succeeds or startup fails.""" + + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + returncode = process.poll() + if returncode is not None: + raise RelayGatewayError( + f"NeMo Relay gateway exited with status {returncode} before becoming ready" + ) + try: + with urllib.request.urlopen(health_url, timeout=1) as response: + if 200 <= response.status < 300: + return + except (OSError, urllib.error.URLError): + pass + time.sleep(0.1) + raise RelayGatewayError(f"NeMo Relay gateway did not become ready at {health_url}") + + +def stop_relay_gateway(process: subprocess.Popen[Any]) -> None: + """Stop a Relay gateway idempotently, escalating when it does not exit.""" + + if process.poll() is not None: + return + try: + process.terminate() + except ProcessLookupError: + return + try: + process.wait(timeout=RELAY_STOP_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: + process.kill() + try: + process.wait(timeout=RELAY_STOP_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired as error: + raise RelayGatewayError( + "NeMo Relay gateway did not stop after kill" + ) from error + + +def start_relay_gateway( + *, + launch: RelayGatewayLaunch, + cwd: Path, +) -> subprocess.Popen[Any]: + """Launch and health-check one invocation-scoped Relay gateway.""" + + if not launch.config_path.is_file(): + raise RelayGatewayError("NeMo Relay gateway configuration was not generated") + launch.log_path.parent.mkdir(parents=True, exist_ok=True) + try: + with launch.log_path.open("wb") as log_stream: + process = subprocess.Popen( + [ + str(launch.executable), + "--config", + str(launch.config_path), + "--bind", + launch.bind, + ], + cwd=cwd, + stdout=log_stream, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + except OSError as error: + raise RelayGatewayError( + f"NeMo Relay gateway could not start; see {launch.log_path}" + ) from error + + try: + wait_for_relay_gateway(process, f"{launch.url.rstrip('/')}/healthz") + except Exception as error: + try: + stop_relay_gateway(process) + except Exception as stop_error: + raise RelayGatewayError( + "NeMo Relay gateway failed to become ready and could not be stopped; " + f"see {launch.log_path}" + ) from ExceptionGroup( + "NeMo Relay gateway startup and cleanup failed", + [error, stop_error], + ) + raise RelayGatewayError( + f"NeMo Relay gateway failed to become ready; see {launch.log_path}" + ) from error + return process diff --git a/adapters/common/src/nemo_fabric_adapters/common/relay_hooks.py b/adapters/common/src/nemo_fabric_adapters/common/relay_hooks.py new file mode 100644 index 000000000..8e8faad4d --- /dev/null +++ b/adapters/common/src/nemo_fabric_adapters/common/relay_hooks.py @@ -0,0 +1,68 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Render NeMo Relay hook documents for supported coding agents.""" + +from __future__ import annotations + +import shlex +from pathlib import Path +from typing import Any +from typing import Literal + + +RelayHookAgent = Literal["claude", "codex"] + +# NeMo Relay currently uses this union for both Claude Code and Codex. Keep the +# snapshot centralized until Relay exposes its hook renderer as a public API. +RELAY_HOOK_EVENTS = ( + "SessionStart", + "UserPromptSubmit", + "UserPromptExpansion", + "PreToolUse", + "PostToolUse", + "PostToolUseFailure", + "PermissionRequest", + "SubagentStart", + "SubagentStop", + "Notification", + "Stop", + "PreCompact", + "PostCompact", + "SessionEnd", +) +RELAY_TOOL_HOOK_EVENTS = frozenset( + { + "PreToolUse", + "PostToolUse", + "PostToolUseFailure", + "PermissionRequest", + } +) + + +def render_relay_hooks( + agent: RelayHookAgent, + executable: Path, +) -> dict[str, Any]: + """Return the native hook document for one Relay-supported coding agent.""" + + if agent not in ("claude", "codex"): + raise ValueError(f"unsupported NeMo Relay hook agent {agent!r}") + + command = f"{shlex.quote(str(executable))} hook-forward {agent}" + hooks: dict[str, list[dict[str, Any]]] = {} + for event in RELAY_HOOK_EVENTS: + group: dict[str, Any] = { + "hooks": [ + { + "type": "command", + "command": command, + "timeout": 30, + } + ] + } + if event in RELAY_TOOL_HOOK_EVENTS: + group["matcher"] = "*" + hooks[event] = [group] + return {"hooks": hooks} diff --git a/adapters/common/src/nemo_fabric_adapters/common/utils.py b/adapters/common/src/nemo_fabric_adapters/common/utils.py index b0f8e944b..50e667ca1 100644 --- a/adapters/common/src/nemo_fabric_adapters/common/utils.py +++ b/adapters/common/src/nemo_fabric_adapters/common/utils.py @@ -5,6 +5,7 @@ from __future__ import annotations +import copy import json import os import sys @@ -421,10 +422,69 @@ def collect_relay_artifacts(plugin_config: dict[str, Any]) -> list[dict[str, str return artifacts +def relay_cli_plugin_config( + plugin_config: dict[str, Any], *, observability_version: int +) -> dict[str, Any]: + """Render normalized Relay intent for the current external CLI contract.""" + + rendered = copy.deepcopy(plugin_config) + if observability_version == 1: + return rendered + if observability_version != 2: + raise ValueError( + f"unsupported NeMo Relay observability config version {observability_version}" + ) + for component in rendered.get("components", []): + if not isinstance(component, dict) or component.get("kind") != "observability": + continue + config = component.get("config") + if not isinstance(config, dict) or int(config.get("version", 1)) != 1: + continue + + atof = config.get("atof") + if isinstance(atof, dict): + sinks = list(atof.get("sinks") or []) + if atof.get("enabled"): + file_sink = without_none( + { + "type": "file", + "output_directory": atof.get("output_directory"), + "filename": atof.get("filename"), + "mode": atof.get("mode", "append"), + } + ) + sinks.append(file_sink) + for endpoint in atof.get("endpoints") or []: + if not isinstance(endpoint, dict): + continue + sinks.append( + without_none( + { + "type": "stream", + "url": endpoint.get("url"), + "transport": endpoint.get("transport", "http_post"), + "headers": endpoint.get("headers", {}), + "header_env": endpoint.get("header_env", {}), + "timeout_millis": endpoint.get("timeout_millis", 3000), + "field_name_policy": endpoint.get( + "field_name_policy", "preserve" + ), + } + ) + ) + config["atof"] = { + "enabled": bool(atof.get("enabled", False)), + "sinks": sinks, + } + config["version"] = 2 + return rendered + + def write_relay_configs( *, relay_config: dict[str, Any] | None = None, plugin_config: dict[str, Any] | None = None, + observability_version: int = 1, ) -> tuple[Path | None, Path | None]: try: import tomli_w @@ -445,7 +505,15 @@ def write_relay_configs( if plugin_config is not None: plugin_config_path = config_dir / "plugins.toml" - plugin_config_path.write_text(tomli_w.dumps(plugin_config), encoding="utf-8") + plugin_config_path.write_text( + tomli_w.dumps( + relay_cli_plugin_config( + plugin_config, + observability_version=observability_version, + ) + ), + encoding="utf-8", + ) return relay_config_path, plugin_config_path except ImportError as e: diff --git a/tests/adapters/test_adapaters_common_utils.py b/tests/adapters/test_adapaters_common_utils.py index 7ac438d3a..7b4716f40 100644 --- a/tests/adapters/test_adapaters_common_utils.py +++ b/tests/adapters/test_adapaters_common_utils.py @@ -409,3 +409,73 @@ def test_write_relay_configs( assert path.parent.name == "relay-config" with path.open("rb") as stream: assert tomllib.load(stream) == config + + +def test_write_relay_configs_migrates_atof_to_current_cli_contract(tmp_path: Path): + os.environ["FABRIC_RELAY_CONFIG_PATH"] = str(tmp_path / "relay.json") + plugin_config = { + "version": 1, + "components": [ + { + "kind": "observability", + "enabled": True, + "config": { + "version": 1, + "atof": { + "enabled": True, + "output_directory": "/tmp/atof", + "filename": "events.jsonl", + "mode": "overwrite", + "endpoints": [ + { + "url": "https://example.test/events", + "transport": "http_post", + "headers": {"x-test": "value"}, + "header_env": {"authorization": "TOKEN"}, + "timeout_millis": 1000, + "field_name_policy": "replace_dots", + } + ], + }, + "atif": {"enabled": True, "output_directory": "/tmp/atif"}, + }, + } + ], + } + + _, plugin_path = common_utils.write_relay_configs( + plugin_config=plugin_config, + observability_version=2, + ) + + assert plugin_path is not None + with plugin_path.open("rb") as stream: + rendered = tomllib.load(stream) + observability = rendered["components"][0]["config"] + assert observability["version"] == 2 + assert observability["atof"] == { + "enabled": True, + "sinks": [ + { + "type": "file", + "output_directory": "/tmp/atof", + "filename": "events.jsonl", + "mode": "overwrite", + }, + { + "type": "stream", + "url": "https://example.test/events", + "transport": "http_post", + "headers": {"x-test": "value"}, + "header_env": {"authorization": "TOKEN"}, + "timeout_millis": 1000, + "field_name_policy": "replace_dots", + }, + ], + } + assert observability["atif"] == { + "enabled": True, + "output_directory": "/tmp/atif", + } + assert plugin_config["components"][0]["config"]["version"] == 1 + assert "sinks" not in plugin_config["components"][0]["config"]["atof"] diff --git a/tests/adapters/test_adapters_common_relay_gateway.py b/tests/adapters/test_adapters_common_relay_gateway.py new file mode 100644 index 000000000..251dd38bd --- /dev/null +++ b/tests/adapters/test_adapters_common_relay_gateway.py @@ -0,0 +1,238 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import subprocess +from unittest.mock import MagicMock, call + +import pytest + +import nemo_fabric_adapters.common.relay_gateway as relay_gateway + + +def test_resolve_relay_command_returns_absolute_executable(monkeypatch, tmp_path): + executable = tmp_path / "bin" / "nemo-relay" + executable.parent.mkdir() + executable.touch() + monkeypatch.setattr( + relay_gateway.shutil, "which", MagicMock(return_value=str(executable)) + ) + + resolved = relay_gateway.resolve_relay_command(tmp_path, "nemo-relay") + + assert resolved == executable.resolve() + + +def test_resolve_relay_command_treats_tilde_path_as_config_relative( + monkeypatch, tmp_path +): + executable = (tmp_path / "~" / "bin" / "nemo-relay").resolve() + mock_which = MagicMock(return_value=str(executable)) + monkeypatch.setattr(relay_gateway.shutil, "which", mock_which) + + resolved = relay_gateway.resolve_relay_command(tmp_path, "~/bin/nemo-relay") + + assert resolved == executable + mock_which.assert_called_once_with(str(executable)) + + +def test_resolve_relay_command_rejects_missing_executable(monkeypatch, tmp_path): + monkeypatch.setattr(relay_gateway.shutil, "which", MagicMock(return_value=None)) + + with pytest.raises( + FileNotFoundError, match="NeMo Relay CLI executable was not found" + ): + relay_gateway.resolve_relay_command(tmp_path, "nemo-relay") + + +@pytest.mark.parametrize( + ("output", "expected"), + [ + ("nemo-relay 0.5.0\n", 1), + ("nemo-relay 0.6.0-alpha.20260714\n", 2), + ("nemo-relay 1.0.0\n", 2), + ], +) +def test_relay_cli_observability_version_selects_compatible_contract( + monkeypatch, tmp_path, output, expected +): + monkeypatch.setattr( + relay_gateway.subprocess, + "run", + MagicMock(return_value=subprocess.CompletedProcess([], 0, stdout=output)), + ) + + assert ( + relay_gateway.relay_cli_observability_version(tmp_path / "nemo-relay") + == expected + ) + + +def test_relay_cli_observability_version_rejects_unparseable_output( + monkeypatch, tmp_path +): + monkeypatch.setattr( + relay_gateway.subprocess, + "run", + MagicMock(return_value=subprocess.CompletedProcess([], 0, stdout="unknown")), + ) + + with pytest.raises( + relay_gateway.RelayGatewayError, match="version could not be determined" + ): + relay_gateway.relay_cli_observability_version(tmp_path / "nemo-relay") + + +def test_start_relay_gateway_captures_logs_and_waits_for_health(monkeypatch, tmp_path): + executable = tmp_path / "nemo-relay" + config_path = tmp_path / "relay-config" / "config.toml" + config_path.parent.mkdir() + config_path.write_text('[agents.claude]\ncommand = "claude"\n', encoding="utf-8") + log_path = config_path.parent / "gateway.log" + process = MagicMock() + mock_popen = MagicMock(return_value=process) + mock_wait = MagicMock() + monkeypatch.setattr(relay_gateway.subprocess, "Popen", mock_popen) + monkeypatch.setattr(relay_gateway, "wait_for_relay_gateway", mock_wait) + launch = relay_gateway.RelayGatewayLaunch( + executable=executable, + config_path=config_path, + bind="127.0.0.1:43210", + url="http://127.0.0.1:43210", + log_path=log_path, + ) + + started = relay_gateway.start_relay_gateway( + launch=launch, + cwd=tmp_path, + ) + + assert started is process + assert mock_popen.call_args.args[0] == [ + str(executable), + "--config", + str(config_path), + "--bind", + "127.0.0.1:43210", + ] + assert mock_popen.call_args.kwargs["cwd"] == tmp_path + assert mock_popen.call_args.kwargs["stderr"] is subprocess.STDOUT + assert mock_popen.call_args.kwargs["start_new_session"] is True + assert mock_popen.call_args.kwargs["stdout"].name == str(log_path) + mock_wait.assert_called_once_with(process, "http://127.0.0.1:43210/healthz") + + +def test_start_relay_gateway_stops_failed_process_and_preserves_log( + monkeypatch, tmp_path +): + executable = tmp_path / "nemo-relay" + config_path = tmp_path / "config.toml" + config_path.write_text("", encoding="utf-8") + log_path = tmp_path / "gateway.log" + process = MagicMock() + monkeypatch.setattr( + relay_gateway.subprocess, "Popen", MagicMock(return_value=process) + ) + monkeypatch.setattr( + relay_gateway, + "wait_for_relay_gateway", + MagicMock(side_effect=relay_gateway.RelayGatewayError("not ready")), + ) + mock_stop = MagicMock() + monkeypatch.setattr(relay_gateway, "stop_relay_gateway", mock_stop) + launch = relay_gateway.RelayGatewayLaunch( + executable=executable, + config_path=config_path, + bind="127.0.0.1:43210", + url="http://127.0.0.1:43210", + log_path=log_path, + ) + + with pytest.raises(relay_gateway.RelayGatewayError, match=str(log_path)): + relay_gateway.start_relay_gateway( + launch=launch, + cwd=tmp_path, + ) + + mock_stop.assert_called_once_with(process) + assert log_path.exists() + + +def test_start_relay_gateway_reports_readiness_and_stop_failures( + monkeypatch, tmp_path +): + config_path = tmp_path / "config.toml" + config_path.write_text("", encoding="utf-8") + readiness_error = relay_gateway.RelayGatewayError("not ready") + stop_error = relay_gateway.RelayGatewayError("could not stop") + process = MagicMock() + monkeypatch.setattr( + relay_gateway.subprocess, "Popen", MagicMock(return_value=process) + ) + monkeypatch.setattr( + relay_gateway, + "wait_for_relay_gateway", + MagicMock(side_effect=readiness_error), + ) + mock_stop = MagicMock(side_effect=stop_error) + monkeypatch.setattr(relay_gateway, "stop_relay_gateway", mock_stop) + launch = relay_gateway.RelayGatewayLaunch( + executable=tmp_path / "nemo-relay", + config_path=config_path, + bind="127.0.0.1:43210", + url="http://127.0.0.1:43210", + log_path=tmp_path / "gateway.log", + ) + + with pytest.raises( + relay_gateway.RelayGatewayError, match="could not be stopped" + ) as captured: + relay_gateway.start_relay_gateway(launch=launch, cwd=tmp_path) + + mock_stop.assert_called_once_with(process) + assert isinstance(captured.value.__cause__, ExceptionGroup) + assert captured.value.__cause__.exceptions == (readiness_error, stop_error) + + +def test_wait_for_relay_gateway_reports_early_exit(): + process = MagicMock() + process.poll.return_value = 17 + + with pytest.raises(relay_gateway.RelayGatewayError, match="status 17"): + relay_gateway.wait_for_relay_gateway( + process, + "http://127.0.0.1:43210/healthz", + ) + + +def test_wait_for_relay_gateway_times_out(): + process = MagicMock() + process.poll.return_value = None + + with pytest.raises(relay_gateway.RelayGatewayError, match="did not become ready"): + relay_gateway.wait_for_relay_gateway( + process, + "http://127.0.0.1:43210/healthz", + timeout=0, + ) + + +def test_stop_relay_gateway_terminates_then_kills_after_timeout(): + process = MagicMock() + process.poll.return_value = None + process.wait.side_effect = [subprocess.TimeoutExpired("nemo-relay", 5), None] + + relay_gateway.stop_relay_gateway(process) + + process.terminate.assert_called_once_with() + process.kill.assert_called_once_with() + assert process.wait.call_args_list == [call(timeout=5), call(timeout=5)] + + +def test_stop_relay_gateway_is_idempotent_for_exited_process(): + process = MagicMock() + process.poll.return_value = 0 + + relay_gateway.stop_relay_gateway(process) + + process.terminate.assert_not_called() + process.kill.assert_not_called() diff --git a/tests/adapters/test_adapters_common_relay_hooks.py b/tests/adapters/test_adapters_common_relay_hooks.py new file mode 100644 index 000000000..c4c1b4743 --- /dev/null +++ b/tests/adapters/test_adapters_common_relay_hooks.py @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path +from typing import cast + +import pytest + +import nemo_fabric_adapters.common.relay_hooks as relay_hooks + + +EXPECTED_EVENTS = ( + "SessionStart", + "UserPromptSubmit", + "UserPromptExpansion", + "PreToolUse", + "PostToolUse", + "PostToolUseFailure", + "PermissionRequest", + "SubagentStart", + "SubagentStop", + "Notification", + "Stop", + "PreCompact", + "PostCompact", + "SessionEnd", +) + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +def test_render_relay_hooks_matches_relay_agent_contract(agent): + executable = Path("/opt/nvidia relay/bin/nemo-relay") + + hooks = relay_hooks.render_relay_hooks(agent, executable)["hooks"] + + assert tuple(hooks) == EXPECTED_EVENTS + assert hooks["SessionStart"] == [ + { + "hooks": [ + { + "type": "command", + "command": f"'{executable}' hook-forward {agent}", + "timeout": 30, + } + ] + } + ] + assert { + event for event, groups in hooks.items() if groups[0].get("matcher") == "*" + } == { + "PreToolUse", + "PostToolUse", + "PostToolUseFailure", + "PermissionRequest", + } + + +def test_render_relay_hooks_rejects_unsupported_agent(): + with pytest.raises(ValueError, match="unsupported NeMo Relay hook agent"): + relay_hooks.render_relay_hooks( + cast(relay_hooks.RelayHookAgent, "other"), + Path("nemo-relay"), + ) diff --git a/tests/adapters/test_claude_adapter.py b/tests/adapters/test_claude_adapter.py index 7cb68b038..edfd9b0f5 100644 --- a/tests/adapters/test_claude_adapter.py +++ b/tests/adapters/test_claude_adapter.py @@ -3,9 +3,10 @@ from __future__ import annotations -import importlib.util +import asyncio import json import os +import tomllib from pathlib import Path from typing import Any from unittest.mock import MagicMock @@ -23,28 +24,9 @@ TextBlock, ) from claude_agent_sdk._errors import MessageParseError +from nemo_fabric_adapters.claude import adapter ROOT = Path(__file__).resolve().parents[2] -ADAPTER_PATH = ( - ROOT - / "adapters" - / "claude" - / "src" - / "nemo_fabric_adapters" - / "claude" - / "adapter.py" -) - - -def load_claude_adapter(): - spec = importlib.util.spec_from_file_location("fabric_claude_adapter", ADAPTER_PATH) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -adapter = load_claude_adapter() def test_claude_descriptor_is_narrow_and_versioned(): @@ -60,7 +42,15 @@ def test_claude_descriptor_is_narrow_and_versioned(): "module": "nemo_fabric_adapters.claude.adapter", "callable": "run", }, - "config": {"accepts": ["models", "tools", "mcp", "skills"]}, + "config": {"accepts": ["models", "tools", "mcp", "skills", "telemetry"]}, + "telemetry": { + "providers": { + "relay": { + "outputs": ["atif", "otel", "openinference"], + "integration_modes": ["hooks", "gateway"], + } + } + }, } @@ -129,12 +119,12 @@ def claude_payload_fixture(tmp_path) -> dict[str, Any]: def test_build_options_maps_normalized_capabilities_and_claude_settings(claude_payload): - skill_path = Path(claude_payload["capability_plan"]["native"]["skill_paths"][0]) - options = adapter.build_options(claude_payload, resume="claude-session") assert options.resume == "claude-session" - assert options.cwd == Path(claude_payload["runtime_context"]["environment"]["workspace"]) + assert options.cwd == Path( + claude_payload["runtime_context"]["environment"]["workspace"] + ) assert options.model == "claude-test-model" assert options.system_prompt == "Review carefully." assert options.tools == ["Read", "Glob", "Grep", "Skill"] @@ -159,6 +149,160 @@ def test_build_options_maps_normalized_capabilities_and_claude_settings(claude_p "docs": {"type": "http", "url": "https://mcp.example.test"}, "repo": {"type": "stdio", "command": "repo-mcp", "args": ["--root", "."]}, } + assert "NEMO_RELAY_GATEWAY_URL" not in options.env + assert "ANTHROPIC_BASE_URL" not in options.env + + +@pytest.fixture(name="relay_payload") +def relay_payload_fixture(claude_payload, tmp_path) -> dict[str, Any]: + relay_intent_path = tmp_path / "relay-config.json" + relay_intent_path.write_text( + json.dumps( + { + "relay": { + "config": { + "atof": {"enabled": True}, + "atif": {"enabled": True}, + } + } + } + ), + encoding="utf-8", + ) + os.environ["FABRIC_RELAY_CONFIG_PATH"] = str(relay_intent_path) + claude_payload["telemetry_plan"] = { + "providers": ["relay"], + "relay_enabled": True, + } + return claude_payload + + +def test_prepare_claude_relay_writes_gateway_config_and_complete_hook_plugin( + relay_payload, monkeypatch, tmp_path +): + executable = tmp_path / "bin" / "nemo-relay" + executable.parent.mkdir() + executable.touch() + monkeypatch.setattr( + adapter.relay_gateway, + "resolve_relay_command", + MagicMock(return_value=executable), + ) + monkeypatch.setattr( + adapter.relay_gateway, + "find_available_tcp_port", + MagicMock(return_value=43210), + ) + monkeypatch.setattr( + adapter.relay_gateway, + "relay_cli_observability_version", + MagicMock(return_value=2), + ) + + relay = adapter.prepare_claude_relay(relay_payload) + + assert relay is not None + assert relay.gateway.executable == executable + assert relay.gateway.bind == "127.0.0.1:43210" + assert relay.gateway.url == "http://127.0.0.1:43210" + assert relay.gateway.log_path == relay.gateway.config_path.parent / "gateway.log" + with relay.gateway.config_path.open("rb") as stream: + assert tomllib.load(stream) == {"agents": {"claude": {"command": "claude"}}} + with (relay.gateway.config_path.parent / "plugins.toml").open("rb") as stream: + plugin_config = tomllib.load(stream) + assert plugin_config["components"][0]["kind"] == "observability" + + manifest = json.loads( + (relay.plugin_path / ".claude-plugin" / "plugin.json").read_text( + encoding="utf-8" + ) + ) + hooks = json.loads( + (relay.plugin_path / "hooks" / "hooks.json").read_text(encoding="utf-8") + )["hooks"] + assert manifest["name"] == "nemo-fabric-relay" + assert set(hooks) == { + "SessionStart", + "UserPromptSubmit", + "UserPromptExpansion", + "PreToolUse", + "PostToolUse", + "PostToolUseFailure", + "PermissionRequest", + "SubagentStart", + "SubagentStop", + "Notification", + "Stop", + "PreCompact", + "PostCompact", + "SessionEnd", + } + assert hooks["SessionStart"][0] == { + "hooks": [ + { + "type": "command", + "command": f"{executable} hook-forward claude", + "timeout": 30, + } + ] + } + assert hooks["PermissionRequest"][0]["matcher"] == "*" + + +def test_build_options_adds_relay_plugin_and_gateway_environment( + relay_payload, monkeypatch, tmp_path +): + executable = tmp_path / "nemo-relay" + executable.touch() + monkeypatch.setattr( + adapter.relay_gateway, + "resolve_relay_command", + MagicMock(return_value=executable), + ) + monkeypatch.setattr( + adapter.relay_gateway, + "find_available_tcp_port", + MagicMock(return_value=43210), + ) + monkeypatch.setattr( + adapter.relay_gateway, + "relay_cli_observability_version", + MagicMock(return_value=2), + ) + relay = adapter.prepare_claude_relay(relay_payload) + + options = adapter.build_options(relay_payload, resume=None, relay=relay) + + assert options.env["NEMO_RELAY_GATEWAY_URL"] == relay.gateway.url + assert options.env["ANTHROPIC_BASE_URL"] == relay.gateway.url + assert len(options.plugins) == 2 + assert Path(options.plugins[1]["path"]) == relay.plugin_path + assert ( + Path(options.plugins[0]["path"]) / "skills" / "review" / "SKILL.md" + ).exists() + + +def test_build_options_does_not_enable_skills_for_relay_plugin_alone( + relay_payload, tmp_path +): + relay_payload["capability_plan"]["native"]["skill_paths"] = [] + relay = adapter.ClaudeRelaySettings( + gateway=adapter.relay_gateway.RelayGatewayLaunch( + executable=tmp_path / "nemo-relay", + config_path=tmp_path / "relay-config" / "config.toml", + bind="127.0.0.1:43210", + url="http://127.0.0.1:43210", + log_path=tmp_path / "relay-config" / "gateway.log", + ), + plugin_config={"version": 1, "components": []}, + plugin_path=tmp_path / "relay-plugin", + ) + + options = adapter.build_options(relay_payload, resume=None, relay=relay) + + assert options.tools == ["Read", "Glob", "Grep"] + assert options.skills is None + assert options.plugins == [{"type": "local", "path": str(relay.plugin_path)}] @pytest.mark.parametrize( @@ -176,7 +320,9 @@ def test_build_options_rejects_normalized_capabilities_in_harness_settings( ): claude_payload["effective_config"]["config"]["harness"]["settings"][name] = [] - with pytest.raises(adapter.AdapterConfigError, match=normalized_field.replace(".", r"\.")): + with pytest.raises( + adapter.AdapterConfigError, match=normalized_field.replace(".", r"\.") + ): adapter.build_options(claude_payload, resume=None) @@ -218,7 +364,9 @@ def test_state_round_trip_is_keyed_by_fabric_runtime(claude_payload): runtime_id = adapter.runtime_id(claude_payload) adapter.save_claude_session_id(claude_payload, runtime_id, "claude-session") - assert adapter.load_claude_session_id(claude_payload, runtime_id) == "claude-session" + assert ( + adapter.load_claude_session_id(claude_payload, runtime_id) == "claude-session" + ) state_path = adapter.runtime_state_path(claude_payload, runtime_id) assert state_path.parent.name == "runtimes" assert runtime_id not in state_path.name @@ -234,7 +382,9 @@ def test_state_loader_rejects_non_object_json(claude_payload): adapter.load_claude_session_id(claude_payload, runtime_id) -def test_normalize_result_exposes_session_usage_cost_and_buffered_events(claude_payload): +def test_normalize_result_exposes_session_usage_cost_and_buffered_events( + claude_payload, +): messages = [ SystemMessage(subtype="init", data={"session_id": "claude-session"}), AssistantMessage( @@ -274,7 +424,9 @@ async def test_run_claude_resumes_and_persists_session(claude_payload, monkeypat async def query_result(*, prompt, options): captured.append((prompt, options.resume, dict(options.env), dict(os.environ))) - yield AssistantMessage(content=[TextBlock(text="done")], model="claude-test-model") + yield AssistantMessage( + content=[TextBlock(text="done")], model="claude-test-model" + ) yield ResultMessage( subtype="success", duration_ms=100, @@ -297,17 +449,306 @@ async def query_result(*, prompt, options): assert first["failed"] is False assert second["failed"] is False - assert [entry[0] for entry in captured] == ["Inspect the patch", "Inspect the patch"] + assert [entry[0] for entry in captured] == [ + "Inspect the patch", + "Inspect the patch", + ] assert [entry[1] for entry in captured] == [None, "claude-session"] assert all(entry[2]["FABRIC_UNRELATED_SECRET"] == "" for entry in captured) - assert all(entry[2]["ANTHROPIC_API_KEY"] == "configured-secret" for entry in captured) - assert all(entry[3]["FABRIC_UNRELATED_SECRET"] == "do-not-forward" for entry in captured) + assert all( + entry[2]["ANTHROPIC_API_KEY"] == "configured-secret" for entry in captured + ) + assert all( + entry[3]["FABRIC_UNRELATED_SECRET"] == "do-not-forward" for entry in captured + ) assert os.environ["FABRIC_UNRELATED_SECRET"] == "do-not-forward" -def test_build_options_forwards_default_anthropic_api_key( - claude_payload, monkeypatch +async def test_run_claude_supervises_relay_and_reports_artifacts( + relay_payload, monkeypatch, tmp_path +): + executable = tmp_path / "nemo-relay" + executable.touch() + relay = adapter.ClaudeRelaySettings( + gateway=adapter.relay_gateway.RelayGatewayLaunch( + executable=executable, + config_path=tmp_path / "relay-config" / "config.toml", + bind="127.0.0.1:43210", + url="http://127.0.0.1:43210", + log_path=tmp_path / "relay-config" / "gateway.log", + ), + plugin_config={ + "version": 1, + "components": [ + { + "kind": "observability", + "enabled": True, + "config": { + "atif": { + "enabled": True, + "output_directory": str(tmp_path / "atif"), + } + }, + } + ], + }, + plugin_path=tmp_path / "relay-plugin", + ) + relay.plugin_path.mkdir() + relay.gateway.log_path.parent.mkdir() + relay.gateway.log_path.write_text("gateway started\n", encoding="utf-8") + atif_path = tmp_path / "atif" / "trajectory-session.atif.json" + atif_path.parent.mkdir() + atif_path.write_text("{}", encoding="utf-8") + process = MagicMock() + mock_start = MagicMock(return_value=process) + mock_stop = MagicMock() + monkeypatch.setattr(adapter, "prepare_claude_relay", MagicMock(return_value=relay)) + monkeypatch.setattr(adapter.relay_gateway, "start_relay_gateway", mock_start) + monkeypatch.setattr(adapter.relay_gateway, "stop_relay_gateway", mock_stop) + + async def query_result(*, prompt, options): + assert options.env["ANTHROPIC_BASE_URL"] == relay.gateway.url + assert Path(options.plugins[-1]["path"]) == relay.plugin_path + yield ResultMessage( + subtype="success", + duration_ms=10, + duration_api_ms=8, + is_error=False, + num_turns=1, + session_id="claude-session", + total_cost_usd=0.01, + usage={"input_tokens": 1, "output_tokens": 1}, + result="done", + ) + + monkeypatch.setattr(adapter, "query", MagicMock(side_effect=query_result)) + + output = await adapter.run_claude(relay_payload) + + assert output["relay_runtime"] == { + "enabled": True, + "emitter": "claude-agent-sdk/nemo-relay", + "config_path": os.environ["FABRIC_RELAY_CONFIG_PATH"], + "gateway_config_path": str(relay.gateway.config_path), + "gateway_url": relay.gateway.url, + "gateway_log_path": str(relay.gateway.log_path), + } + assert output["relay_artifacts"] == [{"kind": "atif", "path": str(atif_path)}] + mock_start.assert_called_once_with( + launch=relay.gateway, + cwd=Path(relay_payload["runtime_context"]["environment"]["workspace"]), + ) + mock_stop.assert_called_once_with(process) + assert not relay.plugin_path.exists() + + +async def test_run_claude_preserves_result_when_relay_stop_fails( + relay_payload, monkeypatch, tmp_path +): + relay = adapter.ClaudeRelaySettings( + gateway=adapter.relay_gateway.RelayGatewayLaunch( + executable=tmp_path / "nemo-relay", + config_path=tmp_path / "relay-config" / "config.toml", + bind="127.0.0.1:43210", + url="http://127.0.0.1:43210", + log_path=tmp_path / "relay-config" / "gateway.log", + ), + plugin_config={"version": 1, "components": []}, + plugin_path=tmp_path / "relay-plugin", + ) + relay.plugin_path.mkdir() + monkeypatch.setattr(adapter, "prepare_claude_relay", MagicMock(return_value=relay)) + monkeypatch.setattr( + adapter.relay_gateway, + "start_relay_gateway", + MagicMock(return_value=MagicMock()), + ) + monkeypatch.setattr( + adapter.relay_gateway, + "stop_relay_gateway", + MagicMock( + side_effect=adapter.relay_gateway.RelayGatewayError("raw stop failure") + ), + ) + + async def query_result(*, prompt, options): + yield ResultMessage( + subtype="success", + duration_ms=10, + duration_api_ms=8, + is_error=False, + num_turns=1, + session_id="claude-session", + total_cost_usd=0.01, + usage={"input_tokens": 1, "output_tokens": 1}, + result="done", + ) + + monkeypatch.setattr(adapter, "query", MagicMock(side_effect=query_result)) + + output = await adapter.run_claude(relay_payload) + + assert output["response"] == "done" + assert output["completed"] is False + assert output["failed"] is True + assert output["error"] == { + "code": "claude_relay_stop_failed", + "message": "NeMo Relay gateway failed to stop", + "retryable": False, + "metadata": {"gateway_log_path": str(relay.gateway.log_path)}, + } + assert output["relay_runtime"]["cleanup_error"] == output["error"] + assert "raw stop failure" not in json.dumps(output) + assert not relay.plugin_path.exists() + + +async def test_run_claude_preserves_result_when_relay_plugin_cleanup_fails( + relay_payload, monkeypatch, tmp_path ): + relay = adapter.ClaudeRelaySettings( + gateway=adapter.relay_gateway.RelayGatewayLaunch( + executable=tmp_path / "nemo-relay", + config_path=tmp_path / "relay-config" / "config.toml", + bind="127.0.0.1:43210", + url="http://127.0.0.1:43210", + log_path=tmp_path / "relay-config" / "gateway.log", + ), + plugin_config={"version": 1, "components": []}, + plugin_path=tmp_path / "relay-plugin", + ) + relay.plugin_path.mkdir() + process = MagicMock() + mock_stop = MagicMock() + mock_rmtree = MagicMock(side_effect=OSError("raw plugin cleanup failure")) + monkeypatch.setattr(adapter, "prepare_claude_relay", MagicMock(return_value=relay)) + monkeypatch.setattr( + adapter.relay_gateway, + "start_relay_gateway", + MagicMock(return_value=process), + ) + monkeypatch.setattr(adapter.relay_gateway, "stop_relay_gateway", mock_stop) + monkeypatch.setattr(adapter.shutil, "rmtree", mock_rmtree) + + async def query_result(**_): + yield ResultMessage( + subtype="success", + duration_ms=10, + duration_api_ms=8, + is_error=False, + num_turns=1, + session_id="claude-session", + total_cost_usd=0.01, + usage={"input_tokens": 1, "output_tokens": 1}, + result="done", + ) + + monkeypatch.setattr(adapter, "query", MagicMock(side_effect=query_result)) + + output = await adapter.run_claude(relay_payload) + + assert output["response"] == "done" + assert output["completed"] is False + assert output["failed"] is True + assert output["error"] == { + "code": "claude_relay_cleanup_failed", + "message": "Claude Relay hook configuration could not be removed", + "retryable": False, + } + assert output["relay_runtime"]["cleanup_error"] == output["error"] + assert "raw plugin cleanup failure" not in json.dumps(output) + mock_stop.assert_called_once_with(process) + mock_rmtree.assert_called_once_with(relay.plugin_path) + assert relay.plugin_path.exists() + + +@pytest.mark.parametrize( + "failure", [ClaudeSDKError("sdk failed"), asyncio.CancelledError()] +) +async def test_run_claude_stops_relay_on_sdk_failure_or_cancellation( + relay_payload, monkeypatch, tmp_path, failure +): + relay = adapter.ClaudeRelaySettings( + gateway=adapter.relay_gateway.RelayGatewayLaunch( + executable=tmp_path / "nemo-relay", + config_path=tmp_path / "relay-config" / "config.toml", + bind="127.0.0.1:43210", + url="http://127.0.0.1:43210", + log_path=tmp_path / "relay-config" / "gateway.log", + ), + plugin_config={"version": 1, "components": []}, + plugin_path=tmp_path / "relay-plugin", + ) + relay.plugin_path.mkdir() + process = MagicMock() + mock_stop = MagicMock() + monkeypatch.setattr(adapter, "prepare_claude_relay", MagicMock(return_value=relay)) + monkeypatch.setattr( + adapter.relay_gateway, + "start_relay_gateway", + MagicMock(return_value=process), + ) + monkeypatch.setattr(adapter.relay_gateway, "stop_relay_gateway", mock_stop) + + async def query_failure(*, prompt, options): + raise failure + yield + + monkeypatch.setattr(adapter, "query", MagicMock(side_effect=query_failure)) + + if isinstance(failure, asyncio.CancelledError): + with pytest.raises(asyncio.CancelledError): + await adapter.run_claude(relay_payload) + else: + output = await adapter.run_claude(relay_payload) + assert output["error"]["code"] == "claude_failed" + assert output["relay_runtime"]["enabled"] is True + + mock_stop.assert_called_once_with(process) + assert not relay.plugin_path.exists() + + +def test_run_reports_relay_start_failure_without_raw_diagnostic( + relay_payload, monkeypatch, tmp_path +): + executable = tmp_path / "nemo-relay" + executable.touch() + relay = adapter.ClaudeRelaySettings( + gateway=adapter.relay_gateway.RelayGatewayLaunch( + executable=executable, + config_path=tmp_path / "relay-config" / "config.toml", + bind="127.0.0.1:43210", + url="http://127.0.0.1:43210", + log_path=tmp_path / "relay-config" / "gateway.log", + ), + plugin_config={"version": 1, "components": []}, + plugin_path=tmp_path / "relay-plugin", + ) + relay.plugin_path.mkdir() + monkeypatch.setattr(adapter, "prepare_claude_relay", MagicMock(return_value=relay)) + monkeypatch.setattr( + adapter.relay_gateway, + "start_relay_gateway", + MagicMock( + side_effect=adapter.relay_gateway.RelayGatewayError( + "raw gateway failure with secret" + ) + ), + ) + + output = adapter.run(relay_payload) + + assert output["error"] == { + "code": "claude_relay_start_failed", + "message": "NeMo Relay gateway failed to start", + "retryable": False, + "metadata": {"gateway_log_path": str(relay.gateway.log_path)}, + } + assert "secret" not in json.dumps(output) + assert not relay.plugin_path.exists() + + +def test_build_options_forwards_default_anthropic_api_key(claude_payload, monkeypatch): model = claude_payload["effective_config"]["config"]["models"]["default"] model.pop("api_key_env") settings = claude_payload["effective_config"]["config"]["harness"]["settings"] @@ -324,9 +765,15 @@ def test_build_options_forwards_default_anthropic_api_key( [ (CLINotFoundError("raw path", "/secret/claude"), "claude_cli_not_found"), (CLIConnectionError("raw connection"), "claude_connection_failed"), - (ProcessError("raw process", exit_code=9, stderr="secret"), "claude_process_failed"), + ( + ProcessError("raw process", exit_code=9, stderr="secret"), + "claude_process_failed", + ), (CLIJSONDecodeError("secret-json", ValueError("bad")), "claude_invalid_json"), - (MessageParseError("raw parse", data={"secret": "value"}), "claude_message_parse_failed"), + ( + MessageParseError("raw parse", data={"secret": "value"}), + "claude_message_parse_failed", + ), (ClaudeSDKError("raw sdk"), "claude_failed"), ], ) diff --git a/tests/adapters/test_codex_cli.py b/tests/adapters/test_codex_cli.py index 322db1ae0..d46f2d734 100644 --- a/tests/adapters/test_codex_cli.py +++ b/tests/adapters/test_codex_cli.py @@ -7,7 +7,6 @@ import tomllib from pathlib import Path from unittest.mock import MagicMock -from unittest.mock import call import pytest from nemo_fabric import Fabric @@ -160,7 +159,9 @@ def test_oneshot_command_uses_fabric_overrides_and_codex_owned_auth( assert "--dangerously-bypass-hook-trust" not in command assert ["--model", "gpt-5.4"] == command[-3:-1] assert command[-1] == "-" - assert tomllib.loads(codex_settings.codex_profile_path.read_text(encoding="utf-8")) == { + assert tomllib.loads( + codex_settings.codex_profile_path.read_text(encoding="utf-8") + ) == { "features": {"web_search": False}, "model_reasoning_effort": "high", } @@ -183,7 +184,9 @@ def test_configured_codex_profile_is_base_for_generated_profile(codex_payload): codex_settings = adapter.write_config_files(codex_payload) assert codex_settings.codex_profile_name == "fabric-runtime-1" - assert tomllib.loads(codex_settings.codex_profile_path.read_text(encoding="utf-8")) == { + assert tomllib.loads( + codex_settings.codex_profile_path.read_text(encoding="utf-8") + ) == { "approval_policy": "never", "model_reasoning_effort": "high", "features": { @@ -218,7 +221,9 @@ def test_relative_codex_command_resolves_from_config_root(codex_payload): def test_codex_home_uses_environment(tmp_path): os.environ["CODEX_HOME"] = str(tmp_path / "custom-codex-home") - name, path = adapter.get_codex_profile_path({"runtime_context": {"runtime_id": "runtime-1"}}) + name, path = adapter.get_codex_profile_path( + {"runtime_context": {"runtime_id": "runtime-1"}} + ) assert name == "fabric-runtime-1" assert path == tmp_path / "custom-codex-home" / "fabric-runtime-1.config.toml" @@ -227,7 +232,9 @@ def test_codex_home_uses_environment(tmp_path): def test_codex_home_defaults_to_user_codex_directory(): os.environ.pop("CODEX_HOME", None) - name, path = adapter.get_codex_profile_path({"runtime_context": {"runtime_id": "runtime-1"}}) + name, path = adapter.get_codex_profile_path( + {"runtime_context": {"runtime_id": "runtime-1"}} + ) assert name == "fabric-runtime-1" assert path == Path.home() / ".codex" / "fabric-runtime-1.config.toml" @@ -243,11 +250,23 @@ def test_relay_routes_codex_through_standalone_gateway( "relay_enabled": True, } mock_find_port = MagicMock(return_value=43210) - monkeypatch.setattr(adapter, "find_available_tcp_port", mock_find_port) + monkeypatch.setattr( + adapter.relay_gateway, "find_available_tcp_port", mock_find_port + ) + relay_executable = tmp_path / "bin" / "nemo-relay" + relay_executable.parent.mkdir() + relay_executable.touch() + monkeypatch.setattr( + adapter.relay_gateway, + "resolve_relay_command", + MagicMock(return_value=relay_executable), + ) relay_plugin_config = {"version": 1, "components": []} relay_config_path = tmp_path / "relay-config" / "config.toml" mock_load_config = MagicMock(return_value=relay_plugin_config) - mock_write_config = MagicMock(return_value=(relay_config_path, tmp_path / "plugins.toml")) + mock_write_config = MagicMock( + return_value=(relay_config_path, tmp_path / "plugins.toml") + ) monkeypatch.setattr( adapter.common_utils, "load_relay_plugin_config", @@ -258,6 +277,11 @@ def test_relay_routes_codex_through_standalone_gateway( "write_relay_configs", mock_write_config, ) + monkeypatch.setattr( + adapter.relay_gateway, + "relay_cli_observability_version", + MagicMock(return_value=2), + ) codex_settings = adapter.write_config_files(codex_payload) command = adapter.build_command( @@ -272,7 +296,9 @@ def test_relay_routes_codex_through_standalone_gateway( assert "--config" not in command assert command[command.index("--profile") + 1] == "fabric-runtime-1" - config = tomllib.loads(codex_settings.codex_profile_path.read_text(encoding="utf-8")) + config = tomllib.loads( + codex_settings.codex_profile_path.read_text(encoding="utf-8") + ) assert config["model_provider"] == "nemo-relay-openai" assert config["model_providers"]["nemo-relay-openai"] == { "name": "NeMo Relay OpenAI", @@ -286,13 +312,15 @@ def test_relay_routes_codex_through_standalone_gateway( assert config["model_reasoning_effort"] == "high" assert config["hooks"]["SessionStart"][0]["hooks"][0] == { "type": "command", - "command": "nemo-relay hook-forward codex", + "command": f"{relay_executable} hook-forward codex", "timeout": 30, } + assert "UserPromptExpansion" in config["hooks"] mock_load_config.assert_called_once_with(codex_payload) mock_write_config.assert_called_once_with( relay_config={"agents": {"codex": {"command": "codex"}}}, plugin_config=relay_plugin_config, + observability_version=2, ) @@ -333,7 +361,9 @@ def test_native_otel_profile_writes_codex_telemetry_config( assert command[command.index("--profile") + 1] == "fabric-runtime-1" assert "--dangerously-bypass-hook-trust" not in command - assert tomllib.loads(codex_settings.codex_profile_path.read_text(encoding="utf-8")) == { + assert tomllib.loads( + codex_settings.codex_profile_path.read_text(encoding="utf-8") + ) == { "otel": { "environment": "dev", "trace_exporter": { @@ -357,16 +387,21 @@ def test_run_codex_configures_relay(codex_payload, monkeypatch, tmp_path): codex_home = tmp_path / "codex-home" profile_name = "fabric-runtime-1" profile_path = codex_home / f"{profile_name}.config.toml" + gateway = adapter.relay_gateway.RelayGatewayLaunch( + executable=tmp_path / "nemo-relay", + config_path=relay_config_path, + bind=gateway_host, + url=gateway_url, + log_path=relay_config_path.parent / "gateway.log", + ) codex_settings = adapter.CodexSettings( telemetry_provider="relay", - relay_enabled=True, codex_profile_name=profile_name, codex_profile_path=profile_path, - relay_gateway_host=gateway_host, - relay_gateway_url=gateway_url, - relay_gateway_port=43210, - relay_config_path=relay_config_path, - relay_plugin_config=relay_plugin_config, + relay=adapter.CodexRelaySettings( + gateway=gateway, + plugin_config=relay_plugin_config, + ), ) mock_write_config_files = MagicMock(return_value=codex_settings) mock_run = MagicMock( @@ -380,9 +415,10 @@ def test_run_codex_configures_relay(codex_payload, monkeypatch, tmp_path): os.environ["FABRIC_RELAY_ENABLED"] = "true" os.environ.pop("CODEX_HOME", None) os.environ["FABRIC_RELAY_CONFIG_PATH"] = str(tmp_path / "relay-config.json") - monkeypatch.setattr(adapter, "start_relay_gateway", mock_start_gateway) - monkeypatch.setattr(adapter, "stop_relay_gateway", mock_stop_gateway) - monkeypatch.setattr(adapter.time, "sleep", MagicMock()) + monkeypatch.setattr( + adapter.relay_gateway, "start_relay_gateway", mock_start_gateway + ) + monkeypatch.setattr(adapter.relay_gateway, "stop_relay_gateway", mock_stop_gateway) monkeypatch.setattr( adapter, "write_config_files", @@ -393,8 +429,14 @@ def test_run_codex_configures_relay(codex_payload, monkeypatch, tmp_path): result = adapter.run_codex(codex_payload) command = mock_run.call_args.args[0] - assert "relay_runtime" in result - assert "relay_artifacts" in result + assert result["relay_runtime"] == { + "enabled": True, + "config_path": os.environ["FABRIC_RELAY_CONFIG_PATH"], + "emitter": "nemo-relay", + "gateway_config_path": str(relay_config_path), + "gateway_log_path": str(gateway.log_path), + } + assert result["relay_artifacts"] == [] assert command[0] == "codex" assert "nemo-relay" not in command assert command[command.index("--profile") + 1] == profile_name @@ -403,89 +445,12 @@ def test_run_codex_configures_relay(codex_payload, monkeypatch, tmp_path): assert not profile_path.exists() mock_write_config_files.assert_called_once_with(codex_payload) mock_start_gateway.assert_called_once_with( - codex_payload, - Path(codex_payload["runtime_context"]["environment"]["workspace"]), - codex_settings, + launch=gateway, + cwd=Path(codex_payload["runtime_context"]["environment"]["workspace"]), ) mock_stop_gateway.assert_called_once_with(mock_gateway) -def test_start_relay_gateway_waits_for_health_and_starts_process_group( - codex_payload, - monkeypatch, - tmp_path, -): - relay_config_path = tmp_path / "relay-config" / "config.toml" - relay_config_path.parent.mkdir() - relay_config_path.write_text('[agents.codex]\ncommand = "codex"\n', encoding="utf-8") - mock_process = MagicMock() - mock_popen = MagicMock(return_value=mock_process) - mock_wait = MagicMock() - gateway_host = "127.0.0.1:43210" - gateway_url = f"http://{gateway_host}" - codex_settings = adapter.CodexSettings( - telemetry_provider="relay", - relay_enabled=True, - codex_profile_name="fabric-runtime-1", - codex_profile_path=tmp_path / "fabric-runtime-1.config.toml", - relay_gateway_host=gateway_host, - relay_gateway_url=gateway_url, - relay_gateway_port=43210, - relay_config_path=relay_config_path, - relay_plugin_config={"version": 1, "components": []}, - ) - monkeypatch.setattr(adapter, "wait_for_relay_gateway", mock_wait) - monkeypatch.setattr(adapter.subprocess, "Popen", mock_popen) - - process = adapter.start_relay_gateway( - codex_payload, - tmp_path, - codex_settings, - ) - - assert process is mock_process - assert mock_popen.call_args.args[0] == [ - "nemo-relay", - "--config", - str(relay_config_path), - "--bind", - gateway_host, - ] - assert mock_popen.call_args.kwargs["start_new_session"] is True - mock_wait.assert_called_once_with(mock_process, f"{gateway_url}/healthz") - - -def test_wait_for_relay_gateway_times_out(): - mock_process = MagicMock() - mock_process.poll.return_value = None - health_url = "http://127.0.0.1:43210/healthz" - - with pytest.raises(RuntimeError, match="gateway did not become ready"): - adapter.wait_for_relay_gateway(mock_process, health_url, timeout=0) - - -def test_stop_relay_gateway_terminates_process(): - mock_process = MagicMock() - mock_process.poll.return_value = None - - adapter.stop_relay_gateway(mock_process) - - mock_process.terminate.assert_called_once_with() - mock_process.wait.assert_called_once_with(timeout=5) - - -def test_stop_relay_gateway_kills_process_after_timeout(): - mock_process = MagicMock() - mock_process.poll.return_value = None - mock_process.wait.side_effect = [subprocess.TimeoutExpired("nemo-relay", 5), None] - - adapter.stop_relay_gateway(mock_process) - - mock_process.terminate.assert_called_once_with() - mock_process.kill.assert_called_once_with() - assert mock_process.wait.call_args_list == [call(timeout=5), call(timeout=5)] - - def test_reported_command_redacts_secret_config_overrides(): command = ["codex", "exec", "--config", 'provider.api_key="secret"', "-"] @@ -510,7 +475,9 @@ def test_config_override_values_reject_nested_non_finite_numbers(value): adapter.toml_value(value) -def test_runtime_reuses_codex_thread_across_invocations(codex_payload, monkeypatch, tmp_path): +def test_runtime_reuses_codex_thread_across_invocations( + codex_payload, monkeypatch, tmp_path +): mock_run = MagicMock( side_effect=[ subprocess.CompletedProcess( @@ -531,7 +498,9 @@ def test_runtime_reuses_codex_thread_across_invocations(codex_payload, monkeypat os.environ.pop("OPENAI_API_KEY", None) os.environ["CODEX_HOME"] = str(tmp_path / "codex-home") os.environ["FABRIC_UNRELATED_SECRET"] = "do-not-forward" - codex_payload["effective_config"]["config"]["harness"]["settings"]["env"] = {"CODEX_EXPLICIT": "forward-me"} + codex_payload["effective_config"]["config"]["harness"]["settings"]["env"] = { + "CODEX_EXPLICIT": "forward-me" + } first = adapter.run_codex(codex_payload) codex_payload["runtime_context"]["invocation_id"] = "invocation-2" @@ -591,7 +560,9 @@ def test_runtime_persists_codex_thread_state(codex_payload, monkeypatch): def test_adapter_rejects_structured_input_until_chat_is_supported(codex_payload): - codex_payload["request"]["input"] = {"messages": [{"role": "user", "content": "Inspect the change."}]} + codex_payload["request"]["input"] = { + "messages": [{"role": "user", "content": "Inspect the change."}] + } with pytest.raises(ValueError, match="requires text input"): adapter.request_to_prompt(codex_payload) @@ -617,7 +588,9 @@ def test_adapter_rejects_non_mapping_env(codex_payload, env): ), ], ) -def test_process_launch_failures_return_structured_results(codex_payload, monkeypatch, error, message, returncode): +def test_process_launch_failures_return_structured_results( + codex_payload, monkeypatch, error, message, returncode +): monkeypatch.setattr(adapter.subprocess, "run", MagicMock(side_effect=error)) output = adapter.run_codex(codex_payload) @@ -659,7 +632,9 @@ def test_adapter_rejects_invalid_timeout(codex_payload, timeout): adapter.run_codex(codex_payload) -def test_runtime_fails_if_codex_does_not_return_thread_identity(codex_payload, monkeypatch): +def test_runtime_fails_if_codex_does_not_return_thread_identity( + codex_payload, monkeypatch +): mock_run = MagicMock( return_value=subprocess.CompletedProcess( args=[], @@ -681,7 +656,9 @@ def test_runtime_fails_if_codex_does_not_return_thread_identity(codex_payload, m assert "thread identity" in output["error"] -def test_successful_process_without_final_response_is_failed(codex_payload, monkeypatch): +def test_successful_process_without_final_response_is_failed( + codex_payload, monkeypatch +): mock_run = MagicMock( return_value=subprocess.CompletedProcess( args=[], @@ -738,7 +715,10 @@ async def test_fabric_oneshot_uses_cached_codex_auth(tmp_path): ) assert report.status == "pass" - assert any(check.name == "requirement.binary" and "codex_command" in check.message for check in report.checks) + assert any( + check.name == "requirement.binary" and "codex_command" in check.message + for check in report.checks + ) assert not any(check.name == "requirement.env" for check in report.checks) assert result.output["response"] == "thread-fake:inspect" assert "--ephemeral" not in result.output["command"] diff --git a/tests/e2e/test_claude.py b/tests/e2e/test_claude.py index 0570eeb55..b7d74a69d 100644 --- a/tests/e2e/test_claude.py +++ b/tests/e2e/test_claude.py @@ -18,6 +18,9 @@ HarnessConfig, MetadataConfig, ModelConfig, + RelayAtifConfig, + RelayAtofConfig, + RelayObservabilityConfig, RuntimeConfig, ) @@ -27,7 +30,44 @@ SESSION_ID = "11111111-1111-4111-8111-111111111111" -def fabric_config(tmp_path, *, cli_path=None): +def write_mock_relay_gateway(path: Path, log_path: Path) -> None: + path.write_text( + f"""#!{sys.executable} +import json +import sys +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path + +args = sys.argv[1:] +if args == ["--version"]: + print("nemo-relay 0.6.0") + raise SystemExit(0) +Path({str(log_path)!r}).write_text(json.dumps(args), encoding="utf-8") +bind = args[args.index("--bind") + 1] +host, port = bind.rsplit(":", 1) + +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200 if self.path == "/healthz" else 404) + self.end_headers() + + def log_message(self, format, *args): + pass + +HTTPServer((host, int(port)), Handler).serve_forever() +""", + encoding="utf-8", + ) + path.chmod(0o755) + + +def fabric_config( + tmp_path, + *, + cli_path=None, + relay=False, + nemo_relay_command=None, +): tmp_path.mkdir(parents=True, exist_ok=True) settings = { "python": sys.executable, @@ -41,9 +81,12 @@ def fabric_config(tmp_path, *, cli_path=None): "env": { "CLAUDE_AGENT_SDK_SKIP_VERSION_CHECK": "1", "MOCK_CLAUDE_CLI_LOG": str(tmp_path / "claude-args.jsonl"), + "MOCK_CLAUDE_CLI_ENV_LOG": str(tmp_path / "claude-env.jsonl"), }, } ) + if nemo_relay_command is not None: + settings["nemo_relay_command"] = str(nemo_relay_command) config = FabricConfig( metadata=MetadataConfig(name="claude-runtime-test"), harness=HarnessConfig( @@ -72,6 +115,13 @@ def fabric_config(tmp_path, *, cli_path=None): transport="streamable-http", url="https://mcp.example.test", ) + if relay: + config.enable_relay( + observability=RelayObservabilityConfig( + atof=RelayAtofConfig(enabled=True), + atif=RelayAtifConfig(enabled=True), + ) + ) return config @@ -85,7 +135,9 @@ async def test_fabric_session_launches_fresh_processes_and_resumes(tmp_path): assert first.status == second.status == "succeeded" assert first.runtime_id == second.runtime_id assert first.output["session_id"] == second.output["session_id"] == SESSION_ID - assert first.output["response"] == second.output["response"] == "mock Claude response" + assert ( + first.output["response"] == second.output["response"] == "mock Claude response" + ) assert first.output["usage"] == {"input_tokens": 1, "output_tokens": 2} assert first.output["cost_usd"] == 0.001 assert [event["type"] for event in first.output["events"]] == ["AssistantMessage"] @@ -97,8 +149,7 @@ async def test_fabric_session_launches_fresh_processes_and_resumes(tmp_path): assert "--resume" not in arguments[0] assert arguments[1][arguments[1].index("--resume") + 1] == SESSION_ID assert all( - args[args.index("--tools") + 1] == "Read,Glob,Grep,Skill" - for args in arguments + args[args.index("--tools") + 1] == "Read,Glob,Grep,Skill" for args in arguments ) assert all("--mcp-config" in args for args in arguments) assert all("--plugin-dir" in args for args in arguments) @@ -108,6 +159,71 @@ async def test_fabric_session_launches_fresh_processes_and_resumes(tmp_path): assert not any(artifact.kind == "stderr" for artifact in second.artifacts.artifacts) +async def test_fabric_claude_relay_supervises_gateway_and_injects_plugin(tmp_path): + mock_relay = tmp_path / "nemo-relay" + relay_args_path = tmp_path / "relay-args.json" + write_mock_relay_gateway(mock_relay, relay_args_path) + config = fabric_config( + tmp_path, + cli_path=MOCK_CLAUDE_CLI, + relay=True, + nemo_relay_command=mock_relay, + ) + + result = await Fabric().run(config, base_dir=tmp_path, input="inspect") + + assert result.status == "succeeded" + assert result.telemetry[0].provider == "relay" + relay_runtime = result.output["relay_runtime"] + assert relay_runtime["enabled"] is True + assert relay_runtime["emitter"] == "claude-agent-sdk/nemo-relay" + assert Path(relay_runtime["gateway_log_path"]).is_file() + assert Path(relay_runtime["gateway_config_path"]).is_file() + assert result.output["relay_artifacts"] == [] + + relay_args = json.loads(relay_args_path.read_text(encoding="utf-8")) + assert relay_args[0] == "--config" + assert relay_args[2] == "--bind" + assert relay_args[3] in relay_runtime["gateway_url"] + claude_args = json.loads((tmp_path / "claude-args.jsonl").read_text()) + assert claude_args.count("--plugin-dir") == 2 + plugin_paths = [ + Path(claude_args[index + 1]) + for index, value in enumerate(claude_args) + if value == "--plugin-dir" + ] + relay_plugin_path = next( + path for path in plugin_paths if path.name == "claude-plugin" + ) + assert relay_plugin_path.name == "claude-plugin" + assert not relay_plugin_path.exists() + claude_env = json.loads((tmp_path / "claude-env.jsonl").read_text()) + assert claude_env == { + "ANTHROPIC_BASE_URL": relay_runtime["gateway_url"], + "NEMO_RELAY_GATEWAY_URL": relay_runtime["gateway_url"], + } + + +@pytest.mark.skipif( + not os.environ.get("FABRIC_NEMO_RELAY_COMMAND"), + reason="set FABRIC_NEMO_RELAY_COMMAND to test an installed NeMo Relay CLI", +) +async def test_fabric_claude_accepts_real_relay_gateway_with_mock_claude(tmp_path): + config = fabric_config( + tmp_path, + cli_path=MOCK_CLAUDE_CLI, + relay=True, + nemo_relay_command=os.environ["FABRIC_NEMO_RELAY_COMMAND"], + ) + + result = await Fabric().run(config, base_dir=tmp_path, input="inspect") + + assert result.status == "succeeded" + assert result.output["relay_runtime"]["enabled"] is True + gateway_log_path = Path(result.output["relay_runtime"]["gateway_log_path"]) + assert gateway_log_path.is_file() + + @pytest.mark.skipif( os.environ.get("RUN_FABRIC_CLAUDE_INTEGRATION") != "1", reason="set RUN_FABRIC_CLAUDE_INTEGRATION=1 to run Claude Agent SDK integration", @@ -126,7 +242,28 @@ async def test_live_claude_one_shot_and_session(tmp_path): fabric_config(session_root), base_dir=session_root ) as session: first = await session.invoke(input="Remember token FABRIC-CONTINUITY-7") - second = await session.invoke(input="Reply only with the token I asked you to remember") + second = await session.invoke( + input="Reply only with the token I asked you to remember" + ) assert first.status == second.status == "succeeded" assert first.output["session_id"] == second.output["session_id"] assert "FABRIC-CONTINUITY-7" in second.output["response"] + + +@pytest.mark.skipif( + os.environ.get("RUN_FABRIC_CLAUDE_RELAY_INTEGRATION") != "1", + reason="set RUN_FABRIC_CLAUDE_RELAY_INTEGRATION=1 to run Claude with NeMo Relay", +) +async def test_live_claude_relay_one_shot(tmp_path): + result = await Fabric().run( + fabric_config(tmp_path, relay=True), + base_dir=tmp_path, + input="Use one simple tool, then reply only with: FABRIC_CLAUDE_RELAY_OK", + ) + + assert result.status == "succeeded" + assert result.output["relay_runtime"]["enabled"] is True + assert {artifact["kind"] for artifact in result.output["relay_artifacts"]} == { + "atof", + "atif", + } diff --git a/tests/fixtures/claude/mock-claude-cli.py b/tests/fixtures/claude/mock-claude-cli.py index 76ee05bde..164ca7f41 100755 --- a/tests/fixtures/claude/mock-claude-cli.py +++ b/tests/fixtures/claude/mock-claude-cli.py @@ -14,6 +14,18 @@ with open(os.environ["MOCK_CLAUDE_CLI_LOG"], "a", encoding="utf-8") as stream: stream.write(json.dumps(sys.argv[1:]) + "\n") +if env_log := os.environ.get("MOCK_CLAUDE_CLI_ENV_LOG"): + with open(env_log, "a", encoding="utf-8") as stream: + stream.write( + json.dumps( + { + "ANTHROPIC_BASE_URL": os.environ.get("ANTHROPIC_BASE_URL"), + "NEMO_RELAY_GATEWAY_URL": os.environ.get("NEMO_RELAY_GATEWAY_URL"), + } + ) + + "\n" + ) + for line in sys.stdin: message = json.loads(line) if message.get("type") == "control_request": diff --git a/uv.lock b/uv.lock index 9c22721c2..3331c0909 100644 --- a/uv.lock +++ b/uv.lock @@ -2132,12 +2132,14 @@ source = { editable = "adapters/claude" } dependencies = [ { name = "claude-agent-sdk" }, { name = "nemo-fabric-adapters-common" }, + { name = "tomli-w" }, ] [package.metadata] requires-dist = [ { name = "claude-agent-sdk", specifier = "==0.2.114" }, { name = "nemo-fabric-adapters-common", editable = "adapters/common" }, + { name = "tomli-w", specifier = "~=1.2" }, ] [[package]]