From 9b4afbd82d3d9ed63c4fb0140cf9a18e3a6cbc3c Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:29:30 -0700 Subject: [PATCH 01/14] Add first-class Claude CLI agent Rename the draft Claude SDK harness, add registered agent serialization, stream Claude CLI steps with trace attribution, and integrate the agent with eval configuration and hosted submission. Co-authored-by: Asadullo Ganiev <62354884+solvemproblr@users.noreply.github.com> --- docs/v6/cookbooks/coding-agent.mdx | 18 +- docs/v6/reference/agents.mdx | 8 +- docs/v6/reference/runtime.mdx | 6 +- hud/agents/__init__.py | 25 +- hud/agents/claude/__init__.py | 6 +- hud/agents/claude/agent.py | 34 +- hud/agents/claude/cli/__init__.py | 5 + hud/agents/claude/cli/agent.py | 549 ++++++++++++++++++ .../claude/{sdk => cli}/computer_mcp.py | 4 +- hud/agents/claude/sdk/__init__.py | 5 - hud/agents/claude/sdk/agent.py | 342 ----------- hud/agents/registry.py | 50 ++ hud/agents/tests/test_base.py | 31 +- ..._sdk_agent.py => test_claude_cli_agent.py} | 415 ++++++++++--- hud/agents/tests/test_tool_agent.py | 4 +- hud/agents/tool_agent.py | 27 +- hud/agents/types.py | 11 +- hud/cli/eval.py | 24 +- hud/cli/tests/test_eval_config.py | 50 ++ hud/eval/run.py | 15 +- hud/eval/runtime/hosted.py | 11 +- hud/eval/tests/test_hosted.py | 51 +- hud/types.py | 49 +- 23 files changed, 1204 insertions(+), 536 deletions(-) create mode 100644 hud/agents/claude/cli/__init__.py create mode 100644 hud/agents/claude/cli/agent.py rename hud/agents/claude/{sdk => cli}/computer_mcp.py (98%) delete mode 100644 hud/agents/claude/sdk/__init__.py delete mode 100644 hud/agents/claude/sdk/agent.py create mode 100644 hud/agents/registry.py rename hud/agents/tests/{test_claude_sdk_agent.py => test_claude_cli_agent.py} (61%) diff --git a/docs/v6/cookbooks/coding-agent.mdx b/docs/v6/cookbooks/coding-agent.mdx index 17c9f0c91..04bb3736a 100644 --- a/docs/v6/cookbooks/coding-agent.mdx +++ b/docs/v6/cookbooks/coding-agent.mdx @@ -24,7 +24,7 @@ CHECKS = Path("checks").resolve() # grader-only, outside the workspace TEST = "from calc import add\n\ndef test_add():\n assert add(2, 3) == 5\n" env = Environment(name="coder") -env.workspace(ROOT) +env.workspace(ROOT, network=True) @env.initialize async def _seed(): @@ -60,16 +60,24 @@ Point a coding agent at the environment. `claude` opens the `ssh` capability, ed hud eval env.py claude ``` -For Claude Code (the `claude` CLI driving the shell over SSH), use the `ClaudeSDKAgent` in code: +To run the `claude` CLI over SSH, select the `claude_cli` agent: + +```bash +hud eval env.py claude_cli --gateway +``` + +The `claude` executable must already be installed in the host or environment image. Pinning it in +the image keeps runs reproducible; `ClaudeCLIAgent` does not install or update it. The equivalent +Python API is: ```python run.py import asyncio -from hud.agents import ClaudeSDKAgent -from hud.agents.types import ClaudeSDKConfig +from hud.agents import ClaudeCLIAgent +from hud.agents.types import ClaudeCLIConfig from env import fix_add async def main(): - agent = ClaudeSDKAgent(ClaudeSDKConfig(model="claude-sonnet-4-5")) + agent = ClaudeCLIAgent(ClaudeCLIConfig(model="claude-sonnet-5")) job = await fix_add().run(agent) print("reward:", job.reward) diff --git a/docs/v6/reference/agents.mdx b/docs/v6/reference/agents.mdx index 217b03158..de18580d9 100644 --- a/docs/v6/reference/agents.mdx +++ b/docs/v6/reference/agents.mdx @@ -59,10 +59,10 @@ agent = ClaudeAgent(ClaudeConfig(model="claude-sonnet-4-5", max_steps=30)) | `OpenAIAgent` | `OpenAIConfig` | `gpt-5.6` | | `GeminiAgent` | `GeminiConfig` | `gemini-3-pro-preview` | | `OpenAIChatAgent` | `OpenAIChatConfig` | `gpt-5.4-mini` | -| `ClaudeSDKAgent` | `ClaudeSDKConfig` | `claude-sonnet-4-6` | +| `ClaudeCLIAgent` | `ClaudeCLIConfig` | `claude-sonnet-5` | Each config lives in `hud.agents.types`. `OpenAIChatAgent` speaks the OpenAI Chat Completions API, so it -points at any compatible server (vLLM, a local model) via `base_url`; `ClaudeSDKAgent` runs the `claude` +points at any compatible server (vLLM, a local model) via `base_url`; `ClaudeCLIAgent` runs the `claude` CLI over an `ssh` capability, against the env's filesystem. SSH-only runs support POSIX and Windows workspaces; computer use over an `rfb` capability currently requires a POSIX workspace. Every knob (`model`, `max_steps`, `timeout_seconds`, `tool_timeout_seconds`, `system_prompt`, `citations_enabled`, `stop_on`) lives on the @@ -100,8 +100,8 @@ A model id maps to one of four gateway agent types (`AgentType`), each a provide | `gemini` | `GeminiAgent` | | `openai_compatible` | `OpenAIChatAgent` | -For a provider key instead of the gateway, or for `ClaudeSDKAgent` (not a gateway type), construct the -provider agent directly. +For a provider key instead of the gateway, or for `ClaudeCLIAgent` (not a gateway shortcut), construct +the agent directly. ## Agent diff --git a/docs/v6/reference/runtime.mdx b/docs/v6/reference/runtime.mdx index eb225d69f..2e7e259fb 100644 --- a/docs/v6/reference/runtime.mdx +++ b/docs/v6/reference/runtime.mdx @@ -226,9 +226,9 @@ provisioning. Where `HUDRuntime` runs the agent loop locally against a tunneled env, `HostedRuntime` runs the **whole rollout** remotely, with the agent running alongside the task environment. This process -only submits the rollout and polls its trace to completion. It supports gateway agents from -[`create_agent`](/v6/reference/agents#create-agent); agents with a custom `model_client` must use -`HUDRuntime` or `LocalRuntime`. +only submits the rollout and polls its trace to completion. It supports registered built-in agents; +[`create_agent`](/v6/reference/agents#create-agent) supplies the gateway-backed ones. Agents with a +custom `model_client` must use `HUDRuntime` or `LocalRuntime`. ### `Runtime` diff --git a/hud/agents/__init__.py b/hud/agents/__init__.py index 7bcaa56bb..b071ee0d0 100644 --- a/hud/agents/__init__.py +++ b/hud/agents/__init__.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, cast +from hud.agents.registry import dump_agent, load_agent from hud.settings import settings from hud.types import AgentType from hud.utils.exceptions import HudAuthenticationError @@ -19,7 +20,7 @@ if TYPE_CHECKING: from typing import TypeAlias - from hud.agents.claude import ClaudeAgent, ClaudeSDKAgent, ClaudeSDKConfig + from hud.agents.claude import ClaudeAgent, ClaudeCLIAgent, ClaudeCLIConfig from hud.agents.gemini import GeminiAgent from hud.agents.openai import OpenAIAgent from hud.agents.openai_compatible import OpenAIChatAgent @@ -51,7 +52,10 @@ def create_agent(model: str, **kwargs: Any) -> GatewayAgent: requested_model = model model = normalize_gateway_model_id(model) - agent_type = next((candidate for candidate in AgentType if candidate.value == model), None) + agent_type = next( + (candidate for candidate in AgentType if not candidate.is_cli and candidate.value == model), + None, + ) if agent_type is not None: model_id = model else: @@ -84,12 +88,14 @@ def create_agent(model: str, **kwargs: Any) -> GatewayAgent: agent_type = AgentType(agent_str) except ValueError as exc: raise ValueError(f"Model '{model}' has invalid agent type metadata") from exc + if agent_type.is_cli: + raise ValueError(f"Model '{model}' has invalid agent type metadata") model_id = gateway_model.model_name or model break else: import difflib - known = [c.value for c in AgentType] + [ + known = [c.value for c in AgentType if not c.is_cli] + [ n for gm in gateway_models for n in (gm.id, gm.name, gm.model_name) @@ -110,15 +116,14 @@ def create_agent(model: str, **kwargs: Any) -> GatewayAgent: raise ValueError(f"Model {requested_model!r} not found in {source}.{hint}") kwargs.setdefault("model", model_id) - # cls/config_cls are matched unions; the pairing is correct by construction. config = agent_type.config_cls(**kwargs) - return agent_type.cls(cast("Any", config)) + return cast("GatewayAgent", agent_type.cls(cast("Any", config))) _LAZY_EXPORTS = { "ClaudeAgent": ("hud.agents.claude", "ClaudeAgent"), - "ClaudeSDKAgent": ("hud.agents.claude", "ClaudeSDKAgent"), - "ClaudeSDKConfig": ("hud.agents.claude", "ClaudeSDKConfig"), + "ClaudeCLIAgent": ("hud.agents.claude", "ClaudeCLIAgent"), + "ClaudeCLIConfig": ("hud.agents.claude", "ClaudeCLIConfig"), "GeminiAgent": ("hud.agents.gemini", "GeminiAgent"), "MCPAgent": ("hud.agents.tool_agent", "ToolAgent"), "OpenAIAgent": ("hud.agents.openai", "OpenAIAgent"), @@ -127,13 +132,15 @@ def create_agent(model: str, **kwargs: Any) -> GatewayAgent: __all__ = [ "ClaudeAgent", - "ClaudeSDKAgent", - "ClaudeSDKConfig", + "ClaudeCLIAgent", + "ClaudeCLIConfig", "GeminiAgent", "MCPAgent", "OpenAIAgent", "OpenAIChatAgent", "create_agent", + "dump_agent", + "load_agent", ] diff --git a/hud/agents/claude/__init__.py b/hud/agents/claude/__init__.py index f5c727565..26e7aae08 100644 --- a/hud/agents/claude/__init__.py +++ b/hud/agents/claude/__init__.py @@ -7,15 +7,15 @@ AsyncAnthropicBedrock, ClaudeAgent, ) -from .sdk import ClaudeSDKAgent, ClaudeSDKConfig +from .cli import ClaudeCLIAgent, ClaudeCLIConfig from .tools import ClaudeToolSearchTool, ClaudeWebFetchTool, ClaudeWebSearchTool __all__ = [ "AsyncAnthropic", "AsyncAnthropicBedrock", "ClaudeAgent", - "ClaudeSDKAgent", - "ClaudeSDKConfig", + "ClaudeCLIAgent", + "ClaudeCLIConfig", "ClaudeToolSearchTool", "ClaudeWebFetchTool", "ClaudeWebSearchTool", diff --git a/hud/agents/claude/agent.py b/hud/agents/claude/agent.py index 4294aa270..7aad09954 100644 --- a/hud/agents/claude/agent.py +++ b/hud/agents/claude/agent.py @@ -257,26 +257,36 @@ async def get_response( if response is None: raise ValueError("Claude response missing after retries") - result = AgentStep(content="", done=True) - result.model = response.model - result.usage = Usage( - prompt_tokens=response.usage.input_tokens, - completion_tokens=response.usage.output_tokens, - cached_tokens=response.usage.cache_read_input_tokens, + return self._message_to_agent_step(response, citations_enabled=citations_enabled) + + @classmethod + def _message_to_agent_step( + cls, + response: BetaMessage, + *, + citations_enabled: bool = False, + ) -> AgentStep: + result = AgentStep( + content="", + done=True, + model=response.model, + usage=Usage( + prompt_tokens=response.usage.input_tokens, + completion_tokens=response.usage.output_tokens, + cached_tokens=response.usage.cache_read_input_tokens, + ), ) text_parts: list[str] = [] thinking_parts: list[str] = [] - citations: list[Citation] = [] for block in response.content: match block.type: case "tool_use": - arguments = dict(block.input) if block.input else {} result.tool_calls.append( MCPToolCall( id=block.id, name=block.name, - arguments=arguments, + arguments=dict(block.input) if block.input else {}, _meta=mcp_types.RequestParams.Meta.model_validate( {"citations_enabled": citations_enabled}, ), @@ -284,9 +294,8 @@ async def get_response( ) result.done = False case "text": - text_block = block - text_parts.append(text_block.text) - citations.extend(self._citation(c) for c in (text_block.citations or [])) + text_parts.append(block.text) + result.citations.extend(cls._citation(c) for c in (block.citations or [])) case "thinking": if block.thinking: thinking_parts.append(block.thinking) @@ -294,7 +303,6 @@ async def get_response( pass result.content = "".join(text_parts) - result.citations = citations if thinking_parts: result.reasoning = "\n".join(thinking_parts) result.finish_reason = response.stop_reason diff --git a/hud/agents/claude/cli/__init__.py b/hud/agents/claude/cli/__init__.py new file mode 100644 index 000000000..9eadbc752 --- /dev/null +++ b/hud/agents/claude/cli/__init__.py @@ -0,0 +1,5 @@ +"""Agent that runs the ``claude`` CLI over SSH.""" + +from .agent import ClaudeCLIAgent, ClaudeCLIConfig + +__all__ = ["ClaudeCLIAgent", "ClaudeCLIConfig"] diff --git a/hud/agents/claude/cli/agent.py b/hud/agents/claude/cli/agent.py new file mode 100644 index 000000000..386ceb04e --- /dev/null +++ b/hud/agents/claude/cli/agent.py @@ -0,0 +1,549 @@ +"""ClaudeCLIAgent — runs ``claude`` CLI over SSH inside the env workspace. + +SSH-execs the ``claude`` CLI on the remote workspace so all built-in tools +(Bash, Read, Write, Edit, Glob, Grep) operate on the env's filesystem. +MCP capabilities from the manifest are written as MCP server config so the +CLI can call env-hosted MCP tools too. + +Inspired by harbor-framework/harbor's ClaudeCode agent. +""" + +from __future__ import annotations + +import asyncio +import base64 +import contextlib +import json +import logging +import shlex +from contextlib import AsyncExitStack +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, cast + +import asyncssh +import mcp.types as mcp_types +from anthropic.types.beta import BetaMessage + +from hud.agents.base import Agent +from hud.agents.claude.agent import ClaudeAgent +from hud.agents.types import ClaudeCLIConfig, ToolStep +from hud.settings import settings +from hud.telemetry.context import get_current_trace_id +from hud.types import MCPToolCall, MCPToolResult, Step +from hud.utils.time import now_iso + +if TYPE_CHECKING: + from hud.capabilities import SSHClient + from hud.eval.run import Run + +logger = logging.getLogger(__name__) + +WINDOWS_SHELLS = ("cmd", "powershell") +_PROMPT_PATH = ".hud_prompt.txt" +_MCP_CONFIG_PATH = ".hud_mcp_config.json" +_RUN_SCRIPT_PATH = ".hud_run.bat" +_PROCESS_CLOSE_TIMEOUT_S = 5.0 + + +@dataclass(slots=True) +class RemoteInvocation: + """How to run an assembled CLI command on the remote workspace shell. + + ``command`` is what gets exec'd over SSH. When ``script_name`` is set, that + file must be written (with ``script_body``) before exec'ing ``command``. + """ + + command: str + script_name: str | None = None + script_body: str | None = None + + +@dataclass(slots=True) +class _PendingToolCall: + call: MCPToolCall + started_at: str + + +class _ClaudeStreamParser: + """Translate Claude CLI stream messages into canonical HUD steps.""" + + def __init__(self, run: Run, *, started_at: str) -> None: + self._run = run + self._agent_started_at = started_at + self._pending_calls: dict[str, _PendingToolCall] = {} + self._last_agent_content = "" + self._message_count = 0 + self._saw_result = False + self._error_recorded = False + + @property + def message_count(self) -> int: + return self._message_count + + def feed_line(self, line: str) -> None: + line = line.strip() + if not line: + return + try: + raw = json.loads(line) + except json.JSONDecodeError: + logger.warning("Ignoring non-JSON Claude stream output") + return + if not isinstance(raw, dict): + logger.warning("Ignoring non-object Claude stream message") + return + + message = cast("dict[str, Any]", raw) + self._message_count += 1 + received_at = now_iso() + match message.get("type"): + case "system" if message.get("subtype") == "init": + self._agent_started_at = received_at + case "assistant": + self._record_assistant(message, received_at) + case "user": + self._record_tool_results(message, received_at) + case "result": + self._record_result(message, received_at) + + def finish(self, *, returncode: int, stderr: str) -> None: + trace = self._run.trace + if returncode != 0: + trace.extra["returncode"] = returncode + if stderr and (returncode != 0 or trace.status == "error"): + trace.extra["stderr"] = stderr + if not trace.content and self._last_agent_content: + trace.content = self._last_agent_content + + if returncode != 0: + trace.status = "error" + self._record_error(stderr.strip() or f"claude CLI exited with return code {returncode}") + elif not self._saw_result: + trace.status = "error" + self._record_error("claude CLI exited without a result event") + elif self._pending_calls: + trace.status = "error" + missing = ", ".join(sorted(self._pending_calls)) + self._record_error(f"claude CLI exited without results for tool calls: {missing}") + + def _record_assistant(self, event: dict[str, Any], received_at: str) -> None: + raw_message = event.get("message") + if not isinstance(raw_message, dict): + raise ValueError("Claude assistant event is missing its message payload") + message = BetaMessage.model_validate(raw_message) + step = ClaudeAgent._message_to_agent_step(message) + step.started_at = self._agent_started_at + step.ended_at = received_at + step.extra = _event_metadata(event, raw_message) + if step.content: + self._last_agent_content = step.content + self._run.record(step) + for call in step.tool_calls: + self._pending_calls[call.id] = _PendingToolCall(call=call, started_at=received_at) + + def _record_tool_results(self, event: dict[str, Any], received_at: str) -> None: + message = event.get("message") + if not isinstance(message, dict): + return + content = message.get("content") + if not isinstance(content, list): + return + + saw_result = False + for raw_block in content: + if not isinstance(raw_block, dict) or raw_block.get("type") != "tool_result": + continue + block = cast("dict[str, Any]", raw_block) + call_id = block.get("tool_use_id") + if not isinstance(call_id, str): + continue + pending = self._pending_calls.pop(call_id, None) + if pending is None: + logger.warning("Claude returned a result for unknown tool call %s", call_id) + continue + saw_result = True + self._run.record( + ToolStep( + call=pending.call, + result=MCPToolResult( + call_id=call_id, + content=_tool_result_content(block.get("content")), + isError=block.get("is_error") is True, + ), + started_at=pending.started_at, + ended_at=received_at, + extra=_event_metadata(event, message), + ) + ) + if saw_result: + self._agent_started_at = received_at + + def _record_result(self, event: dict[str, Any], received_at: str) -> None: + self._saw_result = True + trace = self._run.trace + result = event.get("result") + trace.content = result if isinstance(result, str) else self._last_agent_content + is_error = event.get("is_error") is True + trace.status = "error" if is_error else "completed" + for key in ( + "subtype", + "session_id", + "duration_ms", + "duration_api_ms", + "stop_reason", + "num_turns", + "total_cost_usd", + ): + value = event.get(key) + if value is not None: + trace.extra[key] = value + if is_error: + self._record_error(trace.content or "claude CLI reported an error", received_at) + + def _record_error(self, error: str, at: str | None = None) -> None: + if self._error_recorded: + return + timestamp = at or now_iso() + self._run.record( + Step(source="system", error=error, started_at=timestamp, ended_at=timestamp) + ) + self._error_recorded = True + + +def _tool_result_content(value: Any) -> list[mcp_types.ContentBlock]: + values = value if isinstance(value, list) else [value] + content: list[mcp_types.ContentBlock] = [] + for item in values: + if isinstance(item, str): + content.append(mcp_types.TextContent(type="text", text=item)) + elif ( + isinstance(item, dict) + and item.get("type") == "text" + and isinstance(item.get("text"), str) + ): + content.append(mcp_types.TextContent(type="text", text=item["text"])) + elif isinstance(item, dict) and item.get("type") == "image": + source = item.get("source") + if ( + isinstance(source, dict) + and source.get("type") == "base64" + and isinstance(source.get("data"), str) + and isinstance(source.get("media_type"), str) + ): + content.append( + mcp_types.ImageContent( + type="image", + data=source["data"], + mimeType=source["media_type"], + ) + ) + continue + content.append( + mcp_types.TextContent( + type="text", + text=json.dumps(item, ensure_ascii=False, separators=(",", ":")), + ) + ) + elif item is not None: + content.append( + mcp_types.TextContent( + type="text", + text=json.dumps(item, ensure_ascii=False, separators=(",", ":")), + ) + ) + return content + + +def _event_metadata(event: dict[str, Any], message: dict[str, Any]) -> dict[str, Any]: + metadata: dict[str, Any] = {} + for key in ("session_id", "uuid", "parent_tool_use_id"): + value = event.get(key) + if value is not None: + metadata[key] = value + message_id = message.get("id") + if message_id is not None: + metadata["message_id"] = message_id + return metadata + + +def _powershell_quote(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +def _powershell(script: str) -> str: + encoded = base64.b64encode(script.encode("utf-16-le")).decode("ascii") + return f"powershell -NoProfile -NonInteractive -EncodedCommand {encoded}" + + +def build_remote_invocation(shell: str, run_cmd: str) -> RemoteInvocation: + """Build the remote exec command for ``run_cmd`` under the given login shell. + + Windows shells can't take the assembled command inline — ``cmd.exe`` mangles + the quotes — so it is written to a batch file and invoked through ``cmd /c``. + A bare ``.hud_run.bat`` is rejected as an unknown command, and silently fails + to run under a PowerShell default shell, so ``cmd /c`` is required for both. + POSIX shells take the command inline. + """ + if shell in WINDOWS_SHELLS: + return RemoteInvocation( + command=f"cmd /c {_RUN_SCRIPT_PATH}", + script_name=_RUN_SCRIPT_PATH, + script_body=f"@echo off\r\n{run_cmd}\r\n", + ) + return RemoteInvocation(command=run_cmd) + + +class ClaudeCLIAgent(Agent): + """Runs ``claude`` CLI over SSH inside the env workspace. + + Stateless w.r.t. the env: driven by ``await agent(run)``. SSH is opened + live off the run. Environment MCP bindings are used directly; computer MCP + servers are bridged over the run's SSH connection. + """ + + config: ClaudeCLIConfig + + def __init__(self, config: ClaudeCLIConfig | None = None) -> None: + self.config = config or ClaudeCLIConfig() + + async def __call__(self, run: Run) -> None: + mcp_servers: dict[str, dict[str, Any]] = {} + manifest = run.client.manifest + bindings = manifest.bindings if manifest is not None else [] + families = {c.protocol.split("/", 1)[0] for c in bindings} + + if "ssh" not in families: + raise RuntimeError("ClaudeCLIAgent requires an SSH capability") + ssh = cast("SSHClient", await run.client.open("ssh")) + shell = ssh.capability.params.get("shell", "bash") + + rfb_bindings = [cap for cap in bindings if cap.protocol.split("/", 1)[0] == "rfb"] + async with AsyncExitStack() as resources: + for cap in bindings: + family = cap.protocol.split("/", 1)[0] + if family == "mcp": + token = cap.params.get("auth_token") + transport = "http" if cap.params["transport"] == "streamable-http" else "sse" + server_config: dict[str, Any] = {"type": transport, "url": cap.url} + if token: + server_config["headers"] = {"Authorization": f"Bearer {token}"} + if cap.name in mcp_servers: + raise RuntimeError(f"duplicate MCP server name {cap.name!r}") + mcp_servers[cap.name] = server_config + elif family == "rfb": + from hud.agents.claude.cli.computer_mcp import bridge_computer_mcp + + server_name = ( + "computer-use" if len(rfb_bindings) == 1 else f"computer-use-{cap.name}" + ) + if server_name in mcp_servers: + raise RuntimeError(f"duplicate MCP server name {server_name!r}") + routed = run.client.binding(cap.name) + mcp_servers[server_name] = await resources.enter_async_context( + bridge_computer_mcp( + ssh, + routed, + self.config.screenshot_encoding, + shell=shell, + ) + ) + + await self._exec( + run, + ssh=ssh, + shell=shell, + mcp_servers=mcp_servers, + prompt=run.prompt_text, + max_steps=self.config.max_steps, + system_prompt=self.config.system_prompt, + ) + + async def _exec( + self, + run: Run, + *, + ssh: SSHClient, + shell: str, + mcp_servers: dict[str, dict[str, Any]], + prompt: str, + max_steps: int = -1, + system_prompt: str | None = None, + ) -> None: + runtime_files: list[str] = [] + try: + mcp_config_path = await self._write_mcp_config(ssh, mcp_servers) + if mcp_config_path is not None: + runtime_files.append(mcp_config_path) + if shell in WINDOWS_SHELLS: + await ssh.write_text(_PROMPT_PATH, prompt) + runtime_files.append(_PROMPT_PATH) + + run_cmd = self._build_cli_command( + shell=shell, + prompt=prompt, + max_steps=max_steps, + system_prompt=system_prompt, + mcp_config_path=mcp_config_path, + ) + invocation = build_remote_invocation(shell, run_cmd) + if invocation.script_name is not None: + assert invocation.script_body is not None + await ssh.write_text(invocation.script_name, invocation.script_body) + runtime_files.append(invocation.script_name) + + logger.info("SSH exec claude CLI (%d chars)", len(invocation.command)) + await self._stream_cli(run, ssh, invocation.command) + finally: + await self._remove_runtime_files(ssh, shell, runtime_files) + + async def _stream_cli(self, run: Run, ssh: SSHClient, command: str) -> None: + parser = _ClaudeStreamParser(run, started_at=now_iso()) + process = await ssh.create_process(command) + stderr_task = asyncio.create_task(process.stderr.read()) + try: + while line := await process.stdout.readline(): + parser.feed_line(line.decode(errors="replace")) + await process.wait_closed() + stderr_output = await stderr_task + except BaseException: + try: + process.terminate() + except (OSError, asyncssh.Error): + process.close() + stderr_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await stderr_task + try: + async with asyncio.timeout(_PROCESS_CLOSE_TIMEOUT_S): + await process.wait_closed() + except (OSError, TimeoutError, asyncssh.Error): + process.close() + with contextlib.suppress(OSError, TimeoutError, asyncssh.Error): + async with asyncio.timeout(_PROCESS_CLOSE_TIMEOUT_S): + await process.wait_closed() + raise + + stderr = stderr_output.decode(errors="replace") + returncode = process.returncode + if returncode is None: + raise RuntimeError("claude CLI process closed without an exit status") + logger.info( + "exit=%s events=%d stderr=%d", + returncode, + parser.message_count, + len(stderr), + ) + parser.finish(returncode=returncode, stderr=stderr) + + async def _remove_runtime_files( + self, + ssh: SSHClient, + shell: str, + paths: list[str], + ) -> None: + if not paths: + return + if shell in WINDOWS_SHELLS: + command = f"cmd /c del /f /q {' '.join(paths)} 2>nul" + else: + command = "rm -f -- " + " ".join(shlex.quote(path) for path in paths) + try: + await ssh.run(command, check=False) + except (OSError, asyncssh.Error): + logger.warning("Failed to remove Claude CLI runtime files") + + def _build_env_vars(self) -> dict[str, str]: + env: dict[str, str] = {} + + use_hud_gateway = self.config.use_hud_gateway + if use_hud_gateway is None: + use_hud_gateway = settings.api_key is not None + + if use_hud_gateway: + if not settings.api_key: + raise ValueError("HUD_API_KEY is required for HUD gateway routing") + env["ANTHROPIC_BASE_URL"] = settings.hud_gateway_url + env["ANTHROPIC_API_KEY"] = settings.api_key + if trace_id := get_current_trace_id(): + env["ANTHROPIC_CUSTOM_HEADERS"] = f"Trace-Id: {trace_id}" + elif settings.anthropic_api_key: + env["ANTHROPIC_API_KEY"] = settings.anthropic_api_key + + env["ANTHROPIC_MODEL"] = self.config.model + env["ANTHROPIC_SMALL_FAST_MODEL"] = self.config.model + + # When using a custom base URL, alias all model tiers to the same model + # so the CLI doesn't try to reach Anthropic for background requests. + if "ANTHROPIC_BASE_URL" in env: + env["ANTHROPIC_DEFAULT_SONNET_MODEL"] = self.config.model + env["ANTHROPIC_DEFAULT_OPUS_MODEL"] = self.config.model + env["ANTHROPIC_DEFAULT_HAIKU_MODEL"] = self.config.model + env["CLAUDE_CODE_SUBAGENT_MODEL"] = self.config.model + + env["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC"] = "1" + env["IS_SANDBOX"] = "1" + + return env + + async def _write_mcp_config( + self, + ssh: SSHClient, + mcp_servers: dict[str, dict[str, Any]], + ) -> str | None: + """Write MCP config into the workspace and return its path.""" + if not mcp_servers: + return None + mcp_json = json.dumps({"mcpServers": mcp_servers}, indent=2) + path = _MCP_CONFIG_PATH + await ssh.write_text(path, mcp_json) + logger.info("Wrote MCP config") + return path + + def _build_cli_command( + self, + *, + shell: str, + prompt: str, + max_steps: int, + system_prompt: str | None, + mcp_config_path: str | None = None, + ) -> str: + env_vars = self._build_env_vars() + is_win = shell in WINDOWS_SHELLS + + base_args: list[str] = [ + "claude", + "--verbose", + "--output-format=stream-json", + "--print", + f"--permission-mode={self.config.permission_mode}", + ] + if max_steps > 0: + base_args.append(f"--max-turns={max_steps}") + if system_prompt: + base_args.extend(["--system-prompt", system_prompt]) + for tool in self.config.allowed_tools: + base_args.extend(["--allowedTools", tool]) + if mcp_config_path: + base_args.extend(["--mcp-config", mcp_config_path]) + + if is_win: + script = ";".join( + [ + *(f"$env:{key}={_powershell_quote(value)}" for key, value in env_vars.items()), + f"Get-Content -Raw -Encoding UTF8 {_powershell_quote(_PROMPT_PATH)}" + f" | & claude {' '.join(_powershell_quote(arg) for arg in base_args[1:])}", + "exit $LASTEXITCODE", + ] + ) + return _powershell(script) + + # POSIX path: shell-quote everything and embed prompt inline. + cli_parts = [shlex.quote(a) for a in base_args] + cli_parts.extend(["--", shlex.quote(prompt)]) + cli_cmd = " ".join(cli_parts) + env_prefix = " ".join(f"{k}={shlex.quote(v)}" for k, v in env_vars.items()) + return f'export PATH="$HOME/.local/bin:$PATH"; {env_prefix} {cli_cmd}' + + +__all__ = ["ClaudeCLIAgent", "ClaudeCLIConfig", "RemoteInvocation", "build_remote_invocation"] diff --git a/hud/agents/claude/sdk/computer_mcp.py b/hud/agents/claude/cli/computer_mcp.py similarity index 98% rename from hud/agents/claude/sdk/computer_mcp.py rename to hud/agents/claude/cli/computer_mcp.py index 84d0cc8a0..0b4ef38ac 100644 --- a/hud/agents/claude/sdk/computer_mcp.py +++ b/hud/agents/claude/cli/computer_mcp.py @@ -168,7 +168,7 @@ async def bridge_computer_mcp( ) -> AsyncIterator[dict[str, Any]]: """Bridge a controller-side computer MCP process into a remote POSIX shell.""" if shell in {"cmd", "powershell"}: - raise RuntimeError("ClaudeSDKAgent computer use requires a POSIX workspace") + raise RuntimeError("ClaudeCLIAgent computer use requires a POSIX workspace") token = secrets.token_hex(16) request_path = str(_REMOTE_TMP / f"hud-computer-{token}.request") @@ -190,7 +190,7 @@ async def bridge_computer_mcp( local = await asyncio.create_subprocess_exec( sys.executable, "-m", - "hud.agents.claude.sdk.computer_mcp", + "hud.agents.claude.cli.computer_mcp", stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, diff --git a/hud/agents/claude/sdk/__init__.py b/hud/agents/claude/sdk/__init__.py deleted file mode 100644 index 57fd2773c..000000000 --- a/hud/agents/claude/sdk/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Claude Agent SDK agent.""" - -from .agent import ClaudeSDKAgent, ClaudeSDKConfig - -__all__ = ["ClaudeSDKAgent", "ClaudeSDKConfig"] diff --git a/hud/agents/claude/sdk/agent.py b/hud/agents/claude/sdk/agent.py deleted file mode 100644 index aa62deccb..000000000 --- a/hud/agents/claude/sdk/agent.py +++ /dev/null @@ -1,342 +0,0 @@ -"""ClaudeSDKAgent — runs ``claude`` CLI over SSH inside the env workspace. - -SSH-execs the ``claude`` CLI on the remote workspace so all built-in tools -(Bash, Read, Write, Edit, Glob, Grep) operate on the env's filesystem. -MCP capabilities from the manifest are written as MCP server config so the -CLI can call env-hosted MCP tools too. - -Inspired by harbor-framework/harbor's ClaudeCode agent. -""" - -from __future__ import annotations - -import json -import logging -import shlex -from contextlib import AsyncExitStack -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, cast - -from hud.agents.base import Agent -from hud.agents.types import AgentStep, ClaudeSDKConfig, Usage -from hud.settings import settings -from hud.types import Step - -if TYPE_CHECKING: - from hud.capabilities import SSHClient - from hud.eval.run import Run - -logger = logging.getLogger(__name__) - -WINDOWS_SHELLS = ("cmd", "powershell") -#: Bare ``claude`` install bootstrap for POSIX shells (no-op when already present). -_POSIX_INSTALL_CHECK = ( - "command -v claude >/dev/null 2>&1 || " - "{ curl -fsSL https://claude.ai/install.sh | bash -s -- 2>/dev/null; " - 'export PATH="$HOME/.local/bin:$PATH"; }' -) - - -@dataclass(slots=True) -class RemoteInvocation: - """How to run an assembled CLI command on the remote workspace shell. - - ``command`` is what gets exec'd over SSH. When ``script_name`` is set, that - file must be written (with ``script_body``) before exec'ing ``command``. - """ - - command: str - script_name: str | None = None - script_body: str | None = None - - -def build_remote_invocation(shell: str, run_cmd: str) -> RemoteInvocation: - """Build the remote exec command for ``run_cmd`` under the given login shell. - - Windows shells can't take the assembled command inline — ``cmd.exe`` mangles - the quotes — so it is written to a batch file and invoked through ``cmd /c``. - A bare ``.hud_run.bat`` is rejected as an unknown command, and silently fails - to run under a PowerShell default shell, so ``cmd /c`` is required for both. - POSIX shells take the command inline, prefixed with a one-shot install check. - """ - if shell in WINDOWS_SHELLS: - return RemoteInvocation( - command="cmd /c .hud_run.bat", - script_name=".hud_run.bat", - script_body=f"@echo off\r\n{run_cmd}\r\n", - ) - return RemoteInvocation(command=f"{_POSIX_INSTALL_CHECK} && {run_cmd}") - - -class ClaudeSDKAgent(Agent): - """Runs ``claude`` CLI over SSH inside the env workspace. - - Stateless w.r.t. the env: driven by ``await agent(run)``. SSH is opened - live off the run. Environment MCP bindings are used directly; computer MCP - servers are bridged over the run's SSH connection. - """ - - config: ClaudeSDKConfig - - def __init__(self, config: ClaudeSDKConfig | None = None) -> None: - self.config = config or ClaudeSDKConfig() - - async def __call__(self, run: Run) -> None: - mcp_servers: dict[str, dict[str, Any]] = {} - manifest = run.client.manifest - bindings = manifest.bindings if manifest is not None else [] - families = {c.protocol.split("/", 1)[0] for c in bindings} - - if "ssh" not in families: - raise RuntimeError("ClaudeSDKAgent requires an SSH capability") - ssh = cast("SSHClient", await run.client.open("ssh")) - shell = ssh.capability.params.get("shell", "bash") - - rfb_bindings = [cap for cap in bindings if cap.protocol.split("/", 1)[0] == "rfb"] - async with AsyncExitStack() as resources: - for cap in bindings: - family = cap.protocol.split("/", 1)[0] - if family == "mcp": - token = cap.params.get("auth_token") - transport = "http" if cap.params["transport"] == "streamable-http" else "sse" - server_config: dict[str, Any] = {"type": transport, "url": cap.url} - if token: - server_config["headers"] = {"Authorization": f"Bearer {token}"} - if cap.name in mcp_servers: - raise RuntimeError(f"duplicate MCP server name {cap.name!r}") - mcp_servers[cap.name] = server_config - elif family == "rfb": - from hud.agents.claude.sdk.computer_mcp import bridge_computer_mcp - - server_name = ( - "computer-use" if len(rfb_bindings) == 1 else f"computer-use-{cap.name}" - ) - if server_name in mcp_servers: - raise RuntimeError(f"duplicate MCP server name {server_name!r}") - routed = run.client.binding(cap.name) - mcp_servers[server_name] = await resources.enter_async_context( - bridge_computer_mcp( - ssh, - routed, - self.config.screenshot_encoding, - shell=shell, - ) - ) - - await self._exec( - run, - ssh=ssh, - shell=shell, - mcp_servers=mcp_servers, - prompt=run.prompt_text, - max_steps=self.config.max_steps, - system_prompt=self.config.system_prompt, - ) - - async def _exec( - self, - run: Run, - *, - ssh: SSHClient, - shell: str, - mcp_servers: dict[str, dict[str, Any]], - prompt: str, - max_steps: int = -1, - system_prompt: str | None = None, - ) -> None: - mcp_config_path = await self._write_mcp_config(ssh, mcp_servers) - - await ssh.write_text(".hud_prompt.txt", prompt) - - run_cmd = self._build_cli_command( - shell=shell, - prompt=prompt, - max_steps=max_steps, - system_prompt=system_prompt, - mcp_config_path=mcp_config_path, - ) - - invocation = build_remote_invocation(shell, run_cmd) - if invocation.script_name is not None: - assert invocation.script_body is not None - # cmd.exe mangles inline quotes, so the command rides a batch file. - await ssh.write_text(invocation.script_name, invocation.script_body) - - full_cmd = invocation.command - logger.info("SSH exec claude CLI (%d chars)", len(full_cmd)) - logger.info("Full command: %s", full_cmd) - - completed = await ssh.run(full_cmd, check=False) - stdout = completed.stdout if isinstance(completed.stdout, str) else "" - stderr = completed.stderr if isinstance(completed.stderr, str) else "" - returncode = completed.returncode - - logger.info("returncode=%s stdout=%d stderr=%d", returncode, len(stdout), len(stderr)) - - if returncode != 0 and not stdout.strip(): - error = stderr or f"claude CLI exited with return code {returncode}" - run.trace.status = "error" - run.trace.extra.update({"returncode": returncode, "stderr": stderr}) - run.record(Step(source="system", error=error)) - return - - self._parse_stream_json(run, stdout, stderr) - - def _build_env_vars(self) -> dict[str, str]: - env: dict[str, str] = {} - - if settings.api_key: - env["ANTHROPIC_BASE_URL"] = settings.hud_gateway_url - env["ANTHROPIC_API_KEY"] = settings.api_key - elif settings.anthropic_api_key: - env["ANTHROPIC_API_KEY"] = settings.anthropic_api_key - - env["ANTHROPIC_MODEL"] = self.config.model - env["ANTHROPIC_SMALL_FAST_MODEL"] = self.config.model - - # When using a custom base URL, alias all model tiers to the same model - # so the CLI doesn't try to reach Anthropic for background requests. - if "ANTHROPIC_BASE_URL" in env: - env["ANTHROPIC_DEFAULT_SONNET_MODEL"] = self.config.model - env["ANTHROPIC_DEFAULT_OPUS_MODEL"] = self.config.model - env["ANTHROPIC_DEFAULT_HAIKU_MODEL"] = self.config.model - env["CLAUDE_CODE_SUBAGENT_MODEL"] = self.config.model - - env["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC"] = "1" - env["IS_SANDBOX"] = "1" - - return env - - async def _write_mcp_config( - self, - ssh: SSHClient, - mcp_servers: dict[str, dict[str, Any]], - ) -> str | None: - """Write MCP config into the workspace and return its path.""" - if not mcp_servers: - return None - mcp_json = json.dumps({"mcpServers": mcp_servers}, indent=2) - path = ".hud_mcp_config.json" - await ssh.write_text(path, mcp_json) - logger.info("Wrote MCP config") - return path - - def _build_cli_command( - self, - *, - shell: str, - prompt: str, - max_steps: int, - system_prompt: str | None, - mcp_config_path: str | None = None, - ) -> str: - env_vars = self._build_env_vars() - is_win = shell in WINDOWS_SHELLS - - # Raw args list (no shell quoting) — used directly for Windows Python launcher. - base_args: list[str] = [ - "claude", - "--verbose", - "--output-format=stream-json", - "--print", - f"--permission-mode={self.config.permission_mode}", - ] - if max_steps > 0: - base_args.append(f"--max-turns={max_steps}") - if system_prompt: - base_args.extend(["--system-prompt", system_prompt]) - for tool in self.config.allowed_tools: - base_args.extend(["--allowedTools", tool]) - if mcp_config_path: - base_args.extend(["--mcp-config", mcp_config_path]) - - if is_win: - # On Windows, two problems combine: - # 1. claude is installed as claude.cmd (Node.js wrapper) — Python's - # subprocess.run can't execute .cmd files via CreateProcess directly. - # 2. Embedding the prompt inline in the bat file breaks — cmd.exe parses - # line-by-line, so newlines inside quoted strings split the command. - # Solution: use `cmd /c claude [args]` (no inline prompt) and feed the - # prompt via stdin from .hud_prompt.txt. claude --print reads stdin as - # the initial message when no -- argument is provided. - set_parts = [f"set {k}={v}" for k, v in env_vars.items()] - cmd_args = ["cmd", "/c", "claude"] + base_args[1:] # noqa: RUF005 - py_args_repr = "[" + ",".join(f"'{a}'" for a in cmd_args) + "]" - python_launcher = ( - 'python -c "' - "import subprocess,sys;" - f"r=subprocess.run({py_args_repr},stdin=open('.hud_prompt.txt','rb'));" - 'sys.exit(r.returncode)"' - ) - return " && ".join([*set_parts, python_launcher]) - - # POSIX path: shell-quote everything and embed prompt inline. - cli_parts = [shlex.quote(a) for a in base_args] - cli_parts.extend(["--", shlex.quote(prompt)]) - cli_cmd = " ".join(cli_parts) - env_prefix = " ".join(f"{k}={shlex.quote(v)}" for k, v in env_vars.items()) - return f'export PATH="$HOME/.local/bin:$PATH"; {env_prefix} {cli_cmd}' - - def _parse_stream_json(self, run: Run, stdout: str, stderr: str) -> None: - messages: list[dict[str, Any]] = [] - content_parts: list[str] = [] - is_error = False - info: dict[str, Any] = {} - cost_usd: float | None = None - num_turns: int | None = None - - for line in stdout.splitlines(): - line = line.strip() - if not line: - continue - try: - msg = json.loads(line) - except json.JSONDecodeError: - continue - - messages.append(msg) - msg_type = msg.get("type") - - if msg_type == "assistant" and isinstance(msg.get("message"), dict): - for raw_block in msg["message"].get("content", []): - if not isinstance(raw_block, dict): - continue - block = cast("dict[str, Any]", raw_block) - if block.get("type") == "text" and block.get("text"): - content_parts.append(str(block["text"])) - - elif msg_type == "result": - is_error = msg.get("is_error", False) - result_text = msg.get("result") - if result_text: - content_parts.append(result_text) - info["session_id"] = msg.get("session_id") - info["duration_ms"] = msg.get("duration_ms") - info["stop_reason"] = msg.get("stop_reason") - num_turns = msg.get("num_turns") - cost_usd = msg.get("total_cost_usd") - - content = "\n".join(content_parts) - trace = run.trace - trace.status = "error" if is_error else "completed" - trace.content = content - # Raw CLI stream kept locally; a claude-native serializer can take over - # per-turn fidelity later (the CLI session is its own span vocabulary). - trace.extra["messages"] = messages - if stderr: - trace.extra["stderr"] = stderr - - # The CLI run collapses to one coarse agent step with aggregate usage. - run.record( - AgentStep( - content=content, - done=True, - model=self.config.model, - usage=Usage(cost_usd=cost_usd, llm_call_count=num_turns), - error=content if is_error else None, - extra={k: v for k, v in info.items() if v is not None}, - ), - ) - - -__all__ = ["ClaudeSDKAgent", "ClaudeSDKConfig", "RemoteInvocation", "build_remote_invocation"] diff --git a/hud/agents/registry.py b/hud/agents/registry.py new file mode 100644 index 000000000..bf0380f2b --- /dev/null +++ b/hud/agents/registry.py @@ -0,0 +1,50 @@ +"""Serialization and reconstruction for built-in agents.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, cast + +from hud.agents.types import AgentConfig +from hud.types import AgentType + +if TYPE_CHECKING: + from hud.agents.base import Agent + +_RUNTIME_ONLY_CONFIG_FIELDS = {"model_client", "api_key", "base_url", "hosted_tools"} + + +def dump_agent(agent: Agent) -> dict[str, Any]: + """Serialize a registered agent without credentials or live clients.""" + agent_type = AgentType.of(agent) + config = getattr(agent, "config", None) + if agent_type is None or not isinstance(config, AgentConfig): + raise ValueError( + f"agent must be one of the registered types " + f"({', '.join(member.value for member in AgentType)}); " + f"got {type(agent).__name__}" + ) + if config.model_client is not None: + raise ValueError( + "agents with a custom model_client cannot run remotely; use HUDRuntime or LocalRuntime" + ) + + payload = config.model_dump( + mode="json", + exclude=_RUNTIME_ONLY_CONFIG_FIELDS, + ) + return {"type": agent_type.value, "config": payload} + + +def load_agent(data: Mapping[str, Any]) -> Agent: + """Reconstruct a registered agent from :func:`dump_agent` output.""" + try: + agent_type = AgentType(data["type"]) + except (KeyError, TypeError, ValueError): + raise ValueError(f"unsupported agent type {data.get('type')!r}") from None + + raw_config = data.get("config") + if not isinstance(raw_config, Mapping): + raise ValueError("agent config must be an object") + config = agent_type.config_cls.model_validate(dict(raw_config)) + return cast("Agent", agent_type.cls(cast("Any", config))) diff --git a/hud/agents/tests/test_base.py b/hud/agents/tests/test_base.py index 350a3c544..d6f6427f1 100644 --- a/hud/agents/tests/test_base.py +++ b/hud/agents/tests/test_base.py @@ -11,7 +11,14 @@ import pytest -from hud.agents import OpenAIAgent, OpenAIChatAgent, create_agent +from hud.agents import ( + ClaudeCLIAgent, + OpenAIAgent, + OpenAIChatAgent, + create_agent, + dump_agent, + load_agent, +) from hud.agents.base import Agent from hud.types import AgentType from hud.utils.exceptions import HudAuthenticationError @@ -47,6 +54,28 @@ def test_agent_type_maps_value_to_class_and_provider() -> None: assert isinstance(AgentType("openai").gateway_provider, str) +def test_agent_type_registers_cli_agent() -> None: + assert AgentType("claude_cli").cls is ClaudeCLIAgent + assert AgentType.of(ClaudeCLIAgent()) == AgentType.CLAUDE_CLI + + +def test_cli_agent_round_trips_through_registered_wire_format() -> None: + agent = ClaudeCLIAgent() + spec = dump_agent(agent) + + loaded = load_agent(spec) + + assert spec["type"] == "claude_cli" + assert spec["config"]["model"] == "claude-sonnet-5" + assert isinstance(loaded, ClaudeCLIAgent) + assert loaded.config == agent.config + + +def test_dump_agent_rejects_unregistered_agent() -> None: + with pytest.raises(ValueError, match="registered types"): + dump_agent(_FillingAgent()) + + def test_missing_provider_dependency_points_at_agents_extra( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/hud/agents/tests/test_claude_sdk_agent.py b/hud/agents/tests/test_claude_cli_agent.py similarity index 61% rename from hud/agents/tests/test_claude_sdk_agent.py rename to hud/agents/tests/test_claude_cli_agent.py index fd1af430d..23a8a2326 100644 --- a/hud/agents/tests/test_claude_sdk_agent.py +++ b/hud/agents/tests/test_claude_cli_agent.py @@ -1,4 +1,4 @@ -"""ClaudeSDKAgent remote-command construction over the workspace SSH. +"""ClaudeCLIAgent remote-command construction over the workspace SSH. The agent runs the ``claude`` CLI on the remote workspace. These cover how the command is assembled per login shell — especially the Windows path, where the @@ -19,17 +19,45 @@ from typing import TYPE_CHECKING, Any, Literal, cast from unittest.mock import AsyncMock, Mock +import mcp.types as mcp_types import pytest -from hud.agents.claude.sdk import computer_mcp -from hud.agents.claude.sdk.agent import ClaudeSDKAgent, build_remote_invocation -from hud.agents.types import ClaudeSDKConfig +from hud.agents.claude.cli import computer_mcp +from hud.agents.claude.cli.agent import ( + ClaudeCLIAgent, + _tool_result_content, + build_remote_invocation, +) +from hud.agents.types import AgentStep, ClaudeCLIConfig, ToolStep from hud.capabilities import Capability, SSHClient from hud.capabilities.rfb import WebPScreenshotEncoding +from hud.settings import settings +from hud.telemetry.context import set_trace_context if TYPE_CHECKING: from pathlib import Path + +@pytest.fixture(autouse=True) +def _clear_api_keys(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "api_key", None) + monkeypatch.setattr(settings, "anthropic_api_key", None) + + +def test_env_vars_follow_explicit_gateway_routing(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "api_key", "hud-key") + monkeypatch.setattr(settings, "anthropic_api_key", "anthropic-key") + + gateway = ClaudeCLIAgent(ClaudeCLIConfig(use_hud_gateway=True))._build_env_vars() + provider = ClaudeCLIAgent(ClaudeCLIConfig(use_hud_gateway=False))._build_env_vars() + + assert gateway["ANTHROPIC_BASE_URL"] == settings.hud_gateway_url + assert gateway["ANTHROPIC_API_KEY"] == "hud-key" + assert provider["ANTHROPIC_API_KEY"] == "anthropic-key" + assert "ANTHROPIC_BASE_URL" not in provider + assert provider["ANTHROPIC_MODEL"] == "claude-sonnet-5" + + # ─── build_remote_invocation (pure) ─────────────────────────────────── @@ -43,37 +71,148 @@ def test_windows_shell_runs_batch_file_via_cmd(shell: str) -> None: assert inv.script_body == "@echo off\r\nclaude --print -- hi\r\n" -def test_posix_shell_runs_inline_with_install_check() -> None: +def test_posix_shell_runs_inline() -> None: inv = build_remote_invocation("bash", "claude --print -- hi") assert inv.script_name is None assert inv.script_body is None - assert "install.sh" in inv.command # one-shot bootstrap prefix - assert inv.command.endswith(" && claude --print -- hi") + assert inv.command == "claude --print -- hi" + + +def test_windows_command_encodes_environment_and_arguments( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(settings, "api_key", "hud&key's") + agent = ClaudeCLIAgent(ClaudeCLIConfig(use_hud_gateway=True)) + + command = agent._build_cli_command( + shell="powershell", + prompt="not embedded", + max_steps=3, + system_prompt="don't $expand", + ) + + encoded = command.rsplit(" ", 1)[1] + script = base64.b64decode(encoded).decode("utf-16-le") + assert "$env:ANTHROPIC_API_KEY='hud&key''s'" in script + assert "'--system-prompt' 'don''t $expand'" in script + assert "Get-Content -Raw -Encoding UTF8 '.hud_prompt.txt' | & claude" in script + assert "not embedded" not in script + assert "python" not in script + + +def test_tool_result_content_preserves_images() -> None: + content = _tool_result_content( + [ + {"type": "text", "text": "before"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "aW1hZ2U=", + }, + }, + ] + ) + + assert content[0].type == "text" + assert isinstance(content[1], mcp_types.ImageContent) + assert content[1].mimeType == "image/png" + assert content[1].data == "aW1hZ2U=" # ─── _exec end-to-end over a fake SSH workspace ──────────────────────── +class _FakeReader: + def __init__(self, value: str, *, pause_after: int | None = None) -> None: + self._raw = value.encode() + self._lines = self._raw.splitlines(keepends=True) + self._pause_after = pause_after + self._index = 0 + self.blocked = asyncio.Event() + self.release = asyncio.Event() + + async def readline(self) -> bytes: + if self._pause_after == self._index: + self.blocked.set() + await self.release.wait() + self._pause_after = None + if self._index == len(self._lines): + return b"" + line = self._lines[self._index] + self._index += 1 + return line + + async def read(self) -> bytes: + return self._raw + + +class _FakeStreamProcess: + def __init__( + self, + stdout: str, + *, + stderr: str = "", + exit_status: int | None = 0, + returncode: int | None = None, + pause_after: int | None = None, + ) -> None: + self.stdout = _FakeReader(stdout, pause_after=pause_after) + self.stderr = _FakeReader(stderr) + self.exit_status = exit_status + self.returncode = exit_status if returncode is None else returncode + self.closed = False + self.terminated = False + + def terminate(self) -> None: + self.terminated = True + + def close(self) -> None: + self.closed = True + + async def wait_closed(self) -> None: + pass + + +class _FakeCompletedProcess: + async def wait(self, *, check: bool, **kwargs: Any) -> Any: + del check + assert kwargs == {"timeout": None} + return SimpleNamespace(stdout=b"", stderr=b"", exit_status=0, returncode=0) + + def terminate(self) -> None: + pass + + def close(self) -> None: + pass + + async def wait_closed(self) -> None: + pass + + class _FakeConn: - def __init__(self, sink: dict[str, bytes], result: Any) -> None: + def __init__(self, sink: dict[str, bytes], process: _FakeStreamProcess) -> None: self._sink = sink - self._result = result + self._process = process self.ran: list[str] = [] self.write_commands: list[str] = [] + self.written: dict[str, bytes] = {} + self.deleted: list[str] = [] def is_closed(self) -> bool: return False - async def run( - self, - cmd: str, - *, - input: str | None = None, - check: bool = True, - encoding: str | None = "utf-8", - ) -> Any: - if input is not None or cmd.startswith("powershell "): + async def create_process(self, cmd: str, **kwargs: Any) -> Any: + input_value = kwargs.get("input") + if cmd.startswith(("rm -f -- ", "cmd /c del /f /q ")): + paths = [path for path in self._sink if path in cmd] + for path in paths: + self._sink.pop(path) + self.deleted.extend(paths) + return _FakeCompletedProcess() + if input_value is not None or cmd.startswith("powershell "): self.write_commands.append(cmd) script = cmd if match := re.search(r"-EncodedCommand (\S+)", cmd): @@ -83,37 +222,17 @@ async def run( for path in (".hud_prompt.txt", ".hud_run.bat", ".hud_mcp_config.json") if path in script ) - if input is not None: - self._sink[name] = input.encode() + if input_value is not None: + self._sink[name] = str(input_value).encode() elif match := re.search(r"FromBase64String\('([^']+)'\)", script): self._sink[name] += base64.b64decode(match.group(1)) else: self._sink[name] = b"" - return SimpleNamespace(stdout="", stderr="", exit_status=0, returncode=0) + self.written[name] = self._sink[name] + return _FakeCompletedProcess() + assert kwargs == {"encoding": None} self.ran.append(cmd) - return self._result - - async def create_process(self, cmd: str, **kwargs: Any) -> _FakeProcess: - return _FakeProcess(await self.run(cmd, **kwargs)) - - -class _FakeProcess: - def __init__(self, result: Any) -> None: - self._result = result - - async def wait(self, *, check: bool, **kwargs: Any) -> Any: - del check - assert kwargs == {"timeout": None} - return self._result - - def terminate(self) -> None: - pass - - def close(self) -> None: - pass - - async def wait_closed(self) -> None: - pass + return self._process def _fake_run() -> Any: @@ -123,7 +242,17 @@ def _fake_run() -> Any: _STREAM_JSON = ( - '{"type":"assistant","message":{"content":[{"type":"text","text":"working"}]}}\n' + '{"type":"assistant","message":{"id":"msg-1","type":"message",' + '"role":"assistant","model":"claude-test","content":[{"type":"text",' + '"text":"editing"},{"type":"tool_use","id":"tool-1","name":"Write","input":{}}],' + '"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":11,' + '"output_tokens":7,"cache_read_input_tokens":3}}}\n' + '{"type":"user","message":{"content":[{"type":"tool_result",' + '"tool_use_id":"tool-1","content":"wrote a.txt","is_error":false}]}}\n' + '{"type":"assistant","message":{"id":"msg-2","type":"message",' + '"role":"assistant","model":"claude-test","content":[{"type":"text",' + '"text":"done"}],"stop_reason":"end_turn","stop_sequence":null,' + '"usage":{"input_tokens":11,"output_tokens":7,"cache_read_input_tokens":3}}}\n' '{"type":"result","is_error":false,"result":"done","session_id":"s",' '"duration_ms":5,"num_turns":2,"total_cost_usd":0.01}\n' ) @@ -141,11 +270,8 @@ def _ssh_with_conn(shell: str, conn: _FakeConn) -> SSHClient: async def test_exec_on_windows_writes_batch_and_execs_via_cmd() -> None: sink: dict[str, bytes] = {} - conn = _FakeConn( - sink, - SimpleNamespace(stdout=_STREAM_JSON, stderr="", exit_status=0, returncode=0), - ) - agent = ClaudeSDKAgent() + conn = _FakeConn(sink, _FakeStreamProcess(_STREAM_JSON)) + agent = ClaudeCLIAgent() ssh = _ssh_with_conn("cmd", conn) run = _fake_run() @@ -153,39 +279,147 @@ async def test_exec_on_windows_writes_batch_and_execs_via_cmd() -> None: assert conn.ran == ["cmd /c .hud_run.bat"] assert all(command.startswith("powershell ") for command in conn.write_commands) - assert sink[".hud_run.bat"].startswith(b"@echo off\r\n") - assert sink[".hud_prompt.txt"] == b"build it" + assert conn.written[".hud_run.bat"].startswith(b"@echo off\r\n") + assert conn.written[".hud_prompt.txt"] == b"build it" + assert sink == {} + assert set(conn.deleted) == {".hud_prompt.txt", ".hud_run.bat"} assert run.trace.status == "completed" - assert "done" in run.trace.content + assert run.trace.content == "done" + assert "messages" not in run.trace.extra async def test_exec_on_bash_runs_inline_without_batch() -> None: sink: dict[str, bytes] = {} - conn = _FakeConn( - sink, - SimpleNamespace(stdout=_STREAM_JSON, stderr="", exit_status=0, returncode=0), - ) - agent = ClaudeSDKAgent() + conn = _FakeConn(sink, _FakeStreamProcess(_STREAM_JSON)) + agent = ClaudeCLIAgent() ssh = _ssh_with_conn("bash", conn) run = _fake_run() await agent._exec(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="build it", max_steps=5) - assert ".hud_run.bat" not in sink - assert conn.write_commands == ["cat > .hud_prompt.txt"] + assert sink == {} + assert conn.write_commands == [] + assert conn.deleted == [] assert len(conn.ran) == 1 - assert "install.sh" in conn.ran[0] assert "claude" in conn.ran[0] assert run.trace.status == "completed" + assert run.trace.content == "done" + assert "messages" not in run.trace.extra -async def test_exec_nonzero_exit_with_no_stdout_records_system_error() -> None: +async def test_exec_removes_mcp_config_after_run() -> None: sink: dict[str, bytes] = {} - conn = _FakeConn( - sink, - SimpleNamespace(stdout="", stderr="boom", exit_status=1, returncode=1), + conn = _FakeConn(sink, _FakeStreamProcess(_STREAM_JSON)) + agent = ClaudeCLIAgent() + + await agent._exec( + _fake_run(), + ssh=_ssh_with_conn("bash", conn), + shell="bash", + mcp_servers={"database": {"type": "http", "url": "http://db/mcp"}}, + prompt="build it", + max_steps=5, + ) + + config = json.loads(conn.written[".hud_mcp_config.json"]) + assert config == {"mcpServers": {"database": {"type": "http", "url": "http://db/mcp"}}} + assert sink == {} + assert conn.deleted == [".hud_mcp_config.json"] + assert "--mcp-config .hud_mcp_config.json" in conn.ran[0] + + +async def test_exec_records_steps_before_process_exit() -> None: + process = _FakeStreamProcess(_STREAM_JSON, pause_after=1) + conn = _FakeConn({}, process) + agent = ClaudeCLIAgent() + ssh = _ssh_with_conn("bash", conn) + run = _fake_run() + + execution = asyncio.create_task( + agent._exec(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="edit it", max_steps=5) + ) + await process.stdout.blocked.wait() + + assert not execution.done() + assert len(run.steps) == 1 + first = run.steps[0] + assert isinstance(first, AgentStep) + assert first.content == "editing" + assert first.tool_calls[0].id == "tool-1" + + process.stdout.release.set() + await execution + + assert [type(step) for step in run.steps] == [AgentStep, ToolStep, AgentStep] + tool = cast("ToolStep", run.steps[1]) + assert tool.started_at == first.ended_at + final = cast("AgentStep", run.steps[2]) + assert final.started_at == tool.ended_at + assert run.trace.status == "completed" + assert run.trace.content == "done" + + +async def test_exec_forwards_trace_id_only_to_hud_gateway( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(settings, "api_key", "hud-key") + monkeypatch.setattr(settings, "anthropic_api_key", "anthropic-key") + + gateway_conn = _FakeConn({}, _FakeStreamProcess(_STREAM_JSON)) + gateway = ClaudeCLIAgent(ClaudeCLIConfig(use_hud_gateway=True)) + with set_trace_context("trace-123"): + await gateway._exec( + _fake_run(), + ssh=_ssh_with_conn("bash", gateway_conn), + shell="bash", + mcp_servers={}, + prompt="build it", + max_steps=5, + ) + assert "ANTHROPIC_CUSTOM_HEADERS='Trace-Id: trace-123'" in gateway_conn.ran[0] + + provider_conn = _FakeConn({}, _FakeStreamProcess(_STREAM_JSON)) + provider = ClaudeCLIAgent(ClaudeCLIConfig(use_hud_gateway=False)) + with set_trace_context("trace-123"): + await provider._exec( + _fake_run(), + ssh=_ssh_with_conn("bash", provider_conn), + shell="bash", + mcp_servers={}, + prompt="build it", + max_steps=5, + ) + assert "ANTHROPIC_CUSTOM_HEADERS" not in provider_conn.ran[0] + + +async def test_exec_closes_streaming_process_when_cancelled() -> None: + process = _FakeStreamProcess(_STREAM_JSON, pause_after=0) + conn = _FakeConn({}, process) + agent = ClaudeCLIAgent() + + execution = asyncio.create_task( + agent._exec( + _fake_run(), + ssh=_ssh_with_conn("bash", conn), + shell="bash", + mcp_servers={}, + prompt="build it", + max_steps=5, + ) ) - agent = ClaudeSDKAgent() + await process.stdout.blocked.wait() + execution.cancel() + + with pytest.raises(asyncio.CancelledError): + await execution + + assert process.terminated + + +async def test_exec_nonzero_exit_with_no_stdout_records_system_error() -> None: + sink: dict[str, bytes] = {} + conn = _FakeConn(sink, _FakeStreamProcess("", stderr="boom", exit_status=1)) + agent = ClaudeCLIAgent() ssh = _ssh_with_conn("cmd", conn) run = _fake_run() @@ -200,9 +434,9 @@ async def test_exec_signal_exit_records_the_returncode() -> None: sink: dict[str, bytes] = {} conn = _FakeConn( sink, - SimpleNamespace(stdout="", stderr="", exit_status=None, returncode=-15), + _FakeStreamProcess("", exit_status=None, returncode=-15), ) - agent = ClaudeSDKAgent() + agent = ClaudeCLIAgent() ssh = _ssh_with_conn("bash", conn) run = _fake_run() @@ -213,6 +447,41 @@ async def test_exec_signal_exit_records_the_returncode() -> None: assert run.steps[0].error == "claude CLI exited with return code -15" +async def test_exec_nonzero_exit_with_result_stream_remains_an_error() -> None: + sink: dict[str, bytes] = {} + conn = _FakeConn( + sink, + _FakeStreamProcess(_STREAM_JSON, stderr="transport failed", exit_status=1), + ) + agent = ClaudeCLIAgent() + ssh = _ssh_with_conn("bash", conn) + + run = _fake_run() + await agent._exec(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="x", max_steps=1) + + assert run.trace.status == "error" + assert run.trace.content == "done" + assert run.trace.extra["returncode"] == 1 + assert run.trace.extra["stderr"] == "transport failed" + assert "messages" not in run.trace.extra + assert run.steps[-1].error == "transport failed" + + +async def test_exec_zero_exit_without_result_event_is_an_error() -> None: + sink: dict[str, bytes] = {} + stdout = _STREAM_JSON.rsplit('{"type":"result"', 1)[0] + conn = _FakeConn(sink, _FakeStreamProcess(stdout)) + agent = ClaudeCLIAgent() + ssh = _ssh_with_conn("bash", conn) + + run = _fake_run() + await agent._exec(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="x", max_steps=1) + + assert run.trace.status == "error" + assert run.trace.content == "done" + assert run.steps[-1].error == "claude CLI exited without a result event" + + @pytest.mark.parametrize( ("transport", "claude_type"), [("streamable-http", "http"), ("sse", "sse")], @@ -242,7 +511,7 @@ async def open(self, ref: str) -> SSHClient: assert ref == "ssh" return ssh - agent = ClaudeSDKAgent() + agent = ClaudeCLIAgent() execute = AsyncMock() monkeypatch.setattr(agent, "_exec", execute) @@ -308,7 +577,7 @@ async def bridge( bridge_active = False encoding = WebPScreenshotEncoding(quality=42) - agent = ClaudeSDKAgent(ClaudeSDKConfig(screenshot_encoding=encoding)) + agent = ClaudeCLIAgent(ClaudeCLIConfig(screenshot_encoding=encoding)) async def execute(*_args: Any, **_kwargs: Any) -> None: assert bridge_active @@ -400,7 +669,7 @@ async def execute(*_args: Any, **kwargs: Any) -> None: }, } - agent = ClaudeSDKAgent() + agent = ClaudeCLIAgent() monkeypatch.setattr(computer_mcp, "bridge_computer_mcp", bridge) monkeypatch.setattr(agent, "_exec", execute) @@ -522,7 +791,7 @@ async def test_computer_mcp_bridge_uses_controller_python_and_owns_resources( assert spawn_args[:3] == ( sys.executable, "-m", - "hud.agents.claude.sdk.computer_mcp", + "hud.agents.claude.cli.computer_mcp", ) environ = spawn_call.kwargs["env"] assert json.loads(environ[computer_mcp.RFB_CAPABILITY_ENV]) == screen.to_manifest() @@ -644,7 +913,7 @@ async def execute( first_entered.set() await release_first.wait() - agent = ClaudeSDKAgent() + agent = ClaudeCLIAgent() monkeypatch.setattr(agent, "_exec", execute) run_a = SimpleNamespace(client=Client(shell_a, ssh_a), prompt_text="first") run_b = SimpleNamespace(client=Client(shell_b, ssh_b), prompt_text="second") diff --git a/hud/agents/tests/test_tool_agent.py b/hud/agents/tests/test_tool_agent.py index 9a4049e33..ab9f91089 100644 --- a/hud/agents/tests/test_tool_agent.py +++ b/hud/agents/tests/test_tool_agent.py @@ -24,7 +24,7 @@ from hud.agents.tools.base import AgentToolSpec, result_text from hud.agents.tools.rfb import RFBTool from hud.agents.tools.ssh import SSHInfrastructureErrorResult -from hud.agents.types import AgentConfig, AgentStep, ClaudeConfig, ClaudeSDKConfig, ToolStep +from hud.agents.types import AgentConfig, AgentStep, ClaudeCLIConfig, ClaudeConfig, ToolStep from hud.capabilities import Capability, CapabilityClient, MCPClient, RFBClient, SSHClient from hud.capabilities.rfb import PngScreenshotEncoding, WebPScreenshotEncoding from hud.capabilities.ssh import SSHConnectionError @@ -83,7 +83,7 @@ class WithCatalog(DictAgent): def test_claude_defaults_to_configurable_webp_screenshots() -> None: assert AgentConfig().screenshot_encoding == PngScreenshotEncoding() assert ClaudeConfig().screenshot_encoding == WebPScreenshotEncoding() - assert ClaudeSDKConfig().screenshot_encoding == WebPScreenshotEncoding() + assert ClaudeCLIConfig().screenshot_encoding == WebPScreenshotEncoding() configured = ClaudeConfig.model_validate( {"screenshot_encoding": {"mime_type": "image/webp", "quality": 42}}, diff --git a/hud/agents/tool_agent.py b/hud/agents/tool_agent.py index 5c2b31f53..83cab8710 100644 --- a/hud/agents/tool_agent.py +++ b/hud/agents/tool_agent.py @@ -34,7 +34,7 @@ def _format_result(self, call, result) -> BetaMessageParam | None: ... from hud.agents.types import AgentStep, ToolStep from hud.capabilities import MCPClient, RFBClient from hud.capabilities.ssh import SSHConnectionError -from hud.types import AgentType, MCPToolCall, MCPToolResult, Step, StopCondition +from hud.types import MCPToolCall, MCPToolResult, Step, StopCondition from hud.utils.time import now_iso if TYPE_CHECKING: @@ -92,31 +92,6 @@ def __init_subclass__(cls, **kwargs: Any) -> None: seen.setdefault(t.client_type, None) cls.clients = tuple(seen.keys()) - def hosted_spec(self) -> dict[str, Any]: - """HUD-hosted execution runs the agent remotely, so it is - reconstructed there from this identity (type, model, step budget, system - prompt, provider kwargs) with the model resolved through the HUD gateway. - """ - if self.config.model_client is not None: - raise ValueError( - "hosted execution cannot serialize a custom model_client; " - "use create_agent(model, ...) so the hosted runner rebuilds the " - "gateway client, or run the agent loop locally with HUDRuntime() " - "/ LocalRuntime (recommended for TrainingClient workflows that " - "attach a BYOK client)" - ) - agent_type = AgentType.of(self) - if agent_type is None: - raise ValueError( - f"hosted execution supports the gateway agent types " - f"({', '.join(at.value for at in AgentType)}); got {type(self).__name__}" - ) - config = self.config.model_dump( - mode="json", - exclude={"model_client", "api_key", "base_url", "hosted_tools"}, - ) - return {"type": agent_type.value, "config": config} - async def __call__(self, run: Run) -> None: """Drive this (stateless) agent over a live ``Run``, filling ``run.trace``. diff --git a/hud/agents/types.py b/hud/agents/types.py index fc2978713..768dd8641 100644 --- a/hud/agents/types.py +++ b/hud/agents/types.py @@ -141,19 +141,20 @@ class OpenAIChatConfig(AgentConfig): # ----------------------------------------------------------------------------- -# Claude Code (CLI over SSH) +# Claude CLI (over SSH) # ----------------------------------------------------------------------------- -class ClaudeSDKConfig(AgentConfig): - """Configuration for ClaudeSDKAgent (runs the ``claude`` CLI over SSH). +class ClaudeCLIConfig(AgentConfig): + """Configuration for ClaudeCLIAgent (runs the ``claude`` CLI over SSH). ``system_prompt`` is inherited from ``AgentConfig``. ``max_steps`` maps to the CLI's ``--max-turns``; values <= 0 leave the turn budget to the CLI (unlimited). """ - model_name: str = "Claude Code" - model: str = Field(default="claude-sonnet-4-6", validation_alias=_model_alias) + model_name: str = "Claude CLI" + model: str = Field(default="claude-sonnet-5", validation_alias=_model_alias) + use_hud_gateway: bool | None = None permission_mode: str = "bypassPermissions" max_steps: int = -1 screenshot_encoding: ScreenshotEncoding = Field(default_factory=WebPScreenshotEncoding) diff --git a/hud/cli/eval.py b/hud/cli/eval.py index 598d1c1d5..553e3e57c 100644 --- a/hud/cli/eval.py +++ b/hud/cli/eval.py @@ -119,6 +119,7 @@ class AgentPreset: _AGENT_PRESETS: list[AgentPreset] = [ AgentPreset("Claude Sonnet 4.6", AgentType.CLAUDE, "claude-sonnet-4-6"), AgentPreset("Claude Opus 4.8", AgentType.CLAUDE, "claude-opus-4-8"), + AgentPreset("Claude CLI", AgentType.CLAUDE_CLI, "claude-sonnet-5"), AgentPreset("GPT-5.6", AgentType.OPENAI, "gpt-5.6"), AgentPreset("GPT-5.5", AgentType.OPENAI, "gpt-5.5"), AgentPreset("Gemini 3.1 Pro (Preview)", AgentType.GEMINI, "gemini-3.1-pro-preview"), @@ -176,6 +177,10 @@ class AgentPreset: # max_tokens = 16384 # use_computer_beta = true +[claude_cli] +# model = "claude-sonnet-5" +# permission_mode = "bypassPermissions" + [openai] # model = "gpt-5.6" # temperature = 0.7 @@ -194,6 +199,7 @@ class AgentPreset: # Agent type -> (settings attr, env var name) _API_KEY_REQUIREMENTS: dict[AgentType, tuple[str, str]] = { AgentType.CLAUDE: ("anthropic_api_key", "ANTHROPIC_API_KEY"), + AgentType.CLAUDE_CLI: ("anthropic_api_key", "ANTHROPIC_API_KEY"), AgentType.GEMINI: ("gemini_api_key", "GEMINI_API_KEY"), AgentType.OPENAI: ("openai_api_key", "OPENAI_API_KEY"), } @@ -387,7 +393,7 @@ def validate_api_keys(self) -> None: raise typer.Exit(1) elif self.agent_type == AgentType.CLAUDE and _is_bedrock_arn(self.model): _require_bedrock_credentials() - elif self.agent_type in _API_KEY_REQUIREMENTS: + elif self.agent_type in _API_KEY_REQUIREMENTS and not self.agent_type.is_cli: attr, env_var = _API_KEY_REQUIREMENTS[self.agent_type] if not getattr(settings, attr, None): hud_console.error(f"{env_var} is required for {self.agent_type.value} agent") @@ -672,13 +678,19 @@ def _build_agent(cfg: EvalConfig) -> Any: agent_kwargs = cfg.get_agent_kwargs() if cfg.auto_respond: agent_kwargs["auto_respond"] = True + if cfg.agent_type.is_cli: + if cfg.gateway or cfg.remote: + agent_kwargs["use_hud_gateway"] = True + else: + agent_kwargs.setdefault("use_hud_gateway", False) if cfg.gateway: - from hud.utils.gateway import build_gateway_client + if not cfg.agent_type.is_cli: + from hud.utils.gateway import build_gateway_client - agent_kwargs.setdefault( - "model_client", build_gateway_client(cfg.agent_type.gateway_provider) - ) + agent_kwargs.setdefault( + "model_client", build_gateway_client(cfg.agent_type.gateway_provider) + ) hud_console.info(f"Using HUD Gateway for {cfg.agent_type.gateway_provider} API") config = cfg.agent_type.config_cls(**agent_kwargs) @@ -860,7 +872,7 @@ def eval_command( source: str | None = typer.Argument(None, help="Taskset slug or task JSON file"), agent: str | None = typer.Argument( None, - help="Model name (e.g. claude-sonnet-4-6) or agent type (claude, openai, gemini, openai_compatible)", # noqa: E501 + help="Model name (e.g. claude-sonnet-4-6) or agent type (claude, claude_cli, openai, gemini, openai_compatible)", # noqa: E501 ), all: bool = typer.Option(False, "--all", help="Run all problems instead of just 1"), full: bool = typer.Option( diff --git a/hud/cli/tests/test_eval_config.py b/hud/cli/tests/test_eval_config.py index fad19d88f..5148d6a16 100644 --- a/hud/cli/tests/test_eval_config.py +++ b/hud/cli/tests/test_eval_config.py @@ -31,6 +31,12 @@ def test_parse_agent_type_accepts_known_value() -> None: assert cfg.agent_type.value == "openai" +def test_parse_agent_type_accepts_cli_agent() -> None: + cfg = EvalConfig(agent_type="claude_cli") + assert cfg.agent_type is not None + assert cfg.agent_type.value == "claude_cli" + + def test_parse_agent_type_rejects_unknown() -> None: with pytest.raises(ValueError, match="Invalid agent"): EvalConfig(agent_type="not-an-agent") @@ -121,6 +127,20 @@ def test_validate_api_keys_hud_runtime_keeps_local_gateway( assert cfg.gateway is True +def test_validate_api_keys_allows_cli_workspace_auth( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from hud.settings import settings + + monkeypatch.setattr(settings, "api_key", None) + monkeypatch.setattr(settings, "anthropic_api_key", None) + + cfg = EvalConfig(agent_type="claude_cli", runtime="local") + cfg.validate_api_keys() + + assert cfg.gateway is False + + def test_resolve_placement_runtime_hud_uses_tunnel( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -325,6 +345,36 @@ def test_eval_max_steps_lands_in_agent_config() -> None: assert agent.config.max_steps == 17 +def test_build_agent_constructs_claude_cli() -> None: + from hud.agents.claude import ClaudeCLIAgent + + cfg = EvalConfig(agent_type="claude_cli") + agent = eval_mod._build_agent(cfg) + + assert isinstance(agent, ClaudeCLIAgent) + assert agent.config.model == "claude-sonnet-5" + assert agent.config.use_hud_gateway is False + + +def test_build_agent_routes_hosted_claude_cli_through_gateway() -> None: + cfg = EvalConfig(agent_type="claude_cli", remote=True) + + agent = eval_mod._build_agent(cfg) + + assert agent.config.use_hud_gateway is True + + +def test_build_agent_preserves_claude_cli_gateway_config() -> None: + cfg = EvalConfig( + agent_type="claude_cli", + agent_config={"claude_cli": {"use_hud_gateway": True}}, + ) + + agent = eval_mod._build_agent(cfg) + + assert agent.config.use_hud_gateway is True + + def test_spawn_target_serves_single_file_env(tmp_path: Path) -> None: env_py = tmp_path / "tasks.py" env_py.write_text( diff --git a/hud/eval/run.py b/hud/eval/run.py index bf3cc8fbb..d3ac5c741 100644 --- a/hud/eval/run.py +++ b/hud/eval/run.py @@ -61,9 +61,10 @@ def validate_rollout_timeouts( verifier_runtime_config: RuntimeConfig | None, ) -> float | None: """Validate configured phase limits and return the effective agent timeout.""" - from hud.agents.tool_agent import ToolAgent + from hud.agents.types import AgentConfig - agent_timeout = agent.config.timeout_seconds if isinstance(agent, ToolAgent) else None + config = getattr(agent, "config", None) + agent_timeout = config.timeout_seconds if isinstance(config, AgentConfig) else None if task.agent_config is not None: agent_timeout = task.agent_config.get("timeout_seconds", agent_timeout) @@ -489,12 +490,12 @@ async def rollout( await job_enter(job_id, name=task.id, group=1) trace_id = trace_id or uuid.uuid4().hex # Report the model the agent will sample so the platform attributes the - # trace to it on enter. Only LLM tool agents carry an inference-model slug - # (``config.model``); robot/other agents have none. Local import avoids an - # eval<->agents import cycle. - from hud.agents.tool_agent import ToolAgent + # trace to it on enter. Registered LLM and CLI agents carry it on their + # AgentConfig; robot/custom agents may not. + from hud.agents.types import AgentConfig - agent_model = agent.config.model if isinstance(agent, ToolAgent) else None + config = getattr(agent, "config", None) + agent_model = config.model if isinstance(config, AgentConfig) else None with set_trace_context(trace_id): await trace_enter( trace_id, diff --git a/hud/eval/runtime/hosted.py b/hud/eval/runtime/hosted.py index 9134061d6..f1f11aea8 100644 --- a/hud/eval/runtime/hosted.py +++ b/hud/eval/runtime/hosted.py @@ -29,7 +29,7 @@ class HostedRuntime: agent runs alongside the task environment. This process only submits the rollout and polls the trace to completion, folding the result into a :class:`~hud.eval.run.Run`. Because the agent runs remotely, its identity - travels via :func:`_agent_spec`. + travels via :func:`hud.agents.dump_agent`. ``run_timeout`` is a deprecated constructor alias for ``rollout_timeout``. A local cancel (Ctrl-C) requests remote cancellation before propagating. @@ -119,14 +119,9 @@ async def _submit_and_await( group_id: str | None, trace_id: str, ) -> dict[str, Any]: - from hud.agents.tool_agent import ToolAgent + from hud.agents.registry import dump_agent - if not isinstance(agent, ToolAgent): - raise ValueError( - f"hosted execution requires a gateway agent that can serialize its " - f"identity (Claude/OpenAI/Gemini/OpenAIChat); got {type(agent).__name__}" - ) - spec = agent.hosted_spec() + spec = dump_agent(agent) if task.agent_config: spec = { **spec, diff --git a/hud/eval/tests/test_hosted.py b/hud/eval/tests/test_hosted.py index c554517fa..62d029c48 100644 --- a/hud/eval/tests/test_hosted.py +++ b/hud/eval/tests/test_hosted.py @@ -18,6 +18,8 @@ import pytest +from hud.agents import dump_agent +from hud.agents.claude import ClaudeCLIAgent, ClaudeCLIConfig from hud.agents.openai_compatible import OpenAIChatAgent from hud.agents.types import OpenAIChatConfig from hud.eval.job import Job @@ -91,14 +93,14 @@ def test_runtime_constructor_timeout_is_a_deprecated_alias(runtime_type: type[An assert runtime.run_timeout == 90.0 -def test_hosted_spec_serializes_full_config() -> None: +def test_dump_agent_serializes_full_config() -> None: agent = _agent() agent.config.system_prompt = "be brief" agent.config.max_steps = 7 agent.config.timeout_seconds = 3600 agent.config.tool_timeout_seconds = 1800 - spec = agent.hosted_spec() + spec = dump_agent(agent) assert spec["type"] == "openai_compatible" config = spec["config"] @@ -115,7 +117,7 @@ def test_hosted_spec_serializes_full_config() -> None: assert "hosted_tools" not in config -def test_create_agent_hosted_spec_preserves_training_config( +def test_dump_agent_preserves_training_config( monkeypatch: pytest.MonkeyPatch, ) -> None: """The constructor builds the runtime client without putting it in config.""" @@ -150,7 +152,7 @@ class _GatewayStub: assert agent.config.model_client is None assert agent.oai is client - spec = agent.hosted_spec() + spec = dump_agent(agent) config = spec["config"] assert spec["type"] == "openai_compatible" assert config["model"] == "arith-rl" @@ -162,17 +164,15 @@ class _GatewayStub: assert "model_client" not in config -def test_hosted_spec_rejects_custom_model_client() -> None: +def test_dump_agent_rejects_custom_model_client() -> None: agent = _agent() agent.config = OpenAIChatConfig(model="m", model_client=object()) - with pytest.raises(ValueError, match="custom model_client"): - agent.hosted_spec() - with pytest.raises(ValueError, match="HUDRuntime"): - agent.hosted_spec() + with pytest.raises(ValueError, match=r"custom model_client.*HUDRuntime"): + dump_agent(agent) @pytest.mark.asyncio -async def test_run_rejects_non_gateway_agent() -> None: +async def test_run_rejects_unregistered_agent() -> None: """An agent that can't serialize its identity yields a failed Run, not a crash.""" run = await HostedRuntime(poll_interval=0.0).run( Task(env="e", id="x"), @@ -180,7 +180,7 @@ async def test_run_rejects_non_gateway_agent() -> None: job_id="j", ) assert run.trace.is_error - assert "gateway agent" in (run.trace.error or "") + assert "registered types" in (run.trace.error or "") @pytest.mark.asyncio @@ -253,6 +253,35 @@ async def test_run_submits_and_polls_to_terminal(monkeypatch: pytest.MonkeyPatch assert payload["agent"]["config"]["timeout_seconds"] == 45.0 +@pytest.mark.asyncio +async def test_run_submits_registered_cli_agent(monkeypatch: pytest.MonkeyPatch) -> None: + platform = _FakePlatform([{"status": "completed", "reward": 1.0}]) + monkeypatch.setattr( + "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) + ) + agent = ClaudeCLIAgent( + ClaudeCLIConfig( + model="claude-sonnet-4-6", + max_steps=23, + use_hud_gateway=True, + ) + ) + + run = await HostedRuntime(poll_interval=0.0).run( + Task(env="coding", id="solve"), + agent, + job_id=uuid.uuid4().hex, + trace_id=uuid.uuid4().hex, + ) + + assert run.reward == 1.0 + submitted = platform.posts[0][1]["agent"] + assert submitted["type"] == "claude_cli" + assert submitted["config"]["model"] == "claude-sonnet-4-6" + assert submitted["config"]["max_steps"] == 23 + assert submitted["config"]["use_hud_gateway"] is True + + @pytest.mark.asyncio async def test_run_preserves_runtime_config_null_override( monkeypatch: pytest.MonkeyPatch, diff --git a/hud/types.py b/hud/types.py index 3b530a6e0..a8579c094 100644 --- a/hud/types.py +++ b/hud/types.py @@ -44,15 +44,23 @@ if TYPE_CHECKING: from collections.abc import Callable - from hud.agents.claude import ClaudeAgent + from hud.agents.claude import ClaudeAgent, ClaudeCLIAgent from hud.agents.gemini import GeminiAgent from hud.agents.openai import OpenAIAgent from hud.agents.openai_compatible import OpenAIChatAgent - from hud.agents.types import ClaudeConfig, GeminiConfig, OpenAIChatConfig, OpenAIConfig - - AgentClass: TypeAlias = type[ClaudeAgent | GeminiAgent | OpenAIAgent | OpenAIChatAgent] + from hud.agents.types import ( + ClaudeCLIConfig, + ClaudeConfig, + GeminiConfig, + OpenAIChatConfig, + OpenAIConfig, + ) + + AgentClass: TypeAlias = type[ + ClaudeAgent | ClaudeCLIAgent | GeminiAgent | OpenAIAgent | OpenAIChatAgent + ] AgentConfigClass: TypeAlias = type[ - ClaudeConfig | GeminiConfig | OpenAIConfig | OpenAIChatConfig + ClaudeConfig | ClaudeCLIConfig | GeminiConfig | OpenAIConfig | OpenAIChatConfig ] T = TypeVar("T") @@ -60,10 +68,15 @@ class AgentType(StrEnum): CLAUDE = "claude" + CLAUDE_CLI = "claude_cli" OPENAI = "openai" GEMINI = "gemini" OPENAI_COMPATIBLE = "openai_compatible" + @property + def is_cli(self) -> bool: + return self is AgentType.CLAUDE_CLI + @property def cls(self) -> AgentClass: match self: @@ -71,6 +84,10 @@ def cls(self) -> AgentClass: from hud.agents import ClaudeAgent return ClaudeAgent + case AgentType.CLAUDE_CLI: + from hud.agents import ClaudeCLIAgent + + return ClaudeCLIAgent case AgentType.OPENAI: from hud.agents import OpenAIAgent @@ -87,11 +104,19 @@ def cls(self) -> AgentClass: @property def config_cls(self) -> AgentConfigClass: """Get config class without importing agent (avoids SDK dependency).""" - from hud.agents.types import ClaudeConfig, GeminiConfig, OpenAIChatConfig, OpenAIConfig + from hud.agents.types import ( + ClaudeCLIConfig, + ClaudeConfig, + GeminiConfig, + OpenAIChatConfig, + OpenAIConfig, + ) match self: case AgentType.CLAUDE: return ClaudeConfig + case AgentType.CLAUDE_CLI: + return ClaudeCLIConfig case AgentType.OPENAI: return OpenAIConfig case AgentType.GEMINI: @@ -105,6 +130,8 @@ def gateway_provider(self) -> str: match self: case AgentType.CLAUDE: return "anthropic" + case AgentType.CLAUDE_CLI: + return "anthropic" case AgentType.OPENAI: return "openai" case AgentType.GEMINI: @@ -114,15 +141,15 @@ def gateway_provider(self) -> str: @classmethod def of(cls, agent: object) -> AgentType | None: - """The gateway agent type *agent* is an instance of, or ``None``. + """The registered agent type *agent* is an instance of, or ``None``. - Reverse of :attr:`cls`. Provider extras (anthropic, google-genai, ...) + Reverse of :attr:`cls`. Agent extras (anthropic, google-genai, ...) may be uninstalled, so importing a type's agent class can fail; that - simply means *agent* is not that type. ``None`` for a custom ``Agent`` - subclass that is not one of the gateway shortcuts. + simply means *agent* is not that type. ``None`` means the ``Agent`` + implementation is not registered for reconstruction. """ for agent_type in cls: - with contextlib.suppress(Exception): + with contextlib.suppress(ImportError): if isinstance(agent, agent_type.cls): return agent_type return None From e7f14cedbce4653037faf528b9b0802d5da397f0 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:58:55 -0700 Subject: [PATCH 02/14] Simplify Claude CLI agent --- hud/agents/__init__.py | 2 +- hud/agents/claude/cli/__init__.py | 4 +- hud/agents/claude/cli/agent.py | 366 +++++++--------------- hud/agents/claude/cli/computer_mcp.py | 11 +- hud/agents/registry.py | 4 +- hud/agents/tests/test_base.py | 9 - hud/agents/tests/test_claude_cli_agent.py | 131 +++----- hud/cli/eval.py | 2 +- hud/types.py | 27 +- 9 files changed, 175 insertions(+), 381 deletions(-) diff --git a/hud/agents/__init__.py b/hud/agents/__init__.py index b071ee0d0..382b25331 100644 --- a/hud/agents/__init__.py +++ b/hud/agents/__init__.py @@ -117,7 +117,7 @@ def create_agent(model: str, **kwargs: Any) -> GatewayAgent: kwargs.setdefault("model", model_id) config = agent_type.config_cls(**kwargs) - return cast("GatewayAgent", agent_type.cls(cast("Any", config))) + return cast("GatewayAgent", agent_type.instantiate(config)) _LAZY_EXPORTS = { diff --git a/hud/agents/claude/cli/__init__.py b/hud/agents/claude/cli/__init__.py index 9eadbc752..4511023b2 100644 --- a/hud/agents/claude/cli/__init__.py +++ b/hud/agents/claude/cli/__init__.py @@ -1,5 +1,7 @@ """Agent that runs the ``claude`` CLI over SSH.""" -from .agent import ClaudeCLIAgent, ClaudeCLIConfig +from hud.agents.types import ClaudeCLIConfig + +from .agent import ClaudeCLIAgent __all__ = ["ClaudeCLIAgent", "ClaudeCLIConfig"] diff --git a/hud/agents/claude/cli/agent.py b/hud/agents/claude/cli/agent.py index 386ceb04e..34c462ac9 100644 --- a/hud/agents/claude/cli/agent.py +++ b/hud/agents/claude/cli/agent.py @@ -1,12 +1,4 @@ -"""ClaudeCLIAgent — runs ``claude`` CLI over SSH inside the env workspace. - -SSH-execs the ``claude`` CLI on the remote workspace so all built-in tools -(Bash, Read, Write, Edit, Glob, Grep) operate on the env's filesystem. -MCP capabilities from the manifest are written as MCP server config so the -CLI can call env-hosted MCP tools too. - -Inspired by harbor-framework/harbor's ClaudeCode agent. -""" +"""Claude CLI harness over a workspace SSH capability.""" from __future__ import annotations @@ -17,7 +9,6 @@ import logging import shlex from contextlib import AsyncExitStack -from dataclasses import dataclass from typing import TYPE_CHECKING, Any, cast import asyncssh @@ -45,56 +36,26 @@ _PROCESS_CLOSE_TIMEOUT_S = 5.0 -@dataclass(slots=True) -class RemoteInvocation: - """How to run an assembled CLI command on the remote workspace shell. - - ``command`` is what gets exec'd over SSH. When ``script_name`` is set, that - file must be written (with ``script_body``) before exec'ing ``command``. - """ - - command: str - script_name: str | None = None - script_body: str | None = None - - -@dataclass(slots=True) -class _PendingToolCall: - call: MCPToolCall - started_at: str - - class _ClaudeStreamParser: """Translate Claude CLI stream messages into canonical HUD steps.""" def __init__(self, run: Run, *, started_at: str) -> None: self._run = run self._agent_started_at = started_at - self._pending_calls: dict[str, _PendingToolCall] = {} + self._pending_calls: dict[str, tuple[MCPToolCall, str]] = {} self._last_agent_content = "" - self._message_count = 0 + self.message_count = 0 self._saw_result = False - self._error_recorded = False - - @property - def message_count(self) -> int: - return self._message_count + self._result_error: str | None = None def feed_line(self, line: str) -> None: line = line.strip() if not line: return - try: - raw = json.loads(line) - except json.JSONDecodeError: - logger.warning("Ignoring non-JSON Claude stream output") - return - if not isinstance(raw, dict): - logger.warning("Ignoring non-object Claude stream message") - return - - message = cast("dict[str, Any]", raw) - self._message_count += 1 + message = json.loads(line) + if not isinstance(message, dict): + raise ValueError("Claude stream event must be an object") + self.message_count += 1 received_at = now_iso() match message.get("type"): case "system" if message.get("subtype") == "init": @@ -104,27 +65,31 @@ def feed_line(self, line: str) -> None: case "user": self._record_tool_results(message, received_at) case "result": - self._record_result(message, received_at) + self._record_result(message) def finish(self, *, returncode: int, stderr: str) -> None: trace = self._run.trace + error = self._result_error if returncode != 0: trace.extra["returncode"] = returncode - if stderr and (returncode != 0 or trace.status == "error"): - trace.extra["stderr"] = stderr - if not trace.content and self._last_agent_content: - trace.content = self._last_agent_content - - if returncode != 0: - trace.status = "error" - self._record_error(stderr.strip() or f"claude CLI exited with return code {returncode}") + error = stderr.strip() or f"claude CLI exited with return code {returncode}" elif not self._saw_result: - trace.status = "error" - self._record_error("claude CLI exited without a result event") + error = "claude CLI exited without a result event" elif self._pending_calls: - trace.status = "error" missing = ", ".join(sorted(self._pending_calls)) - self._record_error(f"claude CLI exited without results for tool calls: {missing}") + error = f"claude CLI exited without results for tool calls: {missing}" + + if not trace.content and self._last_agent_content: + trace.content = self._last_agent_content + if error is None: + return + trace.status = "error" + if stderr: + trace.extra["stderr"] = stderr + timestamp = now_iso() + self._run.record( + Step(source="system", error=error, started_at=timestamp, ended_at=timestamp) + ) def _record_assistant(self, event: dict[str, Any], received_at: str) -> None: raw_message = event.get("message") @@ -134,12 +99,11 @@ def _record_assistant(self, event: dict[str, Any], received_at: str) -> None: step = ClaudeAgent._message_to_agent_step(message) step.started_at = self._agent_started_at step.ended_at = received_at - step.extra = _event_metadata(event, raw_message) if step.content: self._last_agent_content = step.content self._run.record(step) for call in step.tool_calls: - self._pending_calls[call.id] = _PendingToolCall(call=call, started_at=received_at) + self._pending_calls[call.id] = (call, received_at) def _record_tool_results(self, event: dict[str, Any], received_at: str) -> None: message = event.get("message") @@ -153,38 +117,62 @@ def _record_tool_results(self, event: dict[str, Any], received_at: str) -> None: for raw_block in content: if not isinstance(raw_block, dict) or raw_block.get("type") != "tool_result": continue - block = cast("dict[str, Any]", raw_block) + block = raw_block call_id = block.get("tool_use_id") if not isinstance(call_id, str): - continue - pending = self._pending_calls.pop(call_id, None) - if pending is None: - logger.warning("Claude returned a result for unknown tool call %s", call_id) - continue + raise ValueError("Claude tool result is missing tool_use_id") + try: + call, started_at = self._pending_calls.pop(call_id) + except KeyError: + raise ValueError( + f"Claude returned a result for unknown tool call {call_id!r}" + ) from None + + raw_result = block.get("content") + raw_items = raw_result if isinstance(raw_result, list) else [raw_result] + result_content: list[mcp_types.ContentBlock] = [] + for item in raw_items: + if isinstance(item, str): + result_content.append(mcp_types.TextContent(type="text", text=item)) + elif isinstance(item, dict) and item.get("type") == "text": + result_content.append(mcp_types.TextContent(type="text", text=item["text"])) + elif isinstance(item, dict) and item.get("type") == "image": + source = item["source"] + result_content.append( + mcp_types.ImageContent( + type="image", + data=source["data"], + mimeType=source["media_type"], + ) + ) + elif item is not None: + raise ValueError(f"unsupported Claude tool result block: {item!r}") + saw_result = True self._run.record( ToolStep( - call=pending.call, + call=call, result=MCPToolResult( call_id=call_id, - content=_tool_result_content(block.get("content")), + content=result_content, isError=block.get("is_error") is True, ), - started_at=pending.started_at, + started_at=started_at, ended_at=received_at, - extra=_event_metadata(event, message), ) ) if saw_result: self._agent_started_at = received_at - def _record_result(self, event: dict[str, Any], received_at: str) -> None: + def _record_result(self, event: dict[str, Any]) -> None: self._saw_result = True trace = self._run.trace result = event.get("result") trace.content = result if isinstance(result, str) else self._last_agent_content is_error = event.get("is_error") is True trace.status = "error" if is_error else "completed" + if is_error: + self._result_error = trace.content or "claude CLI reported an error" for key in ( "subtype", "session_id", @@ -197,73 +185,6 @@ def _record_result(self, event: dict[str, Any], received_at: str) -> None: value = event.get(key) if value is not None: trace.extra[key] = value - if is_error: - self._record_error(trace.content or "claude CLI reported an error", received_at) - - def _record_error(self, error: str, at: str | None = None) -> None: - if self._error_recorded: - return - timestamp = at or now_iso() - self._run.record( - Step(source="system", error=error, started_at=timestamp, ended_at=timestamp) - ) - self._error_recorded = True - - -def _tool_result_content(value: Any) -> list[mcp_types.ContentBlock]: - values = value if isinstance(value, list) else [value] - content: list[mcp_types.ContentBlock] = [] - for item in values: - if isinstance(item, str): - content.append(mcp_types.TextContent(type="text", text=item)) - elif ( - isinstance(item, dict) - and item.get("type") == "text" - and isinstance(item.get("text"), str) - ): - content.append(mcp_types.TextContent(type="text", text=item["text"])) - elif isinstance(item, dict) and item.get("type") == "image": - source = item.get("source") - if ( - isinstance(source, dict) - and source.get("type") == "base64" - and isinstance(source.get("data"), str) - and isinstance(source.get("media_type"), str) - ): - content.append( - mcp_types.ImageContent( - type="image", - data=source["data"], - mimeType=source["media_type"], - ) - ) - continue - content.append( - mcp_types.TextContent( - type="text", - text=json.dumps(item, ensure_ascii=False, separators=(",", ":")), - ) - ) - elif item is not None: - content.append( - mcp_types.TextContent( - type="text", - text=json.dumps(item, ensure_ascii=False, separators=(",", ":")), - ) - ) - return content - - -def _event_metadata(event: dict[str, Any], message: dict[str, Any]) -> dict[str, Any]: - metadata: dict[str, Any] = {} - for key in ("session_id", "uuid", "parent_tool_use_id"): - value = event.get(key) - if value is not None: - metadata[key] = value - message_id = message.get("id") - if message_id is not None: - metadata["message_id"] = message_id - return metadata def _powershell_quote(value: str) -> str: @@ -275,24 +196,6 @@ def _powershell(script: str) -> str: return f"powershell -NoProfile -NonInteractive -EncodedCommand {encoded}" -def build_remote_invocation(shell: str, run_cmd: str) -> RemoteInvocation: - """Build the remote exec command for ``run_cmd`` under the given login shell. - - Windows shells can't take the assembled command inline — ``cmd.exe`` mangles - the quotes — so it is written to a batch file and invoked through ``cmd /c``. - A bare ``.hud_run.bat`` is rejected as an unknown command, and silently fails - to run under a PowerShell default shell, so ``cmd /c`` is required for both. - POSIX shells take the command inline. - """ - if shell in WINDOWS_SHELLS: - return RemoteInvocation( - command=f"cmd /c {_RUN_SCRIPT_PATH}", - script_name=_RUN_SCRIPT_PATH, - script_body=f"@echo off\r\n{run_cmd}\r\n", - ) - return RemoteInvocation(command=run_cmd) - - class ClaudeCLIAgent(Agent): """Runs ``claude`` CLI over SSH inside the env workspace. @@ -348,17 +251,15 @@ async def __call__(self, run: Run) -> None: ) ) - await self._exec( + await self._run_cli( run, ssh=ssh, shell=shell, mcp_servers=mcp_servers, prompt=run.prompt_text, - max_steps=self.config.max_steps, - system_prompt=self.config.system_prompt, ) - async def _exec( + async def _run_cli( self, run: Run, *, @@ -366,35 +267,39 @@ async def _exec( shell: str, mcp_servers: dict[str, dict[str, Any]], prompt: str, - max_steps: int = -1, - system_prompt: str | None = None, ) -> None: - runtime_files: list[str] = [] + files: dict[str, str] = {} + mcp_config_path = None + if mcp_servers: + mcp_config_path = _MCP_CONFIG_PATH + files[mcp_config_path] = json.dumps({"mcpServers": mcp_servers}, indent=2) + if shell in WINDOWS_SHELLS: + files[_PROMPT_PATH] = prompt + + command = self._build_command( + shell=shell, + prompt=prompt, + mcp_config_path=mcp_config_path, + ) + if shell in WINDOWS_SHELLS: + files[_RUN_SCRIPT_PATH] = f"@echo off\r\n{command}\r\n" + command = f"cmd /c {_RUN_SCRIPT_PATH}" + try: - mcp_config_path = await self._write_mcp_config(ssh, mcp_servers) - if mcp_config_path is not None: - runtime_files.append(mcp_config_path) - if shell in WINDOWS_SHELLS: - await ssh.write_text(_PROMPT_PATH, prompt) - runtime_files.append(_PROMPT_PATH) - - run_cmd = self._build_cli_command( - shell=shell, - prompt=prompt, - max_steps=max_steps, - system_prompt=system_prompt, - mcp_config_path=mcp_config_path, - ) - invocation = build_remote_invocation(shell, run_cmd) - if invocation.script_name is not None: - assert invocation.script_body is not None - await ssh.write_text(invocation.script_name, invocation.script_body) - runtime_files.append(invocation.script_name) - - logger.info("SSH exec claude CLI (%d chars)", len(invocation.command)) - await self._stream_cli(run, ssh, invocation.command) + for path, content in files.items(): + await ssh.write_text(path, content) + logger.info("SSH exec claude CLI (%d chars)", len(command)) + await self._stream_cli(run, ssh, command) finally: - await self._remove_runtime_files(ssh, shell, runtime_files) + if files: + if shell in WINDOWS_SHELLS: + cleanup = f"cmd /c del /f /q {' '.join(files)} 2>nul" + else: + cleanup = "rm -f -- " + " ".join(shlex.quote(path) for path in files) + try: + await ssh.run(cleanup, check=False) + except (OSError, asyncssh.Error): + logger.warning("Failed to remove Claude CLI runtime files") async def _stream_cli(self, run: Run, ssh: SSHClient, command: str) -> None: parser = _ClaudeStreamParser(run, started_at=now_iso()) @@ -406,21 +311,13 @@ async def _stream_cli(self, run: Run, ssh: SSHClient, command: str) -> None: await process.wait_closed() stderr_output = await stderr_task except BaseException: - try: - process.terminate() - except (OSError, asyncssh.Error): - process.close() - stderr_task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await stderr_task - try: + process.close() + if not stderr_task.done(): + stderr_task.cancel() + await asyncio.gather(stderr_task, return_exceptions=True) + with contextlib.suppress(OSError, TimeoutError, asyncssh.Error): async with asyncio.timeout(_PROCESS_CLOSE_TIMEOUT_S): await process.wait_closed() - except (OSError, TimeoutError, asyncssh.Error): - process.close() - with contextlib.suppress(OSError, TimeoutError, asyncssh.Error): - async with asyncio.timeout(_PROCESS_CLOSE_TIMEOUT_S): - await process.wait_closed() raise stderr = stderr_output.decode(errors="replace") @@ -435,26 +332,14 @@ async def _stream_cli(self, run: Run, ssh: SSHClient, command: str) -> None: ) parser.finish(returncode=returncode, stderr=stderr) - async def _remove_runtime_files( + def _build_command( self, - ssh: SSHClient, + *, shell: str, - paths: list[str], - ) -> None: - if not paths: - return - if shell in WINDOWS_SHELLS: - command = f"cmd /c del /f /q {' '.join(paths)} 2>nul" - else: - command = "rm -f -- " + " ".join(shlex.quote(path) for path in paths) - try: - await ssh.run(command, check=False) - except (OSError, asyncssh.Error): - logger.warning("Failed to remove Claude CLI runtime files") - - def _build_env_vars(self) -> dict[str, str]: + prompt: str, + mcp_config_path: str | None = None, + ) -> str: env: dict[str, str] = {} - use_hud_gateway = self.config.use_hud_gateway if use_hud_gateway is None: use_hud_gateway = settings.api_key is not None @@ -483,34 +368,6 @@ def _build_env_vars(self) -> dict[str, str]: env["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC"] = "1" env["IS_SANDBOX"] = "1" - return env - - async def _write_mcp_config( - self, - ssh: SSHClient, - mcp_servers: dict[str, dict[str, Any]], - ) -> str | None: - """Write MCP config into the workspace and return its path.""" - if not mcp_servers: - return None - mcp_json = json.dumps({"mcpServers": mcp_servers}, indent=2) - path = _MCP_CONFIG_PATH - await ssh.write_text(path, mcp_json) - logger.info("Wrote MCP config") - return path - - def _build_cli_command( - self, - *, - shell: str, - prompt: str, - max_steps: int, - system_prompt: str | None, - mcp_config_path: str | None = None, - ) -> str: - env_vars = self._build_env_vars() - is_win = shell in WINDOWS_SHELLS - base_args: list[str] = [ "claude", "--verbose", @@ -518,19 +375,19 @@ def _build_cli_command( "--print", f"--permission-mode={self.config.permission_mode}", ] - if max_steps > 0: - base_args.append(f"--max-turns={max_steps}") - if system_prompt: - base_args.extend(["--system-prompt", system_prompt]) + if self.config.max_steps > 0: + base_args.append(f"--max-turns={self.config.max_steps}") + if self.config.system_prompt: + base_args.extend(["--system-prompt", self.config.system_prompt]) for tool in self.config.allowed_tools: base_args.extend(["--allowedTools", tool]) if mcp_config_path: base_args.extend(["--mcp-config", mcp_config_path]) - if is_win: + if shell in WINDOWS_SHELLS: script = ";".join( [ - *(f"$env:{key}={_powershell_quote(value)}" for key, value in env_vars.items()), + *(f"$env:{key}={_powershell_quote(value)}" for key, value in env.items()), f"Get-Content -Raw -Encoding UTF8 {_powershell_quote(_PROMPT_PATH)}" f" | & claude {' '.join(_powershell_quote(arg) for arg in base_args[1:])}", "exit $LASTEXITCODE", @@ -538,12 +395,11 @@ def _build_cli_command( ) return _powershell(script) - # POSIX path: shell-quote everything and embed prompt inline. cli_parts = [shlex.quote(a) for a in base_args] cli_parts.extend(["--", shlex.quote(prompt)]) cli_cmd = " ".join(cli_parts) - env_prefix = " ".join(f"{k}={shlex.quote(v)}" for k, v in env_vars.items()) + env_prefix = " ".join(f"{key}={shlex.quote(value)}" for key, value in env.items()) return f'export PATH="$HOME/.local/bin:$PATH"; {env_prefix} {cli_cmd}' -__all__ = ["ClaudeCLIAgent", "ClaudeCLIConfig", "RemoteInvocation", "build_remote_invocation"] +__all__ = ["ClaudeCLIAgent"] diff --git a/hud/agents/claude/cli/computer_mcp.py b/hud/agents/claude/cli/computer_mcp.py index 0b4ef38ac..36cd8dab4 100644 --- a/hud/agents/claude/cli/computer_mcp.py +++ b/hud/agents/claude/cli/computer_mcp.py @@ -129,23 +129,16 @@ async def computer( return mcp -def _required_env(environ: Mapping[str, str], name: str) -> str: - try: - return environ[name] - except KeyError as exc: - raise RuntimeError(f"missing required environment variable {name}") from exc - - async def run_computer_mcp(environ: Mapping[str, str] = os.environ) -> None: """Run computer-use over stdio in a controller-side child process.""" - raw_manifest = json.loads(_required_env(environ, RFB_CAPABILITY_ENV)) + raw_manifest = json.loads(environ[RFB_CAPABILITY_ENV]) if not isinstance(raw_manifest, dict): raise ValueError(f"{RFB_CAPABILITY_ENV} must contain a JSON object") capability = Capability.from_manifest(raw_manifest) if capability.protocol.split("/", 1)[0] != "rfb": raise ValueError(f"{RFB_CAPABILITY_ENV} must describe an RFB capability") screenshot_encoding = TypeAdapter(ScreenshotEncoding).validate_json( - _required_env(environ, SCREENSHOT_ENCODING_ENV) + environ[SCREENSHOT_ENCODING_ENV] ) rfb = await RFBClient.connect(capability) diff --git a/hud/agents/registry.py b/hud/agents/registry.py index bf0380f2b..10c6f2910 100644 --- a/hud/agents/registry.py +++ b/hud/agents/registry.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any from hud.agents.types import AgentConfig from hud.types import AgentType @@ -47,4 +47,4 @@ def load_agent(data: Mapping[str, Any]) -> Agent: if not isinstance(raw_config, Mapping): raise ValueError("agent config must be an object") config = agent_type.config_cls.model_validate(dict(raw_config)) - return cast("Agent", agent_type.cls(cast("Any", config))) + return agent_type.instantiate(config) diff --git a/hud/agents/tests/test_base.py b/hud/agents/tests/test_base.py index d6f6427f1..b88faf0e0 100644 --- a/hud/agents/tests/test_base.py +++ b/hud/agents/tests/test_base.py @@ -29,9 +29,6 @@ async def __call__(self, run: Any) -> None: run.trace.content = "done" -# ─── the ABC contract ───────────────────────────────────────────────── - - def test_agent_requires_call_implementation() -> None: with pytest.raises(TypeError): Agent() @@ -45,9 +42,6 @@ async def test_agent_call_fills_trace() -> None: assert run.trace.content == "done" -# ─── AgentType resolution ───────────────────────────────────────────── - - def test_agent_type_maps_value_to_class_and_provider() -> None: assert AgentType("openai").cls is OpenAIAgent assert AgentType("openai_compatible").cls is OpenAIChatAgent @@ -104,9 +98,6 @@ def find_spec(self, fullname: str, path: Any = None, target: Any = None) -> None _ = AgentType.CLAUDE.cls -# ─── create_agent routing ───────────────────────────────────────────── - - @pytest.fixture(autouse=True) def gateway_api_key(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr("hud.agents.settings.api_key", "test-key") diff --git a/hud/agents/tests/test_claude_cli_agent.py b/hud/agents/tests/test_claude_cli_agent.py index 23a8a2326..3d3b25ff9 100644 --- a/hud/agents/tests/test_claude_cli_agent.py +++ b/hud/agents/tests/test_claude_cli_agent.py @@ -19,15 +19,11 @@ from typing import TYPE_CHECKING, Any, Literal, cast from unittest.mock import AsyncMock, Mock -import mcp.types as mcp_types import pytest +from mcp.types import ImageContent, TextContent from hud.agents.claude.cli import computer_mcp -from hud.agents.claude.cli.agent import ( - ClaudeCLIAgent, - _tool_result_content, - build_remote_invocation, -) +from hud.agents.claude.cli.agent import ClaudeCLIAgent from hud.agents.types import AgentStep, ClaudeCLIConfig, ToolStep from hud.capabilities import Capability, SSHClient from hud.capabilities.rfb import WebPScreenshotEncoding @@ -44,52 +40,39 @@ def _clear_api_keys(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(settings, "anthropic_api_key", None) -def test_env_vars_follow_explicit_gateway_routing(monkeypatch: pytest.MonkeyPatch) -> None: +def test_command_follows_explicit_gateway_routing(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(settings, "api_key", "hud-key") monkeypatch.setattr(settings, "anthropic_api_key", "anthropic-key") - gateway = ClaudeCLIAgent(ClaudeCLIConfig(use_hud_gateway=True))._build_env_vars() - provider = ClaudeCLIAgent(ClaudeCLIConfig(use_hud_gateway=False))._build_env_vars() + gateway = ClaudeCLIAgent(ClaudeCLIConfig(use_hud_gateway=True))._build_command( + shell="bash", prompt="run", mcp_config_path=None + ) + provider = ClaudeCLIAgent(ClaudeCLIConfig(use_hud_gateway=False))._build_command( + shell="bash", prompt="run", mcp_config_path=None + ) - assert gateway["ANTHROPIC_BASE_URL"] == settings.hud_gateway_url - assert gateway["ANTHROPIC_API_KEY"] == "hud-key" - assert provider["ANTHROPIC_API_KEY"] == "anthropic-key" + assert f"ANTHROPIC_BASE_URL={settings.hud_gateway_url}" in gateway + assert "ANTHROPIC_API_KEY=hud-key" in gateway + assert "ANTHROPIC_API_KEY=anthropic-key" in provider assert "ANTHROPIC_BASE_URL" not in provider - assert provider["ANTHROPIC_MODEL"] == "claude-sonnet-5" - - -# ─── build_remote_invocation (pure) ─────────────────────────────────── - - -@pytest.mark.parametrize("shell", ["cmd", "powershell"]) -def test_windows_shell_runs_batch_file_via_cmd(shell: str) -> None: - inv = build_remote_invocation(shell, "claude --print -- hi") - - # The bare filename is rejected by the remote shell; cmd /c runs it. - assert inv.command == "cmd /c .hud_run.bat" - assert inv.script_name == ".hud_run.bat" - assert inv.script_body == "@echo off\r\nclaude --print -- hi\r\n" - - -def test_posix_shell_runs_inline() -> None: - inv = build_remote_invocation("bash", "claude --print -- hi") - - assert inv.script_name is None - assert inv.script_body is None - assert inv.command == "claude --print -- hi" + assert "ANTHROPIC_MODEL=claude-sonnet-5" in provider def test_windows_command_encodes_environment_and_arguments( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(settings, "api_key", "hud&key's") - agent = ClaudeCLIAgent(ClaudeCLIConfig(use_hud_gateway=True)) + agent = ClaudeCLIAgent( + ClaudeCLIConfig( + use_hud_gateway=True, + max_steps=3, + system_prompt="don't $expand", + ) + ) - command = agent._build_cli_command( + command = agent._build_command( shell="powershell", prompt="not embedded", - max_steps=3, - system_prompt="don't $expand", ) encoded = command.rsplit(" ", 1)[1] @@ -101,30 +84,6 @@ def test_windows_command_encodes_environment_and_arguments( assert "python" not in script -def test_tool_result_content_preserves_images() -> None: - content = _tool_result_content( - [ - {"type": "text", "text": "before"}, - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/png", - "data": "aW1hZ2U=", - }, - }, - ] - ) - - assert content[0].type == "text" - assert isinstance(content[1], mcp_types.ImageContent) - assert content[1].mimeType == "image/png" - assert content[1].data == "aW1hZ2U=" - - -# ─── _exec end-to-end over a fake SSH workspace ──────────────────────── - - class _FakeReader: def __init__(self, value: str, *, pause_after: int | None = None) -> None: self._raw = value.encode() @@ -248,7 +207,9 @@ def _fake_run() -> Any: '"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":11,' '"output_tokens":7,"cache_read_input_tokens":3}}}\n' '{"type":"user","message":{"content":[{"type":"tool_result",' - '"tool_use_id":"tool-1","content":"wrote a.txt","is_error":false}]}}\n' + '"tool_use_id":"tool-1","content":[{"type":"text","text":"wrote a.txt"},' + '{"type":"image","source":{"type":"base64","media_type":"image/png",' + '"data":"aW1hZ2U="}}],"is_error":false}]}}\n' '{"type":"assistant","message":{"id":"msg-2","type":"message",' '"role":"assistant","model":"claude-test","content":[{"type":"text",' '"text":"done"}],"stop_reason":"end_turn","stop_sequence":null,' @@ -275,7 +236,7 @@ async def test_exec_on_windows_writes_batch_and_execs_via_cmd() -> None: ssh = _ssh_with_conn("cmd", conn) run = _fake_run() - await agent._exec(run, ssh=ssh, shell="cmd", mcp_servers={}, prompt="build it", max_steps=5) + await agent._run_cli(run, ssh=ssh, shell="cmd", mcp_servers={}, prompt="build it") assert conn.ran == ["cmd /c .hud_run.bat"] assert all(command.startswith("powershell ") for command in conn.write_commands) @@ -295,7 +256,7 @@ async def test_exec_on_bash_runs_inline_without_batch() -> None: ssh = _ssh_with_conn("bash", conn) run = _fake_run() - await agent._exec(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="build it", max_steps=5) + await agent._run_cli(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="build it") assert sink == {} assert conn.write_commands == [] @@ -312,13 +273,12 @@ async def test_exec_removes_mcp_config_after_run() -> None: conn = _FakeConn(sink, _FakeStreamProcess(_STREAM_JSON)) agent = ClaudeCLIAgent() - await agent._exec( + await agent._run_cli( _fake_run(), ssh=_ssh_with_conn("bash", conn), shell="bash", mcp_servers={"database": {"type": "http", "url": "http://db/mcp"}}, prompt="build it", - max_steps=5, ) config = json.loads(conn.written[".hud_mcp_config.json"]) @@ -336,7 +296,7 @@ async def test_exec_records_steps_before_process_exit() -> None: run = _fake_run() execution = asyncio.create_task( - agent._exec(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="edit it", max_steps=5) + agent._run_cli(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="edit it") ) await process.stdout.blocked.wait() @@ -353,6 +313,14 @@ async def test_exec_records_steps_before_process_exit() -> None: assert [type(step) for step in run.steps] == [AgentStep, ToolStep, AgentStep] tool = cast("ToolStep", run.steps[1]) assert tool.started_at == first.ended_at + assert tool.result is not None + text = tool.result.content[0] + assert isinstance(text, TextContent) + assert text.text == "wrote a.txt" + image = tool.result.content[1] + assert isinstance(image, ImageContent) + assert image.mimeType == "image/png" + assert image.data == "aW1hZ2U=" final = cast("AgentStep", run.steps[2]) assert final.started_at == tool.ended_at assert run.trace.status == "completed" @@ -368,26 +336,24 @@ async def test_exec_forwards_trace_id_only_to_hud_gateway( gateway_conn = _FakeConn({}, _FakeStreamProcess(_STREAM_JSON)) gateway = ClaudeCLIAgent(ClaudeCLIConfig(use_hud_gateway=True)) with set_trace_context("trace-123"): - await gateway._exec( + await gateway._run_cli( _fake_run(), ssh=_ssh_with_conn("bash", gateway_conn), shell="bash", mcp_servers={}, prompt="build it", - max_steps=5, ) assert "ANTHROPIC_CUSTOM_HEADERS='Trace-Id: trace-123'" in gateway_conn.ran[0] provider_conn = _FakeConn({}, _FakeStreamProcess(_STREAM_JSON)) provider = ClaudeCLIAgent(ClaudeCLIConfig(use_hud_gateway=False)) with set_trace_context("trace-123"): - await provider._exec( + await provider._run_cli( _fake_run(), ssh=_ssh_with_conn("bash", provider_conn), shell="bash", mcp_servers={}, prompt="build it", - max_steps=5, ) assert "ANTHROPIC_CUSTOM_HEADERS" not in provider_conn.ran[0] @@ -398,13 +364,12 @@ async def test_exec_closes_streaming_process_when_cancelled() -> None: agent = ClaudeCLIAgent() execution = asyncio.create_task( - agent._exec( + agent._run_cli( _fake_run(), ssh=_ssh_with_conn("bash", conn), shell="bash", mcp_servers={}, prompt="build it", - max_steps=5, ) ) await process.stdout.blocked.wait() @@ -413,7 +378,7 @@ async def test_exec_closes_streaming_process_when_cancelled() -> None: with pytest.raises(asyncio.CancelledError): await execution - assert process.terminated + assert process.closed async def test_exec_nonzero_exit_with_no_stdout_records_system_error() -> None: @@ -423,7 +388,7 @@ async def test_exec_nonzero_exit_with_no_stdout_records_system_error() -> None: ssh = _ssh_with_conn("cmd", conn) run = _fake_run() - await agent._exec(run, ssh=ssh, shell="cmd", mcp_servers={}, prompt="x", max_steps=1) + await agent._run_cli(run, ssh=ssh, shell="cmd", mcp_servers={}, prompt="x") assert run.trace.status == "error" assert run.trace.extra["returncode"] == 1 @@ -440,7 +405,7 @@ async def test_exec_signal_exit_records_the_returncode() -> None: ssh = _ssh_with_conn("bash", conn) run = _fake_run() - await agent._exec(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="x", max_steps=1) + await agent._run_cli(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="x") assert run.trace.status == "error" assert run.trace.extra["returncode"] == -15 @@ -457,7 +422,7 @@ async def test_exec_nonzero_exit_with_result_stream_remains_an_error() -> None: ssh = _ssh_with_conn("bash", conn) run = _fake_run() - await agent._exec(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="x", max_steps=1) + await agent._run_cli(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="x") assert run.trace.status == "error" assert run.trace.content == "done" @@ -475,7 +440,7 @@ async def test_exec_zero_exit_without_result_event_is_an_error() -> None: ssh = _ssh_with_conn("bash", conn) run = _fake_run() - await agent._exec(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="x", max_steps=1) + await agent._run_cli(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="x") assert run.trace.status == "error" assert run.trace.content == "done" @@ -513,7 +478,7 @@ async def open(self, ref: str) -> SSHClient: agent = ClaudeCLIAgent() execute = AsyncMock() - monkeypatch.setattr(agent, "_exec", execute) + monkeypatch.setattr(agent, "_run_cli", execute) await agent( cast( @@ -584,7 +549,7 @@ async def execute(*_args: Any, **_kwargs: Any) -> None: execute_mock = AsyncMock(side_effect=execute) monkeypatch.setattr(computer_mcp, "bridge_computer_mcp", bridge) - monkeypatch.setattr(agent, "_exec", execute_mock) + monkeypatch.setattr(agent, "_run_cli", execute_mock) await agent( cast( @@ -671,7 +636,7 @@ async def execute(*_args: Any, **kwargs: Any) -> None: agent = ClaudeCLIAgent() monkeypatch.setattr(computer_mcp, "bridge_computer_mcp", bridge) - monkeypatch.setattr(agent, "_exec", execute) + monkeypatch.setattr(agent, "_run_cli", execute) await agent(cast("Any", SimpleNamespace(client=Client(), prompt_text="use both screens"))) @@ -914,7 +879,7 @@ async def execute( await release_first.wait() agent = ClaudeCLIAgent() - monkeypatch.setattr(agent, "_exec", execute) + monkeypatch.setattr(agent, "_run_cli", execute) run_a = SimpleNamespace(client=Client(shell_a, ssh_a), prompt_text="first") run_b = SimpleNamespace(client=Client(shell_b, ssh_b), prompt_text="second") diff --git a/hud/cli/eval.py b/hud/cli/eval.py index 553e3e57c..e1fe0b45d 100644 --- a/hud/cli/eval.py +++ b/hud/cli/eval.py @@ -695,7 +695,7 @@ def _build_agent(cfg: EvalConfig) -> Any: config = cfg.agent_type.config_cls(**agent_kwargs) # cls/config_cls are matched unions; the pairing is correct by construction. - return cast("Any", cfg.agent_type.cls)(config=config) + return cfg.agent_type.instantiate(config) def _python_defines_environment(path: Path) -> bool: diff --git a/hud/types.py b/hud/types.py index a8579c094..8ac8428dc 100644 --- a/hud/types.py +++ b/hud/types.py @@ -44,24 +44,8 @@ if TYPE_CHECKING: from collections.abc import Callable - from hud.agents.claude import ClaudeAgent, ClaudeCLIAgent - from hud.agents.gemini import GeminiAgent - from hud.agents.openai import OpenAIAgent - from hud.agents.openai_compatible import OpenAIChatAgent - from hud.agents.types import ( - ClaudeCLIConfig, - ClaudeConfig, - GeminiConfig, - OpenAIChatConfig, - OpenAIConfig, - ) - - AgentClass: TypeAlias = type[ - ClaudeAgent | ClaudeCLIAgent | GeminiAgent | OpenAIAgent | OpenAIChatAgent - ] - AgentConfigClass: TypeAlias = type[ - ClaudeConfig | ClaudeCLIConfig | GeminiConfig | OpenAIConfig | OpenAIChatConfig - ] + from hud.agents.base import Agent + from hud.agents.types import AgentConfig T = TypeVar("T") @@ -78,7 +62,7 @@ def is_cli(self) -> bool: return self is AgentType.CLAUDE_CLI @property - def cls(self) -> AgentClass: + def cls(self) -> type[Agent]: match self: case AgentType.CLAUDE: from hud.agents import ClaudeAgent @@ -102,7 +86,7 @@ def cls(self) -> AgentClass: return OpenAIChatAgent @property - def config_cls(self) -> AgentConfigClass: + def config_cls(self) -> type[AgentConfig]: """Get config class without importing agent (avoids SDK dependency).""" from hud.agents.types import ( ClaudeCLIConfig, @@ -124,6 +108,9 @@ def config_cls(self) -> AgentConfigClass: case AgentType.OPENAI_COMPATIBLE: return OpenAIChatConfig + def instantiate(self, config: AgentConfig) -> Agent: + return cast("Any", self.cls)(config) + @property def gateway_provider(self) -> str: """Default provider client used when this agent type is a gateway shortcut.""" From 2b17f96c038261274056248dbacff72b80847e66 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:46:53 -0700 Subject: [PATCH 03/14] Refine agent execution ownership --- hud/agents/base.py | 6 +- hud/agents/browser_use/agent.py | 8 - hud/agents/claude/cli/agent.py | 79 +++--- hud/agents/claude/cli/computer_mcp.py | 94 +++---- hud/agents/openai_compatible/agent.py | 27 +- hud/agents/registry.py | 4 +- hud/agents/robot/agent.py | 1 - hud/agents/tests/test_claude_cli_agent.py | 52 ++-- .../tests/test_openai_compatible_agent.py | 7 +- hud/agents/tests/test_tool_agent.py | 69 +++-- hud/agents/tool_agent.py | 243 ++++++------------ hud/agents/types.py | 26 +- hud/types.py | 2 +- 13 files changed, 242 insertions(+), 376 deletions(-) diff --git a/hud/agents/base.py b/hud/agents/base.py index 49bdbb499..d9d44814b 100644 --- a/hud/agents/base.py +++ b/hud/agents/base.py @@ -10,13 +10,13 @@ class Agent(ABC): - """Drives a live ``Run`` to completion by filling ``run.trace`` in place. + """Drives a live ``Run`` by recording its trajectory and final answer. Subclasses implement ``__call__(run)``; callers do ``await agent(run)``. Stateless per run — everything comes from ``run`` — so one instance drives many concurrent - rollouts. + rollouts. The caller owns lifecycle status, cancellation, and grading. """ @abstractmethod async def __call__(self, run: Run) -> None: - """Drive ``run`` to completion, filling ``run.trace`` (answer is ``trace.content``).""" + """Fill ``run.trace`` with the trajectory and final answer.""" diff --git a/hud/agents/browser_use/agent.py b/hud/agents/browser_use/agent.py index c9d3c8869..43cae6d37 100644 --- a/hud/agents/browser_use/agent.py +++ b/hud/agents/browser_use/agent.py @@ -24,7 +24,6 @@ from hud.agents.base import Agent from hud.agents.types import AgentStep, BrowserUseConfig from hud.settings import settings -from hud.types import Step if TYPE_CHECKING: from hud.eval.run import Run @@ -107,18 +106,12 @@ async def __call__(self, run: Run) -> None: try: history = await sdk_agent.run(max_steps=self.config.max_steps) - except Exception as exc: - LOGGER.exception("browser-use run failed") - trace.status = "error" - run.record(Step(source="system", error=str(exc))) - return finally: with contextlib.suppress(Exception): await browser.stop() successful = history.is_successful() content = history.final_result() or "" - trace.status = "error" if successful is False else "completed" trace.content = content trace.extra.update( { @@ -133,7 +126,6 @@ async def __call__(self, run: Run) -> None: AgentStep( content=content, done=history.is_done(), - error=content if successful is False else None, ), ) diff --git a/hud/agents/claude/cli/agent.py b/hud/agents/claude/cli/agent.py index 34c462ac9..676d3e001 100644 --- a/hud/agents/claude/cli/agent.py +++ b/hud/agents/claude/cli/agent.py @@ -20,16 +20,18 @@ from hud.agents.types import ClaudeCLIConfig, ToolStep from hud.settings import settings from hud.telemetry.context import get_current_trace_id -from hud.types import MCPToolCall, MCPToolResult, Step +from hud.types import MCPToolCall, MCPToolResult from hud.utils.time import now_iso +from . import computer_mcp + if TYPE_CHECKING: from hud.capabilities import SSHClient from hud.eval.run import Run logger = logging.getLogger(__name__) -WINDOWS_SHELLS = ("cmd", "powershell") +_WINDOWS_SHELLS = ("cmd", "powershell") _PROMPT_PATH = ".hud_prompt.txt" _MCP_CONFIG_PATH = ".hud_mcp_config.json" _RUN_SCRIPT_PATH = ".hud_run.bat" @@ -44,7 +46,6 @@ def __init__(self, run: Run, *, started_at: str) -> None: self._agent_started_at = started_at self._pending_calls: dict[str, tuple[MCPToolCall, str]] = {} self._last_agent_content = "" - self.message_count = 0 self._saw_result = False self._result_error: str | None = None @@ -55,7 +56,6 @@ def feed_line(self, line: str) -> None: message = json.loads(line) if not isinstance(message, dict): raise ValueError("Claude stream event must be an object") - self.message_count += 1 received_at = now_iso() match message.get("type"): case "system" if message.get("subtype") == "init": @@ -81,21 +81,13 @@ def finish(self, *, returncode: int, stderr: str) -> None: if not trace.content and self._last_agent_content: trace.content = self._last_agent_content - if error is None: - return - trace.status = "error" - if stderr: + if error is not None and stderr: trace.extra["stderr"] = stderr - timestamp = now_iso() - self._run.record( - Step(source="system", error=error, started_at=timestamp, ended_at=timestamp) - ) + if error is not None: + raise RuntimeError(error) def _record_assistant(self, event: dict[str, Any], received_at: str) -> None: - raw_message = event.get("message") - if not isinstance(raw_message, dict): - raise ValueError("Claude assistant event is missing its message payload") - message = BetaMessage.model_validate(raw_message) + message = BetaMessage.model_validate(event["message"]) step = ClaudeAgent._message_to_agent_step(message) step.started_at = self._agent_started_at step.ended_at = received_at @@ -117,8 +109,7 @@ def _record_tool_results(self, event: dict[str, Any], received_at: str) -> None: for raw_block in content: if not isinstance(raw_block, dict) or raw_block.get("type") != "tool_result": continue - block = raw_block - call_id = block.get("tool_use_id") + call_id = raw_block.get("tool_use_id") if not isinstance(call_id, str): raise ValueError("Claude tool result is missing tool_use_id") try: @@ -128,7 +119,7 @@ def _record_tool_results(self, event: dict[str, Any], received_at: str) -> None: f"Claude returned a result for unknown tool call {call_id!r}" ) from None - raw_result = block.get("content") + raw_result = raw_block.get("content") raw_items = raw_result if isinstance(raw_result, list) else [raw_result] result_content: list[mcp_types.ContentBlock] = [] for item in raw_items: @@ -155,7 +146,7 @@ def _record_tool_results(self, event: dict[str, Any], received_at: str) -> None: result=MCPToolResult( call_id=call_id, content=result_content, - isError=block.get("is_error") is True, + isError=raw_block.get("is_error") is True, ), started_at=started_at, ended_at=received_at, @@ -169,9 +160,7 @@ def _record_result(self, event: dict[str, Any]) -> None: trace = self._run.trace result = event.get("result") trace.content = result if isinstance(result, str) else self._last_agent_content - is_error = event.get("is_error") is True - trace.status = "error" if is_error else "completed" - if is_error: + if event.get("is_error") is True: self._result_error = trace.content or "claude CLI reported an error" for key in ( "subtype", @@ -211,13 +200,10 @@ def __init__(self, config: ClaudeCLIConfig | None = None) -> None: async def __call__(self, run: Run) -> None: mcp_servers: dict[str, dict[str, Any]] = {} - manifest = run.client.manifest - bindings = manifest.bindings if manifest is not None else [] - families = {c.protocol.split("/", 1)[0] for c in bindings} - - if "ssh" not in families: - raise RuntimeError("ClaudeCLIAgent requires an SSH capability") ssh = cast("SSHClient", await run.client.open("ssh")) + manifest = run.client.manifest + assert manifest is not None + bindings = manifest.bindings shell = ssh.capability.params.get("shell", "bash") rfb_bindings = [cap for cap in bindings if cap.protocol.split("/", 1)[0] == "rfb"] @@ -234,8 +220,6 @@ async def __call__(self, run: Run) -> None: raise RuntimeError(f"duplicate MCP server name {cap.name!r}") mcp_servers[cap.name] = server_config elif family == "rfb": - from hud.agents.claude.cli.computer_mcp import bridge_computer_mcp - server_name = ( "computer-use" if len(rfb_bindings) == 1 else f"computer-use-{cap.name}" ) @@ -243,7 +227,7 @@ async def __call__(self, run: Run) -> None: raise RuntimeError(f"duplicate MCP server name {server_name!r}") routed = run.client.binding(cap.name) mcp_servers[server_name] = await resources.enter_async_context( - bridge_computer_mcp( + computer_mcp.bridge_computer_mcp( ssh, routed, self.config.screenshot_encoding, @@ -269,11 +253,10 @@ async def _run_cli( prompt: str, ) -> None: files: dict[str, str] = {} - mcp_config_path = None + mcp_config_path = _MCP_CONFIG_PATH if mcp_servers else None if mcp_servers: - mcp_config_path = _MCP_CONFIG_PATH - files[mcp_config_path] = json.dumps({"mcpServers": mcp_servers}, indent=2) - if shell in WINDOWS_SHELLS: + files[_MCP_CONFIG_PATH] = json.dumps({"mcpServers": mcp_servers}, indent=2) + if shell in _WINDOWS_SHELLS: files[_PROMPT_PATH] = prompt command = self._build_command( @@ -281,7 +264,7 @@ async def _run_cli( prompt=prompt, mcp_config_path=mcp_config_path, ) - if shell in WINDOWS_SHELLS: + if shell in _WINDOWS_SHELLS: files[_RUN_SCRIPT_PATH] = f"@echo off\r\n{command}\r\n" command = f"cmd /c {_RUN_SCRIPT_PATH}" @@ -292,7 +275,7 @@ async def _run_cli( await self._stream_cli(run, ssh, command) finally: if files: - if shell in WINDOWS_SHELLS: + if shell in _WINDOWS_SHELLS: cleanup = f"cmd /c del /f /q {' '.join(files)} 2>nul" else: cleanup = "rm -f -- " + " ".join(shlex.quote(path) for path in files) @@ -324,12 +307,7 @@ async def _stream_cli(self, run: Run, ssh: SSHClient, command: str) -> None: returncode = process.returncode if returncode is None: raise RuntimeError("claude CLI process closed without an exit status") - logger.info( - "exit=%s events=%d stderr=%d", - returncode, - parser.message_count, - len(stderr), - ) + logger.info("exit=%s stderr=%d", returncode, len(stderr)) parser.finish(returncode=returncode, stderr=stderr) def _build_command( @@ -360,10 +338,13 @@ def _build_command( # When using a custom base URL, alias all model tiers to the same model # so the CLI doesn't try to reach Anthropic for background requests. if "ANTHROPIC_BASE_URL" in env: - env["ANTHROPIC_DEFAULT_SONNET_MODEL"] = self.config.model - env["ANTHROPIC_DEFAULT_OPUS_MODEL"] = self.config.model - env["ANTHROPIC_DEFAULT_HAIKU_MODEL"] = self.config.model - env["CLAUDE_CODE_SUBAGENT_MODEL"] = self.config.model + for name in ( + "ANTHROPIC_DEFAULT_SONNET_MODEL", + "ANTHROPIC_DEFAULT_OPUS_MODEL", + "ANTHROPIC_DEFAULT_HAIKU_MODEL", + "CLAUDE_CODE_SUBAGENT_MODEL", + ): + env[name] = self.config.model env["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC"] = "1" env["IS_SANDBOX"] = "1" @@ -384,7 +365,7 @@ def _build_command( if mcp_config_path: base_args.extend(["--mcp-config", mcp_config_path]) - if shell in WINDOWS_SHELLS: + if shell in _WINDOWS_SHELLS: script = ";".join( [ *(f"$env:{key}={_powershell_quote(value)}" for key, value in env.items()), diff --git a/hud/agents/claude/cli/computer_mcp.py b/hud/agents/claude/cli/computer_mcp.py index 36cd8dab4..8ff30a065 100644 --- a/hud/agents/claude/cli/computer_mcp.py +++ b/hud/agents/claude/cli/computer_mcp.py @@ -19,8 +19,11 @@ import asyncssh import fastmcp +from fastmcp.exceptions import ToolError from pydantic import TypeAdapter +from hud.agents.claude.tools.computer import ClaudeComputerTool +from hud.agents.tools.base import AgentToolSpec, result_text from hud.capabilities import Capability from hud.capabilities.rfb import RFBClient, ScreenshotEncoding, WebPScreenshotEncoding @@ -46,18 +49,23 @@ def create_computer_mcp( """Build a FastMCP server with one ``computer`` tool backed by ``rfb``.""" mcp = fastmcp.FastMCP("computer-use") + tool = ClaudeComputerTool( + spec=AgentToolSpec(api_type="computer", api_name="computer"), + client=rfb, + screenshot_encoding=screenshot_encoding, + ) @mcp.tool() async def computer( action: str, - coordinate: str | None = None, + coordinate: list[int] | None = None, text: str | None = None, scroll_direction: str | None = None, scroll_amount: int | None = None, - start_coordinate: str | None = None, + start_coordinate: list[int] | None = None, duration: float | None = None, repeat: int | None = None, - region: str | None = None, + region: list[int] | None = None, ) -> list[Any]: """Control a remote screen — screenshot, click, type, key, scroll, move, drag, wait, zoom. @@ -67,76 +75,32 @@ async def computer( Returns the resulting screenshot image so you can see the screen state. """ - import mcp.types as mcp_types - - from hud.agents.claude.tools.computer import ClaudeComputerTool - from hud.agents.tools.base import AgentToolSpec - - arguments: dict[str, Any] = {"action": action} - if coordinate is not None: - try: - arguments["coordinate"] = json.loads(coordinate) - except json.JSONDecodeError: - arguments["coordinate"] = coordinate - if text is not None: - arguments["text"] = text - if scroll_direction is not None: - arguments["scroll_direction"] = scroll_direction - if scroll_amount is not None: - arguments["scroll_amount"] = scroll_amount - if start_coordinate is not None: - try: - arguments["start_coordinate"] = json.loads(start_coordinate) - except json.JSONDecodeError: - arguments["start_coordinate"] = start_coordinate - if duration is not None: - arguments["duration"] = duration - if repeat is not None: - arguments["repeat"] = repeat - if region is not None: - try: - arguments["region"] = json.loads(region) - except json.JSONDecodeError: - arguments["region"] = region - - spec = AgentToolSpec(api_type="computer", api_name="computer") - tool = ClaudeComputerTool( - spec=spec, - client=rfb, - screenshot_encoding=screenshot_encoding, - ) + arguments = { + name: value + for name, value in { + "action": action, + "coordinate": coordinate, + "text": text, + "scroll_direction": scroll_direction, + "scroll_amount": scroll_amount, + "start_coordinate": start_coordinate, + "duration": duration, + "repeat": repeat, + "region": region, + }.items() + if value is not None + } result = await tool.execute(arguments) - - # Return content blocks directly so the CLI/model sees real images. - blocks: list[Any] = [] - for block in result.content: - if isinstance(block, mcp_types.ImageContent): - blocks.append( - mcp_types.ImageContent( - type="image", - data=block.data, - mimeType=block.mimeType, - ), - ) - elif isinstance(block, mcp_types.TextContent): - blocks.append(mcp_types.TextContent(type="text", text=block.text)) - if not blocks: - blocks.append(mcp_types.TextContent(type="text", text="ok")) if result.isError: - blocks.insert(0, mcp_types.TextContent(type="text", text="ERROR")) - return blocks + raise ToolError(result_text(result) or "computer action failed") + return result.content return mcp async def run_computer_mcp(environ: Mapping[str, str] = os.environ) -> None: """Run computer-use over stdio in a controller-side child process.""" - raw_manifest = json.loads(environ[RFB_CAPABILITY_ENV]) - if not isinstance(raw_manifest, dict): - raise ValueError(f"{RFB_CAPABILITY_ENV} must contain a JSON object") - capability = Capability.from_manifest(raw_manifest) - if capability.protocol.split("/", 1)[0] != "rfb": - raise ValueError(f"{RFB_CAPABILITY_ENV} must describe an RFB capability") + capability = Capability.from_manifest(json.loads(environ[RFB_CAPABILITY_ENV])) screenshot_encoding = TypeAdapter(ScreenshotEncoding).validate_json( environ[SCREENSHOT_ENCODING_ENV] ) diff --git a/hud/agents/openai_compatible/agent.py b/hud/agents/openai_compatible/agent.py index c2aba7641..08094f578 100644 --- a/hud/agents/openai_compatible/agent.py +++ b/hud/agents/openai_compatible/agent.py @@ -149,23 +149,16 @@ async def get_response( if return_token_ids: request_kwargs.setdefault("logprobs", True) - try: - response: ChatCompletion = await self.oai.chat.completions.create( - model=self.config.model, - messages=( - [{"role": "system", "content": system_prompt}, *messages] - if system_prompt is not None - else messages - ), - stream=False, - **request_kwargs, - ) - except Exception as e: - error_content = f"Error getting response {e}" - if "Invalid JSON" in str(e): - error_content = "Invalid JSON, response was truncated" - logger.warning(error_content) - return AgentStep(error=error_content, done=True) + response: ChatCompletion = await self.oai.chat.completions.create( + model=self.config.model, + messages=( + [{"role": "system", "content": system_prompt}, *messages] + if system_prompt is not None + else messages + ), + stream=False, + **request_kwargs, + ) choice = response.choices[0] message = choice.message diff --git a/hud/agents/registry.py b/hud/agents/registry.py index 10c6f2910..3ccb3e9f3 100644 --- a/hud/agents/registry.py +++ b/hud/agents/registry.py @@ -5,7 +5,7 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Any -from hud.agents.types import AgentConfig +from hud.agents.types import AgentConfig, ToolAgentConfig from hud.types import AgentType if TYPE_CHECKING: @@ -24,7 +24,7 @@ def dump_agent(agent: Agent) -> dict[str, Any]: f"({', '.join(member.value for member in AgentType)}); " f"got {type(agent).__name__}" ) - if config.model_client is not None: + if isinstance(config, ToolAgentConfig) and config.model_client is not None: raise ValueError( "agents with a custom model_client cannot run remotely; use HUDRuntime or LocalRuntime" ) diff --git a/hud/agents/robot/agent.py b/hud/agents/robot/agent.py index c9f27da10..e10454bcc 100644 --- a/hud/agents/robot/agent.py +++ b/hud/agents/robot/agent.py @@ -114,7 +114,6 @@ async def __call__(self, run: Run, *, max_steps: int | None = None) -> None: writer.end_episode() finally: await robot.close() - run.trace.status = "completed" run.trace.content = "done" async def _loop( diff --git a/hud/agents/tests/test_claude_cli_agent.py b/hud/agents/tests/test_claude_cli_agent.py index 3d3b25ff9..cb9ff492a 100644 --- a/hud/agents/tests/test_claude_cli_agent.py +++ b/hud/agents/tests/test_claude_cli_agent.py @@ -19,6 +19,7 @@ from typing import TYPE_CHECKING, Any, Literal, cast from unittest.mock import AsyncMock, Mock +import fastmcp import pytest from mcp.types import ImageContent, TextContent @@ -29,6 +30,7 @@ from hud.capabilities.rfb import WebPScreenshotEncoding from hud.settings import settings from hud.telemetry.context import set_trace_context +from hud.types import MCPToolResult if TYPE_CHECKING: from pathlib import Path @@ -195,7 +197,7 @@ async def create_process(self, cmd: str, **kwargs: Any) -> Any: def _fake_run() -> Any: - trace = SimpleNamespace(status="", content="", extra={}) + trace = SimpleNamespace(status=None, content="", extra={}) steps: list[Any] = [] return SimpleNamespace(trace=trace, record=steps.append, steps=steps) @@ -244,7 +246,7 @@ async def test_exec_on_windows_writes_batch_and_execs_via_cmd() -> None: assert conn.written[".hud_prompt.txt"] == b"build it" assert sink == {} assert set(conn.deleted) == {".hud_prompt.txt", ".hud_run.bat"} - assert run.trace.status == "completed" + assert run.trace.status is None assert run.trace.content == "done" assert "messages" not in run.trace.extra @@ -263,7 +265,7 @@ async def test_exec_on_bash_runs_inline_without_batch() -> None: assert conn.deleted == [] assert len(conn.ran) == 1 assert "claude" in conn.ran[0] - assert run.trace.status == "completed" + assert run.trace.status is None assert run.trace.content == "done" assert "messages" not in run.trace.extra @@ -323,7 +325,7 @@ async def test_exec_records_steps_before_process_exit() -> None: assert image.data == "aW1hZ2U=" final = cast("AgentStep", run.steps[2]) assert final.started_at == tool.ended_at - assert run.trace.status == "completed" + assert run.trace.status is None assert run.trace.content == "done" @@ -381,18 +383,17 @@ async def test_exec_closes_streaming_process_when_cancelled() -> None: assert process.closed -async def test_exec_nonzero_exit_with_no_stdout_records_system_error() -> None: +async def test_exec_nonzero_exit_with_no_stdout_raises() -> None: sink: dict[str, bytes] = {} conn = _FakeConn(sink, _FakeStreamProcess("", stderr="boom", exit_status=1)) agent = ClaudeCLIAgent() ssh = _ssh_with_conn("cmd", conn) run = _fake_run() - await agent._run_cli(run, ssh=ssh, shell="cmd", mcp_servers={}, prompt="x") + with pytest.raises(RuntimeError, match="boom"): + await agent._run_cli(run, ssh=ssh, shell="cmd", mcp_servers={}, prompt="x") - assert run.trace.status == "error" assert run.trace.extra["returncode"] == 1 - assert run.steps[0].error == "boom" async def test_exec_signal_exit_records_the_returncode() -> None: @@ -405,11 +406,10 @@ async def test_exec_signal_exit_records_the_returncode() -> None: ssh = _ssh_with_conn("bash", conn) run = _fake_run() - await agent._run_cli(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="x") + with pytest.raises(RuntimeError, match="return code -15"): + await agent._run_cli(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="x") - assert run.trace.status == "error" assert run.trace.extra["returncode"] == -15 - assert run.steps[0].error == "claude CLI exited with return code -15" async def test_exec_nonzero_exit_with_result_stream_remains_an_error() -> None: @@ -422,14 +422,13 @@ async def test_exec_nonzero_exit_with_result_stream_remains_an_error() -> None: ssh = _ssh_with_conn("bash", conn) run = _fake_run() - await agent._run_cli(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="x") + with pytest.raises(RuntimeError, match="transport failed"): + await agent._run_cli(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="x") - assert run.trace.status == "error" assert run.trace.content == "done" assert run.trace.extra["returncode"] == 1 assert run.trace.extra["stderr"] == "transport failed" assert "messages" not in run.trace.extra - assert run.steps[-1].error == "transport failed" async def test_exec_zero_exit_without_result_event_is_an_error() -> None: @@ -440,11 +439,10 @@ async def test_exec_zero_exit_without_result_event_is_an_error() -> None: ssh = _ssh_with_conn("bash", conn) run = _fake_run() - await agent._run_cli(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="x") + with pytest.raises(RuntimeError, match="without a result event"): + await agent._run_cli(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="x") - assert run.trace.status == "error" assert run.trace.content == "done" - assert run.steps[-1].error == "claude CLI exited without a result event" @pytest.mark.parametrize( @@ -668,6 +666,26 @@ async def test_computer_mcp_stdio_owns_rfb_lifetime( rfb.close.assert_awaited_once() +async def test_computer_mcp_preserves_tool_result(monkeypatch: pytest.MonkeyPatch) -> None: + result = MCPToolResult( + content=[TextContent(type="text", text="failed")], + isError=True, + ) + execute = AsyncMock(return_value=result) + monkeypatch.setattr(computer_mcp.ClaudeComputerTool, "execute", execute) + server = computer_mcp.create_computer_mcp(cast("Any", object())) + + async with fastmcp.Client(server) as client: + received = await client.call_tool_mcp( + "computer", + {"action": "left_click", "coordinate": [10, 20]}, + ) + + execute.assert_awaited_once_with({"action": "left_click", "coordinate": [10, 20]}) + assert received.isError is True + assert received.content == result.content + + class _ByteWriter: def __init__(self) -> None: self.closed = False diff --git a/hud/agents/tests/test_openai_compatible_agent.py b/hud/agents/tests/test_openai_compatible_agent.py index aa8d09114..ea607d773 100644 --- a/hud/agents/tests/test_openai_compatible_agent.py +++ b/hud/agents/tests/test_openai_compatible_agent.py @@ -5,6 +5,8 @@ from types import SimpleNamespace from typing import Any, cast +import pytest + from hud.agents.openai_compatible.agent import OpenAIChatAgent, OpenAIChatRunState from hud.agents.types import OpenAIChatConfig @@ -87,9 +89,8 @@ async def test_get_response_with_tool_call() -> None: async def test_get_response_error_path() -> None: agent = _agent(None, error=RuntimeError("boom")) - result = await agent.get_response(_state(agent)) - assert result.done is True - assert result.error is not None and "boom" in result.error + with pytest.raises(RuntimeError, match="boom"): + await agent.get_response(_state(agent)) async def test_get_response_malformed_tool_args() -> None: diff --git a/hud/agents/tests/test_tool_agent.py b/hud/agents/tests/test_tool_agent.py index ab9f91089..9758042da 100644 --- a/hud/agents/tests/test_tool_agent.py +++ b/hud/agents/tests/test_tool_agent.py @@ -19,13 +19,20 @@ from hud.agents.claude.agent import ClaudeAgent from hud.agents.claude.tools.coding import ClaudeBashTool, ClaudeTextEditorTool from hud.agents.openai.tools.coding import OpenAIShellTool +from hud.agents.openai.tools.computer import OpenAIComputerTool from hud.agents.openai.tools.mcp_proxy import OpenAIMCPProxyTool from hud.agents.tool_agent import RunState, ToolAgent from hud.agents.tools.base import AgentToolSpec, result_text from hud.agents.tools.rfb import RFBTool from hud.agents.tools.ssh import SSHInfrastructureErrorResult -from hud.agents.types import AgentConfig, AgentStep, ClaudeCLIConfig, ClaudeConfig, ToolStep -from hud.capabilities import Capability, CapabilityClient, MCPClient, RFBClient, SSHClient +from hud.agents.types import ( + AgentStep, + ClaudeCLIConfig, + ClaudeConfig, + ToolAgentConfig, + ToolStep, +) +from hud.capabilities import Capability, CapabilityClient, MCPClient, RFBClient from hud.capabilities.rfb import PngScreenshotEncoding, WebPScreenshotEncoding from hud.capabilities.ssh import SSHConnectionError from hud.types import MCPToolCall, MCPToolResult, Step, Trace @@ -46,11 +53,11 @@ def record(self, step: Step) -> None: self.trace.record(step) -class DictAgent(ToolAgent[_Msg, AgentConfig]): +class DictAgent(ToolAgent[_Msg, ToolAgentConfig]): """Minimal concrete ToolAgent over plain-dict messages.""" def __init__(self, turns: list[AgentStep], **config: Any) -> None: - self.config = AgentConfig(model="test-model", **config) + self.config = ToolAgentConfig(model="test-model", **config) self._turns = list(turns) async def _initialize_state(self, *, prompt: Any) -> RunState[_Msg]: @@ -70,18 +77,8 @@ def _format_result( return {"role": "tool", "name": call.name, "isError": result.isError} -# ─── catalog → clients derivation ───────────────────────────────────── - - -def test_init_subclass_derives_clients_from_catalog() -> None: - class WithCatalog(DictAgent): - tool_catalog = (OpenAIShellTool,) - - assert WithCatalog.clients == (SSHClient,) - - def test_claude_defaults_to_configurable_webp_screenshots() -> None: - assert AgentConfig().screenshot_encoding == PngScreenshotEncoding() + assert ToolAgentConfig().screenshot_encoding == PngScreenshotEncoding() assert ClaudeConfig().screenshot_encoding == WebPScreenshotEncoding() assert ClaudeCLIConfig().screenshot_encoding == WebPScreenshotEncoding() @@ -151,7 +148,7 @@ async def open(self, ref: str) -> CapabilityClient: return cast("CapabilityClient", object()) class MultiMCPAgent(DictAgent): - clients = (MCPClient,) + tool_catalog = (OpenAIMCPProxyTool,) class LiveRun(_FakeRun): def __init__(self) -> None: @@ -179,7 +176,7 @@ async def open(self, ref: str) -> CapabilityClient: return cast("CapabilityClient", object()) class ComputerAgent(DictAgent): - clients = (RFBClient,) + tool_catalog = (OpenAIComputerTool,) class LiveRun(_FakeRun): def __init__(self) -> None: @@ -207,7 +204,7 @@ async def open(self, ref: str) -> CapabilityClient: return cast("CapabilityClient", object()) class MCPAndShellAgent(DictAgent): - clients = (MCPClient, SSHClient) + tool_catalog = (OpenAIMCPProxyTool, OpenAIShellTool) class LiveRun(_FakeRun): def __init__(self) -> None: @@ -351,19 +348,12 @@ class MultiMCPAgent(DictAgent): ) -# ─── initial messages / user text formatting ────────────────────────── - - def test_initial_messages_formats_each_turn() -> None: agent = DictAgent([]) turn = mcp_types.PromptMessage( role="user", content=mcp_types.TextContent(type="text", text="a") ) assert agent._initial_messages([turn]) == [{"role": "user", "content": "a"}] - assert agent._format_user_text("hey") == {"role": "user", "content": "hey"} - - -# ─── dispatch + loop ────────────────────────────────────────────────── async def test_dispatch_unknown_tool_returns_error_result() -> None: @@ -516,9 +506,9 @@ async def test_loop_finishes_on_done_response() -> None: agent = DictAgent([AgentStep(content="final answer", done=True)]) run = cast("Run", _FakeRun()) - await agent._loop(run, RunState(), max_steps=3) + await agent._loop(run, RunState()) - assert run.trace.status == "completed" + assert run.trace.status is None assert run.trace.content == "final answer" assert run.trace.is_error is False assert run.trace.stop_reason == "done" @@ -541,7 +531,7 @@ async def test_loop_dispatches_tool_calls_then_finishes() -> None: ) run = cast("Run", _FakeRun()) - await agent._loop(run, RunState(), max_steps=3) + await agent._loop(run, RunState()) assert run.trace.content == "done now" assert [step.source for step in run.trace.steps] == ["agent", "tool", "agent"] @@ -575,12 +565,11 @@ async def test_loop_resets_infrastructure_error_count_after_other_result( monkeypatch.setattr(agent, "_dispatch_call", dispatch) run = cast("Run", _FakeRun()) - await agent._loop(run, RunState(), max_steps=10) + with pytest.raises(RuntimeError, match="SSH tool failure limit reached"): + await agent._loop(run, RunState()) assert dispatch.await_count == 5 - assert run.trace.status == "error" assert run.trace.stop_reason is None - assert run.trace.error == ("SSH tool failure limit reached after 3 consecutive errors") async def test_loop_max_steps_is_normal_termination() -> None: @@ -590,13 +579,13 @@ async def test_loop_max_steps_is_normal_termination() -> None: never_done = [ AgentStep(content="", done=False, tool_calls=[MCPToolCall(name="ghost")]) for _ in range(5) ] - agent = DictAgent(never_done) + agent = DictAgent(never_done, max_steps=2) run = cast("Run", _FakeRun()) - await agent._loop(run, RunState(), max_steps=2) + await agent._loop(run, RunState()) assert run.trace.is_error is False - assert run.trace.status == "completed" + assert run.trace.status is None assert run.trace.stop_reason == "max_steps" assert run.trace.is_truncated is True # No synthetic error step — the trajectory ends on the real agent/tool steps. @@ -611,9 +600,9 @@ async def test_loop_marks_length_finish_as_truncated() -> None: agent = DictAgent([AgentStep(content="partial", done=True, finish_reason=finish_reason)]) run = cast("Run", _FakeRun()) - await agent._loop(run, RunState(), max_steps=3) + await agent._loop(run, RunState()) - assert run.trace.status == "completed" + assert run.trace.status is None assert run.trace.stop_reason == "length" assert run.trace.is_truncated is True @@ -632,7 +621,7 @@ async def test_loop_answers_malformed_call_by_default() -> None: ) run = cast("Run", _FakeRun()) - await agent._loop(run, RunState(), max_steps=3) + await agent._loop(run, RunState()) assert run.trace.content == "recovered" tool_step = run.trace.steps[1] @@ -656,9 +645,9 @@ async def test_loop_stops_on_malformed_call_when_configured() -> None: ) run = cast("Run", _FakeRun()) - await agent._loop(run, RunState(), max_steps=3) + await agent._loop(run, RunState()) - assert run.trace.status == "completed" + assert run.trace.status is None assert run.trace.stop_reason == "malformed_tool_call" assert run.trace.is_truncated is True assert all(not isinstance(step, ToolStep) for step in run.trace.steps) @@ -679,7 +668,7 @@ async def test_loop_stops_on_length_when_configured() -> None: ) run = cast("Run", _FakeRun()) - await agent._loop(run, RunState(), max_steps=3) + await agent._loop(run, RunState()) assert run.trace.stop_reason == "length" assert run.trace.is_truncated is True diff --git a/hud/agents/tool_agent.py b/hud/agents/tool_agent.py index 83cab8710..75b7546de 100644 --- a/hud/agents/tool_agent.py +++ b/hud/agents/tool_agent.py @@ -1,19 +1,4 @@ -"""ToolAgent: catalog-driven provider tool-call loop. - -Subclass contract:: - - class ClaudeAgent(ToolAgent[BetaMessageParam, ClaudeConfig]): - tool_catalog = (ClaudeBashTool, ClaudeTextEditorTool, ClaudeMCPProxyTool) - - async def _initialize_state(self, *, prompt) -> RunState[BetaMessageParam]: ... - async def get_response(self, state, *, system_prompt, citations_enabled): ... - def _format_message(self, role, text) -> BetaMessageParam: ... - def _format_result(self, call, result) -> BetaMessageParam | None: ... - -``RunState`` carries the messages *and* the tools/params built for one run, so a -single agent instance can drive many concurrent ``rollout`` calls with no shared -mutable state. -""" +"""Catalog-driven provider tool-call agents.""" from __future__ import annotations @@ -38,7 +23,7 @@ def _format_result(self, call, result) -> BetaMessageParam | None: ... from hud.utils.time import now_iso if TYPE_CHECKING: - from hud.agents.types import AgentConfig + from hud.agents.types import ToolAgentConfig from hud.capabilities import CapabilityClient from hud.eval.run import Run @@ -50,24 +35,17 @@ def _format_result(self, call, result) -> BetaMessageParam | None: ... MAX_CONSECUTIVE_SSH_FAILURES = 3 MessageT = TypeVar("MessageT") -ConfigT = TypeVar("ConfigT", bound="AgentConfig") +ConfigT = TypeVar("ConfigT", bound="ToolAgentConfig") def _message_text(message: mcp_types.PromptMessage) -> str: - """Best-effort plain text for a prompt message (text content only for now).""" content = message.content - if isinstance(content, mcp_types.TextContent): - return content.text - return getattr(content, "text", "") or "" + return content.text if isinstance(content, mcp_types.TextContent) else "" @dataclass class RunState(Generic[MessageT]): - """Mutable per-run state: messages + the tools/params built for this run. - - Created fresh per ``rollout`` (or ``run``) call, so one agent instance can - drive many concurrent rollouts without shared mutable state. - """ + """Provider messages and tools for one run.""" messages: list[MessageT] = field(default_factory=list[MessageT]) tools: dict[str, AgentTool[Any]] = field(default_factory=dict[str, AgentTool[Any]]) @@ -78,52 +56,24 @@ class ToolAgent(Agent, Generic[MessageT, ConfigT]): """Catalog-driven provider tool-call loop.""" tool_catalog: ClassVar[tuple[type[AgentTool[Any]], ...]] = () - #: Capability-client types this agent can drive (derived from the catalog). - clients: ClassVar[tuple[type[CapabilityClient], ...]] = () - - #: The agent's typed config; set by subclass __init__. config: ConfigT - def __init_subclass__(cls, **kwargs: Any) -> None: - super().__init_subclass__(**kwargs) - if "tool_catalog" in cls.__dict__: - seen: dict[type[CapabilityClient], None] = {} - for t in cls.tool_catalog: - seen.setdefault(t.client_type, None) - cls.clients = tuple(seen.keys()) - async def __call__(self, run: Run) -> None: - """Drive this (stateless) agent over a live ``Run``, filling ``run.trace``. - - Opens the capabilities this agent's catalog supports off the connection, - builds the tools into a fresh ``RunState``, - then runs the loop against ``run.prompt_messages``, accumulating the - trajectory onto ``run.trace``. Loop budget and prompting come from the agent's config - (``max_steps``, ``system_prompt``, ``citations_enabled``). No per-rollout - state is stored on ``self``, so one instance may drive many concurrent - rollouts. - """ connections: dict[str, CapabilityClient] = {} opened_protocols: set[str] = set() manifest = run.client.manifest - if manifest is not None: - wanted = {cls.protocol for cls in type(self).clients} - for cap in manifest.bindings: - if cap.protocol not in wanted: - continue - if cap.protocol != MCPClient.protocol and cap.protocol in opened_protocols: - continue - connections[cap.name] = await run.client.open(cap.name) - opened_protocols.add(cap.protocol) + assert manifest is not None + wanted = {tool.client_type.protocol for tool in self.tool_catalog} + for cap in manifest.bindings: + if cap.protocol not in wanted: + continue + if cap.protocol != MCPClient.protocol and cap.protocol in opened_protocols: + continue + connections[cap.name] = await run.client.open(cap.name) + opened_protocols.add(cap.protocol) state = await self._initialize_state(prompt=run.prompt_messages) state.tools, state.params = await self._build_tools(connections) - await self._loop( - run, - state, - max_steps=self.config.max_steps, - system_prompt=self.config.system_prompt, - citations_enabled=self.config.citations_enabled, - ) + await self._loop(run, state) async def _build_tools( self, @@ -133,16 +83,15 @@ async def _build_tools( tools: dict[str, AgentTool[Any]] = {} params: list[Any] = [] model = self.config.model - hosted_tools = self.config.hosted_tools mcp_clients = [c for c in connections.values() if isinstance(c, MCPClient)] mcp_lists = await asyncio.gather(*(c.list_tools() for c in mcp_clients)) mcp_by_client: dict[MCPClient, list[mcp_types.Tool]] = dict( - zip(mcp_clients, mcp_lists, strict=False), + zip(mcp_clients, mcp_lists, strict=True), ) qualify_mcp_names = len(mcp_clients) > 1 - for tool_cls in type(self).tool_catalog: + for tool_cls in self.tool_catalog: spec = tool_cls.default_spec(model) if spec is None: continue @@ -181,7 +130,11 @@ async def _build_tools( tools[tool.provider_name] = tool params.append(tool.to_params()) - params.extend(hosted.to_params() for hosted in hosted_tools if hosted.supports_model(model)) + params.extend( + hosted.to_params() + for hosted in self.config.hosted_tools + if hosted.supports_model(model) + ) return tools, params @@ -189,92 +142,72 @@ async def _loop( self, run: Run, state: RunState[MessageT], - *, - max_steps: int = 10, - system_prompt: str | None = None, - citations_enabled: bool = False, ) -> None: trace = run.trace - try: - step: AgentStep | None = None - hit_max = False - stopped: StopCondition | None = None - consecutive_ssh_failures = 0 - - for turn in range(1, max_steps + 1): - logger.info("step %d/%d", turn, max_steps) - started_at = now_iso() - step = await self.get_response( - state, - system_prompt=system_prompt, - citations_enabled=citations_enabled, + step: AgentStep | None = None + consecutive_ssh_failures = 0 + + for turn in range(1, self.config.max_steps + 1): + logger.info("step %d/%d", turn, self.config.max_steps) + started_at = now_iso() + step = await self.get_response( + state, + system_prompt=self.config.system_prompt, + citations_enabled=self.config.citations_enabled, + ) + step.started_at = step.started_at or started_at + step.model = step.model or self.config.model + run.record(step) + if step.error: + raise RuntimeError(step.error) + + if step.tool_calls: + logger.info(" → %s", ", ".join(c.name for c in step.tool_calls)) + + if step.done or not step.tool_calls: + follow_up = await auto_respond(step.content, enabled=self.config.auto_respond) + if follow_up is not None: + text = ( + follow_up.content.text + if isinstance(follow_up.content, mcp_types.TextContent) + else "" + ) + state.messages.append(self._format_message("user", text)) + run.record(Step(source="user", messages=[follow_up])) + continue + trace.stop_reason = ( + "length" if step.finish_reason in TRUNCATION_FINISH_REASONS else "done" ) - step.started_at = step.started_at or started_at - step.model = step.model or self.config.model - run.record(step) - - if step.tool_calls: - logger.info(" → %s", ", ".join(c.name for c in step.tool_calls)) - - if step.done or not step.tool_calls: - follow_up = await auto_respond(step.content, enabled=self.config.auto_respond) - if follow_up is not None: - text = ( - follow_up.content.text - if isinstance(follow_up.content, mcp_types.TextContent) - else "" - ) - state.messages.append(self._format_user_text(text)) - run.record(Step(source="user", messages=[follow_up])) - continue - break - - if (stopped := self._stop_condition(step)) is not None: - break - - for call in step.tool_calls: - call_started_at = now_iso() - result = await self._dispatch_call(call, state) - run.record(ToolStep(call=call, result=result, started_at=call_started_at)) - msg = self._format_result(call, result, state) - if isinstance(msg, list): - state.messages.extend(msg) - elif msg is not None: - state.messages.append(cast("MessageT", msg)) - - if isinstance(result, SSHInfrastructureErrorResult): - consecutive_ssh_failures += 1 - else: - consecutive_ssh_failures = 0 - if consecutive_ssh_failures >= MAX_CONSECUTIVE_SSH_FAILURES: - error = ( - "SSH tool failure limit reached " - f"after {MAX_CONSECUTIVE_SSH_FAILURES} consecutive errors" - ) - trace.content = step.content - trace.status = "error" - run.record(Step(source="system", error=error)) - return + break - if turn == max_steps: - hit_max = True - - trace.content = step.content if step else None - trace.status = "error" if step is not None and step.error else "completed" - if stopped is not None: + if stopped := self._stop_condition(step): trace.stop_reason = stopped - elif hit_max: - trace.stop_reason = "max_steps" - elif step is not None and step.finish_reason in TRUNCATION_FINISH_REASONS: - trace.stop_reason = "length" - else: - trace.stop_reason = "done" - except (TimeoutError, asyncio.CancelledError, KeyboardInterrupt): - raise - except Exception as exc: - logger.exception("ToolAgent loop failed") - trace.status = "error" - run.record(Step(source="system", error=str(exc))) + break + + for call in step.tool_calls: + call_started_at = now_iso() + result = await self._dispatch_call(call, state) + run.record(ToolStep(call=call, result=result, started_at=call_started_at)) + msg = self._format_result(call, result, state) + if isinstance(msg, list): + state.messages.extend(msg) + elif msg is not None: + state.messages.append(cast("MessageT", msg)) + + if isinstance(result, SSHInfrastructureErrorResult): + consecutive_ssh_failures += 1 + else: + consecutive_ssh_failures = 0 + if consecutive_ssh_failures >= MAX_CONSECUTIVE_SSH_FAILURES: + trace.content = step.content + raise RuntimeError( + "SSH tool failure limit reached " + f"after {MAX_CONSECUTIVE_SSH_FAILURES} consecutive errors" + ) + else: + trace.stop_reason = "max_steps" + + trace.content = step.content if step else None def _stop_condition(self, step: AgentStep) -> StopCondition | None: """The first configured stop condition this turn trips, if any.""" @@ -351,8 +284,6 @@ async def _dispatch_call( isError=True, ) - # ─── provider hooks ─────────────────────────────────────────────── - def _initial_messages(self, prompt: list[mcp_types.PromptMessage]) -> list[MessageT]: """Map normalized prompt turns onto provider messages.""" return [self._format_message(message.role, _message_text(message)) for message in prompt] @@ -373,14 +304,10 @@ async def get_response( ) -> AgentStep: """Call the provider API and return the model's turn as an ``AgentStep``. - The loop stamps ``started_at``/``model`` fallbacks and records it; - a failed call is an ``AgentStep`` with ``error`` set and ``done=True``. + The loop stamps ``started_at``/``model`` fallbacks, records the step, + and raises its error if present. """ - def _format_user_text(self, text: str) -> MessageT: - """Wrap a plain text string as a provider user message.""" - return self._format_message("user", text) - @abstractmethod def _format_message(self, role: str, text: str) -> MessageT: """Wrap text as a provider message of the given role (``user``/``assistant``).""" diff --git a/hud/agents/types.py b/hud/agents/types.py index 768dd8641..f8103a497 100644 --- a/hud/agents/types.py +++ b/hud/agents/types.py @@ -50,11 +50,16 @@ class AgentConfig(BaseModel): + timeout_seconds: float | None = Field(default=None, gt=0, allow_inf_nan=False) + model_name: str = "Agent" + model: str = Field(default="unknown", validation_alias=_model_alias) + + +class ToolAgentConfig(AgentConfig): model_config = ConfigDict(arbitrary_types_allowed=True) auto_respond: bool = False max_steps: int = 10 - timeout_seconds: float | None = Field(default=None, gt=0, allow_inf_nan=False) tool_timeout_seconds: float | None = Field(default=None, gt=0, allow_inf_nan=False) system_prompt: str | None = None citations_enabled: bool = False @@ -64,8 +69,6 @@ class AgentConfig(BaseModel): hosted_tools: list[HostedTool[object]] = Field(default_factory=list[HostedTool[object]]) screenshot_encoding: ScreenshotEncoding = Field(default_factory=PngScreenshotEncoding) - model_name: str = "Agent" - model: str = Field(default="unknown", validation_alias=_model_alias) #: Provider client (AsyncAnthropic, AsyncOpenAI, genai.Client, ...). When unset, #: agents resolve one from settings (HUD gateway or provider API key). model_client: Any = None @@ -76,7 +79,7 @@ class AgentConfig(BaseModel): # ----------------------------------------------------------------------------- -class ClaudeConfig(AgentConfig): +class ClaudeConfig(ToolAgentConfig): model_name: str = "Claude" model: str = Field(default="claude-sonnet-4-6", validation_alias=_model_alias) tool_timeout_seconds: float | None = Field(default=120, gt=0, allow_inf_nan=False) @@ -90,7 +93,7 @@ class ClaudeConfig(AgentConfig): # ----------------------------------------------------------------------------- -class GeminiConfig(AgentConfig): +class GeminiConfig(ToolAgentConfig): """Configuration for GeminiAgent.""" model_name: str = "Gemini" @@ -109,7 +112,7 @@ class GeminiConfig(AgentConfig): # ----------------------------------------------------------------------------- -class OpenAIConfig(AgentConfig): +class OpenAIConfig(ToolAgentConfig): """Configuration for OpenAIAgent.""" model_name: str = "OpenAI" @@ -123,7 +126,7 @@ class OpenAIConfig(AgentConfig): parallel_tool_calls: bool | None = None -class OpenAIChatConfig(AgentConfig): +class OpenAIChatConfig(ToolAgentConfig): """Configuration for OpenAIChatAgent.""" model_name: str = "OpenAI Chat" @@ -148,10 +151,11 @@ class OpenAIChatConfig(AgentConfig): class ClaudeCLIConfig(AgentConfig): """Configuration for ClaudeCLIAgent (runs the ``claude`` CLI over SSH). - ``system_prompt`` is inherited from ``AgentConfig``. ``max_steps`` maps to the - CLI's ``--max-turns``; values <= 0 leave the turn budget to the CLI (unlimited). + ``max_steps`` maps to the CLI's ``--max-turns``; values <= 0 leave the turn + budget to the CLI (unlimited). """ + system_prompt: str | None = None model_name: str = "Claude CLI" model: str = Field(default="claude-sonnet-5", validation_alias=_model_alias) use_hud_gateway: bool | None = None @@ -181,9 +185,7 @@ class BrowserUseConfig(AgentConfig): """Configuration for BrowserUseAgent. Lives here (not in the agent module) so it can be imported and serialized - without the optional ``browser-use`` dependency installed. The ``auto_respond`` - / ``system_prompt`` / ``hosted_tools`` fields from ``AgentConfig`` do not apply - — browser-use runs its own agent loop. + without the optional ``browser-use`` dependency installed. """ model_name: str = "Browser Use" diff --git a/hud/types.py b/hud/types.py index 8ac8428dc..c697727ee 100644 --- a/hud/types.py +++ b/hud/types.py @@ -316,7 +316,7 @@ def emit(self, *, trace_id: str | None = None) -> None: #: Why the rollout stopped; anything but "done" means a limit cut it off. StopReason: TypeAlias = Literal["done", "max_steps", "length", "timeout", "malformed_tool_call"] -#: The configurable subset of stop reasons (``AgentConfig.stop_on``): policy +#: The configurable subset of stop reasons (``ToolAgentConfig.stop_on``): policy #: conditions the loop may either stop on or answer with an error result. StopCondition: TypeAlias = Literal["length", "malformed_tool_call"] From 0aba334a22066bbf023635262fce0f1ac5cd5d98 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 21 Aug 2026 13:34:22 -0700 Subject: [PATCH 04/14] Update timeout tests for agent ownership --- hud/agents/tests/test_tool_agent.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/hud/agents/tests/test_tool_agent.py b/hud/agents/tests/test_tool_agent.py index 9758042da..11445eb5f 100644 --- a/hud/agents/tests/test_tool_agent.py +++ b/hud/agents/tests/test_tool_agent.py @@ -32,7 +32,7 @@ ToolAgentConfig, ToolStep, ) -from hud.capabilities import Capability, CapabilityClient, MCPClient, RFBClient +from hud.capabilities import Capability, CapabilityClient, MCPClient, RFBClient, SSHClient from hud.capabilities.rfb import PngScreenshotEncoding, WebPScreenshotEncoding from hud.capabilities.ssh import SSHConnectionError from hud.types import MCPToolCall, MCPToolResult, Step, Trace @@ -99,8 +99,8 @@ def test_only_claude_provider_has_a_default_tool_timeout() -> None: assert config.timeout_seconds == 600 assert config.tool_timeout_seconds == 120 assert ClaudeConfig(tool_timeout_seconds=None).tool_timeout_seconds is None - assert AgentConfig().tool_timeout_seconds is None - assert ClaudeSDKConfig().tool_timeout_seconds is None + assert ToolAgentConfig().tool_timeout_seconds is None + assert "tool_timeout_seconds" not in ClaudeCLIConfig.model_fields async def test_agent_passes_screenshot_encoding_to_rfb_tools() -> None: @@ -432,6 +432,7 @@ async def wait(*, check: bool, timeout: None) -> None: # noqa: ASYNC109 ClaudeConfig( model="claude-test", model_client=cast("Any", object()), + max_steps=3, tool_timeout_seconds=0.01, ) ) @@ -451,13 +452,14 @@ async def wait(*, check: bool, timeout: None) -> None: # noqa: ASYNC109 state = RunState(messages=[], tools={"bash": tool}) run = cast("Run", _FakeRun()) - await agent._loop(run, state, max_steps=3) + await agent._loop(run, state) assert started.is_set() process.terminate.assert_called_once_with() process.wait_closed.assert_awaited_once_with() assert responses.await_count == 2 - assert run.trace.status == "completed" + assert run.trace.status is None + assert run.trace.stop_reason == "done" assert run.trace.content == "recovered" tool_result = cast("list[Any]", state.messages[0]["content"])[0] error_block = cast("list[Any]", tool_result["content"])[0] From 1430a8e0015440a29cdf5fb096ce6d3e8c3b8809 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 21 Aug 2026 13:29:42 -0700 Subject: [PATCH 05/14] Add first-class Codex CLI agent --- docs/v6/cookbooks/coding-agent.mdx | 10 +- docs/v6/guides/running-an-eval.mdx | 8 +- docs/v6/reference/agents.mdx | 16 +- hud/agents/__init__.py | 5 + hud/agents/claude/agent.py | 4 +- hud/agents/claude/cli/agent.py | 474 ++++++++++------------ hud/agents/cli.py | 61 +++ hud/agents/codex/__init__.py | 7 + hud/agents/codex/agent.py | 295 ++++++++++++++ hud/agents/tests/test_base.py | 15 + hud/agents/tests/test_claude_cli_agent.py | 87 ++-- hud/agents/tests/test_codex_cli_agent.py | 348 ++++++++++++++++ hud/agents/types.py | 14 + hud/cli/tests/test_eval_config.py | 19 + hud/types.py | 12 +- 15 files changed, 1045 insertions(+), 330 deletions(-) create mode 100644 hud/agents/cli.py create mode 100644 hud/agents/codex/__init__.py create mode 100644 hud/agents/codex/agent.py create mode 100644 hud/agents/tests/test_codex_cli_agent.py diff --git a/docs/v6/cookbooks/coding-agent.mdx b/docs/v6/cookbooks/coding-agent.mdx index 04bb3736a..1e261a915 100644 --- a/docs/v6/cookbooks/coding-agent.mdx +++ b/docs/v6/cookbooks/coding-agent.mdx @@ -66,8 +66,14 @@ To run the `claude` CLI over SSH, select the `claude_cli` agent: hud eval env.py claude_cli --gateway ``` -The `claude` executable must already be installed in the host or environment image. Pinning it in -the image keeps runs reproducible; `ClaudeCLIAgent` does not install or update it. The equivalent +Codex uses the same environment through the `codex_cli` agent: + +```bash +hud eval env.py codex_cli --gateway +``` + +The selected executable must already be installed in the host or environment image. Pinning it in +the image keeps runs reproducible; CLI agents do not install or update it. The equivalent Claude Python API is: ```python run.py diff --git a/docs/v6/guides/running-an-eval.mdx b/docs/v6/guides/running-an-eval.mdx index 9305efc0a..a124e9eec 100644 --- a/docs/v6/guides/running-an-eval.mdx +++ b/docs/v6/guides/running-an-eval.mdx @@ -38,9 +38,11 @@ and launch - no CLI required. ### Choosing an agent -The agent name (`claude`, `openai`, `gemini`) selects a built-in harness and routes calls through the -[HUD gateway](/v6/reference/agents), where one `HUD_API_KEY` covers every provider. Switching models is a -single flag, and `hud models list` shows every model the gateway knows. +The agent name (`claude`, `openai`, `gemini`) selects a provider harness and routes calls through the +[HUD gateway](/v6/reference/agents), where one `HUD_API_KEY` covers every provider. The `claude_cli` and +`codex_cli` harnesses instead run an installed CLI inside the environment; pass `--gateway` to route +their model calls through HUD. Switching models is a single flag, and `hud models list` shows every +model the gateway knows. ```bash hud eval "My Taskset" claude --model claude-haiku-4-5 # a cheaper model for fast iteration diff --git a/docs/v6/reference/agents.mdx b/docs/v6/reference/agents.mdx index de18580d9..ee4ff349a 100644 --- a/docs/v6/reference/agents.mdx +++ b/docs/v6/reference/agents.mdx @@ -60,15 +60,17 @@ agent = ClaudeAgent(ClaudeConfig(model="claude-sonnet-4-5", max_steps=30)) | `GeminiAgent` | `GeminiConfig` | `gemini-3-pro-preview` | | `OpenAIChatAgent` | `OpenAIChatConfig` | `gpt-5.4-mini` | | `ClaudeCLIAgent` | `ClaudeCLIConfig` | `claude-sonnet-5` | +| `CodexCLIAgent` | `CodexCLIConfig` | `gpt-5.6-sol` | Each config lives in `hud.agents.types`. `OpenAIChatAgent` speaks the OpenAI Chat Completions API, so it -points at any compatible server (vLLM, a local model) via `base_url`; `ClaudeCLIAgent` runs the `claude` -CLI over an `ssh` capability, against the env's filesystem. SSH-only runs support POSIX and Windows -workspaces; computer use over an `rfb` capability currently requires a POSIX workspace. Every knob +points at any compatible server (vLLM, a local model) via `base_url`. `ClaudeCLIAgent` and +`CodexCLIAgent` run their respective CLIs over an `ssh` capability against the env's filesystem and +stream the CLI's structured events into the HUD trace. SSH-only runs support POSIX and Windows +workspaces; Claude computer use over an `rfb` capability currently requires a POSIX workspace. Every knob (`model`, `max_steps`, `timeout_seconds`, `tool_timeout_seconds`, `system_prompt`, `citations_enabled`, `stop_on`) lives on the config; `__call__(run)` takes only the run. -`timeout_seconds` bounds the complete agent phase. For provider tool agents, `tool_timeout_seconds` bounds each complete SSH-backed tool call, including multi-operation editor calls. It is unset by default except on `ClaudeConfig`, where it defaults to 120 seconds. A timeout is returned to the model as a tool error so the agent can continue. `ClaudeSDKAgent` does not apply this setting because its SSH process is the complete Claude Code agent, not one tool call. +`timeout_seconds` bounds the complete agent phase. For provider tool agents, `tool_timeout_seconds` bounds each complete SSH-backed tool call, including multi-operation editor calls. It is unset by default except on `ClaudeConfig`, where it defaults to 120 seconds. A timeout is returned to the model as a tool error so the agent can continue. `ClaudeCLIAgent` does not apply this setting because its SSH process is the complete Claude Code agent, not one tool call. ```python agent = OpenAIChatAgent( @@ -100,8 +102,8 @@ A model id maps to one of four gateway agent types (`AgentType`), each a provide | `gemini` | `GeminiAgent` | | `openai_compatible` | `OpenAIChatAgent` | -For a provider key instead of the gateway, or for `ClaudeCLIAgent` (not a gateway shortcut), construct -the agent directly. +For a provider key instead of the gateway, or for a CLI agent (not a gateway shortcut), construct the +agent directly. ## Agent @@ -126,7 +128,7 @@ print(job.reward) ``` **From the CLI**, `hud eval` takes a task source and an agent name (`claude`, `openai`, `gemini`, -`openai_compatible`); see [running an eval](/v6/guides/running-an-eval) for the walkthrough and the +`openai_compatible`, `claude_cli`, `codex_cli`); see [running an eval](/v6/guides/running-an-eval) for the walkthrough and the [CLI reference](/v6/reference/cli#hud-eval) for the full flag set. ## Bring your own harness diff --git a/hud/agents/__init__.py b/hud/agents/__init__.py index 382b25331..b4c0cb97b 100644 --- a/hud/agents/__init__.py +++ b/hud/agents/__init__.py @@ -21,6 +21,7 @@ from typing import TypeAlias from hud.agents.claude import ClaudeAgent, ClaudeCLIAgent, ClaudeCLIConfig + from hud.agents.codex import CodexCLIAgent, CodexCLIConfig from hud.agents.gemini import GeminiAgent from hud.agents.openai import OpenAIAgent from hud.agents.openai_compatible import OpenAIChatAgent @@ -124,6 +125,8 @@ def create_agent(model: str, **kwargs: Any) -> GatewayAgent: "ClaudeAgent": ("hud.agents.claude", "ClaudeAgent"), "ClaudeCLIAgent": ("hud.agents.claude", "ClaudeCLIAgent"), "ClaudeCLIConfig": ("hud.agents.claude", "ClaudeCLIConfig"), + "CodexCLIAgent": ("hud.agents.codex", "CodexCLIAgent"), + "CodexCLIConfig": ("hud.agents.codex", "CodexCLIConfig"), "GeminiAgent": ("hud.agents.gemini", "GeminiAgent"), "MCPAgent": ("hud.agents.tool_agent", "ToolAgent"), "OpenAIAgent": ("hud.agents.openai", "OpenAIAgent"), @@ -134,6 +137,8 @@ def create_agent(model: str, **kwargs: Any) -> GatewayAgent: "ClaudeAgent", "ClaudeCLIAgent", "ClaudeCLIConfig", + "CodexCLIAgent", + "CodexCLIConfig", "GeminiAgent", "MCPAgent", "OpenAIAgent", diff --git a/hud/agents/claude/agent.py b/hud/agents/claude/agent.py index 7aad09954..e3e5d2c84 100644 --- a/hud/agents/claude/agent.py +++ b/hud/agents/claude/agent.py @@ -257,10 +257,10 @@ async def get_response( if response is None: raise ValueError("Claude response missing after retries") - return self._message_to_agent_step(response, citations_enabled=citations_enabled) + return self.message_to_agent_step(response, citations_enabled=citations_enabled) @classmethod - def _message_to_agent_step( + def message_to_agent_step( cls, response: BetaMessage, *, diff --git a/hud/agents/claude/cli/agent.py b/hud/agents/claude/cli/agent.py index 676d3e001..c5b7d0066 100644 --- a/hud/agents/claude/cli/agent.py +++ b/hud/agents/claude/cli/agent.py @@ -2,9 +2,6 @@ from __future__ import annotations -import asyncio -import base64 -import contextlib import json import logging import shlex @@ -17,6 +14,7 @@ from hud.agents.base import Agent from hud.agents.claude.agent import ClaudeAgent +from hud.agents.cli import WINDOWS_SHELLS, powershell, powershell_quote, run_jsonl from hud.agents.types import ClaudeCLIConfig, ToolStep from hud.settings import settings from hud.telemetry.context import get_current_trace_id @@ -31,25 +29,22 @@ logger = logging.getLogger(__name__) -_WINDOWS_SHELLS = ("cmd", "powershell") -_PROMPT_PATH = ".hud_prompt.txt" -_MCP_CONFIG_PATH = ".hud_mcp_config.json" -_RUN_SCRIPT_PATH = ".hud_run.bat" -_PROCESS_CLOSE_TIMEOUT_S = 5.0 +PROMPT_PATH = ".hud_prompt.txt" +MCP_CONFIG_PATH = ".hud_mcp_config.json" +RUN_SCRIPT_PATH = ".hud_run.bat" -class _ClaudeStreamParser: +class ClaudeEvents: """Translate Claude CLI stream messages into canonical HUD steps.""" def __init__(self, run: Run, *, started_at: str) -> None: - self._run = run - self._agent_started_at = started_at - self._pending_calls: dict[str, tuple[MCPToolCall, str]] = {} - self._last_agent_content = "" - self._saw_result = False - self._result_error: str | None = None - - def feed_line(self, line: str) -> None: + self.run = run + self.agent_started_at = started_at + self.pending_calls: dict[str, tuple[MCPToolCall, str]] = {} + self.saw_result = False + self.error: str | None = None + + def consume(self, line: str) -> None: line = line.strip() if not line: return @@ -59,139 +54,218 @@ def feed_line(self, line: str) -> None: received_at = now_iso() match message.get("type"): case "system" if message.get("subtype") == "init": - self._agent_started_at = received_at + self.agent_started_at = received_at case "assistant": - self._record_assistant(message, received_at) + step = ClaudeAgent.message_to_agent_step( + BetaMessage.model_validate(message["message"]) + ) + step.started_at = self.agent_started_at + step.ended_at = received_at + if step.content: + self.run.trace.content = step.content + self.run.record(step) + for call in step.tool_calls: + self.pending_calls[call.id] = (call, received_at) case "user": - self._record_tool_results(message, received_at) + saw_result = False + for block in message["message"]["content"]: + if block["type"] != "tool_result": + continue + call_id = block["tool_use_id"] + try: + call, started_at = self.pending_calls.pop(call_id) + except KeyError: + raise ValueError( + f"Claude returned a result for unknown tool call {call_id!r}" + ) from None + + raw_result = block.get("content") + raw_items = raw_result if isinstance(raw_result, list) else [raw_result] + content: list[mcp_types.ContentBlock] = [] + for item in raw_items: + if isinstance(item, str): + content.append(mcp_types.TextContent(type="text", text=item)) + elif item["type"] == "text": + content.append(mcp_types.TextContent(type="text", text=item["text"])) + elif item["type"] == "image": + source = item["source"] + content.append( + mcp_types.ImageContent( + type="image", + data=source["data"], + mimeType=source["media_type"], + ) + ) + else: + raise ValueError(f"unsupported Claude tool result block: {item!r}") + + self.run.record( + ToolStep( + call=call, + result=MCPToolResult( + call_id=call_id, + content=content, + isError=block.get("is_error") is True, + ), + started_at=started_at, + ended_at=received_at, + ) + ) + saw_result = True + if saw_result: + self.agent_started_at = received_at case "result": - self._record_result(message) + self.saw_result = True + trace = self.run.trace + result = message.get("result") + if isinstance(result, str): + trace.content = result + if message.get("is_error") is True: + self.error = trace.content or "claude CLI reported an error" + for key in ( + "subtype", + "session_id", + "duration_ms", + "duration_api_ms", + "stop_reason", + "num_turns", + "total_cost_usd", + ): + if (value := message.get(key)) is not None: + trace.extra[key] = value def finish(self, *, returncode: int, stderr: str) -> None: - trace = self._run.trace - error = self._result_error + trace = self.run.trace + error = self.error if returncode != 0: trace.extra["returncode"] = returncode error = stderr.strip() or f"claude CLI exited with return code {returncode}" - elif not self._saw_result: + elif not self.saw_result: error = "claude CLI exited without a result event" - elif self._pending_calls: - missing = ", ".join(sorted(self._pending_calls)) + elif self.pending_calls: + missing = ", ".join(sorted(self.pending_calls)) error = f"claude CLI exited without results for tool calls: {missing}" - if not trace.content and self._last_agent_content: - trace.content = self._last_agent_content if error is not None and stderr: trace.extra["stderr"] = stderr if error is not None: raise RuntimeError(error) - def _record_assistant(self, event: dict[str, Any], received_at: str) -> None: - message = BetaMessage.model_validate(event["message"]) - step = ClaudeAgent._message_to_agent_step(message) - step.started_at = self._agent_started_at - step.ended_at = received_at - if step.content: - self._last_agent_content = step.content - self._run.record(step) - for call in step.tool_calls: - self._pending_calls[call.id] = (call, received_at) - - def _record_tool_results(self, event: dict[str, Any], received_at: str) -> None: - message = event.get("message") - if not isinstance(message, dict): - return - content = message.get("content") - if not isinstance(content, list): - return - saw_result = False - for raw_block in content: - if not isinstance(raw_block, dict) or raw_block.get("type") != "tool_result": - continue - call_id = raw_block.get("tool_use_id") - if not isinstance(call_id, str): - raise ValueError("Claude tool result is missing tool_use_id") - try: - call, started_at = self._pending_calls.pop(call_id) - except KeyError: - raise ValueError( - f"Claude returned a result for unknown tool call {call_id!r}" - ) from None - - raw_result = raw_block.get("content") - raw_items = raw_result if isinstance(raw_result, list) else [raw_result] - result_content: list[mcp_types.ContentBlock] = [] - for item in raw_items: - if isinstance(item, str): - result_content.append(mcp_types.TextContent(type="text", text=item)) - elif isinstance(item, dict) and item.get("type") == "text": - result_content.append(mcp_types.TextContent(type="text", text=item["text"])) - elif isinstance(item, dict) and item.get("type") == "image": - source = item["source"] - result_content.append( - mcp_types.ImageContent( - type="image", - data=source["data"], - mimeType=source["media_type"], - ) - ) - elif item is not None: - raise ValueError(f"unsupported Claude tool result block: {item!r}") - - saw_result = True - self._run.record( - ToolStep( - call=call, - result=MCPToolResult( - call_id=call_id, - content=result_content, - isError=raw_block.get("is_error") is True, - ), - started_at=started_at, - ended_at=received_at, - ) - ) - if saw_result: - self._agent_started_at = received_at - - def _record_result(self, event: dict[str, Any]) -> None: - self._saw_result = True - trace = self._run.trace - result = event.get("result") - trace.content = result if isinstance(result, str) else self._last_agent_content - if event.get("is_error") is True: - self._result_error = trace.content or "claude CLI reported an error" - for key in ( - "subtype", - "session_id", - "duration_ms", - "duration_api_ms", - "stop_reason", - "num_turns", - "total_cost_usd", +def claude_command( + config: ClaudeCLIConfig, + shell: str, + prompt: str, + mcp_config_path: str | None = None, +) -> str: + env: dict[str, str] = {} + use_hud_gateway = config.use_hud_gateway + if use_hud_gateway is None: + use_hud_gateway = settings.api_key is not None + + if use_hud_gateway: + if not settings.api_key: + raise ValueError("HUD_API_KEY is required for HUD gateway routing") + env["ANTHROPIC_BASE_URL"] = settings.hud_gateway_url + env["ANTHROPIC_API_KEY"] = settings.api_key + if trace_id := get_current_trace_id(): + env["ANTHROPIC_CUSTOM_HEADERS"] = f"Trace-Id: {trace_id}" + elif settings.anthropic_api_key: + env["ANTHROPIC_API_KEY"] = settings.anthropic_api_key + + env["ANTHROPIC_MODEL"] = config.model + env["ANTHROPIC_SMALL_FAST_MODEL"] = config.model + + # A custom base URL must own every model tier; otherwise background calls + # can escape to Anthropic instead of using the configured gateway. + if "ANTHROPIC_BASE_URL" in env: + for name in ( + "ANTHROPIC_DEFAULT_SONNET_MODEL", + "ANTHROPIC_DEFAULT_OPUS_MODEL", + "ANTHROPIC_DEFAULT_HAIKU_MODEL", + "CLAUDE_CODE_SUBAGENT_MODEL", ): - value = event.get(key) - if value is not None: - trace.extra[key] = value - - -def _powershell_quote(value: str) -> str: - return "'" + value.replace("'", "''") + "'" - - -def _powershell(script: str) -> str: - encoded = base64.b64encode(script.encode("utf-16-le")).decode("ascii") - return f"powershell -NoProfile -NonInteractive -EncodedCommand {encoded}" + env[name] = config.model + + env["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC"] = "1" + env["IS_SANDBOX"] = "1" + + args: list[str] = [ + "claude", + "--verbose", + "--output-format=stream-json", + "--print", + f"--permission-mode={config.permission_mode}", + ] + if config.max_steps > 0: + args.append(f"--max-turns={config.max_steps}") + if config.system_prompt: + args.extend(["--system-prompt", config.system_prompt]) + for tool in config.allowed_tools: + args.extend(["--allowedTools", tool]) + if mcp_config_path: + args.extend(["--mcp-config", mcp_config_path]) + + if shell in WINDOWS_SHELLS: + script = ";".join( + [ + *(f"$env:{key}={powershell_quote(value)}" for key, value in env.items()), + f"Get-Content -Raw -Encoding UTF8 {powershell_quote(PROMPT_PATH)}" + f" | & claude {' '.join(powershell_quote(arg) for arg in args[1:])}", + "exit $LASTEXITCODE", + ] + ) + return powershell(script) + + args.extend(["--", prompt]) + command = " ".join(shlex.quote(arg) for arg in args) + env_prefix = " ".join(f"{key}={shlex.quote(value)}" for key, value in env.items()) + return f'export PATH="$HOME/.local/bin:$PATH"; {env_prefix} {command}' + + +async def run_claude( + config: ClaudeCLIConfig, + run: Run, + *, + ssh: SSHClient, + shell: str, + mcp_servers: dict[str, dict[str, Any]], + prompt: str, +) -> None: + files: dict[str, str] = {} + mcp_config_path = MCP_CONFIG_PATH if mcp_servers else None + if mcp_servers: + files[MCP_CONFIG_PATH] = json.dumps({"mcpServers": mcp_servers}, indent=2) + if shell in WINDOWS_SHELLS: + files[PROMPT_PATH] = prompt + + command = claude_command(config, shell, prompt, mcp_config_path) + if shell in WINDOWS_SHELLS: + files[RUN_SCRIPT_PATH] = f"@echo off\r\n{command}\r\n" + command = f"cmd /c {RUN_SCRIPT_PATH}" + + try: + for path, content in files.items(): + await ssh.write_text(path, content) + logger.info("SSH exec claude CLI (%d chars)", len(command)) + events = ClaudeEvents(run, started_at=now_iso()) + returncode, stderr = await run_jsonl(ssh, command, events.consume) + logger.info("exit=%s stderr=%d", returncode, len(stderr)) + events.finish(returncode=returncode, stderr=stderr) + finally: + if files: + if shell in WINDOWS_SHELLS: + cleanup = f"cmd /c del /f /q {' '.join(files)} 2>nul" + else: + cleanup = "rm -f -- " + " ".join(shlex.quote(path) for path in files) + try: + await ssh.run(cleanup, check=False) + except (OSError, asyncssh.Error): + logger.warning("Failed to remove Claude CLI runtime files") class ClaudeCLIAgent(Agent): - """Runs ``claude`` CLI over SSH inside the env workspace. - - Stateless w.r.t. the env: driven by ``await agent(run)``. SSH is opened - live off the run. Environment MCP bindings are used directly; computer MCP - servers are bridged over the run's SSH connection. - """ + """Runs ``claude`` CLI over SSH inside the environment workspace.""" config: ClaudeCLIConfig @@ -235,7 +309,8 @@ async def __call__(self, run: Run) -> None: ) ) - await self._run_cli( + await run_claude( + self.config, run, ssh=ssh, shell=shell, @@ -243,144 +318,5 @@ async def __call__(self, run: Run) -> None: prompt=run.prompt_text, ) - async def _run_cli( - self, - run: Run, - *, - ssh: SSHClient, - shell: str, - mcp_servers: dict[str, dict[str, Any]], - prompt: str, - ) -> None: - files: dict[str, str] = {} - mcp_config_path = _MCP_CONFIG_PATH if mcp_servers else None - if mcp_servers: - files[_MCP_CONFIG_PATH] = json.dumps({"mcpServers": mcp_servers}, indent=2) - if shell in _WINDOWS_SHELLS: - files[_PROMPT_PATH] = prompt - - command = self._build_command( - shell=shell, - prompt=prompt, - mcp_config_path=mcp_config_path, - ) - if shell in _WINDOWS_SHELLS: - files[_RUN_SCRIPT_PATH] = f"@echo off\r\n{command}\r\n" - command = f"cmd /c {_RUN_SCRIPT_PATH}" - - try: - for path, content in files.items(): - await ssh.write_text(path, content) - logger.info("SSH exec claude CLI (%d chars)", len(command)) - await self._stream_cli(run, ssh, command) - finally: - if files: - if shell in _WINDOWS_SHELLS: - cleanup = f"cmd /c del /f /q {' '.join(files)} 2>nul" - else: - cleanup = "rm -f -- " + " ".join(shlex.quote(path) for path in files) - try: - await ssh.run(cleanup, check=False) - except (OSError, asyncssh.Error): - logger.warning("Failed to remove Claude CLI runtime files") - - async def _stream_cli(self, run: Run, ssh: SSHClient, command: str) -> None: - parser = _ClaudeStreamParser(run, started_at=now_iso()) - process = await ssh.create_process(command) - stderr_task = asyncio.create_task(process.stderr.read()) - try: - while line := await process.stdout.readline(): - parser.feed_line(line.decode(errors="replace")) - await process.wait_closed() - stderr_output = await stderr_task - except BaseException: - process.close() - if not stderr_task.done(): - stderr_task.cancel() - await asyncio.gather(stderr_task, return_exceptions=True) - with contextlib.suppress(OSError, TimeoutError, asyncssh.Error): - async with asyncio.timeout(_PROCESS_CLOSE_TIMEOUT_S): - await process.wait_closed() - raise - - stderr = stderr_output.decode(errors="replace") - returncode = process.returncode - if returncode is None: - raise RuntimeError("claude CLI process closed without an exit status") - logger.info("exit=%s stderr=%d", returncode, len(stderr)) - parser.finish(returncode=returncode, stderr=stderr) - - def _build_command( - self, - *, - shell: str, - prompt: str, - mcp_config_path: str | None = None, - ) -> str: - env: dict[str, str] = {} - use_hud_gateway = self.config.use_hud_gateway - if use_hud_gateway is None: - use_hud_gateway = settings.api_key is not None - - if use_hud_gateway: - if not settings.api_key: - raise ValueError("HUD_API_KEY is required for HUD gateway routing") - env["ANTHROPIC_BASE_URL"] = settings.hud_gateway_url - env["ANTHROPIC_API_KEY"] = settings.api_key - if trace_id := get_current_trace_id(): - env["ANTHROPIC_CUSTOM_HEADERS"] = f"Trace-Id: {trace_id}" - elif settings.anthropic_api_key: - env["ANTHROPIC_API_KEY"] = settings.anthropic_api_key - - env["ANTHROPIC_MODEL"] = self.config.model - env["ANTHROPIC_SMALL_FAST_MODEL"] = self.config.model - - # When using a custom base URL, alias all model tiers to the same model - # so the CLI doesn't try to reach Anthropic for background requests. - if "ANTHROPIC_BASE_URL" in env: - for name in ( - "ANTHROPIC_DEFAULT_SONNET_MODEL", - "ANTHROPIC_DEFAULT_OPUS_MODEL", - "ANTHROPIC_DEFAULT_HAIKU_MODEL", - "CLAUDE_CODE_SUBAGENT_MODEL", - ): - env[name] = self.config.model - - env["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC"] = "1" - env["IS_SANDBOX"] = "1" - - base_args: list[str] = [ - "claude", - "--verbose", - "--output-format=stream-json", - "--print", - f"--permission-mode={self.config.permission_mode}", - ] - if self.config.max_steps > 0: - base_args.append(f"--max-turns={self.config.max_steps}") - if self.config.system_prompt: - base_args.extend(["--system-prompt", self.config.system_prompt]) - for tool in self.config.allowed_tools: - base_args.extend(["--allowedTools", tool]) - if mcp_config_path: - base_args.extend(["--mcp-config", mcp_config_path]) - - if shell in _WINDOWS_SHELLS: - script = ";".join( - [ - *(f"$env:{key}={_powershell_quote(value)}" for key, value in env.items()), - f"Get-Content -Raw -Encoding UTF8 {_powershell_quote(_PROMPT_PATH)}" - f" | & claude {' '.join(_powershell_quote(arg) for arg in base_args[1:])}", - "exit $LASTEXITCODE", - ] - ) - return _powershell(script) - - cli_parts = [shlex.quote(a) for a in base_args] - cli_parts.extend(["--", shlex.quote(prompt)]) - cli_cmd = " ".join(cli_parts) - env_prefix = " ".join(f"{key}={shlex.quote(value)}" for key, value in env.items()) - return f'export PATH="$HOME/.local/bin:$PATH"; {env_prefix} {cli_cmd}' - __all__ = ["ClaudeCLIAgent"] diff --git a/hud/agents/cli.py b/hud/agents/cli.py new file mode 100644 index 000000000..5fc9583f5 --- /dev/null +++ b/hud/agents/cli.py @@ -0,0 +1,61 @@ +"""Process boundary for JSONL CLI agents.""" + +from __future__ import annotations + +import asyncio +import base64 +import contextlib +from typing import TYPE_CHECKING + +import asyncssh + +if TYPE_CHECKING: + from collections.abc import Callable + + from hud.capabilities import SSHClient + +WINDOWS_SHELLS = ("cmd", "powershell") +PROCESS_CLOSE_TIMEOUT_S = 5.0 + + +def powershell_quote(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +def powershell(script: str) -> str: + encoded = base64.b64encode(script.encode("utf-16-le")).decode("ascii") + return f"powershell -NoProfile -NonInteractive -EncodedCommand {encoded}" + + +async def run_jsonl( + ssh: SSHClient, + command: str, + consume: Callable[[str], None], + *, + input_text: str | None = None, +) -> tuple[int, str]: + """Stream one remote JSONL process and own its cancellation cleanup.""" + process = await ssh.create_process(command) + stderr_task = asyncio.create_task(process.stderr.read()) + try: + if input_text is not None: + process.stdin.write(input_text.encode()) + await process.stdin.drain() + process.stdin.write_eof() + while line := await process.stdout.readline(): + consume(line.decode(errors="replace")) + await process.wait_closed() + stderr = (await stderr_task).decode(errors="replace") + except BaseException: + process.close() + if not stderr_task.done(): + stderr_task.cancel() + await asyncio.gather(stderr_task, return_exceptions=True) + with contextlib.suppress(OSError, TimeoutError, asyncssh.Error): + async with asyncio.timeout(PROCESS_CLOSE_TIMEOUT_S): + await process.wait_closed() + raise + + if process.returncode is None: + raise RuntimeError("CLI process closed without an exit status") + return process.returncode, stderr diff --git a/hud/agents/codex/__init__.py b/hud/agents/codex/__init__.py new file mode 100644 index 000000000..0b5e99611 --- /dev/null +++ b/hud/agents/codex/__init__.py @@ -0,0 +1,7 @@ +"""Codex CLI agent.""" + +from hud.agents.types import CodexCLIConfig + +from .agent import CodexCLIAgent + +__all__ = ["CodexCLIAgent", "CodexCLIConfig"] diff --git a/hud/agents/codex/agent.py b/hud/agents/codex/agent.py new file mode 100644 index 000000000..68ba357b4 --- /dev/null +++ b/hud/agents/codex/agent.py @@ -0,0 +1,295 @@ +"""Codex CLI harness over a workspace SSH capability.""" + +from __future__ import annotations + +import json +import logging +import shlex +from typing import TYPE_CHECKING, Any, cast + +import mcp.types as mcp_types + +from hud.agents.base import Agent +from hud.agents.cli import WINDOWS_SHELLS, powershell, powershell_quote, run_jsonl +from hud.agents.types import AgentStep, CodexCLIConfig, ToolStep +from hud.settings import settings +from hud.telemetry.context import get_current_trace_id +from hud.types import MCPToolCall, MCPToolResult, Step +from hud.utils.time import now_iso + +if TYPE_CHECKING: + from hud.capabilities import SSHClient + from hud.eval.run import Run + +logger = logging.getLogger(__name__) + + +class CodexEvents: + """Translate ``codex exec --json`` events into canonical HUD steps.""" + + def __init__(self, run: Run, *, model: str, started_at: str) -> None: + self.run = run + self.model = model + self.agent_started_at = started_at + self.item_started_at: dict[str, str] = {} + self.saw_completion = False + self.error: str | None = None + + def consume(self, line: str) -> None: + line = line.strip() + if not line: + return + event = json.loads(line) + if not isinstance(event, dict): + raise ValueError("Codex stream event must be an object") + + received_at = now_iso() + match event.get("type"): + case "thread.started": + self.run.trace.extra["codex_thread_id"] = event["thread_id"] + case "turn.started": + self.agent_started_at = received_at + case "item.started": + item = event["item"] + self.item_started_at[item["id"]] = received_at + case "item.completed": + self.record(event["item"], received_at) + case "turn.completed": + self.run.trace.extra["usage"] = event["usage"] + self.saw_completion = True + case "turn.failed": + self.error = event["error"]["message"] + case "error": + self.error = event["message"] + + def finish(self, *, returncode: int, stderr: str) -> None: + trace = self.run.trace + error = self.error + if returncode != 0: + trace.extra["returncode"] = returncode + error = error or stderr.strip() or f"codex CLI exited with return code {returncode}" + elif error is None and not self.saw_completion: + error = "codex CLI exited without a turn.completed event" + + if error is not None and stderr and self.error is None: + trace.extra["stderr"] = stderr + if error is not None: + raise RuntimeError(error) + + def record(self, item: dict[str, Any], received_at: str) -> None: + item_id = item["id"] + started_at = self.item_started_at.pop(item_id, self.agent_started_at) + match item["type"]: + case "agent_message": + text = item["text"] + self.run.trace.content = text + self.run.record( + AgentStep( + content=text, + model=self.model, + raw=item, + started_at=started_at, + ended_at=received_at, + ) + ) + case "reasoning": + self.run.record( + AgentStep( + reasoning=item["text"], + model=self.model, + raw=item, + started_at=started_at, + ended_at=received_at, + ) + ) + case "command_execution" | "file_change" | "mcp_tool_call" | "web_search": + self.record_tool(item, started_at, received_at) + case _: + self.run.record( + Step( + source="agent", + extra={"codex_item": item}, + started_at=started_at, + ended_at=received_at, + ) + ) + self.agent_started_at = received_at + + def record_tool(self, item: dict[str, Any], started_at: str, ended_at: str) -> None: + call_id = item["id"] + match item["type"]: + case "command_execution": + call = MCPToolCall( + id=call_id, + name="shell", + arguments={"command": item["command"]}, + ) + result = MCPToolResult( + call_id=call_id, + content=[mcp_types.TextContent(type="text", text=item["aggregated_output"])], + isError=item["status"] != "completed" or item["exit_code"] not in (None, 0), + ) + case "file_change": + changes = item["changes"] + call = MCPToolCall( + id=call_id, + name="apply_patch", + arguments={"changes": changes}, + ) + result = MCPToolResult( + call_id=call_id, + content=[ + mcp_types.TextContent( + type="text", + text="\n".join( + f"{change['kind']}: {change['path']}" for change in changes + ), + ) + ], + isError=item["status"] != "completed", + ) + case "mcp_tool_call": + call = MCPToolCall( + id=call_id, + name=item["tool"], + provider_name=f"{item['server']}.{item['tool']}", + arguments=item["arguments"], + ) + raw_result = item.get("result") or {} + error = item.get("error") + result = MCPToolResult.model_validate( + { + "call_id": call_id, + "content": raw_result.get("content") + or ([{"type": "text", "text": error["message"]}] if error else []), + "structuredContent": raw_result.get("structured_content"), + "_meta": raw_result.get("_meta"), + "isError": item["status"] == "failed", + } + ) + case "web_search": + call = MCPToolCall( + id=call_id, + name="web_search", + arguments={"query": item["query"], "action": item["action"]}, + ) + result = MCPToolResult( + call_id=call_id, + content=[ + mcp_types.TextContent( + type="text", + text=json.dumps(item["action"], separators=(",", ":")), + ) + ], + isError=False, + ) + case _: + raise ValueError(f"unsupported Codex tool item {item['type']!r}") + + self.run.record( + ToolStep( + call=call, + result=result, + extra={"codex_item": item}, + started_at=started_at, + ended_at=ended_at, + ) + ) + + +def codex_command(config: CodexCLIConfig, shell: str) -> str: + env: dict[str, str] = {} + args = [ + "codex", + "exec", + "--json", + "--ephemeral", + "--ignore-user-config", + "--skip-git-repo-check", + "--color", + "never", + "--sandbox", + config.sandbox, + "--model", + config.model, + ] + + use_hud_gateway = config.use_hud_gateway + if use_hud_gateway is None: + use_hud_gateway = settings.api_key is not None + if use_hud_gateway: + if not settings.api_key: + raise ValueError("HUD_API_KEY is required for HUD gateway routing") + env["HUD_API_KEY"] = settings.api_key + overrides = { + "model_provider": "hud", + "model_providers.hud.name": "HUD", + "model_providers.hud.base_url": settings.hud_gateway_url, + "model_providers.hud.env_key": "HUD_API_KEY", + "model_providers.hud.wire_api": "responses", + } + for key, value in overrides.items(): + args.extend(["-c", f"{key}={json.dumps(value)}"]) + if trace_id := get_current_trace_id(): + args.extend( + [ + "-c", + f'model_providers.hud.http_headers={{"Trace-Id"={json.dumps(trace_id)}}}', + ] + ) + elif settings.openai_api_key: + env["CODEX_API_KEY"] = settings.openai_api_key + + args.append("-") + if shell in WINDOWS_SHELLS: + script = ";".join( + [ + *(f"$env:{key}={powershell_quote(value)}" for key, value in env.items()), + f"& codex {' '.join(powershell_quote(arg) for arg in args[1:])}", + "exit $LASTEXITCODE", + ] + ) + return powershell(script) + + command = " ".join(shlex.quote(arg) for arg in args) + env_prefix = " ".join(f"{key}={shlex.quote(value)}" for key, value in env.items()) + invocation = f"{env_prefix} {command}" if env_prefix else command + return f'export PATH="$HOME/.local/bin:$PATH"; {invocation}' + + +async def run_codex( + config: CodexCLIConfig, + run: Run, + *, + ssh: SSHClient, + shell: str, + prompt: str, +) -> None: + command = codex_command(config, shell) + logger.info("SSH exec codex CLI (%d chars)", len(command)) + events = CodexEvents(run, model=config.model, started_at=now_iso()) + returncode, stderr = await run_jsonl(ssh, command, events.consume, input_text=prompt) + logger.info("exit=%s stderr=%d", returncode, len(stderr)) + events.finish(returncode=returncode, stderr=stderr) + + +class CodexCLIAgent(Agent): + """Runs ``codex exec`` over SSH inside the environment workspace.""" + + config: CodexCLIConfig + + def __init__(self, config: CodexCLIConfig | None = None) -> None: + self.config = config or CodexCLIConfig() + + async def __call__(self, run: Run) -> None: + ssh = cast("SSHClient", await run.client.open("ssh")) + await run_codex( + self.config, + run, + ssh=ssh, + shell=ssh.capability.params.get("shell", "bash"), + prompt=run.prompt_text, + ) + + +__all__ = ["CodexCLIAgent"] diff --git a/hud/agents/tests/test_base.py b/hud/agents/tests/test_base.py index b88faf0e0..4af61fe76 100644 --- a/hud/agents/tests/test_base.py +++ b/hud/agents/tests/test_base.py @@ -13,6 +13,7 @@ from hud.agents import ( ClaudeCLIAgent, + CodexCLIAgent, OpenAIAgent, OpenAIChatAgent, create_agent, @@ -51,6 +52,8 @@ def test_agent_type_maps_value_to_class_and_provider() -> None: def test_agent_type_registers_cli_agent() -> None: assert AgentType("claude_cli").cls is ClaudeCLIAgent assert AgentType.of(ClaudeCLIAgent()) == AgentType.CLAUDE_CLI + assert AgentType("codex_cli").cls is CodexCLIAgent + assert AgentType.of(CodexCLIAgent()) == AgentType.CODEX_CLI def test_cli_agent_round_trips_through_registered_wire_format() -> None: @@ -65,6 +68,18 @@ def test_cli_agent_round_trips_through_registered_wire_format() -> None: assert loaded.config == agent.config +def test_codex_cli_agent_round_trips_through_registry() -> None: + agent = CodexCLIAgent() + spec = dump_agent(agent) + + loaded = load_agent(spec) + + assert spec["type"] == "codex_cli" + assert spec["config"]["model"] == "gpt-5.6-sol" + assert isinstance(loaded, CodexCLIAgent) + assert loaded.config == agent.config + + def test_dump_agent_rejects_unregistered_agent() -> None: with pytest.raises(ValueError, match="registered types"): dump_agent(_FillingAgent()) diff --git a/hud/agents/tests/test_claude_cli_agent.py b/hud/agents/tests/test_claude_cli_agent.py index cb9ff492a..fd5864e69 100644 --- a/hud/agents/tests/test_claude_cli_agent.py +++ b/hud/agents/tests/test_claude_cli_agent.py @@ -24,7 +24,7 @@ from mcp.types import ImageContent, TextContent from hud.agents.claude.cli import computer_mcp -from hud.agents.claude.cli.agent import ClaudeCLIAgent +from hud.agents.claude.cli.agent import ClaudeCLIAgent, claude_command, run_claude from hud.agents.types import AgentStep, ClaudeCLIConfig, ToolStep from hud.capabilities import Capability, SSHClient from hud.capabilities.rfb import WebPScreenshotEncoding @@ -46,12 +46,8 @@ def test_command_follows_explicit_gateway_routing(monkeypatch: pytest.MonkeyPatc monkeypatch.setattr(settings, "api_key", "hud-key") monkeypatch.setattr(settings, "anthropic_api_key", "anthropic-key") - gateway = ClaudeCLIAgent(ClaudeCLIConfig(use_hud_gateway=True))._build_command( - shell="bash", prompt="run", mcp_config_path=None - ) - provider = ClaudeCLIAgent(ClaudeCLIConfig(use_hud_gateway=False))._build_command( - shell="bash", prompt="run", mcp_config_path=None - ) + gateway = claude_command(ClaudeCLIConfig(use_hud_gateway=True), "bash", "run") + provider = claude_command(ClaudeCLIConfig(use_hud_gateway=False), "bash", "run") assert f"ANTHROPIC_BASE_URL={settings.hud_gateway_url}" in gateway assert "ANTHROPIC_API_KEY=hud-key" in gateway @@ -64,18 +60,12 @@ def test_windows_command_encodes_environment_and_arguments( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(settings, "api_key", "hud&key's") - agent = ClaudeCLIAgent( - ClaudeCLIConfig( - use_hud_gateway=True, - max_steps=3, - system_prompt="don't $expand", - ) - ) - - command = agent._build_command( - shell="powershell", - prompt="not embedded", + config = ClaudeCLIConfig( + use_hud_gateway=True, + max_steps=3, + system_prompt="don't $expand", ) + command = claude_command(config, "powershell", "not embedded") encoded = command.rsplit(" ", 1)[1] script = base64.b64decode(encoded).decode("utf-16-le") @@ -234,11 +224,12 @@ def _ssh_with_conn(shell: str, conn: _FakeConn) -> SSHClient: async def test_exec_on_windows_writes_batch_and_execs_via_cmd() -> None: sink: dict[str, bytes] = {} conn = _FakeConn(sink, _FakeStreamProcess(_STREAM_JSON)) - agent = ClaudeCLIAgent() ssh = _ssh_with_conn("cmd", conn) run = _fake_run() - await agent._run_cli(run, ssh=ssh, shell="cmd", mcp_servers={}, prompt="build it") + await run_claude( + ClaudeCLIConfig(), run, ssh=ssh, shell="cmd", mcp_servers={}, prompt="build it" + ) assert conn.ran == ["cmd /c .hud_run.bat"] assert all(command.startswith("powershell ") for command in conn.write_commands) @@ -254,11 +245,12 @@ async def test_exec_on_windows_writes_batch_and_execs_via_cmd() -> None: async def test_exec_on_bash_runs_inline_without_batch() -> None: sink: dict[str, bytes] = {} conn = _FakeConn(sink, _FakeStreamProcess(_STREAM_JSON)) - agent = ClaudeCLIAgent() ssh = _ssh_with_conn("bash", conn) run = _fake_run() - await agent._run_cli(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="build it") + await run_claude( + ClaudeCLIConfig(), run, ssh=ssh, shell="bash", mcp_servers={}, prompt="build it" + ) assert sink == {} assert conn.write_commands == [] @@ -273,9 +265,8 @@ async def test_exec_on_bash_runs_inline_without_batch() -> None: async def test_exec_removes_mcp_config_after_run() -> None: sink: dict[str, bytes] = {} conn = _FakeConn(sink, _FakeStreamProcess(_STREAM_JSON)) - agent = ClaudeCLIAgent() - - await agent._run_cli( + await run_claude( + ClaudeCLIConfig(), _fake_run(), ssh=_ssh_with_conn("bash", conn), shell="bash", @@ -293,12 +284,18 @@ async def test_exec_removes_mcp_config_after_run() -> None: async def test_exec_records_steps_before_process_exit() -> None: process = _FakeStreamProcess(_STREAM_JSON, pause_after=1) conn = _FakeConn({}, process) - agent = ClaudeCLIAgent() ssh = _ssh_with_conn("bash", conn) run = _fake_run() execution = asyncio.create_task( - agent._run_cli(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="edit it") + run_claude( + ClaudeCLIConfig(), + run, + ssh=ssh, + shell="bash", + mcp_servers={}, + prompt="edit it", + ) ) await process.stdout.blocked.wait() @@ -336,9 +333,10 @@ async def test_exec_forwards_trace_id_only_to_hud_gateway( monkeypatch.setattr(settings, "anthropic_api_key", "anthropic-key") gateway_conn = _FakeConn({}, _FakeStreamProcess(_STREAM_JSON)) - gateway = ClaudeCLIAgent(ClaudeCLIConfig(use_hud_gateway=True)) + gateway = ClaudeCLIConfig(use_hud_gateway=True) with set_trace_context("trace-123"): - await gateway._run_cli( + await run_claude( + gateway, _fake_run(), ssh=_ssh_with_conn("bash", gateway_conn), shell="bash", @@ -348,9 +346,10 @@ async def test_exec_forwards_trace_id_only_to_hud_gateway( assert "ANTHROPIC_CUSTOM_HEADERS='Trace-Id: trace-123'" in gateway_conn.ran[0] provider_conn = _FakeConn({}, _FakeStreamProcess(_STREAM_JSON)) - provider = ClaudeCLIAgent(ClaudeCLIConfig(use_hud_gateway=False)) + provider = ClaudeCLIConfig(use_hud_gateway=False) with set_trace_context("trace-123"): - await provider._run_cli( + await run_claude( + provider, _fake_run(), ssh=_ssh_with_conn("bash", provider_conn), shell="bash", @@ -363,10 +362,9 @@ async def test_exec_forwards_trace_id_only_to_hud_gateway( async def test_exec_closes_streaming_process_when_cancelled() -> None: process = _FakeStreamProcess(_STREAM_JSON, pause_after=0) conn = _FakeConn({}, process) - agent = ClaudeCLIAgent() - execution = asyncio.create_task( - agent._run_cli( + run_claude( + ClaudeCLIConfig(), _fake_run(), ssh=_ssh_with_conn("bash", conn), shell="bash", @@ -386,12 +384,11 @@ async def test_exec_closes_streaming_process_when_cancelled() -> None: async def test_exec_nonzero_exit_with_no_stdout_raises() -> None: sink: dict[str, bytes] = {} conn = _FakeConn(sink, _FakeStreamProcess("", stderr="boom", exit_status=1)) - agent = ClaudeCLIAgent() ssh = _ssh_with_conn("cmd", conn) run = _fake_run() with pytest.raises(RuntimeError, match="boom"): - await agent._run_cli(run, ssh=ssh, shell="cmd", mcp_servers={}, prompt="x") + await run_claude(ClaudeCLIConfig(), run, ssh=ssh, shell="cmd", mcp_servers={}, prompt="x") assert run.trace.extra["returncode"] == 1 @@ -402,12 +399,11 @@ async def test_exec_signal_exit_records_the_returncode() -> None: sink, _FakeStreamProcess("", exit_status=None, returncode=-15), ) - agent = ClaudeCLIAgent() ssh = _ssh_with_conn("bash", conn) run = _fake_run() with pytest.raises(RuntimeError, match="return code -15"): - await agent._run_cli(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="x") + await run_claude(ClaudeCLIConfig(), run, ssh=ssh, shell="bash", mcp_servers={}, prompt="x") assert run.trace.extra["returncode"] == -15 @@ -418,12 +414,11 @@ async def test_exec_nonzero_exit_with_result_stream_remains_an_error() -> None: sink, _FakeStreamProcess(_STREAM_JSON, stderr="transport failed", exit_status=1), ) - agent = ClaudeCLIAgent() ssh = _ssh_with_conn("bash", conn) run = _fake_run() with pytest.raises(RuntimeError, match="transport failed"): - await agent._run_cli(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="x") + await run_claude(ClaudeCLIConfig(), run, ssh=ssh, shell="bash", mcp_servers={}, prompt="x") assert run.trace.content == "done" assert run.trace.extra["returncode"] == 1 @@ -435,12 +430,11 @@ async def test_exec_zero_exit_without_result_event_is_an_error() -> None: sink: dict[str, bytes] = {} stdout = _STREAM_JSON.rsplit('{"type":"result"', 1)[0] conn = _FakeConn(sink, _FakeStreamProcess(stdout)) - agent = ClaudeCLIAgent() ssh = _ssh_with_conn("bash", conn) run = _fake_run() with pytest.raises(RuntimeError, match="without a result event"): - await agent._run_cli(run, ssh=ssh, shell="bash", mcp_servers={}, prompt="x") + await run_claude(ClaudeCLIConfig(), run, ssh=ssh, shell="bash", mcp_servers={}, prompt="x") assert run.trace.content == "done" @@ -476,7 +470,7 @@ async def open(self, ref: str) -> SSHClient: agent = ClaudeCLIAgent() execute = AsyncMock() - monkeypatch.setattr(agent, "_run_cli", execute) + monkeypatch.setattr("hud.agents.claude.cli.agent.run_claude", execute) await agent( cast( @@ -547,7 +541,7 @@ async def execute(*_args: Any, **_kwargs: Any) -> None: execute_mock = AsyncMock(side_effect=execute) monkeypatch.setattr(computer_mcp, "bridge_computer_mcp", bridge) - monkeypatch.setattr(agent, "_run_cli", execute_mock) + monkeypatch.setattr("hud.agents.claude.cli.agent.run_claude", execute_mock) await agent( cast( @@ -634,7 +628,7 @@ async def execute(*_args: Any, **kwargs: Any) -> None: agent = ClaudeCLIAgent() monkeypatch.setattr(computer_mcp, "bridge_computer_mcp", bridge) - monkeypatch.setattr(agent, "_run_cli", execute) + monkeypatch.setattr("hud.agents.claude.cli.agent.run_claude", execute) await agent(cast("Any", SimpleNamespace(client=Client(), prompt_text="use both screens"))) @@ -883,6 +877,7 @@ async def open(self, ref: str) -> SSHClient: seen: list[tuple[Any, SSHClient, str]] = [] async def execute( + _config: ClaudeCLIConfig, run: Any, *, ssh: SSHClient, @@ -897,7 +892,7 @@ async def execute( await release_first.wait() agent = ClaudeCLIAgent() - monkeypatch.setattr(agent, "_run_cli", execute) + monkeypatch.setattr("hud.agents.claude.cli.agent.run_claude", execute) run_a = SimpleNamespace(client=Client(shell_a, ssh_a), prompt_text="first") run_b = SimpleNamespace(client=Client(shell_b, ssh_b), prompt_text="second") diff --git a/hud/agents/tests/test_codex_cli_agent.py b/hud/agents/tests/test_codex_cli_agent.py new file mode 100644 index 000000000..135690731 --- /dev/null +++ b/hud/agents/tests/test_codex_cli_agent.py @@ -0,0 +1,348 @@ +"""CodexCLIAgent command construction and JSONL trajectory mapping.""" + +from __future__ import annotations + +import asyncio +import base64 +from types import SimpleNamespace +from typing import Any, cast +from unittest.mock import AsyncMock + +import pytest +from mcp.types import TextContent + +from hud.agents.codex import CodexCLIAgent +from hud.agents.codex.agent import codex_command, run_codex +from hud.agents.types import AgentStep, CodexCLIConfig, ToolStep +from hud.capabilities import Capability, SSHClient +from hud.settings import settings +from hud.telemetry.context import set_trace_context + + +@pytest.fixture(autouse=True) +def _clear_api_keys(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "api_key", None) + monkeypatch.setattr(settings, "openai_api_key", None) + + +class _FakeReader: + def __init__(self, value: str, *, pause_after: int | None = None) -> None: + self._raw = value.encode() + self._lines = self._raw.splitlines(keepends=True) + self._pause_after = pause_after + self._index = 0 + self.blocked = asyncio.Event() + self.release = asyncio.Event() + + async def readline(self) -> bytes: + if self._pause_after == self._index: + self.blocked.set() + await self.release.wait() + self._pause_after = None + if self._index == len(self._lines): + return b"" + line = self._lines[self._index] + self._index += 1 + return line + + async def read(self) -> bytes: + return self._raw + + +class _FakeWriter: + def __init__(self) -> None: + self.data = b"" + self.eof = False + + def write(self, data: bytes) -> None: + self.data += data + + async def drain(self) -> None: + pass + + def write_eof(self) -> None: + self.eof = True + + +class _FakeProcess: + def __init__( + self, + stdout: str, + *, + stderr: str = "", + returncode: int | None = 0, + pause_after: int | None = None, + ) -> None: + self.stdin = _FakeWriter() + self.stdout = _FakeReader(stdout, pause_after=pause_after) + self.stderr = _FakeReader(stderr) + self.returncode = returncode + self.closed = False + + def close(self) -> None: + self.closed = True + + async def wait_closed(self) -> None: + pass + + +class _FakeSSH: + def __init__(self, process: _FakeProcess, *, shell: str = "bash") -> None: + self.process = process + self.capability = Capability( + name="shell", + protocol="ssh/2", + url="ssh://localhost:22", + params={"shell": shell}, + ) + self.commands: list[str] = [] + + async def create_process(self, command: str) -> _FakeProcess: + self.commands.append(command) + return self.process + + +def _fake_run() -> Any: + trace = SimpleNamespace(status=None, content="", extra={}) + steps: list[Any] = [] + return SimpleNamespace(trace=trace, record=steps.append, steps=steps) + + +_STREAM_JSON = ( + '{"type":"thread.started","thread_id":"thread-1"}\n' + '{"type":"turn.started"}\n' + '{"type":"item.started","item":{"id":"cmd-1","type":"command_execution",' + '"command":"pytest -q","aggregated_output":"","exit_code":null,' + '"status":"in_progress"}}\n' + '{"type":"item.completed","item":{"id":"cmd-1","type":"command_execution",' + '"command":"pytest -q","aggregated_output":"1 passed\\n","exit_code":0,' + '"status":"completed"}}\n' + '{"type":"item.completed","item":{"id":"patch-1","type":"file_change",' + '"changes":[{"path":"calc.py","kind":"update"}],"status":"completed"}}\n' + '{"type":"item.started","item":{"id":"mcp-1","type":"mcp_tool_call",' + '"server":"db","tool":"query","arguments":{"sql":"select 42"},' + '"result":null,"error":null,"status":"in_progress"}}\n' + '{"type":"item.completed","item":{"id":"mcp-1","type":"mcp_tool_call",' + '"server":"db","tool":"query","arguments":{"sql":"select 42"},' + '"result":{"content":[{"type":"text","text":"42"}],' + '"structured_content":{"answer":42}},"error":null,"status":"completed"}}\n' + '{"type":"item.completed","item":{"id":"search-1","type":"web_search",' + '"query":"HUD evals","action":{"type":"search"}}}\n' + '{"type":"item.completed","item":{"id":"reason-1","type":"reasoning",' + '"text":"The test now passes."}}\n' + '{"type":"item.completed","item":{"id":"message-1","type":"agent_message",' + '"text":"Implemented and verified."}}\n' + '{"type":"turn.completed","usage":{"input_tokens":20,"cached_input_tokens":5,' + '"output_tokens":8,"reasoning_output_tokens":3}}\n' +) + + +def test_command_follows_explicit_gateway_routing(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "api_key", "hud-key") + monkeypatch.setattr(settings, "openai_api_key", "openai-key") + + with set_trace_context("trace-123"): + gateway = codex_command(CodexCLIConfig(use_hud_gateway=True), "bash") + provider = codex_command(CodexCLIConfig(use_hud_gateway=False), "bash") + + assert "HUD_API_KEY=hud-key" in gateway + assert 'model_provider="hud"' in gateway + assert f'model_providers.hud.base_url="{settings.hud_gateway_url}"' in gateway + assert "Trace-Id" in gateway + assert "CODEX_API_KEY=openai-key" in provider + assert "model_provider" not in provider + for command in (gateway, provider): + assert "codex exec" in command + assert "--json" in command + assert "--ephemeral" in command + assert "--sandbox workspace-write" in command + assert "--model gpt-5.6-sol" in command + assert command.endswith(" -") + + +def test_windows_command_encodes_environment_and_arguments( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(settings, "openai_api_key", "key&value's") + config = CodexCLIConfig(use_hud_gateway=False, sandbox="danger-full-access") + command = codex_command(config, "powershell") + + script = base64.b64decode(command.rsplit(" ", 1)[1]).decode("utf-16-le") + assert "$env:CODEX_API_KEY='key&value''s'" in script + assert "'--sandbox' 'danger-full-access'" in script + assert "& codex 'exec'" in script + assert script.endswith(";exit $LASTEXITCODE") + + +async def test_exec_streams_prompt_and_records_codex_items() -> None: + process = _FakeProcess(_STREAM_JSON) + ssh = _FakeSSH(process) + run = _fake_run() + + await run_codex( + CodexCLIConfig(), + run, + ssh=cast("SSHClient", ssh), + shell="bash", + prompt="Fix the failing test", + ) + + assert process.stdin.data == b"Fix the failing test" + assert process.stdin.eof + assert [type(step) for step in run.steps] == [ + ToolStep, + ToolStep, + ToolStep, + ToolStep, + AgentStep, + AgentStep, + ] + command = cast("ToolStep", run.steps[0]) + assert command.call is not None + assert command.call.name == "shell" + assert command.call.arguments == {"command": "pytest -q"} + assert command.result is not None + assert command.result.isError is False + output = command.result.content[0] + assert isinstance(output, TextContent) + assert output.text == "1 passed\n" + patch = cast("ToolStep", run.steps[1]) + assert patch.call is not None + assert patch.call.name == "apply_patch" + mcp = cast("ToolStep", run.steps[2]) + assert mcp.call is not None + assert mcp.call.name == "query" + assert mcp.call.provider_name == "db.query" + assert mcp.result is not None + assert mcp.result.structuredContent == {"answer": 42} + search = cast("ToolStep", run.steps[3]) + assert search.call is not None + assert search.call.name == "web_search" + assert cast("AgentStep", run.steps[4]).reasoning == "The test now passes." + assert cast("AgentStep", run.steps[5]).content == "Implemented and verified." + assert run.trace.content == "Implemented and verified." + assert run.trace.extra["codex_thread_id"] == "thread-1" + assert run.trace.extra["usage"]["cached_input_tokens"] == 5 + assert run.trace.status is None + + +async def test_exec_records_completed_items_before_process_exit() -> None: + process = _FakeProcess(_STREAM_JSON, pause_after=4) + ssh = _FakeSSH(process) + run = _fake_run() + execution = asyncio.create_task( + run_codex( + CodexCLIConfig(), + run, + ssh=cast("SSHClient", ssh), + shell="bash", + prompt="Fix it", + ) + ) + await process.stdout.blocked.wait() + + assert not execution.done() + assert len(run.steps) == 1 + assert isinstance(run.steps[0], ToolStep) + + process.stdout.release.set() + await execution + + +async def test_exec_turn_failure_raises() -> None: + stream = ( + '{"type":"thread.started","thread_id":"thread-1"}\n' + '{"type":"turn.started"}\n' + '{"type":"turn.failed","error":{"message":"model unavailable"}}\n' + ) + run = _fake_run() + + with pytest.raises(RuntimeError, match="model unavailable"): + await run_codex( + CodexCLIConfig(), + run, + ssh=cast("SSHClient", _FakeSSH(_FakeProcess(stream))), + shell="bash", + prompt="Fix it", + ) + + +async def test_exec_nonzero_exit_raises_stderr() -> None: + run = _fake_run() + + with pytest.raises(RuntimeError, match="authentication failed"): + await run_codex( + CodexCLIConfig(), + run, + ssh=cast( + "SSHClient", + _FakeSSH(_FakeProcess("", stderr="authentication failed", returncode=1)), + ), + shell="bash", + prompt="Fix it", + ) + + assert run.trace.extra["returncode"] == 1 + + +async def test_exec_nonzero_exit_prefers_structured_error() -> None: + run = _fake_run() + stream = '{"type":"error","message":"gateway rejected streaming"}\n' + + with pytest.raises(RuntimeError, match="gateway rejected streaming"): + await run_codex( + CodexCLIConfig(), + run, + ssh=cast( + "SSHClient", + _FakeSSH(_FakeProcess(stream, stderr="noisy warning", returncode=1)), + ), + shell="bash", + prompt="Fix it", + ) + + assert "stderr" not in run.trace.extra + + +async def test_exec_closes_process_when_cancelled() -> None: + process = _FakeProcess(_STREAM_JSON, pause_after=0) + execution = asyncio.create_task( + run_codex( + CodexCLIConfig(), + _fake_run(), + ssh=cast("SSHClient", _FakeSSH(process)), + shell="bash", + prompt="Fix it", + ) + ) + await process.stdout.blocked.wait() + execution.cancel() + + with pytest.raises(asyncio.CancelledError): + await execution + + assert process.closed + + +async def test_agent_opens_ssh_and_uses_workspace_prompt(monkeypatch: pytest.MonkeyPatch) -> None: + ssh = _FakeSSH(_FakeProcess(_STREAM_JSON), shell="powershell") + + class Client: + async def open(self, ref: str) -> _FakeSSH: + assert ref == "ssh" + return ssh + + agent = CodexCLIAgent() + execute = AsyncMock() + monkeypatch.setattr("hud.agents.codex.agent.run_codex", execute) + run = SimpleNamespace(client=Client(), prompt_text="Fix it") + + await agent(cast("Any", run)) + + execute.assert_awaited_once_with( + agent.config, + run, + ssh=ssh, + shell="powershell", + prompt="Fix it", + ) diff --git a/hud/agents/types.py b/hud/agents/types.py index f8103a497..4dbf3e3bd 100644 --- a/hud/agents/types.py +++ b/hud/agents/types.py @@ -176,6 +176,20 @@ class ClaudeCLIConfig(AgentConfig): ) +# ----------------------------------------------------------------------------- +# Codex CLI (over SSH) +# ----------------------------------------------------------------------------- + + +class CodexCLIConfig(AgentConfig): + """Configuration for CodexCLIAgent (runs ``codex exec`` over SSH).""" + + model_name: str = "Codex CLI" + model: str = Field(default="gpt-5.6-sol", validation_alias=_model_alias) + use_hud_gateway: bool | None = None + sandbox: Literal["read-only", "workspace-write", "danger-full-access"] = "workspace-write" + + # ----------------------------------------------------------------------------- # Browser Use # ----------------------------------------------------------------------------- diff --git a/hud/cli/tests/test_eval_config.py b/hud/cli/tests/test_eval_config.py index 5148d6a16..e1e521155 100644 --- a/hud/cli/tests/test_eval_config.py +++ b/hud/cli/tests/test_eval_config.py @@ -375,6 +375,25 @@ def test_build_agent_preserves_claude_cli_gateway_config() -> None: assert agent.config.use_hud_gateway is True +def test_build_agent_constructs_codex_cli() -> None: + from hud.agents.codex import CodexCLIAgent + + cfg = EvalConfig(agent_type="codex_cli") + agent = eval_mod._build_agent(cfg) + + assert isinstance(agent, CodexCLIAgent) + assert agent.config.model == "gpt-5.6-sol" + assert agent.config.use_hud_gateway is False + + +def test_build_agent_routes_hosted_codex_cli_through_gateway() -> None: + cfg = EvalConfig(agent_type="codex_cli", remote=True) + + agent = eval_mod._build_agent(cfg) + + assert agent.config.use_hud_gateway is True + + def test_spawn_target_serves_single_file_env(tmp_path: Path) -> None: env_py = tmp_path / "tasks.py" env_py.write_text( diff --git a/hud/types.py b/hud/types.py index c697727ee..05982985d 100644 --- a/hud/types.py +++ b/hud/types.py @@ -53,13 +53,14 @@ class AgentType(StrEnum): CLAUDE = "claude" CLAUDE_CLI = "claude_cli" + CODEX_CLI = "codex_cli" OPENAI = "openai" GEMINI = "gemini" OPENAI_COMPATIBLE = "openai_compatible" @property def is_cli(self) -> bool: - return self is AgentType.CLAUDE_CLI + return self in (AgentType.CLAUDE_CLI, AgentType.CODEX_CLI) @property def cls(self) -> type[Agent]: @@ -72,6 +73,10 @@ def cls(self) -> type[Agent]: from hud.agents import ClaudeCLIAgent return ClaudeCLIAgent + case AgentType.CODEX_CLI: + from hud.agents import CodexCLIAgent + + return CodexCLIAgent case AgentType.OPENAI: from hud.agents import OpenAIAgent @@ -91,6 +96,7 @@ def config_cls(self) -> type[AgentConfig]: from hud.agents.types import ( ClaudeCLIConfig, ClaudeConfig, + CodexCLIConfig, GeminiConfig, OpenAIChatConfig, OpenAIConfig, @@ -101,6 +107,8 @@ def config_cls(self) -> type[AgentConfig]: return ClaudeConfig case AgentType.CLAUDE_CLI: return ClaudeCLIConfig + case AgentType.CODEX_CLI: + return CodexCLIConfig case AgentType.OPENAI: return OpenAIConfig case AgentType.GEMINI: @@ -119,6 +127,8 @@ def gateway_provider(self) -> str: return "anthropic" case AgentType.CLAUDE_CLI: return "anthropic" + case AgentType.CODEX_CLI: + return "openai" case AgentType.OPENAI: return "openai" case AgentType.GEMINI: From a35c6d86ab9b854062ae808acd4d3265ee651d2c Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 21 Aug 2026 15:03:14 -0700 Subject: [PATCH 06/14] Fix Codex CLI config isolation --- hud/agents/codex/agent.py | 21 +++++++++++++++++---- hud/agents/tests/test_codex_cli_agent.py | 9 ++++++++- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/hud/agents/codex/agent.py b/hud/agents/codex/agent.py index 68ba357b4..8cb4ed34e 100644 --- a/hud/agents/codex/agent.py +++ b/hud/agents/codex/agent.py @@ -204,7 +204,6 @@ def codex_command(config: CodexCLIConfig, shell: str) -> str: "exec", "--json", "--ephemeral", - "--ignore-user-config", "--skip-git-repo-check", "--color", "never", @@ -244,9 +243,15 @@ def codex_command(config: CodexCLIConfig, shell: str) -> str: if shell in WINDOWS_SHELLS: script = ";".join( [ + "$codexHome=Join-Path ([System.IO.Path]::GetTempPath()) " + "('hud-codex-' + [System.Guid]::NewGuid())", + "New-Item -ItemType Directory -Force -Path $codexHome | Out-Null", + "$env:CODEX_HOME=$codexHome", *(f"$env:{key}={powershell_quote(value)}" for key, value in env.items()), - f"& codex {' '.join(powershell_quote(arg) for arg in args[1:])}", - "exit $LASTEXITCODE", + f"try {{ & codex {' '.join(powershell_quote(arg) for arg in args[1:])}; " + "$hudExitCode=$LASTEXITCODE } finally { Remove-Item -Recurse -Force " + "$codexHome }", + "exit $hudExitCode", ] ) return powershell(script) @@ -254,7 +259,15 @@ def codex_command(config: CodexCLIConfig, shell: str) -> str: command = " ".join(shlex.quote(arg) for arg in args) env_prefix = " ".join(f"{key}={shlex.quote(value)}" for key, value in env.items()) invocation = f"{env_prefix} {command}" if env_prefix else command - return f'export PATH="$HOME/.local/bin:$PATH"; {invocation}' + return "; ".join( + [ + 'codex_home=$(mktemp -d "${TMPDIR:-/tmp}/hud-codex.XXXXXX") || exit 1', + "trap 'rm -rf -- \"$codex_home\"' EXIT", + 'export CODEX_HOME="$codex_home"', + 'export PATH="$HOME/.local/bin:$PATH"', + invocation, + ] + ) async def run_codex( diff --git a/hud/agents/tests/test_codex_cli_agent.py b/hud/agents/tests/test_codex_cli_agent.py index 135690731..a5ba2a2b8 100644 --- a/hud/agents/tests/test_codex_cli_agent.py +++ b/hud/agents/tests/test_codex_cli_agent.py @@ -155,6 +155,9 @@ def test_command_follows_explicit_gateway_routing(monkeypatch: pytest.MonkeyPatc assert "codex exec" in command assert "--json" in command assert "--ephemeral" in command + assert "--ignore-user-config" not in command + assert "mktemp -d" in command + assert 'export CODEX_HOME="$codex_home"' in command assert "--sandbox workspace-write" in command assert "--model gpt-5.6-sol" in command assert command.endswith(" -") @@ -169,9 +172,13 @@ def test_windows_command_encodes_environment_and_arguments( script = base64.b64decode(command.rsplit(" ", 1)[1]).decode("utf-16-le") assert "$env:CODEX_API_KEY='key&value''s'" in script + assert "$env:CODEX_HOME=$codexHome" in script + assert "[System.Guid]::NewGuid()" in script + assert "Remove-Item -Recurse -Force $codexHome" in script + assert "--ignore-user-config" not in script assert "'--sandbox' 'danger-full-access'" in script assert "& codex 'exec'" in script - assert script.endswith(";exit $LASTEXITCODE") + assert script.endswith(";exit $hudExitCode") async def test_exec_streams_prompt_and_records_codex_items() -> None: From 55b9f0a15c12ad8ad1309764de5832afc39b20fe Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:42:27 -0700 Subject: [PATCH 07/14] Resolve managed CLI agent runtimes --- docs/v6/cookbooks/coding-agent.mdx | 7 +- docs/v6/reference/agents.mdx | 6 +- hud/agents/claude/cli/agent.py | 30 +++++- hud/agents/cli.py | 102 ++++++++++++++++++ hud/agents/codex/agent.py | 30 +++++- hud/agents/tests/test_claude_cli_agent.py | 27 +++-- hud/agents/tests/test_codex_cli_agent.py | 48 ++++++++- hud/capabilities/ssh.py | 88 ++++++++++++++- hud/environment/tests/test_workspace.py | 19 ++++ hud/eval/run.py | 6 +- hud/integrations/harbor/env.py | 1 + .../harbor/tests/test_contract.py | 1 + 12 files changed, 341 insertions(+), 24 deletions(-) diff --git a/docs/v6/cookbooks/coding-agent.mdx b/docs/v6/cookbooks/coding-agent.mdx index 1e261a915..be81fcfcb 100644 --- a/docs/v6/cookbooks/coding-agent.mdx +++ b/docs/v6/cookbooks/coding-agent.mdx @@ -72,9 +72,10 @@ Codex uses the same environment through the `codex_cli` agent: hud eval env.py codex_cli --gateway ``` -The selected executable must already be installed in the host or environment image. Pinning it in -the image keeps runs reproducible; CLI agents do not install or update it. The equivalent Claude -Python API is: +The SSH runtime must expose the selected executable, either as a managed runtime bundle or through +the environment image. The agent validates the effective runtime OS against the live SSH target, +prefers a compatible managed bundle, and otherwise resolves the executable from `PATH`. CLI agents +do not download or update executables. The equivalent Claude Python API is: ```python run.py import asyncio diff --git a/docs/v6/reference/agents.mdx b/docs/v6/reference/agents.mdx index ee4ff349a..2db5f5527 100644 --- a/docs/v6/reference/agents.mdx +++ b/docs/v6/reference/agents.mdx @@ -65,8 +65,10 @@ agent = ClaudeAgent(ClaudeConfig(model="claude-sonnet-4-5", max_steps=30)) Each config lives in `hud.agents.types`. `OpenAIChatAgent` speaks the OpenAI Chat Completions API, so it points at any compatible server (vLLM, a local model) via `base_url`. `ClaudeCLIAgent` and `CodexCLIAgent` run their respective CLIs over an `ssh` capability against the env's filesystem and -stream the CLI's structured events into the HUD trace. SSH-only runs support POSIX and Windows -workspaces; Claude computer use over an `rfb` capability currently requires a POSIX workspace. Every knob +stream the CLI's structured events into the HUD trace. They prefer a compatible managed runtime +bundle and fall back to the environment's `PATH`; they never install or update the executable. +SSH-only runs support POSIX and Windows workspaces; Claude computer use over an `rfb` capability +currently requires a POSIX workspace. Every knob (`model`, `max_steps`, `timeout_seconds`, `tool_timeout_seconds`, `system_prompt`, `citations_enabled`, `stop_on`) lives on the config; `__call__(run)` takes only the run. diff --git a/hud/agents/claude/cli/agent.py b/hud/agents/claude/cli/agent.py index c5b7d0066..3167b5c79 100644 --- a/hud/agents/claude/cli/agent.py +++ b/hud/agents/claude/cli/agent.py @@ -14,7 +14,13 @@ from hud.agents.base import Agent from hud.agents.claude.agent import ClaudeAgent -from hud.agents.cli import WINDOWS_SHELLS, powershell, powershell_quote, run_jsonl +from hud.agents.cli import ( + WINDOWS_SHELLS, + powershell, + powershell_quote, + resolve_executable, + run_jsonl, +) from hud.agents.types import ClaudeCLIConfig, ToolStep from hud.settings import settings from hud.telemetry.context import get_current_trace_id @@ -33,6 +39,11 @@ MCP_CONFIG_PATH = ".hud_mcp_config.json" RUN_SCRIPT_PATH = ".hud_run.bat" +_MANAGED_CLAUDE_PATHS = { + "linux-x64": "/media/hud/bin/claude/linux-x64/claude", + "linux-x64-musl": "/media/hud/bin/claude/linux-x64-musl/claude", +} + class ClaudeEvents: """Translate Claude CLI stream messages into canonical HUD steps.""" @@ -157,6 +168,7 @@ def claude_command( shell: str, prompt: str, mcp_config_path: str | None = None, + executable: str = "claude", ) -> str: env: dict[str, str] = {} use_hud_gateway = config.use_hud_gateway @@ -188,10 +200,11 @@ def claude_command( env[name] = config.model env["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC"] = "1" + env["DISABLE_AUTOUPDATER"] = "1" env["IS_SANDBOX"] = "1" args: list[str] = [ - "claude", + executable, "--verbose", "--output-format=stream-json", "--print", @@ -211,7 +224,8 @@ def claude_command( [ *(f"$env:{key}={powershell_quote(value)}" for key, value in env.items()), f"Get-Content -Raw -Encoding UTF8 {powershell_quote(PROMPT_PATH)}" - f" | & claude {' '.join(powershell_quote(arg) for arg in args[1:])}", + f" | & {powershell_quote(executable)} " + f"{' '.join(powershell_quote(arg) for arg in args[1:])}", "exit $LASTEXITCODE", ] ) @@ -231,6 +245,7 @@ async def run_claude( shell: str, mcp_servers: dict[str, dict[str, Any]], prompt: str, + executable: str = "claude", ) -> None: files: dict[str, str] = {} mcp_config_path = MCP_CONFIG_PATH if mcp_servers else None @@ -239,7 +254,7 @@ async def run_claude( if shell in WINDOWS_SHELLS: files[PROMPT_PATH] = prompt - command = claude_command(config, shell, prompt, mcp_config_path) + command = claude_command(config, shell, prompt, mcp_config_path, executable) if shell in WINDOWS_SHELLS: files[RUN_SCRIPT_PATH] = f"@echo off\r\n{command}\r\n" command = f"cmd /c {RUN_SCRIPT_PATH}" @@ -279,6 +294,12 @@ async def __call__(self, run: Run) -> None: assert manifest is not None bindings = manifest.bindings shell = ssh.capability.params.get("shell", "bash") + executable = await resolve_executable( + ssh, + "claude", + _MANAGED_CLAUDE_PATHS, + run.runtime_config, + ) rfb_bindings = [cap for cap in bindings if cap.protocol.split("/", 1)[0] == "rfb"] async with AsyncExitStack() as resources: @@ -316,6 +337,7 @@ async def __call__(self, run: Run) -> None: shell=shell, mcp_servers=mcp_servers, prompt=run.prompt_text, + executable=executable, ) diff --git a/hud/agents/cli.py b/hud/agents/cli.py index 5fc9583f5..d626c024e 100644 --- a/hud/agents/cli.py +++ b/hud/agents/cli.py @@ -5,6 +5,7 @@ import asyncio import base64 import contextlib +import shlex from typing import TYPE_CHECKING import asyncssh @@ -13,11 +14,112 @@ from collections.abc import Callable from hud.capabilities import SSHClient + from hud.eval.runtime import RuntimeConfig WINDOWS_SHELLS = ("cmd", "powershell") PROCESS_CLOSE_TIMEOUT_S = 5.0 +async def resolve_executable( + ssh: SSHClient, + command: str, + managed_paths: dict[str, str], + runtime_config: RuntimeConfig | None, +) -> str: + """Resolve a CLI against the live SSH target and its declared runtime config.""" + platform = await _runtime_platform(ssh) + _validate_runtime_os(runtime_config, platform.partition("-")[0]) + + managed = managed_paths.get(platform) + if managed is not None: + result = await ssh.run( + f"test -x {shlex.quote(managed)}", + check=False, + encoding=None, + ) + if result.returncode == 0: + return managed + + if platform.startswith("windows-"): + result = await ssh.run(f"where.exe {command}", check=False, encoding=None) + else: + result = await ssh.run(f"command -v -- {command}", check=False, encoding=None) + if result.returncode == 0: + stdout = _output_text(result.stdout) + if path := stdout.splitlines()[0].strip(): + return path + + raise RuntimeError( + f"{command} is unavailable for runtime platform {platform}; " + "install it in the environment or provide a managed runtime bundle" + ) + + +async def _runtime_platform(ssh: SSHClient) -> str: + shell = ssh.capability.params.get("shell", "bash") + if shell in WINDOWS_SHELLS: + result = await ssh.run( + powershell("[System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture"), + check=True, + encoding=None, + ) + arch = _output_text(result.stdout).strip().lower() + return f"windows-{_normalize_arch(arch)}" + + result = await ssh.run( + "uname -s; uname -m; " + "if ls /lib/ld-musl-*.so.1 >/dev/null 2>&1; then echo musl; else echo gnu; fi", + check=True, + encoding=None, + ) + lines = _output_text(result.stdout).splitlines() + if len(lines) != 3: + raise RuntimeError("SSH runtime platform probe returned an invalid response") + system, machine, libc = (line.strip().lower() for line in lines) + os_name = {"darwin": "darwin", "linux": "linux"}.get(system) + if os_name is None: + raise RuntimeError(f"unsupported SSH runtime operating system {system!r}") + platform = f"{os_name}-{_normalize_arch(machine)}" + return f"{platform}-musl" if os_name == "linux" and libc == "musl" else platform + + +def _normalize_arch(value: str) -> str: + normalized = { + "amd64": "x64", + "x86_64": "x64", + "arm64": "arm64", + "aarch64": "arm64", + }.get(value) + if normalized is None: + raise RuntimeError(f"unsupported SSH runtime architecture {value!r}") + return normalized + + +def _output_text(value: bytes | str | None) -> str: + if isinstance(value, bytes): + return value.decode(errors="replace") + return value or "" + + +def _validate_runtime_os(runtime_config: RuntimeConfig | None, actual: str) -> None: + if runtime_config is None or runtime_config.resources is None: + return + declared = runtime_config.resources.os + if declared is None: + return + normalized = { + "darwin": "darwin", + "linux": "linux", + "macos": "darwin", + "windows": "windows", + }.get(declared.lower()) + if normalized is not None and normalized != actual: + raise RuntimeError( + f"runtime_config.resources.os requested {declared!r}, " + f"but the SSH runtime reports {actual!r}" + ) + + def powershell_quote(value: str) -> str: return "'" + value.replace("'", "''") + "'" diff --git a/hud/agents/codex/agent.py b/hud/agents/codex/agent.py index 8cb4ed34e..ed4f6946b 100644 --- a/hud/agents/codex/agent.py +++ b/hud/agents/codex/agent.py @@ -10,7 +10,13 @@ import mcp.types as mcp_types from hud.agents.base import Agent -from hud.agents.cli import WINDOWS_SHELLS, powershell, powershell_quote, run_jsonl +from hud.agents.cli import ( + WINDOWS_SHELLS, + powershell, + powershell_quote, + resolve_executable, + run_jsonl, +) from hud.agents.types import AgentStep, CodexCLIConfig, ToolStep from hud.settings import settings from hud.telemetry.context import get_current_trace_id @@ -23,6 +29,11 @@ logger = logging.getLogger(__name__) +_MANAGED_CODEX_PATHS = { + "linux-x64": "/media/hud/bin/codex/bin/codex", + "linux-x64-musl": "/media/hud/bin/codex/bin/codex", +} + class CodexEvents: """Translate ``codex exec --json`` events into canonical HUD steps.""" @@ -197,10 +208,10 @@ def record_tool(self, item: dict[str, Any], started_at: str, ended_at: str) -> N ) -def codex_command(config: CodexCLIConfig, shell: str) -> str: +def codex_command(config: CodexCLIConfig, shell: str, executable: str = "codex") -> str: env: dict[str, str] = {} args = [ - "codex", + executable, "exec", "--json", "--ephemeral", @@ -248,7 +259,8 @@ def codex_command(config: CodexCLIConfig, shell: str) -> str: "New-Item -ItemType Directory -Force -Path $codexHome | Out-Null", "$env:CODEX_HOME=$codexHome", *(f"$env:{key}={powershell_quote(value)}" for key, value in env.items()), - f"try {{ & codex {' '.join(powershell_quote(arg) for arg in args[1:])}; " + f"try {{ & {powershell_quote(executable)} " + f"{' '.join(powershell_quote(arg) for arg in args[1:])}; " "$hudExitCode=$LASTEXITCODE } finally { Remove-Item -Recurse -Force " "$codexHome }", "exit $hudExitCode", @@ -277,8 +289,9 @@ async def run_codex( ssh: SSHClient, shell: str, prompt: str, + executable: str = "codex", ) -> None: - command = codex_command(config, shell) + command = codex_command(config, shell, executable) logger.info("SSH exec codex CLI (%d chars)", len(command)) events = CodexEvents(run, model=config.model, started_at=now_iso()) returncode, stderr = await run_jsonl(ssh, command, events.consume, input_text=prompt) @@ -296,12 +309,19 @@ def __init__(self, config: CodexCLIConfig | None = None) -> None: async def __call__(self, run: Run) -> None: ssh = cast("SSHClient", await run.client.open("ssh")) + executable = await resolve_executable( + ssh, + "codex", + _MANAGED_CODEX_PATHS, + run.runtime_config, + ) await run_codex( self.config, run, ssh=ssh, shell=ssh.capability.params.get("shell", "bash"), prompt=run.prompt_text, + executable=executable, ) diff --git a/hud/agents/tests/test_claude_cli_agent.py b/hud/agents/tests/test_claude_cli_agent.py index fd5864e69..8248341db 100644 --- a/hud/agents/tests/test_claude_cli_agent.py +++ b/hud/agents/tests/test_claude_cli_agent.py @@ -40,6 +40,10 @@ def _clear_api_keys(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(settings, "api_key", None) monkeypatch.setattr(settings, "anthropic_api_key", None) + monkeypatch.setattr( + "hud.agents.claude.cli.agent.resolve_executable", + AsyncMock(return_value="claude"), + ) def test_command_follows_explicit_gateway_routing(monkeypatch: pytest.MonkeyPatch) -> None: @@ -71,7 +75,7 @@ def test_windows_command_encodes_environment_and_arguments( script = base64.b64decode(encoded).decode("utf-16-le") assert "$env:ANTHROPIC_API_KEY='hud&key''s'" in script assert "'--system-prompt' 'don''t $expand'" in script - assert "Get-Content -Raw -Encoding UTF8 '.hud_prompt.txt' | & claude" in script + assert "Get-Content -Raw -Encoding UTF8 '.hud_prompt.txt' | & 'claude'" in script assert "not embedded" not in script assert "python" not in script @@ -475,7 +479,7 @@ async def open(self, ref: str) -> SSHClient: await agent( cast( "Any", - SimpleNamespace(client=Client(), prompt_text="call the tool"), + SimpleNamespace(client=Client(), prompt_text="call the tool", runtime_config=None), ) ) @@ -546,7 +550,7 @@ async def execute(*_args: Any, **_kwargs: Any) -> None: await agent( cast( "Any", - SimpleNamespace(client=Client(), prompt_text="use the computer"), + SimpleNamespace(client=Client(), prompt_text="use the computer", runtime_config=None), ) ) @@ -630,7 +634,16 @@ async def execute(*_args: Any, **kwargs: Any) -> None: monkeypatch.setattr(computer_mcp, "bridge_computer_mcp", bridge) monkeypatch.setattr("hud.agents.claude.cli.agent.run_claude", execute) - await agent(cast("Any", SimpleNamespace(client=Client(), prompt_text="use both screens"))) + await agent( + cast( + "Any", + SimpleNamespace( + client=Client(), + prompt_text="use both screens", + runtime_config=None, + ), + ) + ) assert bridged == [] @@ -893,8 +906,10 @@ async def execute( agent = ClaudeCLIAgent() monkeypatch.setattr("hud.agents.claude.cli.agent.run_claude", execute) - run_a = SimpleNamespace(client=Client(shell_a, ssh_a), prompt_text="first") - run_b = SimpleNamespace(client=Client(shell_b, ssh_b), prompt_text="second") + run_a = SimpleNamespace(client=Client(shell_a, ssh_a), prompt_text="first", runtime_config=None) + run_b = SimpleNamespace( + client=Client(shell_b, ssh_b), prompt_text="second", runtime_config=None + ) first = asyncio.create_task(agent(cast("Any", run_a))) await first_entered.wait() diff --git a/hud/agents/tests/test_codex_cli_agent.py b/hud/agents/tests/test_codex_cli_agent.py index a5ba2a2b8..53f739f0b 100644 --- a/hud/agents/tests/test_codex_cli_agent.py +++ b/hud/agents/tests/test_codex_cli_agent.py @@ -11,10 +11,12 @@ import pytest from mcp.types import TextContent +from hud.agents.cli import resolve_executable from hud.agents.codex import CodexCLIAgent from hud.agents.codex.agent import codex_command, run_codex from hud.agents.types import AgentStep, CodexCLIConfig, ToolStep from hud.capabilities import Capability, SSHClient +from hud.eval.runtime import RuntimeConfig, RuntimeResources from hud.settings import settings from hud.telemetry.context import set_trace_context @@ -23,6 +25,10 @@ def _clear_api_keys(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(settings, "api_key", None) monkeypatch.setattr(settings, "openai_api_key", None) + monkeypatch.setattr( + "hud.agents.codex.agent.resolve_executable", + AsyncMock(return_value="codex"), + ) class _FakeReader: @@ -177,7 +183,7 @@ def test_windows_command_encodes_environment_and_arguments( assert "Remove-Item -Recurse -Force $codexHome" in script assert "--ignore-user-config" not in script assert "'--sandbox' 'danger-full-access'" in script - assert "& codex 'exec'" in script + assert "& 'codex' 'exec'" in script assert script.endswith(";exit $hudExitCode") @@ -342,7 +348,7 @@ async def open(self, ref: str) -> _FakeSSH: agent = CodexCLIAgent() execute = AsyncMock() monkeypatch.setattr("hud.agents.codex.agent.run_codex", execute) - run = SimpleNamespace(client=Client(), prompt_text="Fix it") + run = SimpleNamespace(client=Client(), prompt_text="Fix it", runtime_config=None) await agent(cast("Any", run)) @@ -352,4 +358,42 @@ async def open(self, ref: str) -> _FakeSSH: ssh=ssh, shell="powershell", prompt="Fix it", + executable="codex", + ) + + +async def test_executable_resolution_prefers_matching_managed_bundle() -> None: + ssh = SimpleNamespace( + capability=Capability.ssh(url="ssh://localhost:22", host_pubkey="key", shell="bash"), + run=AsyncMock( + side_effect=[ + SimpleNamespace(returncode=0, stdout=b"Linux\nx86_64\ngnu\n"), + SimpleNamespace(returncode=0, stdout=b""), + ] + ), + ) + + executable = await resolve_executable( + cast("Any", ssh), + "codex", + {"linux-x64": "/media/hud/bin/codex/bin/codex"}, + RuntimeConfig(resources=RuntimeResources(os="linux")), ) + + assert executable == "/media/hud/bin/codex/bin/codex" + assert ssh.run.await_count == 2 + + +async def test_executable_resolution_rejects_runtime_os_mismatch() -> None: + ssh = SimpleNamespace( + capability=Capability.ssh(url="ssh://localhost:22", host_pubkey="key", shell="bash"), + run=AsyncMock(return_value=SimpleNamespace(returncode=0, stdout=b"Linux\nx86_64\ngnu\n")), + ) + + with pytest.raises(RuntimeError, match=r"requested 'windows'.*reports 'linux'"): + await resolve_executable( + cast("Any", ssh), + "codex", + {}, + RuntimeConfig(resources=RuntimeResources(os="windows")), + ) diff --git a/hud/capabilities/ssh.py b/hud/capabilities/ssh.py index 93b6f062f..fe95b7565 100644 --- a/hud/capabilities/ssh.py +++ b/hud/capabilities/ssh.py @@ -5,12 +5,19 @@ import asyncio import base64 import contextlib +import ntpath +import os +import secrets import shlex -from typing import Any, ClassVar, Self +from pathlib import Path +from typing import TYPE_CHECKING, Any, ClassVar, Self, cast from urllib.parse import urlsplit import asyncssh +if TYPE_CHECKING: + from typing import BinaryIO + from .base import Capability, CapabilityClient SSH_RECONNECT_ATTEMPTS = 3 @@ -220,6 +227,85 @@ def remaining_timeout() -> float | None: run_kwargs["stdin"] = asyncssh.DEVNULL await self.run(f"cat > {shlex.quote(path)}", check=True, timeout=timeout_s, **run_kwargs) + async def upload( + self, + source: str | os.PathLike[str], + destination: str, + *, + executable: bool = False, + timeout_s: float | None = None, + ) -> None: + """Stream a local file into the SSH namespace and install it atomically.""" + source_path = os.fspath(source) + temporary = f"{destination}.hud-upload-{secrets.token_hex(8)}" + if self._is_windows: + parent = ntpath.dirname(destination) or "." + command = _powershell( + f"New-Item -ItemType Directory -Force -Path {_powershell_quote(parent)} " + "| Out-Null;" + f"$out=[IO.File]::Open({_powershell_quote(temporary)}," + "[IO.FileMode]::CreateNew,[IO.FileAccess]::Write,[IO.FileShare]::None);" + "try{[Console]::OpenStandardInput().CopyTo($out)}finally{$out.Dispose()}" + ) + else: + parent = os.path.dirname(destination) or "." + command = f"mkdir -p -- {shlex.quote(parent)} && cat > {shlex.quote(temporary)}" + + source_file = cast( + "BinaryIO", + await asyncio.to_thread(Path(source_path).open, "rb"), + ) + try: + process = await self.create_process(command) + try: + async with asyncio.timeout(timeout_s): + while chunk := await asyncio.to_thread(source_file.read, 1024 * 1024): + process.stdin.write(chunk) + await process.stdin.drain() + process.stdin.write_eof() + completed = await process.wait(check=True, timeout=None) + if completed.returncode is None: + raise SSHConnectionError("SSH upload ended without an exit status") + + if self._is_windows: + install = ( + f"Move-Item -Force -LiteralPath {_powershell_quote(temporary)} " + f"-Destination {_powershell_quote(destination)}" + ) + install_result = await self.run( + _powershell(install), + check=False, + encoding=None, + timeout=timeout_s, + ) + else: + mode = "0555" if executable else "0444" + install_result = await self.run( + f"chmod {mode} {shlex.quote(temporary)} && " + f"mv -f -- {shlex.quote(temporary)} {shlex.quote(destination)}", + check=False, + encoding=None, + timeout=timeout_s, + ) + if install_result.returncode != 0: + message = _decode(install_result.stderr).strip() + raise RuntimeError(f"SSH upload install failed: {message}") + except BaseException: + process.close() + cleanup = ( + _powershell( + "Remove-Item -Force -ErrorAction SilentlyContinue " + f"-LiteralPath {_powershell_quote(temporary)}" + ) + if self._is_windows + else f"rm -f -- {shlex.quote(temporary)}" + ) + with contextlib.suppress(OSError, TimeoutError, asyncssh.Error): + await self.run(cleanup, check=False, timeout=SSH_SESSION_CLOSE_TIMEOUT_S) + raise + finally: + await asyncio.to_thread(source_file.close) + async def listdir(self, path: str, *, timeout_s: float | None = None) -> list[str]: """List direct children through the exec channel.""" if self._is_windows: diff --git a/hud/environment/tests/test_workspace.py b/hud/environment/tests/test_workspace.py index 5b857ab59..fa860ce3a 100644 --- a/hud/environment/tests/test_workspace.py +++ b/hud/environment/tests/test_workspace.py @@ -199,6 +199,25 @@ async def test_file_operations_use_the_exec_channel(tmp_path: Path) -> None: await ws.stop() +@pytest.mark.asyncio +async def test_upload_streams_binary_into_the_session_namespace(tmp_path: Path) -> None: + source = tmp_path / "source.bin" + source.write_bytes(bytes(range(256)) * 8192) + root = tmp_path / "root" + ws = Workspace(root) + await ws.start() + try: + async with await _connect(ws) as conn: + client = SSHClient(ws.capability(), conn) + await client.upload(source, "managed/tool", executable=True) + finally: + await ws.stop() + + uploaded = root / "managed/tool" + assert uploaded.read_bytes() == source.read_bytes() + assert uploaded.stat().st_mode & 0o777 == 0o555 + + @pytest.mark.asyncio async def test_output_arrives_while_the_command_is_still_running(tmp_path: Path) -> None: """Held until exit, a long build tells the agent nothing while it runs and diff --git a/hud/eval/run.py b/hud/eval/run.py index d3ac5c741..839d106e9 100644 --- a/hud/eval/run.py +++ b/hud/eval/run.py @@ -199,11 +199,13 @@ def __init__( args: dict[str, Any], *, best_effort_grade: bool = False, + runtime_config: RuntimeConfig | None = None, ) -> None: self._client = client self._task_id = task_id self._args = args self._best_effort_grade = best_effort_grade + self.runtime_config = runtime_config #: The task's opening prompt as ``tasks.start`` returned it: plain #: text, or a list of message dicts (``{"role", "content"}``) for #: chat-style / multi-turn prompts. Agents consume the normalized @@ -476,11 +478,12 @@ async def rollout( """ from .runtime.core import resolve_runtime_config + actor_runtime_config = resolve_runtime_config(runtime, task) agent_timeout = validate_rollout_timeouts( task, agent, rollout_timeout, - actor_runtime_config=resolve_runtime_config(runtime, task), + actor_runtime_config=actor_runtime_config, verifier_runtime_config=( resolve_runtime_config(runtime, task.verifier) if task.verifier is not None else None ), @@ -539,6 +542,7 @@ async def close_actor() -> None: task.id, task.args, best_effort_grade=task.verifier is not None, + runtime_config=addr.config or actor_runtime_config, ) live._runtime = addr.url # the placement record for the receipt async with live: # start on enter; complete on exit diff --git a/hud/integrations/harbor/env.py b/hud/integrations/harbor/env.py index df8137e83..40f2fb973 100644 --- a/hud/integrations/harbor/env.py +++ b/hud/integrations/harbor/env.py @@ -260,6 +260,7 @@ def home(user_id: int | None, *, root: Path | None = None) -> str | None: ) agent_mounts = ( *harness_mounts, + Mount("ro", src=str(ROOT / "bin"), dst=str(ROOT / "bin")), Mount("tmpfs", dst=str(TESTS)), Mount("tmpfs", dst=str(VERIFIER_LOGS)), Mount("ro", src="/dev/null", dst=str(AGENT_ANSWER)), diff --git a/hud/integrations/harbor/tests/test_contract.py b/hud/integrations/harbor/tests/test_contract.py index 58124a04f..531bf4c55 100644 --- a/hud/integrations/harbor/tests/test_contract.py +++ b/hud/integrations/harbor/tests/test_contract.py @@ -130,6 +130,7 @@ def test_adapt_packages_an_image_task_as_a_compose_project(tmp_path: Path) -> No served = (context / "env.py").read_text(encoding="utf-8") assert f'Environment("{context.name}")' in served assert 'Environment(CONFIG["name"])' not in served + assert 'Mount("ro", src=str(ROOT / "bin"), dst=str(ROOT / "bin"))' in served project_root = context / "compose-project" assert _tree_snapshot(project_root / "environment") == authored_environment payload = project_root / "hud" From 28236fb9abcb50ed6bba52242bfedce052bf40c7 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 21 Aug 2026 20:23:43 -0700 Subject: [PATCH 08/14] Fix Claude CLI prompt transport --- hud/agents/claude/cli/agent.py | 8 ++++++-- hud/agents/tests/test_claude_cli_agent.py | 22 +++++++++++++++++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/hud/agents/claude/cli/agent.py b/hud/agents/claude/cli/agent.py index 3167b5c79..bfc62d007 100644 --- a/hud/agents/claude/cli/agent.py +++ b/hud/agents/claude/cli/agent.py @@ -231,7 +231,6 @@ def claude_command( ) return powershell(script) - args.extend(["--", prompt]) command = " ".join(shlex.quote(arg) for arg in args) env_prefix = " ".join(f"{key}={shlex.quote(value)}" for key, value in env.items()) return f'export PATH="$HOME/.local/bin:$PATH"; {env_prefix} {command}' @@ -264,7 +263,12 @@ async def run_claude( await ssh.write_text(path, content) logger.info("SSH exec claude CLI (%d chars)", len(command)) events = ClaudeEvents(run, started_at=now_iso()) - returncode, stderr = await run_jsonl(ssh, command, events.consume) + returncode, stderr = await run_jsonl( + ssh, + command, + events.consume, + input_text=None if shell in WINDOWS_SHELLS else prompt, + ) logger.info("exit=%s stderr=%d", returncode, len(stderr)) events.finish(returncode=returncode, stderr=stderr) finally: diff --git a/hud/agents/tests/test_claude_cli_agent.py b/hud/agents/tests/test_claude_cli_agent.py index 8248341db..6546b8505 100644 --- a/hud/agents/tests/test_claude_cli_agent.py +++ b/hud/agents/tests/test_claude_cli_agent.py @@ -114,6 +114,7 @@ def __init__( returncode: int | None = None, pause_after: int | None = None, ) -> None: + self.stdin = _FakeWriter() self.stdout = _FakeReader(stdout, pause_after=pause_after) self.stderr = _FakeReader(stderr) self.exit_status = exit_status @@ -131,6 +132,21 @@ async def wait_closed(self) -> None: pass +class _FakeWriter: + def __init__(self) -> None: + self.data = bytearray() + self.eof = False + + def write(self, data: bytes) -> None: + self.data.extend(data) + + async def drain(self) -> None: + pass + + def write_eof(self) -> None: + self.eof = True + + class _FakeCompletedProcess: async def wait(self, *, check: bool, **kwargs: Any) -> Any: del check @@ -248,7 +264,8 @@ async def test_exec_on_windows_writes_batch_and_execs_via_cmd() -> None: async def test_exec_on_bash_runs_inline_without_batch() -> None: sink: dict[str, bytes] = {} - conn = _FakeConn(sink, _FakeStreamProcess(_STREAM_JSON)) + process = _FakeStreamProcess(_STREAM_JSON) + conn = _FakeConn(sink, process) ssh = _ssh_with_conn("bash", conn) run = _fake_run() @@ -261,6 +278,9 @@ async def test_exec_on_bash_runs_inline_without_batch() -> None: assert conn.deleted == [] assert len(conn.ran) == 1 assert "claude" in conn.ran[0] + assert "build it" not in conn.ran[0] + assert process.stdin.data == b"build it" + assert process.stdin.eof is True assert run.trace.status is None assert run.trace.content == "done" assert "messages" not in run.trace.extra From 4749de8c7e95d3e14428741774dd5b0c82a4bcc4 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 21 Aug 2026 21:16:59 -0700 Subject: [PATCH 09/14] Terminate Claude CLI text prompts --- hud/agents/claude/cli/agent.py | 2 +- hud/agents/tests/test_claude_cli_agent.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hud/agents/claude/cli/agent.py b/hud/agents/claude/cli/agent.py index bfc62d007..d52889c8d 100644 --- a/hud/agents/claude/cli/agent.py +++ b/hud/agents/claude/cli/agent.py @@ -267,7 +267,7 @@ async def run_claude( ssh, command, events.consume, - input_text=None if shell in WINDOWS_SHELLS else prompt, + input_text=None if shell in WINDOWS_SHELLS else f"{prompt}\n", ) logger.info("exit=%s stderr=%d", returncode, len(stderr)) events.finish(returncode=returncode, stderr=stderr) diff --git a/hud/agents/tests/test_claude_cli_agent.py b/hud/agents/tests/test_claude_cli_agent.py index 6546b8505..a1080faad 100644 --- a/hud/agents/tests/test_claude_cli_agent.py +++ b/hud/agents/tests/test_claude_cli_agent.py @@ -279,7 +279,7 @@ async def test_exec_on_bash_runs_inline_without_batch() -> None: assert len(conn.ran) == 1 assert "claude" in conn.ran[0] assert "build it" not in conn.ran[0] - assert process.stdin.data == b"build it" + assert process.stdin.data == b"build it\n" assert process.stdin.eof is True assert run.trace.status is None assert run.trace.content == "done" From 48f454ad37ca2e08a0ffc22fc988639c789a2ed4 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 21 Aug 2026 21:27:06 -0700 Subject: [PATCH 10/14] Use Claude stream JSON input --- hud/agents/claude/cli/agent.py | 31 ++++++++++++++++----- hud/agents/tests/test_claude_cli_agent.py | 33 ++++++++++++++++------- 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/hud/agents/claude/cli/agent.py b/hud/agents/claude/cli/agent.py index d52889c8d..7684754f9 100644 --- a/hud/agents/claude/cli/agent.py +++ b/hud/agents/claude/cli/agent.py @@ -35,7 +35,7 @@ logger = logging.getLogger(__name__) -PROMPT_PATH = ".hud_prompt.txt" +INPUT_PATH = ".hud_input.jsonl" MCP_CONFIG_PATH = ".hud_mcp_config.json" RUN_SCRIPT_PATH = ".hud_run.bat" @@ -166,7 +166,6 @@ def finish(self, *, returncode: int, stderr: str) -> None: def claude_command( config: ClaudeCLIConfig, shell: str, - prompt: str, mcp_config_path: str | None = None, executable: str = "claude", ) -> str: @@ -206,6 +205,7 @@ def claude_command( args: list[str] = [ executable, "--verbose", + "--input-format=stream-json", "--output-format=stream-json", "--print", f"--permission-mode={config.permission_mode}", @@ -223,7 +223,7 @@ def claude_command( script = ";".join( [ *(f"$env:{key}={powershell_quote(value)}" for key, value in env.items()), - f"Get-Content -Raw -Encoding UTF8 {powershell_quote(PROMPT_PATH)}" + f"Get-Content -Raw -Encoding UTF8 {powershell_quote(INPUT_PATH)}" f" | & {powershell_quote(executable)} " f"{' '.join(powershell_quote(arg) for arg in args[1:])}", "exit $LASTEXITCODE", @@ -247,13 +247,30 @@ async def run_claude( executable: str = "claude", ) -> None: files: dict[str, str] = {} + input_text = ( + json.dumps( + { + "type": "user", + "message": { + "role": "user", + "content": [{"type": "text", "text": prompt}], + }, + } + ) + + "\n" + ) mcp_config_path = MCP_CONFIG_PATH if mcp_servers else None if mcp_servers: files[MCP_CONFIG_PATH] = json.dumps({"mcpServers": mcp_servers}, indent=2) if shell in WINDOWS_SHELLS: - files[PROMPT_PATH] = prompt - - command = claude_command(config, shell, prompt, mcp_config_path, executable) + files[INPUT_PATH] = input_text + + command = claude_command( + config, + shell, + mcp_config_path=mcp_config_path, + executable=executable, + ) if shell in WINDOWS_SHELLS: files[RUN_SCRIPT_PATH] = f"@echo off\r\n{command}\r\n" command = f"cmd /c {RUN_SCRIPT_PATH}" @@ -267,7 +284,7 @@ async def run_claude( ssh, command, events.consume, - input_text=None if shell in WINDOWS_SHELLS else f"{prompt}\n", + input_text=None if shell in WINDOWS_SHELLS else input_text, ) logger.info("exit=%s stderr=%d", returncode, len(stderr)) events.finish(returncode=returncode, stderr=stderr) diff --git a/hud/agents/tests/test_claude_cli_agent.py b/hud/agents/tests/test_claude_cli_agent.py index a1080faad..daf158e34 100644 --- a/hud/agents/tests/test_claude_cli_agent.py +++ b/hud/agents/tests/test_claude_cli_agent.py @@ -50,8 +50,8 @@ def test_command_follows_explicit_gateway_routing(monkeypatch: pytest.MonkeyPatc monkeypatch.setattr(settings, "api_key", "hud-key") monkeypatch.setattr(settings, "anthropic_api_key", "anthropic-key") - gateway = claude_command(ClaudeCLIConfig(use_hud_gateway=True), "bash", "run") - provider = claude_command(ClaudeCLIConfig(use_hud_gateway=False), "bash", "run") + gateway = claude_command(ClaudeCLIConfig(use_hud_gateway=True), "bash") + provider = claude_command(ClaudeCLIConfig(use_hud_gateway=False), "bash") assert f"ANTHROPIC_BASE_URL={settings.hud_gateway_url}" in gateway assert "ANTHROPIC_API_KEY=hud-key" in gateway @@ -69,14 +69,14 @@ def test_windows_command_encodes_environment_and_arguments( max_steps=3, system_prompt="don't $expand", ) - command = claude_command(config, "powershell", "not embedded") + command = claude_command(config, "powershell") encoded = command.rsplit(" ", 1)[1] script = base64.b64decode(encoded).decode("utf-16-le") assert "$env:ANTHROPIC_API_KEY='hud&key''s'" in script assert "'--system-prompt' 'don''t $expand'" in script - assert "Get-Content -Raw -Encoding UTF8 '.hud_prompt.txt' | & 'claude'" in script - assert "not embedded" not in script + assert "Get-Content -Raw -Encoding UTF8 '.hud_input.jsonl' | & 'claude'" in script + assert "'--input-format=stream-json'" in script assert "python" not in script @@ -190,7 +190,7 @@ async def create_process(self, cmd: str, **kwargs: Any) -> Any: script = base64.b64decode(match.group(1)).decode("utf-16-le") name = next( path - for path in (".hud_prompt.txt", ".hud_run.bat", ".hud_mcp_config.json") + for path in (".hud_input.jsonl", ".hud_run.bat", ".hud_mcp_config.json") if path in script ) if input_value is not None: @@ -254,9 +254,16 @@ async def test_exec_on_windows_writes_batch_and_execs_via_cmd() -> None: assert conn.ran == ["cmd /c .hud_run.bat"] assert all(command.startswith("powershell ") for command in conn.write_commands) assert conn.written[".hud_run.bat"].startswith(b"@echo off\r\n") - assert conn.written[".hud_prompt.txt"] == b"build it" + assert conn.written[".hud_input.jsonl"].endswith(b"\n") + assert json.loads(conn.written[".hud_input.jsonl"]) == { + "type": "user", + "message": { + "role": "user", + "content": [{"type": "text", "text": "build it"}], + }, + } assert sink == {} - assert set(conn.deleted) == {".hud_prompt.txt", ".hud_run.bat"} + assert set(conn.deleted) == {".hud_input.jsonl", ".hud_run.bat"} assert run.trace.status is None assert run.trace.content == "done" assert "messages" not in run.trace.extra @@ -278,8 +285,16 @@ async def test_exec_on_bash_runs_inline_without_batch() -> None: assert conn.deleted == [] assert len(conn.ran) == 1 assert "claude" in conn.ran[0] + assert "--input-format=stream-json" in conn.ran[0] assert "build it" not in conn.ran[0] - assert process.stdin.data == b"build it\n" + assert process.stdin.data.endswith(b"\n") + assert json.loads(process.stdin.data) == { + "type": "user", + "message": { + "role": "user", + "content": [{"type": "text", "text": "build it"}], + }, + } assert process.stdin.eof is True assert run.trace.status is None assert run.trace.content == "done" From f6a6e5fa38d60fe2a8ed7e7bac73d1117d706a15 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 21 Aug 2026 22:20:51 -0700 Subject: [PATCH 11/14] Disable Claude compaction through HUD gateway --- hud/agents/claude/cli/agent.py | 2 ++ hud/agents/tests/test_claude_cli_agent.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/hud/agents/claude/cli/agent.py b/hud/agents/claude/cli/agent.py index 7684754f9..7046dfdd4 100644 --- a/hud/agents/claude/cli/agent.py +++ b/hud/agents/claude/cli/agent.py @@ -179,6 +179,8 @@ def claude_command( raise ValueError("HUD_API_KEY is required for HUD gateway routing") env["ANTHROPIC_BASE_URL"] = settings.hud_gateway_url env["ANTHROPIC_API_KEY"] = settings.api_key + env["CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS"] = "1" + env["DISABLE_AUTO_COMPACT"] = "1" if trace_id := get_current_trace_id(): env["ANTHROPIC_CUSTOM_HEADERS"] = f"Trace-Id: {trace_id}" elif settings.anthropic_api_key: diff --git a/hud/agents/tests/test_claude_cli_agent.py b/hud/agents/tests/test_claude_cli_agent.py index daf158e34..007e1a35c 100644 --- a/hud/agents/tests/test_claude_cli_agent.py +++ b/hud/agents/tests/test_claude_cli_agent.py @@ -55,8 +55,12 @@ def test_command_follows_explicit_gateway_routing(monkeypatch: pytest.MonkeyPatc assert f"ANTHROPIC_BASE_URL={settings.hud_gateway_url}" in gateway assert "ANTHROPIC_API_KEY=hud-key" in gateway + assert "CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS=1" in gateway + assert "DISABLE_AUTO_COMPACT=1" in gateway assert "ANTHROPIC_API_KEY=anthropic-key" in provider assert "ANTHROPIC_BASE_URL" not in provider + assert "CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS" not in provider + assert "DISABLE_AUTO_COMPACT" not in provider assert "ANTHROPIC_MODEL=claude-sonnet-5" in provider @@ -74,6 +78,8 @@ def test_windows_command_encodes_environment_and_arguments( encoded = command.rsplit(" ", 1)[1] script = base64.b64decode(encoded).decode("utf-16-le") assert "$env:ANTHROPIC_API_KEY='hud&key''s'" in script + assert "$env:CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS='1'" in script + assert "$env:DISABLE_AUTO_COMPACT='1'" in script assert "'--system-prompt' 'don''t $expand'" in script assert "Get-Content -Raw -Encoding UTF8 '.hud_input.jsonl' | & 'claude'" in script assert "'--input-format=stream-json'" in script From a79877a028c7960dc5728ac95c865b24b3cc243c Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 27 Aug 2026 22:54:31 -0700 Subject: [PATCH 12/14] Remove unused SSH upload support --- hud/capabilities/ssh.py | 88 +------------------------ hud/environment/tests/test_workspace.py | 19 ------ 2 files changed, 1 insertion(+), 106 deletions(-) diff --git a/hud/capabilities/ssh.py b/hud/capabilities/ssh.py index fe95b7565..93b6f062f 100644 --- a/hud/capabilities/ssh.py +++ b/hud/capabilities/ssh.py @@ -5,19 +5,12 @@ import asyncio import base64 import contextlib -import ntpath -import os -import secrets import shlex -from pathlib import Path -from typing import TYPE_CHECKING, Any, ClassVar, Self, cast +from typing import Any, ClassVar, Self from urllib.parse import urlsplit import asyncssh -if TYPE_CHECKING: - from typing import BinaryIO - from .base import Capability, CapabilityClient SSH_RECONNECT_ATTEMPTS = 3 @@ -227,85 +220,6 @@ def remaining_timeout() -> float | None: run_kwargs["stdin"] = asyncssh.DEVNULL await self.run(f"cat > {shlex.quote(path)}", check=True, timeout=timeout_s, **run_kwargs) - async def upload( - self, - source: str | os.PathLike[str], - destination: str, - *, - executable: bool = False, - timeout_s: float | None = None, - ) -> None: - """Stream a local file into the SSH namespace and install it atomically.""" - source_path = os.fspath(source) - temporary = f"{destination}.hud-upload-{secrets.token_hex(8)}" - if self._is_windows: - parent = ntpath.dirname(destination) or "." - command = _powershell( - f"New-Item -ItemType Directory -Force -Path {_powershell_quote(parent)} " - "| Out-Null;" - f"$out=[IO.File]::Open({_powershell_quote(temporary)}," - "[IO.FileMode]::CreateNew,[IO.FileAccess]::Write,[IO.FileShare]::None);" - "try{[Console]::OpenStandardInput().CopyTo($out)}finally{$out.Dispose()}" - ) - else: - parent = os.path.dirname(destination) or "." - command = f"mkdir -p -- {shlex.quote(parent)} && cat > {shlex.quote(temporary)}" - - source_file = cast( - "BinaryIO", - await asyncio.to_thread(Path(source_path).open, "rb"), - ) - try: - process = await self.create_process(command) - try: - async with asyncio.timeout(timeout_s): - while chunk := await asyncio.to_thread(source_file.read, 1024 * 1024): - process.stdin.write(chunk) - await process.stdin.drain() - process.stdin.write_eof() - completed = await process.wait(check=True, timeout=None) - if completed.returncode is None: - raise SSHConnectionError("SSH upload ended without an exit status") - - if self._is_windows: - install = ( - f"Move-Item -Force -LiteralPath {_powershell_quote(temporary)} " - f"-Destination {_powershell_quote(destination)}" - ) - install_result = await self.run( - _powershell(install), - check=False, - encoding=None, - timeout=timeout_s, - ) - else: - mode = "0555" if executable else "0444" - install_result = await self.run( - f"chmod {mode} {shlex.quote(temporary)} && " - f"mv -f -- {shlex.quote(temporary)} {shlex.quote(destination)}", - check=False, - encoding=None, - timeout=timeout_s, - ) - if install_result.returncode != 0: - message = _decode(install_result.stderr).strip() - raise RuntimeError(f"SSH upload install failed: {message}") - except BaseException: - process.close() - cleanup = ( - _powershell( - "Remove-Item -Force -ErrorAction SilentlyContinue " - f"-LiteralPath {_powershell_quote(temporary)}" - ) - if self._is_windows - else f"rm -f -- {shlex.quote(temporary)}" - ) - with contextlib.suppress(OSError, TimeoutError, asyncssh.Error): - await self.run(cleanup, check=False, timeout=SSH_SESSION_CLOSE_TIMEOUT_S) - raise - finally: - await asyncio.to_thread(source_file.close) - async def listdir(self, path: str, *, timeout_s: float | None = None) -> list[str]: """List direct children through the exec channel.""" if self._is_windows: diff --git a/hud/environment/tests/test_workspace.py b/hud/environment/tests/test_workspace.py index fa860ce3a..5b857ab59 100644 --- a/hud/environment/tests/test_workspace.py +++ b/hud/environment/tests/test_workspace.py @@ -199,25 +199,6 @@ async def test_file_operations_use_the_exec_channel(tmp_path: Path) -> None: await ws.stop() -@pytest.mark.asyncio -async def test_upload_streams_binary_into_the_session_namespace(tmp_path: Path) -> None: - source = tmp_path / "source.bin" - source.write_bytes(bytes(range(256)) * 8192) - root = tmp_path / "root" - ws = Workspace(root) - await ws.start() - try: - async with await _connect(ws) as conn: - client = SSHClient(ws.capability(), conn) - await client.upload(source, "managed/tool", executable=True) - finally: - await ws.stop() - - uploaded = root / "managed/tool" - assert uploaded.read_bytes() == source.read_bytes() - assert uploaded.stat().st_mode & 0o777 == 0o555 - - @pytest.mark.asyncio async def test_output_arrives_while_the_command_is_still_running(tmp_path: Path) -> None: """Held until exit, a long build tells the agent nothing while it runs and From d9940ae16ebc59564890e50df64cfebd5fea7984 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 27 Aug 2026 23:11:37 -0700 Subject: [PATCH 13/14] Simplify CLI agent implementation --- hud/agents/claude/cli/agent.py | 301 ++++++++++++---------- hud/agents/tests/cli_fakes.py | 78 ++++++ hud/agents/tests/test_claude_cli_agent.py | 110 +++----- hud/agents/tests/test_codex_cli_agent.py | 69 +---- 4 files changed, 269 insertions(+), 289 deletions(-) create mode 100644 hud/agents/tests/cli_fakes.py diff --git a/hud/agents/claude/cli/agent.py b/hud/agents/claude/cli/agent.py index 7046dfdd4..4b5b398ff 100644 --- a/hud/agents/claude/cli/agent.py +++ b/hud/agents/claude/cli/agent.py @@ -163,147 +163,13 @@ def finish(self, *, returncode: int, stderr: str) -> None: raise RuntimeError(error) -def claude_command( - config: ClaudeCLIConfig, - shell: str, - mcp_config_path: str | None = None, - executable: str = "claude", -) -> str: - env: dict[str, str] = {} - use_hud_gateway = config.use_hud_gateway - if use_hud_gateway is None: - use_hud_gateway = settings.api_key is not None - - if use_hud_gateway: - if not settings.api_key: - raise ValueError("HUD_API_KEY is required for HUD gateway routing") - env["ANTHROPIC_BASE_URL"] = settings.hud_gateway_url - env["ANTHROPIC_API_KEY"] = settings.api_key - env["CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS"] = "1" - env["DISABLE_AUTO_COMPACT"] = "1" - if trace_id := get_current_trace_id(): - env["ANTHROPIC_CUSTOM_HEADERS"] = f"Trace-Id: {trace_id}" - elif settings.anthropic_api_key: - env["ANTHROPIC_API_KEY"] = settings.anthropic_api_key - - env["ANTHROPIC_MODEL"] = config.model - env["ANTHROPIC_SMALL_FAST_MODEL"] = config.model - - # A custom base URL must own every model tier; otherwise background calls - # can escape to Anthropic instead of using the configured gateway. - if "ANTHROPIC_BASE_URL" in env: - for name in ( - "ANTHROPIC_DEFAULT_SONNET_MODEL", - "ANTHROPIC_DEFAULT_OPUS_MODEL", - "ANTHROPIC_DEFAULT_HAIKU_MODEL", - "CLAUDE_CODE_SUBAGENT_MODEL", - ): - env[name] = config.model - - env["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC"] = "1" - env["DISABLE_AUTOUPDATER"] = "1" - env["IS_SANDBOX"] = "1" - - args: list[str] = [ - executable, - "--verbose", - "--input-format=stream-json", - "--output-format=stream-json", - "--print", - f"--permission-mode={config.permission_mode}", - ] - if config.max_steps > 0: - args.append(f"--max-turns={config.max_steps}") - if config.system_prompt: - args.extend(["--system-prompt", config.system_prompt]) - for tool in config.allowed_tools: - args.extend(["--allowedTools", tool]) - if mcp_config_path: - args.extend(["--mcp-config", mcp_config_path]) - - if shell in WINDOWS_SHELLS: - script = ";".join( - [ - *(f"$env:{key}={powershell_quote(value)}" for key, value in env.items()), - f"Get-Content -Raw -Encoding UTF8 {powershell_quote(INPUT_PATH)}" - f" | & {powershell_quote(executable)} " - f"{' '.join(powershell_quote(arg) for arg in args[1:])}", - "exit $LASTEXITCODE", - ] - ) - return powershell(script) - - command = " ".join(shlex.quote(arg) for arg in args) - env_prefix = " ".join(f"{key}={shlex.quote(value)}" for key, value in env.items()) - return f'export PATH="$HOME/.local/bin:$PATH"; {env_prefix} {command}' - - -async def run_claude( - config: ClaudeCLIConfig, - run: Run, - *, - ssh: SSHClient, - shell: str, - mcp_servers: dict[str, dict[str, Any]], - prompt: str, - executable: str = "claude", -) -> None: - files: dict[str, str] = {} - input_text = ( - json.dumps( - { - "type": "user", - "message": { - "role": "user", - "content": [{"type": "text", "text": prompt}], - }, - } - ) - + "\n" - ) - mcp_config_path = MCP_CONFIG_PATH if mcp_servers else None - if mcp_servers: - files[MCP_CONFIG_PATH] = json.dumps({"mcpServers": mcp_servers}, indent=2) - if shell in WINDOWS_SHELLS: - files[INPUT_PATH] = input_text - - command = claude_command( - config, - shell, - mcp_config_path=mcp_config_path, - executable=executable, - ) - if shell in WINDOWS_SHELLS: - files[RUN_SCRIPT_PATH] = f"@echo off\r\n{command}\r\n" - command = f"cmd /c {RUN_SCRIPT_PATH}" - - try: - for path, content in files.items(): - await ssh.write_text(path, content) - logger.info("SSH exec claude CLI (%d chars)", len(command)) - events = ClaudeEvents(run, started_at=now_iso()) - returncode, stderr = await run_jsonl( - ssh, - command, - events.consume, - input_text=None if shell in WINDOWS_SHELLS else input_text, - ) - logger.info("exit=%s stderr=%d", returncode, len(stderr)) - events.finish(returncode=returncode, stderr=stderr) - finally: - if files: - if shell in WINDOWS_SHELLS: - cleanup = f"cmd /c del /f /q {' '.join(files)} 2>nul" - else: - cleanup = "rm -f -- " + " ".join(shlex.quote(path) for path in files) - try: - await ssh.run(cleanup, check=False) - except (OSError, asyncssh.Error): - logger.warning("Failed to remove Claude CLI runtime files") - - class ClaudeCLIAgent(Agent): - """Runs ``claude`` CLI over SSH inside the environment workspace.""" + """Runs ``claude`` CLI over SSH inside the env workspace. + + Stateless w.r.t. the env: driven by ``await agent(run)``. SSH is opened + live off the run. Environment MCP bindings are used directly; computer MCP + servers are bridged over the run's SSH connection. + """ config: ClaudeCLIConfig @@ -353,8 +219,7 @@ async def __call__(self, run: Run) -> None: ) ) - await run_claude( - self.config, + await self._exec( run, ssh=ssh, shell=shell, @@ -363,5 +228,157 @@ async def __call__(self, run: Run) -> None: executable=executable, ) + async def _exec( + self, + run: Run, + *, + ssh: SSHClient, + shell: str, + mcp_servers: dict[str, dict[str, Any]], + prompt: str, + executable: str = "claude", + ) -> None: + mcp_config_path = await self._write_mcp_config(ssh, mcp_servers) + input_text = ( + json.dumps( + { + "type": "user", + "message": { + "role": "user", + "content": [{"type": "text", "text": prompt}], + }, + } + ) + + "\n" + ) + files = [mcp_config_path] if mcp_config_path else [] + if shell in WINDOWS_SHELLS: + await ssh.write_text(INPUT_PATH, input_text) + files.append(INPUT_PATH) + + command = self._build_cli_command( + shell=shell, + mcp_config_path=mcp_config_path, + executable=executable, + ) + if shell in WINDOWS_SHELLS: + await ssh.write_text(RUN_SCRIPT_PATH, f"@echo off\r\n{command}\r\n") + files.append(RUN_SCRIPT_PATH) + command = f"cmd /c {RUN_SCRIPT_PATH}" + + try: + logger.info("SSH exec claude CLI (%d chars)", len(command)) + events = ClaudeEvents(run, started_at=now_iso()) + returncode, stderr = await run_jsonl( + ssh, + command, + events.consume, + input_text=None if shell in WINDOWS_SHELLS else input_text, + ) + logger.info("exit=%s stderr=%d", returncode, len(stderr)) + events.finish(returncode=returncode, stderr=stderr) + finally: + if files: + if shell in WINDOWS_SHELLS: + cleanup = f"cmd /c del /f /q {' '.join(files)} 2>nul" + else: + cleanup = "rm -f -- " + " ".join(shlex.quote(path) for path in files) + try: + await ssh.run(cleanup, check=False) + except (OSError, asyncssh.Error): + logger.warning("Failed to remove Claude CLI runtime files") + + def _build_env_vars(self) -> dict[str, str]: + env: dict[str, str] = {} + use_hud_gateway = self.config.use_hud_gateway + if use_hud_gateway is None: + use_hud_gateway = settings.api_key is not None + + if use_hud_gateway: + if not settings.api_key: + raise ValueError("HUD_API_KEY is required for HUD gateway routing") + env["ANTHROPIC_BASE_URL"] = settings.hud_gateway_url + env["ANTHROPIC_API_KEY"] = settings.api_key + env["CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS"] = "1" + env["DISABLE_AUTO_COMPACT"] = "1" + if trace_id := get_current_trace_id(): + env["ANTHROPIC_CUSTOM_HEADERS"] = f"Trace-Id: {trace_id}" + elif settings.anthropic_api_key: + env["ANTHROPIC_API_KEY"] = settings.anthropic_api_key + + env["ANTHROPIC_MODEL"] = self.config.model + env["ANTHROPIC_SMALL_FAST_MODEL"] = self.config.model + + # A custom base URL must own every model tier; otherwise background calls + # can escape to Anthropic instead of using the configured gateway. + if "ANTHROPIC_BASE_URL" in env: + for name in ( + "ANTHROPIC_DEFAULT_SONNET_MODEL", + "ANTHROPIC_DEFAULT_OPUS_MODEL", + "ANTHROPIC_DEFAULT_HAIKU_MODEL", + "CLAUDE_CODE_SUBAGENT_MODEL", + ): + env[name] = self.config.model + + env["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC"] = "1" + env["DISABLE_AUTOUPDATER"] = "1" + env["IS_SANDBOX"] = "1" + return env + + async def _write_mcp_config( + self, + ssh: SSHClient, + mcp_servers: dict[str, dict[str, Any]], + ) -> str | None: + """Write MCP config into the workspace and return its path.""" + if not mcp_servers: + return None + await ssh.write_text( + MCP_CONFIG_PATH, + json.dumps({"mcpServers": mcp_servers}, indent=2), + ) + return MCP_CONFIG_PATH + + def _build_cli_command( + self, + *, + shell: str, + mcp_config_path: str | None = None, + executable: str = "claude", + ) -> str: + env = self._build_env_vars() + args: list[str] = [ + executable, + "--verbose", + "--input-format=stream-json", + "--output-format=stream-json", + "--print", + f"--permission-mode={self.config.permission_mode}", + ] + if self.config.max_steps > 0: + args.append(f"--max-turns={self.config.max_steps}") + if self.config.system_prompt: + args.extend(["--system-prompt", self.config.system_prompt]) + for tool in self.config.allowed_tools: + args.extend(["--allowedTools", tool]) + if mcp_config_path: + args.extend(["--mcp-config", mcp_config_path]) + + if shell in WINDOWS_SHELLS: + script = ";".join( + [ + *(f"$env:{key}={powershell_quote(value)}" for key, value in env.items()), + f"Get-Content -Raw -Encoding UTF8 {powershell_quote(INPUT_PATH)}" + f" | & {powershell_quote(executable)} " + f"{' '.join(powershell_quote(arg) for arg in args[1:])}", + "exit $LASTEXITCODE", + ] + ) + return powershell(script) + + command = " ".join(shlex.quote(arg) for arg in args) + env_prefix = " ".join(f"{key}={shlex.quote(value)}" for key, value in env.items()) + return f'export PATH="$HOME/.local/bin:$PATH"; {env_prefix} {command}' + __all__ = ["ClaudeCLIAgent"] diff --git a/hud/agents/tests/cli_fakes.py b/hud/agents/tests/cli_fakes.py new file mode 100644 index 000000000..6cd59bab4 --- /dev/null +++ b/hud/agents/tests/cli_fakes.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from typing import Any + + +class FakeReader: + def __init__(self, value: str, *, pause_after: int | None = None) -> None: + self._raw = value.encode() + self._lines = self._raw.splitlines(keepends=True) + self._pause_after = pause_after + self._index = 0 + self.blocked = asyncio.Event() + self.release = asyncio.Event() + + async def readline(self) -> bytes: + if self._pause_after == self._index: + self.blocked.set() + await self.release.wait() + self._pause_after = None + if self._index == len(self._lines): + return b"" + line = self._lines[self._index] + self._index += 1 + return line + + async def read(self) -> bytes: + return self._raw + + +class FakeWriter: + def __init__(self) -> None: + self.data = bytearray() + self.eof = False + + def write(self, data: bytes) -> None: + self.data.extend(data) + + async def drain(self) -> None: + pass + + def write_eof(self) -> None: + self.eof = True + + +class FakeProcess: + def __init__( + self, + stdout: str, + *, + stderr: str = "", + exit_status: int | None = 0, + returncode: int | None = None, + pause_after: int | None = None, + ) -> None: + self.stdin = FakeWriter() + self.stdout = FakeReader(stdout, pause_after=pause_after) + self.stderr = FakeReader(stderr) + self.exit_status = exit_status + self.returncode = exit_status if returncode is None else returncode + self.closed = False + self.terminated = False + + def terminate(self) -> None: + self.terminated = True + + def close(self) -> None: + self.closed = True + + async def wait_closed(self) -> None: + pass + + +def fake_run() -> Any: + trace = SimpleNamespace(status=None, content="", extra={}) + steps: list[Any] = [] + return SimpleNamespace(trace=trace, record=steps.append, steps=steps) diff --git a/hud/agents/tests/test_claude_cli_agent.py b/hud/agents/tests/test_claude_cli_agent.py index 007e1a35c..ff58e78f5 100644 --- a/hud/agents/tests/test_claude_cli_agent.py +++ b/hud/agents/tests/test_claude_cli_agent.py @@ -24,7 +24,9 @@ from mcp.types import ImageContent, TextContent from hud.agents.claude.cli import computer_mcp -from hud.agents.claude.cli.agent import ClaudeCLIAgent, claude_command, run_claude +from hud.agents.claude.cli.agent import ClaudeCLIAgent +from hud.agents.tests.cli_fakes import FakeProcess as _FakeStreamProcess +from hud.agents.tests.cli_fakes import fake_run as _fake_run from hud.agents.types import AgentStep, ClaudeCLIConfig, ToolStep from hud.capabilities import Capability, SSHClient from hud.capabilities.rfb import WebPScreenshotEncoding @@ -50,8 +52,10 @@ def test_command_follows_explicit_gateway_routing(monkeypatch: pytest.MonkeyPatc monkeypatch.setattr(settings, "api_key", "hud-key") monkeypatch.setattr(settings, "anthropic_api_key", "anthropic-key") - gateway = claude_command(ClaudeCLIConfig(use_hud_gateway=True), "bash") - provider = claude_command(ClaudeCLIConfig(use_hud_gateway=False), "bash") + gateway_agent = ClaudeCLIAgent(ClaudeCLIConfig(use_hud_gateway=True)) + provider_agent = ClaudeCLIAgent(ClaudeCLIConfig(use_hud_gateway=False)) + gateway = gateway_agent._build_cli_command(shell="bash") + provider = provider_agent._build_cli_command(shell="bash") assert f"ANTHROPIC_BASE_URL={settings.hud_gateway_url}" in gateway assert "ANTHROPIC_API_KEY=hud-key" in gateway @@ -73,7 +77,8 @@ def test_windows_command_encodes_environment_and_arguments( max_steps=3, system_prompt="don't $expand", ) - command = claude_command(config, "powershell") + agent = ClaudeCLIAgent(config) + command = agent._build_cli_command(shell="powershell") encoded = command.rsplit(" ", 1)[1] script = base64.b64decode(encoded).decode("utf-16-le") @@ -86,73 +91,6 @@ def test_windows_command_encodes_environment_and_arguments( assert "python" not in script -class _FakeReader: - def __init__(self, value: str, *, pause_after: int | None = None) -> None: - self._raw = value.encode() - self._lines = self._raw.splitlines(keepends=True) - self._pause_after = pause_after - self._index = 0 - self.blocked = asyncio.Event() - self.release = asyncio.Event() - - async def readline(self) -> bytes: - if self._pause_after == self._index: - self.blocked.set() - await self.release.wait() - self._pause_after = None - if self._index == len(self._lines): - return b"" - line = self._lines[self._index] - self._index += 1 - return line - - async def read(self) -> bytes: - return self._raw - - -class _FakeStreamProcess: - def __init__( - self, - stdout: str, - *, - stderr: str = "", - exit_status: int | None = 0, - returncode: int | None = None, - pause_after: int | None = None, - ) -> None: - self.stdin = _FakeWriter() - self.stdout = _FakeReader(stdout, pause_after=pause_after) - self.stderr = _FakeReader(stderr) - self.exit_status = exit_status - self.returncode = exit_status if returncode is None else returncode - self.closed = False - self.terminated = False - - def terminate(self) -> None: - self.terminated = True - - def close(self) -> None: - self.closed = True - - async def wait_closed(self) -> None: - pass - - -class _FakeWriter: - def __init__(self) -> None: - self.data = bytearray() - self.eof = False - - def write(self, data: bytes) -> None: - self.data.extend(data) - - async def drain(self) -> None: - pass - - def write_eof(self) -> None: - self.eof = True - - class _FakeCompletedProcess: async def wait(self, *, check: bool, **kwargs: Any) -> Any: del check @@ -212,10 +150,23 @@ async def create_process(self, cmd: str, **kwargs: Any) -> Any: return self._process -def _fake_run() -> Any: - trace = SimpleNamespace(status=None, content="", extra={}) - steps: list[Any] = [] - return SimpleNamespace(trace=trace, record=steps.append, steps=steps) +async def run_claude( + config: ClaudeCLIConfig, + run: Any, + *, + ssh: SSHClient, + shell: str, + mcp_servers: dict[str, dict[str, Any]], + prompt: str, +) -> None: + agent = ClaudeCLIAgent(config) + await agent._exec( + run, + ssh=ssh, + shell=shell, + mcp_servers=mcp_servers, + prompt=prompt, + ) _STREAM_JSON = ( @@ -515,7 +466,7 @@ async def open(self, ref: str) -> SSHClient: agent = ClaudeCLIAgent() execute = AsyncMock() - monkeypatch.setattr("hud.agents.claude.cli.agent.run_claude", execute) + monkeypatch.setattr(agent, "_exec", execute) await agent( cast( @@ -586,7 +537,7 @@ async def execute(*_args: Any, **_kwargs: Any) -> None: execute_mock = AsyncMock(side_effect=execute) monkeypatch.setattr(computer_mcp, "bridge_computer_mcp", bridge) - monkeypatch.setattr("hud.agents.claude.cli.agent.run_claude", execute_mock) + monkeypatch.setattr(agent, "_exec", execute_mock) await agent( cast( @@ -673,7 +624,7 @@ async def execute(*_args: Any, **kwargs: Any) -> None: agent = ClaudeCLIAgent() monkeypatch.setattr(computer_mcp, "bridge_computer_mcp", bridge) - monkeypatch.setattr("hud.agents.claude.cli.agent.run_claude", execute) + monkeypatch.setattr(agent, "_exec", execute) await agent( cast( @@ -931,7 +882,6 @@ async def open(self, ref: str) -> SSHClient: seen: list[tuple[Any, SSHClient, str]] = [] async def execute( - _config: ClaudeCLIConfig, run: Any, *, ssh: SSHClient, @@ -946,7 +896,7 @@ async def execute( await release_first.wait() agent = ClaudeCLIAgent() - monkeypatch.setattr("hud.agents.claude.cli.agent.run_claude", execute) + monkeypatch.setattr(agent, "_exec", execute) run_a = SimpleNamespace(client=Client(shell_a, ssh_a), prompt_text="first", runtime_config=None) run_b = SimpleNamespace( client=Client(shell_b, ssh_b), prompt_text="second", runtime_config=None diff --git a/hud/agents/tests/test_codex_cli_agent.py b/hud/agents/tests/test_codex_cli_agent.py index 53f739f0b..9819c1827 100644 --- a/hud/agents/tests/test_codex_cli_agent.py +++ b/hud/agents/tests/test_codex_cli_agent.py @@ -14,6 +14,8 @@ from hud.agents.cli import resolve_executable from hud.agents.codex import CodexCLIAgent from hud.agents.codex.agent import codex_command, run_codex +from hud.agents.tests.cli_fakes import FakeProcess as _FakeProcess +from hud.agents.tests.cli_fakes import fake_run as _fake_run from hud.agents.types import AgentStep, CodexCLIConfig, ToolStep from hud.capabilities import Capability, SSHClient from hud.eval.runtime import RuntimeConfig, RuntimeResources @@ -31,67 +33,6 @@ def _clear_api_keys(monkeypatch: pytest.MonkeyPatch) -> None: ) -class _FakeReader: - def __init__(self, value: str, *, pause_after: int | None = None) -> None: - self._raw = value.encode() - self._lines = self._raw.splitlines(keepends=True) - self._pause_after = pause_after - self._index = 0 - self.blocked = asyncio.Event() - self.release = asyncio.Event() - - async def readline(self) -> bytes: - if self._pause_after == self._index: - self.blocked.set() - await self.release.wait() - self._pause_after = None - if self._index == len(self._lines): - return b"" - line = self._lines[self._index] - self._index += 1 - return line - - async def read(self) -> bytes: - return self._raw - - -class _FakeWriter: - def __init__(self) -> None: - self.data = b"" - self.eof = False - - def write(self, data: bytes) -> None: - self.data += data - - async def drain(self) -> None: - pass - - def write_eof(self) -> None: - self.eof = True - - -class _FakeProcess: - def __init__( - self, - stdout: str, - *, - stderr: str = "", - returncode: int | None = 0, - pause_after: int | None = None, - ) -> None: - self.stdin = _FakeWriter() - self.stdout = _FakeReader(stdout, pause_after=pause_after) - self.stderr = _FakeReader(stderr) - self.returncode = returncode - self.closed = False - - def close(self) -> None: - self.closed = True - - async def wait_closed(self) -> None: - pass - - class _FakeSSH: def __init__(self, process: _FakeProcess, *, shell: str = "bash") -> None: self.process = process @@ -108,12 +49,6 @@ async def create_process(self, command: str) -> _FakeProcess: return self.process -def _fake_run() -> Any: - trace = SimpleNamespace(status=None, content="", extra={}) - steps: list[Any] = [] - return SimpleNamespace(trace=trace, record=steps.append, steps=steps) - - _STREAM_JSON = ( '{"type":"thread.started","thread_id":"thread-1"}\n' '{"type":"turn.started"}\n' From 4ebbdd594e4e03b105d08280f6eac116bd067290 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 27 Aug 2026 23:23:56 -0700 Subject: [PATCH 14/14] Narrow Claude CLI package changes --- hud/agents/claude/__init__.py | 2 +- hud/agents/claude/{cli => sdk}/__init__.py | 0 hud/agents/claude/{cli => sdk}/agent.py | 184 ++++-------------- .../claude/{cli => sdk}/computer_mcp.py | 2 +- hud/agents/claude/sdk/events.py | 135 +++++++++++++ hud/agents/tests/test_claude_cli_agent.py | 8 +- 6 files changed, 175 insertions(+), 156 deletions(-) rename hud/agents/claude/{cli => sdk}/__init__.py (100%) rename hud/agents/claude/{cli => sdk}/agent.py (55%) rename hud/agents/claude/{cli => sdk}/computer_mcp.py (99%) create mode 100644 hud/agents/claude/sdk/events.py diff --git a/hud/agents/claude/__init__.py b/hud/agents/claude/__init__.py index 26e7aae08..f61f0add8 100644 --- a/hud/agents/claude/__init__.py +++ b/hud/agents/claude/__init__.py @@ -7,7 +7,7 @@ AsyncAnthropicBedrock, ClaudeAgent, ) -from .cli import ClaudeCLIAgent, ClaudeCLIConfig +from .sdk import ClaudeCLIAgent, ClaudeCLIConfig from .tools import ClaudeToolSearchTool, ClaudeWebFetchTool, ClaudeWebSearchTool __all__ = [ diff --git a/hud/agents/claude/cli/__init__.py b/hud/agents/claude/sdk/__init__.py similarity index 100% rename from hud/agents/claude/cli/__init__.py rename to hud/agents/claude/sdk/__init__.py diff --git a/hud/agents/claude/cli/agent.py b/hud/agents/claude/sdk/agent.py similarity index 55% rename from hud/agents/claude/cli/agent.py rename to hud/agents/claude/sdk/agent.py index 4b5b398ff..76fd85c8e 100644 --- a/hud/agents/claude/cli/agent.py +++ b/hud/agents/claude/sdk/agent.py @@ -1,4 +1,10 @@ -"""Claude CLI harness over a workspace SSH capability.""" +"""ClaudeCLIAgent — runs ``claude`` CLI over SSH inside the env workspace. + +SSH-execs the ``claude`` CLI on the remote workspace so all built-in tools +(Bash, Read, Write, Edit, Glob, Grep) operate on the env's filesystem. +MCP capabilities from the manifest are written as MCP server config so the +CLI can call env-hosted MCP tools too. +""" from __future__ import annotations @@ -9,11 +15,8 @@ from typing import TYPE_CHECKING, Any, cast import asyncssh -import mcp.types as mcp_types -from anthropic.types.beta import BetaMessage from hud.agents.base import Agent -from hud.agents.claude.agent import ClaudeAgent from hud.agents.cli import ( WINDOWS_SHELLS, powershell, @@ -21,13 +24,13 @@ resolve_executable, run_jsonl, ) -from hud.agents.types import ClaudeCLIConfig, ToolStep +from hud.agents.types import ClaudeCLIConfig from hud.settings import settings from hud.telemetry.context import get_current_trace_id -from hud.types import MCPToolCall, MCPToolResult from hud.utils.time import now_iso from . import computer_mcp +from .events import ClaudeEvents if TYPE_CHECKING: from hud.capabilities import SSHClient @@ -45,124 +48,6 @@ } -class ClaudeEvents: - """Translate Claude CLI stream messages into canonical HUD steps.""" - - def __init__(self, run: Run, *, started_at: str) -> None: - self.run = run - self.agent_started_at = started_at - self.pending_calls: dict[str, tuple[MCPToolCall, str]] = {} - self.saw_result = False - self.error: str | None = None - - def consume(self, line: str) -> None: - line = line.strip() - if not line: - return - message = json.loads(line) - if not isinstance(message, dict): - raise ValueError("Claude stream event must be an object") - received_at = now_iso() - match message.get("type"): - case "system" if message.get("subtype") == "init": - self.agent_started_at = received_at - case "assistant": - step = ClaudeAgent.message_to_agent_step( - BetaMessage.model_validate(message["message"]) - ) - step.started_at = self.agent_started_at - step.ended_at = received_at - if step.content: - self.run.trace.content = step.content - self.run.record(step) - for call in step.tool_calls: - self.pending_calls[call.id] = (call, received_at) - case "user": - saw_result = False - for block in message["message"]["content"]: - if block["type"] != "tool_result": - continue - call_id = block["tool_use_id"] - try: - call, started_at = self.pending_calls.pop(call_id) - except KeyError: - raise ValueError( - f"Claude returned a result for unknown tool call {call_id!r}" - ) from None - - raw_result = block.get("content") - raw_items = raw_result if isinstance(raw_result, list) else [raw_result] - content: list[mcp_types.ContentBlock] = [] - for item in raw_items: - if isinstance(item, str): - content.append(mcp_types.TextContent(type="text", text=item)) - elif item["type"] == "text": - content.append(mcp_types.TextContent(type="text", text=item["text"])) - elif item["type"] == "image": - source = item["source"] - content.append( - mcp_types.ImageContent( - type="image", - data=source["data"], - mimeType=source["media_type"], - ) - ) - else: - raise ValueError(f"unsupported Claude tool result block: {item!r}") - - self.run.record( - ToolStep( - call=call, - result=MCPToolResult( - call_id=call_id, - content=content, - isError=block.get("is_error") is True, - ), - started_at=started_at, - ended_at=received_at, - ) - ) - saw_result = True - if saw_result: - self.agent_started_at = received_at - case "result": - self.saw_result = True - trace = self.run.trace - result = message.get("result") - if isinstance(result, str): - trace.content = result - if message.get("is_error") is True: - self.error = trace.content or "claude CLI reported an error" - for key in ( - "subtype", - "session_id", - "duration_ms", - "duration_api_ms", - "stop_reason", - "num_turns", - "total_cost_usd", - ): - if (value := message.get(key)) is not None: - trace.extra[key] = value - - def finish(self, *, returncode: int, stderr: str) -> None: - trace = self.run.trace - error = self.error - if returncode != 0: - trace.extra["returncode"] = returncode - error = stderr.strip() or f"claude CLI exited with return code {returncode}" - elif not self.saw_result: - error = "claude CLI exited without a result event" - elif self.pending_calls: - missing = ", ".join(sorted(self.pending_calls)) - error = f"claude CLI exited without results for tool calls: {missing}" - - if error is not None and stderr: - trace.extra["stderr"] = stderr - if error is not None: - raise RuntimeError(error) - - class ClaudeCLIAgent(Agent): """Runs ``claude`` CLI over SSH inside the env workspace. @@ -309,16 +194,13 @@ def _build_env_vars(self) -> dict[str, str]: env["ANTHROPIC_MODEL"] = self.config.model env["ANTHROPIC_SMALL_FAST_MODEL"] = self.config.model - # A custom base URL must own every model tier; otherwise background calls - # can escape to Anthropic instead of using the configured gateway. + # When using a custom base URL, alias all model tiers to the same model + # so the CLI doesn't try to reach Anthropic for background requests. if "ANTHROPIC_BASE_URL" in env: - for name in ( - "ANTHROPIC_DEFAULT_SONNET_MODEL", - "ANTHROPIC_DEFAULT_OPUS_MODEL", - "ANTHROPIC_DEFAULT_HAIKU_MODEL", - "CLAUDE_CODE_SUBAGENT_MODEL", - ): - env[name] = self.config.model + env["ANTHROPIC_DEFAULT_SONNET_MODEL"] = self.config.model + env["ANTHROPIC_DEFAULT_OPUS_MODEL"] = self.config.model + env["ANTHROPIC_DEFAULT_HAIKU_MODEL"] = self.config.model + env["CLAUDE_CODE_SUBAGENT_MODEL"] = self.config.model env["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC"] = "1" env["DISABLE_AUTOUPDATER"] = "1" @@ -333,11 +215,11 @@ async def _write_mcp_config( """Write MCP config into the workspace and return its path.""" if not mcp_servers: return None - await ssh.write_text( - MCP_CONFIG_PATH, - json.dumps({"mcpServers": mcp_servers}, indent=2), - ) - return MCP_CONFIG_PATH + mcp_json = json.dumps({"mcpServers": mcp_servers}, indent=2) + path = MCP_CONFIG_PATH + await ssh.write_text(path, mcp_json) + logger.info("Wrote MCP config") + return path def _build_cli_command( self, @@ -346,8 +228,9 @@ def _build_cli_command( mcp_config_path: str | None = None, executable: str = "claude", ) -> str: - env = self._build_env_vars() - args: list[str] = [ + env_vars = self._build_env_vars() + is_win = shell in WINDOWS_SHELLS + base_args: list[str] = [ executable, "--verbose", "--input-format=stream-json", @@ -356,29 +239,30 @@ def _build_cli_command( f"--permission-mode={self.config.permission_mode}", ] if self.config.max_steps > 0: - args.append(f"--max-turns={self.config.max_steps}") + base_args.append(f"--max-turns={self.config.max_steps}") if self.config.system_prompt: - args.extend(["--system-prompt", self.config.system_prompt]) + base_args.extend(["--system-prompt", self.config.system_prompt]) for tool in self.config.allowed_tools: - args.extend(["--allowedTools", tool]) + base_args.extend(["--allowedTools", tool]) if mcp_config_path: - args.extend(["--mcp-config", mcp_config_path]) + base_args.extend(["--mcp-config", mcp_config_path]) - if shell in WINDOWS_SHELLS: + if is_win: script = ";".join( [ - *(f"$env:{key}={powershell_quote(value)}" for key, value in env.items()), + *(f"$env:{key}={powershell_quote(value)}" for key, value in env_vars.items()), f"Get-Content -Raw -Encoding UTF8 {powershell_quote(INPUT_PATH)}" f" | & {powershell_quote(executable)} " - f"{' '.join(powershell_quote(arg) for arg in args[1:])}", + f"{' '.join(powershell_quote(arg) for arg in base_args[1:])}", "exit $LASTEXITCODE", ] ) return powershell(script) - command = " ".join(shlex.quote(arg) for arg in args) - env_prefix = " ".join(f"{key}={shlex.quote(value)}" for key, value in env.items()) - return f'export PATH="$HOME/.local/bin:$PATH"; {env_prefix} {command}' + cli_parts = [shlex.quote(a) for a in base_args] + cli_cmd = " ".join(cli_parts) + env_prefix = " ".join(f"{k}={shlex.quote(v)}" for k, v in env_vars.items()) + return f'export PATH="$HOME/.local/bin:$PATH"; {env_prefix} {cli_cmd}' __all__ = ["ClaudeCLIAgent"] diff --git a/hud/agents/claude/cli/computer_mcp.py b/hud/agents/claude/sdk/computer_mcp.py similarity index 99% rename from hud/agents/claude/cli/computer_mcp.py rename to hud/agents/claude/sdk/computer_mcp.py index 8ff30a065..ad51496e4 100644 --- a/hud/agents/claude/cli/computer_mcp.py +++ b/hud/agents/claude/sdk/computer_mcp.py @@ -147,7 +147,7 @@ async def bridge_computer_mcp( local = await asyncio.create_subprocess_exec( sys.executable, "-m", - "hud.agents.claude.cli.computer_mcp", + "hud.agents.claude.sdk.computer_mcp", stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, diff --git a/hud/agents/claude/sdk/events.py b/hud/agents/claude/sdk/events.py new file mode 100644 index 000000000..737cb8776 --- /dev/null +++ b/hud/agents/claude/sdk/events.py @@ -0,0 +1,135 @@ +"""Claude CLI stream translation.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import mcp.types as mcp_types +from anthropic.types.beta import BetaMessage + +from hud.agents.claude.agent import ClaudeAgent +from hud.agents.types import ToolStep +from hud.types import MCPToolCall, MCPToolResult +from hud.utils.time import now_iso + +if TYPE_CHECKING: + from hud.eval.run import Run + + +class ClaudeEvents: + """Translate Claude CLI stream messages into canonical HUD steps.""" + + def __init__(self, run: Run, *, started_at: str) -> None: + self.run = run + self.agent_started_at = started_at + self.pending_calls: dict[str, tuple[MCPToolCall, str]] = {} + self.saw_result = False + self.error: str | None = None + + def consume(self, line: str) -> None: + line = line.strip() + if not line: + return + message = json.loads(line) + if not isinstance(message, dict): + raise ValueError("Claude stream event must be an object") + received_at = now_iso() + match message.get("type"): + case "system" if message.get("subtype") == "init": + self.agent_started_at = received_at + case "assistant": + step = ClaudeAgent.message_to_agent_step( + BetaMessage.model_validate(message["message"]) + ) + step.started_at = self.agent_started_at + step.ended_at = received_at + if step.content: + self.run.trace.content = step.content + self.run.record(step) + for call in step.tool_calls: + self.pending_calls[call.id] = (call, received_at) + case "user": + saw_result = False + for block in message["message"]["content"]: + if block["type"] != "tool_result": + continue + call_id = block["tool_use_id"] + try: + call, started_at = self.pending_calls.pop(call_id) + except KeyError: + raise ValueError( + f"Claude returned a result for unknown tool call {call_id!r}" + ) from None + + raw_result = block.get("content") + raw_items = raw_result if isinstance(raw_result, list) else [raw_result] + content: list[mcp_types.ContentBlock] = [] + for item in raw_items: + if isinstance(item, str): + content.append(mcp_types.TextContent(type="text", text=item)) + elif item["type"] == "text": + content.append(mcp_types.TextContent(type="text", text=item["text"])) + elif item["type"] == "image": + source = item["source"] + content.append( + mcp_types.ImageContent( + type="image", + data=source["data"], + mimeType=source["media_type"], + ) + ) + else: + raise ValueError(f"unsupported Claude tool result block: {item!r}") + + self.run.record( + ToolStep( + call=call, + result=MCPToolResult( + call_id=call_id, + content=content, + isError=block.get("is_error") is True, + ), + started_at=started_at, + ended_at=received_at, + ) + ) + saw_result = True + if saw_result: + self.agent_started_at = received_at + case "result": + self.saw_result = True + trace = self.run.trace + result = message.get("result") + if isinstance(result, str): + trace.content = result + if message.get("is_error") is True: + self.error = trace.content or "claude CLI reported an error" + for key in ( + "subtype", + "session_id", + "duration_ms", + "duration_api_ms", + "stop_reason", + "num_turns", + "total_cost_usd", + ): + if (value := message.get(key)) is not None: + trace.extra[key] = value + + def finish(self, *, returncode: int, stderr: str) -> None: + trace = self.run.trace + error = self.error + if returncode != 0: + trace.extra["returncode"] = returncode + error = stderr.strip() or f"claude CLI exited with return code {returncode}" + elif not self.saw_result: + error = "claude CLI exited without a result event" + elif self.pending_calls: + missing = ", ".join(sorted(self.pending_calls)) + error = f"claude CLI exited without results for tool calls: {missing}" + + if error is not None and stderr: + trace.extra["stderr"] = stderr + if error is not None: + raise RuntimeError(error) diff --git a/hud/agents/tests/test_claude_cli_agent.py b/hud/agents/tests/test_claude_cli_agent.py index ff58e78f5..b6528b24b 100644 --- a/hud/agents/tests/test_claude_cli_agent.py +++ b/hud/agents/tests/test_claude_cli_agent.py @@ -23,8 +23,8 @@ import pytest from mcp.types import ImageContent, TextContent -from hud.agents.claude.cli import computer_mcp -from hud.agents.claude.cli.agent import ClaudeCLIAgent +from hud.agents.claude.sdk import computer_mcp +from hud.agents.claude.sdk.agent import ClaudeCLIAgent from hud.agents.tests.cli_fakes import FakeProcess as _FakeStreamProcess from hud.agents.tests.cli_fakes import fake_run as _fake_run from hud.agents.types import AgentStep, ClaudeCLIConfig, ToolStep @@ -43,7 +43,7 @@ def _clear_api_keys(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(settings, "api_key", None) monkeypatch.setattr(settings, "anthropic_api_key", None) monkeypatch.setattr( - "hud.agents.claude.cli.agent.resolve_executable", + "hud.agents.claude.sdk.agent.resolve_executable", AsyncMock(return_value="claude"), ) @@ -773,7 +773,7 @@ async def test_computer_mcp_bridge_uses_controller_python_and_owns_resources( assert spawn_args[:3] == ( sys.executable, "-m", - "hud.agents.claude.cli.computer_mcp", + "hud.agents.claude.sdk.computer_mcp", ) environ = spawn_call.kwargs["env"] assert json.loads(environ[computer_mcp.RFB_CAPABILITY_ENV]) == screen.to_manifest()