DI-4855 Add Deep Agents Code benchmark runner

README.md

@@ -375,6 +375,7 @@ This will open a local HTML page that includes much more detail, including detai
 ADE-bench currently supports the following agents:
 
 - Claude Code - `--agent claude`
+- Deep Agents Code - `--agent dcode`
 - OpenAI Codex - `--agent codex`
 - Gemini CLI - `--agent gemini`
 
@@ -391,11 +392,15 @@ claude --output-format json -p {task_prompt} --model {model-id} \
 printenv OPENAI_API_KEY | codex login --with-api-key && \
 codex --ask-for-approval never --model {model-id} exec --sandbox workspace-write --skip-git-repo-check {task_prompt}
 
+# DEEP AGENTS CODE
+dcode --no-mcp --no-stream --shell-allow-list all \
+  --model {model-id} --non-interactive {task_prompt}
+
 # GEMINI
 gemini --output-format json --yolo --prompt {task_prompt} --model {model-id}
 ```
 
-Configuration files for each agent are found in the `/shared/config` directory. You can use `CLAUDE.md` to configure Claude Code, `AGENTS.md` to configure Codex, and `GEMINI.md` to configure Gemini.
+Configuration files for each agent are found in the `/shared/config` directory. You can use `CLAUDE.md` to configure Claude Code, `AGENTS.md` to configure Codex and Deep Agents Code, and `GEMINI.md` to configure Gemini.
 
 ### Plugin sets
 

ade_bench/agents/agent_factory.py

@@ -6,6 +6,9 @@
 from ade_bench.agents.installed_agents.claude_code.claude_code_agent import (
     ClaudeCodeAgent,
 )
+from ade_bench.agents.installed_agents.deep_agents_code.deep_agents_code_agent import (
+    DeepAgentsCodeAgent,
+)
 from ade_bench.agents.installed_agents.gemini_cli.gemini_cli_agent import (
     GeminiCLIAgent,
 )
@@ -35,6 +38,7 @@ class NamedAgentFactory(AgentFactory):
         NoneAgent.NAME: NoneAgent,
         SageAgent.NAME: SageAgent,
         ClaudeCodeAgent.NAME: ClaudeCodeAgent,
+        DeepAgentsCodeAgent.NAME: DeepAgentsCodeAgent,
         OpenAICodexAgent.NAME: OpenAICodexAgent,
         GeminiCLIAgent.NAME: GeminiCLIAgent,
         MacroAgent.NAME: MacroAgent,

ade_bench/agents/agent_name.py

@@ -10,6 +10,7 @@ class AgentName(Enum):
     SAGE = "sage"
     CLAUDE_CODE = "claude"
     OPENAI_CODEX = "codex"
+    DEEP_AGENTS_CODE = "dcode"
     GEMINI_CLI = "gemini"
     MACRO = "macro"
 

ade_bench/agents/installed_agents/abstract_installed_agent.py

@@ -11,6 +11,7 @@
 """
 
 import shlex
+import time
 from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import Any
@@ -226,6 +227,7 @@ def perform_task(
         agent_output_file = "/tmp/agent_output.log"
 
         run_agent_commands = self._run_agent_commands(task_prompt)
+        agent_started_at = time.monotonic()
         for command in run_agent_commands:
             log_harness_info(
                 logger,
@@ -251,6 +253,8 @@ def perform_task(
 
         # Try to extract just the JSON part from the output
         parsed_metrics = self._parse_agent_output(output)
+        if not parsed_metrics.get("runtime_ms"):
+            parsed_metrics["runtime_ms"] = round((time.monotonic() - agent_started_at) * 1000)
 
         # Log the agent response metrics if we have a task name
         if parsed_metrics:

ade_bench/agents/installed_agents/deep_agents_code/deep_agents_code-setup.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+
+echo "Setup Deep Agents Code"
+uv tool install --prerelease=allow deepagents-code==0.1.45
+export PATH="/root/.local/bin:${PATH}"
+dcode --version

ade_bench/agents/installed_agents/deep_agents_code/deep_agents_code_agent.py

@@ -0,0 +1,100 @@
+import json
+import os
+import re
+import shlex
+from pathlib import Path
+from typing import Any
+
+from ade_bench.agents.agent_name import AgentName
+from ade_bench.agents.installed_agents.abstract_installed_agent import (
+    AbstractInstalledAgent,
+)
+from ade_bench.config import config
+from ade_bench.harness_models import TerminalCommand
+
+
+class DeepAgentsCodeAgent(AbstractInstalledAgent):
+    """Run LangChain's Deep Agents Code CLI inside an ADE-Bench task container."""
+
+    NAME = AgentName.DEEP_AGENTS_CODE
+    _REASONING_EFFORT_ENV_VAR = "DEEPAGENTS_CODE_REASONING_EFFORT"
+    _PROVIDER_ENV_VARS = (
+        "ANTHROPIC_API_KEY",
+        "GOOGLE_API_KEY",
+        "GOOGLE_CLOUD_PROJECT",
+        "OPENAI_API_KEY",
+    )
+    _USAGE_ROW = re.compile(
+        r"^\s*\S+\s+\S+\s+(?P<requests>\d+)\s+"
+        r"(?P<input>[\d.]+[KMB]?)\s+(?P<output>[\d.]+[KMB]?)\s*$",
+        re.MULTILINE,
+    )
+
+    @property
+    def _env(self) -> dict[str, str]:
+        return {name: value for name in self._PROVIDER_ENV_VARS if (value := os.environ.get(name))}
+
+    @property
+    def _install_agent_script(self) -> Path:
+        return Path(__file__).parent / "deep_agents_code-setup.sh"
+
+    def _run_agent_commands(self, task_prompt: str) -> list[TerminalCommand]:
+        command_parts = [
+            "echo 'AGENT RESPONSE: '",
+            "dcode",
+            "--no-mcp",
+            "--no-stream",
+            "--shell-allow-list all",
+        ]
+
+        if self._model_name:
+            command_parts.append(f"--model {shlex.quote(self._model_name)}")
+
+        if effort := os.environ.get(self._REASONING_EFFORT_ENV_VAR):
+            model_params = json.dumps({"reasoning": {"effort": effort}})
+            command_parts.append(f"--model-params {shlex.quote(model_params)}")
+
+        command_parts.append(f"--non-interactive {shlex.quote(task_prompt)}")
+        command = " && ".join(command_parts[:2]) + " " + " ".join(command_parts[2:])
+
+        return [
+            TerminalCommand(
+                command=command,
+                min_timeout_sec=0.0,
+                max_timeout_sec=config.default_agent_timeout_sec,
+                block=True,
+                append_enter=True,
+            )
+        ]
+
+    def _parse_agent_output(self, output: str) -> dict[str, Any]:
+        # dcode prints a human-readable Usage Stats table after successful
+        # non-interactive runs. Its compact K/M/B values are rounded by dcode,
+        # so the parsed token counts are approximate rather than exact.
+        usage_output = output.partition("Usage Stats")[2]
+        usage_match = self._USAGE_ROW.search(usage_output)
+        input_tokens = 0
+        output_tokens = 0
+        num_turns = 0
+        if usage_match:
+            input_tokens = self._parse_compact_token_count(usage_match["input"])
+            output_tokens = self._parse_compact_token_count(usage_match["output"])
+            num_turns = int(usage_match["requests"])
+
+        return {
+            "input_tokens": input_tokens,
+            "output_tokens": output_tokens,
+            "cache_tokens": 0,
+            "num_turns": num_turns,
+            "runtime_ms": 0,
+            "cost_usd": 0.0,
+            "model_name": self._model_name,
+        }
+
+    @staticmethod
+    def _parse_compact_token_count(value: str) -> int:
+        multiplier = {"K": 1_000, "M": 1_000_000, "B": 1_000_000_000}
+        suffix = value[-1]
+        if suffix in multiplier:
+            return round(float(value[:-1]) * multiplier[suffix])
+        return int(value)

ade_bench/setup/agent_setup.py

@@ -38,7 +38,7 @@ def setup_agent_config(terminal, task_id: str, trial_handler, logger) -> None:
         _copy_config_file(terminal, trial_handler, "CLAUDE.md")
     elif agent_name == AgentName.GEMINI_CLI:
         _copy_config_file(terminal, trial_handler, "GEMINI.md")
-    elif agent_name == AgentName.OPENAI_CODEX:
+    elif agent_name in {AgentName.OPENAI_CODEX, AgentName.DEEP_AGENTS_CODE}:
         _copy_config_file(terminal, trial_handler, "AGENTS.md")
     elif agent_name == AgentName.MACRO:
         _copy_config_file(terminal, trial_handler, "MACRO.md")

tests/agents/installed_agents/test_deep_agents_code_agent.py

@@ -0,0 +1,71 @@
+from unittest.mock import MagicMock, patch
+
+from ade_bench.agents.agent_factory import NamedAgentFactory
+from ade_bench.agents.agent_name import AgentName
+
+
+def test_named_factory_creates_deep_agents_code_agent():
+    agent = NamedAgentFactory(AgentName.DEEP_AGENTS_CODE).get_agent()
+
+    assert agent.NAME is AgentName.DEEP_AGENTS_CODE
+
+
+def test_perform_task_runs_dcode_headlessly_with_model_and_reasoning_effort():
+    session = MagicMock()
+    session.container.exec_run.return_value = MagicMock(exit_code=0, output=b"completed")
+    agent = NamedAgentFactory(AgentName.DEEP_AGENTS_CODE).get_agent(
+        model_name="gpt-5.4",
+    )
+
+    with (
+        patch.dict(
+            "os.environ",
+            {
+                "OPENAI_API_KEY": "test-key",
+                "DEEPAGENTS_CODE_REASONING_EFFORT": "low",
+            },
+            clear=True,
+        ),
+        patch(
+            "ade_bench.agents.installed_agents.abstract_installed_agent.time.monotonic",
+            side_effect=[10.0, 10.5],
+        ),
+    ):
+        result = agent.perform_task("fix user's model", session)
+
+    command = session.send_command.call_args.args[0].command
+    assert "dcode" in command
+    assert "--non-interactive 'fix user'\"'\"'s model'" in command
+    assert "--model gpt-5.4" in command
+    assert "--model-params" in command
+    assert '"effort": "low"' in command
+    assert "--shell-allow-list all" in command
+    assert "--no-mcp" in command
+    assert result.model_name == "gpt-5.4"
+    assert result.runtime_ms == 500
+
+
+def test_perform_task_captures_dcode_usage_stats():
+    session = MagicMock()
+    session.container.exec_run.return_value = MagicMock(
+        exit_code=0,
+        output=b"""Task completed
+
+Usage Stats
+Provider  Model        Reqs  InputTok  OutputTok
+openai    gpt-5.6-sol     9    158.3K        903
+
+Agent active  18.7s
+""",
+    )
+    agent = NamedAgentFactory(AgentName.DEEP_AGENTS_CODE).get_agent(
+        model_name="gpt-5.6-sol",
+    )
+
+    with patch.dict("os.environ", {"OPENAI_API_KEY": "test-key"}, clear=True):
+        result = agent.perform_task("fix the model", session)
+
+    assert result.input_tokens == 158_300
+    assert result.output_tokens == 903
+    assert result.num_turns == 9
+    assert result.model_name == "gpt-5.6-sol"