diff --git a/DOCFS_PILOT.md b/DOCFS_PILOT.md new file mode 100644 index 0000000..ef59f4f --- /dev/null +++ b/DOCFS_PILOT.md @@ -0,0 +1,105 @@ +# docfs Pilot + +This repository's main benchmark measures URL discovery through a local HTTP +proxy. `docfs` presents the same `llms.txt`-publishing documentation sites as +a local, read-only filesystem, so it needs a separate runner and different +efficiency metrics. + +`python -m url_discovery_bench.docfs_run` runs the existing questions against +one docfs mount. It preserves exact-path grading, records the agent's shell +command count, and records the pages and bytes docfs cached. It intentionally +does not report HTTP fetch or 404 counts: the agent never fetches HTTP URLs. + +## Pilot setup + +- Dataset: one pre-existing question from each of the 20 sites in + `dataset/full.json`. +- Agents: `claude-sonnet-5` and `gpt-5.5`, one attempt per task. +- Each attempt evicted its domain before the agent started, preventing a + previous task from warming that site's docfs cache. +- Claude could use local read/search shell commands only. Codex ran with + `sandbox_workspace_write.network_access=false`. +- The prompt requires the site-relative answer path, including a site's base + path such as `/docs`. + +Example: + +```bash +cargo build --release --locked --manifest-path ../docfs/Cargo.toml +XDG_CACHE_HOME=/tmp/udb-docfs-cache \ + ../docfs/target/release/docfs mount --path /tmp/udb-docfs + +python -m url_discovery_bench.docfs_run \ + --dataset dataset/full.json \ + --agents claude,codex \ + --job docfs-pilot \ + --docfs-bin ../docfs/target/release/docfs \ + --mount-path /tmp/udb-docfs \ + --cache-home /tmp/udb-docfs-cache + +XDG_CACHE_HOME=/tmp/udb-docfs-cache \ + ../docfs/target/release/docfs unmount +``` + +The runner is serial by design. A shared mount/cache cannot be cold per +attempt when attempts execute concurrently. + +## Results + +Five base-path tasks were re-run after correcting the answer contract: the +first prompt requested a path relative to the local base directory while the +dataset expects a site-relative path. The table uses those corrected rows. + +| agent | n | accuracy | shell commands/task | pages cached/site | cache MiB/site | input tokens (k) | seconds/task | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| Claude | 20 | 90% | 4.9 | 173.3 | 12.4 | 227.2 | 35.7 | +| Codex | 20 | 95% | 7.8 | 174.5 | 12.2 | 162.8 | 45.9 | + +No audited external fetches occurred. The committed full-study `md-link` arm, +filtered to the same task IDs (three attempts per task), is a useful but not +directly equivalent comparison: + +| agent | n | accuracy | 404s/task | HTTP fetches/task | input tokens (k) | seconds/task | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Claude | 60 | 95% | 0.15 | 4.6 | 187.1 | 26.2 | +| Codex | 60 | 95% | 0.13 | 17.8 | 128.6 | 54.1 | + +docfs's first directory traversal initiates background hydration. The roughly +12 MiB/site cache cost is therefore real upstream transfer, despite agents +only receiving local shell output. + +## Token audit + +The pilot means are not representative of a normal docfs navigation. Two +runaway attempts per agent dominate the token totals: + +| agent | reported mean | median | mean excluding two outliers | +| --- | ---: | ---: | ---: | +| Claude | 227.2k | 139.3k | 140.3k | +| Codex | 162.8k | 81.2k | 91.4k | + +The Claude `trigger` attempt consumed 1.34M tokens across 30 turns, including +1.28M cache-read tokens; the `stytch` attempt used 682k across 16 turns. +Codex used 551k and 1.06M tokens respectively. These are repeated +conversation-context reads, not a megabyte-scale prompt or a large document +being injected into context. + +The `stytch` domain resolved a shallow root manifest with two pages rather +than the expected `/docs` corpus, so both agents searched an incomplete tree. +The `trigger` agent spent 29 shell commands navigating the mounted layout. +Neither behavior is representative of the straightforward tasks. + +For a report-quality docfs study: + +- Preflight that the resolved tree contains every expected path. +- Set a finite `--max-turns` value to bound unsuccessful navigation. +- Run multiple attempts per task and report medians as well as means. +- Treat cache-hydration bytes as a separate transport-cost metric, rather than + comparing them to proxy HTTP request counts. + +## Local CLI configuration + +The pilot uses the same runner defaults as the existing study. Claude does +not ignore local CLI configuration in either runner, while Codex uses +`--ignore-user-config` in both. The outlier pattern occurs in both agents, +including Codex, so it is not explained solely by local Claude configuration. diff --git a/url_discovery_bench/docfs_run.py b/url_discovery_bench/docfs_run.py new file mode 100644 index 0000000..a43cc26 --- /dev/null +++ b/url_discovery_bench/docfs_run.py @@ -0,0 +1,194 @@ +"""Run URL-discovery questions against a local docfs mount. + +This is deliberately separate from ``run.py``: docfs gives agents files rather +than HTTP URLs, so HTTP fetch and 404 metrics do not apply. Results instead +record shell-command count and the docfs cache bytes/pages fetched per attempt. +""" + +import argparse +import json +import os +import re +import subprocess +import threading +from datetime import datetime +from pathlib import Path +from urllib.parse import urlparse + +from .grade import grade +from .runners import AgentRunOpts, RUNNERS + + +_write_lock = threading.Lock() +_DOMAIN_STATUS = re.compile(r"^\s*(\S+)\s+(\d+) pages\s+(\d+) bytes cached$") +_FILESYSTEM_TOOLS = ",".join([ + "Bash(ls:*)", "Bash(find:*)", "Bash(rg:*)", "Bash(grep:*)", "Bash(cat:*)", + "Bash(sed:*)", "Bash(head:*)", "Bash(tail:*)", "Bash(wc:*)", "Bash(pwd)", +]) + + +def build_prompt(source: Path, base_path: str, question: str) -> str: + root = source / base_path.lstrip("/") + url_prefix = base_path or "/" + return ( + f"A documentation site is available locally at {root}.\n\n" + f"Question: {question}\n\n" + "Find the specific documentation page that answers the question. Explore only " + "the local documentation directory with shell commands; do not use the network, " + "curl, or prior knowledge. The files are Markdown and their paths map to the " + "site's URL paths.\n\n" + f"When you have found the page, reply with its documentation-site-relative path " + f"on the final line. Include the site's URL prefix {url_prefix!r}, even though " + "the local directory starts after that prefix; omit the .md suffix:\n" + "ANSWER: /path/to/page" + ) + + +def docfs_command(docfs_bin: str, cache_home: str, *args: str) -> str: + env = os.environ.copy() + env["XDG_CACHE_HOME"] = cache_home + result = subprocess.run( + [docfs_bin, *args], env=env, capture_output=True, text=True, check=False, + ) + if result.returncode: + raise RuntimeError((result.stderr or result.stdout).strip()) + return result.stdout + + +def evict_and_status(docfs_bin: str, cache_home: str, domain: str) -> dict: + docfs_command(docfs_bin, cache_home, "evict", domain) + status = docfs_command(docfs_bin, cache_home, "status") + for line in status.splitlines(): + match = _DOMAIN_STATUS.match(line) + if match and match.group(1) == domain: + return {"pagesCached": int(match.group(2)), "cacheBytes": int(match.group(3))} + return {"pagesCached": 0, "cacheBytes": 0} + + +def main() -> None: + ap = argparse.ArgumentParser(prog="url_discovery_bench.docfs_run") + ap.add_argument("--dataset", default="dataset/full.json") + ap.add_argument("--agents", default="claude,codex") + ap.add_argument("--attempts", type=int, default=1) + ap.add_argument("--tasks") + ap.add_argument("--sites") + ap.add_argument("--claude-model", default="claude-sonnet-5") + ap.add_argument("--codex-model", default="gpt-5.5") + ap.add_argument("--job") + ap.add_argument("--max-turns", type=int) + ap.add_argument("--timeout-min", type=int, default=8) + ap.add_argument("--docfs-bin", required=True) + ap.add_argument("--mount-path", required=True) + ap.add_argument("--cache-home", required=True) + args = ap.parse_args() + + dataset = json.loads(Path(args.dataset).read_text(encoding="utf-8")) + sites = dataset["sites"] + if args.sites: + wanted = set(args.sites.split(",")) + sites = [s for s in sites if s["name"] in wanted] + if args.tasks: + wanted = set(args.tasks.split(",")) + for site in sites: + site["tasks"] = [task for task in site["tasks"] if task["id"] in wanted] + + agents = args.agents.split(",") + for agent in agents: + if agent not in RUNNERS: + raise SystemExit(f"unknown agent {agent}") + + mount_path = Path(args.mount_path) + if not mount_path.is_dir(): + raise SystemExit(f"docfs mount is not available at {mount_path}") + + job_name = args.job or datetime.now().strftime("docfs-%Y-%m-%dT%H-%M-%S") + job_dir = Path("jobs") / job_name + (job_dir / "transcripts").mkdir(parents=True, exist_ok=True) + results_file = job_dir / "results.jsonl" + + queue = [ + {"site": site, "task": task, "agent": agent, "attempt": attempt} + for site in sites + for task in site["tasks"] + for agent in agents + for attempt in range(1, args.attempts + 1) + ] + done = set() + if results_file.exists(): + for line in results_file.read_text(encoding="utf-8").splitlines(): + if line.strip(): + row = json.loads(line) + if not row.get("error"): + done.add(f"{row['site']}|{row['task']}|{row['agent']}|{row['attempt']}") + + def key_of(item: dict) -> str: + return f"{item['site']['name']}|{item['task']['id']}|{item['agent']}|{item['attempt']}" + + pending = [item for item in queue if key_of(item) not in done] + print(f"job {job_name}: {len(pending)} docfs attempts to run ({len(done)} already done)") + + def append_row(row: dict) -> None: + with _write_lock: + with open(results_file, "a", encoding="utf-8") as output: + output.write(json.dumps(row) + "\n") + + # A shared mount/cache must be cold per attempt, so keep attempts serial. + for item in pending: + site, task = item["site"], item["task"] + domain = urlparse(site["target"]).netloc + slug = f"{site['name']}-{task['id']}-docfs-{item['agent']}-{item['attempt']}" + try: + evict_and_status(args.docfs_bin, args.cache_home, domain) + prompt = build_prompt(mount_path / domain, site["basePath"], task["question"]) + result = RUNNERS[item["agent"]](prompt, AgentRunOpts( + model=args.claude_model if item["agent"] == "claude" else args.codex_model, + max_turns=args.max_turns, + timeout_ms=args.timeout_min * 60_000, + transcript_file=str(job_dir / "transcripts" / f"{slug}.jsonl"), + allowed_tools=_FILESYSTEM_TOOLS, + network_access=False, + )) + metrics = docfs_command(args.docfs_bin, args.cache_home, "status") + cache = {"pagesCached": 0, "cacheBytes": 0} + for line in metrics.splitlines(): + match = _DOMAIN_STATUS.match(line) + if match and match.group(1) == domain: + cache = {"pagesCached": int(match.group(2)), "cacheBytes": int(match.group(3))} + break + graded = grade(result.answer, task.get("expected"), "http://docfs.local") + row = { + "site": site["name"], "task": task["id"], "arm": "docfs", + "agent": item["agent"], "attempt": item["attempt"], "correct": graded["correct"], + "answerUrl": graded["answer_url"], "model": result.model, + "expected": task.get("expected"), "shellCommands": len(result.commands or []), + "pagesCached": cache["pagesCached"], "cacheBytes": cache["cacheBytes"], + "inputTokens": result.input_tokens, "outputTokens": result.output_tokens, + "durationMs": result.duration_ms, "costUsd": result.cost_usd, + "externalFetches": result.external_fetches, + } + append_row(row) + verdict = "PASS" if graded["correct"] else "FAIL" + print(f"{verdict} {slug} commands={row['shellCommands']} pages={row['pagesCached']}") + except Exception as err: + append_row({"site": site["name"], "task": task["id"], "arm": "docfs", + "agent": item["agent"], "attempt": item["attempt"], "error": str(err)}) + print(f"ERROR {slug}: {str(err)[:200]}") + + rows = [json.loads(line) for line in results_file.read_text(encoding="utf-8").splitlines() if line.strip()] + groups = {} + for row in rows: + if not row.get("error"): + groups.setdefault(row["agent"], []).append(row) + print("agent | n | accuracy % | shell commands | pages cached | cache MiB | input tok (k) | sec") + print("-+-" * 8) + for agent, group in sorted(groups.items()): + graded = [row for row in group if row.get("correct") is not None] + accuracy = 100 * sum(row["correct"] for row in graded) / len(graded) if graded else None + average = lambda field: sum(row[field] for row in group) / len(group) + print(f"{agent} | {len(group)} | {accuracy:.1f} | {average('shellCommands'):.1f} | " + f"{average('pagesCached'):.1f} | {average('cacheBytes') / 1024 / 1024:.1f} | " + f"{average('inputTokens') / 1000:.1f} | {average('durationMs') / 1000:.1f}") + + +if __name__ == "__main__": + main() diff --git a/url_discovery_bench/runners.py b/url_discovery_bench/runners.py index a2d7a55..1858448 100644 --- a/url_discovery_bench/runners.py +++ b/url_discovery_bench/runners.py @@ -14,6 +14,8 @@ class AgentRunOpts: model: Optional[str] = None max_turns: Optional[int] = None transcript_file: Optional[str] = None + allowed_tools: Optional[str] = None + network_access: bool = True @dataclass @@ -28,6 +30,7 @@ class AgentResult: external_samples: list unaudited_fetches: int cost_usd: Optional[float] = None + commands: list[str] = None def run_claude(prompt: str, opts: AgentRunOpts) -> AgentResult: @@ -36,7 +39,7 @@ def run_claude(prompt: str, opts: AgentRunOpts) -> AgentResult: "-p", prompt, "--output-format", "stream-json", "--verbose", - "--allowedTools", "Bash(curl:*)", + "--allowedTools", opts.allowed_tools or "Bash(curl:*)", "--disallowedTools", "WebFetch,WebSearch,Task,Read,Write,Edit,Glob,Grep,NotebookEdit,TodoWrite", ] if opts.max_turns: @@ -77,6 +80,7 @@ def run_claude(prompt: str, opts: AgentRunOpts) -> AgentResult: external_fetches=audit["external_fetches"], external_samples=audit["samples"], unaudited_fetches=audit["unaudited"], + commands=commands, ) finally: shutil.rmtree(cwd, ignore_errors=True) @@ -87,7 +91,7 @@ def run_codex(prompt: str, opts: AgentRunOpts) -> AgentResult: args = [ "exec", "--json", "--ephemeral", "--ignore-user-config", "--skip-git-repo-check", "--sandbox", "workspace-write", - "-c", "sandbox_workspace_write.network_access=true", + "-c", f"sandbox_workspace_write.network_access={'true' if opts.network_access else 'false'}", "-c", 'approval_policy="never"', ] if opts.model: @@ -121,6 +125,7 @@ def run_codex(prompt: str, opts: AgentRunOpts) -> AgentResult: external_fetches=audit["external_fetches"], external_samples=audit["samples"], unaudited_fetches=audit["unaudited"], + commands=[str((e.get("item") or {}).get("command", "")) for e in commands], ) finally: shutil.rmtree(cwd, ignore_errors=True)