diff --git a/examples/sandbox_harbor/.gitignore b/examples/sandbox_harbor/.gitignore new file mode 100644 index 0000000..127e60c --- /dev/null +++ b/examples/sandbox_harbor/.gitignore @@ -0,0 +1,12 @@ +# Toolkit + harbor_sandbox staged into a harness build context by build_push.sh +harness/*/_toolkit/ +harness/*/harbor_sandbox/ +# sandboxd health-shim binary is staged from the repo-root sandboxd/, not vendored +harbor_sandbox/wrapper/agentcore-sandboxd-linux-* +# real account-specific values (copy .env.example); never committed +harbor_sandbox/.env +# Run outputs +*.jsonl +# Standalone uv project: users run `uv sync` themselves; examples don't track lockfiles +uv.lock +__pycache__/ diff --git a/examples/sandbox_harbor/README.md b/examples/sandbox_harbor/README.md new file mode 100644 index 0000000..9ee85ac --- /dev/null +++ b/examples/sandbox_harbor/README.md @@ -0,0 +1,86 @@ +# Harbor on AgentCore Runtime + +Harbor support easy evaluation on popular benchmarks on infra such as Daytona, Modal, LangSmith, Blaxel, and Novita Sandbox: +``` +harbor run -d "" -m "" -a "" +``` +This example shows how to enable similar, convenient evaluation on Harbor benchmarks on AgentCore Runtime. + +one command that evaluates a whole [Harbor](https://harborframework.com) benchmark on AgentCore Runtime: + +```bash +uv run python bench.py --benchmark tmax/TMax-15K-Harbor --task-root ./tasks \ + --agent claude-code --model us.anthropic.claude-sonnet-4-6 +``` + +That runs one rollout per task: for each task the agent gets its own fresh +sandbox, does the work, and is graded by the task's own tests. You get a solve +rate and one result record per task. + +## How it fits together + +A Harbor benchmark is a folder of tasks. Each task ships a container image, an +instruction, and hidden tests. Evaluating it on AgentCore Runtime takes three +pieces, all in this folder: + +- **`harbor_sandbox/`** — turns each task into a runnable sandbox. `build.py` + packages every task image and pushes it to ECR; `HarborSandboxClient` creates + (and later removes) a task's runtime on demand. +- **`harness/`** — the agent. Two interchangeable ones: `strands` and + `claude-code` (Claude Code co-located in the box). A harness is deployed once + as a long-lived runtime and reused for every task. +- **`bench.py`** — the entrypoint above. It hands each task to the harness, + collects results, and prints the summary. + +## Setup + +```bash +uv sync # installs this example (and its harbor_sandbox package) into ./.venv +cp harbor_sandbox/.env.example harbor_sandbox/.env # then fill in your account/region/bucket +``` + +`.env` holds your account-specific values (git-ignored, never committed); it is +read by `harbor_sandbox/config.py` and sourced by `harness/build_push.sh`. + +## Steps + +1. **Build the task images** (once per benchmark): + + ```bash + uv run python -m harbor_sandbox.build --task-root ./tasks \ + --benchmark tmax/TMax-15K-Harbor --arch arm64 + ``` + +2. **Deploy an agent harness** (once): + + ```bash + (cd harness && ./build_push.sh claude_code) + uv run python harness/deploy_harness.py --agent claude-code --image-tag claude-code + ``` + +3. **Run the benchmark**: + + ```bash + uv run python bench.py --benchmark tmax/TMax-15K-Harbor --task-root ./tasks \ + --agent claude-code --model us.anthropic.claude-sonnet-4-6 + ``` + +## Runtime + +Tasks run on the serverless arm64 microVM substrate — fast cold start, PUBLIC +network. Build the task images for arm64 (`--arch arm64`); that is the substrate +this example targets end to end. For now, AgentCore Runtime microVM only support arm64, with upcoming support on x86 microVM, we'll update to support once the infra is available. + +## Notes + +- The harness needs no credentials injected: each task runtime uses its own IAM + role. +- By default a task's runtime is removed right after its rollout, so a large + benchmark only ever holds a handful of runtimes at once. Alternatively, one can just raise up AgentCore Runtime quota, and keep the runtime deployed to achieve a faster start. +- Restrict a run with `--tasks` / `--exclude` (a comma list or `@file`) and + `--limit`. +- `bench.py` reads tasks from `--task-root`; if that dir is empty it auto-pulls + them from the Harbor registry, which needs the optional `harbor` package + (`uv sync --extra harbor`). With the tasks already on disk this is skipped. + +Known limits on the terminal-bench-2 benchmark are tracked in [TODO.md](TODO.md). diff --git a/examples/sandbox_harbor/TODO.md b/examples/sandbox_harbor/TODO.md new file mode 100644 index 0000000..262c765 --- /dev/null +++ b/examples/sandbox_harbor/TODO.md @@ -0,0 +1,33 @@ +# TODO — current limits on terminal-bench-2 + +We can run terminal-bench-2 end to end on AgentCore Runtime today (serverless +arm64 microVM). A handful of tasks still don't grade cleanly. These are the +known limits, and none of them is a problem with the agent or the scoring — +they're environment limits. + +## 1. Long silent commands time out + +A few heavy tasks run a single command that works for a long time while printing +nothing (for example, compiling a large library from source). The connection +that carries output has an idle limit, so after enough silence it gives up and +the task is dropped instead of being scored. + +**Fix:** keep the connection alive during silent commands — send a heartbeat, or +start the command in the background and poll it. Not done yet. + +## 2. One command can't run longer than an hour + +A single command in a sandbox is capped at one hour by the service. Tasks that +ask for more are stopped at the cap and graded on whatever they finished. This +mostly overlaps with limit #1. + +## Future: x86 / EC2 + +This example runs arm64-only. An earlier version could also place tasks on an +x86 EC2 capacity provider; it was removed to keep the code simple, since it +didn't improve results and added a lot of complexity (a bare x86 image ships no +`curl`/`python`, and its root is capability-stripped, so the harness had to +inject a static `curl` + CA bundle before anything could run). Re-adding x86 +would also have to handle a per-image size limit — a few x86 images come out +around 6 GB, over the limit, so their runtime can't be created (the same tasks +are smaller and run fine on arm64). diff --git a/examples/sandbox_harbor/bench.py b/examples/sandbox_harbor/bench.py new file mode 100644 index 0000000..bbac034 --- /dev/null +++ b/examples/sandbox_harbor/bench.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +"""bench — one-command evaluation of a Harbor benchmark on AgentCore Runtime. + + python bench.py --benchmark tmax/TMax-15K-Harbor --task-root ./tasks \ + --agent claude-code --model us.anthropic.claude-sonnet-4-6 + +One rollout per task: the payload carries Harbor coordinates (benchmark + +task_id) — the HARNESS creates/ensures the task sandbox itself via +HarborSandboxClient (idempotent), runs the agent, grades with the task's own +shipped verifier, saves the record to S3, and (lease mode, default) removes the +runtime afterwards. No pre-created task runtimes: quota use is transient +(~ n-concurrent), and this launcher only invokes + polls. + +Restrict the task set with --tasks / --exclude (comma list or @file) and --limit. +Results: a jsonl record per task plus a solve-rate summary. +""" +from __future__ import annotations + +import argparse +import base64 +import io +import json +import logging +import re +import sys +import tarfile +import time +from pathlib import Path + +import boto3 +import tomllib +from harbor_sandbox import REGION, S3_BUCKET, ensure_dataset + +from agentcore_rl_toolkit import RolloutClient + +ROOT = Path(__file__).resolve().parent + +# --agent choices. A harness deploys under `harness__v1` (see +# harness/deploy_harness.py); we resolve that name to an ARN at startup. +AGENTS = ("strands", "claude-code") + +# transient infra failures worth ONE re-run. +_TRANSIENT_ERR = re.compile( + r"read timed out|timed out|ConnectionPool|Failed to launch|" + r"ResourceNotFound|No endpoint|no agent found|Throttl|TooManyRequests|" + r"runtimeClientError|ServiceException|InternalServer", + re.I, +) + +_SDK_KEYS = ("status_code", "input_id", "s3_bucket", "result_key", "payload") + + +def harness_arn(agent: str) -> str: + """Resolve --agent to its deployed harness runtime ARN. Deploy the harness + first (see harness/deploy_harness.py).""" + wanted = f"harness_{agent.replace('-', '_')}_v1" + ctrl = boto3.client("bedrock-agentcore-control", region_name=REGION) + kw: dict = {"maxResults": 100} + while True: + resp = ctrl.list_agent_runtimes(**kw) + for r in resp.get("agentRuntimes", []): + if r.get("agentRuntimeName") == wanted: + return r["agentRuntimeArn"] + kw["nextToken"] = resp.get("nextToken") + if not kw["nextToken"]: + break + raise SystemExit( + f"no harness runtime named {wanted!r} for agent {agent!r} " f"— deploy it first (harness/deploy_harness.py)" + ) + + +def tests_tar_b64(tests_dir: Path) -> str: + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w:gz") as tf: + for p in sorted(tests_dir.rglob("*")): + tf.add(p, arcname=str(p.relative_to(tests_dir))) + return base64.b64encode(buf.getvalue()).decode() + + +# Fallback budgets for tasks whose task.toml omits a per-task timeout. Threading +# the task's own [agent]/[verifier].timeout_sec into the harness reproduces the +# leaderboard's per-task AgentTimeout semantics (stop at the budget, still grade +# partial work); these defaults only apply when a task declares neither. +DEFAULT_AGENT_TIMEOUT_S = 3300.0 +DEFAULT_VERIFIER_TIMEOUT_S = 900.0 +# head-room the OUTER client wedge gets over the largest per-task budget, so the +# faithful per-task budget always fires first and the wedge only catches a hang. +WEDGE_MARGIN_S = 900.0 + + +def _task_section_timeout_s(task_dir: Path, section: str, default: float) -> float: + """`[
].timeout_sec` from the task's `task.toml`, else `default`.""" + toml = task_dir / "task.toml" + if toml.exists(): + try: + v = (tomllib.loads(toml.read_text()).get(section) or {}).get("timeout_sec") + if v: + return float(v) + except Exception: + pass + return default + + +def build_payload(task_dir: Path, args) -> dict: + return { + "benchmark": args.benchmark, + "task_id": task_dir.name, + "lease": not args.keep_runtimes, + "instruction": (task_dir / "instruction.md").read_text(), + "tests_tar_b64": tests_tar_b64(task_dir / "tests"), + "model": args.model, + "max_steps": args.max_steps, + "agent_timeout_s": _task_section_timeout_s(task_dir, "agent", args.agent_timeout_default), + "verifier_timeout_s": _task_section_timeout_s(task_dir, "verifier", DEFAULT_VERIFIER_TIMEOUT_S), + } + + +def normalize(item) -> dict: + if not item.success: + return {"error": str(item.error)[:300]} + doc = item.result + if doc.get("status_code") == 500: + return {"error": str(doc.get("stop_reason", ""))[:300], "traceback": str(doc.get("traceback", ""))[:1500]} + return {k: v for k, v in doc.items() if k not in _SDK_KEYS} + + +def run_jobs(client, jobs, n_concurrent, timeout) -> list: + """jobs = [(task_id, payload)]; returns records in job order, prints live.""" + records = {} + for item in client.run_batch([p for _, p in jobs], max_concurrent_sessions=n_concurrent, timeout=timeout): + tid = jobs[item.index][0] + rec = normalize(item) + rec.update({"task_id": tid, "elapsed_s": round(item.elapsed or 0.0, 1)}) + records[item.index] = rec + n = len(records) + if "error" in rec: + print(f" [{n}/{len(jobs)}] {tid} ERROR ({rec['elapsed_s']}s): {rec['error'][:90]}") + elif n % 25 == 0 or n <= 3: + print( + f" [{n}/{len(jobs)}] {tid} reward={rec.get('reward')} " + f"steps={rec.get('steps')} ({rec['elapsed_s']}s)" + ) + return [records[i] for i in sorted(records)] + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--benchmark", required=True, help="Harbor benchmark identifier ('org/name')") + p.add_argument( + "--task-root", + required=True, + dest="task_root", + help="Harbor dataset dir whose immediate subdirectories ARE the " + "tasks (each with instruction.md, environment/, tests/, " + "task.toml). If empty, the tasks are pulled from the Harbor " + "registry (needs the 'harbor' package; see README).", + ) + p.add_argument("--agent", default="claude-code", choices=sorted(AGENTS)) + p.add_argument("--model", default="us.anthropic.claude-sonnet-4-6") + p.add_argument("--n-concurrent", type=int, default=48, dest="n_concurrent") + p.add_argument("--max-steps", type=int, default=100, dest="max_steps") + p.add_argument( + "--timeout", + type=float, + default=None, + help="OUTER client wedge deadline per rollout. Default: auto = " + "max per-task agent budget + margin, so the faithful " + "per-task budget (task.toml [agent].timeout_sec, enforced " + "in the harness) always fires first.", + ) + + p.add_argument( + "--agent-timeout-default", + type=float, + default=DEFAULT_AGENT_TIMEOUT_S, + dest="agent_timeout_default", + help="agent wall-clock budget for tasks whose task.toml omits " "[agent].timeout_sec (e.g. the tmax corpus)", + ) + p.add_argument( + "--keep-runtimes", + action="store_true", + dest="keep_runtimes", + help="leave each task's runtime alive after its rollout " "(default: delete it, so quota use stays transient)", + ) + p.add_argument("--tasks", default="", help="comma list or @file to restrict") + p.add_argument("--exclude", default="", help="comma list or @file to skip") + p.add_argument("--limit", type=int, default=0, help="first N tasks (0 = all)") + p.add_argument("--exp-id", default=None, dest="exp_id") + p.add_argument("--out", default=str(ROOT / "bench_results.jsonl")) + args = p.parse_args() + sys.stdout.reconfigure(line_buffering=True) + logging.basicConfig(level=logging.WARNING, format="%(message)s") + logging.getLogger("agentcore_rl_toolkit.client").setLevel(logging.INFO) + + # ---- task scope --------------------------------------------------------- + def idset(spec): + if not spec: + return set() + return set(Path(spec[1:]).read_text().split() if spec.startswith("@") else spec.split(",")) + + task_root = Path(args.task_root) + ensure_dataset(args.benchmark, task_root) # pull tasks if not already on disk + only, excl = idset(args.tasks), idset(args.exclude) + tasks = [ + d + for d in sorted(task_root.iterdir()) + if d.is_dir() and (d / "instruction.md").exists() and d.name not in excl and (not only or d.name in only) + ] + if args.limit: + tasks = tasks[: args.limit] + + print( + f"bench: {len(tasks)} tasks | benchmark={args.benchmark} agent={args.agent} " + f"model={args.model} lease={not args.keep_runtimes} " + f"n-concurrent={args.n_concurrent}" + ) + + t_payload = time.time() + jobs = [(d.name, build_payload(d, args)) for d in tasks] + print(f"payloads built in {time.time()-t_payload:.0f}s") + + # per-task agent budgets (faithful, enforced harness-side); size the OUTER + # client wedge above the largest so the per-task budget always fires first. + budgets = sorted(p["agent_timeout_s"] for _, p in jobs) or [DEFAULT_AGENT_TIMEOUT_S] + print( + f"agent budgets (task.toml [agent].timeout_sec): min {budgets[0]:.0f}s " + f"p50 {budgets[len(budgets)//2]:.0f}s max {budgets[-1]:.0f}s" + ) + if args.timeout is None: + args.timeout = budgets[-1] + WEDGE_MARGIN_S + print( + f"outer wedge auto-set to {args.timeout:.0f}s " + f"(max budget {budgets[-1]:.0f}s + {WEDGE_MARGIN_S:.0f}s margin)" + ) + elif args.timeout <= budgets[-1]: + print( + f"WARNING: --timeout {args.timeout:.0f}s <= max per-task budget " + f"{budgets[-1]:.0f}s — the client wedge may fire before the faithful " + f"per-task budget, abandoning rollouts as errors instead of grading them" + ) + + exp_id = args.exp_id or f"harbor-bench/{time.strftime('%Y%m%d-%H%M%S', time.gmtime())}" + client = RolloutClient( + agent_runtime_arn=harness_arn(args.agent), + s3_bucket=S3_BUCKET, + exp_id=exp_id, + max_pool_connections=max(10, args.n_concurrent), + ) + + # ---- run (wall measured) ------------------------------------------------ + t0 = time.time() + records = run_jobs(client, jobs, args.n_concurrent, args.timeout) + + retry_idx = [i for i, r in enumerate(records) if "error" in r and _TRANSIENT_ERR.search(r["error"])] + if retry_idx: + print(f"\nretrying {len(retry_idx)} transient failure(s) ...") + for i, rec in zip( + retry_idx, run_jobs(client, [jobs[i] for i in retry_idx], args.n_concurrent, args.timeout), strict=True + ): + records[i] = rec + wall = time.time() - t0 + + # ---- results + summary -------------------------------------------------- + Path(args.out).write_text("\n".join(json.dumps(r) for r in records) + "\n") + ok = [r for r in records if "error" not in r] + solved = [r for r in ok if r.get("reward") == 1] + el = sorted(r["elapsed_s"] for r in ok) or [0] + + print("\n================ bench summary ================") + print( + f"wall: {wall/3600:.2f} h ({wall:.0f}s) for {len(records)} tasks " + f"@ n-concurrent {args.n_concurrent} ({len(records)/max(wall/3600,1e-9):.0f} tasks/h)" + ) + print(f"graded: {len(ok)}/{len(records)} errors: {len(records)-len(ok)}") + print(f"solve: {len(solved)}/{len(ok)} ({100*len(solved)/max(1,len(ok)):.1f}%)") + print(f"rollout p50 {el[len(el)//2]:.0f}s p90 {el[int(0.9*(len(el)-1))]:.0f}s max {el[-1]:.0f}s") + print(f"records -> {args.out} | S3 -> s3://{S3_BUCKET}/{exp_id}/") + + +if __name__ == "__main__": + main() diff --git a/examples/sandbox_harbor/harbor_sandbox/.env.example b/examples/sandbox_harbor/harbor_sandbox/.env.example new file mode 100644 index 0000000..62c148f --- /dev/null +++ b/examples/sandbox_harbor/harbor_sandbox/.env.example @@ -0,0 +1,15 @@ +# Copy this file to `.env` (git-ignored) and fill in your own deployment values, +# so the real account-specific identifiers never land in the committed repo. +# Loaded by harbor_sandbox/config.py and sourced by harness/build_push.sh. +# Format: one KEY=value per line, no inline comments (keeps the parser + shell +# `source` trivial); put any note on its own # line above the value. + +REGION=us-west-2 +# your AWS account id +ACCOUNT=123456789012 +# bucket for rollout records + run outputs +S3_BUCKET=my-rollout-bucket +# runtime execution role name (lives in ACCOUNT) +ROLE_NAME=my-runtime-role +# ECR repo holding the harness images +HARNESS_REPO=my-harness-repo diff --git a/examples/sandbox_harbor/harbor_sandbox/__init__.py b/examples/sandbox_harbor/harbor_sandbox/__init__.py new file mode 100644 index 0000000..b5a6b1d --- /dev/null +++ b/examples/sandbox_harbor/harbor_sandbox/__init__.py @@ -0,0 +1,27 @@ +"""Harbor-on-AgentCore glue: naming convention + sandbox lifecycle + image build. + +Built on the toolkit's generic ``SandboxClient`` (session API), adding the +Harbor-benchmark-specific half: one naming convention (``resolve``), a client +that creates/releases a task's runtime on demand (``HarborSandboxClient``), a +builder that turns a Harbor dataset into runnable images (``build``), and a +puller that fetches a benchmark's tasks from the Harbor registry +(``ensure_dataset``). Deployment constants live in ``config``; shared exceptions +in ``errors``. +""" + +from .client import HarborSandboxClient +from .config import REGION, S3_BUCKET +from .dataset import ensure_dataset +from .errors import ImageNotFoundError, ValidationError +from .naming import SandboxNames, resolve + +__all__ = [ + "resolve", + "SandboxNames", + "ensure_dataset", + "ValidationError", + "REGION", + "S3_BUCKET", + "HarborSandboxClient", + "ImageNotFoundError", +] diff --git a/examples/sandbox_harbor/harbor_sandbox/build.py b/examples/sandbox_harbor/harbor_sandbox/build.py new file mode 100644 index 0000000..782cfb5 --- /dev/null +++ b/examples/sandbox_harbor/harbor_sandbox/build.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +"""Build a Harbor task or dataset into ECR under the ratified naming scheme. + +Benchmark-agnostic: every image URI comes from naming.resolve, so the SAME +builder serves tmax, terminal-bench-2, and any future org/name dataset. The +build is deliberately minimal — the task's ORIGINAL environment/ image, then a +COPY-only wrapper that adds just the sandboxd binary (the AgentCore runtime +contract: port 8080, no logic). Nothing else is injected, so the numbers +reflect the most original image. COPY-only means the wrap stage needs no qemu; +the env stage still runs the task's own Dockerfile (RUN lines there need qemu +when cross-building arm64-on-x86). + +Idempotent by ECR tag (existing tags are listed up front and skipped), so a run +is safely resumable. Drive a remote NATIVE builder with DOCKER_HOST=ssh://host +(docker runs there; ECR auth stays client-side, so the builder needs no IAM). + + python -m harbor_sandbox.build \ + --task-root /tmp/tb2/terminal-bench-2 \ + --benchmark terminal-bench/terminal-bench-2 --arch arm64 +""" +from __future__ import annotations + +import argparse +import concurrent.futures as cf +import json +import shutil +import subprocess +import sys +import time +from pathlib import Path + +from .config import REGION +from .naming import resolve + +WRAPPER = Path(__file__).resolve().parent / "wrapper" +# Single source of truth for the health-shim binary is the repo-root sandboxd/ +# (source + build.sh + prebuilt dist/); we stage from it rather than vendor a copy. +SANDBOXD = Path(__file__).resolve().parents[3] / "sandboxd" + + +def ensure_sandboxd(arch: str) -> None: + """Stage the sandboxd health-shim binary into the wrapper build context. + + Reuses the repo's prebuilt ``sandboxd/dist/`` binary if present, else builds + it via ``sandboxd/build.sh`` (local Go toolchain or a golang container). The + binary is git-ignored here — never vendored.""" + binary = f"agentcore-sandboxd-linux-{arch}" + dst = WRAPPER / binary + if dst.exists(): + return + src = SANDBOXD / "dist" / binary + if src.exists(): + shutil.copy2(src, dst) + else: + subprocess.run([str(SANDBOXD / "build.sh"), "--arch", arch, "--stage", str(WRAPPER)], check=True) + + +def sh(cmd: list[str]) -> tuple[int, str]: + p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + return p.returncode, p.stdout + + +def ecr_login(benchmark: str, arch: str) -> tuple[str, set[str]]: + """Ensure the repo exists + docker login; return (repo, existing tags).""" + n = resolve(benchmark, "_probe_", arch=arch) + repo, registry = n.ecr_repo, n.image_uri.split("/")[0] + subprocess.run( + ["aws", "ecr", "describe-repositories", "--region", REGION, "--repository-names", repo], capture_output=True + ).returncode == 0 or subprocess.run( + ["aws", "ecr", "create-repository", "--region", REGION, "--repository-name", repo], capture_output=True + ) + pw = subprocess.run( + ["aws", "ecr", "get-login-password", "--region", REGION], capture_output=True, text=True, check=True + ).stdout.strip() + subprocess.run( + ["docker", "login", "--username", "AWS", "--password-stdin", registry], + input=pw, + text=True, + check=True, + capture_output=True, + ) + tags, token = set(), None + while True: + cmd = [ + "aws", + "ecr", + "list-images", + "--region", + REGION, + "--repository-name", + repo, + "--max-results", + "1000", + "--query", + "{t:imageIds[].imageTag,n:nextToken}", + "--output", + "json", + ] + if token: + cmd += ["--next-token", token] + out = json.loads(subprocess.run(cmd, capture_output=True, text=True, check=True).stdout) + tags.update(x for x in (out.get("t") or []) if x) + token = out.get("n") + if not token: + return repo, tags + + +def build_task(task_dir: Path, benchmark: str, *, arch: str = "arm64", push: bool = True) -> tuple[str, str]: + """Build one task image (original env stage + COPY-only sandboxd wrap) and push.""" + tid = task_dir.name + env_ctx = task_dir / "environment" + if not (env_ctx / "Dockerfile").exists(): + return tid, "skip: no environment/Dockerfile" + uri = resolve(benchmark, tid, arch=arch).image_uri + plat = ["--platform", f"linux/{arch}", "--provenance=false"] + env_tag = f"harbor-env-{arch}:{tid}" + + rc, out = sh(["docker", "build", *plat, "-t", env_tag, str(env_ctx)]) + if rc: + return tid, f"env build failed: {out[-250:]}" + + wrap = [ + "docker", + "build", + *plat, + "--build-arg", + f"BASE={env_tag}", + "--build-arg", + f"SANDBOXD=agentcore-sandboxd-linux-{arch}", + "-t", + uri, + str(WRAPPER), + ] + rc, out = sh(wrap) + if rc: + return tid, f"wrap failed: {out[-250:]}" + + if push: + rc, out = sh(["docker", "push", uri]) + if rc: + return tid, f"push failed: {out[-250:]}" + sh(["docker", "rmi", "-f", env_tag, uri]) # drop tags, keep shared layer cache + return tid, "ok" + + +def _idset(spec: str) -> set[str]: + if not spec: + return set() + return set(Path(spec[1:]).read_text().split() if spec.startswith("@") else spec.split(",")) + + +def main(argv=None): + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + p.add_argument("--task-root", required=True, help="Harbor dataset dir (task dirs inside)") + p.add_argument("--benchmark", required=True, help="Harbor org/name identifier") + p.add_argument("--arch", default="arm64", choices=["arm64"]) + p.add_argument("--tasks", default="", help="restrict to these task ids: comma list or @file") + p.add_argument("--exclude", default="", help="skip these task ids: comma list or @file") + p.add_argument("--limit", type=int, default=0, help="first N tasks (0 = all)") + p.add_argument("--concurrency", type=int, default=8) + p.add_argument("--no-skip-existing", action="store_true", help="rebuild even if the ECR tag exists") + args = p.parse_args(argv) + sys.stdout.reconfigure(line_buffering=True) + ensure_sandboxd(args.arch) # stage the wrap-stage binary from repo-root sandboxd/ + + root = Path(args.task_root) + only, excl = _idset(args.tasks), _idset(args.exclude) + tasks = sorted( + d + for d in root.iterdir() + if d.is_dir() + and (d / "environment" / "Dockerfile").exists() + and d.name not in excl + and (not only or d.name in only) + ) + if args.limit: + tasks = tasks[: args.limit] + + repo, existing = ecr_login(args.benchmark, args.arch) + if not args.no_skip_existing: + skip = {resolve(args.benchmark, d.name, arch=args.arch).image_tag for d in tasks} & existing + tasks = [d for d in tasks if resolve(args.benchmark, d.name, arch=args.arch).image_tag not in existing] + else: + skip = set() + print( + f"benchmark={args.benchmark} repo={repo} arch={args.arch} " + f"todo={len(tasks)} skip-existing={len(skip)} concurrency={args.concurrency}", + flush=True, + ) + + t0, n, fails = time.time(), 0, [] + with cf.ThreadPoolExecutor(max_workers=args.concurrency) as ex: + futs = [ex.submit(build_task, d, args.benchmark, arch=args.arch) for d in tasks] + for fut in cf.as_completed(futs): + tid, status = fut.result() + n += 1 + if status != "ok": + fails.append((tid, status)) + if n % 10 == 0 or status != "ok": + rate = n / max(1e-9, time.time() - t0) * 3600 + print( + f" [{n}/{len(tasks)}] {tid:32} {status[:60]} " + f"({rate:.0f}/h, eta {(len(tasks)-n)/max(1e-9,rate):.1f}h)", + flush=True, + ) + + print(f"\ndone: {len(tasks)-len(fails)} ok, {len(fails)} failed in {(time.time()-t0)/60:.1f}m") + for tid, s in fails[:20]: + print(f" FAIL {tid}: {s}") + return 1 if fails else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/sandbox_harbor/harbor_sandbox/client.py b/examples/sandbox_harbor/harbor_sandbox/client.py new file mode 100644 index 0000000..431020a --- /dev/null +++ b/examples/sandbox_harbor/harbor_sandbox/client.py @@ -0,0 +1,143 @@ +"""HarborSandboxClient — SandboxClient plus Harbor-aware runtime lifecycle. + +Subclass of the toolkit's ``SandboxClient`` (session API: start/exec/stop) that +adds create/release for a task's AgentCore runtime: + + sb = HarborSandboxClient.create("tmax/TMax-15K-Harbor", "task_000606_03976796") + with sb.start() as s: + s.exec("uname -m") + sb.release() # delete THIS client's runtime (the lease pattern) + +Self-contained: no local corpus or config files. A task's existence and +built-ness are answered by ONE remote source of truth — the conventional ECR +tag (see ``naming.resolve``). ``create`` is idempotent (create-if-missing / +reuse-if-present) and does NOT build images: a missing tag raises +``ImageNotFoundError``. Runtime naming: +``sb__``. +""" +from __future__ import annotations + +import logging +import time +import uuid + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + +from agentcore_rl_toolkit.sandbox import SandboxClient + +from .config import IDLE_SESSION_TIMEOUT_S, MAX_LIFETIME_S, NETWORK_CONFIG, REGION, ROLE_ARN +from .errors import ImageNotFoundError +from .naming import resolve + +logger = logging.getLogger(__name__) + + +def _control(region: str = REGION): + return boto3.client( + "bedrock-agentcore-control", region_name=region, config=Config(retries={"max_attempts": 10, "mode": "adaptive"}) + ) + + +def _find_runtime_id(ctrl, name: str) -> str | None: + kw: dict = {"maxResults": 100} + while True: + resp = ctrl.list_agent_runtimes(**kw) + for r in resp.get("agentRuntimes", []): + if r.get("agentRuntimeName") == name: + return r["agentRuntimeId"] + kw["nextToken"] = resp.get("nextToken") + if not kw["nextToken"]: + return None + + +class HarborSandboxClient(SandboxClient): + """SandboxClient that also owns runtime lifecycle for Harbor benchmarks. + Instantiable anywhere with AWS credentials — dev host, harness container, CI + — since all validation runs against ECR + the AgentCore control plane.""" + + @classmethod + def create( + cls, + benchmark: str, + task_id: str, + arch: str = "arm64", + wait_ready_s: int = 180, + unique: bool = False, + **client_kwargs, + ) -> "HarborSandboxClient": + """Ensure the task's runtime exists and is READY; return a client bound + to it (identical whether it created the runtime or reused a live one). + + ``unique=True`` creates a PRIVATE runtime (name gets a random suffix) + instead of the shared per-task one. Required whenever the caller will + ``release()`` while concurrent rollouts of the same task may be live + (e.g. GRPO groups): releasing the shared deterministic name deletes the + runtime out from under the siblings. + + Raises ImageNotFoundError if the conventional ECR tag is absent. + """ + names = resolve(benchmark, task_id, arch=arch, suffix=uuid.uuid4().hex[:8] if unique else None) + + # the ECR tag is the single source of truth for "this task exists AND is + # built" (task existence + built-ness in one check). + ecr = boto3.client("ecr", region_name=REGION) + try: + ecr.describe_images(repositoryName=names.ecr_repo, imageIds=[{"imageTag": names.image_tag}]) + except (ecr.exceptions.ImageNotFoundException, ecr.exceptions.RepositoryNotFoundException): + raise ImageNotFoundError( + f"{names.image_uri} not in ECR — unknown task or not built " f"for {arch}" + ) from None + + body = dict( + agentRuntimeArtifact={"containerConfiguration": {"containerUri": names.image_uri}}, + roleArn=ROLE_ARN, + protocolConfiguration={"serverProtocol": "HTTP"}, + lifecycleConfiguration={"idleRuntimeSessionTimeout": IDLE_SESSION_TIMEOUT_S, "maxLifetime": MAX_LIFETIME_S}, + **NETWORK_CONFIG, + ) + name = names.runtime_name + ctrl = _control() + deadline = time.time() + wait_ready_s + + # create-or-reuse: a lost create race, or a create right after release() + # (same name still DELETING), both resolve to one live runtime. + while True: + try: + arn = ctrl.create_agent_runtime(agentRuntimeName=name, **body)["agentRuntimeArn"] + logger.info(f"created runtime {name}") + break + except ClientError as e: + if e.response.get("Error", {}).get("Code") != "ConflictException": + raise + rid = _find_runtime_id(ctrl, name) + info = ctrl.get_agent_runtime(agentRuntimeId=rid) if rid else None + if info and info["status"] != "DELETING": + arn = info["agentRuntimeArn"] + break + if time.time() > deadline: + raise RuntimeError(f"runtime {name}: create conflict unresolved " f"after {wait_ready_s}s") + time.sleep(3) + + # wait for control-plane READY (seconds on both substrates) + rid = arn.split("/")[-1] + while ctrl.get_agent_runtime(agentRuntimeId=rid)["status"] != "READY": + if time.time() > deadline: + raise RuntimeError(f"runtime {name} not READY after {wait_ready_s}s") + time.sleep(3) + return cls(runtime_arn=arn, **client_kwargs) + + def release(self) -> bool: + """Delete THIS client's runtime (the lease pattern: the object knows its + own ARN, so no coordinates are re-derived). Idempotent — returns False if + the runtime was already gone or still mid-deletion.""" + ctrl = _control(self._parse_region_from_arn(self.runtime_arn)) + try: + ctrl.delete_agent_runtime(agentRuntimeId=self.runtime_arn.split("/")[-1]) + return True + except ClientError as e: + code = e.response.get("Error", {}).get("Code") + if code == "ResourceNotFoundException" or (code == "ConflictException" and "DELETING" in str(e)): + return False + raise diff --git a/examples/sandbox_harbor/harbor_sandbox/config.py b/examples/sandbox_harbor/harbor_sandbox/config.py new file mode 100644 index 0000000..fe31e24 --- /dev/null +++ b/examples/sandbox_harbor/harbor_sandbox/config.py @@ -0,0 +1,48 @@ +"""Deployment constants for the Harbor-on-AgentCore example — one source of truth. + +Account-specific identifiers (account id, role, bucket) are NOT committed: the +values below are placeholders, overridden at import time from a git-ignored +``.env`` beside this file (copy ``.env.example`` and fill it in). That +``.env`` is staged into the harness image by build_push.sh, so the same values +reach the in-container code, and build_push.sh sources it too — one file feeds +both shell and Python. Everything here is deployment-level, NOT per-benchmark: +benchmark-specific names are DERIVED in ``naming.resolve()``. +""" +from __future__ import annotations + +import os +from pathlib import Path + + +def _load_env(path: Path) -> None: + """Minimal .env reader: ``KEY=value`` lines, ``#`` comment lines, optional + quotes. Real environment variables win (``setdefault``), so an exported var + or deploy-time injection overrides the file.""" + if not path.exists(): + return + for line in path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, val = line.split("=", 1) + os.environ.setdefault(key.strip(), val.strip().strip('"').strip("'")) + + +_load_env(Path(__file__).with_name(".env")) + +# -- overridable scalars (placeholders until .env / the environment supplies them) -- +REGION = os.environ.get("REGION", "us-west-2") +ACCOUNT = os.environ.get("ACCOUNT", "000000000000") +S3_BUCKET = os.environ.get("S3_BUCKET", "your-rollout-bucket") # rollout records + run outputs +ROLE_NAME = os.environ.get("ROLE_NAME", "your-runtime-role") # runtime execution role (in ACCOUNT) +HARNESS_REPO = os.environ.get("HARNESS_REPO", "your-harness-repo") # ECR repo for the harness images +IDLE_SESSION_TIMEOUT_S = 900 # runtime lifecycle: idle -> stop +MAX_LIFETIME_S = 28800 # runtime lifecycle: hard cap (8h) + +# -- derived (built from the resolved values above) -- +ECR_REGISTRY = f"{ACCOUNT}.dkr.ecr.{REGION}.amazonaws.com" +ROLE_ARN = f"arn:aws:iam::{ACCOUNT}:role/{ROLE_NAME}" + +# Network config for the serverless arm64 microVM runtime, passed to +# create_agent_runtime. +NETWORK_CONFIG = {"networkConfiguration": {"networkMode": "PUBLIC"}} diff --git a/examples/sandbox_harbor/harbor_sandbox/dataset.py b/examples/sandbox_harbor/harbor_sandbox/dataset.py new file mode 100644 index 0000000..938d7e9 --- /dev/null +++ b/examples/sandbox_harbor/harbor_sandbox/dataset.py @@ -0,0 +1,60 @@ +"""ensure_dataset — pull a Harbor benchmark's tasks to a local dir on demand. + +A Harbor benchmark ("org/name") is a dataset in the hosted Harbor registry +(hub.harborframework.com, public read). ``ensure_dataset`` downloads it into a +task_root where each task is a subdirectory (instruction.md, environment/, +tests/, task.toml) — the layout both the image builder and bench.py read. + +Runs launcher-side, never in the harness container (the container gets a task's +instruction + tests via the rollout payload, and its image from ECR). Needs the +``harbor`` package, imported lazily so importing this module stays cheap. +""" +from __future__ import annotations + +import logging +from pathlib import Path + +logger = logging.getLogger(__name__) + + +def _local_task_count(task_root: Path) -> int: + if not task_root.is_dir(): + return 0 + return sum(1 for d in task_root.iterdir() if (d / "instruction.md").is_file()) + + +def ensure_dataset(benchmark: str, task_root: str | Path, *, ref: str = "latest", overwrite: bool = False) -> Path: + """Ensure benchmark's tasks live under task_root (one subdir per task), + pulling them from the Harbor registry if absent. Returns task_root. + + Idempotent: when task_root already holds tasks the download is skipped + entirely (pass overwrite=True to force a re-pull). ``ref`` selects a version + ("latest", a tag, or "sha256:..."). + """ + task_root = Path(task_root) + have = _local_task_count(task_root) + if have and not overwrite: + logger.info(f"{benchmark}: {have} tasks already in {task_root}") + return task_root + + try: + from harbor.cli.utils import run_async + from harbor.registry.client.package import PackageDatasetClient + except ImportError as e: + raise RuntimeError( + "ensure_dataset needs the 'harbor' package importable on the launcher " + "host — install it into this venv: `uv sync --extra harbor` (or " + "`uv pip install harbor`)" + ) from e + + task_root.mkdir(parents=True, exist_ok=True) + logger.info(f"pulling {benchmark}@{ref} -> {task_root}") + # download the client directly (not the CLI) so tasks land straight under + # task_root without the CLI's extra / wrapper directory. + items = run_async( + PackageDatasetClient().download_dataset( + f"{benchmark}@{ref}", overwrite=overwrite, output_dir=task_root, export=True + ) + ) + logger.info(f"{benchmark}: {len(items)} tasks ready in {task_root}") + return task_root diff --git a/examples/sandbox_harbor/harbor_sandbox/errors.py b/examples/sandbox_harbor/harbor_sandbox/errors.py new file mode 100644 index 0000000..f3fda99 --- /dev/null +++ b/examples/sandbox_harbor/harbor_sandbox/errors.py @@ -0,0 +1,11 @@ +"""Exceptions raised by harbor_sandbox — gathered here so every module's failure +modes are visible in one place.""" +from __future__ import annotations + + +class ValidationError(ValueError): + """Malformed benchmark id ('org/name' expected).""" + + +class ImageNotFoundError(RuntimeError): + """No conventional ECR tag for (task, arch) — task unknown or not built yet.""" diff --git a/examples/sandbox_harbor/harbor_sandbox/naming.py b/examples/sandbox_harbor/harbor_sandbox/naming.py new file mode 100644 index 0000000..0f1e5bd --- /dev/null +++ b/examples/sandbox_harbor/harbor_sandbox/naming.py @@ -0,0 +1,59 @@ +"""resolve() maps a (benchmark, task_id) to the URIs that locate its resources — +the ECR image and the AgentCore runtime — so any script can check whether a +resource exists and address it by benchmark + task id. + +AgentCore runtime names must be < 40 chars, so the runtime name is a fixed-length +hash of the image URI rather than the raw ``-``; that rule +lives here so callers never synthesize names by hand. + + >>> n = resolve("tmax/TMax-15K-Harbor", "task_000606_03976796") + >>> n.image_uri # ...amazonaws.com/harbor_bench/tmax/tmax-15k-harbor:task_000606_03976796-arm64 + >>> n.runtime_name # sb_tmax15kh_39102efe2637 + +Convention: repo = harbor_bench// lowercased (ECR forbids uppercase); +tag = -; runtime = sb__. +""" +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass + +from .config import ECR_REGISTRY +from .errors import ValidationError + + +@dataclass(frozen=True) +class SandboxNames: + benchmark: str + task_id: str + arch: str + ecr_repo: str + image_tag: str + image_uri: str + runtime_name: str + + +def resolve(benchmark: str, task_id: str, *, arch: str = "arm64", suffix: str | None = None) -> SandboxNames: + """Map (benchmark, task_id) to its ECR image + AgentCore runtime names. + + ``benchmark`` ('org/name') is taken verbatim — split only to build the repo + path, never matched against a list. Whether it actually EXISTS is Harbor + Hub's call (ensure_dataset pulls it and fails if unknown), so there is one + source of truth; here we only check it parses as 'org/name'. + + ``suffix`` appends a per-caller discriminator to the runtime name + (``sb___``): concurrent rollouts of the SAME task + (e.g. a GRPO group) each get a private runtime instead of colliding on the + deterministic name. The image URIs are unchanged. + """ + org, _, name = benchmark.partition("/") + if not org or not name or "/" in name: + raise ValidationError(f"{benchmark!r} is not a Harbor id ('org/name')") + repo = f"harbor_bench/{org.lower()}/{name.lower()}" + uri = f"{ECR_REGISTRY}/{repo}:{task_id}-{arch}" + code = re.sub(r"[^a-z0-9]", "", name.lower())[:8] or "bench" + runtime = f"sb_{code}_{hashlib.sha256(uri.encode()).hexdigest()[:12]}" + if suffix: + runtime = f"{runtime}_{re.sub(r'[^a-zA-Z0-9_]', '', suffix)[:8]}" + return SandboxNames(benchmark, task_id, arch, repo, f"{task_id}-{arch}", uri, runtime) diff --git a/examples/sandbox_harbor/harbor_sandbox/wrapper/Dockerfile b/examples/sandbox_harbor/harbor_sandbox/wrapper/Dockerfile new file mode 100644 index 0000000..2b95881 --- /dev/null +++ b/examples/sandbox_harbor/harbor_sandbox/wrapper/Dockerfile @@ -0,0 +1,12 @@ +# Sandbox wrapper: layer the AgentCore sandboxd health shim onto a task env image. +# COPY-only (no RUN) -> no shell execution -> no qemu even when cross-building. +# SANDBOXD selects the shim arch (arm64 for the serverless microVM runtime); +# pass the matching --platform too. build.py always sets this explicitly. +ARG BASE +FROM ${BASE} + +ARG SANDBOXD=agentcore-sandboxd-linux-arm64 +COPY --chmod=0755 ${SANDBOXD} /opt/agentcore-sandbox/agentcore-sandboxd + +EXPOSE 8080 +CMD ["/opt/agentcore-sandbox/agentcore-sandboxd"] diff --git a/examples/sandbox_harbor/harness/build_push.sh b/examples/sandbox_harbor/harness/build_push.sh new file mode 100755 index 0000000..859558c --- /dev/null +++ b/examples/sandbox_harbor/harness/build_push.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Build ONE harness image and push to ECR as $HARNESS_REPO: (repo from .env). +# +# build_push.sh strands -> $HARNESS_REPO:strands +# build_push.sh claude_code -> $HARNESS_REPO:claude-code +# build_push.sh all -> both +# +# Each harness is a (Dockerfile + harness.py) pair in its own subdirectory; the +# build stages in the toolkit source and the local harbor_sandbox package (both +# imported by harness.py). All build for linux/arm64 (the default serverless +# MicroVM runtime); qemu binfmt handles the cross-build on this x86 host — the +# images are pure Python so emulated builds are fine. +# +# Optional 3rd arg overrides the image tag (default: the agent name). Use a +# versioned tag (e.g. claude-code-v1) to publish a NEW harness build WITHOUT +# clobbering the tag a live runtime is pulling from. +# build_push.sh claude_code linux/arm64 claude-code-v1 -> $HARNESS_REPO:claude-code-v1 +set -euo pipefail + +HARNESS=${1:?usage: build_push.sh [platform] [tag]} +PLATFORM=${2:-linux/arm64} +TAG_OVERRIDE=${3:-} + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # examples/sandbox_harbor/harness +EXAMPLE="$(cd "$HERE/.." && pwd)" # examples/sandbox_harbor +TOOLKIT_SRC="$(cd "$HERE/../../.." && pwd)" # repo root (this example lives inside it) + +# Same .env the Python config reads (no hand-synced duplicate). Full-line comments +# + KEY=value only, so it sources cleanly. REPO mirrors config's HARNESS_REPO. +ENV_FILE="$EXAMPLE/harbor_sandbox/.env" +[ -f "$ENV_FILE" ] || { echo "missing $ENV_FILE (copy harbor_sandbox/.env.example)"; exit 1; } +# shellcheck disable=SC1090 +source "$ENV_FILE" +REPO="$HARNESS_REPO" +REGISTRY="${ACCOUNT}.dkr.ecr.${REGION}.amazonaws.com" + +if [ "$HARNESS" = "all" ]; then + for h in strands claude_code; do + "$0" "$h" "$PLATFORM" + done + exit 0 +fi +[ -d "$HERE/$HARNESS" ] || { echo "unknown harness '$HARNESS'"; exit 1; } + +TAG="${TAG_OVERRIDE:-${HARNESS//_/-}}" # default claude_code -> claude-code; override for versioned builds +IMAGE="${REGISTRY}/${REPO}:${TAG}" +CTX="$HERE/$HARNESS" + +# Stage clean copies of the toolkit source + the local harbor_sandbox package +# into the build context (no venv/git/caches). harbor_sandbox drops the wrapper/ +# sandboxd binaries — those are for the image builder, not the harness runtime. +# NB: rsync ignores .gitignore, so the git-ignored .env IS copied in — intentional: +# it's how the harness container gets the real account/region. +echo ">> staging toolkit source into $HARNESS/_toolkit" +rm -rf "$CTX/_toolkit" "$CTX/harbor_sandbox" +mkdir -p "$CTX/_toolkit" +rsync -a --exclude '.venv' --exclude '.git' --exclude '__pycache__' \ + --exclude '*.egg-info' --exclude 'examples' \ + "$TOOLKIT_SRC/" "$CTX/_toolkit/" +rsync -a --exclude '__pycache__' --exclude 'wrapper' \ + "$EXAMPLE/harbor_sandbox/" "$CTX/harbor_sandbox/" + +# Ensure the ECR repo exists. +aws ecr describe-repositories --region "$REGION" --repository-names "$REPO" >/dev/null 2>&1 \ + || aws ecr create-repository --region "$REGION" --repository-name "$REPO" >/dev/null + +echo ">> docker login to ECR" +aws ecr get-login-password --region "$REGION" \ + | docker login --username AWS --password-stdin "$REGISTRY" + +echo ">> building $IMAGE ($PLATFORM)" +docker build --platform "$PLATFORM" -t "$IMAGE" "$CTX" + +echo ">> pushing $IMAGE" +docker push "$IMAGE" + +rm -rf "$CTX/_toolkit" "$CTX/harbor_sandbox" +echo ">> done: $IMAGE" diff --git a/examples/sandbox_harbor/harness/claude_code/Dockerfile b/examples/sandbox_harbor/harness/claude_code/Dockerfile new file mode 100644 index 0000000..197dd1e --- /dev/null +++ b/examples/sandbox_harbor/harness/claude_code/Dockerfile @@ -0,0 +1,29 @@ +# tmax harness image — agent_type "claude_code" (Claude Code co-located in the +# task sandbox; this image only orchestrates — node + claude are bootstrapped +# INTO the task box at session start). +# Built for linux/arm64: runs on the default serverless MicroVM runtime. +FROM python:3.12-slim + +ENV PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + AWS_REGION=us-west-2 \ + AWS_DEFAULT_REGION=us-west-2 + +WORKDIR /app + +# Core server + AWS SDK first (cached layer). bedrock-agentcore provides the +# /ping + /invocations HTTP contract on :8080. +RUN pip install "bedrock-agentcore>=1.0.3" "boto3>=1.43.68" + +# The toolkit source (staged by build_push.sh) provides SandboxClient + the +# @rollout_entrypoint SDK (AgentCoreRLApp). +COPY _toolkit /toolkit +RUN pip install /toolkit + +# The local harbor_sandbox package (staged by build_push.sh) provides +# HarborSandboxClient; /app is on sys.path when `python harness.py` runs. +COPY harbor_sandbox /app/harbor_sandbox +COPY harness.py /app/harness.py + +EXPOSE 8080 +CMD ["python", "harness.py"] diff --git a/examples/sandbox_harbor/harness/claude_code/harness.py b/examples/sandbox_harbor/harness/claude_code/harness.py new file mode 100644 index 0000000..847add4 --- /dev/null +++ b/examples/sandbox_harbor/harness/claude_code/harness.py @@ -0,0 +1,336 @@ +#!/usr/bin/env python3 +"""Harbor harness — agent_type "claude_code": Claude Code CO-LOCATED in the box. + +Claude Code runs *inside* the task sandbox: this app bootstraps it into the +session and runs ``claude -p`` there, so its native Bash/Read/Write tools +operate directly on the task's own filesystem — the same box the verifier will +grade. (This harness process only orchestrates; the agent lives in the box.) + + launcher ──invoke──▶ THIS app ──sb.exec──▶ task runtime [node + claude -p] + +Flow per rollout: + 1. open a session on the task runtime + 2. bootstrap: fetch a pinned Node build (the task images ship no node) and + ``npm i -g @anthropic-ai/claude-code`` (~5-10s) + 3. run ``claude -p `` over Bedrock — creds come from the task + runtime's OWN IAM role (verified reachable in-box), nothing is injected. + stream-json output doubles as the per-turn transcript AND keepalive bytes + on the exec stream, so long runs never trip the SandboxClient's 900s idle read. + 4. grade with the task's shipped verifier (injected only now, agent frozen) + +The prompt is EXACTLY the task instruction (Claude Code's own system prompt; no +scaffolding added, no "remote sandbox" note — co-located, the box IS local). + +Async contract (same for both harness images): @rollout_entrypoint — the +invoke ACKS immediately; the record (or a status_code=500 error doc) is saved +to s3://///.json. + +Payload: task_runtime_arn, instruction (required); tests_tar_b64, model, + max_steps (-> --max-turns), agent_timeout_s (per-task agent wall-clock + budget; on expiry the agent is stopped and the verifier still grades its + partial work — the leaderboard's AgentTimeout semantics), _rollout. +""" +from __future__ import annotations + +import base64 +import json +import time + +from harbor_sandbox import REGION, HarborSandboxClient + +from agentcore_rl_toolkit import AgentCoreRLApp +from agentcore_rl_toolkit.sandbox import SandboxClient + +# REGION is imported from harbor_sandbox.config (.env-driven) so the in-box +# Bedrock calls follow the deploy region, not the Dockerfile-baked AWS_REGION. +DEFAULT_MODEL = "us.anthropic.claude-sonnet-4-6" +# agent wall-clock budget when the payload omits one (was the old hardcoded cap). +DEFAULT_AGENT_TIMEOUT_S = 3300 +# verifier (test.sh) budget when the payload omits [verifier].timeout_sec. Was a +# hardcoded 180s that truncated heavy verifiers mid-build -> reward=None. +DEFAULT_VERIFIER_TIMEOUT_S = 900 +# SandboxClient.exec enforces a service-side timeout ceiling of 3600s, so a single +# `claude -p` exec cannot run longer than this. Tasks whose task.toml budget +# exceeds it (TB-2: 7200s, 12000s) are CAPPED here — the run is stopped at 3600s +# and still graded (partial work). We record requested-vs-applied so the cap is +# auditable rather than silent. (>3600s budgets need a background-run+poll loop.) +EXEC_TIMEOUT_MAX_S = 3600 + +# Client-side botocore read_timeout for the sandbox connection. This is a PER-READ +# idle timeout (max silence between streamed stream-json events), NOT a total cap. +# The SandboxClient default is 900s (sized for cold-start start()); that is fatal +# for the long agent exec: a task may legitimately run one silent shell op (compile, +# train, MCMC) for its ENTIRE budget with zero stream-json, so the idle timer must +# outlast the longest possible command. The longest command is the 3600s service +# cap itself, so read_timeout must be STRICTLY greater than it — equal is a dead +# heat (read_timeout==budget is exactly what killed the 900s-budget tasks and would +# re-kill train-fasttext at 3600). +300s covers the terminal result's round-trip +# after the service kills the command. Over-provisioning is free: read_timeout only +# bites during silence, and the 3600s service cap still bounds a genuinely-hung one. +SANDBOX_READ_TIMEOUT_S = EXEC_TIMEOUT_MAX_S + 300 # 3900 + +# Pinned Node build fetched into the box at session start (the task images ship +# no node). NA is picked in-box from uname -m (arm64 on the microVM substrate). +NODE_VER = "v20.18.1" + +# The harness assumes ONLY a Linux container with a shell, not whatever the TASK +# image happens to ship, so the fetcher falls back curl -> wget -> python -> +# `apt-get install curl`; `echo DL=` records which path was taken. Kept +# single-quote-free (python fetchers use \" not ') so the sandbox exec wrapper +# stays a simple `/bin/sh -c '...'`. +_BOOTSTRAP = r""" +set -e +case "$(uname -m)" in aarch64|arm64) NA=arm64;; *) NA=x64;; esac +URL="https://nodejs.org/dist/NODEVER/node-NODEVER-linux-$NA.tar.gz" +have(){ command -v "$1" >/dev/null 2>&1; } +if have node && have npm; then + echo "DL=preinstalled" +else + # no fetcher in the image? install curl via the package manager on the box. + if ! { have curl || have wget || have python3 || have python; }; then + if have apt-get; then + echo "DL=apt-installing-curl" + apt-get update >/dev/null 2>&1 || true + DEBIAN_FRONTEND=noninteractive apt-get install -y curl >/dev/null 2>&1 || true + fi + fi + if have curl; then echo "DL=curl"; curl -fsSL "$URL" -o /tmp/node.tgz; + elif have wget; then echo "DL=wget"; wget -qO /tmp/node.tgz "$URL"; + elif have python3; then echo "DL=python3"; + python3 -c "import urllib.request as u; u.urlretrieve(\"$URL\",\"/tmp/node.tgz\")"; + elif have python; then echo "DL=python"; python -c "import urllib; urllib.urlretrieve(\"$URL\",\"/tmp/node.tgz\")"; + else echo "no fetcher and apt-get install curl unavailable/failed" >&2; exit 3; + fi + mkdir -p /opt/node + # --no-same-owner: even with caps, avoid chowning to the uid in the tarball + tar xzf /tmp/node.tgz -C /opt/node --strip-components=1 --no-same-owner >/dev/null +fi +export PATH=/opt/node/bin:$PATH +# stdout silenced, stderr KEPT: on failure the npm error must reach the +# bootstrap-failed exception detail instead of vanishing into /dev/null. +npm i -g @anthropic-ai/claude-code >/dev/null +echo BOOTSTRAP_OK $(node --version) $(claude --version) +""".replace("NODEVER", NODE_VER) + +app = AgentCoreRLApp() + + +def _resolve_sandbox(payload): + """Start a task's sandbox. Two addressing modes: + * legacy: payload["task_runtime_arn"] — a pre-created runtime; open a session. + * harbor: payload["benchmark"] + payload["task_id"] — ensure the runtime via + HarborSandboxClient.create (idempotent). With payload["lease"]=true the + harness deletes it after the rollout (the lease pattern). + + Returns (sandbox_client, release_fn) — release_fn is a no-op unless leased. + """ + arn = payload.get("task_runtime_arn") + if arn: + return SandboxClient(runtime_arn=arn, read_timeout=SANDBOX_READ_TIMEOUT_S), (lambda: None) + bench, task = payload.get("benchmark"), payload.get("task_id") + if not (bench and task): + raise ValueError("payload must include either 'task_runtime_arn' or " "'benchmark' + 'task_id'") + # read_timeout forwards through **client_kwargs to the SandboxClient ctor. + client = HarborSandboxClient.create(bench, task, read_timeout=SANDBOX_READ_TIMEOUT_S) + if payload.get("lease"): + return client, client.release # instance path: deletes its OWN runtime + return client, (lambda: None) + + +def _claude_env(model: str) -> dict: + return { + "PATH": "/opt/node/bin:/usr/local/bin:/usr/bin:/bin", + "CLAUDE_CODE_USE_BEDROCK": "1", # creds = the TASK runtime's own role + "ANTHROPIC_MODEL": model, + "AWS_REGION": REGION, + "IS_SANDBOX": "1", # allow --dangerously-skip-permissions as root in a sandbox + "HOME": "/root", + } + + +def _parse_stream(stdout: str) -> dict: + """Pull the agent's Bash commands and the final result out of Claude Code's + stream-json transcript.""" + commands = [] + result, num_turns, cost = None, None, None + for line in stdout.splitlines(): + line = line.strip() + if not line.startswith("{"): + continue + try: + obj = json.loads(line) + except Exception: + continue + t = obj.get("type") + if t == "assistant": + for blk in obj.get("message", {}).get("content", []): + if blk.get("type") == "tool_use" and blk.get("name") == "Bash": + cmd = blk.get("input", {}).get("command") + if cmd: + commands.append(cmd) + elif t == "result": + result = obj.get("result") + num_turns = obj.get("num_turns") + cost = obj.get("total_cost_usd") + return {"commands": commands, "result": result, "num_turns": num_turns, "cost_usd": cost} + + +def _stage_b64(sb, b64: str, dest: str, chunk: int = 50000) -> None: + """Decode a base64 blob to INSIDE the box, written in slices. + + A single ``printf '%s' '' | base64 -d`` puts the whole blob in one + command, but SandboxClient.exec caps body.command at 65536 bytes, so big + test tarballs (build-pov-ray, sam-cell-seg, video-processing, ...) blow the + cap and die with a ValidationException. Append the base64 in <=50000-char + slices (the base64 alphabet has no single-quotes, so '...'-wrapping is safe + and never re-expanded) and decode once at the end.""" + tmp = dest + ".b64" + sb.exec(f"rm -f {tmp}", timeout=30) + for i in range(0, len(b64), chunk): + sb.exec(f"printf '%s' '{b64[i:i + chunk]}' >> {tmp}", timeout=60) + sb.exec(f"base64 -d {tmp} > {dest} && rm -f {tmp}", timeout=60) + + +def grade_with_verifier(sb, tests_tar_b64: str, verifier_timeout_s: float = DEFAULT_VERIFIER_TIMEOUT_S) -> dict: + """Run the task's own shipped verifier in the sandbox (uniform corpus + contract: tests/test.sh -> /logs/verifier/reward.txt). + + The test.sh budget is the task's OWN `[verifier].timeout_sec` (threaded via + the payload), clamped to exec's 3600s ceiling. A hardcoded cap here (was + 180s) silently truncates heavy verifiers mid-build — they never write + reward.txt, so the row grades reward=None (indeterminate, NOT a real fail). + Faithful to the leaderboard, which gives each verifier its declared budget.""" + if not tests_tar_b64: + return {"reward": None, "verifier_tail": "(no tests supplied)"} + verify_s = min(int(verifier_timeout_s), EXEC_TIMEOUT_MAX_S) + sb.exec("rm -rf /tests && mkdir -p /tests /logs/verifier", timeout=30) + _stage_b64(sb, tests_tar_b64, "/tmp/_tests.tgz") + unpack = sb.exec("tar xzf /tmp/_tests.tgz -C /tests 2>&1", timeout=120) + run = sb.exec("bash /tests/test.sh 2>&1", timeout=verify_s) + reward_txt = sb.exec("cat /logs/verifier/reward.txt 2>/dev/null", timeout=30) + reward = None + try: + reward = int(reward_txt.stdout.strip()) + except Exception: + pass + return { + "reward": reward, + "verifier_tail": (run.stdout or "")[-1500:], + "unpack_err": (unpack.stdout or "")[:300] if unpack.exit_code != 0 else "", + "verifier_budget_s": verify_s, + "verifier_timed_out": bool(getattr(run, "timed_out", False)), + } + + +def run_rollout( + sb_client, + instruction, + model, + max_steps, + tests_tar_b64, + agent_timeout_s, + verifier_timeout_s=DEFAULT_VERIFIER_TIMEOUT_S, +) -> dict: + client = sb_client + timing = {} + + t0 = time.time() + with client.start() as sb: + timing["start_s"] = round(time.time() - t0, 1) + + # --- bootstrap Claude Code into the box --- + t0 = time.time() + boot = sb.exec(_BOOTSTRAP, timeout=300) + timing["bootstrap_s"] = round(time.time() - t0, 1) + if "BOOTSTRAP_OK" not in (boot.stdout or ""): + raise RuntimeError(f"claude-code bootstrap failed: {(boot.stderr or boot.stdout or '')[-300:]}") + + # --- stage the instruction as the raw prompt (base64 avoids all quoting) --- + # chunked so a very long instruction can't blow exec's 64KB command cap. + b64 = base64.b64encode(instruction.encode()).decode() + _stage_b64(sb, b64, "/tmp/prompt.txt") + + # --- run Claude Code; prompt = the instruction verbatim, nothing added --- + # cwd = the first task-like dir that exists, so relative paths resolve. + cwd = ( + "d=$(for x in /home/user /app /workspace /root; do " + '[ -d "$x" ] && echo "$x" && break; done); cd "${d:-/}" && ' + ) + run_cmd = ( + cwd + "cat /tmp/prompt.txt | claude -p " + "--output-format stream-json --verbose " + f"--max-turns {max_steps} --dangerously-skip-permissions" + ) + t0 = time.time() + # per-task agent budget: on expiry sb.exec stops the agent (timed_out) + # and we STILL grade its partial work below — faithful AgentTimeout. + # int(): body.timeout is validated as an integer (float 900.0 is rejected). + # min(..., EXEC_TIMEOUT_MAX_S): exec's own 3600s ceiling; larger budgets + # are capped (and recorded below) since one exec cannot outlive it. + budget_s = int(agent_timeout_s) + exec_timeout_s = min(budget_s, EXEC_TIMEOUT_MAX_S) + run = sb.exec(run_cmd, timeout=exec_timeout_s, env=_claude_env(model)) + timing["agent_s"] = round(time.time() - t0, 1) + parsed = _parse_stream(run.stdout or "") + + # --- grade with the shipped verifier (agent is frozen) --- + t0 = time.time() + result = grade_with_verifier(sb, tests_tar_b64, verifier_timeout_s) + timing["grade_s"] = round(time.time() - t0, 1) + + result.update( + { + "agent_type": "claude_code", + "model": model, + "stop_reason": "agent_timed_out" if run.timed_out else "end_turn", + "agent_budget_s": budget_s, + "agent_budget_applied_s": exec_timeout_s, + "budget_capped_by_exec_max": exec_timeout_s < budget_s, + "steps": len(parsed["commands"]), + "num_turns": parsed["num_turns"], + "agent_result": (parsed["result"] or "")[:400], + "cost_usd": parsed["cost_usd"], + "timing": timing, + "commands": [c[:500] for c in parsed["commands"]], + } + ) + return result + + +@app.rollout_entrypoint +def invoke(payload: dict) -> dict: + """One rollout per invocation, run as a background task by the SDK. + + Exceptions are NOT caught here on purpose: the SDK's error path saves + {"status_code": 500, "stop_reason": str(e), "traceback": ...} to the same + S3 key, so the launcher learns about failures the same way as results. + """ + instruction = payload.get("instruction") + if not instruction: + raise ValueError("payload must include 'instruction'") + sb_client, release = _resolve_sandbox(payload) + try: + return run_rollout( + sb_client=sb_client, + instruction=instruction, + model=payload.get("model", DEFAULT_MODEL), + max_steps=int(payload.get("max_steps", 25)), + tests_tar_b64=payload.get("tests_tar_b64", ""), + agent_timeout_s=float(payload.get("agent_timeout_s", DEFAULT_AGENT_TIMEOUT_S)), + verifier_timeout_s=float(payload.get("verifier_timeout_s", DEFAULT_VERIFIER_TIMEOUT_S)), + ) + finally: + # no-op unless payload["lease"]: frees the runtime slot. A delete failure + # (AccessDenied, or ResourceNotFound when the runtime never became ready) + # must NOT run as an unguarded finally — that would REPLACE the rollout's + # real return value / real exception with the cleanup error (this is how + # circuit-fibsqrt's true ResourceNotFound got masked as DeleteAgentRuntime + # AccessDenied). Swallow it so the genuine result/cause is what's recorded. + try: + release() + except Exception as e: # noqa: BLE001 - cleanup best-effort + print(f"lease release failed (ignored, slot may leak): {e!r}") + + +if __name__ == "__main__": + app.run() diff --git a/examples/sandbox_harbor/harness/deploy_harness.py b/examples/sandbox_harbor/harness/deploy_harness.py new file mode 100644 index 0000000..19ce7e8 --- /dev/null +++ b/examples/sandbox_harbor/harness/deploy_harness.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Deploy an agent HARNESS as an AgentCore runtime under its CANONICAL name. + + python deploy_harness.py --agent claude-code --image-tag claude-code-v1 + +Creates the runtime ``harness__`` (benchmark-agnostic) pointing +at ``$HARNESS_REPO:`` (repo from .env), on the serverless arm64 MicroVM recipe +(PUBLIC network, HTTP, idle 900s / maxLifetime 8h). Idempotent: if the name +already exists it prints the ARN and exits without change, so it never disturbs +a runtime an in-flight run is using. +""" +from __future__ import annotations + +import argparse +import sys +import time + +import boto3 +from harbor_sandbox.config import ECR_REGISTRY, HARNESS_REPO, IDLE_SESSION_TIMEOUT_S, MAX_LIFETIME_S, REGION, ROLE_ARN + + +def find(ctrl, name: str): + kw = {"maxResults": 100} + while True: + r = ctrl.list_agent_runtimes(**kw) + for rt in r.get("agentRuntimes", []): + if rt.get("agentRuntimeName") == name: + return rt["agentRuntimeArn"] + kw["nextToken"] = r.get("nextToken") + if not kw["nextToken"]: + return None + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--agent", required=True, help="strands | claude-code") + ap.add_argument("--image-tag", required=True, dest="image_tag", help="$HARNESS_REPO: to point the runtime at") + ap.add_argument("--version", default="v1") + a = ap.parse_args() + + name = f"harness_{a.agent.replace('-', '_')}_{a.version}" + uri = f"{ECR_REGISTRY}/{HARNESS_REPO}:{a.image_tag}" + ctrl = boto3.client("bedrock-agentcore-control", region_name=REGION) + + existing = find(ctrl, name) + if existing: + print(f"exists (no change): {name} -> {existing}") + return + + resp = ctrl.create_agent_runtime( + agentRuntimeName=name, + agentRuntimeArtifact={"containerConfiguration": {"containerUri": uri}}, + roleArn=ROLE_ARN, + networkConfiguration={"networkMode": "PUBLIC"}, + protocolConfiguration={"serverProtocol": "HTTP"}, + lifecycleConfiguration={"idleRuntimeSessionTimeout": IDLE_SESSION_TIMEOUT_S, "maxLifetime": MAX_LIFETIME_S}, + ) + arn = resp["agentRuntimeArn"] + rid = arn.split("/")[-1] + print(f"creating {name}\n image {uri}\n arn {arn}") + + for _ in range(90): + st = ctrl.get_agent_runtime(agentRuntimeId=rid).get("status") + print(f" status: {st}") + if st == "READY": + print(f"READY: {name} -> {arn}") + return + if st and ("FAIL" in st or st == "DELETING"): + sys.exit(f"deploy failed: status={st}") + time.sleep(10) + sys.exit("timed out waiting for READY") + + +if __name__ == "__main__": + main() diff --git a/examples/sandbox_harbor/harness/strands/Dockerfile b/examples/sandbox_harbor/harness/strands/Dockerfile new file mode 100644 index 0000000..0fa7d39 --- /dev/null +++ b/examples/sandbox_harbor/harness/strands/Dockerfile @@ -0,0 +1,28 @@ +# tmax harness image — agent_type "strands" (Strands Agent, bash tool -> sb.exec). +# Built for linux/arm64: runs on the default serverless MicroVM runtime. +FROM python:3.12-slim + +ENV PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + AWS_REGION=us-west-2 \ + AWS_DEFAULT_REGION=us-west-2 + +WORKDIR /app + +# Core server + AWS SDK first (cached layer). bedrock-agentcore provides the +# /ping + /invocations HTTP contract on :8080. strands powers the agent +# (1.51.0 = the version the prototype was validated against). +RUN pip install "bedrock-agentcore>=1.0.3" "boto3>=1.43.68" "strands-agents[openai]>=1.51,<2" + +# The toolkit source (staged by build_push.sh) provides SandboxClient + the +# @rollout_entrypoint SDK (AgentCoreRLApp). +COPY _toolkit /toolkit +RUN pip install /toolkit + +# The local harbor_sandbox package (staged by build_push.sh) provides +# HarborSandboxClient; /app is on sys.path when `python harness.py` runs. +COPY harbor_sandbox /app/harbor_sandbox +COPY harness.py /app/harness.py + +EXPOSE 8080 +CMD ["python", "harness.py"] diff --git a/examples/sandbox_harbor/harness/strands/harness.py b/examples/sandbox_harbor/harness/strands/harness.py new file mode 100644 index 0000000..0569536 --- /dev/null +++ b/examples/sandbox_harbor/harness/strands/harness.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +"""Harbor harness — agent_type "strands": Strands Agent, bash tool -> sb.exec. + +SEPARATED agent, framework-run: a Strands ``Agent`` — backed by a Bedrock +``BedrockModel`` for evaluation, or by an ``OpenAIModel`` wired from the +trainer-injected ``_rollout`` config (base_url/model_id/api_key) for RL +training — reasons IN THIS PROCESS (arm64 microVM); its single tool, +``bash``, is overridden to ship each command into the task session via +``SandboxClient.exec`` and return the output: + + launcher ──invoke──▶ THIS app (Strands Agent) ──bash tool = sb.exec──▶ task runtime + +The system prompt introduces exactly ONE thing — that a remote sandbox holds +the task and the ``bash`` tool runs there. The user message is EXACTLY the task +instruction. Grading is identical to the other harness, so results are directly +comparable. + +Async contract (same for both harness images): the entrypoint is the +toolkit's @rollout_entrypoint — the invoke ACKS immediately and the rollout runs +as a background task whose record (or a status_code=500 error doc) is saved to +s3://///.json. + +Payload: task_runtime_arn, instruction (required); tests_tar_b64, model, + max_steps, agent_timeout_s, verifier_timeout_s, _rollout. +""" +from __future__ import annotations + +import time + +from harbor_sandbox import REGION, HarborSandboxClient +from strands import Agent, tool +from strands.hooks import BeforeToolCallEvent, HookProvider +from strands.models import BedrockModel +from strands.models.openai import OpenAIModel + +from agentcore_rl_toolkit import AgentCoreRLApp +from agentcore_rl_toolkit.sandbox import SandboxClient + +# REGION is imported from harbor_sandbox.config (.env-driven) so the model calls +# follow the deploy region, not the Dockerfile-baked AWS_REGION. +DEFAULT_MODEL = "us.anthropic.claude-sonnet-4-6" + +# Default agent wall-clock ceiling when the payload omits a per-task budget. The +# step cap alone is not enough: a task whose every command times out could burn +# max_steps*120s. The launcher normally passes the task's own agent_timeout_s +# (task.toml [agent].timeout_sec); either way the agent stops at the budget and +# we still GRADE its partial work (faithful AgentTimeout). +DEFAULT_AGENT_TIMEOUT_S = 3300 +# Verifier (test.sh) budget when the payload omits [verifier].timeout_sec, and +# the exec service ceiling it is clamped to — same as claude_code, so grading +# stays directly comparable (a hardcoded 180s silently truncated heavy verifiers). +DEFAULT_VERIFIER_TIMEOUT_S = 900 +EXEC_TIMEOUT_MAX_S = 3600 +# Per-read idle timeout: must outlast the longest single command (the 3600s +# verifier) so a long silent verifier isn't killed mid-run. +SANDBOX_READ_TIMEOUT_S = EXEC_TIMEOUT_MAX_S + 300 # 3900 + +SYSTEM_PROMPT = ( + "You are an autonomous software-engineering agent. Use the `bash` tool to run " + "commands in a remote Linux sandbox — that is your only way to act. Complete " + "the task, then stop." +) + +app = AgentCoreRLApp() + + +def _resolve_sandbox(payload): + """Start a task's sandbox. Two addressing modes: + * legacy: payload["task_runtime_arn"] — a pre-created runtime; open a session. + * harbor: payload["benchmark"] + payload["task_id"] — ensure the runtime via + HarborSandboxClient.create (idempotent). With payload["lease"]=true the + harness deletes it after the rollout (the lease pattern). + + Returns (sandbox_client, release_fn) — release_fn is a no-op unless leased. + """ + arn = payload.get("task_runtime_arn") + if arn: + return SandboxClient(runtime_arn=arn, read_timeout=SANDBOX_READ_TIMEOUT_S), (lambda: None) + bench, task = payload.get("benchmark"), payload.get("task_id") + if not (bench and task): + raise ValueError("payload must include either 'task_runtime_arn' or " "'benchmark' + 'task_id'") + # lease => a PRIVATE uniquely-named runtime: concurrent rollouts of the same + # task (GRPO groups) must not share a deletable runtime, or the first + # release() kills the siblings mid-rollout. + lease = bool(payload.get("lease")) + client = HarborSandboxClient.create(bench, task, read_timeout=SANDBOX_READ_TIMEOUT_S, unique=lease) + if lease: + return client, client.release # instance path: deletes its OWN runtime + return client, (lambda: None) + + +class _Budget(HookProvider): + """Hard cap on BOTH tool count and wall clock, so a stuck agent cannot loop + forever or burn time on a task whose every command times out. Raises once + either budget is exceeded; the caller catches it and grades what exists.""" + + class Exceeded(Exception): + pass + + def __init__(self, limit: int, deadline: float): + self.limit, self.deadline, self.n = limit, deadline, 0 + + def register_hooks(self, registry, **_): + registry.add_callback(BeforeToolCallEvent, self._before) + + def _before(self, event): + self.n += 1 + if self.n > self.limit or time.time() > self.deadline: + raise _Budget.Exceeded() + + +def _stage_b64(sb, b64: str, dest: str, chunk: int = 50000) -> None: + """Decode a base64 blob to inside the box, written in <=50000-char + slices so a large test tarball can't blow exec's 64KB command cap (the + base64 alphabet has no single-quotes, so '...'-wrapping is safe).""" + tmp = dest + ".b64" + sb.exec(f"rm -f {tmp}", timeout=30) + for i in range(0, len(b64), chunk): + sb.exec(f"printf '%s' '{b64[i:i + chunk]}' >> {tmp}", timeout=60) + sb.exec(f"base64 -d {tmp} > {dest} && rm -f {tmp}", timeout=60) + + +def grade_with_verifier(sb, tests_tar_b64: str, verifier_timeout_s: float = DEFAULT_VERIFIER_TIMEOUT_S) -> dict: + """Run the task's own shipped verifier in the sandbox (uniform corpus + contract: tests/test.sh -> /logs/verifier/reward.txt). The test.sh budget is + the task's own [verifier].timeout_sec, clamped to exec's 3600s ceiling — a + hardcoded cap would silently truncate heavy verifiers into reward=None.""" + if not tests_tar_b64: + return {"reward": None, "rewards": None, "verifier_tail": "(no tests supplied)"} + verify_s = min(int(verifier_timeout_s), EXEC_TIMEOUT_MAX_S) + sb.exec("rm -rf /tests && mkdir -p /tests /logs/verifier", timeout=30) + _stage_b64(sb, tests_tar_b64, "/tmp/_tests.tgz") + unpack = sb.exec("tar xzf /tmp/_tests.tgz -C /tests 2>&1", timeout=120) + run = sb.exec("bash /tests/test.sh 2>&1", timeout=verify_s) + reward_txt = sb.exec("cat /logs/verifier/reward.txt 2>/dev/null", timeout=30) + reward = None + try: + reward = int(reward_txt.stdout.strip()) + except Exception: + pass + return { + "reward": reward, + # Plural alias: training backends read result["rewards"] (e.g. + # backends/experimental/verl/agent_loop.py); the singular key stays for + # the eval launcher. None (verifier wrote no reward.txt) scores 0. + "rewards": reward, + "verifier_tail": (run.stdout or "")[-1500:], + "unpack_err": (unpack.stdout or "")[:300] if unpack.exit_code != 0 else "", + "verifier_budget_s": verify_s, + "verifier_timed_out": bool(getattr(run, "timed_out", False)), + } + + +def run_rollout( + sb_client, + instruction, + model, + max_steps, + tests_tar_b64, + agent_timeout_s, + verifier_timeout_s=DEFAULT_VERIFIER_TIMEOUT_S, + rollout_cfg=None, +) -> dict: + client = sb_client + commands: list[str] = [] + timing = {} + + t0 = time.time() + with client.start() as sb: + timing["start_s"] = round(time.time() - t0, 1) + + @tool + def bash(command: str) -> str: + """Run a shell command in the remote task sandbox and return its + combined result (exit code, stdout, stderr).""" + commands.append(command) + r = sb.exec(command, timeout=120) + return f"exit={r.exit_code}\nstdout:\n{r.stdout[:4000]}\nstderr:\n{r.stderr[:1500]}" + + rc = rollout_cfg or {} + if rc.get("base_url"): + # Training: the trainer injects the inference endpoint (the rollout + # gateway) via _rollout, and the api-key slot carries the trajectory- + # capture session key — it MUST reach the LLM client or every rollout + # degenerates into one shared gateway session. "EMPTY" keeps plain + # OpenAI-compatible eval endpoints (vLLM etc.) working unchanged. + model = rc["model_id"] + model_obj = OpenAIModel( + client_args={"api_key": rc.get("api_key") or "EMPTY", "base_url": rc["base_url"]}, + model_id=model, + params=rc.get("sampling_params", {}), + ) + else: + model_obj = BedrockModel(model_id=model, region_name=REGION, max_tokens=4096, temperature=1.0) + t0 = time.time() + limiter = _Budget(max_steps, deadline=t0 + agent_timeout_s) + agent = Agent( + model=model_obj, tools=[bash], system_prompt=SYSTEM_PROMPT, hooks=[limiter], callback_handler=None + ) + stop = "end_turn" + try: + agent(instruction) + except _Budget.Exceeded: + stop = "max_steps" if limiter.n > limiter.limit else "max_wall" + except Exception as e: # still grade whatever the agent left behind + stop = f"agent_error:{type(e).__name__}" + timing["agent_s"] = round(time.time() - t0, 1) + + t0 = time.time() + result = grade_with_verifier(sb, tests_tar_b64, verifier_timeout_s) + timing["grade_s"] = round(time.time() - t0, 1) + + result.update( + { + "agent_type": "strands", + "model": model, + "stop_reason": stop, + "steps": len(commands), + "timing": timing, + "commands": [c[:500] for c in commands], + } + ) + return result + + +@app.rollout_entrypoint +def invoke(payload: dict) -> dict: + """One rollout per invocation, run as a background task by the SDK. + + Exceptions are NOT caught here on purpose: the SDK's error path saves + {"status_code": 500, "stop_reason": str(e), "traceback": ...} to the same + S3 key, so the launcher learns about failures the same way as results. + """ + instruction = payload.get("instruction") + if not instruction: + raise ValueError("payload must include 'instruction'") + sb_client, release = _resolve_sandbox(payload) + try: + return run_rollout( + sb_client=sb_client, + instruction=instruction, + model=payload.get("model", DEFAULT_MODEL), + max_steps=int(payload.get("max_steps", 25)), + tests_tar_b64=payload.get("tests_tar_b64", ""), + agent_timeout_s=float(payload.get("agent_timeout_s", DEFAULT_AGENT_TIMEOUT_S)), + verifier_timeout_s=float(payload.get("verifier_timeout_s", DEFAULT_VERIFIER_TIMEOUT_S)), + rollout_cfg=payload.get("_rollout"), + ) + finally: + # guarded so a delete failure (AccessDenied, or ResourceNotFound when the + # runtime never became ready) can't REPLACE the rollout's real result or + # exception with the cleanup error. Swallow it; the slot may leak. + try: + release() + except Exception as e: # noqa: BLE001 - cleanup best-effort + print(f"lease release failed (ignored, slot may leak): {e!r}") + + +if __name__ == "__main__": + app.run() diff --git a/examples/sandbox_harbor/pyproject.toml b/examples/sandbox_harbor/pyproject.toml new file mode 100644 index 0000000..594f0bb --- /dev/null +++ b/examples/sandbox_harbor/pyproject.toml @@ -0,0 +1,22 @@ +[project] +name = "sandbox-harbor-example" +version = "0.1.0" +description = "One-command evaluation of a Harbor benchmark on AgentCore Runtime" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "agentcore-rl-toolkit>=0.1.3", + "boto3>=1.43.68", +] + +[project.optional-dependencies] +# Only needed to auto-pull tasks from the Harbor registry (bench.py / +# ensure_dataset when --task-root is empty). Install with `uv sync --extra harbor`. +harbor = ["harbor"] + +[tool.setuptools] +packages = ["harbor_sandbox"] +py-modules = ["bench"] + +[tool.setuptools.package-data] +harbor_sandbox = ["wrapper/*"] diff --git a/src/agentcore_rl_toolkit/sandbox/client.py b/src/agentcore_rl_toolkit/sandbox/client.py index f8de3e8..6428824 100644 --- a/src/agentcore_rl_toolkit/sandbox/client.py +++ b/src/agentcore_rl_toolkit/sandbox/client.py @@ -116,6 +116,10 @@ class SandboxClient: qualifier: Runtime endpoint qualifier. max_retry_attempts: Max boto3 retry attempts (adaptive mode). max_pool_connections: Max boto3 connection pool size. + read_timeout: Per-request socket read timeout in seconds. Defaults to + 900s so a long, silent command (a heavy verifier, or a runtime still + warming up) is not cut off by boto3's short default. + connect_timeout: TCP connect timeout in seconds. shell: Shell used to interpret ``exec()`` commands in the container. Defaults to ``/bin/sh`` (present in any image with a shell, including busybox/alpine); set to ``/bin/bash`` if your image has @@ -150,6 +154,8 @@ def __init__( qualifier: str = "DEFAULT", max_retry_attempts: int = 5, max_pool_connections: int = 10, + read_timeout: int = 900, + connect_timeout: int = 15, shell: str = "/bin/sh", ): self.runtime_arn = runtime_arn @@ -160,6 +166,8 @@ def __init__( config = Config( retries={"max_attempts": max_retry_attempts, "mode": "adaptive"}, max_pool_connections=max_pool_connections, + read_timeout=read_timeout, + connect_timeout=connect_timeout, ) self._client = boto3.client("bedrock-agentcore", region_name=self.region, config=config)