diff --git a/README.md b/README.md index eed2e718..3cd81a0a 100644 --- a/README.md +++ b/README.md @@ -382,6 +382,7 @@ stagehand = await Stagehand(...).init() page = wrap_stagehand(stagehand.page, tracer, name="my run") await page.goto("https://example.com") +await page.observe("find the login button") # auto-recorded await page.act("click the login button") # auto-recorded await page.extract("get the headline") # auto-recorded page.bt_run.close() @@ -389,6 +390,9 @@ page.bt_run.close() The wrapper records method args/kwargs as `model_input`, then writes the successful Stagehand return value back to the same step as `model_output`. +It also summarizes Stagehand-shaped evidence such as observe candidates, +selectors, extracted text, action/status fields, retries, and verification or +tool-call metadata under `model_output["stagehand_evidence"]`. For Stagehand `act` and `extract` debugging, see [Debug Stagehand runs with BrowserTrace](https://aaronlab.github.io/browsertrace/stagehand-debugging.html). diff --git a/browsertrace/integrations/stagehand.py b/browsertrace/integrations/stagehand.py index 0dc529ff..1d85c167 100644 --- a/browsertrace/integrations/stagehand.py +++ b/browsertrace/integrations/stagehand.py @@ -26,6 +26,7 @@ from __future__ import annotations +from collections.abc import Mapping from typing import Any, Optional from ..tracer import Run, Tracer @@ -62,6 +63,8 @@ async def traced(*args: Any, **kwargs: Any) -> Any: url=getattr(self._page, "url", "") or "", screenshot=shot, model_input={"method": name, "args": list(args), "kwargs": kwargs}, + stagehand_method=name, + instruction=str(instr), ) try: result = await fn(*args, **kwargs) @@ -72,7 +75,7 @@ async def traced(*args: Any, **kwargs: Any) -> Any: raise output = _serialize_result(result) model_output = {"result": output} - evidence = _extract_stagehand_evidence(output) + evidence = _extract_stagehand_evidence(output, method=name) if evidence: model_output["stagehand_evidence"] = evidence self.bt_run.update_step(step_id, model_output=model_output) @@ -93,28 +96,53 @@ def wrap_stagehand(page: Any, tracer: Tracer, name: str = "stagehand run") -> _T def _serialize_result(result: Any) -> Any: - if hasattr(result, "model_dump"): + return _to_plain(result, set()) + + +def _to_plain(value: Any, seen: set[int]) -> Any: + if isinstance(value, (str, int, float, bool)) or value is None: + return value + if isinstance(value, bytes): + return repr(value) + value_id = id(value) + if value_id in seen: + return "" + seen.add(value_id) + + if hasattr(value, "model_dump"): try: - return result.model_dump(exclude_none=True) + return _to_plain(value.model_dump(exclude_none=True), seen) except Exception: pass - if hasattr(result, "dict"): + if hasattr(value, "dict"): try: - return result.dict() + return _to_plain(value.dict(), seen) except Exception: pass - if isinstance(result, (dict, list, tuple, str, int, float, bool)) or result is None: - return result - if hasattr(result, "__dict__"): - return vars(result) - return str(result) + if isinstance(value, Mapping): + return {str(key): _to_plain(child, seen) for key, child in value.items()} + if isinstance(value, (list, tuple, set)): + return [_to_plain(child, seen) for child in value] + if hasattr(value, "__dict__"): + return _to_plain(vars(value), seen) + return str(value) -def _extract_stagehand_evidence(output: Any) -> dict[str, Any]: +def _extract_stagehand_evidence(output: Any, *, method: str = "") -> dict[str, Any]: evidence: dict[str, Any] = {} selectors = _find_values(output, "selector", "selectors", "css_selector", "cssSelector") descriptions = _find_values(output, "description") methods = _find_values(output, "method") + elements = _find_values(output, "element", "elements", "target", "targets") + extracted_text = _find_values( + output, "text", "content", "extracted_text", "extractedText", "extracted_content" + ) + actions = _find_values(output, "action", "actions") + statuses = _find_values(output, "status", "success", "ok") + attempts = _find_values(output, "attempt", "attempts", "retry_count", "retries") + verifications = _find_values( + output, "verification", "verified", "validation", "tool_call", "tool_calls", "toolCalls" + ) if selectors: evidence["selectors"] = selectors @@ -122,10 +150,42 @@ def _extract_stagehand_evidence(output: Any) -> dict[str, Any]: evidence["descriptions"] = descriptions if methods: evidence["methods"] = methods + if elements: + evidence["elements"] = elements + if extracted_text: + evidence["extracted_text"] = extracted_text + if actions: + evidence["actions"] = actions + if statuses: + evidence["statuses"] = statuses + if attempts: + evidence["attempts"] = attempts + if verifications: + evidence["verification"] = verifications + if method == "observe": + candidates = _observe_candidates(output) + if candidates: + evidence["observe_candidates"] = candidates return evidence +def _observe_candidates(output: Any) -> list[dict[str, Any]]: + candidates = output if isinstance(output, list) else output.get("candidates", []) if isinstance(output, dict) else [] + summary: list[dict[str, Any]] = [] + for candidate in candidates: + if not isinstance(candidate, dict): + continue + item = { + key: candidate[key] + for key in ("selector", "description", "method", "action") + if candidate.get(key) is not None + } + if item: + summary.append(item) + return summary + + def _find_values(value: Any, *keys: str) -> list[str]: found: list[str] = [] seen: set[str] = set() diff --git a/docs/stagehand-debugging.html b/docs/stagehand-debugging.html index 59afd044..2bec14e9 100644 --- a/docs/stagehand-debugging.html +++ b/docs/stagehand-debugging.html @@ -273,11 +273,12 @@

Wrap a Stagehand page

page = wrap_stagehand(stagehand.page, tracer, name="stagehand checkout run") await page.goto("https://example.com") +await page.observe("find the checkout button") await page.act("click the checkout button") await page.extract("get the order total") page.bt_run.close()

The wrapper records goto, act, extract, observe, and click calls while preserving the original page methods.

-

Arguments and keyword arguments are saved as model_input. Successful Stagehand return values are written back to the same trace step as model_output.

+

Arguments and keyword arguments are saved as model_input. Successful Stagehand return values are written back to the same trace step as model_output, with a compact stagehand_evidence summary for selectors, observe candidates, extracted text, retries, and verification or tool-call metadata.

@@ -286,7 +287,7 @@

What the trace captures

  • The Stagehand method and instruction.
  • The current page URL.
  • A screenshot before the action when available.
    • -
  • The successful Stagehand result, including observe or extract output when returned by the wrapped method.
    • +
  • The successful Stagehand result and compact evidence, including observe candidates or extract text when returned by the wrapped method.
  • Step status and exception text if the action fails.
  • Exportable HTML with optional model I/O redaction.
    • diff --git a/examples/README.md b/examples/README.md index f661f98e..b48c4891 100644 --- a/examples/README.md +++ b/examples/README.md @@ -171,7 +171,8 @@ BROWSERTRACE_HOME=/tmp/browsertrace-demo browsertrace show Expect `browsertrace list --limit 5` to show a recent `demo: stagehand checkout flow` run, and `browsertrace show ` to list -two successful steps: `act: click the checkout button` and +three successful steps: `observe: find the checkout button`, +`act: click the checkout button`, and `extract: extract the order total`. For a no-service Browser Use-shaped callback demo, run: diff --git a/examples/stagehand_wrapper_example.py b/examples/stagehand_wrapper_example.py index 96b83ce2..bfbe209e 100644 --- a/examples/stagehand_wrapper_example.py +++ b/examples/stagehand_wrapper_example.py @@ -26,16 +26,34 @@ async def screenshot(self) -> bytes: async def act(self, instruction: str) -> dict[str, str]: await asyncio.sleep(0) - return {"status": "completed", "instruction": instruction} + return { + "status": "completed", + "instruction": instruction, + "action": "click", + "selector": "button.checkout.primary", + "attempts": 1, + } async def extract(self, instruction: str) -> dict[str, str]: await asyncio.sleep(0) return { "status": "completed", "instruction": instruction, - "order_total": "$42.00", + "text": "Order total: $42.00", + "tool_calls": [{"name": "extract_text", "status": "ok"}], } + async def observe(self, instruction: str) -> list[dict[str, str]]: + await asyncio.sleep(0) + return [ + { + "selector": "button.checkout.primary", + "description": "Checkout button", + "method": "click", + "instruction": instruction, + } + ] + async def main() -> None: tracer = Tracer(home=os.environ.get("BROWSERTRACE_HOME")) @@ -45,12 +63,13 @@ async def main() -> None: name="demo: stagehand checkout flow", ) + await page.observe("find the checkout button") await page.act("click the checkout button") await page.extract("extract the order total") page.bt_run.close() print(f"BrowserTrace run id: {page.bt_run.id}") - print("Recorded Stagehand calls: act, extract") + print("Recorded Stagehand calls: observe, act, extract") print("Open the local UI with: browsertrace") diff --git a/tests/test_examples.py b/tests/test_examples.py index d9c857f7..04863b20 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -67,7 +67,7 @@ def test_stagehand_wrapper_example_creates_completed_trace(tmp_path, monkeypatch "SELECT id, name, status, error FROM runs ORDER BY started_at DESC LIMIT 1" ).fetchone() steps = c.execute( - "SELECT action, url, status, model_input, screenshot_path " + "SELECT action, url, status, model_input, screenshot_path, model_output " "FROM steps WHERE run_id=? ORDER BY step_index", (run[0],), ).fetchall() @@ -76,13 +76,17 @@ def test_stagehand_wrapper_example_creates_completed_trace(tmp_path, monkeypatch assert run[2] == "completed" assert run[3] is None assert [step[0] for step in steps] == [ + "observe: find the checkout button", "act: click the checkout button", "extract: extract the order total", ] assert steps[0][1] == "https://shop.example.test/cart" assert steps[0][2] == "ok" - assert json.loads(steps[0][3])["method"] == "act" + assert json.loads(steps[0][3])["method"] == "observe" assert steps[0][4].endswith(".png") + assert json.loads(steps[0][5])["stagehand_evidence"]["selectors"] == [ + "button.checkout.primary" + ] def test_browser_use_callback_demo_creates_completed_trace(tmp_path, monkeypatch): diff --git a/tests/test_stagehand_integration.py b/tests/test_stagehand_integration.py index aa4eff43..c9fe3f72 100644 --- a/tests/test_stagehand_integration.py +++ b/tests/test_stagehand_integration.py @@ -17,7 +17,22 @@ async def screenshot(self): return b"\x89PNG\r\n\x1a\n" + b"\x00" * 16 async def act(self, instruction: str): - return {"ok": True, "instruction": instruction} + return { + "ok": True, + "instruction": instruction, + "action": "click", + "selector": "button.login", + "attempts": 2, + "verification": {"status": "passed"}, + } + + async def extract(self, instruction: str): + return { + "status": "completed", + "instruction": instruction, + "data": {"text": "Welcome back", "selector": "h1"}, + "tool_calls": [{"name": "extract_text", "status": "ok"}], + } async def observe(self, instruction: str): return [ @@ -30,6 +45,27 @@ async def observe(self, instruction: str): ] +class NonSerializableResult: + def __init__(self): + self.selector = "button.nonserial" + self.payload = object() + + +class FakeNoScreenshotPage: + url = "https://example.com/no-shot" + + async def screenshot(self): + raise RuntimeError("screenshots disabled") + + async def act(self, instruction: str): + return NonSerializableResult() + + +class FakeFailurePage(FakeStagehandPage): + async def act(self, instruction: str): + raise RuntimeError(f"could not {instruction}") + + def test_stagehand_documented_bt_run_close_marks_completed(tmp_path): tracer = Tracer(home=tmp_path) page = wrap_stagehand(FakeStagehandPage(), tracer, name="stagehand close") @@ -59,7 +95,7 @@ def test_stagehand_records_method_result_as_model_output(tmp_path): model_input = json.loads(row[0]) model_output = json.loads(row[1]) - assert result == {"ok": True, "instruction": "click the login button"} + assert result["instruction"] == "click the login button" assert model_input["method"] == "act" assert model_output["result"] == result @@ -83,5 +119,93 @@ def test_stagehand_records_compact_evidence_from_observe_result(tmp_path): assert evidence["selectors"] == ["button.checkout.primary"] assert evidence["descriptions"] == ["Checkout button"] assert evidence["methods"] == ["click"] + assert evidence["observe_candidates"] == [ + { + "selector": "button.checkout.primary", + "description": "Checkout button", + "method": "click", + } + ] + + page.bt_run.close() + + +def test_stagehand_records_act_extract_evidence_and_metadata(tmp_path): + tracer = Tracer(home=tmp_path) + page = wrap_stagehand(FakeStagehandPage(), tracer, name="stagehand rich evidence") + + asyncio.run(page.act("click the login button")) + asyncio.run(page.extract("read the heading")) + + with sqlite3.connect(tmp_path / "db.sqlite") as c: + rows = c.execute( + "SELECT model_output, metadata FROM steps WHERE run_id=? ORDER BY step_index", + (page.bt_run.id,), + ).fetchall() + + act_output = json.loads(rows[0][0]) + act_metadata = json.loads(rows[0][1]) + extract_output = json.loads(rows[1][0]) + extract_metadata = json.loads(rows[1][1]) + + assert act_metadata == { + "stagehand_method": "act", + "instruction": "click the login button", + } + assert act_output["stagehand_evidence"]["selectors"] == ["button.login"] + assert act_output["stagehand_evidence"]["actions"] == ["click"] + assert act_output["stagehand_evidence"]["statuses"] == ["True", "passed"] + assert act_output["stagehand_evidence"]["attempts"] == ["2"] + assert act_output["stagehand_evidence"]["verification"] == ["{'status': 'passed'}"] + + assert extract_metadata["stagehand_method"] == "extract" + assert extract_output["stagehand_evidence"]["extracted_text"] == ["Welcome back"] + assert extract_output["stagehand_evidence"]["verification"] == [ + "{'name': 'extract_text', 'status': 'ok'}" + ] + + page.bt_run.close() + + +def test_stagehand_marks_failed_step_error(tmp_path): + tracer = Tracer(home=tmp_path) + page = wrap_stagehand(FakeFailurePage(), tracer, name="stagehand failure") + + try: + asyncio.run(page.act("click missing button")) + except RuntimeError: + pass + else: + raise AssertionError("expected RuntimeError") + + with sqlite3.connect(tmp_path / "db.sqlite") as c: + status, error = c.execute( + "SELECT status, error FROM steps WHERE run_id=?", + (page.bt_run.id,), + ).fetchone() + + assert status == "error" + assert "RuntimeError: could not click missing button" in error + + page.bt_run.close() + + +def test_stagehand_handles_non_serializable_result_and_missing_screenshot(tmp_path): + tracer = Tracer(home=tmp_path) + page = wrap_stagehand(FakeNoScreenshotPage(), tracer, name="stagehand no screenshot") + + asyncio.run(page.act("click nonserial")) + + with sqlite3.connect(tmp_path / "db.sqlite") as c: + model_output, screenshot_path = c.execute( + "SELECT model_output, screenshot_path FROM steps WHERE run_id=?", + (page.bt_run.id,), + ).fetchone() + + output = json.loads(model_output) + assert output["result"]["selector"] == "button.nonserial" + assert isinstance(output["result"]["payload"], str) + assert output["stagehand_evidence"]["selectors"] == ["button.nonserial"] + assert screenshot_path is None page.bt_run.close()