Skip to content

Commit babd023

Browse files
ralyodioclaude
andcommitted
feat(m3.2): page snapshots and ref actions (tron snapshot/click/fill)
Adds CDP-driven page automation on the M3.1 managed session (PRD M3.2): tron snapshot [--json] [--include-hidden] tron click @e3 tron fill @E4 "hi@example.com" Runtime pivot: snapshots/actions need programmatic CDP over a WebSocket, which the PRD's TS package layout (browser-core/sdk) already assumes. So these subcommands are implemented in TypeScript/Node and the shell `tron` dispatcher delegates to them, attaching to the session via the descriptor's webSocketDebuggerUrl (the M3.1 attach point). packages/browser-core/src/automation: - cdp-client.ts CDP JSON-RPC over Node's global WebSocket (no dependency). - snapshot-script.ts / action-script.ts in-page scripts. Snapshot tags each element with data-tron-ref so a later `tron click @e3` (a separate process) resolves the ref by attribute selector; a vanished element -> STALE_REF. - page.ts evaluate + StaleRefError + compact text formatting. - page-target.ts pick the session's current page target to drive. packages/browser-core/src/automate-cli.ts + automate-bin.ts: the `tron-automate` Node entry (snapshot/click/fill), deps injectable. Packaging: build-release.sh ships browser-core's self-contained dist tree as `automate/` (with a {"type":"module"} marker); the dispatcher runs it via node. Tests (37 new): CDP client over a real WebSocket, the in-page scripts against a real DOM (happy-dom), orchestration + CLI with fakes, and an end-to-end run of the real fetch + CdpClient transport against a mock DevTools server. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent ab04e2f commit babd023

19 files changed

Lines changed: 1537 additions & 24 deletions

apps/desktop/scripts/build-release.sh

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,25 @@ fetch_marksyncr() {
5151
[ -n "$MKS_SRC" ] || echo " ! MarkSyncr fetch skipped (non-fatal)"
5252
}
5353

54+
# The Node automation runtime for `tron snapshot|click|fill` (PRD M3.2). The
55+
# @tronbrowser/browser-core source has no runtime deps, so its compiled dist tree
56+
# is self-contained; ship it with a {"type":"module"} marker and the shell
57+
# dispatcher runs it via node. Best-effort like the extension fetches — a build
58+
# host without node/pnpm simply omits it (the CLI then reports "run tron upgrade").
59+
stage_automation() { # dest dir
60+
local s="$1"
61+
command -v node >/dev/null 2>&1 && command -v pnpm >/dev/null 2>&1 || {
62+
echo " ! automation runtime skipped (needs node + pnpm)"; return; }
63+
if ( cd "$REPO_ROOT" && pnpm --filter @tronbrowser/browser-core build >/dev/null 2>&1 ); then
64+
rm -rf "$s/automate"
65+
cp -R "$REPO_ROOT/packages/browser-core/dist" "$s/automate"
66+
printf '{\n "type": "module"\n}\n' > "$s/automate/package.json"
67+
echo " + bundled automation runtime (tron snapshot/click/fill)"
68+
else
69+
echo " ! automation runtime skipped (browser-core build failed)"
70+
fi
71+
}
72+
5473
stage() { # dest dir
5574
local s="$1"
5675
mkdir -p "$s/extensions"
@@ -62,6 +81,7 @@ stage() { # dest dir
6281
# Managed-session engine for `tron browser …` / `tron open` (PRD M3.1). Sits
6382
# next to the shim; the `tron` dispatcher resolves it relative to $CURRENT.
6483
install -m 0755 "$DESKTOP/launcher/tron-session" "$s/tron-session"
84+
stage_automation "$s"
6585
# -L dereferences the branding symlinks (icons/logo.svg -> repo-root logo.svg)
6686
# so the package contains real files, not dangling links.
6787
cp -RL "$DESKTOP/extensions/ai-sidebar" "$s/extensions/ai-sidebar"

apps/web/public/install.sh

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,9 @@ Usage:
8484
tron browser status Show managed-session status (--json for machine output)
8585
tron browser tabs List tabs in the managed session (--json)
8686
tron browser close Close the managed session
87+
tron snapshot Structured, ref-tagged page snapshot (--json)
88+
tron click <ref> Click a snapshot ref, e.g. @e3
89+
tron fill <ref> <val> Fill an input by ref, e.g. tron fill @e4 "hi@x.com"
8790
tron upgrade Update to the latest release
8891
tron remove Uninstall TronBrowser (keeps your profile data)
8992
tron version Print the installed version
@@ -131,6 +134,20 @@ session_bin() {
131134
echo "$_ld/tron-session"
132135
}
133136
137+
# Node automation runtime entry for `tron snapshot|click|fill|type` (M3.2).
138+
automate_entry() {
139+
_ld="$(dirname "$(readlink -f "$CURRENT" 2>/dev/null || echo "$CURRENT")")"
140+
echo "$_ld/automate/automate-bin.js"
141+
}
142+
143+
# Route a CDP automation subcommand to the Node runtime, or explain what's missing.
144+
run_automation() {
145+
ENTRY="$(automate_entry)"
146+
command -v node >/dev/null 2>&1 || { echo "tron $1 needs Node.js (>=22) on PATH." >&2; exit 1; }
147+
[ -f "$ENTRY" ] || { echo "This TronBrowser build lacks the automation runtime. Run: tron upgrade" >&2; exit 1; }
148+
exec node "$ENTRY" "$@"
149+
}
150+
134151
case "${1:-}" in
135152
open)
136153
shift
@@ -151,6 +168,9 @@ case "${1:-}" in
151168
SESSION="$(session_bin)"
152169
[ -x "$SESSION" ] || { echo "This TronBrowser build has no managed-session support (missing tron-session). Run: tron upgrade" >&2; exit 1; }
153170
exec "$SESSION" browser "$@" ;;
171+
snapshot|click|fill|type)
172+
# CDP automation on the managed session's current page (PRD M3.2).
173+
run_automation "$@" ;;
154174
restart)
155175
# Force-quit any running TronBrowser, then launch fresh. Chromium forwards a
156176
# new launch to an already-running instance (which keeps the OLD extension

docs/snapshots-and-refs.md

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# Snapshots and refs (M3.2)
2+
3+
Once a managed session is running (`tron browser launch`, see
4+
[managed-sessions.md](./managed-sessions.md)), the `tron` CLI can read the
5+
current page as a compact, ref-tagged structure and act on it by ref.
6+
7+
```sh
8+
tron snapshot # compact text snapshot of the current page
9+
tron snapshot --json # machine-readable snapshot
10+
tron snapshot --include-hidden
11+
tron click @e3 # click a ref from the last snapshot
12+
tron fill @e4 "hi@example.com" # fill an input/textarea by ref
13+
```
14+
15+
Text output:
16+
17+
```txt
18+
Page: Contact Us
19+
URL: https://example.com/contact
20+
21+
@e1 heading "Contact Us"
22+
@e2 textbox "Name"
23+
@e3 textbox "Email"
24+
@e4 link "Privacy" -> https://example.com/privacy
25+
@e5 button "Submit"
26+
```
27+
28+
## Refs
29+
30+
A snapshot assigns `@e1`, `@e2`, … to visible interactive elements (and
31+
headings) in document order and tags each element in the page with a
32+
`data-tron-ref` attribute. Because the ref lives in the DOM, a later
33+
`tron click @e3` — a separate process — resolves it with a plain attribute
34+
selector. If the element is gone (navigation, re-render), the action returns a
35+
recoverable **STALE_REF** error (exit code 5) telling you to re-`snapshot`,
36+
rather than acting on the wrong node. Prefer refs over CSS selectors for agents.
37+
38+
Password values are never echoed in snapshots; `--json` includes `role`,
39+
`name`, `value`, `href`, visibility, and interactivity per element.
40+
41+
## How it works
42+
43+
- `snapshot`/`click`/`fill` are Node subcommands the shell `tron` dispatcher
44+
delegates to. They attach to the session's current page via the descriptor's
45+
`webSocketDebuggerUrl` and drive it over the Chrome DevTools Protocol
46+
(`Runtime.evaluate`).
47+
- The CDP client uses Node's global `WebSocket` (Node >= 22) — no dependency.
48+
The runtime is `@tronbrowser/browser-core`'s compiled tree, shipped in the
49+
launcher payload; the dispatcher runs it with `node`.
50+
- Everything stays on `127.0.0.1` — no page content leaves the machine.
51+
52+
## Scope / limitations
53+
54+
- Requires Node.js (>= 22) on PATH, plus a running managed session.
55+
- The snapshot targets the session's current tab (`tron browser use <id>` to
56+
switch). Shadow DOM and cross-origin iframes are out of scope for M3.2.
57+
- Contracts and CDP/DOM logic are unit-tested in
58+
`packages/browser-core/src/automation` and `src/automate-*.test.ts`.

packages/browser-core/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
"lint": "eslint src"
2020
},
2121
"devDependencies": {
22+
"happy-dom": "^20.10.6",
2223
"typescript": "^5.6.3",
2324
"vitest": "^2.1.4"
2425
}
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
/**
2+
* Executable wrapper around the automation CLI. Built into a self-contained
3+
* `automate.js` (see apps/desktop/scripts/build-release.sh) that the shell
4+
* `tron` dispatcher runs via `node`.
5+
*/
6+
import { run } from './automate-cli.js';
7+
8+
run(process.argv.slice(2)).then(
9+
(code) => process.exit(code),
10+
(err: unknown) => {
11+
process.stderr.write(`tron: ${err instanceof Error ? err.message : String(err)}\n`);
12+
process.exit(1);
13+
},
14+
);
Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
import { describe, expect, it, vi } from 'vitest';
2+
import { EXIT, run, type CliDeps } from './automate-cli.js';
3+
import type { CdpConnection } from './automation/cdp-client.js';
4+
import type { AgentSnapshot } from './automation/snapshot-script.js';
5+
import type { SessionDescriptor } from './automation/types.js';
6+
7+
const descriptor: SessionDescriptor = {
8+
version: 1,
9+
pid: 1,
10+
host: '127.0.0.1',
11+
port: 9222,
12+
profileDir: '/x',
13+
profileName: 'agent',
14+
headless: false,
15+
ephemeral: false,
16+
createdAt: '2026-07-04T00:00:00.000Z',
17+
activeTabId: 'p1',
18+
};
19+
20+
const snap: AgentSnapshot = {
21+
url: 'https://example.com',
22+
title: 'Example',
23+
timestamp: '2026-07-04T00:00:00.000Z',
24+
elements: [
25+
{ ref: '@e1', role: 'link', name: 'More', tag: 'a', interactive: true, visible: true, href: 'https://x' },
26+
],
27+
};
28+
29+
/** A CdpConnection whose Runtime.evaluate yields `evalValue`. */
30+
function conn(evalValue: unknown): CdpConnection {
31+
return {
32+
send: (async (method: string) =>
33+
method === 'Runtime.evaluate' ? { result: { value: evalValue } } : {}) as CdpConnection['send'],
34+
on: vi.fn(),
35+
close: vi.fn(),
36+
};
37+
}
38+
39+
function harness(overrides: Partial<CliDeps> = {}) {
40+
const out: string[] = [];
41+
const err: string[] = [];
42+
const deps: Partial<CliDeps> = {
43+
env: {},
44+
loadDescriptor: async () => descriptor,
45+
fetchTargets: async () => [
46+
{ id: 'p1', type: 'page', url: 'https://example.com', webSocketDebuggerUrl: 'ws://x/p1' },
47+
],
48+
connect: async () => conn(snap),
49+
out: (t) => out.push(t),
50+
err: (t) => err.push(t),
51+
...overrides,
52+
};
53+
return { deps, out, err };
54+
}
55+
56+
describe('automate-cli run', () => {
57+
it('prints a text snapshot', async () => {
58+
const { deps, out } = harness();
59+
const code = await run(['snapshot'], deps);
60+
expect(code).toBe(EXIT.ok);
61+
expect(out.join('\n')).toContain('@e1 link "More"');
62+
});
63+
64+
it('prints JSON with --json', async () => {
65+
const { deps, out } = harness();
66+
await run(['snapshot', '--json'], deps);
67+
expect(JSON.parse(out.join('\n')).title).toBe('Example');
68+
});
69+
70+
it('clicks a ref', async () => {
71+
const { deps, out } = harness({ connect: async () => conn({ ok: true, ref: '@e1' }) });
72+
const code = await run(['click', '@e1'], deps);
73+
expect(code).toBe(EXIT.ok);
74+
expect(out.join('\n')).toContain('clicked @e1');
75+
});
76+
77+
it('fills a ref', async () => {
78+
const { deps, out } = harness({ connect: async () => conn({ ok: true, ref: '@e2' }) });
79+
const code = await run(['fill', '@e2', 'hello'], deps);
80+
expect(code).toBe(EXIT.ok);
81+
expect(out.join('\n')).toContain('filled @e2');
82+
});
83+
84+
it('exits staleRef when a ref no longer resolves', async () => {
85+
const { deps, err } = harness({
86+
connect: async () => conn({ ok: false, error: 'STALE_REF', ref: '@e9' }),
87+
});
88+
const code = await run(['click', '@e9'], deps);
89+
expect(code).toBe(EXIT.staleRef);
90+
expect(err.join('\n')).toMatch(/stale/i);
91+
});
92+
93+
it('exits noSession when there is no descriptor', async () => {
94+
const { deps, err } = harness({
95+
loadDescriptor: async () => {
96+
throw new Error('ENOENT');
97+
},
98+
});
99+
const code = await run(['snapshot'], deps);
100+
expect(code).toBe(EXIT.noSession);
101+
expect(err.join('\n')).toContain('tron browser launch');
102+
});
103+
104+
it('exits usage when click is missing a ref', async () => {
105+
const { deps } = harness();
106+
expect(await run(['click'], deps)).toBe(EXIT.usage);
107+
});
108+
});

0 commit comments

Comments
 (0)