diff --git a/packages/skills/skills/argent-tv-interact/SKILL.md b/packages/skills/skills/argent-tv-interact/SKILL.md index 918594ba2..cb2a50551 100644 --- a/packages/skills/skills/argent-tv-interact/SKILL.md +++ b/packages/skills/skills/argent-tv-interact/SKILL.md @@ -14,15 +14,16 @@ description: Control and inspect TV apps via argent — Apple TV (tvOS), Android ## The navigation loop 1. `describe` — find the cursor and your target (returns the focused element + all focusable ones, not a tap tree). -2. `tv-remote` — move focus toward the target. Prefer **one** call with a path ending in `select`, e.g. `{button:["down","right","select"]}`; count rows/columns from the frames to build the path. +2. `tv-remote` — move focus toward the target. Prefer **one** call with a path ending in `select`, e.g. `{button:["down","right","select"]}`; count rows/columns from the order of the focusable list (the cursor is marked) to build the path. 3. `describe` again to confirm. On a miss, repeat. ## Tools -- `describe {udid}` — focus view: the focused / `[selected]` element + focusable elements with labels and normalized frames. The discovery tool — call before and after navigating. Empty tree → see the per-platform notes. +- `describe {udid}` — focus view: the focused / `[selected]` element + focusable elements with labels, traits and values. It does not print coordinates — a TV is navigated with the D-pad, never by tapping. The discovery tool — call before and after navigating. Empty tree → see the per-platform notes. - `tv-remote {udid, button}` — D-pad / remote. `button` is one key **or a whole path** (run in one call). Keys: `up`/`down`/`left`/`right`, `select`, `back`, `menu`, `home`, `playPause`, plus media keys `rewind`/`fastForward`/`next`/`previous`/`volumeUp`/`volumeDown`/`mute`. Single: `{button:"down"}`; repeat: `{button:"down", repeat:3}`; path: `{button:["up","right","select"]}`. - `keyboard {udid, text}` — type into the focused field (focus it with `tv-remote` first). Named `key` presses (e.g. `{key:"enter"}`) work on Vega; on Apple TV / Android TV move focus with `tv-remote` instead. - `launch-app` / `restart-app` / `reinstall-app {udid, bundleId}` — `bundleId` from the app manifest. Vega `reinstall-app` takes `appPath` = a `.vpkg`. +- `await-ui-element {udid, …}` / `await-screen-idle {udid}` — wait for the TV to be ready instead of guessing a delay. They poll the same focus view `describe` reads. `visible` means the same as `exists` there (the focus engine only enumerates what is on screen and reachable); wait for the cursor with `{condition:"exists", selector:{text:"X", role:"focused"}}`. `await-screen-idle` settles once the app, the focusable set and the cursor stop changing — playback or animation the focus engine cannot see will not hold it unsettled. - `screenshot {udid, scale?}` — Apple TV via `xcrun simctl io` (downscaled); Android TV / Vega host-side via `adb` / `screencap`. ## Per-platform diff --git a/packages/tool-server/src/blueprints/tv-control-types.ts b/packages/tool-server/src/blueprints/tv-control-types.ts index 5356389b1..29d2e52e3 100644 --- a/packages/tool-server/src/blueprints/tv-control-types.ts +++ b/packages/tool-server/src/blueprints/tv-control-types.ts @@ -16,6 +16,16 @@ export interface TvElement { traits?: string[]; value?: string; isFocused?: boolean; + /** + * Normalized 0..1 rect. The tvOS daemon has always reported this (and a + * `tapPoint`) — the field simply went undeclared, while `describe`'s focus + * rendering drops it because a TV is navigated with the D-pad rather than by + * coordinate. Declared now because the wait tools adapt this element into a + * describe tree, where the frame drives visibility and reading order. + * Absent on backends that report no bounds (Android TV's focus view) and for + * zero-size elements, so every consumer must tolerate it missing. + */ + frame?: { x: number; y: number; width: number; height: number }; } export interface TvDescribeResponse { diff --git a/packages/tool-server/src/tools/await-screen-idle/index.ts b/packages/tool-server/src/tools/await-screen-idle/index.ts index fb3bf5dbf..1a6852afb 100644 --- a/packages/tool-server/src/tools/await-screen-idle/index.ts +++ b/packages/tool-server/src/tools/await-screen-idle/index.ts @@ -18,6 +18,8 @@ import type { DescribeNode, DescribeTreeData } from "../describe/contract"; import { describeIos, iosRequires } from "../describe/platforms/ios"; import { describeAndroid, androidRequires } from "../describe/platforms/android"; import { describeChromium } from "../describe/platforms/chromium"; +import { describeTvFocus } from "../describe/platforms/tv-focus"; +import { resolveTvApi } from "../tv/tv-service"; export const AWAIT_SCREEN_IDLE_TOOL_ID = "await-screen-idle"; @@ -66,6 +68,13 @@ interface IdleResult { waitedMs: number; /** Number of tree reads taken. */ polls: number; + /** + * Why it did not settle, when the last read said something useful — a + * degraded accessibility read, a still-launching TV app. Absent on success. + * Without it an unsettled result is a silent stall: the caller waits the full + * budget and is told only `settled: false` (#620). + */ + note?: string; } const capability: ToolCapability = { @@ -92,18 +101,52 @@ function treeSignature(root: DescribeNode): string { return parts.join("\n"); } +/** + * Explain an unsettled wait from whatever the last read reported. Mirrors the + * diagnostics `await-ui-element` folds onto its timeout note — this tool had no + * equivalent, so a caller got a silent stall on a degraded read. + */ +function unsettledNote( + lastData: DescribeTreeData | null, + lastError: string | undefined +): string | undefined { + if (lastError) return `last tree read failed: ${lastError}`; + if (!lastData) return undefined; + // A non-empty tree that never held still is self-explanatory: the screen was + // genuinely moving. Only an empty one needs a reason. + if (lastData.tree.children.length > 0) return undefined; + const parts: string[] = ["the screen reported no content"]; + if (lastData.should_restart) { + parts.push("the foreground app may need a restart for native inspection"); + } + if (lastData.hint) parts.push(lastData.hint); + return parts.join("; "); +} + // `await-screen-idle` waits for the screen to *settle* — render content and stop // changing — rather than for a named element like `await-ui-element`. The MCP // layer uses it to time its auto-screenshot: capture once the screen is stable // instead of after a fixed delay. export function createAwaitScreenIdleTool(registry: Registry): ToolDefinition { - function fetchTree( + async function fetchTree( device: DeviceInfo, services: Record, isTvOs: boolean, androidIsTv: boolean ): Promise { if (device.platform === "ios") { + // Apple TV: `describeIos` short-circuits every tvOS read to an empty tree, + // so this tool could never settle there (#620). Poll the focus view the + // `describe` tool already uses successfully instead. + // + // Resolved lazily, INSIDE the fetch, on purpose: the first resolution + // spawns the tvOS ax/HID daemons and can take seconds. pollDescribeTree + // already races each fetch against the remaining deadline, and the + // registry caches the running service, so only the first poll pays — and + // it can never overrun the caller's budget. Resolving up front and + // bounding it separately would double-count the wait against a timeout + // this tool exists to respect. + if (isTvOs) return describeTvFocus(await resolveTvApi(registry, device.id)); return describeIos(registry, device, {}, { isTvOs }); } if (device.platform === "android") { @@ -125,8 +168,12 @@ export function createAwaitScreenIdleTool(registry: Registry): ToolDefinition, isTvOs: boolean, - androidIsTv: boolean + androidIsTv: boolean, + tvApi: TvControlApi | null ): Promise { if (device.platform === "ios") { + // Apple TV: match against the focus view. `describeIos` short-circuits + // every tvOS read to an empty tree, so no selector could ever resolve + // (#620). Resolution happens once, up front (see execute) so a backend + // failure still throws rather than being reported as an unmet condition. + if (isTvOs && tvApi) return describeTvFocus(tvApi); return describeIos(registry, device, { bundleId: params.bundleId }, { isTvOs }); } if (device.platform === "android") { @@ -271,9 +280,14 @@ The selector is { text?, identifier?, role? }; every provided field must match. case-insensitive substrings of the element's label/value and role; identifier matches exactly (case-insensitive), also accepting the unqualified Android resource-id name ('submit' matches 'com.example.app:id/submit'). It polls the same accessibility / DOM tree as \`describe\` -(iOS AXRuntime, Android uiautomator, Chromium CDP, Vega automation toolkit) every pollIntervalMs +(iOS AXRuntime, Android uiautomator, Chromium CDP, Apple TV focus engine, Vega automation toolkit) every pollIntervalMs (default ${DEFAULT_POLL_INTERVAL_MS}ms) until timeoutMs (default ${DEFAULT_TIMEOUT_MS}ms). +On an Apple TV the tree is the focus view: \`visible\` means the same as \`exists\` there, because the +focus engine only enumerates what is on screen and reachable with the D-pad; \`role\` matches the +element's accessibility traits, and \`{role:"focused"}\` targets whichever element currently holds +the cursor — the wait to use between \`tv-remote\` and \`select\`. \`identifier\` does not apply. +Android TV and Vega keep their full element trees. Returns { success: boolean, elapsed: number } — success=false means the condition never held before the timeout (a \`note\` then explains what was seen). Use this after a tap/navigation to wait for the next screen, or before tapping an element that appears asynchronously.`, @@ -304,6 +318,12 @@ or before tapping an element that appears asynchronously.`, // the Android TV probe: a serial that isn't listed is never cached, so // leaving it inside `describeAndroid` would spawn `adb devices` per poll. const isTvOs = device.platform === "ios" && (await isTvOsSimulator(device.id)); + // Resolved before the clock starts, like every other setup step here: the + // first resolution spawns the tvOS daemons and can take seconds, which + // must not eat the caller's wait budget. A failure here is an + // infrastructure problem, so it throws — reporting it as `success: false` + // would make run-sequence call it an unmet condition. + const tvApi = isTvOs ? await resolveTvApi(registry, device.id) : null; const androidIsTv = device.platform === "android" && (await isAndroidTv(device.id)); // Start the wait clock after setup so its fixed cost isn't charged against @@ -325,7 +345,7 @@ or before tapping an element that appears asynchronously.`, let everMatched = false; const poll = await pollDescribeTree({ - fetchTree: () => fetchTree(device, params, services, isTvOs, androidIsTv), + fetchTree: () => fetchTree(device, params, services, isTvOs, androidIsTv, tvApi), timeoutMs, pollIntervalMs, signal, diff --git a/packages/tool-server/src/tools/describe/platforms/tv-focus.ts b/packages/tool-server/src/tools/describe/platforms/tv-focus.ts new file mode 100644 index 000000000..7b2d69142 --- /dev/null +++ b/packages/tool-server/src/tools/describe/platforms/tv-focus.ts @@ -0,0 +1,144 @@ +import type { + TvControlApi, + TvDescribeResponse, + TvElement, +} from "../../../blueprints/tv-control-types"; +import type { DescribeNode, DescribeTreeData } from "../contract"; + +/** + * The TV focus view, adapted into the ordinary `DescribeNode` tree so the wait + * tools can poll it. + * + * `describe` renders this same source for humans (see `./tv.ts`); this module + * exists because `await-screen-idle` / `await-ui-element` need a *tree* to + * fingerprint and match against, and because they must NOT inherit describe's + * retry-and-recycle behaviour — see {@link describeTvFocus}. + * + * Why they need it at all: `describeIos` short-circuits every tvOS read to an + * empty tree (the iOS accessibility service cannot drive an Apple TV), so both + * wait tools saw a permanently empty screen and could never settle or match — + * issue #620. + */ + +/** Synthetic root the focusables hang off, mirroring the other platforms' shape. */ +function focusRoot(): DescribeNode { + return { role: "AXGroup", frame: { x: 0, y: 0, width: 1, height: 1 }, children: [] }; +} + +/** + * Shared cause text for an empty focus set, so `describe` and the wait tools + * explain it the same way. Kept separate from the advice, which differs: only + * `describe` actually performs the retry-and-recycle it can then talk about. + */ +export const TV_EMPTY_FOCUS_CAUSE = + "The app is most likely still launching (splash / loading screen) or mid-transition — a React " + + "Native app only exposes focus once its JS bundle has rendered."; + +/** What a wait tool says: it diagnoses, and points at the tool that repairs. */ +export const TV_FOCUS_WAIT_EMPTY_HINT = + `The TV focus engine reported no focusable elements. ${TV_EMPTY_FOCUS_CAUSE} ` + + "Call `describe` once — it retries and recycles the tvOS read path — then wait again."; + +/** A focus read is "empty" when nothing actionable was reported. */ +export function isEmptyFocus(res: TvDescribeResponse): boolean { + return res.focusable.length === 0 && !res.focused; +} + +/** + * The tvOS daemon reports a normalized frame per element, but `TvElement` has + * historically not declared it (the JSON is passed through by reference, so the + * data is there at runtime). Read it defensively: Android TV's focus backend + * genuinely omits bounds, and the tvOS daemon drops `frame` for a zero-size + * element. + */ +function frameOf(element: TvElement, index: number, total: number): DescribeNode["frame"] { + const raw = element.frame; + if (raw && raw.width > 0 && raw.height > 0) { + return { x: raw.x, y: raw.y, width: raw.width, height: raw.height }; + } + // Fallback: a non-degenerate band per element, ordered by enumeration index. + // `isVisible` requires a non-zero area, and both backends enumerate in + // traversal order, so index order IS reading order (android-tv-control.ts + // reverses its child push specifically to guarantee that). + const slots = Math.max(total, 1); + return { x: 0, y: index / slots, width: 1, height: 1 / slots }; +} + +/** + * Synthetic trait marking the cursor. Carried in `role` — rather than only in + * the `focused` field — so it does two jobs the field cannot: + * + * - it is selectable, making "wait until focus lands on X" expressible as + * `{ selector: { text: "X", role: "focused" }, condition: "exists" }`, which + * is the wait a TV `run-sequence` actually needs between `tv-remote` and + * `select`; + * - it puts the cursor into the idle fingerprint, so a screen whose focus is + * still moving does not read as settled. + * + * Safe as a `role` token: role matching is a case-insensitive substring, and no + * real trait on either backend contains "focused" (`_focusGuide` and + * `_tvFocusable` do not). + */ +const FOCUSED_TRAIT = "focused"; + +function toNode(element: TvElement, index: number, total: number): DescribeNode { + const traits = [...(element.traits ?? [])]; + if (element.isFocused) traits.push(FOCUSED_TRAIT); + return { + // Traits are what a selector's `role` matches, exactly as on a phone. + role: traits.length > 0 ? traits.join(",") : "element", + frame: frameOf(element, index, total), + children: [], + ...(element.label ? { label: element.label } : {}), + ...(element.value ? { value: element.value } : {}), + // The cursor. `format-tree` already renders this as [focused], and it is + // what makes "wait until focus lands on X" expressible. + ...(element.isFocused ? { focused: true } : {}), + }; +} + +/** + * Adapt a focus read into a describe tree. + * + * The root carries the foreground bundle id, so a wait notices the app itself + * changing underneath it — a TV transition often swaps the whole app, not just + * the focusable set. + */ +export function tvFocusTree(res: TvDescribeResponse): DescribeTreeData { + if (isEmptyFocus(res)) { + // The hint is load-bearing, not decoration: `await-ui-element` treats an + // empty tree as an untrustworthy read ONLY when a hint (or a prior match) + // says so. Without it, `condition: "hidden"` would report success on the + // very first poll of a still-launching app — a false pass that would + // release a gated interaction. + return { tree: focusRoot(), source: "tv-focus", hint: TV_FOCUS_WAIT_EMPTY_HINT }; + } + + const elements = [...res.focusable]; + // Some reads report a focused element that is absent from the focusable list; + // it still has to be matchable, and it is the single most useful node here. + // + // Detect that by looking for a focusable already MARKED focused, not by object + // identity: the two arrive as separate objects from the same JSON payload, so + // an identity check would append a duplicate of an element that is already + // there — and the cursor would then match twice. + if (res.focused && !res.focusable.some((e) => e.isFocused)) elements.push(res.focused); + + const root = focusRoot(); + root.children = elements.map((el, i) => toNode(el, i, elements.length)); + if (res.bundleId) root.label = res.bundleId; + return { tree: root, source: "tv-focus" }; +} + +/** + * One bare focus read, for the wait tools. + * + * Deliberately NOT `describeTv`: that sleeps between empty probes and can + * respawn the tvOS ax daemon (`recycleAx`). Inside a 200ms poll loop the sleeps + * are redundant and the respawn is destructive — it would drop the very state + * the caller is waiting on. Repair stays in the one-shot tool; a wait only + * observes and reports. + */ +export async function describeTvFocus(api: TvControlApi): Promise { + return tvFocusTree(await api.describe()); +} diff --git a/packages/tool-server/src/tools/describe/platforms/tv.ts b/packages/tool-server/src/tools/describe/platforms/tv.ts index 7502cae74..2d54f5e41 100644 --- a/packages/tool-server/src/tools/describe/platforms/tv.ts +++ b/packages/tool-server/src/tools/describe/platforms/tv.ts @@ -1,3 +1,4 @@ +import { isEmptyFocus } from "./tv-focus"; import type { DeviceInfo, Registry } from "@argent/registry"; import type { DescribeResult } from "../contract"; import { formatDescribeTree } from "../format-tree"; @@ -47,10 +48,11 @@ const ANDROID_FOCUS_EMPTY_HINT = "these screens even though the labels aren't enumerable, so you can drive blind + screenshot " + "to confirm."; -/** A describe result is "empty" when the focus engine reports nothing actionable. */ -function isEmpty(res: TvDescribeResponse): boolean { - return res.focusable.length === 0 && !res.focused; -} +/** + * Shared with the wait tools' focus adapter so the two paths can never disagree + * about what "empty" means — the hint they each attach depends on it. + */ +const isEmpty = isEmptyFocus; /** * tvOS AX labels are often compound multi-line strings, e.g. diff --git a/packages/tool-server/test/await-tv-focus.test.ts b/packages/tool-server/test/await-tv-focus.test.ts new file mode 100644 index 000000000..38b5fbc72 --- /dev/null +++ b/packages/tool-server/test/await-tv-focus.test.ts @@ -0,0 +1,267 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { __primeDepCacheForTests, __resetDepCacheForTests } from "../src/utils/check-deps"; +import type { TvControlApi, TvDescribeResponse } from "../src/blueprints/tv-control-types"; + +/** + * Issue #620: on an Apple TV both wait tools polled `describeIos`, which + * short-circuits every tvOS read to an empty tree. `await-screen-idle` therefore + * could never settle — it reset on every poll and burned the whole budget + * without saying why — and no selector could ever match. + * + * These pin the routing (focus view, not the iOS AX service) and the two + * properties that make it safe: the tvOS daemon is never repaired from inside a + * poll loop, and Android TV keeps its full uiautomator tree. + */ + +// Pin the form-factor probe: the real one shells out to `xcrun simctl list`. +vi.mock("../src/utils/ios-devices", async () => { + const actual = await vi.importActual( + "../src/utils/ios-devices" + ); + return { ...actual, isTvOsSimulator: async () => true }; +}); + +const describeAndroidMock = vi.fn(); +vi.mock("../src/tools/describe/platforms/android", async () => { + const actual = await vi.importActual( + "../src/tools/describe/platforms/android" + ); + return { ...actual, describeAndroid: (...a: unknown[]) => describeAndroidMock(...a) }; +}); + +// describeIos must NOT be reached for a tvOS target — that is the bug. +const describeIosMock = vi.fn(); +vi.mock("../src/tools/describe/platforms/ios", async () => { + const actual = await vi.importActual( + "../src/tools/describe/platforms/ios" + ); + return { ...actual, describeIos: (...a: unknown[]) => describeIosMock(...a) }; +}); + +const resolveTvApiMock = vi.fn(); +vi.mock("../src/tools/tv/tv-service", () => ({ + resolveTvApi: (...a: unknown[]) => resolveTvApiMock(...a), + tvServiceRef: vi.fn(), +})); + +import { createAwaitScreenIdleTool } from "../src/tools/await-screen-idle"; +import { createAwaitUiElementTool } from "../src/tools/await-ui-element"; + +const TV_UDID = "DDDDDDDD-DDDD-DDDD-DDDD-DDDDDDDDDDDD"; + +function row(label: string, y: number, extra: Record = {}) { + return { + label, + frame: { x: 0.55, y, width: 0.4, height: 0.06 }, + traits: ["button", "_focusGuide"], + ...extra, + }; +} + +const SCREEN: TvDescribeResponse = { + bundleId: "com.apple.TVSettings", + focused: row("About", 0.175, { isFocused: true }), + focusable: [ + row("About", 0.175, { isFocused: true }), + row("Appearance", 0.249, { value: "Dark" }), + ], +}; + +/** A TvControlApi that walks `frames`, repeating the last. */ +function makeTvApi(frames: TvDescribeResponse[]): TvControlApi & { calls: () => number } { + let i = 0; + const describe = vi.fn(async () => frames[Math.min(i++, frames.length - 1)]!); + const recycleAx = vi.fn(async () => {}); + return { + describe, + recycleAx, + navigate: vi.fn(), + type: vi.fn(), + calls: () => describe.mock.calls.length, + } as unknown as TvControlApi & { calls: () => number }; +} + +const registry = { resolveService: vi.fn() } as never; + +beforeEach(() => { + vi.clearAllMocks(); + __primeDepCacheForTests(["xcrun", "adb"]); +}); + +afterEach(() => { + __resetDepCacheForTests(); +}); + +async function runIdle(api: TvControlApi, params: Record = {}) { + resolveTvApiMock.mockResolvedValue(api); + const tool = createAwaitScreenIdleTool(registry); + return tool.execute({}, { udid: TV_UDID, timeoutMs: 2000, ...params } as never); +} + +async function runElement(api: TvControlApi, params: Record) { + resolveTvApiMock.mockResolvedValue(api); + const tool = createAwaitUiElementTool(registry); + return tool.execute({}, { udid: TV_UDID, timeoutMs: 1500, ...params } as never); +} + +describe("await-screen-idle on an Apple TV", () => { + it("settles on a static focus view", async () => { + // Was structurally impossible: every poll saw an empty tree and reset. + const api = makeTvApi([SCREEN]); + + const res = await runIdle(api); + + expect(res.settled).toBe(true); + expect(res.note).toBeUndefined(); + expect(describeIosMock).not.toHaveBeenCalled(); + }); + + it("does not settle while the cursor is still moving", async () => { + const moved: TvDescribeResponse = { + ...SCREEN, + focused: row("Appearance", 0.249, { isFocused: true }), + focusable: [row("About", 0.175), row("Appearance", 0.249, { isFocused: true })], + }; + // Alternate so the fingerprint never repeats. + const api = makeTvApi([SCREEN, moved, SCREEN, moved, SCREEN, moved, SCREEN, moved]); + + expect((await runIdle(api, { timeoutMs: 900 })).settled).toBe(false); + }); + + it("does not settle while the focusable set is still growing", async () => { + const grown: TvDescribeResponse = { + ...SCREEN, + focusable: [...SCREEN.focusable, row("Region", 0.323, { value: "Poland" })], + }; + const api = makeTvApi([SCREEN, grown, SCREEN, grown, SCREEN, grown, SCREEN, grown]); + + expect((await runIdle(api, { timeoutMs: 900 })).settled).toBe(false); + }); + + it("explains an empty focus view instead of stalling silently", async () => { + // The reported symptom was `{settled:false}` with no note at all after a + // full 30s — nothing for the agent to act on. + const api = makeTvApi([{ focused: null, focusable: [] }]); + + const res = await runIdle(api, { timeoutMs: 600 }); + + expect(res.settled).toBe(false); + expect(res.note).toMatch(/no focusable elements|launching|transition/i); + // …and not the old advice to go and use describe instead of this tool. + expect(res.note).not.toMatch(/accessibility service does not support/i); + }); + + it("never repairs the read path from inside the poll loop", async () => { + // describeTv retries and can respawn the tvOS ax daemon. Doing that mid-wait + // would drop the very state being watched, so the wait path takes one bare + // read per poll and points at `describe` for the repair. + const api = makeTvApi([{ focused: null, focusable: [] }]); + + const res = await runIdle(api, { timeoutMs: 600 }); + + expect( + (api as unknown as { recycleAx: { mock: { calls: unknown[] } } }).recycleAx.mock.calls + ).toHaveLength(0); + expect(api.calls()).toBe(res.polls); + }); +}); + +describe("await-ui-element on an Apple TV", () => { + it("matches a focusable by label", async () => { + const res = await runElement(makeTvApi([SCREEN]), { + condition: "exists", + selector: { text: "Appearance" }, + }); + + expect(res.success).toBe(true); + expect(describeIosMock).not.toHaveBeenCalled(); + }); + + it("treats every enumerated element as visible", async () => { + // A focus view has no "present but invisible" state — what it reports is + // exactly the on-screen, D-pad-reachable set. So `visible` matches an + // element the cursor is not on. + const res = await runElement(makeTvApi([SCREEN]), { + condition: "visible", + selector: { text: "Appearance" }, + }); + + expect(res.success).toBe(true); + }); + + it("reads a row's value for the text condition", async () => { + const res = await runElement(makeTvApi([SCREEN]), { + condition: "text", + selector: { text: "Appearance" }, + expectedText: "Dark", + }); + + expect(res.success).toBe(true); + }); + + it("waits for the cursor to land on a specific element", async () => { + const moved: TvDescribeResponse = { + ...SCREEN, + focused: row("Appearance", 0.249, { isFocused: true }), + focusable: [row("About", 0.175), row("Appearance", 0.249, { isFocused: true })], + }; + const api = makeTvApi([SCREEN, SCREEN, moved]); + + const res = await runElement(api, { + condition: "exists", + selector: { text: "Appearance", role: "focused" }, + }); + + expect(res.success).toBe(true); + }); + + it("does not report `hidden` satisfied on an empty focus view", async () => { + // The dangerous false pass: a still-launching app reports nothing, and + // "nothing matched" would read as "the element is gone" — releasing an + // interaction that was deliberately gated. + const api = makeTvApi([{ focused: null, focusable: [] }]); + + const res = await runElement(api, { + condition: "hidden", + selector: { text: "About" }, + timeoutMs: 600, + }); + + expect(res.success).toBe(false); + expect(res.note).toMatch(/focus|launching|empty/i); + }); + + it("reports `hidden` satisfied when the element leaves a populated view", async () => { + const without: TvDescribeResponse = { + ...SCREEN, + focused: row("Appearance", 0.249, { isFocused: true }), + focusable: [row("Appearance", 0.249, { value: "Dark", isFocused: true })], + }; + const api = makeTvApi([SCREEN, without]); + + const res = await runElement(api, { condition: "hidden", selector: { text: "About" } }); + + expect(res.success).toBe(true); + }); +}); + +describe("Android TV keeps its full tree", () => { + it("is not routed onto the focus view", async () => { + // An empty focus set is STEADY STATE on Android TV — react-native-tvos + // screens drive focus with RN's own engine, invisible to the OS tree. Moving + // it onto the focus source would import the never-settles bug onto a + // platform that works. + describeAndroidMock.mockResolvedValue({ + tree: { role: "root", frame: { x: 0, y: 0, width: 1, height: 1 }, children: [] }, + source: "uiautomator", + }); + const api = makeTvApi([SCREEN]); + resolveTvApiMock.mockResolvedValue(api); + + const tool = createAwaitScreenIdleTool(registry); + await tool.execute({}, { udid: "emulator-5554", timeoutMs: 400 } as never); + + expect(describeAndroidMock).toHaveBeenCalled(); + expect(api.calls()).toBe(0); + }); +}); diff --git a/packages/tool-server/test/tv-focus-tree.test.ts b/packages/tool-server/test/tv-focus-tree.test.ts new file mode 100644 index 000000000..7afba787d --- /dev/null +++ b/packages/tool-server/test/tv-focus-tree.test.ts @@ -0,0 +1,163 @@ +import { describe, it, expect } from "vitest"; +import { + tvFocusTree, + isEmptyFocus, + TV_FOCUS_WAIT_EMPTY_HINT, +} from "../src/tools/describe/platforms/tv-focus"; +import type { TvDescribeResponse } from "../src/blueprints/tv-control-types"; +import { findAll, isVisible, firstInReadingOrder } from "../src/utils/ui-tree-match"; + +/** + * The adapter behind issue #620: `describeIos` short-circuits every tvOS read to + * an empty tree, so the wait tools could never settle or match on an Apple TV. + * This turns the focus view — the source `describe` already uses successfully — + * into a tree they can poll. + * + * Payloads below mirror a real tvOS daemon response (captured from + * com.apple.TVSettings): normalized frames, compound traits, values on rows. + */ + +const SETTINGS: TvDescribeResponse = { + bundleId: "com.apple.TVSettings", + focused: { + label: "About", + frame: { x: 0.552, y: 0.175, width: 0.406, height: 0.061 }, + traits: ["button", "_focusGuide"], + isFocused: true, + }, + focusable: [ + { + label: "About", + frame: { x: 0.552, y: 0.175, width: 0.406, height: 0.061 }, + traits: ["button", "_focusGuide"], + isFocused: true, + }, + { + label: "Appearance", + value: "Dark", + frame: { x: 0.552, y: 0.249, width: 0.406, height: 0.061 }, + traits: ["button", "_focusGuide"], + }, + { + label: "Region", + value: "Poland", + frame: { x: 0.552, y: 0.323, width: 0.406, height: 0.061 }, + traits: ["button", "_focusGuide"], + }, + ], +}; + +describe("tvFocusTree — the focus view as a describe tree", () => { + it("keeps the real frames the daemon reports", () => { + // The frames were always in the payload; only the TS type and describe's + // rendering dropped them. Using them means isVisible and reading order mean + // the same thing here as on a phone. + const first = tvFocusTree(SETTINGS).tree.children[0]!; + + expect(first.frame).toEqual({ x: 0.552, y: 0.175, width: 0.406, height: 0.061 }); + expect(isVisible(first)).toBe(true); + }); + + it("falls back to an ordered, non-degenerate frame when a backend reports none", () => { + // Android TV's focus view genuinely has no bounds, and the tvOS daemon omits + // the frame for a zero-size element. Neither may end up invisible or + // unordered. + const noFrames: TvDescribeResponse = { + focused: null, + focusable: [{ label: "One" }, { label: "Two" }, { label: "Three" }], + }; + + const nodes = tvFocusTree(noFrames).tree.children; + + expect(nodes.every((n) => n.frame.width > 0 && n.frame.height > 0)).toBe(true); + expect(nodes.every(isVisible)).toBe(true); + expect(nodes.map((n) => n.frame.y)).toEqual([0, 1 / 3, 2 / 3]); + expect(firstInReadingOrder(nodes)?.label).toBe("One"); + }); + + it("carries the app id on the root so a whole-app swap is noticed", () => { + const { tree } = tvFocusTree(SETTINGS); + + expect(tree.label).toBe("com.apple.TVSettings"); + // …but the root itself must not be matchable, or every selector would hit it. + expect(findAll(tree, { text: "com.apple.TVSettings" })).toHaveLength(0); + }); + + it("exposes labels, values and traits to the selector matcher", () => { + const { tree } = tvFocusTree(SETTINGS); + + expect(findAll(tree, { text: "Appearance" })).toHaveLength(1); + // Values matter: a settings row's state lives there, and `condition: "text"` + // reads it. + expect(findAll(tree, { text: "Dark" })).toHaveLength(1); + expect(findAll(tree, { role: "button" })).toHaveLength(3); + }); + + it("marks the cursor so it is both selectable and fingerprinted", () => { + const { tree } = tvFocusTree(SETTINGS); + + const focused = findAll(tree, { role: "focused" }); + expect(focused).toHaveLength(1); + expect(focused[0]!.label).toBe("About"); + // The field is set too, so format-tree renders [focused] as on Vega. + expect(focused[0]!.focused).toBe(true); + // And it lands in `role`, which the idle fingerprint hashes — so a cursor + // move alone makes the screen unsettled. + expect(focused[0]!.role).toContain("focused"); + }); + + it("does not duplicate the cursor when it also appears in the focusable list", () => { + // `focused` and its twin in `focusable` arrive as separate objects from the + // same JSON, so an identity check would append a second copy and the cursor + // would match twice. + const { tree } = tvFocusTree(SETTINGS); + + expect(tree.children).toHaveLength(3); + expect(tree.children.filter((c) => c.role.includes("focused"))).toHaveLength(1); + }); + + it("still surfaces a cursor that is missing from the focusable list", () => { + const orphan: TvDescribeResponse = { + focused: { label: "Orphan", traits: ["button"], isFocused: true }, + focusable: [{ label: "Other", traits: ["button"] }], + }; + + const { tree } = tvFocusTree(orphan); + + expect(tree.children).toHaveLength(2); + expect(findAll(tree, { text: "Orphan", role: "focused" })).toHaveLength(1); + }); +}); + +describe("tvFocusTree — the empty read", () => { + it("is empty only when nothing actionable was reported", () => { + expect(isEmptyFocus({ focused: null, focusable: [] })).toBe(true); + // A cursor with no enumerable siblings is still something to act on. + expect(isEmptyFocus({ focused: { label: "X" }, focusable: [] })).toBe(false); + }); + + it("attaches a hint, which is what stops `hidden` false-passing", () => { + // Load-bearing, not decoration: await-ui-element treats an empty tree as an + // untrustworthy read ONLY when a hint says so. Without this, a wait for + // `hidden` would succeed on the first poll of a still-launching app and + // release a gated interaction. + const empty = tvFocusTree({ focused: null, focusable: [] }); + + expect(empty.tree.children).toHaveLength(0); + expect(empty.hint).toBe(TV_FOCUS_WAIT_EMPTY_HINT); + expect(empty.hint).toMatch(/launching|transition/i); + }); + + it("does not tell the agent to use describe instead — the wait tool works now", () => { + // The old tvOS note redirected to describe/tv-remote because nothing else + // worked. Repeating that here would be advice to abandon a tool that has + // just been fixed. + const empty = tvFocusTree({ focused: null, focusable: [] }); + + expect(empty.hint).not.toMatch(/accessibility service does not support/i); + }); + + it("never attaches a hint to a populated read", () => { + expect(tvFocusTree(SETTINGS).hint).toBeUndefined(); + }); +});