fix(tv): give Apple TV a working wait primitive

packages/skills/skills/argent-tv-interact/SKILL.md

@@ -14,15 +14,16 @@ description: Control and inspect TV apps via argent — Apple TV (tvOS), Android
 ## The navigation loop
 
 1. `describe` — find the cursor and your target (returns the focused element + all focusable ones, not a tap tree).
-2. `tv-remote` — move focus toward the target. Prefer **one** call with a path ending in `select`, e.g. `{button:["down","right","select"]}`; count rows/columns from the frames to build the path.
+2. `tv-remote` — move focus toward the target. Prefer **one** call with a path ending in `select`, e.g. `{button:["down","right","select"]}`; count rows/columns from the order of the focusable list (the cursor is marked) to build the path.
 3. `describe` again to confirm. On a miss, repeat.
 
 ## Tools
 
-- `describe {udid}` — focus view: the focused / `[selected]` element + focusable elements with labels and normalized frames. The discovery tool — call before and after navigating. Empty tree → see the per-platform notes.
+- `describe {udid}` — focus view: the focused / `[selected]` element + focusable elements with labels, traits and values. It does not print coordinates — a TV is navigated with the D-pad, never by tapping. The discovery tool — call before and after navigating. Empty tree → see the per-platform notes.
 - `tv-remote {udid, button}` — D-pad / remote. `button` is one key **or a whole path** (run in one call). Keys: `up`/`down`/`left`/`right`, `select`, `back`, `menu`, `home`, `playPause`, plus media keys `rewind`/`fastForward`/`next`/`previous`/`volumeUp`/`volumeDown`/`mute`. Single: `{button:"down"}`; repeat: `{button:"down", repeat:3}`; path: `{button:["up","right","select"]}`.
 - `keyboard {udid, text}` — type into the focused field (focus it with `tv-remote` first). Named `key` presses (e.g. `{key:"enter"}`) work on Vega; on Apple TV / Android TV move focus with `tv-remote` instead.
 - `launch-app` / `restart-app` / `reinstall-app {udid, bundleId}` — `bundleId` from the app manifest. Vega `reinstall-app` takes `appPath` = a `.vpkg`.
+- `await-ui-element {udid, …}` / `await-screen-idle {udid}` — wait for the TV to be ready instead of guessing a delay. They poll the same focus view `describe` reads. `visible` means the same as `exists` there (the focus engine only enumerates what is on screen and reachable); wait for the cursor with `{condition:"exists", selector:{text:"X", role:"focused"}}`. `await-screen-idle` settles once the app, the focusable set and the cursor stop changing — playback or animation the focus engine cannot see will not hold it unsettled.
 - `screenshot {udid, scale?}` — Apple TV via `xcrun simctl io` (downscaled); Android TV / Vega host-side via `adb` / `screencap`.
 
 ## Per-platform

packages/tool-server/src/blueprints/tv-control-types.ts

@@ -16,6 +16,16 @@ export interface TvElement {
   traits?: string[];
   value?: string;
   isFocused?: boolean;
+  /**
+   * Normalized 0..1 rect. The tvOS daemon has always reported this (and a
+   * `tapPoint`) — the field simply went undeclared, while `describe`'s focus
+   * rendering drops it because a TV is navigated with the D-pad rather than by
+   * coordinate. Declared now because the wait tools adapt this element into a
+   * describe tree, where the frame drives visibility and reading order.
+   * Absent on backends that report no bounds (Android TV's focus view) and for
+   * zero-size elements, so every consumer must tolerate it missing.
+   */
+  frame?: { x: number; y: number; width: number; height: number };
 }
 
 export interface TvDescribeResponse {

packages/tool-server/src/tools/await-screen-idle/index.ts

@@ -18,6 +18,8 @@ import type { DescribeNode, DescribeTreeData } from "../describe/contract";
 import { describeIos, iosRequires } from "../describe/platforms/ios";
 import { describeAndroid, androidRequires } from "../describe/platforms/android";
 import { describeChromium } from "../describe/platforms/chromium";
+import { describeTvFocus } from "../describe/platforms/tv-focus";
+import { resolveTvApi } from "../tv/tv-service";
 
 export const AWAIT_SCREEN_IDLE_TOOL_ID = "await-screen-idle";
 
@@ -66,6 +68,13 @@ interface IdleResult {
   waitedMs: number;
   /** Number of tree reads taken. */
   polls: number;
+  /**
+   * Why it did not settle, when the last read said something useful — a
+   * degraded accessibility read, a still-launching TV app. Absent on success.
+   * Without it an unsettled result is a silent stall: the caller waits the full
+   * budget and is told only `settled: false` (#620).
+   */
+  note?: string;
 }
 
 const capability: ToolCapability = {
@@ -92,18 +101,52 @@ function treeSignature(root: DescribeNode): string {
   return parts.join("\n");
 }
 
+/**
+ * Explain an unsettled wait from whatever the last read reported. Mirrors the
+ * diagnostics `await-ui-element` folds onto its timeout note — this tool had no
+ * equivalent, so a caller got a silent stall on a degraded read.
+ */
+function unsettledNote(
+  lastData: DescribeTreeData | null,
+  lastError: string | undefined
+): string | undefined {
+  if (lastError) return `last tree read failed: ${lastError}`;
+  if (!lastData) return undefined;
+  // A non-empty tree that never held still is self-explanatory: the screen was
+  // genuinely moving. Only an empty one needs a reason.
+  if (lastData.tree.children.length > 0) return undefined;
+  const parts: string[] = ["the screen reported no content"];
+  if (lastData.should_restart) {
+    parts.push("the foreground app may need a restart for native inspection");
+  }
+  if (lastData.hint) parts.push(lastData.hint);
+  return parts.join("; ");
+}
+
 // `await-screen-idle` waits for the screen to *settle* — render content and stop
 // changing — rather than for a named element like `await-ui-element`. The MCP
 // layer uses it to time its auto-screenshot: capture once the screen is stable
 // instead of after a fixed delay.
 export function createAwaitScreenIdleTool(registry: Registry): ToolDefinition<Params, IdleResult> {
-  function fetchTree(
+  async function fetchTree(
     device: DeviceInfo,
     services: Record<string, unknown>,
     isTvOs: boolean,
     androidIsTv: boolean
   ): Promise<DescribeTreeData> {
     if (device.platform === "ios") {
+      // Apple TV: `describeIos` short-circuits every tvOS read to an empty tree,
+      // so this tool could never settle there (#620). Poll the focus view the
+      // `describe` tool already uses successfully instead.
+      //
+      // Resolved lazily, INSIDE the fetch, on purpose: the first resolution
+      // spawns the tvOS ax/HID daemons and can take seconds. pollDescribeTree
+      // already races each fetch against the remaining deadline, and the
+      // registry caches the running service, so only the first poll pays — and
+      // it can never overrun the caller's budget. Resolving up front and
+      // bounding it separately would double-count the wait against a timeout
+      // this tool exists to respect.
+      if (isTvOs) return describeTvFocus(await resolveTvApi(registry, device.id));
       return describeIos(registry, device, {}, { isTvOs });
     }
     if (device.platform === "android") {
@@ -125,8 +168,12 @@ export function createAwaitScreenIdleTool(registry: Registry): ToolDefinition<Pa
 
 Polls the same accessibility / DOM tree as \`describe\` every pollIntervalMs (default ${DEFAULT_POLL_INTERVAL_MS}ms) until it
 has content and that content holds identical for minStableMs (default ${DEFAULT_MIN_STABLE_MS}ms), or timeoutMs (default
-${DEFAULT_TIMEOUT_MS}ms) is reached. Returns { settled, waitedMs, polls } — settled=false means the screen never went
-still before the timeout. Use after a launch/navigation to wait for the UI to render before screenshotting or tapping.`,
+${DEFAULT_TIMEOUT_MS}ms) is reached. Returns { settled, waitedMs, polls, note? } — settled=false means the screen never went
+still before the timeout, and \`note\` explains why when the last read said something useful
+(a degraded accessibility read, an app still launching).
+On an Apple TV it polls the focus view rather than a pixel-backed tree: settled means the app,
+the focusable set and the cursor stopped changing, so playback or animation the focus engine
+cannot see will not hold it unsettled. Use after a launch/navigation to wait for the UI to render before screenshotting or tapping.`,
     searchHint:
       "wait until screen settles idle stable stops changing animation transition rendered ready before screenshot",
     longRunning: true,
@@ -179,7 +226,19 @@ still before the timeout. Use after a launch/navigation to wait for the UI to re
         },
       });
 
-      return { settled: poll.result === true, waitedMs: poll.elapsedMs, polls: poll.polls };
+      const settled = poll.result === true;
+      // Only ever explain a FAILURE, and only from a read that came back empty:
+      // a populated tree that simply kept changing has no diagnosis to offer,
+      // and some hints (Android TV's "prefer tv-remote over taps") are standing
+      // advice rather than a reason, so folding them unconditionally would bury
+      // the real cause in noise.
+      const note = settled ? undefined : unsettledNote(poll.lastData, poll.lastError);
+      return {
+        settled,
+        waitedMs: poll.elapsedMs,
+        polls: poll.polls,
+        ...(note ? { note } : {}),
+      };
     },
   };
 }

packages/tool-server/src/tools/await-ui-element/index.ts

@@ -10,6 +10,9 @@ import type {
 import { chromiumCdpRef, type ChromiumCdpApi } from "../../blueprints/chromium-cdp";
 import { resolveDevice } from "../../utils/device-info";
 import { isTvOsSimulator } from "../../utils/ios-devices";
+import { describeTvFocus } from "../describe/platforms/tv-focus";
+import { resolveTvApi } from "../tv/tv-service";
+import type { TvControlApi } from "../../blueprints/tv-control-types";
 import { isAndroidTv } from "../../utils/adb";
 import { assertSupported } from "../../utils/capability";
 import { ensureDeps } from "../../utils/check-deps";
@@ -232,9 +235,15 @@ export function createAwaitUiElementTool(registry: Registry): ToolDefinition<Par
     params: Params,
     services: Record<string, unknown>,
     isTvOs: boolean,
-    androidIsTv: boolean
+    androidIsTv: boolean,
+    tvApi: TvControlApi | null
   ): Promise<DescribeTreeData> {
     if (device.platform === "ios") {
+      // Apple TV: match against the focus view. `describeIos` short-circuits
+      // every tvOS read to an empty tree, so no selector could ever resolve
+      // (#620). Resolution happens once, up front (see execute) so a backend
+      // failure still throws rather than being reported as an unmet condition.
+      if (isTvOs && tvApi) return describeTvFocus(tvApi);
       return describeIos(registry, device, { bundleId: params.bundleId }, { isTvOs });
     }
     if (device.platform === "android") {
@@ -271,9 +280,14 @@ The selector is { text?, identifier?, role? }; every provided field must match.
 case-insensitive substrings of the element's label/value and role; identifier matches exactly (case-insensitive),
 also accepting the unqualified Android resource-id name ('submit' matches 'com.example.app:id/submit').
 It polls the same accessibility / DOM tree as \`describe\`
-(iOS AXRuntime, Android uiautomator, Chromium CDP, Vega automation toolkit) every pollIntervalMs
+(iOS AXRuntime, Android uiautomator, Chromium CDP, Apple TV focus engine, Vega automation toolkit) every pollIntervalMs
 (default ${DEFAULT_POLL_INTERVAL_MS}ms) until timeoutMs (default ${DEFAULT_TIMEOUT_MS}ms).
 
+On an Apple TV the tree is the focus view: \`visible\` means the same as \`exists\` there, because the
+focus engine only enumerates what is on screen and reachable with the D-pad; \`role\` matches the
+element's accessibility traits, and \`{role:"focused"}\` targets whichever element currently holds
+the cursor — the wait to use between \`tv-remote\` and \`select\`. \`identifier\` does not apply.
+Android TV and Vega keep their full element trees.
 Returns { success: boolean, elapsed: number } — success=false means the condition never held before the
 timeout (a \`note\` then explains what was seen). Use this after a tap/navigation to wait for the next screen,
 or before tapping an element that appears asynchronously.`,
@@ -304,6 +318,12 @@ or before tapping an element that appears asynchronously.`,
       // the Android TV probe: a serial that isn't listed is never cached, so
       // leaving it inside `describeAndroid` would spawn `adb devices` per poll.
       const isTvOs = device.platform === "ios" && (await isTvOsSimulator(device.id));
+      // Resolved before the clock starts, like every other setup step here: the
+      // first resolution spawns the tvOS daemons and can take seconds, which
+      // must not eat the caller's wait budget. A failure here is an
+      // infrastructure problem, so it throws — reporting it as `success: false`
+      // would make run-sequence call it an unmet condition.
+      const tvApi = isTvOs ? await resolveTvApi(registry, device.id) : null;
       const androidIsTv = device.platform === "android" && (await isAndroidTv(device.id));
 
       // Start the wait clock after setup so its fixed cost isn't charged against
@@ -325,7 +345,7 @@ or before tapping an element that appears asynchronously.`,
       let everMatched = false;
 
       const poll = await pollDescribeTree<WaitResult>({
-        fetchTree: () => fetchTree(device, params, services, isTvOs, androidIsTv),
+        fetchTree: () => fetchTree(device, params, services, isTvOs, androidIsTv, tvApi),
         timeoutMs,
         pollIntervalMs,
         signal,

packages/tool-server/src/tools/describe/platforms/tv-focus.ts

@@ -0,0 +1,144 @@
+import type {
+  TvControlApi,
+  TvDescribeResponse,
+  TvElement,
+} from "../../../blueprints/tv-control-types";
+import type { DescribeNode, DescribeTreeData } from "../contract";
+
+/**
+ * The TV focus view, adapted into the ordinary `DescribeNode` tree so the wait
+ * tools can poll it.
+ *
+ * `describe` renders this same source for humans (see `./tv.ts`); this module
+ * exists because `await-screen-idle` / `await-ui-element` need a *tree* to
+ * fingerprint and match against, and because they must NOT inherit describe's
+ * retry-and-recycle behaviour — see {@link describeTvFocus}.
+ *
+ * Why they need it at all: `describeIos` short-circuits every tvOS read to an
+ * empty tree (the iOS accessibility service cannot drive an Apple TV), so both
+ * wait tools saw a permanently empty screen and could never settle or match —
+ * issue #620.
+ */
+
+/** Synthetic root the focusables hang off, mirroring the other platforms' shape. */
+function focusRoot(): DescribeNode {
+  return { role: "AXGroup", frame: { x: 0, y: 0, width: 1, height: 1 }, children: [] };
+}
+
+/**
+ * Shared cause text for an empty focus set, so `describe` and the wait tools
+ * explain it the same way. Kept separate from the advice, which differs: only
+ * `describe` actually performs the retry-and-recycle it can then talk about.
+ */
+export const TV_EMPTY_FOCUS_CAUSE =
+  "The app is most likely still launching (splash / loading screen) or mid-transition — a React " +
+  "Native app only exposes focus once its JS bundle has rendered.";
+
+/** What a wait tool says: it diagnoses, and points at the tool that repairs. */
+export const TV_FOCUS_WAIT_EMPTY_HINT =
+  `The TV focus engine reported no focusable elements. ${TV_EMPTY_FOCUS_CAUSE} ` +
+  "Call `describe` once — it retries and recycles the tvOS read path — then wait again.";
+
+/** A focus read is "empty" when nothing actionable was reported. */
+export function isEmptyFocus(res: TvDescribeResponse): boolean {
+  return res.focusable.length === 0 && !res.focused;
+}
+
+/**
+ * The tvOS daemon reports a normalized frame per element, but `TvElement` has
+ * historically not declared it (the JSON is passed through by reference, so the
+ * data is there at runtime). Read it defensively: Android TV's focus backend
+ * genuinely omits bounds, and the tvOS daemon drops `frame` for a zero-size
+ * element.
+ */
+function frameOf(element: TvElement, index: number, total: number): DescribeNode["frame"] {
+  const raw = element.frame;
+  if (raw && raw.width > 0 && raw.height > 0) {
+    return { x: raw.x, y: raw.y, width: raw.width, height: raw.height };
+  }
+  // Fallback: a non-degenerate band per element, ordered by enumeration index.
+  // `isVisible` requires a non-zero area, and both backends enumerate in
+  // traversal order, so index order IS reading order (android-tv-control.ts
+  // reverses its child push specifically to guarantee that).
+  const slots = Math.max(total, 1);
+  return { x: 0, y: index / slots, width: 1, height: 1 / slots };
+}
+
+/**
+ * Synthetic trait marking the cursor. Carried in `role` — rather than only in
+ * the `focused` field — so it does two jobs the field cannot:
+ *
+ *  - it is selectable, making "wait until focus lands on X" expressible as
+ *    `{ selector: { text: "X", role: "focused" }, condition: "exists" }`, which
+ *    is the wait a TV `run-sequence` actually needs between `tv-remote` and
+ *    `select`;
+ *  - it puts the cursor into the idle fingerprint, so a screen whose focus is
+ *    still moving does not read as settled.
+ *
+ * Safe as a `role` token: role matching is a case-insensitive substring, and no
+ * real trait on either backend contains "focused" (`_focusGuide` and
+ * `_tvFocusable` do not).
+ */
+const FOCUSED_TRAIT = "focused";
+
+function toNode(element: TvElement, index: number, total: number): DescribeNode {
+  const traits = [...(element.traits ?? [])];
+  if (element.isFocused) traits.push(FOCUSED_TRAIT);
+  return {
+    // Traits are what a selector's `role` matches, exactly as on a phone.
+    role: traits.length > 0 ? traits.join(",") : "element",
+    frame: frameOf(element, index, total),
+    children: [],
+    ...(element.label ? { label: element.label } : {}),
+    ...(element.value ? { value: element.value } : {}),
+    // The cursor. `format-tree` already renders this as [focused], and it is
+    // what makes "wait until focus lands on X" expressible.
+    ...(element.isFocused ? { focused: true } : {}),
+  };
+}
+
+/**
+ * Adapt a focus read into a describe tree.
+ *
+ * The root carries the foreground bundle id, so a wait notices the app itself
+ * changing underneath it — a TV transition often swaps the whole app, not just
+ * the focusable set.
+ */
+export function tvFocusTree(res: TvDescribeResponse): DescribeTreeData {
+  if (isEmptyFocus(res)) {
+    // The hint is load-bearing, not decoration: `await-ui-element` treats an
+    // empty tree as an untrustworthy read ONLY when a hint (or a prior match)
+    // says so. Without it, `condition: "hidden"` would report success on the
+    // very first poll of a still-launching app — a false pass that would
+    // release a gated interaction.
+    return { tree: focusRoot(), source: "tv-focus", hint: TV_FOCUS_WAIT_EMPTY_HINT };
+  }
+
+  const elements = [...res.focusable];
+  // Some reads report a focused element that is absent from the focusable list;
+  // it still has to be matchable, and it is the single most useful node here.
+  //
+  // Detect that by looking for a focusable already MARKED focused, not by object
+  // identity: the two arrive as separate objects from the same JSON payload, so
+  // an identity check would append a duplicate of an element that is already
+  // there — and the cursor would then match twice.
+  if (res.focused && !res.focusable.some((e) => e.isFocused)) elements.push(res.focused);
+
+  const root = focusRoot();
+  root.children = elements.map((el, i) => toNode(el, i, elements.length));
+  if (res.bundleId) root.label = res.bundleId;
+  return { tree: root, source: "tv-focus" };
+}
+
+/**
+ * One bare focus read, for the wait tools.
+ *
+ * Deliberately NOT `describeTv`: that sleeps between empty probes and can
+ * respawn the tvOS ax daemon (`recycleAx`). Inside a 200ms poll loop the sleeps
+ * are redundant and the respawn is destructive — it would drop the very state
+ * the caller is waiting on. Repair stays in the one-shot tool; a wait only
+ * observes and reports.
+ */
+export async function describeTvFocus(api: TvControlApi): Promise<DescribeTreeData> {
+  return tvFocusTree(await api.describe());
+}