fix(ui-automation): Bound recovery candidates

cameroncooke · codex · cameroncooke · commit 4ba5046d4ba0 · 2026-05-22T09:06:39.000+01:00
Cap compact recoverable UI error candidates so large accessibility trees do
not emit unbounded actionable rows in structured output. Also classify macOS
launch rendering from artifacts instead of coupling it to an exact error
message string.

Co-Authored-By: Codex &lt;codex@openai.com&gt;
diff --git a/src/mcp/tools/ui-automation/__tests__/snapshot-ui-state.test.ts b/src/mcp/tools/ui-automation/__tests__/snapshot-ui-state.test.ts
@@ -1,5 +1,6 @@
 import { beforeEach, describe, expect, it } from 'vitest';
 import type { AccessibilityNode } from '../../../../types/domain-results.ts';
+import { COMPACT_RUNTIME_TARGET_LIMIT } from '../../../../types/ui-snapshot.ts';
 import { createRuntimeSnapshotRecord } from '../shared/runtime-snapshot.ts';
 import {
   __resetRuntimeSnapshotStoreForTests,
@@ -103,6 +104,46 @@ describe('runtime snapshot store', () => {
     });
   });
 
+  it('caps not-actionable candidate lists at the compact runtime target limit', () => {
+    const textFields: AccessibilityNode[] = Array.from(
+      { length: COMPACT_RUNTIME_TARGET_LIMIT + 10 },
+      (_, index) => ({
+        type: 'TextField',
+        role: 'AXTextField',
+        frame: { x: 10, y: 80 + index, width: 200, height: 40 },
+        children: [],
+        enabled: true,
+        custom_actions: [],
+        AXLabel: `Field ${index + 1}`,
+      }),
+    );
+    const snapshot = createRuntimeSnapshotRecord({
+      simulatorId,
+      uiHierarchy: [node, ...textFields],
+      nowMs: 1_000,
+    });
+    recordRuntimeSnapshot(snapshot);
+
+    const result = resolveElementRef(simulatorId, 'e1', 'typeText', 2_000);
+
+    expect(result).toEqual({
+      ok: false,
+      error: expect.objectContaining({
+        code: 'TARGET_NOT_ACTIONABLE',
+        message: "Element ref 'e1' does not support 'typeText'.",
+        elementRef: 'e1',
+        candidates: expect.any(Array),
+      }),
+    });
+    if (!result.ok) {
+      expect(result.error.candidates).toHaveLength(COMPACT_RUNTIME_TARGET_LIMIT);
+      expect(result.error.candidates?.[0]?.ref).toBe('e2');
+      expect(result.error.candidates?.[COMPACT_RUNTIME_TARGET_LIMIT - 1]?.ref).toBe(
+        `e${COMPACT_RUNTIME_TARGET_LIMIT + 1}`,
+      );
+    }
+  });
+
   it('returns typed recoverable errors for missing, expired, not-found, and not-actionable refs', () => {
     expect(resolveElementRef(simulatorId, 'e1', 'tap', 1_000)).toEqual({
       ok: false,
diff --git a/src/mcp/tools/ui-automation/shared/snapshot-ui-state.ts b/src/mcp/tools/ui-automation/shared/snapshot-ui-state.ts
@@ -1,3 +1,4 @@
+import { COMPACT_RUNTIME_TARGET_LIMIT } from '../../../../types/ui-snapshot.ts';
 import type {
   RuntimeActionNameV1,
   RuntimeElementResolution,
@@ -124,9 +125,11 @@ export function resolveElementRefForAnyAction(
             ? 'Choose an elementRef that lists the required action, or refresh with snapshot_ui.'
             : `Choose an elementRef that lists ${requiredActionText}, or refresh with snapshot_ui.`,
         elementRef,
-        candidates: snapshot.payload.elements.filter((candidate) =>
-          requiredActions.some((action) => candidate.actions.includes(action)),
-        ),
+        candidates: snapshot.payload.elements
+          .filter((candidate) =>
+            requiredActions.some((action) => candidate.actions.includes(action)),
+          )
+          .slice(0, COMPACT_RUNTIME_TARGET_LIMIT),
         snapshotAgeMs: ageMs,
       },
     };
diff --git a/src/types/ui-snapshot.ts b/src/types/ui-snapshot.ts
@@ -3,6 +3,8 @@ import type { AccessibilityNode, Frame, Point } from './domain-results.ts';
 export type RuntimeSnapshotProtocol = 'rs/1';
 export type RuntimeSnapshotCaptureType = 'runtime-snapshot';
 
+export const COMPACT_RUNTIME_TARGET_LIMIT = 64;
+
 export type RuntimeActionNameV1 = 'tap' | 'typeText' | 'longPress' | 'touch' | 'swipeWithin';
 
 export type RuntimeElementRoleV1 =
diff --git a/src/utils/__tests__/structured-output-envelope.test.ts b/src/utils/__tests__/structured-output-envelope.test.ts
@@ -5,6 +5,7 @@ import type {
   CaptureResultDomainResult,
   DeviceListDomainResult,
 } from '../../types/domain-results.ts';
+import { COMPACT_RUNTIME_TARGET_LIMIT } from '../../types/ui-snapshot.ts';
 
 describe('toStructuredEnvelope', () => {
   it('strips kind, didError, and error from the data payload', () => {
@@ -410,6 +411,43 @@ describe('toStructuredEnvelope', () => {
     });
   });
 
+  it('caps compact runtime snapshot candidates inside recoverable UI errors', () => {
+    const candidates = Array.from({ length: COMPACT_RUNTIME_TARGET_LIMIT + 16 }, (_, index) => ({
+      ref: `e${index + 1}`,
+      role: 'button' as const,
+      label: `Candidate ${index + 1}`,
+      frame: { x: 0, y: index, width: 100, height: 40 },
+      actions: ['tap' as const],
+    }));
+    const result: CaptureResultDomainResult = {
+      kind: 'capture-result',
+      didError: true,
+      error: 'Element ref is not actionable.',
+      summary: { status: 'FAILED' },
+      artifacts: { simulatorId: 'SIMULATOR-1' },
+      uiError: {
+        code: 'TARGET_NOT_ACTIONABLE',
+        message: 'Element ref is not actionable.',
+        recoveryHint: 'Choose another elementRef.',
+        elementRef: 'e404',
+        candidates,
+      },
+    };
+
+    const envelope = toStructuredEnvelope(result, 'xcodebuildmcp.output.capture-result', '2');
+    const data = envelope.data as {
+      uiError: { candidates: string[]; message: string; elementRef: string };
+    };
+
+    expect(data.uiError.message).toBe('Element ref is not actionable.');
+    expect(data.uiError.elementRef).toBe('e404');
+    expect(data.uiError.candidates).toHaveLength(COMPACT_RUNTIME_TARGET_LIMIT);
+    expect(data.uiError.candidates[0]).toBe('e1|tap|button|Candidate 1||');
+    expect(data.uiError.candidates[COMPACT_RUNTIME_TARGET_LIMIT - 1]).toBe(
+      `e${COMPACT_RUNTIME_TARGET_LIMIT}|tap|button|Candidate ${COMPACT_RUNTIME_TARGET_LIMIT}||`,
+    );
+  });
+
   it('can keep full runtime snapshots and candidates for verbose callers', () => {
     const result: CaptureResultDomainResult = {
       kind: 'capture-result',
diff --git a/src/utils/renderers/__tests__/domain-result-text.test.ts b/src/utils/renderers/__tests__/domain-result-text.test.ts
@@ -1,5 +1,8 @@
 import { describe, expect, it } from 'vitest';
-import type { UiActionResultDomainResult } from '../../../types/domain-results.ts';
+import type {
+  LaunchResultDomainResult,
+  UiActionResultDomainResult,
+} from '../../../types/domain-results.ts';
 import { renderDomainResultTextItems } from '../domain-result-text.ts';
 
 function uiActionResult(action: UiActionResultDomainResult['action']): UiActionResultDomainResult {
@@ -13,7 +16,73 @@ function uiActionResult(action: UiActionResultDomainResult['action']): UiActionR
   };
 }
 
+function launchResult(
+  artifacts: LaunchResultDomainResult['artifacts'],
+  error: string,
+): LaunchResultDomainResult {
+  return {
+    kind: 'launch-result',
+    didError: true,
+    error,
+    summary: { status: 'FAILED' },
+    artifacts,
+    diagnostics: { warnings: [], errors: [] },
+  };
+}
+
 describe('renderDomainResultTextItems', () => {
+  it('renders macOS launch errors from artifacts instead of exact error text', () => {
+    expect(
+      renderDomainResultTextItems(
+        launchResult({ appPath: '/tmp/Test.app' }, 'Custom launch failure.'),
+      ),
+    ).toMatchInlineSnapshot(`
+      [
+        {
+          "operation": "Launch macOS App",
+          "params": [
+            {
+              "label": "App",
+              "value": "/tmp/Test.app",
+            },
+          ],
+          "type": "header",
+        },
+        {
+          "level": "error",
+          "message": "Custom launch failure.",
+          "type": "status",
+        },
+      ]
+    `);
+  });
+
+  it('does not classify targetless simulator launch errors as macOS without app artifacts', () => {
+    expect(
+      renderDomainResultTextItems(
+        launchResult({ bundleId: 'com.example.App' }, 'Failed to launch app.'),
+      ),
+    ).toMatchInlineSnapshot(`
+      [
+        {
+          "operation": "Launch App",
+          "params": [
+            {
+              "label": "Bundle ID",
+              "value": "com.example.App",
+            },
+          ],
+          "type": "header",
+        },
+        {
+          "level": "error",
+          "message": "Failed to launch app.",
+          "type": "status",
+        },
+      ]
+    `);
+  });
+
   it('renders drag UI action results', () => {
     expect(
       renderDomainResultTextItems(
diff --git a/src/utils/renderers/domain-result-text.ts b/src/utils/renderers/domain-result-text.ts
@@ -796,10 +796,7 @@ function createLaunchResultItems(
 ): TextRenderableItem[] {
   const isSimulator = typeof result.artifacts.simulatorId === 'string';
   const isDevice = typeof result.artifacts.deviceId === 'string';
-  const isMac =
-    !isSimulator &&
-    !isDevice &&
-    (!result.didError || result.error === 'Failed to launch macOS app.');
+  const isMac = !isSimulator && !isDevice && typeof result.artifacts.appPath === 'string';
   const title = isMac ? 'Launch macOS App' : 'Launch App';
   const params: HeaderRenderItem['params'] = [];
 
diff --git a/src/utils/structured-output-envelope.ts b/src/utils/structured-output-envelope.ts
@@ -2,6 +2,7 @@ import type { RuntimeKind } from '../runtime/types.ts';
 import type { NextStep, OutputStyle } from '../types/common.ts';
 import type { ToolDomainResult } from '../types/domain-results.ts';
 import type { StructuredOutputEnvelope } from '../types/structured-output.ts';
+import { COMPACT_RUNTIME_TARGET_LIMIT } from '../types/ui-snapshot.ts';
 import type {
   RuntimeActionNameV1,
   RuntimeElementV1,
@@ -47,7 +48,6 @@ type RuntimeSnapshotUnchangedCompactCapture = {
 };
 
 const MINIMAL_DATA_PRUNE_KEYS = ['request'] as const;
-const COMPACT_RUNTIME_TARGET_LIMIT = 64;
 const COMPACT_RUNTIME_SCROLL_LIMIT = 32;
 const COMPACT_RUNTIME_TEXT_LIMIT = 64;
 const HIDDEN_RUNTIME_TARGET_LABELS = new Set(['sheet grabber']);
@@ -350,9 +350,11 @@ function projectRuntimeSnapshotData<TData>(
   const uiError = Array.isArray(dataWithRuntimeRows.uiError?.candidates)
     ? {
         ...dataWithRuntimeRows.uiError,
-        candidates: dataWithRuntimeRows.uiError.candidates.map((candidate) =>
-          isRuntimeElement(candidate) ? compactRuntimeElementCandidate(candidate) : candidate,
-        ),
+        candidates: dataWithRuntimeRows.uiError.candidates
+          .slice(0, COMPACT_RUNTIME_TARGET_LIMIT)
+          .map((candidate) =>
+            isRuntimeElement(candidate) ? compactRuntimeElementCandidate(candidate) : candidate,
+          ),
       }
     : dataWithRuntimeRows.uiError;
   const waitMatch = Array.isArray(dataWithRuntimeRows.waitMatch?.matches)
@@ -382,7 +384,7 @@ export function toStructuredEnvelope<TResult extends ToolDomainResult>(
   options: StructuredEnvelopeOptions = {},
 ): StructuredOutputEnvelope<unknown> {
   const { nextSteps, nextStepRuntime = 'cli', outputStyle = 'normal' } = options;
-  const { kind: _kind, didError, error, ...data } = result;
+  const { kind: neverKind, didError, error, ...data } = result;
   const projectedData = projectRuntimeSnapshotData(data as DomainResultData<TResult>, options);
   const serializedNextSteps =
     schema === 'xcodebuildmcp.output.error'
