Skip to content

Commit 4ba5046

Browse files
cameroncookecodex
andcommitted
fix(ui-automation): Bound recovery candidates
Cap compact recoverable UI error candidates so large accessibility trees do not emit unbounded actionable rows in structured output. Also classify macOS launch rendering from artifacts instead of coupling it to an exact error message string. Co-Authored-By: Codex <codex@openai.com>
1 parent 699ab74 commit 4ba5046

7 files changed

Lines changed: 165 additions & 13 deletions

File tree

src/mcp/tools/ui-automation/__tests__/snapshot-ui-state.test.ts

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { beforeEach, describe, expect, it } from 'vitest';
22
import type { AccessibilityNode } from '../../../../types/domain-results.ts';
3+
import { COMPACT_RUNTIME_TARGET_LIMIT } from '../../../../types/ui-snapshot.ts';
34
import { createRuntimeSnapshotRecord } from '../shared/runtime-snapshot.ts';
45
import {
56
__resetRuntimeSnapshotStoreForTests,
@@ -103,6 +104,46 @@ describe('runtime snapshot store', () => {
103104
});
104105
});
105106

107+
it('caps not-actionable candidate lists at the compact runtime target limit', () => {
108+
const textFields: AccessibilityNode[] = Array.from(
109+
{ length: COMPACT_RUNTIME_TARGET_LIMIT + 10 },
110+
(_, index) => ({
111+
type: 'TextField',
112+
role: 'AXTextField',
113+
frame: { x: 10, y: 80 + index, width: 200, height: 40 },
114+
children: [],
115+
enabled: true,
116+
custom_actions: [],
117+
AXLabel: `Field ${index + 1}`,
118+
}),
119+
);
120+
const snapshot = createRuntimeSnapshotRecord({
121+
simulatorId,
122+
uiHierarchy: [node, ...textFields],
123+
nowMs: 1_000,
124+
});
125+
recordRuntimeSnapshot(snapshot);
126+
127+
const result = resolveElementRef(simulatorId, 'e1', 'typeText', 2_000);
128+
129+
expect(result).toEqual({
130+
ok: false,
131+
error: expect.objectContaining({
132+
code: 'TARGET_NOT_ACTIONABLE',
133+
message: "Element ref 'e1' does not support 'typeText'.",
134+
elementRef: 'e1',
135+
candidates: expect.any(Array),
136+
}),
137+
});
138+
if (!result.ok) {
139+
expect(result.error.candidates).toHaveLength(COMPACT_RUNTIME_TARGET_LIMIT);
140+
expect(result.error.candidates?.[0]?.ref).toBe('e2');
141+
expect(result.error.candidates?.[COMPACT_RUNTIME_TARGET_LIMIT - 1]?.ref).toBe(
142+
`e${COMPACT_RUNTIME_TARGET_LIMIT + 1}`,
143+
);
144+
}
145+
});
146+
106147
it('returns typed recoverable errors for missing, expired, not-found, and not-actionable refs', () => {
107148
expect(resolveElementRef(simulatorId, 'e1', 'tap', 1_000)).toEqual({
108149
ok: false,

src/mcp/tools/ui-automation/shared/snapshot-ui-state.ts

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { COMPACT_RUNTIME_TARGET_LIMIT } from '../../../../types/ui-snapshot.ts';
12
import type {
23
RuntimeActionNameV1,
34
RuntimeElementResolution,
@@ -124,9 +125,11 @@ export function resolveElementRefForAnyAction(
124125
? 'Choose an elementRef that lists the required action, or refresh with snapshot_ui.'
125126
: `Choose an elementRef that lists ${requiredActionText}, or refresh with snapshot_ui.`,
126127
elementRef,
127-
candidates: snapshot.payload.elements.filter((candidate) =>
128-
requiredActions.some((action) => candidate.actions.includes(action)),
129-
),
128+
candidates: snapshot.payload.elements
129+
.filter((candidate) =>
130+
requiredActions.some((action) => candidate.actions.includes(action)),
131+
)
132+
.slice(0, COMPACT_RUNTIME_TARGET_LIMIT),
130133
snapshotAgeMs: ageMs,
131134
},
132135
};

src/types/ui-snapshot.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ import type { AccessibilityNode, Frame, Point } from './domain-results.ts';
33
export type RuntimeSnapshotProtocol = 'rs/1';
44
export type RuntimeSnapshotCaptureType = 'runtime-snapshot';
55

6+
export const COMPACT_RUNTIME_TARGET_LIMIT = 64;
7+
68
export type RuntimeActionNameV1 = 'tap' | 'typeText' | 'longPress' | 'touch' | 'swipeWithin';
79

810
export type RuntimeElementRoleV1 =

src/utils/__tests__/structured-output-envelope.test.ts

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import type {
55
CaptureResultDomainResult,
66
DeviceListDomainResult,
77
} from '../../types/domain-results.ts';
8+
import { COMPACT_RUNTIME_TARGET_LIMIT } from '../../types/ui-snapshot.ts';
89

910
describe('toStructuredEnvelope', () => {
1011
it('strips kind, didError, and error from the data payload', () => {
@@ -410,6 +411,43 @@ describe('toStructuredEnvelope', () => {
410411
});
411412
});
412413

414+
it('caps compact runtime snapshot candidates inside recoverable UI errors', () => {
415+
const candidates = Array.from({ length: COMPACT_RUNTIME_TARGET_LIMIT + 16 }, (_, index) => ({
416+
ref: `e${index + 1}`,
417+
role: 'button' as const,
418+
label: `Candidate ${index + 1}`,
419+
frame: { x: 0, y: index, width: 100, height: 40 },
420+
actions: ['tap' as const],
421+
}));
422+
const result: CaptureResultDomainResult = {
423+
kind: 'capture-result',
424+
didError: true,
425+
error: 'Element ref is not actionable.',
426+
summary: { status: 'FAILED' },
427+
artifacts: { simulatorId: 'SIMULATOR-1' },
428+
uiError: {
429+
code: 'TARGET_NOT_ACTIONABLE',
430+
message: 'Element ref is not actionable.',
431+
recoveryHint: 'Choose another elementRef.',
432+
elementRef: 'e404',
433+
candidates,
434+
},
435+
};
436+
437+
const envelope = toStructuredEnvelope(result, 'xcodebuildmcp.output.capture-result', '2');
438+
const data = envelope.data as {
439+
uiError: { candidates: string[]; message: string; elementRef: string };
440+
};
441+
442+
expect(data.uiError.message).toBe('Element ref is not actionable.');
443+
expect(data.uiError.elementRef).toBe('e404');
444+
expect(data.uiError.candidates).toHaveLength(COMPACT_RUNTIME_TARGET_LIMIT);
445+
expect(data.uiError.candidates[0]).toBe('e1|tap|button|Candidate 1||');
446+
expect(data.uiError.candidates[COMPACT_RUNTIME_TARGET_LIMIT - 1]).toBe(
447+
`e${COMPACT_RUNTIME_TARGET_LIMIT}|tap|button|Candidate ${COMPACT_RUNTIME_TARGET_LIMIT}||`,
448+
);
449+
});
450+
413451
it('can keep full runtime snapshots and candidates for verbose callers', () => {
414452
const result: CaptureResultDomainResult = {
415453
kind: 'capture-result',

src/utils/renderers/__tests__/domain-result-text.test.ts

Lines changed: 70 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
import { describe, expect, it } from 'vitest';
2-
import type { UiActionResultDomainResult } from '../../../types/domain-results.ts';
2+
import type {
3+
LaunchResultDomainResult,
4+
UiActionResultDomainResult,
5+
} from '../../../types/domain-results.ts';
36
import { renderDomainResultTextItems } from '../domain-result-text.ts';
47

58
function uiActionResult(action: UiActionResultDomainResult['action']): UiActionResultDomainResult {
@@ -13,7 +16,73 @@ function uiActionResult(action: UiActionResultDomainResult['action']): UiActionR
1316
};
1417
}
1518

19+
function launchResult(
20+
artifacts: LaunchResultDomainResult['artifacts'],
21+
error: string,
22+
): LaunchResultDomainResult {
23+
return {
24+
kind: 'launch-result',
25+
didError: true,
26+
error,
27+
summary: { status: 'FAILED' },
28+
artifacts,
29+
diagnostics: { warnings: [], errors: [] },
30+
};
31+
}
32+
1633
describe('renderDomainResultTextItems', () => {
34+
it('renders macOS launch errors from artifacts instead of exact error text', () => {
35+
expect(
36+
renderDomainResultTextItems(
37+
launchResult({ appPath: '/tmp/Test.app' }, 'Custom launch failure.'),
38+
),
39+
).toMatchInlineSnapshot(`
40+
[
41+
{
42+
"operation": "Launch macOS App",
43+
"params": [
44+
{
45+
"label": "App",
46+
"value": "/tmp/Test.app",
47+
},
48+
],
49+
"type": "header",
50+
},
51+
{
52+
"level": "error",
53+
"message": "Custom launch failure.",
54+
"type": "status",
55+
},
56+
]
57+
`);
58+
});
59+
60+
it('does not classify targetless simulator launch errors as macOS without app artifacts', () => {
61+
expect(
62+
renderDomainResultTextItems(
63+
launchResult({ bundleId: 'com.example.App' }, 'Failed to launch app.'),
64+
),
65+
).toMatchInlineSnapshot(`
66+
[
67+
{
68+
"operation": "Launch App",
69+
"params": [
70+
{
71+
"label": "Bundle ID",
72+
"value": "com.example.App",
73+
},
74+
],
75+
"type": "header",
76+
},
77+
{
78+
"level": "error",
79+
"message": "Failed to launch app.",
80+
"type": "status",
81+
},
82+
]
83+
`);
84+
});
85+
1786
it('renders drag UI action results', () => {
1887
expect(
1988
renderDomainResultTextItems(

src/utils/renderers/domain-result-text.ts

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -796,10 +796,7 @@ function createLaunchResultItems(
796796
): TextRenderableItem[] {
797797
const isSimulator = typeof result.artifacts.simulatorId === 'string';
798798
const isDevice = typeof result.artifacts.deviceId === 'string';
799-
const isMac =
800-
!isSimulator &&
801-
!isDevice &&
802-
(!result.didError || result.error === 'Failed to launch macOS app.');
799+
const isMac = !isSimulator && !isDevice && typeof result.artifacts.appPath === 'string';
803800
const title = isMac ? 'Launch macOS App' : 'Launch App';
804801
const params: HeaderRenderItem['params'] = [];
805802

src/utils/structured-output-envelope.ts

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import type { RuntimeKind } from '../runtime/types.ts';
22
import type { NextStep, OutputStyle } from '../types/common.ts';
33
import type { ToolDomainResult } from '../types/domain-results.ts';
44
import type { StructuredOutputEnvelope } from '../types/structured-output.ts';
5+
import { COMPACT_RUNTIME_TARGET_LIMIT } from '../types/ui-snapshot.ts';
56
import type {
67
RuntimeActionNameV1,
78
RuntimeElementV1,
@@ -47,7 +48,6 @@ type RuntimeSnapshotUnchangedCompactCapture = {
4748
};
4849

4950
const MINIMAL_DATA_PRUNE_KEYS = ['request'] as const;
50-
const COMPACT_RUNTIME_TARGET_LIMIT = 64;
5151
const COMPACT_RUNTIME_SCROLL_LIMIT = 32;
5252
const COMPACT_RUNTIME_TEXT_LIMIT = 64;
5353
const HIDDEN_RUNTIME_TARGET_LABELS = new Set(['sheet grabber']);
@@ -350,9 +350,11 @@ function projectRuntimeSnapshotData<TData>(
350350
const uiError = Array.isArray(dataWithRuntimeRows.uiError?.candidates)
351351
? {
352352
...dataWithRuntimeRows.uiError,
353-
candidates: dataWithRuntimeRows.uiError.candidates.map((candidate) =>
354-
isRuntimeElement(candidate) ? compactRuntimeElementCandidate(candidate) : candidate,
355-
),
353+
candidates: dataWithRuntimeRows.uiError.candidates
354+
.slice(0, COMPACT_RUNTIME_TARGET_LIMIT)
355+
.map((candidate) =>
356+
isRuntimeElement(candidate) ? compactRuntimeElementCandidate(candidate) : candidate,
357+
),
356358
}
357359
: dataWithRuntimeRows.uiError;
358360
const waitMatch = Array.isArray(dataWithRuntimeRows.waitMatch?.matches)
@@ -382,7 +384,7 @@ export function toStructuredEnvelope<TResult extends ToolDomainResult>(
382384
options: StructuredEnvelopeOptions = {},
383385
): StructuredOutputEnvelope<unknown> {
384386
const { nextSteps, nextStepRuntime = 'cli', outputStyle = 'normal' } = options;
385-
const { kind: _kind, didError, error, ...data } = result;
387+
const { kind: neverKind, didError, error, ...data } = result;
386388
const projectedData = projectRuntimeSnapshotData(data as DomainResultData<TResult>, options);
387389
const serializedNextSteps =
388390
schema === 'xcodebuildmcp.output.error'

0 commit comments

Comments
 (0)