Skip to content

Commit 9fa2dd5

Browse files
committed
refactor(mcp): project interaction family output schemas from their owning module
Moves the 6 hand-authored MCP outputSchema entries for press, click, fill, longpress, hover, and find out of command-output-schemas.ts into a frozen INTERACTION_COMMAND_OUTPUT_SCHEMAS map exported from src/commands/interaction/index.ts, following the #2810 projection seam. Registers the family in the PROJECTED_FAMILIES disjointness check and adds a colocated command-tools-interaction-schemas.test.ts proving reference equality with the module object, including the settle-observation copy for the five settle-capable entries (find has no such trait). postActionSurfaceChangeSchema is duplicated locally rather than imported back from command-output-schemas.ts (which still needs it for the generic --settle observation), matching the constSchema duplication precedent from the system family's move; responseCostSchema and the other interaction-only helpers had no other consumer and moved outright. Part of #2819.
1 parent a595a82 commit 9fa2dd5

4 files changed

Lines changed: 356 additions & 256 deletions

File tree

‎src/commands/interaction/index.ts‎

Lines changed: 291 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,21 @@ import type {
2020
TypeTextOptions,
2121
} from '@agent-device/contracts/client';
2222
import type { CommandSchemaOverride } from '@agent-device/command-registry/command-schema';
23+
import type { CommandResultMap } from '@agent-device/command-registry/command-result';
2324
import {
2425
REPEATED_TOUCH_FLAGS,
2526
SELECTOR_SNAPSHOT_FLAGS,
2627
} from '@agent-device/command-registry/flag-groups';
2728
import { postActionObservationCliFlags } from '../post-action-observation-grammar.ts';
29+
import type { JsonSchema } from '../command-contract.ts';
2830
import {
31+
booleanSchema,
32+
enumSchema,
33+
looseObjectSchema,
34+
numberSchema,
35+
objectSchema,
36+
stringArraySchema,
37+
stringSchema,
2938
toClientElementTarget,
3039
toClientInteractionTarget,
3140
toRepeatedOptions,
@@ -54,6 +63,288 @@ import {
5463
import { interactionCliOutputFormatters } from './output.ts';
5564
import { selectorCliReaders, selectorDaemonWriters } from './selectors.ts';
5665

66+
function constSchema(value: string): JsonSchema {
67+
return { type: 'string', const: value };
68+
}
69+
70+
function nullableStringSchema(description?: string): JsonSchema {
71+
return { type: ['string', 'null'], ...(description ? { description } : {}) };
72+
}
73+
74+
// PostActionSurfaceChange (packages/contracts/src/interaction.ts) — the post-action capture
75+
// describes a different surface than the pre-action baseline (#2438), so no same-surface
76+
// comparison is presented across it. Mirrored in src/mcp/command-output-schemas.ts for the
77+
// generic `--settle` observation, which every family (not only this one) can carry.
78+
const postActionSurfaceChangeSchema: JsonSchema = objectSchema(
79+
{
80+
from: stringSchema('Surface the pre-action baseline described: a host bundle id, or app.'),
81+
to: stringSchema('Surface the post-action capture describes: a host bundle id, or app.'),
82+
disclosure: stringSchema('Agent-facing sentence explaining the surface transition.'),
83+
},
84+
['from', 'to', 'disclosure'],
85+
'Present when an in-place system surface (web sign-in or Apple Pay sheet) was presented over the app, or left it.',
86+
);
87+
88+
// InteractionEvidence (packages/contracts/src/interaction.ts) — opt-in `--verify` cheap
89+
// post-condition evidence (#1047).
90+
const interactionEvidenceSchema: JsonSchema = objectSchema(
91+
{
92+
foregroundApp: stringSchema('Foreground app bundle id or name, when the capture carries it.'),
93+
nodeCount: numberSchema('Node count in the post-action interactive-only capture.'),
94+
interactiveNodeCount: numberSchema('Subset of nodeCount the platform reports as hittable.'),
95+
digest: stringSchema('Order-independent digest of the post-action node multiset.'),
96+
changedFromBefore: booleanSchema(
97+
'Whether the post-action digest differs from the pre-action capture digest. false is evidence, not failure. With surfaceChange present, no digest comparison is made: it reports that surface transition.',
98+
),
99+
surfaceChange: postActionSurfaceChangeSchema,
100+
},
101+
['nodeCount', 'interactiveNodeCount', 'digest', 'changedFromBefore'],
102+
);
103+
104+
const responseCostSchema: JsonSchema = objectSchema(
105+
{
106+
wallClockMs: numberSchema('Total wall-clock time for the request in milliseconds.'),
107+
runnerRoundTrips: numberSchema(
108+
'Number of real runner round-trips made while serving the request.',
109+
),
110+
nodeCount: numberSchema(
111+
'Number of nodes in the original node tree when the response carries one.',
112+
),
113+
},
114+
['wallClockMs', 'runnerRoundTrips'],
115+
);
116+
117+
// ResolutionDiagnosticEntry (packages/contracts/src/interaction.ts) — a disambiguation
118+
// winner or losing alternative. Never a snapshot ref.
119+
const resolutionDiagnosticEntrySchema: JsonSchema = objectSchema(
120+
{
121+
diagnosticRef: stringSchema(
122+
'Opaque non-@ diagnostic token. Never a snapshot ref: not issued via refsGeneration and cannot be pinned or reused as an @ref target. UTF-8 truncated to 256 bytes.',
123+
),
124+
role: stringSchema('UTF-8 truncated to 256 bytes.'),
125+
label: stringSchema('UTF-8 truncated to 256 bytes.'),
126+
},
127+
['diagnosticRef'],
128+
);
129+
130+
// ResolutionDisclosure (packages/contracts/src/interaction.ts) — never ref-issuing;
131+
// absent on paths where the guarantee is inapplicable (ADR 0012 decision 2).
132+
// `alternatives` rides default/full levels only; the digest view omits it.
133+
const resolutionDisclosureSchema: JsonSchema = {
134+
type: 'object',
135+
description:
136+
'Pre-action disclosure of how the acting path resolved its target. Absent when resolutionDisclosure is inapplicable for the path.',
137+
oneOf: [
138+
objectSchema(
139+
{
140+
source: constSchema('runtime'),
141+
phase: constSchema('pre-action'),
142+
kind: constSchema('unique'),
143+
},
144+
['source', 'phase', 'kind'],
145+
),
146+
objectSchema(
147+
{
148+
source: constSchema('runtime'),
149+
phase: constSchema('pre-action'),
150+
kind: constSchema('disambiguated'),
151+
matchCount: numberSchema('Total matches resolveSelectorChain found before disambiguation.'),
152+
winnerDiagnostic: resolutionDiagnosticEntrySchema,
153+
tiebreak: enumSchema(
154+
['visible', 'deepest', 'smallest-area', 'structural-equivalence'],
155+
'The comparison that decided the winner.',
156+
),
157+
alternatives: {
158+
type: 'array',
159+
description:
160+
'At most 5 losing candidates, document order. Present at default/full response levels and omitted in digest. The winner is never included.',
161+
items: resolutionDiagnosticEntrySchema,
162+
},
163+
},
164+
['source', 'phase', 'kind', 'matchCount', 'winnerDiagnostic', 'tiebreak'],
165+
),
166+
objectSchema(
167+
{ source: constSchema('ref'), phase: constSchema('pre-action'), kind: constSchema('exact') },
168+
['source', 'phase', 'kind'],
169+
),
170+
objectSchema(
171+
{
172+
source: constSchema('ref'),
173+
phase: constSchema('pre-action'),
174+
kind: constSchema('label-fallback'),
175+
},
176+
['source', 'phase', 'kind'],
177+
),
178+
objectSchema({ source: constSchema('direct-ios'), kind: constSchema('not-observed') }, [
179+
'source',
180+
'kind',
181+
]),
182+
],
183+
};
184+
185+
type InteractionExtra = {
186+
properties?: Record<string, JsonSchema>;
187+
required?: readonly string[];
188+
};
189+
190+
/**
191+
* Canonical interaction response data built by buildInteractionResponseData:
192+
* shared target/coordinate/evidence fields plus per-command extras. The runtime
193+
* result still has richer internal node/backend data; this schema documents the
194+
* JSON payload returned to clients.
195+
*/
196+
function interactionResponseDataSchema(extra: InteractionExtra = {}): JsonSchema {
197+
const extraProperties = extra.properties ?? {};
198+
const extraRequired = extra.required ?? [];
199+
return objectSchema(
200+
{
201+
targetKind: enumSchema(['point', 'ref', 'selector'], 'Resolved interaction target kind.'),
202+
x: numberSchema('Resolved interaction x coordinate when available.'),
203+
y: numberSchema('Resolved interaction y coordinate when available.'),
204+
referenceWidth: numberSchema('Reference frame width for visualizing the interaction point.'),
205+
referenceHeight: numberSchema(
206+
'Reference frame height for visualizing the interaction point.',
207+
),
208+
ref: stringSchema('Snapshot ref without the @ prefix when the target was an @ref.'),
209+
selector: stringSchema('Selector expression when the target was a selector.'),
210+
selectorChain: stringArraySchema(),
211+
refLabel: stringSchema(),
212+
targetHittable: booleanSchema(),
213+
hint: stringSchema(),
214+
warning: stringSchema(),
215+
message: stringSchema(),
216+
evidence: interactionEvidenceSchema,
217+
resolution: resolutionDisclosureSchema,
218+
cost: responseCostSchema,
219+
maestroNonHittableCoordinateFallbackAllowed: booleanSchema(
220+
'Whether the direct iOS Maestro coordinate fallback was allowed for this selector.',
221+
),
222+
maestroNonHittableCoordinateFallbackUsed: booleanSchema(
223+
'Whether the direct iOS Maestro coordinate fallback was actually used.',
224+
),
225+
maestroFallbackReason: constSchema('non-hittable-coordinate'),
226+
...extraProperties,
227+
},
228+
['targetKind', ...extraRequired],
229+
);
230+
}
231+
232+
const tapInteractionResponseDataSchema = interactionResponseDataSchema({
233+
properties: {
234+
evidence: interactionEvidenceSchema,
235+
button: enumSchema(['secondary', 'middle']),
236+
count: numberSchema('Number of press/click repetitions.'),
237+
intervalMs: numberSchema('Delay between repeated press/click actions.'),
238+
holdMs: numberSchema('Hold duration for each action.'),
239+
jitterPx: numberSchema('Randomization radius in pixels.'),
240+
doubleTap: booleanSchema('Whether the command requested a double-tap action.'),
241+
},
242+
});
243+
244+
const fillResponseProperties = {
245+
text: stringSchema('Text submitted to the field.'),
246+
delayMs: numberSchema('Delay between typed characters in milliseconds.'),
247+
evidence: interactionEvidenceSchema,
248+
};
249+
250+
const fillVerificationTargetSchema = objectSchema(
251+
{
252+
resourceId: nullableStringSchema('Android resource id of the exact field that changed.'),
253+
className: nullableStringSchema('Android class name of the exact field that changed.'),
254+
packageName: nullableStringSchema('Android package name that owns the exact field.'),
255+
rect: objectSchema(
256+
{
257+
x: numberSchema(),
258+
y: numberSchema(),
259+
width: numberSchema(),
260+
height: numberSchema(),
261+
},
262+
['x', 'y', 'width', 'height'],
263+
'Screen-space rectangle of the exact field that changed.',
264+
),
265+
},
266+
['resourceId', 'className', 'packageName', 'rect'],
267+
'Target identity captured before fill and matched after fill.',
268+
);
269+
270+
const confirmedFillResponseSchema: JsonSchema = {
271+
...interactionResponseDataSchema({
272+
properties: fillResponseProperties,
273+
required: ['text'],
274+
}),
275+
// The public result contract omits verification evidence on an ordinary
276+
// confirmed fill. Keep this branch disjoint from the unconfirmed branch
277+
// without making the response strict to unrelated additive fields.
278+
not: objectSchema({}, ['verification']),
279+
};
280+
281+
const unconfirmedFillResponseSchema = interactionResponseDataSchema({
282+
properties: {
283+
...fillResponseProperties,
284+
verification: constSchema('unconfirmed'),
285+
requested: stringSchema('Literal text requested by the fill command.'),
286+
before: nullableStringSchema('Raw target text captured before the fill.'),
287+
after: nullableStringSchema('Raw target text captured after the fill.'),
288+
target: fillVerificationTargetSchema,
289+
},
290+
required: ['text', 'verification', 'requested', 'before', 'after', 'target'],
291+
});
292+
293+
/**
294+
* This family's advertised MCP `outputSchema`s, keyed by daemon command name and projected into
295+
* the command map by `src/mcp/command-output-schemas.ts`. Non-strict like every other entry: no
296+
* `additionalProperties: false`, so additive response fields such as `settle`/`cost` keep
297+
* validating. #1652: the opt-in `settle` observation is NOT listed here — the trait derivation
298+
* pass in that file grafts it onto settle-capable entries.
299+
*/
300+
export const INTERACTION_COMMAND_OUTPUT_SCHEMAS = {
301+
press: tapInteractionResponseDataSchema,
302+
click: tapInteractionResponseDataSchema,
303+
fill: {
304+
type: 'object',
305+
description:
306+
'Fill response. Android may return target-bound unconfirmed evidence when the exact app-owned field changed but formatting prevented raw equality.',
307+
oneOf: [confirmedFillResponseSchema, unconfirmedFillResponseSchema],
308+
},
309+
longpress: interactionResponseDataSchema({
310+
properties: {
311+
durationMs: numberSchema(),
312+
gesture: constSchema('longpress'),
313+
},
314+
}),
315+
hover: interactionResponseDataSchema({
316+
properties: {
317+
gesture: constSchema('hover'),
318+
},
319+
}),
320+
find: objectSchema(
321+
{
322+
ref: stringSchema('Snapshot ref without the @ prefix when the find action returns one.'),
323+
refsGeneration: numberSchema('ADR 0014 ref frame epoch for read-only find actions.'),
324+
found: booleanSchema('Whether a wait/exists/read-only find satisfied its condition.'),
325+
waitedMs: numberSchema('Milliseconds waited for a read-only find condition.'),
326+
text: stringSchema('Text value returned by find get_text.'),
327+
node: looseObjectSchema('Snapshot node for find get_attrs/get_text.'),
328+
matches: {
329+
type: 'array',
330+
description: 'Every match for the read-only find list action (#1625): { ref, node } each.',
331+
items: looseObjectSchema('One listed match with its snapshot ref and node.'),
332+
},
333+
locator: stringSchema('Locator kind used for the find action.'),
334+
query: stringSchema('Query argument used for the find action.'),
335+
x: numberSchema('Resolved x coordinate for mutating find actions.'),
336+
y: numberSchema('Resolved y coordinate for mutating find actions.'),
337+
message: stringSchema('Diagnostic message for mutating find actions.'),
338+
cost: responseCostSchema,
339+
},
340+
[],
341+
'Daemon response data for the find command.',
342+
),
343+
} satisfies Pick<
344+
Record<keyof CommandResultMap, JsonSchema>,
345+
'press' | 'click' | 'fill' | 'longpress' | 'hover' | 'find'
346+
>;
347+
57348
const interactionCliSchemas = {
58349
get: {
59350
usageOverride: 'get text|attrs <@ref|selector>',
Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import assert from 'node:assert/strict';
2+
import { test } from 'vitest';
3+
import { ownerFilesForCommand } from '@agent-device/command-registry/owner-files';
4+
import { INTERACTION_COMMAND_OUTPUT_SCHEMAS } from '../../commands/interaction/index.ts';
5+
import { COMMAND_OUTPUT_SCHEMAS } from '../command-output-schemas.ts';
6+
7+
const INTERACTION_COMMANDS = Object.keys(INTERACTION_COMMAND_OUTPUT_SCHEMAS) as Array<
8+
keyof typeof INTERACTION_COMMAND_OUTPUT_SCHEMAS
9+
>;
10+
11+
// press, click, fill, longpress, and hover carry the post-action observation trait (#1652): the
12+
// composed map grafts a `settle` property onto a COPY, so they are not reference-equal to the
13+
// module's own object. find is read-only and carries no such trait.
14+
const SETTLE_DERIVED_COMMANDS = new Set(['press', 'click', 'fill', 'longpress', 'hover']);
15+
16+
test('MCP interaction family output schemas are the family module entries, not copies', () => {
17+
for (const command of INTERACTION_COMMANDS) {
18+
if (SETTLE_DERIVED_COMMANDS.has(command)) continue;
19+
assert.equal(
20+
COMMAND_OUTPUT_SCHEMAS[command],
21+
INTERACTION_COMMAND_OUTPUT_SCHEMAS[command],
22+
`${command} is not reference-equal to the interaction module's own schema object`,
23+
);
24+
}
25+
});
26+
27+
test('a settle-derived interaction entry still grafts onto the module object, not a foreign copy', () => {
28+
const modulePress = INTERACTION_COMMAND_OUTPUT_SCHEMAS.press;
29+
const derivedPress = COMMAND_OUTPUT_SCHEMAS.press;
30+
assert.notEqual(
31+
derivedPress,
32+
modulePress,
33+
'the derivation pass must copy, never mutate in place',
34+
);
35+
assert.equal(derivedPress.required, modulePress.required, 'required list is not re-derived');
36+
assert.equal(
37+
derivedPress.properties?.targetKind,
38+
modulePress.properties?.targetKind,
39+
'a field object nested under the derived copy must still be the module property, not a rebuild',
40+
);
41+
assert.ok(
42+
derivedPress.properties?.settle,
43+
'the settle-capable command must carry the grafted observation property',
44+
);
45+
});
46+
47+
test('press and click share the same tap response schema object', () => {
48+
assert.equal(INTERACTION_COMMAND_OUTPUT_SCHEMAS.press, INTERACTION_COMMAND_OUTPUT_SCHEMAS.click);
49+
});
50+
51+
test('the projected family map declares exactly the commands its module owns', () => {
52+
for (const command of INTERACTION_COMMANDS) {
53+
assert.ok(
54+
ownerFilesForCommand(command).includes('src/commands/interaction/index.ts'),
55+
`${command} projects its output schema from this module but does not name it as its owner`,
56+
);
57+
}
58+
});

‎src/mcp/__tests__/command-tools-replay-schemas.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { test } from 'vitest';
33
import type { ReplayCommandResult, ReplaySuiteResult } from '@agent-device/contracts/replay';
44
import { ownerFilesForCommand } from '@agent-device/command-registry/owner-files';
55
import { commandSupportsSettleObservation } from '@agent-device/command-registry/registry';
6+
import { INTERACTION_COMMAND_OUTPUT_SCHEMAS } from '../../commands/interaction/index.ts';
67
import { REPLAY_COMMAND_OUTPUT_SCHEMAS } from '../../commands/replay/index.ts';
78
import { SYSTEM_COMMAND_OUTPUT_SCHEMAS } from '../../commands/system/index.ts';
89
import { COMMAND_OUTPUT_SCHEMAS } from '../command-output-schemas.ts';
@@ -71,6 +72,7 @@ test('the projected family map declares exactly the commands its module owns', (
7172
const PROJECTED_FAMILIES = [
7273
{ name: 'replay', schemas: REPLAY_COMMAND_OUTPUT_SCHEMAS },
7374
{ name: 'system', schemas: SYSTEM_COMMAND_OUTPUT_SCHEMAS },
75+
{ name: 'interaction', schemas: INTERACTION_COMMAND_OUTPUT_SCHEMAS },
7476
] as const;
7577

7678
test('projected output-schema families claim disjoint commands and survive the composition intact', () => {

0 commit comments

Comments
 (0)