@@ -20,12 +20,21 @@ import type {
2020 TypeTextOptions ,
2121} from '@agent-device/contracts/client' ;
2222import type { CommandSchemaOverride } from '@agent-device/command-registry/command-schema' ;
23+ import type { CommandResultMap } from '@agent-device/command-registry/command-result' ;
2324import {
2425 REPEATED_TOUCH_FLAGS ,
2526 SELECTOR_SNAPSHOT_FLAGS ,
2627} from '@agent-device/command-registry/flag-groups' ;
2728import { postActionObservationCliFlags } from '../post-action-observation-grammar.ts' ;
29+ import type { JsonSchema } from '../command-contract.ts' ;
2830import {
31+ booleanSchema ,
32+ enumSchema ,
33+ looseObjectSchema ,
34+ numberSchema ,
35+ objectSchema ,
36+ stringArraySchema ,
37+ stringSchema ,
2938 toClientElementTarget ,
3039 toClientInteractionTarget ,
3140 toRepeatedOptions ,
@@ -54,6 +63,288 @@ import {
5463import { interactionCliOutputFormatters } from './output.ts' ;
5564import { selectorCliReaders , selectorDaemonWriters } from './selectors.ts' ;
5665
66+ function constSchema ( value : string ) : JsonSchema {
67+ return { type : 'string' , const : value } ;
68+ }
69+
70+ function nullableStringSchema ( description ?: string ) : JsonSchema {
71+ return { type : [ 'string' , 'null' ] , ...( description ? { description } : { } ) } ;
72+ }
73+
74+ // PostActionSurfaceChange (packages/contracts/src/interaction.ts) — the post-action capture
75+ // describes a different surface than the pre-action baseline (#2438), so no same-surface
76+ // comparison is presented across it. Mirrored in src/mcp/command-output-schemas.ts for the
77+ // generic `--settle` observation, which every family (not only this one) can carry.
78+ const postActionSurfaceChangeSchema : JsonSchema = objectSchema (
79+ {
80+ from : stringSchema ( 'Surface the pre-action baseline described: a host bundle id, or app.' ) ,
81+ to : stringSchema ( 'Surface the post-action capture describes: a host bundle id, or app.' ) ,
82+ disclosure : stringSchema ( 'Agent-facing sentence explaining the surface transition.' ) ,
83+ } ,
84+ [ 'from' , 'to' , 'disclosure' ] ,
85+ 'Present when an in-place system surface (web sign-in or Apple Pay sheet) was presented over the app, or left it.' ,
86+ ) ;
87+
88+ // InteractionEvidence (packages/contracts/src/interaction.ts) — opt-in `--verify` cheap
89+ // post-condition evidence (#1047).
90+ const interactionEvidenceSchema : JsonSchema = objectSchema (
91+ {
92+ foregroundApp : stringSchema ( 'Foreground app bundle id or name, when the capture carries it.' ) ,
93+ nodeCount : numberSchema ( 'Node count in the post-action interactive-only capture.' ) ,
94+ interactiveNodeCount : numberSchema ( 'Subset of nodeCount the platform reports as hittable.' ) ,
95+ digest : stringSchema ( 'Order-independent digest of the post-action node multiset.' ) ,
96+ changedFromBefore : booleanSchema (
97+ 'Whether the post-action digest differs from the pre-action capture digest. false is evidence, not failure. With surfaceChange present, no digest comparison is made: it reports that surface transition.' ,
98+ ) ,
99+ surfaceChange : postActionSurfaceChangeSchema ,
100+ } ,
101+ [ 'nodeCount' , 'interactiveNodeCount' , 'digest' , 'changedFromBefore' ] ,
102+ ) ;
103+
104+ const responseCostSchema : JsonSchema = objectSchema (
105+ {
106+ wallClockMs : numberSchema ( 'Total wall-clock time for the request in milliseconds.' ) ,
107+ runnerRoundTrips : numberSchema (
108+ 'Number of real runner round-trips made while serving the request.' ,
109+ ) ,
110+ nodeCount : numberSchema (
111+ 'Number of nodes in the original node tree when the response carries one.' ,
112+ ) ,
113+ } ,
114+ [ 'wallClockMs' , 'runnerRoundTrips' ] ,
115+ ) ;
116+
117+ // ResolutionDiagnosticEntry (packages/contracts/src/interaction.ts) — a disambiguation
118+ // winner or losing alternative. Never a snapshot ref.
119+ const resolutionDiagnosticEntrySchema : JsonSchema = objectSchema (
120+ {
121+ diagnosticRef : stringSchema (
122+ 'Opaque non-@ diagnostic token. Never a snapshot ref: not issued via refsGeneration and cannot be pinned or reused as an @ref target. UTF-8 truncated to 256 bytes.' ,
123+ ) ,
124+ role : stringSchema ( 'UTF-8 truncated to 256 bytes.' ) ,
125+ label : stringSchema ( 'UTF-8 truncated to 256 bytes.' ) ,
126+ } ,
127+ [ 'diagnosticRef' ] ,
128+ ) ;
129+
130+ // ResolutionDisclosure (packages/contracts/src/interaction.ts) — never ref-issuing;
131+ // absent on paths where the guarantee is inapplicable (ADR 0012 decision 2).
132+ // `alternatives` rides default/full levels only; the digest view omits it.
133+ const resolutionDisclosureSchema : JsonSchema = {
134+ type : 'object' ,
135+ description :
136+ 'Pre-action disclosure of how the acting path resolved its target. Absent when resolutionDisclosure is inapplicable for the path.' ,
137+ oneOf : [
138+ objectSchema (
139+ {
140+ source : constSchema ( 'runtime' ) ,
141+ phase : constSchema ( 'pre-action' ) ,
142+ kind : constSchema ( 'unique' ) ,
143+ } ,
144+ [ 'source' , 'phase' , 'kind' ] ,
145+ ) ,
146+ objectSchema (
147+ {
148+ source : constSchema ( 'runtime' ) ,
149+ phase : constSchema ( 'pre-action' ) ,
150+ kind : constSchema ( 'disambiguated' ) ,
151+ matchCount : numberSchema ( 'Total matches resolveSelectorChain found before disambiguation.' ) ,
152+ winnerDiagnostic : resolutionDiagnosticEntrySchema ,
153+ tiebreak : enumSchema (
154+ [ 'visible' , 'deepest' , 'smallest-area' , 'structural-equivalence' ] ,
155+ 'The comparison that decided the winner.' ,
156+ ) ,
157+ alternatives : {
158+ type : 'array' ,
159+ description :
160+ 'At most 5 losing candidates, document order. Present at default/full response levels and omitted in digest. The winner is never included.' ,
161+ items : resolutionDiagnosticEntrySchema ,
162+ } ,
163+ } ,
164+ [ 'source' , 'phase' , 'kind' , 'matchCount' , 'winnerDiagnostic' , 'tiebreak' ] ,
165+ ) ,
166+ objectSchema (
167+ { source : constSchema ( 'ref' ) , phase : constSchema ( 'pre-action' ) , kind : constSchema ( 'exact' ) } ,
168+ [ 'source' , 'phase' , 'kind' ] ,
169+ ) ,
170+ objectSchema (
171+ {
172+ source : constSchema ( 'ref' ) ,
173+ phase : constSchema ( 'pre-action' ) ,
174+ kind : constSchema ( 'label-fallback' ) ,
175+ } ,
176+ [ 'source' , 'phase' , 'kind' ] ,
177+ ) ,
178+ objectSchema ( { source : constSchema ( 'direct-ios' ) , kind : constSchema ( 'not-observed' ) } , [
179+ 'source' ,
180+ 'kind' ,
181+ ] ) ,
182+ ] ,
183+ } ;
184+
185+ type InteractionExtra = {
186+ properties ?: Record < string , JsonSchema > ;
187+ required ?: readonly string [ ] ;
188+ } ;
189+
190+ /**
191+ * Canonical interaction response data built by buildInteractionResponseData:
192+ * shared target/coordinate/evidence fields plus per-command extras. The runtime
193+ * result still has richer internal node/backend data; this schema documents the
194+ * JSON payload returned to clients.
195+ */
196+ function interactionResponseDataSchema ( extra : InteractionExtra = { } ) : JsonSchema {
197+ const extraProperties = extra . properties ?? { } ;
198+ const extraRequired = extra . required ?? [ ] ;
199+ return objectSchema (
200+ {
201+ targetKind : enumSchema ( [ 'point' , 'ref' , 'selector' ] , 'Resolved interaction target kind.' ) ,
202+ x : numberSchema ( 'Resolved interaction x coordinate when available.' ) ,
203+ y : numberSchema ( 'Resolved interaction y coordinate when available.' ) ,
204+ referenceWidth : numberSchema ( 'Reference frame width for visualizing the interaction point.' ) ,
205+ referenceHeight : numberSchema (
206+ 'Reference frame height for visualizing the interaction point.' ,
207+ ) ,
208+ ref : stringSchema ( 'Snapshot ref without the @ prefix when the target was an @ref.' ) ,
209+ selector : stringSchema ( 'Selector expression when the target was a selector.' ) ,
210+ selectorChain : stringArraySchema ( ) ,
211+ refLabel : stringSchema ( ) ,
212+ targetHittable : booleanSchema ( ) ,
213+ hint : stringSchema ( ) ,
214+ warning : stringSchema ( ) ,
215+ message : stringSchema ( ) ,
216+ evidence : interactionEvidenceSchema ,
217+ resolution : resolutionDisclosureSchema ,
218+ cost : responseCostSchema ,
219+ maestroNonHittableCoordinateFallbackAllowed : booleanSchema (
220+ 'Whether the direct iOS Maestro coordinate fallback was allowed for this selector.' ,
221+ ) ,
222+ maestroNonHittableCoordinateFallbackUsed : booleanSchema (
223+ 'Whether the direct iOS Maestro coordinate fallback was actually used.' ,
224+ ) ,
225+ maestroFallbackReason : constSchema ( 'non-hittable-coordinate' ) ,
226+ ...extraProperties ,
227+ } ,
228+ [ 'targetKind' , ...extraRequired ] ,
229+ ) ;
230+ }
231+
232+ const tapInteractionResponseDataSchema = interactionResponseDataSchema ( {
233+ properties : {
234+ evidence : interactionEvidenceSchema ,
235+ button : enumSchema ( [ 'secondary' , 'middle' ] ) ,
236+ count : numberSchema ( 'Number of press/click repetitions.' ) ,
237+ intervalMs : numberSchema ( 'Delay between repeated press/click actions.' ) ,
238+ holdMs : numberSchema ( 'Hold duration for each action.' ) ,
239+ jitterPx : numberSchema ( 'Randomization radius in pixels.' ) ,
240+ doubleTap : booleanSchema ( 'Whether the command requested a double-tap action.' ) ,
241+ } ,
242+ } ) ;
243+
244+ const fillResponseProperties = {
245+ text : stringSchema ( 'Text submitted to the field.' ) ,
246+ delayMs : numberSchema ( 'Delay between typed characters in milliseconds.' ) ,
247+ evidence : interactionEvidenceSchema ,
248+ } ;
249+
250+ const fillVerificationTargetSchema = objectSchema (
251+ {
252+ resourceId : nullableStringSchema ( 'Android resource id of the exact field that changed.' ) ,
253+ className : nullableStringSchema ( 'Android class name of the exact field that changed.' ) ,
254+ packageName : nullableStringSchema ( 'Android package name that owns the exact field.' ) ,
255+ rect : objectSchema (
256+ {
257+ x : numberSchema ( ) ,
258+ y : numberSchema ( ) ,
259+ width : numberSchema ( ) ,
260+ height : numberSchema ( ) ,
261+ } ,
262+ [ 'x' , 'y' , 'width' , 'height' ] ,
263+ 'Screen-space rectangle of the exact field that changed.' ,
264+ ) ,
265+ } ,
266+ [ 'resourceId' , 'className' , 'packageName' , 'rect' ] ,
267+ 'Target identity captured before fill and matched after fill.' ,
268+ ) ;
269+
270+ const confirmedFillResponseSchema : JsonSchema = {
271+ ...interactionResponseDataSchema ( {
272+ properties : fillResponseProperties ,
273+ required : [ 'text' ] ,
274+ } ) ,
275+ // The public result contract omits verification evidence on an ordinary
276+ // confirmed fill. Keep this branch disjoint from the unconfirmed branch
277+ // without making the response strict to unrelated additive fields.
278+ not : objectSchema ( { } , [ 'verification' ] ) ,
279+ } ;
280+
281+ const unconfirmedFillResponseSchema = interactionResponseDataSchema ( {
282+ properties : {
283+ ...fillResponseProperties ,
284+ verification : constSchema ( 'unconfirmed' ) ,
285+ requested : stringSchema ( 'Literal text requested by the fill command.' ) ,
286+ before : nullableStringSchema ( 'Raw target text captured before the fill.' ) ,
287+ after : nullableStringSchema ( 'Raw target text captured after the fill.' ) ,
288+ target : fillVerificationTargetSchema ,
289+ } ,
290+ required : [ 'text' , 'verification' , 'requested' , 'before' , 'after' , 'target' ] ,
291+ } ) ;
292+
293+ /**
294+ * This family's advertised MCP `outputSchema`s, keyed by daemon command name and projected into
295+ * the command map by `src/mcp/command-output-schemas.ts`. Non-strict like every other entry: no
296+ * `additionalProperties: false`, so additive response fields such as `settle`/`cost` keep
297+ * validating. #1652: the opt-in `settle` observation is NOT listed here — the trait derivation
298+ * pass in that file grafts it onto settle-capable entries.
299+ */
300+ export const INTERACTION_COMMAND_OUTPUT_SCHEMAS = {
301+ press : tapInteractionResponseDataSchema ,
302+ click : tapInteractionResponseDataSchema ,
303+ fill : {
304+ type : 'object' ,
305+ description :
306+ 'Fill response. Android may return target-bound unconfirmed evidence when the exact app-owned field changed but formatting prevented raw equality.' ,
307+ oneOf : [ confirmedFillResponseSchema , unconfirmedFillResponseSchema ] ,
308+ } ,
309+ longpress : interactionResponseDataSchema ( {
310+ properties : {
311+ durationMs : numberSchema ( ) ,
312+ gesture : constSchema ( 'longpress' ) ,
313+ } ,
314+ } ) ,
315+ hover : interactionResponseDataSchema ( {
316+ properties : {
317+ gesture : constSchema ( 'hover' ) ,
318+ } ,
319+ } ) ,
320+ find : objectSchema (
321+ {
322+ ref : stringSchema ( 'Snapshot ref without the @ prefix when the find action returns one.' ) ,
323+ refsGeneration : numberSchema ( 'ADR 0014 ref frame epoch for read-only find actions.' ) ,
324+ found : booleanSchema ( 'Whether a wait/exists/read-only find satisfied its condition.' ) ,
325+ waitedMs : numberSchema ( 'Milliseconds waited for a read-only find condition.' ) ,
326+ text : stringSchema ( 'Text value returned by find get_text.' ) ,
327+ node : looseObjectSchema ( 'Snapshot node for find get_attrs/get_text.' ) ,
328+ matches : {
329+ type : 'array' ,
330+ description : 'Every match for the read-only find list action (#1625): { ref, node } each.' ,
331+ items : looseObjectSchema ( 'One listed match with its snapshot ref and node.' ) ,
332+ } ,
333+ locator : stringSchema ( 'Locator kind used for the find action.' ) ,
334+ query : stringSchema ( 'Query argument used for the find action.' ) ,
335+ x : numberSchema ( 'Resolved x coordinate for mutating find actions.' ) ,
336+ y : numberSchema ( 'Resolved y coordinate for mutating find actions.' ) ,
337+ message : stringSchema ( 'Diagnostic message for mutating find actions.' ) ,
338+ cost : responseCostSchema ,
339+ } ,
340+ [ ] ,
341+ 'Daemon response data for the find command.' ,
342+ ) ,
343+ } satisfies Pick <
344+ Record < keyof CommandResultMap , JsonSchema > ,
345+ 'press' | 'click' | 'fill' | 'longpress' | 'hover' | 'find'
346+ > ;
347+
57348const interactionCliSchemas = {
58349 get : {
59350 usageOverride : 'get text|attrs <@ref|selector>' ,
0 commit comments