Skip to content

Commit b476c94

Browse files
cameroncookecodex
andcommitted
fix(ui-automation): Report semantic touch commands accurately
Use the actual AXe command name when executing semantic tap commands so switch-backed touch interactions log and report as touch instead of tap. Co-Authored-By: Codex <noreply@openai.com>
1 parent c7e93f9 commit b476c94

2 files changed

Lines changed: 61 additions & 2 deletions

File tree

src/mcp/tools/ui-automation/__tests__/semantic-tap.test.ts

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,44 @@ describe('semantic tap helpers', () => {
9191
]);
9292
});
9393

94+
it('uses the executed command name for switch touch commands', async () => {
95+
const [element] = createElements([
96+
createNode({
97+
type: 'Switch',
98+
role: 'AXSwitch',
99+
AXLabel: 'Alerts',
100+
frame: { x: 10, y: 20, width: 200, height: 40 },
101+
}),
102+
]);
103+
const command = createSemanticTapCommand(element!, 'e1');
104+
const { calls, executor } = createSequencedExecutor([{ success: true, output: 'ok' }]);
105+
106+
await executeSemanticTapWithAmbiguityFallback({
107+
command,
108+
simulatorId,
109+
executor,
110+
axeHelpers: createMockAxeHelpers(),
111+
});
112+
113+
expect(calls[0]).toEqual(
114+
expect.objectContaining({
115+
command: [
116+
'/mocked/axe/path',
117+
'touch',
118+
'-x',
119+
'158',
120+
'-y',
121+
'40',
122+
'--down',
123+
'--up',
124+
'--udid',
125+
simulatorId,
126+
],
127+
logPrefix: '[AXe]: touch',
128+
}),
129+
);
130+
});
131+
94132
it('retries recoverable selector failures with coordinates', async () => {
95133
const [element] = createElements([
96134
createNode({ AXUniqueId: 'continue.button', AXLabel: 'Continue' }),
@@ -112,6 +150,7 @@ describe('semantic tap helpers', () => {
112150
['tap', '--id', 'continue.button', '--element-type', 'Button'],
113151
['tap', '-x', '60', '-y', '40'],
114152
]);
153+
expect(calls.map((call) => call.logPrefix)).toEqual(['[AXe]: tap', '[AXe]: tap']);
115154
});
116155

117156
it('does not retry unrecoverable selector failures', async () => {

src/mcp/tools/ui-automation/shared/semantic-tap.ts

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,14 @@ export function createSemanticTapCommand(
118118
};
119119
}
120120

121+
function readAxeCommandName(args: readonly string[]): string {
122+
const commandName = args[0];
123+
if (!commandName) {
124+
throw new Error('Semantic tap command has no AXe command name.');
125+
}
126+
return commandName;
127+
}
128+
121129
export function createSemanticTapBatchSteps(command: SemanticTapCommand): string[] {
122130
if (command.coordinateArgs[0] !== 'touch') {
123131
return [command.coordinateArgs.join(' ')];
@@ -136,12 +144,24 @@ export async function executeSemanticTapWithAmbiguityFallback(params: {
136144
const { command, simulatorId, executor, axeHelpers } = params;
137145

138146
try {
139-
await executeAxeCommand(command.primaryArgs, simulatorId, 'tap', executor, axeHelpers);
147+
await executeAxeCommand(
148+
command.primaryArgs,
149+
simulatorId,
150+
readAxeCommandName(command.primaryArgs),
151+
executor,
152+
axeHelpers,
153+
);
140154
} catch (error) {
141155
if (!command.selectorArgs || !isRecoverableAxeSelectorError(error)) {
142156
throw error;
143157
}
144158

145-
await executeAxeCommand(command.coordinateArgs, simulatorId, 'tap', executor, axeHelpers);
159+
await executeAxeCommand(
160+
command.coordinateArgs,
161+
simulatorId,
162+
readAxeCommandName(command.coordinateArgs),
163+
executor,
164+
axeHelpers,
165+
);
146166
}
147167
}

0 commit comments

Comments
 (0)