fix(ios): say when described coordinates are not the ones taps use

packages/skills/skills/argent-device-interact/SKILL.md

@@ -232,6 +232,8 @@ When using `screenshot` for permission or native modal navigation:
 
 Optional rotation parameter: `{ "udid": "<UDID>", "rotation": "LandscapeLeft" }` — rotates the capture without changing simulator orientation.
 
+On a rotated iOS simulator the capture is composited in the device's unrotated space, so it looks sideways. `rotation` makes it readable, but that image is then in a different coordinate space from `describe` frames and from where taps land — so never read tap coordinates off a rotated capture. Use `describe` for anything you intend to tap.
+
 Screenshots are downscaled by default (30% of original resolution) to reduce context size. Use the normal downscaled screenshot for UI context and state checks. `scale` accepts values from 0.01 to 1.0, but do not use `scale: 1.0` as a general readability or tapping aid.
 
 Use full-resolution screenshots only when saving baseline/current PNG files for comparison. In that case, suppress the image block so the full-size PNG is not loaded into agent context:

packages/tool-server/src/tools/describe/platforms/ios/index.ts

@@ -9,6 +9,7 @@ import {
 import { resolveNativeTargetApp } from "../../../../utils/native-target-app";
 import { isTvOsSimulator } from "../../../../utils/ios-devices";
 import { parseNativeDescribeScreenResult } from "../../../native-devtools/native-describe-contract";
+import { withLandscapeHint } from "../../../../utils/ios-orientation-hint";
 import { DescribeTreeData, parseDescribeResult, type DescribeNode } from "../../contract";
 import { adaptAXDescribeToDescribeResult } from "./ios-ax-adapter";
 import { adaptNativeDescribeToDescribeResult } from "./ios-native-adapter";
@@ -158,7 +159,15 @@ export async function describeIos(
 
     const parsed = parseNativeDescribeScreenResult(rawResult);
     const nativeTree = adaptNativeDescribeToDescribeResult(parsed);
-    return { tree: nativeTree, source: "native-devtools", hint };
+    // This branch reports the app's own coordinate space, which on a rotated
+    // device is upright — unlike the ax-service branch above and unlike where
+    // taps land. Say so rather than silently handing back coordinates that miss
+    // (#609).
+    return {
+      tree: nativeTree,
+      source: "native-devtools",
+      hint: withLandscapeHint(hint, parsed.screenFrame),
+    };
   } catch {
     // Native devtools unavailable or no connected app — return the empty AX result
     return { tree, source: "ax-service", hint };

packages/tool-server/src/tools/native-devtools/native-describe-screen.ts

@@ -12,6 +12,7 @@ import {
   parseNativeDescribeScreenResult,
   type NativeDescribeScreenResult,
 } from "./native-describe-contract";
+import { withLandscapeHint } from "../../utils/ios-orientation-hint";
 
 const zodSchema = z.object({
   udid: z.string().describe("Simulator UDID"),
@@ -96,6 +97,11 @@ If status is restart_required: call restart-app then retry.`,
     }
 
     const parsed = parseNativeDescribeScreenResult(result);
-    return { status: "ok", ...parsed };
+    // normalizedFrame / normalizedTapPoint are relative to the app's own screen
+    // bounds, which follow the device rotation. Touch input does not, so on a
+    // rotated device these are not tap coordinates (#609). Flag it rather than
+    // leave the caller to discover it by missing.
+    const hint = withLandscapeHint(undefined, parsed.screenFrame);
+    return { status: "ok", ...parsed, ...(hint ? { hint } : {}) };
   },
 };

packages/tool-server/src/tools/rotate/index.ts

@@ -3,6 +3,7 @@ import type { ToolCapability, ToolDefinition } from "@argent/registry";
 import { simulatorServerRef, type SimulatorServerApi } from "../../blueprints/simulator-server";
 import { resolveDevice } from "../../utils/device-info";
 import { sendCommand } from "../../utils/simulator-client";
+import { IOS_ROTATED_CAPTURE_NOTE } from "../../utils/ios-orientation-hint";
 
 const zodSchema = z.object({
   udid: z.string().describe("Target device id from `list-devices` (iOS UDID or Android serial)."),
@@ -15,6 +16,8 @@ type Params = z.infer<typeof zodSchema>;
 
 interface Result {
   orientation: string;
+  /** Present on iOS, where rotating leaves the capture in a different space. */
+  note?: string;
 }
 
 const capability: ToolCapability = {
@@ -32,15 +35,26 @@ export const rotateTool: ToolDefinition<Params, Result> = {
   },
   description: `Set the device orientation to Portrait, LandscapeLeft, LandscapeRight, or PortraitUpsideDown.
 Use to test layout in a different orientation. Re-run \`describe\` afterwards — frame coordinates change with the orientation.
-Returns { orientation }. Fails if the target device is not booted.`,
+On iOS a rotated simulator still captures in its unrotated space, so the screenshot looks sideways; \`describe\` stays the source of tap coordinates.
+Returns { orientation, note }. Fails if the target device is not booted.`,
   zodSchema,
   capability,
   services: (params) => ({
     simulatorServer: simulatorServerRef(resolveDevice(params.udid)),
   }),
   async execute(services, params) {
     const api = services.simulatorServer as SimulatorServerApi;
+    const device = resolveDevice(params.udid);
     sendCommand(api, { cmd: "rotate", direction: params.orientation });
-    return { orientation: params.orientation };
+    // On iOS the capture is composited in the device's unrotated space, so it
+    // comes back sideways after a rotation. `rotation` on `screenshot` will make
+    // it readable, but that image is then in a different space from `describe`
+    // frames and from where taps land — so say both halves rather than
+    // recommending a flag that silently breaks coordinates (#609).
+    const note =
+      device.platform === "ios" || device.platform === "ios-remote"
+        ? IOS_ROTATED_CAPTURE_NOTE
+        : undefined;
+    return { orientation: params.orientation, ...(note ? { note } : {}) };
   },
 };

packages/tool-server/src/utils/ios-orientation-hint.ts

@@ -0,0 +1,68 @@
+/**
+ * Advisory for the iOS description paths that report coordinates in the app's
+ * upright space while the device's touch space is portrait-native.
+ *
+ * Background (#609). On a rotated iOS simulator the surfaces disagree:
+ *
+ *   `describe` via ax-service   portrait-native  — matches where taps land
+ *   `describe` via native-devtools  upright      — does NOT match
+ *   `native-describe-screen`        upright      — does NOT match
+ *   gesture-* input                 portrait-native
+ *
+ * Measured on a landscape iPad Pro 13-inch: the same button reads
+ * `(0.238, 0.342)` from ax-service and `(0.342, 0.762)` from native-devtools,
+ * related by `ax.x = 1 - native.y`, `ax.y = native.x` — a 90° rotation.
+ *
+ * These coordinates are deliberately NOT transformed here. The only orientation
+ * signal available is the screen's aspect, which gives the axis but not the
+ * sense — it cannot separate LandscapeLeft from LandscapeRight, and those differ
+ * by 180°. Rotating on a guess would turn a visible mismatch into taps that land
+ * somewhere plausible but wrong, which is far harder to notice. Saying so is the
+ * honest fix until the orientation itself is on the wire.
+ */
+
+export interface ScreenFrameLike {
+  width: number;
+  height: number;
+}
+
+/** Is the app reporting a screen wider than it is tall, i.e. a rotated device? */
+export function isLandscapeScreenFrame(frame: ScreenFrameLike | undefined): boolean {
+  if (!frame) return false;
+  return frame.width > frame.height;
+}
+
+export const LANDSCAPE_COORDINATE_HINT =
+  "This device is rotated, and these coordinates are in the app's upright space. " +
+  "Touch input is in the device's unrotated space, so tapping these values directly " +
+  "will miss. Prefer `describe` (which reads the accessibility tree in touch space) " +
+  "for anything you intend to tap.";
+
+/**
+ * Combine an existing hint with the landscape advisory.
+ *
+ * Existing hints stay first: `describe`'s other hints tell the caller the
+ * simulator needs rebooting or the app cannot be inspected at all, and those are
+ * blocking problems where this advisory would only be noise.
+ */
+/**
+ * What `rotate` tells the caller on iOS.
+ *
+ * It deliberately does not stop at "pass `rotation` to fix the image". Doing
+ * only that is the trap the issue reports: the readable capture is then in the
+ * app's upright space while `describe` frames and touch input stay in the
+ * device's unrotated one, so coordinates read off that image miss.
+ */
+export const IOS_ROTATED_CAPTURE_NOTE =
+  "On iOS the screen is captured in the device's unrotated space, so `screenshot` will look " +
+  "sideways after this. Passing `rotation` to `screenshot` makes it readable, but that image " +
+  "is then in a different space from `describe` frames and from where taps land — do not read " +
+  "coordinates off it. Use `describe` for anything you intend to tap.";
+
+export function withLandscapeHint(
+  existing: string | undefined,
+  frame: ScreenFrameLike | undefined
+): string | undefined {
+  if (!isLandscapeScreenFrame(frame)) return existing;
+  return existing ? `${existing} ${LANDSCAPE_COORDINATE_HINT}` : LANDSCAPE_COORDINATE_HINT;
+}

packages/tool-server/test/ios-orientation-hint.test.ts

@@ -0,0 +1,83 @@
+import { describe, it, expect } from "vitest";
+import {
+  IOS_ROTATED_CAPTURE_NOTE,
+  LANDSCAPE_COORDINATE_HINT,
+  isLandscapeScreenFrame,
+  withLandscapeHint,
+} from "../src/utils/ios-orientation-hint";
+
+/**
+ * Issue #609. On a rotated iOS simulator argent reports coordinates in two
+ * different spaces depending on which path served the tree:
+ *
+ *   describe via ax-service       portrait-native  — agrees with where taps land
+ *   describe via native-devtools  upright          — does not
+ *   native-describe-screen        upright          — does not
+ *
+ * Measured on a landscape iPad Pro 13-inch, the same button reads (0.238, 0.342)
+ * from ax-service and (0.342, 0.762) from native-devtools.
+ *
+ * These tests pin the advisory and — just as importantly — pin that the numbers
+ * themselves are left alone. The screen aspect gives the axis but not the sense,
+ * so a transform here would be a guess between two possibilities 180° apart.
+ */
+
+const LANDSCAPE = { width: 1376, height: 1032 }; // rotated iPad Pro 13-inch
+const PORTRAIT = { width: 1032, height: 1376 };
+
+describe("isLandscapeScreenFrame", () => {
+  it("recognises a rotated device by its reported screen", () => {
+    expect(isLandscapeScreenFrame(LANDSCAPE)).toBe(true);
+  });
+
+  it("does not flag an upright device", () => {
+    expect(isLandscapeScreenFrame(PORTRAIT)).toBe(false);
+  });
+
+  it("does not flag a square screen, which says nothing either way", () => {
+    expect(isLandscapeScreenFrame({ width: 1024, height: 1024 })).toBe(false);
+  });
+
+  it("does not flag a missing screen frame", () => {
+    expect(isLandscapeScreenFrame(undefined)).toBe(false);
+  });
+});
+
+describe("withLandscapeHint", () => {
+  it("adds the advisory when the device is rotated", () => {
+    expect(withLandscapeHint(undefined, LANDSCAPE)).toBe(LANDSCAPE_COORDINATE_HINT);
+  });
+
+  it("adds nothing when the device is upright", () => {
+    expect(withLandscapeHint(undefined, PORTRAIT)).toBeUndefined();
+  });
+
+  it("leaves an existing hint untouched when the device is upright", () => {
+    expect(withLandscapeHint("reboot the simulator", PORTRAIT)).toBe("reboot the simulator");
+  });
+
+  it("keeps a blocking hint first when both apply", () => {
+    // describe's other hints say the simulator needs rebooting or the app cannot
+    // be inspected at all. Those are blocking; a coordinate-space note is only
+    // useful once you have a tree, so it must not be what the caller reads first.
+    const combined = withLandscapeHint("reboot the simulator", LANDSCAPE)!;
+    expect(combined.startsWith("reboot the simulator")).toBe(true);
+    expect(combined).toContain(LANDSCAPE_COORDINATE_HINT);
+  });
+
+  it("points the caller at describe rather than at a coordinate transform", () => {
+    expect(LANDSCAPE_COORDINATE_HINT).toMatch(/describe/);
+    expect(LANDSCAPE_COORDINATE_HINT).toMatch(/will miss/);
+  });
+});
+
+describe("the rotate note", () => {
+  it("does not stop at 'pass rotation', which is the trap being reported", () => {
+    // Recommending `rotation` alone yields a readable image whose coordinates
+    // silently disagree with describe and with touch — worse than a sideways
+    // image, because the failure is invisible.
+    expect(IOS_ROTATED_CAPTURE_NOTE).toContain("rotation");
+    expect(IOS_ROTATED_CAPTURE_NOTE).toMatch(/different space/);
+    expect(IOS_ROTATED_CAPTURE_NOTE).toMatch(/do not read coordinates off it/i);
+  });
+});