chore(gates): integrate merged proxy package declarations

Author: thymikee

.fallowrc.json

@@ -20,6 +20,9 @@
     "src/sdk/selectors.ts",
     "src/sdk/finders.ts",
     "src/ai-sdk/index.ts",
+    "packages/proxy/src/index.ts",
+    "packages/proxy/src/node.ts",
+    "packages/proxy/tsdown.config.ts",
     "src/bin.ts",
     "src/client/companion-tunnel.ts",
     "src/daemon.ts",
@@ -290,8 +293,8 @@
       "exports": ["*"]
     },
     {
-      "comment": "Published package surface (see package.json#exports); consumed by SDK users, not internally.",
-      "file": "{src/sdk/*.ts,src/ai-sdk/index.ts}",
+      "comment": "Published package surfaces (see each package.json#exports); consumers may be external or reach them through workspace package exports Fallow does not follow.",
+      "file": "{src/sdk/*.ts,src/ai-sdk/index.ts,packages/proxy/src/index.ts,packages/proxy/src/node.ts}",
       "exports": ["*"]
     },
     {
@@ -301,7 +304,7 @@
     },
     {
       "comment": "Tool config default exports, loaded by the tool rather than imported.",
-      "file": "{oxlint.config.ts,tsdown.config.ts,packages/provider-*/tsdown.config.ts,vitest.mutation.config.ts,website/rspress.config.ts}",
+      "file": "{oxlint.config.ts,tsdown.config.ts,packages/provider-*/tsdown.config.ts,packages/proxy/tsdown.config.ts,vitest.mutation.config.ts,website/rspress.config.ts}",
       "exports": ["default"]
     },
     {

android/snapshot-helper/README.md

@@ -106,6 +106,9 @@ The XML node attributes intentionally mirror acquisition facts decoded by the ho
 metadata on window roots. The helper emits `drawing-order` on Android API 24+ and omits it on API
 23, where the platform API is unavailable. The host keeps that fact as private capture evidence;
 the daemon uses it to annotate covered actions without adding it to normalized snapshot nodes.
+On API 33+ an input method window root also carries `window-region-rect`, whether its touchable
+region is a single rectangle (`window-bounds` is that region's bounding box). It is omitted on other
+window types, on API 32 and lower, and when the region reads empty.
 
 Each XML chunk is sent with:
 

android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java

@@ -5,6 +5,7 @@
 import android.os.Build;
 import android.view.accessibility.AccessibilityNodeInfo;
 import android.view.accessibility.AccessibilityWindowInfo;
+import java.util.ArrayList;
 import java.util.List;
 
 /** Captures and serializes the Android accessibility window tree. */
@@ -25,7 +26,10 @@ private static Result captureOnce(UiAutomation automation, int maxDepth, int max
     StringBuilder xml = new StringBuilder();
     xml.append("<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>");
     xml.append("<hierarchy rotation=\"0\">");
-    int windowCount = appendInteractiveWindowRoots(xml, automation, maxDepth, maxNodes, stats);
+    List<Integer> missingRootWindowTypes = new ArrayList<>();
+    int windowCount =
+        appendInteractiveWindowRoots(
+            xml, automation, maxDepth, maxNodes, stats, missingRootWindowTypes);
     String captureMode = "interactive-windows";
     if (AccessibilityCaptureStabilizer.requiresActiveWindowFallback(
         windowCount, stats.activeWindowRootMissing)) {
@@ -61,7 +65,8 @@ private static Result captureOnce(UiAutomation automation, int maxDepth, int max
         captureMode,
         windowCount,
         stats.nodeCount,
-        stats.truncated);
+        stats.truncated,
+        missingRootWindowTypes);
   }
 
   private static void clearAccessibilityCache(UiAutomation automation) {
@@ -118,7 +123,8 @@ private static int appendInteractiveWindowRoots(
       UiAutomation automation,
       int maxDepth,
       int maxNodes,
-      AccessibilityTreeXml.Stats stats) {
+      AccessibilityTreeXml.Stats stats,
+      List<Integer> missingRootWindowTypes) {
     List<AccessibilityWindowInfo> windows;
     try {
       windows = automation.getWindows();
@@ -140,6 +146,7 @@ private static int appendInteractiveWindowRoots(
         windowMetadata = AccessibilityTreeXml.readWindowMetadata(window, windowCount);
         root = window.getRoot();
         if (root == null) {
+          missingRootWindowTypes.add(windowMetadata.type);
           stats.activeWindowRootMissing |= activeWindow;
           stats.focusedNonActiveWindowRootMissing |= focusedNonActiveWindow;
           if (activeWindow) {
@@ -165,6 +172,9 @@ private static int appendInteractiveWindowRoots(
         // Accessibility windows can disappear while traversing; keep the rest of the snapshot.
         stats.activeWindowRootMissing |= activeWindow;
         stats.focusedNonActiveWindowRootMissing |= focusedNonActiveWindow;
+        if (windowMetadata != null) {
+          missingRootWindowTypes.add(windowMetadata.type);
+        }
         if (activeWindow && windowMetadata != null) {
           stats.activeWindowMetadata = windowMetadata;
         }
@@ -202,6 +212,9 @@ static final class Result implements AccessibilityCaptureStabilizer.Capture {
     final int windowCount;
     final int nodeCount;
     final boolean truncated;
+    // AccessibilityWindowInfo types of listed windows whose root read null or threw, so the host
+    // can tell a window it could not read (an input method, say) from one that was not on screen.
+    final List<Integer> missingRootWindowTypes;
 
     Result(
         String xml,
@@ -210,14 +223,16 @@ static final class Result implements AccessibilityCaptureStabilizer.Capture {
         String captureMode,
         int windowCount,
         int nodeCount,
-        boolean truncated) {
+        boolean truncated,
+        List<Integer> missingRootWindowTypes) {
       this.xml = xml;
       this.rootPresent = rootPresent;
       this.foregroundWindowRootsPresent = foregroundWindowRootsPresent;
       this.captureMode = captureMode;
       this.windowCount = windowCount;
       this.nodeCount = nodeCount;
       this.truncated = truncated;
+      this.missingRootWindowTypes = missingRootWindowTypes;
     }
 
     @Override

android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java

@@ -1,6 +1,7 @@
 package com.callstack.agentdevice.snapshothelper;
 
 import android.graphics.Rect;
+import android.graphics.Region;
 import android.os.Build;
 import android.view.accessibility.AccessibilityNodeInfo;
 import android.view.accessibility.AccessibilityNodeInfo.AccessibilityAction;
@@ -133,8 +134,29 @@ static void appendNode(
   static WindowMetadata readWindowMetadata(AccessibilityWindowInfo window, int index) {
     Rect bounds = new Rect();
     window.getBoundsInScreen(bounds);
+    int type = window.getType();
     return new WindowMetadata(
-        index, window.getType(), window.getLayer(), window.isActive(), window.isFocused(), bounds);
+        index,
+        type,
+        window.getLayer(),
+        window.isActive(),
+        window.isFocused(),
+        bounds,
+        readInputMethodRegionRect(window, type));
+  }
+
+  // The window bounds are the box around the window's touchable region. A floating keyboard's region
+  // is several rects (the panel and the gesture strip), so the box claims app content between them;
+  // the host must not read such a box as the keyboard band. getRegionInScreen arrived in API 33, so
+  // older releases, and a region that reads empty, report nothing and keep the bounds-only answer.
+  private static Boolean readInputMethodRegionRect(AccessibilityWindowInfo window, int type) {
+    if (type != AccessibilityWindowInfo.TYPE_INPUT_METHOD
+        || Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU) {
+      return null;
+    }
+    Region region = new Region();
+    window.getRegionInScreen(region);
+    return region.isEmpty() ? null : region.isRect();
   }
 
   private static void appendNonEmptyAttribute(
@@ -194,6 +216,9 @@ private static void appendWindowMetadata(StringBuilder xml, WindowMetadata metad
             metadata.bounds.top,
             metadata.bounds.right,
             metadata.bounds.bottom));
+    if (metadata.regionRect != null) {
+      appendAttribute(xml, "window-region-rect", Boolean.toString(metadata.regionRect));
+    }
   }
 
   private static void appendAttribute(StringBuilder xml, String name, CharSequence value) {
@@ -279,18 +304,28 @@ static final class WindowMetadata {
     final boolean active;
     final boolean focused;
     final Rect bounds;
+    // Null unless an input method window's touchable region was read (API 33+).
+    final Boolean regionRect;
 
-    WindowMetadata(int index, int type, int layer, boolean active, boolean focused, Rect bounds) {
+    WindowMetadata(
+        int index,
+        int type,
+        int layer,
+        boolean active,
+        boolean focused,
+        Rect bounds,
+        Boolean regionRect) {
       this.index = index;
       this.type = type;
       this.layer = layer;
       this.active = active;
       this.focused = focused;
       this.bounds = bounds;
+      this.regionRect = regionRect;
     }
 
     WindowMetadata withIndex(int nextIndex) {
-      return new WindowMetadata(nextIndex, type, layer, active, focused, bounds);
+      return new WindowMetadata(nextIndex, type, layer, active, focused, bounds, regionRect);
     }
   }
 }

android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java

@@ -4,6 +4,7 @@
 import android.app.UiAutomation;
 import android.content.res.Resources;
 import android.os.Bundle;
+import android.text.TextUtils;
 import android.util.Base64;
 import android.util.DisplayMetrics;
 import java.io.BufferedReader;
@@ -125,6 +126,8 @@ private static void putCaptureMetadata(
     result.putString("rootPresent", Boolean.toString(capture.rootPresent));
     result.putString("captureMode", capture.captureMode);
     result.putString("windowCount", Integer.toString(capture.windowCount));
+    result.putString(
+        "missingRootWindowTypes", TextUtils.join(",", capture.missingRootWindowTypes));
     result.putString("nodeCount", Integer.toString(capture.nodeCount));
     result.putString("truncated", Boolean.toString(capture.truncated));
     result.putString("elapsedMs", Long.toString(elapsedMs));

package.json

@@ -179,7 +179,7 @@
     "check:unit": "pnpm test:unit && pnpm check:tmpdir-leaks && pnpm test:smoke",
     "check": "pnpm check:tooling && pnpm check:fallow && pnpm check:unit",
     "prepack": "npm_config_dry_run=false pnpm release:prepare",
-    "typecheck": "tsc -b packages/xml packages/kernel packages/contracts packages/device-selection packages/host-kit packages/capture-kit packages/managed-allocation packages/provision-kit packages/platform-apple packages/platform-android packages/platform-harmonyos packages/platform-vega packages/platform-linux packages/platform-web packages/ad-script packages/selectors packages/command-registry packages/session-journal packages/ad-replay packages/maestro packages/replay-port packages/replay-test packages/provider-webdriver packages/provider-limrun packages/provider-testmu && tsc -p tsconfig.json && tsc -p examples/sdk/tsconfig.json",
+    "typecheck": "tsc -b packages/xml packages/kernel packages/contracts packages/device-selection packages/host-kit packages/capture-kit packages/managed-allocation packages/provision-kit packages/platform-apple packages/platform-android packages/platform-harmonyos packages/platform-vega packages/platform-linux packages/platform-web packages/ad-script packages/selectors packages/command-registry packages/session-journal packages/ad-replay packages/maestro packages/replay-port packages/replay-test packages/provider-webdriver packages/provider-limrun packages/proxy packages/provider-testmu && tsc -p tsconfig.json && tsc -p examples/sdk/tsconfig.json",
     "test-app:install": "pnpm install --dir examples/test-app",
     "test-app:start": "pnpm --dir examples/test-app start",
     "test-app:ios": "pnpm --dir examples/test-app ios",
@@ -308,14 +308,15 @@
     "@agent-device/provider-limrun": "workspace:*",
     "@agent-device/provider-webdriver": "workspace:*",
     "@agent-device/provision-kit": "workspace:*",
+    "@agent-device/proxy": "workspace:*",
     "@agent-device/replay-port": "workspace:*",
     "@agent-device/replay-test": "workspace:*",
     "@agent-device/selectors": "workspace:*",
     "@agent-device/session-journal": "workspace:*",
     "@agent-device/xml": "workspace:*",
     "@arethetypeswrong/cli": "^0.18.5",
     "@chenglou/freerange": "^0.0.4",
-    "@limrun/api": "^0.49.3",
+    "@limrun/api": "^0.59.0",
     "@nkzw/eslint-plugin": "^2.0.0",
     "@nkzw/oxlint-config": "^1.3.0",
     "@stryker-mutator/core": "10.0.0",

packages/capture-kit/src/ios-snapshot-engine/transitions.ts

@@ -1,5 +1,5 @@
 import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot';
-import { rectContains } from '@agent-device/kernel/rect';
+import { rectContains, unionRects } from '@agent-device/kernel/rect';
 import { extractNodeText, normalizeType } from '@agent-device/contracts/snapshot';
 import { collectChildrenByParent, mergeReplacement, type SnapshotTreeRuleContext } from './tree.ts';
 import { collectIosReplacedActionShelves } from './action-shelf.ts';
@@ -116,11 +116,3 @@ function formsNavigationTitleAffordance(image: Rect, field: Rect, title: Rect, b
 function verticallyOverlaps(left: Rect, right: Rect): boolean {
   return Math.max(left.y, right.y) <= Math.min(left.y + left.height, right.y + right.height);
 }
-
-function unionRects(rects: Rect[]): Rect {
-  const x = Math.min(...rects.map((rect) => rect.x));
-  const y = Math.min(...rects.map((rect) => rect.y));
-  const right = Math.max(...rects.map((rect) => rect.x + rect.width));
-  const bottom = Math.max(...rects.map((rect) => rect.y + rect.height));
-  return { x, y, width: right - x, height: bottom - y };
-}

packages/capture-kit/src/screenshot-overlay-android.ts

@@ -1,6 +1,7 @@
 import type { Rect, SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot';
 import { isViewportRootNode, normalizeType } from '@agent-device/contracts/snapshot';
-import { hasPositiveRect, rectArea, rectContains, unionRects } from './screenshot-overlay-rects.ts';
+import { unionRects } from '@agent-device/kernel/rect';
+import { hasPositiveRect, rectArea, rectContains } from './screenshot-overlay-rects.ts';
 
 /**
  * Android overlay policy (#1983): which Android nodes earn an overlay ref, and what rectangle

packages/capture-kit/src/screenshot-overlay-rects.ts

@@ -15,26 +15,3 @@ export function clamp(value: number, min: number, max: number): number {
 export function rectArea(rect: Rect): number {
   return rect.width * rect.height;
 }
-
-export function unionRects(rects: Rect[]): Rect {
-  const firstRect = rects[0];
-  if (firstRect === undefined) {
-    throw new Error('unionRects requires at least one rect');
-  }
-  let minX = firstRect.x;
-  let minY = firstRect.y;
-  let maxRight = firstRect.x + firstRect.width;
-  let maxBottom = firstRect.y + firstRect.height;
-  for (const rect of rects.slice(1)) {
-    minX = Math.min(minX, rect.x);
-    minY = Math.min(minY, rect.y);
-    maxRight = Math.max(maxRight, rect.x + rect.width);
-    maxBottom = Math.max(maxBottom, rect.y + rect.height);
-  }
-  return {
-    x: minX,
-    y: minY,
-    width: maxRight - minX,
-    height: maxBottom - minY,
-  };
-}

packages/contracts/src/snapshot-types.ts

@@ -73,6 +73,12 @@ export type AndroidSnapshotBackendMetadata = {
   captureMode?: string;
   systemSurfaceOnly?: boolean;
   windowCount?: number;
+  /**
+   * `AccessibilityWindowInfo` types of the listed windows the helper could not serialize because
+   * reading their root returned null or threw (`2` is an input method window); `windowCount` counts
+   * only the roots it serialized. Empty when every listed window was read. Absent on an older helper.
+   */
+  missingRootWindowTypes?: number[];
   nodeCount?: number;
   helperTruncated?: boolean;
   elapsedMs?: number;