diff --git a/src/input/backend-resolver.ts b/src/input/backend-resolver.ts index d88fa866..2d2721f9 100644 --- a/src/input/backend-resolver.ts +++ b/src/input/backend-resolver.ts @@ -17,12 +17,12 @@ import { SimctlExecutor } from '../simulator/simctl'; import type { BrowserBackend } from '../types/browser-backend'; -import { FlutterVMInputBackend } from '../tools/flutter-vm-input-backend'; -import { tryCreateSimulatorKitHIDBackend } from '../tools/sim-hid-input-backend'; +import { FlutterVMInputBackend } from './flutter-vm-backend'; +import { tryCreateSimulatorKitHIDBackend } from './sim-hid-backend'; import { isPointerServiceEnabled, tryCreatePointerServiceBackend, -} from '../tools/pointer-service-input-backend'; +} from './pointer-service-backend'; import type { InputBackend } from './backend'; import { SimctlInputBackend } from './simctl-backend'; import { AppleScriptInputBackend } from './applescript-backend'; diff --git a/src/input/flutter-vm-backend.ts b/src/input/flutter-vm-backend.ts new file mode 100644 index 00000000..2a8f4c54 --- /dev/null +++ b/src/input/flutter-vm-backend.ts @@ -0,0 +1,525 @@ +/** + * FlutterVMInputBackend — Tier-0 headless input backend for Flutter apps. + * + * Dispatches pointer/keyboard/text events directly to the Flutter engine via + * the Dart VM Service (`FlutterVMClient.evaluate`). Because the events are + * synthesised inside the running Dart isolate and fed straight into + * `PlatformDispatcher.onPointerDataPacket`, this backend: + * + * - **Does not move the physical mouse cursor** (no CGEvent) + * - **Does not bring Simulator.app to the foreground** (no AppleScript + * activation) + * - **Requires no opt-in env var** — it is truly headless + * + * Compared to the three existing tiers (simctl → webkit → applescript), this + * path is picked first whenever the target device is running a Flutter app in + * debug/profile mode and the VM Service URL can be discovered. Native UIKit + * apps continue to flow through the existing tiers unchanged. + * + * Coordinate system: iOS AX frames are expressed in logical points (the same + * units Flutter calls "logical pixels"). The Dart payload multiplies by the + * implicit view's `devicePixelRatio` to land on the engine's `physicalX/Y` + * expectations. + * + * See issue #481 for the motivation and rollout checklist. + * + * Moved from `src/tools/flutter-vm-input-backend.ts` as part of the #707 (b) + * consolidation. The old path is kept as a re-export shim for compatibility. + */ + +import type { FlutterVMClient } from '../flutter'; +import { FlutterVMError } from '../flutter'; +import type { InputBackend, InputBackendKind } from './backend'; +import { timedInput } from '../metrics/input-telemetry'; + +/** + * Structured error surfaced by FlutterVMInputBackend when the underlying VM + * Service call fails (connection drop, Dart exception, timeout, etc). Carries + * the originating op so observability layers can attribute the failure. + */ +/** + * Structured error codes attached to `FlutterVMInputBackendError`. + * + * Today the code table is deliberately small — callers branch only on + * `VM_NO_EVALUATE` (release-build / no-DDS fallback signal) vs "anything + * else" which is surfaced to the user as a concrete failure. Expand this + * as we learn which failure modes the callers actually need to + * discriminate. + */ +export type FlutterVMInputBackendErrorCode = + /** `evaluate` rejected with code 113 — VM cannot compile expressions. */ + | 'VM_NO_EVALUATE' + /** Dart code ran but raised / returned an @Error. */ + | 'DART_ERROR' + /** Any other cause (connection drop, timeout, unknown). */ + | 'UNKNOWN'; + +export class FlutterVMInputBackendError extends Error { + readonly name = 'FlutterVMInputBackendError' as const; + readonly op: 'tap' | 'swipe' | 'typeText' | 'keypress' | 'sendKey'; + readonly cause: unknown; + readonly code: FlutterVMInputBackendErrorCode; + + constructor( + op: FlutterVMInputBackendError['op'], + cause: unknown, + ) { + const msg = cause instanceof Error ? cause.message : String(cause); + const code: FlutterVMInputBackendErrorCode = classifyFlutterVMCause(cause); + // Release / no-DDS builds emit a verbose JSON-RPC string — replace it + // with a short, actionable diagnostic so tool consumers see a single + // clean message instead of the compiler's internal error payload. + const userMessage = + code === 'VM_NO_EVALUATE' + ? `FlutterVMInputBackend.${op} failed: VM cannot compile expressions ` + + `(code 113). This app is likely a release build or was launched ` + + `with \`simctl launch\` instead of \`flutter run\`. ` + + `Use Tier 1.5 AX press (app_tap_element / app_type_element) or ` + + `relaunch under \`flutter run --debug\` for full gesture coverage. ` + + `See docs/ci-recipes.md#qa-ready-flutter-build for the ` + + `simulator (--debug) and physical-device (--profile) recipes ` + + `that keep Tier 0 available.` + : `FlutterVMInputBackend.${op} failed: ${msg}`; + super(userMessage); + this.op = op; + this.cause = cause; + this.code = code; + Object.setPrototypeOf(this, FlutterVMInputBackendError.prototype); + } +} + +function classifyFlutterVMCause(cause: unknown): FlutterVMInputBackendErrorCode { + const msg = cause instanceof Error ? cause.message : String(cause); + if (/\(code:\s*113\)/.test(msg)) return 'VM_NO_EVALUATE'; + // The inner `evalOrThrow` wraps Dart-side errors with code `DART_ERROR` + // via `FlutterVMError`; preserve that classification when bubbling up. + if ( + cause && + typeof cause === 'object' && + 'code' in cause && + (cause as { code?: string }).code === 'DART_ERROR' + ) { + return 'DART_ERROR'; + } + return 'UNKNOWN'; +} + +/** + * HID key-code → Dart `LogicalKeyboardKey` identifier. The keyIds match the + * values exposed by `package:flutter/services.dart` so the Dart payload can + * materialise a `KeyDownEvent` / `KeyUpEvent` pair. + */ +const HID_TO_LOGICAL_KEY: Record = { + '40': { keyId: 'LogicalKeyboardKey.enter', keyLabel: 'Enter', physicalKey: 'PhysicalKeyboardKey.enter' }, + '41': { keyId: 'LogicalKeyboardKey.escape', keyLabel: 'Escape', physicalKey: 'PhysicalKeyboardKey.escape' }, + '42': { keyId: 'LogicalKeyboardKey.backspace', keyLabel: 'Backspace', physicalKey: 'PhysicalKeyboardKey.backspace' }, + '43': { keyId: 'LogicalKeyboardKey.tab', keyLabel: 'Tab', physicalKey: 'PhysicalKeyboardKey.tab' }, + '44': { keyId: 'LogicalKeyboardKey.space', keyLabel: ' ', physicalKey: 'PhysicalKeyboardKey.space' }, + '74': { keyId: 'LogicalKeyboardKey.home', keyLabel: 'Home', physicalKey: 'PhysicalKeyboardKey.home' }, + '79': { keyId: 'LogicalKeyboardKey.arrowRight', keyLabel: 'ArrowRight', physicalKey: 'PhysicalKeyboardKey.arrowRight' }, + '80': { keyId: 'LogicalKeyboardKey.arrowLeft', keyLabel: 'ArrowLeft', physicalKey: 'PhysicalKeyboardKey.arrowLeft' }, + '81': { keyId: 'LogicalKeyboardKey.arrowDown', keyLabel: 'ArrowDown', physicalKey: 'PhysicalKeyboardKey.arrowDown' }, + '82': { keyId: 'LogicalKeyboardKey.arrowUp', keyLabel: 'ArrowUp', physicalKey: 'PhysicalKeyboardKey.arrowUp' }, +}; + +const SENDKEY_TO_LOGICAL_KEY: Record = { + Return: { keyId: 'LogicalKeyboardKey.enter', keyLabel: 'Enter', physicalKey: 'PhysicalKeyboardKey.enter' }, + Enter: { keyId: 'LogicalKeyboardKey.enter', keyLabel: 'Enter', physicalKey: 'PhysicalKeyboardKey.enter' }, + Escape: { keyId: 'LogicalKeyboardKey.escape', keyLabel: 'Escape', physicalKey: 'PhysicalKeyboardKey.escape' }, + Tab: { keyId: 'LogicalKeyboardKey.tab', keyLabel: 'Tab', physicalKey: 'PhysicalKeyboardKey.tab' }, + Space: { keyId: 'LogicalKeyboardKey.space', keyLabel: ' ', physicalKey: 'PhysicalKeyboardKey.space' }, + Delete: { keyId: 'LogicalKeyboardKey.backspace', keyLabel: 'Backspace', physicalKey: 'PhysicalKeyboardKey.backspace' }, + Backspace: { keyId: 'LogicalKeyboardKey.backspace', keyLabel: 'Backspace', physicalKey: 'PhysicalKeyboardKey.backspace' }, + Home: { keyId: 'LogicalKeyboardKey.home', keyLabel: 'Home', physicalKey: 'PhysicalKeyboardKey.home' }, + ArrowRight: { keyId: 'LogicalKeyboardKey.arrowRight', keyLabel: 'ArrowRight', physicalKey: 'PhysicalKeyboardKey.arrowRight' }, + ArrowLeft: { keyId: 'LogicalKeyboardKey.arrowLeft', keyLabel: 'ArrowLeft', physicalKey: 'PhysicalKeyboardKey.arrowLeft' }, + ArrowDown: { keyId: 'LogicalKeyboardKey.arrowDown', keyLabel: 'ArrowDown', physicalKey: 'PhysicalKeyboardKey.arrowDown' }, + ArrowUp: { keyId: 'LogicalKeyboardKey.arrowUp', keyLabel: 'ArrowUp', physicalKey: 'PhysicalKeyboardKey.arrowUp' }, +}; + +/** + * Format a finite number for interpolation into a Dart literal. Reject NaN / + * ±Infinity so the VM Service never receives a syntactically invalid + * expression (e.g. `Offset(NaN, NaN)` would confuse the analyser). + */ +function dartNum(value: number, label: string): string { + if (!Number.isFinite(value)) { + throw new Error(`Invalid ${label}: ${value} (must be finite)`); + } + // toString() preserves sufficient precision for pixel coordinates. + return value.toString(); +} + +/** + * Escape a JS string for safe embedding inside a Dart single-quoted literal. + * Dart string escape rules are similar to JS but the safest approach is to + * emit a Dart list-of-codeUnits from JSON.stringify, avoiding any ambiguity + * over dollar interpolation, backslashes, quotes, or non-ASCII. + */ +function dartStringLiteral(value: string): string { + // Encode via a Dart `String.fromCharCodes` so we never have to worry about + // dollar-sign interpolation, adjacent quotes, or embedded newlines. + const codeUnits: number[] = []; + for (let i = 0; i < value.length; i++) { + codeUnits.push(value.charCodeAt(i)); + } + return `String.fromCharCodes(const [${codeUnits.join(',')}])`; +} + +/** + * FlutterVMInputBackend — implements the InputBackend contract by evaluating + * Dart expressions inside the target app's main isolate. + */ +export class FlutterVMInputBackend implements InputBackend { + readonly kind: InputBackendKind = 'flutter-vm'; + private libIdCache: Map = new Map(); + + constructor(private vmClient: FlutterVMClient) {} + + /** + * Resolve a Flutter library URI to its VM Service library id, caching the + * result per URI. Each operation targets a different library so that the + * required symbols are in lexical scope: + * - pointer dispatch → `mouse_tracker.dart` (bare `import 'dart:ui'`) + * - text input → `editable_text.dart` + * - key events → `hardware_keyboard.dart` + * + * Same pattern used by `FlutterVMClient.selectWidgetAtPoint` for the + * inspector library (see `vm-service-client.ts:386-411`). + */ + private async resolveLibId(uri: string): Promise { + const cached = this.libIdCache.get(uri); + if (cached) return cached; + + const isolateId = this.vmClient.getState()?.mainIsolateId; + if (!isolateId) { + throw new FlutterVMError('No main isolate', 'NO_ISOLATE'); + } + const isolate = await this.vmClient.callMethod('getIsolate', { isolateId }); + const libs = + (isolate as { libraries?: Array<{ uri?: string; id?: string }> }).libraries ?? []; + const lib = libs.find((l) => l.uri === uri); + if (!lib?.id) { + throw new FlutterVMError( + `${uri} library not loaded in isolate — is this a Flutter app?`, + 'NO_BINDING_LIB', + ); + } + this.libIdCache.set(uri, lib.id); + return lib.id; + } + + /** + * Synthesise a pointer down → up sequence. When `duration` (in seconds) is + * positive the up event is timestamped `duration * 1000` ms after the down + * event so Flutter's gesture arena treats it as a long-press rather than a + * tap. The Dart payload ends with + * `SchedulerBinding.instance.scheduleFrame()` to ensure the engine pumps + * the event queue even in a quiescent state. + */ + async tap( + deviceId: string, + x: number, + y: number, + duration?: number, + ): Promise { + await timedInput(this.kind, 'tap', deviceId, () => this.tapInternal(x, y, duration)); + } + + private async tapInternal( + x: number, + y: number, + duration?: number, + ): Promise { + let xStr: string; + let yStr: string; + try { + xStr = dartNum(x, 'x'); + yStr = dartNum(y, 'y'); + } catch (err) { + throw new FlutterVMInputBackendError('tap', err); + } + const durMs = duration && duration > 0 ? Math.round(duration * 1000) : 0; + + const expression = + '(() {' + + ' final dispatcher = PlatformDispatcher.instance;' + + ' final view = dispatcher.implicitView;' + + ' if (view == null) { return false; }' + + ' final dpr = view.devicePixelRatio;' + + ` final double px = ${xStr} * dpr;` + + ` final double py = ${yStr} * dpr;` + + ` final int downUs = 0;` + + ` final int upUs = ${durMs} * 1000;` + + ' void dispatch(int tUs, PointerChange change) {' + + ' final packet = PointerDataPacket(data: [' + + ' PointerData(' + + ' timeStamp: Duration(microseconds: tUs),' + + ' change: change,' + + ' kind: PointerDeviceKind.touch,' + + ' device: 1,' + + ' pointerIdentifier: 1,' + + ' physicalX: px,' + + ' physicalY: py,' + + ' buttons: change == PointerChange.up ? 0 : 1,' + + ' pressure: change == PointerChange.up ? 0.0 : 1.0,' + + ' pressureMax: 1.0,' + + ' ),' + + ' ]);' + + ' dispatcher.onPointerDataPacket?.call(packet);' + + ' }' + + ' dispatch(downUs, PointerChange.add);' + + ' dispatch(downUs, PointerChange.down);' + + ' dispatch(upUs, PointerChange.up);' + + ' dispatch(upUs, PointerChange.remove);' + + ' return true;' + + '})()'; + + await this.evalOrThrow('tap', expression, 'package:flutter/src/rendering/mouse_tracker.dart'); + } + + /** + * Synthesise a drag gesture as a down → N×move → up sequence. `duration` + * (seconds) spreads the move events evenly across the requested window so + * the gesture arena classifies it as a swipe rather than a flick or tap. + */ + async swipe( + deviceId: string, + startX: number, + startY: number, + endX: number, + endY: number, + duration?: number, + ): Promise { + await timedInput(this.kind, 'swipe', deviceId, () => + this.swipeInternal(startX, startY, endX, endY, duration), + ); + } + + private async swipeInternal( + startX: number, + startY: number, + endX: number, + endY: number, + duration?: number, + ): Promise { + const sxStr = dartNum(startX, 'startX'); + const syStr = dartNum(startY, 'startY'); + const exStr = dartNum(endX, 'endX'); + const eyStr = dartNum(endY, 'endY'); + const totalMs = Math.max(1, Math.round((duration ?? 0.5) * 1000)); + const steps = 20; + const stepUs = Math.round((totalMs * 1000) / steps); + + const expression = + '(() {' + + ' final dispatcher = PlatformDispatcher.instance;' + + ' final view = dispatcher.implicitView;' + + ' if (view == null) { return false; }' + + ' final dpr = view.devicePixelRatio;' + + ` final double sx = ${sxStr} * dpr;` + + ` final double sy = ${syStr} * dpr;` + + ` final double ex = ${exStr} * dpr;` + + ` final double ey = ${eyStr} * dpr;` + + ` const int steps = ${steps};` + + ` const int stepUs = ${stepUs};` + + ' void post(int tUs, PointerChange change, double x, double y) {' + + ' final packet = PointerDataPacket(data: [' + + ' PointerData(' + + ' timeStamp: Duration(microseconds: tUs),' + + ' change: change,' + + ' kind: PointerDeviceKind.touch,' + + ' device: 1,' + + ' pointerIdentifier: 1,' + + ' physicalX: x,' + + ' physicalY: y,' + + ' buttons: change == PointerChange.up ? 0 : 1,' + + ' pressure: change == PointerChange.up ? 0.0 : 1.0,' + + ' pressureMax: 1.0,' + + ' ),' + + ' ]);' + + ' dispatcher.onPointerDataPacket?.call(packet);' + + ' }' + + ' post(0, PointerChange.add, sx, sy);' + + ' post(0, PointerChange.down, sx, sy);' + + ' for (int i = 1; i <= steps; i++) {' + + ' final double t = i / steps;' + + ' final double x = sx + (ex - sx) * t;' + + ' final double y = sy + (ey - sy) * t;' + + ' post(stepUs * i, PointerChange.move, x, y);' + + ' }' + + ' final int endUs = stepUs * steps;' + + ' post(endUs, PointerChange.up, ex, ey);' + + ' post(endUs, PointerChange.remove, ex, ey);' + + ' return true;' + + '})()'; + + await this.evalOrThrow('swipe', expression, 'package:flutter/src/rendering/mouse_tracker.dart'); + } + + /** + * Inject text into the currently-focused `EditableText` via Flutter's + * `TextInput` channel. This mirrors what the iOS IME would send when the + * user types on the system keyboard, so controllers and `onChanged` + * callbacks fire naturally. Falls through silently (no-op) if nothing is + * focused — same behaviour as WebKitInputBackend. + */ + async typeText(deviceId: string, text: string): Promise { + await timedInput(this.kind, 'typeText', deviceId, () => this.typeTextInternal(text)); + } + + private async typeTextInternal(text: string): Promise { + const textLit = dartStringLiteral(text); + + // Read the live TextInputConnection client id so the platform message + // targets the correct connection. The Flutter framework drops messages + // where args[0] != _currentConnection._id, so hardcoding -1 would be + // a silent no-op. We read the id via TextInput._currentConnection._id, + // which is private but accessible via evaluate on the binding library. + const expression = + '(() async {' + + ` final String newText = ${textLit};` + + ' // Read the current TextInputConnection client id.' + + ' // If nothing is focused, fall back to -1 (message is dropped, same' + + ' // as typing on a hardware keyboard with no focused field).' + + ' int clientId = -1;' + + ' try {' + + ' // Fallback: just check if the primary focus accepts text.' + + ' final focused = FocusManager.instance.primaryFocus;' + + ' if (focused != null && focused.context != null) {' + + ' final editable = focused.context!.findAncestorStateOfType();' + + ' if (editable != null) {' + + ' // Force update through the editable directly — this is' + + ' // the most reliable path since it bypasses the private _id.' + + ' final ctrl = editable.textEditingValue;' + + ' editable.userUpdateTextEditingValue(' + + ' TextEditingValue(' + + ' text: ctrl.text + newText,' + + ' selection: TextSelection.collapsed(offset: ctrl.text.length + newText.length),' + + ' ),' + + ' SelectionChangedCause.keyboard,' + + ' );' + + ' return true;' + + ' }' + + ' }' + + ' } catch (_) {}' + + ' // Fallback: deliver via platform channel with best-effort client id.' + + ' final Map state = {' + + ' "text": newText,' + + ' "selectionBase": newText.length,' + + ' "selectionExtent": newText.length,' + + ' "selectionAffinity": "TextAffinity.downstream",' + + ' "selectionIsDirectional": false,' + + ' "composingBase": -1,' + + ' "composingExtent": -1,' + + ' };' + + ' final message = const JSONMethodCodec().encodeMethodCall(' + + ' MethodCall("TextInputClient.updateEditingState", [clientId, state]),' + + ' );' + + ' await WidgetsBinding.instance!.defaultBinaryMessenger.handlePlatformMessage(' + + ' "flutter/textinput",' + + ' message,' + + ' (dynamic _) {},' + + ' );' + + ' return true;' + + '})()'; + + await this.evalOrThrow('typeText', expression, 'package:flutter/src/widgets/editable_text.dart'); + } + + /** + * Dispatch a HID key code through `HardwareKeyboard`. Only a curated set of + * control keys is supported — matches the WebKit/AppleScript backends. + */ + async keypress(deviceId: string, keyCode: string): Promise { + await timedInput(this.kind, 'keypress', deviceId, () => this.keypressInternal(keyCode)); + } + + private async keypressInternal(keyCode: string): Promise { + const entry = HID_TO_LOGICAL_KEY[keyCode]; + if (!entry) { + throw new Error( + `Unknown HID key code "${keyCode}" for FlutterVM backend. ` + + `Supported: ${Object.keys(HID_TO_LOGICAL_KEY).join(', ')}`, + ); + } + await this.dispatchKey('keypress', entry.keyId, entry.keyLabel, entry.physicalKey); + } + + /** Dispatch a named key ("Return", "Escape", ...) through HardwareKeyboard. */ + async sendKey(deviceId: string, keyName: string): Promise { + await timedInput(this.kind, 'sendKey', deviceId, () => this.sendKeyInternal(keyName)); + } + + private async sendKeyInternal(keyName: string): Promise { + const entry = SENDKEY_TO_LOGICAL_KEY[keyName]; + if (!entry) { + throw new Error( + `Unknown key name "${keyName}" for FlutterVM backend. ` + + `Supported: ${Object.keys(SENDKEY_TO_LOGICAL_KEY).join(', ')}`, + ); + } + await this.dispatchKey('sendKey', entry.keyId, entry.keyLabel, entry.physicalKey); + } + + // ── internals ────────────────────────────────────────────────────────── + + private async dispatchKey( + op: 'keypress' | 'sendKey', + logicalKeyExpr: string, + keyLabel: string, + physicalKeyExpr: string, + ): Promise { + const labelLit = dartStringLiteral(keyLabel); + // Emit a KeyDown event then a KeyUp through HardwareKeyboard so downstream + // focus nodes observe a complete press. `timeStamp` uses the default + // (zero) — the event queue does not require strict monotonicity. + const expression = + '(() {' + + ` final label = ${labelLit};` + + ` final logical = ${logicalKeyExpr};` + + ` final physical = ${physicalKeyExpr};` + + ' final down = KeyDownEvent(' + + ' physicalKey: physical,' + + ' logicalKey: logical,' + + ' timeStamp: Duration.zero,' + + ' character: label.length == 1 ? label : null,' + + ' );' + + ' final up = KeyUpEvent(' + + ' physicalKey: physical,' + + ' logicalKey: logical,' + + ' timeStamp: Duration.zero,' + + ' );' + + ' HardwareKeyboard.instance.handleKeyEvent(down);' + + ' HardwareKeyboard.instance.handleKeyEvent(up);' + + ' return true;' + + '})()'; + + await this.evalOrThrow(op, expression, 'package:flutter/src/services/hardware_keyboard.dart'); + } + + private async evalOrThrow( + op: FlutterVMInputBackendError['op'], + expression: string, + libraryUri: string, + ): Promise { + try { + // Scope the evaluate to the per-operation Flutter library so all + // required symbols are in lexical scope. + const targetId = await this.resolveLibId(libraryUri); + const result = await this.vmClient.evaluate(expression, { targetId }); + // VM returns an @Error shape instead of throwing when the expression + // itself compiled but raised a Dart exception. Surface that as a + // structured InputBackendError. + const errType = (result as { type?: string }).type; + if (errType === '@Error' || errType === 'Error') { + const message = + (result as { message?: string }).message ?? JSON.stringify(result); + throw new FlutterVMError(message, 'DART_ERROR'); + } + } catch (err) { + if (err instanceof FlutterVMInputBackendError) throw err; + throw new FlutterVMInputBackendError(op, err); + } + } +} diff --git a/src/input/pointer-service-backend.ts b/src/input/pointer-service-backend.ts new file mode 100644 index 00000000..975d8e33 --- /dev/null +++ b/src/input/pointer-service-backend.ts @@ -0,0 +1,223 @@ +/** + * PointerServiceInputBackend — Phase 1 opt-in wrapper around the + * `sim-hid-bridge tap-ps` subcommand for Xcode 26+ coordinate tap. + * + * Motivation (issue #590): Apple dropped `IndigoHIDMessageForMouseNSEvent` + * handling in CoreSimulator on Xcode 26. The Tier-1 SimulatorKitHID bare + * mouse path has therefore been gated off for tap/swipe, forcing + * coordinate-based `app_tap` / `app_swipe_native` to fall through to the + * focus-stealing AppleScript / CGEvent backend. The PointerService probe + * shipped in #555 wraps the same mouse events with + * `IndigoHIDMessageToCreatePointerService` / `RemovePointerService` + * brackets; the synthesis doc (#557) falsified this as a *fix*, but it + * remains the most-likely-to-help interim stop-gap because: + * + * - It requires no simulator-side changes. + * - Telemetry is cheap to collect under an opt-in flag. + * - If field data shows ≥ 99 % success, #590 Phase 2 promotes it to the + * default Tier-1 path. + * + * Status: **opt-in experimental**. Activated only when + * `OPENSAFARI_ENABLE_POINTERSERVICE=1` is set. Defaults off so CI that + * has already adapted to the AppleScript fallback is unaffected. + * + * Scope: only `tap` is routed through the `tap-ps` subcommand. The Swift + * bridge does not yet expose `swipe-ps`; swipe / typeText / keypress / + * sendKey delegate to the underlying `SimulatorKitHIDInputBackend`, which + * keeps non-tap input paths on their existing (keyboard-safe) Tier-1 + * route. On Xcode 26+ that delegated SimHID swipe path is itself gated + * off and throws `HeadlessInputUnavailableError`; the PointerService + * backend is cached as the selected backend by `getInputBackend`, so the + * throw does NOT re-enter the tier chain. See the comment on `swipe()` + * below and issue #649 for the caller-visible contract. Extending + * `sim-hid-bridge` with pointer-service-bracketed swipe and promoting + * the backend to the default chain are tracked as Phase 2 follow-ups in + * #590. + * + * Moved from `src/tools/pointer-service-input-backend.ts` as part of the + * #707 (b) consolidation. The old path is kept as a re-export shim. + */ + +import { execFile } from 'child_process'; +import { existsSync } from 'fs'; +import * as path from 'path'; +import { promisify } from 'util'; +import type { InputBackend } from './backend'; +import { + SimulatorKitHIDInputBackend, + InputBackendError, +} from './sim-hid-backend'; +import { timedInput } from '../metrics/input-telemetry'; + +const execFileAsync = promisify(execFile); + +/** + * Env flag that enables the PointerService backend (Phase 1 opt-in). + * Accepted values: `1`, `true`. Anything else is ignored. + */ +export const OPENSAFARI_ENABLE_POINTERSERVICE_ENV = + 'OPENSAFARI_ENABLE_POINTERSERVICE'; + +export function isPointerServiceEnabled(): boolean { + const value = process.env[OPENSAFARI_ENABLE_POINTERSERVICE_ENV]; + return value === '1' || value === 'true'; +} + +/** + * PointerService tap + delegated swipe/keys. + * + * Composes a `SimulatorKitHIDInputBackend` for the subset of methods that + * still share the default `sim-hid-bridge` subcommand set, and shells out + * directly to `sim-hid-bridge tap-ps ...` for `tap`. + */ +export class PointerServiceInputBackend implements InputBackend { + readonly kind = 'pointer-service' as const; + + constructor( + private readonly bridgePath: string, + private readonly delegate: SimulatorKitHIDInputBackend, + ) {} + + async tap(deviceId: string, x: number, y: number, duration?: number): Promise { + await timedInput(this.kind, 'tap', deviceId, async () => { + const args = [deviceId, 'tap-ps', String(x), String(y)]; + if (duration !== undefined && duration > 0) { + args.push(String(duration)); + } + await runTapPs(this.bridgePath, args); + }); + } + + async swipe( + deviceId: string, + startX: number, + startY: number, + endX: number, + endY: number, + duration?: number, + ): Promise { + // Phase 1: no swipe-ps subcommand exists yet. Delegate straight to the + // underlying SimulatorKitHIDInputBackend. On Xcode 26+ the `sim-hid-bridge + // swipe` subcommand exits with `SIMULATORKIT_UNAVAILABLE` (or another + // non-zero SimulatorKit code), and the delegate surfaces that to the caller + // as an `InputBackendError` — not `HeadlessInputUnavailableError`, which is + // produced one layer up in `native-input-backend.getInputBackend` when + // selecting a backend, never from a backend's own swipe() method. + // + // That error does NOT re-enter the tier chain because + // PointerServiceInputBackend is cached as the selected backend in + // getInputBackend once OPENSAFARI_ENABLE_POINTERSERVICE=1 resolves it. For + // completeness, any other error type the delegate may raise in the future + // is also passed through unchanged; see the "swipe propagates … without + // wrapping" tests for the frozen contract. + // + // Callers that need swipe fallback on Xcode 26+ must either (a) leave + // OPENSAFARI_ENABLE_POINTERSERVICE unset so the standard Tier-1 SimHID / + // focus-input chain is selected for every call, or (b) invoke an + // element-targeted swipe so the AX-press tier handles the gesture. + // Promoting this to a real in-tool tier downgrade is tracked under #590 + // Phase 2 alongside `sim-hid-bridge swipe-ps`. See #649 for the decision + // record. + await this.delegate.swipe(deviceId, startX, startY, endX, endY, duration); + } + + async typeText(deviceId: string, text: string): Promise { + await this.delegate.typeText(deviceId, text); + } + + async keypress(deviceId: string, keyCode: string): Promise { + await this.delegate.keypress(deviceId, keyCode); + } + + async sendKey(deviceId: string, keyName: string): Promise { + await this.delegate.sendKey(deviceId, keyName); + } +} + +/** + * Execute `sim-hid-bridge` with the given argv. Mirrors the spawn/parse + * contract of `SimulatorKitHIDInputBackend.run` but is narrowed to the + * single `tap-ps` subcommand used by the PointerService backend, so the + * failure-handling path stays local and auditable. + */ +async function runTapPs(bridgePath: string, args: string[]): Promise { + const isSwiftSource = bridgePath.endsWith('.swift'); + const cmd = isSwiftSource ? 'swift' : bridgePath; + const cmdArgs = isSwiftSource ? [bridgePath, ...args] : args; + + let stdout = ''; + let stderr = ''; + try { + const result = await execFileAsync(cmd, cmdArgs, { + timeout: 10_000, + maxBuffer: 1 * 1024 * 1024, + }); + stdout = result.stdout ?? ''; + stderr = result.stderr ?? ''; + } catch (err) { + const e = err as NodeJS.ErrnoException & { + stdout?: string; + stderr?: string; + code?: number | string; + killed?: boolean; + }; + stdout = e.stdout ?? ''; + stderr = e.stderr ?? ''; + const exit = typeof e.code === 'number' ? e.code : undefined; + const hint = stderr.trim() || stdout.trim() || e.message; + throw new InputBackendError( + `sim-hid-bridge tap-ps exited ${exit ?? '?'}: ${hint}`, + 'UNKNOWN', + stderr, + ); + } + + if (!stdout.trim()) return; + try { + const parsed = JSON.parse(stdout) as { ok?: boolean; error?: string }; + if (parsed.ok === false) { + throw new InputBackendError( + parsed.error ?? 'sim-hid-bridge tap-ps reported ok=false', + 'UNKNOWN', + stderr, + ); + } + } catch (err) { + if (err instanceof InputBackendError) throw err; + const safeStdout = stdout + .slice(0, 200) + .replace(/[\x00-\x1f\x7f]/g, '?'); + throw new InputBackendError( + `sim-hid-bridge tap-ps produced non-JSON stdout: ${safeStdout}`, + 'JSON_PARSE_FAILURE', + stderr, + ); + } +} + +/** + * Factory mirroring `tryCreateSimulatorKitHIDBackend`: locate a usable + * `sim-hid-bridge` helper and wrap it as a `PointerServiceInputBackend`. + * Returns `null` when the helper is not installed — callers are expected + * to fall through to the default SimHID tier. + */ +export async function tryCreatePointerServiceBackend(): Promise { + const candidates = [ + path.resolve(__dirname, '..', 'sim-hid-bridge'), + path.resolve(__dirname, 'sim-hid-bridge'), + path.resolve(__dirname, '..', 'sim-hid-bridge.swift'), + path.resolve(__dirname, 'sim-hid-bridge.swift'), + ]; + if (process.env.OPENSAFARI_ALLOW_SWIFT_INTERPRETER === '1') { + candidates.push( + path.resolve(__dirname, '..', '..', 'src', 'native', 'sim-hid-bridge.swift'), + ); + } + for (const candidate of candidates) { + if (existsSync(candidate)) { + const delegate = new SimulatorKitHIDInputBackend(candidate); + return new PointerServiceInputBackend(candidate, delegate); + } + } + return null; +} diff --git a/src/input/sim-hid-backend.ts b/src/input/sim-hid-backend.ts new file mode 100644 index 00000000..4b7ef66c --- /dev/null +++ b/src/input/sim-hid-backend.ts @@ -0,0 +1,480 @@ +/** + * SimulatorKitHIDInputBackend — Node wrapper around the `sim-hid-bridge` + * Swift helper described in issue #483. + * + * Status: PoC. Backend class is shipped for integration and unit testing, but + * routing in `native-input-backend.ts` is intentionally NOT wired up yet. See + * the `TODO(#483)` comment there. + * + * The Swift bridge spawns as a short-lived child process and communicates via + * argv (command) + stdout (newline-terminated JSON). Exit codes are the + * contract between Swift and Node: + * + * 0 — success + * 64 — BAD_ARGS (EX_USAGE) + * 69 — DEVICE_NOT_BOOTED (EX_UNAVAILABLE) + * 78 — SIMULATORKIT_UNAVAILABLE (EX_CONFIG — dlopen failed) + * 99 — NOT_IMPLEMENTED (PoC stub path) + * * — UNKNOWN (stderr surfaced verbatim) + * + * The current Swift implementation is a PoC stub that proves the dlopen path + * works and always exits with 99 NOT_IMPLEMENTED. This wrapper classifies that + * (and every other documented exit code) into a structured `InputBackendError` + * so the routing layer can decide to fall through to the next tier. + * + * Moved from `src/tools/sim-hid-input-backend.ts` as part of the #707 (b) + * consolidation. The old path is kept as a re-export shim for compatibility. + */ + +import { execFile } from 'child_process'; +import { promisify } from 'util'; +import { existsSync } from 'fs'; +import * as path from 'path'; +import type { InputBackend } from './backend'; +import { timedInput } from '../metrics/input-telemetry'; + +const execFileAsync = promisify(execFile); + +/** Reference appended to error messages for private-framework failures. */ +const PRIVATE_API_DOC_REF = 'See docs/private-apis.md'; + +/** Latch so the private-API warning is emitted only once per process. */ +let warnedAboutPrivateAPI = false; + +/** + * Reset the private-API warning latch. Exported for unit tests only — do not + * call from production code. + */ +export function resetSimHidPrivateAPIWarning(): void { + warnedAboutPrivateAPI = false; +} + +/** Spawn timeout for the Swift helper. Matches idb's default. */ +const SPAWN_TIMEOUT_MS = 10_000; + + +/** HID usage page 0x07 (Keyboard/Keypad) — subset we map for pressKey(). */ +const KEY_NAME_TO_HID_USAGE: Record = { + Enter: 0x28, + Return: 0x28, + Escape: 0x29, + Backspace: 0x2a, + Delete: 0x2a, + Tab: 0x2b, + Space: 0x2c, + ArrowRight: 0x4f, + ArrowLeft: 0x50, + ArrowDown: 0x51, + ArrowUp: 0x52, + Home: 0x4a, +}; + +/** + * HID usage of the LeftShift modifier (Keyboard/Keypad page 0x07). + * Sent alongside a character key via the bridge's `key-mod` subcommand for + * every ASCII symbol that requires Shift on a US keyboard (uppercase letters, + * `!@#$%^&*()_+{}|:"<>?~`). + */ +const HID_USAGE_LEFT_SHIFT = 0xe1; + +/** + * US-keyboard printable ASCII → HID usage + whether Shift must be held. + * + * Covers U+0020 (space) through U+007E (tilde) — i.e. every character produced + * by a US layout without dead keys or IME. Returns null for everything else, + * including control characters (tab, newline), DEL, and any non-ASCII byte. + * + * Reference: USB HID Usage Tables v1.21, §10 Keyboard/Keypad (page 0x07). + */ +function asciiToHidKey(ch: string): { usage: number; shift: boolean } | null { + if (ch.length !== 1) return null; + const code = ch.charCodeAt(0); + if (code < 0x20 || code > 0x7e) return null; + // Lowercase letters → HID 0x04..0x1D + if (code >= 0x61 && code <= 0x7a) return { usage: 0x04 + (code - 0x61), shift: false }; + // Uppercase letters → same keys, but Shift is required + if (code >= 0x41 && code <= 0x5a) return { usage: 0x04 + (code - 0x41), shift: true }; + // Digits '1'..'9' → 0x1E..0x26 + if (code >= 0x31 && code <= 0x39) return { usage: 0x1e + (code - 0x31), shift: false }; + if (code === 0x30) return { usage: 0x27, shift: false }; // '0' + switch (ch) { + case ' ': return { usage: 0x2c, shift: false }; + case '-': return { usage: 0x2d, shift: false }; + case '_': return { usage: 0x2d, shift: true }; + case '=': return { usage: 0x2e, shift: false }; + case '+': return { usage: 0x2e, shift: true }; + case '[': return { usage: 0x2f, shift: false }; + case '{': return { usage: 0x2f, shift: true }; + case ']': return { usage: 0x30, shift: false }; + case '}': return { usage: 0x30, shift: true }; + case '\\': return { usage: 0x31, shift: false }; + case '|': return { usage: 0x31, shift: true }; + case ';': return { usage: 0x33, shift: false }; + case ':': return { usage: 0x33, shift: true }; + case "'": return { usage: 0x34, shift: false }; + case '"': return { usage: 0x34, shift: true }; + case '`': return { usage: 0x35, shift: false }; + case '~': return { usage: 0x35, shift: true }; + case ',': return { usage: 0x36, shift: false }; + case '<': return { usage: 0x36, shift: true }; + case '.': return { usage: 0x37, shift: false }; + case '>': return { usage: 0x37, shift: true }; + case '/': return { usage: 0x38, shift: false }; + case '?': return { usage: 0x38, shift: true }; + case '!': return { usage: 0x1e, shift: true }; + case '@': return { usage: 0x1f, shift: true }; + case '#': return { usage: 0x20, shift: true }; + case '$': return { usage: 0x21, shift: true }; + case '%': return { usage: 0x22, shift: true }; + case '^': return { usage: 0x23, shift: true }; + case '&': return { usage: 0x24, shift: true }; + case '*': return { usage: 0x25, shift: true }; + case '(': return { usage: 0x26, shift: true }; + case ')': return { usage: 0x27, shift: true }; + } + return null; +} + +/** + * Error emitted by `SimulatorKitHIDInputBackend`. Mirrors the convention used + * by `AccessibilityBridgeError` (see `src/native/accessibility-bridge.ts`): + * a stable machine-readable `code` plus the human-readable `message`. + */ +export class InputBackendError extends Error { + readonly name = 'InputBackendError' as const; + constructor( + message: string, + public readonly code: InputBackendErrorCode, + public readonly stderr?: string, + ) { + super(message); + Object.setPrototypeOf(this, InputBackendError.prototype); + } +} + +export type InputBackendErrorCode = + | 'BAD_ARGS' + | 'DEVICE_NOT_BOOTED' + | 'SIMULATORKIT_UNAVAILABLE' + | 'NOT_IMPLEMENTED' + | 'SPAWN_TIMEOUT' + | 'BRIDGE_NOT_FOUND' + | 'HID_BRIDGE_MISSING' + | 'JSON_PARSE_FAILURE' + | 'UNKNOWN'; + +/** Map Swift bridge exit codes to structured error codes. */ +function codeForExit(exit: number | undefined): InputBackendErrorCode { + switch (exit) { + case 64: return 'BAD_ARGS'; + case 69: return 'DEVICE_NOT_BOOTED'; + case 78: return 'SIMULATORKIT_UNAVAILABLE'; + case 99: return 'NOT_IMPLEMENTED'; + default: return 'UNKNOWN'; + } +} + +/** + * SimulatorKit HID input backend. Spawns `sim-hid-bridge` per call and parses + * the JSON status envelope. All methods throw `InputBackendError` on failure. + */ +export class SimulatorKitHIDInputBackend implements InputBackend { + readonly kind = 'simhid' as const; + + constructor(private readonly bridgePath: string) {} + + async tap(deviceId: string, x: number, y: number, duration?: number): Promise { + await timedInput(this.kind, 'tap', deviceId, async () => { + const args = [deviceId, 'tap', String(x), String(y)]; + if (duration !== undefined && duration > 0) { + args.push(String(duration)); + } + await this.run(args); + }); + } + + async swipe( + deviceId: string, + startX: number, + startY: number, + endX: number, + endY: number, + duration?: number, + ): Promise { + await timedInput(this.kind, 'swipe', deviceId, async () => { + const args = [ + deviceId, 'swipe', + String(startX), String(startY), + String(endX), String(endY), + ]; + if (duration !== undefined && duration > 0) { + args.push(String(duration)); + } + await this.run(args); + }); + } + + async typeText(deviceId: string, text: string, delayMs = 0): Promise { + await timedInput(this.kind, 'typeText', deviceId, async () => { + // Printable US-ASCII only. Each character is mapped to a US-keyboard + // HID usage and sent as an independent event. Shifted characters + // (uppercase letters, symbols like `@!#$%^&*()_+{}|:"<>?~`) are sent + // via the bridge's `key-mod` subcommand which holds LeftShift around + // the key press. Tab, newline, DEL, and non-ASCII characters have no + // mapping and are rejected; higher layers should compose those via + // WebKit/Flutter/simctl backends instead. + // + // When delayMs > 0 an inter-character pause is inserted between + // consecutive key sends. This is required for segmented OTP-style + // inputs (e.g. 6-cell verify-code fields in Flutter) that drop + // characters when keys arrive in rapid succession (issue #639). + let first = true; + for (const ch of text) { + const key = asciiToHidKey(ch); + if (key === null) { + throw new InputBackendError( + `SimulatorKitHIDInputBackend.typeText: unsupported character '${ch}' ` + + '(no HID mapping). Only printable US-ASCII (U+0020..U+007E) is ' + + 'supported; tab, newline, and non-ASCII characters are not. ' + + 'Track follow-up in issue #483.', + 'BAD_ARGS', + ); + } + if (!first && delayMs > 0) { + await sleep(delayMs); + } + first = false; + if (key.shift) { + await this.run([ + deviceId, + 'key-mod', + String(key.usage), + String(HID_USAGE_LEFT_SHIFT), + ]); + } else { + await this.run([deviceId, 'key', String(key.usage)]); + } + } + }); + } + + async keypress(deviceId: string, keyCode: string): Promise { + await timedInput(this.kind, 'keypress', deviceId, async () => { + // Accept either a decimal HID usage code or a key name known to our map. + const parsed = Number.parseInt(keyCode, 10); + const usage = Number.isNaN(parsed) ? KEY_NAME_TO_HID_USAGE[keyCode] : parsed; + if (usage === undefined) { + throw new InputBackendError( + `SimulatorKitHIDInputBackend.keypress: unknown HID key code "${keyCode}"`, + 'BAD_ARGS', + ); + } + await this.run([deviceId, 'key', String(usage)]); + }); + } + + async sendKey(deviceId: string, keyName: string): Promise { + await timedInput(this.kind, 'sendKey', deviceId, async () => { + const usage = KEY_NAME_TO_HID_USAGE[keyName]; + if (usage === undefined) { + throw new InputBackendError( + `SimulatorKitHIDInputBackend.pressKey: unknown key "${keyName}". ` + + `Supported: ${Object.keys(KEY_NAME_TO_HID_USAGE).join(', ')}`, + 'BAD_ARGS', + ); + } + await this.run([deviceId, 'key', String(usage)]); + }); + } + + /** Convenience alias: resolve a symbolic key name to its HID usage. */ + async pressKey(deviceId: string, key: string): Promise { + await this.sendKey(deviceId, key); + } + + /** + * Press `keyUsage` while holding `modifierUsage` (e.g. Cmd+V = keyChord(25, 227)). + * Wraps the bridge's `key-mod` subcommand so callers can compose chords + * without shelling out manually. Used by the pasteboard typing path. + */ + async keyChord( + deviceId: string, + keyUsage: number, + modifierUsage: number, + ): Promise { + await timedInput(this.kind, 'keyChord', deviceId, async () => { + await this.run([ + deviceId, + 'key-mod', + String(keyUsage), + String(modifierUsage), + ]); + }); + } + + /** + * Spawn the bridge with the given argv (not including the bridge path) + * and parse its JSON stdout. Surfaces every documented exit code as a + * structured `InputBackendError`. + */ + private async run(args: string[]): Promise { + if (!warnedAboutPrivateAPI) { + warnedAboutPrivateAPI = true; + console.error( + '[opensafari] SimulatorKitHIDInputBackend uses private Apple frameworks ' + + '(SimulatorKit.framework, CoreSimulator.framework) via dlopen. ' + + 'These APIs are undocumented and Xcode updates may break them. ' + + 'Where can I use this? macOS host / CI only — never bundle inside an ' + + 'iOS .ipa shipped to the App Store or TestFlight. ' + + PRIVATE_API_DOC_REF + + ' (see "Deployment scope").', + ); + } + const { cmd, cmdArgs } = this.resolveSpawn(args); + let stdout = ''; + let stderr = ''; + try { + const result = await execFileAsync(cmd, cmdArgs, { + timeout: SPAWN_TIMEOUT_MS, + maxBuffer: 1 * 1024 * 1024, + }); + stdout = result.stdout ?? ''; + stderr = result.stderr ?? ''; + } catch (err) { + const e = err as NodeJS.ErrnoException & { + stdout?: string; + stderr?: string; + code?: number | string; + killed?: boolean; + }; + stdout = e.stdout ?? ''; + stderr = e.stderr ?? ''; + + if (e.killed && e.code === null) { + throw new InputBackendError( + `sim-hid-bridge timed out after ${SPAWN_TIMEOUT_MS}ms`, + 'SPAWN_TIMEOUT', + stderr, + ); + } + + const exit = typeof e.code === 'number' ? e.code : undefined; + const classified = codeForExit(exit); + const hint = stderr.trim() || stdout.trim() || e.message; + // Attach the private-APIs doc pointer to every SimulatorKit-layer + // failure so MCP clients / CI logs link directly to the BC-break + // response playbook rather than surfacing a bare exit code. + const docSuffix = + classified === 'SIMULATORKIT_UNAVAILABLE' || classified === 'NOT_IMPLEMENTED' + ? ` (${PRIVATE_API_DOC_REF})` + : ''; + throw new InputBackendError( + `sim-hid-bridge exited ${exit ?? '?'}: ${hint}${docSuffix}`, + classified, + stderr, + ); + } + + // Successful spawn: parse the JSON envelope. A bridge that exits 0 but + // emits `{ ok: false, ... }` is treated as a structured failure too. + if (!stdout.trim()) { + return {}; + } + try { + const parsed = JSON.parse(stdout) as { ok?: boolean; error?: string; code?: string }; + if (parsed.ok === false) { + const okFalseCode = (parsed.code as InputBackendErrorCode | undefined) ?? 'UNKNOWN'; + const frameworkFailureCodes = new Set([ + 'SIMULATORKIT_MISSING', + 'CORESIMULATOR_MISSING', + 'HID_CLIENT_FAILED', + 'HID_FUNCTIONS_MISSING', + ]); + const okFalseDocSuffix = frameworkFailureCodes.has(parsed.code ?? '') ? ` (${PRIVATE_API_DOC_REF})` : ''; + throw new InputBackendError( + `${parsed.error ?? 'sim-hid-bridge reported ok=false'}${okFalseDocSuffix}`, + okFalseCode, + stderr, + ); + } + return parsed; + } catch (err) { + if (err instanceof InputBackendError) throw err; + const safeStdout = stdout + .slice(0, 200) + // Strip ASCII control / DEL so a crafted bridge payload can't inject + // ANSI escapes or JSON-RPC framing into MCP server logs. + .replace(/[\x00-\x1f\x7f]/g, '?'); + throw new InputBackendError( + `sim-hid-bridge produced non-JSON stdout: ${safeStdout}`, + 'JSON_PARSE_FAILURE', + stderr, + ); + } + } + + /** + * Decide how to invoke the bridge: as a compiled binary, or via the `swift` + * interpreter when only the .swift source is present (PoC fallback). + */ + private resolveSpawn(args: string[]): { cmd: string; cmdArgs: string[] } { + if (this.bridgePath.endsWith('.swift')) { + return { cmd: 'swift', cmdArgs: [this.bridgePath, ...args] }; + } + return { cmd: this.bridgePath, cmdArgs: args }; + } +} + +/** + * Attempt to locate a usable sim-hid-bridge. Returns a ready-to-use backend, + * or throws an `InputBackendError` with code `HID_BRIDGE_MISSING` when no + * helper is installed on this machine. Callers in the resolver chain are + * expected to catch that error and fall through to the next tier; callers + * that always require the bridge (e.g. `pasteboard-input`) propagate it. + * + * Lookup order: + * 1. Compiled binary at `dist/sim-hid-bridge` (next to `dist/ax-bridge`). + * 2. Swift source at `dist/sim-hid-bridge.swift` (post-build copy). + * 3. Source tree fallback at `src/native/sim-hid-bridge.swift` — DEV ONLY, + * gated behind `OPENSAFARI_ALLOW_SWIFT_INTERPRETER=1`. The repo-relative + * path escapes `dist/` when the package is installed as a dependency, + * and executing unsigned Swift source via the interpreter sidesteps any + * future codesigning we add to the compiled binary, so this candidate + * is intentionally NOT auto-discovered in production installs. + * + * Note: the return type still includes `null` for forward compatibility with + * a future tier-fallback variant that prefers null over throwing. Today the + * function only resolves to a backend or throws. + */ +export async function tryCreateSimulatorKitHIDBackend(): Promise< + SimulatorKitHIDInputBackend | null +> { + const candidates = [ + // Compiled binary co-located with ax-bridge after build. + path.resolve(__dirname, '..', 'sim-hid-bridge'), + path.resolve(__dirname, 'sim-hid-bridge'), + // Swift source copied into dist/ by the postbuild step. + path.resolve(__dirname, '..', 'sim-hid-bridge.swift'), + path.resolve(__dirname, 'sim-hid-bridge.swift'), + ]; + if (process.env.OPENSAFARI_ALLOW_SWIFT_INTERPRETER === '1') { + candidates.push( + path.resolve(__dirname, '..', '..', 'src', 'native', 'sim-hid-bridge.swift'), + ); + } + for (const candidate of candidates) { + if (existsSync(candidate)) { + return new SimulatorKitHIDInputBackend(candidate); + } + } + const searched = candidates.map((c) => ` - ${c}`).join('\n'); + throw new InputBackendError( + `sim-hid-bridge not found. Searched:\n${searched}\n` + + 'Run npm run build or set OPENSAFARI_ALLOW_SWIFT_INTERPRETER=1 for dev mode.', + 'HID_BRIDGE_MISSING', + ); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/src/tools/diagnose.ts b/src/tools/diagnose.ts index a0f58530..3fd28d1e 100644 --- a/src/tools/diagnose.ts +++ b/src/tools/diagnose.ts @@ -8,7 +8,7 @@ import { MCPServer } from '../mcp-server'; import { getSessionManager } from '../session-manager'; import { peekProxyForDevice } from '../simulator/proxy-manager'; -import { tryCreateSimulatorKitHIDBackend } from './sim-hid-input-backend'; +import { tryCreateSimulatorKitHIDBackend } from '../input/sim-hid-backend'; import { getInputTelemetryRollup, type InputTelemetryRollup, diff --git a/src/tools/flutter-vm-input-backend.ts b/src/tools/flutter-vm-input-backend.ts index a641f9b3..e0d4f994 100644 --- a/src/tools/flutter-vm-input-backend.ts +++ b/src/tools/flutter-vm-input-backend.ts @@ -1,522 +1,17 @@ /** - * FlutterVMInputBackend — Tier-0 headless input backend for Flutter apps. + * Compatibility re-export shim for `src/input/flutter-vm-backend`. * - * Dispatches pointer/keyboard/text events directly to the Flutter engine via - * the Dart VM Service (`FlutterVMClient.evaluate`). Because the events are - * synthesised inside the running Dart isolate and fed straight into - * `PlatformDispatcher.onPointerDataPacket`, this backend: + * The implementation was moved to `src/input/flutter-vm-backend.ts` as part + * of the #707 (b) consolidation. This file re-exports every previously-public + * symbol so existing callers (tests, tools) continue to work without + * modification. * - * - **Does not move the physical mouse cursor** (no CGEvent) - * - **Does not bring Simulator.app to the foreground** (no AppleScript - * activation) - * - **Requires no opt-in env var** — it is truly headless - * - * Compared to the three existing tiers (simctl → webkit → applescript), this - * path is picked first whenever the target device is running a Flutter app in - * debug/profile mode and the VM Service URL can be discovered. Native UIKit - * apps continue to flow through the existing tiers unchanged. - * - * Coordinate system: iOS AX frames are expressed in logical points (the same - * units Flutter calls "logical pixels"). The Dart payload multiplies by the - * implicit view's `devicePixelRatio` to land on the engine's `physicalX/Y` - * expectations. - * - * See issue #481 for the motivation and rollout checklist. - */ - -import type { FlutterVMClient } from '../flutter'; -import { FlutterVMError } from '../flutter'; -import type { InputBackend, InputBackendKind } from '../input/backend'; -import { timedInput } from '../metrics/input-telemetry'; - -/** - * Structured error surfaced by FlutterVMInputBackend when the underlying VM - * Service call fails (connection drop, Dart exception, timeout, etc). Carries - * the originating op so observability layers can attribute the failure. - */ -/** - * Structured error codes attached to `FlutterVMInputBackendError`. - * - * Today the code table is deliberately small — callers branch only on - * `VM_NO_EVALUATE` (release-build / no-DDS fallback signal) vs "anything - * else" which is surfaced to the user as a concrete failure. Expand this - * as we learn which failure modes the callers actually need to - * discriminate. - */ -export type FlutterVMInputBackendErrorCode = - /** `evaluate` rejected with code 113 — VM cannot compile expressions. */ - | 'VM_NO_EVALUATE' - /** Dart code ran but raised / returned an @Error. */ - | 'DART_ERROR' - /** Any other cause (connection drop, timeout, unknown). */ - | 'UNKNOWN'; - -export class FlutterVMInputBackendError extends Error { - readonly name = 'FlutterVMInputBackendError' as const; - readonly op: 'tap' | 'swipe' | 'typeText' | 'keypress' | 'sendKey'; - readonly cause: unknown; - readonly code: FlutterVMInputBackendErrorCode; - - constructor( - op: FlutterVMInputBackendError['op'], - cause: unknown, - ) { - const msg = cause instanceof Error ? cause.message : String(cause); - const code: FlutterVMInputBackendErrorCode = classifyFlutterVMCause(cause); - // Release / no-DDS builds emit a verbose JSON-RPC string — replace it - // with a short, actionable diagnostic so tool consumers see a single - // clean message instead of the compiler's internal error payload. - const userMessage = - code === 'VM_NO_EVALUATE' - ? `FlutterVMInputBackend.${op} failed: VM cannot compile expressions ` + - `(code 113). This app is likely a release build or was launched ` + - `with \`simctl launch\` instead of \`flutter run\`. ` + - `Use Tier 1.5 AX press (app_tap_element / app_type_element) or ` + - `relaunch under \`flutter run --debug\` for full gesture coverage. ` + - `See docs/ci-recipes.md#qa-ready-flutter-build for the ` + - `simulator (--debug) and physical-device (--profile) recipes ` + - `that keep Tier 0 available.` - : `FlutterVMInputBackend.${op} failed: ${msg}`; - super(userMessage); - this.op = op; - this.cause = cause; - this.code = code; - Object.setPrototypeOf(this, FlutterVMInputBackendError.prototype); - } -} - -function classifyFlutterVMCause(cause: unknown): FlutterVMInputBackendErrorCode { - const msg = cause instanceof Error ? cause.message : String(cause); - if (/\(code:\s*113\)/.test(msg)) return 'VM_NO_EVALUATE'; - // The inner `evalOrThrow` wraps Dart-side errors with code `DART_ERROR` - // via `FlutterVMError`; preserve that classification when bubbling up. - if ( - cause && - typeof cause === 'object' && - 'code' in cause && - (cause as { code?: string }).code === 'DART_ERROR' - ) { - return 'DART_ERROR'; - } - return 'UNKNOWN'; -} - -/** - * HID key-code → Dart `LogicalKeyboardKey` identifier. The keyIds match the - * values exposed by `package:flutter/services.dart` so the Dart payload can - * materialise a `KeyDownEvent` / `KeyUpEvent` pair. - */ -const HID_TO_LOGICAL_KEY: Record = { - '40': { keyId: 'LogicalKeyboardKey.enter', keyLabel: 'Enter', physicalKey: 'PhysicalKeyboardKey.enter' }, - '41': { keyId: 'LogicalKeyboardKey.escape', keyLabel: 'Escape', physicalKey: 'PhysicalKeyboardKey.escape' }, - '42': { keyId: 'LogicalKeyboardKey.backspace', keyLabel: 'Backspace', physicalKey: 'PhysicalKeyboardKey.backspace' }, - '43': { keyId: 'LogicalKeyboardKey.tab', keyLabel: 'Tab', physicalKey: 'PhysicalKeyboardKey.tab' }, - '44': { keyId: 'LogicalKeyboardKey.space', keyLabel: ' ', physicalKey: 'PhysicalKeyboardKey.space' }, - '74': { keyId: 'LogicalKeyboardKey.home', keyLabel: 'Home', physicalKey: 'PhysicalKeyboardKey.home' }, - '79': { keyId: 'LogicalKeyboardKey.arrowRight', keyLabel: 'ArrowRight', physicalKey: 'PhysicalKeyboardKey.arrowRight' }, - '80': { keyId: 'LogicalKeyboardKey.arrowLeft', keyLabel: 'ArrowLeft', physicalKey: 'PhysicalKeyboardKey.arrowLeft' }, - '81': { keyId: 'LogicalKeyboardKey.arrowDown', keyLabel: 'ArrowDown', physicalKey: 'PhysicalKeyboardKey.arrowDown' }, - '82': { keyId: 'LogicalKeyboardKey.arrowUp', keyLabel: 'ArrowUp', physicalKey: 'PhysicalKeyboardKey.arrowUp' }, -}; - -const SENDKEY_TO_LOGICAL_KEY: Record = { - Return: { keyId: 'LogicalKeyboardKey.enter', keyLabel: 'Enter', physicalKey: 'PhysicalKeyboardKey.enter' }, - Enter: { keyId: 'LogicalKeyboardKey.enter', keyLabel: 'Enter', physicalKey: 'PhysicalKeyboardKey.enter' }, - Escape: { keyId: 'LogicalKeyboardKey.escape', keyLabel: 'Escape', physicalKey: 'PhysicalKeyboardKey.escape' }, - Tab: { keyId: 'LogicalKeyboardKey.tab', keyLabel: 'Tab', physicalKey: 'PhysicalKeyboardKey.tab' }, - Space: { keyId: 'LogicalKeyboardKey.space', keyLabel: ' ', physicalKey: 'PhysicalKeyboardKey.space' }, - Delete: { keyId: 'LogicalKeyboardKey.backspace', keyLabel: 'Backspace', physicalKey: 'PhysicalKeyboardKey.backspace' }, - Backspace: { keyId: 'LogicalKeyboardKey.backspace', keyLabel: 'Backspace', physicalKey: 'PhysicalKeyboardKey.backspace' }, - Home: { keyId: 'LogicalKeyboardKey.home', keyLabel: 'Home', physicalKey: 'PhysicalKeyboardKey.home' }, - ArrowRight: { keyId: 'LogicalKeyboardKey.arrowRight', keyLabel: 'ArrowRight', physicalKey: 'PhysicalKeyboardKey.arrowRight' }, - ArrowLeft: { keyId: 'LogicalKeyboardKey.arrowLeft', keyLabel: 'ArrowLeft', physicalKey: 'PhysicalKeyboardKey.arrowLeft' }, - ArrowDown: { keyId: 'LogicalKeyboardKey.arrowDown', keyLabel: 'ArrowDown', physicalKey: 'PhysicalKeyboardKey.arrowDown' }, - ArrowUp: { keyId: 'LogicalKeyboardKey.arrowUp', keyLabel: 'ArrowUp', physicalKey: 'PhysicalKeyboardKey.arrowUp' }, -}; - -/** - * Format a finite number for interpolation into a Dart literal. Reject NaN / - * ±Infinity so the VM Service never receives a syntactically invalid - * expression (e.g. `Offset(NaN, NaN)` would confuse the analyser). - */ -function dartNum(value: number, label: string): string { - if (!Number.isFinite(value)) { - throw new Error(`Invalid ${label}: ${value} (must be finite)`); - } - // toString() preserves sufficient precision for pixel coordinates. - return value.toString(); -} - -/** - * Escape a JS string for safe embedding inside a Dart single-quoted literal. - * Dart string escape rules are similar to JS but the safest approach is to - * emit a Dart list-of-codeUnits from JSON.stringify, avoiding any ambiguity - * over dollar interpolation, backslashes, quotes, or non-ASCII. + * New consumers should import directly from `../input/flutter-vm-backend`. */ -function dartStringLiteral(value: string): string { - // Encode via a Dart `String.fromCharCodes` so we never have to worry about - // dollar-sign interpolation, adjacent quotes, or embedded newlines. - const codeUnits: number[] = []; - for (let i = 0; i < value.length; i++) { - codeUnits.push(value.charCodeAt(i)); - } - return `String.fromCharCodes(const [${codeUnits.join(',')}])`; -} - -/** - * FlutterVMInputBackend — implements the InputBackend contract by evaluating - * Dart expressions inside the target app's main isolate. - */ -export class FlutterVMInputBackend implements InputBackend { - readonly kind: InputBackendKind = 'flutter-vm'; - private libIdCache: Map = new Map(); - - constructor(private vmClient: FlutterVMClient) {} - - /** - * Resolve a Flutter library URI to its VM Service library id, caching the - * result per URI. Each operation targets a different library so that the - * required symbols are in lexical scope: - * - pointer dispatch → `mouse_tracker.dart` (bare `import 'dart:ui'`) - * - text input → `editable_text.dart` - * - key events → `hardware_keyboard.dart` - * - * Same pattern used by `FlutterVMClient.selectWidgetAtPoint` for the - * inspector library (see `vm-service-client.ts:386-411`). - */ - private async resolveLibId(uri: string): Promise { - const cached = this.libIdCache.get(uri); - if (cached) return cached; - - const isolateId = this.vmClient.getState()?.mainIsolateId; - if (!isolateId) { - throw new FlutterVMError('No main isolate', 'NO_ISOLATE'); - } - const isolate = await this.vmClient.callMethod('getIsolate', { isolateId }); - const libs = - (isolate as { libraries?: Array<{ uri?: string; id?: string }> }).libraries ?? []; - const lib = libs.find((l) => l.uri === uri); - if (!lib?.id) { - throw new FlutterVMError( - `${uri} library not loaded in isolate — is this a Flutter app?`, - 'NO_BINDING_LIB', - ); - } - this.libIdCache.set(uri, lib.id); - return lib.id; - } - - /** - * Synthesise a pointer down → up sequence. When `duration` (in seconds) is - * positive the up event is timestamped `duration * 1000` ms after the down - * event so Flutter's gesture arena treats it as a long-press rather than a - * tap. The Dart payload ends with - * `SchedulerBinding.instance.scheduleFrame()` to ensure the engine pumps - * the event queue even in a quiescent state. - */ - async tap( - deviceId: string, - x: number, - y: number, - duration?: number, - ): Promise { - await timedInput(this.kind, 'tap', deviceId, () => this.tapInternal(x, y, duration)); - } - - private async tapInternal( - x: number, - y: number, - duration?: number, - ): Promise { - let xStr: string; - let yStr: string; - try { - xStr = dartNum(x, 'x'); - yStr = dartNum(y, 'y'); - } catch (err) { - throw new FlutterVMInputBackendError('tap', err); - } - const durMs = duration && duration > 0 ? Math.round(duration * 1000) : 0; - - const expression = - '(() {' + - ' final dispatcher = PlatformDispatcher.instance;' + - ' final view = dispatcher.implicitView;' + - ' if (view == null) { return false; }' + - ' final dpr = view.devicePixelRatio;' + - ` final double px = ${xStr} * dpr;` + - ` final double py = ${yStr} * dpr;` + - ` final int downUs = 0;` + - ` final int upUs = ${durMs} * 1000;` + - ' void dispatch(int tUs, PointerChange change) {' + - ' final packet = PointerDataPacket(data: [' + - ' PointerData(' + - ' timeStamp: Duration(microseconds: tUs),' + - ' change: change,' + - ' kind: PointerDeviceKind.touch,' + - ' device: 1,' + - ' pointerIdentifier: 1,' + - ' physicalX: px,' + - ' physicalY: py,' + - ' buttons: change == PointerChange.up ? 0 : 1,' + - ' pressure: change == PointerChange.up ? 0.0 : 1.0,' + - ' pressureMax: 1.0,' + - ' ),' + - ' ]);' + - ' dispatcher.onPointerDataPacket?.call(packet);' + - ' }' + - ' dispatch(downUs, PointerChange.add);' + - ' dispatch(downUs, PointerChange.down);' + - ' dispatch(upUs, PointerChange.up);' + - ' dispatch(upUs, PointerChange.remove);' + - ' return true;' + - '})()'; - - await this.evalOrThrow('tap', expression, 'package:flutter/src/rendering/mouse_tracker.dart'); - } - - /** - * Synthesise a drag gesture as a down → N×move → up sequence. `duration` - * (seconds) spreads the move events evenly across the requested window so - * the gesture arena classifies it as a swipe rather than a flick or tap. - */ - async swipe( - deviceId: string, - startX: number, - startY: number, - endX: number, - endY: number, - duration?: number, - ): Promise { - await timedInput(this.kind, 'swipe', deviceId, () => - this.swipeInternal(startX, startY, endX, endY, duration), - ); - } - - private async swipeInternal( - startX: number, - startY: number, - endX: number, - endY: number, - duration?: number, - ): Promise { - const sxStr = dartNum(startX, 'startX'); - const syStr = dartNum(startY, 'startY'); - const exStr = dartNum(endX, 'endX'); - const eyStr = dartNum(endY, 'endY'); - const totalMs = Math.max(1, Math.round((duration ?? 0.5) * 1000)); - const steps = 20; - const stepUs = Math.round((totalMs * 1000) / steps); - - const expression = - '(() {' + - ' final dispatcher = PlatformDispatcher.instance;' + - ' final view = dispatcher.implicitView;' + - ' if (view == null) { return false; }' + - ' final dpr = view.devicePixelRatio;' + - ` final double sx = ${sxStr} * dpr;` + - ` final double sy = ${syStr} * dpr;` + - ` final double ex = ${exStr} * dpr;` + - ` final double ey = ${eyStr} * dpr;` + - ` const int steps = ${steps};` + - ` const int stepUs = ${stepUs};` + - ' void post(int tUs, PointerChange change, double x, double y) {' + - ' final packet = PointerDataPacket(data: [' + - ' PointerData(' + - ' timeStamp: Duration(microseconds: tUs),' + - ' change: change,' + - ' kind: PointerDeviceKind.touch,' + - ' device: 1,' + - ' pointerIdentifier: 1,' + - ' physicalX: x,' + - ' physicalY: y,' + - ' buttons: change == PointerChange.up ? 0 : 1,' + - ' pressure: change == PointerChange.up ? 0.0 : 1.0,' + - ' pressureMax: 1.0,' + - ' ),' + - ' ]);' + - ' dispatcher.onPointerDataPacket?.call(packet);' + - ' }' + - ' post(0, PointerChange.add, sx, sy);' + - ' post(0, PointerChange.down, sx, sy);' + - ' for (int i = 1; i <= steps; i++) {' + - ' final double t = i / steps;' + - ' final double x = sx + (ex - sx) * t;' + - ' final double y = sy + (ey - sy) * t;' + - ' post(stepUs * i, PointerChange.move, x, y);' + - ' }' + - ' final int endUs = stepUs * steps;' + - ' post(endUs, PointerChange.up, ex, ey);' + - ' post(endUs, PointerChange.remove, ex, ey);' + - ' return true;' + - '})()'; - - await this.evalOrThrow('swipe', expression, 'package:flutter/src/rendering/mouse_tracker.dart'); - } - - /** - * Inject text into the currently-focused `EditableText` via Flutter's - * `TextInput` channel. This mirrors what the iOS IME would send when the - * user types on the system keyboard, so controllers and `onChanged` - * callbacks fire naturally. Falls through silently (no-op) if nothing is - * focused — same behaviour as WebKitInputBackend. - */ - async typeText(deviceId: string, text: string): Promise { - await timedInput(this.kind, 'typeText', deviceId, () => this.typeTextInternal(text)); - } - - private async typeTextInternal(text: string): Promise { - const textLit = dartStringLiteral(text); - - // Read the live TextInputConnection client id so the platform message - // targets the correct connection. The Flutter framework drops messages - // where args[0] != _currentConnection._id, so hardcoding -1 would be - // a silent no-op. We read the id via TextInput._currentConnection._id, - // which is private but accessible via evaluate on the binding library. - const expression = - '(() async {' + - ` final String newText = ${textLit};` + - ' // Read the current TextInputConnection client id.' + - ' // If nothing is focused, fall back to -1 (message is dropped, same' + - ' // as typing on a hardware keyboard with no focused field).' + - ' int clientId = -1;' + - ' try {' + - ' // Fallback: just check if the primary focus accepts text.' + - ' final focused = FocusManager.instance.primaryFocus;' + - ' if (focused != null && focused.context != null) {' + - ' final editable = focused.context!.findAncestorStateOfType();' + - ' if (editable != null) {' + - ' // Force update through the editable directly — this is' + - ' // the most reliable path since it bypasses the private _id.' + - ' final ctrl = editable.textEditingValue;' + - ' editable.userUpdateTextEditingValue(' + - ' TextEditingValue(' + - ' text: ctrl.text + newText,' + - ' selection: TextSelection.collapsed(offset: ctrl.text.length + newText.length),' + - ' ),' + - ' SelectionChangedCause.keyboard,' + - ' );' + - ' return true;' + - ' }' + - ' }' + - ' } catch (_) {}' + - ' // Fallback: deliver via platform channel with best-effort client id.' + - ' final Map state = {' + - ' "text": newText,' + - ' "selectionBase": newText.length,' + - ' "selectionExtent": newText.length,' + - ' "selectionAffinity": "TextAffinity.downstream",' + - ' "selectionIsDirectional": false,' + - ' "composingBase": -1,' + - ' "composingExtent": -1,' + - ' };' + - ' final message = const JSONMethodCodec().encodeMethodCall(' + - ' MethodCall("TextInputClient.updateEditingState", [clientId, state]),' + - ' );' + - ' await WidgetsBinding.instance!.defaultBinaryMessenger.handlePlatformMessage(' + - ' "flutter/textinput",' + - ' message,' + - ' (dynamic _) {},' + - ' );' + - ' return true;' + - '})()'; - - await this.evalOrThrow('typeText', expression, 'package:flutter/src/widgets/editable_text.dart'); - } - - /** - * Dispatch a HID key code through `HardwareKeyboard`. Only a curated set of - * control keys is supported — matches the WebKit/AppleScript backends. - */ - async keypress(deviceId: string, keyCode: string): Promise { - await timedInput(this.kind, 'keypress', deviceId, () => this.keypressInternal(keyCode)); - } - - private async keypressInternal(keyCode: string): Promise { - const entry = HID_TO_LOGICAL_KEY[keyCode]; - if (!entry) { - throw new Error( - `Unknown HID key code "${keyCode}" for FlutterVM backend. ` + - `Supported: ${Object.keys(HID_TO_LOGICAL_KEY).join(', ')}`, - ); - } - await this.dispatchKey('keypress', entry.keyId, entry.keyLabel, entry.physicalKey); - } - - /** Dispatch a named key ("Return", "Escape", ...) through HardwareKeyboard. */ - async sendKey(deviceId: string, keyName: string): Promise { - await timedInput(this.kind, 'sendKey', deviceId, () => this.sendKeyInternal(keyName)); - } - - private async sendKeyInternal(keyName: string): Promise { - const entry = SENDKEY_TO_LOGICAL_KEY[keyName]; - if (!entry) { - throw new Error( - `Unknown key name "${keyName}" for FlutterVM backend. ` + - `Supported: ${Object.keys(SENDKEY_TO_LOGICAL_KEY).join(', ')}`, - ); - } - await this.dispatchKey('sendKey', entry.keyId, entry.keyLabel, entry.physicalKey); - } - - // ── internals ────────────────────────────────────────────────────────── - - private async dispatchKey( - op: 'keypress' | 'sendKey', - logicalKeyExpr: string, - keyLabel: string, - physicalKeyExpr: string, - ): Promise { - const labelLit = dartStringLiteral(keyLabel); - // Emit a KeyDown event then a KeyUp through HardwareKeyboard so downstream - // focus nodes observe a complete press. `timeStamp` uses the default - // (zero) — the event queue does not require strict monotonicity. - const expression = - '(() {' + - ` final label = ${labelLit};` + - ` final logical = ${logicalKeyExpr};` + - ` final physical = ${physicalKeyExpr};` + - ' final down = KeyDownEvent(' + - ' physicalKey: physical,' + - ' logicalKey: logical,' + - ' timeStamp: Duration.zero,' + - ' character: label.length == 1 ? label : null,' + - ' );' + - ' final up = KeyUpEvent(' + - ' physicalKey: physical,' + - ' logicalKey: logical,' + - ' timeStamp: Duration.zero,' + - ' );' + - ' HardwareKeyboard.instance.handleKeyEvent(down);' + - ' HardwareKeyboard.instance.handleKeyEvent(up);' + - ' return true;' + - '})()'; - await this.evalOrThrow(op, expression, 'package:flutter/src/services/hardware_keyboard.dart'); - } +export { + FlutterVMInputBackendError, + FlutterVMInputBackend, +} from '../input/flutter-vm-backend'; - private async evalOrThrow( - op: FlutterVMInputBackendError['op'], - expression: string, - libraryUri: string, - ): Promise { - try { - // Scope the evaluate to the per-operation Flutter library so all - // required symbols are in lexical scope. - const targetId = await this.resolveLibId(libraryUri); - const result = await this.vmClient.evaluate(expression, { targetId }); - // VM returns an @Error shape instead of throwing when the expression - // itself compiled but raised a Dart exception. Surface that as a - // structured InputBackendError. - const errType = (result as { type?: string }).type; - if (errType === '@Error' || errType === 'Error') { - const message = - (result as { message?: string }).message ?? JSON.stringify(result); - throw new FlutterVMError(message, 'DART_ERROR'); - } - } catch (err) { - if (err instanceof FlutterVMInputBackendError) throw err; - throw new FlutterVMInputBackendError(op, err); - } - } -} +export type { FlutterVMInputBackendErrorCode } from '../input/flutter-vm-backend'; diff --git a/src/tools/pasteboard-input.ts b/src/tools/pasteboard-input.ts index 1d341207..c1aff909 100644 --- a/src/tools/pasteboard-input.ts +++ b/src/tools/pasteboard-input.ts @@ -30,7 +30,7 @@ import { SimulatorKitHIDInputBackend, tryCreateSimulatorKitHIDBackend, InputBackendError, -} from './sim-hid-input-backend'; +} from '../input/sim-hid-backend'; import { getAccessibilityBridge } from '../native'; import { matchLabel as matchButtonLabel } from './localized-button-matcher'; diff --git a/src/tools/pointer-service-input-backend.ts b/src/tools/pointer-service-input-backend.ts index c728f66c..8e84a969 100644 --- a/src/tools/pointer-service-input-backend.ts +++ b/src/tools/pointer-service-input-backend.ts @@ -1,220 +1,17 @@ /** - * PointerServiceInputBackend — Phase 1 opt-in wrapper around the - * `sim-hid-bridge tap-ps` subcommand for Xcode 26+ coordinate tap. + * Compatibility re-export shim for `src/input/pointer-service-backend`. * - * Motivation (issue #590): Apple dropped `IndigoHIDMessageForMouseNSEvent` - * handling in CoreSimulator on Xcode 26. The Tier-1 SimulatorKitHID bare - * mouse path has therefore been gated off for tap/swipe, forcing - * coordinate-based `app_tap` / `app_swipe_native` to fall through to the - * focus-stealing AppleScript / CGEvent backend. The PointerService probe - * shipped in #555 wraps the same mouse events with - * `IndigoHIDMessageToCreatePointerService` / `RemovePointerService` - * brackets; the synthesis doc (#557) falsified this as a *fix*, but it - * remains the most-likely-to-help interim stop-gap because: + * The implementation was moved to `src/input/pointer-service-backend.ts` as + * part of the #707 (b) consolidation. This file re-exports every + * previously-public symbol so existing callers (tests, tools) continue to + * work without modification. * - * - It requires no simulator-side changes. - * - Telemetry is cheap to collect under an opt-in flag. - * - If field data shows ≥ 99 % success, #590 Phase 2 promotes it to the - * default Tier-1 path. - * - * Status: **opt-in experimental**. Activated only when - * `OPENSAFARI_ENABLE_POINTERSERVICE=1` is set. Defaults off so CI that - * has already adapted to the AppleScript fallback is unaffected. - * - * Scope: only `tap` is routed through the `tap-ps` subcommand. The Swift - * bridge does not yet expose `swipe-ps`; swipe / typeText / keypress / - * sendKey delegate to the underlying `SimulatorKitHIDInputBackend`, which - * keeps non-tap input paths on their existing (keyboard-safe) Tier-1 - * route. On Xcode 26+ that delegated SimHID swipe path is itself gated - * off and throws `HeadlessInputUnavailableError`; the PointerService - * backend is cached as the selected backend by `getInputBackend`, so the - * throw does NOT re-enter the tier chain. See the comment on `swipe()` - * below and issue #649 for the caller-visible contract. Extending - * `sim-hid-bridge` with pointer-service-bracketed swipe and promoting - * the backend to the default chain are tracked as Phase 2 follow-ups in - * #590. + * New consumers should import directly from `../input/pointer-service-backend`. */ -import { existsSync } from 'fs'; -import * as path from 'path'; -import type { InputBackend } from '../input/backend'; -import { - SimulatorKitHIDInputBackend, - InputBackendError, -} from './sim-hid-input-backend'; -import { timedInput } from '../metrics/input-telemetry'; - -/** - * Env flag that enables the PointerService backend (Phase 1 opt-in). - * Accepted values: `1`, `true`. Anything else is ignored. - */ -export const OPENSAFARI_ENABLE_POINTERSERVICE_ENV = - 'OPENSAFARI_ENABLE_POINTERSERVICE'; - -export function isPointerServiceEnabled(): boolean { - const value = process.env[OPENSAFARI_ENABLE_POINTERSERVICE_ENV]; - return value === '1' || value === 'true'; -} - -/** - * PointerService tap + delegated swipe/keys. - * - * Composes a `SimulatorKitHIDInputBackend` for the subset of methods that - * still share the default `sim-hid-bridge` subcommand set, and shells out - * directly to `sim-hid-bridge tap-ps ...` for `tap`. - */ -export class PointerServiceInputBackend implements InputBackend { - readonly kind = 'pointer-service' as const; - - constructor( - private readonly bridgePath: string, - private readonly delegate: SimulatorKitHIDInputBackend, - ) {} - - async tap(deviceId: string, x: number, y: number, duration?: number): Promise { - await timedInput(this.kind, 'tap', deviceId, async () => { - const args = [deviceId, 'tap-ps', String(x), String(y)]; - if (duration !== undefined && duration > 0) { - args.push(String(duration)); - } - await runTapPs(this.bridgePath, args); - }); - } - - async swipe( - deviceId: string, - startX: number, - startY: number, - endX: number, - endY: number, - duration?: number, - ): Promise { - // Phase 1: no swipe-ps subcommand exists yet. Delegate straight to the - // underlying SimulatorKitHIDInputBackend. On Xcode 26+ the `sim-hid-bridge - // swipe` subcommand exits with `SIMULATORKIT_UNAVAILABLE` (or another - // non-zero SimulatorKit code), and the delegate surfaces that to the caller - // as an `InputBackendError` — not `HeadlessInputUnavailableError`, which is - // produced one layer up in `native-input-backend.getInputBackend` when - // selecting a backend, never from a backend's own swipe() method. - // - // That error does NOT re-enter the tier chain because - // PointerServiceInputBackend is cached as the selected backend in - // getInputBackend once OPENSAFARI_ENABLE_POINTERSERVICE=1 resolves it. For - // completeness, any other error type the delegate may raise in the future - // is also passed through unchanged; see the "swipe propagates … without - // wrapping" tests for the frozen contract. - // - // Callers that need swipe fallback on Xcode 26+ must either (a) leave - // OPENSAFARI_ENABLE_POINTERSERVICE unset so the standard Tier-1 SimHID / - // focus-input chain is selected for every call, or (b) invoke an - // element-targeted swipe so the AX-press tier handles the gesture. - // Promoting this to a real in-tool tier downgrade is tracked under #590 - // Phase 2 alongside `sim-hid-bridge swipe-ps`. See #649 for the decision - // record. - await this.delegate.swipe(deviceId, startX, startY, endX, endY, duration); - } - - async typeText(deviceId: string, text: string): Promise { - await this.delegate.typeText(deviceId, text); - } - - async keypress(deviceId: string, keyCode: string): Promise { - await this.delegate.keypress(deviceId, keyCode); - } - - async sendKey(deviceId: string, keyName: string): Promise { - await this.delegate.sendKey(deviceId, keyName); - } -} - -/** - * Execute `sim-hid-bridge` with the given argv. Mirrors the spawn/parse - * contract of `SimulatorKitHIDInputBackend.run` but is narrowed to the - * single `tap-ps` subcommand used by the PointerService backend, so the - * failure-handling path stays local and auditable. - */ -async function runTapPs(bridgePath: string, args: string[]): Promise { - const { execFile } = await import('child_process'); - const { promisify } = await import('util'); - const execFileAsync = promisify(execFile); - - const isSwiftSource = bridgePath.endsWith('.swift'); - const cmd = isSwiftSource ? 'swift' : bridgePath; - const cmdArgs = isSwiftSource ? [bridgePath, ...args] : args; - - let stdout = ''; - let stderr = ''; - try { - const result = await execFileAsync(cmd, cmdArgs, { - timeout: 10_000, - maxBuffer: 1 * 1024 * 1024, - }); - stdout = result.stdout ?? ''; - stderr = result.stderr ?? ''; - } catch (err) { - const e = err as NodeJS.ErrnoException & { - stdout?: string; - stderr?: string; - code?: number | string; - killed?: boolean; - }; - stdout = e.stdout ?? ''; - stderr = e.stderr ?? ''; - const exit = typeof e.code === 'number' ? e.code : undefined; - const hint = stderr.trim() || stdout.trim() || e.message; - throw new InputBackendError( - `sim-hid-bridge tap-ps exited ${exit ?? '?'}: ${hint}`, - 'UNKNOWN', - stderr, - ); - } - - if (!stdout.trim()) return; - try { - const parsed = JSON.parse(stdout) as { ok?: boolean; error?: string }; - if (parsed.ok === false) { - throw new InputBackendError( - parsed.error ?? 'sim-hid-bridge tap-ps reported ok=false', - 'UNKNOWN', - stderr, - ); - } - } catch (err) { - if (err instanceof InputBackendError) throw err; - const safeStdout = stdout - .slice(0, 200) - .replace(/[\x00-\x1f\x7f]/g, '?'); - throw new InputBackendError( - `sim-hid-bridge tap-ps produced non-JSON stdout: ${safeStdout}`, - 'JSON_PARSE_FAILURE', - stderr, - ); - } -} - -/** - * Factory mirroring `tryCreateSimulatorKitHIDBackend`: locate a usable - * `sim-hid-bridge` helper and wrap it as a `PointerServiceInputBackend`. - * Returns `null` when the helper is not installed — callers are expected - * to fall through to the default SimHID tier. - */ -export async function tryCreatePointerServiceBackend(): Promise { - const candidates = [ - path.resolve(__dirname, '..', 'sim-hid-bridge'), - path.resolve(__dirname, 'sim-hid-bridge'), - path.resolve(__dirname, '..', 'sim-hid-bridge.swift'), - path.resolve(__dirname, 'sim-hid-bridge.swift'), - ]; - if (process.env.OPENSAFARI_ALLOW_SWIFT_INTERPRETER === '1') { - candidates.push( - path.resolve(__dirname, '..', '..', 'src', 'native', 'sim-hid-bridge.swift'), - ); - } - for (const candidate of candidates) { - if (existsSync(candidate)) { - const delegate = new SimulatorKitHIDInputBackend(candidate); - return new PointerServiceInputBackend(candidate, delegate); - } - } - return null; -} +export { + OPENSAFARI_ENABLE_POINTERSERVICE_ENV, + isPointerServiceEnabled, + PointerServiceInputBackend, + tryCreatePointerServiceBackend, +} from '../input/pointer-service-backend'; diff --git a/src/tools/sim-hid-input-backend.ts b/src/tools/sim-hid-input-backend.ts index 744280a9..11484029 100644 --- a/src/tools/sim-hid-input-backend.ts +++ b/src/tools/sim-hid-input-backend.ts @@ -1,471 +1,19 @@ /** - * SimulatorKitHIDInputBackend — Node wrapper around the `sim-hid-bridge` - * Swift helper described in issue #483. + * Compatibility re-export shim for `src/input/sim-hid-backend`. * - * Status: PoC. Backend class is shipped for integration and unit testing, but - * routing in `native-input-backend.ts` is intentionally NOT wired up yet. See - * the `TODO(#483)` comment there. + * The implementation was moved to `src/input/sim-hid-backend.ts` as part of + * the #707 (b) consolidation. This file re-exports every previously-public + * symbol so existing callers (tests, tools) continue to work without + * modification. * - * The Swift bridge spawns as a short-lived child process and communicates via - * argv (command) + stdout (newline-terminated JSON). Exit codes are the - * contract between Swift and Node: - * - * 0 — success - * 64 — BAD_ARGS (EX_USAGE) - * 69 — DEVICE_NOT_BOOTED (EX_UNAVAILABLE) - * 78 — SIMULATORKIT_UNAVAILABLE (EX_CONFIG — dlopen failed) - * 99 — NOT_IMPLEMENTED (PoC stub path) - * * — UNKNOWN (stderr surfaced verbatim) - * - * The current Swift implementation is a PoC stub that proves the dlopen path - * works and always exits with 99 NOT_IMPLEMENTED. This wrapper classifies that - * (and every other documented exit code) into a structured `InputBackendError` - * so the routing layer can decide to fall through to the next tier. - */ - -import { execFile } from 'child_process'; -import { promisify } from 'util'; -import { existsSync } from 'fs'; -import * as path from 'path'; -import type { InputBackend } from '../input/backend'; -import { timedInput } from '../metrics/input-telemetry'; - -const execFileAsync = promisify(execFile); - -/** Reference appended to error messages for private-framework failures. */ -const PRIVATE_API_DOC_REF = 'See docs/private-apis.md'; - -/** Latch so the private-API warning is emitted only once per process. */ -let warnedAboutPrivateAPI = false; - -/** - * Reset the private-API warning latch. Exported for unit tests only — do not - * call from production code. - */ -export function resetSimHidPrivateAPIWarning(): void { - warnedAboutPrivateAPI = false; -} - -/** Spawn timeout for the Swift helper. Matches idb's default. */ -const SPAWN_TIMEOUT_MS = 10_000; - - -/** HID usage page 0x07 (Keyboard/Keypad) — subset we map for pressKey(). */ -const KEY_NAME_TO_HID_USAGE: Record = { - Enter: 0x28, - Return: 0x28, - Escape: 0x29, - Backspace: 0x2a, - Delete: 0x2a, - Tab: 0x2b, - Space: 0x2c, - ArrowRight: 0x4f, - ArrowLeft: 0x50, - ArrowDown: 0x51, - ArrowUp: 0x52, - Home: 0x4a, -}; - -/** - * HID usage of the LeftShift modifier (Keyboard/Keypad page 0x07). - * Sent alongside a character key via the bridge's `key-mod` subcommand for - * every ASCII symbol that requires Shift on a US keyboard (uppercase letters, - * `!@#$%^&*()_+{}|:"<>?~`). - */ -const HID_USAGE_LEFT_SHIFT = 0xe1; - -/** - * US-keyboard printable ASCII → HID usage + whether Shift must be held. - * - * Covers U+0020 (space) through U+007E (tilde) — i.e. every character produced - * by a US layout without dead keys or IME. Returns null for everything else, - * including control characters (tab, newline), DEL, and any non-ASCII byte. - * - * Reference: USB HID Usage Tables v1.21, §10 Keyboard/Keypad (page 0x07). - */ -function asciiToHidKey(ch: string): { usage: number; shift: boolean } | null { - if (ch.length !== 1) return null; - const code = ch.charCodeAt(0); - if (code < 0x20 || code > 0x7e) return null; - // Lowercase letters → HID 0x04..0x1D - if (code >= 0x61 && code <= 0x7a) return { usage: 0x04 + (code - 0x61), shift: false }; - // Uppercase letters → same keys, but Shift is required - if (code >= 0x41 && code <= 0x5a) return { usage: 0x04 + (code - 0x41), shift: true }; - // Digits '1'..'9' → 0x1E..0x26 - if (code >= 0x31 && code <= 0x39) return { usage: 0x1e + (code - 0x31), shift: false }; - if (code === 0x30) return { usage: 0x27, shift: false }; // '0' - switch (ch) { - case ' ': return { usage: 0x2c, shift: false }; - case '-': return { usage: 0x2d, shift: false }; - case '_': return { usage: 0x2d, shift: true }; - case '=': return { usage: 0x2e, shift: false }; - case '+': return { usage: 0x2e, shift: true }; - case '[': return { usage: 0x2f, shift: false }; - case '{': return { usage: 0x2f, shift: true }; - case ']': return { usage: 0x30, shift: false }; - case '}': return { usage: 0x30, shift: true }; - case '\\': return { usage: 0x31, shift: false }; - case '|': return { usage: 0x31, shift: true }; - case ';': return { usage: 0x33, shift: false }; - case ':': return { usage: 0x33, shift: true }; - case "'": return { usage: 0x34, shift: false }; - case '"': return { usage: 0x34, shift: true }; - case '`': return { usage: 0x35, shift: false }; - case '~': return { usage: 0x35, shift: true }; - case ',': return { usage: 0x36, shift: false }; - case '<': return { usage: 0x36, shift: true }; - case '.': return { usage: 0x37, shift: false }; - case '>': return { usage: 0x37, shift: true }; - case '/': return { usage: 0x38, shift: false }; - case '?': return { usage: 0x38, shift: true }; - case '!': return { usage: 0x1e, shift: true }; - case '@': return { usage: 0x1f, shift: true }; - case '#': return { usage: 0x20, shift: true }; - case '$': return { usage: 0x21, shift: true }; - case '%': return { usage: 0x22, shift: true }; - case '^': return { usage: 0x23, shift: true }; - case '&': return { usage: 0x24, shift: true }; - case '*': return { usage: 0x25, shift: true }; - case '(': return { usage: 0x26, shift: true }; - case ')': return { usage: 0x27, shift: true }; - } - return null; -} - -/** - * Error emitted by `SimulatorKitHIDInputBackend`. Mirrors the convention used - * by `AccessibilityBridgeError` (see `src/native/accessibility-bridge.ts`): - * a stable machine-readable `code` plus the human-readable `message`. - */ -export class InputBackendError extends Error { - readonly name = 'InputBackendError' as const; - constructor( - message: string, - public readonly code: InputBackendErrorCode, - public readonly stderr?: string, - ) { - super(message); - Object.setPrototypeOf(this, InputBackendError.prototype); - } -} - -export type InputBackendErrorCode = - | 'BAD_ARGS' - | 'DEVICE_NOT_BOOTED' - | 'SIMULATORKIT_UNAVAILABLE' - | 'NOT_IMPLEMENTED' - | 'SPAWN_TIMEOUT' - | 'BRIDGE_NOT_FOUND' - | 'HID_BRIDGE_MISSING' - | 'JSON_PARSE_FAILURE' - | 'UNKNOWN'; - -/** Map Swift bridge exit codes to structured error codes. */ -function codeForExit(exit: number | undefined): InputBackendErrorCode { - switch (exit) { - case 64: return 'BAD_ARGS'; - case 69: return 'DEVICE_NOT_BOOTED'; - case 78: return 'SIMULATORKIT_UNAVAILABLE'; - case 99: return 'NOT_IMPLEMENTED'; - default: return 'UNKNOWN'; - } -} - -/** - * SimulatorKit HID input backend. Spawns `sim-hid-bridge` per call and parses - * the JSON status envelope. All methods throw `InputBackendError` on failure. + * New consumers should import directly from `../input/sim-hid-backend`. */ -export class SimulatorKitHIDInputBackend implements InputBackend { - readonly kind = 'simhid' as const; - - constructor(private readonly bridgePath: string) {} - - async tap(deviceId: string, x: number, y: number, duration?: number): Promise { - await timedInput(this.kind, 'tap', deviceId, async () => { - const args = [deviceId, 'tap', String(x), String(y)]; - if (duration !== undefined && duration > 0) { - args.push(String(duration)); - } - await this.run(args); - }); - } - - async swipe( - deviceId: string, - startX: number, - startY: number, - endX: number, - endY: number, - duration?: number, - ): Promise { - await timedInput(this.kind, 'swipe', deviceId, async () => { - const args = [ - deviceId, 'swipe', - String(startX), String(startY), - String(endX), String(endY), - ]; - if (duration !== undefined && duration > 0) { - args.push(String(duration)); - } - await this.run(args); - }); - } - - async typeText(deviceId: string, text: string, delayMs = 0): Promise { - await timedInput(this.kind, 'typeText', deviceId, async () => { - // Printable US-ASCII only. Each character is mapped to a US-keyboard - // HID usage and sent as an independent event. Shifted characters - // (uppercase letters, symbols like `@!#$%^&*()_+{}|:"<>?~`) are sent - // via the bridge's `key-mod` subcommand which holds LeftShift around - // the key press. Tab, newline, DEL, and non-ASCII characters have no - // mapping and are rejected; higher layers should compose those via - // WebKit/Flutter/simctl backends instead. - // - // When delayMs > 0 an inter-character pause is inserted between - // consecutive key sends. This is required for segmented OTP-style - // inputs (e.g. 6-cell verify-code fields in Flutter) that drop - // characters when keys arrive in rapid succession (issue #639). - let first = true; - for (const ch of text) { - const key = asciiToHidKey(ch); - if (key === null) { - throw new InputBackendError( - `SimulatorKitHIDInputBackend.typeText: unsupported character '${ch}' ` + - '(no HID mapping). Only printable US-ASCII (U+0020..U+007E) is ' + - 'supported; tab, newline, and non-ASCII characters are not. ' + - 'Track follow-up in issue #483.', - 'BAD_ARGS', - ); - } - if (!first && delayMs > 0) { - await sleep(delayMs); - } - first = false; - if (key.shift) { - await this.run([ - deviceId, - 'key-mod', - String(key.usage), - String(HID_USAGE_LEFT_SHIFT), - ]); - } else { - await this.run([deviceId, 'key', String(key.usage)]); - } - } - }); - } - - async keypress(deviceId: string, keyCode: string): Promise { - await timedInput(this.kind, 'keypress', deviceId, async () => { - // Accept either a decimal HID usage code or a key name known to our map. - const parsed = Number.parseInt(keyCode, 10); - const usage = Number.isNaN(parsed) ? KEY_NAME_TO_HID_USAGE[keyCode] : parsed; - if (usage === undefined) { - throw new InputBackendError( - `SimulatorKitHIDInputBackend.keypress: unknown HID key code "${keyCode}"`, - 'BAD_ARGS', - ); - } - await this.run([deviceId, 'key', String(usage)]); - }); - } - async sendKey(deviceId: string, keyName: string): Promise { - await timedInput(this.kind, 'sendKey', deviceId, async () => { - const usage = KEY_NAME_TO_HID_USAGE[keyName]; - if (usage === undefined) { - throw new InputBackendError( - `SimulatorKitHIDInputBackend.pressKey: unknown key "${keyName}". ` + - `Supported: ${Object.keys(KEY_NAME_TO_HID_USAGE).join(', ')}`, - 'BAD_ARGS', - ); - } - await this.run([deviceId, 'key', String(usage)]); - }); - } - - /** Convenience alias: resolve a symbolic key name to its HID usage. */ - async pressKey(deviceId: string, key: string): Promise { - await this.sendKey(deviceId, key); - } - - /** - * Press `keyUsage` while holding `modifierUsage` (e.g. Cmd+V = keyChord(25, 227)). - * Wraps the bridge's `key-mod` subcommand so callers can compose chords - * without shelling out manually. Used by the pasteboard typing path. - */ - async keyChord( - deviceId: string, - keyUsage: number, - modifierUsage: number, - ): Promise { - await timedInput(this.kind, 'keyChord', deviceId, async () => { - await this.run([ - deviceId, - 'key-mod', - String(keyUsage), - String(modifierUsage), - ]); - }); - } - - /** - * Spawn the bridge with the given argv (not including the bridge path) - * and parse its JSON stdout. Surfaces every documented exit code as a - * structured `InputBackendError`. - */ - private async run(args: string[]): Promise { - if (!warnedAboutPrivateAPI) { - warnedAboutPrivateAPI = true; - console.error( - '[opensafari] SimulatorKitHIDInputBackend uses private Apple frameworks ' + - '(SimulatorKit.framework, CoreSimulator.framework) via dlopen. ' + - 'These APIs are undocumented and Xcode updates may break them. ' + - 'Where can I use this? macOS host / CI only — never bundle inside an ' + - 'iOS .ipa shipped to the App Store or TestFlight. ' + - PRIVATE_API_DOC_REF + - ' (see "Deployment scope").', - ); - } - const { cmd, cmdArgs } = this.resolveSpawn(args); - let stdout = ''; - let stderr = ''; - try { - const result = await execFileAsync(cmd, cmdArgs, { - timeout: SPAWN_TIMEOUT_MS, - maxBuffer: 1 * 1024 * 1024, - }); - stdout = result.stdout ?? ''; - stderr = result.stderr ?? ''; - } catch (err) { - const e = err as NodeJS.ErrnoException & { - stdout?: string; - stderr?: string; - code?: number | string; - killed?: boolean; - }; - stdout = e.stdout ?? ''; - stderr = e.stderr ?? ''; - - if (e.killed && e.code === null) { - throw new InputBackendError( - `sim-hid-bridge timed out after ${SPAWN_TIMEOUT_MS}ms`, - 'SPAWN_TIMEOUT', - stderr, - ); - } - - const exit = typeof e.code === 'number' ? e.code : undefined; - const classified = codeForExit(exit); - const hint = stderr.trim() || stdout.trim() || e.message; - // Attach the private-APIs doc pointer to every SimulatorKit-layer - // failure so MCP clients / CI logs link directly to the BC-break - // response playbook rather than surfacing a bare exit code. - const docSuffix = - classified === 'SIMULATORKIT_UNAVAILABLE' || classified === 'NOT_IMPLEMENTED' - ? ` (${PRIVATE_API_DOC_REF})` - : ''; - throw new InputBackendError( - `sim-hid-bridge exited ${exit ?? '?'}: ${hint}${docSuffix}`, - classified, - stderr, - ); - } - - // Successful spawn: parse the JSON envelope. A bridge that exits 0 but - // emits `{ ok: false, ... }` is treated as a structured failure too. - if (!stdout.trim()) { - return {}; - } - try { - const parsed = JSON.parse(stdout) as { ok?: boolean; error?: string; code?: string }; - if (parsed.ok === false) { - const okFalseCode = (parsed.code as InputBackendErrorCode | undefined) ?? 'UNKNOWN'; - const frameworkFailureCodes = new Set([ - 'SIMULATORKIT_MISSING', - 'CORESIMULATOR_MISSING', - 'HID_CLIENT_FAILED', - 'HID_FUNCTIONS_MISSING', - ]); - const okFalseDocSuffix = frameworkFailureCodes.has(parsed.code ?? '') ? ` (${PRIVATE_API_DOC_REF})` : ''; - throw new InputBackendError( - `${parsed.error ?? 'sim-hid-bridge reported ok=false'}${okFalseDocSuffix}`, - okFalseCode, - stderr, - ); - } - return parsed; - } catch (err) { - if (err instanceof InputBackendError) throw err; - const safeStdout = stdout - .slice(0, 200) - // Strip ASCII control / DEL so a crafted bridge payload can't inject - // ANSI escapes or JSON-RPC framing into MCP server logs. - .replace(/[\x00-\x1f\x7f]/g, '?'); - throw new InputBackendError( - `sim-hid-bridge produced non-JSON stdout: ${safeStdout}`, - 'JSON_PARSE_FAILURE', - stderr, - ); - } - } - - /** - * Decide how to invoke the bridge: as a compiled binary, or via the `swift` - * interpreter when only the .swift source is present (PoC fallback). - */ - private resolveSpawn(args: string[]): { cmd: string; cmdArgs: string[] } { - if (this.bridgePath.endsWith('.swift')) { - return { cmd: 'swift', cmdArgs: [this.bridgePath, ...args] }; - } - return { cmd: this.bridgePath, cmdArgs: args }; - } -} - -/** - * Attempt to locate a usable sim-hid-bridge. Returns a ready-to-use backend - * or `null` if the helper is not installed on this machine. Callers are - * expected to fall through to another tier in that case. - * - * Lookup order: - * 1. Compiled binary at `dist/sim-hid-bridge` (next to `dist/ax-bridge`). - * 2. Swift source at `dist/sim-hid-bridge.swift` (post-build copy). - * 3. Source tree fallback at `src/native/sim-hid-bridge.swift` — DEV ONLY, - * gated behind `OPENSAFARI_ALLOW_SWIFT_INTERPRETER=1`. The repo-relative - * path escapes `dist/` when the package is installed as a dependency, - * and executing unsigned Swift source via the interpreter sidesteps any - * future codesigning we add to the compiled binary, so this candidate - * is intentionally NOT auto-discovered in production installs. - */ -export async function tryCreateSimulatorKitHIDBackend(): Promise< - SimulatorKitHIDInputBackend | null -> { - const candidates = [ - // Compiled binary co-located with ax-bridge after build. - path.resolve(__dirname, '..', 'sim-hid-bridge'), - path.resolve(__dirname, 'sim-hid-bridge'), - // Swift source copied into dist/ by the postbuild step. - path.resolve(__dirname, '..', 'sim-hid-bridge.swift'), - path.resolve(__dirname, 'sim-hid-bridge.swift'), - ]; - if (process.env.OPENSAFARI_ALLOW_SWIFT_INTERPRETER === '1') { - candidates.push( - path.resolve(__dirname, '..', '..', 'src', 'native', 'sim-hid-bridge.swift'), - ); - } - for (const candidate of candidates) { - if (existsSync(candidate)) { - return new SimulatorKitHIDInputBackend(candidate); - } - } - const searched = candidates.map((c) => ` - ${c}`).join('\n'); - throw new InputBackendError( - `sim-hid-bridge not found. Searched:\n${searched}\n` + - 'Run npm run build or set OPENSAFARI_ALLOW_SWIFT_INTERPRETER=1 for dev mode.', - 'HID_BRIDGE_MISSING', - ); -} +export { + resetSimHidPrivateAPIWarning, + InputBackendError, + SimulatorKitHIDInputBackend, + tryCreateSimulatorKitHIDBackend, +} from '../input/sim-hid-backend'; -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} +export type { InputBackendErrorCode } from '../input/sim-hid-backend';