fix(ios-runner): dispose a busy runner on close so open boots clean (#2552) (#2605)

* fix(ios-runner): dispose a busy runner on close so open boots clean (#2552)

A runner with watchdog-abandoned main-thread work refuses every command until it drains or escalates to RUNNER_WEDGED. Plain close retained it and the next open reused it, so close recovered nothing. The runner now stamps its live main-thread occupancy on every successful response; the daemon mirrors it on the session and, at the close retention decision, disposes a runner reported busy instead of pooling it back. Killing the process is the only way to abort uncancellable XCTest work.

* fix(ios-runner): tag the watchdog timeout, drain-clear busy, stop busy at close (#2552)

Address review on #2552:
- Runner tags the execution-watchdog timeout with the typed MAIN_THREAD_TIMEOUT code so the
  stalling command itself reports main-thread occupancy, not only a later RUNNER_BUSY refusal.
- Host mirrors occupancy: set on RUNNER_BUSY or MAIN_THREAD_TIMEOUT, cleared by any other served
  reply (which reached the main thread and drained), left intact by transport failures.
- Close that would retain a runner first stops it when occupied, awaited and lease-released via
  the new stopRunnerSessionIfBusy seam, before the retain-vs-stop decision.
- Cover the production close route with lifecycle finalize tests and the Swift wire-code mapping.

* refactor(ios-runner): own the close release decision in the runner module (#2552)

Collapse the retain-vs-stop close decision into the module that owns the occupancy bit.
Close now calls one intent, releaseRunnerOnClose(deviceId, { retain }), instead of asking
stopRunnerSessionIfBusy for a boolean and re-branching in lifecycle. The runner module keeps
warm reuse for an idle runner and stops one still draining, so the busy fact never crosses the
package boundary, the fire-and-forget scheduling path is gone, and the close route stays awaited.

* test(ios-runner): chain close release from a real MAIN_THREAD_TIMEOUT, pin daemon close by call arguments (#2552)

The release-on-close test now trips MAIN_THREAD_TIMEOUT through executeRunnerCommandWithSession instead
of setting runnerMainThreadBusy by hand, so it fails if the catch-side busy marking stops reaching close.

The daemon mocks for releaseIosRunnerOnClose no longer restate the retain policy; the close tests assert
the { retain } argument instead. The unrelated session-test-harness un-exports are reverted.

Author: thymikee

.fallowrc.json

@@ -95,6 +95,7 @@
       "exports": [
         "detachIosSimulatorRunnerSessionsForShutdown",
         "hasLiveIosRunnerSession",
+        "releaseIosRunnerOnClose",
         "releaseSpeculativeIosRunnerSessionFor",
         "stopAllIosRunnerSessions"
       ]
@@ -160,7 +161,6 @@
         "resolveRunnerAppBundleId",
         "detachIosSimulatorRunnerSessionsForShutdown",
         "getRunnerSessionSnapshot",
-        "scheduleIosRunnerIdleStop",
         "stopIosRunnerSession",
         "stopAllIosRunnerSessions",
         "readStaleRunnerLease",

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -999,6 +999,14 @@ extension RunnerTests {
     case idle
     case busy(abandonedForSeconds: TimeInterval)
     case wedged(abandonedForSeconds: TimeInterval)
+
+    /// Whether the main thread is occupied by watchdog-abandoned work, for the occupancy stamp that
+    /// every successful response carries. Wedged is still occupied: it only differs in that a
+    /// restart, not waiting, is the cure.
+    var reportsMainThreadBusy: Bool {
+      if case .idle = self { return false }
+      return true
+    }
   }
 
   func currentMainThreadBusyState() -> MainThreadBusyState {

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift

@@ -183,6 +183,52 @@ extension RunnerTests {
     XCTAssertEqual(stamped.error?.message, "boom")
   }
 
+  func testStampingCurrentMainThreadBusyPreservesPayload() {
+    let stamped = Response(ok: true, data: DataPayload(nodes: [], truncated: false))
+      .stampingCurrentMainThreadBusy(true)
+
+    XCTAssertEqual(stamped.ok, true)
+    XCTAssertEqual(stamped.data?.runnerMainThreadBusy, true)
+  }
+
+  func testStampingCurrentMainThreadBusySkipsErrorResponses() {
+    let response = Response(ok: false, error: ErrorPayload(code: "RUNNER_BUSY", message: "busy"))
+    let stamped = response.stampingCurrentMainThreadBusy(true)
+
+    XCTAssertEqual(stamped.ok, false)
+    XCTAssertNil(stamped.data)
+    XCTAssertEqual(stamped.error?.code, "RUNNER_BUSY")
+  }
+
+  func testMainThreadBusyStateReportsOccupancy() {
+    XCTAssertFalse(MainThreadBusyState.idle.reportsMainThreadBusy)
+    XCTAssertTrue(MainThreadBusyState.busy(abandonedForSeconds: 5).reportsMainThreadBusy)
+    XCTAssertTrue(MainThreadBusyState.wedged(abandonedForSeconds: 200).reportsMainThreadBusy)
+    XCTAssertEqual(
+      Response(ok: true).stampingCurrentMainThreadBusy(false).data?.runnerMainThreadBusy, false)
+  }
+
+  func testCommandFailedResponseTagsMainThreadTimeoutWithTypedCode() {
+    let timeout = NSError(
+      domain: RunnerErrorDomain.general,
+      code: RunnerErrorCode.mainThreadExecutionTimedOut,
+      userInfo: [NSLocalizedDescriptionKey: "main thread execution timed out"]
+    )
+
+    let response = commandFailedResponse(from: timeout)
+
+    XCTAssertEqual(response.ok, false)
+    XCTAssertEqual(response.error?.code, RunnerWireErrorCode.mainThreadTimeout)
+  }
+
+  func testCommandFailedResponseKeepsGenericCodeForOtherErrors() {
+    let other = NSError(domain: "SomeOtherDomain", code: 99, userInfo: nil)
+
+    let response = commandFailedResponse(from: other)
+
+    XCTAssertEqual(response.error?.code, "COMMAND_FAILED")
+  }
+
   func testJournalStoredResponseStaysUnstamped() throws {
     let journal = RunnerCommandJournal()
     let recordStart = runnerJournalCommand("recordStart", id: "record-start-anchor")

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift

@@ -229,6 +229,16 @@ extension Response {
     payload.currentUptimeMs = value
     return Response(ok: ok, data: payload, error: error)
   }
+
+  // The daemon reads this occupancy flag to decide whether a healthy response proves the runner
+  // drained its watchdog-abandoned main-thread work. Only successful responses carry it; a refusal
+  // is itself the busy signal and needs no stamp.
+  func stampingCurrentMainThreadBusy(_ value: Bool) -> Response {
+    guard ok else { return self }
+    var payload = data ?? DataPayload()
+    payload.runnerMainThreadBusy = value
+    return Response(ok: ok, data: payload, error: error)
+  }
 }
 
 struct DataPayload: Codable {
@@ -276,6 +286,11 @@ struct DataPayload: Codable {
   var textEntryRoute: String?
   var runnerFatal: Bool?
   var runnerFatalReason: String?
+  /// Whether main-thread XCTest work past the execution watchdog is still draining when this
+  /// response is written. A private-AX snapshot can be served successfully while an abandoned tree
+  /// crawl still grinds, so the healthy response must carry the live occupancy rather than let the
+  /// daemon read `ok` as proof the runner drained (#2552).
+  var runnerMainThreadBusy: Bool?
   var completedSteps: Int?
   var failedStepIndex: Int?
   var sequenceResults: [SequenceStepResult]?

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift

@@ -152,14 +152,7 @@ extension RunnerTests {
           self.deliverCommandResult(
             command: command,
             result: (
-              self.jsonResponse(
-                status: 500,
-                response: self.errorResponse(
-                  code: "COMMAND_FAILED",
-                  message: error.localizedDescription,
-                  hint: "Check the runner log for XCTest details, then retry after the app is foregrounded if this was a timeout or activation failure."
-                )
-              ),
+              self.jsonResponse(status: 500, response: self.commandFailedResponse(from: error)),
               false
             ),
             completion: completion
@@ -274,7 +267,9 @@ extension RunnerTests {
     // rather than pairing a stale uptime with a much-later receipt time.
     let stamped =
       response.ok
-      ? response.stampingCurrentUptimeMs(ProcessInfo.processInfo.systemUptime * 1000)
+      ? response
+        .stampingCurrentUptimeMs(ProcessInfo.processInfo.systemUptime * 1000)
+        .stampingCurrentMainThreadBusy(currentMainThreadBusyState().reportsMainThreadBusy)
       : response
     let encoder = JSONEncoder()
     let body = (try? encoder.encode(stamped)).flatMap { String(data: $0, encoding: .utf8) } ?? "{}"
@@ -285,6 +280,32 @@ extension RunnerTests {
     Response(ok: false, error: ErrorPayload(code: code, message: message, hint: hint))
   }
 
+  /// Turns a thrown command error into its wire response. The execution-watchdog timeout keeps its
+  /// own typed code so the daemon records the runner as main-thread-occupied from the stalling
+  /// command itself, not only from a later `RUNNER_BUSY` refusal (#2552); every other throw stays the
+  /// generic `COMMAND_FAILED`.
+  func commandFailedResponse(from error: Error) -> Response {
+    let nsError = error as NSError
+    if nsError.domain == RunnerErrorDomain.general,
+      nsError.code == RunnerErrorCode.mainThreadExecutionTimedOut
+    {
+      return Response(
+        ok: false,
+        error: ErrorPayload(
+          code: RunnerWireErrorCode.mainThreadTimeout,
+          message: nsError.localizedDescription,
+          hint:
+            "The runner abandoned this command's main-thread work past its execution watchdog and it is still draining. Wait and retry, or use a screenshot and interact by coordinates."
+        )
+      )
+    }
+    return errorResponse(
+      code: "COMMAND_FAILED",
+      message: error.localizedDescription,
+      hint: "Check the runner log for XCTest details, then retry after the app is foregrounded if this was a timeout or activation failure."
+    )
+  }
+
   private func httpResponse(status: Int, body: String) -> Data {
     let headers = [
       "HTTP/1.1 \(status) OK",

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift

@@ -28,6 +28,13 @@ final class RunnerTests: XCTestCase {
     static let objcException = 1
   }
 
+  /// String codes the daemon keys behavior on. `RUNNER_BUSY` and `RUNNER_WEDGED` come from the busy
+  /// gate; `MAIN_THREAD_TIMEOUT` is emitted by the transport when a command trips the execution
+  /// watchdog, so the daemon can tell "the main thread is now occupied" from a generic failure.
+  enum RunnerWireErrorCode {
+    static let mainThreadTimeout = "MAIN_THREAD_TIMEOUT"
+  }
+
   static let springboardBundleId = "com.apple.springboard"
   // SpringBoard hosts blocking system modals on iOS/visionOS; tvOS (PineBoard/HeadBoard)
   // and macOS have no such host, so there is nothing to probe there.

docs/adr/0005-ios-runner-interaction-lifecycle.md

@@ -56,6 +56,22 @@ whether the runner is alive and accepting new HTTP requests.
 Dead cached runner processes are invalidated without graceful `shutdown`. A process that already
 stopped cannot answer the shutdown request, so graceful cleanup only adds stale-listener delay.
 
+The runner stamps its live main-thread occupancy (`runnerMainThreadBusy`) onto every successful
+response and tags the execution-watchdog timeout with the typed `MAIN_THREAD_TIMEOUT` code, so the
+daemon learns the main thread is occupied from the stalling command itself, not only from a later
+`RUNNER_BUSY` refusal. The daemon mirrors that occupancy on the `RunnerSession`: set on `RUNNER_BUSY`
+or `MAIN_THREAD_TIMEOUT`, cleared by any other served runner reply (which reached the main thread and
+therefore drained), and left intact by a transport failure or an unstamped recovered response. A
+healthy `ok` is never read as proof of drain because a private-AX snapshot can be served while an
+abandoned tree crawl still grinds on the XCTest main thread (#2552).
+
+Close that would retain a runner for reuse first stops it when that occupancy is set, awaiting the
+stop so the lease is released before the next request, because a runner still finishing
+watchdog-abandoned work refuses every command until it drains or escalates to `RUNNER_WEDGED`; pooling
+it back to the next `open` hands the same stalled runner to the caller and `close` recovers nothing.
+Killing the process is the only way to abort uncancellable XCTest work. This is the `RUNNER_WEDGED`
+restart from #1105 applied at the close boundary rather than after the wedge threshold elapses.
+
 When XCTest reports a root accessibility snapshot failure such as `kAXErrorIllegalArgument`, the
 runner treats the cached app target as suspect. Interactive snapshots fail closed to a truncated
 root-only payload instead of issuing more flat fallback queries against the same broken tree, and
@@ -84,6 +100,10 @@ If xcodebuild still exits for another reason, the next command detects the stale
 process/liveness checks and avoids the old 15-second graceful-shutdown wait. The remaining latency is
 fresh xcodebuild runner startup, not a stale transport stall.
 
+A `close` then `open` on a runner still draining abandoned main-thread work now pays a fresh runner boot
+instead of inheriting the stalled one, so the wedge no longer survives the close/open cycle. Runners
+that were never reported busy keep the existing warm-reuse path unchanged.
+
 The daemon no longer models a generic "recent success" cache as a runner-health signal. A proven
 healthy mutating response for the same app — recorded only after the `runnerFatal` check and only
 for allowlisted interactions — is now a real end-to-end liveness proof (HTTP listener through to the

packages/contracts/src/application-lifecycle-runtime.ts

@@ -315,7 +315,13 @@ export type AppleApplicationTools = Readonly<{
     device: DeviceInfo,
     execution: Readonly<{ requestId?: string }>,
   ): Promise<boolean>;
-  scheduleRunnerIdleStop(deviceId: string): void;
+  /**
+   * Releases this device's runner at session close. When `retain` is set and the runner is idle it
+   * keeps warm reuse under an idle-stop timer; otherwise it stops now. A runner whose last exchange
+   * reported main-thread work still draining is never retained, so a stalled process is not pooled
+   * back out to the next `open` (#2552). Awaited so `close` returns only once the lease is gone.
+   */
+  releaseRunnerOnClose(deviceId: string, options: Readonly<{ retain: boolean }>): Promise<void>;
   prepareRunner(
     device: DeviceInfo,
     input: PrepareAppleRunnerInput,

packages/platform-apple/src/core/runner-client.ts

@@ -37,8 +37,8 @@ export const detachIosSimulatorRunnerSessionsForShutdown: AppleRunnerClient['det
   client.detachIosSimulatorRunnerSessionsForShutdown;
 export const getRunnerSessionSnapshot: AppleRunnerClient['getRunnerSessionSnapshot'] =
   client.getRunnerSessionSnapshot;
-export const scheduleIosRunnerIdleStop: AppleRunnerClient['scheduleIosRunnerIdleStop'] =
-  client.scheduleIosRunnerIdleStop;
+export const releaseIosRunnerOnClose: AppleRunnerClient['releaseIosRunnerOnClose'] =
+  client.releaseIosRunnerOnClose;
 export const stopIosRunnerSession: AppleRunnerClient['stopIosRunnerSession'] =
   client.stopIosRunnerSession;
 export const stopAllIosRunnerSessions: AppleRunnerClient['stopAllIosRunnerSessions'] =

packages/platform-apple/src/lifecycle.test.ts

@@ -244,6 +244,56 @@ test('discards a retained physical iOS runner when relaunch fails and preserves
   expect(notifyRunnerAppRelaunched).not.toHaveBeenCalled();
 });
 
+test.each([true, false])(
+  'close finalization delegates the runner release to the runner module with retain=%s (#2552)',
+  async (retainRunner) => {
+    const signal = new AbortController().signal;
+    const baseHost = platformRuntimeHostFixture();
+    const releaseRunnerOnClose = vi.fn(async () => {});
+    const dismissCloseAlerts = vi.fn(async () => {});
+    const host = {
+      ...baseHost,
+      appleApplications: {
+        ...baseHost.appleApplications,
+        releaseRunnerOnClose,
+        dismissCloseAlerts,
+      },
+    } as unknown as PlatformRuntimeHost;
+    const lifecycle = bindAppleApplicationLifecycle({ host, device, signal });
+
+    await lifecycle.finalizeApplicationClose({ surface: 'app', retainRunner, stateDir: '/tmp' });
+
+    expect(releaseRunnerOnClose).toHaveBeenCalledWith(device.id, { retain: retainRunner });
+    expect(dismissCloseAlerts).toHaveBeenCalled();
+  },
+);
+
+test('daemon-shutdown finalization dismisses alerts and defers the runner release to the gateway (#2552)', async () => {
+  const signal = new AbortController().signal;
+  const baseHost = platformRuntimeHostFixture();
+  const releaseRunnerOnClose = vi.fn(async () => {});
+  const dismissCloseAlerts = vi.fn(async () => {});
+  const host = {
+    ...baseHost,
+    appleApplications: {
+      ...baseHost.appleApplications,
+      releaseRunnerOnClose,
+      dismissCloseAlerts,
+    },
+  } as unknown as PlatformRuntimeHost;
+  const lifecycle = bindAppleApplicationLifecycle({ host, device, signal });
+
+  await lifecycle.finalizeApplicationClose({
+    surface: 'app',
+    retainRunner: true,
+    stateDir: '/tmp',
+    daemonShutdown: true,
+  });
+
+  expect(releaseRunnerOnClose).not.toHaveBeenCalled();
+  expect(dismissCloseAlerts).toHaveBeenCalled();
+});
+
 test('prepare shares one startup budget across the Simulator boot and the runner preparation', async () => {
   vi.useFakeTimers();
   try {