From be787aece9bc112bb388fba31a2058ffcc273300 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 9 Oct 2026 00:15:31 +0200 Subject: [PATCH 01/20] fix(ios): stop a warm runner from silently rebooting a shut-down simulator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A runner retained after `close` owns its Simulator destination: when the Simulator is shut down externally, the retained xcodebuild's destination machinery silently reboots it and the device stays powered on until that runner dies (#3321). While a runner is warm-retained, hold one idle TCP connection to the runner's listener as a push signal. When an established watch connection closes during retention, read the device's boot identity once (bounded recheck, never a sample): a boot newer than the retention window proves Xcode rebooted the destination, so stop the retained runner — which powers the rebooted device back off — and record the typed `runner_destination_lost` reason the next `open` reports as a response warning. A same-boot close re-arms on the restarted generation; a never-established connection retries on a bounded backoff before being called `runner_unreachable`. Healthy warm reuse is untouched: killing a retained runner whose device was not rebooted leaves the device on (negative control), and the idle-stop policy still stands behind it. Mechanism evidence and per-claim measurements: PR body, issue #3321. --- .../platform-apple/src/core/runner-client.ts | 14 + .../platform-apple/src/core/runner-host.ts | 2 + .../src/runner-operations-facade.ts | 2 + packages/platform-apple/src/runner/host.ts | 10 + .../src/runner/runner-destination-watch.ts | 367 ++++++++++++++++++ .../src/runner/runner-session.ts | 70 +++- .../internal/session-open-execution.ts | 34 ++ src/platform-runtime-warm-runner-notice.ts | 24 ++ 8 files changed, 521 insertions(+), 2 deletions(-) create mode 100644 packages/platform-apple/src/runner/runner-destination-watch.ts create mode 100644 src/platform-runtime-warm-runner-notice.ts diff --git a/packages/platform-apple/src/core/runner-client.ts b/packages/platform-apple/src/core/runner-client.ts index 5aff792e8d..d43a39a3bd 100644 --- a/packages/platform-apple/src/core/runner-client.ts +++ b/packages/platform-apple/src/core/runner-client.ts @@ -34,6 +34,20 @@ export { stopAllIosRunnerSessions, stopIosRunnerSession, } from '../runner/runner-session.ts'; +import type { RunnerWarmLossNotice } from '../runner/runner-destination-watch.ts'; + +/** + * Takes the #3321 warm-runner loss notice recorded for this device, once, if a retained runner was + * stopped because its Simulator destination was replaced under it. Loaded on call, not at import: + * the watcher module must not join the façade closures the eager-closure budget holds flat, and + * the read happens once per `open` at most. + */ +export async function takeRunnerWarmLossNotice( + deviceId: string, +): Promise { + const { takeRunnerWarmLossNotice: take } = await import('../runner/runner-destination-watch.ts'); + return take(deviceId); +} export { hasCachedAppleRunnerArtifact, resolveRunnerAppBundleId, diff --git a/packages/platform-apple/src/core/runner-host.ts b/packages/platform-apple/src/core/runner-host.ts index 6790d89a7c..73ba0151b0 100644 --- a/packages/platform-apple/src/core/runner-host.ts +++ b/packages/platform-apple/src/core/runner-host.ts @@ -109,4 +109,6 @@ export const appleRunnerHost: AppleRunnerHost = { visitXmlPlistEntries, leaseOwnerStateDir: getRunnerLeaseOwnerStateDir, hasDeviceClaimAuthority: (device) => getRunnerDeviceClaimAuthorityProbe()?.(device) ?? false, + observeSimulatorBootTimeMs: async (device) => + (await import('../simulator-boot.ts')).observeSimulatorBootTimeMs(device), }; diff --git a/packages/platform-apple/src/runner-operations-facade.ts b/packages/platform-apple/src/runner-operations-facade.ts index 7c826e11ee..2eb60bb60a 100644 --- a/packages/platform-apple/src/runner-operations-facade.ts +++ b/packages/platform-apple/src/runner-operations-facade.ts @@ -26,9 +26,11 @@ export { runAppleRunnerCommand, stopAllIosRunnerSessions, stopIosRunnerSession, + takeRunnerWarmLossNotice, verifyLeaseRunnerPidIdentity, writeRunnerCacheMetadataForArtifacts, } from './core/runner-client.ts'; +export type { RunnerWarmLossNotice } from './runner/runner-destination-watch.ts'; export { queryAppleRunnerSelector } from './core/runner-selector-query.ts'; export async function cleanupRunnerLeasesForOwner( diff --git a/packages/platform-apple/src/runner/host.ts b/packages/platform-apple/src/runner/host.ts index 37be783617..29943382a3 100644 --- a/packages/platform-apple/src/runner/host.ts +++ b/packages/platform-apple/src/runner/host.ts @@ -16,6 +16,7 @@ import type { IosPhysicalDeviceRunnerControl } from '../core/physical-device-rou import type * as ApplePlistXml from '../core/plist-xml.ts'; import type * as AppleRunnerOwnerState from '../core/runner-owner-state.ts'; import type * as AppleSimctl from '../core/simctl.ts'; +import type * as AppleSimulatorBoot from '../simulator-boot.ts'; import type * as AppleToolProvider from '../core/tool-provider.ts'; /** @@ -103,6 +104,14 @@ export type AppleRunnerHost = Pick< * family/OS/id, never a bare id). Embedders without a claim store answer false. */ hasDeviceClaimAuthority: AppleRunnerOwnerState.RunnerDeviceClaimAuthorityProbe; + /** + * When this Simulator's current boot began, read from the host (`launchd_sim`). The warm-loss + * watcher classifies destination loss by device identity rather than port liveness, because + * Xcode restarts the runner app onto the same port after rebooting the destination (#3321). + * The production binding loads the observer on first call, not at binding time: that module + * sits outside the façade closures this port's eager budget holds flat. + */ + observeSimulatorBootTimeMs: typeof AppleSimulatorBoot.observeSimulatorBootTimeMs; }; /** The runner's deadline type is the host's read side; the {@link Deadline} shim below builds them. */ @@ -195,6 +204,7 @@ export const visitXmlPlistEntries = delegate('visitXmlPlistEntries'); export const resolveIosPhysicalDeviceControl = delegate('resolveIosPhysicalDeviceControl'); export const leaseOwnerStateDir = delegate('leaseOwnerStateDir'); export const hasDeviceClaimAuthority = delegate('hasDeviceClaimAuthority'); +export const observeSimulatorBootTimeMs = delegate('observeSimulatorBootTimeMs'); /** * Deadline keeps its root call-site shape (`Deadline.fromTimeoutMs(...)`); diff --git a/packages/platform-apple/src/runner/runner-destination-watch.ts b/packages/platform-apple/src/runner/runner-destination-watch.ts new file mode 100644 index 0000000000..68b5a151a0 --- /dev/null +++ b/packages/platform-apple/src/runner/runner-destination-watch.ts @@ -0,0 +1,367 @@ +import net from 'node:net'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { emitDiagnostic, isProcessAlive, observeSimulatorBootTimeMs } from './host.ts'; + +/** + * Why a runner retained after `close` was stopped before the session asked for it again. + * Keyed behavior: consumers branch on these values, never on prose. + */ +export type RunnerWarmLossReason = 'runner_destination_lost' | 'runner_unreachable'; + +/** + * What a retained runner's owner records when the watcher stops it: the typed reason, the session + * the runner served, and the moment the loss was proven. Read once by the next `open`. + */ +export type RunnerWarmLossNotice = Readonly<{ + reason: RunnerWarmLossReason; + deviceId: string; + sessionId: string; + atMs: number; +}>; + +/** + * A push watcher for a runner retained after `close` (#3321). + * + * A retained `xcodebuild test-without-building` owns its Simulator destination: when something + * outside agent-device shuts that Simulator down, the runner app dies (`ipc/mig server died`) and + * Xcode's destination machinery silently reboots the device to keep its own session alive, with + * nobody asking agent-device. The device stays powered on for as long as that runner process + * lives — potentially the whole idle-retention window. Killing the runner is what powers a + * rebooted device back off (measured: within ~2s of the kill on a device Xcode had rebooted), + * while killing a runner whose device was NOT rebooted leaves the device powered on — so stopping + * on loss cannot harm a healthy warm runner. + * + * The watcher holds ONE idle TCP connection to the runner's listener for the length of the + * retention window and writes nothing on it. The runner's listener serves many connections and + * never closes an idle one on its own (measured: alive past 180s while commands answered in + * single-digit milliseconds beside it), so a connection that closes AFTER it was established is + * the push signal that the runner's app generation ended — arriving within ~1s of an external + * shutdown. No timer samples a quiet runner: every read below happens once, at an event. + * + * "A connected socket closed" cannot by itself tell a reboot from a crash, because Xcode restarts + * the runner app onto the same port after rebooting the destination. The classification therefore + * reads device identity, not port liveness: a boot that began after the watch was armed, or no + * boot observable while the runner is still there to serve one, means the destination was replaced + * under the session and the retained runner must go. + * + * An attach that never connected (ECONNREFUSED) carries no information about WHY nothing was + * listening — the restart after a reboot lands on the same port within seconds — so it is retried + * on a bounded backoff instead of being classified; only an established connection that later + * closes is evidence of a runner-generation change. + */ + +/** How long an attach keeps retrying a refused port before the runner is called unreachable. */ +const ATTACH_RETRY_BUDGET_DEFAULT_MS = 15_000; +const ATTACH_RETRY_BASE_DELAY_MS = 1_000; + +/** How long a connected-socket close waits before the boot identity is read (see below). */ +const DESTINATION_CONFIRM_DEFAULT_MS = 2_500; +const DESTINATION_RECHECK_DEFAULT_MS = 1_000; + +type LossAction = 'stop' | 'rearm'; + +type LossVerdict = Readonly<{ + action: LossAction; + /** Present only when the loss is worth telling the next `open` about. */ + noticeReason?: RunnerWarmLossReason; +}>; + +type DestinationWatch = { + device: DeviceInfo; + sessionId: string; + port: number; + runnerPid: number | undefined; + /** When this retention window began; a boot starting later belongs to a replacement device state. */ + armedAtMs: number; + /** Re-read at every decision: false once any path has left the idle-retention window. */ + isArmed: () => boolean; + /** Stops the retained runner and releases its lease. */ + onStop: () => Promise; + socket: net.Socket; + /** True once this connection was established: only then can its close mean runner death. */ + connected: boolean; + detached: boolean; +}; + +const destinationWatches = new Map(); +const pendingWarmLossNotices = new Map(); + +/** + * How long a loss event waits before reading the device's boot identity. A shutdown is not over + * when the runner app dies: the old `launchd_sim` may still be visible for a moment, and Xcode's + * reboot starts within ~1s of the shutdown returning. One bounded re-read per event (never a + * sample of a quiet runner) lets that churn settle so a mid-shutdown read cannot misclassify a + * crash. A stop that lands after the reboot has finished still powers the device back off. + */ +function resolveConfirmDelayMs(env: NodeJS.ProcessEnv = process.env): number { + const raw = env.AGENT_DEVICE_IOS_RUNNER_DESTINATION_CONFIRM_MS?.trim(); + const parsed = raw ? Number(raw) : Number.NaN; + return Number.isFinite(parsed) && parsed >= 0 + ? Math.floor(parsed) + : DESTINATION_CONFIRM_DEFAULT_MS; +} + +function resolveRecheckDelayMs(env: NodeJS.ProcessEnv = process.env): number { + const raw = env.AGENT_DEVICE_IOS_RUNNER_DESTINATION_RECHECK_MS?.trim(); + const parsed = raw ? Number(raw) : Number.NaN; + return Number.isFinite(parsed) && parsed >= 0 + ? Math.floor(parsed) + : DESTINATION_RECHECK_DEFAULT_MS; +} + +function resolveAttachRetryBudgetMs(env: NodeJS.ProcessEnv = process.env): number { + const raw = env.AGENT_DEVICE_IOS_RUNNER_DESTINATION_ATTACH_RETRY_MS?.trim(); + const parsed = raw ? Number(raw) : Number.NaN; + return Number.isFinite(parsed) && parsed >= 0 + ? Math.floor(parsed) + : ATTACH_RETRY_BUDGET_DEFAULT_MS; +} + +function delay(ms: number): Promise { + return new Promise((resolve) => { + const timer = setTimeout(resolve, ms); + timer.unref?.(); + }); +} + +export type RunnerDestinationWatchParams = { + device: DeviceInfo; + sessionId: string; + port: number; + runnerPid: number | undefined; + isArmed: () => boolean; + onStop: () => Promise; + /** + * Remaining attach-retry budget for a refused port, carried across the re-attaches of one loss + * event so a runner that never answers is bounded; a fresh retention window starts a fresh + * budget. + */ + attachRetryBudgetMs?: number; +}; + +/** + * Attaches the watcher for one retention window. Idempotent per device: a fresh retention window + * re-arms the existing watch rather than stacking another. `isArmed` is consulted again at every + * decision, so an attach that lands after its retention window already ended arms nothing that + * can act. + */ +export function attachRunnerDestinationWatch(params: RunnerDestinationWatchParams): void { + const existing = destinationWatches.get(params.device.id); + if (existing && !existing.detached) { + existing.sessionId = params.sessionId; + existing.runnerPid = params.runnerPid; + existing.armedAtMs = Date.now(); + existing.isArmed = params.isArmed; + existing.onStop = params.onStop; + return; + } + const watch: DestinationWatch = { + device: params.device, + sessionId: params.sessionId, + port: params.port, + runnerPid: params.runnerPid, + armedAtMs: Date.now(), + isArmed: params.isArmed, + onStop: params.onStop, + socket: net.connect(params.port, '127.0.0.1'), + connected: false, + detached: false, + }; + watch.socket.unref(); + watch.socket.on('connect', () => { + watch.connected = true; + }); + watch.socket.on('close', () => { + void handleSocketClosed(watch, params.attachRetryBudgetMs); + }); + // `close` always follows an `error`; classification happens once, there. + watch.socket.on('error', () => {}); + destinationWatches.set(params.device.id, watch); +} + +/** + * Ends the watch without acting. Every path that leaves the idle-retention window calls it; even + * a missed call is inert, because `isArmed` re-reads the retention state at each decision. + */ +export function closeRunnerDestinationWatch(deviceId: string): void { + const watch = destinationWatches.get(deviceId); + if (watch) detachWatch(watch); +} + +function detachWatch(watch: DestinationWatch): void { + watch.detached = true; + destinationWatches.delete(watch.device.id); + watch.socket.removeAllListeners('connect'); + watch.socket.removeAllListeners('close'); + watch.socket.removeAllListeners('error'); + watch.socket.destroy(); +} + +async function handleSocketClosed( + watch: DestinationWatch, + attachRetryBudgetMs: number | undefined, +): Promise { + if (watch.detached) return; + if (!watch.isArmed()) { + detachWatch(watch); + return; + } + if (!watch.connected) { + await handleRefusedAttach(watch, attachRetryBudgetMs); + return; + } + await delay(resolveConfirmDelayMs()); + if (watch.detached || !watch.isArmed()) { + detachWatch(watch); + return; + } + const verdict = await classifyConnectedLoss(watch); + if (watch.detached || !watch.isArmed()) { + detachWatch(watch); + return; + } + if (verdict.action === 'rearm') { + rearmOnSameBoot(watch); + return; + } + stopRetainedRunner(watch, verdict.noticeReason); +} + +/** + * The attach never reached a listener. A rebooting destination is usually unreachable for a + * couple of seconds while Xcode restarts the app onto the same port, so retry with a growing + * delay inside a bounded budget. Only when the budget runs out is the retained runner called + * what it has proven to be: unable to answer anything. By then the destination process's own + * liveness decides whether this was a crash (Xcode gave up with it, the device lost its boot + * owner) or a listener that died while Xcode kept the device — the latter is exactly the silent + * power-on #3321 is about, so the runner goes with a notice. + */ +async function handleRefusedAttach( + watch: DestinationWatch, + attachRetryBudgetMs: number | undefined, +): Promise { + const budgetMs = attachRetryBudgetMs ?? resolveAttachRetryBudgetMs(); + if (budgetMs > 0) { + const nextDelayMs = Math.min(ATTACH_RETRY_BASE_DELAY_MS, budgetMs); + await delay(nextDelayMs); + if (watch.detached || !watch.isArmed()) { + detachWatch(watch); + return; + } + detachWatch(watch); + attachRunnerDestinationWatch({ + device: watch.device, + sessionId: watch.sessionId, + port: watch.port, + runnerPid: watch.runnerPid, + isArmed: watch.isArmed, + onStop: watch.onStop, + attachRetryBudgetMs: budgetMs - nextDelayMs, + }); + return; + } + if (watch.runnerPid !== undefined && !isProcessAlive(watch.runnerPid)) { + // Xcode itself gave up on this runner generation: the retention is over, the lease is stale, + // and nothing agent-device owns is powering the device any more. Plain cleanup, no notice. + stopRetainedRunner(watch, undefined); + return; + } + stopRetainedRunner(watch, 'runner_unreachable'); +} + +/** + * Reads whether the device behind the runner stopped being the device this session was armed on. + * A dead destination process answers for itself before the probe: Xcode cannot reboot from a dead + * process, so stopping is plain lease cleanup. A boot observed newer than the armed window is + * Xcode's reboot; a boot unobservable twice while the destination process is still alive is a + * device that is down with a reboot promise that may already be in flight. Both are destination + * loss, and stopping still pre-empts the reboot in the second case. + */ +async function classifyConnectedLoss(watch: DestinationWatch): Promise { + if (watch.runnerPid !== undefined && !isProcessAlive(watch.runnerPid)) { + return { action: 'stop' }; + } + const boot = await observeSimulatorBootTimeMs(watch.device); + if (boot.observed && boot.bootedAtMs <= watch.armedAtMs) { + return { action: 'rearm' }; + } + if (!boot.observed) { + await delay(resolveRecheckDelayMs()); + const recheck = await observeSimulatorBootTimeMs(watch.device); + if (recheck.observed && recheck.bootedAtMs <= watch.armedAtMs) { + return { action: 'rearm' }; + } + } + return { action: 'stop', noticeReason: 'runner_destination_lost' }; +} + +/** + * The runner's app generation died while the device kept this session's boot: a crash Xcode may + * restart onto the same port. Re-arm the push signal on the replacement. Each cycle waits on the + * new socket's own close event — an established close here is a real runner death, not a sample, + * and Xcode's restart budget bounds how many there can be; a refused port runs the bounded + * attach-retry above instead of spinning here. + */ +function rearmOnSameBoot(watch: DestinationWatch): void { + emitDiagnostic({ + level: 'info', + phase: 'ios_runner_destination_watch_rearmed', + data: { deviceId: watch.device.id, sessionId: watch.sessionId }, + }); + const { device, sessionId, port, runnerPid, isArmed, onStop } = watch; + detachWatch(watch); + if (!isArmed()) return; + attachRunnerDestinationWatch({ device, sessionId, port, runnerPid, isArmed, onStop }); +} + +/** + * Stops the retained runner, recording the notice the next `open` must carry before any notice + * consumer can race this path. A `undefined` reason is silent lease cleanup for a runner + * generation Xcode already buried. + */ +function stopRetainedRunner(watch: DestinationWatch, noticeReason: RunnerWarmLossReason | undefined): void { + detachWatch(watch); + if (noticeReason) { + pendingWarmLossNotices.set(watch.device.id, { + reason: noticeReason, + deviceId: watch.device.id, + sessionId: watch.sessionId, + atMs: Date.now(), + }); + } + emitDiagnostic({ + level: 'warn', + phase: 'ios_runner_warm_stop', + data: { + deviceId: watch.device.id, + sessionId: watch.sessionId, + reason: noticeReason ?? 'runner_process_gone', + }, + }); + void watch.onStop().catch((error: unknown) => { + emitDiagnostic({ + level: 'warn', + phase: 'ios_runner_warm_stop_failed', + data: { + deviceId: watch.device.id, + error: error instanceof Error ? error.message : String(error), + }, + }); + }); +} + +/** + * Takes the destination-loss notice recorded for this device, once. A notice exists only when a + * retained runner was stopped because the destination was replaced under it — the state the next + * `open` must explain rather than leave the caller to infer from a cold start. + */ +export function takeRunnerWarmLossNotice(deviceId: string): RunnerWarmLossNotice | undefined { + const notice = pendingWarmLossNotices.get(deviceId); + if (notice) pendingWarmLossNotices.delete(deviceId); + return notice; +} + +/** Test seam: whether a watcher currently holds a socket for the device. */ +export function hasRunnerDestinationWatch(deviceId: string): boolean { + return destinationWatches.has(deviceId); +} diff --git a/packages/platform-apple/src/runner/runner-session.ts b/packages/platform-apple/src/runner/runner-session.ts index 1200c5dc9f..39a530bbf0 100644 --- a/packages/platform-apple/src/runner/runner-session.ts +++ b/packages/platform-apple/src/runner/runner-session.ts @@ -7,7 +7,7 @@ import { buildSimctlArgsForDevice, runXcrun, } from './host.ts'; -import { isApplePlatform, type DeviceInfo } from '@agent-device/kernel/device'; +import { isApplePlatform, isIosFamily, type DeviceInfo } from '@agent-device/kernel/device'; import type { RunnerLogicalLeaseContext } from '@agent-device/contracts/runner-lease-context'; import type { AppleRunnerLifecycleOptions } from './runner-provider.ts'; import { flushRunnerLogAppends, getFreePort, resolveRunnerLaunchLogPath } from './runner-io.ts'; @@ -82,6 +82,13 @@ export type RunnerSessionOptions = AppleRunnerLifecycleOptions; const runnerSessions = new Map(); const runnerSessionLocks = new Map>(); const runnerIdleStopTimers = new Map(); +/** + * Devices whose runner a `close` retained for warm reuse, until the window ends. Owns the + * retention fact separately from the idle-stop timer: the timer is disabled when + * `AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS=0`, and the destination watcher's duty (#3321) covers the + * whole retain-until-daemon-exit behavior just the same. + */ +const runnerWarmRetainedDevices = new Set(); const RUNNER_RETAINED_IDLE_STOP_DEFAULT_MS = 5 * 60_000; const RUNNER_STALE_BUNDLE_UNINSTALL_TIMEOUT_MS = 10_000; @@ -652,12 +659,67 @@ export function scheduleIosRunnerIdleStop(deviceId: string): void { } export function cancelIosRunnerIdleStop(deviceId: string): void { + endRunnerWarmRetention(deviceId); const timer = runnerIdleStopTimers.get(deviceId); if (!timer) return; clearTimeout(timer); runnerIdleStopTimers.delete(deviceId); } +/** + * Ends this device's warm-retention window. The destination watcher's duty (#3321) covers exactly + * the time between a retaining `close` and the moment the runner is used or stopped again; after + * that point nothing may act on a stale watch. Retention always begins in code that has already + * loaded the watcher module, so the close here reads it from the module cache. + */ +function endRunnerWarmRetention(deviceId: string): void { + if (!runnerWarmRetainedDevices.delete(deviceId)) return; + void import('./runner-destination-watch.ts').then( + ({ closeRunnerDestinationWatch }) => closeRunnerDestinationWatch(deviceId), + () => {}, + ); +} + +/** + * Whether this session is the device's runner sitting in the post-`close` idle-retention window: + * retained by a close and nothing has used or stopped it since, still registered, and not being + * torn down. Any use (`ensureRunnerSession`) or teardown leaves the window, which is exactly when + * the destination watcher must stop acting without stopping anything itself. + */ +function isRunnerIdleRetained(deviceId: string, session: RunnerSession | undefined): boolean { + return ( + runnerWarmRetainedDevices.has(deviceId) && + session !== undefined && + runnerSessions.get(deviceId) === session && + canWorkWithRunnerSession(session) + ); +} + +/** + * Arms the #3321 push watcher over a runner this close retained. Reached through a function-scoped + * import: the watcher module is needed only at the start of a retention window and must not join + * the façade closures the eager-closure budget holds at merge-base size. + */ +async function armRunnerDestinationWatch(session: RunnerSession | undefined): Promise { + // Local Simulators only: the push signal is the runner's loopback listener, the reboot owner is + // Xcode's Simulator destination machinery, and `launchd_sim` is the boot witness. A retained + // physical-device runner has none of these and stays exactly as it was. + if (session?.device.kind !== 'simulator' || !isIosFamily(session.device)) return; + if (!isRunnerIdleRetained(session.deviceId, session)) return; + const { attachRunnerDestinationWatch } = await import('./runner-destination-watch.ts'); + const deviceId = session.deviceId; + attachRunnerDestinationWatch({ + device: session.device, + sessionId: session.sessionId, + port: session.port, + runnerPid: session.child.pid, + isArmed: () => isRunnerIdleRetained(deviceId, session), + onStop: async () => { + await stopIosRunnerSession(deviceId); + }, + }); +} + function resolveRunnerIdleStopMs(env: NodeJS.ProcessEnv = process.env): number { const raw = env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS?.trim(); if (raw) { @@ -743,8 +805,12 @@ export async function releaseIosRunnerOnClose( options: { retain: boolean }, ): Promise { const session = runnerSessions.get(deviceId); - if (options.retain && !isRunnerMainThreadOccupied(session)) { + if (options.retain && session && !isRunnerMainThreadOccupied(session)) { + // The flag is set after the scheduler because the scheduler's own cancel step ends any + // retention the previous window left open, and marking first would erase this one. scheduleIosRunnerIdleStop(deviceId); + runnerWarmRetainedDevices.add(deviceId); + await armRunnerDestinationWatch(session); return; } if (options.retain) { diff --git a/src/daemon/session-lifecycle/internal/session-open-execution.ts b/src/daemon/session-lifecycle/internal/session-open-execution.ts index 5764438cf0..97e1dac2b4 100644 --- a/src/daemon/session-lifecycle/internal/session-open-execution.ts +++ b/src/daemon/session-lifecycle/internal/session-open-execution.ts @@ -56,6 +56,7 @@ import { } from '../../device/device-claims.ts'; import type { TakenOverDeviceClaim } from '../../device/device-claim-reboot.ts'; import { deviceBootObservation } from '../../../platform-runtime-device-boot.ts'; +import type { RunnerWarmLossNotice } from '../../../platform-runtime-warm-runner-notice.ts'; import { appendResponseWarning } from './session-open-warnings.ts'; import { buildAllocatorHeldRefusal, @@ -121,6 +122,35 @@ function deviceClaimTakeoverWarning(tookOver: TakenOverDeviceClaim): string { ); } +/** + * Reads and consumes the warm-runner loss notice #3321's destination watcher recorded for this + * device. Reached through the root's platform-runtime seam by function-scoped import: the notice + * exists only on iOS simulators, and only when a retained runner was stopped, so the common open + * must not pay for the platform module's evaluation. + */ +async function readWarmRunnerLossNotice( + device: DeviceInfo, +): Promise { + const { takeWarmRunnerLossNotice } = await import( + '../../../platform-runtime-warm-runner-notice.ts' + ); + return await takeWarmRunnerLossNotice(device); +} + +/** + * What the caller's `open` output says when the runner this session left warm after `close` had to + * be stopped because the Simulator was shut down externally while it was retained (#3321). Names + * the typed reason so automation can key on it without parsing prose. + */ +function warmRunnerLossWarning(notice: RunnerWarmLossNotice): string { + return ( + 'The warm iOS runner left by the previous close was stopped: the simulator was shut down ' + + 'externally while it was retained, and the retained runner had begun rebooting it. ' + + 'This open started a fresh runner and left the simulator booted from now on. ' + + `reason=${notice.reason}` + ); +} + // Default-on for emulators, opt-in via --test-ime on real devices; --no-test-ime forces off. function shouldActivateAndroidTestIme(device: DeviceInfo, req: DaemonRequest): boolean { if (device.platform !== 'android') return false; @@ -310,6 +340,10 @@ export async function completeOpenCommand(params: { if (tookOverDeviceClaim) { appendResponseWarning(openResult, deviceClaimTakeoverWarning(tookOverDeviceClaim)); } + const warmLossNotice = await readWarmRunnerLossNotice(device); + if (warmLossNotice) { + appendResponseWarning(openResult, warmRunnerLossWarning(warmLossNotice)); + } const nextRef = publishOpenSession({ req, sessionStore, diff --git a/src/platform-runtime-warm-runner-notice.ts b/src/platform-runtime-warm-runner-notice.ts new file mode 100644 index 0000000000..689617f193 --- /dev/null +++ b/src/platform-runtime-warm-runner-notice.ts @@ -0,0 +1,24 @@ +import type { RunnerWarmLossNotice } from '@agent-device/platform-apple/runner/operations'; +import { isIosFamily, type DeviceInfo } from '@agent-device/kernel/device'; + +export type { RunnerWarmLossNotice }; + +/** + * Takes the warm-runner loss notice recorded for this device by the Apple runner's destination + * watcher, once, if there is one (#3321). A notice exists only when a runner retained after + * `close` had to be stopped because something shut the Simulator down under it and the retained + * Xcode destination had begun rebooting it — the state the next `open` reports rather than leaves + * the caller to infer from a cold start. Families that retain no warm runner record no notice. + * + * The notice is read through the platform's public operations surface from this root module: the + * daemon must not import a concrete platform package (R65), and the notice is a runner fact, not a + * session fact — it exists precisely in the gap where no session is live. + */ +export async function takeWarmRunnerLossNotice( + device: DeviceInfo, +): Promise { + if (!isIosFamily(device) || device.kind !== 'simulator') return undefined; + const { takeRunnerWarmLossNotice } = + await import('@agent-device/platform-apple/runner/operations'); + return takeRunnerWarmLossNotice(device.id); +} From 4315ff96f52da206dca708867e9c16123552ac42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 9 Oct 2026 21:02:31 +0200 Subject: [PATCH 02/20] test(ios): cover the warm-runner destination watch (#3321) Co-Authored-By: Claude Sonnet 5.5 --- .../runner-destination-watch.test.ts | 266 ++++++++++++++++++ .../src/runner/runner-destination-watch.ts | 10 +- .../internal/session-open-execution.ts | 5 +- 3 files changed, 272 insertions(+), 9 deletions(-) create mode 100644 packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts diff --git a/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts b/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts new file mode 100644 index 0000000000..c5d44de41c --- /dev/null +++ b/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts @@ -0,0 +1,266 @@ +import { afterEach, beforeEach, test, vi } from 'vitest'; +import assert from 'node:assert/strict'; +import net from 'node:net'; +import { appleRunnerTestHost } from '../test-host.ts'; +import { + attachRunnerDestinationWatch, + closeRunnerDestinationWatch, + takeRunnerWarmLossNotice, +} from '../runner-destination-watch.ts'; +import { IOS_SIMULATOR } from './device-fixtures.ts'; + +const DEVICE = IOS_SIMULATOR; +const RUNNER_PID = 4242; + +type FakeListener = { + port: number; + connections: net.Socket[]; + nextConnection(): Promise; + close(): Promise; +}; + +const listeners: FakeListener[] = []; + +async function listen(): Promise { + const connections: net.Socket[] = []; + const waiting: Array<(socket: net.Socket) => void> = []; + const server = net.createServer((socket) => { + socket.on('error', () => {}); + const waiter = waiting.shift(); + if (waiter) waiter(socket); + else connections.push(socket); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const listener: FakeListener = { + port: (server.address() as net.AddressInfo).port, + connections, + nextConnection: () => { + const queued = connections.shift(); + return queued + ? Promise.resolve(queued) + : new Promise((resolve) => waiting.push(resolve)); + }, + close: () => + new Promise((resolve) => { + for (const socket of connections) socket.destroy(); + server.close(() => resolve()); + }), + }; + listeners.push(listener); + return listener; +} + +async function unusedPort(): Promise { + const probe = await listen(); + const { port } = probe; + await probe.close(); + return port; +} + +function watchParams(port: number, overrides: { isArmed?: () => boolean } = {}) { + const onStop = vi.fn(async () => {}); + return { + onStop, + params: { + device: DEVICE, + sessionId: 'session-1', + port, + runnerPid: RUNNER_PID, + isArmed: overrides.isArmed ?? (() => true), + onStop, + }, + }; +} + +function stubHost(options: { + boots: Array<{ observed: true; bootedAtMs: number } | { observed: false; reason: 'unobserved' }>; + processAlive?: boolean; +}) { + const observeSimulatorBootTimeMs = vi.fn(async () => { + const next = options.boots.length > 1 ? options.boots.shift() : options.boots[0]; + return next ?? { observed: false as const, reason: 'unobserved' as const }; + }); + appleRunnerTestHost.update({ + observeSimulatorBootTimeMs, + isProcessAlive: () => options.processAlive ?? true, + }); + return observeSimulatorBootTimeMs; +} + +beforeEach(() => { + vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_DESTINATION_CONFIRM_MS', '5'); + vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_DESTINATION_RECHECK_MS', '5'); + vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_DESTINATION_ATTACH_RETRY_MS', '0'); +}); + +afterEach(async () => { + closeRunnerDestinationWatch(DEVICE.id); + takeRunnerWarmLossNotice(DEVICE.id); + vi.unstubAllEnvs(); + await Promise.all(listeners.splice(0).map((listener) => listener.close())); +}); + +test('an established connection that closes after a newer boot stops the runner with a notice', async () => { + const listener = await listen(); + stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() + 60_000 }] }); + const { params, onStop } = watchParams(listener.port); + + attachRunnerDestinationWatch(params); + (await listener.nextConnection()).destroy(); + await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); + + const notice = takeRunnerWarmLossNotice(DEVICE.id); + assert.equal(notice?.reason, 'runner_destination_lost'); + assert.equal(notice?.sessionId, 'session-1'); + assert.equal(takeRunnerWarmLossNotice(DEVICE.id), undefined); +}); + +test('a close on the boot the watch was armed on re-arms instead of stopping', async () => { + const listener = await listen(); + const observe = stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() - 60_000 }] }); + const { params, onStop } = watchParams(listener.port); + + attachRunnerDestinationWatch(params); + (await listener.nextConnection()).destroy(); + const replacement = await listener.nextConnection(); + + assert.equal(observe.mock.calls.length, 1); + assert.equal(onStop.mock.calls.length, 0); + assert.equal(takeRunnerWarmLossNotice(DEVICE.id), undefined); + replacement.destroy(); +}); + +test('a boot unobservable on both reads while the destination process lives is a loss', async () => { + const listener = await listen(); + const observe = stubHost({ boots: [{ observed: false, reason: 'unobserved' }] }); + const { params, onStop } = watchParams(listener.port); + + attachRunnerDestinationWatch(params); + (await listener.nextConnection()).destroy(); + await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); + + assert.equal(observe.mock.calls.length, 2); + assert.equal(takeRunnerWarmLossNotice(DEVICE.id)?.reason, 'runner_destination_lost'); +}); + +test('a boot that reappears on the recheck with the armed boot re-arms', async () => { + const listener = await listen(); + stubHost({ + boots: [ + { observed: false, reason: 'unobserved' }, + { observed: true, bootedAtMs: Date.now() - 60_000 }, + ], + }); + const { params, onStop } = watchParams(listener.port); + + attachRunnerDestinationWatch(params); + (await listener.nextConnection()).destroy(); + const replacement = await listener.nextConnection(); + + assert.equal(onStop.mock.calls.length, 0); + replacement.destroy(); +}); + +test('a dead runner process is plain cleanup without a notice', async () => { + const listener = await listen(); + const observe = stubHost({ boots: [{ observed: true, bootedAtMs: 0 }], processAlive: false }); + const { params, onStop } = watchParams(listener.port); + + attachRunnerDestinationWatch(params); + (await listener.nextConnection()).destroy(); + await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); + + assert.equal(observe.mock.calls.length, 0); + assert.equal(takeRunnerWarmLossNotice(DEVICE.id), undefined); +}); + +test('a port that never answers is runner_unreachable once the retry budget is spent', async () => { + stubHost({ boots: [{ observed: true, bootedAtMs: 0 }] }); + const { params, onStop } = watchParams(await unusedPort()); + + attachRunnerDestinationWatch(params); + await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); + + assert.equal(takeRunnerWarmLossNotice(DEVICE.id)?.reason, 'runner_unreachable'); +}); + +test('a refused attach whose runner process is gone is silent cleanup', async () => { + stubHost({ boots: [{ observed: true, bootedAtMs: 0 }], processAlive: false }); + const { params, onStop } = watchParams(await unusedPort()); + + attachRunnerDestinationWatch(params); + await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); + + assert.equal(takeRunnerWarmLossNotice(DEVICE.id), undefined); +}); + +test('a refused attach retries inside its budget and watches the runner once it listens', async () => { + vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_DESTINATION_ATTACH_RETRY_MS', '5000'); + const port = await unusedPort(); + stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() + 60_000 }] }); + const { params, onStop } = watchParams(port); + + attachRunnerDestinationWatch(params); + await new Promise((resolve) => setTimeout(resolve, 50)); + assert.equal(onStop.mock.calls.length, 0); + + const server = net.createServer((socket) => { + socket.on('error', () => {}); + socket.destroy(); + }); + await new Promise((resolve) => server.listen(port, '127.0.0.1', resolve)); + try { + await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1), { timeout: 4000 }); + assert.equal(takeRunnerWarmLossNotice(DEVICE.id)?.reason, 'runner_destination_lost'); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + } +}); + +test('a close after the retention window ended acts on nothing', async () => { + const listener = await listen(); + const observe = stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() + 60_000 }] }); + let armed = true; + const { params, onStop } = watchParams(listener.port, { isArmed: () => armed }); + + attachRunnerDestinationWatch(params); + const socket = await listener.nextConnection(); + armed = false; + socket.destroy(); + await new Promise((resolve) => setTimeout(resolve, 60)); + + assert.equal(observe.mock.calls.length, 0); + assert.equal(onStop.mock.calls.length, 0); + assert.equal(takeRunnerWarmLossNotice(DEVICE.id), undefined); +}); + +test('closing the watch detaches it so a later runner death stops nothing', async () => { + const listener = await listen(); + const observe = stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() + 60_000 }] }); + const { params, onStop } = watchParams(listener.port); + + attachRunnerDestinationWatch(params); + const socket = await listener.nextConnection(); + closeRunnerDestinationWatch(DEVICE.id); + socket.destroy(); + await new Promise((resolve) => setTimeout(resolve, 60)); + + assert.equal(observe.mock.calls.length, 0); + assert.equal(onStop.mock.calls.length, 0); +}); + +test('a second retention window re-arms the existing watch instead of stacking another', async () => { + const listener = await listen(); + stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() + 60_000 }] }); + const first = watchParams(listener.port); + const second = watchParams(listener.port); + + attachRunnerDestinationWatch(first.params); + const socket = await listener.nextConnection(); + attachRunnerDestinationWatch(second.params); + socket.destroy(); + await vi.waitFor(() => assert.equal(second.onStop.mock.calls.length, 1)); + + assert.equal(first.onStop.mock.calls.length, 0); + assert.equal(listener.connections.length, 0); +}); diff --git a/packages/platform-apple/src/runner/runner-destination-watch.ts b/packages/platform-apple/src/runner/runner-destination-watch.ts index 68b5a151a0..2d22f29a85 100644 --- a/packages/platform-apple/src/runner/runner-destination-watch.ts +++ b/packages/platform-apple/src/runner/runner-destination-watch.ts @@ -319,7 +319,10 @@ function rearmOnSameBoot(watch: DestinationWatch): void { * consumer can race this path. A `undefined` reason is silent lease cleanup for a runner * generation Xcode already buried. */ -function stopRetainedRunner(watch: DestinationWatch, noticeReason: RunnerWarmLossReason | undefined): void { +function stopRetainedRunner( + watch: DestinationWatch, + noticeReason: RunnerWarmLossReason | undefined, +): void { detachWatch(watch); if (noticeReason) { pendingWarmLossNotices.set(watch.device.id, { @@ -360,8 +363,3 @@ export function takeRunnerWarmLossNotice(deviceId: string): RunnerWarmLossNotice if (notice) pendingWarmLossNotices.delete(deviceId); return notice; } - -/** Test seam: whether a watcher currently holds a socket for the device. */ -export function hasRunnerDestinationWatch(deviceId: string): boolean { - return destinationWatches.has(deviceId); -} diff --git a/src/daemon/session-lifecycle/internal/session-open-execution.ts b/src/daemon/session-lifecycle/internal/session-open-execution.ts index 97e1dac2b4..2b5014e133 100644 --- a/src/daemon/session-lifecycle/internal/session-open-execution.ts +++ b/src/daemon/session-lifecycle/internal/session-open-execution.ts @@ -131,9 +131,8 @@ function deviceClaimTakeoverWarning(tookOver: TakenOverDeviceClaim): string { async function readWarmRunnerLossNotice( device: DeviceInfo, ): Promise { - const { takeWarmRunnerLossNotice } = await import( - '../../../platform-runtime-warm-runner-notice.ts' - ); + const { takeWarmRunnerLossNotice } = + await import('../../../platform-runtime-warm-runner-notice.ts'); return await takeWarmRunnerLossNotice(device); } From 9f58f7f6385decfb4b5991b6004f590ec8259499 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 9 Oct 2026 21:11:57 +0200 Subject: [PATCH 03/20] chore(gates): classify the warm-runner notice module and edge (#3321) Co-Authored-By: Claude Sonnet 5.5 --- .../daemon-platform-runtime-inventory.ts | 10 ++++ scripts/layering/root-module-zones.ts | 1 + .../request-router-open-claim.test.ts | 46 +++++++++++++++++++ website/docs/docs/sessions.md | 4 ++ 4 files changed, 61 insertions(+) diff --git a/scripts/layering/daemon-platform-runtime-inventory.ts b/scripts/layering/daemon-platform-runtime-inventory.ts index 67cc9e1be5..33077ffde0 100644 --- a/scripts/layering/daemon-platform-runtime-inventory.ts +++ b/scripts/layering/daemon-platform-runtime-inventory.ts @@ -191,6 +191,16 @@ export const DAEMON_PLATFORM_RUNTIME_EDGES: readonly DaemonPlatformRuntimeEdge[] 'foreign claim can still describe live ownership; the per-family probe mechanics stay in the ' + 'Apple and Android packages behind the neutral observation contract (#2538).', }, + { + file: 'src/daemon/session-lifecycle/internal/session-open-execution.ts', + target: 'src/platform-runtime-warm-runner-notice.ts', + symbols: ['RunnerWarmLossNotice', 'takeWarmRunnerLossNotice'], + classification: 'daemon-policy-essential', + rationale: + 'daemon-owned open reports that a runner left warm by close was stopped because its ' + + 'simulator was shut down externally (#3321); the watcher and the typed reason stay ' + + 'Apple-owned behind the neutral notice read.', + }, { file: 'src/daemon/handlers/session-selector-dispatch.ts', target: 'src/platform-runtime-open-target.ts', diff --git a/scripts/layering/root-module-zones.ts b/scripts/layering/root-module-zones.ts index fdf1cf4d3b..e9bf6cfe2c 100644 --- a/scripts/layering/root-module-zones.ts +++ b/scripts/layering/root-module-zones.ts @@ -62,6 +62,7 @@ export const ROOT_MODULE_ZONES: Readonly> = { 'src/platform-runtime-managed-web-backend.ts', 'src/platform-runtime-open-target.ts', 'src/platform-runtime-resource-cleanup.ts', + 'src/platform-runtime-warm-runner-notice.ts', 'src/platform-runtime.ts', 'src/provider-credential-fingerprint.ts', 'src/provider-device-runtimes.ts', diff --git a/src/daemon/__tests__/request-router-open-claim.test.ts b/src/daemon/__tests__/request-router-open-claim.test.ts index eabd859560..b5ac0220c0 100644 --- a/src/daemon/__tests__/request-router-open-claim.test.ts +++ b/src/daemon/__tests__/request-router-open-claim.test.ts @@ -11,6 +11,9 @@ vi.mock('@agent-device/host-kit/process', async (importOriginal) => { const actual = await importOriginal(); return { ...actual, readProcessStartTime: vi.fn(() => 'test-process-start') }; }); +const mockTakeRunnerWarmLossNotice = vi.hoisted(() => + vi.fn(async (): Promise => undefined), +); vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { const actual = await importOriginal(); @@ -20,6 +23,7 @@ vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) notifyIosRunnerAppRelaunched: vi.fn(async () => {}), prewarmAppleRunnerCache: vi.fn(async () => {}), prewarmIosRunner: vi.fn(async () => {}), + prewarmIosRunnerSession: vi.fn(async () => {}), prepareIosRunner: vi.fn(async () => ({ runner: { currentUptimeMs: 42 }, connectMs: 0, @@ -28,6 +32,7 @@ vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) resolveRunnerAppBundleId: vi.fn(() => 'com.callstack.agentdevice.runner'), stopIosRunnerSession: vi.fn(async () => {}), stopAllIosRunnerSessions: vi.fn(async () => {}), + takeRunnerWarmLossNotice: mockTakeRunnerWarmLossNotice, }; }); vi.mock('@agent-device/platform-apple/app-lifecycle', async (importOriginal) => { @@ -67,6 +72,7 @@ import { import { ensureDeviceReady } from '../device/device-ready.ts'; import type { DeviceInfo } from '@agent-device/kernel/device'; import type { DeviceBootObservation } from '@agent-device/contracts/device-boot'; +import type { RunnerWarmLossNotice } from '@agent-device/platform-apple/runner/operations'; import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; import { inspectDeviceClaims } from '../device/device-claim-inspection.ts'; import { makeIosDevice, openRequest, storedClaimUpdatedAt } from './request-router-open-harness.ts'; @@ -98,6 +104,8 @@ beforeEach(() => { mockEnsureDeviceReady.mockResolvedValue(undefined); mockAwaitFixtureReadiness.mockReset(); mockAwaitFixtureReadiness.mockResolvedValue(undefined); + mockTakeRunnerWarmLossNotice.mockReset(); + mockTakeRunnerWarmLossNotice.mockResolvedValue(undefined); mockObserveSimulatorBoot.mockReset(); mockObserveSimulatorBoot.mockImplementation(async () => ({ observed: false, @@ -167,6 +175,44 @@ test('open takes a live foreign claim whose device rebooted after the claim was } }); +test('open warns with the typed reason when a warm runner was stopped for a lost destination', async () => { + const sessionStore = makeSessionStore('agent-device-router-open-warm-loss-'); + const device = makeIosDevice('SIM-WARM-LOSS'); + mockResolveTargetDevice.mockResolvedValue(device); + mockTakeRunnerWarmLossNotice.mockResolvedValueOnce({ + reason: 'runner_destination_lost', + deviceId: device.id, + sessionId: 'warm-loss', + atMs: Date.now(), + }); + const claimsDir = mkdtempForTestSync('agent-device-router-open-warm-loss-claims-'); + const previousClaimsDir = process.env.AGENT_DEVICE_CLAIMS_DIR; + process.env.AGENT_DEVICE_CLAIMS_DIR = claimsDir; + + try { + const opened = await createOpenHandler(sessionStore)( + openRequest('warm-loss', { platform: 'ios' }, 'req-open-warm-loss', {}, ['FixtureApp']), + ); + + expect(opened.ok).toBe(true); + if (!opened.ok) return; + expect(mockTakeRunnerWarmLossNotice).toHaveBeenCalledWith(device.id); + expect(opened.data?.warnings).toEqual([ + expect.stringContaining('reason=runner_destination_lost'), + ]); + + const reopened = await createOpenHandler(sessionStore)( + openRequest('warm-loss', { platform: 'ios' }, 'req-open-warm-loss-again', {}, ['FixtureApp']), + ); + expect(reopened.ok).toBe(true); + if (reopened.ok) expect(reopened.data?.warnings).toBeUndefined(); + } finally { + if (previousClaimsDir === undefined) delete process.env.AGENT_DEVICE_CLAIMS_DIR; + else process.env.AGENT_DEVICE_CLAIMS_DIR = previousClaimsDir; + fs.rmSync(claimsDir, { recursive: true, force: true }); + } +}); + // The production reopen path never re-acquires the claim, so renewal has to ride the successful // existing-session open itself: without it, an owner that came back after a reboot still carries a // pre-reboot stamp and loses the device to the next caller that asks. diff --git a/website/docs/docs/sessions.md b/website/docs/docs/sessions.md index 6a8b3ee14e..8ebf052cf8 100644 --- a/website/docs/docs/sessions.md +++ b/website/docs/docs/sessions.md @@ -114,6 +114,10 @@ in CI and on shared hosts so devices don't keep running after the run: agent-device close --shutdown ``` +Without `--shutdown`, `close` keeps the iOS runner warm for the next `open`. If you shut that simulator +down yourself (`xcrun simctl shutdown`) while the runner is waiting, agent-device stops the runner so it +doesn't boot the simulator again, and the next `open` returns a warning with `reason=runner_destination_lost`. + ## Wait for a slow or busy device A never-booted iOS Simulator can take several minutes to finish its first boot. Give `open` (or From bc01130bf237a9982d0930d8e82dcd76b9cac987 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 9 Oct 2026 21:53:19 +0200 Subject: [PATCH 04/20] fix(ios): classify destination loss by device state and kill the retained runner outright (#3321) Live runs showed three gaps: attach retries reset the window start, so a reboot looked older than the window; a shut-down device keeps its old launchd_sim listed for seconds, so the boot witness alone read a crash; and a graceful stop let the dying xcodebuild reboot the device again. Co-Authored-By: Claude Sonnet 5.5 --- .../platform-apple/src/core/runner-host.ts | 16 ++++++- .../runner-destination-watch.test.ts | 47 ++++++++++++++++++- packages/platform-apple/src/runner/host.ts | 7 +++ .../src/runner/runner-destination-watch.ts | 36 ++++++++++++-- .../src/runner/runner-session.ts | 2 +- .../internal/session-open-execution.ts | 10 ++-- 6 files changed, 106 insertions(+), 12 deletions(-) diff --git a/packages/platform-apple/src/core/runner-host.ts b/packages/platform-apple/src/core/runner-host.ts index 73ba0151b0..b22ec694aa 100644 --- a/packages/platform-apple/src/core/runner-host.ts +++ b/packages/platform-apple/src/core/runner-host.ts @@ -43,7 +43,7 @@ import { getRunnerDeviceClaimAuthorityProbe, getRunnerLeaseOwnerStateDir, } from './runner-owner-state.ts'; -import { buildSimctlArgsForDevice, simulatorAddressFor } from './simctl.ts'; +import { buildSimctlArgsForDevice, readSimctlDeviceState, simulatorAddressFor } from './simctl.ts'; import { hasScopedAppleToolProvider, readApplePlistJson, @@ -51,6 +51,9 @@ import { runXcrun, } from './tool-provider.ts'; +/** The state probe answers in tens of milliseconds and must not become the reason a stop waits. */ +const SIMULATOR_STATE_PROBE_TIMEOUT_MS = 2_000; + /** * The real host capabilities for `@agent-device/platform-apple/runner`: the one place * the runner package's port meets the root-owned utilities. Consumed by the @@ -109,6 +112,17 @@ export const appleRunnerHost: AppleRunnerHost = { visitXmlPlistEntries, leaseOwnerStateDir: getRunnerLeaseOwnerStateDir, hasDeviceClaimAuthority: (device) => getRunnerDeviceClaimAuthorityProbe()?.(device) ?? false, + observeSimulatorState: async (device) => { + try { + const result = await runXcrun(buildSimctlArgsForDevice(device, ['list', 'devices', '-j']), { + allowFailure: true, + timeoutMs: SIMULATOR_STATE_PROBE_TIMEOUT_MS, + }); + return result.exitCode === 0 ? readSimctlDeviceState(result.stdout, device.id) : null; + } catch { + return null; + } + }, observeSimulatorBootTimeMs: async (device) => (await import('../simulator-boot.ts')).observeSimulatorBootTimeMs(device), }; diff --git a/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts b/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts index c5d44de41c..5f7131d3fd 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts @@ -75,6 +75,7 @@ function watchParams(port: number, overrides: { isArmed?: () => boolean } = {}) function stubHost(options: { boots: Array<{ observed: true; bootedAtMs: number } | { observed: false; reason: 'unobserved' }>; processAlive?: boolean; + state?: string | null; }) { const observeSimulatorBootTimeMs = vi.fn(async () => { const next = options.boots.length > 1 ? options.boots.shift() : options.boots[0]; @@ -82,6 +83,7 @@ function stubHost(options: { }); appleRunnerTestHost.update({ observeSimulatorBootTimeMs, + observeSimulatorState: async () => (options.state === undefined ? 'Booted' : options.state), isProcessAlive: () => options.processAlive ?? true, }); return observeSimulatorBootTimeMs; @@ -130,6 +132,48 @@ test('a close on the boot the watch was armed on re-arms instead of stopping', a replacement.destroy(); }); +test('a device listed as shut down is a loss even while its old boot is still observable', async () => { + const listener = await listen(); + const observe = stubHost({ + boots: [{ observed: true, bootedAtMs: Date.now() - 60_000 }], + state: 'Shutdown', + }); + const { params, onStop } = watchParams(listener.port); + + attachRunnerDestinationWatch(params); + (await listener.nextConnection()).destroy(); + await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); + + assert.equal(observe.mock.calls.length, 0); + assert.equal(takeRunnerWarmLossNotice(DEVICE.id)?.reason, 'runner_destination_lost'); +}); + +test('a device that is still booting down is a loss, not a crash', async () => { + const listener = await listen(); + stubHost({ + boots: [{ observed: true, bootedAtMs: Date.now() - 60_000 }], + state: 'Shutting Down', + }); + const { params, onStop } = watchParams(listener.port); + + attachRunnerDestinationWatch(params); + (await listener.nextConnection()).destroy(); + await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); +}); + +test('an unreadable device state falls back to the boot witness', async () => { + const listener = await listen(); + stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() - 60_000 }], state: null }); + const { params, onStop } = watchParams(listener.port); + + attachRunnerDestinationWatch(params); + (await listener.nextConnection()).destroy(); + const replacement = await listener.nextConnection(); + + assert.equal(onStop.mock.calls.length, 0); + replacement.destroy(); +}); + test('a boot unobservable on both reads while the destination process lives is a loss', async () => { const listener = await listen(); const observe = stubHost({ boots: [{ observed: false, reason: 'unobserved' }] }); @@ -197,7 +241,8 @@ test('a refused attach whose runner process is gone is silent cleanup', async () test('a refused attach retries inside its budget and watches the runner once it listens', async () => { vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_DESTINATION_ATTACH_RETRY_MS', '5000'); const port = await unusedPort(); - stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() + 60_000 }] }); + // A reboot that began after the window opened but before the retries reached the runner. + stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() + 20 }] }); const { params, onStop } = watchParams(port); attachRunnerDestinationWatch(params); diff --git a/packages/platform-apple/src/runner/host.ts b/packages/platform-apple/src/runner/host.ts index 29943382a3..9008a34951 100644 --- a/packages/platform-apple/src/runner/host.ts +++ b/packages/platform-apple/src/runner/host.ts @@ -112,6 +112,12 @@ export type AppleRunnerHost = Pick< * sits outside the façade closures this port's eager budget holds flat. */ observeSimulatorBootTimeMs: typeof AppleSimulatorBoot.observeSimulatorBootTimeMs; + /** + * The Simulator's listed power state (`Booted`, `Shutting Down`, `Shutdown`), or null when the + * listing is unavailable. A shut-down device can keep its old `launchd_sim` visible for several + * seconds, so the boot witness alone cannot tell a crash from an external shutdown (#3321). + */ + observeSimulatorState: (device: DeviceInfo) => Promise; }; /** The runner's deadline type is the host's read side; the {@link Deadline} shim below builds them. */ @@ -205,6 +211,7 @@ export const resolveIosPhysicalDeviceControl = delegate('resolveIosPhysicalDevic export const leaseOwnerStateDir = delegate('leaseOwnerStateDir'); export const hasDeviceClaimAuthority = delegate('hasDeviceClaimAuthority'); export const observeSimulatorBootTimeMs = delegate('observeSimulatorBootTimeMs'); +export const observeSimulatorState = delegate('observeSimulatorState'); /** * Deadline keeps its root call-site shape (`Deadline.fromTimeoutMs(...)`); diff --git a/packages/platform-apple/src/runner/runner-destination-watch.ts b/packages/platform-apple/src/runner/runner-destination-watch.ts index 2d22f29a85..169eb93822 100644 --- a/packages/platform-apple/src/runner/runner-destination-watch.ts +++ b/packages/platform-apple/src/runner/runner-destination-watch.ts @@ -1,6 +1,11 @@ import net from 'node:net'; import type { DeviceInfo } from '@agent-device/kernel/device'; -import { emitDiagnostic, isProcessAlive, observeSimulatorBootTimeMs } from './host.ts'; +import { + emitDiagnostic, + isProcessAlive, + observeSimulatorBootTimeMs, + observeSimulatorState, +} from './host.ts'; /** * Why a runner retained after `close` was stopped before the session asked for it again. @@ -58,6 +63,9 @@ const ATTACH_RETRY_BASE_DELAY_MS = 1_000; const DESTINATION_CONFIRM_DEFAULT_MS = 2_500; const DESTINATION_RECHECK_DEFAULT_MS = 1_000; +/** `simctl list` name of a powered-on Simulator. */ +const SIMULATOR_BOOTED_STATE = 'Booted'; + type LossAction = 'stop' | 'rearm'; type LossVerdict = Readonly<{ @@ -137,6 +145,8 @@ export type RunnerDestinationWatchParams = { * budget. */ attachRetryBudgetMs?: number; + /** When the retention window began; re-attaches within one window carry it forward unchanged. */ + armedAtMs?: number; }; /** @@ -160,7 +170,7 @@ export function attachRunnerDestinationWatch(params: RunnerDestinationWatchParam sessionId: params.sessionId, port: params.port, runnerPid: params.runnerPid, - armedAtMs: Date.now(), + armedAtMs: params.armedAtMs ?? Date.now(), isArmed: params.isArmed, onStop: params.onStop, socket: net.connect(params.port, '127.0.0.1'), @@ -257,6 +267,7 @@ async function handleRefusedAttach( isArmed: watch.isArmed, onStop: watch.onStop, attachRetryBudgetMs: budgetMs - nextDelayMs, + armedAtMs: watch.armedAtMs, }); return; } @@ -272,7 +283,10 @@ async function handleRefusedAttach( /** * Reads whether the device behind the runner stopped being the device this session was armed on. * A dead destination process answers for itself before the probe: Xcode cannot reboot from a dead - * process, so stopping is plain lease cleanup. A boot observed newer than the armed window is + * process, so stopping is plain lease cleanup. A device that is no longer `Booted` was shut down + * under the session (measured: its old `launchd_sim` stays listed for ~6s after `simctl shutdown`, + * so the boot witness alone would read the old boot and call it a crash). A boot observed newer than + * the armed window is * Xcode's reboot; a boot unobservable twice while the destination process is still alive is a * device that is down with a reboot promise that may already be in flight. Both are destination * loss, and stopping still pre-empts the reboot in the second case. @@ -281,6 +295,10 @@ async function classifyConnectedLoss(watch: DestinationWatch): Promise isRunnerIdleRetained(deviceId, session), onStop: async () => { - await stopIosRunnerSession(deviceId); + await invalidateRunnerSession(session, 'warm_runner_destination_lost'); }, }); } diff --git a/src/daemon/session-lifecycle/internal/session-open-execution.ts b/src/daemon/session-lifecycle/internal/session-open-execution.ts index 2b5014e133..7897d6f3a2 100644 --- a/src/daemon/session-lifecycle/internal/session-open-execution.ts +++ b/src/daemon/session-lifecycle/internal/session-open-execution.ts @@ -142,11 +142,13 @@ async function readWarmRunnerLossNotice( * the typed reason so automation can key on it without parsing prose. */ function warmRunnerLossWarning(notice: RunnerWarmLossNotice): string { + const cause = + notice.reason === 'runner_destination_lost' + ? 'the simulator was shut down externally while it was retained, so the runner was stopped before it could boot the simulator again' + : 'it stopped answering while it was retained'; return ( - 'The warm iOS runner left by the previous close was stopped: the simulator was shut down ' + - 'externally while it was retained, and the retained runner had begun rebooting it. ' + - 'This open started a fresh runner and left the simulator booted from now on. ' + - `reason=${notice.reason}` + `The warm iOS runner left by the previous close was stopped: ${cause}. ` + + `This open started a fresh runner. reason=${notice.reason}` ); } From 68b92eeb255b64aba83fb33773a7cf424103b132 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 9 Oct 2026 21:54:09 +0200 Subject: [PATCH 05/20] refactor(ios): split destination-loss classification into a boot read and a verdict (#3321) Co-Authored-By: Claude Sonnet 5.5 --- .../src/runner/runner-destination-watch.ts | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/packages/platform-apple/src/runner/runner-destination-watch.ts b/packages/platform-apple/src/runner/runner-destination-watch.ts index 169eb93822..520750c383 100644 --- a/packages/platform-apple/src/runner/runner-destination-watch.ts +++ b/packages/platform-apple/src/runner/runner-destination-watch.ts @@ -74,6 +74,8 @@ type LossVerdict = Readonly<{ noticeReason?: RunnerWarmLossReason; }>; +const DESTINATION_LOST: LossVerdict = { action: 'stop', noticeReason: 'runner_destination_lost' }; + type DestinationWatch = { device: DeviceInfo; sessionId: string; @@ -296,21 +298,21 @@ async function classifyConnectedLoss(watch: DestinationWatch): Promise { + const boot = await observeSimulatorBootTimeMs(watch.device); + if (!boot.observed) return 'unobserved'; + return boot.bootedAtMs <= watch.armedAtMs ? 'same-boot' : 'newer-boot'; } /** From 5eb630b640125adf3e850fc669eb572dc48e9063 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 9 Oct 2026 23:05:41 +0200 Subject: [PATCH 06/20] fix(ios): own warm-runner retention in one place and classify replacement generations (#3321) Review round on #3357. Retention (idle stop, destination watch, retention fact) now enters through one retainRunnerForReuse and a failed ensureRunnerSession restores it, a re-attach that connects classifies the replacement generation, an unreadable device state counts as loss, the notice read waits for an in-flight decision on an open window, and a stale handler no longer deletes a newer watch. Co-Authored-By: Claude Sonnet 5.5 --- .../runner-close-finalization.test.ts | 19 +++ .../runner-destination-watch.test.ts | 124 ++++++++++++++---- .../src/runner/runner-destination-watch.ts | 101 ++++++++++---- .../src/runner/runner-session.ts | 52 ++++++-- src/commands/schema/cli-help.ts | 2 +- ...latform-runtime-warm-runner-notice.test.ts | 39 ++++++ website/docs/docs/sessions.md | 3 +- 7 files changed, 276 insertions(+), 64 deletions(-) create mode 100644 src/platform-runtime-warm-runner-notice.test.ts diff --git a/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts b/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts index e5a7f7d729..2af8ec4eb6 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts @@ -262,14 +262,33 @@ test('a retained close with no runner in memory starts none and claims no lease const events: string[] = []; const lifecycle = closeFinalizationLifecycle(device, events); + const phases: string[] = []; + appleRunnerTestHost.update({ emitDiagnostic: (event) => phases.push(event.phase) }); + await lifecycle.finalizeApplicationClose(closeInput(true)); assert.deepEqual(events, ['release', 'released', 'alerts']); + assert.deepEqual(phases, []); assert.equal(readRunnerSessionLiveness(device.id), null); assert.equal(mockRunCmdBackground.mock.calls.length, 0); assert.equal(runnerLeaseExists(device.id), false); }); +test('a start that fails without using a retained runner leaves it retained', async () => { + const device = deviceNamed('close-finalize-failed-start-retention'); + const session = await ensureRunnerSession(device, {}); + await closeFinalizationLifecycle(device, []).finalizeApplicationClose(closeInput(true)); + + const diagnostics = await captureDiagnostics(async () => { + await assert.rejects( + ensureRunnerSession(device, { expectedRunnerSessionId: 'another-runner-session' }), + ); + }); + + assert.match(diagnostics, /"phase":"ios_runner_idle_stop_scheduled"/); + assert.equal(readRunnerSessionLiveness(device.id)?.sessionId, session.sessionId); +}); + test('a daemon-shutdown close never issues the ordinary close release (#2615)', async () => { const device = deviceNamed('close-finalize-daemon-shutdown'); const session = await ensureRunnerSession(device, {}); diff --git a/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts b/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts index 5f7131d3fd..d62e47ad97 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts @@ -97,7 +97,7 @@ beforeEach(() => { afterEach(async () => { closeRunnerDestinationWatch(DEVICE.id); - takeRunnerWarmLossNotice(DEVICE.id); + await takeRunnerWarmLossNotice(DEVICE.id); vi.unstubAllEnvs(); await Promise.all(listeners.splice(0).map((listener) => listener.close())); }); @@ -111,10 +111,10 @@ test('an established connection that closes after a newer boot stops the runner (await listener.nextConnection()).destroy(); await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); - const notice = takeRunnerWarmLossNotice(DEVICE.id); + const notice = await takeRunnerWarmLossNotice(DEVICE.id); assert.equal(notice?.reason, 'runner_destination_lost'); assert.equal(notice?.sessionId, 'session-1'); - assert.equal(takeRunnerWarmLossNotice(DEVICE.id), undefined); + assert.equal(await takeRunnerWarmLossNotice(DEVICE.id), undefined); }); test('a close on the boot the watch was armed on re-arms instead of stopping', async () => { @@ -126,9 +126,9 @@ test('a close on the boot the watch was armed on re-arms instead of stopping', a (await listener.nextConnection()).destroy(); const replacement = await listener.nextConnection(); - assert.equal(observe.mock.calls.length, 1); + assert.ok(observe.mock.calls.length >= 1); assert.equal(onStop.mock.calls.length, 0); - assert.equal(takeRunnerWarmLossNotice(DEVICE.id), undefined); + assert.equal(await takeRunnerWarmLossNotice(DEVICE.id), undefined); replacement.destroy(); }); @@ -145,10 +145,10 @@ test('a device listed as shut down is a loss even while its old boot is still ob await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); assert.equal(observe.mock.calls.length, 0); - assert.equal(takeRunnerWarmLossNotice(DEVICE.id)?.reason, 'runner_destination_lost'); + assert.equal((await takeRunnerWarmLossNotice(DEVICE.id))?.reason, 'runner_destination_lost'); }); -test('a device that is still booting down is a loss, not a crash', async () => { +test('a device listed as shutting down is a loss, not a crash', async () => { const listener = await listen(); stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() - 60_000 }], @@ -159,19 +159,20 @@ test('a device that is still booting down is a loss, not a crash', async () => { attachRunnerDestinationWatch(params); (await listener.nextConnection()).destroy(); await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); + + assert.equal((await takeRunnerWarmLossNotice(DEVICE.id))?.reason, 'runner_destination_lost'); }); -test('an unreadable device state falls back to the boot witness', async () => { +test('an unreadable device state is no evidence of a healthy device', async () => { const listener = await listen(); stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() - 60_000 }], state: null }); const { params, onStop } = watchParams(listener.port); attachRunnerDestinationWatch(params); (await listener.nextConnection()).destroy(); - const replacement = await listener.nextConnection(); + await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); - assert.equal(onStop.mock.calls.length, 0); - replacement.destroy(); + assert.equal((await takeRunnerWarmLossNotice(DEVICE.id))?.reason, 'runner_destination_lost'); }); test('a boot unobservable on both reads while the destination process lives is a loss', async () => { @@ -184,7 +185,7 @@ test('a boot unobservable on both reads while the destination process lives is a await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); assert.equal(observe.mock.calls.length, 2); - assert.equal(takeRunnerWarmLossNotice(DEVICE.id)?.reason, 'runner_destination_lost'); + assert.equal((await takeRunnerWarmLossNotice(DEVICE.id))?.reason, 'runner_destination_lost'); }); test('a boot that reappears on the recheck with the armed boot re-arms', async () => { @@ -215,17 +216,18 @@ test('a dead runner process is plain cleanup without a notice', async () => { await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); assert.equal(observe.mock.calls.length, 0); - assert.equal(takeRunnerWarmLossNotice(DEVICE.id), undefined); + assert.equal(await takeRunnerWarmLossNotice(DEVICE.id), undefined); }); test('a port that never answers is runner_unreachable once the retry budget is spent', async () => { + vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_DESTINATION_ATTACH_RETRY_MS', '20'); stubHost({ boots: [{ observed: true, bootedAtMs: 0 }] }); const { params, onStop } = watchParams(await unusedPort()); attachRunnerDestinationWatch(params); await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); - assert.equal(takeRunnerWarmLossNotice(DEVICE.id)?.reason, 'runner_unreachable'); + assert.equal((await takeRunnerWarmLossNotice(DEVICE.id))?.reason, 'runner_unreachable'); }); test('a refused attach whose runner process is gone is silent cleanup', async () => { @@ -235,33 +237,84 @@ test('a refused attach whose runner process is gone is silent cleanup', async () attachRunnerDestinationWatch(params); await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); - assert.equal(takeRunnerWarmLossNotice(DEVICE.id), undefined); + assert.equal(await takeRunnerWarmLossNotice(DEVICE.id), undefined); }); -test('a refused attach retries inside its budget and watches the runner once it listens', async () => { +async function listenOnHeld(port: number): Promise { + const server = net.createServer((socket) => socket.on('error', () => {})); + await new Promise((resolve) => server.listen(port, '127.0.0.1', resolve)); + return server; +} + +test('a runner generation that answers after refusals is classified at the connect', async () => { vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_DESTINATION_ATTACH_RETRY_MS', '5000'); const port = await unusedPort(); - // A reboot that began after the window opened but before the retries reached the runner. - stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() + 20 }] }); + // The device rebooted while the port was refused; the replacement connection never closes. + stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() + 60_000 }] }); const { params, onStop } = watchParams(port); attachRunnerDestinationWatch(params); await new Promise((resolve) => setTimeout(resolve, 50)); assert.equal(onStop.mock.calls.length, 0); - const server = net.createServer((socket) => { - socket.on('error', () => {}); - socket.destroy(); - }); - await new Promise((resolve) => server.listen(port, '127.0.0.1', resolve)); + const server = await listenOnHeld(port); try { await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1), { timeout: 4000 }); - assert.equal(takeRunnerWarmLossNotice(DEVICE.id)?.reason, 'runner_destination_lost'); + assert.equal((await takeRunnerWarmLossNotice(DEVICE.id))?.reason, 'runner_destination_lost'); } finally { await new Promise((resolve) => server.close(() => resolve())); } }); +test('a runner generation that answers on the armed boot keeps the watch', async () => { + vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_DESTINATION_ATTACH_RETRY_MS', '5000'); + const port = await unusedPort(); + const observe = stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() - 60_000 }] }); + const { params, onStop } = watchParams(port); + + attachRunnerDestinationWatch(params); + await new Promise((resolve) => setTimeout(resolve, 50)); + const server = await listenOnHeld(port); + try { + await vi.waitFor(() => assert.equal(observe.mock.calls.length, 1), { timeout: 4000 }); + await new Promise((resolve) => setTimeout(resolve, 30)); + assert.equal(onStop.mock.calls.length, 0); + } finally { + closeRunnerDestinationWatch(DEVICE.id); + await new Promise((resolve) => server.close(() => resolve())); + } +}); + +test('a notice read waits for a loss decision still in flight for an open window', async () => { + vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_DESTINATION_CONFIRM_MS', '80'); + const listener = await listen(); + stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() + 60_000 }] }); + const { params } = watchParams(listener.port); + + attachRunnerDestinationWatch(params); + (await listener.nextConnection()).destroy(); + await new Promise((resolve) => setTimeout(resolve, 20)); + + assert.equal((await takeRunnerWarmLossNotice(DEVICE.id))?.reason, 'runner_destination_lost'); +}); + +test('a notice read does not wait once the retention window has ended', async () => { + vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_DESTINATION_CONFIRM_MS', '2000'); + const listener = await listen(); + stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() + 60_000 }] }); + let armed = true; + const { params } = watchParams(listener.port, { isArmed: () => armed }); + + attachRunnerDestinationWatch(params); + (await listener.nextConnection()).destroy(); + await new Promise((resolve) => setTimeout(resolve, 20)); + armed = false; + const startedAt = Date.now(); + + assert.equal(await takeRunnerWarmLossNotice(DEVICE.id), undefined); + assert.ok(Date.now() - startedAt < 500); +}); + test('a close after the retention window ended acts on nothing', async () => { const listener = await listen(); const observe = stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() + 60_000 }] }); @@ -276,7 +329,7 @@ test('a close after the retention window ended acts on nothing', async () => { assert.equal(observe.mock.calls.length, 0); assert.equal(onStop.mock.calls.length, 0); - assert.equal(takeRunnerWarmLossNotice(DEVICE.id), undefined); + assert.equal(await takeRunnerWarmLossNotice(DEVICE.id), undefined); }); test('closing the watch detaches it so a later runner death stops nothing', async () => { @@ -309,3 +362,24 @@ test('a second retention window re-arms the existing watch instead of stacking a assert.equal(first.onStop.mock.calls.length, 0); assert.equal(listener.connections.length, 0); }); + +test('a handler resuming after its window was replaced leaves the newer watch tracked', async () => { + vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_DESTINATION_CONFIRM_MS', '60'); + const listener = await listen(); + stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() + 60_000 }] }); + const first = watchParams(listener.port); + attachRunnerDestinationWatch(first.params); + (await listener.nextConnection()).destroy(); + await new Promise((resolve) => setTimeout(resolve, 10)); + + closeRunnerDestinationWatch(DEVICE.id); + const second = watchParams(listener.port); + attachRunnerDestinationWatch(second.params); + await listener.nextConnection(); + await new Promise((resolve) => setTimeout(resolve, 120)); + attachRunnerDestinationWatch(watchParams(listener.port).params); + await new Promise((resolve) => setTimeout(resolve, 30)); + + assert.equal(first.onStop.mock.calls.length, 0); + assert.equal(listener.connections.length, 0); +}); diff --git a/packages/platform-apple/src/runner/runner-destination-watch.ts b/packages/platform-apple/src/runner/runner-destination-watch.ts index 520750c383..8e2c886a19 100644 --- a/packages/platform-apple/src/runner/runner-destination-watch.ts +++ b/packages/platform-apple/src/runner/runner-destination-watch.ts @@ -90,6 +90,13 @@ type DestinationWatch = { socket: net.Socket; /** True once this connection was established: only then can its close mean runner death. */ connected: boolean; + /** + * True when this attach follows a refusal or a same-boot re-arm, so what answers may be a new + * runner generation started after the device changed under the session. + */ + replacement: boolean; + /** The in-flight loss decision, which a notice reader waits for while the window is still open. */ + deciding: Promise | undefined; detached: boolean; }; @@ -149,6 +156,8 @@ export type RunnerDestinationWatchParams = { attachRetryBudgetMs?: number; /** When the retention window began; re-attaches within one window carry it forward unchanged. */ armedAtMs?: number; + /** Set by re-attaches within one window: the listener that answers may be a new generation. */ + replacement?: boolean; }; /** @@ -177,11 +186,14 @@ export function attachRunnerDestinationWatch(params: RunnerDestinationWatchParam onStop: params.onStop, socket: net.connect(params.port, '127.0.0.1'), connected: false, + replacement: params.replacement ?? false, + deciding: undefined, detached: false, }; watch.socket.unref(); watch.socket.on('connect', () => { watch.connected = true; + if (watch.replacement) void decide(watch, () => confirmReplacementGeneration(watch)); }); watch.socket.on('close', () => { void handleSocketClosed(watch, params.attachRetryBudgetMs); @@ -202,7 +214,8 @@ export function closeRunnerDestinationWatch(deviceId: string): void { function detachWatch(watch: DestinationWatch): void { watch.detached = true; - destinationWatches.delete(watch.device.id); + // A handler resuming after an await may belong to a watch a newer window has replaced. + if (destinationWatches.get(watch.device.id) === watch) destinationWatches.delete(watch.device.id); watch.socket.removeAllListeners('connect'); watch.socket.removeAllListeners('close'); watch.socket.removeAllListeners('error'); @@ -222,27 +235,56 @@ async function handleSocketClosed( await handleRefusedAttach(watch, attachRetryBudgetMs); return; } - await delay(resolveConfirmDelayMs()); - if (watch.detached || !watch.isArmed()) { - detachWatch(watch); - return; - } + await decide(watch, async () => { + await delay(resolveConfirmDelayMs()); + if (watch.detached || !watch.isArmed()) { + detachWatch(watch); + return; + } + const verdict = await classifyConnectedLoss(watch); + if (watch.detached || !watch.isArmed()) { + detachWatch(watch); + return; + } + if (verdict.action === 'rearm') { + rearmOnSameBoot(watch); + return; + } + stopRetainedRunner(watch, verdict.noticeReason); + }); +} + +/** + * A listener answered after the device may have been replaced: the refused attaches that preceded + * it span exactly the gap in which Xcode reboots a shut-down destination and restarts the runner + * app onto the same port, and a connection that reaches that new generation never closes on its + * own. One classification at the connect decides whether the window's boot is still the device's. + * A same-boot answer keeps this connection as the watch. + */ +async function confirmReplacementGeneration(watch: DestinationWatch): Promise { const verdict = await classifyConnectedLoss(watch); if (watch.detached || !watch.isArmed()) { detachWatch(watch); return; } - if (verdict.action === 'rearm') { - rearmOnSameBoot(watch); - return; + if (verdict.action === 'stop') stopRetainedRunner(watch, verdict.noticeReason); +} + +/** Runs one loss decision and publishes it, so {@link takeRunnerWarmLossNotice} can wait for it. */ +async function decide(watch: DestinationWatch, run: () => Promise): Promise { + const deciding = run(); + watch.deciding = deciding; + try { + await deciding; + } finally { + if (watch.deciding === deciding) watch.deciding = undefined; } - stopRetainedRunner(watch, verdict.noticeReason); } /** * The attach never reached a listener. A rebooting destination is usually unreachable for a - * couple of seconds while Xcode restarts the app onto the same port, so retry with a growing - * delay inside a bounded budget. Only when the budget runs out is the retained runner called + * couple of seconds while Xcode restarts the app onto the same port, so retry at a flat one-second + * cadence inside a bounded budget; a retry that connects classifies the replacement generation. Only when the budget runs out is the retained runner called * what it has proven to be: unable to answer anything. By then the destination process's own * liveness decides whether this was a crash (Xcode gave up with it, the device lost its boot * owner) or a listener that died while Xcode kept the device — the latter is exactly the silent @@ -270,6 +312,7 @@ async function handleRefusedAttach( onStop: watch.onStop, attachRetryBudgetMs: budgetMs - nextDelayMs, armedAtMs: watch.armedAtMs, + replacement: true, }); return; } @@ -285,20 +328,20 @@ async function handleRefusedAttach( /** * Reads whether the device behind the runner stopped being the device this session was armed on. * A dead destination process answers for itself before the probe: Xcode cannot reboot from a dead - * process, so stopping is plain lease cleanup. A device that is no longer `Booted` was shut down - * under the session (measured: its old `launchd_sim` stays listed for ~6s after `simctl shutdown`, - * so the boot witness alone would read the old boot and call it a crash). A boot observed newer than - * the armed window is - * Xcode's reboot; a boot unobservable twice while the destination process is still alive is a - * device that is down with a reboot promise that may already be in flight. Both are destination - * loss, and stopping still pre-empts the reboot in the second case. + * process, so stopping is plain lease cleanup. Otherwise the device must be listed `Booted` on the + * boot the window began with, and anything else is destination loss: a shut-down device keeps its + * old `launchd_sim` listed for ~6s, so the boot witness alone would read the old boot and call it + * a crash, and an unreadable listing is no evidence of a healthy device. A boot newer than the + * window is Xcode's reboot; a boot unobservable twice is a device that is down with a reboot + * promise that may already be in flight. Stopping pre-empts that promise in every loss case. */ async function classifyConnectedLoss(watch: DestinationWatch): Promise { if (watch.runnerPid !== undefined && !isProcessAlive(watch.runnerPid)) { return { action: 'stop' }; } - const state = await observeSimulatorState(watch.device); - if (state !== null && state !== SIMULATOR_BOOTED_STATE) return DESTINATION_LOST; + if ((await observeSimulatorState(watch.device)) !== SIMULATOR_BOOTED_STATE) { + return DESTINATION_LOST; + } let boot = await readBootAgainstWindow(watch); if (boot === 'unobserved') { await delay(resolveRecheckDelayMs()); @@ -339,6 +382,7 @@ function rearmOnSameBoot(watch: DestinationWatch): void { isArmed, onStop, armedAtMs, + replacement: true, }); } @@ -384,9 +428,20 @@ function stopRetainedRunner( /** * Takes the destination-loss notice recorded for this device, once. A notice exists only when a * retained runner was stopped because the destination was replaced under it — the state the next - * `open` must explain rather than leave the caller to infer from a cold start. + * `open` must explain rather than leave the caller to infer from a cold start. A decision still in + * flight for a window that is still open is waited for, so a notice cannot land after the open it + * belongs to has already read; once any use of the runner has ended the window nothing is waited + * for, because nothing can be recorded any more. */ -export function takeRunnerWarmLossNotice(deviceId: string): RunnerWarmLossNotice | undefined { +export async function takeRunnerWarmLossNotice( + deviceId: string, +): Promise { + for (let watch = destinationWatches.get(deviceId); watch?.deciding;) { + if (!watch.isArmed()) break; + await watch.deciding; + const next = destinationWatches.get(deviceId); + watch = next === watch ? undefined : next; + } const notice = pendingWarmLossNotices.get(deviceId); if (notice) pendingWarmLossNotices.delete(deviceId); return notice; diff --git a/packages/platform-apple/src/runner/runner-session.ts b/packages/platform-apple/src/runner/runner-session.ts index 588daeb803..bf584db969 100644 --- a/packages/platform-apple/src/runner/runner-session.ts +++ b/packages/platform-apple/src/runner/runner-session.ts @@ -102,6 +102,9 @@ export async function ensureRunnerSession( ): Promise { // Any runner use means the device is active again: a pending idle stop // from a retained-after-close runner no longer applies. + const retainedSession = runnerWarmRetainedDevices.has(device.id) + ? runnerSessions.get(device.id) + : undefined; cancelIosRunnerIdleStop(device.id); // This start's admission: the loop that owns the start across retries supplies one, and every // other start takes a token for itself. Both are taken synchronously, before the first await and @@ -147,14 +150,23 @@ export async function ensureRunnerSession( releaseOwnerInterest(); } }); - const { raceRunnerStartAgainstCaller } = await import('./runner-start-budget.ts'); - return await raceRunnerStartAgainstCaller( - // A start that minted its own token releases it; a loop-supplied token belongs to the loop, - // which finishes it when the loop itself is done retrying (#3220). - ownedAdmission ? start : start.finally(() => finishRunnerStartAdmission(startAdmission)), - startAdmission, - options.signal, - ); + try { + const { raceRunnerStartAgainstCaller } = await import('./runner-start-budget.ts'); + return await raceRunnerStartAgainstCaller( + // A start that minted its own token releases it; a loop-supplied token belongs to the loop, + // which finishes it when the loop itself is done retrying (#3220). + ownedAdmission ? start : start.finally(() => finishRunnerStartAdmission(startAdmission)), + startAdmission, + options.signal, + ); + } catch (error) { + // A start that failed without using the retained runner leaves it exactly as idle as it was, + // so it goes back to the retention that carries both its idle stop and its destination watch. + if (retainedSession && runnerSessions.get(device.id) === retainedSession) { + await retainRunnerForReuse(device.id, retainedSession); + } + throw error; + } } /** How long the device-readiness probe may take, bounded by the startup budget it runs inside. */ @@ -695,6 +707,22 @@ function isRunnerIdleRetained(deviceId: string, session: RunnerSession | undefin ); } +/** + * The one way a runner enters post-`close` retention: the idle-stop timer, the retention fact the + * destination watch reads, and the watch itself are set together, so no retained runner is left + * with one but not the others. Retention marks the device after the scheduler because the + * scheduler's own cancel step ends any window the previous retention left open. + */ +async function retainRunnerForReuse( + deviceId: string, + session: RunnerSession | undefined, +): Promise { + scheduleIosRunnerIdleStop(deviceId); + if (!session || !canWorkWithRunnerSession(session)) return; + runnerWarmRetainedDevices.add(deviceId); + await armRunnerDestinationWatch(session); +} + /** * Arms the #3321 push watcher over a runner this close retained. Reached through a function-scoped * import: the watcher module is needed only at the start of a retention window and must not join @@ -805,12 +833,8 @@ export async function releaseIosRunnerOnClose( options: { retain: boolean }, ): Promise { const session = runnerSessions.get(deviceId); - if (options.retain && session && !isRunnerMainThreadOccupied(session)) { - // The flag is set after the scheduler because the scheduler's own cancel step ends any - // retention the previous window left open, and marking first would erase this one. - scheduleIosRunnerIdleStop(deviceId); - runnerWarmRetainedDevices.add(deviceId); - await armRunnerDestinationWatch(session); + if (options.retain && !isRunnerMainThreadOccupied(session)) { + await retainRunnerForReuse(deviceId, session); return; } if (options.retain) { diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index dbf334bbbe..aa3735c20f 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -530,7 +530,7 @@ Android physical-device prerequisites: Runner and daemon lifecycle (applies to simulators too): open without --relaunch is idempotent-foreground for an already-running app (it brings the process forward; it does not restart it). open --relaunch restarts the app; on iOS simulators this collapses to one simctl launch --terminate-running-process call instead of a separate terminate-then-launch. No runner read launches a session app that is not running: snapshot, wait, is, get, a reading find, and an interaction's leading reads (a gesture's viewport read, the capture that resolves a selector click/fill) answer the retriable APP_NOT_RUNNING instead of bare-launching over a launch SpringBoard still holds behind its deep-link confirmation. Only open, activate, and a command that mutates without a leading read bring a stopped app up. - close keeps a healthy iOS simulator XCTest runner warm by default so the next open on that simulator (same udid in the same simulator set) skips the runner build, unless --shutdown was requested, the session was recording, or the session held a device lease. A retained runner auto-stops after an idle window (default 5 minutes); set AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS to override, or 0 to disable idle stop and retain until daemon exit. + close keeps a healthy iOS simulator XCTest runner warm by default so the next open on that simulator (same udid in the same simulator set) skips the runner build, unless --shutdown was requested, the session was recording, or the session held a device lease. A retained runner auto-stops after an idle window (default 5 minutes); set AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS to override, or 0 to disable idle stop and retain until daemon exit. A simulator shut down externally while its runner is retained is detected and the retained runner is stopped, because the retained Xcode destination would otherwise reboot the simulator and leave it powered on; the next open reports reason=runner_destination_lost (or reason=runner_unreachable when the runner stopped answering) in its warnings. Each AGENT_DEVICE_STATE_DIR runs its own daemon. It self-exits after an idle window (default 5 minutes, matching the runner idle-stop default) once it has no open sessions, no in-flight requests, and no active recording; set AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS to override, or 0 to disable idle reap. On a machine shared by several agents, set AGENT_DEVICE_SESSION_IDLE_TIMEOUT_MS to also expire an individual session that has taken a host-global device claim and then received no commands for that long; the claim is released and the next command on that session answers SESSION_NOT_FOUND with details.reason SESSION_IDLE_EXPIRED naming the window and the device. Off by default, because on a shared host "idle" and "thinking" are indistinguishable: only set it where an abandoned claim blocking every other agent is the worse failure. Sessions holding a remote lease or any active capture (recording, logs, audio, performance, or trace) are never expired this way. A stale iOS runner lease — its owner process dead, or its AGENT_DEVICE_STATE_DIR deleted — is reclaimed automatically instead of failing with "is already owned by another agent-device daemon". A live owner's runner is also reclaimed when the requesting daemon holds the host-global device claim for that device: claims are exclusive, so holding one proves the runner's owner released the device and merely kept the runner warm. The error remains only for owners outside claim arbitration (a pre-claims build, or daemons pointed at different claim stores). diff --git a/src/platform-runtime-warm-runner-notice.test.ts b/src/platform-runtime-warm-runner-notice.test.ts new file mode 100644 index 0000000000..8465c9eefa --- /dev/null +++ b/src/platform-runtime-warm-runner-notice.test.ts @@ -0,0 +1,39 @@ +import { beforeEach, expect, test, vi } from 'vitest'; +import type { RunnerWarmLossNotice } from '@agent-device/platform-apple/runner/operations'; + +const NOTICE: RunnerWarmLossNotice = { + reason: 'runner_destination_lost', + deviceId: 'sim-1', + sessionId: 'session-1', + atMs: 1, +}; + +const mocks = vi.hoisted(() => ({ + takeRunnerWarmLossNotice: vi.fn(async (): Promise => undefined), +})); + +vi.mock('@agent-device/platform-apple/runner/operations', () => mocks); + +import { IOS_DEVICE, IOS_SIMULATOR, MACOS_DEVICE } from './__tests__/test-utils/device-fixtures.ts'; +import { takeWarmRunnerLossNotice } from './platform-runtime-warm-runner-notice.ts'; + +beforeEach(() => { + vi.clearAllMocks(); +}); + +test('an iOS Simulator reads the notice recorded for its id', async () => { + mocks.takeRunnerWarmLossNotice.mockResolvedValueOnce(NOTICE); + + expect(await takeWarmRunnerLossNotice(IOS_SIMULATOR)).toEqual(NOTICE); + expect(mocks.takeRunnerWarmLossNotice).toHaveBeenCalledWith(IOS_SIMULATOR.id); +}); + +test('a physical iOS device retains no warm runner and reads nothing', async () => { + expect(await takeWarmRunnerLossNotice(IOS_DEVICE)).toBeUndefined(); + expect(mocks.takeRunnerWarmLossNotice).not.toHaveBeenCalled(); +}); + +test('a macOS device retains no warm runner and reads nothing', async () => { + expect(await takeWarmRunnerLossNotice(MACOS_DEVICE)).toBeUndefined(); + expect(mocks.takeRunnerWarmLossNotice).not.toHaveBeenCalled(); +}); diff --git a/website/docs/docs/sessions.md b/website/docs/docs/sessions.md index 8ebf052cf8..3aa3ccf8f7 100644 --- a/website/docs/docs/sessions.md +++ b/website/docs/docs/sessions.md @@ -116,7 +116,8 @@ agent-device close --shutdown Without `--shutdown`, `close` keeps the iOS runner warm for the next `open`. If you shut that simulator down yourself (`xcrun simctl shutdown`) while the runner is waiting, agent-device stops the runner so it -doesn't boot the simulator again, and the next `open` returns a warning with `reason=runner_destination_lost`. +doesn't boot the simulator again, and the next `open` returns a warning that ends in +`reason=runner_destination_lost`, or `reason=runner_unreachable` when the runner stopped answering. ## Wait for a slow or busy device From 97360e123a73e996ebe3bc231426ada21d457147 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 9 Oct 2026 23:27:30 +0200 Subject: [PATCH 07/20] test(ios): pin that a destination lost during retention stops the runner and releases its lease (#3321) Co-Authored-By: Claude Sonnet 5.5 --- .../runner-close-finalization.test.ts | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts b/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts index 2af8ec4eb6..51d2594210 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; +import net from 'node:net'; import path from 'node:path'; import { beforeEach, test, vi } from 'vitest'; import type { AppleApplicationTools } from '@agent-device/contracts/application-lifecycle-runtime'; @@ -18,6 +19,7 @@ import { RUNNER_CACHE_KEY_FIXTURE, } from './runner-session-fixtures.ts'; import { mkdtempForTestSync } from './tmp-dir.ts'; +import { takeRunnerWarmLossNotice } from '../runner-destination-watch.ts'; import { bindAppleApplicationLifecycle } from '../../lifecycle.ts'; import { platformRuntimeHostFixture } from '../../runtime.fixtures.ts'; import { runnerOwnerToken, writeRunnerLease } from '../runner-lease.ts'; @@ -351,6 +353,39 @@ test('a drain that lands after close is issued cannot return a busy runner to re assert.equal(readRunnerSessionLiveness(device.id), null); }); +test('a destination lost during retention takes the retained runner and its lease down (#3321)', async () => { + const device = deviceNamed('close-finalize-destination-lost'); + const connections: net.Socket[] = []; + const destination = net.createServer((socket) => { + socket.on('error', () => {}); + connections.push(socket); + }); + await new Promise((resolve) => destination.listen(0, '127.0.0.1', resolve)); + mockGetFreePort.mockResolvedValue((destination.address() as net.AddressInfo).port); + vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_DESTINATION_CONFIRM_MS', '5'); + appleRunnerTestHost.update({ + observeSimulatorState: async () => 'Shutdown', + observeSimulatorBootTimeMs: async () => ({ observed: true, bootedAtMs: 1 }), + }); + try { + const session = await ensureRunnerSession(device, {}); + await closeFinalizationLifecycle(device, []).finalizeApplicationClose(closeInput(true)); + await vi.waitFor(() => assert.equal(connections.length, 1)); + + for (const socket of connections) socket.destroy(); + + await vi.waitFor(() => assert.equal(readRunnerSessionLiveness(device.id), null), { + timeout: 5000, + }); + assert.equal(runnerLeaseExists(device.id), false); + assert.equal(session.state, 'stopped'); + assert.equal((await takeRunnerWarmLossNotice(device.id))?.reason, 'runner_destination_lost'); + } finally { + vi.unstubAllEnvs(); + await new Promise((resolve) => destination.close(() => resolve())); + } +}); + /** The close path's Apple tools, composed exactly as the root's lazy tools compose them. */ function closeFinalizationLifecycle( device: DeviceInfo, From b43943037a7196b537bf685032f3cfbaf29a96ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 10 Oct 2026 00:09:37 +0200 Subject: [PATCH 08/20] fix(ios): restore a released retention window only while no one has touched it (#3321) A failed start that released a retained runner could re-arm retention over a runner a concurrent start was already using, because the restore keyed on session identity. Retention state changes now advance a per-device epoch and a restore is valid only at the epoch it released at. retainRunnerForReuse checks the session before it touches any retention state. Co-Authored-By: Claude Sonnet 5.5 --- .../runner-close-finalization.test.ts | 17 +++++++ .../src/runner/runner-session.ts | 51 +++++++++++++++---- 2 files changed, 58 insertions(+), 10 deletions(-) diff --git a/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts b/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts index 51d2594210..79dd94cfd9 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts @@ -353,6 +353,23 @@ test('a drain that lands after close is issued cannot return a busy runner to re assert.equal(readRunnerSessionLiveness(device.id), null); }); +test('a failed start does not re-retain a runner a concurrent start already took over', async () => { + const device = deviceNamed('close-finalize-concurrent-start'); + const session = await ensureRunnerSession(device, {}); + await closeFinalizationLifecycle(device, []).finalizeApplicationClose(closeInput(true)); + const phases: string[] = []; + appleRunnerTestHost.update({ emitDiagnostic: (event) => phases.push(event.phase) }); + + const [failed, reused] = await Promise.allSettled([ + ensureRunnerSession(device, { expectedRunnerSessionId: 'another-runner-session' }), + ensureRunnerSession(device, {}), + ]); + + assert.equal(failed.status, 'rejected'); + assert.equal(reused.status === 'fulfilled' && reused.value.sessionId, session.sessionId); + assert.doesNotMatch(phases.join('\n'), /ios_runner_idle_stop_scheduled/); +}); + test('a destination lost during retention takes the retained runner and its lease down (#3321)', async () => { const device = deviceNamed('close-finalize-destination-lost'); const connections: net.Socket[] = []; diff --git a/packages/platform-apple/src/runner/runner-session.ts b/packages/platform-apple/src/runner/runner-session.ts index bf584db969..84fb1d4b92 100644 --- a/packages/platform-apple/src/runner/runner-session.ts +++ b/packages/platform-apple/src/runner/runner-session.ts @@ -89,6 +89,12 @@ const runnerIdleStopTimers = new Map(); * whole retain-until-daemon-exit behavior just the same. */ const runnerWarmRetainedDevices = new Set(); +/** + * Advances every time a device's retention state is ended, so "nothing has touched this device's + * retention since I released it" is a fact a caller can hold: a start that released a retained + * runner may put it back only while the epoch it released at is still current. + */ +const runnerRetentionEpochs = new Map(); const RUNNER_RETAINED_IDLE_STOP_DEFAULT_MS = 5 * 60_000; const RUNNER_STALE_BUNDLE_UNINSTALL_TIMEOUT_MS = 10_000; @@ -102,10 +108,7 @@ export async function ensureRunnerSession( ): Promise { // Any runner use means the device is active again: a pending idle stop // from a retained-after-close runner no longer applies. - const retainedSession = runnerWarmRetainedDevices.has(device.id) - ? runnerSessions.get(device.id) - : undefined; - cancelIosRunnerIdleStop(device.id); + const released = releaseRetainedRunnerForUse(device.id); // This start's admission: the loop that owns the start across retries supplies one, and every // other start takes a token for itself. Both are taken synchronously, before the first await and // registered with the device, so a teardown beginning in this same turn closes it even while @@ -162,9 +165,7 @@ export async function ensureRunnerSession( } catch (error) { // A start that failed without using the retained runner leaves it exactly as idle as it was, // so it goes back to the retention that carries both its idle stop and its destination watch. - if (retainedSession && runnerSessions.get(device.id) === retainedSession) { - await retainRunnerForReuse(device.id, retainedSession); - } + if (released) await restoreReleasedRunnerRetention(device.id, released); throw error; } } @@ -671,6 +672,7 @@ export function scheduleIosRunnerIdleStop(deviceId: string): void { } export function cancelIosRunnerIdleStop(deviceId: string): void { + runnerRetentionEpochs.set(deviceId, (runnerRetentionEpochs.get(deviceId) ?? 0) + 1); endRunnerWarmRetention(deviceId); const timer = runnerIdleStopTimers.get(deviceId); if (!timer) return; @@ -710,19 +712,48 @@ function isRunnerIdleRetained(deviceId: string, session: RunnerSession | undefin /** * The one way a runner enters post-`close` retention: the idle-stop timer, the retention fact the * destination watch reads, and the watch itself are set together, so no retained runner is left - * with one but not the others. Retention marks the device after the scheduler because the - * scheduler's own cancel step ends any window the previous retention left open. + * with one but not the others. A runner that cannot be retained changes nothing. Retention marks + * the device after the scheduler because the scheduler's own cancel step ends any window the + * previous retention left open. */ async function retainRunnerForReuse( deviceId: string, session: RunnerSession | undefined, ): Promise { - scheduleIosRunnerIdleStop(deviceId); if (!session || !canWorkWithRunnerSession(session)) return; + scheduleIosRunnerIdleStop(deviceId); runnerWarmRetainedDevices.add(deviceId); await armRunnerDestinationWatch(session); } +type ReleasedRunnerRetention = Readonly<{ session: RunnerSession; epoch: number }>; + +/** + * A start is about to use the device's runner, which ends any retention window on it. Returns what + * was released, for {@link restoreReleasedRunnerRetention} to put back if the start fails. + */ +function releaseRetainedRunnerForUse(deviceId: string): ReleasedRunnerRetention | undefined { + const session = runnerWarmRetainedDevices.has(deviceId) + ? runnerSessions.get(deviceId) + : undefined; + cancelIosRunnerIdleStop(deviceId); + return session && { session, epoch: runnerRetentionEpochs.get(deviceId) ?? 0 }; +} + +/** + * Puts a released window back only while no one has touched the device's retention since the + * release: a later start that used the runner, a close that retained it anew, or a stop each end + * or replace the window, and a failed start must not re-arm a runner someone else now owns. + */ +async function restoreReleasedRunnerRetention( + deviceId: string, + released: ReleasedRunnerRetention, +): Promise { + if ((runnerRetentionEpochs.get(deviceId) ?? 0) !== released.epoch) return; + if (runnerSessions.get(deviceId) !== released.session) return; + await retainRunnerForReuse(deviceId, released.session); +} + /** * Arms the #3321 push watcher over a runner this close retained. Reached through a function-scoped * import: the watcher module is needed only at the start of a retention window and must not join From fbbf157c484762298f74f9722cea44d3eb6817f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 10 Oct 2026 00:18:46 +0200 Subject: [PATCH 09/20] test(ios): assert the typed ownership reason and the reused session separately (#3321) Co-Authored-By: Claude Sonnet 5.5 --- .../runner/__tests__/runner-close-finalization.test.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts b/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts index 79dd94cfd9..a60356a854 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts @@ -366,7 +366,15 @@ test('a failed start does not re-retain a runner a concurrent start already took ]); assert.equal(failed.status, 'rejected'); - assert.equal(reused.status === 'fulfilled' && reused.value.sessionId, session.sessionId); + assert.equal( + (failed as PromiseRejectedResult).reason?.details?.reason, + 'runner_session_ownership_changed', + ); + assert.equal(reused.status, 'fulfilled'); + assert.equal( + (reused as PromiseFulfilledResult).value.sessionId, + session.sessionId, + ); assert.doesNotMatch(phases.join('\n'), /ios_runner_idle_stop_scheduled/); }); From dd52602c3c1a292d1b0b1de8223ecf458ff9464e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 10 Oct 2026 04:56:11 +0200 Subject: [PATCH 10/20] refactor(ios): one Simulator state probe, one env reader, and a watch that replaces rather than amends (#3321) Co-Authored-By: Claude Sonnet 5.5 --- .../platform-apple/src/core/runner-host.ts | 9 +-- packages/platform-apple/src/core/simulator.ts | 14 +++-- .../runner-destination-watch.test.ts | 24 ++++---- .../src/runner/runner-destination-watch.ts | 57 ++++++++++--------- .../internal/session-open-execution.ts | 4 +- 5 files changed, 58 insertions(+), 50 deletions(-) diff --git a/packages/platform-apple/src/core/runner-host.ts b/packages/platform-apple/src/core/runner-host.ts index b22ec694aa..db94ba5a61 100644 --- a/packages/platform-apple/src/core/runner-host.ts +++ b/packages/platform-apple/src/core/runner-host.ts @@ -43,7 +43,7 @@ import { getRunnerDeviceClaimAuthorityProbe, getRunnerLeaseOwnerStateDir, } from './runner-owner-state.ts'; -import { buildSimctlArgsForDevice, readSimctlDeviceState, simulatorAddressFor } from './simctl.ts'; +import { buildSimctlArgsForDevice, simulatorAddressFor } from './simctl.ts'; import { hasScopedAppleToolProvider, readApplePlistJson, @@ -114,11 +114,8 @@ export const appleRunnerHost: AppleRunnerHost = { hasDeviceClaimAuthority: (device) => getRunnerDeviceClaimAuthorityProbe()?.(device) ?? false, observeSimulatorState: async (device) => { try { - const result = await runXcrun(buildSimctlArgsForDevice(device, ['list', 'devices', '-j']), { - allowFailure: true, - timeoutMs: SIMULATOR_STATE_PROBE_TIMEOUT_MS, - }); - return result.exitCode === 0 ? readSimctlDeviceState(result.stdout, device.id) : null; + const { getSimulatorState } = await import('./simulator.ts'); + return await getSimulatorState(device, { timeoutMs: SIMULATOR_STATE_PROBE_TIMEOUT_MS }); } catch { return null; } diff --git a/packages/platform-apple/src/core/simulator.ts b/packages/platform-apple/src/core/simulator.ts index 7317ada69a..5c4b71365e 100644 --- a/packages/platform-apple/src/core/simulator.ts +++ b/packages/platform-apple/src/core/simulator.ts @@ -85,7 +85,7 @@ export async function ensureBootedSimulator( const state = wasSimulatorRecentlyObservedBooted(device) ? 'Booted' - : await getSimulatorState(device, options.signal); + : await getSimulatorState(device, { signal: options.signal }); if (state === 'Booted') { markSimulatorBooted(device); return; @@ -148,7 +148,7 @@ export async function ensureBootedSimulator( requireExecSuccess(bootStatusResult, 'simctl bootstatus failed'); - const nextState = await getSimulatorState(device, options.signal); + const nextState = await getSimulatorState(device, { signal: options.signal }); if (nextState !== 'Booted') { throw new AppError('COMMAND_FAILED', 'Simulator is still booting', { state: nextState }); } @@ -205,12 +205,16 @@ export async function ensureBootedSimulator( await openIosSimulatorApp({ signal: options.signal }); } -async function getSimulatorState(device: DeviceInfo, signal?: AbortSignal): Promise { +/** The listed power state of one Simulator (`Booted`, `Shutting Down`, `Shutdown`), or null when unreadable. */ +export async function getSimulatorState( + device: DeviceInfo, + options: { signal?: AbortSignal; timeoutMs?: number } = {}, +): Promise { const simctlArgs = buildSimctlArgsForDevice(device, ['list', 'devices', '-j']); const result = await runXcrun(simctlArgs, { allowFailure: true, - signal, - timeoutMs: IOS_SIMCTL_LIST_TIMEOUT_MS, + signal: options.signal, + timeoutMs: options.timeoutMs ?? IOS_SIMCTL_LIST_TIMEOUT_MS, }); if (result.exitCode !== 0) return null; return readSimctlDeviceState(result.stdout, device.id); diff --git a/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts b/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts index d62e47ad97..0efbd573d1 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts @@ -347,20 +347,21 @@ test('closing the watch detaches it so a later runner death stops nothing', asyn assert.equal(onStop.mock.calls.length, 0); }); -test('a second retention window re-arms the existing watch instead of stacking another', async () => { +test('a second retention window replaces the existing watch instead of stacking another', async () => { const listener = await listen(); stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() + 60_000 }] }); const first = watchParams(listener.port); const second = watchParams(listener.port); attachRunnerDestinationWatch(first.params); - const socket = await listener.nextConnection(); + const firstSocket = await listener.nextConnection(); attachRunnerDestinationWatch(second.params); - socket.destroy(); + const secondSocket = await listener.nextConnection(); + firstSocket.destroy(); + secondSocket.destroy(); await vi.waitFor(() => assert.equal(second.onStop.mock.calls.length, 1)); assert.equal(first.onStop.mock.calls.length, 0); - assert.equal(listener.connections.length, 0); }); test('a handler resuming after its window was replaced leaves the newer watch tracked', async () => { @@ -373,13 +374,16 @@ test('a handler resuming after its window was replaced leaves the newer watch tr await new Promise((resolve) => setTimeout(resolve, 10)); closeRunnerDestinationWatch(DEVICE.id); - const second = watchParams(listener.port); - attachRunnerDestinationWatch(second.params); - await listener.nextConnection(); - await new Promise((resolve) => setTimeout(resolve, 120)); attachRunnerDestinationWatch(watchParams(listener.port).params); - await new Promise((resolve) => setTimeout(resolve, 30)); + const newer = await listener.nextConnection(); + let newerClosed = false; + newer.on('close', () => { + newerClosed = true; + }); + await new Promise((resolve) => setTimeout(resolve, 120)); + assert.equal(newerClosed, false); + closeRunnerDestinationWatch(DEVICE.id); + await vi.waitFor(() => assert.equal(newerClosed, true)); assert.equal(first.onStop.mock.calls.length, 0); - assert.equal(listener.connections.length, 0); }); diff --git a/packages/platform-apple/src/runner/runner-destination-watch.ts b/packages/platform-apple/src/runner/runner-destination-watch.ts index 8e2c886a19..33927244ff 100644 --- a/packages/platform-apple/src/runner/runner-destination-watch.ts +++ b/packages/platform-apple/src/runner/runner-destination-watch.ts @@ -109,29 +109,37 @@ const pendingWarmLossNotices = new Map(); * reboot starts within ~1s of the shutdown returning. One bounded re-read per event (never a * sample of a quiet runner) lets that churn settle so a mid-shutdown read cannot misclassify a * crash. A stop that lands after the reboot has finished still powers the device back off. + * + * The confirm delay, the recheck delay and the attach-retry budget are fixed in production; whole + * milliseconds in `AGENT_DEVICE_IOS_RUNNER_DESTINATION_CONFIRM_MS`, `..._RECHECK_MS` and + * `..._ATTACH_RETRY_MS` shorten them so tests do not wait production time. */ -function resolveConfirmDelayMs(env: NodeJS.ProcessEnv = process.env): number { - const raw = env.AGENT_DEVICE_IOS_RUNNER_DESTINATION_CONFIRM_MS?.trim(); - const parsed = raw ? Number(raw) : Number.NaN; - return Number.isFinite(parsed) && parsed >= 0 - ? Math.floor(parsed) - : DESTINATION_CONFIRM_DEFAULT_MS; +function resolveConfirmDelayMs(): number { + return readDurationEnvMs( + 'AGENT_DEVICE_IOS_RUNNER_DESTINATION_CONFIRM_MS', + DESTINATION_CONFIRM_DEFAULT_MS, + ); } -function resolveRecheckDelayMs(env: NodeJS.ProcessEnv = process.env): number { - const raw = env.AGENT_DEVICE_IOS_RUNNER_DESTINATION_RECHECK_MS?.trim(); - const parsed = raw ? Number(raw) : Number.NaN; - return Number.isFinite(parsed) && parsed >= 0 - ? Math.floor(parsed) - : DESTINATION_RECHECK_DEFAULT_MS; +function resolveRecheckDelayMs(): number { + return readDurationEnvMs( + 'AGENT_DEVICE_IOS_RUNNER_DESTINATION_RECHECK_MS', + DESTINATION_RECHECK_DEFAULT_MS, + ); } -function resolveAttachRetryBudgetMs(env: NodeJS.ProcessEnv = process.env): number { - const raw = env.AGENT_DEVICE_IOS_RUNNER_DESTINATION_ATTACH_RETRY_MS?.trim(); +function resolveAttachRetryBudgetMs(): number { + return readDurationEnvMs( + 'AGENT_DEVICE_IOS_RUNNER_DESTINATION_ATTACH_RETRY_MS', + ATTACH_RETRY_BUDGET_DEFAULT_MS, + ); +} + +/** A non-negative whole-millisecond override; anything else keeps the default. */ +function readDurationEnvMs(key: string, fallbackMs: number): number { + const raw = process.env[key]?.trim(); const parsed = raw ? Number(raw) : Number.NaN; - return Number.isFinite(parsed) && parsed >= 0 - ? Math.floor(parsed) - : ATTACH_RETRY_BUDGET_DEFAULT_MS; + return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : fallbackMs; } function delay(ms: number): Promise { @@ -161,21 +169,16 @@ export type RunnerDestinationWatchParams = { }; /** - * Attaches the watcher for one retention window. Idempotent per device: a fresh retention window - * re-arms the existing watch rather than stacking another. `isArmed` is consulted again at every + * Attaches the watcher for one retention window. A device has one watch: a fresh window replaces + * the existing one rather than stacking another. `isArmed` is consulted again at every * decision, so an attach that lands after its retention window already ended arms nothing that * can act. */ export function attachRunnerDestinationWatch(params: RunnerDestinationWatchParams): void { + // One watch per device, and a new attach replaces rather than amends: nothing carries over + // from a window another path already ended, so no caller has to have detached first. const existing = destinationWatches.get(params.device.id); - if (existing && !existing.detached) { - existing.sessionId = params.sessionId; - existing.runnerPid = params.runnerPid; - existing.armedAtMs = Date.now(); - existing.isArmed = params.isArmed; - existing.onStop = params.onStop; - return; - } + if (existing) detachWatch(existing); const watch: DestinationWatch = { device: params.device, sessionId: params.sessionId, diff --git a/src/daemon/session-lifecycle/internal/session-open-execution.ts b/src/daemon/session-lifecycle/internal/session-open-execution.ts index 7897d6f3a2..adb329c9bf 100644 --- a/src/daemon/session-lifecycle/internal/session-open-execution.ts +++ b/src/daemon/session-lifecycle/internal/session-open-execution.ts @@ -147,8 +147,8 @@ function warmRunnerLossWarning(notice: RunnerWarmLossNotice): string { ? 'the simulator was shut down externally while it was retained, so the runner was stopped before it could boot the simulator again' : 'it stopped answering while it was retained'; return ( - `The warm iOS runner left by the previous close was stopped: ${cause}. ` + - `This open started a fresh runner. reason=${notice.reason}` + `The warm iOS runner left by a previous close was stopped at ${new Date(notice.atMs).toISOString()}: ${cause}. ` + + `reason=${notice.reason}` ); } From 97f47a283ba2d4a216449b8489af5458f3573ee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 10 Oct 2026 05:06:36 +0200 Subject: [PATCH 11/20] docs(ios): say what the state probe throws, what the env overrides accept, and why two probe seams exist (#3321) Co-Authored-By: Claude Sonnet 5.5 --- packages/platform-apple/src/core/runner-host.ts | 4 ++-- packages/platform-apple/src/core/simulator.ts | 8 +++++++- .../src/runner/runner-destination-watch.ts | 9 +++++---- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/packages/platform-apple/src/core/runner-host.ts b/packages/platform-apple/src/core/runner-host.ts index db94ba5a61..b912bb88c6 100644 --- a/packages/platform-apple/src/core/runner-host.ts +++ b/packages/platform-apple/src/core/runner-host.ts @@ -51,8 +51,8 @@ import { runXcrun, } from './tool-provider.ts'; -/** The state probe answers in tens of milliseconds and must not become the reason a stop waits. */ -const SIMULATOR_STATE_PROBE_TIMEOUT_MS = 2_000; +/** `simctl list` takes ~0.7s per spawn on an idle host; an unanswered probe reads as a lost device, so the budget leaves room for a loaded one. */ +const SIMULATOR_STATE_PROBE_TIMEOUT_MS = 5_000; /** * The real host capabilities for `@agent-device/platform-apple/runner`: the one place diff --git a/packages/platform-apple/src/core/simulator.ts b/packages/platform-apple/src/core/simulator.ts index 5c4b71365e..276465441a 100644 --- a/packages/platform-apple/src/core/simulator.ts +++ b/packages/platform-apple/src/core/simulator.ts @@ -205,7 +205,13 @@ export async function ensureBootedSimulator( await openIosSimulatorApp({ signal: options.signal }); } -/** The listed power state of one Simulator (`Booted`, `Shutting Down`, `Shutdown`), or null when unreadable. */ +/** + * The listed power state of one Simulator (`Booted`, `Shutting Down`, `Shutdown`), or null when the + * listing exits non-zero or omits the device. A listing that exceeds `timeoutMs` rejects. This + * runs through the global tool provider; the readiness and shutdown runtimes read the same listing + * through their injected `AppleToolHost` in `simulator-state.ts`, which is a different execution + * seam and not interchangeable with this one. + */ export async function getSimulatorState( device: DeviceInfo, options: { signal?: AbortSignal; timeoutMs?: number } = {}, diff --git a/packages/platform-apple/src/runner/runner-destination-watch.ts b/packages/platform-apple/src/runner/runner-destination-watch.ts index 33927244ff..742c9f0adc 100644 --- a/packages/platform-apple/src/runner/runner-destination-watch.ts +++ b/packages/platform-apple/src/runner/runner-destination-watch.ts @@ -110,9 +110,10 @@ const pendingWarmLossNotices = new Map(); * sample of a quiet runner) lets that churn settle so a mid-shutdown read cannot misclassify a * crash. A stop that lands after the reboot has finished still powers the device back off. * - * The confirm delay, the recheck delay and the attach-retry budget are fixed in production; whole - * milliseconds in `AGENT_DEVICE_IOS_RUNNER_DESTINATION_CONFIRM_MS`, `..._RECHECK_MS` and - * `..._ATTACH_RETRY_MS` shorten them so tests do not wait production time. + * * The confirm delay, the recheck delay and the attach-retry budget have production defaults; a + * non-negative millisecond value in `AGENT_DEVICE_IOS_RUNNER_DESTINATION_CONFIRM_MS`, + * `..._RECHECK_MS` or `..._ATTACH_RETRY_MS` replaces one. Only tests set them, to avoid waiting + * production time. */ function resolveConfirmDelayMs(): number { return readDurationEnvMs( @@ -135,7 +136,7 @@ function resolveAttachRetryBudgetMs(): number { ); } -/** A non-negative whole-millisecond override; anything else keeps the default. */ +/** A non-negative millisecond override, fractions floored; anything else keeps the default. */ function readDurationEnvMs(key: string, fallbackMs: number): number { const raw = process.env[key]?.trim(); const parsed = raw ? Number(raw) : Number.NaN; From e2fb9559ea53cbcad7fba351537e6ddbf18d4112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 10 Oct 2026 05:11:13 +0200 Subject: [PATCH 12/20] fix(ios): a state probe that times out stops the runner as runner_unreachable, not as a shutdown (#3321) Co-Authored-By: Claude Sonnet 5.5 --- .../__tests__/runner-destination-watch.test.ts | 4 ++-- .../src/runner/runner-destination-watch.ts | 15 +++++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts b/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts index 0efbd573d1..3385031b4a 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts @@ -163,7 +163,7 @@ test('a device listed as shutting down is a loss, not a crash', async () => { assert.equal((await takeRunnerWarmLossNotice(DEVICE.id))?.reason, 'runner_destination_lost'); }); -test('an unreadable device state is no evidence of a healthy device', async () => { +test('an unreadable device state stops the runner without claiming a shutdown', async () => { const listener = await listen(); stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() - 60_000 }], state: null }); const { params, onStop } = watchParams(listener.port); @@ -172,7 +172,7 @@ test('an unreadable device state is no evidence of a healthy device', async () = (await listener.nextConnection()).destroy(); await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); - assert.equal((await takeRunnerWarmLossNotice(DEVICE.id))?.reason, 'runner_destination_lost'); + assert.equal((await takeRunnerWarmLossNotice(DEVICE.id))?.reason, 'runner_unreachable'); }); test('a boot unobservable on both reads while the destination process lives is a loss', async () => { diff --git a/packages/platform-apple/src/runner/runner-destination-watch.ts b/packages/platform-apple/src/runner/runner-destination-watch.ts index 742c9f0adc..4226c13696 100644 --- a/packages/platform-apple/src/runner/runner-destination-watch.ts +++ b/packages/platform-apple/src/runner/runner-destination-watch.ts @@ -76,6 +76,12 @@ type LossVerdict = Readonly<{ const DESTINATION_LOST: LossVerdict = { action: 'stop', noticeReason: 'runner_destination_lost' }; +/** + * The listing timed out or was unreadable, so nothing proves the device changed. Stopping is still + * the safe side, but the notice must not claim a shutdown nobody observed. + */ +const STATE_UNVERIFIABLE: LossVerdict = { action: 'stop', noticeReason: 'runner_unreachable' }; + type DestinationWatch = { device: DeviceInfo; sessionId: string; @@ -335,7 +341,8 @@ async function handleRefusedAttach( * process, so stopping is plain lease cleanup. Otherwise the device must be listed `Booted` on the * boot the window began with, and anything else is destination loss: a shut-down device keeps its * old `launchd_sim` listed for ~6s, so the boot witness alone would read the old boot and call it - * a crash, and an unreadable listing is no evidence of a healthy device. A boot newer than the + * a crash. A listing that timed out or was unreadable is no evidence of a healthy device, so it + * stops the runner too, but as `runner_unreachable`: it cannot support a claim about the device. A boot newer than the * window is Xcode's reboot; a boot unobservable twice is a device that is down with a reboot * promise that may already be in flight. Stopping pre-empts that promise in every loss case. */ @@ -343,9 +350,9 @@ async function classifyConnectedLoss(watch: DestinationWatch): Promise Date: Sat, 10 Oct 2026 05:15:09 +0200 Subject: [PATCH 13/20] docs(ios): drop a stray bullet marker in the watch comment (#3321) Co-Authored-By: Claude Sonnet 5.5 --- packages/platform-apple/src/runner/runner-destination-watch.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/platform-apple/src/runner/runner-destination-watch.ts b/packages/platform-apple/src/runner/runner-destination-watch.ts index 4226c13696..37320a37fd 100644 --- a/packages/platform-apple/src/runner/runner-destination-watch.ts +++ b/packages/platform-apple/src/runner/runner-destination-watch.ts @@ -116,7 +116,7 @@ const pendingWarmLossNotices = new Map(); * sample of a quiet runner) lets that churn settle so a mid-shutdown read cannot misclassify a * crash. A stop that lands after the reboot has finished still powers the device back off. * - * * The confirm delay, the recheck delay and the attach-retry budget have production defaults; a + * The confirm delay, the recheck delay and the attach-retry budget have production defaults; a * non-negative millisecond value in `AGENT_DEVICE_IOS_RUNNER_DESTINATION_CONFIRM_MS`, * `..._RECHECK_MS` or `..._ATTACH_RETRY_MS` replaces one. Only tests set them, to avoid waiting * production time. From 9c71e667378ad2c346befda9dadd19e4ea41af99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 10 Oct 2026 05:20:14 +0200 Subject: [PATCH 14/20] fix(ios): give an unverifiable simulator state its own typed reason and cause text (#3321) runner_unreachable meant both 'the runner stopped answering' and 'our own state probe could not be read'. The probe verdict is now runner_destination_unverified, and the open warning's cause text is a total map over the reasons. Co-Authored-By: Claude Sonnet 5.5 --- .../runner-destination-watch.test.ts | 5 ++- .../src/runner/runner-destination-watch.ts | 12 ++++-- src/commands/schema/cli-help.ts | 2 +- .../request-router-open-claim.test.ts | 43 +++++++++++++++++++ .../internal/session-open-execution.ts | 16 ++++--- website/docs/docs/sessions.md | 2 +- 6 files changed, 68 insertions(+), 12 deletions(-) diff --git a/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts b/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts index 3385031b4a..75c9918a30 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts @@ -172,7 +172,10 @@ test('an unreadable device state stops the runner without claiming a shutdown', (await listener.nextConnection()).destroy(); await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); - assert.equal((await takeRunnerWarmLossNotice(DEVICE.id))?.reason, 'runner_unreachable'); + assert.equal( + (await takeRunnerWarmLossNotice(DEVICE.id))?.reason, + 'runner_destination_unverified', + ); }); test('a boot unobservable on both reads while the destination process lives is a loss', async () => { diff --git a/packages/platform-apple/src/runner/runner-destination-watch.ts b/packages/platform-apple/src/runner/runner-destination-watch.ts index 37320a37fd..c7332eba0f 100644 --- a/packages/platform-apple/src/runner/runner-destination-watch.ts +++ b/packages/platform-apple/src/runner/runner-destination-watch.ts @@ -11,7 +11,10 @@ import { * Why a runner retained after `close` was stopped before the session asked for it again. * Keyed behavior: consumers branch on these values, never on prose. */ -export type RunnerWarmLossReason = 'runner_destination_lost' | 'runner_unreachable'; +export type RunnerWarmLossReason = + | 'runner_destination_lost' + | 'runner_unreachable' + | 'runner_destination_unverified'; /** * What a retained runner's owner records when the watcher stops it: the typed reason, the session @@ -80,7 +83,10 @@ const DESTINATION_LOST: LossVerdict = { action: 'stop', noticeReason: 'runner_de * The listing timed out or was unreadable, so nothing proves the device changed. Stopping is still * the safe side, but the notice must not claim a shutdown nobody observed. */ -const STATE_UNVERIFIABLE: LossVerdict = { action: 'stop', noticeReason: 'runner_unreachable' }; +const STATE_UNVERIFIABLE: LossVerdict = { + action: 'stop', + noticeReason: 'runner_destination_unverified', +}; type DestinationWatch = { device: DeviceInfo; @@ -342,7 +348,7 @@ async function handleRefusedAttach( * boot the window began with, and anything else is destination loss: a shut-down device keeps its * old `launchd_sim` listed for ~6s, so the boot witness alone would read the old boot and call it * a crash. A listing that timed out or was unreadable is no evidence of a healthy device, so it - * stops the runner too, but as `runner_unreachable`: it cannot support a claim about the device. A boot newer than the + * stops the runner too, but as `runner_destination_unverified`: it cannot support a claim about the device. A boot newer than the * window is Xcode's reboot; a boot unobservable twice is a device that is down with a reboot * promise that may already be in flight. Stopping pre-empts that promise in every loss case. */ diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index aa3735c20f..d49efce122 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -530,7 +530,7 @@ Android physical-device prerequisites: Runner and daemon lifecycle (applies to simulators too): open without --relaunch is idempotent-foreground for an already-running app (it brings the process forward; it does not restart it). open --relaunch restarts the app; on iOS simulators this collapses to one simctl launch --terminate-running-process call instead of a separate terminate-then-launch. No runner read launches a session app that is not running: snapshot, wait, is, get, a reading find, and an interaction's leading reads (a gesture's viewport read, the capture that resolves a selector click/fill) answer the retriable APP_NOT_RUNNING instead of bare-launching over a launch SpringBoard still holds behind its deep-link confirmation. Only open, activate, and a command that mutates without a leading read bring a stopped app up. - close keeps a healthy iOS simulator XCTest runner warm by default so the next open on that simulator (same udid in the same simulator set) skips the runner build, unless --shutdown was requested, the session was recording, or the session held a device lease. A retained runner auto-stops after an idle window (default 5 minutes); set AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS to override, or 0 to disable idle stop and retain until daemon exit. A simulator shut down externally while its runner is retained is detected and the retained runner is stopped, because the retained Xcode destination would otherwise reboot the simulator and leave it powered on; the next open reports reason=runner_destination_lost (or reason=runner_unreachable when the runner stopped answering) in its warnings. + close keeps a healthy iOS simulator XCTest runner warm by default so the next open on that simulator (same udid in the same simulator set) skips the runner build, unless --shutdown was requested, the session was recording, or the session held a device lease. A retained runner auto-stops after an idle window (default 5 minutes); set AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS to override, or 0 to disable idle stop and retain until daemon exit. A simulator shut down externally while its runner is retained is detected and the retained runner is stopped, because the retained Xcode destination would otherwise reboot the simulator and leave it powered on; the next open reports reason=runner_destination_lost in its warnings, or reason=runner_unreachable when the runner stopped answering, or reason=runner_destination_unverified when the runner's connection closed and the simulator's state could not be read. Each AGENT_DEVICE_STATE_DIR runs its own daemon. It self-exits after an idle window (default 5 minutes, matching the runner idle-stop default) once it has no open sessions, no in-flight requests, and no active recording; set AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS to override, or 0 to disable idle reap. On a machine shared by several agents, set AGENT_DEVICE_SESSION_IDLE_TIMEOUT_MS to also expire an individual session that has taken a host-global device claim and then received no commands for that long; the claim is released and the next command on that session answers SESSION_NOT_FOUND with details.reason SESSION_IDLE_EXPIRED naming the window and the device. Off by default, because on a shared host "idle" and "thinking" are indistinguishable: only set it where an abandoned claim blocking every other agent is the worse failure. Sessions holding a remote lease or any active capture (recording, logs, audio, performance, or trace) are never expired this way. A stale iOS runner lease — its owner process dead, or its AGENT_DEVICE_STATE_DIR deleted — is reclaimed automatically instead of failing with "is already owned by another agent-device daemon". A live owner's runner is also reclaimed when the requesting daemon holds the host-global device claim for that device: claims are exclusive, so holding one proves the runner's owner released the device and merely kept the runner warm. The error remains only for owners outside claim arbitration (a pre-claims build, or daemons pointed at different claim stores). diff --git a/src/daemon/__tests__/request-router-open-claim.test.ts b/src/daemon/__tests__/request-router-open-claim.test.ts index b5ac0220c0..98485a531c 100644 --- a/src/daemon/__tests__/request-router-open-claim.test.ts +++ b/src/daemon/__tests__/request-router-open-claim.test.ts @@ -213,6 +213,49 @@ test('open warns with the typed reason when a warm runner was stopped for a lost } }); +test('open warns with its own typed reason when a warm runner was stopped on an unverifiable state', async () => { + const sessionStore = makeSessionStore('agent-device-router-open-warm-unverified-'); + const device = makeIosDevice('SIM-WARM-UNVERIFIED'); + mockResolveTargetDevice.mockResolvedValue(device); + mockTakeRunnerWarmLossNotice.mockResolvedValueOnce({ + reason: 'runner_destination_unverified', + deviceId: device.id, + sessionId: 'warm-unverified', + atMs: Date.now(), + }); + const claimsDir = mkdtempForTestSync('agent-device-router-open-warm-unverified-claims-'); + const previousClaimsDir = process.env.AGENT_DEVICE_CLAIMS_DIR; + process.env.AGENT_DEVICE_CLAIMS_DIR = claimsDir; + + try { + const opened = await createOpenHandler(sessionStore)( + openRequest('warm-unverified', { platform: 'ios' }, 'req-open-warm-unverified', {}, [ + 'FixtureApp', + ]), + ); + + expect(opened.ok).toBe(true); + if (!opened.ok) return; + expect(mockTakeRunnerWarmLossNotice).toHaveBeenCalledWith(device.id); + expect(opened.data?.warnings).toEqual([ + expect.stringContaining('reason=runner_destination_unverified'), + ]); + expect(String(opened.data?.warnings)).not.toContain('stopped answering'); + + const reopened = await createOpenHandler(sessionStore)( + openRequest('warm-unverified', { platform: 'ios' }, 'req-open-warm-unverified-again', {}, [ + 'FixtureApp', + ]), + ); + expect(reopened.ok).toBe(true); + if (reopened.ok) expect(reopened.data?.warnings).toBeUndefined(); + } finally { + if (previousClaimsDir === undefined) delete process.env.AGENT_DEVICE_CLAIMS_DIR; + else process.env.AGENT_DEVICE_CLAIMS_DIR = previousClaimsDir; + fs.rmSync(claimsDir, { recursive: true, force: true }); + } +}); + // The production reopen path never re-acquires the claim, so renewal has to ride the successful // existing-session open itself: without it, an owner that came back after a reboot still carries a // pre-reboot stamp and loses the device to the next caller that asks. diff --git a/src/daemon/session-lifecycle/internal/session-open-execution.ts b/src/daemon/session-lifecycle/internal/session-open-execution.ts index adb329c9bf..0ef8809dc0 100644 --- a/src/daemon/session-lifecycle/internal/session-open-execution.ts +++ b/src/daemon/session-lifecycle/internal/session-open-execution.ts @@ -141,14 +141,18 @@ async function readWarmRunnerLossNotice( * be stopped because the Simulator was shut down externally while it was retained (#3321). Names * the typed reason so automation can key on it without parsing prose. */ +const WARM_RUNNER_LOSS_CAUSE: Record = { + runner_destination_lost: + 'the simulator was shut down externally while it was retained, so the runner was stopped before it could boot the simulator again', + runner_unreachable: 'it stopped answering while it was retained', + runner_destination_unverified: + "its connection closed while it was retained and the simulator's state could not be read, so the runner was stopped as a precaution", +}; + function warmRunnerLossWarning(notice: RunnerWarmLossNotice): string { - const cause = - notice.reason === 'runner_destination_lost' - ? 'the simulator was shut down externally while it was retained, so the runner was stopped before it could boot the simulator again' - : 'it stopped answering while it was retained'; return ( - `The warm iOS runner left by a previous close was stopped at ${new Date(notice.atMs).toISOString()}: ${cause}. ` + - `reason=${notice.reason}` + `The warm iOS runner left by a previous close was stopped at ${new Date(notice.atMs).toISOString()}: ` + + `${WARM_RUNNER_LOSS_CAUSE[notice.reason]}. reason=${notice.reason}` ); } diff --git a/website/docs/docs/sessions.md b/website/docs/docs/sessions.md index 3aa3ccf8f7..e6e6734ac1 100644 --- a/website/docs/docs/sessions.md +++ b/website/docs/docs/sessions.md @@ -117,7 +117,7 @@ agent-device close --shutdown Without `--shutdown`, `close` keeps the iOS runner warm for the next `open`. If you shut that simulator down yourself (`xcrun simctl shutdown`) while the runner is waiting, agent-device stops the runner so it doesn't boot the simulator again, and the next `open` returns a warning that ends in -`reason=runner_destination_lost`, or `reason=runner_unreachable` when the runner stopped answering. +`reason=runner_destination_lost`. It ends in `reason=runner_unreachable` when the runner stopped answering, or `reason=runner_destination_unverified` when its connection closed and the simulator's state could not be read. ## Wait for a slow or busy device From dd9c0ae57748a162eb531962bb0fcc6ec77b51ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 10 Oct 2026 05:28:28 +0200 Subject: [PATCH 15/20] docs(ios): wrap the watch and probe comments to the 100-column width (#3321) Comment-only; no behaviour change. Co-Authored-By: Claude Sonnet 5.5 --- .../platform-apple/src/core/runner-host.ts | 5 ++- .../src/runner/runner-destination-watch.ts | 34 ++++++++++--------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/packages/platform-apple/src/core/runner-host.ts b/packages/platform-apple/src/core/runner-host.ts index b912bb88c6..b66d2ee08f 100644 --- a/packages/platform-apple/src/core/runner-host.ts +++ b/packages/platform-apple/src/core/runner-host.ts @@ -51,7 +51,10 @@ import { runXcrun, } from './tool-provider.ts'; -/** `simctl list` takes ~0.7s per spawn on an idle host; an unanswered probe reads as a lost device, so the budget leaves room for a loaded one. */ +/** + * `simctl list` takes ~0.7s per spawn on an idle host; an unanswered probe reads as an unverified + * device, so the budget leaves room for a loaded one. + */ const SIMULATOR_STATE_PROBE_TIMEOUT_MS = 5_000; /** diff --git a/packages/platform-apple/src/runner/runner-destination-watch.ts b/packages/platform-apple/src/runner/runner-destination-watch.ts index c7332eba0f..5cd50f7fa9 100644 --- a/packages/platform-apple/src/runner/runner-destination-watch.ts +++ b/packages/platform-apple/src/runner/runner-destination-watch.ts @@ -93,7 +93,7 @@ type DestinationWatch = { sessionId: string; port: number; runnerPid: number | undefined; - /** When this retention window began; a boot starting later belongs to a replacement device state. */ + /** When this retention window began; a boot starting later is a replacement device state. */ armedAtMs: number; /** Re-read at every decision: false once any path has left the idle-retention window. */ isArmed: () => boolean; @@ -107,7 +107,7 @@ type DestinationWatch = { * runner generation started after the device changed under the session. */ replacement: boolean; - /** The in-flight loss decision, which a notice reader waits for while the window is still open. */ + /** The in-flight loss decision, which a notice reader waits for while the window is open. */ deciding: Promise | undefined; detached: boolean; }; @@ -298,13 +298,14 @@ async function decide(watch: DestinationWatch, run: () => Promise): Promis } /** - * The attach never reached a listener. A rebooting destination is usually unreachable for a - * couple of seconds while Xcode restarts the app onto the same port, so retry at a flat one-second - * cadence inside a bounded budget; a retry that connects classifies the replacement generation. Only when the budget runs out is the retained runner called - * what it has proven to be: unable to answer anything. By then the destination process's own - * liveness decides whether this was a crash (Xcode gave up with it, the device lost its boot - * owner) or a listener that died while Xcode kept the device — the latter is exactly the silent - * power-on #3321 is about, so the runner goes with a notice. + * The attach never reached a listener. A rebooting destination is usually unreachable for a couple + * of seconds while Xcode restarts the app onto the same port, so retry at a flat one-second cadence + * inside a bounded budget; a retry that connects classifies the replacement generation. Only when + * the budget runs out is the retained runner called what it has proven to be: unable to answer + * anything. By then the destination process's own liveness decides whether this was a crash (Xcode + * gave up with it, the device lost its boot owner) or a listener that died while Xcode kept the + * device — the latter is exactly the silent power-on #3321 is about, so the runner goes with a + * notice. */ async function handleRefusedAttach( watch: DestinationWatch, @@ -342,15 +343,16 @@ async function handleRefusedAttach( } /** - * Reads whether the device behind the runner stopped being the device this session was armed on. - * A dead destination process answers for itself before the probe: Xcode cannot reboot from a dead + * Reads whether the device behind the runner stopped being the device this session was armed on. A + * dead destination process answers for itself before the probe: Xcode cannot reboot from a dead * process, so stopping is plain lease cleanup. Otherwise the device must be listed `Booted` on the * boot the window began with, and anything else is destination loss: a shut-down device keeps its - * old `launchd_sim` listed for ~6s, so the boot witness alone would read the old boot and call it - * a crash. A listing that timed out or was unreadable is no evidence of a healthy device, so it - * stops the runner too, but as `runner_destination_unverified`: it cannot support a claim about the device. A boot newer than the - * window is Xcode's reboot; a boot unobservable twice is a device that is down with a reboot - * promise that may already be in flight. Stopping pre-empts that promise in every loss case. + * old `launchd_sim` listed for ~6s, so the boot witness alone would read the old boot and call it a + * crash. A listing that timed out or was unreadable is no evidence of a healthy device, so it stops + * the runner too, but as `runner_destination_unverified`: it cannot support a claim about the + * device. A boot newer than the window is Xcode's reboot; a boot unobservable twice is a device + * that is down with a reboot promise that may already be in flight. Stopping pre-empts that promise + * in every loss case. */ async function classifyConnectedLoss(watch: DestinationWatch): Promise { if (watch.runnerPid !== undefined && !isProcessAlive(watch.runnerPid)) { From fe6a625d95950dcd899c192e3b3a2e15ede16218 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 10 Oct 2026 08:10:16 +0200 Subject: [PATCH 16/20] refactor(ios): stop a retained runner on any lost listener instead of classifying the loss (#3321) Drops the boot-identity classification, same-boot re-arm, replacement-generation retry, confirm/recheck timers, their env vars, the two host ports and the extra probe export. The watcher now stops the retained runner when its idle connection closes, or when the port stays refused past a bounded retry budget; the next open reports reason=runner_connection_lost. Co-Authored-By: Claude Sonnet 5.5 --- .../platform-apple/src/core/runner-client.ts | 2 +- .../platform-apple/src/core/runner-host.ts | 16 - packages/platform-apple/src/core/simulator.ts | 20 +- .../runner-close-finalization.test.ts | 10 +- .../runner-destination-watch.test.ts | 328 +++--------- packages/platform-apple/src/runner/host.ts | 17 - .../src/runner/runner-destination-watch.ts | 476 +++--------------- .../src/runner/runner-session.ts | 6 +- .../daemon-platform-runtime-inventory.ts | 2 +- src/commands/schema/cli-help.ts | 2 +- .../request-router-open-claim.test.ts | 54 +- .../internal/session-open-execution.ts | 19 +- ...latform-runtime-warm-runner-notice.test.ts | 7 +- src/platform-runtime-warm-runner-notice.ts | 6 +- website/docs/docs/sessions.md | 2 +- 15 files changed, 161 insertions(+), 806 deletions(-) diff --git a/packages/platform-apple/src/core/runner-client.ts b/packages/platform-apple/src/core/runner-client.ts index d43a39a3bd..4b6ecb827e 100644 --- a/packages/platform-apple/src/core/runner-client.ts +++ b/packages/platform-apple/src/core/runner-client.ts @@ -38,7 +38,7 @@ import type { RunnerWarmLossNotice } from '../runner/runner-destination-watch.ts /** * Takes the #3321 warm-runner loss notice recorded for this device, once, if a retained runner was - * stopped because its Simulator destination was replaced under it. Loaded on call, not at import: + * stopped because its connection closed during retention. Loaded on call, not at import: * the watcher module must not join the façade closures the eager-closure budget holds flat, and * the read happens once per `open` at most. */ diff --git a/packages/platform-apple/src/core/runner-host.ts b/packages/platform-apple/src/core/runner-host.ts index b66d2ee08f..6790d89a7c 100644 --- a/packages/platform-apple/src/core/runner-host.ts +++ b/packages/platform-apple/src/core/runner-host.ts @@ -51,12 +51,6 @@ import { runXcrun, } from './tool-provider.ts'; -/** - * `simctl list` takes ~0.7s per spawn on an idle host; an unanswered probe reads as an unverified - * device, so the budget leaves room for a loaded one. - */ -const SIMULATOR_STATE_PROBE_TIMEOUT_MS = 5_000; - /** * The real host capabilities for `@agent-device/platform-apple/runner`: the one place * the runner package's port meets the root-owned utilities. Consumed by the @@ -115,14 +109,4 @@ export const appleRunnerHost: AppleRunnerHost = { visitXmlPlistEntries, leaseOwnerStateDir: getRunnerLeaseOwnerStateDir, hasDeviceClaimAuthority: (device) => getRunnerDeviceClaimAuthorityProbe()?.(device) ?? false, - observeSimulatorState: async (device) => { - try { - const { getSimulatorState } = await import('./simulator.ts'); - return await getSimulatorState(device, { timeoutMs: SIMULATOR_STATE_PROBE_TIMEOUT_MS }); - } catch { - return null; - } - }, - observeSimulatorBootTimeMs: async (device) => - (await import('../simulator-boot.ts')).observeSimulatorBootTimeMs(device), }; diff --git a/packages/platform-apple/src/core/simulator.ts b/packages/platform-apple/src/core/simulator.ts index 276465441a..7317ada69a 100644 --- a/packages/platform-apple/src/core/simulator.ts +++ b/packages/platform-apple/src/core/simulator.ts @@ -85,7 +85,7 @@ export async function ensureBootedSimulator( const state = wasSimulatorRecentlyObservedBooted(device) ? 'Booted' - : await getSimulatorState(device, { signal: options.signal }); + : await getSimulatorState(device, options.signal); if (state === 'Booted') { markSimulatorBooted(device); return; @@ -148,7 +148,7 @@ export async function ensureBootedSimulator( requireExecSuccess(bootStatusResult, 'simctl bootstatus failed'); - const nextState = await getSimulatorState(device, { signal: options.signal }); + const nextState = await getSimulatorState(device, options.signal); if (nextState !== 'Booted') { throw new AppError('COMMAND_FAILED', 'Simulator is still booting', { state: nextState }); } @@ -205,22 +205,12 @@ export async function ensureBootedSimulator( await openIosSimulatorApp({ signal: options.signal }); } -/** - * The listed power state of one Simulator (`Booted`, `Shutting Down`, `Shutdown`), or null when the - * listing exits non-zero or omits the device. A listing that exceeds `timeoutMs` rejects. This - * runs through the global tool provider; the readiness and shutdown runtimes read the same listing - * through their injected `AppleToolHost` in `simulator-state.ts`, which is a different execution - * seam and not interchangeable with this one. - */ -export async function getSimulatorState( - device: DeviceInfo, - options: { signal?: AbortSignal; timeoutMs?: number } = {}, -): Promise { +async function getSimulatorState(device: DeviceInfo, signal?: AbortSignal): Promise { const simctlArgs = buildSimctlArgsForDevice(device, ['list', 'devices', '-j']); const result = await runXcrun(simctlArgs, { allowFailure: true, - signal: options.signal, - timeoutMs: options.timeoutMs ?? IOS_SIMCTL_LIST_TIMEOUT_MS, + signal, + timeoutMs: IOS_SIMCTL_LIST_TIMEOUT_MS, }); if (result.exitCode !== 0) return null; return readSimctlDeviceState(result.stdout, device.id); diff --git a/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts b/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts index a60356a854..4702a3ad8e 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts @@ -378,7 +378,7 @@ test('a failed start does not re-retain a runner a concurrent start already took assert.doesNotMatch(phases.join('\n'), /ios_runner_idle_stop_scheduled/); }); -test('a destination lost during retention takes the retained runner and its lease down (#3321)', async () => { +test('a runner connection lost during retention takes the retained runner and its lease down (#3321)', async () => { const device = deviceNamed('close-finalize-destination-lost'); const connections: net.Socket[] = []; const destination = net.createServer((socket) => { @@ -387,11 +387,6 @@ test('a destination lost during retention takes the retained runner and its leas }); await new Promise((resolve) => destination.listen(0, '127.0.0.1', resolve)); mockGetFreePort.mockResolvedValue((destination.address() as net.AddressInfo).port); - vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_DESTINATION_CONFIRM_MS', '5'); - appleRunnerTestHost.update({ - observeSimulatorState: async () => 'Shutdown', - observeSimulatorBootTimeMs: async () => ({ observed: true, bootedAtMs: 1 }), - }); try { const session = await ensureRunnerSession(device, {}); await closeFinalizationLifecycle(device, []).finalizeApplicationClose(closeInput(true)); @@ -404,9 +399,8 @@ test('a destination lost during retention takes the retained runner and its leas }); assert.equal(runnerLeaseExists(device.id), false); assert.equal(session.state, 'stopped'); - assert.equal((await takeRunnerWarmLossNotice(device.id))?.reason, 'runner_destination_lost'); + assert.ok(takeRunnerWarmLossNotice(device.id)); } finally { - vi.unstubAllEnvs(); await new Promise((resolve) => destination.close(() => resolve())); } }); diff --git a/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts b/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts index 75c9918a30..7aa9e5c1d1 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts @@ -1,7 +1,6 @@ -import { afterEach, beforeEach, test, vi } from 'vitest'; +import { afterEach, test, vi } from 'vitest'; import assert from 'node:assert/strict'; import net from 'node:net'; -import { appleRunnerTestHost } from '../test-host.ts'; import { attachRunnerDestinationWatch, closeRunnerDestinationWatch, @@ -10,7 +9,6 @@ import { import { IOS_SIMULATOR } from './device-fixtures.ts'; const DEVICE = IOS_SIMULATOR; -const RUNNER_PID = 4242; type FakeListener = { port: number; @@ -21,7 +19,7 @@ type FakeListener = { const listeners: FakeListener[] = []; -async function listen(): Promise { +async function listen(port = 0): Promise { const connections: net.Socket[] = []; const waiting: Array<(socket: net.Socket) => void> = []; const server = net.createServer((socket) => { @@ -30,7 +28,7 @@ async function listen(): Promise { if (waiter) waiter(socket); else connections.push(socket); }); - await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + await new Promise((resolve) => server.listen(port, '127.0.0.1', resolve)); const listener: FakeListener = { port: (server.address() as net.AddressInfo).port, connections, @@ -65,328 +63,134 @@ function watchParams(port: number, overrides: { isArmed?: () => boolean } = {}) device: DEVICE, sessionId: 'session-1', port, - runnerPid: RUNNER_PID, isArmed: overrides.isArmed ?? (() => true), onStop, }, }; } -function stubHost(options: { - boots: Array<{ observed: true; bootedAtMs: number } | { observed: false; reason: 'unobserved' }>; - processAlive?: boolean; - state?: string | null; -}) { - const observeSimulatorBootTimeMs = vi.fn(async () => { - const next = options.boots.length > 1 ? options.boots.shift() : options.boots[0]; - return next ?? { observed: false as const, reason: 'unobserved' as const }; - }); - appleRunnerTestHost.update({ - observeSimulatorBootTimeMs, - observeSimulatorState: async () => (options.state === undefined ? 'Booted' : options.state), - isProcessAlive: () => options.processAlive ?? true, - }); - return observeSimulatorBootTimeMs; -} - -beforeEach(() => { - vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_DESTINATION_CONFIRM_MS', '5'); - vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_DESTINATION_RECHECK_MS', '5'); - vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_DESTINATION_ATTACH_RETRY_MS', '0'); -}); +const realSetTimeout = globalThis.setTimeout; +const realTick = () => new Promise((resolve) => realSetTimeout(resolve, 20)); afterEach(async () => { + vi.useRealTimers(); closeRunnerDestinationWatch(DEVICE.id); - await takeRunnerWarmLossNotice(DEVICE.id); - vi.unstubAllEnvs(); + takeRunnerWarmLossNotice(DEVICE.id); await Promise.all(listeners.splice(0).map((listener) => listener.close())); }); -test('an established connection that closes after a newer boot stops the runner with a notice', async () => { +test('an established connection that closes stops the runner and leaves one notice', async () => { const listener = await listen(); - stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() + 60_000 }] }); const { params, onStop } = watchParams(listener.port); attachRunnerDestinationWatch(params); (await listener.nextConnection()).destroy(); await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); - const notice = await takeRunnerWarmLossNotice(DEVICE.id); - assert.equal(notice?.reason, 'runner_destination_lost'); - assert.equal(notice?.sessionId, 'session-1'); - assert.equal(await takeRunnerWarmLossNotice(DEVICE.id), undefined); + assert.ok(takeRunnerWarmLossNotice(DEVICE.id)); + assert.equal(takeRunnerWarmLossNotice(DEVICE.id), undefined); }); -test('a close on the boot the watch was armed on re-arms instead of stopping', async () => { +test('a quiet established connection stops nothing', async () => { const listener = await listen(); - const observe = stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() - 60_000 }] }); const { params, onStop } = watchParams(listener.port); attachRunnerDestinationWatch(params); - (await listener.nextConnection()).destroy(); - const replacement = await listener.nextConnection(); + await listener.nextConnection(); + await realTick(); - assert.ok(observe.mock.calls.length >= 1); assert.equal(onStop.mock.calls.length, 0); - assert.equal(await takeRunnerWarmLossNotice(DEVICE.id), undefined); - replacement.destroy(); -}); - -test('a device listed as shut down is a loss even while its old boot is still observable', async () => { - const listener = await listen(); - const observe = stubHost({ - boots: [{ observed: true, bootedAtMs: Date.now() - 60_000 }], - state: 'Shutdown', - }); - const { params, onStop } = watchParams(listener.port); - - attachRunnerDestinationWatch(params); - (await listener.nextConnection()).destroy(); - await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); - - assert.equal(observe.mock.calls.length, 0); - assert.equal((await takeRunnerWarmLossNotice(DEVICE.id))?.reason, 'runner_destination_lost'); -}); - -test('a device listed as shutting down is a loss, not a crash', async () => { - const listener = await listen(); - stubHost({ - boots: [{ observed: true, bootedAtMs: Date.now() - 60_000 }], - state: 'Shutting Down', - }); - const { params, onStop } = watchParams(listener.port); - - attachRunnerDestinationWatch(params); - (await listener.nextConnection()).destroy(); - await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); - - assert.equal((await takeRunnerWarmLossNotice(DEVICE.id))?.reason, 'runner_destination_lost'); -}); - -test('an unreadable device state stops the runner without claiming a shutdown', async () => { - const listener = await listen(); - stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() - 60_000 }], state: null }); - const { params, onStop } = watchParams(listener.port); - - attachRunnerDestinationWatch(params); - (await listener.nextConnection()).destroy(); - await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); - - assert.equal( - (await takeRunnerWarmLossNotice(DEVICE.id))?.reason, - 'runner_destination_unverified', - ); + assert.equal(takeRunnerWarmLossNotice(DEVICE.id), undefined); }); -test('a boot unobservable on both reads while the destination process lives is a loss', async () => { +test('a close after the retention window ended acts on nothing', async () => { const listener = await listen(); - const observe = stubHost({ boots: [{ observed: false, reason: 'unobserved' }] }); - const { params, onStop } = watchParams(listener.port); + let armed = true; + const { params, onStop } = watchParams(listener.port, { isArmed: () => armed }); attachRunnerDestinationWatch(params); - (await listener.nextConnection()).destroy(); - await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); + const connection = await listener.nextConnection(); + armed = false; + connection.destroy(); + await realTick(); - assert.equal(observe.mock.calls.length, 2); - assert.equal((await takeRunnerWarmLossNotice(DEVICE.id))?.reason, 'runner_destination_lost'); + assert.equal(onStop.mock.calls.length, 0); + assert.equal(takeRunnerWarmLossNotice(DEVICE.id), undefined); }); -test('a boot that reappears on the recheck with the armed boot re-arms', async () => { +test('closing the watch detaches it so a later runner death stops nothing', async () => { const listener = await listen(); - stubHost({ - boots: [ - { observed: false, reason: 'unobserved' }, - { observed: true, bootedAtMs: Date.now() - 60_000 }, - ], - }); const { params, onStop } = watchParams(listener.port); attachRunnerDestinationWatch(params); - (await listener.nextConnection()).destroy(); - const replacement = await listener.nextConnection(); + const connection = await listener.nextConnection(); + closeRunnerDestinationWatch(DEVICE.id); + connection.destroy(); + await realTick(); assert.equal(onStop.mock.calls.length, 0); - replacement.destroy(); + assert.equal(takeRunnerWarmLossNotice(DEVICE.id), undefined); }); -test('a dead runner process is plain cleanup without a notice', async () => { +test('a second retention window replaces the existing watch instead of stacking another', async () => { const listener = await listen(); - const observe = stubHost({ boots: [{ observed: true, bootedAtMs: 0 }], processAlive: false }); - const { params, onStop } = watchParams(listener.port); - - attachRunnerDestinationWatch(params); - (await listener.nextConnection()).destroy(); - await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); - - assert.equal(observe.mock.calls.length, 0); - assert.equal(await takeRunnerWarmLossNotice(DEVICE.id), undefined); -}); - -test('a port that never answers is runner_unreachable once the retry budget is spent', async () => { - vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_DESTINATION_ATTACH_RETRY_MS', '20'); - stubHost({ boots: [{ observed: true, bootedAtMs: 0 }] }); - const { params, onStop } = watchParams(await unusedPort()); - - attachRunnerDestinationWatch(params); - await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); - - assert.equal((await takeRunnerWarmLossNotice(DEVICE.id))?.reason, 'runner_unreachable'); -}); - -test('a refused attach whose runner process is gone is silent cleanup', async () => { - stubHost({ boots: [{ observed: true, bootedAtMs: 0 }], processAlive: false }); - const { params, onStop } = watchParams(await unusedPort()); + const first = watchParams(listener.port); + const second = watchParams(listener.port); - attachRunnerDestinationWatch(params); - await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); + attachRunnerDestinationWatch(first.params); + const firstConnection = await listener.nextConnection(); + attachRunnerDestinationWatch(second.params); + const secondConnection = await listener.nextConnection(); + firstConnection.destroy(); + await realTick(); + assert.equal(first.onStop.mock.calls.length + second.onStop.mock.calls.length, 0); - assert.equal(await takeRunnerWarmLossNotice(DEVICE.id), undefined); + secondConnection.destroy(); + await vi.waitFor(() => assert.equal(second.onStop.mock.calls.length, 1)); + assert.equal(first.onStop.mock.calls.length, 0); }); -async function listenOnHeld(port: number): Promise { - const server = net.createServer((socket) => socket.on('error', () => {})); - await new Promise((resolve) => server.listen(port, '127.0.0.1', resolve)); - return server; -} - -test('a runner generation that answers after refusals is classified at the connect', async () => { - vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_DESTINATION_ATTACH_RETRY_MS', '5000'); +test('a refused attach is retried, and a listener that answers keeps the runner', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); const port = await unusedPort(); - // The device rebooted while the port was refused; the replacement connection never closes. - stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() + 60_000 }] }); const { params, onStop } = watchParams(port); attachRunnerDestinationWatch(params); - await new Promise((resolve) => setTimeout(resolve, 50)); - assert.equal(onStop.mock.calls.length, 0); + await realTick(); + const listener = await listen(port); + await vi.advanceTimersByTimeAsync(1_000); + const connection = await listener.nextConnection(); + await realTick(); - const server = await listenOnHeld(port); - try { - await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1), { timeout: 4000 }); - assert.equal((await takeRunnerWarmLossNotice(DEVICE.id))?.reason, 'runner_destination_lost'); - } finally { - await new Promise((resolve) => server.close(() => resolve())); - } + assert.equal(onStop.mock.calls.length, 0); + connection.destroy(); + await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); }); -test('a runner generation that answers on the armed boot keeps the watch', async () => { - vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_DESTINATION_ATTACH_RETRY_MS', '5000'); - const port = await unusedPort(); - const observe = stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() - 60_000 }] }); - const { params, onStop } = watchParams(port); +test('a port that never answers stops the runner once the retry budget is spent', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const { params, onStop } = watchParams(await unusedPort()); attachRunnerDestinationWatch(params); - await new Promise((resolve) => setTimeout(resolve, 50)); - const server = await listenOnHeld(port); - try { - await vi.waitFor(() => assert.equal(observe.mock.calls.length, 1), { timeout: 4000 }); - await new Promise((resolve) => setTimeout(resolve, 30)); - assert.equal(onStop.mock.calls.length, 0); - } finally { - closeRunnerDestinationWatch(DEVICE.id); - await new Promise((resolve) => server.close(() => resolve())); + for (let attempt = 0; attempt < 20 && onStop.mock.calls.length === 0; attempt += 1) { + await realTick(); + await vi.advanceTimersByTimeAsync(1_000); } -}); - -test('a notice read waits for a loss decision still in flight for an open window', async () => { - vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_DESTINATION_CONFIRM_MS', '80'); - const listener = await listen(); - stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() + 60_000 }] }); - const { params } = watchParams(listener.port); - attachRunnerDestinationWatch(params); - (await listener.nextConnection()).destroy(); - await new Promise((resolve) => setTimeout(resolve, 20)); - - assert.equal((await takeRunnerWarmLossNotice(DEVICE.id))?.reason, 'runner_destination_lost'); + assert.equal(onStop.mock.calls.length, 1); + assert.ok(takeRunnerWarmLossNotice(DEVICE.id)); }); -test('a notice read does not wait once the retention window has ended', async () => { - vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_DESTINATION_CONFIRM_MS', '2000'); - const listener = await listen(); - stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() + 60_000 }] }); +test('a refused attach whose window ended stops retrying', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); let armed = true; - const { params } = watchParams(listener.port, { isArmed: () => armed }); + const { params, onStop } = watchParams(await unusedPort(), { isArmed: () => armed }); attachRunnerDestinationWatch(params); - (await listener.nextConnection()).destroy(); - await new Promise((resolve) => setTimeout(resolve, 20)); + await realTick(); armed = false; - const startedAt = Date.now(); - - assert.equal(await takeRunnerWarmLossNotice(DEVICE.id), undefined); - assert.ok(Date.now() - startedAt < 500); -}); + await vi.advanceTimersByTimeAsync(60_000); -test('a close after the retention window ended acts on nothing', async () => { - const listener = await listen(); - const observe = stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() + 60_000 }] }); - let armed = true; - const { params, onStop } = watchParams(listener.port, { isArmed: () => armed }); - - attachRunnerDestinationWatch(params); - const socket = await listener.nextConnection(); - armed = false; - socket.destroy(); - await new Promise((resolve) => setTimeout(resolve, 60)); - - assert.equal(observe.mock.calls.length, 0); assert.equal(onStop.mock.calls.length, 0); - assert.equal(await takeRunnerWarmLossNotice(DEVICE.id), undefined); -}); - -test('closing the watch detaches it so a later runner death stops nothing', async () => { - const listener = await listen(); - const observe = stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() + 60_000 }] }); - const { params, onStop } = watchParams(listener.port); - - attachRunnerDestinationWatch(params); - const socket = await listener.nextConnection(); - closeRunnerDestinationWatch(DEVICE.id); - socket.destroy(); - await new Promise((resolve) => setTimeout(resolve, 60)); - - assert.equal(observe.mock.calls.length, 0); - assert.equal(onStop.mock.calls.length, 0); -}); - -test('a second retention window replaces the existing watch instead of stacking another', async () => { - const listener = await listen(); - stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() + 60_000 }] }); - const first = watchParams(listener.port); - const second = watchParams(listener.port); - - attachRunnerDestinationWatch(first.params); - const firstSocket = await listener.nextConnection(); - attachRunnerDestinationWatch(second.params); - const secondSocket = await listener.nextConnection(); - firstSocket.destroy(); - secondSocket.destroy(); - await vi.waitFor(() => assert.equal(second.onStop.mock.calls.length, 1)); - - assert.equal(first.onStop.mock.calls.length, 0); -}); - -test('a handler resuming after its window was replaced leaves the newer watch tracked', async () => { - vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_DESTINATION_CONFIRM_MS', '60'); - const listener = await listen(); - stubHost({ boots: [{ observed: true, bootedAtMs: Date.now() + 60_000 }] }); - const first = watchParams(listener.port); - attachRunnerDestinationWatch(first.params); - (await listener.nextConnection()).destroy(); - await new Promise((resolve) => setTimeout(resolve, 10)); - - closeRunnerDestinationWatch(DEVICE.id); - attachRunnerDestinationWatch(watchParams(listener.port).params); - const newer = await listener.nextConnection(); - let newerClosed = false; - newer.on('close', () => { - newerClosed = true; - }); - await new Promise((resolve) => setTimeout(resolve, 120)); - assert.equal(newerClosed, false); - - closeRunnerDestinationWatch(DEVICE.id); - await vi.waitFor(() => assert.equal(newerClosed, true)); - assert.equal(first.onStop.mock.calls.length, 0); + assert.equal(vi.getTimerCount(), 0); }); diff --git a/packages/platform-apple/src/runner/host.ts b/packages/platform-apple/src/runner/host.ts index 9008a34951..37be783617 100644 --- a/packages/platform-apple/src/runner/host.ts +++ b/packages/platform-apple/src/runner/host.ts @@ -16,7 +16,6 @@ import type { IosPhysicalDeviceRunnerControl } from '../core/physical-device-rou import type * as ApplePlistXml from '../core/plist-xml.ts'; import type * as AppleRunnerOwnerState from '../core/runner-owner-state.ts'; import type * as AppleSimctl from '../core/simctl.ts'; -import type * as AppleSimulatorBoot from '../simulator-boot.ts'; import type * as AppleToolProvider from '../core/tool-provider.ts'; /** @@ -104,20 +103,6 @@ export type AppleRunnerHost = Pick< * family/OS/id, never a bare id). Embedders without a claim store answer false. */ hasDeviceClaimAuthority: AppleRunnerOwnerState.RunnerDeviceClaimAuthorityProbe; - /** - * When this Simulator's current boot began, read from the host (`launchd_sim`). The warm-loss - * watcher classifies destination loss by device identity rather than port liveness, because - * Xcode restarts the runner app onto the same port after rebooting the destination (#3321). - * The production binding loads the observer on first call, not at binding time: that module - * sits outside the façade closures this port's eager budget holds flat. - */ - observeSimulatorBootTimeMs: typeof AppleSimulatorBoot.observeSimulatorBootTimeMs; - /** - * The Simulator's listed power state (`Booted`, `Shutting Down`, `Shutdown`), or null when the - * listing is unavailable. A shut-down device can keep its old `launchd_sim` visible for several - * seconds, so the boot witness alone cannot tell a crash from an external shutdown (#3321). - */ - observeSimulatorState: (device: DeviceInfo) => Promise; }; /** The runner's deadline type is the host's read side; the {@link Deadline} shim below builds them. */ @@ -210,8 +195,6 @@ export const visitXmlPlistEntries = delegate('visitXmlPlistEntries'); export const resolveIosPhysicalDeviceControl = delegate('resolveIosPhysicalDeviceControl'); export const leaseOwnerStateDir = delegate('leaseOwnerStateDir'); export const hasDeviceClaimAuthority = delegate('hasDeviceClaimAuthority'); -export const observeSimulatorBootTimeMs = delegate('observeSimulatorBootTimeMs'); -export const observeSimulatorState = delegate('observeSimulatorState'); /** * Deadline keeps its root call-site shape (`Deadline.fromTimeoutMs(...)`); diff --git a/packages/platform-apple/src/runner/runner-destination-watch.ts b/packages/platform-apple/src/runner/runner-destination-watch.ts index 5cd50f7fa9..b7e6d195e7 100644 --- a/packages/platform-apple/src/runner/runner-destination-watch.ts +++ b/packages/platform-apple/src/runner/runner-destination-watch.ts @@ -1,467 +1,129 @@ import net from 'node:net'; import type { DeviceInfo } from '@agent-device/kernel/device'; -import { - emitDiagnostic, - isProcessAlive, - observeSimulatorBootTimeMs, - observeSimulatorState, -} from './host.ts'; +import { emitDiagnostic } from './host.ts'; /** - * Why a runner retained after `close` was stopped before the session asked for it again. - * Keyed behavior: consumers branch on these values, never on prose. + * What the watcher records when it stops a retained runner: the moment the loss was seen. Read + * once by the next `open`. */ -export type RunnerWarmLossReason = - | 'runner_destination_lost' - | 'runner_unreachable' - | 'runner_destination_unverified'; - -/** - * What a retained runner's owner records when the watcher stops it: the typed reason, the session - * the runner served, and the moment the loss was proven. Read once by the next `open`. - */ -export type RunnerWarmLossNotice = Readonly<{ - reason: RunnerWarmLossReason; - deviceId: string; - sessionId: string; - atMs: number; -}>; +export type RunnerWarmLossNotice = Readonly<{ atMs: number }>; /** * A push watcher for a runner retained after `close` (#3321). * * A retained `xcodebuild test-without-building` owns its Simulator destination: when something - * outside agent-device shuts that Simulator down, the runner app dies (`ipc/mig server died`) and - * Xcode's destination machinery silently reboots the device to keep its own session alive, with - * nobody asking agent-device. The device stays powered on for as long as that runner process - * lives — potentially the whole idle-retention window. Killing the runner is what powers a - * rebooted device back off (measured: within ~2s of the kill on a device Xcode had rebooted), - * while killing a runner whose device was NOT rebooted leaves the device powered on — so stopping - * on loss cannot harm a healthy warm runner. - * - * The watcher holds ONE idle TCP connection to the runner's listener for the length of the - * retention window and writes nothing on it. The runner's listener serves many connections and - * never closes an idle one on its own (measured: alive past 180s while commands answered in - * single-digit milliseconds beside it), so a connection that closes AFTER it was established is - * the push signal that the runner's app generation ended — arriving within ~1s of an external - * shutdown. No timer samples a quiet runner: every read below happens once, at an event. - * - * "A connected socket closed" cannot by itself tell a reboot from a crash, because Xcode restarts - * the runner app onto the same port after rebooting the destination. The classification therefore - * reads device identity, not port liveness: a boot that began after the watch was armed, or no - * boot observable while the runner is still there to serve one, means the destination was replaced - * under the session and the retained runner must go. + * outside agent-device shuts that Simulator down, the runner app dies and Xcode silently reboots + * the device to keep its own session alive, with nobody asking agent-device. The device then stays + * powered on for as long as the runner process lives. Stopping the runner is what powers it back + * off, and stopping a runner whose device was not rebooted only costs its warm reuse. * - * An attach that never connected (ECONNREFUSED) carries no information about WHY nothing was - * listening — the restart after a reboot lands on the same port within seconds — so it is retried - * on a bounded backoff instead of being classified; only an established connection that later - * closes is evidence of a runner-generation change. + * The watcher holds one idle TCP connection to the runner's loopback listener for the retention + * window and writes nothing on it. The listener never closes an idle connection on its own, so a + * connection that closes after it was established means the runner app ended, and the retained + * runner is stopped. Nothing samples a quiet runner. A refused attach is retried on a flat cadence + * inside a bounded budget, because the listener can refuse for a few seconds right after `close`; + * a port that stays refused is a runner that cannot answer, and is stopped the same way. */ -/** How long an attach keeps retrying a refused port before the runner is called unreachable. */ -const ATTACH_RETRY_BUDGET_DEFAULT_MS = 15_000; -const ATTACH_RETRY_BASE_DELAY_MS = 1_000; - -/** How long a connected-socket close waits before the boot identity is read (see below). */ -const DESTINATION_CONFIRM_DEFAULT_MS = 2_500; -const DESTINATION_RECHECK_DEFAULT_MS = 1_000; - -/** `simctl list` name of a powered-on Simulator. */ -const SIMULATOR_BOOTED_STATE = 'Booted'; - -type LossAction = 'stop' | 'rearm'; - -type LossVerdict = Readonly<{ - action: LossAction; - /** Present only when the loss is worth telling the next `open` about. */ - noticeReason?: RunnerWarmLossReason; -}>; - -const DESTINATION_LOST: LossVerdict = { action: 'stop', noticeReason: 'runner_destination_lost' }; - -/** - * The listing timed out or was unreadable, so nothing proves the device changed. Stopping is still - * the safe side, but the notice must not claim a shutdown nobody observed. - */ -const STATE_UNVERIFIABLE: LossVerdict = { - action: 'stop', - noticeReason: 'runner_destination_unverified', -}; +const ATTACH_RETRY_DELAY_MS = 1_000; +const ATTACH_RETRY_ATTEMPTS = 15; -type DestinationWatch = { +export type RunnerDestinationWatchParams = { device: DeviceInfo; sessionId: string; port: number; - runnerPid: number | undefined; - /** When this retention window began; a boot starting later is a replacement device state. */ - armedAtMs: number; /** Re-read at every decision: false once any path has left the idle-retention window. */ isArmed: () => boolean; /** Stops the retained runner and releases its lease. */ onStop: () => Promise; - socket: net.Socket; - /** True once this connection was established: only then can its close mean runner death. */ - connected: boolean; - /** - * True when this attach follows a refusal or a same-boot re-arm, so what answers may be a new - * runner generation started after the device changed under the session. - */ - replacement: boolean; - /** The in-flight loss decision, which a notice reader waits for while the window is open. */ - deciding: Promise | undefined; - detached: boolean; + /** Attach attempts left for a refused port; a fresh retention window starts a fresh budget. */ + attemptsLeft?: number; }; +type DestinationWatch = { socket: net.Socket; retryTimer?: NodeJS.Timeout }; + const destinationWatches = new Map(); const pendingWarmLossNotices = new Map(); -/** - * How long a loss event waits before reading the device's boot identity. A shutdown is not over - * when the runner app dies: the old `launchd_sim` may still be visible for a moment, and Xcode's - * reboot starts within ~1s of the shutdown returning. One bounded re-read per event (never a - * sample of a quiet runner) lets that churn settle so a mid-shutdown read cannot misclassify a - * crash. A stop that lands after the reboot has finished still powers the device back off. - * - * The confirm delay, the recheck delay and the attach-retry budget have production defaults; a - * non-negative millisecond value in `AGENT_DEVICE_IOS_RUNNER_DESTINATION_CONFIRM_MS`, - * `..._RECHECK_MS` or `..._ATTACH_RETRY_MS` replaces one. Only tests set them, to avoid waiting - * production time. - */ -function resolveConfirmDelayMs(): number { - return readDurationEnvMs( - 'AGENT_DEVICE_IOS_RUNNER_DESTINATION_CONFIRM_MS', - DESTINATION_CONFIRM_DEFAULT_MS, - ); -} - -function resolveRecheckDelayMs(): number { - return readDurationEnvMs( - 'AGENT_DEVICE_IOS_RUNNER_DESTINATION_RECHECK_MS', - DESTINATION_RECHECK_DEFAULT_MS, - ); -} - -function resolveAttachRetryBudgetMs(): number { - return readDurationEnvMs( - 'AGENT_DEVICE_IOS_RUNNER_DESTINATION_ATTACH_RETRY_MS', - ATTACH_RETRY_BUDGET_DEFAULT_MS, - ); -} - -/** A non-negative millisecond override, fractions floored; anything else keeps the default. */ -function readDurationEnvMs(key: string, fallbackMs: number): number { - const raw = process.env[key]?.trim(); - const parsed = raw ? Number(raw) : Number.NaN; - return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : fallbackMs; -} - -function delay(ms: number): Promise { - return new Promise((resolve) => { - const timer = setTimeout(resolve, ms); - timer.unref?.(); - }); -} - -export type RunnerDestinationWatchParams = { - device: DeviceInfo; - sessionId: string; - port: number; - runnerPid: number | undefined; - isArmed: () => boolean; - onStop: () => Promise; - /** - * Remaining attach-retry budget for a refused port, carried across the re-attaches of one loss - * event so a runner that never answers is bounded; a fresh retention window starts a fresh - * budget. - */ - attachRetryBudgetMs?: number; - /** When the retention window began; re-attaches within one window carry it forward unchanged. */ - armedAtMs?: number; - /** Set by re-attaches within one window: the listener that answers may be a new generation. */ - replacement?: boolean; -}; - /** * Attaches the watcher for one retention window. A device has one watch: a fresh window replaces - * the existing one rather than stacking another. `isArmed` is consulted again at every - * decision, so an attach that lands after its retention window already ended arms nothing that - * can act. + * the existing one. `isArmed` is consulted again at every decision, so an attach that lands after + * its retention window already ended arms nothing that can act. */ export function attachRunnerDestinationWatch(params: RunnerDestinationWatchParams): void { - // One watch per device, and a new attach replaces rather than amends: nothing carries over - // from a window another path already ended, so no caller has to have detached first. - const existing = destinationWatches.get(params.device.id); - if (existing) detachWatch(existing); - const watch: DestinationWatch = { - device: params.device, - sessionId: params.sessionId, - port: params.port, - runnerPid: params.runnerPid, - armedAtMs: params.armedAtMs ?? Date.now(), - isArmed: params.isArmed, - onStop: params.onStop, - socket: net.connect(params.port, '127.0.0.1'), - connected: false, - replacement: params.replacement ?? false, - deciding: undefined, - detached: false, - }; + const deviceId = params.device.id; + closeRunnerDestinationWatch(deviceId); + const watch: DestinationWatch = { socket: net.connect(params.port, '127.0.0.1') }; + let connected = false; watch.socket.unref(); watch.socket.on('connect', () => { - watch.connected = true; - if (watch.replacement) void decide(watch, () => confirmReplacementGeneration(watch)); + connected = true; }); + // `close` always follows an `error`; the decision is made once, there. + watch.socket.on('error', () => {}); watch.socket.on('close', () => { - void handleSocketClosed(watch, params.attachRetryBudgetMs); + if (destinationWatches.get(deviceId) !== watch) return; + if (!params.isArmed()) { + closeRunnerDestinationWatch(deviceId); + } else if (connected) { + stopRetainedRunner(params); + } else { + retryRefusedAttach(watch, params); + } }); - // `close` always follows an `error`; classification happens once, there. - watch.socket.on('error', () => {}); - destinationWatches.set(params.device.id, watch); + destinationWatches.set(deviceId, watch); } /** - * Ends the watch without acting. Every path that leaves the idle-retention window calls it; even - * a missed call is inert, because `isArmed` re-reads the retention state at each decision. + * Ends the watch without acting. Every path that leaves the idle-retention window calls it; even a + * missed call is inert, because `isArmed` re-reads the retention state at each decision. */ export function closeRunnerDestinationWatch(deviceId: string): void { const watch = destinationWatches.get(deviceId); - if (watch) detachWatch(watch); -} - -function detachWatch(watch: DestinationWatch): void { - watch.detached = true; - // A handler resuming after an await may belong to a watch a newer window has replaced. - if (destinationWatches.get(watch.device.id) === watch) destinationWatches.delete(watch.device.id); - watch.socket.removeAllListeners('connect'); - watch.socket.removeAllListeners('close'); - watch.socket.removeAllListeners('error'); + if (!watch) return; + destinationWatches.delete(deviceId); + clearTimeout(watch.retryTimer); + watch.socket.removeAllListeners(); + watch.socket.on('error', () => {}); watch.socket.destroy(); } -async function handleSocketClosed( - watch: DestinationWatch, - attachRetryBudgetMs: number | undefined, -): Promise { - if (watch.detached) return; - if (!watch.isArmed()) { - detachWatch(watch); - return; - } - if (!watch.connected) { - await handleRefusedAttach(watch, attachRetryBudgetMs); - return; - } - await decide(watch, async () => { - await delay(resolveConfirmDelayMs()); - if (watch.detached || !watch.isArmed()) { - detachWatch(watch); - return; - } - const verdict = await classifyConnectedLoss(watch); - if (watch.detached || !watch.isArmed()) { - detachWatch(watch); - return; - } - if (verdict.action === 'rearm') { - rearmOnSameBoot(watch); - return; - } - stopRetainedRunner(watch, verdict.noticeReason); - }); -} - -/** - * A listener answered after the device may have been replaced: the refused attaches that preceded - * it span exactly the gap in which Xcode reboots a shut-down destination and restarts the runner - * app onto the same port, and a connection that reaches that new generation never closes on its - * own. One classification at the connect decides whether the window's boot is still the device's. - * A same-boot answer keeps this connection as the watch. - */ -async function confirmReplacementGeneration(watch: DestinationWatch): Promise { - const verdict = await classifyConnectedLoss(watch); - if (watch.detached || !watch.isArmed()) { - detachWatch(watch); - return; - } - if (verdict.action === 'stop') stopRetainedRunner(watch, verdict.noticeReason); -} - -/** Runs one loss decision and publishes it, so {@link takeRunnerWarmLossNotice} can wait for it. */ -async function decide(watch: DestinationWatch, run: () => Promise): Promise { - const deciding = run(); - watch.deciding = deciding; - try { - await deciding; - } finally { - if (watch.deciding === deciding) watch.deciding = undefined; - } -} - -/** - * The attach never reached a listener. A rebooting destination is usually unreachable for a couple - * of seconds while Xcode restarts the app onto the same port, so retry at a flat one-second cadence - * inside a bounded budget; a retry that connects classifies the replacement generation. Only when - * the budget runs out is the retained runner called what it has proven to be: unable to answer - * anything. By then the destination process's own liveness decides whether this was a crash (Xcode - * gave up with it, the device lost its boot owner) or a listener that died while Xcode kept the - * device — the latter is exactly the silent power-on #3321 is about, so the runner goes with a - * notice. - */ -async function handleRefusedAttach( - watch: DestinationWatch, - attachRetryBudgetMs: number | undefined, -): Promise { - const budgetMs = attachRetryBudgetMs ?? resolveAttachRetryBudgetMs(); - if (budgetMs > 0) { - const nextDelayMs = Math.min(ATTACH_RETRY_BASE_DELAY_MS, budgetMs); - await delay(nextDelayMs); - if (watch.detached || !watch.isArmed()) { - detachWatch(watch); - return; - } - detachWatch(watch); - attachRunnerDestinationWatch({ - device: watch.device, - sessionId: watch.sessionId, - port: watch.port, - runnerPid: watch.runnerPid, - isArmed: watch.isArmed, - onStop: watch.onStop, - attachRetryBudgetMs: budgetMs - nextDelayMs, - armedAtMs: watch.armedAtMs, - replacement: true, - }); +function retryRefusedAttach(watch: DestinationWatch, params: RunnerDestinationWatchParams): void { + const attemptsLeft = params.attemptsLeft ?? ATTACH_RETRY_ATTEMPTS; + if (attemptsLeft <= 0) { + stopRetainedRunner(params); return; } - if (watch.runnerPid !== undefined && !isProcessAlive(watch.runnerPid)) { - // Xcode itself gave up on this runner generation: the retention is over, the lease is stale, - // and nothing agent-device owns is powering the device any more. Plain cleanup, no notice. - stopRetainedRunner(watch, undefined); - return; - } - stopRetainedRunner(watch, 'runner_unreachable'); -} - -/** - * Reads whether the device behind the runner stopped being the device this session was armed on. A - * dead destination process answers for itself before the probe: Xcode cannot reboot from a dead - * process, so stopping is plain lease cleanup. Otherwise the device must be listed `Booted` on the - * boot the window began with, and anything else is destination loss: a shut-down device keeps its - * old `launchd_sim` listed for ~6s, so the boot witness alone would read the old boot and call it a - * crash. A listing that timed out or was unreadable is no evidence of a healthy device, so it stops - * the runner too, but as `runner_destination_unverified`: it cannot support a claim about the - * device. A boot newer than the window is Xcode's reboot; a boot unobservable twice is a device - * that is down with a reboot promise that may already be in flight. Stopping pre-empts that promise - * in every loss case. - */ -async function classifyConnectedLoss(watch: DestinationWatch): Promise { - if (watch.runnerPid !== undefined && !isProcessAlive(watch.runnerPid)) { - return { action: 'stop' }; - } - const state = await observeSimulatorState(watch.device); - if (state === null) return STATE_UNVERIFIABLE; - if (state !== SIMULATOR_BOOTED_STATE) return DESTINATION_LOST; - let boot = await readBootAgainstWindow(watch); - if (boot === 'unobserved') { - await delay(resolveRecheckDelayMs()); - boot = await readBootAgainstWindow(watch); - } - return boot === 'same-boot' ? { action: 'rearm' } : DESTINATION_LOST; -} - -async function readBootAgainstWindow( - watch: DestinationWatch, -): Promise<'same-boot' | 'newer-boot' | 'unobserved'> { - const boot = await observeSimulatorBootTimeMs(watch.device); - if (!boot.observed) return 'unobserved'; - return boot.bootedAtMs <= watch.armedAtMs ? 'same-boot' : 'newer-boot'; -} - -/** - * The runner's app generation died while the device kept this session's boot: a crash Xcode may - * restart onto the same port. Re-arm the push signal on the replacement. Each cycle waits on the - * new socket's own close event — an established close here is a real runner death, not a sample, - * and Xcode's restart budget bounds how many there can be; a refused port runs the bounded - * attach-retry above instead of spinning here. - */ -function rearmOnSameBoot(watch: DestinationWatch): void { - emitDiagnostic({ - level: 'info', - phase: 'ios_runner_destination_watch_rearmed', - data: { deviceId: watch.device.id, sessionId: watch.sessionId }, - }); - const { device, sessionId, port, runnerPid, isArmed, onStop, armedAtMs } = watch; - detachWatch(watch); - if (!isArmed()) return; - attachRunnerDestinationWatch({ - device, - sessionId, - port, - runnerPid, - isArmed, - onStop, - armedAtMs, - replacement: true, - }); -} - -/** - * Stops the retained runner, recording the notice the next `open` must carry before any notice - * consumer can race this path. A `undefined` reason is silent lease cleanup for a runner - * generation Xcode already buried. - */ -function stopRetainedRunner( - watch: DestinationWatch, - noticeReason: RunnerWarmLossReason | undefined, -): void { - detachWatch(watch); - if (noticeReason) { - pendingWarmLossNotices.set(watch.device.id, { - reason: noticeReason, - deviceId: watch.device.id, - sessionId: watch.sessionId, - atMs: Date.now(), - }); - } + watch.retryTimer = setTimeout(() => { + if (params.isArmed()) + attachRunnerDestinationWatch({ ...params, attemptsLeft: attemptsLeft - 1 }); + else closeRunnerDestinationWatch(params.device.id); + }, ATTACH_RETRY_DELAY_MS); + watch.retryTimer.unref(); +} + +/** Stops the retained runner, recording the notice before the stop so an `open` cannot miss it. */ +function stopRetainedRunner(params: RunnerDestinationWatchParams): void { + const deviceId = params.device.id; + closeRunnerDestinationWatch(deviceId); + pendingWarmLossNotices.set(deviceId, { atMs: Date.now() }); emitDiagnostic({ level: 'warn', phase: 'ios_runner_warm_stop', - data: { - deviceId: watch.device.id, - sessionId: watch.sessionId, - reason: noticeReason ?? 'runner_process_gone', - }, + data: { deviceId, sessionId: params.sessionId }, }); - void watch.onStop().catch((error: unknown) => { + void params.onStop().catch((error: unknown) => { emitDiagnostic({ level: 'warn', phase: 'ios_runner_warm_stop_failed', - data: { - deviceId: watch.device.id, - error: error instanceof Error ? error.message : String(error), - }, + data: { deviceId, error: error instanceof Error ? error.message : String(error) }, }); }); } -/** - * Takes the destination-loss notice recorded for this device, once. A notice exists only when a - * retained runner was stopped because the destination was replaced under it — the state the next - * `open` must explain rather than leave the caller to infer from a cold start. A decision still in - * flight for a window that is still open is waited for, so a notice cannot land after the open it - * belongs to has already read; once any use of the runner has ended the window nothing is waited - * for, because nothing can be recorded any more. - */ -export async function takeRunnerWarmLossNotice( - deviceId: string, -): Promise { - for (let watch = destinationWatches.get(deviceId); watch?.deciding;) { - if (!watch.isArmed()) break; - await watch.deciding; - const next = destinationWatches.get(deviceId); - watch = next === watch ? undefined : next; - } +/** Takes the loss notice recorded for this device, once. */ +export function takeRunnerWarmLossNotice(deviceId: string): RunnerWarmLossNotice | undefined { const notice = pendingWarmLossNotices.get(deviceId); - if (notice) pendingWarmLossNotices.delete(deviceId); + pendingWarmLossNotices.delete(deviceId); return notice; } diff --git a/packages/platform-apple/src/runner/runner-session.ts b/packages/platform-apple/src/runner/runner-session.ts index 84fb1d4b92..4f908714fb 100644 --- a/packages/platform-apple/src/runner/runner-session.ts +++ b/packages/platform-apple/src/runner/runner-session.ts @@ -760,9 +760,8 @@ async function restoreReleasedRunnerRetention( * the façade closures the eager-closure budget holds at merge-base size. */ async function armRunnerDestinationWatch(session: RunnerSession | undefined): Promise { - // Local Simulators only: the push signal is the runner's loopback listener, the reboot owner is - // Xcode's Simulator destination machinery, and `launchd_sim` is the boot witness. A retained - // physical-device runner has none of these and stays exactly as it was. + // Local Simulators only: the reboot comes from Xcode's Simulator destination machinery. A + // retained physical-device runner has no such destination and stays exactly as it was. if (session?.device.kind !== 'simulator' || !isIosFamily(session.device)) return; if (!isRunnerIdleRetained(session.deviceId, session)) return; const { attachRunnerDestinationWatch } = await import('./runner-destination-watch.ts'); @@ -771,7 +770,6 @@ async function armRunnerDestinationWatch(session: RunnerSession | undefined): Pr device: session.device, sessionId: session.sessionId, port: session.port, - runnerPid: session.child.pid, isArmed: () => isRunnerIdleRetained(deviceId, session), onStop: async () => { await invalidateRunnerSession(session, 'warm_runner_destination_lost'); diff --git a/scripts/layering/daemon-platform-runtime-inventory.ts b/scripts/layering/daemon-platform-runtime-inventory.ts index 33077ffde0..5bbe8b0d7f 100644 --- a/scripts/layering/daemon-platform-runtime-inventory.ts +++ b/scripts/layering/daemon-platform-runtime-inventory.ts @@ -198,7 +198,7 @@ export const DAEMON_PLATFORM_RUNTIME_EDGES: readonly DaemonPlatformRuntimeEdge[] classification: 'daemon-policy-essential', rationale: 'daemon-owned open reports that a runner left warm by close was stopped because its ' + - 'simulator was shut down externally (#3321); the watcher and the typed reason stay ' + + 'connection closed, typically a simulator shut down externally (#3321); the watcher stays ' + 'Apple-owned behind the neutral notice read.', }, { diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index d49efce122..e46b3ae5da 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -530,7 +530,7 @@ Android physical-device prerequisites: Runner and daemon lifecycle (applies to simulators too): open without --relaunch is idempotent-foreground for an already-running app (it brings the process forward; it does not restart it). open --relaunch restarts the app; on iOS simulators this collapses to one simctl launch --terminate-running-process call instead of a separate terminate-then-launch. No runner read launches a session app that is not running: snapshot, wait, is, get, a reading find, and an interaction's leading reads (a gesture's viewport read, the capture that resolves a selector click/fill) answer the retriable APP_NOT_RUNNING instead of bare-launching over a launch SpringBoard still holds behind its deep-link confirmation. Only open, activate, and a command that mutates without a leading read bring a stopped app up. - close keeps a healthy iOS simulator XCTest runner warm by default so the next open on that simulator (same udid in the same simulator set) skips the runner build, unless --shutdown was requested, the session was recording, or the session held a device lease. A retained runner auto-stops after an idle window (default 5 minutes); set AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS to override, or 0 to disable idle stop and retain until daemon exit. A simulator shut down externally while its runner is retained is detected and the retained runner is stopped, because the retained Xcode destination would otherwise reboot the simulator and leave it powered on; the next open reports reason=runner_destination_lost in its warnings, or reason=runner_unreachable when the runner stopped answering, or reason=runner_destination_unverified when the runner's connection closed and the simulator's state could not be read. + close keeps a healthy iOS simulator XCTest runner warm by default so the next open on that simulator (same udid in the same simulator set) skips the runner build, unless --shutdown was requested, the session was recording, or the session held a device lease. A retained runner auto-stops after an idle window (default 5 minutes); set AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS to override, or 0 to disable idle stop and retain until daemon exit. A simulator shut down externally while its runner is retained is detected and the retained runner is stopped, because the retained Xcode destination would otherwise reboot the simulator and leave it powered on; the next open reports reason=runner_connection_lost in its warnings. Each AGENT_DEVICE_STATE_DIR runs its own daemon. It self-exits after an idle window (default 5 minutes, matching the runner idle-stop default) once it has no open sessions, no in-flight requests, and no active recording; set AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS to override, or 0 to disable idle reap. On a machine shared by several agents, set AGENT_DEVICE_SESSION_IDLE_TIMEOUT_MS to also expire an individual session that has taken a host-global device claim and then received no commands for that long; the claim is released and the next command on that session answers SESSION_NOT_FOUND with details.reason SESSION_IDLE_EXPIRED naming the window and the device. Off by default, because on a shared host "idle" and "thinking" are indistinguishable: only set it where an abandoned claim blocking every other agent is the worse failure. Sessions holding a remote lease or any active capture (recording, logs, audio, performance, or trace) are never expired this way. A stale iOS runner lease — its owner process dead, or its AGENT_DEVICE_STATE_DIR deleted — is reclaimed automatically instead of failing with "is already owned by another agent-device daemon". A live owner's runner is also reclaimed when the requesting daemon holds the host-global device claim for that device: claims are exclusive, so holding one proves the runner's owner released the device and merely kept the runner warm. The error remains only for owners outside claim arbitration (a pre-claims build, or daemons pointed at different claim stores). diff --git a/src/daemon/__tests__/request-router-open-claim.test.ts b/src/daemon/__tests__/request-router-open-claim.test.ts index 98485a531c..0334ad455f 100644 --- a/src/daemon/__tests__/request-router-open-claim.test.ts +++ b/src/daemon/__tests__/request-router-open-claim.test.ts @@ -175,16 +175,11 @@ test('open takes a live foreign claim whose device rebooted after the claim was } }); -test('open warns with the typed reason when a warm runner was stopped for a lost destination', async () => { +test('open warns with the typed reason when a warm runner was stopped for a lost connection', async () => { const sessionStore = makeSessionStore('agent-device-router-open-warm-loss-'); const device = makeIosDevice('SIM-WARM-LOSS'); mockResolveTargetDevice.mockResolvedValue(device); - mockTakeRunnerWarmLossNotice.mockResolvedValueOnce({ - reason: 'runner_destination_lost', - deviceId: device.id, - sessionId: 'warm-loss', - atMs: Date.now(), - }); + mockTakeRunnerWarmLossNotice.mockResolvedValueOnce({ atMs: Date.now() }); const claimsDir = mkdtempForTestSync('agent-device-router-open-warm-loss-claims-'); const previousClaimsDir = process.env.AGENT_DEVICE_CLAIMS_DIR; process.env.AGENT_DEVICE_CLAIMS_DIR = claimsDir; @@ -198,7 +193,7 @@ test('open warns with the typed reason when a warm runner was stopped for a lost if (!opened.ok) return; expect(mockTakeRunnerWarmLossNotice).toHaveBeenCalledWith(device.id); expect(opened.data?.warnings).toEqual([ - expect.stringContaining('reason=runner_destination_lost'), + expect.stringContaining('reason=runner_connection_lost'), ]); const reopened = await createOpenHandler(sessionStore)( @@ -213,49 +208,6 @@ test('open warns with the typed reason when a warm runner was stopped for a lost } }); -test('open warns with its own typed reason when a warm runner was stopped on an unverifiable state', async () => { - const sessionStore = makeSessionStore('agent-device-router-open-warm-unverified-'); - const device = makeIosDevice('SIM-WARM-UNVERIFIED'); - mockResolveTargetDevice.mockResolvedValue(device); - mockTakeRunnerWarmLossNotice.mockResolvedValueOnce({ - reason: 'runner_destination_unverified', - deviceId: device.id, - sessionId: 'warm-unverified', - atMs: Date.now(), - }); - const claimsDir = mkdtempForTestSync('agent-device-router-open-warm-unverified-claims-'); - const previousClaimsDir = process.env.AGENT_DEVICE_CLAIMS_DIR; - process.env.AGENT_DEVICE_CLAIMS_DIR = claimsDir; - - try { - const opened = await createOpenHandler(sessionStore)( - openRequest('warm-unverified', { platform: 'ios' }, 'req-open-warm-unverified', {}, [ - 'FixtureApp', - ]), - ); - - expect(opened.ok).toBe(true); - if (!opened.ok) return; - expect(mockTakeRunnerWarmLossNotice).toHaveBeenCalledWith(device.id); - expect(opened.data?.warnings).toEqual([ - expect.stringContaining('reason=runner_destination_unverified'), - ]); - expect(String(opened.data?.warnings)).not.toContain('stopped answering'); - - const reopened = await createOpenHandler(sessionStore)( - openRequest('warm-unverified', { platform: 'ios' }, 'req-open-warm-unverified-again', {}, [ - 'FixtureApp', - ]), - ); - expect(reopened.ok).toBe(true); - if (reopened.ok) expect(reopened.data?.warnings).toBeUndefined(); - } finally { - if (previousClaimsDir === undefined) delete process.env.AGENT_DEVICE_CLAIMS_DIR; - else process.env.AGENT_DEVICE_CLAIMS_DIR = previousClaimsDir; - fs.rmSync(claimsDir, { recursive: true, force: true }); - } -}); - // The production reopen path never re-acquires the claim, so renewal has to ride the successful // existing-session open itself: without it, an owner that came back after a reboot still carries a // pre-reboot stamp and loses the device to the next caller that asks. diff --git a/src/daemon/session-lifecycle/internal/session-open-execution.ts b/src/daemon/session-lifecycle/internal/session-open-execution.ts index 0ef8809dc0..748c4392b6 100644 --- a/src/daemon/session-lifecycle/internal/session-open-execution.ts +++ b/src/daemon/session-lifecycle/internal/session-open-execution.ts @@ -137,22 +137,15 @@ async function readWarmRunnerLossNotice( } /** - * What the caller's `open` output says when the runner this session left warm after `close` had to - * be stopped because the Simulator was shut down externally while it was retained (#3321). Names - * the typed reason so automation can key on it without parsing prose. + * What the caller's `open` output says when the runner this session left warm after `close` was + * stopped because its connection closed while it was retained (#3321), typically a Simulator shut + * down externally. The reason is named so automation can key on it without parsing prose. */ -const WARM_RUNNER_LOSS_CAUSE: Record = { - runner_destination_lost: - 'the simulator was shut down externally while it was retained, so the runner was stopped before it could boot the simulator again', - runner_unreachable: 'it stopped answering while it was retained', - runner_destination_unverified: - "its connection closed while it was retained and the simulator's state could not be read, so the runner was stopped as a precaution", -}; - function warmRunnerLossWarning(notice: RunnerWarmLossNotice): string { return ( - `The warm iOS runner left by a previous close was stopped at ${new Date(notice.atMs).toISOString()}: ` + - `${WARM_RUNNER_LOSS_CAUSE[notice.reason]}. reason=${notice.reason}` + `The warm iOS runner left by a previous close was stopped at ${new Date(notice.atMs).toISOString()} ` + + 'because its connection closed while it was retained, typically because the simulator was shut ' + + 'down externally; stopping it keeps Xcode from rebooting the simulator. reason=runner_connection_lost' ); } diff --git a/src/platform-runtime-warm-runner-notice.test.ts b/src/platform-runtime-warm-runner-notice.test.ts index 8465c9eefa..2ff39bec75 100644 --- a/src/platform-runtime-warm-runner-notice.test.ts +++ b/src/platform-runtime-warm-runner-notice.test.ts @@ -1,12 +1,7 @@ import { beforeEach, expect, test, vi } from 'vitest'; import type { RunnerWarmLossNotice } from '@agent-device/platform-apple/runner/operations'; -const NOTICE: RunnerWarmLossNotice = { - reason: 'runner_destination_lost', - deviceId: 'sim-1', - sessionId: 'session-1', - atMs: 1, -}; +const NOTICE: RunnerWarmLossNotice = { atMs: 1 }; const mocks = vi.hoisted(() => ({ takeRunnerWarmLossNotice: vi.fn(async (): Promise => undefined), diff --git a/src/platform-runtime-warm-runner-notice.ts b/src/platform-runtime-warm-runner-notice.ts index 689617f193..a5548b0140 100644 --- a/src/platform-runtime-warm-runner-notice.ts +++ b/src/platform-runtime-warm-runner-notice.ts @@ -6,9 +6,9 @@ export type { RunnerWarmLossNotice }; /** * Takes the warm-runner loss notice recorded for this device by the Apple runner's destination * watcher, once, if there is one (#3321). A notice exists only when a runner retained after - * `close` had to be stopped because something shut the Simulator down under it and the retained - * Xcode destination had begun rebooting it — the state the next `open` reports rather than leaves - * the caller to infer from a cold start. Families that retain no warm runner record no notice. + * `close` was stopped because its connection closed, typically because something shut the Simulator + * down under it — the state the next `open` reports rather than leaves the caller to infer from a + * cold start. Families that retain no warm runner record no notice. * * The notice is read through the platform's public operations surface from this root module: the * daemon must not import a concrete platform package (R65), and the notice is a runner fact, not a diff --git a/website/docs/docs/sessions.md b/website/docs/docs/sessions.md index e6e6734ac1..db1627d4be 100644 --- a/website/docs/docs/sessions.md +++ b/website/docs/docs/sessions.md @@ -117,7 +117,7 @@ agent-device close --shutdown Without `--shutdown`, `close` keeps the iOS runner warm for the next `open`. If you shut that simulator down yourself (`xcrun simctl shutdown`) while the runner is waiting, agent-device stops the runner so it doesn't boot the simulator again, and the next `open` returns a warning that ends in -`reason=runner_destination_lost`. It ends in `reason=runner_unreachable` when the runner stopped answering, or `reason=runner_destination_unverified` when its connection closed and the simulator's state could not be read. +`reason=runner_connection_lost`. ## Wait for a slow or busy device From c1d59f1d9c8edb71deedcc3b7a157aff9bb7e727 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 10 Oct 2026 08:44:36 +0200 Subject: [PATCH 17/20] refactor(apple): give retained runners one lifecycle owner --- .../0005-ios-runner-interaction-lifecycle.md | 6 +- docs/agents/device-verification.md | 5 +- packages/kernel/src/keyed-lock.test.ts | 41 +++ packages/kernel/src/keyed-lock.ts | 15 +- .../platform-apple/src/core/runner-client.ts | 14 - .../src/runner-operations-facade.ts | 2 - .../runner-close-finalization.test.ts | 197 ++++++++++-- .../runner-destination-watch.test.ts | 196 ------------ .../runner/__tests__/runner-retention.test.ts | 122 ++++++++ .../runner-session-close-prep-fence.test.ts | 105 ++++--- .../__tests__/runner-session-close.test.ts | 30 +- .../runner-session-lifecycle.test.ts | 10 +- .../runner/__tests__/runner-session.test.ts | 54 ---- .../src/runner/runner-adoption.ts | 1 + .../src/runner/runner-destination-watch.ts | 129 -------- .../src/runner/runner-disposal.ts | 4 + .../src/runner/runner-lifecycle.ts | 2 + .../src/runner/runner-retention.ts | 129 ++++++++ .../src/runner/runner-session-types.ts | 8 +- .../src/runner/runner-session.ts | 296 +++++------------- src/commands/schema/cli-help.ts | 2 +- .../request-router-open-claim.test.ts | 41 --- src/daemon/server/daemon-idle-reap.ts | 10 +- .../internal/session-open-execution.ts | 32 -- ...latform-runtime-warm-runner-notice.test.ts | 34 -- src/platform-runtime-warm-runner-notice.ts | 24 -- website/docs/docs/sessions.md | 8 +- 27 files changed, 658 insertions(+), 859 deletions(-) delete mode 100644 packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts create mode 100644 packages/platform-apple/src/runner/__tests__/runner-retention.test.ts delete mode 100644 packages/platform-apple/src/runner/runner-destination-watch.ts create mode 100644 packages/platform-apple/src/runner/runner-retention.ts delete mode 100644 src/platform-runtime-warm-runner-notice.test.ts delete mode 100644 src/platform-runtime-warm-runner-notice.ts diff --git a/docs/adr/0005-ios-runner-interaction-lifecycle.md b/docs/adr/0005-ios-runner-interaction-lifecycle.md index 5c539144ce..7e9533effc 100644 --- a/docs/adr/0005-ios-runner-interaction-lifecycle.md +++ b/docs/adr/0005-ios-runner-interaction-lifecycle.md @@ -117,7 +117,11 @@ fresh xcodebuild runner startup, not a stale transport stall. A `close` then `open` on a runner still draining abandoned main-thread work now pays a fresh runner boot instead of inheriting the stalled one, so the wedge no longer survives the close/open cycle. Runners -that were never reported busy keep the existing warm-reuse path unchanged. +that are ready, have no unanswered exchanges, and have an observed live Simulator listener can +remain warm after close. Close retires unfinished startup and its recovery authority. The retained +generation owns its idle timer and listener observation; successful reuse ends retention, listener +loss or idle expiry disposes that generation, and daemon shutdown stops idle retained generations +before their observation ends. Active ready generations retain the existing handoff policy. The daemon no longer models a generic "recent success" cache as a runner-health signal. A proven healthy mutating response for the same app — recorded only after the `runnerFatal` check and only diff --git a/docs/agents/device-verification.md b/docs/agents/device-verification.md index ce843cbe9f..ce14954333 100644 --- a/docs/agents/device-verification.md +++ b/docs/agents/device-verification.md @@ -10,8 +10,9 @@ physical devices. Live verification steps apply when exercising a device-facing - Before any Android verification from source: `pnpm build`, `pnpm build:android`, `pnpm clean:daemon`. `build:android` refreshes and verifies both bundled Android helper artifacts for the current package version. -- Graceful `shutdown` hands off a healthy runner that already answered a command, on the simulator - and physical iOS lanes alike; a new daemon may adopt the old binary. After +- Graceful `shutdown` hands off an active healthy runner that already answered a command, on the + simulator and physical iOS lanes alike; a new daemon may adopt the old binary. A runner retained + idle after `close` is stopped before the daemon exits. After Swift runner changes, run `pnpm build:xcuitest` before verification. Use the session cleanup procedure below if ownership is stuck. The physical handoff's device steps are `docs/evidence/ios-physical-runner-handoff-2026-09-19.md`; nothing in it is proven until someone diff --git a/packages/kernel/src/keyed-lock.test.ts b/packages/kernel/src/keyed-lock.test.ts index 5fc2bf8259..3f87fbd652 100644 --- a/packages/kernel/src/keyed-lock.test.ts +++ b/packages/kernel/src/keyed-lock.test.ts @@ -64,3 +64,44 @@ test('withKeyedLock allows reentrant work for the same key while holding the out assert.deepEqual(order, ['outer-start', 'inner', 'outer-end', 'competing']); }); + +test('a deferred descendant of a released lock queues behind its current owner', async () => { + const locks = new Map>(); + const order: string[] = []; + let timerAttempted!: () => void; + const attempted = new Promise((resolve) => { + timerAttempted = resolve; + }); + let descendant!: Promise; + await withKeyedLock(locks, 'device-a', async () => { + descendant = new Promise((resolve, reject) => { + setTimeout(() => { + timerAttempted(); + withKeyedLock(locks, 'device-a', async () => { + order.push('descendant'); + }).then(resolve, reject); + }, 0); + }); + }); + let releaseOwner!: () => void; + const ownerGate = new Promise((resolve) => { + releaseOwner = resolve; + }); + let ownerStarted!: () => void; + const started = new Promise((resolve) => { + ownerStarted = resolve; + }); + const owner = withKeyedLock(locks, 'device-a', async () => { + order.push('owner'); + ownerStarted(); + await ownerGate; + order.push('released'); + }); + await started; + await attempted; + await new Promise((resolve) => setImmediate(resolve)); + assert.deepEqual(order, ['owner']); + releaseOwner(); + await Promise.all([owner, descendant]); + assert.deepEqual(order, ['owner', 'released', 'descendant']); +}); diff --git a/packages/kernel/src/keyed-lock.ts b/packages/kernel/src/keyed-lock.ts index de1e820184..edf4a6c828 100644 --- a/packages/kernel/src/keyed-lock.ts +++ b/packages/kernel/src/keyed-lock.ts @@ -3,6 +3,7 @@ import { AsyncLocalStorage } from 'node:async_hooks'; type LockFrame = { locks: Map>; key: string; + active: boolean; }; const keyedLockStorage = new AsyncLocalStorage(); @@ -13,13 +14,23 @@ export async function withKeyedLock( task: () => Promise, ): Promise { const activeLocks = keyedLockStorage.getStore() ?? []; - if (activeLocks.some((entry) => entry.locks === locks && entry.key === key)) { + if (activeLocks.some((entry) => entry.active && entry.locks === locks && entry.key === key)) { return await task(); } const previous = locks.get(key) ?? Promise.resolve(); const current = previous .catch(() => {}) - .then(() => keyedLockStorage.run([...activeLocks, { locks, key }], task)); + .then(async () => { + const frame: LockFrame = { locks, key, active: true }; + try { + return await keyedLockStorage.run( + [...activeLocks.filter((entry) => entry.active), frame], + task, + ); + } finally { + frame.active = false; + } + }); locks.set(key, current); return current.finally(() => { if (locks.get(key) === current) { diff --git a/packages/platform-apple/src/core/runner-client.ts b/packages/platform-apple/src/core/runner-client.ts index 4b6ecb827e..5aff792e8d 100644 --- a/packages/platform-apple/src/core/runner-client.ts +++ b/packages/platform-apple/src/core/runner-client.ts @@ -34,20 +34,6 @@ export { stopAllIosRunnerSessions, stopIosRunnerSession, } from '../runner/runner-session.ts'; -import type { RunnerWarmLossNotice } from '../runner/runner-destination-watch.ts'; - -/** - * Takes the #3321 warm-runner loss notice recorded for this device, once, if a retained runner was - * stopped because its connection closed during retention. Loaded on call, not at import: - * the watcher module must not join the façade closures the eager-closure budget holds flat, and - * the read happens once per `open` at most. - */ -export async function takeRunnerWarmLossNotice( - deviceId: string, -): Promise { - const { takeRunnerWarmLossNotice: take } = await import('../runner/runner-destination-watch.ts'); - return take(deviceId); -} export { hasCachedAppleRunnerArtifact, resolveRunnerAppBundleId, diff --git a/packages/platform-apple/src/runner-operations-facade.ts b/packages/platform-apple/src/runner-operations-facade.ts index 2eb60bb60a..7c826e11ee 100644 --- a/packages/platform-apple/src/runner-operations-facade.ts +++ b/packages/platform-apple/src/runner-operations-facade.ts @@ -26,11 +26,9 @@ export { runAppleRunnerCommand, stopAllIosRunnerSessions, stopIosRunnerSession, - takeRunnerWarmLossNotice, verifyLeaseRunnerPidIdentity, writeRunnerCacheMetadataForArtifacts, } from './core/runner-client.ts'; -export type { RunnerWarmLossNotice } from './runner/runner-destination-watch.ts'; export { queryAppleRunnerSelector } from './core/runner-selector-query.ts'; export async function cleanupRunnerLeasesForOwner( diff --git a/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts b/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts index 4702a3ad8e..a5ecc5270f 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts @@ -1,8 +1,9 @@ import assert from 'node:assert/strict'; +import { AsyncResource } from 'node:async_hooks'; import fs from 'node:fs'; import net from 'node:net'; import path from 'node:path'; -import { beforeEach, test, vi } from 'vitest'; +import { afterEach, beforeEach, test, vi } from 'vitest'; import type { AppleApplicationTools } from '@agent-device/contracts/application-lifecycle-runtime'; import type { PlatformRuntimeHost } from '@agent-device/contracts/platform-runtime-operations'; import type { DeviceInfo } from '@agent-device/kernel/device'; @@ -13,23 +14,27 @@ import { makeClassifyOwnerLivenessViaMocks, makeBackgroundRunner, makeRunnerLease, + runnerConnectFailure, runnerError, runnerResponse, RUNNER_CACHE_METADATA_FIXTURE, RUNNER_CACHE_KEY_FIXTURE, } from './runner-session-fixtures.ts'; import { mkdtempForTestSync } from './tmp-dir.ts'; -import { takeRunnerWarmLossNotice } from '../runner-destination-watch.ts'; import { bindAppleApplicationLifecycle } from '../../lifecycle.ts'; +import { createRequestCanceledError } from '@agent-device/kernel/errors'; +import { prewarmIosRunnerSession } from '../runner-client.ts'; import { platformRuntimeHostFixture } from '../../runtime.fixtures.ts'; import { runnerOwnerToken, writeRunnerLease } from '../runner-lease.ts'; import { abortAllIosRunnerSessions, + detachIosRunnerSessionsForShutdown, ensureRunnerSession, executeRunnerCommandWithSession, readRunnerSessionLiveness, releaseIosRunnerOnClose, stopIosRunnerSession, + stopAllIosRunnerSessions, type RunnerSession, } from '../runner-session.ts'; @@ -127,6 +132,14 @@ vi.mock('../runner-xctestrun.ts', async () => { const IDLE_WINDOW_OUTLIVES_RUN_MS = '600000'; let stateDir: string; +let destination: net.Server; +let destinationConnections: net.Socket[]; + +afterEach(async () => { + await abortAllIosRunnerSessions(); + for (const socket of destinationConnections) socket.destroy(); + await new Promise((resolve) => destination.close(() => resolve())); +}); beforeEach(async () => { appleRunnerTestHost.update({ @@ -162,7 +175,10 @@ beforeEach(async () => { buildMs: 12, xctestrunPathSource: 'build', }); - mockGetFreePort.mockResolvedValue(8123); + destinationConnections = []; + destination = net.createServer((socket) => destinationConnections.push(socket)); + await new Promise((resolve) => destination.listen(0, '127.0.0.1', resolve)); + mockGetFreePort.mockResolvedValue((destination.address() as net.AddressInfo).port); mockPrepareXctestrunWithEnv.mockResolvedValue({ xctestrunPath: '/tmp/session-runner.xctestrun', jsonPath: '/tmp/session-runner.json', @@ -184,7 +200,7 @@ beforeEach(async () => { test('a retained close over an idle runner keeps that runner warm for the next open (#2615)', async () => { const device = deviceNamed('close-finalize-warm-reuse'); - const session = await ensureRunnerSession(device, {}); + const session = await ensureReadyRunner(device); const events: string[] = []; const lifecycle = closeFinalizationLifecycle(device, events); @@ -204,7 +220,7 @@ test('a retained close over an idle runner keeps that runner warm for the next o test('a retained close disposes a runner still draining so the next open boots a clean one (#2552, #2615)', async () => { const device = deviceNamed('close-finalize-busy-dispose'); - const session = await ensureRunnerSession(device, {}); + const session = await ensureReadyRunner(device); await refuseWithRunnerBusy(device, session); const events: string[] = []; const lifecycle = closeFinalizationLifecycle(device, events); @@ -224,7 +240,7 @@ test('a retained close disposes a runner still draining so the next open boots a test('a non-retained close stops the runner and cancels the idle stop a retained close armed (#2615)', async () => { const device = deviceNamed('close-finalize-cancel-idle'); - await ensureRunnerSession(device, {}); + await ensureReadyRunner(device); const lifecycle = closeFinalizationLifecycle(device, []); process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS = '40'; @@ -276,24 +292,160 @@ test('a retained close with no runner in memory starts none and claims no lease assert.equal(runnerLeaseExists(device.id), false); }); -test('a start that fails without using a retained runner leaves it retained', async () => { - const device = deviceNamed('close-finalize-failed-start-retention'); - const session = await ensureRunnerSession(device, {}); +test('close retires a pending prewarm before its health failure can launch a replacement (#3359)', async () => { + const device = deviceNamed('close-finalize-pending-prewarm'); + let rejectHealth!: (error: Error) => void; + mockWaitForRunner.mockImplementationOnce( + () => + new Promise((_resolve, reject) => { + rejectHealth = reject; + }), + ); + const prewarm = prewarmIosRunnerSession(device); + await vi.waitFor(() => assert.equal(mockWaitForRunner.mock.calls.length, 1)); + await closeFinalizationLifecycle(device, []).finalizeApplicationClose(closeInput(true)); + rejectHealth(runnerConnectFailure('runner_connect_refused')); + await prewarm; - const diagnostics = await captureDiagnostics(async () => { - await assert.rejects( - ensureRunnerSession(device, { expectedRunnerSessionId: 'another-runner-session' }), + assert.equal(mockRunCmdBackground.mock.calls.length, 1); + assert.equal(readRunnerSessionLiveness(device.id), null); + assert.equal(runnerLeaseExists(device.id), false); + assert.equal( + mockRunXcrun.mock.calls.some(([args]) => args.includes('bootstatus')), + false, + ); +}); + +test.each(['transport', 'restored_artifact', 'canceled'] as const)( + 'a retired prewarm cannot invalidate a retained ready generation after %s failure', + async (failure) => { + const device = deviceNamed('close-finalize-ready-prewarm'); + const connections: net.Socket[] = []; + const listener = net.createServer((socket) => connections.push(socket)); + await new Promise((resolve) => listener.listen(0, '127.0.0.1', resolve)); + mockGetFreePort.mockResolvedValue((listener.address() as net.AddressInfo).port); + if (failure === 'restored_artifact') { + mockEnsureXctestrunArtifact.mockResolvedValue({ + xctestrunPath: '/tmp/base-runner.xctestrun', + derived: '/tmp/derived', + cacheKey: RUNNER_CACHE_KEY_FIXTURE, + cache: 'exact', + artifact: 'valid', + buildMs: 0, + xctestrunPathSource: 'build', + }); + } + let rejectHealth!: (error: Error) => void; + mockWaitForRunner.mockImplementationOnce( + () => + new Promise((_resolve, reject) => { + rejectHealth = reject; + }), ); - }); + try { + const prewarm = prewarmIosRunnerSession(device); + await vi.waitFor(() => assert.equal(mockWaitForRunner.mock.calls.length, 1)); + const session = await ensureRunnerSession(device, {}); + await executeRunnerCommandWithSession( + device, + session, + { command: 'uptime' }, + undefined, + 1_000, + ); + await closeFinalizationLifecycle(device, []).finalizeApplicationClose(closeInput(true)); + await vi.waitFor(() => assert.equal(connections.length, 1)); + rejectHealth( + failure === 'canceled' + ? createRequestCanceledError() + : runnerConnectFailure('runner_connect_refused'), + ); + await prewarm; + + assert.equal(mockRunCmdBackground.mock.calls.length, 1); + assert.equal(readRunnerSessionLiveness(device.id)?.sessionId, session.sessionId); + assert.equal(mockSignalPidsBestEffort.mock.calls.length, 0); + } finally { + await stopIosRunnerSession(device.id); + for (const socket of connections) socket.destroy(); + await new Promise((resolve) => listener.close(() => resolve())); + } + }, +); + +test('graceful shutdown stops idle retained generations and hands off active ready generations', async () => { + const idleDevice = deviceNamed('close-finalize-idle-handoff'); + const activeDevice = deviceNamed('close-finalize-active-handoff'); + const idle = await ensureReadyRunner(idleDevice); + const active = await ensureReadyRunner(activeDevice); + await closeFinalizationLifecycle(idleDevice, []).finalizeApplicationClose(closeInput(true)); + + assert.equal(await detachIosRunnerSessionsForShutdown(), 1); + assert.equal(active.state, 'stopped'); + assert.equal(idle.state, 'ready'); + await stopAllIosRunnerSessions(); + assert.equal(idle.state, 'stopped'); + assert.equal(runnerLeaseExists(idleDevice.id), false); + assert.equal(runnerLeaseExists(activeDevice.id), true); +}); - assert.match(diagnostics, /"phase":"ios_runner_idle_stop_scheduled"/); +test('an idle expiry disposes its retained generation and releases its lease', async () => { + const device = deviceNamed('close-finalize-idle-expiry'); + const session = await ensureReadyRunner(device); + vi.useFakeTimers(); + try { + process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS = '40'; + await closeFinalizationLifecycle(device, []).finalizeApplicationClose(closeInput(true)); + await vi.advanceTimersByTimeAsync(40); + assert.equal(readRunnerSessionLiveness(device.id), null); + assert.equal(session.state, 'stopped'); + assert.equal(runnerLeaseExists(device.id), false); + } finally { + vi.useRealTimers(); + } +}); + +test('an idle expiry queued during reuse cannot stop the generation that was just claimed', async () => { + const device = deviceNamed('close-finalize-expiry-during-reuse'); + const session = await ensureReadyRunner(device); + vi.useFakeTimers(); + try { + process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS = '40'; + await closeFinalizationLifecycle(device, []).finalizeApplicationClose(closeInput(true)); + const expire = AsyncResource.bind(() => vi.advanceTimersByTime(40)); + appleRunnerTestHost.update({ + emitDiagnostic: (event) => { + if (event.phase === 'ios_runner_session_reuse') expire(); + }, + }); + assert.equal(await ensureRunnerSession(device, {}), session); + await vi.runAllTimersAsync(); + assert.equal(readRunnerSessionLiveness(device.id)?.sessionId, session.sessionId); + assert.equal(session.retention, undefined); + assert.equal(mockRunCmdBackground.mock.calls.length, 1); + } finally { + vi.useRealTimers(); + } +}); + +test('a start that fails without using a retained runner leaves it retained', async () => { + const device = deviceNamed('close-finalize-failed-start-retention'); + const session = await ensureReadyRunner(device); + await closeFinalizationLifecycle(device, []).finalizeApplicationClose(closeInput(true)); + + const retention = session.retention; + await assert.rejects( + ensureRunnerSession(device, { expectedRunnerSessionId: 'another-runner-session' }), + ); + assert.equal(session.retention, retention); + assert.ok(session.retention); assert.equal(readRunnerSessionLiveness(device.id)?.sessionId, session.sessionId); }); test('a daemon-shutdown close never issues the ordinary close release (#2615)', async () => { const device = deviceNamed('close-finalize-daemon-shutdown'); - const session = await ensureRunnerSession(device, {}); + const session = await ensureReadyRunner(device); const events: string[] = []; const lifecycle = closeFinalizationLifecycle(device, events); @@ -324,7 +476,7 @@ test('a runner disposal failure propagates out of close before close alerts run test('a drain that lands after close is issued cannot return a busy runner to reuse (#2615)', async () => { const device = deviceNamed('close-finalize-queued-drain'); - const session = await ensureRunnerSession(device, {}); + const session = await ensureReadyRunner(device); await refuseWithRunnerBusy(device, session); // Hold the disposal open so a served reply that clears occupancy lands while close is in // flight, rather than counting microtasks around the check. @@ -355,7 +507,7 @@ test('a drain that lands after close is issued cannot return a busy runner to re test('a failed start does not re-retain a runner a concurrent start already took over', async () => { const device = deviceNamed('close-finalize-concurrent-start'); - const session = await ensureRunnerSession(device, {}); + const session = await ensureReadyRunner(device); await closeFinalizationLifecycle(device, []).finalizeApplicationClose(closeInput(true)); const phases: string[] = []; appleRunnerTestHost.update({ emitDiagnostic: (event) => phases.push(event.phase) }); @@ -388,7 +540,7 @@ test('a runner connection lost during retention takes the retained runner and it await new Promise((resolve) => destination.listen(0, '127.0.0.1', resolve)); mockGetFreePort.mockResolvedValue((destination.address() as net.AddressInfo).port); try { - const session = await ensureRunnerSession(device, {}); + const session = await ensureReadyRunner(device); await closeFinalizationLifecycle(device, []).finalizeApplicationClose(closeInput(true)); await vi.waitFor(() => assert.equal(connections.length, 1)); @@ -399,7 +551,7 @@ test('a runner connection lost during retention takes the retained runner and it }); assert.equal(runnerLeaseExists(device.id), false); assert.equal(session.state, 'stopped'); - assert.ok(takeRunnerWarmLossNotice(device.id)); + assert.equal(session.retention, undefined); } finally { await new Promise((resolve) => destination.close(() => resolve())); } @@ -479,3 +631,10 @@ function runnerLeaseExists(deviceId: string): boolean { path.join(process.env.AGENT_DEVICE_IOS_RUNNER_LEASE_DIR ?? '', `${deviceId}.json`), ); } + +async function ensureReadyRunner(device: DeviceInfo): Promise { + const session = await ensureRunnerSession(device, {}); + await executeRunnerCommandWithSession(device, session, { command: 'uptime' }, undefined, 1_000); + await session.listenerWatch?.ready; + return session; +} diff --git a/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts b/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts deleted file mode 100644 index 7aa9e5c1d1..0000000000 --- a/packages/platform-apple/src/runner/__tests__/runner-destination-watch.test.ts +++ /dev/null @@ -1,196 +0,0 @@ -import { afterEach, test, vi } from 'vitest'; -import assert from 'node:assert/strict'; -import net from 'node:net'; -import { - attachRunnerDestinationWatch, - closeRunnerDestinationWatch, - takeRunnerWarmLossNotice, -} from '../runner-destination-watch.ts'; -import { IOS_SIMULATOR } from './device-fixtures.ts'; - -const DEVICE = IOS_SIMULATOR; - -type FakeListener = { - port: number; - connections: net.Socket[]; - nextConnection(): Promise; - close(): Promise; -}; - -const listeners: FakeListener[] = []; - -async function listen(port = 0): Promise { - const connections: net.Socket[] = []; - const waiting: Array<(socket: net.Socket) => void> = []; - const server = net.createServer((socket) => { - socket.on('error', () => {}); - const waiter = waiting.shift(); - if (waiter) waiter(socket); - else connections.push(socket); - }); - await new Promise((resolve) => server.listen(port, '127.0.0.1', resolve)); - const listener: FakeListener = { - port: (server.address() as net.AddressInfo).port, - connections, - nextConnection: () => { - const queued = connections.shift(); - return queued - ? Promise.resolve(queued) - : new Promise((resolve) => waiting.push(resolve)); - }, - close: () => - new Promise((resolve) => { - for (const socket of connections) socket.destroy(); - server.close(() => resolve()); - }), - }; - listeners.push(listener); - return listener; -} - -async function unusedPort(): Promise { - const probe = await listen(); - const { port } = probe; - await probe.close(); - return port; -} - -function watchParams(port: number, overrides: { isArmed?: () => boolean } = {}) { - const onStop = vi.fn(async () => {}); - return { - onStop, - params: { - device: DEVICE, - sessionId: 'session-1', - port, - isArmed: overrides.isArmed ?? (() => true), - onStop, - }, - }; -} - -const realSetTimeout = globalThis.setTimeout; -const realTick = () => new Promise((resolve) => realSetTimeout(resolve, 20)); - -afterEach(async () => { - vi.useRealTimers(); - closeRunnerDestinationWatch(DEVICE.id); - takeRunnerWarmLossNotice(DEVICE.id); - await Promise.all(listeners.splice(0).map((listener) => listener.close())); -}); - -test('an established connection that closes stops the runner and leaves one notice', async () => { - const listener = await listen(); - const { params, onStop } = watchParams(listener.port); - - attachRunnerDestinationWatch(params); - (await listener.nextConnection()).destroy(); - await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); - - assert.ok(takeRunnerWarmLossNotice(DEVICE.id)); - assert.equal(takeRunnerWarmLossNotice(DEVICE.id), undefined); -}); - -test('a quiet established connection stops nothing', async () => { - const listener = await listen(); - const { params, onStop } = watchParams(listener.port); - - attachRunnerDestinationWatch(params); - await listener.nextConnection(); - await realTick(); - - assert.equal(onStop.mock.calls.length, 0); - assert.equal(takeRunnerWarmLossNotice(DEVICE.id), undefined); -}); - -test('a close after the retention window ended acts on nothing', async () => { - const listener = await listen(); - let armed = true; - const { params, onStop } = watchParams(listener.port, { isArmed: () => armed }); - - attachRunnerDestinationWatch(params); - const connection = await listener.nextConnection(); - armed = false; - connection.destroy(); - await realTick(); - - assert.equal(onStop.mock.calls.length, 0); - assert.equal(takeRunnerWarmLossNotice(DEVICE.id), undefined); -}); - -test('closing the watch detaches it so a later runner death stops nothing', async () => { - const listener = await listen(); - const { params, onStop } = watchParams(listener.port); - - attachRunnerDestinationWatch(params); - const connection = await listener.nextConnection(); - closeRunnerDestinationWatch(DEVICE.id); - connection.destroy(); - await realTick(); - - assert.equal(onStop.mock.calls.length, 0); - assert.equal(takeRunnerWarmLossNotice(DEVICE.id), undefined); -}); - -test('a second retention window replaces the existing watch instead of stacking another', async () => { - const listener = await listen(); - const first = watchParams(listener.port); - const second = watchParams(listener.port); - - attachRunnerDestinationWatch(first.params); - const firstConnection = await listener.nextConnection(); - attachRunnerDestinationWatch(second.params); - const secondConnection = await listener.nextConnection(); - firstConnection.destroy(); - await realTick(); - assert.equal(first.onStop.mock.calls.length + second.onStop.mock.calls.length, 0); - - secondConnection.destroy(); - await vi.waitFor(() => assert.equal(second.onStop.mock.calls.length, 1)); - assert.equal(first.onStop.mock.calls.length, 0); -}); - -test('a refused attach is retried, and a listener that answers keeps the runner', async () => { - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); - const port = await unusedPort(); - const { params, onStop } = watchParams(port); - - attachRunnerDestinationWatch(params); - await realTick(); - const listener = await listen(port); - await vi.advanceTimersByTimeAsync(1_000); - const connection = await listener.nextConnection(); - await realTick(); - - assert.equal(onStop.mock.calls.length, 0); - connection.destroy(); - await vi.waitFor(() => assert.equal(onStop.mock.calls.length, 1)); -}); - -test('a port that never answers stops the runner once the retry budget is spent', async () => { - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); - const { params, onStop } = watchParams(await unusedPort()); - - attachRunnerDestinationWatch(params); - for (let attempt = 0; attempt < 20 && onStop.mock.calls.length === 0; attempt += 1) { - await realTick(); - await vi.advanceTimersByTimeAsync(1_000); - } - - assert.equal(onStop.mock.calls.length, 1); - assert.ok(takeRunnerWarmLossNotice(DEVICE.id)); -}); - -test('a refused attach whose window ended stops retrying', async () => { - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); - let armed = true; - const { params, onStop } = watchParams(await unusedPort(), { isArmed: () => armed }); - - attachRunnerDestinationWatch(params); - await realTick(); - armed = false; - await vi.advanceTimersByTimeAsync(60_000); - - assert.equal(onStop.mock.calls.length, 0); - assert.equal(vi.getTimerCount(), 0); -}); diff --git a/packages/platform-apple/src/runner/__tests__/runner-retention.test.ts b/packages/platform-apple/src/runner/__tests__/runner-retention.test.ts new file mode 100644 index 0000000000..fbbf754613 --- /dev/null +++ b/packages/platform-apple/src/runner/__tests__/runner-retention.test.ts @@ -0,0 +1,122 @@ +import assert from 'node:assert/strict'; +import net from 'node:net'; +import { afterEach, test, vi } from 'vitest'; +import { observeRunnerListener, retainRunnerSession } from '../runner-retention.ts'; +import { makeRunnerSession } from './runner-session-fixtures.ts'; +import type { RunnerSession } from '../runner-session-types.ts'; + +const listeners: net.Server[] = []; +const connections: net.Socket[] = []; +const sessions: RunnerSession[] = []; + +afterEach(async () => { + vi.useRealTimers(); + vi.unstubAllEnvs(); + for (const session of sessions.splice(0)) { + session.retention?.cancel(); + session.listenerWatch?.close(); + } + for (const socket of connections.splice(0)) socket.destroy(); + await Promise.all( + listeners + .splice(0) + .map((server) => new Promise((resolve) => server.close(() => resolve()))), + ); +}); + +async function servingSession(): Promise { + const listener = net.createServer((socket) => connections.push(socket)); + await new Promise((resolve) => listener.listen(0, '127.0.0.1', resolve)); + listeners.push(listener); + const session = makeRunnerSession({ + state: 'ready', + port: (listener.address() as net.AddressInfo).port, + }); + sessions.push(session); + return session; +} + +async function observe(session: RunnerSession, onLost = vi.fn()): Promise { + observeRunnerListener(session, onLost); + assert.equal(await session.listenerWatch?.ready, true); + await vi.waitFor(() => assert.ok(connections.length)); +} + +test('one generation has one listener watch, and loss before close prevents retention', async () => { + const session = await servingSession(); + const onLost = vi.fn(); + await observe(session, onLost); + observeRunnerListener(session, onLost); + assert.equal(connections.length, 1); + connections[0]!.destroy(); + await vi.waitFor(() => assert.equal(onLost.mock.calls.length, 1)); + assert.equal(await retainRunnerSession(session, vi.fn()), false); +}); + +test('a refused watch is never attached to a later listener on the same port', async () => { + const session = await servingSession(); + const listener = listeners.pop()!; + await new Promise((resolve) => listener.close(() => resolve())); + observeRunnerListener(session, vi.fn()); + assert.equal(await session.listenerWatch?.ready, false); + const replacement = net.createServer((socket) => connections.push(socket)); + await new Promise((resolve) => replacement.listen(session.port, '127.0.0.1', resolve)); + listeners.push(replacement); + observeRunnerListener(session, vi.fn()); + assert.equal(await retainRunnerSession(session, vi.fn()), false); + assert.equal(connections.length, 0); +}); + +test('the idle timer expires once and cancellation of an old window preserves its replacement', async () => { + const session = await servingSession(); + await observe(session); + vi.useFakeTimers(); + vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS', '40'); + const stop = vi.fn(async () => {}); + await retainRunnerSession(session, stop); + const previous = session.retention!; + await vi.advanceTimersByTimeAsync(20); + await retainRunnerSession(session, stop); + const current = session.retention; + previous.cancel(); + assert.equal(session.retention, current); + await vi.advanceTimersByTimeAsync(39); + assert.equal(stop.mock.calls.length, 0); + await vi.advanceTimersByTimeAsync(1); + assert.deepEqual(stop.mock.calls, [[current, 'idle_timeout']]); +}); + +test('ending retention cancels its idle timer while preserving observation', async () => { + const session = await servingSession(); + await observe(session); + vi.useFakeTimers(); + vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS', '40'); + const stop = vi.fn(async () => {}); + await retainRunnerSession(session, stop); + session.retention!.cancel(); + await vi.advanceTimersByTimeAsync(40); + assert.equal(stop.mock.calls.length, 0); + assert.equal(session.retention, undefined); + assert.equal(session.listenerWatch?.lost, false); +}); + +test('zero disables only the idle timer, and unready or occupied generations cannot be retained', async () => { + const session = await servingSession(); + await observe(session); + vi.useFakeTimers(); + vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS', '0'); + const stop = vi.fn(async () => {}); + assert.equal(await retainRunnerSession(session, stop), true); + await vi.advanceTimersByTimeAsync(600_000); + assert.equal(stop.mock.calls.length, 0); + assert.ok(session.retention); + session.retention!.cancel(); + session.state = 'starting'; + assert.equal(await retainRunnerSession(session, stop), false); + session.state = 'ready'; + session.runnerMainThreadBusy = true; + assert.equal(await retainRunnerSession(session, stop), false); + session.runnerMainThreadBusy = false; + session.commandCharges.charge('still-in-flight'); + assert.equal(await retainRunnerSession(session, stop), false); +}); diff --git a/packages/platform-apple/src/runner/__tests__/runner-session-close-prep-fence.test.ts b/packages/platform-apple/src/runner/__tests__/runner-session-close-prep-fence.test.ts index 9540029d88..3cd2699ff6 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-session-close-prep-fence.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-session-close-prep-fence.test.ts @@ -181,60 +181,67 @@ beforeEach(async () => { ); }); -test('close during a cold build leaves the start no way to spawn a replacement build', async () => { - const device = IOS_ADMISSION_SIMULATOR; - // Park close inside its prep stop: the kill is attempted, but the stop has not returned, so - // close provably has not reached the session lock yet (the start holds it) and no later fence - // could have run. That is the moment the device must already be fenced. - let releasePrepStop: () => void = () => {}; - const prepStopParked = new Promise((resolve) => { - releasePrepStop = resolve; - }); - let signalAttempted = () => {}; - const killAttempted = new Promise((resolve) => { - signalAttempted = resolve; - }); - mockRunAppleToolCommand.mockImplementation(async (tool: string, args: string[]) => { - // `killRunnerProcessTree` is the only pkill that signals by parent pid (`-P`); the lease - // cleanup's stale-launch sweep matches argv (`-f`), so only the prep tree-kill parks. - if (tool === 'pkill' && args.includes('-P')) { - signalAttempted(); - await prepStopParked; - } - return { exitCode: 0, stdout: '', stderr: '' }; - }); +test.each([false, true])( + 'close (retain=%s) during a cold build leaves no replacement build', + async (retain) => { + const device = IOS_ADMISSION_SIMULATOR; + // Park close inside its prep stop: the kill is attempted, but the stop has not returned, so + // close provably has not reached the session lock yet (the start holds it) and no later fence + // could have run. That is the moment the device must already be fenced. + let releasePrepStop: () => void = () => {}; + const prepStopParked = new Promise((resolve) => { + releasePrepStop = resolve; + }); + let signalAttempted = () => {}; + const killAttempted = new Promise((resolve) => { + signalAttempted = resolve; + }); + mockRunAppleToolCommand.mockImplementation(async (tool: string, args: string[]) => { + // `killRunnerProcessTree` is the only pkill that signals by parent pid (`-P`); the lease + // cleanup's stale-launch sweep matches argv (`-f`), so only the prep tree-kill parks. + if (tool === 'pkill' && args.includes('-P')) { + signalAttempted(); + await prepStopParked; + } + return { exitCode: 0, stdout: '', stderr: '' }; + }); - // A cold start: holds the session lock for the whole build, as a real one does. - const firstStart = ensureRunnerSession(device, {}).catch((error: unknown) => error); - await vi.waitFor(() => assert.equal(builds.length, 1)); - assert.equal(runnerPrepProcessChildren(device.id).length, 1, 'the build is on the prep ledger'); + // A cold start: holds the session lock for the whole build, as a real one does. + const firstStart = ensureRunnerSession(device, {}).catch((error: unknown) => error); + await vi.waitFor(() => assert.equal(builds.length, 1)); + assert.equal(runnerPrepProcessChildren(device.id).length, 1, 'the build is on the prep ledger'); - const closing = releaseIosRunnerOnClose(device.id, { retain: false }); - await killAttempted; - assert.equal( - runnerStartTeardownPending(device.id), - true, - 'the device was fenced before the prep stop finished', - ); + const closing = releaseIosRunnerOnClose(device.id, { retain }); + await killAttempted; + assert.equal( + runnerStartTeardownPending(device.id), + true, + 'the device was fenced before the prep stop finished', + ); - // The prewarm health retry of the incident: the same caller starts over while close waits. - const retriedStart = ensureRunnerSession(device, {}).catch((error: unknown) => error); - releasePrepStop(); - builds[0]!.kill(); + // The prewarm health retry of the incident: the same caller starts over while close waits. + const retriedStart = ensureRunnerSession(device, {}).catch((error: unknown) => error); + releasePrepStop(); + builds[0]!.kill(); - const [first, retried] = await Promise.all([firstStart, retriedStart, closing]); + const [first, retried] = await Promise.all([firstStart, retriedStart, closing]); - assert.ok(first instanceof Error, 'the start whose build was killed failed'); - assert.ok(isRequestCanceledError(retried), 'the retry is refused as a canceled start'); - assert.equal( - (retried as { details?: { runnerStartRetirementReason?: unknown } }).details - ?.runnerStartRetirementReason, - 'device_teardown', - 'the refusal carries the typed retirement reason, not just a cancellation', - ); - assert.equal(mockBuildForTesting.mock.calls.length, 1, 'no replacement build was ever spawned'); - assert.equal(runnerPrepProcessChildren(device.id).length, 0, 'the killed build left the ledger'); -}); + assert.ok(first instanceof Error, 'the start whose build was killed failed'); + assert.ok(isRequestCanceledError(retried), 'the retry is refused as a canceled start'); + assert.equal( + (retried as { details?: { runnerStartRetirementReason?: unknown } }).details + ?.runnerStartRetirementReason, + 'device_teardown', + 'the refusal carries the typed retirement reason, not just a cancellation', + ); + assert.equal(mockBuildForTesting.mock.calls.length, 1, 'no replacement build was ever spawned'); + assert.equal( + runnerPrepProcessChildren(device.id).length, + 0, + 'the killed build left the ledger', + ); + }, +); /** * The reviewer's counter-case to the fence: a caller that merely QUEUED behind the close — it diff --git a/packages/platform-apple/src/runner/__tests__/runner-session-close.test.ts b/packages/platform-apple/src/runner/__tests__/runner-session-close.test.ts index 67b43e3133..66528aaf1c 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-session-close.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-session-close.test.ts @@ -288,38 +288,14 @@ test('a served non-busy error clears a stale busy report so close keeps a draine assert.equal(session.runnerMainThreadBusy, false); }); -test('releaseIosRunnerOnClose retains an idle runner, disposes a busy one, and the reopen boots fresh (#2552)', async () => { +test('close stops an unready generation and the next use starts fresh', async () => { const device = { ...IOS_SIMULATOR, id: 'runner-session-release-on-close-sim' }; const session = await ensureRunnerSession(device, {}); - - // Idle + retain: warm reuse, same session comes back for the next open. - await releaseIosRunnerOnClose(device.id, { retain: true }); - assert.ok(readRunnerSessionLiveness(device.id)); - assert.equal((await ensureRunnerSession(device, {})).sessionId, session.sessionId); - - // Busy + retain: a command that stalls answers MAIN_THREAD_TIMEOUT, so the stalled runner is - // disposed and the next open boots a clean one. - // The session has not served a command yet, so the read-only snapshot answers over the startup - // transport with no preflight. - mockWaitForRunner.mockResolvedValueOnce( - runnerError({ code: 'MAIN_THREAD_TIMEOUT', message: 'main thread execution timed out' }), - ); - await assert.rejects(() => - executeRunnerCommandWithSession( - device, - session, - { command: 'snapshot', appBundleId: 'com.example.demo' }, - '/tmp/runner.log', - 30_000, - ), - ); await releaseIosRunnerOnClose(device.id, { retain: true }); assert.equal(readRunnerSessionLiveness(device.id), null); + assert.equal(session.state, 'stopped'); + assert.equal(mockWaitForRunner.mock.calls.length, 0); assert.notEqual((await ensureRunnerSession(device, {})).sessionId, session.sessionId); - - // Non-retained close: stops regardless of occupancy. - await releaseIosRunnerOnClose(device.id, { retain: false }); - assert.equal(readRunnerSessionLiveness(device.id), null); }); /** diff --git a/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts b/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts index 048bb20b5c..6237bfe7fb 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import { beforeEach, test, vi } from 'vitest'; +import { retainRunnerSession } from '../runner-retention.ts'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { IOS_DEVICE, IOS_SIMULATOR, MACOS_DEVICE } from './device-fixtures.ts'; import { appleRunnerTestHost } from '../test-host.ts'; @@ -136,7 +137,6 @@ import { executeRunnerCommandWithSession, invalidateRunnerSession, readRunnerSessionLiveness, - scheduleIosRunnerIdleStop, stopIosRunnerSession, } from '../runner-session.ts'; @@ -244,7 +244,13 @@ test('an idle stop moves a ready session through disposal to stopped', async () 30_000, ); - scheduleIosRunnerIdleStop(device.id); + session.listenerWatch?.close(); + session.listenerWatch = { + lost: false, + ready: Promise.resolve(true), + close() {}, + }; + await retainRunnerSession(session, async () => stopIosRunnerSession(device.id)); await vi.waitFor(() => assert.equal(session.state, 'stopped')); assert.equal( diff --git a/packages/platform-apple/src/runner/__tests__/runner-session.test.ts b/packages/platform-apple/src/runner/__tests__/runner-session.test.ts index 775e284c9f..facfb59a32 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-session.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-session.test.ts @@ -117,9 +117,7 @@ vi.mock('../runner-xctestrun.ts', async () => { import { abortAllIosRunnerSessions, - cancelIosRunnerIdleStop, ensureRunnerSession, - scheduleIosRunnerIdleStop, executeRunnerCommandWithSession, readRunnerSessionLiveness, invalidateRunnerSession, @@ -404,58 +402,6 @@ test('runner session does not require Apple developer mode for iOS simulators', assert.equal(mockRunAppleToolCommand.mock.calls.some(isDevToolsSecurityStatusCall), false); }); -test('idle stop tears down a retained runner after the idle window', async () => { - const device = { ...IOS_SIMULATOR, id: 'runner-session-idle-stop-sim' }; - const previousIdleMs = process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS; - process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS = '40'; - try { - await ensureRunnerSession(device, {}); - scheduleIosRunnerIdleStop(device.id); - - await new Promise((resolve) => setTimeout(resolve, 150)); - assert.equal(readRunnerSessionLiveness(device.id), null); - } finally { - cancelIosRunnerIdleStop(device.id); - if (previousIdleMs === undefined) delete process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS; - else process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS = previousIdleMs; - } -}); - -test('any runner use cancels a pending idle stop', async () => { - const device = { ...IOS_SIMULATOR, id: 'runner-session-idle-cancel-sim' }; - const previousIdleMs = process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS; - process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS = '40'; - try { - await ensureRunnerSession(device, {}); - scheduleIosRunnerIdleStop(device.id); - await ensureRunnerSession(device, {}); - - await new Promise((resolve) => setTimeout(resolve, 150)); - assert.ok(readRunnerSessionLiveness(device.id)); - } finally { - cancelIosRunnerIdleStop(device.id); - if (previousIdleMs === undefined) delete process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS; - else process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS = previousIdleMs; - } -}); - -test('idle stop is disabled when the window is zero', async () => { - const device = { ...IOS_SIMULATOR, id: 'runner-session-idle-disabled-sim' }; - const previousIdleMs = process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS; - process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS = '0'; - try { - await ensureRunnerSession(device, {}); - scheduleIosRunnerIdleStop(device.id); - - await new Promise((resolve) => setTimeout(resolve, 100)); - assert.ok(readRunnerSessionLiveness(device.id)); - } finally { - cancelIosRunnerIdleStop(device.id); - if (previousIdleMs === undefined) delete process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS; - else process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS = previousIdleMs; - } -}); - test('runner session startup kills legacy ownerless xcodebuild before launching a new runner', async () => { const device = { ...IOS_SIMULATOR, id: 'runner-session-startup-stale-sim' }; diff --git a/packages/platform-apple/src/runner/runner-adoption.ts b/packages/platform-apple/src/runner/runner-adoption.ts index 4ea3d0316f..5511eff1fc 100644 --- a/packages/platform-apple/src/runner/runner-adoption.ts +++ b/packages/platform-apple/src/runner/runner-adoption.ts @@ -159,6 +159,7 @@ function detachRunnerSessionForShutdown(session: RunnerSession): RunnerDetachOut // the runner's log: until that write lands the session is still owned, and an owned session that // stopped following its runner's output is worse off than one that never handed anything off. // The runner holds its own descriptor, so this cannot disturb it either way (#2681). + session.listenerWatch?.close(); session.endOutputObservation?.(); advanceRunnerSessionState(session, 'stopped'); return { detached: true, lane }; diff --git a/packages/platform-apple/src/runner/runner-destination-watch.ts b/packages/platform-apple/src/runner/runner-destination-watch.ts deleted file mode 100644 index b7e6d195e7..0000000000 --- a/packages/platform-apple/src/runner/runner-destination-watch.ts +++ /dev/null @@ -1,129 +0,0 @@ -import net from 'node:net'; -import type { DeviceInfo } from '@agent-device/kernel/device'; -import { emitDiagnostic } from './host.ts'; - -/** - * What the watcher records when it stops a retained runner: the moment the loss was seen. Read - * once by the next `open`. - */ -export type RunnerWarmLossNotice = Readonly<{ atMs: number }>; - -/** - * A push watcher for a runner retained after `close` (#3321). - * - * A retained `xcodebuild test-without-building` owns its Simulator destination: when something - * outside agent-device shuts that Simulator down, the runner app dies and Xcode silently reboots - * the device to keep its own session alive, with nobody asking agent-device. The device then stays - * powered on for as long as the runner process lives. Stopping the runner is what powers it back - * off, and stopping a runner whose device was not rebooted only costs its warm reuse. - * - * The watcher holds one idle TCP connection to the runner's loopback listener for the retention - * window and writes nothing on it. The listener never closes an idle connection on its own, so a - * connection that closes after it was established means the runner app ended, and the retained - * runner is stopped. Nothing samples a quiet runner. A refused attach is retried on a flat cadence - * inside a bounded budget, because the listener can refuse for a few seconds right after `close`; - * a port that stays refused is a runner that cannot answer, and is stopped the same way. - */ - -const ATTACH_RETRY_DELAY_MS = 1_000; -const ATTACH_RETRY_ATTEMPTS = 15; - -export type RunnerDestinationWatchParams = { - device: DeviceInfo; - sessionId: string; - port: number; - /** Re-read at every decision: false once any path has left the idle-retention window. */ - isArmed: () => boolean; - /** Stops the retained runner and releases its lease. */ - onStop: () => Promise; - /** Attach attempts left for a refused port; a fresh retention window starts a fresh budget. */ - attemptsLeft?: number; -}; - -type DestinationWatch = { socket: net.Socket; retryTimer?: NodeJS.Timeout }; - -const destinationWatches = new Map(); -const pendingWarmLossNotices = new Map(); - -/** - * Attaches the watcher for one retention window. A device has one watch: a fresh window replaces - * the existing one. `isArmed` is consulted again at every decision, so an attach that lands after - * its retention window already ended arms nothing that can act. - */ -export function attachRunnerDestinationWatch(params: RunnerDestinationWatchParams): void { - const deviceId = params.device.id; - closeRunnerDestinationWatch(deviceId); - const watch: DestinationWatch = { socket: net.connect(params.port, '127.0.0.1') }; - let connected = false; - watch.socket.unref(); - watch.socket.on('connect', () => { - connected = true; - }); - // `close` always follows an `error`; the decision is made once, there. - watch.socket.on('error', () => {}); - watch.socket.on('close', () => { - if (destinationWatches.get(deviceId) !== watch) return; - if (!params.isArmed()) { - closeRunnerDestinationWatch(deviceId); - } else if (connected) { - stopRetainedRunner(params); - } else { - retryRefusedAttach(watch, params); - } - }); - destinationWatches.set(deviceId, watch); -} - -/** - * Ends the watch without acting. Every path that leaves the idle-retention window calls it; even a - * missed call is inert, because `isArmed` re-reads the retention state at each decision. - */ -export function closeRunnerDestinationWatch(deviceId: string): void { - const watch = destinationWatches.get(deviceId); - if (!watch) return; - destinationWatches.delete(deviceId); - clearTimeout(watch.retryTimer); - watch.socket.removeAllListeners(); - watch.socket.on('error', () => {}); - watch.socket.destroy(); -} - -function retryRefusedAttach(watch: DestinationWatch, params: RunnerDestinationWatchParams): void { - const attemptsLeft = params.attemptsLeft ?? ATTACH_RETRY_ATTEMPTS; - if (attemptsLeft <= 0) { - stopRetainedRunner(params); - return; - } - watch.retryTimer = setTimeout(() => { - if (params.isArmed()) - attachRunnerDestinationWatch({ ...params, attemptsLeft: attemptsLeft - 1 }); - else closeRunnerDestinationWatch(params.device.id); - }, ATTACH_RETRY_DELAY_MS); - watch.retryTimer.unref(); -} - -/** Stops the retained runner, recording the notice before the stop so an `open` cannot miss it. */ -function stopRetainedRunner(params: RunnerDestinationWatchParams): void { - const deviceId = params.device.id; - closeRunnerDestinationWatch(deviceId); - pendingWarmLossNotices.set(deviceId, { atMs: Date.now() }); - emitDiagnostic({ - level: 'warn', - phase: 'ios_runner_warm_stop', - data: { deviceId, sessionId: params.sessionId }, - }); - void params.onStop().catch((error: unknown) => { - emitDiagnostic({ - level: 'warn', - phase: 'ios_runner_warm_stop_failed', - data: { deviceId, error: error instanceof Error ? error.message : String(error) }, - }); - }); -} - -/** Takes the loss notice recorded for this device, once. */ -export function takeRunnerWarmLossNotice(deviceId: string): RunnerWarmLossNotice | undefined { - const notice = pendingWarmLossNotices.get(deviceId); - pendingWarmLossNotices.delete(deviceId); - return notice; -} diff --git a/packages/platform-apple/src/runner/runner-disposal.ts b/packages/platform-apple/src/runner/runner-disposal.ts index a046d1bfdb..f4d74ed16e 100644 --- a/packages/platform-apple/src/runner/runner-disposal.ts +++ b/packages/platform-apple/src/runner/runner-disposal.ts @@ -65,6 +65,8 @@ export async function disposeRunnerSession( ): Promise { // From here the session is going away: it still owns the lease and the process is still up, so // it can answer a request, and it must not be chosen for new work while it is being taken down. + session.retention?.cancel(); + session.listenerWatch?.close(); advanceRunnerSessionState(session, 'draining'); let processExitHandled = false; if (options.graceful !== false) { @@ -97,6 +99,8 @@ export async function abortRunnerSessionsAndPrepProcesses( const macOsSessions = activeSessions.filter((session) => isMacOs(session.device)); const otherSessions = activeSessions.filter((session) => !isMacOs(session.device)); for (const session of activeSessions) { + session.retention?.cancel(); + session.listenerWatch?.close(); advanceRunnerSessionState(session, 'draining'); } await signalRunnerSessions(otherSessions, 'SIGINT'); diff --git a/packages/platform-apple/src/runner/runner-lifecycle.ts b/packages/platform-apple/src/runner/runner-lifecycle.ts index 76c5218e4d..92305a836c 100644 --- a/packages/platform-apple/src/runner/runner-lifecycle.ts +++ b/packages/platform-apple/src/runner/runner-lifecycle.ts @@ -44,6 +44,7 @@ import type { AppleRunnerPrepareResult, } from './runner-provider.ts'; import { + assertRunnerStartAdmitsPreparation, finishRunnerStartAdmission, markRunnerXctestrunArtifactBadForRun, openRunnerStartLoopAdmission, @@ -156,6 +157,7 @@ async function handlePrepareHealthFailure(params: { error: unknown; }): Promise { const { device, session, command, options, signal, attempt, error } = params; + assertRunnerStartAdmitsPreparation(device.id, options.startAdmission); const appErr = asAppError(error, 'COMMAND_FAILED'); if (isRequestCanceledError(appErr)) { // The owning request was canceled mid-startup (client disconnect): stop the diff --git a/packages/platform-apple/src/runner/runner-retention.ts b/packages/platform-apple/src/runner/runner-retention.ts new file mode 100644 index 0000000000..8257aba5cc --- /dev/null +++ b/packages/platform-apple/src/runner/runner-retention.ts @@ -0,0 +1,129 @@ +import net from 'node:net'; +import { isIosFamily } from '@agent-device/kernel/device'; +import { emitDiagnostic } from './host.ts'; +import { isRunnerMainThreadOccupied, type RunnerSession } from './runner-session-types.ts'; + +const LISTENER_CONNECT_TIMEOUT_MS = 500; +const RETAINED_IDLE_STOP_DEFAULT_MS = 5 * 60_000; + +export type RunnerListenerWatch = { + lost: boolean; + ready: Promise; + close(): void; +}; + +export type RunnerRetention = { cancel(): void }; +type StopRetainedRunner = ( + retention: RunnerRetention, + reason: 'idle_timeout' | 'listener_lost', +) => Promise; + +/** Observes one serving generation; a lost connection is never attached to a replacement. */ +export function observeRunnerListener( + session: RunnerSession, + onLost: () => Promise | undefined, +): void { + if (session.state !== 'ready' || !needsListenerWatch(session) || session.listenerWatch) return; + const socket = net.connect(session.port, '127.0.0.1'); + socket.unref(); + let settle!: (connected: boolean) => void; + const ready = new Promise((resolve) => { + settle = resolve; + }); + const connectTimer = setTimeout(() => socket.destroy(), LISTENER_CONNECT_TIMEOUT_MS); + connectTimer.unref(); + const watch: RunnerListenerWatch = { + lost: false, + ready, + close() { + clearTimeout(connectTimer); + watch.lost = true; + settle(false); + socket.removeAllListeners(); + socket.on('error', () => {}); + socket.destroy(); + }, + }; + session.listenerWatch = watch; + socket.on('connect', () => { + clearTimeout(connectTimer); + settle(true); + }); + socket.on('error', () => {}); + socket.on('close', () => { + clearTimeout(connectTimer); + watch.lost = true; + settle(false); + void onLost()?.catch((error: unknown) => { + emitDiagnostic({ + level: 'warn', + phase: 'ios_runner_warm_stop_failed', + data: { deviceId: session.deviceId, sessionId: session.sessionId, error: String(error) }, + }); + }); + }); +} + +/** Retains a ready generation already observed by its listener connection, with one idle timer. */ +export async function retainRunnerSession( + session: RunnerSession, + stop: StopRetainedRunner, +): Promise { + if ( + session.state !== 'ready' || + isRunnerMainThreadOccupied(session) || + session.commandCharges.hasOutstandingCharges + ) + return false; + if (needsListenerWatch(session)) { + const watch = session.listenerWatch; + if (!watch || !(await watch.ready) || watch.lost) return false; + } + if ( + session.state !== 'ready' || + isRunnerMainThreadOccupied(session) || + session.commandCharges.hasOutstandingCharges + ) + return false; + session.retention?.cancel(); + let timer: NodeJS.Timeout | undefined; + const retention: RunnerRetention = { + cancel() { + clearTimeout(timer); + if (session.retention === retention) session.retention = undefined; + }, + }; + session.retention = retention; + const idleMs = resolveRunnerIdleStopMs(); + if (idleMs > 0) { + timer = setTimeout(() => { + void stop(retention, 'idle_timeout').catch((error: unknown) => { + emitDiagnostic({ + level: 'warn', + phase: 'ios_runner_idle_stop_failed', + data: { deviceId: session.deviceId, sessionId: session.sessionId, error: String(error) }, + }); + }); + }, idleMs); + timer.unref(); + emitDiagnostic({ + level: 'debug', + phase: 'ios_runner_idle_stop_scheduled', + data: { deviceId: session.deviceId, sessionId: session.sessionId, idleMs }, + }); + } + return true; +} + +function needsListenerWatch(session: RunnerSession): boolean { + return session.device.kind === 'simulator' && isIosFamily(session.device); +} + +function resolveRunnerIdleStopMs(env: NodeJS.ProcessEnv = process.env): number { + const raw = env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS?.trim(); + if (raw) { + const parsed = Number(raw); + if (Number.isFinite(parsed) && parsed >= 0) return Math.floor(parsed); + } + return RETAINED_IDLE_STOP_DEFAULT_MS; +} diff --git a/packages/platform-apple/src/runner/runner-session-types.ts b/packages/platform-apple/src/runner/runner-session-types.ts index 4ea18af424..edad7cda39 100644 --- a/packages/platform-apple/src/runner/runner-session-types.ts +++ b/packages/platform-apple/src/runner/runner-session-types.ts @@ -1,3 +1,4 @@ +import type { RunnerListenerWatch, RunnerRetention } from './runner-retention.ts'; import type { RunnerLogicalLeaseContext } from '@agent-device/contracts/runner-lease-context'; import type { ExecResult } from '@agent-device/host-kit/command'; import type { DeviceInfo } from '@agent-device/kernel/device'; @@ -79,6 +80,8 @@ export type RunnerSession = { readLogTail?: (maxBytes: number) => string; /** Moves only through {@link advanceRunnerSessionState}. */ state: RunnerSessionState; + listenerWatch?: RunnerListenerWatch; + retention?: RunnerRetention; /** Wakes one startup retry when the listener becomes ready or its process exits. */ startupRetryWake?: AbortSignal; /** @@ -174,6 +177,8 @@ export function isRunnerMainThreadOccupied( export type RunnerDetachRefusal = /** The runner never answered a command, so nothing proves it serves requests (#2681). */ | 'runner_never_served_a_command' + /** Idle retention depends on this daemon's timer and listener observation. */ + | 'retained_idle' /** A command is still owed a response, so the runner is busy whatever its last report says (#2681). */ | 'command_in_flight' /** The runner reported main-thread work still draining as of its last exchange. */ @@ -319,11 +324,12 @@ function normalizeRunnerChargeId(commandId: string | undefined): string { * command the next daemon sends it. */ export function resolveRunnerDetachDecision( - session: Pick, + session: Pick, ): RunnerDetachDecision { if (session.state !== 'ready') { return { detach: false, reason: 'runner_never_served_a_command' }; } + if (session.retention) return { detach: false, reason: 'retained_idle' }; if (session.commandCharges.hasOutstandingCharges) { return { detach: false, reason: 'command_in_flight' }; } diff --git a/packages/platform-apple/src/runner/runner-session.ts b/packages/platform-apple/src/runner/runner-session.ts index 4f908714fb..e6e783d282 100644 --- a/packages/platform-apple/src/runner/runner-session.ts +++ b/packages/platform-apple/src/runner/runner-session.ts @@ -7,7 +7,7 @@ import { buildSimctlArgsForDevice, runXcrun, } from './host.ts'; -import { isApplePlatform, isIosFamily, type DeviceInfo } from '@agent-device/kernel/device'; +import { isApplePlatform, type DeviceInfo } from '@agent-device/kernel/device'; import type { RunnerLogicalLeaseContext } from '@agent-device/contracts/runner-lease-context'; import type { AppleRunnerLifecycleOptions } from './runner-provider.ts'; import { flushRunnerLogAppends, getFreePort, resolveRunnerLaunchLogPath } from './runner-io.ts'; @@ -81,21 +81,6 @@ export type RunnerSessionOptions = AppleRunnerLifecycleOptions; const runnerSessions = new Map(); const runnerSessionLocks = new Map>(); -const runnerIdleStopTimers = new Map(); -/** - * Devices whose runner a `close` retained for warm reuse, until the window ends. Owns the - * retention fact separately from the idle-stop timer: the timer is disabled when - * `AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS=0`, and the destination watcher's duty (#3321) covers the - * whole retain-until-daemon-exit behavior just the same. - */ -const runnerWarmRetainedDevices = new Set(); -/** - * Advances every time a device's retention state is ended, so "nothing has touched this device's - * retention since I released it" is a fact a caller can hold: a start that released a retained - * runner may put it back only while the epoch it released at is still current. - */ -const runnerRetentionEpochs = new Map(); -const RUNNER_RETAINED_IDLE_STOP_DEFAULT_MS = 5 * 60_000; const RUNNER_STALE_BUNDLE_UNINSTALL_TIMEOUT_MS = 10_000; function withRunnerSessionLock(deviceId: string, task: () => Promise): Promise { @@ -106,9 +91,6 @@ export async function ensureRunnerSession( device: DeviceInfo, options: RunnerSessionOptions, ): Promise { - // Any runner use means the device is active again: a pending idle stop - // from a retained-after-close runner no longer applies. - const released = releaseRetainedRunnerForUse(device.id); // This start's admission: the loop that owns the start across retries supplies one, and every // other start takes a token for itself. Both are taken synchronously, before the first await and // registered with the device, so a teardown beginning in this same turn closes it even while @@ -139,7 +121,10 @@ export async function ensureRunnerSession( if (existing) { assertExpectedRunnerSession(existing, options.expectedRunnerSessionId); const reusable = await resolveReusableRunnerSession(device, existing, budget.phase); - if (reusable) return reusable; + if (reusable) { + reusable.retention?.cancel(); + return reusable; + } } return await withRunnerLeaseLock( @@ -153,21 +138,12 @@ export async function ensureRunnerSession( releaseOwnerInterest(); } }); - try { - const { raceRunnerStartAgainstCaller } = await import('./runner-start-budget.ts'); - return await raceRunnerStartAgainstCaller( - // A start that minted its own token releases it; a loop-supplied token belongs to the loop, - // which finishes it when the loop itself is done retrying (#3220). - ownedAdmission ? start : start.finally(() => finishRunnerStartAdmission(startAdmission)), - startAdmission, - options.signal, - ); - } catch (error) { - // A start that failed without using the retained runner leaves it exactly as idle as it was, - // so it goes back to the retention that carries both its idle stop and its destination watch. - if (released) await restoreReleasedRunnerRetention(device.id, released); - throw error; - } + const { raceRunnerStartAgainstCaller } = await import('./runner-start-budget.ts'); + return await raceRunnerStartAgainstCaller( + ownedAdmission ? start : start.finally(() => finishRunnerStartAdmission(startAdmission)), + startAdmission, + options.signal, + ); } /** How long the device-readiness probe may take, bounded by the startup budget it runs inside. */ @@ -218,6 +194,7 @@ async function startRunnerSessionWithLease( await prepareRunnerLeaseForStartup(device, runnerLeaseCleanupAdapter, logicalLeaseContext); }); await measureRunnerStartupStep(startupTimings, 'ensure_booted', async () => { + assertRunnerStartAdmitsPreparation(device.id, startAdmission); await ensureBootedIfNeeded(device); }); // Device first, host second: both answers can be wrong at once, and the phone's own state is the @@ -313,6 +290,7 @@ async function startRunnerSessionWithLease( // where this generation's output starts is only trustworthy once those bytes have landed below // it; otherwise a queued build line reads as the runner's own failure output (#2681). await flushRunnerLogAppends(runnerLogPath).catch(() => {}); + assertRunnerStartAdmitsPreparation(device.id, startAdmission); return await launchRunnerProcess({ device, port, @@ -632,160 +610,25 @@ async function stopRunnerSessionInternal( } } -// Bounds the lifetime of a runner retained after session close: the retained -// runner holds the device's runner lease, which blocks every other daemon on -// the machine from using the device. If nothing touches the runner within the -// idle window, stop it and release the lease. Any ensureRunnerSession call -// cancels the pending stop. AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS overrides -// the window; 0 disables idle stops (retain until daemon exit, the pre-idle -// behavior). -export function scheduleIosRunnerIdleStop(deviceId: string): void { - cancelIosRunnerIdleStop(deviceId); - const idleMs = resolveRunnerIdleStopMs(); - if (idleMs <= 0) return; - if (!runnerSessions.has(deviceId)) return; - const timer = setTimeout(() => { - runnerIdleStopTimers.delete(deviceId); +async function stopRetainedRunner( + session: RunnerSession, + retention: NonNullable, + reason: 'idle_timeout' | 'listener_lost', +): Promise { + await withRunnerSessionLock(session.deviceId, async () => { + if (runnerSessions.get(session.deviceId) !== session || session.retention !== retention) return; emitDiagnostic({ - level: 'info', - phase: 'ios_runner_idle_stop', - data: { deviceId, idleMs }, + level: reason === 'listener_lost' ? 'warn' : 'info', + phase: reason === 'listener_lost' ? 'ios_runner_warm_stop' : 'ios_runner_idle_stop', + data: { deviceId: session.deviceId, sessionId: session.sessionId, reason }, }); - stopIosRunnerSession(deviceId).catch((error: unknown) => { - emitDiagnostic({ - level: 'warn', - phase: 'ios_runner_idle_stop_failed', - data: { - deviceId, - error: error instanceof Error ? error.message : String(error), - }, - }); + await stopRunnerSessionInternal(session.deviceId, session, { + graceful: reason !== 'listener_lost', + waitTimeoutMs: RUNNER_INVALIDATE_WAIT_TIMEOUT_MS, }); - }, idleMs); - timer.unref?.(); - runnerIdleStopTimers.set(deviceId, timer); - emitDiagnostic({ - level: 'debug', - phase: 'ios_runner_idle_stop_scheduled', - data: { deviceId, idleMs }, }); } -export function cancelIosRunnerIdleStop(deviceId: string): void { - runnerRetentionEpochs.set(deviceId, (runnerRetentionEpochs.get(deviceId) ?? 0) + 1); - endRunnerWarmRetention(deviceId); - const timer = runnerIdleStopTimers.get(deviceId); - if (!timer) return; - clearTimeout(timer); - runnerIdleStopTimers.delete(deviceId); -} - -/** - * Ends this device's warm-retention window. The destination watcher's duty (#3321) covers exactly - * the time between a retaining `close` and the moment the runner is used or stopped again; after - * that point nothing may act on a stale watch. Retention always begins in code that has already - * loaded the watcher module, so the close here reads it from the module cache. - */ -function endRunnerWarmRetention(deviceId: string): void { - if (!runnerWarmRetainedDevices.delete(deviceId)) return; - void import('./runner-destination-watch.ts').then( - ({ closeRunnerDestinationWatch }) => closeRunnerDestinationWatch(deviceId), - () => {}, - ); -} - -/** - * Whether this session is the device's runner sitting in the post-`close` idle-retention window: - * retained by a close and nothing has used or stopped it since, still registered, and not being - * torn down. Any use (`ensureRunnerSession`) or teardown leaves the window, which is exactly when - * the destination watcher must stop acting without stopping anything itself. - */ -function isRunnerIdleRetained(deviceId: string, session: RunnerSession | undefined): boolean { - return ( - runnerWarmRetainedDevices.has(deviceId) && - session !== undefined && - runnerSessions.get(deviceId) === session && - canWorkWithRunnerSession(session) - ); -} - -/** - * The one way a runner enters post-`close` retention: the idle-stop timer, the retention fact the - * destination watch reads, and the watch itself are set together, so no retained runner is left - * with one but not the others. A runner that cannot be retained changes nothing. Retention marks - * the device after the scheduler because the scheduler's own cancel step ends any window the - * previous retention left open. - */ -async function retainRunnerForReuse( - deviceId: string, - session: RunnerSession | undefined, -): Promise { - if (!session || !canWorkWithRunnerSession(session)) return; - scheduleIosRunnerIdleStop(deviceId); - runnerWarmRetainedDevices.add(deviceId); - await armRunnerDestinationWatch(session); -} - -type ReleasedRunnerRetention = Readonly<{ session: RunnerSession; epoch: number }>; - -/** - * A start is about to use the device's runner, which ends any retention window on it. Returns what - * was released, for {@link restoreReleasedRunnerRetention} to put back if the start fails. - */ -function releaseRetainedRunnerForUse(deviceId: string): ReleasedRunnerRetention | undefined { - const session = runnerWarmRetainedDevices.has(deviceId) - ? runnerSessions.get(deviceId) - : undefined; - cancelIosRunnerIdleStop(deviceId); - return session && { session, epoch: runnerRetentionEpochs.get(deviceId) ?? 0 }; -} - -/** - * Puts a released window back only while no one has touched the device's retention since the - * release: a later start that used the runner, a close that retained it anew, or a stop each end - * or replace the window, and a failed start must not re-arm a runner someone else now owns. - */ -async function restoreReleasedRunnerRetention( - deviceId: string, - released: ReleasedRunnerRetention, -): Promise { - if ((runnerRetentionEpochs.get(deviceId) ?? 0) !== released.epoch) return; - if (runnerSessions.get(deviceId) !== released.session) return; - await retainRunnerForReuse(deviceId, released.session); -} - -/** - * Arms the #3321 push watcher over a runner this close retained. Reached through a function-scoped - * import: the watcher module is needed only at the start of a retention window and must not join - * the façade closures the eager-closure budget holds at merge-base size. - */ -async function armRunnerDestinationWatch(session: RunnerSession | undefined): Promise { - // Local Simulators only: the reboot comes from Xcode's Simulator destination machinery. A - // retained physical-device runner has no such destination and stays exactly as it was. - if (session?.device.kind !== 'simulator' || !isIosFamily(session.device)) return; - if (!isRunnerIdleRetained(session.deviceId, session)) return; - const { attachRunnerDestinationWatch } = await import('./runner-destination-watch.ts'); - const deviceId = session.deviceId; - attachRunnerDestinationWatch({ - device: session.device, - sessionId: session.sessionId, - port: session.port, - isArmed: () => isRunnerIdleRetained(deviceId, session), - onStop: async () => { - await invalidateRunnerSession(session, 'warm_runner_destination_lost'); - }, - }); -} - -function resolveRunnerIdleStopMs(env: NodeJS.ProcessEnv = process.env): number { - const raw = env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS?.trim(); - if (raw) { - const parsed = Number(raw); - if (Number.isFinite(parsed) && parsed >= 0) return Math.floor(parsed); - } - return RUNNER_RETAINED_IDLE_STOP_DEFAULT_MS; -} - /** The first command that is not a readiness probe makes the session the caller's, not a guess. */ export function markRunnerSessionServed(session: RunnerSession, command: RunnerCommand): void { if (session.speculative && !isRunnerReadinessProbeCommand(command)) { @@ -831,11 +674,17 @@ async function stopIosRunnerSessionDevice( deviceId: string, fenceSettle?: () => void, ): Promise { - cancelIosRunnerIdleStop(deviceId); try { await withRunnerSessionLock(deviceId, async () => { await withRunnerLeaseLock(deviceId, async () => { - await stopRunnerSessionInternal(deviceId, undefined, { leaseLockHeld: true }); + const session = runnerSessions.get(deviceId); + await stopRunnerSessionInternal(deviceId, session, { + leaseLockHeld: true, + graceful: + session?.state === 'ready' && + !isRunnerMainThreadOccupied(session) && + !session.listenerWatch?.lost, + }); await cleanupOwnedIosRunnerLease(deviceId); }); fenceSettle?.(); @@ -845,44 +694,48 @@ async function stopIosRunnerSessionDevice( } } -/** - * Releases a runner at session close, preferring warm reuse only when the runner is actually - * reusable. A non-retained close, or a retained close over a runner whose last exchange reported - * main-thread work still draining, stops it now: a busy runner refuses every command until it drains - * or wedges, so pooling it back hands the same stalled process to the next `open` (#2552). An idle - * retained runner keeps warm reuse via the idle-stop timer. The decision is owned here because the - * occupancy fact lives on the session, and awaited so `close` returns only once the lease is gone. - * Non-retained close first fences the device's start admission, then stops the device's current - * prep processes, before taking the session lock an in-flight cold start holds through its build: - * a start that answers the kill by retrying its build finds the spawn refused, which is what keeps - * close from waiting on the replacement build (#3220). - */ +/** Retires startup recovery at close; only a serving, observed generation can remain idle. */ export async function releaseIosRunnerOnClose( deviceId: string, options: { retain: boolean }, ): Promise { - const session = runnerSessions.get(deviceId); - if (options.retain && !isRunnerMainThreadOccupied(session)) { - await retainRunnerForReuse(deviceId, session); - return; - } - if (options.retain) { - emitDiagnostic({ - level: 'info', - phase: 'ios_runner_retain_skipped_busy', - data: { deviceId }, - }); - } - // First the fence, then the kill: an in-flight start that answers the kill by retrying its - // build finds admission closed and stops, and the fence outlives the kill and the session stop - // until this close settles — while it stands, nothing prepares this device (#3220). + const candidate = runnerSessions.get(deviceId); + const busyAtClose = isRunnerMainThreadOccupied(candidate); const settleFence = fenceRunnerStartAdmissionsForTeardown(deviceId); try { + if (options.retain && candidate?.state === 'ready' && !busyAtClose) { + const retained = await withRunnerSessionLock(deviceId, async () => { + const session = runnerSessions.get(deviceId); + if (!session) return false; + const { retainRunnerSession } = await import('./runner-retention.ts'); + if ( + await retainRunnerSession(session, (retention, reason) => + stopRetainedRunner(session, retention, reason), + ) + ) { + settleFence(); + return true; + } + emitDiagnostic({ + level: 'info', + phase: isRunnerMainThreadOccupied(session) + ? 'ios_runner_retain_skipped_busy' + : 'ios_runner_retain_skipped', + data: { deviceId, sessionId: session.sessionId, state: session.state }, + }); + return false; + }); + if (retained) return; + } else if (options.retain && busyAtClose && candidate) { + emitDiagnostic({ + level: 'info', + phase: 'ios_runner_retain_skipped_busy', + data: { deviceId, sessionId: candidate.sessionId, state: candidate.state }, + }); + } await stopRunnerPrepProcesses(deviceId); await stopIosRunnerSessionDevice(deviceId, settleFence); } finally { - // The error path settles too: a close that throws mid-teardown must not fence the device - // past its own failure. settleFence(); } } @@ -909,12 +762,11 @@ export async function abortAllIosRunnerSessions(): Promise { } // The detach decision itself lives in runner-adoption.ts beside the adoption it hands off to -// (#2681); this is the map side: a detached session leaves the registry and drops its idle -// timer (the detached module gives up the log observation and marks the session stopped). +// (#2681); detached active generations leave the registry. Idle retained generations stay +// owned until shutdown stops them. export async function detachIosRunnerSessionsForShutdown(): Promise { return await detachRunnerSessionsForShutdown(runnerSessions, (deviceId) => { runnerSessions.delete(deviceId); - cancelIosRunnerIdleStop(deviceId); }); } @@ -984,7 +836,7 @@ export async function executeRunnerCommandWithSession( ): Promise> { emitRunnerStartupTimings(session, command.command); const { executeRunnerExchange } = await import('./runner-exchange.ts'); - return executeRunnerExchange( + const data = await executeRunnerExchange( device, session, command, @@ -993,6 +845,14 @@ export async function executeRunnerCommandWithSession( (reason) => invalidateRunnerSession(session, reason), signal, ); + if (runnerSessions.get(device.id) === session && session.state === 'ready') { + const { observeRunnerListener } = await import('./runner-retention.ts'); + observeRunnerListener(session, () => { + const retention = session.retention; + if (retention) return stopRetainedRunner(session, retention, 'listener_lost'); + }); + } + return data; } /** diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index e46b3ae5da..c7ab8e0693 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -530,7 +530,7 @@ Android physical-device prerequisites: Runner and daemon lifecycle (applies to simulators too): open without --relaunch is idempotent-foreground for an already-running app (it brings the process forward; it does not restart it). open --relaunch restarts the app; on iOS simulators this collapses to one simctl launch --terminate-running-process call instead of a separate terminate-then-launch. No runner read launches a session app that is not running: snapshot, wait, is, get, a reading find, and an interaction's leading reads (a gesture's viewport read, the capture that resolves a selector click/fill) answer the retriable APP_NOT_RUNNING instead of bare-launching over a launch SpringBoard still holds behind its deep-link confirmation. Only open, activate, and a command that mutates without a leading read bring a stopped app up. - close keeps a healthy iOS simulator XCTest runner warm by default so the next open on that simulator (same udid in the same simulator set) skips the runner build, unless --shutdown was requested, the session was recording, or the session held a device lease. A retained runner auto-stops after an idle window (default 5 minutes); set AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS to override, or 0 to disable idle stop and retain until daemon exit. A simulator shut down externally while its runner is retained is detected and the retained runner is stopped, because the retained Xcode destination would otherwise reboot the simulator and leave it powered on; the next open reports reason=runner_connection_lost in its warnings. + close keeps a healthy iOS simulator XCTest runner warm by default so the next open on that simulator (same udid in the same simulator set) skips the runner build, unless --shutdown was requested, the session was recording, or the session held a device lease. A retained runner auto-stops after an idle window (default 5 minutes); set AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS to override, or 0 to disable idle stop and retain until daemon exit. Only a ready runner with an observed listener is retained; close retires unfinished prewarm recovery. Losing the retained listener stops that generation, and daemon shutdown stops idle retained runners rather than handing them off without supervision. Each AGENT_DEVICE_STATE_DIR runs its own daemon. It self-exits after an idle window (default 5 minutes, matching the runner idle-stop default) once it has no open sessions, no in-flight requests, and no active recording; set AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS to override, or 0 to disable idle reap. On a machine shared by several agents, set AGENT_DEVICE_SESSION_IDLE_TIMEOUT_MS to also expire an individual session that has taken a host-global device claim and then received no commands for that long; the claim is released and the next command on that session answers SESSION_NOT_FOUND with details.reason SESSION_IDLE_EXPIRED naming the window and the device. Off by default, because on a shared host "idle" and "thinking" are indistinguishable: only set it where an abandoned claim blocking every other agent is the worse failure. Sessions holding a remote lease or any active capture (recording, logs, audio, performance, or trace) are never expired this way. A stale iOS runner lease — its owner process dead, or its AGENT_DEVICE_STATE_DIR deleted — is reclaimed automatically instead of failing with "is already owned by another agent-device daemon". A live owner's runner is also reclaimed when the requesting daemon holds the host-global device claim for that device: claims are exclusive, so holding one proves the runner's owner released the device and merely kept the runner warm. The error remains only for owners outside claim arbitration (a pre-claims build, or daemons pointed at different claim stores). diff --git a/src/daemon/__tests__/request-router-open-claim.test.ts b/src/daemon/__tests__/request-router-open-claim.test.ts index 0334ad455f..eabd859560 100644 --- a/src/daemon/__tests__/request-router-open-claim.test.ts +++ b/src/daemon/__tests__/request-router-open-claim.test.ts @@ -11,9 +11,6 @@ vi.mock('@agent-device/host-kit/process', async (importOriginal) => { const actual = await importOriginal(); return { ...actual, readProcessStartTime: vi.fn(() => 'test-process-start') }; }); -const mockTakeRunnerWarmLossNotice = vi.hoisted(() => - vi.fn(async (): Promise => undefined), -); vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { const actual = await importOriginal(); @@ -23,7 +20,6 @@ vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) notifyIosRunnerAppRelaunched: vi.fn(async () => {}), prewarmAppleRunnerCache: vi.fn(async () => {}), prewarmIosRunner: vi.fn(async () => {}), - prewarmIosRunnerSession: vi.fn(async () => {}), prepareIosRunner: vi.fn(async () => ({ runner: { currentUptimeMs: 42 }, connectMs: 0, @@ -32,7 +28,6 @@ vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) resolveRunnerAppBundleId: vi.fn(() => 'com.callstack.agentdevice.runner'), stopIosRunnerSession: vi.fn(async () => {}), stopAllIosRunnerSessions: vi.fn(async () => {}), - takeRunnerWarmLossNotice: mockTakeRunnerWarmLossNotice, }; }); vi.mock('@agent-device/platform-apple/app-lifecycle', async (importOriginal) => { @@ -72,7 +67,6 @@ import { import { ensureDeviceReady } from '../device/device-ready.ts'; import type { DeviceInfo } from '@agent-device/kernel/device'; import type { DeviceBootObservation } from '@agent-device/contracts/device-boot'; -import type { RunnerWarmLossNotice } from '@agent-device/platform-apple/runner/operations'; import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; import { inspectDeviceClaims } from '../device/device-claim-inspection.ts'; import { makeIosDevice, openRequest, storedClaimUpdatedAt } from './request-router-open-harness.ts'; @@ -104,8 +98,6 @@ beforeEach(() => { mockEnsureDeviceReady.mockResolvedValue(undefined); mockAwaitFixtureReadiness.mockReset(); mockAwaitFixtureReadiness.mockResolvedValue(undefined); - mockTakeRunnerWarmLossNotice.mockReset(); - mockTakeRunnerWarmLossNotice.mockResolvedValue(undefined); mockObserveSimulatorBoot.mockReset(); mockObserveSimulatorBoot.mockImplementation(async () => ({ observed: false, @@ -175,39 +167,6 @@ test('open takes a live foreign claim whose device rebooted after the claim was } }); -test('open warns with the typed reason when a warm runner was stopped for a lost connection', async () => { - const sessionStore = makeSessionStore('agent-device-router-open-warm-loss-'); - const device = makeIosDevice('SIM-WARM-LOSS'); - mockResolveTargetDevice.mockResolvedValue(device); - mockTakeRunnerWarmLossNotice.mockResolvedValueOnce({ atMs: Date.now() }); - const claimsDir = mkdtempForTestSync('agent-device-router-open-warm-loss-claims-'); - const previousClaimsDir = process.env.AGENT_DEVICE_CLAIMS_DIR; - process.env.AGENT_DEVICE_CLAIMS_DIR = claimsDir; - - try { - const opened = await createOpenHandler(sessionStore)( - openRequest('warm-loss', { platform: 'ios' }, 'req-open-warm-loss', {}, ['FixtureApp']), - ); - - expect(opened.ok).toBe(true); - if (!opened.ok) return; - expect(mockTakeRunnerWarmLossNotice).toHaveBeenCalledWith(device.id); - expect(opened.data?.warnings).toEqual([ - expect.stringContaining('reason=runner_connection_lost'), - ]); - - const reopened = await createOpenHandler(sessionStore)( - openRequest('warm-loss', { platform: 'ios' }, 'req-open-warm-loss-again', {}, ['FixtureApp']), - ); - expect(reopened.ok).toBe(true); - if (reopened.ok) expect(reopened.data?.warnings).toBeUndefined(); - } finally { - if (previousClaimsDir === undefined) delete process.env.AGENT_DEVICE_CLAIMS_DIR; - else process.env.AGENT_DEVICE_CLAIMS_DIR = previousClaimsDir; - fs.rmSync(claimsDir, { recursive: true, force: true }); - } -}); - // The production reopen path never re-acquires the claim, so renewal has to ride the successful // existing-session open itself: without it, an owner that came back after a reboot still carries a // pre-reboot stamp and loses the device to the next caller that asks. diff --git a/src/daemon/server/daemon-idle-reap.ts b/src/daemon/server/daemon-idle-reap.ts index 2b7b9afbd5..61e085b29a 100644 --- a/src/daemon/server/daemon-idle-reap.ts +++ b/src/daemon/server/daemon-idle-reap.ts @@ -11,13 +11,9 @@ import { isUncommittedRepairSession } from '../session-replay-transaction.ts'; // way also keeps holding its iOS runner lease, which blocks a fresh daemon for // the same device (see runner-lease.ts stale-lease takeover). // -// The window mirrors the iOS runner idle-stop default -// (AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS, 5 minutes, see runner-session.ts): -// graceful daemon shutdown already hands off a healthy retained runner - -// simulator or physical iOS - for the next daemon to adopt -// (detachIosRunnerSessionsForShutdown), so reaping the daemon process itself on -// the same timescale does not force a runner rebuild in the common case - it -// only pays the cheap daemon bootstrap (socket/HTTP listen) on the next command. +// The window mirrors the runner idle-stop default. Idle retained runners stop with their daemon: +// their timer and listener observation cannot survive a handoff. Active eligible runners retain +// the graceful-shutdown adoption policy. const DAEMON_IDLE_REAP_DEFAULT_MS = 5 * 60_000; // AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS overrides the window; 0 disables idle diff --git a/src/daemon/session-lifecycle/internal/session-open-execution.ts b/src/daemon/session-lifecycle/internal/session-open-execution.ts index 748c4392b6..5764438cf0 100644 --- a/src/daemon/session-lifecycle/internal/session-open-execution.ts +++ b/src/daemon/session-lifecycle/internal/session-open-execution.ts @@ -56,7 +56,6 @@ import { } from '../../device/device-claims.ts'; import type { TakenOverDeviceClaim } from '../../device/device-claim-reboot.ts'; import { deviceBootObservation } from '../../../platform-runtime-device-boot.ts'; -import type { RunnerWarmLossNotice } from '../../../platform-runtime-warm-runner-notice.ts'; import { appendResponseWarning } from './session-open-warnings.ts'; import { buildAllocatorHeldRefusal, @@ -122,33 +121,6 @@ function deviceClaimTakeoverWarning(tookOver: TakenOverDeviceClaim): string { ); } -/** - * Reads and consumes the warm-runner loss notice #3321's destination watcher recorded for this - * device. Reached through the root's platform-runtime seam by function-scoped import: the notice - * exists only on iOS simulators, and only when a retained runner was stopped, so the common open - * must not pay for the platform module's evaluation. - */ -async function readWarmRunnerLossNotice( - device: DeviceInfo, -): Promise { - const { takeWarmRunnerLossNotice } = - await import('../../../platform-runtime-warm-runner-notice.ts'); - return await takeWarmRunnerLossNotice(device); -} - -/** - * What the caller's `open` output says when the runner this session left warm after `close` was - * stopped because its connection closed while it was retained (#3321), typically a Simulator shut - * down externally. The reason is named so automation can key on it without parsing prose. - */ -function warmRunnerLossWarning(notice: RunnerWarmLossNotice): string { - return ( - `The warm iOS runner left by a previous close was stopped at ${new Date(notice.atMs).toISOString()} ` + - 'because its connection closed while it was retained, typically because the simulator was shut ' + - 'down externally; stopping it keeps Xcode from rebooting the simulator. reason=runner_connection_lost' - ); -} - // Default-on for emulators, opt-in via --test-ime on real devices; --no-test-ime forces off. function shouldActivateAndroidTestIme(device: DeviceInfo, req: DaemonRequest): boolean { if (device.platform !== 'android') return false; @@ -338,10 +310,6 @@ export async function completeOpenCommand(params: { if (tookOverDeviceClaim) { appendResponseWarning(openResult, deviceClaimTakeoverWarning(tookOverDeviceClaim)); } - const warmLossNotice = await readWarmRunnerLossNotice(device); - if (warmLossNotice) { - appendResponseWarning(openResult, warmRunnerLossWarning(warmLossNotice)); - } const nextRef = publishOpenSession({ req, sessionStore, diff --git a/src/platform-runtime-warm-runner-notice.test.ts b/src/platform-runtime-warm-runner-notice.test.ts deleted file mode 100644 index 2ff39bec75..0000000000 --- a/src/platform-runtime-warm-runner-notice.test.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { beforeEach, expect, test, vi } from 'vitest'; -import type { RunnerWarmLossNotice } from '@agent-device/platform-apple/runner/operations'; - -const NOTICE: RunnerWarmLossNotice = { atMs: 1 }; - -const mocks = vi.hoisted(() => ({ - takeRunnerWarmLossNotice: vi.fn(async (): Promise => undefined), -})); - -vi.mock('@agent-device/platform-apple/runner/operations', () => mocks); - -import { IOS_DEVICE, IOS_SIMULATOR, MACOS_DEVICE } from './__tests__/test-utils/device-fixtures.ts'; -import { takeWarmRunnerLossNotice } from './platform-runtime-warm-runner-notice.ts'; - -beforeEach(() => { - vi.clearAllMocks(); -}); - -test('an iOS Simulator reads the notice recorded for its id', async () => { - mocks.takeRunnerWarmLossNotice.mockResolvedValueOnce(NOTICE); - - expect(await takeWarmRunnerLossNotice(IOS_SIMULATOR)).toEqual(NOTICE); - expect(mocks.takeRunnerWarmLossNotice).toHaveBeenCalledWith(IOS_SIMULATOR.id); -}); - -test('a physical iOS device retains no warm runner and reads nothing', async () => { - expect(await takeWarmRunnerLossNotice(IOS_DEVICE)).toBeUndefined(); - expect(mocks.takeRunnerWarmLossNotice).not.toHaveBeenCalled(); -}); - -test('a macOS device retains no warm runner and reads nothing', async () => { - expect(await takeWarmRunnerLossNotice(MACOS_DEVICE)).toBeUndefined(); - expect(mocks.takeRunnerWarmLossNotice).not.toHaveBeenCalled(); -}); diff --git a/src/platform-runtime-warm-runner-notice.ts b/src/platform-runtime-warm-runner-notice.ts deleted file mode 100644 index a5548b0140..0000000000 --- a/src/platform-runtime-warm-runner-notice.ts +++ /dev/null @@ -1,24 +0,0 @@ -import type { RunnerWarmLossNotice } from '@agent-device/platform-apple/runner/operations'; -import { isIosFamily, type DeviceInfo } from '@agent-device/kernel/device'; - -export type { RunnerWarmLossNotice }; - -/** - * Takes the warm-runner loss notice recorded for this device by the Apple runner's destination - * watcher, once, if there is one (#3321). A notice exists only when a runner retained after - * `close` was stopped because its connection closed, typically because something shut the Simulator - * down under it — the state the next `open` reports rather than leaves the caller to infer from a - * cold start. Families that retain no warm runner record no notice. - * - * The notice is read through the platform's public operations surface from this root module: the - * daemon must not import a concrete platform package (R65), and the notice is a runner fact, not a - * session fact — it exists precisely in the gap where no session is live. - */ -export async function takeWarmRunnerLossNotice( - device: DeviceInfo, -): Promise { - if (!isIosFamily(device) || device.kind !== 'simulator') return undefined; - const { takeRunnerWarmLossNotice } = - await import('@agent-device/platform-apple/runner/operations'); - return takeRunnerWarmLossNotice(device.id); -} diff --git a/website/docs/docs/sessions.md b/website/docs/docs/sessions.md index db1627d4be..87d09070a2 100644 --- a/website/docs/docs/sessions.md +++ b/website/docs/docs/sessions.md @@ -114,10 +114,10 @@ in CI and on shared hosts so devices don't keep running after the run: agent-device close --shutdown ``` -Without `--shutdown`, `close` keeps the iOS runner warm for the next `open`. If you shut that simulator -down yourself (`xcrun simctl shutdown`) while the runner is waiting, agent-device stops the runner so it -doesn't boot the simulator again, and the next `open` returns a warning that ends in -`reason=runner_connection_lost`. +Without `--shutdown`, `close` keeps a ready iOS simulator runner warm for the next `open`. +Unfinished runner startup is stopped. If you shut that simulator down yourself while the runner is +retained, losing its listener connection stops the retained runner. Idle retained runners also stop +when their daemon exits. ## Wait for a slow or busy device From f234d548df5956f27605541f352cff33d06cf016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 10 Oct 2026 08:50:22 +0200 Subject: [PATCH 18/20] chore(gates): remove retired warm-loss notice ownership --- scripts/layering/daemon-platform-runtime-inventory.ts | 10 ---------- scripts/layering/root-module-zones.ts | 1 - 2 files changed, 11 deletions(-) diff --git a/scripts/layering/daemon-platform-runtime-inventory.ts b/scripts/layering/daemon-platform-runtime-inventory.ts index 5bbe8b0d7f..67cc9e1be5 100644 --- a/scripts/layering/daemon-platform-runtime-inventory.ts +++ b/scripts/layering/daemon-platform-runtime-inventory.ts @@ -191,16 +191,6 @@ export const DAEMON_PLATFORM_RUNTIME_EDGES: readonly DaemonPlatformRuntimeEdge[] 'foreign claim can still describe live ownership; the per-family probe mechanics stay in the ' + 'Apple and Android packages behind the neutral observation contract (#2538).', }, - { - file: 'src/daemon/session-lifecycle/internal/session-open-execution.ts', - target: 'src/platform-runtime-warm-runner-notice.ts', - symbols: ['RunnerWarmLossNotice', 'takeWarmRunnerLossNotice'], - classification: 'daemon-policy-essential', - rationale: - 'daemon-owned open reports that a runner left warm by close was stopped because its ' + - 'connection closed, typically a simulator shut down externally (#3321); the watcher stays ' + - 'Apple-owned behind the neutral notice read.', - }, { file: 'src/daemon/handlers/session-selector-dispatch.ts', target: 'src/platform-runtime-open-target.ts', diff --git a/scripts/layering/root-module-zones.ts b/scripts/layering/root-module-zones.ts index e9bf6cfe2c..fdf1cf4d3b 100644 --- a/scripts/layering/root-module-zones.ts +++ b/scripts/layering/root-module-zones.ts @@ -62,7 +62,6 @@ export const ROOT_MODULE_ZONES: Readonly> = { 'src/platform-runtime-managed-web-backend.ts', 'src/platform-runtime-open-target.ts', 'src/platform-runtime-resource-cleanup.ts', - 'src/platform-runtime-warm-runner-notice.ts', 'src/platform-runtime.ts', 'src/provider-credential-fingerprint.ts', 'src/provider-device-runtimes.ts', From 06362cacdd988b347780250f7ab4b0516952f0ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 10 Oct 2026 09:27:34 +0200 Subject: [PATCH 19/20] fix(apple): address retained runner review findings --- .../0005-ios-runner-interaction-lifecycle.md | 8 +- packages/kernel/src/keyed-lock.test.ts | 48 ++++++++ packages/kernel/src/keyed-lock.ts | 21 +++- .../runner/__tests__/runner-retention.test.ts | 15 +++ .../runner-session-close-prep-fence.test.ts | 105 ++++++++---------- .../runner-session-lifecycle.test.ts | 31 ------ .../src/runner/runner-adoption.ts | 2 +- .../src/runner/runner-lifecycle.ts | 3 - .../src/runner/runner-retention.ts | 16 ++- .../src/runner/runner-session-types.ts | 9 +- website/docs/docs/sessions.md | 5 +- 11 files changed, 153 insertions(+), 110 deletions(-) diff --git a/docs/adr/0005-ios-runner-interaction-lifecycle.md b/docs/adr/0005-ios-runner-interaction-lifecycle.md index 7e9533effc..396bdae7cd 100644 --- a/docs/adr/0005-ios-runner-interaction-lifecycle.md +++ b/docs/adr/0005-ios-runner-interaction-lifecycle.md @@ -117,11 +117,11 @@ fresh xcodebuild runner startup, not a stale transport stall. A `close` then `open` on a runner still draining abandoned main-thread work now pays a fresh runner boot instead of inheriting the stalled one, so the wedge no longer survives the close/open cycle. Runners -that are ready, have no unanswered exchanges, and have an observed live Simulator listener can -remain warm after close. Close retires unfinished startup and its recovery authority. The retained +that are ready and have no unanswered exchanges can remain warm when retention is requested; +iOS-family simulators additionally require an observed live listener. Close retires unfinished startup and its recovery authority. The retained generation owns its idle timer and listener observation; successful reuse ends retention, listener -loss or idle expiry disposes that generation, and daemon shutdown stops idle retained generations -before their observation ends. Active ready generations retain the existing handoff policy. +loss or idle expiry disposes that generation, and graceful daemon shutdown stops idle retained +generations before exiting. Active ready generations retain the existing handoff policy. The daemon no longer models a generic "recent success" cache as a runner-health signal. A proven healthy mutating response for the same app — recorded only after the `runnerFatal` check and only diff --git a/packages/kernel/src/keyed-lock.test.ts b/packages/kernel/src/keyed-lock.test.ts index 3f87fbd652..610aa9e54a 100644 --- a/packages/kernel/src/keyed-lock.test.ts +++ b/packages/kernel/src/keyed-lock.test.ts @@ -105,3 +105,51 @@ test('a deferred descendant of a released lock queues behind its current owner', await Promise.all([owner, descendant]); assert.deepEqual(order, ['owner', 'released', 'descendant']); }); + +test.each([false, true])( + 'reentrant descendants settle before the next owner (reject=%s)', + async (reject) => { + const locks = new Map>(); + const order: string[] = []; + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let markStarted!: () => void; + const started = new Promise((resolve) => { + markStarted = resolve; + }); + let descendant!: Promise; + const owner = withKeyedLock(locks, 'device-a', async () => { + queueMicrotask(() => { + descendant = withKeyedLock(locks, 'device-a', async () => { + order.push('descendant'); + markStarted(); + await gate; + order.push('finished'); + void withKeyedLock(locks, 'device-a', async () => { + await new Promise((resolve) => setImmediate(resolve)); + order.push('nested'); + }); + if (reject) throw new Error('descendant failed'); + }); + void descendant.catch(() => {}); + }); + }); + const competing = withKeyedLock(locks, 'device-a', async () => { + order.push('competing'); + }); + await started; + await new Promise((resolve) => setImmediate(resolve)); + const beforeRelease = [...order]; + release(); + const [, outcome] = await Promise.all([ + owner, + descendant.catch((error: Error) => error), + competing, + ]); + assert.equal(outcome instanceof Error, reject); + assert.deepEqual(beforeRelease, ['descendant']); + assert.deepEqual(order, ['descendant', 'finished', 'nested', 'competing']); + }, +); diff --git a/packages/kernel/src/keyed-lock.ts b/packages/kernel/src/keyed-lock.ts index edf4a6c828..c8a5e55b06 100644 --- a/packages/kernel/src/keyed-lock.ts +++ b/packages/kernel/src/keyed-lock.ts @@ -4,6 +4,7 @@ type LockFrame = { locks: Map>; key: string; active: boolean; + descendants: Set>; }; const keyedLockStorage = new AsyncLocalStorage(); @@ -14,20 +15,34 @@ export async function withKeyedLock( task: () => Promise, ): Promise { const activeLocks = keyedLockStorage.getStore() ?? []; - if (activeLocks.some((entry) => entry.active && entry.locks === locks && entry.key === key)) { - return await task(); + const held = activeLocks.find( + (entry) => entry.active && entry.locks === locks && entry.key === key, + ); + if (held) { + let markSettled!: () => void; + const settled = new Promise((resolve) => { + markSettled = resolve; + }); + held.descendants.add(settled); + try { + return await task(); + } finally { + held.descendants.delete(settled); + markSettled(); + } } const previous = locks.get(key) ?? Promise.resolve(); const current = previous .catch(() => {}) .then(async () => { - const frame: LockFrame = { locks, key, active: true }; + const frame: LockFrame = { locks, key, active: true, descendants: new Set() }; try { return await keyedLockStorage.run( [...activeLocks.filter((entry) => entry.active), frame], task, ); } finally { + while (frame.descendants.size > 0) await Promise.all(frame.descendants); frame.active = false; } }); diff --git a/packages/platform-apple/src/runner/__tests__/runner-retention.test.ts b/packages/platform-apple/src/runner/__tests__/runner-retention.test.ts index fbbf754613..c23ee4302f 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-retention.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-retention.test.ts @@ -46,7 +46,9 @@ test('one generation has one listener watch, and loss before close prevents rete const session = await servingSession(); const onLost = vi.fn(); await observe(session, onLost); + const watch = session.listenerWatch; observeRunnerListener(session, onLost); + assert.equal(session.listenerWatch, watch); assert.equal(connections.length, 1); connections[0]!.destroy(); await vi.waitFor(() => assert.equal(onLost.mock.calls.length, 1)); @@ -120,3 +122,16 @@ test('zero disables only the idle timer, and unready or occupied generations can session.commandCharges.charge('still-in-flight'); assert.equal(await retainRunnerSession(session, stop), false); }); + +test('an overflowing idle duration does not become an immediate stop', async () => { + const session = await servingSession(); + await observe(session); + vi.useFakeTimers(); + vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS', '2147483648'); + const stop = vi.fn(async () => {}); + await retainRunnerSession(session, stop); + await vi.advanceTimersByTimeAsync(1); + assert.equal(stop.mock.calls.length, 0); + await vi.advanceTimersByTimeAsync(2147483646); + assert.deepEqual(stop.mock.calls, [[session.retention, 'idle_timeout']]); +}); diff --git a/packages/platform-apple/src/runner/__tests__/runner-session-close-prep-fence.test.ts b/packages/platform-apple/src/runner/__tests__/runner-session-close-prep-fence.test.ts index 3cd2699ff6..12e20b154e 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-session-close-prep-fence.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-session-close-prep-fence.test.ts @@ -181,67 +181,60 @@ beforeEach(async () => { ); }); -test.each([false, true])( - 'close (retain=%s) during a cold build leaves no replacement build', - async (retain) => { - const device = IOS_ADMISSION_SIMULATOR; - // Park close inside its prep stop: the kill is attempted, but the stop has not returned, so - // close provably has not reached the session lock yet (the start holds it) and no later fence - // could have run. That is the moment the device must already be fenced. - let releasePrepStop: () => void = () => {}; - const prepStopParked = new Promise((resolve) => { - releasePrepStop = resolve; - }); - let signalAttempted = () => {}; - const killAttempted = new Promise((resolve) => { - signalAttempted = resolve; - }); - mockRunAppleToolCommand.mockImplementation(async (tool: string, args: string[]) => { - // `killRunnerProcessTree` is the only pkill that signals by parent pid (`-P`); the lease - // cleanup's stale-launch sweep matches argv (`-f`), so only the prep tree-kill parks. - if (tool === 'pkill' && args.includes('-P')) { - signalAttempted(); - await prepStopParked; - } - return { exitCode: 0, stdout: '', stderr: '' }; - }); +test.each([false, true])('cold close (retain=%s) fences replacement builds', async (retain) => { + const device = IOS_ADMISSION_SIMULATOR; + // Park close inside its prep stop: the kill is attempted, but the stop has not returned, so + // close provably has not reached the session lock yet (the start holds it) and no later fence + // could have run. That is the moment the device must already be fenced. + let releasePrepStop: () => void = () => {}; + const prepStopParked = new Promise((resolve) => { + releasePrepStop = resolve; + }); + let signalAttempted = () => {}; + const killAttempted = new Promise((resolve) => { + signalAttempted = resolve; + }); + mockRunAppleToolCommand.mockImplementation(async (tool: string, args: string[]) => { + // `killRunnerProcessTree` is the only pkill that signals by parent pid (`-P`); the lease + // cleanup's stale-launch sweep matches argv (`-f`), so only the prep tree-kill parks. + if (tool === 'pkill' && args.includes('-P')) { + signalAttempted(); + await prepStopParked; + } + return { exitCode: 0, stdout: '', stderr: '' }; + }); - // A cold start: holds the session lock for the whole build, as a real one does. - const firstStart = ensureRunnerSession(device, {}).catch((error: unknown) => error); - await vi.waitFor(() => assert.equal(builds.length, 1)); - assert.equal(runnerPrepProcessChildren(device.id).length, 1, 'the build is on the prep ledger'); + // A cold start: holds the session lock for the whole build, as a real one does. + const firstStart = ensureRunnerSession(device, {}).catch((error: unknown) => error); + await vi.waitFor(() => assert.equal(builds.length, 1)); + assert.equal(runnerPrepProcessChildren(device.id).length, 1, 'the build is on the prep ledger'); - const closing = releaseIosRunnerOnClose(device.id, { retain }); - await killAttempted; - assert.equal( - runnerStartTeardownPending(device.id), - true, - 'the device was fenced before the prep stop finished', - ); + const closing = releaseIosRunnerOnClose(device.id, { retain }); + await killAttempted; + assert.equal( + runnerStartTeardownPending(device.id), + true, + 'the device was fenced before the prep stop finished', + ); - // The prewarm health retry of the incident: the same caller starts over while close waits. - const retriedStart = ensureRunnerSession(device, {}).catch((error: unknown) => error); - releasePrepStop(); - builds[0]!.kill(); + // The prewarm health retry of the incident: the same caller starts over while close waits. + const retriedStart = ensureRunnerSession(device, {}).catch((error: unknown) => error); + releasePrepStop(); + builds[0]!.kill(); - const [first, retried] = await Promise.all([firstStart, retriedStart, closing]); + const [first, retried] = await Promise.all([firstStart, retriedStart, closing]); - assert.ok(first instanceof Error, 'the start whose build was killed failed'); - assert.ok(isRequestCanceledError(retried), 'the retry is refused as a canceled start'); - assert.equal( - (retried as { details?: { runnerStartRetirementReason?: unknown } }).details - ?.runnerStartRetirementReason, - 'device_teardown', - 'the refusal carries the typed retirement reason, not just a cancellation', - ); - assert.equal(mockBuildForTesting.mock.calls.length, 1, 'no replacement build was ever spawned'); - assert.equal( - runnerPrepProcessChildren(device.id).length, - 0, - 'the killed build left the ledger', - ); - }, -); + assert.ok(first instanceof Error, 'the start whose build was killed failed'); + assert.ok(isRequestCanceledError(retried), 'the retry is refused as a canceled start'); + assert.equal( + (retried as { details?: { runnerStartRetirementReason?: unknown } }).details + ?.runnerStartRetirementReason, + 'device_teardown', + 'the refusal carries the typed retirement reason, not just a cancellation', + ); + assert.equal(mockBuildForTesting.mock.calls.length, 1, 'no replacement build was ever spawned'); + assert.equal(runnerPrepProcessChildren(device.id).length, 0, 'the killed build left the ledger'); +}); /** * The reviewer's counter-case to the fence: a caller that merely QUEUED behind the close — it diff --git a/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts b/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts index 6237bfe7fb..b9d41fa96a 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts @@ -2,7 +2,6 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import { beforeEach, test, vi } from 'vitest'; -import { retainRunnerSession } from '../runner-retention.ts'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { IOS_DEVICE, IOS_SIMULATOR, MACOS_DEVICE } from './device-fixtures.ts'; import { appleRunnerTestHost } from '../test-host.ts'; @@ -231,36 +230,6 @@ test('a startup publishes starting and its first answer publishes ready', async assert.equal(hasLiveIosRunnerSession(device), true); }); -test('an idle stop moves a ready session through disposal to stopped', async () => { - const device = { ...IOS_SIMULATOR, id: 'runner-lifecycle-idle-stop' }; - process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS = '1'; - const session = await ensureRunnerSession(device, {}); - mockWaitForRunner.mockResolvedValueOnce(runnerResponse({ nodes: [], truncated: false })); - await executeRunnerCommandWithSession( - device, - session, - { command: 'snapshot', appBundleId: 'com.example.demo' }, - '/tmp/runner.log', - 30_000, - ); - - session.listenerWatch?.close(); - session.listenerWatch = { - lost: false, - ready: Promise.resolve(true), - close() {}, - }; - await retainRunnerSession(session, async () => stopIosRunnerSession(device.id)); - await vi.waitFor(() => assert.equal(session.state, 'stopped')); - - assert.equal( - readRunnerSessionLiveness(device.id), - null, - 'the idle stop removes the session from the device registry', - ); - assert.equal(hasLiveIosRunnerSession(device), false); -}); - test('stopping the same registered runner twice tears it down once', async () => { const device = { ...IOS_SIMULATOR, id: 'runner-lifecycle-stop-twice' }; const session = await ensureRunnerSession(device, {}); diff --git a/packages/platform-apple/src/runner/runner-adoption.ts b/packages/platform-apple/src/runner/runner-adoption.ts index 5511eff1fc..5d0583f431 100644 --- a/packages/platform-apple/src/runner/runner-adoption.ts +++ b/packages/platform-apple/src/runner/runner-adoption.ts @@ -155,11 +155,11 @@ function detachRunnerSessionForShutdown(session: RunnerSession): RunnerDetachOut } catch { return { detached: false, lane, reason: 'lease_write_failed' }; } + session.listenerWatch?.close(); // Only once the lease says the runner is handed over does this process give up its own sides of // the runner's log: until that write lands the session is still owned, and an owned session that // stopped following its runner's output is worse off than one that never handed anything off. // The runner holds its own descriptor, so this cannot disturb it either way (#2681). - session.listenerWatch?.close(); session.endOutputObservation?.(); advanceRunnerSessionState(session, 'stopped'); return { detached: true, lane }; diff --git a/packages/platform-apple/src/runner/runner-lifecycle.ts b/packages/platform-apple/src/runner/runner-lifecycle.ts index 92305a836c..1ff50b905e 100644 --- a/packages/platform-apple/src/runner/runner-lifecycle.ts +++ b/packages/platform-apple/src/runner/runner-lifecycle.ts @@ -160,9 +160,6 @@ async function handlePrepareHealthFailure(params: { assertRunnerStartAdmitsPreparation(device.id, options.startAdmission); const appErr = asAppError(error, 'COMMAND_FAILED'); if (isRequestCanceledError(appErr)) { - // The owning request was canceled mid-startup (client disconnect): stop the - // just-created session so a canceled prep never leaves a runner retained for - // reuse. Scoped to this request's device only. await invalidateRunnerSessionBestEffort(session, 'prepare_runner_request_canceled'); throw error; } diff --git a/packages/platform-apple/src/runner/runner-retention.ts b/packages/platform-apple/src/runner/runner-retention.ts index 8257aba5cc..a6affef723 100644 --- a/packages/platform-apple/src/runner/runner-retention.ts +++ b/packages/platform-apple/src/runner/runner-retention.ts @@ -1,18 +1,16 @@ import net from 'node:net'; import { isIosFamily } from '@agent-device/kernel/device'; import { emitDiagnostic } from './host.ts'; -import { isRunnerMainThreadOccupied, type RunnerSession } from './runner-session-types.ts'; +import { + isRunnerMainThreadOccupied, + type RunnerSession, + type RunnerListenerWatch, + type RunnerRetention, +} from './runner-session-types.ts'; const LISTENER_CONNECT_TIMEOUT_MS = 500; const RETAINED_IDLE_STOP_DEFAULT_MS = 5 * 60_000; -export type RunnerListenerWatch = { - lost: boolean; - ready: Promise; - close(): void; -}; - -export type RunnerRetention = { cancel(): void }; type StopRetainedRunner = ( retention: RunnerRetention, reason: 'idle_timeout' | 'listener_lost', @@ -123,7 +121,7 @@ function resolveRunnerIdleStopMs(env: NodeJS.ProcessEnv = process.env): number { const raw = env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS?.trim(); if (raw) { const parsed = Number(raw); - if (Number.isFinite(parsed) && parsed >= 0) return Math.floor(parsed); + if (Number.isFinite(parsed) && parsed >= 0) return Math.min(Math.floor(parsed), 2 ** 31 - 1); } return RETAINED_IDLE_STOP_DEFAULT_MS; } diff --git a/packages/platform-apple/src/runner/runner-session-types.ts b/packages/platform-apple/src/runner/runner-session-types.ts index edad7cda39..a1b7b7df84 100644 --- a/packages/platform-apple/src/runner/runner-session-types.ts +++ b/packages/platform-apple/src/runner/runner-session-types.ts @@ -1,4 +1,3 @@ -import type { RunnerListenerWatch, RunnerRetention } from './runner-retention.ts'; import type { RunnerLogicalLeaseContext } from '@agent-device/contracts/runner-lease-context'; import type { ExecResult } from '@agent-device/host-kit/command'; import type { DeviceInfo } from '@agent-device/kernel/device'; @@ -51,6 +50,14 @@ export type RunnerProcessHandle = { exitCode: number | null; }; +export type RunnerListenerWatch = { + lost: boolean; + ready: Promise; + close(): void; +}; + +export type RunnerRetention = { cancel(): void }; + export type RunnerSession = { sessionId: string; device: DeviceInfo; diff --git a/website/docs/docs/sessions.md b/website/docs/docs/sessions.md index 87d09070a2..7821fe9237 100644 --- a/website/docs/docs/sessions.md +++ b/website/docs/docs/sessions.md @@ -114,10 +114,11 @@ in CI and on shared hosts so devices don't keep running after the run: agent-device close --shutdown ``` -Without `--shutdown`, `close` keeps a ready iOS simulator runner warm for the next `open`. +Without `--shutdown`, `close` can keep a ready, idle iOS simulator runner warm for the next `open` +when its listener connection is still live and the session has no screen recording or device lease. Unfinished runner startup is stopped. If you shut that simulator down yourself while the runner is retained, losing its listener connection stops the retained runner. Idle retained runners also stop -when their daemon exits. +when their daemon shuts down gracefully. ## Wait for a slow or busy device From 8afe9598ef3995dbffcde946479e0f929aa984fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 10 Oct 2026 09:35:16 +0200 Subject: [PATCH 20/20] docs(apple): wrap retained runner lifecycle explanation --- docs/adr/0005-ios-runner-interaction-lifecycle.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/adr/0005-ios-runner-interaction-lifecycle.md b/docs/adr/0005-ios-runner-interaction-lifecycle.md index 396bdae7cd..01d91a6836 100644 --- a/docs/adr/0005-ios-runner-interaction-lifecycle.md +++ b/docs/adr/0005-ios-runner-interaction-lifecycle.md @@ -115,13 +115,14 @@ If xcodebuild still exits for another reason, the next command detects the stale process/liveness checks and avoids the old 15-second graceful-shutdown wait. The remaining latency is fresh xcodebuild runner startup, not a stale transport stall. -A `close` then `open` on a runner still draining abandoned main-thread work now pays a fresh runner boot -instead of inheriting the stalled one, so the wedge no longer survives the close/open cycle. Runners -that are ready and have no unanswered exchanges can remain warm when retention is requested; -iOS-family simulators additionally require an observed live listener. Close retires unfinished startup and its recovery authority. The retained -generation owns its idle timer and listener observation; successful reuse ends retention, listener -loss or idle expiry disposes that generation, and graceful daemon shutdown stops idle retained -generations before exiting. Active ready generations retain the existing handoff policy. +A `close` then `open` on a runner still draining abandoned main-thread work now pays a fresh runner +boot instead of inheriting the stalled one, so the wedge no longer survives the close/open cycle. +Runners that are ready and have no unanswered exchanges can remain warm when retention is requested; +iOS-family simulators additionally require an observed live listener. Close retires unfinished +startup and its recovery authority. The retained generation owns its idle timer and listener +observation; successful reuse ends retention, listener loss or idle expiry disposes that generation, +and graceful daemon shutdown stops idle retained generations before exiting. Active ready +generations retain the existing handoff policy. The daemon no longer models a generic "recent success" cache as a runner-health signal. A proven healthy mutating response for the same app — recorded only after the `runnerFatal` check and only