diff --git a/docs/adr/0005-ios-runner-interaction-lifecycle.md b/docs/adr/0005-ios-runner-interaction-lifecycle.md index 5c539144ce..01d91a6836 100644 --- a/docs/adr/0005-ios-runner-interaction-lifecycle.md +++ b/docs/adr/0005-ios-runner-interaction-lifecycle.md @@ -115,9 +115,14 @@ If xcodebuild still exits for another reason, the next command detects the stale process/liveness checks and avoids the old 15-second graceful-shutdown wait. The remaining latency is fresh xcodebuild runner startup, not a stale transport stall. -A `close` then `open` on a runner still draining abandoned main-thread work now pays a fresh runner boot -instead of inheriting the stalled one, so the wedge no longer survives the close/open cycle. Runners -that were never reported busy keep the existing warm-reuse path unchanged. +A `close` then `open` on a runner still draining abandoned main-thread work now pays a fresh runner +boot instead of inheriting the stalled one, so the wedge no longer survives the close/open cycle. +Runners that are ready and have no unanswered exchanges can remain warm when retention is requested; +iOS-family simulators additionally require an observed live listener. Close retires unfinished +startup and its recovery authority. The retained generation owns its idle timer and listener +observation; successful reuse ends retention, listener loss or idle expiry disposes that generation, +and graceful daemon shutdown stops idle retained generations before exiting. Active ready +generations retain the existing handoff policy. The daemon no longer models a generic "recent success" cache as a runner-health signal. A proven healthy mutating response for the same app — recorded only after the `runnerFatal` check and only diff --git a/docs/agents/device-verification.md b/docs/agents/device-verification.md index ce843cbe9f..ce14954333 100644 --- a/docs/agents/device-verification.md +++ b/docs/agents/device-verification.md @@ -10,8 +10,9 @@ physical devices. Live verification steps apply when exercising a device-facing - Before any Android verification from source: `pnpm build`, `pnpm build:android`, `pnpm clean:daemon`. `build:android` refreshes and verifies both bundled Android helper artifacts for the current package version. -- Graceful `shutdown` hands off a healthy runner that already answered a command, on the simulator - and physical iOS lanes alike; a new daemon may adopt the old binary. After +- Graceful `shutdown` hands off an active healthy runner that already answered a command, on the + simulator and physical iOS lanes alike; a new daemon may adopt the old binary. A runner retained + idle after `close` is stopped before the daemon exits. After Swift runner changes, run `pnpm build:xcuitest` before verification. Use the session cleanup procedure below if ownership is stuck. The physical handoff's device steps are `docs/evidence/ios-physical-runner-handoff-2026-09-19.md`; nothing in it is proven until someone diff --git a/packages/kernel/src/keyed-lock.test.ts b/packages/kernel/src/keyed-lock.test.ts index 5fc2bf8259..610aa9e54a 100644 --- a/packages/kernel/src/keyed-lock.test.ts +++ b/packages/kernel/src/keyed-lock.test.ts @@ -64,3 +64,92 @@ test('withKeyedLock allows reentrant work for the same key while holding the out assert.deepEqual(order, ['outer-start', 'inner', 'outer-end', 'competing']); }); + +test('a deferred descendant of a released lock queues behind its current owner', async () => { + const locks = new Map>(); + const order: string[] = []; + let timerAttempted!: () => void; + const attempted = new Promise((resolve) => { + timerAttempted = resolve; + }); + let descendant!: Promise; + await withKeyedLock(locks, 'device-a', async () => { + descendant = new Promise((resolve, reject) => { + setTimeout(() => { + timerAttempted(); + withKeyedLock(locks, 'device-a', async () => { + order.push('descendant'); + }).then(resolve, reject); + }, 0); + }); + }); + let releaseOwner!: () => void; + const ownerGate = new Promise((resolve) => { + releaseOwner = resolve; + }); + let ownerStarted!: () => void; + const started = new Promise((resolve) => { + ownerStarted = resolve; + }); + const owner = withKeyedLock(locks, 'device-a', async () => { + order.push('owner'); + ownerStarted(); + await ownerGate; + order.push('released'); + }); + await started; + await attempted; + await new Promise((resolve) => setImmediate(resolve)); + assert.deepEqual(order, ['owner']); + releaseOwner(); + await Promise.all([owner, descendant]); + assert.deepEqual(order, ['owner', 'released', 'descendant']); +}); + +test.each([false, true])( + 'reentrant descendants settle before the next owner (reject=%s)', + async (reject) => { + const locks = new Map>(); + const order: string[] = []; + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let markStarted!: () => void; + const started = new Promise((resolve) => { + markStarted = resolve; + }); + let descendant!: Promise; + const owner = withKeyedLock(locks, 'device-a', async () => { + queueMicrotask(() => { + descendant = withKeyedLock(locks, 'device-a', async () => { + order.push('descendant'); + markStarted(); + await gate; + order.push('finished'); + void withKeyedLock(locks, 'device-a', async () => { + await new Promise((resolve) => setImmediate(resolve)); + order.push('nested'); + }); + if (reject) throw new Error('descendant failed'); + }); + void descendant.catch(() => {}); + }); + }); + const competing = withKeyedLock(locks, 'device-a', async () => { + order.push('competing'); + }); + await started; + await new Promise((resolve) => setImmediate(resolve)); + const beforeRelease = [...order]; + release(); + const [, outcome] = await Promise.all([ + owner, + descendant.catch((error: Error) => error), + competing, + ]); + assert.equal(outcome instanceof Error, reject); + assert.deepEqual(beforeRelease, ['descendant']); + assert.deepEqual(order, ['descendant', 'finished', 'nested', 'competing']); + }, +); diff --git a/packages/kernel/src/keyed-lock.ts b/packages/kernel/src/keyed-lock.ts index de1e820184..c8a5e55b06 100644 --- a/packages/kernel/src/keyed-lock.ts +++ b/packages/kernel/src/keyed-lock.ts @@ -3,6 +3,8 @@ import { AsyncLocalStorage } from 'node:async_hooks'; type LockFrame = { locks: Map>; key: string; + active: boolean; + descendants: Set>; }; const keyedLockStorage = new AsyncLocalStorage(); @@ -13,13 +15,37 @@ export async function withKeyedLock( task: () => Promise, ): Promise { const activeLocks = keyedLockStorage.getStore() ?? []; - if (activeLocks.some((entry) => entry.locks === locks && entry.key === key)) { - return await task(); + const held = activeLocks.find( + (entry) => entry.active && entry.locks === locks && entry.key === key, + ); + if (held) { + let markSettled!: () => void; + const settled = new Promise((resolve) => { + markSettled = resolve; + }); + held.descendants.add(settled); + try { + return await task(); + } finally { + held.descendants.delete(settled); + markSettled(); + } } const previous = locks.get(key) ?? Promise.resolve(); const current = previous .catch(() => {}) - .then(() => keyedLockStorage.run([...activeLocks, { locks, key }], task)); + .then(async () => { + const frame: LockFrame = { locks, key, active: true, descendants: new Set() }; + try { + return await keyedLockStorage.run( + [...activeLocks.filter((entry) => entry.active), frame], + task, + ); + } finally { + while (frame.descendants.size > 0) await Promise.all(frame.descendants); + frame.active = false; + } + }); locks.set(key, current); return current.finally(() => { if (locks.get(key) === current) { diff --git a/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts b/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts index e5a7f7d729..a5ecc5270f 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-close-finalization.test.ts @@ -1,7 +1,9 @@ import assert from 'node:assert/strict'; +import { AsyncResource } from 'node:async_hooks'; import fs from 'node:fs'; +import net from 'node:net'; import path from 'node:path'; -import { beforeEach, test, vi } from 'vitest'; +import { afterEach, beforeEach, test, vi } from 'vitest'; import type { AppleApplicationTools } from '@agent-device/contracts/application-lifecycle-runtime'; import type { PlatformRuntimeHost } from '@agent-device/contracts/platform-runtime-operations'; import type { DeviceInfo } from '@agent-device/kernel/device'; @@ -12,6 +14,7 @@ import { makeClassifyOwnerLivenessViaMocks, makeBackgroundRunner, makeRunnerLease, + runnerConnectFailure, runnerError, runnerResponse, RUNNER_CACHE_METADATA_FIXTURE, @@ -19,15 +22,19 @@ import { } from './runner-session-fixtures.ts'; import { mkdtempForTestSync } from './tmp-dir.ts'; import { bindAppleApplicationLifecycle } from '../../lifecycle.ts'; +import { createRequestCanceledError } from '@agent-device/kernel/errors'; +import { prewarmIosRunnerSession } from '../runner-client.ts'; import { platformRuntimeHostFixture } from '../../runtime.fixtures.ts'; import { runnerOwnerToken, writeRunnerLease } from '../runner-lease.ts'; import { abortAllIosRunnerSessions, + detachIosRunnerSessionsForShutdown, ensureRunnerSession, executeRunnerCommandWithSession, readRunnerSessionLiveness, releaseIosRunnerOnClose, stopIosRunnerSession, + stopAllIosRunnerSessions, type RunnerSession, } from '../runner-session.ts'; @@ -125,6 +132,14 @@ vi.mock('../runner-xctestrun.ts', async () => { const IDLE_WINDOW_OUTLIVES_RUN_MS = '600000'; let stateDir: string; +let destination: net.Server; +let destinationConnections: net.Socket[]; + +afterEach(async () => { + await abortAllIosRunnerSessions(); + for (const socket of destinationConnections) socket.destroy(); + await new Promise((resolve) => destination.close(() => resolve())); +}); beforeEach(async () => { appleRunnerTestHost.update({ @@ -160,7 +175,10 @@ beforeEach(async () => { buildMs: 12, xctestrunPathSource: 'build', }); - mockGetFreePort.mockResolvedValue(8123); + destinationConnections = []; + destination = net.createServer((socket) => destinationConnections.push(socket)); + await new Promise((resolve) => destination.listen(0, '127.0.0.1', resolve)); + mockGetFreePort.mockResolvedValue((destination.address() as net.AddressInfo).port); mockPrepareXctestrunWithEnv.mockResolvedValue({ xctestrunPath: '/tmp/session-runner.xctestrun', jsonPath: '/tmp/session-runner.json', @@ -182,7 +200,7 @@ beforeEach(async () => { test('a retained close over an idle runner keeps that runner warm for the next open (#2615)', async () => { const device = deviceNamed('close-finalize-warm-reuse'); - const session = await ensureRunnerSession(device, {}); + const session = await ensureReadyRunner(device); const events: string[] = []; const lifecycle = closeFinalizationLifecycle(device, events); @@ -202,7 +220,7 @@ test('a retained close over an idle runner keeps that runner warm for the next o test('a retained close disposes a runner still draining so the next open boots a clean one (#2552, #2615)', async () => { const device = deviceNamed('close-finalize-busy-dispose'); - const session = await ensureRunnerSession(device, {}); + const session = await ensureReadyRunner(device); await refuseWithRunnerBusy(device, session); const events: string[] = []; const lifecycle = closeFinalizationLifecycle(device, events); @@ -222,7 +240,7 @@ test('a retained close disposes a runner still draining so the next open boots a test('a non-retained close stops the runner and cancels the idle stop a retained close armed (#2615)', async () => { const device = deviceNamed('close-finalize-cancel-idle'); - await ensureRunnerSession(device, {}); + await ensureReadyRunner(device); const lifecycle = closeFinalizationLifecycle(device, []); process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS = '40'; @@ -262,17 +280,172 @@ test('a retained close with no runner in memory starts none and claims no lease const events: string[] = []; const lifecycle = closeFinalizationLifecycle(device, events); + const phases: string[] = []; + appleRunnerTestHost.update({ emitDiagnostic: (event) => phases.push(event.phase) }); + await lifecycle.finalizeApplicationClose(closeInput(true)); assert.deepEqual(events, ['release', 'released', 'alerts']); + assert.deepEqual(phases, []); assert.equal(readRunnerSessionLiveness(device.id), null); assert.equal(mockRunCmdBackground.mock.calls.length, 0); assert.equal(runnerLeaseExists(device.id), false); }); +test('close retires a pending prewarm before its health failure can launch a replacement (#3359)', async () => { + const device = deviceNamed('close-finalize-pending-prewarm'); + let rejectHealth!: (error: Error) => void; + mockWaitForRunner.mockImplementationOnce( + () => + new Promise((_resolve, reject) => { + rejectHealth = reject; + }), + ); + const prewarm = prewarmIosRunnerSession(device); + await vi.waitFor(() => assert.equal(mockWaitForRunner.mock.calls.length, 1)); + + await closeFinalizationLifecycle(device, []).finalizeApplicationClose(closeInput(true)); + rejectHealth(runnerConnectFailure('runner_connect_refused')); + await prewarm; + + assert.equal(mockRunCmdBackground.mock.calls.length, 1); + assert.equal(readRunnerSessionLiveness(device.id), null); + assert.equal(runnerLeaseExists(device.id), false); + assert.equal( + mockRunXcrun.mock.calls.some(([args]) => args.includes('bootstatus')), + false, + ); +}); + +test.each(['transport', 'restored_artifact', 'canceled'] as const)( + 'a retired prewarm cannot invalidate a retained ready generation after %s failure', + async (failure) => { + const device = deviceNamed('close-finalize-ready-prewarm'); + const connections: net.Socket[] = []; + const listener = net.createServer((socket) => connections.push(socket)); + await new Promise((resolve) => listener.listen(0, '127.0.0.1', resolve)); + mockGetFreePort.mockResolvedValue((listener.address() as net.AddressInfo).port); + if (failure === 'restored_artifact') { + mockEnsureXctestrunArtifact.mockResolvedValue({ + xctestrunPath: '/tmp/base-runner.xctestrun', + derived: '/tmp/derived', + cacheKey: RUNNER_CACHE_KEY_FIXTURE, + cache: 'exact', + artifact: 'valid', + buildMs: 0, + xctestrunPathSource: 'build', + }); + } + let rejectHealth!: (error: Error) => void; + mockWaitForRunner.mockImplementationOnce( + () => + new Promise((_resolve, reject) => { + rejectHealth = reject; + }), + ); + try { + const prewarm = prewarmIosRunnerSession(device); + await vi.waitFor(() => assert.equal(mockWaitForRunner.mock.calls.length, 1)); + const session = await ensureRunnerSession(device, {}); + await executeRunnerCommandWithSession( + device, + session, + { command: 'uptime' }, + undefined, + 1_000, + ); + await closeFinalizationLifecycle(device, []).finalizeApplicationClose(closeInput(true)); + await vi.waitFor(() => assert.equal(connections.length, 1)); + rejectHealth( + failure === 'canceled' + ? createRequestCanceledError() + : runnerConnectFailure('runner_connect_refused'), + ); + await prewarm; + + assert.equal(mockRunCmdBackground.mock.calls.length, 1); + assert.equal(readRunnerSessionLiveness(device.id)?.sessionId, session.sessionId); + assert.equal(mockSignalPidsBestEffort.mock.calls.length, 0); + } finally { + await stopIosRunnerSession(device.id); + for (const socket of connections) socket.destroy(); + await new Promise((resolve) => listener.close(() => resolve())); + } + }, +); + +test('graceful shutdown stops idle retained generations and hands off active ready generations', async () => { + const idleDevice = deviceNamed('close-finalize-idle-handoff'); + const activeDevice = deviceNamed('close-finalize-active-handoff'); + const idle = await ensureReadyRunner(idleDevice); + const active = await ensureReadyRunner(activeDevice); + await closeFinalizationLifecycle(idleDevice, []).finalizeApplicationClose(closeInput(true)); + + assert.equal(await detachIosRunnerSessionsForShutdown(), 1); + assert.equal(active.state, 'stopped'); + assert.equal(idle.state, 'ready'); + await stopAllIosRunnerSessions(); + assert.equal(idle.state, 'stopped'); + assert.equal(runnerLeaseExists(idleDevice.id), false); + assert.equal(runnerLeaseExists(activeDevice.id), true); +}); + +test('an idle expiry disposes its retained generation and releases its lease', async () => { + const device = deviceNamed('close-finalize-idle-expiry'); + const session = await ensureReadyRunner(device); + vi.useFakeTimers(); + try { + process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS = '40'; + await closeFinalizationLifecycle(device, []).finalizeApplicationClose(closeInput(true)); + await vi.advanceTimersByTimeAsync(40); + assert.equal(readRunnerSessionLiveness(device.id), null); + assert.equal(session.state, 'stopped'); + assert.equal(runnerLeaseExists(device.id), false); + } finally { + vi.useRealTimers(); + } +}); + +test('an idle expiry queued during reuse cannot stop the generation that was just claimed', async () => { + const device = deviceNamed('close-finalize-expiry-during-reuse'); + const session = await ensureReadyRunner(device); + vi.useFakeTimers(); + try { + process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS = '40'; + await closeFinalizationLifecycle(device, []).finalizeApplicationClose(closeInput(true)); + const expire = AsyncResource.bind(() => vi.advanceTimersByTime(40)); + appleRunnerTestHost.update({ + emitDiagnostic: (event) => { + if (event.phase === 'ios_runner_session_reuse') expire(); + }, + }); + assert.equal(await ensureRunnerSession(device, {}), session); + await vi.runAllTimersAsync(); + assert.equal(readRunnerSessionLiveness(device.id)?.sessionId, session.sessionId); + assert.equal(session.retention, undefined); + assert.equal(mockRunCmdBackground.mock.calls.length, 1); + } finally { + vi.useRealTimers(); + } +}); + +test('a start that fails without using a retained runner leaves it retained', async () => { + const device = deviceNamed('close-finalize-failed-start-retention'); + const session = await ensureReadyRunner(device); + await closeFinalizationLifecycle(device, []).finalizeApplicationClose(closeInput(true)); + + const retention = session.retention; + await assert.rejects( + ensureRunnerSession(device, { expectedRunnerSessionId: 'another-runner-session' }), + ); + assert.equal(session.retention, retention); + assert.ok(session.retention); + assert.equal(readRunnerSessionLiveness(device.id)?.sessionId, session.sessionId); +}); + test('a daemon-shutdown close never issues the ordinary close release (#2615)', async () => { const device = deviceNamed('close-finalize-daemon-shutdown'); - const session = await ensureRunnerSession(device, {}); + const session = await ensureReadyRunner(device); const events: string[] = []; const lifecycle = closeFinalizationLifecycle(device, events); @@ -303,7 +476,7 @@ test('a runner disposal failure propagates out of close before close alerts run test('a drain that lands after close is issued cannot return a busy runner to reuse (#2615)', async () => { const device = deviceNamed('close-finalize-queued-drain'); - const session = await ensureRunnerSession(device, {}); + const session = await ensureReadyRunner(device); await refuseWithRunnerBusy(device, session); // Hold the disposal open so a served reply that clears occupancy lands while close is in // flight, rather than counting microtasks around the check. @@ -332,6 +505,58 @@ test('a drain that lands after close is issued cannot return a busy runner to re assert.equal(readRunnerSessionLiveness(device.id), null); }); +test('a failed start does not re-retain a runner a concurrent start already took over', async () => { + const device = deviceNamed('close-finalize-concurrent-start'); + const session = await ensureReadyRunner(device); + await closeFinalizationLifecycle(device, []).finalizeApplicationClose(closeInput(true)); + const phases: string[] = []; + appleRunnerTestHost.update({ emitDiagnostic: (event) => phases.push(event.phase) }); + + const [failed, reused] = await Promise.allSettled([ + ensureRunnerSession(device, { expectedRunnerSessionId: 'another-runner-session' }), + ensureRunnerSession(device, {}), + ]); + + assert.equal(failed.status, 'rejected'); + assert.equal( + (failed as PromiseRejectedResult).reason?.details?.reason, + 'runner_session_ownership_changed', + ); + assert.equal(reused.status, 'fulfilled'); + assert.equal( + (reused as PromiseFulfilledResult).value.sessionId, + session.sessionId, + ); + assert.doesNotMatch(phases.join('\n'), /ios_runner_idle_stop_scheduled/); +}); + +test('a runner connection lost during retention takes the retained runner and its lease down (#3321)', async () => { + const device = deviceNamed('close-finalize-destination-lost'); + const connections: net.Socket[] = []; + const destination = net.createServer((socket) => { + socket.on('error', () => {}); + connections.push(socket); + }); + await new Promise((resolve) => destination.listen(0, '127.0.0.1', resolve)); + mockGetFreePort.mockResolvedValue((destination.address() as net.AddressInfo).port); + try { + const session = await ensureReadyRunner(device); + await closeFinalizationLifecycle(device, []).finalizeApplicationClose(closeInput(true)); + await vi.waitFor(() => assert.equal(connections.length, 1)); + + for (const socket of connections) socket.destroy(); + + await vi.waitFor(() => assert.equal(readRunnerSessionLiveness(device.id), null), { + timeout: 5000, + }); + assert.equal(runnerLeaseExists(device.id), false); + assert.equal(session.state, 'stopped'); + assert.equal(session.retention, undefined); + } finally { + await new Promise((resolve) => destination.close(() => resolve())); + } +}); + /** The close path's Apple tools, composed exactly as the root's lazy tools compose them. */ function closeFinalizationLifecycle( device: DeviceInfo, @@ -406,3 +631,10 @@ function runnerLeaseExists(deviceId: string): boolean { path.join(process.env.AGENT_DEVICE_IOS_RUNNER_LEASE_DIR ?? '', `${deviceId}.json`), ); } + +async function ensureReadyRunner(device: DeviceInfo): Promise { + const session = await ensureRunnerSession(device, {}); + await executeRunnerCommandWithSession(device, session, { command: 'uptime' }, undefined, 1_000); + await session.listenerWatch?.ready; + return session; +} diff --git a/packages/platform-apple/src/runner/__tests__/runner-retention.test.ts b/packages/platform-apple/src/runner/__tests__/runner-retention.test.ts new file mode 100644 index 0000000000..c23ee4302f --- /dev/null +++ b/packages/platform-apple/src/runner/__tests__/runner-retention.test.ts @@ -0,0 +1,137 @@ +import assert from 'node:assert/strict'; +import net from 'node:net'; +import { afterEach, test, vi } from 'vitest'; +import { observeRunnerListener, retainRunnerSession } from '../runner-retention.ts'; +import { makeRunnerSession } from './runner-session-fixtures.ts'; +import type { RunnerSession } from '../runner-session-types.ts'; + +const listeners: net.Server[] = []; +const connections: net.Socket[] = []; +const sessions: RunnerSession[] = []; + +afterEach(async () => { + vi.useRealTimers(); + vi.unstubAllEnvs(); + for (const session of sessions.splice(0)) { + session.retention?.cancel(); + session.listenerWatch?.close(); + } + for (const socket of connections.splice(0)) socket.destroy(); + await Promise.all( + listeners + .splice(0) + .map((server) => new Promise((resolve) => server.close(() => resolve()))), + ); +}); + +async function servingSession(): Promise { + const listener = net.createServer((socket) => connections.push(socket)); + await new Promise((resolve) => listener.listen(0, '127.0.0.1', resolve)); + listeners.push(listener); + const session = makeRunnerSession({ + state: 'ready', + port: (listener.address() as net.AddressInfo).port, + }); + sessions.push(session); + return session; +} + +async function observe(session: RunnerSession, onLost = vi.fn()): Promise { + observeRunnerListener(session, onLost); + assert.equal(await session.listenerWatch?.ready, true); + await vi.waitFor(() => assert.ok(connections.length)); +} + +test('one generation has one listener watch, and loss before close prevents retention', async () => { + const session = await servingSession(); + const onLost = vi.fn(); + await observe(session, onLost); + const watch = session.listenerWatch; + observeRunnerListener(session, onLost); + assert.equal(session.listenerWatch, watch); + assert.equal(connections.length, 1); + connections[0]!.destroy(); + await vi.waitFor(() => assert.equal(onLost.mock.calls.length, 1)); + assert.equal(await retainRunnerSession(session, vi.fn()), false); +}); + +test('a refused watch is never attached to a later listener on the same port', async () => { + const session = await servingSession(); + const listener = listeners.pop()!; + await new Promise((resolve) => listener.close(() => resolve())); + observeRunnerListener(session, vi.fn()); + assert.equal(await session.listenerWatch?.ready, false); + const replacement = net.createServer((socket) => connections.push(socket)); + await new Promise((resolve) => replacement.listen(session.port, '127.0.0.1', resolve)); + listeners.push(replacement); + observeRunnerListener(session, vi.fn()); + assert.equal(await retainRunnerSession(session, vi.fn()), false); + assert.equal(connections.length, 0); +}); + +test('the idle timer expires once and cancellation of an old window preserves its replacement', async () => { + const session = await servingSession(); + await observe(session); + vi.useFakeTimers(); + vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS', '40'); + const stop = vi.fn(async () => {}); + await retainRunnerSession(session, stop); + const previous = session.retention!; + await vi.advanceTimersByTimeAsync(20); + await retainRunnerSession(session, stop); + const current = session.retention; + previous.cancel(); + assert.equal(session.retention, current); + await vi.advanceTimersByTimeAsync(39); + assert.equal(stop.mock.calls.length, 0); + await vi.advanceTimersByTimeAsync(1); + assert.deepEqual(stop.mock.calls, [[current, 'idle_timeout']]); +}); + +test('ending retention cancels its idle timer while preserving observation', async () => { + const session = await servingSession(); + await observe(session); + vi.useFakeTimers(); + vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS', '40'); + const stop = vi.fn(async () => {}); + await retainRunnerSession(session, stop); + session.retention!.cancel(); + await vi.advanceTimersByTimeAsync(40); + assert.equal(stop.mock.calls.length, 0); + assert.equal(session.retention, undefined); + assert.equal(session.listenerWatch?.lost, false); +}); + +test('zero disables only the idle timer, and unready or occupied generations cannot be retained', async () => { + const session = await servingSession(); + await observe(session); + vi.useFakeTimers(); + vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS', '0'); + const stop = vi.fn(async () => {}); + assert.equal(await retainRunnerSession(session, stop), true); + await vi.advanceTimersByTimeAsync(600_000); + assert.equal(stop.mock.calls.length, 0); + assert.ok(session.retention); + session.retention!.cancel(); + session.state = 'starting'; + assert.equal(await retainRunnerSession(session, stop), false); + session.state = 'ready'; + session.runnerMainThreadBusy = true; + assert.equal(await retainRunnerSession(session, stop), false); + session.runnerMainThreadBusy = false; + session.commandCharges.charge('still-in-flight'); + assert.equal(await retainRunnerSession(session, stop), false); +}); + +test('an overflowing idle duration does not become an immediate stop', async () => { + const session = await servingSession(); + await observe(session); + vi.useFakeTimers(); + vi.stubEnv('AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS', '2147483648'); + const stop = vi.fn(async () => {}); + await retainRunnerSession(session, stop); + await vi.advanceTimersByTimeAsync(1); + assert.equal(stop.mock.calls.length, 0); + await vi.advanceTimersByTimeAsync(2147483646); + assert.deepEqual(stop.mock.calls, [[session.retention, 'idle_timeout']]); +}); diff --git a/packages/platform-apple/src/runner/__tests__/runner-session-close-prep-fence.test.ts b/packages/platform-apple/src/runner/__tests__/runner-session-close-prep-fence.test.ts index 9540029d88..12e20b154e 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-session-close-prep-fence.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-session-close-prep-fence.test.ts @@ -181,7 +181,7 @@ beforeEach(async () => { ); }); -test('close during a cold build leaves the start no way to spawn a replacement build', async () => { +test.each([false, true])('cold close (retain=%s) fences replacement builds', async (retain) => { const device = IOS_ADMISSION_SIMULATOR; // Park close inside its prep stop: the kill is attempted, but the stop has not returned, so // close provably has not reached the session lock yet (the start holds it) and no later fence @@ -209,7 +209,7 @@ test('close during a cold build leaves the start no way to spawn a replacement b await vi.waitFor(() => assert.equal(builds.length, 1)); assert.equal(runnerPrepProcessChildren(device.id).length, 1, 'the build is on the prep ledger'); - const closing = releaseIosRunnerOnClose(device.id, { retain: false }); + const closing = releaseIosRunnerOnClose(device.id, { retain }); await killAttempted; assert.equal( runnerStartTeardownPending(device.id), diff --git a/packages/platform-apple/src/runner/__tests__/runner-session-close.test.ts b/packages/platform-apple/src/runner/__tests__/runner-session-close.test.ts index 67b43e3133..66528aaf1c 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-session-close.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-session-close.test.ts @@ -288,38 +288,14 @@ test('a served non-busy error clears a stale busy report so close keeps a draine assert.equal(session.runnerMainThreadBusy, false); }); -test('releaseIosRunnerOnClose retains an idle runner, disposes a busy one, and the reopen boots fresh (#2552)', async () => { +test('close stops an unready generation and the next use starts fresh', async () => { const device = { ...IOS_SIMULATOR, id: 'runner-session-release-on-close-sim' }; const session = await ensureRunnerSession(device, {}); - - // Idle + retain: warm reuse, same session comes back for the next open. - await releaseIosRunnerOnClose(device.id, { retain: true }); - assert.ok(readRunnerSessionLiveness(device.id)); - assert.equal((await ensureRunnerSession(device, {})).sessionId, session.sessionId); - - // Busy + retain: a command that stalls answers MAIN_THREAD_TIMEOUT, so the stalled runner is - // disposed and the next open boots a clean one. - // The session has not served a command yet, so the read-only snapshot answers over the startup - // transport with no preflight. - mockWaitForRunner.mockResolvedValueOnce( - runnerError({ code: 'MAIN_THREAD_TIMEOUT', message: 'main thread execution timed out' }), - ); - await assert.rejects(() => - executeRunnerCommandWithSession( - device, - session, - { command: 'snapshot', appBundleId: 'com.example.demo' }, - '/tmp/runner.log', - 30_000, - ), - ); await releaseIosRunnerOnClose(device.id, { retain: true }); assert.equal(readRunnerSessionLiveness(device.id), null); + assert.equal(session.state, 'stopped'); + assert.equal(mockWaitForRunner.mock.calls.length, 0); assert.notEqual((await ensureRunnerSession(device, {})).sessionId, session.sessionId); - - // Non-retained close: stops regardless of occupancy. - await releaseIosRunnerOnClose(device.id, { retain: false }); - assert.equal(readRunnerSessionLiveness(device.id), null); }); /** diff --git a/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts b/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts index 048bb20b5c..b9d41fa96a 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts @@ -136,7 +136,6 @@ import { executeRunnerCommandWithSession, invalidateRunnerSession, readRunnerSessionLiveness, - scheduleIosRunnerIdleStop, stopIosRunnerSession, } from '../runner-session.ts'; @@ -231,30 +230,6 @@ test('a startup publishes starting and its first answer publishes ready', async assert.equal(hasLiveIosRunnerSession(device), true); }); -test('an idle stop moves a ready session through disposal to stopped', async () => { - const device = { ...IOS_SIMULATOR, id: 'runner-lifecycle-idle-stop' }; - process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS = '1'; - const session = await ensureRunnerSession(device, {}); - mockWaitForRunner.mockResolvedValueOnce(runnerResponse({ nodes: [], truncated: false })); - await executeRunnerCommandWithSession( - device, - session, - { command: 'snapshot', appBundleId: 'com.example.demo' }, - '/tmp/runner.log', - 30_000, - ); - - scheduleIosRunnerIdleStop(device.id); - await vi.waitFor(() => assert.equal(session.state, 'stopped')); - - assert.equal( - readRunnerSessionLiveness(device.id), - null, - 'the idle stop removes the session from the device registry', - ); - assert.equal(hasLiveIosRunnerSession(device), false); -}); - test('stopping the same registered runner twice tears it down once', async () => { const device = { ...IOS_SIMULATOR, id: 'runner-lifecycle-stop-twice' }; const session = await ensureRunnerSession(device, {}); diff --git a/packages/platform-apple/src/runner/__tests__/runner-session.test.ts b/packages/platform-apple/src/runner/__tests__/runner-session.test.ts index 775e284c9f..facfb59a32 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-session.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-session.test.ts @@ -117,9 +117,7 @@ vi.mock('../runner-xctestrun.ts', async () => { import { abortAllIosRunnerSessions, - cancelIosRunnerIdleStop, ensureRunnerSession, - scheduleIosRunnerIdleStop, executeRunnerCommandWithSession, readRunnerSessionLiveness, invalidateRunnerSession, @@ -404,58 +402,6 @@ test('runner session does not require Apple developer mode for iOS simulators', assert.equal(mockRunAppleToolCommand.mock.calls.some(isDevToolsSecurityStatusCall), false); }); -test('idle stop tears down a retained runner after the idle window', async () => { - const device = { ...IOS_SIMULATOR, id: 'runner-session-idle-stop-sim' }; - const previousIdleMs = process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS; - process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS = '40'; - try { - await ensureRunnerSession(device, {}); - scheduleIosRunnerIdleStop(device.id); - - await new Promise((resolve) => setTimeout(resolve, 150)); - assert.equal(readRunnerSessionLiveness(device.id), null); - } finally { - cancelIosRunnerIdleStop(device.id); - if (previousIdleMs === undefined) delete process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS; - else process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS = previousIdleMs; - } -}); - -test('any runner use cancels a pending idle stop', async () => { - const device = { ...IOS_SIMULATOR, id: 'runner-session-idle-cancel-sim' }; - const previousIdleMs = process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS; - process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS = '40'; - try { - await ensureRunnerSession(device, {}); - scheduleIosRunnerIdleStop(device.id); - await ensureRunnerSession(device, {}); - - await new Promise((resolve) => setTimeout(resolve, 150)); - assert.ok(readRunnerSessionLiveness(device.id)); - } finally { - cancelIosRunnerIdleStop(device.id); - if (previousIdleMs === undefined) delete process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS; - else process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS = previousIdleMs; - } -}); - -test('idle stop is disabled when the window is zero', async () => { - const device = { ...IOS_SIMULATOR, id: 'runner-session-idle-disabled-sim' }; - const previousIdleMs = process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS; - process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS = '0'; - try { - await ensureRunnerSession(device, {}); - scheduleIosRunnerIdleStop(device.id); - - await new Promise((resolve) => setTimeout(resolve, 100)); - assert.ok(readRunnerSessionLiveness(device.id)); - } finally { - cancelIosRunnerIdleStop(device.id); - if (previousIdleMs === undefined) delete process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS; - else process.env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS = previousIdleMs; - } -}); - test('runner session startup kills legacy ownerless xcodebuild before launching a new runner', async () => { const device = { ...IOS_SIMULATOR, id: 'runner-session-startup-stale-sim' }; diff --git a/packages/platform-apple/src/runner/runner-adoption.ts b/packages/platform-apple/src/runner/runner-adoption.ts index 4ea3d0316f..5d0583f431 100644 --- a/packages/platform-apple/src/runner/runner-adoption.ts +++ b/packages/platform-apple/src/runner/runner-adoption.ts @@ -155,6 +155,7 @@ function detachRunnerSessionForShutdown(session: RunnerSession): RunnerDetachOut } catch { return { detached: false, lane, reason: 'lease_write_failed' }; } + session.listenerWatch?.close(); // Only once the lease says the runner is handed over does this process give up its own sides of // the runner's log: until that write lands the session is still owned, and an owned session that // stopped following its runner's output is worse off than one that never handed anything off. diff --git a/packages/platform-apple/src/runner/runner-disposal.ts b/packages/platform-apple/src/runner/runner-disposal.ts index a046d1bfdb..f4d74ed16e 100644 --- a/packages/platform-apple/src/runner/runner-disposal.ts +++ b/packages/platform-apple/src/runner/runner-disposal.ts @@ -65,6 +65,8 @@ export async function disposeRunnerSession( ): Promise { // From here the session is going away: it still owns the lease and the process is still up, so // it can answer a request, and it must not be chosen for new work while it is being taken down. + session.retention?.cancel(); + session.listenerWatch?.close(); advanceRunnerSessionState(session, 'draining'); let processExitHandled = false; if (options.graceful !== false) { @@ -97,6 +99,8 @@ export async function abortRunnerSessionsAndPrepProcesses( const macOsSessions = activeSessions.filter((session) => isMacOs(session.device)); const otherSessions = activeSessions.filter((session) => !isMacOs(session.device)); for (const session of activeSessions) { + session.retention?.cancel(); + session.listenerWatch?.close(); advanceRunnerSessionState(session, 'draining'); } await signalRunnerSessions(otherSessions, 'SIGINT'); diff --git a/packages/platform-apple/src/runner/runner-lifecycle.ts b/packages/platform-apple/src/runner/runner-lifecycle.ts index 76c5218e4d..1ff50b905e 100644 --- a/packages/platform-apple/src/runner/runner-lifecycle.ts +++ b/packages/platform-apple/src/runner/runner-lifecycle.ts @@ -44,6 +44,7 @@ import type { AppleRunnerPrepareResult, } from './runner-provider.ts'; import { + assertRunnerStartAdmitsPreparation, finishRunnerStartAdmission, markRunnerXctestrunArtifactBadForRun, openRunnerStartLoopAdmission, @@ -156,11 +157,9 @@ async function handlePrepareHealthFailure(params: { error: unknown; }): Promise { const { device, session, command, options, signal, attempt, error } = params; + assertRunnerStartAdmitsPreparation(device.id, options.startAdmission); const appErr = asAppError(error, 'COMMAND_FAILED'); if (isRequestCanceledError(appErr)) { - // The owning request was canceled mid-startup (client disconnect): stop the - // just-created session so a canceled prep never leaves a runner retained for - // reuse. Scoped to this request's device only. await invalidateRunnerSessionBestEffort(session, 'prepare_runner_request_canceled'); throw error; } diff --git a/packages/platform-apple/src/runner/runner-retention.ts b/packages/platform-apple/src/runner/runner-retention.ts new file mode 100644 index 0000000000..a6affef723 --- /dev/null +++ b/packages/platform-apple/src/runner/runner-retention.ts @@ -0,0 +1,127 @@ +import net from 'node:net'; +import { isIosFamily } from '@agent-device/kernel/device'; +import { emitDiagnostic } from './host.ts'; +import { + isRunnerMainThreadOccupied, + type RunnerSession, + type RunnerListenerWatch, + type RunnerRetention, +} from './runner-session-types.ts'; + +const LISTENER_CONNECT_TIMEOUT_MS = 500; +const RETAINED_IDLE_STOP_DEFAULT_MS = 5 * 60_000; + +type StopRetainedRunner = ( + retention: RunnerRetention, + reason: 'idle_timeout' | 'listener_lost', +) => Promise; + +/** Observes one serving generation; a lost connection is never attached to a replacement. */ +export function observeRunnerListener( + session: RunnerSession, + onLost: () => Promise | undefined, +): void { + if (session.state !== 'ready' || !needsListenerWatch(session) || session.listenerWatch) return; + const socket = net.connect(session.port, '127.0.0.1'); + socket.unref(); + let settle!: (connected: boolean) => void; + const ready = new Promise((resolve) => { + settle = resolve; + }); + const connectTimer = setTimeout(() => socket.destroy(), LISTENER_CONNECT_TIMEOUT_MS); + connectTimer.unref(); + const watch: RunnerListenerWatch = { + lost: false, + ready, + close() { + clearTimeout(connectTimer); + watch.lost = true; + settle(false); + socket.removeAllListeners(); + socket.on('error', () => {}); + socket.destroy(); + }, + }; + session.listenerWatch = watch; + socket.on('connect', () => { + clearTimeout(connectTimer); + settle(true); + }); + socket.on('error', () => {}); + socket.on('close', () => { + clearTimeout(connectTimer); + watch.lost = true; + settle(false); + void onLost()?.catch((error: unknown) => { + emitDiagnostic({ + level: 'warn', + phase: 'ios_runner_warm_stop_failed', + data: { deviceId: session.deviceId, sessionId: session.sessionId, error: String(error) }, + }); + }); + }); +} + +/** Retains a ready generation already observed by its listener connection, with one idle timer. */ +export async function retainRunnerSession( + session: RunnerSession, + stop: StopRetainedRunner, +): Promise { + if ( + session.state !== 'ready' || + isRunnerMainThreadOccupied(session) || + session.commandCharges.hasOutstandingCharges + ) + return false; + if (needsListenerWatch(session)) { + const watch = session.listenerWatch; + if (!watch || !(await watch.ready) || watch.lost) return false; + } + if ( + session.state !== 'ready' || + isRunnerMainThreadOccupied(session) || + session.commandCharges.hasOutstandingCharges + ) + return false; + session.retention?.cancel(); + let timer: NodeJS.Timeout | undefined; + const retention: RunnerRetention = { + cancel() { + clearTimeout(timer); + if (session.retention === retention) session.retention = undefined; + }, + }; + session.retention = retention; + const idleMs = resolveRunnerIdleStopMs(); + if (idleMs > 0) { + timer = setTimeout(() => { + void stop(retention, 'idle_timeout').catch((error: unknown) => { + emitDiagnostic({ + level: 'warn', + phase: 'ios_runner_idle_stop_failed', + data: { deviceId: session.deviceId, sessionId: session.sessionId, error: String(error) }, + }); + }); + }, idleMs); + timer.unref(); + emitDiagnostic({ + level: 'debug', + phase: 'ios_runner_idle_stop_scheduled', + data: { deviceId: session.deviceId, sessionId: session.sessionId, idleMs }, + }); + } + return true; +} + +function needsListenerWatch(session: RunnerSession): boolean { + return session.device.kind === 'simulator' && isIosFamily(session.device); +} + +function resolveRunnerIdleStopMs(env: NodeJS.ProcessEnv = process.env): number { + const raw = env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS?.trim(); + if (raw) { + const parsed = Number(raw); + if (Number.isFinite(parsed) && parsed >= 0) return Math.min(Math.floor(parsed), 2 ** 31 - 1); + } + return RETAINED_IDLE_STOP_DEFAULT_MS; +} diff --git a/packages/platform-apple/src/runner/runner-session-types.ts b/packages/platform-apple/src/runner/runner-session-types.ts index 4ea18af424..a1b7b7df84 100644 --- a/packages/platform-apple/src/runner/runner-session-types.ts +++ b/packages/platform-apple/src/runner/runner-session-types.ts @@ -50,6 +50,14 @@ export type RunnerProcessHandle = { exitCode: number | null; }; +export type RunnerListenerWatch = { + lost: boolean; + ready: Promise; + close(): void; +}; + +export type RunnerRetention = { cancel(): void }; + export type RunnerSession = { sessionId: string; device: DeviceInfo; @@ -79,6 +87,8 @@ export type RunnerSession = { readLogTail?: (maxBytes: number) => string; /** Moves only through {@link advanceRunnerSessionState}. */ state: RunnerSessionState; + listenerWatch?: RunnerListenerWatch; + retention?: RunnerRetention; /** Wakes one startup retry when the listener becomes ready or its process exits. */ startupRetryWake?: AbortSignal; /** @@ -174,6 +184,8 @@ export function isRunnerMainThreadOccupied( export type RunnerDetachRefusal = /** The runner never answered a command, so nothing proves it serves requests (#2681). */ | 'runner_never_served_a_command' + /** Idle retention depends on this daemon's timer and listener observation. */ + | 'retained_idle' /** A command is still owed a response, so the runner is busy whatever its last report says (#2681). */ | 'command_in_flight' /** The runner reported main-thread work still draining as of its last exchange. */ @@ -319,11 +331,12 @@ function normalizeRunnerChargeId(commandId: string | undefined): string { * command the next daemon sends it. */ export function resolveRunnerDetachDecision( - session: Pick, + session: Pick, ): RunnerDetachDecision { if (session.state !== 'ready') { return { detach: false, reason: 'runner_never_served_a_command' }; } + if (session.retention) return { detach: false, reason: 'retained_idle' }; if (session.commandCharges.hasOutstandingCharges) { return { detach: false, reason: 'command_in_flight' }; } diff --git a/packages/platform-apple/src/runner/runner-session.ts b/packages/platform-apple/src/runner/runner-session.ts index 1200c5dc9f..e6e783d282 100644 --- a/packages/platform-apple/src/runner/runner-session.ts +++ b/packages/platform-apple/src/runner/runner-session.ts @@ -81,8 +81,6 @@ export type RunnerSessionOptions = AppleRunnerLifecycleOptions; const runnerSessions = new Map(); const runnerSessionLocks = new Map>(); -const runnerIdleStopTimers = new Map(); -const RUNNER_RETAINED_IDLE_STOP_DEFAULT_MS = 5 * 60_000; const RUNNER_STALE_BUNDLE_UNINSTALL_TIMEOUT_MS = 10_000; function withRunnerSessionLock(deviceId: string, task: () => Promise): Promise { @@ -93,9 +91,6 @@ export async function ensureRunnerSession( device: DeviceInfo, options: RunnerSessionOptions, ): Promise { - // Any runner use means the device is active again: a pending idle stop - // from a retained-after-close runner no longer applies. - cancelIosRunnerIdleStop(device.id); // This start's admission: the loop that owns the start across retries supplies one, and every // other start takes a token for itself. Both are taken synchronously, before the first await and // registered with the device, so a teardown beginning in this same turn closes it even while @@ -126,7 +121,10 @@ export async function ensureRunnerSession( if (existing) { assertExpectedRunnerSession(existing, options.expectedRunnerSessionId); const reusable = await resolveReusableRunnerSession(device, existing, budget.phase); - if (reusable) return reusable; + if (reusable) { + reusable.retention?.cancel(); + return reusable; + } } return await withRunnerLeaseLock( @@ -142,8 +140,6 @@ export async function ensureRunnerSession( }); const { raceRunnerStartAgainstCaller } = await import('./runner-start-budget.ts'); return await raceRunnerStartAgainstCaller( - // A start that minted its own token releases it; a loop-supplied token belongs to the loop, - // which finishes it when the loop itself is done retrying (#3220). ownedAdmission ? start : start.finally(() => finishRunnerStartAdmission(startAdmission)), startAdmission, options.signal, @@ -198,6 +194,7 @@ async function startRunnerSessionWithLease( await prepareRunnerLeaseForStartup(device, runnerLeaseCleanupAdapter, logicalLeaseContext); }); await measureRunnerStartupStep(startupTimings, 'ensure_booted', async () => { + assertRunnerStartAdmitsPreparation(device.id, startAdmission); await ensureBootedIfNeeded(device); }); // Device first, host second: both answers can be wrong at once, and the phone's own state is the @@ -293,6 +290,7 @@ async function startRunnerSessionWithLease( // where this generation's output starts is only trustworthy once those bytes have landed below // it; otherwise a queued build line reads as the runner's own failure output (#2681). await flushRunnerLogAppends(runnerLogPath).catch(() => {}); + assertRunnerStartAdmitsPreparation(device.id, startAdmission); return await launchRunnerProcess({ device, port, @@ -612,61 +610,25 @@ async function stopRunnerSessionInternal( } } -// Bounds the lifetime of a runner retained after session close: the retained -// runner holds the device's runner lease, which blocks every other daemon on -// the machine from using the device. If nothing touches the runner within the -// idle window, stop it and release the lease. Any ensureRunnerSession call -// cancels the pending stop. AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS overrides -// the window; 0 disables idle stops (retain until daemon exit, the pre-idle -// behavior). -export function scheduleIosRunnerIdleStop(deviceId: string): void { - cancelIosRunnerIdleStop(deviceId); - const idleMs = resolveRunnerIdleStopMs(); - if (idleMs <= 0) return; - if (!runnerSessions.has(deviceId)) return; - const timer = setTimeout(() => { - runnerIdleStopTimers.delete(deviceId); +async function stopRetainedRunner( + session: RunnerSession, + retention: NonNullable, + reason: 'idle_timeout' | 'listener_lost', +): Promise { + await withRunnerSessionLock(session.deviceId, async () => { + if (runnerSessions.get(session.deviceId) !== session || session.retention !== retention) return; emitDiagnostic({ - level: 'info', - phase: 'ios_runner_idle_stop', - data: { deviceId, idleMs }, + level: reason === 'listener_lost' ? 'warn' : 'info', + phase: reason === 'listener_lost' ? 'ios_runner_warm_stop' : 'ios_runner_idle_stop', + data: { deviceId: session.deviceId, sessionId: session.sessionId, reason }, }); - stopIosRunnerSession(deviceId).catch((error: unknown) => { - emitDiagnostic({ - level: 'warn', - phase: 'ios_runner_idle_stop_failed', - data: { - deviceId, - error: error instanceof Error ? error.message : String(error), - }, - }); + await stopRunnerSessionInternal(session.deviceId, session, { + graceful: reason !== 'listener_lost', + waitTimeoutMs: RUNNER_INVALIDATE_WAIT_TIMEOUT_MS, }); - }, idleMs); - timer.unref?.(); - runnerIdleStopTimers.set(deviceId, timer); - emitDiagnostic({ - level: 'debug', - phase: 'ios_runner_idle_stop_scheduled', - data: { deviceId, idleMs }, }); } -export function cancelIosRunnerIdleStop(deviceId: string): void { - const timer = runnerIdleStopTimers.get(deviceId); - if (!timer) return; - clearTimeout(timer); - runnerIdleStopTimers.delete(deviceId); -} - -function resolveRunnerIdleStopMs(env: NodeJS.ProcessEnv = process.env): number { - const raw = env.AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS?.trim(); - if (raw) { - const parsed = Number(raw); - if (Number.isFinite(parsed) && parsed >= 0) return Math.floor(parsed); - } - return RUNNER_RETAINED_IDLE_STOP_DEFAULT_MS; -} - /** The first command that is not a readiness probe makes the session the caller's, not a guess. */ export function markRunnerSessionServed(session: RunnerSession, command: RunnerCommand): void { if (session.speculative && !isRunnerReadinessProbeCommand(command)) { @@ -712,11 +674,17 @@ async function stopIosRunnerSessionDevice( deviceId: string, fenceSettle?: () => void, ): Promise { - cancelIosRunnerIdleStop(deviceId); try { await withRunnerSessionLock(deviceId, async () => { await withRunnerLeaseLock(deviceId, async () => { - await stopRunnerSessionInternal(deviceId, undefined, { leaseLockHeld: true }); + const session = runnerSessions.get(deviceId); + await stopRunnerSessionInternal(deviceId, session, { + leaseLockHeld: true, + graceful: + session?.state === 'ready' && + !isRunnerMainThreadOccupied(session) && + !session.listenerWatch?.lost, + }); await cleanupOwnedIosRunnerLease(deviceId); }); fenceSettle?.(); @@ -726,44 +694,48 @@ async function stopIosRunnerSessionDevice( } } -/** - * Releases a runner at session close, preferring warm reuse only when the runner is actually - * reusable. A non-retained close, or a retained close over a runner whose last exchange reported - * main-thread work still draining, stops it now: a busy runner refuses every command until it drains - * or wedges, so pooling it back hands the same stalled process to the next `open` (#2552). An idle - * retained runner keeps warm reuse via the idle-stop timer. The decision is owned here because the - * occupancy fact lives on the session, and awaited so `close` returns only once the lease is gone. - * Non-retained close first fences the device's start admission, then stops the device's current - * prep processes, before taking the session lock an in-flight cold start holds through its build: - * a start that answers the kill by retrying its build finds the spawn refused, which is what keeps - * close from waiting on the replacement build (#3220). - */ +/** Retires startup recovery at close; only a serving, observed generation can remain idle. */ export async function releaseIosRunnerOnClose( deviceId: string, options: { retain: boolean }, ): Promise { - const session = runnerSessions.get(deviceId); - if (options.retain && !isRunnerMainThreadOccupied(session)) { - scheduleIosRunnerIdleStop(deviceId); - return; - } - if (options.retain) { - emitDiagnostic({ - level: 'info', - phase: 'ios_runner_retain_skipped_busy', - data: { deviceId }, - }); - } - // First the fence, then the kill: an in-flight start that answers the kill by retrying its - // build finds admission closed and stops, and the fence outlives the kill and the session stop - // until this close settles — while it stands, nothing prepares this device (#3220). + const candidate = runnerSessions.get(deviceId); + const busyAtClose = isRunnerMainThreadOccupied(candidate); const settleFence = fenceRunnerStartAdmissionsForTeardown(deviceId); try { + if (options.retain && candidate?.state === 'ready' && !busyAtClose) { + const retained = await withRunnerSessionLock(deviceId, async () => { + const session = runnerSessions.get(deviceId); + if (!session) return false; + const { retainRunnerSession } = await import('./runner-retention.ts'); + if ( + await retainRunnerSession(session, (retention, reason) => + stopRetainedRunner(session, retention, reason), + ) + ) { + settleFence(); + return true; + } + emitDiagnostic({ + level: 'info', + phase: isRunnerMainThreadOccupied(session) + ? 'ios_runner_retain_skipped_busy' + : 'ios_runner_retain_skipped', + data: { deviceId, sessionId: session.sessionId, state: session.state }, + }); + return false; + }); + if (retained) return; + } else if (options.retain && busyAtClose && candidate) { + emitDiagnostic({ + level: 'info', + phase: 'ios_runner_retain_skipped_busy', + data: { deviceId, sessionId: candidate.sessionId, state: candidate.state }, + }); + } await stopRunnerPrepProcesses(deviceId); await stopIosRunnerSessionDevice(deviceId, settleFence); } finally { - // The error path settles too: a close that throws mid-teardown must not fence the device - // past its own failure. settleFence(); } } @@ -790,12 +762,11 @@ export async function abortAllIosRunnerSessions(): Promise { } // The detach decision itself lives in runner-adoption.ts beside the adoption it hands off to -// (#2681); this is the map side: a detached session leaves the registry and drops its idle -// timer (the detached module gives up the log observation and marks the session stopped). +// (#2681); detached active generations leave the registry. Idle retained generations stay +// owned until shutdown stops them. export async function detachIosRunnerSessionsForShutdown(): Promise { return await detachRunnerSessionsForShutdown(runnerSessions, (deviceId) => { runnerSessions.delete(deviceId); - cancelIosRunnerIdleStop(deviceId); }); } @@ -865,7 +836,7 @@ export async function executeRunnerCommandWithSession( ): Promise> { emitRunnerStartupTimings(session, command.command); const { executeRunnerExchange } = await import('./runner-exchange.ts'); - return executeRunnerExchange( + const data = await executeRunnerExchange( device, session, command, @@ -874,6 +845,14 @@ export async function executeRunnerCommandWithSession( (reason) => invalidateRunnerSession(session, reason), signal, ); + if (runnerSessions.get(device.id) === session && session.state === 'ready') { + const { observeRunnerListener } = await import('./runner-retention.ts'); + observeRunnerListener(session, () => { + const retention = session.retention; + if (retention) return stopRetainedRunner(session, retention, 'listener_lost'); + }); + } + return data; } /** diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index dbf334bbbe..c7ab8e0693 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -530,7 +530,7 @@ Android physical-device prerequisites: Runner and daemon lifecycle (applies to simulators too): open without --relaunch is idempotent-foreground for an already-running app (it brings the process forward; it does not restart it). open --relaunch restarts the app; on iOS simulators this collapses to one simctl launch --terminate-running-process call instead of a separate terminate-then-launch. No runner read launches a session app that is not running: snapshot, wait, is, get, a reading find, and an interaction's leading reads (a gesture's viewport read, the capture that resolves a selector click/fill) answer the retriable APP_NOT_RUNNING instead of bare-launching over a launch SpringBoard still holds behind its deep-link confirmation. Only open, activate, and a command that mutates without a leading read bring a stopped app up. - close keeps a healthy iOS simulator XCTest runner warm by default so the next open on that simulator (same udid in the same simulator set) skips the runner build, unless --shutdown was requested, the session was recording, or the session held a device lease. A retained runner auto-stops after an idle window (default 5 minutes); set AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS to override, or 0 to disable idle stop and retain until daemon exit. + close keeps a healthy iOS simulator XCTest runner warm by default so the next open on that simulator (same udid in the same simulator set) skips the runner build, unless --shutdown was requested, the session was recording, or the session held a device lease. A retained runner auto-stops after an idle window (default 5 minutes); set AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS to override, or 0 to disable idle stop and retain until daemon exit. Only a ready runner with an observed listener is retained; close retires unfinished prewarm recovery. Losing the retained listener stops that generation, and daemon shutdown stops idle retained runners rather than handing them off without supervision. Each AGENT_DEVICE_STATE_DIR runs its own daemon. It self-exits after an idle window (default 5 minutes, matching the runner idle-stop default) once it has no open sessions, no in-flight requests, and no active recording; set AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS to override, or 0 to disable idle reap. On a machine shared by several agents, set AGENT_DEVICE_SESSION_IDLE_TIMEOUT_MS to also expire an individual session that has taken a host-global device claim and then received no commands for that long; the claim is released and the next command on that session answers SESSION_NOT_FOUND with details.reason SESSION_IDLE_EXPIRED naming the window and the device. Off by default, because on a shared host "idle" and "thinking" are indistinguishable: only set it where an abandoned claim blocking every other agent is the worse failure. Sessions holding a remote lease or any active capture (recording, logs, audio, performance, or trace) are never expired this way. A stale iOS runner lease — its owner process dead, or its AGENT_DEVICE_STATE_DIR deleted — is reclaimed automatically instead of failing with "is already owned by another agent-device daemon". A live owner's runner is also reclaimed when the requesting daemon holds the host-global device claim for that device: claims are exclusive, so holding one proves the runner's owner released the device and merely kept the runner warm. The error remains only for owners outside claim arbitration (a pre-claims build, or daemons pointed at different claim stores). diff --git a/src/daemon/server/daemon-idle-reap.ts b/src/daemon/server/daemon-idle-reap.ts index 2b7b9afbd5..61e085b29a 100644 --- a/src/daemon/server/daemon-idle-reap.ts +++ b/src/daemon/server/daemon-idle-reap.ts @@ -11,13 +11,9 @@ import { isUncommittedRepairSession } from '../session-replay-transaction.ts'; // way also keeps holding its iOS runner lease, which blocks a fresh daemon for // the same device (see runner-lease.ts stale-lease takeover). // -// The window mirrors the iOS runner idle-stop default -// (AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS, 5 minutes, see runner-session.ts): -// graceful daemon shutdown already hands off a healthy retained runner - -// simulator or physical iOS - for the next daemon to adopt -// (detachIosRunnerSessionsForShutdown), so reaping the daemon process itself on -// the same timescale does not force a runner rebuild in the common case - it -// only pays the cheap daemon bootstrap (socket/HTTP listen) on the next command. +// The window mirrors the runner idle-stop default. Idle retained runners stop with their daemon: +// their timer and listener observation cannot survive a handoff. Active eligible runners retain +// the graceful-shutdown adoption policy. const DAEMON_IDLE_REAP_DEFAULT_MS = 5 * 60_000; // AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS overrides the window; 0 disables idle diff --git a/website/docs/docs/sessions.md b/website/docs/docs/sessions.md index 6a8b3ee14e..7821fe9237 100644 --- a/website/docs/docs/sessions.md +++ b/website/docs/docs/sessions.md @@ -114,6 +114,12 @@ in CI and on shared hosts so devices don't keep running after the run: agent-device close --shutdown ``` +Without `--shutdown`, `close` can keep a ready, idle iOS simulator runner warm for the next `open` +when its listener connection is still live and the session has no screen recording or device lease. +Unfinished runner startup is stopped. If you shut that simulator down yourself while the runner is +retained, losing its listener connection stops the retained runner. Idle retained runners also stop +when their daemon shuts down gracefully. + ## Wait for a slow or busy device A never-booted iOS Simulator can take several minutes to finish its first boot. Give `open` (or