diff --git a/src/daemon/handlers/session-open.ts b/src/daemon/handlers/session-open.ts index 1e3e17720c..53e3dcf8c7 100644 --- a/src/daemon/handlers/session-open.ts +++ b/src/daemon/handlers/session-open.ts @@ -12,7 +12,7 @@ import { createAppleRunnerCacheColdBootPrewarmForOpen, } from '../apple-runner-options.ts'; import { applyRuntimeHintsToApp } from '../runtime-hints.ts'; -import { isIosFamily, type DeviceInfo } from '../../kernel/device.ts'; +import { isApplePlatform, isIosFamily, type DeviceInfo } from '../../kernel/device.ts'; import type { DaemonRequest, DaemonResponse, SessionRuntimeHints, SessionState } from '../types.ts'; import { resolveSessionRequestLogPath, @@ -72,13 +72,13 @@ async function relaunchCloseApp(params: { context: Parameters[4]; }): Promise { const { device, closeTarget, outFlag, context } = params; - // Simulator close/open go through simctl and never touch the XCUITest - // runner, so a healthy runner session survives the relaunch (~6s saved per - // open --relaunch). A runner that does go stale is caught by the readiness - // preflight and restarted via invalidateRunnerSession/restart-and-replay. - // Real devices keep the conservative teardown: their runner transport rides - // the device tunnel, which app relaunches can disturb. - if (device.platform !== 'android' && !isIosSimulator(device)) { + // Only Apple targets have an XCUITest runner to tear down, and simulators + // keep theirs hot: their close/open go through simctl and never touch the + // runner (~6s saved per open --relaunch); one that goes stale is caught by + // the readiness preflight and restarted via invalidateRunnerSession. + // macOS and real iOS devices keep the conservative teardown — the device + // transport rides the tunnel, which app relaunches can disturb. + if (isApplePlatform(device.platform) && !isIosSimulator(device)) { await stopIosRunnerSession(device.id); } await dispatchCommand(device, 'close', [closeTarget], outFlag, context); @@ -167,6 +167,47 @@ async function completeOpenCommand(params: { const openCommandStartedAtMs = Date.now(); const timing: OpenTiming = {}; + const shouldPrewarmIosRunner = + isIosFamily(device) && + surface === 'app' && + openPositionals.length > 0 && + Boolean(sessionAppBundleId); + const runnerPrewarmOptions = buildAppleRunnerSessionOptions({ + req, + logPath, + appBundleId: sessionAppBundleId, + traceLogPath, + }); + const shouldPrewarmRunnerBeforeOpen = req.flags?.maestro?.prewarmRunnerBeforeOpen === true; + let runnerPrewarm: Promise | undefined; + // Tracked separately from `runnerPrewarm`: prewarmIosRunnerSession may + // return undefined (prewarm unavailable), and one attempt is one attempt. + let runnerPrewarmScheduled = false; + let runnerPrewarmAwaited = false; + const schedulePrewarm = ( + options: Parameters[1] = runnerPrewarmOptions, + ): void => { + runnerPrewarmScheduled = true; + timing.runnerPrewarmKind = 'session'; + timing.runnerPrewarmScheduled = true; + runnerPrewarm = prewarmIosRunnerSession(device, options); + }; + const awaitPrewarm = async (): Promise => { + if (!runnerPrewarm || runnerPrewarmAwaited) return; + runnerPrewarmAwaited = true; + const startedAtMs = Date.now(); + await runnerPrewarm; + timing.runnerPrewarmWaited = true; + timing.runnerPrewarmDurationMs = Math.max(0, Date.now() - startedAtMs); + }; + // Start the runner spin-up before close/open dispatch on simulators: neither + // touches the runner there (both ride simctl), so the xcodebuild ramp + // overlaps the app relaunch instead of following it. Real devices tear the + // runner down in relaunchCloseApp, so their prewarm stays post-open. + if (shouldPrewarmIosRunner && isIosSimulator(device) && !shouldPrewarmRunnerBeforeOpen) { + schedulePrewarm(); + } + if (shouldRelaunch && openTarget) { const closeTarget = sessionAppBundleId ?? openTarget; const closeStartedAtMs = Date.now(); @@ -193,29 +234,9 @@ async function completeOpenCommand(params: { runtime, }); timing.runtimeHintsDurationMs = Math.max(0, Date.now() - runtimeHintsStartedAtMs); - const shouldPrewarmIosRunner = - isIosFamily(device) && surface === 'app' && openPositionals.length > 0; - const runnerPrewarmOptions = buildAppleRunnerSessionOptions({ - req, - logPath, - appBundleId: sessionAppBundleId, - traceLogPath, - }); - const shouldPrewarmRunnerBeforeOpen = req.flags?.maestro?.prewarmRunnerBeforeOpen === true; - let runnerPrewarm: Promise | undefined; - if (shouldPrewarmIosRunner && sessionAppBundleId) { - timing.runnerPrewarmKind = 'session'; - timing.runnerPrewarmScheduled = true; - if (shouldPrewarmRunnerBeforeOpen) { - runnerPrewarm = prewarmIosRunnerSession(device, { - ...runnerPrewarmOptions, - propagateError: true, - }); - const runnerPrewarmStartedAtMs = Date.now(); - await runnerPrewarm; - timing.runnerPrewarmWaited = true; - timing.runnerPrewarmDurationMs = Math.max(0, Date.now() - runnerPrewarmStartedAtMs); - } + if (shouldPrewarmIosRunner && shouldPrewarmRunnerBeforeOpen) { + schedulePrewarm({ ...runnerPrewarmOptions, propagateError: true }); + await awaitPrewarm(); } const openStartedAtMs = Date.now(); const provisionalSession = await prepareOpenDispatchSession({ @@ -247,15 +268,12 @@ async function completeOpenCommand(params: { openPositionals, }); timing.launchUrlDurationMs = Math.max(0, Date.now() - launchUrlStartedAtMs); - if (shouldPrewarmIosRunner && sessionAppBundleId && !runnerPrewarm) { - runnerPrewarm = prewarmIosRunnerSession(device, runnerPrewarmOptions); + if (shouldPrewarmIosRunner && !runnerPrewarmScheduled) { + schedulePrewarm(); } - if (shouldRelaunch && runnerPrewarm && timing.runnerPrewarmWaited !== true) { - const runnerPrewarmStartedAtMs = Date.now(); - await runnerPrewarm; - timing.runnerPrewarmWaited = true; - timing.runnerPrewarmDurationMs = Math.max(0, Date.now() - runnerPrewarmStartedAtMs); - } else if (runnerPrewarm && timing.runnerPrewarmWaited !== true) { + if (shouldRelaunch) { + await awaitPrewarm(); + } else if (runnerPrewarm && !runnerPrewarmAwaited) { timing.runnerPrewarmWaited = false; } sessionAppBundleId = await inferAndroidPackageAfterOpen(device, openTarget, sessionAppBundleId); diff --git a/src/daemon/server/daemon-runtime.ts b/src/daemon/server/daemon-runtime.ts index d78ecaf404..8ec6de177f 100644 --- a/src/daemon/server/daemon-runtime.ts +++ b/src/daemon/server/daemon-runtime.ts @@ -239,12 +239,18 @@ export async function startDaemonRuntime( await emitFatalDiagnostic(shutdownOptions.cause); } await closeDaemonServers(servers); + // Hand healthy simulator runners off to the next daemon before session + // teardown gets a chance to kill them; everything left after this + // (real devices, unhealthy runners) goes through the normal stop path. + const { detachIosSimulatorRunnerSessionsForShutdown, stopAllIosRunnerSessions } = + await import('../../platforms/apple/core/runner/runner-client.ts'); + try { + await detachIosSimulatorRunnerSessionsForShutdown(); + } catch {} await teardownDaemonSessions(); await Promise.allSettled( providerDeviceRuntimes.map(async (runtime) => await runtime.shutdown()), ); - const { stopAllIosRunnerSessions } = - await import('../../platforms/apple/core/runner/runner-client.ts'); await stopAllIosRunnerSessions(); // Best effort: stop the PNG worker so an in-flight job cannot delay exit. await Promise.race([ diff --git a/src/platforms/apple/core/__tests__/runner-adoption.test.ts b/src/platforms/apple/core/__tests__/runner-adoption.test.ts new file mode 100644 index 0000000000..e6fe3e1d3d --- /dev/null +++ b/src/platforms/apple/core/__tests__/runner-adoption.test.ts @@ -0,0 +1,173 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, expect, test, vi } from 'vitest'; +import type { DeviceInfo } from '../../../../kernel/device.ts'; +import { + buildDetachedRunnerLease, + buildRunnerLease, + readStaleRunnerLease, + writeRunnerLease, + type RunnerLease, +} from '../runner/runner-lease.ts'; +import { + isIosRunnerDetachEnabled, + tryAdoptRunnerSessionFromLease, +} from '../runner/runner-adoption.ts'; +import { sendRunnerCommandOnce } from '../runner/runner-transport.ts'; +import { isProcessAlive } from '../../../../utils/process-identity.ts'; +import { + resolveExpectedRunnerCacheMetadata, + resolveRunnerDerivedPath, +} from '../runner/runner-xctestrun.ts'; + +vi.mock('../runner/runner-transport.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, sendRunnerCommandOnce: vi.fn() }; +}); +vi.mock('../../../../utils/process-identity.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, isProcessAlive: vi.fn(() => false) }; +}); + +const mockSendRunnerCommandOnce = vi.mocked(sendRunnerCommandOnce); +const mockIsProcessAlive = vi.mocked(isProcessAlive); + +const simulator: DeviceInfo = { + platform: 'apple', + id: 'adopt-sim-1', + name: 'iPhone 17 Pro', + kind: 'simulator', + target: 'mobile', + booted: true, +}; + +let leaseDir: string; +let expectedDerived: string; + +function writeStaleLease(overrides: Partial = {}): RunnerLease { + const lease: RunnerLease = { + ...buildRunnerLease({ + deviceId: simulator.id, + sessionId: `${simulator.id}:50700:1`, + runnerPid: 424242, + port: 50700, + xctestrunPath: path.join(expectedDerived, 'Build', 'Products', 'env.session.xctestrun'), + jsonPath: path.join(expectedDerived, 'Build', 'Products', 'env.session.json'), + }), + // A pid+start-time that cannot belong to a live process makes the lease + // owner dead, i.e. the lease classifies as stale. + ownerToken: 'owner-99999-deadbeef', + ownerPid: 99999, + ownerStartTime: 'not-a-real-start-time', + ...overrides, + }; + writeRunnerLease(lease); + return lease; +} + +beforeEach(() => { + leaseDir = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-lease-test-')); + process.env.AGENT_DEVICE_IOS_RUNNER_LEASE_DIR = leaseDir; + expectedDerived = resolveRunnerDerivedPath( + simulator, + resolveExpectedRunnerCacheMetadata(simulator), + ); + mockSendRunnerCommandOnce.mockReset(); + mockIsProcessAlive.mockReset(); + mockIsProcessAlive.mockReturnValue(false); + delete process.env.AGENT_DEVICE_IOS_RUNNER_DETACH; +}); + +afterEach(() => { + delete process.env.AGENT_DEVICE_IOS_RUNNER_LEASE_DIR; + delete process.env.AGENT_DEVICE_IOS_RUNNER_DETACH; + fs.rmSync(leaseDir, { recursive: true, force: true }); +}); + +test('isIosRunnerDetachEnabled honors the kill switch', () => { + expect(isIosRunnerDetachEnabled({})).toBe(true); + expect(isIosRunnerDetachEnabled({ AGENT_DEVICE_IOS_RUNNER_DETACH: '0' })).toBe(false); + expect(isIosRunnerDetachEnabled({ AGENT_DEVICE_IOS_RUNNER_DETACH: 'false' })).toBe(false); + expect(isIosRunnerDetachEnabled({ AGENT_DEVICE_IOS_RUNNER_DETACH: '1' })).toBe(true); +}); + +test('readStaleRunnerLease returns dead-owner leases and skips owned ones', () => { + writeStaleLease(); + expect(readStaleRunnerLease(simulator.id)?.port).toBe(50700); + + // A lease written by this process is owned, not stale. + writeRunnerLease( + buildRunnerLease({ + deviceId: simulator.id, + sessionId: `${simulator.id}:50700:2`, + runnerPid: 424242, + port: 50700, + xctestrunPath: '/tmp/x.xctestrun', + jsonPath: '/tmp/x.json', + }), + ); + expect(readStaleRunnerLease(simulator.id)).toBeNull(); +}); + +test('buildDetachedRunnerLease rewrites the token', () => { + const lease = writeStaleLease(); + expect(buildDetachedRunnerLease(lease).ownerToken).toBe(`detached-${lease.ownerToken}`); +}); + +test('adoption succeeds for a live, matching, probe-healthy runner', async () => { + const lease = writeStaleLease(); + mockIsProcessAlive.mockReturnValue(true); + mockSendRunnerCommandOnce.mockResolvedValue(new Response(JSON.stringify({ ok: true }))); + + const session = await tryAdoptRunnerSessionFromLease(simulator, {}); + + expect(session).not.toBeNull(); + expect(session?.port).toBe(lease.port); + expect(session?.ready).toBe(true); + expect(session?.child.pid).toBe(424242); + expect(session?.xctestrunArtifact?.reason).toBe('adopted_from_lease'); + // Adoption transfers ownership: the lease on disk now belongs to us. + expect(readStaleRunnerLease(simulator.id)).toBeNull(); +}); + +test('adoption is skipped for devices in a custom simulator set', async () => { + writeStaleLease(); + mockIsProcessAlive.mockReturnValue(true); + + const scopedDevice = { ...simulator, simulatorSetPath: '/custom/device-set' }; + expect(await tryAdoptRunnerSessionFromLease(scopedDevice, {})).toBeNull(); + expect(mockSendRunnerCommandOnce).not.toHaveBeenCalled(); +}); + +test('adoption is skipped when the runner process is dead', async () => { + writeStaleLease(); + mockIsProcessAlive.mockReturnValue(false); + + expect(await tryAdoptRunnerSessionFromLease(simulator, {})).toBeNull(); + expect(mockSendRunnerCommandOnce).not.toHaveBeenCalled(); +}); + +test('adoption is skipped on artifact fingerprint mismatch', async () => { + writeStaleLease({ xctestrunPath: '/somewhere/else/Build/Products/env.xctestrun' }); + mockIsProcessAlive.mockReturnValue(true); + + expect(await tryAdoptRunnerSessionFromLease(simulator, {})).toBeNull(); + expect(mockSendRunnerCommandOnce).not.toHaveBeenCalled(); +}); + +test('adoption is skipped when the probe fails', async () => { + writeStaleLease(); + mockIsProcessAlive.mockReturnValue(true); + mockSendRunnerCommandOnce.mockRejectedValue(new Error('connection refused')); + + expect(await tryAdoptRunnerSessionFromLease(simulator, {})).toBeNull(); +}); + +test('adoption is disabled by the kill switch', async () => { + writeStaleLease(); + mockIsProcessAlive.mockReturnValue(true); + process.env.AGENT_DEVICE_IOS_RUNNER_DETACH = '0'; + + expect(await tryAdoptRunnerSessionFromLease(simulator, {})).toBeNull(); +}); diff --git a/src/platforms/apple/core/__tests__/runner-session.test.ts b/src/platforms/apple/core/__tests__/runner-session.test.ts index f4364016c6..a64ef8882a 100644 --- a/src/platforms/apple/core/__tests__/runner-session.test.ts +++ b/src/platforms/apple/core/__tests__/runner-session.test.ts @@ -109,6 +109,7 @@ vi.mock('../runner/runner-xctestrun.ts', async () => { import { abortAllIosRunnerSessions, + detachIosSimulatorRunnerSessionsForShutdown, ensureRunnerSession, executeRunnerCommandWithSession, getRunnerSessionSnapshot, @@ -725,6 +726,42 @@ test('runner session does not require Apple developer mode for iOS simulators', assert.equal(mockRunAppleToolCommand.mock.calls.some(isDevToolsSecurityStatusCall), false); }); +test('shutdown detach hands off default-set simulator runner sessions', async () => { + const device = { ...IOS_SIMULATOR, id: 'runner-session-detach-default-sim' }; + // Default simulator set: no XCTestDevices redirect is held. + mockAcquireXcodebuildSimulatorSetRedirect.mockResolvedValue(null); + await ensureRunnerSession(device, {}); + + const detached = await detachIosSimulatorRunnerSessionsForShutdown(); + + assert.equal(detached, 1); + assert.equal(getRunnerSessionSnapshot(device.id), null); + const leaseRaw = fs.readFileSync( + path.join(process.env.AGENT_DEVICE_IOS_RUNNER_LEASE_DIR ?? '', `${device.id}.json`), + 'utf8', + ); + const lease = JSON.parse(leaseRaw) as { ownerToken: string }; + assert.match(lease.ownerToken, /^detached-owner-/); +}); + +test('shutdown detach keeps scoped simulator-set runner sessions for the kill path', async () => { + const device = { + ...IOS_SIMULATOR, + id: 'runner-session-detach-scoped-sim', + simulatorSetPath: '/tmp/custom-device-set', + }; + await ensureRunnerSession(device, {}); + assert.equal(mockAcquireXcodebuildSimulatorSetRedirect.mock.calls.length, 1); + + const detached = await detachIosSimulatorRunnerSessionsForShutdown(); + + // The redirect-holding session must stay for disposal, which restores the + // XCTestDevices symlink; detach never releases the redirect itself. + assert.equal(detached, 0); + assert.ok(getRunnerSessionSnapshot(device.id)); + assert.equal(mockRedirectRelease.mock.calls.length, 0); +}); + test('runner session startup kills legacy ownerless xcodebuild before launching a new runner', async () => { const device = { ...IOS_SIMULATOR, id: 'runner-session-startup-stale-sim' }; diff --git a/src/platforms/apple/core/__tests__/simulator-booted-memo.test.ts b/src/platforms/apple/core/__tests__/simulator-booted-memo.test.ts index 5f615b1b57..872455dff9 100644 --- a/src/platforms/apple/core/__tests__/simulator-booted-memo.test.ts +++ b/src/platforms/apple/core/__tests__/simulator-booted-memo.test.ts @@ -3,6 +3,7 @@ import type { DeviceInfo } from '../../../../kernel/device.ts'; import { __resetSimulatorBootedMemoForTests, ensureBootedSimulator, + markSimulatorBooted, shutdownSimulator, SIMULATOR_BOOTED_MEMO_TTL_MS, } from '../simulator.ts'; @@ -71,6 +72,13 @@ test('shutdownSimulator invalidates the booted memo', async () => { expect(countSimctlListCalls()).toBe(2); }); +test('markSimulatorBooted seeds the memo so the first boot check skips the listing', async () => { + markSimulatorBooted(simulator); + + await ensureBootedSimulator(simulator); + expect(countSimctlListCalls()).toBe(0); +}); + test('booted memo is scoped per simulator set path', async () => { await ensureBootedSimulator(simulator); expect(countSimctlListCalls()).toBe(1); diff --git a/src/platforms/apple/core/devices.ts b/src/platforms/apple/core/devices.ts index 5d008d09b1..23ff67b144 100644 --- a/src/platforms/apple/core/devices.ts +++ b/src/platforms/apple/core/devices.ts @@ -12,6 +12,7 @@ import { import { resolveIosSimulatorDeviceSetPath } from '../../../utils/device-isolation.ts'; import { buildHostMacDevice } from '../os/macos/devices.ts'; import { buildSimctlArgs } from './simctl.ts'; +import { markSimulatorBooted } from './simulator.ts'; import { resolveAppleToolProvider, runXcrun } from './tool-provider.ts'; export { createLocalAppleToolProvider, withAppleToolProvider } from './tool-provider.ts'; @@ -212,7 +213,7 @@ function parseSimctlAppleDevices( for (const device of runtimes) { if (!device.isAvailable) continue; const target = resolveAppleTargetFromRuntime(runtime); - devices.push({ + const parsed: DeviceInfo = { platform: 'apple', id: device.udid, name: device.name, @@ -223,7 +224,9 @@ function parseSimctlAppleDevices( appleOs: resolveAppleOs(target, [device.deviceTypeIdentifier ?? '', device.name]), booted: device.state === 'Booted', ...(simulatorSetPath ? { simulatorSetPath } : {}), - }); + }; + if (parsed.booted) markSimulatorBooted(parsed); + devices.push(parsed); } } return devices; diff --git a/src/platforms/apple/core/runner/runner-adoption.ts b/src/platforms/apple/core/runner/runner-adoption.ts new file mode 100644 index 0000000000..c36d1c5df8 --- /dev/null +++ b/src/platforms/apple/core/runner/runner-adoption.ts @@ -0,0 +1,184 @@ +import path from 'node:path'; +import { resolveIosSimulatorDeviceSetPath } from '../../../../utils/device-isolation.ts'; +import { emitDiagnostic } from '../../../../utils/diagnostics.ts'; +import { isProcessAlive } from '../../../../utils/process-identity.ts'; +import { parseBooleanLiteral } from '../../../../utils/source-value.ts'; +import type { DeviceInfo } from '../../../../kernel/device.ts'; +import type { ExecResult } from '../../../../utils/exec.ts'; +import { sendRunnerCommandOnce } from './runner-transport.ts'; +import { withRunnerCommandId } from './runner-contract.ts'; +import { + buildRunnerLease, + readStaleRunnerLease, + writeRunnerLease, + type RunnerLease, +} from './runner-lease.ts'; +import { + resolveExpectedRunnerCacheMetadata, + resolveRunnerDerivedPath, + type RunnerXctestrunArtifact, +} from './runner-xctestrun.ts'; +import { + buildRunnerSessionId, + normalizeRunnerStartupTimeoutMs, + type RunnerProcessHandle, + type RunnerSession, +} from './runner-session-types.ts'; + +// A healthy localhost runner answers uptime in tens of milliseconds and a dead +// port refuses immediately; the timeout only bounds the wedged-runner case, +// where giving up fast matters — the probe runs under the lease lock, in +// series before the restart it would otherwise avoid. +const RUNNER_ADOPTION_PROBE_TIMEOUT_MS = 500; +const RUNNER_ADOPTION_EXIT_POLL_INTERVAL_MS = 1_000; + +// Kill switch for the runner handoff across daemon restarts: disables both +// detaching healthy simulator runners on graceful shutdown and adopting them +// on the next startup. +export function isIosRunnerDetachEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + return parseBooleanLiteral(env.AGENT_DEVICE_IOS_RUNNER_DETACH ?? '') !== false; +} + +// Adopts a still-running runner left behind by a dead daemon (crash or +// graceful detach) instead of killing and restarting it: the lease must be +// stale, the xcodebuild process alive, the artifact fingerprint current, and +// the runner must answer an uptime probe. Any miss returns null and the +// normal cleanup-and-start path takes over. Must run under the runner lease +// lock, like the rest of session startup. +export async function tryAdoptRunnerSessionFromLease( + device: DeviceInfo, + options: { startupTimeoutMs?: number }, +): Promise { + if (device.kind !== 'simulator' || !isIosRunnerDetachEnabled()) return null; + // Custom simulator sets run behind the XCTestDevices redirect, whose + // symlink+lock lifetime is bound to the owning session and cannot be + // carried across daemons; scoped-set runners always restart fresh. + if (resolveIosSimulatorDeviceSetPath(device.simulatorSetPath)) return null; + const lease = readStaleRunnerLease(device.id); + if (!lease) return null; + + const skip = (reason: string): null => { + emitDiagnostic({ + level: 'debug', + phase: 'ios_runner_lease_adoption_skipped', + data: { deviceId: device.id, runnerPid: lease.runnerPid, port: lease.port, reason }, + }); + return null; + }; + + const runnerPid = lease.runnerPid; + if (!runnerPid) return skip('runner_pid_missing'); + if (!isProcessAlive(runnerPid)) return skip('runner_process_dead'); + const expectedDerived = resolveExpectedDerivedPath(device); + if (!expectedDerived) return skip('expected_derived_unresolved'); + if (!lease.xctestrunPath.startsWith(`${expectedDerived}${path.sep}`)) { + return skip('artifact_fingerprint_mismatch'); + } + if (!(await probeRunnerAnswersUptime(device, lease.port))) return skip('probe_failed'); + + const session = buildAdoptedRunnerSession(device, lease, runnerPid, expectedDerived, options); + try { + writeRunnerLease(session.lease); + } catch { + return null; + } + emitDiagnostic({ + level: 'info', + phase: 'ios_runner_lease_adopted', + data: { + deviceId: device.id, + sessionId: session.sessionId, + runnerPid, + port: lease.port, + previousOwnerPid: lease.ownerPid, + }, + }); + return session; +} + +async function probeRunnerAnswersUptime(device: DeviceInfo, port: number): Promise { + try { + const response = await sendRunnerCommandOnce( + device, + port, + withRunnerCommandId({ command: 'uptime' }), + RUNNER_ADOPTION_PROBE_TIMEOUT_MS, + ); + const payload = JSON.parse(await response.text()) as { ok?: unknown }; + return payload?.ok === true; + } catch { + return false; + } +} + +function resolveExpectedDerivedPath(device: DeviceInfo): string | null { + try { + return resolveRunnerDerivedPath(device, resolveExpectedRunnerCacheMetadata(device)); + } catch { + return null; + } +} + +function buildAdoptedRunnerSession( + device: DeviceInfo, + lease: RunnerLease, + runnerPid: number, + expectedDerived: string, + options: { startupTimeoutMs?: number }, +): RunnerSession & { lease: RunnerLease } { + const sessionId = buildRunnerSessionId(device.id, lease.port); + const artifact: RunnerXctestrunArtifact = { + xctestrunPath: lease.xctestrunPath, + derived: expectedDerived, + cache: 'exact', + artifact: 'valid', + buildMs: 0, + xctestrunPathSource: 'manifest', + reason: 'adopted_from_lease', + }; + const { child, wait } = watchDetachedRunnerProcess(runnerPid); + return { + sessionId, + device, + deviceId: device.id, + port: lease.port, + xctestrunPath: lease.xctestrunPath, + xctestrunArtifact: artifact, + jsonPath: lease.jsonPath, + testPromise: wait, + child, + // The probe already proved the runner answers commands. + ready: true, + startupTimeoutMs: normalizeRunnerStartupTimeoutMs(options.startupTimeoutMs), + lease: buildRunnerLease({ + deviceId: device.id, + sessionId, + runnerPid, + port: lease.port, + xctestrunPath: lease.xctestrunPath, + jsonPath: lease.jsonPath, + }), + }; +} + +// The adopted xcodebuild was spawned by a dead process, so there is no +// ChildProcess to hold — just a pid-backed RunnerProcessHandle. A +// low-frequency poll flips exitCode and settles testPromise when the process +// actually exits, which is what the transport's early-exit detection and +// disposal wait on. +function watchDetachedRunnerProcess(pid: number): { + child: RunnerProcessHandle; + wait: Promise; +} { + const child: RunnerProcessHandle = { pid, exitCode: null }; + const wait = new Promise((resolve) => { + const timer = setInterval(() => { + if (isProcessAlive(pid)) return; + clearInterval(timer); + child.exitCode = -1; + resolve({ stdout: '', stderr: '', exitCode: -1 }); + }, RUNNER_ADOPTION_EXIT_POLL_INTERVAL_MS); + timer.unref?.(); + }); + return { child, wait }; +} diff --git a/src/platforms/apple/core/runner/runner-client.ts b/src/platforms/apple/core/runner/runner-client.ts index ab5bdc8a58..d53f145680 100644 --- a/src/platforms/apple/core/runner/runner-client.ts +++ b/src/platforms/apple/core/runner/runner-client.ts @@ -185,6 +185,7 @@ export { } from './runner-xctestrun.ts'; export { + detachIosSimulatorRunnerSessionsForShutdown, getRunnerSessionSnapshot, stopIosRunnerSession, abortAllIosRunnerSessions, diff --git a/src/platforms/apple/core/runner/runner-disposal.ts b/src/platforms/apple/core/runner/runner-disposal.ts index e0848098a5..9cb4d84309 100644 --- a/src/platforms/apple/core/runner/runner-disposal.ts +++ b/src/platforms/apple/core/runner/runner-disposal.ts @@ -1,6 +1,7 @@ import { emitDiagnostic } from '../../../../utils/diagnostics.ts'; import type { DeviceInfo } from '../../../../kernel/device.ts'; import { isProcessAlive, isProcessGroupAlive } from '../../../../utils/process-identity.ts'; +import type { ExecBackgroundResult } from '../../../../utils/exec.ts'; import { cleanupTempFile, waitForRunner } from './runner-transport.ts'; import { withRunnerCommandId, type RunnerCommand } from './runner-contract.ts'; import { @@ -165,7 +166,7 @@ async function signalRunnerSessions( } async function signalRunnerPrepProcesses( - prepProcesses: readonly RunnerSession['child'][], + prepProcesses: readonly ExecBackgroundResult['child'][], signal: 'SIGINT' | 'SIGTERM' | 'SIGKILL', ): Promise { await Promise.allSettled( diff --git a/src/platforms/apple/core/runner/runner-lease.ts b/src/platforms/apple/core/runner/runner-lease.ts index 7d7f584a1e..cc1d07fdde 100644 --- a/src/platforms/apple/core/runner/runner-lease.ts +++ b/src/platforms/apple/core/runner/runner-lease.ts @@ -226,6 +226,22 @@ function shellQuote(value: string): string { return `'${value.replaceAll("'", "'\\''")}'`; } +// A lease whose owner process is gone but whose runner may still be running: +// the adoption path probes it instead of killing it. Detached leases (graceful +// daemon shutdown rewrote the token) classify as stale too once the owner pid +// dies, so crash-orphans and deliberate handoffs share one recovery path. +export function readStaleRunnerLease(deviceId: string): RunnerLease | null { + const state = classifyRunnerLease(readRunnerLease(deviceId)); + return state.type === 'stale' ? state.lease : null; +} + +// Marks a lease as handed off during graceful shutdown: the token no longer +// matches this daemon, so the shutdown's own lease-cleanup paths skip it, and +// once this process exits the lease classifies as stale for the next adopter. +export function buildDetachedRunnerLease(lease: RunnerLease): RunnerLease { + return { ...lease, ownerToken: `detached-${lease.ownerToken}` }; +} + export async function cleanupOwnedRunnerLease( deviceId: string, cleanup: RunnerLeaseCleanupAdapter, diff --git a/src/platforms/apple/core/runner/runner-session-types.ts b/src/platforms/apple/core/runner/runner-session-types.ts index e29eb9a213..725f893d00 100644 --- a/src/platforms/apple/core/runner/runner-session-types.ts +++ b/src/platforms/apple/core/runner/runner-session-types.ts @@ -1,9 +1,18 @@ import type { RunnerLogicalLeaseContext } from '../../../../core/runner-lease-context.ts'; -import type { ExecResult, ExecBackgroundResult } from '../../../../utils/exec.ts'; +import type { ExecResult } from '../../../../utils/exec.ts'; import type { DeviceInfo } from '../../../../kernel/device.ts'; import type { RunnerXctestrunArtifact } from './runner-xctestrun.ts'; import type { RunnerLease } from './runner-lease.ts'; +// The runner process seen through the session: pid for liveness/kill-tree and +// exitCode for early-exit detection. A spawned ChildProcess satisfies this +// structurally; adopted runners (whose spawner died) provide a pid-backed +// surrogate — which is why the session must not assume streams or kill() here. +export type RunnerProcessHandle = { + pid?: number | undefined; + exitCode: number | null; +}; + export type RunnerSession = { sessionId: string; device: DeviceInfo; @@ -13,7 +22,7 @@ export type RunnerSession = { xctestrunArtifact?: RunnerXctestrunArtifact; jsonPath: string; testPromise: Promise; - child: ExecBackgroundResult['child']; + child: RunnerProcessHandle; ready: boolean; startupTimeoutMs?: number; // Records the last allowlisted mutating interaction that the runner confirmed @@ -26,3 +35,13 @@ export type RunnerSession = { simulatorSetRedirect?: { release: () => Promise }; lease?: RunnerLease; }; + +export function buildRunnerSessionId(deviceId: string, port: number): string { + return `${deviceId}:${port}:${Date.now()}`; +} + +export function normalizeRunnerStartupTimeoutMs(value: number | undefined): number | undefined { + return typeof value === 'number' && Number.isFinite(value) && value > 0 + ? Math.floor(value) + : undefined; +} diff --git a/src/platforms/apple/core/runner/runner-session.ts b/src/platforms/apple/core/runner/runner-session.ts index 6a0d2373f3..f641ad1f05 100644 --- a/src/platforms/apple/core/runner/runner-session.ts +++ b/src/platforms/apple/core/runner/runner-session.ts @@ -38,12 +38,14 @@ import { isRunnerReadinessProbeCommand, } from './runner-command-traits.ts'; import { + buildDetachedRunnerLease, buildRunnerLease, prepareRunnerLeaseForStartup, RUNNER_OWNER_TOKEN, withRunnerLeaseLock, writeRunnerLease, } from './runner-lease.ts'; +import { isIosRunnerDetachEnabled, tryAdoptRunnerSessionFromLease } from './runner-adoption.ts'; import { abortRunnerSessionsAndPrepProcesses, cleanupOwnedIosRunnerLease, @@ -54,7 +56,11 @@ import { stopRunnerPrepProcesses, } from './runner-disposal.ts'; import { enrichRunnerFailureFromLog } from './runner-failure-diagnostics.ts'; -import type { RunnerSession } from './runner-session-types.ts'; +import { + buildRunnerSessionId, + normalizeRunnerStartupTimeoutMs, + type RunnerSession, +} from './runner-session-types.ts'; export type { RunnerSession } from './runner-session-types.ts'; @@ -126,6 +132,20 @@ async function startRunnerSessionWithLease( logicalLeaseContext, }, }); + const adopted = await measureRunnerStartupStep( + startupTimings, + 'adopt_detached_runner', + async () => + await tryAdoptRunnerSessionFromLease(device, { + startupTimeoutMs: options.startupTimeoutMs, + }), + ); + if (adopted) { + adopted.startupTimings = startupTimings; + adopted.logicalLeaseContext = logicalLeaseContext; + runnerSessions.set(device.id, adopted); + return adopted; + } await measureRunnerStartupStep(startupTimings, 'cleanup_stale_xcodebuild', async () => { await prepareRunnerLeaseForStartup(device.id, runnerLeaseCleanupAdapter, logicalLeaseContext); }); @@ -225,7 +245,7 @@ async function startRunnerSessionWithLease( logChunk(chunk, options.logPath, options.traceLogPath, options.verbose); }); - const sessionId = `${device.id}:${port}:${Date.now()}`; + const sessionId = buildRunnerSessionId(device.id, port); const lease = buildRunnerLease({ deviceId: device.id, sessionId, @@ -443,6 +463,46 @@ export async function abortAllIosRunnerSessions(): Promise { } } +// Graceful daemon shutdown hands healthy simulator runners off to the next +// daemon instead of paying the ~5s xcodebuild ramp again: the lease token is +// rewritten to a detached form (so this daemon's own teardown paths no longer +// classify it as owned) and the session simply leaves the in-memory map. Once +// this process exits the lease is stale and the adoption path picks it up. +// Explicit cleanup still works: clean:daemon kills by the lease's runnerPid, +// and the runner's XCTWaiter self-expires after 24h. +export async function detachIosSimulatorRunnerSessionsForShutdown(): Promise { + if (!isIosRunnerDetachEnabled()) return 0; + let detached = 0; + for (const [deviceId, session] of runnerSessions) { + if (session.device.kind !== 'simulator') continue; + // A held device-set redirect means this runner depends on the global + // XCTestDevices symlink pointing at a custom simulator set for its whole + // lifetime. Handing it off would either restore the symlink under a live + // runner or leak the redirect lock, so scoped-set runners keep the normal + // dispose-and-restore path. + if (session.simulatorSetRedirect) continue; + if (!session.lease || !isRunnerProcessAlive(session.child.pid)) continue; + try { + writeRunnerLease(buildDetachedRunnerLease(session.lease)); + } catch { + continue; // Could not mark the handoff; leave it for the kill path. + } + runnerSessions.delete(deviceId); + detached += 1; + emitDiagnostic({ + level: 'info', + phase: 'ios_runner_session_detached', + data: { + deviceId, + sessionId: session.sessionId, + runnerPid: session.child.pid, + port: session.port, + }, + }); + } + return detached; +} + export async function stopAllIosRunnerSessions(): Promise { await abortAllIosRunnerSessions(); const pending = Array.from(runnerSessions.keys()); @@ -880,12 +940,6 @@ export function readRunnerStartupTimeoutMs( return session.startupTimeoutMs ?? RUNNER_STARTUP_TIMEOUT_MS; } -function normalizeRunnerStartupTimeoutMs(value: number | undefined): number | undefined { - return typeof value === 'number' && Number.isFinite(value) && value > 0 - ? Math.floor(value) - : undefined; -} - async function measureRunnerStartupStep( timings: Record, phase: string, diff --git a/src/platforms/apple/core/simulator.ts b/src/platforms/apple/core/simulator.ts index 82ce13eadb..f1e1f89010 100644 --- a/src/platforms/apple/core/simulator.ts +++ b/src/platforms/apple/core/simulator.ts @@ -49,7 +49,12 @@ function readSimulatorBootedMemo(device: DeviceInfo): boolean { return true; } -function markSimulatorBooted(device: DeviceInfo): void { +// Also called by the device-inventory parser: a `simctl list` that reports a +// simulator Booted is the same observation ensureBootedSimulator would make, +// so resolving a device seeds the memo and the boot checks that follow in the +// same request cost nothing. Callers must only pass FRESH observations — +// seeding from a cached or persisted device listing would poison the memo. +export function markSimulatorBooted(device: DeviceInfo): void { simulatorBootedMemo.set(simulatorBootedMemoKey(device), Date.now()); } diff --git a/src/utils/source-value.ts b/src/utils/source-value.ts index 4a8c37cf58..b92a4b6128 100644 --- a/src/utils/source-value.ts +++ b/src/utils/source-value.ts @@ -108,7 +108,7 @@ function parseBooleanValue(value: unknown, sourceLabel: string, rawKey: string): ); } -function parseBooleanLiteral(value: string): boolean | undefined { +export function parseBooleanLiteral(value: string): boolean | undefined { const normalized = value.trim().toLowerCase(); if (BOOLEAN_TRUE_VALUES.has(normalized)) return true; if (BOOLEAN_FALSE_VALUES.has(normalized)) return false;