diff --git a/.changeset/serialize-event-claim-recovery.md b/.changeset/serialize-event-claim-recovery.md new file mode 100644 index 000000000..e6b8abba4 --- /dev/null +++ b/.changeset/serialize-event-claim-recovery.md @@ -0,0 +1,5 @@ +--- +"agent-bundle": patch +--- + +Serialize Linux event-runtime orphan claim reclamation behind an automatically released kernel gate, and reclaim claims owned by zombie processes. diff --git a/packages/agent-bundle/src/events/ipc.ts b/packages/agent-bundle/src/events/ipc.ts index 7fc75e365..c6f98ea86 100644 --- a/packages/agent-bundle/src/events/ipc.ts +++ b/packages/agent-bundle/src/events/ipc.ts @@ -231,6 +231,12 @@ type EndpointProbe = 'live' | 'missing' | 'stale'; export interface EventRuntimeServerTestHooks { readonly afterEndpointProbe?: (state: EndpointProbe) => Promise; + readonly afterEndpointClaimAcquired?: () => Promise; + readonly afterEndpointClaimReclamation?: (removed: boolean) => Promise; + readonly afterEndpointClaimReclamationSnapshot?: ( + identity: Readonly<{ readonly device: number; readonly inode: number }>, + ) => Promise; + readonly beforeEndpointClaimRemoval?: () => Promise; } interface EndpointClaim { @@ -249,6 +255,11 @@ interface EndpointClaimSnapshot { readonly owner: EndpointClaimOwner; } +interface LinuxProcessStat { + readonly startTime: string; + readonly state: string; +} + const endpointClaimOwnerSchema = z.object({ linuxStartTime: z.string().regex(/^\d+$/u).optional(), pid: z.number().int().positive(), @@ -292,16 +303,20 @@ const probeEndpoint = (endpoint: string): Effect.Effect => { +const linuxProcessStat = async (pid: number): Promise => { const processStat = await readFile(`/proc/${pid}/stat`, 'utf8'); const commEnd = processStat.lastIndexOf(')'); if (commEnd === -1) throw new Error(`Unable to parse process stat for pid ${pid}.`); const fieldsAfterComm = processStat.slice(commEnd + 1).trim().split(/\s+/u); + const state = fieldsAfterComm[0]; const startTime = fieldsAfterComm[19]; + if (state === undefined) throw new Error(`Process stat for pid ${pid} has no state.`); if (startTime === undefined) throw new Error(`Process stat for pid ${pid} has no start time.`); - return startTime; + return { startTime, state }; }; +const linuxProcessStartTime = async (pid: number): Promise => (await linuxProcessStat(pid)).startTime; + const currentEndpointClaimOwner = async (): Promise => ({ ...(process.platform === 'linux' ? { linuxStartTime: await linuxProcessStartTime(process.pid) } : {}), pid: process.pid, @@ -385,23 +400,136 @@ const isEndpointClaimOwnerProvablyDead = async (owner: EndpointClaimOwner): Prom } if (process.platform !== 'linux' || owner.linuxStartTime === undefined) return false; try { - return await linuxProcessStartTime(owner.pid) !== owner.linuxStartTime; + const processStat = await linuxProcessStat(owner.pid); + return processStat.state === 'Z' + || processStat.state === 'X' + || processStat.state === 'x' + || processStat.startTime !== owner.linuxStartTime; } catch { return false; } }; -const reclaimOrphanedEndpointClaim = (path: string): Effect.Effect => - liftPromise(async () => { - const snapshot = await readEndpointClaimSnapshot(path); - if (snapshot === 'missing') return true; - if (snapshot === undefined || !await isEndpointClaimOwnerProvablyDead(snapshot.owner)) return false; - try { - return await removeFileIfIdentityMatches(path, snapshot.identity); - } catch { - return false; +const endpointRecoveryGate = (endpoint: string): string => { + const hash = createHash('sha256').update(endpoint, 'utf8').digest('hex').slice(0, 32); + const user = typeof process.getuid === 'function' ? String(process.getuid()) : 'user'; + return `\0agent-bundle-${user}-event-recovery-${hash}`; +}; + +const tryAcquireEndpointRecoveryGate = ( + endpoint: string, +): Effect.Effect => + Effect.callback((resume) => { + const server = createServer(); + const cleanup = (): void => { + server.removeListener('error', onError); + server.removeListener('listening', onListening); + }; + const onError = (error: NodeJS.ErrnoException): void => { + cleanup(); + if (error.code === 'EADDRINUSE') { + resume(Effect.succeed(undefined)); + return; + } + resume(Effect.fail(transportError( + 'runtime-failed', + 'Unable to claim the event runtime recovery gate.', + error, + ))); + }; + const onListening = (): void => { + cleanup(); + resume(Effect.succeed(server)); + }; + server.once('error', onError); + server.once('listening', onListening); + server.listen(endpointRecoveryGate(endpoint)); + return Effect.sync(() => { + cleanup(); + if (server.listening) server.close(); + }); + }); + +const releaseEndpointRecoveryGate = (server: Server): Effect.Effect => + Effect.callback((resume) => { + if (!server.listening) { + resume(Effect.void); + return undefined; } - }).pipe(Effect.catch(() => Effect.succeed(false))); + server.close(() => resume(Effect.void)); + return undefined; + }).pipe(Effect.ignore); + +const reclaimOrphanedEndpointClaim = Effect.fnUntraced(function*( + endpoint: string, + testHooks?: EventRuntimeServerTestHooks, +): Effect.fn.Return { + const path = `${endpoint}.lock`; + const snapshot = yield* liftPromise(async () => { + const candidate = await readEndpointClaimSnapshot(path); + if (candidate === 'missing') return 'missing' as const; + if (candidate === undefined || !await isEndpointClaimOwnerProvablyDead(candidate.owner)) return undefined; + return candidate; + }).pipe(Effect.catch(() => Effect.succeed(undefined))); + if (snapshot === 'missing') return true; + if (snapshot === undefined) return false; + + const afterSnapshot = testHooks?.afterEndpointClaimReclamationSnapshot; + if (afterSnapshot !== undefined) { + yield* liftPromise(() => afterSnapshot(snapshot.identity)).pipe( + Effect.mapError((error) => transportError('runtime-failed', 'Event runtime claim snapshot hook failed.', error)), + ); + } + + let removed: boolean; + if (process.platform !== 'linux') { + // Other POSIX platforms retain the existing identity-checked fallback: + // Node exposes no auto-released filesystem gate, and a pathname gate can + // itself become an unrecoverable orphan. + removed = yield* liftPromise(() => removeFileIfIdentityMatches(path, snapshot.identity)).pipe( + Effect.catch(() => Effect.succeed(false)), + ); + } else { + // Abstract socket binds serialize every Linux reclamation vacancy and are + // released by the kernel on process death. A namespace squatter can only + // force bounded fail-closed retries, never concurrent claim ownership. + removed = yield* Effect.acquireUseRelease( + tryAcquireEndpointRecoveryGate(endpoint), + (gate) => { + if (gate === undefined) return Effect.succeed(false); + return Effect.gen(function*() { + const freshSnapshot = yield* liftPromise(async () => { + const candidate = await readEndpointClaimSnapshot(path); + if (candidate === 'missing') return 'missing' as const; + if (candidate === undefined || !await isEndpointClaimOwnerProvablyDead(candidate.owner)) return undefined; + return candidate; + }).pipe(Effect.catch(() => Effect.succeed(undefined))); + if (freshSnapshot === 'missing') return true; + if (freshSnapshot === undefined) return false; + + const beforeRemoval = testHooks?.beforeEndpointClaimRemoval; + if (beforeRemoval !== undefined) { + yield* liftPromise(beforeRemoval).pipe( + Effect.mapError((error) => transportError('runtime-failed', 'Event runtime claim removal hook failed.', error)), + ); + } + return yield* liftPromise(() => removeFileIfIdentityMatches(path, freshSnapshot.identity)).pipe( + Effect.catch(() => Effect.succeed(false)), + ); + }); + }, + (gate) => gate === undefined ? Effect.void : releaseEndpointRecoveryGate(gate), + ); + } + + const afterReclamation = testHooks?.afterEndpointClaimReclamation; + if (afterReclamation !== undefined) { + yield* liftPromise(() => afterReclamation(removed)).pipe( + Effect.mapError((error) => transportError('runtime-failed', 'Event runtime claim reclamation hook failed.', error)), + ); + } + return removed; +}); const releaseEndpointClaim = (claim: EndpointClaim): Effect.Effect => liftPromise(async () => { @@ -411,10 +539,19 @@ const releaseEndpointClaim = (claim: EndpointClaim): Effect.Effect => const claimEndpoint = Effect.fnUntraced(function*( endpoint: string, + testHooks?: EventRuntimeServerTestHooks, ): Effect.fn.Return { for (let attempt = 0; attempt < ENDPOINT_CLAIM_RETRY_COUNT; attempt += 1) { const claim = yield* tryClaimEndpoint(endpoint); - if (claim !== undefined) return claim; + if (claim !== undefined) { + const afterClaimAcquired = testHooks?.afterEndpointClaimAcquired; + if (afterClaimAcquired !== undefined) { + yield* liftPromise(afterClaimAcquired).pipe( + Effect.mapError((error) => transportError('runtime-failed', 'Event runtime claim hook failed.', error)), + ); + } + return claim; + } const endpointState = yield* probeEndpoint(endpoint); if (endpointState === 'live') { @@ -423,7 +560,7 @@ const claimEndpoint = Effect.fnUntraced(function*( 'Event runtime endpoint already has a live server.', )); } - if (yield* reclaimOrphanedEndpointClaim(`${endpoint}.lock`)) continue; + if (yield* reclaimOrphanedEndpointClaim(endpoint, testHooks)) continue; if (attempt + 1 < ENDPOINT_CLAIM_RETRY_COUNT) { yield* Effect.sleep(ENDPOINT_CLAIM_RETRY_DELAY); } @@ -517,7 +654,7 @@ const openServer = ( } return yield* Effect.acquireUseRelease( - claimEndpoint(endpoint), + claimEndpoint(endpoint, testHooks), () => Effect.gen(function*() { const claimedEndpointState = yield* probeEndpoint(endpoint); if (claimedEndpointState === 'live') { diff --git a/packages/agent-bundle/tests/event-ipc.test.ts b/packages/agent-bundle/tests/event-ipc.test.ts index c3319910d..590aa4992 100644 --- a/packages/agent-bundle/tests/event-ipc.test.ts +++ b/packages/agent-bundle/tests/event-ipc.test.ts @@ -3,6 +3,7 @@ import { once } from 'node:events'; import { mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { createConnection, type Socket } from 'node:net'; import { dirname } from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; import { Effect } from 'effect'; import { expect, it } from 'effect-rstest'; @@ -20,16 +21,25 @@ interface EndpointClaimOwner { readonly pid: number; } -const linuxProcessStartTime = async (pid: number): Promise => { +interface LinuxProcessStat { + readonly startTime: string; + readonly state: string; +} + +const linuxProcessStat = async (pid: number): Promise => { const processStat = await readFile(`/proc/${pid}/stat`, 'utf8'); const commEnd = processStat.lastIndexOf(')'); if (commEnd === -1) throw new Error(`Unable to parse process stat for pid ${pid}.`); const fieldsAfterComm = processStat.slice(commEnd + 1).trim().split(/\s+/u); + const state = fieldsAfterComm[0]; const startTime = fieldsAfterComm[19]; + if (state === undefined) throw new Error(`Process stat for pid ${pid} has no state.`); if (startTime === undefined) throw new Error(`Process stat for pid ${pid} has no start time.`); - return startTime; + return { startTime, state }; }; +const linuxProcessStartTime = async (pid: number): Promise => (await linuxProcessStat(pid)).startTime; + const currentProcessOwner = async (): Promise => ({ ...(process.platform === 'linux' ? { linuxStartTime: await linuxProcessStartTime(process.pid) } : {}), pid: process.pid, @@ -67,6 +77,37 @@ const deadChildOwner = async (): Promise => { return owner; }; +const spawnZombieOwner = async (): Promise> => { + const child = spawn('sh', ['-c', 'p=$$; { while [ "$(cat /proc/$p/comm 2>/dev/null)" != "sleep" ]; do :; done; } & echo $!; exec sleep 300'], { + stdio: ['ignore', 'pipe', 'ignore'], + }); + const pidLine = new Promise((resolve, reject) => { + child.stdout?.once('data', (chunk: Buffer) => resolve(chunk.toString('utf8').trim())); + child.stdout?.once('error', reject); + }); + await once(child, 'spawn'); + const zombiePid = Number(await pidLine); + if (!Number.isInteger(zombiePid) || zombiePid <= 0) { + await killChild(child); + throw new Error('Zombie child pid was not reported.'); + } + for (let attempt = 0; attempt < 200; attempt += 1) { + const processStat = await linuxProcessStat(zombiePid).catch(() => undefined); + if (processStat?.state === 'Z') { + return { + child, + owner: { linuxStartTime: processStat.startTime, pid: zombiePid }, + }; + } + await delay(10); + } + await killChild(child); + throw new Error(`Process ${zombiePid} did not become a zombie.`); +}; + const writeEndpointClaim = async (endpointId: string, contents: string): Promise => { const endpoint = eventRuntimeEndpoint(endpointId); await mkdir(dirname(endpoint), { mode: 0o700, recursive: true }); @@ -300,6 +341,262 @@ it.live('reclaims an endpoint claim whose owner process was killed', () => Effec yield* Effect.promise(() => rm(claimPath, { force: true })); })); +it.live('serializes concurrent reclamation before either contender can unlink a replacement claim', () => Effect.gen(function*() { + if (process.platform !== 'linux') return; + const endpointId = `event-ipc-reclaim-race-${crypto.randomUUID()}`; + const endpoint = yield* Effect.acquireUseRelease( + Effect.promise(() => createEventRuntimeServer({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => undefined, + })), + (server) => Effect.succeed(server.endpoint), + (server) => Effect.promise(() => server.close()), + ); + yield* Effect.promise(() => writeFile(endpoint, 'stale socket')); + const deadOwner = yield* Effect.promise(deadChildOwner); + const claimPath = yield* Effect.promise(() => writeEndpointClaim(endpointId, JSON.stringify(deadOwner))); + const originalClaim = yield* Effect.promise(() => stat(claimPath)); + + let firstSnapshotIdentity: Readonly<{ readonly device: number; readonly inode: number }> | undefined; + let markFirstSnapshot!: () => void; + let releaseFirstSnapshot!: () => void; + let markFirstClaimed!: () => void; + let releaseFirstClaim!: () => void; + const firstSnapshot = new Promise((resolve) => { markFirstSnapshot = resolve; }); + const firstCanReclaim = new Promise((resolve) => { releaseFirstSnapshot = resolve; }); + const firstClaimed = new Promise((resolve) => { markFirstClaimed = resolve; }); + const firstCanBind = new Promise((resolve) => { releaseFirstClaim = resolve; }); + const firstServer = createEventRuntimeServerForTest({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => ({ owner: 'first' }), + }, { + afterEndpointClaimAcquired: async () => { + markFirstClaimed(); + await firstCanBind; + }, + afterEndpointClaimReclamationSnapshot: async (identity) => { + firstSnapshotIdentity = identity; + markFirstSnapshot(); + await firstCanReclaim; + }, + }); + + let secondSnapshotIdentity: Readonly<{ readonly device: number; readonly inode: number }> | undefined; + let markSecondSnapshot!: () => void; + let releaseSecondSnapshot!: () => void; + let markSecondReclamation!: () => void; + let secondRemovedClaim: boolean | undefined; + const secondSnapshot = new Promise((resolve) => { markSecondSnapshot = resolve; }); + const secondCanReclaim = new Promise((resolve) => { releaseSecondSnapshot = resolve; }); + const secondReclamation = new Promise((resolve) => { markSecondReclamation = resolve; }); + const secondServer = createEventRuntimeServerForTest({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => ({ owner: 'second' }), + }, { + afterEndpointClaimReclamation: async (removed) => { + secondRemovedClaim = removed; + markSecondReclamation(); + }, + afterEndpointClaimReclamationSnapshot: async (identity) => { + secondSnapshotIdentity = identity; + markSecondSnapshot(); + await secondCanReclaim; + }, + }); + + yield* Effect.promise(() => Promise.all([firstSnapshot, secondSnapshot])); + expect(firstSnapshotIdentity).toEqual({ device: originalClaim.dev, inode: originalClaim.ino }); + expect(secondSnapshotIdentity).toEqual(firstSnapshotIdentity); + + releaseFirstSnapshot(); + yield* Effect.promise(() => firstClaimed); + const replacementBeforeSecond = yield* Effect.promise(() => readFile(claimPath, 'utf8')); + expect(JSON.parse(replacementBeforeSecond)).toEqual(yield* Effect.promise(currentProcessOwner)); + + releaseSecondSnapshot(); + yield* Effect.promise(() => secondReclamation); + expect(secondRemovedClaim).toBe(false); + const replacementAfterSecond = yield* Effect.promise(() => readFile(claimPath, 'utf8')); + expect(replacementAfterSecond).toBe(replacementBeforeSecond); + + releaseFirstClaim(); + const winner = yield* Effect.acquireRelease( + Effect.promise(() => firstServer), + (server) => Effect.promise(() => server.close()), + ); + const loser = yield* Effect.promise(async () => { + try { + return { server: await secondServer, status: 'opened' as const }; + } catch (error) { + return { error, status: 'failed' as const }; + } + }); + if (loser.status === 'opened') { + yield* Effect.promise(() => loser.server.close()); + expect(loser.status).toBe('failed'); + return; + } + expect(loser.error).toMatchObject({ + code: 'runtime-failed', + message: expect.stringMatching(/already has a live server/u), + name: EventRuntimeTransportError.name, + }); + const response = yield* Effect.promise(() => requestEventRuntime({ + artifactEpoch: 'epoch-1', + endpointId, + event: 'session/start', + hostContractRevision: '2.1.250', + native: { hook_event_name: 'SessionStart' }, + signal: new AbortController().signal, + target: 'claude', + timeoutMs: 1_000, + })); + expect(response).toEqual({ owner: 'first' }); + expect(winner.endpoint).toBe(endpoint); +})); + +it.live('excludes a second orphan-claim remover while the recovery gate is held', () => Effect.gen(function*() { + if (process.platform !== 'linux') return; + const endpointId = `event-ipc-recovery-gate-${crypto.randomUUID()}`; + const endpoint = yield* Effect.acquireUseRelease( + Effect.promise(() => createEventRuntimeServer({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => undefined, + })), + (server) => Effect.succeed(server.endpoint), + (server) => Effect.promise(() => server.close()), + ); + yield* Effect.promise(() => writeFile(endpoint, 'stale socket')); + const deadOwner = yield* Effect.promise(deadChildOwner); + const claimPath = yield* Effect.promise(() => writeEndpointClaim(endpointId, JSON.stringify(deadOwner))); + const originalClaim = yield* Effect.promise(() => stat(claimPath)); + + let markFirstSnapshot!: () => void; + let releaseFirstSnapshot!: () => void; + let markFirstInsideGate!: () => void; + let releaseFirstInsideGate!: () => void; + const firstSnapshot = new Promise((resolve) => { markFirstSnapshot = resolve; }); + const firstCanEnterGate = new Promise((resolve) => { releaseFirstSnapshot = resolve; }); + const firstInsideGate = new Promise((resolve) => { markFirstInsideGate = resolve; }); + const firstCanRemove = new Promise((resolve) => { releaseFirstInsideGate = resolve; }); + const firstServer = createEventRuntimeServerForTest({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => ({ owner: 'first' }), + }, { + afterEndpointClaimReclamationSnapshot: async () => { + markFirstSnapshot(); + await firstCanEnterGate; + }, + beforeEndpointClaimRemoval: async () => { + markFirstInsideGate(); + await firstCanRemove; + }, + }); + + let markSecondSnapshot!: () => void; + let releaseSecondSnapshot!: () => void; + let markSecondReclamation!: () => void; + let releaseSecondReclamation!: () => void; + const secondRemovalResults: boolean[] = []; + const secondSnapshot = new Promise((resolve) => { markSecondSnapshot = resolve; }); + const secondCanAttemptReclamation = new Promise((resolve) => { releaseSecondSnapshot = resolve; }); + const secondReclamation = new Promise((resolve) => { markSecondReclamation = resolve; }); + const secondCanRetry = new Promise((resolve) => { releaseSecondReclamation = resolve; }); + const secondServer = createEventRuntimeServerForTest({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => ({ owner: 'second' }), + }, { + afterEndpointClaimReclamation: async (removed) => { + secondRemovalResults.push(removed); + markSecondReclamation(); + await secondCanRetry; + }, + afterEndpointClaimReclamationSnapshot: async () => { + markSecondSnapshot(); + await secondCanAttemptReclamation; + }, + }); + + yield* Effect.promise(() => Promise.all([firstSnapshot, secondSnapshot])); + releaseFirstSnapshot(); + yield* Effect.promise(() => firstInsideGate); + + releaseSecondSnapshot(); + yield* Effect.promise(() => secondReclamation); + expect(secondRemovalResults).toEqual([false]); + const claimWhileGateHeld = yield* Effect.promise(() => stat(claimPath)); + expect({ + device: claimWhileGateHeld.dev, + inode: claimWhileGateHeld.ino, + }).toEqual({ + device: originalClaim.dev, + inode: originalClaim.ino, + }); + + releaseFirstInsideGate(); + const winner = yield* Effect.acquireRelease( + Effect.promise(() => firstServer), + (server) => Effect.promise(() => server.close()), + ); + releaseSecondReclamation(); + const loser = yield* Effect.promise(async () => { + try { + return { server: await secondServer, status: 'opened' as const }; + } catch (error) { + return { error, status: 'failed' as const }; + } + }); + if (loser.status === 'opened') { + yield* Effect.promise(() => loser.server.close()); + expect(loser.status).toBe('failed'); + return; + } + expect(loser.error).toMatchObject({ + code: 'runtime-failed', + message: expect.stringMatching(/already has a live server/u), + name: EventRuntimeTransportError.name, + }); + const response = yield* Effect.promise(() => requestEventRuntime({ + artifactEpoch: 'epoch-1', + endpointId, + event: 'session/start', + hostContractRevision: '2.1.250', + native: { hook_event_name: 'SessionStart' }, + signal: new AbortController().signal, + target: 'claude', + timeoutMs: 1_000, + })); + expect(response).toEqual({ owner: 'first' }); + expect(winner.endpoint).toBe(endpoint); +})); + +it.live('reclaims an endpoint claim owned by a zombie process', () => Effect.gen(function*() { + if (process.platform !== 'linux') return; + const endpointId = `event-ipc-zombie-claim-${crypto.randomUUID()}`; + const { child, owner } = yield* Effect.acquireRelease( + Effect.promise(spawnZombieOwner), + ({ child }) => Effect.promise(() => killChild(child)), + ); + const claimPath = yield* Effect.promise(() => writeEndpointClaim(endpointId, JSON.stringify(owner))); + const server = yield* Effect.acquireRelease( + Effect.promise(() => createEventRuntimeServer({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => undefined, + })), + (server) => Effect.promise(() => server.close()), + ); + expect(server.endpoint).toBe(eventRuntimeEndpoint(endpointId)); + expect(child.exitCode).toBeNull(); + yield* Effect.promise(() => rm(claimPath, { force: true })); +})); + it.live('fails closed when an endpoint claim owner is still alive', () => Effect.gen(function*() { if (process.platform === 'win32') return; const endpointId = `event-ipc-live-claim-${crypto.randomUUID()}`;