diff --git a/packages/contracts/package.json b/packages/contracts/package.json index 9032f1ee18..5d53316bab 100644 --- a/packages/contracts/package.json +++ b/packages/contracts/package.json @@ -55,6 +55,14 @@ "./replay": { "types": "./src/facades/replay.ts", "default": "./src/facades/replay.ts" + }, + "./divergence": { + "types": "./src/facades/divergence.ts", + "default": "./src/facades/divergence.ts" + }, + "./progress": { + "types": "./src/facades/progress.ts", + "default": "./src/facades/progress.ts" } } } diff --git a/packages/contracts/src/facades/divergence.ts b/packages/contracts/src/facades/divergence.ts new file mode 100644 index 0000000000..86f161b8ff --- /dev/null +++ b/packages/contracts/src/facades/divergence.ts @@ -0,0 +1 @@ +export * from '../replay-divergence.ts'; diff --git a/packages/contracts/src/facades/progress.ts b/packages/contracts/src/facades/progress.ts new file mode 100644 index 0000000000..b2575ad43e --- /dev/null +++ b/packages/contracts/src/facades/progress.ts @@ -0,0 +1 @@ +export * from '../request-progress.ts'; diff --git a/src/replay/__tests__/divergence.test.ts b/packages/contracts/src/replay-divergence.test.ts similarity index 95% rename from src/replay/__tests__/divergence.test.ts rename to packages/contracts/src/replay-divergence.test.ts index b8c5f81eca..4d38a6bdfe 100644 --- a/src/replay/__tests__/divergence.test.ts +++ b/packages/contracts/src/replay-divergence.test.ts @@ -11,7 +11,7 @@ import { REPLAY_DIVERGENCE_SUGGESTION_LIMIT, truncateUtf8Field, type ReplayDivergence, -} from '../divergence.ts'; +} from './replay-divergence.ts'; function buildDivergence(overrides: Partial = {}): ReplayDivergence { return { @@ -239,7 +239,7 @@ test('sanitizeReplayDivergenceField redacts sensitive content even when no trunc // --- Text report carries the repair data (bounded refs + unavailable hint) --- test('formatReplayDivergenceReport lists a bounded ref/role/label subset for an available screen', async () => { - const { formatReplayDivergenceReport } = await import('../divergence.ts'); + const { formatReplayDivergenceReport } = await import('./replay-divergence.ts'); const report = formatReplayDivergenceReport({ divergence: { version: 1, @@ -272,7 +272,7 @@ test('formatReplayDivergenceReport lists a bounded ref/role/label subset for an }); test('formatReplayDivergenceReport carries the unavailable-screen hint', async () => { - const { formatReplayDivergenceReport } = await import('../divergence.ts'); + const { formatReplayDivergenceReport } = await import('./replay-divergence.ts'); const report = formatReplayDivergenceReport({ divergence: { version: 1, @@ -295,7 +295,7 @@ test('formatReplayDivergenceReport carries the unavailable-screen hint', async ( }); test('scrubReplayVarValues replaces every occurrence with a named marker, longest value first', async () => { - const { scrubReplayVarValues } = await import('../divergence.ts'); + const { scrubReplayVarValues } = await import('./replay-divergence.ts'); const entries = [ { name: 'LONG', value: 'abc-def' }, { name: 'SHORT', value: 'abc' }, @@ -414,7 +414,7 @@ test('boundReplayDivergence keeps targetBinding on the minimal overflow fallback }); test('formatReplayDivergenceReport renders matchCount, mismatches, and candidates for a target-binding divergence', async () => { - const { formatReplayDivergenceReport } = await import('../divergence.ts'); + const { formatReplayDivergenceReport } = await import('./replay-divergence.ts'); const report = formatReplayDivergenceReport({ divergence: { version: 1, @@ -442,7 +442,7 @@ test('formatReplayDivergenceReport renders matchCount, mismatches, and candidate }); test('formatReplayDivergenceReport lists candidates for an identity-unverifiable target-binding divergence', async () => { - const { formatReplayDivergenceReport } = await import('../divergence.ts'); + const { formatReplayDivergenceReport } = await import('./replay-divergence.ts'); const report = formatReplayDivergenceReport({ divergence: { version: 1, @@ -477,7 +477,7 @@ test('formatReplayDivergenceReport lists candidates for an identity-unverifiable // refused — it surfaces `resume.reason` instead. --- test('formatReplayDivergenceReport embeds the concrete resume command for an allowed record-and-heal divergence', async () => { - const { formatReplayDivergenceReport } = await import('../divergence.ts'); + const { formatReplayDivergenceReport } = await import('./replay-divergence.ts'); const report = formatReplayDivergenceReport({ divergence: { version: 1, @@ -502,7 +502,7 @@ test('formatReplayDivergenceReport embeds the concrete resume command for an all }); test('formatReplayDivergenceReport never renders a --from command when resume is NOT allowed, surfacing the reason instead', async () => { - const { formatReplayDivergenceReport } = await import('../divergence.ts'); + const { formatReplayDivergenceReport } = await import('./replay-divergence.ts'); const report = formatReplayDivergenceReport({ divergence: { version: 1, @@ -532,7 +532,7 @@ test('formatReplayDivergenceReport never renders a --from command when resume is }); test('formatReplayDivergenceReport falls back to a generic non-resumable sentence when resume carries no reason', async () => { - const { formatReplayDivergenceReport } = await import('../divergence.ts'); + const { formatReplayDivergenceReport } = await import('./replay-divergence.ts'); const report = formatReplayDivergenceReport({ divergence: { version: 1, @@ -561,7 +561,7 @@ test('formatReplayDivergenceReport falls back to a generic non-resumable sentenc // NEVER re-derives resumability, so text and structured wire never disagree. --- test('formatReplayDivergenceReport embeds BOTH concrete resume commands for a caution divergence whose wire carries alternateFrom', async () => { - const { formatReplayDivergenceReport } = await import('../divergence.ts'); + const { formatReplayDivergenceReport } = await import('./replay-divergence.ts'); const report = formatReplayDivergenceReport({ divergence: { version: 1, @@ -594,7 +594,7 @@ test('formatReplayDivergenceReport embeds BOTH concrete resume commands for a ca }); test('formatReplayDivergenceReport embeds BOTH concrete resume commands for a manual divergence whose wire carries alternateFrom', async () => { - const { formatReplayDivergenceReport } = await import('../divergence.ts'); + const { formatReplayDivergenceReport } = await import('./replay-divergence.ts'); const report = formatReplayDivergenceReport({ divergence: { version: 1, @@ -622,7 +622,7 @@ test('formatReplayDivergenceReport embeds BOTH concrete resume commands for a ma }); test('formatReplayDivergenceReport renders ONLY the state-fix command for a caution divergence WITHOUT alternateFrom (the diverged step is not skip-safe)', async () => { - const { formatReplayDivergenceReport } = await import('../divergence.ts'); + const { formatReplayDivergenceReport } = await import('./replay-divergence.ts'); // resume.allowed is true (resuming AT N is fine), but the diverged step N // is a runScript/control-flow action, so `--from N + 1` would be refused — // the daemon omits alternateFrom, and the text must NOT offer `--from N + 1`. @@ -654,7 +654,7 @@ test('formatReplayDivergenceReport renders ONLY the state-fix command for a caut }); test('formatReplayDivergenceReport renders neither caution command when resume is NOT allowed', async () => { - const { formatReplayDivergenceReport } = await import('../divergence.ts'); + const { formatReplayDivergenceReport } = await import('./replay-divergence.ts'); const report = formatReplayDivergenceReport({ divergence: { version: 1, @@ -695,7 +695,7 @@ const REPAIR_DIAGNOSTICS_CLAUSE_PATTERN = /Read-only inspection while armed \(snapshot -i, get attrs, find, is\) is excluded from the healed script by default — no --no-record needed\. If the step you are repairing is itself a read, add --record to that command so it lands in the heal\./; test('formatReplayDivergenceReport appends the diagnostics default-exclusion clause for record-and-heal when repairSessionHeld is true', async () => { - const { formatReplayDivergenceReport } = await import('../divergence.ts'); + const { formatReplayDivergenceReport } = await import('./replay-divergence.ts'); const report = formatReplayDivergenceReport({ divergence: { version: 1, @@ -718,7 +718,7 @@ test('formatReplayDivergenceReport appends the diagnostics default-exclusion cla }); test('formatReplayDivergenceReport OMITS the diagnostics clause for record-and-heal when repairSessionHeld is absent (plain, non-repair divergence)', async () => { - const { formatReplayDivergenceReport } = await import('../divergence.ts'); + const { formatReplayDivergenceReport } = await import('./replay-divergence.ts'); const report = formatReplayDivergenceReport({ divergence: { version: 1, @@ -741,7 +741,7 @@ test('formatReplayDivergenceReport OMITS the diagnostics clause for record-and-h }); test('formatReplayDivergenceReport appends the diagnostics clause for state-repair when armed, distinct from the existing app-state --no-record clause', async () => { - const { formatReplayDivergenceReport } = await import('../divergence.ts'); + const { formatReplayDivergenceReport } = await import('./replay-divergence.ts'); const report = formatReplayDivergenceReport({ divergence: { version: 1, @@ -765,7 +765,7 @@ test('formatReplayDivergenceReport appends the diagnostics clause for state-repa }); test('formatReplayDivergenceReport OMITS the diagnostics clause for state-repair when repairSessionHeld is absent', async () => { - const { formatReplayDivergenceReport } = await import('../divergence.ts'); + const { formatReplayDivergenceReport } = await import('./replay-divergence.ts'); const report = formatReplayDivergenceReport({ divergence: { version: 1, @@ -786,7 +786,7 @@ test('formatReplayDivergenceReport OMITS the diagnostics clause for state-repair }); test('formatReplayDivergenceReport appends the diagnostics clause for caution when armed, alongside the dual-path resume commands', async () => { - const { formatReplayDivergenceReport } = await import('../divergence.ts'); + const { formatReplayDivergenceReport } = await import('./replay-divergence.ts'); const report = formatReplayDivergenceReport({ divergence: { version: 1, @@ -816,7 +816,7 @@ test('formatReplayDivergenceReport appends the diagnostics clause for caution wh }); test('formatReplayDivergenceReport OMITS the diagnostics clause for caution when repairSessionHeld is absent', async () => { - const { formatReplayDivergenceReport } = await import('../divergence.ts'); + const { formatReplayDivergenceReport } = await import('./replay-divergence.ts'); const report = formatReplayDivergenceReport({ divergence: { version: 1, @@ -836,7 +836,7 @@ test('formatReplayDivergenceReport OMITS the diagnostics clause for caution when }); test('formatReplayDivergenceReport appends the diagnostics clause for manual when armed, even when resume is NOT allowed', async () => { - const { formatReplayDivergenceReport } = await import('../divergence.ts'); + const { formatReplayDivergenceReport } = await import('./replay-divergence.ts'); // `repairSessionHeld` reports the daemon KEPT THE SESSION LIVE, independent // of `resume.allowed` (plan-resumability) — the diagnostics clause must // still render here: the agent may still inspect the (held) session while @@ -868,7 +868,7 @@ test('formatReplayDivergenceReport appends the diagnostics clause for manual whe }); test('formatReplayDivergenceReport OMITS the diagnostics clause for manual when repairSessionHeld is absent', async () => { - const { formatReplayDivergenceReport } = await import('../divergence.ts'); + const { formatReplayDivergenceReport } = await import('./replay-divergence.ts'); const report = formatReplayDivergenceReport({ divergence: { version: 1, diff --git a/src/replay/divergence.ts b/packages/contracts/src/replay-divergence.ts similarity index 100% rename from src/replay/divergence.ts rename to packages/contracts/src/replay-divergence.ts diff --git a/packages/contracts/src/request-progress.ts b/packages/contracts/src/request-progress.ts new file mode 100644 index 0000000000..6c4ec3e4a2 --- /dev/null +++ b/packages/contracts/src/request-progress.ts @@ -0,0 +1,58 @@ +/** + * Progress events the daemon streams to a connected client while a request runs. + * + * These are wire values: `src/daemon/request-progress-protocol.ts` serializes them onto the + * response stream and the CLI reconstructs them before handing them to the replay-test + * reporter registry. They therefore belong to neither side — the daemon's request-global + * progress plumbing carries them, and the replay-test scheduler produces and interprets + * them, so the vocabulary sits below both. + */ + +export type ReplayTestSuiteProgressEvent = { + type: 'replay-test-suite'; + status: 'start'; + total: number; + runnable: number; + skipped: number; + artifactsDir: string; + shardMode?: 'all' | 'split'; + shardCount?: number; +}; + +export type ReplayTestProgressEvent = { + type: 'replay-test'; + file: string; + title?: string; + status: 'start' | 'progress' | 'pass' | 'fail' | 'skip'; + index: number; + total: number; + stepIndex?: number; + stepTotal?: number; + stepCommand?: string; + stepValue?: string; + attempt?: number; + maxAttempts?: number; + durationMs?: number; + retrying?: boolean; + message?: string; + hint?: string; + session?: string; + artifactsDir?: string; + shardIndex?: number; + shardCount?: number; + deviceId?: string; + deviceName?: string; +}; + +export type CommandProgressEvent = { + type: 'command'; + status: 'progress'; + message: string; +}; + +export type RequestProgressEvent = + | ReplayTestSuiteProgressEvent + | ReplayTestProgressEvent + | CommandProgressEvent; + +export type RequestProgressSink = (event: RequestProgressEvent) => void; diff --git a/scripts/layering/daemon-modularity.test.ts b/scripts/layering/daemon-modularity.test.ts index 27dca1ffcd..563dc49bf4 100644 --- a/scripts/layering/daemon-modularity.test.ts +++ b/scripts/layering/daemon-modularity.test.ts @@ -128,21 +128,20 @@ test('replay-test may still import its own files inside the wider replay engine assert.deepEqual(checkDaemonModularityRatchets([...baselineEdges(), ...edges], []), []); }); -test('recorded replay-test migration imports are exempt until the import is deleted', () => { - const recorded = LOGICAL_MODULE_POLICIES.find( - ({ name }) => name === 'replay-test', - )?.recordedMigrationImports; - assert.deepEqual(recorded, [ - 'src/replay/test/reporters/default.ts -> src/replay/divergence.ts', - 'src/replay/test/reporters/progress.ts -> src/request/progress.ts', - 'src/replay/test/reporters/registry.ts -> src/request/progress.ts', - 'src/replay/test/reporting.ts -> src/request/progress.ts', - ]); +// #1478 P3 cleared every recorded replay-test migration import: the ADR 0012 divergence +// vocabulary became a neutral contracts leaf, and the reporter tree now reads the progress +// wire vocabulary from contracts instead of request-global plumbing. The rule enforces +// unconditionally for replay-test from here on. +test('replay-test carries no recorded migration imports', () => { + assert.equal( + LOGICAL_MODULE_POLICIES.find(({ name }) => name === 'replay-test')?.recordedMigrationImports, + undefined, + ); + assert.deepEqual( + LOGICAL_MODULE_POLICIES.flatMap((module) => module.recordedMigrationImports ?? []), + [], + ); assert.deepEqual(checkDaemonModularityRatchets(baselineEdges(), []), []); - - const withoutOne = checkDaemonModularityRatchets(baselineEdges().slice(0, -1), []); - assert.equal(withoutOne.length, 1); - assert.match(withoutOne[0]!.message, /delete it from replay-test's recordedMigrationImports/); }); test('internal trees reject deep imports globally, including from daemon', () => { diff --git a/scripts/layering/daemon-modularity.ts b/scripts/layering/daemon-modularity.ts index d582a3eba1..0b129348fe 100644 --- a/scripts/layering/daemon-modularity.ts +++ b/scripts/layering/daemon-modularity.ts @@ -83,12 +83,6 @@ export const LOGICAL_MODULE_POLICIES: readonly LogicalModulePolicy[] = [ 'src/maestro/', 'src/ad-replay/', ], - recordedMigrationImports: [ - 'src/replay/test/reporters/default.ts -> src/replay/divergence.ts', - 'src/replay/test/reporters/progress.ts -> src/request/progress.ts', - 'src/replay/test/reporters/registry.ts -> src/request/progress.ts', - 'src/replay/test/reporting.ts -> src/request/progress.ts', - ], }, ]; diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index e99e2943b1..39eeb107e5 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -40,9 +40,11 @@ const CONTRACT_EXPORTS = [ '@agent-device/contracts/client', '@agent-device/contracts/command', '@agent-device/contracts/device', + '@agent-device/contracts/divergence', '@agent-device/contracts/interaction', '@agent-device/contracts/observability', '@agent-device/contracts/platform', + '@agent-device/contracts/progress', '@agent-device/contracts/recording', '@agent-device/contracts/remote', '@agent-device/contracts/replay', diff --git a/src/daemon/handlers/__tests__/session-replay-divergence-publication.test.ts b/src/daemon/handlers/__tests__/session-replay-divergence-publication.test.ts index 8edef1d926..696b4d382b 100644 --- a/src/daemon/handlers/__tests__/session-replay-divergence-publication.test.ts +++ b/src/daemon/handlers/__tests__/session-replay-divergence-publication.test.ts @@ -11,7 +11,7 @@ vi.mock('../../../core/dispatch.ts', async (importOriginal) => { import { dispatchCommand } from '../../../core/dispatch.ts'; import { makeIosSession } from '../../../__tests__/test-utils/session-factories.ts'; import type { SnapshotState } from '@agent-device/kernel/snapshot'; -import type { ReplayDivergence } from '../../../replay/divergence.ts'; +import type { ReplayDivergence } from '@agent-device/contracts/divergence'; import { bindInternalObservationAuthority } from '../../internal-observation.ts'; import { expireRefFrame } from '../../ref-frame.ts'; import { markSessionPartialRefsIssued, setSessionSnapshot } from '../../session-snapshot.ts'; diff --git a/src/daemon/handlers/__tests__/session-replay-runtime-failure.test.ts b/src/daemon/handlers/__tests__/session-replay-runtime-failure.test.ts index dc19a3869f..3fafe51b1d 100644 --- a/src/daemon/handlers/__tests__/session-replay-runtime-failure.test.ts +++ b/src/daemon/handlers/__tests__/session-replay-runtime-failure.test.ts @@ -12,7 +12,7 @@ import { SessionStore } from '../../session-store.ts'; import type { DaemonResponse } from '../../types.ts'; import { dispatchCommand } from '../../../core/dispatch.ts'; import { makeIosSession } from '../../../__tests__/test-utils/session-factories.ts'; -import { formatReplayDivergenceReport } from '../../../replay/divergence.ts'; +import { formatReplayDivergenceReport } from '@agent-device/contracts/divergence'; import { baseReplayRequest as baseReq, writeReplayFile, diff --git a/src/daemon/handlers/__tests__/session-test-artifacts.test.ts b/src/daemon/handlers/__tests__/session-test-artifacts.test.ts index 14482e4036..36beb140eb 100644 --- a/src/daemon/handlers/__tests__/session-test-artifacts.test.ts +++ b/src/daemon/handlers/__tests__/session-test-artifacts.test.ts @@ -7,6 +7,7 @@ import { materializeReplayTestAttemptArtifacts, prepareReplayTestAttemptArtifacts, } from '../session-test-artifacts.ts'; +import { toReplayTestAttemptOutcome } from '../session-test-outcome.ts'; import type { DaemonResponse } from '../../types.ts'; test('materializeReplayTestAttemptArtifacts writes replay and result manifests for passing attempts', () => { @@ -27,7 +28,7 @@ test('materializeReplayTestAttemptArtifacts writes replay and result manifests f }, }; materializeReplayTestAttemptArtifacts({ - response, + outcome: toReplayTestAttemptOutcome(response), filePath: replayPath, sessionName: 'default:test:suite:1', attempts: 1, @@ -83,7 +84,7 @@ test('materializeReplayTestAttemptArtifacts writes failure manifest and copies l }, }; materializeReplayTestAttemptArtifacts({ - response, + outcome: toReplayTestAttemptOutcome(response), filePath: replayPath, sessionName: 'default:test:suite:2', attempts: 2, diff --git a/src/daemon/handlers/__tests__/session-test-reporter-values-maestro.test.ts b/src/daemon/handlers/__tests__/session-test-reporter-values-maestro.test.ts new file mode 100644 index 0000000000..16e0db2bcc --- /dev/null +++ b/src/daemon/handlers/__tests__/session-test-reporter-values-maestro.test.ts @@ -0,0 +1,286 @@ +// Characterization of the shipped reporter contract for the MAESTRO adapter (#1478 P3). +// +// The sibling session-test-reporter-values.test.ts covers the same contract for native `.ad`. +// Both are needed, and neither substitutes for the other: P3 replaced a request-global +// AsyncLocalStorage with an explicit per-attempt `onStep` port, and that port has two +// independent forwarding chains. The native chain is +// scheduler sink -> runReplayScriptFile -> executeReplayActions -> onStep?.(...) +// and the Maestro chain is a different set of links entirely: +// scheduler sink -> runReplayScriptFile -> runTypedMaestroReplayFile +// -> createMaestroReplayObserver({ onStep }) -> actionStarted -> onStep?.(...) +// +// An `.ad`-only reporter test stays green if any Maestro link is deleted, so `onTestStep` +// would silently stop firing for every `test --maestro` run with no gate objecting. That is +// the same defect class as the dropped diagnosticId/logPath (#1501) and the dropped reporter +// `hint` (#1505): a ratchet that keeps passing while a shipped value disappears. +// +// These tests document today's behavior; they do not propose behavior. + +// Maestro replay resolves a target device and captures view hierarchies through +// core/dispatch. These fixtures model no runner, so both are stubbed: the device resolves to a +// fixed Android emulator, and `snapshot` returns an empty hierarchy (which is what makes +// `assertNotVisible` pass deterministically without a screen). +import { expect, test, vi } from 'vitest'; + +vi.mock('../../../core/dispatch.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + resolveTargetDevice: vi.fn(async () => ({ + platform: 'android', + id: 'emulator-5554', + name: 'Pixel 8', + kind: 'emulator', + booted: true, + })), + dispatchCommand: vi.fn(async () => { + throw new Error('no device runner available in this test'); + }), + dispatchGestureViewport: vi.fn(async () => ({ x: 0, y: 0, width: 400, height: 800 })), + }; +}); + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import type { ReplaySuiteResult } from '@agent-device/contracts/replay'; +import { handleSessionCommands } from '../session.ts'; +import { SessionStore } from '../../session-store.ts'; +import type { DaemonRequest, DaemonResponse } from '../../types.ts'; +import { withRequestProgressSink } from '../../../request/progress.ts'; +import { + getReplayTestReporterExitCode, + runReplayTestReporterProgress, + runReplayTestReporters, +} from '../../../replay/test/reporters/registry.ts'; +import type { + ReplayTestReporter, + ReplayTestReporterContext, +} from '../../../replay/test/reporters/types.ts'; + +type RecordedHook = { + hook: 'onSuiteStart' | 'onTestStart' | 'onTestStep' | 'onTestResult' | 'onSuiteEnd'; + value: Record; +}; + +const reporterContext: ReplayTestReporterContext = { + stdout: { isTTY: false, write() {} }, + stderr: { isTTY: false, write() {} }, +}; + +/** + * Two steps, so `stepIndex`/`stepTotal` are distinguishable and one step carries a value while + * the other does not. The flow `name` is what populates the reporter `title` — a value only the + * Maestro path can produce, since `.ad` scripts have no title. + */ +const MAESTRO_FLOW = [ + 'appId: demo.app', + 'name: Login flow', + '---', + '- assertNotVisible: "Nope"', + '- back', + '', +].join('\n'); + +function createRecordingReporter(): { reporter: ReplayTestReporter; hooks: RecordedHook[] } { + const hooks: RecordedHook[] = []; + return { + hooks, + reporter: { + name: 'recording', + onSuiteStart: (suite) => void hooks.push({ hook: 'onSuiteStart', value: { ...suite } }), + onTestStart: (testCase) => void hooks.push({ hook: 'onTestStart', value: { ...testCase } }), + onTestStep: (testCase) => void hooks.push({ hook: 'onTestStep', value: { ...testCase } }), + onTestResult: (testCase) => void hooks.push({ hook: 'onTestResult', value: { ...testCase } }), + onSuiteEnd: (suite) => void hooks.push({ hook: 'onSuiteEnd', value: { ...suite } }), + getExitCode: (suite) => (suite.failed > 0 ? 1 : 0), + }, + }; +} + +function makeSessionStore(root: string): SessionStore { + return new SessionStore(path.join(root, 'sessions')); +} + +/** + * Runs `test --maestro` exactly as the daemon does, feeding every progress event through the + * shipped reporter registry — the same translation and dispatch a `--reporter ./mine.mjs` + * module gets. + */ +async function runMaestroSuiteThroughReporter(params: { + root: string; + requestId: string; + flags?: DaemonRequest['flags']; + invoke: (req: DaemonRequest) => Promise; +}): Promise<{ hooks: RecordedHook[]; suite: ReplaySuiteResult; exitCode: number }> { + const { reporter, hooks } = createRecordingReporter(); + const reporters = [reporter]; + const response = await withRequestProgressSink( + (event) => runReplayTestReporterProgress(reporters, event, reporterContext), + async () => + await handleSessionCommands({ + req: { + token: 't', + session: 'default', + command: 'test', + positionals: [params.root], + meta: { cwd: params.root, requestId: params.requestId }, + flags: { replayBackend: 'maestro', platform: 'android', ...params.flags }, + }, + sessionName: 'default', + logPath: path.join(os.tmpdir(), 'daemon.log'), + sessionStore: makeSessionStore(params.root), + invoke: params.invoke, + }), + ); + + expect(response?.ok, JSON.stringify(response)).toBeTruthy(); + if (!response?.ok) throw new Error('Expected a successful suite response.'); + const suite = response.data as unknown as ReplaySuiteResult; + await runReplayTestReporters(reporters, suite, reporterContext); + return { hooks, suite, exitCode: getReplayTestReporterExitCode(reporters, suite) }; +} + +/** `snapshot` returns an empty hierarchy; every other command succeeds. */ +function maestroInvoke(req: DaemonRequest): DaemonResponse { + return req.command === 'snapshot' + ? { ok: true, data: { createdAt: 0, nodes: [] } } + : { ok: true, data: {} }; +} + +test('a Maestro suite reaches the reporter with step payloads and attempt-scoped identity', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-reporter-maestro-')); + fs.writeFileSync(path.join(root, '01-login.yaml'), MAESTRO_FLOW); + + const { hooks, suite, exitCode } = await runMaestroSuiteThroughReporter({ + root, + requestId: 'suite-maestro', + invoke: async (req) => maestroInvoke(req), + }); + + expect(hooks.map((entry) => entry.hook)).toEqual([ + 'onSuiteStart', + 'onTestStart', + 'onTestStep', + 'onTestStep', + 'onTestResult', + 'onSuiteEnd', + ]); + + const suiteArtifactsDir = path.join(root, '.agent-device', 'test-artifacts', 'suite-maestro'); + const testArtifactsDir = path.join(suiteArtifactsDir, '01-login.yaml'); + const session = 'default:test:suite-maestro:1-01-login:attempt-1'; + + // The Maestro flow's `name` becomes the reporter `title`. Discovery reads it through + // `inspectMaestroFlow`, so this is a shipped reporter value the native scenario cannot cover. + expect(hooks[1]?.value).toMatchObject({ + file: path.join(root, '01-login.yaml'), + title: 'Login flow', + index: 1, + total: 1, + attempt: undefined, + maxAttempts: 1, + session, + artifactsDir: testArtifactsDir, + }); + + // The two step events: payload comes from the Maestro engine through the observer's + // `onStep`, while file/index/total/attempt/session/artifactsDir come from the scheduler's + // per-attempt context. Before P3 the latter half came from request-global AsyncLocalStorage. + const steps = hooks.filter((entry) => entry.hook === 'onTestStep').map((entry) => entry.value); + expect(steps).toHaveLength(2); + expect('status' in steps[0]!).toBe(false); + expect(steps[0]).toMatchObject({ + file: path.join(root, '01-login.yaml'), + title: 'Login flow', + index: 1, + total: 1, + attempt: 1, + maxAttempts: 1, + session, + artifactsDir: testArtifactsDir, + stepIndex: 1, + stepTotal: 2, + stepCommand: 'assertNotVisible', + stepValue: 'Nope', + }); + expect(steps[1]).toMatchObject({ + index: 1, + total: 1, + attempt: 1, + session, + artifactsDir: testArtifactsDir, + stepIndex: 2, + stepTotal: 2, + stepCommand: 'back', + }); + // `back` carries no progress value, and the observer must not invent one. + expect(steps[1]!.stepValue).toBe(undefined); + + expect(hooks[4]?.value).toMatchObject({ + status: 'pass', + title: 'Login flow', + attempt: 1, + maxAttempts: 1, + session, + artifactsDir: testArtifactsDir, + }); + expect(suite.passed).toBe(1); + expect(exitCode).toBe(0); +}); + +test('Maestro step sessions track the running attempt across a retry', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-reporter-maestro-retry-')); + fs.writeFileSync(path.join(root, '01-retry.yaml'), MAESTRO_FLOW); + + // The first attempt's `back` fails, so attempt 1 fails at step 2 and attempt 2 passes. + let backCalls = 0; + const { hooks, suite } = await runMaestroSuiteThroughReporter({ + root, + requestId: 'suite-maestro-retry', + flags: { retries: 1 }, + invoke: async (req) => { + if (req.command !== 'snapshot') { + backCalls += 1; + if (backCalls === 1) { + return { ok: false, error: { code: 'COMMAND_FAILED', message: 'first attempt failed' } }; + } + } + return maestroInvoke(req); + }, + }); + + const steps = hooks.filter((entry) => entry.hook === 'onTestStep').map((entry) => entry.value); + + // Every step event of attempt 1 carries attempt-1's session, and every step event of + // attempt 2 carries attempt-2's. This is the identity the removed AsyncLocalStorage used to + // supply, now threaded from the scheduler's attempt context through the Maestro observer. + const attempts = steps.map((step) => step.attempt); + const sessions = steps.map((step) => step.session); + expect(new Set(attempts)).toEqual(new Set([1, 2])); + for (const [index, step] of steps.entries()) { + expect(step.session).toBe( + `default:test:suite-maestro-retry:1-01-retry:attempt-${String(attempts[index])}`, + ); + } + expect(new Set(sessions).size).toBe(2); + expect(steps.every((step) => step.maxAttempts === 2)).toBe(true); + + // The retry result and the final result, both Maestro-produced. + const results = hooks + .filter((entry) => entry.hook === 'onTestResult') + .map((entry) => entry.value); + expect(results[0]).toMatchObject({ + status: 'fail', + attempt: 1, + retrying: true, + session: 'default:test:suite-maestro-retry:1-01-retry:attempt-1', + }); + expect(results[1]).toMatchObject({ + status: 'pass', + attempt: 2, + retrying: undefined, + session: 'default:test:suite-maestro-retry:1-01-retry:attempt-2', + }); + expect(suite.passed).toBe(1); +}); diff --git a/src/daemon/handlers/__tests__/session-test-runtime.test.ts b/src/daemon/handlers/__tests__/session-test-runtime.test.ts index a1f09e7ac3..86ee3ac114 100644 --- a/src/daemon/handlers/__tests__/session-test-runtime.test.ts +++ b/src/daemon/handlers/__tests__/session-test-runtime.test.ts @@ -2,9 +2,17 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { afterEach, expect, test, vi } from 'vitest'; -import type { DaemonResponse } from '../../types.ts'; import { isRequestCanceled } from '../../../request/cancel.ts'; import { runReplayTestAttempt } from '../session-test-runtime.ts'; +import type { ReplayTestAttemptOutcome } from '../session-test-types.ts'; + +const PASSED: ReplayTestAttemptOutcome = { + status: 'passed', + replayed: 1, + healed: 0, + warnings: [], + artifactPaths: [], +}; afterEach(() => { vi.useRealTimers(); @@ -25,8 +33,8 @@ function readTimingEventTypes(artifactsDir: string): string[] { test('runReplayTestAttempt keeps cancellation active until a timed-out replay settles', async () => { vi.useFakeTimers(); - let resolveReplay: ((response: DaemonResponse) => void) | undefined; - const replayPromise = new Promise((resolve) => { + let resolveReplay: ((outcome: ReplayTestAttemptOutcome) => void) | undefined; + const replayPromise = new Promise((resolve) => { resolveReplay = resolve; }); const replaySettled = replayPromise.then(() => undefined); @@ -55,11 +63,14 @@ test('runReplayTestAttempt keeps cancellation active until a timed-out replay se await vi.advanceTimersByTimeAsync(2_000); const result = await attemptPromise; - expect(result.ok).toBe(false); - if (!result.ok) { + expect(result.status).toBe('failed'); + if (result.status === 'failed') { expect(result.error.message).toContain('TIMEOUT after 10ms'); expect(result.error.details?.reason).toBe('timeout_cleanup_pending'); expect(result.error.details?.timeoutCleanupPending).toBe(true); + // The abandoned replay still owns the device, so the outcome is tagged infrastructure and + // the scheduler stops the suite instead of retrying into a contended session. + expect(result.infrastructure).toBe(true); } expect(cleanupSession).toHaveBeenCalledWith('default:test:timeout'); expect(finalizeAttempt).toHaveBeenCalledWith( @@ -76,8 +87,10 @@ test('runReplayTestAttempt keeps cancellation active until a timed-out replay se expect(cleanupSession).toHaveBeenCalledTimes(1); resolveReplay?.({ - ok: false, + status: 'failed', error: { code: 'COMMAND_FAILED', message: 'request canceled' }, + artifactPaths: [], + infrastructure: false, }); await replaySettled; await vi.waitFor(() => { @@ -97,19 +110,19 @@ test('runReplayTestAttempt keeps a passing replay passed when finalization fails filePath: '01-pass.ad', sessionName: 'default:test:pass', requestId: 'req-pass', - runReplay: async () => ({ ok: true, data: { replayed: 1, healed: 0 } }), + runReplay: async () => PASSED, finalizeAttempt: async () => ({ - ok: false, + status: 'failed', error: { code: 'COMMAND_FAILED', message: 'failed to stop recording' }, + artifactPaths: [], + infrastructure: false, }), cleanupSession, }); - expect(result.ok).toBe(true); - if (!result.ok) throw new Error(result.error.message); - expect(result.data?.warnings).toEqual([ - 'Replay test finalization failed: failed to stop recording', - ]); + expect(result.status).toBe('passed'); + if (result.status !== 'passed') throw new Error(result.error.message); + expect(result.warnings).toEqual(['Replay test finalization failed: failed to stop recording']); expect(cleanupSession).toHaveBeenCalledWith('default:test:pass'); }); @@ -127,7 +140,7 @@ test('runReplayTestAttempt finalizes before cleanup and records that order in th artifactsDir, runReplay: async () => { lifecycleEvents.push('replay'); - return { ok: true, data: { replayed: 1, healed: 0 } }; + return PASSED; }, finalizeAttempt: async () => { lifecycleEvents.push('finalize'); @@ -138,7 +151,7 @@ test('runReplayTestAttempt finalizes before cleanup and records that order in th }, }); - expect(result.ok).toBe(true); + expect(result.status).toBe('passed'); expect(lifecycleEvents).toEqual(['replay', 'finalize', 'cleanup']); expect(readTimingEventTypes(artifactsDir)).toEqual([ 'replay_test_attempt_start', @@ -154,8 +167,8 @@ test('runReplayTestAttempt cleans up once when a timed-out replay settles inside vi.useFakeTimers(); const artifactsDir = makeArtifactsDir('grace'); - let resolveReplay: ((response: DaemonResponse) => void) | undefined; - const replayPromise = new Promise((resolve) => { + let resolveReplay: ((outcome: ReplayTestAttemptOutcome) => void) | undefined; + const replayPromise = new Promise((resolve) => { resolveReplay = resolve; }); const lifecycleEvents: string[] = []; @@ -179,16 +192,17 @@ test('runReplayTestAttempt cleans up once when a timed-out replay settles inside await vi.advanceTimersByTimeAsync(10); // The replay comes back inside the 2s grace window, so no cleanup race is declared. - resolveReplay?.({ ok: true, data: { replayed: 1, healed: 0 } }); + resolveReplay?.(PASSED); const result = await attemptPromise; - // The raced timeout response still wins: a late success does not un-fail the attempt. - expect(result.ok).toBe(false); - if (!result.ok) { + // The raced timeout outcome still wins: a late success does not un-fail the attempt. + expect(result.status).toBe('failed'); + if (result.status === 'failed') { expect(result.error.message).toBe('TIMEOUT after 10ms'); expect(result.error.details?.reason).toBe('timeout'); expect(result.error.details?.timeoutCleanupPending).toBe(undefined); expect(result.error.details?.timeoutMode).toBe('cooperative'); + expect(result.infrastructure).toBe(false); } expect(lifecycleEvents).toEqual(['finalize', 'cleanup']); expect(cleanupSession).toHaveBeenCalledTimes(1); @@ -209,12 +223,12 @@ test('runReplayTestAttempt cleans up without a finalizer and adds no finalizatio filePath: '01-no-finalizer.ad', sessionName: 'default:test:no-finalizer', requestId: 'req-no-finalizer', - runReplay: async () => ({ ok: true, data: { replayed: 1, healed: 0 } }), + runReplay: async () => PASSED, cleanupSession, }); - expect(result.ok).toBe(true); - if (!result.ok) throw new Error(result.error.message); - expect(result.data?.warnings).toBe(undefined); + expect(result.status).toBe('passed'); + if (result.status !== 'passed') throw new Error(result.error.message); + expect(result.warnings).toEqual([]); expect(cleanupSession).toHaveBeenCalledWith('default:test:no-finalizer'); }); diff --git a/src/daemon/handlers/session-replay-divergence-publication.ts b/src/daemon/handlers/session-replay-divergence-publication.ts index 4862d6d09b..160fd369e7 100644 --- a/src/daemon/handlers/session-replay-divergence-publication.ts +++ b/src/daemon/handlers/session-replay-divergence-publication.ts @@ -2,7 +2,7 @@ import fs from 'node:fs'; import path from 'node:path'; import type { ResponseLevel } from '@agent-device/kernel/contracts'; import { redactDiagnosticData } from '@agent-device/kernel/redaction'; -import { boundReplayDivergence, type ReplayDivergence } from '../../replay/divergence.ts'; +import { boundReplayDivergence, type ReplayDivergence } from '@agent-device/contracts/divergence'; import { bindInternalObservationAuthority, type InternalObservationEvidence, diff --git a/src/daemon/handlers/session-replay-divergence.ts b/src/daemon/handlers/session-replay-divergence.ts index daf91881fe..4ae9113feb 100644 --- a/src/daemon/handlers/session-replay-divergence.ts +++ b/src/daemon/handlers/session-replay-divergence.ts @@ -41,7 +41,7 @@ import { type ReplayDivergenceSuggestion, type ReplayDivergenceSuggestionBasis, type ReplayVarScrubEntry, -} from '../../replay/divergence.ts'; +} from '@agent-device/contracts/divergence'; export type DivergenceFieldSanitizer = (value: string, limit?: number) => string; diff --git a/src/daemon/handlers/session-replay-maestro-failure.ts b/src/daemon/handlers/session-replay-maestro-failure.ts index 1e3ddf6c16..1fbf403d23 100644 --- a/src/daemon/handlers/session-replay-maestro-failure.ts +++ b/src/daemon/handlers/session-replay-maestro-failure.ts @@ -7,7 +7,7 @@ import { createReplayDivergenceSanitizer, type ReplayDivergence, type ReplayVarScrubEntry, -} from '../../replay/divergence.ts'; +} from '@agent-device/contracts/divergence'; import { formatScriptArg } from '../../replay/script-utils.ts'; import { getRequestSignal } from '../../request/cancel.ts'; import { SessionStore } from '../session-store.ts'; diff --git a/src/daemon/handlers/session-replay-maestro-observer.ts b/src/daemon/handlers/session-replay-maestro-observer.ts index df4a5a92ee..d5776de6ae 100644 --- a/src/daemon/handlers/session-replay-maestro-observer.ts +++ b/src/daemon/handlers/session-replay-maestro-observer.ts @@ -5,20 +5,21 @@ import type { MaestroFailedAction, } from '@agent-device/maestro'; import { AppError } from '@agent-device/kernel/errors'; -import { emitRequestProgress, readReplayTestActionProgress } from '../../request/progress.ts'; +import type { ReplayTestAttemptStepSink } from './session-test-types.ts'; import { stripUndefined } from '../../utils/parsing.ts'; import { appendReplayTraceEvent } from './session-replay-trace.ts'; export function createMaestroReplayObserver(params: { filePath: string; tracePath: string | undefined; + onStep?: ReplayTestAttemptStepSink; }): MaestroExecutionObserver { - const { filePath, tracePath } = params; + const { filePath, tracePath, onStep } = params; const traceStarts = new Map(); return { actionStarted: (event) => { traceStarts.set(event.stepIndex, event); - runTelemetrySink(() => emitMaestroProgress(filePath, event)); + runTelemetrySink(() => emitMaestroStep(onStep, event)); runTelemetrySink(() => appendMaestroTraceStart(tracePath, filePath, event)); }, actionCompleted: (event) => { @@ -64,18 +65,15 @@ function traceStopEvent( }); } -function emitMaestroProgress(file: string, event: MaestroActionEvent): void { - const progress = readReplayTestActionProgress(); - if (!progress) return; - emitRequestProgress({ - type: 'replay-test', - ...progress, - file: progress.file || file, - status: 'progress', - stepIndex: event.stepIndex, - stepTotal: event.stepTotal, - stepCommand: event.action, - ...(event.value ? { stepValue: event.value } : {}), +function emitMaestroStep( + onStep: ReplayTestAttemptStepSink | undefined, + event: MaestroActionEvent, +): void { + onStep?.({ + index: event.stepIndex, + total: event.stepTotal, + command: event.action, + ...(event.value ? { value: event.value } : {}), }); } diff --git a/src/daemon/handlers/session-replay-maestro-runtime.ts b/src/daemon/handlers/session-replay-maestro-runtime.ts index 3f8b99576f..ce084c9c54 100644 --- a/src/daemon/handlers/session-replay-maestro-runtime.ts +++ b/src/daemon/handlers/session-replay-maestro-runtime.ts @@ -28,6 +28,7 @@ import { SessionStore } from '../session-store.ts'; import { errorResponse } from './response.ts'; import { buildReplayBuiltinVars } from './session-replay-vars.ts'; import { createMaestroReplayObserver } from './session-replay-maestro-observer.ts'; +import type { ReplayTestAttemptStepSink } from './session-test-types.ts'; import { buildTypedMaestroReplayErrorResponse, buildTypedMaestroSuccessResponse, @@ -42,6 +43,7 @@ type TypedMaestroReplayParams = { logPath: string; sessionStore: SessionStore; tracePath?: string; + onStep?: ReplayTestAttemptStepSink; invoke: DaemonInvokeFn; }; @@ -104,7 +106,7 @@ async function executeTypedMaestroReplay( state: TypedMaestroReplayState; }, ): Promise { - const { req, sessionName, sessionStore, tracePath, invoke, state } = params; + const { req, sessionName, sessionStore, tracePath, onStep, invoke, state } = params; const context = await prepareTypedMaestroReplay(params); const port = createMaestroReplayPort({ req, @@ -130,6 +132,7 @@ async function executeTypedMaestroReplay( observer: createMaestroReplayObserver({ filePath: context.filePath, tracePath, + onStep, }), }); if (!outcome.ok) { diff --git a/src/daemon/handlers/session-replay-repair-hint.ts b/src/daemon/handlers/session-replay-repair-hint.ts index c31bf07f87..47815de8d3 100644 --- a/src/daemon/handlers/session-replay-repair-hint.ts +++ b/src/daemon/handlers/session-replay-repair-hint.ts @@ -22,7 +22,7 @@ */ import type { SnapshotNode } from '@agent-device/kernel/snapshot'; -import type { ReplayDivergenceKind, ReplayRepairHint } from '../../replay/divergence.ts'; +import type { ReplayDivergenceKind, ReplayRepairHint } from '@agent-device/contracts/divergence'; import { matchesAncestryPrefix, type TargetAnnotationV1, diff --git a/src/daemon/handlers/session-replay-resume.ts b/src/daemon/handlers/session-replay-resume.ts index ebdd64d501..c36b10cce1 100644 --- a/src/daemon/handlers/session-replay-resume.ts +++ b/src/daemon/handlers/session-replay-resume.ts @@ -1,5 +1,5 @@ import type { SessionAction, SessionState } from '../types.ts'; -import type { ReplayDivergenceResume, ReplayRepairHint } from '../../replay/divergence.ts'; +import type { ReplayDivergenceResume, ReplayRepairHint } from '@agent-device/contracts/divergence'; import { SessionStore } from '../session-store.ts'; export function buildAndPersistReplayDivergenceResume(params: { @@ -40,7 +40,7 @@ export function buildAndPersistReplayDivergenceResume(params: { * `failedIndex` would re-diverge on the step the agent already performed). * Every other repair hint (including a plain `action-failure`) resumes AT * `failedIndex` unchanged. This must agree with the text guidance rendered by - * `formatReplayDivergenceReport` (`src/replay/divergence.ts`) — both are + * `formatReplayDivergenceReport` (`packages/contracts/src/replay-divergence.ts`) — both are * derived from the same computed `from` value. * * `failedIndex` is always a valid 1-based index into `actions` (both call diff --git a/src/daemon/handlers/session-replay-runtime-failure-response.ts b/src/daemon/handlers/session-replay-runtime-failure-response.ts index 1aae0a2bb2..4c14e954e1 100644 --- a/src/daemon/handlers/session-replay-runtime-failure-response.ts +++ b/src/daemon/handlers/session-replay-runtime-failure-response.ts @@ -1,4 +1,4 @@ -import { scrubReplayVarValues, type ReplayVarScrubEntry } from '../../replay/divergence.ts'; +import { scrubReplayVarValues, type ReplayVarScrubEntry } from '@agent-device/contracts/divergence'; import { formatDivergenceActionLabel } from '../../replay/script-utils.ts'; import type { SnapshotDiagnosticsSummary } from '@agent-device/contracts/capture'; import { buildDisplayPositionals } from '../session-event-action.ts'; diff --git a/src/daemon/handlers/session-replay-runtime.ts b/src/daemon/handlers/session-replay-runtime.ts index 3ae666d94d..ba0f79bd34 100644 --- a/src/daemon/handlers/session-replay-runtime.ts +++ b/src/daemon/handlers/session-replay-runtime.ts @@ -9,11 +9,6 @@ import type { SessionAction, SessionState, } from '../types.ts'; -import { - emitRequestProgress, - readReplayTestActionProgress, - type ReplayTestProgressEvent, -} from '../../request/progress.ts'; import { SessionStore } from '../session-store.ts'; import { clearPendingRecordAndHealWatermark } from './session-replay-resume.ts'; import { expandSessionPath } from '../session-paths.ts'; @@ -38,7 +33,7 @@ import { type SnapshotTimingSample, } from '@agent-device/contracts/capture'; import type { ReplayCommandResult } from '@agent-device/contracts/replay'; -import type { ReplayDivergenceResume } from '../../replay/divergence.ts'; +import type { ReplayDivergenceResume } from '@agent-device/contracts/divergence'; import { isMaestroYamlPath, maestroBackendRequiredMessage, @@ -62,6 +57,7 @@ import { } from './session-replay-target-verification.ts'; import { buildReplayBuiltinVars } from './session-replay-vars.ts'; import { runTypedMaestroReplayFile } from './session-replay-maestro-runtime.ts'; +import type { ReplayTestAttemptStep, ReplayTestAttemptStepSink } from './session-test-types.ts'; import { getRequestSignal } from '../../request/cancel.ts'; /** Per-run invariants for a single replay step (ADR 0012 step 4 verify + dispatch + guard). */ @@ -200,9 +196,15 @@ export async function runReplayScriptFile(params: { logPath: string; sessionStore: SessionStore; tracePath?: string; + /** + * Per-attempt step sink supplied by the replay-test scheduler through its host (#1478 P3). + * Threaded alongside `tracePath` rather than read from request-global storage, so a direct + * `replay` simply has no sink and emits nothing. + */ + onStep?: ReplayTestAttemptStepSink; invoke: DaemonInvokeFn; }): Promise { - const { req, sessionName, logPath, sessionStore, tracePath, invoke } = params; + const { req, sessionName, logPath, sessionStore, tracePath, onStep, invoke } = params; const filePath = req.positionals?.[0]; if (!filePath) { return errorResponse('INVALID_ARGS', 'replay requires a path'); @@ -285,6 +287,7 @@ export async function runReplayScriptFile(params: { stepContext, artifactPaths, snapshotDiagnosticSamples, + onStep, armSaveScript: sessionPreparation.armSaveScript, }); if (failure) return failure; @@ -323,6 +326,7 @@ type ReplayActionExecution = { stepContext: ReplayStepContext; artifactPaths: Set; snapshotDiagnosticSamples: SnapshotTimingSample[]; + onStep: ReplayTestAttemptStepSink | undefined; armSaveScript: () => void; }; @@ -332,12 +336,12 @@ async function executeReplayActions( const { sessionName, sessionStore, - resolved, actions, entryIndex, stepContext, artifactPaths, snapshotDiagnosticSamples, + onStep, armSaveScript, } = params; for (let index = entryIndex; index < actions.length; index += 1) { @@ -357,7 +361,7 @@ async function executeReplayActions( ) { continue; } - emitReplayTestActionProgress(resolved, index, actions.length, action); + onStep?.(replayActionStep(index, actions.length, action)); const sampleStart = readSessionSnapshotSampleCount(sessionStore, sessionName); const response = await resolveReplayStepResponse(stepContext, action, index, [ ...artifactPaths, @@ -452,42 +456,25 @@ function completeReplayRun(params: { }; } -function emitReplayTestActionProgress( - file: string, +function replayActionStep( actionIndex: number, actionTotal: number, action: SessionAction, -): void { - const progress = readReplayTestActionProgress(); - if (!progress) return; - emitRequestProgress({ - type: 'replay-test', - ...progress, - file: progress.file || file, - status: 'progress', - stepIndex: actionIndex + 1, - stepTotal: actionTotal, - ...formatReplayTestActionProgress(action), - }); -} - -function formatReplayTestActionProgress( - action: SessionAction, -): Pick { +): ReplayTestAttemptStep { return { - stepCommand: action.command, - ...formatReplayTestProgressValue(action), + index: actionIndex + 1, + total: actionTotal, + command: action.command, + ...replayActionStepValue(action), }; } -function formatReplayTestProgressValue( - action: SessionAction, -): Pick { +function replayActionStepValue(action: SessionAction): Pick { const positionals = action.positionals ?? []; const selectorValue = readSelectorDisplayValue(positionals[0]); - if (selectorValue) return { stepValue: selectorValue }; + if (selectorValue) return { value: selectorValue }; if (positionals.length === 0) return {}; - return { stepValue: positionals.join(' ') }; + return { value: positionals.join(' ') }; } function readSelectorDisplayValue(selector: string | undefined): string | undefined { diff --git a/src/daemon/handlers/session-replay-suggestion-ranking.ts b/src/daemon/handlers/session-replay-suggestion-ranking.ts index c2121b5131..3a20d53ebb 100644 --- a/src/daemon/handlers/session-replay-suggestion-ranking.ts +++ b/src/daemon/handlers/session-replay-suggestion-ranking.ts @@ -1,4 +1,4 @@ -import type { ReplayDivergenceSuggestionBasis } from '../../replay/divergence.ts'; +import type { ReplayDivergenceSuggestionBasis } from '@agent-device/contracts/divergence'; const BASIS_RANK: Record = { id: 0, diff --git a/src/daemon/handlers/session-replay-target-classification.ts b/src/daemon/handlers/session-replay-target-classification.ts index 7102282273..b49e647620 100644 --- a/src/daemon/handlers/session-replay-target-classification.ts +++ b/src/daemon/handlers/session-replay-target-classification.ts @@ -58,7 +58,7 @@ import { type LocalIdentity, type TargetAnnotationV1, } from '../../replay/target-identity.ts'; -import type { ReplayDivergenceTargetBindingKind } from '../../replay/divergence.ts'; +import type { ReplayDivergenceTargetBindingKind } from '@agent-device/contracts/divergence'; // --------------------------------------------------------------------------- // Pure classification core — no capture, no session, no wire shaping. diff --git a/src/daemon/handlers/session-replay-target-verification.ts b/src/daemon/handlers/session-replay-target-verification.ts index 97390a6d14..3aca8a3a22 100644 --- a/src/daemon/handlers/session-replay-target-verification.ts +++ b/src/daemon/handlers/session-replay-target-verification.ts @@ -19,7 +19,7 @@ import { type ReplayDivergenceTargetBindingKind, type ReplayDivergenceTargetCandidate, type ReplayDivergenceTargetIdentity, -} from '../../replay/divergence.ts'; +} from '@agent-device/contracts/divergence'; import { readNodeStructuralDenotation, REPLAY_TARGET_GUARD_MISMATCH_REASON, diff --git a/src/daemon/handlers/session-replay.ts b/src/daemon/handlers/session-replay.ts index b2aa99ab28..46cac94d1f 100644 --- a/src/daemon/handlers/session-replay.ts +++ b/src/daemon/handlers/session-replay.ts @@ -8,6 +8,7 @@ import { collectReplayActionArtifactPaths } from './session-replay-runtime-artif import { errorResponse } from './response.ts'; import type { ReplayScriptMetadata } from '../../replay/script.ts'; import { buildReplayTestShardFlags, type ReplayTestShardContext } from './session-test-sharding.ts'; +import { toReplayTestAttemptOutcome, toReplayTestFinalizeFailure } from './session-test-outcome.ts'; import type { LeaseRegistry } from '../lease-registry.ts'; import { buildReplayTestVideoOpenLifecycle, @@ -104,6 +105,7 @@ export async function handleSessionReplayCommands(params: { artifactPaths, tracePath, shard, + onStep, }) => { const captureArtifacts = (response: DaemonResponse): DaemonResponse => { if (!artifactPaths) return response; @@ -152,6 +154,7 @@ export async function handleSessionReplayCommands(params: { logPath, sessionStore, tracePath, + onStep, invoke: async (nestedReq) => { const startResponse = await startReplayTestVideoRecordingIfReady(videoRecordingParams); if (startResponse && !startResponse.ok) return startResponse; @@ -159,7 +162,7 @@ export async function handleSessionReplayCommands(params: { return response; }, }); - return replayResponse; + return toReplayTestAttemptOutcome(replayResponse); }, finalizeAttempt: async ({ sessionName: testSessionName, @@ -167,15 +170,17 @@ export async function handleSessionReplayCommands(params: { artifactsDir, tracePath, }) => - await finalizeReplayTestVideoRecording({ - req, - sessionName: testSessionName, - logPath, - sessionStore, - artifactsDir, - tracePath, - artifactPaths, - }), + toReplayTestFinalizeFailure( + await finalizeReplayTestVideoRecording({ + req, + sessionName: testSessionName, + logPath, + sessionStore, + artifactsDir, + tracePath, + artifactPaths, + }), + ), cleanupSession: async (testSessionName) => { if (!sessionStore.get(testSessionName)) return; await handleCloseCommand({ diff --git a/src/daemon/handlers/session-test-artifacts.ts b/src/daemon/handlers/session-test-artifacts.ts index 5239adf605..a439ef6f33 100644 --- a/src/daemon/handlers/session-test-artifacts.ts +++ b/src/daemon/handlers/session-test-artifacts.ts @@ -1,7 +1,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { isMaestroYamlPath } from '../../replay/format.ts'; -import type { DaemonResponse } from '../types.ts'; +import type { ReplayTestAttemptOutcome } from './session-test-types.ts'; import { SessionStore } from '../session-store.ts'; const DEFAULT_TEST_ARTIFACTS_ROOT = '.agent-device/test-artifacts'; @@ -40,18 +40,18 @@ export function prepareReplayTestAttemptArtifacts( } export function materializeReplayTestAttemptArtifacts(params: { - response: DaemonResponse; + outcome: ReplayTestAttemptOutcome; filePath: string; sessionName: string; attempts: number; maxAttempts: number; attemptArtifactsDir: string; }): void { - const { response, filePath, sessionName, attempts, maxAttempts, attemptArtifactsDir } = params; - const artifactPaths = getReplayTestArtifactPaths(response); - const sourcePaths = [...artifactPaths]; - if (!response.ok && typeof response.error.logPath === 'string') { - sourcePaths.push(response.error.logPath); + const { outcome, filePath, sessionName, attempts, maxAttempts, attemptArtifactsDir } = params; + const passed = outcome.status === 'passed'; + const sourcePaths = [...new Set(outcome.artifactPaths)]; + if (outcome.status === 'failed' && typeof outcome.error.logPath === 'string') { + sourcePaths.push(outcome.error.logPath); } const copiedArtifacts = copyReplayTestArtifacts(sourcePaths, attemptArtifactsDir); @@ -59,19 +59,17 @@ export function materializeReplayTestAttemptArtifacts(params: { `file: ${filePath}`, `session: ${sessionName}`, `attempt: ${attempts}/${maxAttempts}`, - `status: ${response.ok ? 'passed' : 'failed'}`, + `status: ${passed ? 'passed' : 'failed'}`, ]; - if (response.ok) { - const replayed = typeof response.data?.replayed === 'number' ? response.data.replayed : 0; - const healed = typeof response.data?.healed === 'number' ? response.data.healed : 0; - lines.push(`replayed: ${replayed}`, `healed: ${healed}`); + if (outcome.status === 'passed') { + lines.push(`replayed: ${outcome.replayed}`, `healed: ${outcome.healed}`); } else { - lines.push(`code: ${response.error.code}`, `message: ${response.error.message}`); - if (response.error.hint) lines.push(`hint: ${response.error.hint}`); - if (response.error.diagnosticId) lines.push(`diagnosticId: ${response.error.diagnosticId}`); - if (response.error.logPath) lines.push(`logPath: ${response.error.logPath}`); - if (response.error.details?.reason === 'timeout') { + lines.push(`code: ${outcome.error.code}`, `message: ${outcome.error.message}`); + if (outcome.error.hint) lines.push(`hint: ${outcome.error.hint}`); + if (outcome.error.diagnosticId) lines.push(`diagnosticId: ${outcome.error.diagnosticId}`); + if (outcome.error.logPath) lines.push(`logPath: ${outcome.error.logPath}`); + if (outcome.error.details?.reason === 'timeout') { lines.push('timeoutMode: cooperative'); } } @@ -85,19 +83,11 @@ export function materializeReplayTestAttemptArtifacts(params: { const resultPath = path.join(attemptArtifactsDir, 'result.txt'); const output = `${lines.join('\n')}\n`; fs.writeFileSync(resultPath, output); - if (!response.ok) { + if (!passed) { fs.writeFileSync(path.join(attemptArtifactsDir, 'failure.txt'), output); } } -function getReplayTestArtifactPaths(response: DaemonResponse): string[] { - const raw = response.ok - ? (response.data as Record | undefined)?.artifactPaths - : response.error.details?.artifactPaths; - if (!Array.isArray(raw)) return []; - return [...new Set(raw.filter((entry): entry is string => typeof entry === 'string'))]; -} - function copyReplayTestArtifacts(paths: string[], attemptArtifactsDir: string): string[] { const copiedPaths: string[] = []; const usedNames = new Map(); diff --git a/src/daemon/handlers/session-test-attempt.ts b/src/daemon/handlers/session-test-attempt.ts index a2e1a9656d..a9d82ac404 100644 --- a/src/daemon/handlers/session-test-attempt.ts +++ b/src/daemon/handlers/session-test-attempt.ts @@ -1,7 +1,7 @@ import path from 'node:path'; import { emitRequestProgress } from '../../request/progress.ts'; -import type { DaemonResponse } from '../types.ts'; import type { ReplaySuiteTestFailed, ReplaySuiteTestResult } from '@agent-device/contracts/replay'; +import type { ReplayTestProgressEvent } from '@agent-device/contracts/progress'; import { buildReplayTestArtifactSlug, materializeReplayTestAttemptArtifacts, @@ -12,24 +12,38 @@ import { buildReplayTestSessionName, type ReplayTestRunEntry, } from './session-test-discovery.ts'; -import { isReplayInfrastructureFailure } from './session-test-infrastructure.ts'; import { runReplayTestAttempt } from './session-test-runtime.ts'; -import type { ReplayTestRuntimeDependencies } from './session-test-types.ts'; +import type { + ReplayTestAttemptOutcome, + ReplayTestRuntimeDependencies, +} from './session-test-types.ts'; import type { ReplayTestShardContext } from './session-test-sharding.ts'; import { isRequestCanceled } from '../../request/cancel.ts'; -import { readSnapshotDiagnosticsSummary } from '@agent-device/contracts/capture'; type ReplayTestCaseResult = Extract; type ReplayTestAttemptFailure = NonNullable< Extract['attemptFailures'] >[number]; +/** + * A finished test case plus the one scheduling fact the public result cannot carry: whether the + * final attempt failed for environmental reasons, which stops the suite instead of continuing. + */ +export type ReplayTestCaseReport = { + result: ReplayTestCaseResult; + infrastructure: boolean; +}; + export async function runReplayTestCase( params: ReplayTestCaseParams, -): Promise { +): Promise { const context = buildReplayTestCaseContext(params); const outcome = await runReplayTestCaseAttempts(params, context); - return buildReplayTestCaseResult(params, context, outcome); + return { + result: buildReplayTestCaseResult(params, context, outcome), + infrastructure: + outcome.finalOutcome?.status === 'failed' && outcome.finalOutcome.infrastructure, + }; } type ReplayTestCaseParams = { @@ -54,14 +68,14 @@ type ReplayTestCaseContext = { }; type ReplayTestAttemptResult = { - response: DaemonResponse; + outcome: ReplayTestAttemptOutcome; sessionName: string; attempt: number; durationMs: number; }; type ReplayTestCaseOutcome = { - finalResponse?: DaemonResponse; + finalOutcome?: ReplayTestAttemptOutcome; finalSessionName: string; attempts: number; finalAttemptDurationMs: number; @@ -97,7 +111,7 @@ async function runReplayTestCaseAttempts( if (isRequestCanceled(params.requestId)) break; const attempt = await runSingleReplayTestAttempt(params, context, attemptIndex); updateReplayTestCaseOutcome(outcome, attempt); - if (shouldStopReplayTestAttempts(params, attempt.response, attemptIndex)) break; + if (shouldStopReplayTestAttempts(params, attempt.outcome, attemptIndex)) break; emitReplayTestRetryProgress(params, context, attempt); } @@ -139,8 +153,19 @@ async function runSingleReplayTestAttempt( attemptIndex, shardIndex: shard?.shardIndex, }); + const attemptProgress: ReplayTestAttemptProgressContext = { + file: entry.path, + title: entry.title, + index: suiteIndex, + total: suiteTotal, + attempt, + maxAttempts: context.maxAttempts, + session: testSessionName, + artifactsDir: context.testArtifactsDir, + ...replayTestProgressShardMetadata(shard), + }; - const response = await runReplayTestAttempt({ + const outcome = await runReplayTestAttempt({ filePath: entry.path, sessionName: testSessionName, requestId: attemptRequestId, @@ -150,16 +175,16 @@ async function runSingleReplayTestAttempt( target: entry.metadata.target, artifactsDir: attemptArtifactsDir, shard, - progress: { - file: entry.path, - title: entry.title, - index: suiteIndex, - total: suiteTotal, - attempt, - maxAttempts: context.maxAttempts, - session: testSessionName, - artifactsDir: context.testArtifactsDir, - ...replayTestProgressShardMetadata(shard), + onStep: (step) => { + emitRequestProgress({ + type: 'replay-test', + ...attemptProgress, + status: 'progress', + stepIndex: step.index, + stepTotal: step.total, + ...(step.command !== undefined ? { stepCommand: step.command } : {}), + ...(step.value !== undefined ? { stepValue: step.value } : {}), + }); }, runReplay: params.runReplay, cleanupSession: params.cleanupSession, @@ -167,16 +192,31 @@ async function runSingleReplayTestAttempt( }); const durationMs = Date.now() - startedAt; materializeReplayTestAttemptArtifacts({ - response, + outcome, filePath: entry.path, sessionName: testSessionName, attempts: attempt, maxAttempts: context.maxAttempts, attemptArtifactsDir, }); - return { response, sessionName: testSessionName, attempt, durationMs }; + return { outcome, sessionName: testSessionName, attempt, durationMs }; } +/** The per-attempt reporter context every progress event for that attempt shares. */ +type ReplayTestAttemptProgressContext = Omit< + ReplayTestProgressEvent, + | 'type' + | 'status' + | 'stepIndex' + | 'stepTotal' + | 'stepCommand' + | 'stepValue' + | 'durationMs' + | 'retrying' + | 'message' + | 'hint' +>; + function emitReplayTestStartProgress( params: ReplayTestCaseParams, context: ReplayTestCaseContext, @@ -201,27 +241,27 @@ function updateReplayTestCaseOutcome( outcome: ReplayTestCaseOutcome, attempt: ReplayTestAttemptResult, ): void { - outcome.finalResponse = attempt.response; + outcome.finalOutcome = attempt.outcome; outcome.finalSessionName = attempt.sessionName; outcome.attempts = attempt.attempt; outcome.finalAttemptDurationMs = attempt.durationMs; - if (attempt.response.ok) return; + if (attempt.outcome.status === 'passed') return; outcome.attemptFailures.push({ attempt: attempt.attempt, - message: attempt.response.error.message, + message: attempt.outcome.error.message, durationMs: attempt.durationMs, }); } function shouldStopReplayTestAttempts( params: ReplayTestCaseParams, - response: DaemonResponse, + outcome: ReplayTestAttemptOutcome, attemptIndex: number, ): boolean { return ( - response.ok || + outcome.status === 'passed' || isRequestCanceled(params.requestId) || - isReplayInfrastructureFailure(response) || + outcome.infrastructure || attemptIndex >= params.retries ); } @@ -231,7 +271,7 @@ function emitReplayTestRetryProgress( context: ReplayTestCaseContext, attempt: ReplayTestAttemptResult, ): void { - if (attempt.response.ok) return; + if (attempt.outcome.status === 'passed') return; emitRequestProgress({ type: 'replay-test', file: params.entry.path, @@ -243,8 +283,8 @@ function emitReplayTestRetryProgress( maxAttempts: context.maxAttempts, durationMs: attempt.durationMs, retrying: true, - message: attempt.response.error.message, - hint: attempt.response.error.hint, + message: attempt.outcome.error.message, + hint: attempt.outcome.error.hint, session: attempt.sessionName, artifactsDir: context.testArtifactsDir, ...replayTestProgressShardMetadata(params.shard), @@ -257,7 +297,7 @@ function buildReplayTestCaseResult( outcome: ReplayTestCaseOutcome, ): ReplayTestCaseResult { const durationMs = Date.now() - context.testStartedAt; - if (outcome.finalResponse?.ok) { + if (outcome.finalOutcome?.status === 'passed') { return buildReplayTestPassedResult(params, context, outcome, durationMs); } return buildReplayTestFailedResult(params, context, outcome, durationMs); @@ -270,8 +310,8 @@ function buildReplayTestPassedResult( durationMs: number, ): Extract { const { entry, suiteIndex, suiteTotal, shard } = params; - const response = outcome.finalResponse; - if (!response?.ok) throw new Error('Expected passing replay test response.'); + const attemptOutcome = outcome.finalOutcome; + if (attemptOutcome?.status !== 'passed') throw new Error('Expected passing replay test outcome.'); emitRequestProgress({ type: 'replay-test', file: entry.path, @@ -295,9 +335,12 @@ function buildReplayTestPassedResult( finalAttemptDurationMs: outcome.finalAttemptDurationMs, attempts: outcome.attempts, artifactsDir: context.testArtifactsDir, - ...replayTestResponseMetrics(response), - ...replayTestWarningsResultMetadata(response.data?.warnings), - ...replayTestSnapshotDiagnosticsResultMetadata(response.data?.snapshotDiagnostics), + replayed: attemptOutcome.replayed, + healed: attemptOutcome.healed, + ...(attemptOutcome.warnings.length > 0 ? { warnings: [...attemptOutcome.warnings] } : {}), + ...(attemptOutcome.snapshotDiagnostics + ? { snapshotDiagnostics: attemptOutcome.snapshotDiagnostics } + : {}), ...replayTestShardResultMetadata(shard), ...(outcome.attemptFailures.length > 0 ? { attemptFailures: outcome.attemptFailures } : {}), }; @@ -310,7 +353,11 @@ function buildReplayTestFailedResult( durationMs: number, ): Extract { const { entry, suiteIndex, suiteTotal, shard } = params; - const error = replayTestFailureError(outcome.finalResponse); + const attemptOutcome = outcome.finalOutcome; + const error = + attemptOutcome?.status === 'failed' + ? attemptOutcome.error + : { code: 'COMMAND_FAILED', message: 'Unknown replay test failure' }; emitRequestProgress({ type: 'replay-test', file: entry.path, @@ -336,50 +383,13 @@ function buildReplayTestFailedResult( attempts: outcome.attempts, artifactsDir: context.testArtifactsDir, error, - ...replayTestSnapshotDiagnosticsResultMetadata( - readReplayResponseSnapshotDiagnostics(outcome.finalResponse), - ), + ...(attemptOutcome?.snapshotDiagnostics + ? { snapshotDiagnostics: attemptOutcome.snapshotDiagnostics } + : {}), ...replayTestShardResultMetadata(shard), }; } -function replayTestFailureError( - response: DaemonResponse | undefined, -): Extract['error'] { - if (response && !response.ok) return response.error; - return { code: 'COMMAND_FAILED', message: 'Unknown replay test failure' }; -} - -function replayTestResponseMetrics( - response: Extract, -): Pick, 'replayed' | 'healed'> { - return { - replayed: typeof response.data?.replayed === 'number' ? response.data.replayed : 0, - healed: typeof response.data?.healed === 'number' ? response.data.healed : 0, - }; -} - -function replayTestWarningsResultMetadata( - warnings: unknown, -): Pick, 'warnings'> { - if (!Array.isArray(warnings)) return {}; - const filtered = warnings.filter((entry): entry is string => typeof entry === 'string'); - return filtered.length > 0 ? { warnings: filtered } : {}; -} - -function replayTestSnapshotDiagnosticsResultMetadata( - value: unknown, -): Pick { - const snapshotDiagnostics = readSnapshotDiagnosticsSummary(value); - return snapshotDiagnostics ? { snapshotDiagnostics } : {}; -} - -function readReplayResponseSnapshotDiagnostics(response: DaemonResponse | undefined): unknown { - return response?.ok - ? response.data?.snapshotDiagnostics - : response?.error.details?.snapshotDiagnostics; -} - function replayTestShardResultMetadata( shard: ReplayTestShardContext | undefined, ): Pick { diff --git a/src/daemon/handlers/session-test-outcome.ts b/src/daemon/handlers/session-test-outcome.ts new file mode 100644 index 0000000000..d4b68d5a09 --- /dev/null +++ b/src/daemon/handlers/session-test-outcome.ts @@ -0,0 +1,63 @@ +import { readSnapshotDiagnosticsSummary } from '@agent-device/contracts/capture'; +import type { DaemonResponse } from '../types.ts'; +import { isReplayInfrastructureFailure } from './session-test-infrastructure.ts'; +import type { ReplayTestAttemptFailed, ReplayTestAttemptOutcome } from './session-test-types.ts'; + +/** + * The one place a `DaemonResponse` becomes a neutral replay-test attempt outcome (#1478 P3). + * + * The scheduler owns retries, fail-fast, artifacts, and result aggregation, but it must not + * read a daemon response shape to do so: `ok`, `data`, and `error.details` are the daemon's + * projection vocabulary, not the scheduler's. Everything the scheduler actually consumes is + * pulled out here into an explicit tagged value, including the infrastructure verdict, which + * needs platform boot-diagnostic vocabulary the scheduler may not import. + */ +export function toReplayTestAttemptOutcome(response: DaemonResponse): ReplayTestAttemptOutcome { + if (!response.ok) { + return { + status: 'failed', + error: response.error, + artifactPaths: readArtifactPaths(response.error.details?.artifactPaths), + infrastructure: isReplayInfrastructureFailure(response), + ...snapshotDiagnostics(response.error.details?.snapshotDiagnostics), + }; + } + const data = response.data as Record | undefined; + return { + status: 'passed', + replayed: typeof data?.replayed === 'number' ? data.replayed : 0, + healed: typeof data?.healed === 'number' ? data.healed : 0, + warnings: readStringArray(data?.warnings), + artifactPaths: readArtifactPaths(data?.artifactPaths), + ...snapshotDiagnostics(data?.snapshotDiagnostics), + }; +} + +/** + * Attempt finalization reports only failure: a successful finalization has nothing the + * scheduler can act on, and `undefined` is what its port already means by "nothing to report". + */ +export function toReplayTestFinalizeFailure( + response: DaemonResponse | undefined, +): ReplayTestAttemptFailed | undefined { + if (!response || response.ok) return undefined; + const outcome = toReplayTestAttemptOutcome(response); + return outcome.status === 'failed' ? outcome : undefined; +} + +function readStringArray(value: unknown): readonly string[] { + if (!Array.isArray(value)) return []; + return value.filter((entry): entry is string => typeof entry === 'string'); +} + +/** Artifact paths are copied by name, so the same path twice would collide on the copy. */ +function readArtifactPaths(value: unknown): readonly string[] { + return [...new Set(readStringArray(value))]; +} + +function snapshotDiagnostics( + value: unknown, +): Pick { + const summary = readSnapshotDiagnosticsSummary(value); + return summary ? { snapshotDiagnostics: summary } : {}; +} diff --git a/src/daemon/handlers/session-test-runtime.ts b/src/daemon/handlers/session-test-runtime.ts index cfd98103fc..f0be12c772 100644 --- a/src/daemon/handlers/session-test-runtime.ts +++ b/src/daemon/handlers/session-test-runtime.ts @@ -9,15 +9,14 @@ import { markRequestCanceled, registerRequestAbort, } from '../../request/cancel.ts'; -import { - type ReplayTestActionProgressContext, - withReplayTestActionProgress, -} from '../../request/progress.ts'; -import type { DaemonResponse } from '../types.ts'; import type { ReplayScriptMetadata } from '../../replay/script.ts'; -import type { - ReplayTestRunReplayParams, - ReplayTestRuntimeDependencies, +import { + replayTestAttemptFailure, + type ReplayTestAttemptFailed, + type ReplayTestAttemptOutcome, + type ReplayTestAttemptStepSink, + type ReplayTestRunReplayParams, + type ReplayTestRuntimeDependencies, } from './session-test-types.ts'; const REPLAY_TIMEOUT_CLEANUP_GRACE_MS = 2_000; @@ -36,9 +35,9 @@ export async function runReplayTestAttempt( target?: ReplayScriptMetadata['target']; artifactsDir?: string; shard?: ReplayTestRunReplayParams['shard']; - progress?: ReplayTestActionProgressContext; + onStep?: ReplayTestAttemptStepSink; } & ReplayTestRuntimeDependencies, -): Promise { +): Promise { const { filePath, sessionName, @@ -49,7 +48,7 @@ export async function runReplayTestAttempt( target, artifactsDir, shard, - progress, + onStep, runReplay, cleanupSession, finalizeAttempt, @@ -59,7 +58,7 @@ export async function runReplayTestAttempt( const artifactPaths = new Set(); let timeoutHandle: ReturnType | undefined; let timedOut = false; - let response: DaemonResponse | undefined; + let outcome: ReplayTestAttemptOutcome | undefined; const attemptStartedAt = Date.now(); const tracePath = prepareReplayTestTimingTrace({ artifactsDir, @@ -71,43 +70,34 @@ export async function runReplayTestAttempt( platform, target, }); - const replayPromise = withReplayTestActionProgress( - progress, - async () => - await runReplay({ - filePath, - sessionName, - platform, - target, - requestId, - artifactsDir, - artifactPaths, - tracePath, - shard, - }), - ) - .catch((error) => { - const appErr = normalizeError(error); - return { - ok: false, - error: appErr, - } satisfies DaemonResponse; - }) + const replayPromise = runReplay({ + filePath, + sessionName, + platform, + target, + requestId, + artifactsDir, + artifactPaths, + tracePath, + shard, + onStep, + }) + .catch((error) => replayTestAttemptFailure({ error: normalizeError(error) })) .finally(() => { clearParentAbortRelay(); clearRequestCanceled(requestId); }); try { - response = + outcome = typeof timeoutMs === 'number' ? await Promise.race([ replayPromise, - new Promise((resolve) => { + new Promise((resolve) => { timeoutHandle = setTimeout(() => { timedOut = true; markRequestCanceled(requestId); - resolve(createReplayTestTimeoutResponse(timeoutMs, [...artifactPaths])); + resolve(createReplayTestTimeoutOutcome(timeoutMs, [...artifactPaths])); }, timeoutMs); }), ]) @@ -116,17 +106,17 @@ export async function runReplayTestAttempt( type: 'replay_test_attempt_stop', ts: new Date().toISOString(), session: sessionName, - ok: response.ok, + ok: outcome.status === 'passed', timedOut, durationMs: Date.now() - attemptStartedAt, - errorCode: response.ok ? undefined : response.error.code, + errorCode: outcome.status === 'passed' ? undefined : outcome.error.code, }); } finally { if (timeoutHandle) clearTimeout(timeoutHandle); if (timedOut) { const settled = await waitForReplayAfterTimeout(replayPromise); if (!settled) { - markReplayTimeoutCleanupPending(response); + outcome = markReplayTimeoutCleanupPending(outcome); emitDiagnostic({ level: 'warn', phase: 'test_timeout_cleanup_race', @@ -144,17 +134,17 @@ export async function runReplayTestAttempt( }); } } - const finalizedResponse = await finalizeReplayTestAttempt({ + const finalizeFailure = await finalizeReplayTestAttempt({ finalizeAttempt, sessionName, artifactPaths, artifactsDir, tracePath, }); - if (response?.ok && finalizedResponse && !finalizedResponse.ok) { - appendReplayTestWarning( - response, - `Replay test finalization failed: ${finalizedResponse.error.message}`, + if (outcome?.status === 'passed' && finalizeFailure) { + outcome = appendReplayTestWarning( + outcome, + `Replay test finalization failed: ${finalizeFailure.error.message}`, ); } const cleanupStartedAt = Date.now(); @@ -193,13 +183,10 @@ export async function runReplayTestAttempt( } } return ( - response ?? { - ok: false, - error: { - code: 'COMMAND_FAILED', - message: 'Unknown replay test failure', - }, - } + outcome ?? + replayTestAttemptFailure({ + error: { code: 'COMMAND_FAILED', message: 'Unknown replay test failure' }, + }) ); } @@ -224,7 +211,9 @@ function relayReplayTestAbortFromParent( }; } -async function waitForReplayAfterTimeout(replayPromise: Promise): Promise { +async function waitForReplayAfterTimeout( + replayPromise: Promise, +): Promise { return await Promise.race([ replayPromise.then(() => true), sleep(REPLAY_TIMEOUT_CLEANUP_GRACE_MS).then(() => false), @@ -232,7 +221,7 @@ async function waitForReplayAfterTimeout(replayPromise: Promise) } async function cleanupSessionAfterLateReplay(params: { - replayPromise: Promise; + replayPromise: Promise; cleanupSession: ReplayTestRuntimeDependencies['cleanupSession']; sessionName: string; requestId: string; @@ -264,7 +253,7 @@ async function finalizeReplayTestAttempt(params: { artifactPaths: Set; artifactsDir?: string; tracePath?: string; -}): Promise { +}): Promise { const { finalizeAttempt, sessionName, artifactPaths, artifactsDir, tracePath } = params; if (!finalizeAttempt) return undefined; const finalizeStartedAt = Date.now(); @@ -284,9 +273,9 @@ async function finalizeReplayTestAttempt(params: { type: 'replay_test_finalize_stop', ts: new Date().toISOString(), session: sessionName, - ok: finalized?.ok ?? true, + ok: finalized === undefined, durationMs: Date.now() - finalizeStartedAt, - errorCode: finalized?.ok === false ? finalized.error.code : undefined, + errorCode: finalized?.error.code, }); return finalized; } catch (error) { @@ -307,28 +296,35 @@ async function finalizeReplayTestAttempt(params: { error: appErr.message, }, }); - return { ok: false, error: appErr }; + return replayTestAttemptFailure({ error: appErr }); } } -function markReplayTimeoutCleanupPending(response: DaemonResponse | undefined): void { - if (!response || response.ok) return; - response.error.details = { - ...(response.error.details ?? {}), - reason: REPLAY_TIMEOUT_CLEANUP_PENDING_REASON, - timeoutCleanupPending: true, +function markReplayTimeoutCleanupPending( + outcome: ReplayTestAttemptOutcome | undefined, +): ReplayTestAttemptOutcome | undefined { + if (!outcome || outcome.status === 'passed') return outcome; + return { + ...outcome, + // The abandoned replay is still running against the device, so the attempt is an + // environmental failure rather than a test failure: the suite must stop rather than retry. + infrastructure: true, + error: { + ...outcome.error, + details: { + ...(outcome.error.details ?? {}), + reason: REPLAY_TIMEOUT_CLEANUP_PENDING_REASON, + timeoutCleanupPending: true, + }, + }, }; } function appendReplayTestWarning( - response: Extract, + outcome: Extract, warning: string, -): void { - const data = (response.data ??= {}); - const warnings = Array.isArray(data.warnings) - ? data.warnings.filter((entry): entry is string => typeof entry === 'string') - : []; - data.warnings = [...warnings, warning]; +): ReplayTestAttemptOutcome { + return { ...outcome, warnings: [...outcome.warnings, warning] }; } function prepareReplayTestTimingTrace(params: { @@ -377,12 +373,12 @@ export function appendReplayTestTimingEvent( fs.appendFileSync(tracePath, `${JSON.stringify(event)}\n`); } -function createReplayTestTimeoutResponse( +function createReplayTestTimeoutOutcome( timeoutMs: number, artifactPaths: string[] = [], -): DaemonResponse { - return { - ok: false, +): ReplayTestAttemptOutcome { + return replayTestAttemptFailure({ + artifactPaths, error: { code: 'COMMAND_FAILED', message: `TIMEOUT after ${timeoutMs}ms`, @@ -394,5 +390,5 @@ function createReplayTestTimeoutResponse( artifactPaths, }, }, - }; + }); } diff --git a/src/daemon/handlers/session-test-types.ts b/src/daemon/handlers/session-test-types.ts index 04c98cc2ee..47d1d22d8f 100644 --- a/src/daemon/handlers/session-test-types.ts +++ b/src/daemon/handlers/session-test-types.ts @@ -1,7 +1,58 @@ -import type { DaemonResponse } from '../types.ts'; +import type { ReplaySuiteTestFailed } from '@agent-device/contracts/replay'; +import type { SnapshotDiagnosticsSummary } from '@agent-device/contracts/capture'; import type { ReplayScriptMetadata } from '../../replay/script.ts'; import type { ReplayTestShardContext } from './session-test-sharding.ts'; +/** + * One execution step an engine reports while an attempt runs (#1478 P3, finding 1). + * + * Step payloads originate below the attempt boundary, inside engine execution, and used to + * reach the reporter through a request-global `AsyncLocalStorage` seeded per attempt. The + * scheduler now hands each attempt a narrow sink instead, so step progress is an explicit + * per-attempt port with two real adapters (native `.ad` and Maestro) rather than ambient + * request state the scheduler cannot see. + */ +export type ReplayTestAttemptStep = { + index: number; + total: number; + command?: string; + value?: string; +}; + +export type ReplayTestAttemptStepSink = (step: ReplayTestAttemptStep) => void; + +/** + * ADR 0010 error fields exactly as the public suite result publishes them. This is the + * neutral wire error, not `DaemonResponse`: the scheduler never sees a daemon response shape. + */ +export type ReplayTestAttemptError = ReplaySuiteTestFailed['error']; + +export type ReplayTestAttemptPassed = { + status: 'passed'; + replayed: number; + healed: number; + warnings: readonly string[]; + artifactPaths: readonly string[]; + snapshotDiagnostics?: SnapshotDiagnosticsSummary; +}; + +export type ReplayTestAttemptFailed = { + status: 'failed'; + error: ReplayTestAttemptError; + artifactPaths: readonly string[]; + snapshotDiagnostics?: SnapshotDiagnosticsSummary; + /** + * The host's verdict that this failure is environmental (device/runner/boot) rather than a + * test failure, so retrying and continuing the suite cannot help. Classification needs + * platform boot-diagnostic vocabulary, which the scheduler must not import, so the host + * tags the outcome and the scheduler only reads the tag. + */ + infrastructure: boolean; +}; + +/** Every expected attempt state resolves as a tagged outcome; nothing throws across the seam. */ +export type ReplayTestAttemptOutcome = ReplayTestAttemptPassed | ReplayTestAttemptFailed; + export type ReplayTestRunReplayParams = { filePath: string; sessionName: string; @@ -12,21 +63,44 @@ export type ReplayTestRunReplayParams = { artifactPaths?: Set; tracePath?: string; shard?: ReplayTestShardContext; + onStep?: ReplayTestAttemptStepSink; }; -export type ReplayTestRunReplay = (params: ReplayTestRunReplayParams) => Promise; +export type ReplayTestRunReplay = ( + params: ReplayTestRunReplayParams, +) => Promise; export type ReplayTestCleanupSession = (sessionName: string) => Promise; +/** + * Runs after the attempt settles and before cleanup. Returns a failure outcome when + * finalization itself failed, or `undefined` when there was nothing to finalize. + */ export type ReplayTestFinalizeAttempt = (params: { sessionName: string; artifactPaths: Set; artifactsDir?: string; tracePath?: string; -}) => Promise; +}) => Promise; export type ReplayTestRuntimeDependencies = { runReplay: ReplayTestRunReplay; cleanupSession: ReplayTestCleanupSession; finalizeAttempt?: ReplayTestFinalizeAttempt; }; + +/** Neutral failure outcome helper; keeps timeout/unknown construction in one place. */ +export function replayTestAttemptFailure(params: { + error: ReplayTestAttemptError; + artifactPaths?: readonly string[]; + infrastructure?: boolean; + snapshotDiagnostics?: SnapshotDiagnosticsSummary; +}): ReplayTestAttemptFailed { + return { + status: 'failed', + error: params.error, + artifactPaths: params.artifactPaths ?? [], + infrastructure: params.infrastructure ?? false, + ...(params.snapshotDiagnostics ? { snapshotDiagnostics: params.snapshotDiagnostics } : {}), + }; +} diff --git a/src/daemon/handlers/session-test.ts b/src/daemon/handlers/session-test.ts index 67f0b78fff..e9a1b9c37a 100644 --- a/src/daemon/handlers/session-test.ts +++ b/src/daemon/handlers/session-test.ts @@ -15,8 +15,7 @@ import { resolveReplayTestRetries, resolveReplayTestTimeout, } from './session-test-discovery.ts'; -import { isReplayInfrastructureFailure } from './session-test-infrastructure.ts'; -import { runReplayTestCase } from './session-test-attempt.ts'; +import { runReplayTestCase, type ReplayTestCaseReport } from './session-test-attempt.ts'; import type { ReplayTestRuntimeDependencies } from './session-test-types.ts'; import { buildReplayTestShardPlan, @@ -309,7 +308,7 @@ async function runReplayTestEntriesInDiscoveryOrder( continue; } executed += 1; - const result = await runReplayTestCase({ + const report = await runReplayTestCase({ entry, sessionName, suiteInvocationId, @@ -325,8 +324,8 @@ async function runReplayTestEntriesInDiscoveryOrder( cleanupSession, finalizeAttempt, }); - results.push(result); - if (shouldStopReplayTestExecution(result, flags, requestId)) break; + results.push(report.result); + if (shouldStopReplayTestExecution(report, flags, requestId)) break; } return results; } @@ -362,7 +361,7 @@ async function runReplayTestEntries( for (const [entryIndex, queued] of entries.entries()) { if (isRequestCanceled(requestId)) break; const { entry, suiteIndex } = queued; - const result = await runReplayTestCase({ + const report = await runReplayTestCase({ entry, sessionName, suiteInvocationId, @@ -379,21 +378,21 @@ async function runReplayTestEntries( cleanupSession, finalizeAttempt, }); - results.push(result); - if (shouldStopReplayTestExecution(result, flags, requestId)) break; + results.push(report.result); + if (shouldStopReplayTestExecution(report, flags, requestId)) break; } return results; } function shouldStopReplayTestExecution( - result: ReplaySuiteTestResult, + report: ReplayTestCaseReport, flags: DaemonRequest['flags'], requestId: string | undefined, ): boolean { return ( isRequestCanceled(requestId) || - (flags?.failFast === true && result.status === 'failed') || - isReplayInfrastructureFailure(result) + (flags?.failFast === true && report.result.status === 'failed') || + report.infrastructure ); } diff --git a/src/mcp/tool-error.ts b/src/mcp/tool-error.ts index 9145409ed7..04f163881b 100644 --- a/src/mcp/tool-error.ts +++ b/src/mcp/tool-error.ts @@ -1,5 +1,5 @@ import { normalizeError, type NormalizedError } from '@agent-device/kernel/errors'; -import { formatReplayDivergenceReport } from '../replay/divergence.ts'; +import { formatReplayDivergenceReport } from '@agent-device/contracts/divergence'; /** * Shared MCP error normalization + text rendering (executor and router diff --git a/src/replay/test/reporters/__tests__/progress.test.ts b/src/replay/test/reporters/__tests__/progress.test.ts index 0916c72759..c583975adc 100644 --- a/src/replay/test/reporters/__tests__/progress.test.ts +++ b/src/replay/test/reporters/__tests__/progress.test.ts @@ -7,7 +7,7 @@ // `in`, so a key that exists today with an `undefined` value is part of the contract. import assert from 'node:assert/strict'; import { test } from 'vitest'; -import type { RequestProgressEvent } from '../../../../request/progress.ts'; +import type { RequestProgressEvent } from '@agent-device/contracts/progress'; import { toReplayTestReporterProgressEvent } from '../progress.ts'; const SESSION = 'default:test:req-7:1-checkout:attempt-2'; diff --git a/src/replay/test/reporters/default.ts b/src/replay/test/reporters/default.ts index 69dfb1d08c..67eb585f54 100644 --- a/src/replay/test/reporters/default.ts +++ b/src/replay/test/reporters/default.ts @@ -6,7 +6,7 @@ import { } from '../progress.ts'; import { formatDurationSeconds } from '../../../utils/duration-format.ts'; import { colorize, supportsColor } from '../../../utils/output.ts'; -import { formatReplayDivergenceReport } from '../../divergence.ts'; +import { formatReplayDivergenceReport } from '@agent-device/contracts/divergence'; import type { ReplayTestReporter, ReplayTestReporterContext, diff --git a/src/replay/test/reporters/progress.ts b/src/replay/test/reporters/progress.ts index 7fbda64c99..10c53e8c88 100644 --- a/src/replay/test/reporters/progress.ts +++ b/src/replay/test/reporters/progress.ts @@ -1,4 +1,4 @@ -import type { RequestProgressEvent } from '../../../request/progress.ts'; +import type { RequestProgressEvent } from '@agent-device/contracts/progress'; import type { ReplayTestReporterProgressEvent } from './types.ts'; export function toReplayTestReporterProgressEvent( diff --git a/src/replay/test/reporters/registry.ts b/src/replay/test/reporters/registry.ts index 564feb182a..e712af1c4d 100644 --- a/src/replay/test/reporters/registry.ts +++ b/src/replay/test/reporters/registry.ts @@ -1,5 +1,5 @@ import type { ReplaySuiteResult } from '@agent-device/contracts/replay'; -import type { RequestProgressEvent } from '../../../request/progress.ts'; +import type { RequestProgressEvent } from '@agent-device/contracts/progress'; import { createCustomReplayTestReporter } from './custom.ts'; import { createDefaultReplayTestReporter } from './default.ts'; import { getReplayTestExitCode } from './format.ts'; diff --git a/src/replay/test/reporting.ts b/src/replay/test/reporting.ts index 6ab915b67b..ae574f0f83 100644 --- a/src/replay/test/reporting.ts +++ b/src/replay/test/reporting.ts @@ -1,4 +1,4 @@ -import type { RequestProgressEvent } from '../../request/progress.ts'; +import type { RequestProgressEvent } from '@agent-device/contracts/progress'; import type { ReplaySuiteResult } from '@agent-device/contracts/replay'; import { getReplayTestReporterExitCode, diff --git a/src/request/progress.ts b/src/request/progress.ts index d6fdd0bd89..0b48a783ba 100644 --- a/src/request/progress.ts +++ b/src/request/progress.ts @@ -1,69 +1,12 @@ import { AsyncLocalStorage } from 'node:async_hooks'; +import type { RequestProgressEvent, RequestProgressSink } from '@agent-device/contracts/progress'; -export type ReplayTestSuiteProgressEvent = { - type: 'replay-test-suite'; - status: 'start'; - total: number; - runnable: number; - skipped: number; - artifactsDir: string; - shardMode?: 'all' | 'split'; - shardCount?: number; -}; - -export type ReplayTestProgressEvent = { - type: 'replay-test'; - file: string; - title?: string; - status: 'start' | 'progress' | 'pass' | 'fail' | 'skip'; - index: number; - total: number; - stepIndex?: number; - stepTotal?: number; - stepCommand?: string; - stepValue?: string; - attempt?: number; - maxAttempts?: number; - durationMs?: number; - retrying?: boolean; - message?: string; - hint?: string; - session?: string; - artifactsDir?: string; - shardIndex?: number; - shardCount?: number; - deviceId?: string; - deviceName?: string; -}; - -export type CommandProgressEvent = { - type: 'command'; - status: 'progress'; - message: string; -}; - -export type RequestProgressEvent = - | ReplayTestSuiteProgressEvent - | ReplayTestProgressEvent - | CommandProgressEvent; -export type RequestProgressSink = (event: RequestProgressEvent) => void; -export type ReplayTestActionProgressContext = Omit< - ReplayTestProgressEvent, - | 'type' - | 'status' - | 'stepIndex' - | 'stepTotal' - | 'stepCommand' - | 'stepValue' - | 'durationMs' - | 'retrying' - | 'message' ->; +// The event vocabulary is a wire contract shared with the CLI reporter path, so it lives in +// `@agent-device/contracts/progress`. This module owns only the request-global plumbing that +// carries it: the per-request sink and its AsyncLocalStorage binding. +export type { RequestProgressEvent, RequestProgressSink } from '@agent-device/contracts/progress'; const requestProgress = new AsyncLocalStorage(); -const replayTestActionProgress = new AsyncLocalStorage< - ReplayTestActionProgressContext | undefined ->(); export async function withRequestProgressSink( sink: RequestProgressSink | undefined, @@ -75,14 +18,3 @@ export async function withRequestProgressSink( export function emitRequestProgress(event: RequestProgressEvent): void { requestProgress.getStore()?.(event); } - -export async function withReplayTestActionProgress( - context: ReplayTestActionProgressContext | undefined, - run: () => Promise, -): Promise { - return await replayTestActionProgress.run(context, run); -} - -export function readReplayTestActionProgress(): ReplayTestActionProgressContext | undefined { - return replayTestActionProgress.getStore(); -} diff --git a/src/utils/output.ts b/src/utils/output.ts index 68dea1f54a..28340ea5d6 100644 --- a/src/utils/output.ts +++ b/src/utils/output.ts @@ -5,7 +5,7 @@ import { } from './android-helper-snapshot-presentation.ts'; import { AppError, normalizeError, type NormalizedError } from '@agent-device/kernel/errors'; import { detectPossibleRepeatedNavSubtree } from './repeated-nav-subtree.ts'; -import { formatReplayDivergenceReport } from '../replay/divergence.ts'; +import { formatReplayDivergenceReport } from '@agent-device/contracts/divergence'; import { buildSnapshotDisplayLines, formatSnapshotLine } from '../snapshot/snapshot-lines.ts'; import { isSnapshotBackend,