From af2518bdaaf37835db98c553a2b45e4f90fdd318 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 29 Aug 2026 19:10:50 +0000 Subject: [PATCH] test: fail the runtime capture fast with phase attribution instead of an opaque timeout The runtime-playground capture child process had bounded browser waits but no overall deadline and an unbounded cleanup path, so any wedge (fixture start, a stuck retry loop, a hung dev-server close) surfaced as the calling test's 600s rstest timeout with zero child output - exactly what the Verify (Node 24) failure on the #35 merge commit showed. Add an in-process 480s watchdog that exits with the current capture phase on stderr, phase markers across the capture stages, and a 30s bound per cleanup step so a wedged close records a failed step instead of holding the process open. --- .../scripts/capture-runtime-playground.mjs | 64 +++++++++++++++++-- .../tests/runtime-playground-capture.test.ts | 10 +++ 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/packages/workbench/scripts/capture-runtime-playground.mjs b/packages/workbench/scripts/capture-runtime-playground.mjs index 139c4a710..730782d97 100644 --- a/packages/workbench/scripts/capture-runtime-playground.mjs +++ b/packages/workbench/scripts/capture-runtime-playground.mjs @@ -8,7 +8,41 @@ import { timeScale } from '../../agent-bundle/tests/support/time-scale.ts'; import { startRuntimePlaygroundFixture } from '../tests/helpers/runtime-playground-fixture.ts'; const browserTimeout = 30_000 * timeScale; +/** + * Hard ceiling for the whole capture, comfortably under the calling test's + * 600s budget so a wedge fails HERE with the current phase on stderr instead + * of as an opaque rstest timeout that surfaces no child output at all. + */ +const captureDeadline = 480_000; +/** Budget per cleanup step; a wedged dev-server close must not hold the process. */ +const cleanupStepTimeout = 30_000; const desktopViewport = Object.freeze({ height: 900, width: 1440 }); + +let currentPhase = 'parse-arguments'; +/** Marks capture progress so a watchdog failure can say where the run wedged. */ +const phase = (name) => { currentPhase = name; }; + +/** + * Bounds one cleanup action. `action` is invoked lazily so a step that loses + * the race can still have its eventual rejection observed instead of + * surfacing as an unhandled rejection. + */ +const boundedStep = async (name, action, stepTimeout) => { + const timedOut = Symbol(name); + let timer; + const work = (async () => action())(); + try { + const outcome = await Promise.race([ + work, + new Promise((resolveTimeout) => { timer = setTimeout(() => { resolveTimeout(timedOut); }, stepTimeout); }), + ]); + if (outcome !== timedOut) return outcome; + } finally { + clearTimeout(timer); + } + work.catch(() => {}); + throw new Error(`Capture cleanup step ${name} exceeded ${stepTimeout}ms.`); +}; const outputFlags = Object.freeze([ '--desktop', '--hmr-before', @@ -87,19 +121,22 @@ const writeEvidence = (path, evidence) => atomically(path, async (temporary) => await writeFile(temporary, `${JSON.stringify(evidence, null, 2)}\n`, 'utf8'); }); -export const cleanupCaptureResources = async ({ browser, fixture, restores }) => { - const settledRestores = await Promise.allSettled(restores.map(async (restore) => restore())); +export const cleanupCaptureResources = async ({ browser, fixture, restores, stepTimeout = cleanupStepTimeout }) => { + phase('cleanup'); + const settledRestores = await Promise.allSettled( + restores.map(async (restore, index) => boundedStep(`restore-${index + 1}`, restore, stepTimeout)), + ); const failedSteps = settledRestores.flatMap((result, index) => result.status === 'rejected' ? [`restore-${index + 1}`] : []); if (browser !== undefined) { try { - await browser.close(); + await boundedStep('browser.close', () => browser.close(), stepTimeout); } catch { failedSteps.push('browser.close'); } } if (fixture !== undefined) { try { - await fixture.close(); + await boundedStep('fixture.close', () => fixture.close(), stepTimeout); } catch { failedSteps.push('fixture.close'); } @@ -359,17 +396,20 @@ const capture = async (outputs) => { let primaryFailure; let evidence; try { + phase('start-fixture'); fixture = await startRuntimePlaygroundFixture(); originals = await Promise.all([ readFile(fixture.serverComponentSource, 'utf8'), readFile(fixture.widgetAppSource, 'utf8'), readFile(fixture.appStyles, 'utf8'), ]); + phase('launch-browser'); browser = await chromium.launch({ channel: 'chrome', headless: true }); const context = await browser.newContext({ viewport: desktopViewport }); const page = await context.newPage(); const pageErrors = []; page.on('pageerror', (error) => pageErrors.push(error.message)); + phase('load-workbench'); await page.goto(`${fixture.url}#runtime`, { waitUntil: 'domcontentloaded' }); await page.getByRole('heading', { name: 'Runtime Playground' }).waitFor({ state: 'visible', timeout: browserTimeout }); const identity = page.locator('[data-runtime-provider-session]'); @@ -389,6 +429,7 @@ const capture = async (outputs) => { const documentMarker = 'runtime-capture-document'; await page.evaluate((value) => { globalThis.document.documentElement.dataset.runtimeCaptureDocument = value; }, documentMarker); + phase('initial-run'); const runBefore = await runSurface(page, 'mcp.render_edit_timeline', {}); await selectRun(page, runBefore); await showRuntimeApp(page, runBefore); @@ -399,6 +440,7 @@ const capture = async (outputs) => { if (editedServer === originals[0]) throw new Error('Capture fixture server source did not contain the expected HMR literal.'); const repairedServer = editedServer.replace('Live runtime state now contains', 'Recovered runtime state now contains'); if (repairedServer === editedServer) throw new Error('Capture fixture server source did not contain the expected repair literal.'); + phase('hmr-edit'); const history = page.getByRole('region', { name: 'Runtime run history' }).locator('ol > li'); const historyBeforeHmr = await history.count(); const runIdsBeforeHmr = await runtimeRunIds(page); @@ -416,6 +458,7 @@ const capture = async (outputs) => { const appVisibleAfter = true; await screenshot(page, outputs.hmrAfter); + phase('compact-run'); const compactRunId = await runSurface(page, 'mcp.recent_edits', {}); await selectRun(page, compactRunId); await page.waitForFunction( @@ -428,6 +471,7 @@ const capture = async (outputs) => { throw new Error('Runtime capture compact run omitted its generation.'); } const lastGoodGenerationDuringError = compactRunGeneration; + phase('compile-error'); const historyBeforeError = await history.count(); const eventSequenceBeforeError = Number((await attributes(identity))['data-runtime-event-sequence']); if (!Number.isFinite(eventSequenceBeforeError)) throw new Error('Runtime identity omitted its event sequence.'); @@ -463,6 +507,7 @@ const capture = async (outputs) => { const compileErrorLayout = await captureCompileErrorLayout(page, compactRunGeneration); await screenshot(page, outputs.compileError); + phase('recovery'); await replaceWatchedSource(fixture.root, fixture.serverComponentSource, repairedServer); const generationRecovered = await waitForNewGeneration(page, lastGoodGenerationDuringError); await page.waitForFunction( @@ -476,6 +521,7 @@ const capture = async (outputs) => { { timeout: browserTimeout }, ); + phase('app-refresh'); const runWithApp = await runSurface(page, 'mcp.render_edit_timeline', {}); if (runWithApp === runAfter) throw new Error('Runtime App capture did not create a fresh explicit run after recovery.'); await selectRun(page, runWithApp); @@ -528,6 +574,7 @@ const capture = async (outputs) => { && element.getAttribute('data-runtime-capture-outer') === value, outerMarker) && await outerHandle.getAttribute('src') === outerSource; if (!appRefreshPreservedDocument) throw new Error('Runtime App refresh replaced the Workbench document or outer frame.'); + phase('desktop-capture'); await outerFrame.scrollIntoViewIfNeeded(); await marker.scrollIntoViewIfNeeded(); const desktopControlColumns = await page.evaluate(() => { @@ -540,6 +587,7 @@ const capture = async (outputs) => { if (desktopControlColumns !== 4) throw new Error(`Runtime capture expected four desktop control columns, received ${desktopControlColumns}.`); await screenshot(page, outputs.desktop); + phase('restore-sources'); await Promise.all([ restore(fixture.root, fixture.serverComponentSource, originals[0]), restore(fixture.root, fixture.widgetAppSource, originals[1]), @@ -604,6 +652,14 @@ const run = async () => { }; if (process.argv[1] !== undefined && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + // A wedge anywhere (fixture start, a stuck retry loop, cleanup) must fail + // fast with attribution: the calling test only surfaces this process's + // stderr when it exits, so an in-process deadline beats an opaque outer + // test timeout. unref keeps healthy runs exiting naturally. + setTimeout(() => { + process.stderr.write(`Capture watchdog: run exceeded ${captureDeadline}ms during phase ${currentPhase}.\n`); + process.exit(1); + }, captureDeadline).unref(); run().catch((error) => { process.stderr.write(`${formatCaptureFailure(error)}\n`); process.exitCode = 1; diff --git a/packages/workbench/tests/runtime-playground-capture.test.ts b/packages/workbench/tests/runtime-playground-capture.test.ts index 1c4868846..9bbfc8b4b 100644 --- a/packages/workbench/tests/runtime-playground-capture.test.ts +++ b/packages/workbench/tests/runtime-playground-capture.test.ts @@ -121,6 +121,16 @@ test('settles every capture cleanup action without masking the primary failure', } }); +test('bounds a wedged cleanup step instead of holding the capture process open', async () => { + const cleanup = await cleanupCaptureResources({ + browser: { close: async () => new Promise(() => {}) }, + fixture: { close: async () => {} }, + restores: [async () => new Promise(() => {})], + stepTimeout: 50, + }); + expect(cleanup).toEqual({ attemptedRestores: 1, failedSteps: ['restore-1', 'browser.close'] }); +}); + test('captures identity-backed HMR, last-good, recovery, and desktop browser evidence', { timeout: 600_000 }, async () => { const outputRoot = await mkdtemp(join(tmpdir(), 'agent-bundle-runtime-capture-')); const outputs = Object.freeze({