From 698af52de7999f57ee0bad13fd3329aab81151b3 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 5 Sep 2026 02:13:30 +0000 Subject: [PATCH 01/15] test(proxy): make the client-surface proxy upstream timeout injectable; bound the deadline tests at 500 ms (#576) --- .../src/dev/runtime-client-surface-proxy.ts | 29 ++++++++++++-- .../runtime-client-surface-proxy.test.ts | 40 +++++++++++++++---- 2 files changed, 59 insertions(+), 10 deletions(-) diff --git a/packages/agent-bundle/src/dev/runtime-client-surface-proxy.ts b/packages/agent-bundle/src/dev/runtime-client-surface-proxy.ts index dbf38d172..2b60e2c72 100644 --- a/packages/agent-bundle/src/dev/runtime-client-surface-proxy.ts +++ b/packages/agent-bundle/src/dev/runtime-client-surface-proxy.ts @@ -23,7 +23,10 @@ import { const appAssetLimit = 4 * 1024 * 1024; const headerLimit = 16 * 1024; -const upstreamRequestTimeout = 15_000; +/** Bound on each upstream compiler request unless `RuntimeClientSurfaceProxyOptions` overrides it. */ +export const defaultRuntimeClientSurfaceUpstreamRequestTimeoutMs = 15_000; +/** Node collapses longer `setTimeout` delays to 1 ms, which would silently drop the bound. */ +const maximumTimerDelayMs = 2_147_483_647; const loopbackHosts = new Set(['127.0.0.1', '::1']); /** * Proxy-owned browser push channel. The proxy authors both ends: its server @@ -60,6 +63,16 @@ export interface RuntimeClientSurfaceConnectionEvent { readonly type: 'connected' | 'disconnected'; } +/** Server-only tuning for `RuntimeClientSurfaceProxy.open`; production callers take the defaults. */ +export interface RuntimeClientSurfaceProxyOptions { + /** + * Bound on each upstream compiler request — the bootstrap entry fetch and + * every proxied asset — from dispatch until its body has been read. A + * request still open at the deadline is aborted and answered 502. + */ + readonly upstreamRequestTimeoutMs?: number; +} + interface ValidatedEndpoint { readonly entryPath: string; readonly host: string; @@ -134,6 +147,14 @@ const contentSecurityPolicy = (input: RuntimeClientSurfaceContentPolicy): string return value; }; +const upstreamRequestTimeoutMs = (options: RuntimeClientSurfaceProxyOptions): number => { + const timeout = options.upstreamRequestTimeoutMs ?? defaultRuntimeClientSurfaceUpstreamRequestTimeoutMs; + if (!Number.isSafeInteger(timeout) || timeout < 1 || timeout > maximumTimerDelayMs) { + throw new TypeError(`Runtime client surface proxy options must use an integer upstreamRequestTimeoutMs from 1 to ${String(maximumTimerDelayMs)}.`); + } + return timeout; +}; + const response = (target: ServerResponse, status: number): void => { if (target.destroyed || target.writableEnded) return; target.writeHead(status, { 'content-type': 'text/plain; charset=utf-8', 'x-content-type-options': 'nosniff' }); @@ -591,10 +612,12 @@ export class RuntimeClientSurfaceProxy { listener: (event: RuntimeClientSurfaceConnectionEvent) => void, hostOrigin: string, policy: RuntimeClientSurfaceContentPolicy = strictRuntimeClientSurfaceContentPolicy, + options: RuntimeClientSurfaceProxyOptions = {}, ): Promise { const trusted = endpoint(input); const trustedHostOrigin = canonicalHostOrigin(hostOrigin); const trustedContentSecurityPolicy = contentSecurityPolicy(policy); + const trustedUpstreamRequestTimeoutMs = upstreamRequestTimeoutMs(options); const bootstrapCapability = randomBytes(32).toString('base64url'); const sessionCapability = randomBytes(32).toString('base64url'); const bootstrapPath = `/__agent_bundle_runtime/bootstrap/${bootstrapCapability}`; @@ -656,7 +679,7 @@ export class RuntimeClientSurfaceProxy { const deadline = setTimeout(() => { abort(); rejectPromise(new Error('Runtime client entry timed out.')); - }, upstreamRequestTimeout); + }, trustedUpstreamRequestTimeoutMs); const upstreamRequest = requestUpstream({ agent: upstreamAgent, headers: { accept: 'text/html' }, @@ -772,7 +795,7 @@ export class RuntimeClientSurfaceProxy { timedOut = true; response(target, 502); abort(); - }, upstreamRequestTimeout); + }, trustedUpstreamRequestTimeoutMs); const upstreamRequest = requestUpstream({ agent: upstreamAgent, headers: { diff --git a/packages/agent-bundle/tests/runtime-client-surface-proxy.test.ts b/packages/agent-bundle/tests/runtime-client-surface-proxy.test.ts index ccbc99194..2656e2a0e 100644 --- a/packages/agent-bundle/tests/runtime-client-surface-proxy.test.ts +++ b/packages/agent-bundle/tests/runtime-client-surface-proxy.test.ts @@ -10,10 +10,20 @@ import { type DevRuntimeClientSurfaceEndpoint, } from '../src/dev/index.ts'; import { runtimeAppMessageLimits } from '../src/dev/runtime-app-message-limits.ts'; +import { + defaultRuntimeClientSurfaceUpstreamRequestTimeoutMs, + type RuntimeClientSurfaceProxyOptions, +} from '../src/dev/runtime-client-surface-proxy.ts'; const foregroundOrigin = 'http://127.0.0.1:41999'; const noopSubscribeReload = (): (() => void) => () => undefined; const reloadFrame = (generation: number): string => JSON.stringify({ generation, kind: 'runtime-app-reload' }); +/** + * Deadline tests bound the upstream far below the production default: long + * enough for the loopback bootstrap fetch that shares the bound, short enough + * that the file no longer waits on the real 15 s. + */ +const shortUpstreamRequestTimeout: RuntimeClientSurfaceProxyOptions = Object.freeze({ upstreamRequestTimeoutMs: 500 }); /** Provider-side reload authority stub: the trusted channel the proxy relays. */ const createReloadSource = () => { @@ -31,7 +41,8 @@ const RuntimeClientSurfaceProxy = Object.freeze({ open: ( input: DevRuntimeClientSurfaceEndpoint, listener: Parameters[1], - ) => RuntimeClientSurfaceProxyImplementation.open(input, listener, foregroundOrigin), + options?: RuntimeClientSurfaceProxyOptions, + ) => RuntimeClientSurfaceProxyImplementation.open(input, listener, foregroundOrigin, undefined, options), }); const listen = async (server: ReturnType): Promise => { @@ -573,6 +584,19 @@ it('rejects a custom-prototype child policy before opening a proxy binding', asy }, () => undefined, foregroundOrigin, policy as never)).rejects.toThrow('plain policy record'); }); +it('bounds upstream requests at 15 s by default and rejects out-of-range overrides before opening', async () => { + expect(defaultRuntimeClientSurfaceUpstreamRequestTimeoutMs).toBe(15_000); + for (const upstreamRequestTimeoutMs of [0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY, 2_147_483_648]) { + await expect(RuntimeClientSurfaceProxy.open({ + entryPath: '/app/index.html', + httpOrigin: 'http://127.0.0.1:41998', + httpPathPrefixes: ['/app/'], + surfaceId: 'app.weather', + subscribeReload: noopSubscribeReload, + }, () => undefined, { upstreamRequestTimeoutMs })).rejects.toThrow('upstreamRequestTimeoutMs from 1 to'); + } +}); + it('does not reinstall the opaque child when a held refresh fetch resolves after pagehide', async () => { const upstream = createServer((request, response) => { if (serveBootstrapEntry(request, response)) return; @@ -1020,19 +1044,20 @@ it('bounds an upstream HTTP request before headers arrive', async () => { httpPathPrefixes: ['/app/'], surfaceId: 'app.weather', subscribeReload: noopSubscribeReload, - }, () => undefined); + }, () => undefined, shortUpstreamRequestTimeout); try { const cookie = await bootstrapCookie(binding); const pending = fetch(`${binding.origin}/app/index.html`, { headers: { cookie } }); void pending.catch(() => undefined); - await expect(within(pending, 16_000)).resolves.toMatchObject({ status: 502 }); + // Well under the 15 s default: a proxy that ignored the override would fail here. + await expect(within(pending, 2_000)).resolves.toMatchObject({ status: 502 }); } finally { await binding.close(); upstream.closeAllConnections(); await close(upstream); } -}, 20_000); +}); it('releases a reload-channel client that writes into the strictly one-way channel', async () => { const upstream = createServer((request, response) => { @@ -1162,7 +1187,7 @@ it('keeps a completed 502 response intact when a response body stalls after head httpPathPrefixes: ['/app/'], surfaceId: 'app.weather', subscribeReload: noopSubscribeReload, - }, () => undefined); + }, () => undefined, shortUpstreamRequestTimeout); try { const cookie = await bootstrapCookie(binding); @@ -1171,14 +1196,15 @@ it('keeps a completed 502 response intact when a response body stalls after head status: response.status, })); void pending.catch(() => undefined); - await expect(within(pending, 16_000)).resolves.toEqual({ body: 'Not Found', status: 502 }); + // Well under the 15 s default: a proxy that ignored the override would fail here. + await expect(within(pending, 2_000)).resolves.toEqual({ body: 'Not Found', status: 502 }); await expect(within(socketClosed, 250)).resolves.toBeUndefined(); } finally { await binding.close(); upstream.closeAllConnections(); await close(upstream); } -}, 20_000); +}); it('bounds chunked upstream assets and releases their socket immediately', async () => { let resolveSocketClosed: (() => void) | undefined; From a67e3c82871f0e79c8a2028e2e43cf0703e6b4a5 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 5 Sep 2026 02:13:30 +0000 Subject: [PATCH 02/15] test(dev-host-install): stub the coordinator watcher in the Cursor re-sync test (#576) --- packages/agent-bundle/tests/dev-host-install.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/agent-bundle/tests/dev-host-install.test.ts b/packages/agent-bundle/tests/dev-host-install.test.ts index 0ecc39858..c3bd0cb7d 100644 --- a/packages/agent-bundle/tests/dev-host-install.test.ts +++ b/packages/agent-bundle/tests/dev-host-install.test.ts @@ -316,6 +316,10 @@ it('re-syncs the isolated Cursor install from coordinator epochs and ignores a f }); const coordinator = new DevCoordinator({ acquireLock: async () => ({ close: async () => undefined }), + // A no-op watcher: the real ProjectWatcher would turn this test's own source + // writes into an unrequested second rebuild that races the explicit + // rebuild() calls and can rewrite the install marker after settled() (#576). + createWatcher: () => ({ close: async () => undefined }), epochStore, eventHub, prepareCommand: 'dev', From 60c64635c7b83e74e319239ad60199a72440fa9f Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 5 Sep 2026 02:13:30 +0000 Subject: [PATCH 03/15] test(scaffold-matrix): assert the scaffolded pools through Rstest's json report, not reporter prose (#576) --- .../tests/scaffold-packed-matrix.e2e.test.ts | 19 ++++--- .../tests/support/scaffold-fixture.ts | 52 +++++++++++++++++++ 2 files changed, 61 insertions(+), 10 deletions(-) diff --git a/packages/create-agent-bundle/tests/scaffold-packed-matrix.e2e.test.ts b/packages/create-agent-bundle/tests/scaffold-packed-matrix.e2e.test.ts index b425bd3c6..a9aea40e9 100644 --- a/packages/create-agent-bundle/tests/scaffold-packed-matrix.e2e.test.ts +++ b/packages/create-agent-bundle/tests/scaffold-packed-matrix.e2e.test.ts @@ -11,6 +11,7 @@ import { installedEnvironment, packOutputFromJson } from '../../agent-bundle/tes import { cleanupScaffoldFixture, expectCleanValidate, + expectPassedPool, installScaffoldedProject, npmRun, scaffoldProject, @@ -45,12 +46,10 @@ it.concurrent('scaffolds the mcp-server template and serves the conventional ent expect(checked).toContain('tests/projection/mcp-in-memory.test.ts'); // The route-unit pool renders through the framework's own generated setup, // resolved from the packed tarball's `agent-bundle/rstest` export. - const routes = await npmRun(projectRoot, 'test:routes'); - expect(routes).toContain('renders a known service into a final Agent Document'); - expect(routes).toContain('"failedTests": 0'); - const projection = await npmRun(projectRoot, 'test:projection'); - expect(projection).toContain('projects the rendered document into the protocol result the server returns'); - expect(projection).toContain('"failedTests": 0'); + await expectPassedPool(projectRoot, 'test:routes', ['renders a known service into a final Agent Document']); + await expectPassedPool(projectRoot, 'test:projection', [ + 'projects the rendered document into the protocol result the server returns', + ]); const artifact = join(projectRoot, 'artifact'); const manifest = JSON.parse(await readFile(join(artifact, 'portable', 'mcp.json'), 'utf8')) as { @@ -92,10 +91,10 @@ it.concurrent('scaffolds the cli-tool template with a routed bin, lib, and artif await npmRun(projectRoot, 'prepack'); // The projection pool dispatches through the framework's own generated // setup, resolved from the packed tarball's `agent-bundle/rstest` export. - const projection = await npmRun(projectRoot, 'test:projection'); - expect(projection).toContain('greets through the routed CLI shell and prints one canonical JSON line'); - expect(projection).toContain('greets through the main process envelope'); - expect(projection).toContain('"failedTests": 0'); + await expectPassedPool(projectRoot, 'test:projection', [ + 'greets through the routed CLI shell and prints one canonical JSON line', + 'greets through the main process envelope', + ]); // The src/cli/** convention produced the routed executable package bin: // generated help, the compiled argv grammar, and one canonical JSON line. diff --git a/packages/create-agent-bundle/tests/support/scaffold-fixture.ts b/packages/create-agent-bundle/tests/support/scaffold-fixture.ts index af0ef6fdd..b80e99452 100644 --- a/packages/create-agent-bundle/tests/support/scaffold-fixture.ts +++ b/packages/create-agent-bundle/tests/support/scaffold-fixture.ts @@ -146,6 +146,58 @@ export const npmRun = async (projectRoot: string, script: string): Promise => { + const stdout = await execFile('npm', ['run', script, '--', '--reporter=json'], { + cwd: projectRoot, + env: installedEnvironment(), + }).then((result) => result.stdout, (error: unknown) => { + const failed = error as { readonly stdout?: string }; + if (typeof failed.stdout !== 'string') throw error; + return failed.stdout; + }); + // npm's script banner and Rstest's own precede the report on stdout; the + // report is the only thing there that opens a line with `{`. + const start = stdout.search(/^\{$/mu); + if (start === -1) throw new Error(`\`npm run ${script}\` wrote no Rstest JSON report:\n${stdout}`); + return JSON.parse(stdout.slice(start)) as PoolReport; +}; + +/** + * The pool passed and ran the named tests. Failing entries come first — a + * test's, then a file's, for a file that failed before it had tests — because + * they carry the error where the counts would only say that something + * failed. The names catch a dropped or empty pool, which Rstest reports as + * `fail` with zero tests — and `failedTests: 0`. + */ +export const expectPassedPool = async ( + projectRoot: string, + script: string, + testNames: readonly string[], +): Promise => { + const report = await poolReport(projectRoot, script); + expect(report.tests.filter((test) => test.status === 'fail')).toEqual([]); + expect(report.files.filter((file) => file.status === 'fail')).toEqual([]); + expect(report.tests.map((test) => test.name)).toEqual(expect.arrayContaining([...testNames])); + expect(report).toMatchObject({ status: 'pass', summary: { failedTests: 0 } }); +}; + /** Zero diagnostics — including the informational AB473x migration nudges. */ export const expectCleanValidate = async (projectRoot: string): Promise => { const cli = join(projectRoot, 'node_modules', '.bin', 'agent-bundle'); From 7ac23599492e64ce63c55217006d10e67a6440b2 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 5 Sep 2026 02:13:30 +0000 Subject: [PATCH 04/15] test(mcp-app-real): publish the relay state on the iframe and wait on it before closing (#576) --- packages/workbench/src/mcp/mcp-app-frame.tsx | 24 ++++++++++ .../workbench/tests/mcp-app-real.e2e.test.ts | 48 +++++++++++++------ 2 files changed, 58 insertions(+), 14 deletions(-) diff --git a/packages/workbench/src/mcp/mcp-app-frame.tsx b/packages/workbench/src/mcp/mcp-app-frame.tsx index 7c84c2454..cc8186835 100644 --- a/packages/workbench/src/mcp/mcp-app-frame.tsx +++ b/packages/workbench/src/mcp/mcp-app-frame.tsx @@ -29,6 +29,8 @@ export interface McpAppFrameTarget { export interface McpAppFrameIframe { readonly contentWindow: McpAppFrameTarget | null; + /** A DOM iframe publishes the relay lifecycle through this (`relayStateAttribute`); fakes may omit it. */ + setAttribute?(name: string, value: string): void; } export interface McpAppFrameWindow { @@ -75,6 +77,18 @@ export interface SecureAppRendererProps { type RelayState = 'closed' | 'closing' | 'open'; +/** + * Relay lifecycle as published on the outer iframe's `data-mcp-app-relay-state` + * attribute, so the host UI and browser tests observe the state the relay is + * in instead of inferring it from route traffic. `loading`: listening, but the + * proxy has not signalled readiness, so `close()` releases the binding with a + * forced DELETE. `ready`: the proxy holds the resource, so `close()` runs the + * graceful `POST …/close` teardown handshake. `closing` and `closed` mirror + * `RelayState`. + */ +type RelayFrameState = 'closed' | 'closing' | 'loading' | 'ready'; +const relayStateAttribute = 'data-mcp-app-relay-state'; + interface CanonicalResource { readonly csp?: McpAppJsonValue; readonly html: string; @@ -193,6 +207,7 @@ export class McpAppFrameRelay { if (this.#state !== 'open' || this.#listening) return false; this.#window.addEventListener('message', this.#listener); this.#listening = true; + this.#publishFrameState(); return true; } @@ -209,6 +224,7 @@ export class McpAppFrameRelay { if (isProxyReady(message)) { if (this.#resourceProvided) return false; this.#resourceProvided = true; + this.#publishFrameState(); return this.#post(messageForResource(this.#frame, this.#resource), false); } if (!this.#resourceProvided) return false; @@ -219,6 +235,7 @@ export class McpAppFrameRelay { if (this.#state === 'closed') return closedRelay; if (this.#closePromise !== undefined) return this.#closePromise; this.#state = 'closing'; + this.#publishFrameState(); this.#closePromise = new Promise((resolve) => { this.#finishClose = resolve; }); // Before the proxy signals readiness there is no app to tear down and no // window that can acknowledge a teardown frame: the proxy document is @@ -361,6 +378,7 @@ export class McpAppFrameRelay { #completeClose(): void { if (this.#state === 'closed') return; this.#state = 'closed'; + this.#publishFrameState(); this.#queue.length = 0; if (this.#closeTimer !== undefined) clearTimeout(this.#closeTimer); this.#closeTimer = undefined; @@ -370,6 +388,12 @@ export class McpAppFrameRelay { this.#finishClose = undefined; } + /** Mirrors `#state` and `#resourceProvided` onto the iframe on the same event that changes them. */ + #publishFrameState(): void { + const state: RelayFrameState = this.#state === 'open' ? (this.#resourceProvided ? 'ready' : 'loading') : this.#state; + this.#iframe.setAttribute?.(relayStateAttribute, state); + } + #teardownId(): string { return `mcp-app-frame-close:${this.#bindingId}`; } diff --git a/packages/workbench/tests/mcp-app-real.e2e.test.ts b/packages/workbench/tests/mcp-app-real.e2e.test.ts index d335b4c87..6771a0005 100644 --- a/packages/workbench/tests/mcp-app-real.e2e.test.ts +++ b/packages/workbench/tests/mcp-app-real.e2e.test.ts @@ -164,6 +164,7 @@ const writeBundledAppProject = async (root: string): Promise => { interface AppRouteRequest { readonly body: unknown; + readonly method: string; readonly path: string; } @@ -173,7 +174,6 @@ interface AppRouteResponse extends AppRouteRequest { interface RuntimeAppRouteRequest extends AppRouteRequest { readonly headers: Readonly>; - readonly method: string; } interface RuntimeAppRouteResponse extends RuntimeAppRouteRequest { @@ -232,7 +232,7 @@ e2e('runs a generated SDK-v2 App through the real foreground session and separat page.on('request', (request) => { const requestUrl = new URL(request.url()); if (requestUrl.origin !== foregroundOrigin || !requestUrl.pathname.startsWith('/api/mcp/apps/')) return; - appRequests.push({ body: requestBody(request.postData()), path: requestUrl.pathname }); + appRequests.push({ body: requestBody(request.postData()), method: request.method(), path: requestUrl.pathname }); }); page.on('response', (response) => { const responseUrl = new URL(response.url()); @@ -243,7 +243,7 @@ e2e('runs a generated SDK-v2 App through the real foreground session and separat const responseUrl = new URL(response.url()); if (responseUrl.origin !== foregroundOrigin || !responseUrl.pathname.endsWith('/messages')) return; void response.json().then((body) => { - appResponses.push({ body: requestBody(response.request().postData()), path: responseUrl.pathname, response: body }); + appResponses.push({ body: requestBody(response.request().postData()), method: response.request().method(), path: responseUrl.pathname, response: body }); }).catch(() => undefined); }); @@ -431,22 +431,42 @@ e2e('runs a generated SDK-v2 App through the real foreground session and separat await expect(historyEntries).toHaveCount(2, { timeout: browserTimeout }); await expect(page.locator('.mcp-page-phase')).toContainText('Session ready', { timeout: browserTimeout }); + const reopenedPreview = page.waitForResponse((response) => + response.url() === `${foregroundOrigin}/api/mcp/sessions/${openedSession.session.id}/apps` && response.request().method() === 'POST', { timeout: browserTimeout }); await page.getByRole('button', { name: 'Open App preview for mcp-page-1' }).click(); + const reopened = await (await reopenedPreview).json() as Readonly<{ readonly preview: Readonly<{ readonly bindingId: string }> }>; + const reopenedAppPath = `/api/mcp/apps/${encodeURIComponent(reopened.preview.bindingId)}`; await expect(outerFrame).toBeVisible({ timeout: browserTimeout }); - // A visible iframe only proves the element mounted. The graceful /close - // handshake below needs a proxy that has loaded and an app that has - // initialized (a preview closed before its proxy signals readiness is - // released by DELETE, with nothing to acknowledge a teardown), so wait for - // the reopened binding's own `initialized` notification first. - await expect.poll(() => appRequests.filter((request) => { + // A visible iframe only proves the element mounted. The relay sends the + // graceful POST …/close only once it has seen the proxy's ready + // notification; before that, close() releases the binding with a forced + // DELETE and nothing can acknowledge a teardown. McpAppFrameRelay publishes + // that state on the iframe, so wait on it directly, then on the reopened + // App's own `initialized` so the teardown below is acknowledged instead of + // riding out the force-close timer. + await expect(outerFrame).toHaveAttribute('data-mcp-app-relay-state', 'ready', { timeout: browserTimeout }); + await expect.poll(() => appRequests.some((request) => { const message = (request.body as { readonly message?: { readonly method?: string } } | undefined)?.message; - return request.path.endsWith('/messages') && message?.method === 'ui/notifications/initialized'; - }).length, { timeout: browserTimeout }).toBe(2); - const secondClose = page.waitForRequest((request) => request.url().startsWith(`${foregroundOrigin}/api/mcp/apps/`) && request.url().endsWith('/close'), { timeout: 30_000 * timeScale }); + return request.path === `${reopenedAppPath}/messages` && message?.method === 'ui/notifications/initialized'; + }), { message: 'The reopened App preview never sent ui/notifications/initialized.', timeout: browserTimeout }).toBe(true); const closedSession = page.waitForRequest((request) => - request.url() === `${foregroundOrigin}/api/mcp/sessions/${openedSession.session.id}` && request.method() === 'DELETE', { timeout: 30_000 * timeScale }); + request.url() === `${foregroundOrigin}/api/mcp/sessions/${openedSession.session.id}` && request.method() === 'DELETE', { timeout: browserTimeout }); await page.getByRole('button', { name: 'Close MCP session' }).click(); - await secondClose; + // The first route call the close makes for this binding decides its path. + // Observing the DELETE too makes a force-close fail here, in milliseconds, + // instead of waiting out a /close that will never be sent. + const reopenedClose = () => appRequests.find((request) => + (request.method === 'POST' && request.path === `${reopenedAppPath}/close`) || (request.method === 'DELETE' && request.path === reopenedAppPath)); + await expect.poll(reopenedClose, { message: 'Closing the session sent no close request for the reopened App preview.', timeout: browserTimeout }).toBeDefined(); + const secondClose = reopenedClose(); + if (secondClose?.method !== 'POST') { + throw new Error(`Expected the reopened App preview to close gracefully (POST ${reopenedAppPath}/close); the relay sent ${secondClose === undefined ? 'no close request' : `${secondClose.method} ${secondClose.path}`} instead.`); + } + const secondCloseBody = secondClose.body as Readonly<{ readonly id: string }>; + await expect.poll(() => appRequests.some((request) => { + const message = (request.body as { readonly message?: { readonly id?: string; readonly result?: unknown } } | undefined)?.message; + return request.path === `${reopenedAppPath}/messages` && message?.id === secondCloseBody.id && message.result !== undefined; + }), { message: 'The reopened App preview never acknowledged the graceful teardown.', timeout: browserTimeout }).toBe(true); await closedSession; await expect(page.locator('.mcp-page-phase')).toContainText('Session closed', { timeout: browserTimeout }); await expect(outerFrame).toBeHidden({ timeout: browserTimeout }); From d0367b7c9b8e56f64bf0192a289d3570d0bd5346 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 5 Sep 2026 02:13:30 +0000 Subject: [PATCH 05/15] test(mcp-app-preview-browser): poll the bootstrap request log before pinning it (#576) --- .../tests/mcp-app-preview-browser.test.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/workbench/tests/mcp-app-preview-browser.test.ts b/packages/workbench/tests/mcp-app-preview-browser.test.ts index af15cba02..6a3b50fa1 100644 --- a/packages/workbench/tests/mcp-app-preview-browser.test.ts +++ b/packages/workbench/tests/mcp-app-preview-browser.test.ts @@ -206,6 +206,15 @@ describe('MCP App preview browser', () => { } }, 30_000); + // The page logs `create:*` inside the mocked fetch and `factory` in the + // passive effect of the commit that arms the iframe `src`; the bootstrap GET + // reaches the Node handler behind `bootstrapRequests` on its own network + // round-trip, so wait for the count before pinning the exact log. + const expectBootstrapRequests = async (requests: readonly string[], expected: readonly string[]) => { + await expect.poll(() => requests.length, { timeout: 5_000 }).toBe(expected.length); + expect(requests).toEqual(expected); + }; + it('keeps one runtime owner across same-authority StrictMode renders and drains the old owner before replacement', async () => { const fixture = await mountedPreviewFixture(); const browser = await chromium.launch({ channel: 'chrome' }); @@ -244,7 +253,7 @@ describe('MCP App preview browser', () => { expect(policyTrace.map((entry) => entry.name)).toEqual(['src', 'allow', 'referrerpolicy', 'sandbox', 'src']); expect(policyTrace[0]?.value).toBe('about:blank'); expect(policyTrace.at(-1)?.value).toBeDefined(); - expect(fixture.bootstrapRequests).toEqual(['/runtime-bootstrap']); + await expectBootstrapRequests(fixture.bootstrapRequests, ['/runtime-bootstrap']); const simulatedProfile = page.getByLabel('Simulated MCP App profile'); expect(await simulatedProfile.isVisible()).toBe(true); @@ -329,7 +338,7 @@ describe('MCP App preview browser', () => { expect(newCreate).toBeGreaterThan(oldUnregister); expect(replaced.events.filter((entry) => entry === 'register:first')).toHaveLength(1); expect(replaced.events.filter((entry) => entry === 'register:second')).toHaveLength(1); - expect(fixture.bootstrapRequests).toEqual(['/runtime-bootstrap', '/runtime-bootstrap']); + await expectBootstrapRequests(fixture.bootstrapRequests, ['/runtime-bootstrap', '/runtime-bootstrap']); expect(await implementationEvidence.count()).toBe(0); await runtime('publishOperationTraceWithUnexpectedEpoch'); expect(await implementationEvidence.count()).toBe(0); @@ -443,7 +452,7 @@ describe('MCP App preview browser', () => { const abandoned = await stats(); expect(lifecycleEvents(abandoned.events)).toEqual(lifecycleEvents(stable.events)); expect(abandoned.iframeNodes).toBe(stable.iframeNodes); - expect(fixture.bootstrapRequests).toEqual(['/runtime-bootstrap']); + await expectBootstrapRequests(fixture.bootstrapRequests, ['/runtime-bootstrap']); await runtime('failClose'); await runtime('throwUnregister'); @@ -473,7 +482,7 @@ describe('MCP App preview browser', () => { expect(await page.getByLabel('Runtime App result').textContent()).toContain('22'); expect(await page.getByLabel('Runtime App result').textContent()).not.toContain('Mutated'); expect(await page.getByLabel('Runtime App result').textContent()).not.toContain('999'); - expect(fixture.bootstrapRequests).toEqual(['/runtime-bootstrap', '/runtime-bootstrap']); + await expectBootstrapRequests(fixture.bootstrapRequests, ['/runtime-bootstrap', '/runtime-bootstrap']); await runtime('failClose'); await runtime('unmountRuntime'); From dd80b84260848934ff107f5a464191ca666de280 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 5 Sep 2026 02:13:30 +0000 Subject: [PATCH 06/15] test(packed-release): derive the outage-ledger windows from wire entries; accept pre-header replay aborts (#576) --- .../tests/packed-release.e2e.test.ts | 38 +++++++- .../tests/support/packed-outage-ledger.ts | 87 ++++++++++++++----- 2 files changed, 100 insertions(+), 25 deletions(-) diff --git a/packages/workbench/tests/packed-release.e2e.test.ts b/packages/workbench/tests/packed-release.e2e.test.ts index 1dc856142..b6a5d20d4 100644 --- a/packages/workbench/tests/packed-release.e2e.test.ts +++ b/packages/workbench/tests/packed-release.e2e.test.ts @@ -398,9 +398,16 @@ e2e('runs every Agent API tool from the installed tarball', { timeout: 360_000 * await expect(appFrame.locator('#view')).toHaveText('packed release dashboard', { timeout: browserTimeout }); phase = 'Logs, Evals, and Comparisons pages'; + // The heading renders before the page's mount replay is answered; leaving + // on the heading alone cancels that replay mid-flight on a loaded server. + // The visit is complete once the replay has responded. + const logsReplayListing = page.waitForResponse((response) => new URL(response.url()).pathname === '/api/logs/replay'); await page.goto(workbenchUrl(origin, 'logs')); phase = 'Logs page heading'; await expect(page.getByRole('heading', { name: 'Logs' })).toBeVisible({ timeout: browserTimeout }); + phase = 'Logs page replay'; + const logsReplayListingResponse = await logsReplayListing; + if (!logsReplayListingResponse.ok()) throw new Error(`The Logs page replay route failed with ${logsReplayListingResponse.status()}: ${await logsReplayListingResponse.text()}`); const initialEvalsRequestIndex = browserRequests.length; await page.goto(workbenchUrl(origin, 'evals')); phase = 'Evals page heading'; @@ -960,18 +967,37 @@ e2e('runs every Agent API tool from the installed tarball', { timeout: 360_000 * response.request().method() === 'POST' && response.ok(), ); await page.getByRole('button', { name: 'Call show-dashboard' }).click(); - await browserMcpSessionBOperation; + const browserMcpSessionBOperationResponse = await browserMcpSessionBOperation; await expect(page.getByRole('region', { name: 'Invocation history' })).toContainText('packed dashboard ready', { timeout: browserTimeout }); const closedBrowserMcpSessionB = page.waitForResponse((response) => response.url() === `${origin}/api/mcp/sessions/${encodeURIComponent(browserMcpSessionBId)}` && response.request().method() === 'DELETE' && response.ok(), ); - const browserMcpSessionBCloseStartedAt = Date.now(); await page.getByRole('button', { name: 'Close MCP session' }).click(); const closedBrowserMcpSessionBResponse = await closedBrowserMcpSessionB; expect(closedBrowserMcpSessionBResponse.request().headers()['x-agent-bundle-session']).toBe(browserGenerationBToken); await expect(page.locator('.mcp-page-phase')).toContainText('Session closed', { timeout: browserTimeout }); - const browserMcpSessionBCloseCompletedAt = Date.now(); + // The ledger's close window is bounded by the session's own wire entries, + // not by clock stamps around the click: it opens when the last operation + // the session served completed and closes when its DELETE completed. The + // page aborts both session streams before it issues that DELETE, so the + // close-induced aborts are delivered inside the window however late + // Playwright hands them over. Completion events trail their responses, + // so the two entries are awaited rather than read at once. + const browserMcpSessionBOperationRequest = browserRequestByPlaywrightRequest.get(browserMcpSessionBOperationResponse.request()); + const browserMcpSessionBCloseRequest = browserRequestByPlaywrightRequest.get(closedBrowserMcpSessionBResponse.request()); + if (browserMcpSessionBOperationRequest === undefined || browserMcpSessionBCloseRequest === undefined) { + throw new Error('The fresh B browser MCP operation or session close was not recorded in the network ledger.'); + } + const browserMcpSessionBCloseWindowRequests = [browserMcpSessionBOperationRequest, browserMcpSessionBCloseRequest]; + await expect.poll(() => browserMcpSessionBCloseWindowRequests.filter((request) => request.completedAt === undefined) + .map((request) => `${request.method} ${request.url}`), { timeout: browserTimeout }).toEqual([]); + const browserMcpSessionBCloseStartedAt = browserMcpSessionBOperationRequest.completedAt; + const browserMcpSessionBCloseCompletedAt = browserMcpSessionBCloseRequest.completedAt; + // Narrowing only: the poll above settled both entries. + if (browserMcpSessionBCloseStartedAt === undefined || browserMcpSessionBCloseCompletedAt === undefined) { + throw new Error('The fresh B browser MCP close window is missing a completed wire entry.'); + } phase = 'desktop navigation floor'; const navigationFloorRequestIndex = browserRequests.length; @@ -1006,6 +1032,12 @@ e2e('runs every Agent API tool from the installed tarball', { timeout: 360_000 * })); activeNavigationRoute = undefined; }; + // A departure is a test action with no wire event of its own — the next + // route's first request is not ordered against the aborts it provokes — + // so the arrival and departure instants are clock stamps taken before + // each click. The ledger treats both as inclusive bounds: a request the + // page issued before the click can still be handed over in the same + // millisecond as the stamp when Playwright delivers a batch of events. for (const route of navigationRoutes) { const openedAt = Date.now(); leaveActiveNavigationRoute(openedAt); diff --git a/packages/workbench/tests/support/packed-outage-ledger.ts b/packages/workbench/tests/support/packed-outage-ledger.ts index 05fef215f..bc3069d39 100644 --- a/packages/workbench/tests/support/packed-outage-ledger.ts +++ b/packages/workbench/tests/support/packed-outage-ledger.ts @@ -22,8 +22,24 @@ export interface OutageLedger { readonly origin: string; readonly outageStartedAt: number; readonly postRecovery?: Readonly<{ + /** + * The B-generation browser MCP session, located by its own wire entries: + * `openedAt` is the `POST /api/mcp/sessions` request instant, and the + * close window `[closeStartedAt, closeCompletedAt]` spans from the + * completion of the session's last operation to the completion of its + * `DELETE`. The page aborts both of its session streams before it issues + * that `DELETE`, so every close-induced stream abort lands inside the + * window while a mid-session or post-close abort does not. + */ readonly freshMcpSession: Readonly<{ readonly closeCompletedAt: number; readonly closeStartedAt: number; readonly id: string; readonly openedAt: number }>; - /** Exact page-owned requests cancelled when the test deliberately navigated away. */ + /** + * Exact page-owned requests cancelled when the test deliberately navigated + * away. A request belongs to the visit when it was observed no earlier + * than `openedAt` and no later than `leftAt`; both bounds are inclusive + * because Playwright hands the ledger batches of network events, so a + * request, its response, and the test's departure stamp can all share one + * millisecond. + */ readonly navigation: readonly Readonly<{ readonly leftAt: number; readonly openedAt: number; @@ -131,16 +147,30 @@ const isPlaygroundSessionReplayPath = (path: string): boolean => { segments[2] === 'sessions' && segments[3]!.length > 0 && segments[4] === 'replay'; }; +const isSuccessStatus = (status: number | undefined): status is number => status !== undefined && status >= 200 && status < 300; + +/** The abort landed before any response headers: Chromium reports neither a status nor a response instant. */ +const responseIsAbsent = (request: NetworkLedgerEntry): boolean => request.respondedAt === undefined && request.status === undefined; + const isPlaygroundSessionReadCancellation = (request: NetworkLedgerEntry): boolean => { const segments = request.path.split('/').filter((segment) => segment.length > 0); - const responseIsAbsent = request.respondedAt === undefined && request.status === undefined; - const responseIsSuccessful = request.respondedAt !== undefined && request.status !== undefined && - request.status >= 200 && request.status < 300; + const responseIsSuccessful = request.respondedAt !== undefined && isSuccessStatus(request.status); return segments.length === 4 && segments[0] === 'api' && segments[1] === 'playground' && segments[2] === 'sessions' && segments[3]!.length > 0 && request.url === `${request.origin}${request.path}` && - request.completedAt !== undefined && request.at <= request.completedAt && (responseIsAbsent || responseIsSuccessful); + request.completedAt !== undefined && request.at <= request.completedAt && (responseIsAbsent(request) || responseIsSuccessful); }; +/** + * The Logs page issues `/api/logs/replay` from its mount effect and aborts it + * from the effect's cleanup, so leaving the page cancels the replay in + * whichever state it is in: after a 2xx arrived (the body read is cut short) + * or before any headers arrived (a loaded server has not answered yet, and + * Chromium reports the abort with no status at all). Both are the same + * deliberate navigation; an abort after a non-2xx answer is still rejected. + */ +const isLogsReplayCancellation = (request: NetworkLedgerEntry): boolean => + request.path === '/api/logs/replay' && (responseIsAbsent(request) || isSuccessStatus(request.status)); + /** * The playground screen retires a superseded in-flight catalog request when * its effect re-runs (one AbortController per effect), and route changes abort @@ -152,7 +182,7 @@ const isKnownPreOutageClientCancellation = (request: NetworkLedgerEntry): boolea request.path === '/api/playground/catalog' || isPlaygroundSessionReadCancellation(request) || isPlaygroundSessionReplayPath(request.path) || - (request.path === '/api/logs/replay' && request.status !== undefined && request.status >= 200 && request.status < 300) + isLogsReplayCancellation(request) ); export const hasCanonicalAfterCursor = (url: URL): boolean => { @@ -206,6 +236,14 @@ export const validateOutageLedger = (ledger: OutageLedger): void => { const postRecovery = ledger.postRecovery; const freshMcpSession = postRecovery.freshMcpSession; const freshMcpStreamPath = `/api/mcp/sessions/${encodeURIComponent(freshMcpSession.id)}/stream`; + // The MCP page keeps two readers on one session: the transport opens + // `stream?after=0` as soon as the POST returns (AgentBundleRemoteTransport + // #start) and the session controller subscribes to `stream?after=N` once + // the trace refresh has settled (McpSessionController #subscribeTrace). + // Closing the session aborts both — the controller's subscription, then + // the transport's stream — before the DELETE goes out, so a close leaves + // one or two same-path aborts, each carrying the 2xx headers it had + // already received, inside the close window. const freshMcpStreamFailures = postRecoveryFailures.filter((request) => request.path === freshMcpStreamPath); assertOutageLedger(freshMcpStreamFailures.length >= 1 && freshMcpStreamFailures.length <= 2, `fresh B MCP stream did not terminate exactly once or twice: ${JSON.stringify(freshMcpStreamFailures)}`); @@ -216,26 +254,26 @@ export const validateOutageLedger = (ledger: OutageLedger): void => { assertOutageLedger( failure.origin === ledger.origin && url.origin === ledger.origin && url.pathname === freshMcpStreamPath && hasCanonicalAfterCursor(url) && failure.method === 'GET' && failure.error === 'net::ERR_ABORTED' && failure.at >= freshMcpSession.openedAt && - failure.respondedAt !== undefined && failure.respondedAt <= ledgerFailureAt(failure) && - failure.status !== undefined && failure.status >= 200 && failure.status < 300 && + failure.respondedAt !== undefined && failure.respondedAt <= ledgerFailureAt(failure) && isSuccessStatus(failure.status) && ledgerFailureAt(failure) >= freshMcpSession.closeStartedAt && ledgerFailureAt(failure) <= freshMcpSession.closeCompletedAt, `fresh B MCP stream cancellation is not action-induced: ${JSON.stringify(failure)}`, ); } - const navigationFailures: NetworkLedgerEntry[] = []; + const navigationFailures = new Set(); for (const navigation of postRecovery.navigation) { + // Inclusive on both ends (see OutageLedger.postRecovery.navigation): a + // request delivered in the same millisecond the test stamped its + // departure still belongs to the visit it was issued from. const failures = postRecoveryFailures.filter((request) => - request.url === navigation.url && request.at >= navigation.openedAt && request.at < navigation.leftAt && + request.url === navigation.url && request.at >= navigation.openedAt && request.at <= navigation.leftAt && ledgerFailureAt(request) >= navigation.leftAt, ); assertOutageLedger(failures.length <= 1, `multiple action-induced navigation cancellations: ${JSON.stringify({ failures, navigation })}`); for (const failure of failures) { - const responseIsAbsent = failure.respondedAt === undefined && failure.status === undefined; const responseIsSuccessful = failure.respondedAt !== undefined && failure.respondedAt >= failure.at && - failure.respondedAt <= ledgerFailureAt(failure) && failure.status !== undefined && - failure.status >= 200 && failure.status < 300; - let validResponse = responseIsAbsent || responseIsSuccessful; + failure.respondedAt <= ledgerFailureAt(failure) && isSuccessStatus(failure.status); + let validResponse = responseIsAbsent(failure) || responseIsSuccessful; if (navigation.respondedStream === true) { let url: URL; try { url = new URL(failure.url); } @@ -249,11 +287,18 @@ export const validateOutageLedger = (ledger: OutageLedger): void => { `navigation cancellation did not match its exact pending-or-stream response contract: ${JSON.stringify({ failure, navigation })}`, ); } - navigationFailures.push(...failures); + for (const failure of failures) navigationFailures.add(failure); } - const recognizedPostRecoveryFailures = [...freshMcpStreamFailures, ...navigationFailures]; - assertOutageLedger(recognizedPostRecoveryFailures.length === postRecoveryFailures.length, - `unknown post-recovery failure: ${JSON.stringify(postRecoveryFailures)}`); + // Two adjacent routes may list the same URL, and the departure stamp of one + // is the arrival stamp of the next, so one abort can satisfy two records; + // what must hold is that every post-recovery failure is claimed by some + // contract. Report only the unclaimed ones — a dump of every failure reads + // as if the recognized ones were at fault. + const unrecognizedPostRecoveryFailures = postRecoveryFailures.filter((request) => + !freshMcpStreamFailures.includes(request) && !navigationFailures.has(request), + ); + assertOutageLedger(unrecognizedPostRecoveryFailures.length === 0, + `unknown post-recovery failure: ${JSON.stringify(unrecognizedPostRecoveryFailures)}`); const postRecoveryConsoleErrors = ledger.consoleErrors.filter((consoleError) => consoleError.at >= ledger.recoveredAt); assertOutageLedger(postRecoveryConsoleErrors.length === 0, `post-recovery console errors: ${JSON.stringify(postRecoveryConsoleErrors)}`); } @@ -297,7 +342,7 @@ export const validateOutageLedger = (ledger: OutageLedger): void => { const oldSessionDeletes = sameOriginRequests.filter((request) => request.method === 'DELETE' && request.path === oldSessionPath); assertOutageLedger(oldSessionDeletes.length === 1, `expected exactly one old-session DELETE attempt: ${JSON.stringify(oldSessionDeletes)}`); const oldSessionDelete = oldSessionDeletes[0]!; - const deleteSucceeded = oldSessionDelete.status !== undefined && oldSessionDelete.status >= 200 && oldSessionDelete.status < 300; + const deleteSucceeded = isSuccessStatus(oldSessionDelete.status); const deleteRefused = oldSessionDelete.error === 'net::ERR_CONNECTION_REFUSED'; assertOutageLedger((deleteSucceeded ? 1 : 0) + (deleteRefused ? 1 : 0) === 1 && oldSessionDelete.completedAt !== undefined, `old-session DELETE must succeed or fail exactly with ERR_CONNECTION_REFUSED: ${JSON.stringify(oldSessionDelete)}`); @@ -305,9 +350,7 @@ export const validateOutageLedger = (ledger: OutageLedger): void => { const projectSessionAttempts = sameOriginRequests.filter((request) => request.method === 'GET' && request.path === '/api/project/session' && request.at >= ledger.outageStartedAt, ).sort((left, right) => left.at - right.at); - const successfulSessions = projectSessionAttempts.filter((request) => - request.status !== undefined && request.status >= 200 && request.status < 300, - ); + const successfulSessions = projectSessionAttempts.filter((request) => isSuccessStatus(request.status)); assertOutageLedger(successfulSessions.length >= 1, `the browser did not complete a B-generation project session: ${JSON.stringify(projectSessionAttempts)}`); const firstSuccessfulBSession = successfulSessions[0]!; assertOutageLedger(firstSuccessfulBSession.completedAt === ledger.recoveredAt, From f70574fa1c1f44418b94ee0db74a0038febcf38e Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 5 Sep 2026 02:24:17 +0000 Subject: [PATCH 07/15] changeset: agent-bundle patch for the injectable client-surface proxy timeout (#584) --- .changeset/576-proxy-upstream-timeout.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/576-proxy-upstream-timeout.md diff --git a/.changeset/576-proxy-upstream-timeout.md b/.changeset/576-proxy-upstream-timeout.md new file mode 100644 index 000000000..d6d3d9afc --- /dev/null +++ b/.changeset/576-proxy-upstream-timeout.md @@ -0,0 +1,5 @@ +--- +"agent-bundle": patch +--- + +Make the dev runtime client-surface proxy's upstream request timeout configurable: `RuntimeClientSurfaceProxy.open` accepts a trailing `RuntimeClientSurfaceProxyOptions` with `upstreamRequestTimeoutMs` (default `defaultRuntimeClientSurfaceUpstreamRequestTimeoutMs`, 15 000 ms; a value that is not a positive safe integer within the `setTimeout` ceiling is rejected before the proxy opens). The dev server keeps the 15 s default; the proxy's own deadline tests no longer wait on the real timeout (#584) From 504351f8c4332a1a6916405c2987121967ba7af9 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 5 Sep 2026 03:51:55 +0000 Subject: [PATCH 08/15] test(mcp-app-real): settle the scroll before the Close MCP session click and assert it landed (#576) --- packages/workbench/tests/mcp-app-real.e2e.test.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/workbench/tests/mcp-app-real.e2e.test.ts b/packages/workbench/tests/mcp-app-real.e2e.test.ts index 6771a0005..d0f615970 100644 --- a/packages/workbench/tests/mcp-app-real.e2e.test.ts +++ b/packages/workbench/tests/mcp-app-real.e2e.test.ts @@ -451,7 +451,18 @@ e2e('runs a generated SDK-v2 App through the real foreground session and separat }), { message: 'The reopened App preview never sent ui/notifications/initialized.', timeout: browserTimeout }).toBe(true); const closedSession = page.waitForRequest((request) => request.url() === `${foregroundOrigin}/api/mcp/sessions/${openedSession.session.id}` && request.method() === 'DELETE', { timeout: browserTimeout }); - await page.getByRole('button', { name: 'Close MCP session' }).click(); + // The session controls sit ~2000 px above the reopened App's cross-origin + // iframe. Letting click() scroll that far and dispatch in the same breath + // lets Chromium route the pointer to the frame that used to occupy the + // point (its hit-test regions update asynchronously), so the click is + // swallowed under load. Settle the scroll first, then confirm the click + // landed: run('close') disables the button synchronously and it stays + // disabled through the terminal phase. + const closeSession = page.getByRole('button', { name: 'Close MCP session' }); + await closeSession.scrollIntoViewIfNeeded(); + await expect(closeSession).toBeInViewport({ timeout: browserTimeout }); + await closeSession.click(); + await expect(closeSession, 'The Close MCP session click did not start the close action.').toBeDisabled({ timeout: browserTimeout }); // The first route call the close makes for this binding decides its path. // Observing the DELETE too makes a force-close fail here, in milliseconds, // instead of waiting out a /close that will never be sent. From cbdf3738c5adaa4eeda0be3596a8307c0b2ddfa1 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 5 Sep 2026 04:08:42 +0000 Subject: [PATCH 09/15] test(packed-release): assert the project/session retry cadence as a mean over the sequence, not per delivered gap (#576) --- .../tests/packed-outage-ledger.test.ts | 23 +++++++++++++++++++ .../tests/support/packed-outage-ledger.ts | 16 +++++++++---- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/packages/workbench/tests/packed-outage-ledger.test.ts b/packages/workbench/tests/packed-outage-ledger.test.ts index e3cdc1976..431284954 100644 --- a/packages/workbench/tests/packed-outage-ledger.test.ts +++ b/packages/workbench/tests/packed-outage-ledger.test.ts @@ -238,6 +238,26 @@ test('outage ledger rejects the legacy duplicate, cross-origin, and missing-clea )), }), }); + // Replaces the fixture's single project/session probe with `probeAts` refused + // probes (each paired with its console error) and a success at `successAt`. + const withRetryProbes = (probeAts: readonly number[], successAt: number): OutageLedger => Object.freeze({ + ...valid, + consoleErrors: Object.freeze([ + ...valid.consoleErrors.filter((consoleError) => consoleError.url !== `${valid.origin}/api/project/session`), + ...probeAts.map((at) => Object.freeze({ at: at + 2, text: 'Failed to load resource: net::ERR_CONNECTION_REFUSED', url: `${valid.origin}/api/project/session` })), + ]), + recoveredAt: successAt + 1, + requests: Object.freeze([ + ...valid.requests.filter((request) => request.path !== '/api/project/session'), + ...probeAts.map((at) => ledgerRequest({ at, completedAt: at + 1, error: 'net::ERR_CONNECTION_REFUSED', method: 'GET', path: '/api/project/session' })), + ledgerRequest({ at: successAt, completedAt: successAt + 1, method: 'GET', path: '/api/project/session', status: 200 }), + ]), + }); + // One probe's `request` event delivered 34 ms late: the gap before it reads + // 284 ms and the gap after it 222 ms, while the page kept its 250 ms delay. + const lateDeliveredRetry = withRetryProbes([1_010, 1_260, 1_544, 1_766], 2_016); + const burstRetry = withRetryProbes([1_010, 1_013], 1_263); + const underpacedRetries = withRetryProbes([1_010, 1_160, 1_310], 1_460); const malformedLedgers = [duplicateConsole, crossOriginConsole, missingCleanup]; expect(malformedLedgers.map(legacyOutageLedgerPasses)).toEqual([true, true, true]); @@ -253,6 +273,9 @@ test('outage ledger rejects the legacy duplicate, cross-origin, and missing-clea expect(() => validateOutageLedger(validPostRecovery)).not.toThrow(); expect(() => validateOutageLedger(navigationLiveStreamCancellation)).not.toThrow(); expect(() => validateOutageLedger(navigationRespondedCatalogCancellation)).not.toThrow(); + expect(() => validateOutageLedger(lateDeliveredRetry)).not.toThrow(); + expect(() => validateOutageLedger(burstRetry)).toThrow(/project\/session retries began too quickly/u); + expect(() => validateOutageLedger(underpacedRetries)).toThrow(/project\/session retries were paced below the client's 250 ms delay/u); for (const malformed of malformedLedgers) expect(() => validateOutageLedger(malformed)).toThrow(/Foreground outage ledger rejected/u); for (const malformed of [ resetWithAlteredQuery, resetWithResponse, resetWithUnknownSession, resetWithForeignOrigin, resetWithMismatchedConsoleUrl, diff --git a/packages/workbench/tests/support/packed-outage-ledger.ts b/packages/workbench/tests/support/packed-outage-ledger.ts index bc3069d39..090b240ae 100644 --- a/packages/workbench/tests/support/packed-outage-ledger.ts +++ b/packages/workbench/tests/support/packed-outage-ledger.ts @@ -363,10 +363,18 @@ export const validateOutageLedger = (ledger: OutageLedger): void => { `project/session retry had a non-connection failure: ${JSON.stringify(retryAttempts)}`); assertOutageLedger(retryAttempts.at(-1) === firstSuccessfulBSession && firstSuccessfulBSession.status === 200, `project/session recovery did not finish with the first successful B session: ${JSON.stringify(retryAttempts)}`); - for (const [index, attempt] of retryAttempts.entries()) { - if (index > 0) assertOutageLedger(attempt.at - retryAttempts[index - 1]!.at >= 225, - `project/session retries began too quickly: ${JSON.stringify(retryAttempts)}`); - } + // `at` is stamped when Playwright delivers the `request` event to Node, not + // when the page issued the probe, and a delivery can only run late: one + // held-back event shortens the next measured gap by exactly what it + // lengthened its own (a 284 ms / 222 ms pair on a loaded host). The client + // waits 250 ms after every failed probe, so the cadence is asserted as a mean + // over the retry sequence, where delivery latency cancels, while a burst — + // two probes issued without the delay — still fails on its own gap. + const retryGaps = retryAttempts.slice(1).map((attempt, index) => attempt.at - retryAttempts[index]!.at); + assertOutageLedger(retryGaps.every((gap) => gap >= 125), + `project/session retries began too quickly: ${JSON.stringify(retryAttempts)}`); + assertOutageLedger(retryGaps.reduce((sum, gap) => sum + gap, 0) >= 225 * retryGaps.length, + `project/session retries were paced below the client's 250 ms delay: ${JSON.stringify(retryAttempts)}`); const retryTimeline = retryAttempts.flatMap((request) => [ Object.freeze({ at: request.at, delta: 1 }), Object.freeze({ at: request.completedAt!, delta: -1 }), From 7d40b22798abd4c65008844aebd7476502982abe Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 5 Sep 2026 04:16:45 +0000 Subject: [PATCH 10/15] test(packed-release): open the fresh-B close window at the click, claim the Skills suites abort, cover the relay-state attribute (#576) --- .../workbench/tests/mcp-app-frame.test.ts | 77 +++++++++++++++++++ .../tests/packed-outage-ledger.test.ts | 35 +++++++++ .../tests/packed-release.e2e.test.ts | 38 +++++---- .../tests/support/packed-outage-ledger.ts | 18 +++-- 4 files changed, 140 insertions(+), 28 deletions(-) diff --git a/packages/workbench/tests/mcp-app-frame.test.ts b/packages/workbench/tests/mcp-app-frame.test.ts index aa28a37c2..868782b5d 100644 --- a/packages/workbench/tests/mcp-app-frame.test.ts +++ b/packages/workbench/tests/mcp-app-frame.test.ts @@ -18,6 +18,7 @@ import { applyMcpAppFramePolicy, McpAppFrame, SecureAppRenderer, + type McpAppFrameIframe, type McpAppFrameMessageListener, type McpAppFrameRelayRoutes, type McpAppFrameWindow, @@ -365,6 +366,82 @@ describe('MCP App frame relay', () => { expect(messages).toHaveLength(1); }); + // The attribute is what mcp-app-real.e2e waits on before it closes a + // preview; each transition is published on the event that changes it. + const publishedRelayStates = (browser: ReturnType): Readonly<{ readonly iframe: McpAppFrameIframe; readonly states: readonly string[] }> => { + const states: string[] = []; + return Object.freeze({ + iframe: Object.freeze({ + contentWindow: browser.iframe.contentWindow, + setAttribute: (name: string, value: string) => { + expect(name).toBe('data-mcp-app-relay-state'); + states.push(value); + }, + }), + states, + }); + }; + + it('publishes loading, ready, closing, and closed on the iframe as the relay moves through a graceful close', async () => { + const browser = fakeBrowser(); + const published = publishedRelayStates(browser); + const closeCalls: { readonly id: string }[] = []; + const routes: McpAppFrameRelayRoutes = { + close: async (_bindingId, options) => { + closeCalls.push(options); + return closeResult({ id: options.id, jsonrpc: '2.0', method: 'ui/resource-teardown', params: {} }); + }, + forceClose: async () => true, + message: async () => messageResult([], 'closed'), + }; + const relay = createMcpAppFrameRelay({ bindingId: 'binding-weather', closeTimeoutMs: 100, frame, iframe: published.iframe, resource, routes, window: browser.window }); + expect(published.states).toEqual([]); + expect(relay.start()).toBe(true); + expect(published.states).toEqual(['loading']); + expect(relay.start()).toBe(false); + expect(published.states).toEqual(['loading']); + browser.emit({ data: proxyReady(), origin: frame.targetOrigin, source: browser.child }); + expect(published.states).toEqual(['loading', 'ready']); + browser.emit({ data: proxyReady(), origin: frame.targetOrigin, source: browser.child }); + expect(published.states).toEqual(['loading', 'ready']); + + const closing = relay.close(); + expect(published.states).toEqual(['loading', 'ready', 'closing']); + await eventually(() => closeCalls.length === 1); + expect(published.states).toEqual(['loading', 'ready', 'closing']); + browser.emit({ data: { id: closeCalls[0]!.id, jsonrpc: '2.0', result: {} }, origin: frame.targetOrigin, source: browser.child }); + await closing; + expect(published.states).toEqual(['loading', 'ready', 'closing', 'closed']); + await relay.close(); + expect(published.states).toEqual(['loading', 'ready', 'closing', 'closed']); + }); + + it('publishes closing and closed, never ready, around the forced DELETE of a proxy that never signaled readiness', async () => { + const browser = fakeBrowser(); + const published = publishedRelayStates(browser); + let closeCalls = 0; + let forceCloseCalls = 0; + const routes: McpAppFrameRelayRoutes = { + close: async () => { + closeCalls += 1; + return closeResult(); + }, + forceClose: async () => { + forceCloseCalls += 1; + return true; + }, + message: async () => messageResult(), + }; + const relay = createMcpAppFrameRelay({ bindingId: 'binding-weather', closeTimeoutMs: 30_000, frame, iframe: published.iframe, resource, routes, window: browser.window }); + relay.start(); + const closing = relay.close(); + expect(published.states).toEqual(['loading', 'closing']); + await closing; + expect(published.states).toEqual(['loading', 'closing', 'closed']); + expect(closeCalls).toBe(0); + expect(forceCloseCalls).toBe(1); + }); + it('force-deletes a closing binding when the ready proxy never acknowledges the teardown frame', async () => { const browser = fakeBrowser(); let closeCalls = 0; diff --git a/packages/workbench/tests/packed-outage-ledger.test.ts b/packages/workbench/tests/packed-outage-ledger.test.ts index 431284954..109a0adaf 100644 --- a/packages/workbench/tests/packed-outage-ledger.test.ts +++ b/packages/workbench/tests/packed-outage-ledger.test.ts @@ -157,6 +157,29 @@ test('outage ledger rejects the legacy duplicate, cross-origin, and missing-clea ...valid.requests, ]), }); + // Leaving the Logs page before the replay answered: Chromium reports the + // abort with neither a status nor a response instant. + const knownPreOutageLogsReplayPreHeaderCancellation = Object.freeze({ + ...valid, + requests: Object.freeze([ + ledgerRequest({ at: 900, completedAt: 901, error: 'net::ERR_ABORTED', method: 'GET', path: '/api/logs/replay' }), + ...valid.requests, + ]), + }); + const logsReplayCancellationWithoutTerminal = Object.freeze({ + ...valid, + requests: Object.freeze([ + ledgerRequest({ at: 900, error: 'net::ERR_ABORTED', method: 'GET', path: '/api/logs/replay' }), + ...valid.requests, + ]), + }); + const logsReplayCancellationAfterFailure = Object.freeze({ + ...valid, + requests: Object.freeze([ + ledgerRequest({ at: 900, completedAt: 901, error: 'net::ERR_ABORTED', method: 'GET', path: '/api/logs/replay', respondedAt: 900, status: 500 }), + ...valid.requests, + ]), + }); const preStartedOutageStreamTermination = Object.freeze({ ...valid, requests: Object.freeze(valid.requests.map((request) => @@ -229,6 +252,14 @@ test('outage ledger rejects the legacy duplicate, cross-origin, and missing-clea ledgerRequest({ at: 1_335, completedAt: 1_336, error: 'net::ERR_ABORTED', method: 'POST', path: '/api/playground/runs' }), ]), }); + // A fresh-B stream abort delivered before the test clicked Close is not the + // close's doing, however close to the click it lands. + const preCloseFreshStreamCancellation = Object.freeze({ + ...validPostRecovery, + requests: Object.freeze(validPostRecovery.requests.map((request) => + request.path.startsWith('/api/mcp/sessions/fresh-browser-mcp-session/') ? ledgerRequest({ ...request, completedAt: 1_319 }) : request, + )), + }); const navigationCancellationBeforeDeparture = Object.freeze({ ...validPostRecovery, postRecovery: Object.freeze({ @@ -270,6 +301,10 @@ test('outage ledger rejects the legacy duplicate, cross-origin, and missing-clea expect(() => validateOutageLedger(knownPreOutageSessionReplayCancellation)).not.toThrow(); expect(() => validateOutageLedger(knownPreOutageSessionReadCancellation)).not.toThrow(); expect(() => validateOutageLedger(knownPreOutageLogsReplayCancellation)).not.toThrow(); + expect(() => validateOutageLedger(knownPreOutageLogsReplayPreHeaderCancellation)).not.toThrow(); + expect(() => validateOutageLedger(logsReplayCancellationWithoutTerminal)).toThrow(/unexpected pre-outage failures/u); + expect(() => validateOutageLedger(logsReplayCancellationAfterFailure)).toThrow(/unexpected pre-outage failures/u); + expect(() => validateOutageLedger(preCloseFreshStreamCancellation)).toThrow(/fresh B MCP stream cancellation is not action-induced/u); expect(() => validateOutageLedger(validPostRecovery)).not.toThrow(); expect(() => validateOutageLedger(navigationLiveStreamCancellation)).not.toThrow(); expect(() => validateOutageLedger(navigationRespondedCatalogCancellation)).not.toThrow(); diff --git a/packages/workbench/tests/packed-release.e2e.test.ts b/packages/workbench/tests/packed-release.e2e.test.ts index 6b13e933a..fa5d64634 100644 --- a/packages/workbench/tests/packed-release.e2e.test.ts +++ b/packages/workbench/tests/packed-release.e2e.test.ts @@ -968,37 +968,30 @@ e2e('runs every Agent API tool from the installed tarball', { timeout: 360_000 * response.request().method() === 'POST' && response.ok(), ); await page.getByRole('button', { name: 'Call show-dashboard' }).click(); - const browserMcpSessionBOperationResponse = await browserMcpSessionBOperation; + await browserMcpSessionBOperation; await expect(page.getByRole('region', { name: 'Invocation history' })).toContainText('packed dashboard ready', { timeout: browserTimeout }); const closedBrowserMcpSessionB = page.waitForResponse((response) => response.url() === `${origin}/api/mcp/sessions/${encodeURIComponent(browserMcpSessionBId)}` && response.request().method() === 'DELETE' && response.ok(), ); + // The close window opens on a stamp taken before the click — the click + // is issued from here, so nothing it causes can reach the ledger earlier + // — and closes on the DELETE's own completion entry: the page aborts both + // session streams before it issues that DELETE, so the close-induced + // aborts are delivered inside the window however late Playwright hands + // them over, while an abort before the click stays outside it. Completion + // events trail their responses, so the DELETE entry is awaited. + const browserMcpSessionBCloseStartedAt = Date.now(); await page.getByRole('button', { name: 'Close MCP session' }).click(); const closedBrowserMcpSessionBResponse = await closedBrowserMcpSessionB; expect(closedBrowserMcpSessionBResponse.request().headers()['x-agent-bundle-session']).toBe(browserGenerationBToken); await expect(page.locator('.mcp-page-phase')).toContainText('Session closed', { timeout: browserTimeout }); - // The ledger's close window is bounded by the session's own wire entries, - // not by clock stamps around the click: it opens when the last operation - // the session served completed and closes when its DELETE completed. The - // page aborts both session streams before it issues that DELETE, so the - // close-induced aborts are delivered inside the window however late - // Playwright hands them over. Completion events trail their responses, - // so the two entries are awaited rather than read at once. - const browserMcpSessionBOperationRequest = browserRequestByPlaywrightRequest.get(browserMcpSessionBOperationResponse.request()); const browserMcpSessionBCloseRequest = browserRequestByPlaywrightRequest.get(closedBrowserMcpSessionBResponse.request()); - if (browserMcpSessionBOperationRequest === undefined || browserMcpSessionBCloseRequest === undefined) { - throw new Error('The fresh B browser MCP operation or session close was not recorded in the network ledger.'); - } - const browserMcpSessionBCloseWindowRequests = [browserMcpSessionBOperationRequest, browserMcpSessionBCloseRequest]; - await expect.poll(() => browserMcpSessionBCloseWindowRequests.filter((request) => request.completedAt === undefined) - .map((request) => `${request.method} ${request.url}`), { timeout: browserTimeout }).toEqual([]); - const browserMcpSessionBCloseStartedAt = browserMcpSessionBOperationRequest.completedAt; + if (browserMcpSessionBCloseRequest === undefined) throw new Error('The fresh B browser MCP session close was not recorded in the network ledger.'); + await expect.poll(() => browserMcpSessionBCloseRequest.completedAt, { timeout: browserTimeout }).toBeDefined(); const browserMcpSessionBCloseCompletedAt = browserMcpSessionBCloseRequest.completedAt; - // Narrowing only: the poll above settled both entries. - if (browserMcpSessionBCloseStartedAt === undefined || browserMcpSessionBCloseCompletedAt === undefined) { - throw new Error('The fresh B browser MCP close window is missing a completed wire entry.'); - } + // Narrowing only: the poll above settled the entry. + if (browserMcpSessionBCloseCompletedAt === undefined) throw new Error('The fresh B browser MCP close window is missing its completed DELETE entry.'); phase = 'desktop navigation floor'; const navigationFloorRequestIndex = browserRequests.length; @@ -1007,7 +1000,12 @@ e2e('runs every Agent API tool from the installed tarball', { timeout: 360_000 * { heading: 'MCP playground', label: 'MCP playground' }, { heading: 'Artifacts', label: 'Artifacts' }, { heading: 'Playground', label: 'Playground' }, { heading: 'Logs', label: 'Logs' }, { heading: 'Evals', label: 'Evals' }, { heading: 'Comparisons', label: 'Comparisons' }, ]; + // Every abortable request a route issues on mount, so leaving the route + // before a loaded server answers is claimed by the route's record rather + // than reported as an unknown post-recovery failure. const postRecoveryNavigationUrls = new Map([ + // Skills lists the authored suites for its eval-coverage column. + ['Skills', [`${origin}/api/evals/suites`]], ['Hooks', [`${origin}/api/hooks?epochId=${encodeURIComponent(recoveredEpochId)}`]], ['MCP playground', [`${origin}/api/artifacts/epochs/${encodeURIComponent(recoveredEpochId)}`]], ['Playground', [`${origin}/api/playground/catalog?epochId=${encodeURIComponent(recoveredEpochId)}`]], diff --git a/packages/workbench/tests/support/packed-outage-ledger.ts b/packages/workbench/tests/support/packed-outage-ledger.ts index 090b240ae..27a131588 100644 --- a/packages/workbench/tests/support/packed-outage-ledger.ts +++ b/packages/workbench/tests/support/packed-outage-ledger.ts @@ -23,13 +23,14 @@ export interface OutageLedger { readonly outageStartedAt: number; readonly postRecovery?: Readonly<{ /** - * The B-generation browser MCP session, located by its own wire entries: - * `openedAt` is the `POST /api/mcp/sessions` request instant, and the - * close window `[closeStartedAt, closeCompletedAt]` spans from the - * completion of the session's last operation to the completion of its - * `DELETE`. The page aborts both of its session streams before it issues - * that `DELETE`, so every close-induced stream abort lands inside the - * window while a mid-session or post-close abort does not. + * The B-generation browser MCP session: `openedAt` is the + * `POST /api/mcp/sessions` request instant, and the close window + * `[closeStartedAt, closeCompletedAt]` spans from the stamp the test took + * before clicking Close (the click is issued from the test, so nothing it + * causes can be delivered earlier) to the completion of the session's + * `DELETE` wire entry. The page aborts both of its session streams before + * it issues that `DELETE`, so every close-induced stream abort lands inside + * the window while a pre-click or post-close abort does not. */ readonly freshMcpSession: Readonly<{ readonly closeCompletedAt: number; readonly closeStartedAt: number; readonly id: string; readonly openedAt: number }>; /** @@ -169,7 +170,8 @@ const isPlaygroundSessionReadCancellation = (request: NetworkLedgerEntry): boole * deliberate navigation; an abort after a non-2xx answer is still rejected. */ const isLogsReplayCancellation = (request: NetworkLedgerEntry): boolean => - request.path === '/api/logs/replay' && (responseIsAbsent(request) || isSuccessStatus(request.status)); + request.path === '/api/logs/replay' && request.completedAt !== undefined && request.at <= request.completedAt && + (responseIsAbsent(request) || isSuccessStatus(request.status)); /** * The playground screen retires a superseded in-flight catalog request when From bd25a7c2faa501bfb145f6c41e93990945f4d868 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 5 Sep 2026 04:24:59 +0000 Subject: [PATCH 11/15] test(packed-release): attribute the recovered Comparisons runs listing abort to the departure for Overview (#576) --- .../workbench/tests/packed-release.e2e.test.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/packages/workbench/tests/packed-release.e2e.test.ts b/packages/workbench/tests/packed-release.e2e.test.ts index fa5d64634..3215d479a 100644 --- a/packages/workbench/tests/packed-release.e2e.test.ts +++ b/packages/workbench/tests/packed-release.e2e.test.ts @@ -886,6 +886,7 @@ e2e('runs every Agent API tool from the installed tarball', { timeout: 360_000 * await expect(page.locator('.eval-raw-result')).toContainText('The deterministic packed fixture passed.', { timeout: browserTimeout }); await waitForBrowserRequestsAfter(evalsBrowserRequestIndex); phase = 'Evals comparison run availability'; + const comparisonsOpenedAt = Date.now(); await page.getByRole('link', { name: 'Comparisons', exact: true }).click(); await expect(page.getByRole('heading', { name: 'Comparisons' })).toBeVisible({ timeout: browserTimeout }); await expect.poll(async () => page.locator('#comparison-base option').count(), { timeout: browserTimeout }).toBeGreaterThanOrEqual(2); @@ -930,6 +931,16 @@ e2e('runs every Agent API tool from the installed tarball', { timeout: 360_000 * const rebuiltWithRecoveredSession = page.waitForResponse((response) => response.url() === `${origin}/api/project/rebuild` && response.request().method() === 'POST' && response.ok(), ); + // The recovered session hands the Comparisons page new clients, and its + // effect re-lists /api/evals/runs; leaving for Overview before a loaded + // server answers aborts that listing, which the ledger must be able to + // attribute to this departure like every later one. + const postRecoveryNavigation: Array> = [Object.freeze({ leftAt: Date.now(), openedAt: comparisonsOpenedAt, url: `${origin}/api/evals/runs` })]; await page.getByRole('link', { name: 'Overview', exact: true }).click(); await page.getByRole('button', { name: 'Rebuild' }).click(); const rebuiltWithRecoveredSessionResponse = await rebuiltWithRecoveredSession; @@ -1014,12 +1025,6 @@ e2e('runs every Agent API tool from the installed tarball', { timeout: 360_000 * ['Comparisons', [`${origin}/api/evals/runs`]], ]); const respondedNavigationStreams = new Set(); - const postRecoveryNavigation: Array> = []; let activeNavigationRoute: Readonly<{ openedAt: number; urls?: readonly string[] }> | undefined; const leaveActiveNavigationRoute = (leftAt: number): void => { if (activeNavigationRoute === undefined) return; From d13d8a2bfb6fd6086b07513fbf1f909a93efb433 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 5 Sep 2026 04:32:30 +0000 Subject: [PATCH 12/15] test(packed-release): own navigation aborts by ledger order, not by millisecond; explain short retry gaps only by accumulated delivery lateness (#576) --- .../tests/packed-outage-ledger.test.ts | 43 ++++++++++- .../tests/packed-release.e2e.test.ts | 41 +++++----- .../tests/support/packed-outage-ledger.ts | 77 ++++++++++++------- 3 files changed, 110 insertions(+), 51 deletions(-) diff --git a/packages/workbench/tests/packed-outage-ledger.test.ts b/packages/workbench/tests/packed-outage-ledger.test.ts index 109a0adaf..6378805c4 100644 --- a/packages/workbench/tests/packed-outage-ledger.test.ts +++ b/packages/workbench/tests/packed-outage-ledger.test.ts @@ -202,6 +202,8 @@ test('outage ledger rejects the legacy duplicate, cross-origin, and missing-clea ledgerRequest({ at: 1_350, completedAt: 1_351, error: 'net::ERR_ABORTED', method: 'GET', path: '/api/unknown/stream' }), ]), }); + // Index of a request appended after the post-recovery fixture's own entries. + const appendedIndex = validPostRecovery.requests.length; const navigationLiveStreamCancellation = Object.freeze({ ...validPostRecovery, postRecovery: Object.freeze({ @@ -210,7 +212,8 @@ test('outage ledger rejects the legacy duplicate, cross-origin, and missing-clea ...validPostRecovery.postRecovery!.navigation, Object.freeze({ leftAt: 1_350, - openedAt: 1_345, + leftIndex: appendedIndex + 1, + openedIndex: appendedIndex, respondedStream: true as const, url: `${valid.origin}/api/logs/stream?after=32`, }), @@ -232,7 +235,8 @@ test('outage ledger rejects the legacy duplicate, cross-origin, and missing-clea ...validPostRecovery.postRecovery!.navigation, Object.freeze({ leftAt: 1_350, - openedAt: 1_345, + leftIndex: appendedIndex + 1, + openedIndex: appendedIndex, url: `${valid.origin}/api/playground/catalog?epochId=recovered-epoch`, }), ]), @@ -260,6 +264,27 @@ test('outage ledger rejects the legacy duplicate, cross-origin, and missing-clea request.path.startsWith('/api/mcp/sessions/fresh-browser-mcp-session/') ? ledgerRequest({ ...request, completedAt: 1_319 }) : request, )), }); + const hooksNavigation = validPostRecovery.postRecovery!.navigation[0]!; + const hooksRequest = validPostRecovery.requests.at(-1)!; + // The CI shape: the Hooks request and its response arrived in one batch and + // the awaiting test stamped its departure in the same millisecond. Delivery + // order still places the request before the stamp, so the visit owns it. + const sameMillisecondDepartedRequest = Object.freeze({ + ...validPostRecovery, + requests: Object.freeze([ + ...validPostRecovery.requests.slice(0, -1), + ledgerRequest({ ...hooksRequest, at: hooksNavigation.leftAt }), + ]), + }); + // The mirror image: a request handed over after the departure stamp is the + // next page's, however it is timestamped, and its abort stays unexplained. + const sameMillisecondNextPageRequest = Object.freeze({ + ...validPostRecovery, + requests: Object.freeze([ + ...validPostRecovery.requests, + ledgerRequest({ ...hooksRequest, at: hooksNavigation.leftAt, completedAt: hooksNavigation.leftAt + 1 }), + ]), + }); const navigationCancellationBeforeDeparture = Object.freeze({ ...validPostRecovery, postRecovery: Object.freeze({ @@ -287,8 +312,14 @@ test('outage ledger rejects the legacy duplicate, cross-origin, and missing-clea // One probe's `request` event delivered 34 ms late: the gap before it reads // 284 ms and the gap after it 222 ms, while the page kept its 250 ms delay. const lateDeliveredRetry = withRetryProbes([1_010, 1_260, 1_544, 1_766], 2_016); + // The first probe's event delivered 100 ms late shortens only the first gap. + const lateDeliveredFirstRetry = withRetryProbes([1_110, 1_260], 1_510); const burstRetry = withRetryProbes([1_010, 1_013], 1_263); const underpacedRetries = withRetryProbes([1_010, 1_160, 1_310], 1_460); + // Two short gaps then a long one: lateness cannot be borrowed from a gap that + // has not happened yet, so the second short gap is rejected even though the + // three gaps average well above the client's delay. + const borrowedRetryLateness = withRetryProbes([1_010, 1_135, 1_260], 1_735); const malformedLedgers = [duplicateConsole, crossOriginConsole, missingCleanup]; expect(malformedLedgers.map(legacyOutageLedgerPasses)).toEqual([true, true, true]); @@ -308,9 +339,13 @@ test('outage ledger rejects the legacy duplicate, cross-origin, and missing-clea expect(() => validateOutageLedger(validPostRecovery)).not.toThrow(); expect(() => validateOutageLedger(navigationLiveStreamCancellation)).not.toThrow(); expect(() => validateOutageLedger(navigationRespondedCatalogCancellation)).not.toThrow(); + expect(() => validateOutageLedger(sameMillisecondDepartedRequest)).not.toThrow(); + expect(() => validateOutageLedger(sameMillisecondNextPageRequest)).toThrow(/unknown post-recovery failure/u); expect(() => validateOutageLedger(lateDeliveredRetry)).not.toThrow(); - expect(() => validateOutageLedger(burstRetry)).toThrow(/project\/session retries began too quickly/u); - expect(() => validateOutageLedger(underpacedRetries)).toThrow(/project\/session retries were paced below the client's 250 ms delay/u); + expect(() => validateOutageLedger(lateDeliveredFirstRetry)).not.toThrow(); + expect(() => validateOutageLedger(burstRetry)).toThrow(/project\/session retries began too quickly \(attempt 2 arrived 122 ms before/u); + expect(() => validateOutageLedger(underpacedRetries)).toThrow(/project\/session retries began too quickly \(attempt 3 arrived 75 ms before/u); + expect(() => validateOutageLedger(borrowedRetryLateness)).toThrow(/project\/session retries began too quickly \(attempt 3 arrived 125 ms before/u); for (const malformed of malformedLedgers) expect(() => validateOutageLedger(malformed)).toThrow(/Foreground outage ledger rejected/u); for (const malformed of [ resetWithAlteredQuery, resetWithResponse, resetWithUnknownSession, resetWithForeignOrigin, resetWithMismatchedConsoleUrl, diff --git a/packages/workbench/tests/packed-release.e2e.test.ts b/packages/workbench/tests/packed-release.e2e.test.ts index 3215d479a..fcae978d8 100644 --- a/packages/workbench/tests/packed-release.e2e.test.ts +++ b/packages/workbench/tests/packed-release.e2e.test.ts @@ -886,7 +886,7 @@ e2e('runs every Agent API tool from the installed tarball', { timeout: 360_000 * await expect(page.locator('.eval-raw-result')).toContainText('The deterministic packed fixture passed.', { timeout: browserTimeout }); await waitForBrowserRequestsAfter(evalsBrowserRequestIndex); phase = 'Evals comparison run availability'; - const comparisonsOpenedAt = Date.now(); + const comparisonsOpenedIndex = browserRequests.length; await page.getByRole('link', { name: 'Comparisons', exact: true }).click(); await expect(page.getByRole('heading', { name: 'Comparisons' })).toBeVisible({ timeout: browserTimeout }); await expect.poll(async () => page.locator('#comparison-base option').count(), { timeout: browserTimeout }).toBeGreaterThanOrEqual(2); @@ -934,13 +934,16 @@ e2e('runs every Agent API tool from the installed tarball', { timeout: 360_000 * // The recovered session hands the Comparisons page new clients, and its // effect re-lists /api/evals/runs; leaving for Overview before a loaded // server answers aborts that listing, which the ledger must be able to - // attribute to this departure like every later one. + // attribute to this departure like every later one. A visit owns the + // requests recorded between its arrival and departure stamps — delivery + // order, which a millisecond cannot establish (see the ledger). const postRecoveryNavigation: Array> = [Object.freeze({ leftAt: Date.now(), openedAt: comparisonsOpenedAt, url: `${origin}/api/evals/runs` })]; + }>> = [Object.freeze({ leftAt: Date.now(), leftIndex: browserRequests.length, openedIndex: comparisonsOpenedIndex, url: `${origin}/api/evals/runs` })]; await page.getByRole('link', { name: 'Overview', exact: true }).click(); await page.getByRole('button', { name: 'Rebuild' }).click(); const rebuiltWithRecoveredSessionResponse = await rebuiltWithRecoveredSession; @@ -1025,26 +1028,28 @@ e2e('runs every Agent API tool from the installed tarball', { timeout: 360_000 * ['Comparisons', [`${origin}/api/evals/runs`]], ]); const respondedNavigationStreams = new Set(); - let activeNavigationRoute: Readonly<{ openedAt: number; urls?: readonly string[] }> | undefined; - const leaveActiveNavigationRoute = (leftAt: number): void => { + let activeNavigationRoute: Readonly<{ openedIndex: number; urls?: readonly string[] }> | undefined; + // A departure is a test action with no wire event of its own, so it is + // stamped twice before the click: the instant, which the abort it + // provokes must not precede, and the ledger length, which orders the + // requests already handed over (the departed page's) against those that + // follow (the next page's) even when both share the instant's millisecond. + const leaveActiveNavigationRoute = (): void => { if (activeNavigationRoute === undefined) return; + const leftAt = Date.now(); + const leftIndex = browserRequests.length; for (const url of activeNavigationRoute.urls ?? []) postRecoveryNavigation.push(Object.freeze({ leftAt, - openedAt: activeNavigationRoute.openedAt, + leftIndex, + openedIndex: activeNavigationRoute.openedIndex, ...(respondedNavigationStreams.has(url) ? { respondedStream: true as const } : {}), url, })); activeNavigationRoute = undefined; }; - // A departure is a test action with no wire event of its own — the next - // route's first request is not ordered against the aborts it provokes — - // so the arrival and departure instants are clock stamps taken before - // each click. The ledger treats both as inclusive bounds: a request the - // page issued before the click can still be handed over in the same - // millisecond as the stamp when Playwright delivers a batch of events. for (const route of navigationRoutes) { - const openedAt = Date.now(); - leaveActiveNavigationRoute(openedAt); + leaveActiveNavigationRoute(); + const openedIndex = browserRequests.length; const logsStreamResponse = route.label === 'Logs' ? page.waitForResponse((response) => { const url = new URL(response.url()); @@ -1064,15 +1069,15 @@ e2e('runs every Agent API tool from the installed tarball', { timeout: 360_000 * const streamUrl = response.url(); const stream = browserRequestByPlaywrightRequest.get(response.request()); if ( - stream === undefined || stream.at < openedAt || stream.respondedAt === undefined || + stream === undefined || browserRequests.indexOf(stream) < openedIndex || stream.respondedAt === undefined || stream.status !== response.status() ) throw new Error('The Logs navigation stream response was not recorded in the network ledger.'); respondedNavigationStreams.add(streamUrl); routeUrls.push(streamUrl); } - activeNavigationRoute = Object.freeze({ openedAt, urls: Object.freeze(routeUrls) }); + activeNavigationRoute = Object.freeze({ openedIndex, urls: Object.freeze(routeUrls) }); } - leaveActiveNavigationRoute(Date.now()); + leaveActiveNavigationRoute(); await page.getByRole('link', { name: 'Overview', exact: true }).focus(); await page.keyboard.press('Enter'); await expect(page.getByRole('heading', { name: 'Bundle dashboard' })).toBeVisible({ timeout: browserTimeout }); diff --git a/packages/workbench/tests/support/packed-outage-ledger.ts b/packages/workbench/tests/support/packed-outage-ledger.ts index 27a131588..32292d54d 100644 --- a/packages/workbench/tests/support/packed-outage-ledger.ts +++ b/packages/workbench/tests/support/packed-outage-ledger.ts @@ -35,15 +35,19 @@ export interface OutageLedger { readonly freshMcpSession: Readonly<{ readonly closeCompletedAt: number; readonly closeStartedAt: number; readonly id: string; readonly openedAt: number }>; /** * Exact page-owned requests cancelled when the test deliberately navigated - * away. A request belongs to the visit when it was observed no earlier - * than `openedAt` and no later than `leftAt`; both bounds are inclusive - * because Playwright hands the ledger batches of network events, so a - * request, its response, and the test's departure stamp can all share one - * millisecond. + * away. Membership is by delivery order, not by clock: `requests` grows in + * the order Playwright hands events over, and `openedIndex`/`leftIndex` + * are its length when the test stamped the arrival and the departure, so + * a request belongs to the visit when it sits in `[openedIndex, leftIndex)`. + * A millisecond cannot order a request against a departure — a batch of + * events and the stamp that follows it share one — but the array can. The + * abort itself must complete no earlier than `leftAt`: an abort that + * finished before the departure was not the departure's doing. */ readonly navigation: readonly Readonly<{ readonly leftAt: number; - readonly openedAt: number; + readonly leftIndex: number; + readonly openedIndex: number; readonly respondedStream?: true; readonly url: string; }>[]; @@ -93,8 +97,9 @@ export const postRecoveryCancellationFixture = (): OutageLedger => { ...base, postRecovery: Object.freeze({ freshMcpSession: Object.freeze({ closeCompletedAt: 1_321, closeStartedAt: 1_320, id: freshMcpSessionId, openedAt: 1_310 }), + // The Hooks visit owns exactly the last request of the ledger below. navigation: Object.freeze([ - Object.freeze({ leftAt: 1_340, openedAt: 1_330, url: hooksUrl }), + Object.freeze({ leftAt: 1_340, leftIndex: base.requests.length + 2, openedIndex: base.requests.length + 1, url: hooksUrl }), ]), }), requests: Object.freeze([ @@ -198,6 +203,16 @@ export const hasCanonicalAfterCursor = (url: URL): boolean => { /** A probe against a dying server can hit its half-open socket (RESET) instead of a closed port (REFUSED). */ const downServerProbeCodes: ReadonlySet = new Set(['net::ERR_CONNECTION_REFUSED', 'net::ERR_CONNECTION_RESET']); +/** The wait the Workbench project client observes after every failed session probe (`project-client.ts`). */ +const projectSessionRetryDelayMs = 250; + +/** + * How late Playwright may hand a network event to the ledger on a loaded + * runner before the ledger stops explaining a short retry gap with it: half + * the client's delay, so a probe issued without any delay can never pass. + */ +const maximumDeliveryDelayMs = 125; + /** Chromium reports a severed old-stream socket as RESET or, when the reconnect never attached, SOCKET_NOT_CONNECTED. */ const oldStreamSeveranceCodes: ReadonlySet = new Set(['net::ERR_CONNECTION_RESET', 'net::ERR_SOCKET_NOT_CONNECTED']); @@ -263,13 +278,14 @@ export const validateOutageLedger = (ledger: OutageLedger): void => { } const navigationFailures = new Set(); for (const navigation of postRecovery.navigation) { - // Inclusive on both ends (see OutageLedger.postRecovery.navigation): a - // request delivered in the same millisecond the test stamped its - // departure still belongs to the visit it was issued from. - const failures = postRecoveryFailures.filter((request) => - request.url === navigation.url && request.at >= navigation.openedAt && request.at <= navigation.leftAt && - ledgerFailureAt(request) >= navigation.leftAt, - ); + // Membership by delivery order (see OutageLedger.postRecovery.navigation): + // the visit owns the requests handed over between its arrival and its + // departure stamps, whatever millisecond they carry. + const failures = postRecoveryFailures.filter((request) => { + const index = ledger.requests.indexOf(request); + return request.url === navigation.url && index >= navigation.openedIndex && index < navigation.leftIndex && + ledgerFailureAt(request) >= navigation.leftAt; + }); assertOutageLedger(failures.length <= 1, `multiple action-induced navigation cancellations: ${JSON.stringify({ failures, navigation })}`); for (const failure of failures) { @@ -291,9 +307,7 @@ export const validateOutageLedger = (ledger: OutageLedger): void => { } for (const failure of failures) navigationFailures.add(failure); } - // Two adjacent routes may list the same URL, and the departure stamp of one - // is the arrival stamp of the next, so one abort can satisfy two records; - // what must hold is that every post-recovery failure is claimed by some + // What must hold is that every post-recovery failure is claimed by some // contract. Report only the unclaimed ones — a dump of every failure reads // as if the recognized ones were at fault. const unrecognizedPostRecoveryFailures = postRecoveryFailures.filter((request) => @@ -366,17 +380,22 @@ export const validateOutageLedger = (ledger: OutageLedger): void => { assertOutageLedger(retryAttempts.at(-1) === firstSuccessfulBSession && firstSuccessfulBSession.status === 200, `project/session recovery did not finish with the first successful B session: ${JSON.stringify(retryAttempts)}`); // `at` is stamped when Playwright delivers the `request` event to Node, not - // when the page issued the probe, and a delivery can only run late: one - // held-back event shortens the next measured gap by exactly what it - // lengthened its own (a 284 ms / 222 ms pair on a loaded host). The client - // waits 250 ms after every failed probe, so the cadence is asserted as a mean - // over the retry sequence, where delivery latency cancels, while a burst — - // two probes issued without the delay — still fails on its own gap. - const retryGaps = retryAttempts.slice(1).map((attempt, index) => attempt.at - retryAttempts[index]!.at); - assertOutageLedger(retryGaps.every((gap) => gap >= 125), - `project/session retries began too quickly: ${JSON.stringify(retryAttempts)}`); - assertOutageLedger(retryGaps.reduce((sum, gap) => sum + gap, 0) >= 225 * retryGaps.length, - `project/session retries were paced below the client's 250 ms delay: ${JSON.stringify(retryAttempts)}`); + // when the page issued the probe, and a delivery can only run late — by at + // most `maximumDeliveryDelayMs` here — so a held-back event lengthens its own + // gap and shortens the next by the same amount (a 284 ms / 222 ms pair on a + // loaded host). A gap may therefore fall short of the client's 250 ms only by + // lateness already accumulated: the credit starts at the ceiling (the first + // probe may itself have been delivered late), every gap adds or spends its + // difference from 250 ms, capped at the ceiling, and it may never go + // negative. A burst (one probe issued without the delay) overspends at once; + // an under-paced client cannot borrow from gaps it has not produced yet. + let deliveryLatenessCredit = maximumDeliveryDelayMs; + for (const [index, attempt] of retryAttempts.entries()) { + if (index === 0) continue; + deliveryLatenessCredit = Math.min(maximumDeliveryDelayMs, deliveryLatenessCredit + (attempt.at - retryAttempts[index - 1]!.at) - projectSessionRetryDelayMs); + assertOutageLedger(deliveryLatenessCredit >= 0, + `project/session retries began too quickly (attempt ${String(index + 1)} arrived ${String(-deliveryLatenessCredit)} ms before any delivery lateness could explain): ${JSON.stringify(retryAttempts)}`); + } const retryTimeline = retryAttempts.flatMap((request) => [ Object.freeze({ at: request.at, delta: 1 }), Object.freeze({ at: request.completedAt!, delta: -1 }), @@ -388,7 +407,7 @@ export const validateOutageLedger = (ledger: OutageLedger): void => { maxInFlight = Math.max(maxInFlight, inFlight); } assertOutageLedger(maxInFlight <= 1 && inFlight === 0, `project/session retry concurrency exceeded one: ${JSON.stringify(retryAttempts)}`); - const retryUpperBound = 2 + Math.ceil((ledger.recoveredAt - ledger.outageStartedAt) / 250); + const retryUpperBound = 2 + Math.ceil((ledger.recoveredAt - ledger.outageStartedAt) / projectSessionRetryDelayMs); assertOutageLedger(retryAttempts.length <= retryUpperBound, `project/session retries exceeded the bounded cadence (${String(retryUpperBound)}): ${JSON.stringify(retryAttempts)}`); From 879411485af7e72549aad2c7bc9227e5a40eaf58 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 5 Sep 2026 05:21:25 +0000 Subject: [PATCH 13/15] test(packed-release): accept ERR_SOCKET_NOT_CONNECTED as a dying-server probe failure in the outage ledger (#576) CI (Release gates, runs 33933481002 and 33936651225) rejected the foreground outage ledger with "project/session retry had a non-connection failure": the first GET /api/project/session probe of the outage, issued while closeChild was SIGTERM-ing the server, went out over a keep-alive connection the server had already closed and failed 9 ms / 18 ms in with net::ERR_SOCKET_NOT_CONNECTED (no response headers); every later probe was REFUSED and recovery answered 200. Add the code to downServerProbeCodes beside ERR_CONNECTION_RESET, so it is accepted exactly where RESET is: a same-origin GET /api/project/session probe inside the outage window with one terminal state, paced by the client's 250 ms cadence, and paired 1:1 with a console error carrying the same code and URL. The old-session DELETE, non-GET probes, probes carrying response headers, probes outside the window, and unpaired console errors stay rejected with their existing messages; the unit test gains a row for each plus the CI-shaped first-probe ledger. --- .../tests/packed-outage-ledger.test.ts | 93 +++++++++++++++++-- .../tests/support/packed-outage-ledger.ts | 12 ++- 2 files changed, 95 insertions(+), 10 deletions(-) diff --git a/packages/workbench/tests/packed-outage-ledger.test.ts b/packages/workbench/tests/packed-outage-ledger.test.ts index 6378805c4..7ee462ede 100644 --- a/packages/workbench/tests/packed-outage-ledger.test.ts +++ b/packages/workbench/tests/packed-outage-ledger.test.ts @@ -13,7 +13,8 @@ import { test('outage ledger rejects the legacy duplicate, cross-origin, and missing-cleanup false positives', () => { const valid = outageLedgerFixture(); - const oldStreamPath = `/api/mcp/sessions/${encodeURIComponent(valid.oldSessionId)}/stream`; + const oldSessionPath = `/api/mcp/sessions/${encodeURIComponent(valid.oldSessionId)}`; + const oldStreamPath = `${oldSessionPath}/stream`; const resetRequest = ledgerRequest({ at: 999, completedAt: 1_008, @@ -106,19 +107,77 @@ test('outage ledger rejects the legacy duplicate, cross-origin, and missing-clea ...valid.requests, ]), }); - const resetSessionProbe = Object.freeze({ + // The fixture's single failed project/session probe, and the console error + // it logged, failing with `code` instead of REFUSED. + const withSessionProbeCode = (code: string): OutageLedger => Object.freeze({ ...valid, consoleErrors: Object.freeze(valid.consoleErrors.map((consoleError) => consoleError.url === `${valid.origin}/api/project/session` - ? Object.freeze({ ...consoleError, text: 'Failed to load resource: net::ERR_CONNECTION_RESET' }) + ? Object.freeze({ ...consoleError, text: `Failed to load resource: ${code}` }) : consoleError, )), requests: Object.freeze(valid.requests.map((request) => request.path === '/api/project/session' && request.error !== undefined - ? ledgerRequest({ ...request, error: 'net::ERR_CONNECTION_RESET' }) + ? ledgerRequest({ ...request, error: code }) : request, )), }); + const resetSessionProbe = withSessionProbeCode('net::ERR_CONNECTION_RESET'); + const socketNotConnectedSessionProbe = withSessionProbeCode('net::ERR_SOCKET_NOT_CONNECTED'); + // SOCKET_NOT_CONNECTED is a dying-server probe failure and nothing else: the + // old-session DELETE, a non-GET on the probe path, a probe that carries + // response headers, and a probe outside the outage window all stay rejected + // with their existing messages. + const socketNotConnectedDelete = Object.freeze({ + ...valid, + consoleErrors: Object.freeze(valid.consoleErrors.map((consoleError) => + consoleError.url === `${valid.origin}${oldSessionPath}` + ? Object.freeze({ ...consoleError, text: 'Failed to load resource: net::ERR_SOCKET_NOT_CONNECTED' }) + : consoleError, + )), + requests: Object.freeze(valid.requests.map((request) => + request.method === 'DELETE' ? ledgerRequest({ ...request, error: 'net::ERR_SOCKET_NOT_CONNECTED' }) : request, + )), + }); + const socketNotConnectedSessionPost = Object.freeze({ + ...valid, + consoleErrors: Object.freeze([ + ...valid.consoleErrors, + Object.freeze({ at: 1_022, text: 'Failed to load resource: net::ERR_SOCKET_NOT_CONNECTED', url: `${valid.origin}/api/project/session` }), + ]), + requests: Object.freeze([ + ...valid.requests, + ledgerRequest({ at: 1_020, completedAt: 1_021, error: 'net::ERR_SOCKET_NOT_CONNECTED', method: 'POST', path: '/api/project/session' }), + ]), + }); + const socketNotConnectedRespondedProbe = Object.freeze({ + ...socketNotConnectedSessionProbe, + requests: Object.freeze(socketNotConnectedSessionProbe.requests.map((request) => + request.error === 'net::ERR_SOCKET_NOT_CONNECTED' ? ledgerRequest({ ...request, respondedAt: request.at, status: 503 }) : request, + )), + }); + const socketNotConnectedPreOutageProbe = Object.freeze({ + ...valid, + consoleErrors: Object.freeze([ + Object.freeze({ at: 992, text: 'Failed to load resource: net::ERR_SOCKET_NOT_CONNECTED', url: `${valid.origin}/api/project/session` }), + ...valid.consoleErrors, + ]), + requests: Object.freeze([ + ledgerRequest({ at: 990, completedAt: 991, error: 'net::ERR_SOCKET_NOT_CONNECTED', method: 'GET', path: '/api/project/session' }), + ...valid.requests, + ]), + }); + // The code on the console error alone, or on the probe alone, pairs with + // nothing: a console error still needs its own request failure and a + // request failure its own console error. + const socketNotConnectedConsoleWithoutFailure = Object.freeze({ + ...valid, + consoleErrors: socketNotConnectedSessionProbe.consoleErrors, + }); + const socketNotConnectedProbeWithoutConsole = Object.freeze({ + ...socketNotConnectedSessionProbe, + consoleErrors: Object.freeze(valid.consoleErrors.filter((consoleError) => consoleError.url !== `${valid.origin}/api/project/session`)), + }); const knownPreOutageCatalogCancellation = Object.freeze({ ...valid, requests: Object.freeze([ @@ -294,21 +353,31 @@ test('outage ledger rejects the legacy duplicate, cross-origin, and missing-clea )), }), }); - // Replaces the fixture's single project/session probe with `probeAts` refused + // Replaces the fixture's single project/session probe with `probeAts` failed // probes (each paired with its console error) and a success at `successAt`. - const withRetryProbes = (probeAts: readonly number[], successAt: number): OutageLedger => Object.freeze({ + // A probe is refused unless `codes` names its failure at the same index. + const withRetryProbes = (probeAts: readonly number[], successAt: number, codes: readonly string[] = []): OutageLedger => Object.freeze({ ...valid, consoleErrors: Object.freeze([ ...valid.consoleErrors.filter((consoleError) => consoleError.url !== `${valid.origin}/api/project/session`), - ...probeAts.map((at) => Object.freeze({ at: at + 2, text: 'Failed to load resource: net::ERR_CONNECTION_REFUSED', url: `${valid.origin}/api/project/session` })), + ...probeAts.map((at, index) => Object.freeze({ + at: at + 2, text: `Failed to load resource: ${codes[index] ?? 'net::ERR_CONNECTION_REFUSED'}`, url: `${valid.origin}/api/project/session`, + })), ]), recoveredAt: successAt + 1, requests: Object.freeze([ ...valid.requests.filter((request) => request.path !== '/api/project/session'), - ...probeAts.map((at) => ledgerRequest({ at, completedAt: at + 1, error: 'net::ERR_CONNECTION_REFUSED', method: 'GET', path: '/api/project/session' })), + ...probeAts.map((at, index) => ledgerRequest({ + at, completedAt: at + 1, error: codes[index] ?? 'net::ERR_CONNECTION_REFUSED', method: 'GET', path: '/api/project/session', + })), ledgerRequest({ at: successAt, completedAt: successAt + 1, method: 'GET', path: '/api/project/session', status: 200 }), ]), }); + // The CI shape (Release gates, runs 33933481002 and 33936651225): the first + // probe of the outage went out over a keep-alive connection the closing + // server had already shut and failed with SOCKET_NOT_CONNECTED; every later + // probe found the port closed, and recovery answered 200. + const socketNotConnectedFirstRetry = withRetryProbes([1_010, 1_266, 1_518], 1_770, ['net::ERR_SOCKET_NOT_CONNECTED']); // One probe's `request` event delivered 34 ms late: the gap before it reads // 284 ms and the gap after it 222 ms, while the page kept its 250 ms delay. const lateDeliveredRetry = withRetryProbes([1_010, 1_260, 1_544, 1_766], 2_016); @@ -327,6 +396,14 @@ test('outage ledger rejects the legacy duplicate, cross-origin, and missing-clea expect(() => validateOutageLedger(validOldStreamReset)).not.toThrow(); expect(() => validateOutageLedger(validOldStreamSocketNotConnected)).not.toThrow(); expect(() => validateOutageLedger(resetSessionProbe)).not.toThrow(); + expect(() => validateOutageLedger(socketNotConnectedSessionProbe)).not.toThrow(); + expect(() => validateOutageLedger(socketNotConnectedFirstRetry)).not.toThrow(); + expect(() => validateOutageLedger(socketNotConnectedDelete)).toThrow(/old-session DELETE must succeed or fail exactly with ERR_CONNECTION_REFUSED/u); + expect(() => validateOutageLedger(socketNotConnectedSessionPost)).toThrow(/unknown outage failure/u); + expect(() => validateOutageLedger(socketNotConnectedRespondedProbe)).toThrow(/project\/session retry is missing or has multiple terminal states/u); + expect(() => validateOutageLedger(socketNotConnectedPreOutageProbe)).toThrow(/unexpected pre-outage failures/u); + expect(() => validateOutageLedger(socketNotConnectedConsoleWithoutFailure)).toThrow(/console error does not uniquely pair with an outage request failure/u); + expect(() => validateOutageLedger(socketNotConnectedProbeWithoutConsole)).toThrow(/outage request failures lack a unique paired console error/u); expect(() => validateOutageLedger(preStartedOutageStreamTermination)).not.toThrow(); expect(() => validateOutageLedger(knownPreOutageCatalogCancellation)).not.toThrow(); expect(() => validateOutageLedger(knownPreOutageSessionReplayCancellation)).not.toThrow(); diff --git a/packages/workbench/tests/support/packed-outage-ledger.ts b/packages/workbench/tests/support/packed-outage-ledger.ts index 32292d54d..601707c8f 100644 --- a/packages/workbench/tests/support/packed-outage-ledger.ts +++ b/packages/workbench/tests/support/packed-outage-ledger.ts @@ -200,8 +200,16 @@ export const hasCanonicalAfterCursor = (url: URL): boolean => { return Number.isSafeInteger(parsed) && parsed >= 0 && String(parsed) === after && url.search === `?after=${after}`; }; -/** A probe against a dying server can hit its half-open socket (RESET) instead of a closed port (REFUSED). */ -const downServerProbeCodes: ReadonlySet = new Set(['net::ERR_CONNECTION_REFUSED', 'net::ERR_CONNECTION_RESET']); +/** + * A probe against a dying server can hit its half-open socket (RESET), or go + * out over a keep-alive connection the server has already closed — Chromium + * reports that as SOCKET_NOT_CONNECTED — instead of a closed port (REFUSED). + * `closeChild` (`packed-release-harness.ts`) stops the server with SIGTERM, so + * the first probe of an outage races the server closing its idle sockets by + * sub-millisecond ordering (CI runs 33933481002 and 33936651225: 9 ms and + * 18 ms in, no response headers, every later probe REFUSED). + */ +const downServerProbeCodes: ReadonlySet = new Set(['net::ERR_CONNECTION_REFUSED', 'net::ERR_CONNECTION_RESET', 'net::ERR_SOCKET_NOT_CONNECTED']); /** The wait the Workbench project client observes after every failed session probe (`project-client.ts`). */ const projectSessionRetryDelayMs = 250; From b9a34080dbd1a3e119cc5dd412d23515cd179b19 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 5 Sep 2026 05:28:01 +0000 Subject: [PATCH 14/15] test(packed-release): reject a fresh-B stream abort that completes after the DELETE; trim the changeset summary to the user-facing sentence (#584) --- .changeset/576-proxy-upstream-timeout.md | 2 +- packages/workbench/tests/packed-outage-ledger.test.ts | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.changeset/576-proxy-upstream-timeout.md b/.changeset/576-proxy-upstream-timeout.md index d6d3d9afc..05fa83f1e 100644 --- a/.changeset/576-proxy-upstream-timeout.md +++ b/.changeset/576-proxy-upstream-timeout.md @@ -2,4 +2,4 @@ "agent-bundle": patch --- -Make the dev runtime client-surface proxy's upstream request timeout configurable: `RuntimeClientSurfaceProxy.open` accepts a trailing `RuntimeClientSurfaceProxyOptions` with `upstreamRequestTimeoutMs` (default `defaultRuntimeClientSurfaceUpstreamRequestTimeoutMs`, 15 000 ms; a value that is not a positive safe integer within the `setTimeout` ceiling is rejected before the proxy opens). The dev server keeps the 15 s default; the proxy's own deadline tests no longer wait on the real timeout (#584) +Make the dev runtime client-surface proxy's upstream request timeout configurable: `RuntimeClientSurfaceProxy.open` accepts a trailing `RuntimeClientSurfaceProxyOptions` with `upstreamRequestTimeoutMs` (default `defaultRuntimeClientSurfaceUpstreamRequestTimeoutMs`, 15 000 ms; a value that is not a positive safe integer within the `setTimeout` ceiling is rejected before the proxy opens). The dev server keeps the 15 s default (#584) diff --git a/packages/workbench/tests/packed-outage-ledger.test.ts b/packages/workbench/tests/packed-outage-ledger.test.ts index 7ee462ede..eee242364 100644 --- a/packages/workbench/tests/packed-outage-ledger.test.ts +++ b/packages/workbench/tests/packed-outage-ledger.test.ts @@ -323,6 +323,14 @@ test('outage ledger rejects the legacy duplicate, cross-origin, and missing-clea request.path.startsWith('/api/mcp/sessions/fresh-browser-mcp-session/') ? ledgerRequest({ ...request, completedAt: 1_319 }) : request, )), }); + // The other edge: the page aborts both streams before it issues the DELETE, + // so an abort that completes after the DELETE completed was not the close's. + const postCloseFreshStreamCancellation = Object.freeze({ + ...validPostRecovery, + requests: Object.freeze(validPostRecovery.requests.map((request) => + request.path.startsWith('/api/mcp/sessions/fresh-browser-mcp-session/') ? ledgerRequest({ ...request, completedAt: 1_322 }) : request, + )), + }); const hooksNavigation = validPostRecovery.postRecovery!.navigation[0]!; const hooksRequest = validPostRecovery.requests.at(-1)!; // The CI shape: the Hooks request and its response arrived in one batch and @@ -413,6 +421,7 @@ test('outage ledger rejects the legacy duplicate, cross-origin, and missing-clea expect(() => validateOutageLedger(logsReplayCancellationWithoutTerminal)).toThrow(/unexpected pre-outage failures/u); expect(() => validateOutageLedger(logsReplayCancellationAfterFailure)).toThrow(/unexpected pre-outage failures/u); expect(() => validateOutageLedger(preCloseFreshStreamCancellation)).toThrow(/fresh B MCP stream cancellation is not action-induced/u); + expect(() => validateOutageLedger(postCloseFreshStreamCancellation)).toThrow(/fresh B MCP stream cancellation is not action-induced/u); expect(() => validateOutageLedger(validPostRecovery)).not.toThrow(); expect(() => validateOutageLedger(navigationLiveStreamCancellation)).not.toThrow(); expect(() => validateOutageLedger(navigationRespondedCatalogCancellation)).not.toThrow(); From 91a2485b9400a47164f5bf462c7dd0ffabc8a48d Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 5 Sep 2026 05:52:58 +0000 Subject: [PATCH 15/15] test(scaffold-matrix): require a zero npm exit behind a passing pool report; pin expectPassedPool's fence in a unit test (#584) --- .../tests/scaffold-fixture.test.ts | 89 +++++++++++++++++++ .../tests/support/scaffold-fixture.ts | 43 ++++++--- 2 files changed, 119 insertions(+), 13 deletions(-) create mode 100644 packages/create-agent-bundle/tests/scaffold-fixture.test.ts diff --git a/packages/create-agent-bundle/tests/scaffold-fixture.test.ts b/packages/create-agent-bundle/tests/scaffold-fixture.test.ts new file mode 100644 index 000000000..136bfd547 --- /dev/null +++ b/packages/create-agent-bundle/tests/scaffold-fixture.test.ts @@ -0,0 +1,89 @@ +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { afterAll, beforeAll, describe, expect, it } from '@rstest/core'; + +import { expectPassedPool } from './support/scaffold-fixture.ts'; + +/** + * `expectPassedPool` is the release matrix's verdict on a scaffolded pool, so + * its own fence is pinned here against a stub project whose npm scripts print + * a Rstest-shaped JSON report and exit as instructed. The scaffolded pools + * themselves run in scaffold-packed-matrix.e2e.test.ts. + */ +const poolScript = ` +const scenario = process.argv[2]; +const report = (tests) => JSON.stringify({ + files: [{ status: tests.some((test) => test.status === 'fail') ? 'fail' : 'pass' }], + status: tests.some((test) => test.status === 'fail') ? 'fail' : 'pass', + summary: { failedTests: tests.filter((test) => test.status === 'fail').length }, + tests, +}, null, 2); +console.log('Rstest v0.0.0'); +switch (scenario) { + case 'pass': + console.log(report([{ name: 'greets', status: 'pass' }])); + break; + case 'pass-exit-1': + console.log(report([{ name: 'greets', status: 'pass' }])); + process.exitCode = 1; + break; + case 'fail': + console.log(report([{ name: 'greets', status: 'fail' }])); + process.exitCode = 1; + break; + case 'no-report': + console.error('failed to load rstest.config.ts'); + process.exitCode = 2; + break; + default: + throw new Error('unknown scenario ' + String(scenario)); +} +`; + +describe('expectPassedPool', () => { + let projectRoot = ''; + + beforeAll(async () => { + projectRoot = await mkdtemp(join(tmpdir(), 'scaffold-fixture-pool-')); + await writeFile(join(projectRoot, 'pool.mjs'), poolScript); + await writeFile(join(projectRoot, 'package.json'), JSON.stringify({ + name: 'pool-fixture', + private: true, + scripts: { + 'pool:fail': 'node pool.mjs fail', + 'pool:no-report': 'node pool.mjs no-report', + 'pool:pass': 'node pool.mjs pass', + 'pool:pass-exit-1': 'node pool.mjs pass-exit-1', + }, + version: '0.0.0', + }, null, 2)); + }); + + afterAll(async () => { + await rm(projectRoot, { force: true, recursive: true }); + }); + + it('accepts a passing report whose script exited 0 and names the expected tests', async () => { + await expect(expectPassedPool(projectRoot, 'pool:pass', ['greets'])).resolves.toBeUndefined(); + }); + + it('rejects a passing report whose script exited non-zero', async () => { + await expect(expectPassedPool(projectRoot, 'pool:pass-exit-1', ['greets'])) + .rejects.toThrow(/`npm run pool:pass-exit-1` exited 1 although its report says pass/u); + }); + + it('rejects a passing report that does not name an expected test', async () => { + await expect(expectPassedPool(projectRoot, 'pool:pass', ['greets', 'lists'])).rejects.toThrow(/lists/u); + }); + + it('reports the failing test entry before the exit code', async () => { + await expect(expectPassedPool(projectRoot, 'pool:fail', ['greets'])).rejects.toThrow(/greets[\s\S]*to deeply equal \[\]/u); + }); + + it('rejects a script that wrote no report, quoting its exit and stderr', async () => { + await expect(expectPassedPool(projectRoot, 'pool:no-report', ['greets'])) + .rejects.toThrow(/wrote no Rstest JSON report \(exit 2\)[\s\S]*failed to load rstest\.config\.ts/u); + }); +}); diff --git a/packages/create-agent-bundle/tests/support/scaffold-fixture.ts b/packages/create-agent-bundle/tests/support/scaffold-fixture.ts index b80e99452..7b0977706 100644 --- a/packages/create-agent-bundle/tests/support/scaffold-fixture.ts +++ b/packages/create-agent-bundle/tests/support/scaffold-fixture.ts @@ -154,7 +154,9 @@ export const npmRun = async (projectRoot: string, script: string): Promise => { - const stdout = await execFile('npm', ['run', script, '--', '--reporter=json'], { +interface PoolRun { + /** `0`, or the exit code — or the signal or spawn error code when there is none. */ + readonly exit: number | string; + readonly report: PoolReport; + readonly stderr: string; +} + +const poolRun = async (projectRoot: string, script: string): Promise => { + const run = await execFile('npm', ['run', script, '--', '--reporter=json'], { cwd: projectRoot, env: installedEnvironment(), - }).then((result) => result.stdout, (error: unknown) => { - const failed = error as { readonly stdout?: string }; - if (typeof failed.stdout !== 'string') throw error; - return failed.stdout; - }); + }).then( + (result) => ({ exit: 0, stderr: result.stderr, stdout: result.stdout }), + (error: unknown) => { + const failed = error as { readonly code?: number | string; readonly signal?: string; readonly stderr?: string; readonly stdout?: string }; + if (typeof failed.stdout !== 'string') throw error; + return { exit: failed.code ?? failed.signal ?? 'unknown', stderr: failed.stderr ?? '', stdout: failed.stdout }; + }, + ); // npm's script banner and Rstest's own precede the report on stdout; the // report is the only thing there that opens a line with `{`. - const start = stdout.search(/^\{$/mu); - if (start === -1) throw new Error(`\`npm run ${script}\` wrote no Rstest JSON report:\n${stdout}`); - return JSON.parse(stdout.slice(start)) as PoolReport; + const start = run.stdout.search(/^\{$/mu); + if (start === -1) { + throw new Error(`\`npm run ${script}\` wrote no Rstest JSON report (exit ${String(run.exit)}):\n${run.stdout}${run.stderr}`); + } + return { exit: run.exit, report: JSON.parse(run.stdout.slice(start)) as PoolReport, stderr: run.stderr }; }; /** @@ -184,18 +198,21 @@ const poolReport = async (projectRoot: string, script: string): Promise => { - const report = await poolReport(projectRoot, script); + const { exit, report, stderr } = await poolRun(projectRoot, script); expect(report.tests.filter((test) => test.status === 'fail')).toEqual([]); expect(report.files.filter((file) => file.status === 'fail')).toEqual([]); expect(report.tests.map((test) => test.name)).toEqual(expect.arrayContaining([...testNames])); expect(report).toMatchObject({ status: 'pass', summary: { failedTests: 0 } }); + if (exit !== 0) throw new Error(`\`npm run ${script}\` exited ${String(exit)} although its report says pass:\n${stderr}`); }; /** Zero diagnostics — including the informational AB473x migration nudges. */