test: fix the #576 test-level P1/P2 items — injectable proxy timeout, stubbed watcher, json-report scaffold assertions, relay-state readiness, polled request log, wire-derived outage fences

.changeset/576-proxy-upstream-timeout.md

@@ -0,0 +1,5 @@
+---
+"agent-bundle": patch
+---
+
+Make the dev runtime client-surface proxy's upstream request timeout configurable: `RuntimeClientSurfaceProxy.open` accepts a trailing `RuntimeClientSurfaceProxyOptions` with `upstreamRequestTimeoutMs` (default `defaultRuntimeClientSurfaceUpstreamRequestTimeoutMs`, 15 000 ms; a value that is not a positive safe integer within the `setTimeout` ceiling is rejected before the proxy opens). The dev server keeps the 15 s default (#584)

packages/agent-bundle/src/dev/runtime-client-surface-proxy.ts

@@ -23,7 +23,10 @@ import {
 
 const appAssetLimit = 4 * 1024 * 1024;
 const headerLimit = 16 * 1024;
-const upstreamRequestTimeout = 15_000;
+/** Bound on each upstream compiler request unless `RuntimeClientSurfaceProxyOptions` overrides it. */
+export const defaultRuntimeClientSurfaceUpstreamRequestTimeoutMs = 15_000;
+/** Node collapses longer `setTimeout` delays to 1 ms, which would silently drop the bound. */
+const maximumTimerDelayMs = 2_147_483_647;
 const loopbackHosts = new Set(['127.0.0.1', '::1']);
 /**
  * Proxy-owned browser push channel. The proxy authors both ends: its server
@@ -60,6 +63,16 @@ export interface RuntimeClientSurfaceConnectionEvent {
   readonly type: 'connected' | 'disconnected';
 }
 
+/** Server-only tuning for `RuntimeClientSurfaceProxy.open`; production callers take the defaults. */
+export interface RuntimeClientSurfaceProxyOptions {
+  /**
+   * Bound on each upstream compiler request — the bootstrap entry fetch and
+   * every proxied asset — from dispatch until its body has been read. A
+   * request still open at the deadline is aborted and answered 502.
+   */
+  readonly upstreamRequestTimeoutMs?: number;
+}
+
 interface ValidatedEndpoint {
   readonly entryPath: string;
   readonly host: string;
@@ -134,6 +147,14 @@ const contentSecurityPolicy = (input: RuntimeClientSurfaceContentPolicy): string
   return value;
 };
 
+const upstreamRequestTimeoutMs = (options: RuntimeClientSurfaceProxyOptions): number => {
+  const timeout = options.upstreamRequestTimeoutMs ?? defaultRuntimeClientSurfaceUpstreamRequestTimeoutMs;
+  if (!Number.isSafeInteger(timeout) || timeout < 1 || timeout > maximumTimerDelayMs) {
+    throw new TypeError(`Runtime client surface proxy options must use an integer upstreamRequestTimeoutMs from 1 to ${String(maximumTimerDelayMs)}.`);
+  }
+  return timeout;
+};
+
 const response = (target: ServerResponse, status: number): void => {
   if (target.destroyed || target.writableEnded) return;
   target.writeHead(status, { 'content-type': 'text/plain; charset=utf-8', 'x-content-type-options': 'nosniff' });
@@ -591,10 +612,12 @@ export class RuntimeClientSurfaceProxy {
     listener: (event: RuntimeClientSurfaceConnectionEvent) => void,
     hostOrigin: string,
     policy: RuntimeClientSurfaceContentPolicy = strictRuntimeClientSurfaceContentPolicy,
+    options: RuntimeClientSurfaceProxyOptions = {},
   ): Promise<DevRuntimeClientSurfaceProxyBinding> {
     const trusted = endpoint(input);
     const trustedHostOrigin = canonicalHostOrigin(hostOrigin);
     const trustedContentSecurityPolicy = contentSecurityPolicy(policy);
+    const trustedUpstreamRequestTimeoutMs = upstreamRequestTimeoutMs(options);
     const bootstrapCapability = randomBytes(32).toString('base64url');
     const sessionCapability = randomBytes(32).toString('base64url');
     const bootstrapPath = `/__agent_bundle_runtime/bootstrap/${bootstrapCapability}`;
@@ -656,7 +679,7 @@ export class RuntimeClientSurfaceProxy {
       const deadline = setTimeout(() => {
         abort();
         rejectPromise(new Error('Runtime client entry timed out.'));
-      }, upstreamRequestTimeout);
+      }, trustedUpstreamRequestTimeoutMs);
       const upstreamRequest = requestUpstream({
         agent: upstreamAgent,
         headers: { accept: 'text/html' },
@@ -772,7 +795,7 @@ export class RuntimeClientSurfaceProxy {
           timedOut = true;
           response(target, 502);
           abort();
-        }, upstreamRequestTimeout);
+        }, trustedUpstreamRequestTimeoutMs);
         const upstreamRequest = requestUpstream({
           agent: upstreamAgent,
           headers: {

packages/agent-bundle/tests/dev-host-install.test.ts

@@ -316,6 +316,10 @@ it('re-syncs the isolated Cursor install from coordinator epochs and ignores a f
   });
   const coordinator = new DevCoordinator({
     acquireLock: async () => ({ close: async () => undefined }),
+    // A no-op watcher: the real ProjectWatcher would turn this test's own source
+    // writes into an unrequested second rebuild that races the explicit
+    // rebuild() calls and can rewrite the install marker after settled() (#576).
+    createWatcher: () => ({ close: async () => undefined }),
     epochStore,
     eventHub,
     prepareCommand: 'dev',

packages/agent-bundle/tests/runtime-client-surface-proxy.test.ts

@@ -10,10 +10,20 @@ import {
   type DevRuntimeClientSurfaceEndpoint,
 } from '../src/dev/index.ts';
 import { runtimeAppMessageLimits } from '../src/dev/runtime-app-message-limits.ts';
+import {
+  defaultRuntimeClientSurfaceUpstreamRequestTimeoutMs,
+  type RuntimeClientSurfaceProxyOptions,
+} from '../src/dev/runtime-client-surface-proxy.ts';
 
 const foregroundOrigin = 'http://127.0.0.1:41999';
 const noopSubscribeReload = (): (() => void) => () => undefined;
 const reloadFrame = (generation: number): string => JSON.stringify({ generation, kind: 'runtime-app-reload' });
+/**
+ * Deadline tests bound the upstream far below the production default: long
+ * enough for the loopback bootstrap fetch that shares the bound, short enough
+ * that the file no longer waits on the real 15 s.
+ */
+const shortUpstreamRequestTimeout: RuntimeClientSurfaceProxyOptions = Object.freeze({ upstreamRequestTimeoutMs: 500 });
 
 /** Provider-side reload authority stub: the trusted channel the proxy relays. */
 const createReloadSource = () => {
@@ -31,7 +41,8 @@ const RuntimeClientSurfaceProxy = Object.freeze({
   open: (
     input: DevRuntimeClientSurfaceEndpoint,
     listener: Parameters<typeof RuntimeClientSurfaceProxyImplementation.open>[1],
-  ) => RuntimeClientSurfaceProxyImplementation.open(input, listener, foregroundOrigin),
+    options?: RuntimeClientSurfaceProxyOptions,
+  ) => RuntimeClientSurfaceProxyImplementation.open(input, listener, foregroundOrigin, undefined, options),
 });
 
 const listen = async (server: ReturnType<typeof createServer>): Promise<string> => {
@@ -573,6 +584,19 @@ it('rejects a custom-prototype child policy before opening a proxy binding', asy
   }, () => undefined, foregroundOrigin, policy as never)).rejects.toThrow('plain policy record');
 });
 
+it('bounds upstream requests at 15 s by default and rejects out-of-range overrides before opening', async () => {
+  expect(defaultRuntimeClientSurfaceUpstreamRequestTimeoutMs).toBe(15_000);
+  for (const upstreamRequestTimeoutMs of [0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY, 2_147_483_648]) {
+    await expect(RuntimeClientSurfaceProxy.open({
+      entryPath: '/app/index.html',
+      httpOrigin: 'http://127.0.0.1:41998',
+      httpPathPrefixes: ['/app/'],
+      surfaceId: 'app.weather',
+      subscribeReload: noopSubscribeReload,
+    }, () => undefined, { upstreamRequestTimeoutMs })).rejects.toThrow('upstreamRequestTimeoutMs from 1 to');
+  }
+});
+
 it('does not reinstall the opaque child when a held refresh fetch resolves after pagehide', async () => {
   const upstream = createServer((request, response) => {
     if (serveBootstrapEntry(request, response)) return;
@@ -1020,19 +1044,20 @@ it('bounds an upstream HTTP request before headers arrive', async () => {
     httpPathPrefixes: ['/app/'],
     surfaceId: 'app.weather',
     subscribeReload: noopSubscribeReload,
-  }, () => undefined);
+  }, () => undefined, shortUpstreamRequestTimeout);
 
   try {
     const cookie = await bootstrapCookie(binding);
     const pending = fetch(`${binding.origin}/app/index.html`, { headers: { cookie } });
     void pending.catch(() => undefined);
-    await expect(within(pending, 16_000)).resolves.toMatchObject({ status: 502 });
+    // Well under the 15 s default: a proxy that ignored the override would fail here.
+    await expect(within(pending, 2_000)).resolves.toMatchObject({ status: 502 });
   } finally {
     await binding.close();
     upstream.closeAllConnections();
     await close(upstream);
   }
-}, 20_000);
+});
 
 it('releases a reload-channel client that writes into the strictly one-way channel', async () => {
   const upstream = createServer((request, response) => {
@@ -1162,7 +1187,7 @@ it('keeps a completed 502 response intact when a response body stalls after head
     httpPathPrefixes: ['/app/'],
     surfaceId: 'app.weather',
     subscribeReload: noopSubscribeReload,
-  }, () => undefined);
+  }, () => undefined, shortUpstreamRequestTimeout);
 
   try {
     const cookie = await bootstrapCookie(binding);
@@ -1171,14 +1196,15 @@ it('keeps a completed 502 response intact when a response body stalls after head
       status: response.status,
     }));
     void pending.catch(() => undefined);
-    await expect(within(pending, 16_000)).resolves.toEqual({ body: 'Not Found', status: 502 });
+    // Well under the 15 s default: a proxy that ignored the override would fail here.
+    await expect(within(pending, 2_000)).resolves.toEqual({ body: 'Not Found', status: 502 });
     await expect(within(socketClosed, 250)).resolves.toBeUndefined();
   } finally {
     await binding.close();
     upstream.closeAllConnections();
     await close(upstream);
   }
-}, 20_000);
+});
 
 it('bounds chunked upstream assets and releases their socket immediately', async () => {
   let resolveSocketClosed: (() => void) | undefined;

packages/create-agent-bundle/tests/scaffold-fixture.test.ts

@@ -0,0 +1,89 @@
+import { mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+import { afterAll, beforeAll, describe, expect, it } from '@rstest/core';
+
+import { expectPassedPool } from './support/scaffold-fixture.ts';
+
+/**
+ * `expectPassedPool` is the release matrix's verdict on a scaffolded pool, so
+ * its own fence is pinned here against a stub project whose npm scripts print
+ * a Rstest-shaped JSON report and exit as instructed. The scaffolded pools
+ * themselves run in scaffold-packed-matrix.e2e.test.ts.
+ */
+const poolScript = `
+const scenario = process.argv[2];
+const report = (tests) => JSON.stringify({
+  files: [{ status: tests.some((test) => test.status === 'fail') ? 'fail' : 'pass' }],
+  status: tests.some((test) => test.status === 'fail') ? 'fail' : 'pass',
+  summary: { failedTests: tests.filter((test) => test.status === 'fail').length },
+  tests,
+}, null, 2);
+console.log('Rstest v0.0.0');
+switch (scenario) {
+  case 'pass':
+    console.log(report([{ name: 'greets', status: 'pass' }]));
+    break;
+  case 'pass-exit-1':
+    console.log(report([{ name: 'greets', status: 'pass' }]));
+    process.exitCode = 1;
+    break;
+  case 'fail':
+    console.log(report([{ name: 'greets', status: 'fail' }]));
+    process.exitCode = 1;
+    break;
+  case 'no-report':
+    console.error('failed to load rstest.config.ts');
+    process.exitCode = 2;
+    break;
+  default:
+    throw new Error('unknown scenario ' + String(scenario));
+}
+`;
+
+describe('expectPassedPool', () => {
+  let projectRoot = '';
+
+  beforeAll(async () => {
+    projectRoot = await mkdtemp(join(tmpdir(), 'scaffold-fixture-pool-'));
+    await writeFile(join(projectRoot, 'pool.mjs'), poolScript);
+    await writeFile(join(projectRoot, 'package.json'), JSON.stringify({
+      name: 'pool-fixture',
+      private: true,
+      scripts: {
+        'pool:fail': 'node pool.mjs fail',
+        'pool:no-report': 'node pool.mjs no-report',
+        'pool:pass': 'node pool.mjs pass',
+        'pool:pass-exit-1': 'node pool.mjs pass-exit-1',
+      },
+      version: '0.0.0',
+    }, null, 2));
+  });
+
+  afterAll(async () => {
+    await rm(projectRoot, { force: true, recursive: true });
+  });
+
+  it('accepts a passing report whose script exited 0 and names the expected tests', async () => {
+    await expect(expectPassedPool(projectRoot, 'pool:pass', ['greets'])).resolves.toBeUndefined();
+  });
+
+  it('rejects a passing report whose script exited non-zero', async () => {
+    await expect(expectPassedPool(projectRoot, 'pool:pass-exit-1', ['greets']))
+      .rejects.toThrow(/`npm run pool:pass-exit-1` exited 1 although its report says pass/u);
+  });
+
+  it('rejects a passing report that does not name an expected test', async () => {
+    await expect(expectPassedPool(projectRoot, 'pool:pass', ['greets', 'lists'])).rejects.toThrow(/lists/u);
+  });
+
+  it('reports the failing test entry before the exit code', async () => {
+    await expect(expectPassedPool(projectRoot, 'pool:fail', ['greets'])).rejects.toThrow(/greets[\s\S]*to deeply equal \[\]/u);
+  });
+
+  it('rejects a script that wrote no report, quoting its exit and stderr', async () => {
+    await expect(expectPassedPool(projectRoot, 'pool:no-report', ['greets']))
+      .rejects.toThrow(/wrote no Rstest JSON report \(exit 2\)[\s\S]*failed to load rstest\.config\.ts/u);
+  });
+});

packages/create-agent-bundle/tests/scaffold-packed-matrix.e2e.test.ts

@@ -11,6 +11,7 @@ import { installedEnvironment, packOutputFromJson } from '../../agent-bundle/tes
 import {
   cleanupScaffoldFixture,
   expectCleanValidate,
+  expectPassedPool,
   installScaffoldedProject,
   npmRun,
   scaffoldProject,
@@ -45,12 +46,10 @@ it.concurrent('scaffolds the mcp-server template and serves the conventional ent
   expect(checked).toContain('tests/projection/mcp-in-memory.test.ts');
   // The route-unit pool renders through the framework's own generated setup,
   // resolved from the packed tarball's `agent-bundle/rstest` export.
-  const routes = await npmRun(projectRoot, 'test:routes');
-  expect(routes).toContain('renders a known service into a final Agent Document');
-  expect(routes).toContain('"failedTests": 0');
-  const projection = await npmRun(projectRoot, 'test:projection');
-  expect(projection).toContain('projects the rendered document into the protocol result the server returns');
-  expect(projection).toContain('"failedTests": 0');
+  await expectPassedPool(projectRoot, 'test:routes', ['renders a known service into a final Agent Document']);
+  await expectPassedPool(projectRoot, 'test:projection', [
+    'projects the rendered document into the protocol result the server returns',
+  ]);
 
   const artifact = join(projectRoot, 'artifact');
   const manifest = JSON.parse(await readFile(join(artifact, 'portable', 'mcp.json'), 'utf8')) as {
@@ -92,10 +91,10 @@ it.concurrent('scaffolds the cli-tool template with a routed bin, lib, and artif
   await npmRun(projectRoot, 'prepack');
   // The projection pool dispatches through the framework's own generated
   // setup, resolved from the packed tarball's `agent-bundle/rstest` export.
-  const projection = await npmRun(projectRoot, 'test:projection');
-  expect(projection).toContain('greets through the routed CLI shell and prints one canonical JSON line');
-  expect(projection).toContain('greets through the main process envelope');
-  expect(projection).toContain('"failedTests": 0');
+  await expectPassedPool(projectRoot, 'test:projection', [
+    'greets through the routed CLI shell and prints one canonical JSON line',
+    'greets through the main process envelope',
+  ]);
 
   // The src/cli/** convention produced the routed executable package bin:
   // generated help, the compiled argv grammar, and one canonical JSON line.

packages/create-agent-bundle/tests/support/scaffold-fixture.ts

@@ -146,6 +146,75 @@ export const npmRun = async (projectRoot: string, script: string): Promise<strin
   return `${stdout}${stderr}`;
 };
 
+/**
+ * Rstest's `json` report for one pool of a scaffolded project, run through
+ * the project's own npm script with the reporter requested on the command
+ * line. The default reporter's prose is not a contract: it expands per-test
+ * lines only for single-file runs, and under an AI-agent environment Rstest
+ * swaps in the `md` reporter altogether (which is where a `"failedTests"`
+ * key used to come from). A failing pool exits non-zero with its report
+ * already written, and the report names the failing test where the exit code
+ * alone would not, so a non-zero exit is read rather than thrown — and kept,
+ * because the exit is npm's verdict on the whole script where the report is
+ * only Rstest's.
+ */
+interface PoolReport {
+  readonly files: readonly { readonly status: string }[];
+  readonly status: 'fail' | 'pass';
+  readonly summary: { readonly failedTests: number };
+  readonly tests: readonly { readonly name: string; readonly status: string }[];
+}
+
+interface PoolRun {
+  /** `0`, or the exit code — or the signal or spawn error code when there is none. */
+  readonly exit: number | string;
+  readonly report: PoolReport;
+  readonly stderr: string;
+}
+
+const poolRun = async (projectRoot: string, script: string): Promise<PoolRun> => {
+  const run = await execFile('npm', ['run', script, '--', '--reporter=json'], {
+    cwd: projectRoot,
+    env: installedEnvironment(),
+  }).then(
+    (result) => ({ exit: 0, stderr: result.stderr, stdout: result.stdout }),
+    (error: unknown) => {
+      const failed = error as { readonly code?: number | string; readonly signal?: string; readonly stderr?: string; readonly stdout?: string };
+      if (typeof failed.stdout !== 'string') throw error;
+      return { exit: failed.code ?? failed.signal ?? 'unknown', stderr: failed.stderr ?? '', stdout: failed.stdout };
+    },
+  );
+  // npm's script banner and Rstest's own precede the report on stdout; the
+  // report is the only thing there that opens a line with `{`.
+  const start = run.stdout.search(/^\{$/mu);
+  if (start === -1) {
+    throw new Error(`\`npm run ${script}\` wrote no Rstest JSON report (exit ${String(run.exit)}):\n${run.stdout}${run.stderr}`);
+  }
+  return { exit: run.exit, report: JSON.parse(run.stdout.slice(start)) as PoolReport, stderr: run.stderr };
+};
+
+/**
+ * The pool passed and ran the named tests. Failing entries come first — a
+ * test's, then a file's, for a file that failed before it had tests — because
+ * they carry the error where the counts would only say that something
+ * failed. The names catch a dropped or empty pool, which Rstest reports as
+ * `fail` with zero tests — and `failedTests: 0`. Last, the script itself must
+ * have exited 0: a report that says `pass` while npm exited non-zero (a
+ * lifecycle script, a crash after the report was written) is not a pass.
+ */
+export const expectPassedPool = async (
+  projectRoot: string,
+  script: string,
+  testNames: readonly string[],
+): Promise<void> => {
+  const { exit, report, stderr } = await poolRun(projectRoot, script);
+  expect(report.tests.filter((test) => test.status === 'fail')).toEqual([]);
+  expect(report.files.filter((file) => file.status === 'fail')).toEqual([]);
+  expect(report.tests.map((test) => test.name)).toEqual(expect.arrayContaining([...testNames]));
+  expect(report).toMatchObject({ status: 'pass', summary: { failedTests: 0 } });
+  if (exit !== 0) throw new Error(`\`npm run ${script}\` exited ${String(exit)} although its report says pass:\n${stderr}`);
+};
+
 /** Zero diagnostics — including the informational AB473x migration nudges. */
 export const expectCleanValidate = async (projectRoot: string): Promise<void> => {
   const cli = join(projectRoot, 'node_modules', '.bin', 'agent-bundle');