fix(test): make the sweep's flaky e2e and packed suites deterministic

.changeset/workbench-app-preview-close-before-ready.md

@@ -0,0 +1,5 @@
+---
+"agent-bundle": patch
+---
+
+Close a Workbench MCP App preview immediately when its sandbox proxy has not loaded yet: closing the preview (or switching its profile, deactivating the MCP page, or ending the session) before the proxy signals readiness now releases the binding at once instead of posting a teardown no window can acknowledge and holding the preview in its closing state for the full five-second force-close budget (#435)

docs/local-ci.md

@@ -187,6 +187,41 @@ write for a second epoch. Those edits now go through
 one atomic replacement, then a wait on the coordinator's published build
 attempt, so one edit is exactly one build.
 
+## Infrastructure failures and their retry policy
+
+Two failure shapes in the hosted Release gates are registry or runner
+infrastructure, not the tree under test. Neither gets a code-level retry,
+and neither is a reason to weaken the gate; the policy is to re-run the job
+once the cause has cleared, then treat a repeat as a real signal.
+
+- **`npm audit signatures` → `EATTESTATIONVERIFY`** (from
+  `scripts/audit-packed-release.mjs`, reached through `pnpm audit:release`).
+  Example (CI run 33584654855, 2026-09-02, Release gates on Node 22.19):
+  `@modelcontextprotocol/server@2.0.0 failed to verify attestation:
+  Unexpected end of JSON input`. npm fetched a truncated attestation bundle
+  from `registry.npmjs.org` for a dependency this repository does not
+  publish; the same pinned version and integrity verify on every later run
+  of the same gate without any lockfile change. The audit deliberately
+  installs against live registry metadata (no `--prefer-offline`), so a
+  registry-side transient reaches it unfiltered. Policy: read the JSON in the
+  step log first; if the `invalid` entry names a third-party package with an
+  unchanged pinned version and a parse-shaped message (`Unexpected end of
+  JSON input`, `Unexpected token`, a 5xx), re-run the failed job (`gh run
+  rerun <id> --failed`). If the same package fails twice in a row, or the
+  message is a genuine signature mismatch (`EATTESTATIONSIGNATURE`,
+  `EINTEGRITY`), stop and investigate the dependency before merging: that is
+  the supply-chain check doing its job. Do not add retries around the audit
+  command and do not relax `--json` parsing to tolerate the error.
+- **Runner network stalls during `npm install`** in the packed pool. The
+  pool's consumer installs are cache-backed per worker
+  (`rstest.worker-isolation.ts`): each worker pays for one cold download of
+  the packed dependency tree (about 180 MB), and `public-api-packed` warms
+  that cache in a `beforeAll` with its own budget so no per-test budget spans
+  a cold network. A remaining timeout inside that `beforeAll` on a hosted
+  runner is a registry or runner-network stall; re-run the job. A timeout in
+  a test body after the warm-up is not: read the failure text, it names the
+  step (install, `tsc`, CLI) that overran.
+
 ## What is deliberately not covered
 
 - **dependency-review** runs as a GitHub-side action against the GitHub

packages/agent-bundle/rslib.config.ts

@@ -32,6 +32,18 @@ const esmNodeGlobalsPlugin = (rspack: typeof RspackInstance): Rspack.RspackPlugi
   },
 });
 
+/**
+ * Rslib enables Rspack's persistent build cache by default and keys its
+ * directory by this config's root (`node_modules/.cache/rspack`), never by
+ * `--dist-path`. Two builds of this config running at once — the packed pool's
+ * packed-consumer and dev-workbench-packaging suites each rebuild it into an
+ * isolated dist from parallel workers — would then contend for a single cache
+ * lock ("Transaction already in progress by process … in directory …"). Test
+ * harnesses hand every spawned build its own directory through this variable
+ * (rstest.worker-isolation.ts); `pnpm build` keeps the default warm cache.
+ */
+const buildCacheDirectory = process.env['AGENT_BUNDLE_RSLIB_CACHE_DIRECTORY'];
+
 export default defineConfig({
   lib: [
     {
@@ -50,6 +62,9 @@ export default defineConfig({
     legalComments: 'linked',
     target: 'node',
   },
+  ...(buildCacheDirectory === undefined || buildCacheDirectory.length === 0
+    ? {}
+    : { performance: { buildCache: { cacheDirectory: buildCacheDirectory } } }),
   // Suggestions stay informational; errors and warnings block publishing.
   plugins: [pluginPublint({ throwOn: 'warning' })],
   root: import.meta.dirname,

packages/agent-bundle/tests/dev-workbench-packaging.test.ts

@@ -60,11 +60,14 @@ it('prunes stale copied workbench assets without removing the package library ou
     await mkdir(join(workbench, 'static', 'js', 'async'), { recursive: true });
     await writeFile(stale, 'obsolete workbench output\n');
     await expect(access(stale)).resolves.toBeUndefined();
+    // installedEnvironment() gives this build its own persistent-cache
+    // directory: packed-consumer rebuilds the same config from another
+    // worker, and Rslib keys the shared default by the config root.
     await execFile(join(workspaceRoot, 'node_modules', '.bin', 'rslib'), [
       'build',
       '--config', join(packageRoot, 'rslib.config.ts'),
       '--dist-path', isolatedDist,
-    ], { cwd: workspaceRoot });
+    ], { cwd: workspaceRoot, env: installedEnvironment() });
     await expect(access(stale)).rejects.toThrow();
     await expect(access(join(isolatedDist, 'cli.js'))).resolves.toBeUndefined();
     expect(await readdir(workbench, { recursive: true })).not.toContain('index.js.map');

packages/agent-bundle/tests/dev-workbench.test.ts

@@ -1,4 +1,4 @@
-import { access, mkdtemp, mkdir, readFile, rm, symlink, writeFile } from 'node:fs/promises';
+import { access, mkdtemp, mkdir, readdir, readFile, rm, symlink, writeFile } from 'node:fs/promises';
 import { get as httpGet } from 'node:http';
 import { tmpdir } from 'node:os';
 import { join, relative } from 'node:path';
@@ -1110,17 +1110,28 @@ it('prepares the optional runtime once with the development config context befor
   });
   let proxyCalls = 0;
   let surfaceCloseCalls = 0;
+  // Both fixture records live outside the watched project source. The dev
+  // watcher treats every non-ignored path under the project root as source
+  // (the project snapshot is broad by design), so a config that appended its
+  // calls inside the root, or a provider that wrote its start context there,
+  // would itself be a source change: the watcher rebuilds, the rebuild
+  // re-loads the config, the load appends again, and the dev-load count
+  // climbs until the test happens to read it. The provider uses its
+  // contract-provided storageRoot (`.agent-bundle/runtime/<session>`, excluded
+  // from watching and snapshots); the config logs beside the workbench assets.
+  const configCallsPath = join(assetsRoot, 'config-calls.ndjson');
   await mkdir(join(project.root, 'src', 'dev'), { recursive: true });
   await Promise.all([
     writeFile(join(assetsRoot, 'index.html'), '<!doctype html><title>Agent Bundle workbench</title>'),
     writeFile(join(project.root, 'src', 'dev', 'provider.ts'), [
-      "import { writeFile } from 'node:fs/promises';",
+      "import { mkdir, writeFile } from 'node:fs/promises';",
       "import { join } from 'node:path';",
       '',
       'export const createDevRuntimeProvider = () => ({',
       "  descriptor: { environmentVariables: [], id: 'fixture-runtime', label: 'Fixture runtime', schemaVersion: 1 },",
       '  start: async (context) => {',
-      "    await writeFile(join(context.projectRoot, 'provider-context.json'), JSON.stringify({",
+      '    await mkdir(context.storageRoot, { recursive: true });',
+      "    await writeFile(join(context.storageRoot, 'provider-context.json'), JSON.stringify({",
       '      artifact: context.artifactStatus(),',
       '      environment: context.environment,',
       '      preparedRuntime: context.preparedRuntime,',
@@ -1142,11 +1153,10 @@ it('prepares the optional runtime once with the development config context befor
     ].join('\n')),
     writeFile(project.configPath, [
       "import { appendFile } from 'node:fs/promises';",
-      "import { join } from 'node:path';",
       "import { defineConfig } from 'agent-bundle';",
       '',
-      'export default defineConfig(async ({ command, mode, projectRoot }) => {',
-      "  await appendFile(join(projectRoot, 'config-calls.ndjson'), JSON.stringify({ command, mode }) + '\\n');",
+      'export default defineConfig(async ({ command, mode }) => {',
+      `  await appendFile(${JSON.stringify(configCallsPath)}, JSON.stringify({ command, mode }) + '\\n');`,
       '  return {',
       "    dev: { runtime: { provider: './src/dev/provider.ts' } },",
       "    plugin: { name: 'runtime-fixture', version: '1.0.0' },",
@@ -1171,9 +1181,13 @@ it('prepares the optional runtime once with the development config context befor
       },
     });
 
+    const runtimeStorageRoot = join(project.root, '.agent-bundle', 'runtime');
+    const [providerSessionDirectory, ...otherSessionDirectories] = await readdir(runtimeStorageRoot);
+    if (providerSessionDirectory === undefined) throw new Error('The fixture provider did not create its storage root.');
+    expect(otherSessionDirectories).toEqual([]);
     const [calls, context, runtimeStatus, projectStatus] = await Promise.all([
-      readFile(join(project.root, 'config-calls.ndjson'), 'utf8'),
-      readFile(join(project.root, 'provider-context.json'), 'utf8').then(JSON.parse) as Promise<Record<string, unknown>>,
+      readFile(configCallsPath, 'utf8'),
+      readFile(join(runtimeStorageRoot, providerSessionDirectory, 'provider-context.json'), 'utf8').then(JSON.parse) as Promise<Record<string, unknown>>,
       fetch(`${server.url}/api/runtime/status`).then((response) => response.json()),
       fetch(`${server.url}/api/project/status`).then((response) => response.json()),
     ]);
@@ -1186,7 +1200,7 @@ it('prepares the optional runtime once with the development config context befor
       environment: {},
       preparedRuntime: { provider: './src/dev/provider.ts' },
       projectRoot: project.root,
-      storageRoot: expect.stringMatching(new RegExp(`^${join(project.root, '.agent-bundle', 'runtime').replace(/[.*+?^${}()|[\]\\]/gu, '\\$&')}/`)),
+      storageRoot: join(runtimeStorageRoot, providerSessionDirectory),
     });
     expect(runtimeStatus).toMatchObject({ status: { descriptor: { id: 'fixture-runtime' }, state: 'active' } });
     expect(projectStatus).toMatchObject({ status: { runtime: { state: 'configured' } } });

packages/agent-bundle/tests/public-api-packed.test.ts

@@ -4,7 +4,7 @@ import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { promisify } from 'node:util';
 
-import { expect, it } from '@rstest/core';
+import { beforeAll, expect, it } from '@rstest/core';
 
 import { isolatedCommandEnvironment } from '../../../rstest.worker-isolation.ts';
 import { writeFixtureManifest } from './support/manifest.ts';
@@ -51,6 +51,31 @@ const producerFrom = async (output: string): Promise<{ readonly name: string; re
   return manifest.producer;
 };
 
+/**
+ * Warms this worker's npm cache with the packed tarball's full dependency
+ * tree once, so the three consumer installs below are cache-backed
+ * (`--prefer-offline` then serves every tarball and metadata record from
+ * disk). The download is the one network-bound step in this file: about
+ * 180 MB of registry tarballs, and a CI runner's npm cache is empty at job
+ * start (pnpm/setup caches only the pnpm store), so the Release gates and
+ * Verify jobs always pay it exactly once here, never inside a test's 30 s
+ * budget. The budget below covers that cold download on a slow runner
+ * network; a warm worker cache finishes in a few seconds.
+ */
+beforeAll(async () => {
+  const { tarball } = await sharedPackedTarball('agent-bundle');
+  const warmRoot = await mkdtemp(join(tmpdir(), 'agent-bundle-packed-npm-warm-'));
+  try {
+    await writeFile(join(warmRoot, 'package.json'), '{"type":"module"}\n');
+    await execFile(
+      'npm', ['install', ...cachedNpmInstallArguments, tarball],
+      { cwd: warmRoot, env: isolatedCommandEnvironment() },
+    );
+  } finally {
+    await rm(warmRoot, { force: true, recursive: true });
+  }
+}, 180_000);
+
 it('writes the package version as the producer of a packed CLI manifest', async () => {
   const { tarball } = await sharedPackedTarball('agent-bundle');
 

packages/workbench/src/mcp/mcp-app-frame.tsx

@@ -220,6 +220,16 @@ export class McpAppFrameRelay {
     if (this.#closePromise !== undefined) return this.#closePromise;
     this.#state = 'closing';
     this.#closePromise = new Promise<void>((resolve) => { this.#finishClose = resolve; });
+    // Before the proxy signals readiness there is no app to tear down and no
+    // window that can acknowledge a teardown frame: the proxy document is
+    // still loading (or is the initial about:blank, whose origin never matches
+    // targetOrigin, so postMessage drops the frame silently). The graceful
+    // handshake could only wait out the force timer, so release the binding
+    // now instead of holding the closing state for the whole budget.
+    if (!this.#resourceProvided) {
+      void this.#forceClose();
+      return this.#closePromise;
+    }
     this.#closeTimer = setTimeout(() => { void this.#forceClose(); }, this.#closeTimeoutMs);
     this.#enqueue(() => this.#beginClose(), true);
     return this.#closePromise;

packages/workbench/tests/mcp-app-frame.test.ts

@@ -385,11 +385,15 @@ describe('MCP App frame relay', () => {
     expect(messages).toHaveLength(1);
   });
 
-  it('force-deletes a closing binding when the proxy never acknowledges the teardown frame', async () => {
+  it('force-deletes a closing binding when the ready proxy never acknowledges the teardown frame', async () => {
     const browser = fakeBrowser();
+    let closeCalls = 0;
     let forceClosed = false;
     const routes: McpAppFrameRelayRoutes = {
-      close: async (_bindingId, options) => closeResult({ id: options.id, jsonrpc: '2.0', method: 'ui/resource-teardown', params: {} }),
+      close: async (_bindingId, options) => {
+        closeCalls += 1;
+        return closeResult({ id: options.id, jsonrpc: '2.0', method: 'ui/resource-teardown', params: {} });
+      },
       forceClose: async () => {
         forceClosed = true;
         return true;
@@ -398,12 +402,44 @@ describe('MCP App frame relay', () => {
     };
     const relay = createMcpAppFrameRelay({ bindingId: 'binding-weather', closeTimeoutMs: 1, frame, iframe: browser.iframe, resource, routes, window: browser.window });
     relay.start();
+    relay.receive({ data: proxyReady(), origin: frame.targetOrigin, source: browser.child });
 
     await relay.close();
 
+    expect(closeCalls).toBe(1);
     expect(forceClosed).toBe(true);
   });
 
+  it('force-deletes immediately, without a teardown handshake or the timer, when the proxy never signaled readiness', async () => {
+    const browser = fakeBrowser();
+    let closeCalls = 0;
+    let forceCloseCalls = 0;
+    const routes: McpAppFrameRelayRoutes = {
+      close: async (_bindingId, options) => {
+        closeCalls += 1;
+        return closeResult({ id: options.id, jsonrpc: '2.0', method: 'ui/resource-teardown', params: {} });
+      },
+      forceClose: async () => {
+        forceCloseCalls += 1;
+        return true;
+      },
+      message: async () => messageResult(),
+    };
+    // The full 30 s budget: a timer-driven fallback would fail this test.
+    const relay = createMcpAppFrameRelay({ bindingId: 'binding-weather', closeTimeoutMs: 30_000, frame, iframe: browser.iframe, resource, routes, window: browser.window });
+    relay.start();
+
+    await relay.close();
+
+    expect(closeCalls).toBe(0);
+    expect(forceCloseCalls).toBe(1);
+    expect(relay.state).toBe('closed');
+    expect(browser.child.posts).toEqual([]);
+    // A proxy that reports ready after the close began is a late arrival, not
+    // a reopened relay.
+    expect(relay.receive({ data: proxyReady(), origin: frame.targetOrigin, source: browser.child })).toBe(false);
+  });
+
   it('renders the exact server-issued sandbox URL and no inline document or credential-bearing attribute', () => {
     const markup = renderToStaticMarkup(createElement(McpAppFrame, {
       bindingId: 'binding-weather',