perf(build): lex compiler bundles once instead of re-parsing every module with acorn

.changeset/validate-artifact-lex-once.md

@@ -0,0 +1,5 @@
+---
+"agent-bundle": patch
+---
+
+Cut artifact validation time in `agent-bundle build` and `agent-bundle validate --artifact` without dropping a check: modules the framework compiled (manifest kind `bundle`) are no longer re-parsed in full with `acorn` to prove they are JavaScript — the ESM lexer that drives the import-graph walk is their only syntax pass, while copied and generated modules the framework did not compile keep the full parse, as do all bundles of a build whose `tools` hatch could have rewritten the emitted assets — and each module's imports are read once per process by the digest of the bytes actually read, so the post-compile self-containment check and the two validation passes of one build share one lex. `AB6005` codes and messages are unchanged; `validate --artifact` results are identical for every emitted artifact. `examples/host-test`: build 40 s → 12.5 s, `validate --artifact` 17 s → 3.6 s; `examples/audiobook-curator`: build 12.7 s → 6.8 s. (#521)

packages/agent-bundle/src/build/artifact-validation-types.ts

@@ -6,11 +6,20 @@ import type {
   ArtifactHook,
 } from './emit.ts';
 import type { ArtifactManifest } from './manifest.ts';
+import type { ModuleSyntaxCheck } from './module-imports.ts';
 
 export interface ValidateArtifactOptions {
   /** Enables the one store-owned epoch staging marker after its exact schema validates. */
   readonly allowEpochStagingMarker?: true;
   readonly artifactRoot: string;
+  /**
+   * How the syntax of a module the framework compiled (manifest kind
+   * `bundle`) is checked: `lexed` (the default) trusts the bundler's own
+   * output to the ESM lexer; `parsed` runs the full parse a build selects
+   * when a consumer bundler hatch may have rewritten the emitted assets.
+   * Every other module is always parsed in full.
+   */
+  readonly bundleSyntaxCheck?: ModuleSyntaxCheck;
   /**
    * Artifact-relative paths of prebuilt payload files for pre-manifest
    * validation. Prebuilt files are integrity-checked but never subjected to

packages/agent-bundle/src/build/build.ts

@@ -508,8 +508,16 @@ export const build = async (options: BuildOptions): Promise<BuildResult> => {
       files: await listArtifactFiles(stageRoot),
       outputProvenance,
     });
+    // The bundler's own output is trusted to the ESM lexer; once a consumer
+    // hatch can rewrite emitted assets (a banner, a processAssets pass), the
+    // final bytes are no longer the bundler's proof and are parsed in full.
+    const bundleSyntaxCheck = options.tools?.rspack === undefined && options.tools?.rsbuild === undefined
+      ? 'lexed'
+      : 'parsed';
     const preManifestDiagnostics = await validateArtifactFiles({
       artifactRoot: stageRoot,
+      bundleSyntaxCheck,
+      manifestFiles: files,
       prebuiltPaths: new Set(outputProvenance
         .filter((output) => output.kind === 'prebuilt')
         .map((output) => output.path)),
@@ -527,7 +535,7 @@ export const build = async (options: BuildOptions): Promise<BuildResult> => {
         targets: stagedTargets,
       }),
     });
-    const diagnostics = await validateArtifact({ artifactRoot: stageRoot, registry: options.registry });
+    const diagnostics = await validateArtifact({ artifactRoot: stageRoot, bundleSyntaxCheck, registry: options.registry });
     if (diagnostics.some((entry) => entry.severity === 'error')) {
       throw new DiagnosticError(diagnostics);
     }

packages/agent-bundle/src/build/mcp-apps.ts

@@ -14,29 +14,23 @@ import {
   generatedMetaModuleSource,
   generatedModulesRoot,
   metaModuleSpecifier,
+  virtualModulesPluginConstructor,
 } from './meta.ts';
 import { collectBundledOutputEvidence } from './provenance.ts';
 
 export const mcpAppMimeType = 'text/html;profile=mcp-app';
 
 /**
- * Rsbuild and Rslib carry independent Rspack copies (the dual-engine reality
- * documented on `AgentBundleToolsConfig`), so the browser path checks its own
- * engine for the experimental virtual-module surface rather than borrowing
- * the Rslib guard.
+ * This engine's virtual-module plugin: the browser path checks the workspace
+ * `@rsbuild/core`'s own Rspack rather than borrowing the Rslib guard (see
+ * {@link virtualModulesPluginConstructor}).
  */
-const virtualModulesPluginConstructor = (): typeof rspack.experiments.VirtualModulesPlugin => {
-  const constructor = (rspack as { readonly experiments?: { readonly VirtualModulesPlugin?: unknown } })
-    .experiments?.VirtualModulesPlugin;
-  if (typeof constructor !== 'function') {
-    throw new Error(
-      'The Rspack engine nested in @rsbuild/core no longer exposes experiments.VirtualModulesPlugin, '
-      + 'which agent-bundle uses to serve the generated agent-bundle/meta module to browser MCP App bundles. '
-      + 'Pin @rsbuild/core to a version whose Rspack ships the plugin, or update agent-bundle.',
-    );
-  }
-  return constructor as typeof rspack.experiments.VirtualModulesPlugin;
-};
+const rsbuildVirtualModulesPlugin = (): typeof rspack.experiments.VirtualModulesPlugin =>
+  virtualModulesPluginConstructor(
+    rspack,
+    '@rsbuild/core',
+    'serve the generated agent-bundle/meta module to browser MCP App bundles',
+  );
 
 export interface CompiledMcpApp {
   readonly _meta?: Readonly<Record<string, unknown>>;
@@ -225,7 +219,7 @@ export const composeMcpAppsRsbuildConfig = (
     };
     // Added after the hatch mutator (this hook is merged last), so a consumer
     // cannot strip the generated identity module out of the compiler.
-    const VirtualModulesPlugin = virtualModulesPluginConstructor();
+    const VirtualModulesPlugin = rsbuildVirtualModulesPlugin();
     config.plugins = [
       ...(config.plugins ?? []),
       new VirtualModulesPlugin({ [metaModulePath]: generatedMetaModuleSource(options.meta) }),

packages/agent-bundle/src/build/meta.ts

@@ -109,3 +109,36 @@ export const generatedMetaModuleSource = (meta: AgentBundleMeta): string => [
   'export default meta;',
   '',
 ].join('\n');
+
+/** The shape both Rspack engines expose the experimental virtual-module surface through. */
+interface VirtualModulesEngine {
+  readonly experiments?: { readonly VirtualModulesPlugin?: unknown };
+}
+
+/**
+ * Generated sources ride Rspack's `experiments.VirtualModulesPlugin` instead
+ * of throwaway files on disk — an accepted design decision: the experimental
+ * surface is the cost of never touching the artifact tree with build-time
+ * scratch files. This narrow feature check turns an upstream rename or
+ * removal into an actionable diagnostic instead of an opaque resolution
+ * failure deep inside a build. Rsbuild and Rslib carry independent Rspack
+ * copies (the dual-engine reality documented on `AgentBundleToolsConfig`),
+ * so each build path checks its own `rspack` and names its own package.
+ */
+export const virtualModulesPluginConstructor = <Engine extends VirtualModulesEngine>(
+  rspack: Engine,
+  /** The package whose nested Rspack is checked, as the consumer pins it. */
+  packageName: string,
+  /** What agent-bundle serves through the plugin on this path, completing "uses to …". */
+  purpose: string,
+): NonNullable<NonNullable<Engine['experiments']>['VirtualModulesPlugin']> => {
+  const constructor = rspack.experiments?.VirtualModulesPlugin;
+  if (typeof constructor !== 'function') {
+    throw new Error(
+      `The Rspack engine nested in ${packageName} no longer exposes experiments.VirtualModulesPlugin, `
+        + `which agent-bundle uses to ${purpose}. `
+        + `Pin ${packageName} to a version whose Rspack ships the plugin, or update agent-bundle.`,
+    );
+  }
+  return constructor as NonNullable<NonNullable<Engine['experiments']>['VirtualModulesPlugin']>;
+};

packages/agent-bundle/src/build/module-imports.ts

@@ -0,0 +1,75 @@
+import { parse as parseJavaScript } from 'acorn';
+import { init, parse } from 'es-module-lexer';
+
+/**
+ * One import of an ES module as the lexer reports it: `specifier` is the
+ * literal module specifier (absent for a non-literal dynamic import), and
+ * `kind` tells a static import from a dynamic `import()` and from the
+ * `import.meta` pseudo-import that carries no module at all.
+ */
+export interface ModuleImport {
+  readonly kind: 'dynamic' | 'meta' | 'static';
+  readonly specifier: string | undefined;
+}
+
+/**
+ * How thoroughly a module's syntax is checked before its imports are read.
+ *
+ * - `lexed`: the ESM lexer is the only pass. It rejects unterminated strings,
+ *   templates, comments, and regexps and unbalanced braces — enough for a
+ *   module the framework's own bundler emitted, whose syntax is the
+ *   bundler's to guarantee. Re-parsing megabytes of bundler output to prove
+ *   it is JavaScript was the dominant cost of every build.
+ * - `parsed`: a full `acorn` parse runs first, so a module the framework did
+ *   not compile — a copied consumer script, a generated installer — keeps
+ *   the complete syntax check.
+ */
+export type ModuleSyntaxCheck = 'lexed' | 'parsed';
+
+const importKind = (dynamic: number): ModuleImport['kind'] =>
+  dynamic === -2 ? 'meta' : dynamic === -1 ? 'static' : 'dynamic';
+
+/**
+ * Imports already read from bytes with a known SHA-256, keyed by check level
+ * and digest. Within one process the same emitted bundle is scanned by the
+ * post-compile self-containment check and then by artifact validation, twice
+ * (before and after the manifest is written); the bytes never change between
+ * those passes, so the imports of a multi-megabyte bundle are lexed once.
+ * The records are a few dozen specifiers per module; the map stays bounded.
+ */
+const importsByDigest = new Map<string, readonly ModuleImport[]>();
+const importsByDigestLimit = 512;
+
+const remember = (key: string, imports: readonly ModuleImport[]): void => {
+  if (importsByDigest.size >= importsByDigestLimit) {
+    const oldest = importsByDigest.keys().next();
+    if (!oldest.done) importsByDigest.delete(oldest.value);
+  }
+  importsByDigest.set(key, imports);
+};
+
+/** Imports previously read (this process) from bytes with this digest at this check level. */
+export const rememberedModuleImports = (
+  check: ModuleSyntaxCheck,
+  sha256: string,
+): readonly ModuleImport[] | undefined => importsByDigest.get(`${check}:${sha256}`);
+
+/**
+ * Reads the imports of one ES module source, throwing on invalid syntax
+ * (the lexer's or, for `parsed`, acorn's). When the source's SHA-256 is
+ * known the result is remembered for the next pass over the same bytes.
+ */
+export const readModuleImports = async (
+  source: string,
+  options: { readonly check: ModuleSyntaxCheck; readonly sha256?: string },
+): Promise<readonly ModuleImport[]> => {
+  await init;
+  if (options.check === 'parsed') parseJavaScript(source, { ecmaVersion: 'latest', sourceType: 'module' });
+  const [records] = parse(source);
+  const imports = Object.freeze(records.map((record) => Object.freeze({
+    kind: importKind(record.d),
+    specifier: record.n,
+  })));
+  if (options.sha256 !== undefined) remember(`${options.check}:${options.sha256}`, imports);
+  return imports;
+};

packages/agent-bundle/src/build/rslib.ts

@@ -3,10 +3,10 @@
 // (https://rslib.rs/api/javascript-api/core).
 import { pluginReact } from '@rsbuild/plugin-react';
 import { createRslib, mergeRslibConfig, rspack, type LibConfig, type Rspack } from '@rslib/core';
-import { init, parse } from 'es-module-lexer';
 import { readFile, realpath } from 'node:fs/promises';
 import { join, resolve } from 'node:path';
 
+import { sha256Hex } from '../core/digest.ts';
 import { isErrno } from '../core/errors.ts';
 import { isRecord } from '../core/strict-json.ts';
 import type { AgentBundleToolsConfig } from '../core/types.ts';
@@ -19,7 +19,9 @@ import {
   generatedMetaModuleSource,
   generatedModulesRoot,
   metaModuleSpecifier,
+  virtualModulesPluginConstructor,
 } from './meta.ts';
+import { readModuleImports, type ModuleImport } from './module-imports.ts';
 import { collectBundledOutputEvidence, type BundledOutputEvidence } from './provenance.ts';
 
 export interface RslibVirtualModule {
@@ -61,26 +63,9 @@ interface RslibDependencies {
   readonly createRslib?: (options: Parameters<typeof createRslib>[0]) => Promise<Pick<RslibInstance, 'build' | 'inspectConfig'>>;
 }
 
-/**
- * Generated sources ride Rspack's `experiments.VirtualModulesPlugin` instead
- * of throwaway files on disk — an accepted design decision: the experimental
- * surface is the cost of never touching the artifact tree with build-time
- * scratch files. This narrow feature check turns an upstream rename or
- * removal into an actionable diagnostic instead of an opaque resolution
- * failure deep inside a build.
- */
-const virtualModulesPluginConstructor = (): typeof rspack.experiments.VirtualModulesPlugin => {
-  const constructor = (rspack as { readonly experiments?: { readonly VirtualModulesPlugin?: unknown } })
-    .experiments?.VirtualModulesPlugin;
-  if (typeof constructor !== 'function') {
-    throw new Error(
-      'The Rspack engine nested in @rslib/core no longer exposes experiments.VirtualModulesPlugin, '
-      + 'which agent-bundle uses to serve generated wrapper and registry modules. '
-      + 'Pin @rslib/core to a version whose Rspack ships the plugin, or update agent-bundle.',
-    );
-  }
-  return constructor as typeof rspack.experiments.VirtualModulesPlugin;
-};
+/** This engine's virtual-module plugin (see {@link virtualModulesPluginConstructor}). */
+const rslibVirtualModulesPlugin = (): typeof rspack.experiments.VirtualModulesPlugin =>
+  virtualModulesPluginConstructor(rspack, '@rslib/core', 'serve generated wrapper and registry modules');
 
 /**
  * The tools escape hatch is typed against the workspace `@rsbuild/core` (the
@@ -273,28 +258,26 @@ const reservedAliasViolation = (
  * Fail-closed self-containment check on the emitted bundles themselves:
  * no reserved specifier may survive bundling as a live import. This is the
  * belt behind the static externals check and the function-external guard.
- * The bundle is parsed as an ES module (the emitted format by contract), so
+ * The bundle is lexed as an ES module (the emitted format by contract), so
  * string literals or comments that merely mention a reserved specifier are
- * not violations.
+ * not violations. The lex is keyed by the bundle's digest, so artifact
+ * validation, which scans these same bytes next, reads the imports once.
  */
 const assertNoResidualReservedImports = async (
   entries: readonly RslibEntry[],
   outputRoot: string,
 ): Promise<void> => {
-  await init;
   await Promise.all(entries.map(async (entry) => {
     const reserved = reservedSpecifiers(entry);
-    const bundle = await readFile(resolve(outputRoot, entry.outputRelativePath), 'utf8');
-    // A bin banner shebang is legal for Node but not for the ESM lexer.
-    const source = bundle.startsWith('#!') ? bundle.slice(bundle.indexOf('\n') + 1) : bundle;
-    let imports: ReturnType<typeof parse>[0];
+    const bytes = await readFile(resolve(outputRoot, entry.outputRelativePath));
+    let imports: readonly ModuleImport[];
     try {
-      [imports] = parse(source);
+      imports = await readModuleImports(bytes.toString('utf8'), { check: 'lexed', sha256: sha256Hex(bytes) });
     } catch {
       throw new Error(`Generated executable ${JSON.stringify(entry.outputRelativePath)} did not parse as an ES module.`);
     }
     const residual = imports
-      .map((record) => record.n)
+      .map((record) => record.specifier)
       .find((specifier) => specifier !== undefined && reserved.includes(specifier));
     if (residual !== undefined) {
       throw new Error(
@@ -396,7 +379,7 @@ const assertExecutableConfig = (
     // resolved config without the plugin instance would resolve the
     // reserved generated paths against the real filesystem.
     const registryModules = virtualRegistryModules(cwd, entry, meta);
-    const constructor = virtualModulesPluginConstructor();
+    const constructor = rslibVirtualModulesPlugin();
     if (config.plugins?.some((plugin) => plugin instanceof constructor) !== true) {
       throw new Error('Rslib resolved a generated executable environment without its virtual modules.');
     }
@@ -468,12 +451,6 @@ export const composeEntryLibConfig = (
   ]);
   const enforceInvariants = (config: Rspack.Configuration): Rspack.Configuration => {
     config.output = { ...config.output, asyncChunks: false };
-    if (entry.rscManifest === true) {
-      config.plugins = [
-        ...(config.plugins ?? []),
-        new rspack.DefinePlugin({ __rspack_rsc_manifest__: JSON.stringify({ clientManifest: {}, cssLinkProps: {}, entryCssFiles: {}, entryJsFiles: [], moduleLoading: { prefix: '' }, serverConsumerModuleMap: {}, serverManifest: {} }) }),
-      ];
-    }
     if (entry.reactServer === true) {
       config.resolve = { ...config.resolve, conditionNames: ['react-server', '...'] };
     }
@@ -500,15 +477,18 @@ export const composeEntryLibConfig = (
         },
       };
     }
-    if (generatedModules.length > 0) {
-      // Added after the hatch mutator (this hook is merged last), so a
-      // consumer cannot strip the generated sources out of the compiler.
-      const VirtualModulesPlugin = virtualModulesPluginConstructor();
-      config.plugins = [
-        ...(config.plugins ?? []),
-        new VirtualModulesPlugin(Object.fromEntries(generatedModules.map((module) => [module.path, module.source]))),
-      ];
-    }
+    // Framework plugins are added after the hatch mutator (this hook is
+    // merged last), so a consumer cannot strip the RSC manifest stub or the
+    // generated sources out of the compiler.
+    const frameworkPlugins = [
+      ...(entry.rscManifest === true
+        ? [new rspack.DefinePlugin({ __rspack_rsc_manifest__: JSON.stringify({ clientManifest: {}, cssLinkProps: {}, entryCssFiles: {}, entryJsFiles: [], moduleLoading: { prefix: '' }, serverConsumerModuleMap: {}, serverManifest: {} }) })]
+        : []),
+      ...(generatedModules.length > 0
+        ? [new (rslibVirtualModulesPlugin())(Object.fromEntries(generatedModules.map((module) => [module.path, module.source])))]
+        : []),
+    ];
+    if (frameworkPlugins.length > 0) config.plugins = [...(config.plugins ?? []), ...frameworkPlugins];
     if (virtualSource !== undefined) {
       // Rslib validates `source.entry` against the real filesystem before
       // Rspack exists, so the profile keys the entry on the authored program