diff --git a/.changeset/validate-artifact-lex-once.md b/.changeset/validate-artifact-lex-once.md new file mode 100644 index 000000000..becf2c6e2 --- /dev/null +++ b/.changeset/validate-artifact-lex-once.md @@ -0,0 +1,5 @@ +--- +"agent-bundle": patch +--- + +Cut artifact validation time in `agent-bundle build` and `agent-bundle validate --artifact` without dropping a check: modules the framework compiled (manifest kind `bundle`) are no longer re-parsed in full with `acorn` to prove they are JavaScript — the ESM lexer that drives the import-graph walk is their only syntax pass, while copied and generated modules the framework did not compile keep the full parse, as do all bundles of a build whose `tools` hatch could have rewritten the emitted assets — and each module's imports are read once per process by the digest of the bytes actually read, so the post-compile self-containment check and the two validation passes of one build share one lex. `AB6005` codes and messages are unchanged; `validate --artifact` results are identical for every emitted artifact. `examples/host-test`: build 40 s → 12.5 s, `validate --artifact` 17 s → 3.6 s; `examples/audiobook-curator`: build 12.7 s → 6.8 s. (#521) diff --git a/packages/agent-bundle/src/build/artifact-validation-types.ts b/packages/agent-bundle/src/build/artifact-validation-types.ts index 98c1120be..4c3ddcb94 100644 --- a/packages/agent-bundle/src/build/artifact-validation-types.ts +++ b/packages/agent-bundle/src/build/artifact-validation-types.ts @@ -6,11 +6,20 @@ import type { ArtifactHook, } from './emit.ts'; import type { ArtifactManifest } from './manifest.ts'; +import type { ModuleSyntaxCheck } from './module-imports.ts'; export interface ValidateArtifactOptions { /** Enables the one store-owned epoch staging marker after its exact schema validates. */ readonly allowEpochStagingMarker?: true; readonly artifactRoot: string; + /** + * How the syntax of a module the framework compiled (manifest kind + * `bundle`) is checked: `lexed` (the default) trusts the bundler's own + * output to the ESM lexer; `parsed` runs the full parse a build selects + * when a consumer bundler hatch may have rewritten the emitted assets. + * Every other module is always parsed in full. + */ + readonly bundleSyntaxCheck?: ModuleSyntaxCheck; /** * Artifact-relative paths of prebuilt payload files for pre-manifest * validation. Prebuilt files are integrity-checked but never subjected to diff --git a/packages/agent-bundle/src/build/build.ts b/packages/agent-bundle/src/build/build.ts index b37bd1ac7..be26fff1f 100644 --- a/packages/agent-bundle/src/build/build.ts +++ b/packages/agent-bundle/src/build/build.ts @@ -508,8 +508,16 @@ export const build = async (options: BuildOptions): Promise => { files: await listArtifactFiles(stageRoot), outputProvenance, }); + // The bundler's own output is trusted to the ESM lexer; once a consumer + // hatch can rewrite emitted assets (a banner, a processAssets pass), the + // final bytes are no longer the bundler's proof and are parsed in full. + const bundleSyntaxCheck = options.tools?.rspack === undefined && options.tools?.rsbuild === undefined + ? 'lexed' + : 'parsed'; const preManifestDiagnostics = await validateArtifactFiles({ artifactRoot: stageRoot, + bundleSyntaxCheck, + manifestFiles: files, prebuiltPaths: new Set(outputProvenance .filter((output) => output.kind === 'prebuilt') .map((output) => output.path)), @@ -527,7 +535,7 @@ export const build = async (options: BuildOptions): Promise => { targets: stagedTargets, }), }); - const diagnostics = await validateArtifact({ artifactRoot: stageRoot, registry: options.registry }); + const diagnostics = await validateArtifact({ artifactRoot: stageRoot, bundleSyntaxCheck, registry: options.registry }); if (diagnostics.some((entry) => entry.severity === 'error')) { throw new DiagnosticError(diagnostics); } diff --git a/packages/agent-bundle/src/build/mcp-apps.ts b/packages/agent-bundle/src/build/mcp-apps.ts index eb7c29712..a0b06ffd6 100644 --- a/packages/agent-bundle/src/build/mcp-apps.ts +++ b/packages/agent-bundle/src/build/mcp-apps.ts @@ -14,29 +14,23 @@ import { generatedMetaModuleSource, generatedModulesRoot, metaModuleSpecifier, + virtualModulesPluginConstructor, } from './meta.ts'; import { collectBundledOutputEvidence } from './provenance.ts'; export const mcpAppMimeType = 'text/html;profile=mcp-app'; /** - * Rsbuild and Rslib carry independent Rspack copies (the dual-engine reality - * documented on `AgentBundleToolsConfig`), so the browser path checks its own - * engine for the experimental virtual-module surface rather than borrowing - * the Rslib guard. + * This engine's virtual-module plugin: the browser path checks the workspace + * `@rsbuild/core`'s own Rspack rather than borrowing the Rslib guard (see + * {@link virtualModulesPluginConstructor}). */ -const virtualModulesPluginConstructor = (): typeof rspack.experiments.VirtualModulesPlugin => { - const constructor = (rspack as { readonly experiments?: { readonly VirtualModulesPlugin?: unknown } }) - .experiments?.VirtualModulesPlugin; - if (typeof constructor !== 'function') { - throw new Error( - 'The Rspack engine nested in @rsbuild/core no longer exposes experiments.VirtualModulesPlugin, ' - + 'which agent-bundle uses to serve the generated agent-bundle/meta module to browser MCP App bundles. ' - + 'Pin @rsbuild/core to a version whose Rspack ships the plugin, or update agent-bundle.', - ); - } - return constructor as typeof rspack.experiments.VirtualModulesPlugin; -}; +const rsbuildVirtualModulesPlugin = (): typeof rspack.experiments.VirtualModulesPlugin => + virtualModulesPluginConstructor( + rspack, + '@rsbuild/core', + 'serve the generated agent-bundle/meta module to browser MCP App bundles', + ); export interface CompiledMcpApp { readonly _meta?: Readonly>; @@ -225,7 +219,7 @@ export const composeMcpAppsRsbuildConfig = ( }; // Added after the hatch mutator (this hook is merged last), so a consumer // cannot strip the generated identity module out of the compiler. - const VirtualModulesPlugin = virtualModulesPluginConstructor(); + const VirtualModulesPlugin = rsbuildVirtualModulesPlugin(); config.plugins = [ ...(config.plugins ?? []), new VirtualModulesPlugin({ [metaModulePath]: generatedMetaModuleSource(options.meta) }), diff --git a/packages/agent-bundle/src/build/meta.ts b/packages/agent-bundle/src/build/meta.ts index 0c7a49b0b..7187b23bb 100644 --- a/packages/agent-bundle/src/build/meta.ts +++ b/packages/agent-bundle/src/build/meta.ts @@ -109,3 +109,36 @@ export const generatedMetaModuleSource = (meta: AgentBundleMeta): string => [ 'export default meta;', '', ].join('\n'); + +/** The shape both Rspack engines expose the experimental virtual-module surface through. */ +interface VirtualModulesEngine { + readonly experiments?: { readonly VirtualModulesPlugin?: unknown }; +} + +/** + * Generated sources ride Rspack's `experiments.VirtualModulesPlugin` instead + * of throwaway files on disk — an accepted design decision: the experimental + * surface is the cost of never touching the artifact tree with build-time + * scratch files. This narrow feature check turns an upstream rename or + * removal into an actionable diagnostic instead of an opaque resolution + * failure deep inside a build. Rsbuild and Rslib carry independent Rspack + * copies (the dual-engine reality documented on `AgentBundleToolsConfig`), + * so each build path checks its own `rspack` and names its own package. + */ +export const virtualModulesPluginConstructor = ( + rspack: Engine, + /** The package whose nested Rspack is checked, as the consumer pins it. */ + packageName: string, + /** What agent-bundle serves through the plugin on this path, completing "uses to …". */ + purpose: string, +): NonNullable['VirtualModulesPlugin']> => { + const constructor = rspack.experiments?.VirtualModulesPlugin; + if (typeof constructor !== 'function') { + throw new Error( + `The Rspack engine nested in ${packageName} no longer exposes experiments.VirtualModulesPlugin, ` + + `which agent-bundle uses to ${purpose}. ` + + `Pin ${packageName} to a version whose Rspack ships the plugin, or update agent-bundle.`, + ); + } + return constructor as NonNullable['VirtualModulesPlugin']>; +}; diff --git a/packages/agent-bundle/src/build/module-imports.ts b/packages/agent-bundle/src/build/module-imports.ts new file mode 100644 index 000000000..691fd12d2 --- /dev/null +++ b/packages/agent-bundle/src/build/module-imports.ts @@ -0,0 +1,75 @@ +import { parse as parseJavaScript } from 'acorn'; +import { init, parse } from 'es-module-lexer'; + +/** + * One import of an ES module as the lexer reports it: `specifier` is the + * literal module specifier (absent for a non-literal dynamic import), and + * `kind` tells a static import from a dynamic `import()` and from the + * `import.meta` pseudo-import that carries no module at all. + */ +export interface ModuleImport { + readonly kind: 'dynamic' | 'meta' | 'static'; + readonly specifier: string | undefined; +} + +/** + * How thoroughly a module's syntax is checked before its imports are read. + * + * - `lexed`: the ESM lexer is the only pass. It rejects unterminated strings, + * templates, comments, and regexps and unbalanced braces — enough for a + * module the framework's own bundler emitted, whose syntax is the + * bundler's to guarantee. Re-parsing megabytes of bundler output to prove + * it is JavaScript was the dominant cost of every build. + * - `parsed`: a full `acorn` parse runs first, so a module the framework did + * not compile — a copied consumer script, a generated installer — keeps + * the complete syntax check. + */ +export type ModuleSyntaxCheck = 'lexed' | 'parsed'; + +const importKind = (dynamic: number): ModuleImport['kind'] => + dynamic === -2 ? 'meta' : dynamic === -1 ? 'static' : 'dynamic'; + +/** + * Imports already read from bytes with a known SHA-256, keyed by check level + * and digest. Within one process the same emitted bundle is scanned by the + * post-compile self-containment check and then by artifact validation, twice + * (before and after the manifest is written); the bytes never change between + * those passes, so the imports of a multi-megabyte bundle are lexed once. + * The records are a few dozen specifiers per module; the map stays bounded. + */ +const importsByDigest = new Map(); +const importsByDigestLimit = 512; + +const remember = (key: string, imports: readonly ModuleImport[]): void => { + if (importsByDigest.size >= importsByDigestLimit) { + const oldest = importsByDigest.keys().next(); + if (!oldest.done) importsByDigest.delete(oldest.value); + } + importsByDigest.set(key, imports); +}; + +/** Imports previously read (this process) from bytes with this digest at this check level. */ +export const rememberedModuleImports = ( + check: ModuleSyntaxCheck, + sha256: string, +): readonly ModuleImport[] | undefined => importsByDigest.get(`${check}:${sha256}`); + +/** + * Reads the imports of one ES module source, throwing on invalid syntax + * (the lexer's or, for `parsed`, acorn's). When the source's SHA-256 is + * known the result is remembered for the next pass over the same bytes. + */ +export const readModuleImports = async ( + source: string, + options: { readonly check: ModuleSyntaxCheck; readonly sha256?: string }, +): Promise => { + await init; + if (options.check === 'parsed') parseJavaScript(source, { ecmaVersion: 'latest', sourceType: 'module' }); + const [records] = parse(source); + const imports = Object.freeze(records.map((record) => Object.freeze({ + kind: importKind(record.d), + specifier: record.n, + }))); + if (options.sha256 !== undefined) remember(`${options.check}:${options.sha256}`, imports); + return imports; +}; diff --git a/packages/agent-bundle/src/build/rslib.ts b/packages/agent-bundle/src/build/rslib.ts index 8eb7285e0..ec8b6663a 100644 --- a/packages/agent-bundle/src/build/rslib.ts +++ b/packages/agent-bundle/src/build/rslib.ts @@ -3,10 +3,10 @@ // (https://rslib.rs/api/javascript-api/core). import { pluginReact } from '@rsbuild/plugin-react'; import { createRslib, mergeRslibConfig, rspack, type LibConfig, type Rspack } from '@rslib/core'; -import { init, parse } from 'es-module-lexer'; import { readFile, realpath } from 'node:fs/promises'; import { join, resolve } from 'node:path'; +import { sha256Hex } from '../core/digest.ts'; import { isErrno } from '../core/errors.ts'; import { isRecord } from '../core/strict-json.ts'; import type { AgentBundleToolsConfig } from '../core/types.ts'; @@ -19,7 +19,9 @@ import { generatedMetaModuleSource, generatedModulesRoot, metaModuleSpecifier, + virtualModulesPluginConstructor, } from './meta.ts'; +import { readModuleImports, type ModuleImport } from './module-imports.ts'; import { collectBundledOutputEvidence, type BundledOutputEvidence } from './provenance.ts'; export interface RslibVirtualModule { @@ -61,26 +63,9 @@ interface RslibDependencies { readonly createRslib?: (options: Parameters[0]) => Promise>; } -/** - * Generated sources ride Rspack's `experiments.VirtualModulesPlugin` instead - * of throwaway files on disk — an accepted design decision: the experimental - * surface is the cost of never touching the artifact tree with build-time - * scratch files. This narrow feature check turns an upstream rename or - * removal into an actionable diagnostic instead of an opaque resolution - * failure deep inside a build. - */ -const virtualModulesPluginConstructor = (): typeof rspack.experiments.VirtualModulesPlugin => { - const constructor = (rspack as { readonly experiments?: { readonly VirtualModulesPlugin?: unknown } }) - .experiments?.VirtualModulesPlugin; - if (typeof constructor !== 'function') { - throw new Error( - 'The Rspack engine nested in @rslib/core no longer exposes experiments.VirtualModulesPlugin, ' - + 'which agent-bundle uses to serve generated wrapper and registry modules. ' - + 'Pin @rslib/core to a version whose Rspack ships the plugin, or update agent-bundle.', - ); - } - return constructor as typeof rspack.experiments.VirtualModulesPlugin; -}; +/** This engine's virtual-module plugin (see {@link virtualModulesPluginConstructor}). */ +const rslibVirtualModulesPlugin = (): typeof rspack.experiments.VirtualModulesPlugin => + virtualModulesPluginConstructor(rspack, '@rslib/core', 'serve generated wrapper and registry modules'); /** * The tools escape hatch is typed against the workspace `@rsbuild/core` (the @@ -273,28 +258,26 @@ const reservedAliasViolation = ( * Fail-closed self-containment check on the emitted bundles themselves: * no reserved specifier may survive bundling as a live import. This is the * belt behind the static externals check and the function-external guard. - * The bundle is parsed as an ES module (the emitted format by contract), so + * The bundle is lexed as an ES module (the emitted format by contract), so * string literals or comments that merely mention a reserved specifier are - * not violations. + * not violations. The lex is keyed by the bundle's digest, so artifact + * validation, which scans these same bytes next, reads the imports once. */ const assertNoResidualReservedImports = async ( entries: readonly RslibEntry[], outputRoot: string, ): Promise => { - await init; await Promise.all(entries.map(async (entry) => { const reserved = reservedSpecifiers(entry); - const bundle = await readFile(resolve(outputRoot, entry.outputRelativePath), 'utf8'); - // A bin banner shebang is legal for Node but not for the ESM lexer. - const source = bundle.startsWith('#!') ? bundle.slice(bundle.indexOf('\n') + 1) : bundle; - let imports: ReturnType[0]; + const bytes = await readFile(resolve(outputRoot, entry.outputRelativePath)); + let imports: readonly ModuleImport[]; try { - [imports] = parse(source); + imports = await readModuleImports(bytes.toString('utf8'), { check: 'lexed', sha256: sha256Hex(bytes) }); } catch { throw new Error(`Generated executable ${JSON.stringify(entry.outputRelativePath)} did not parse as an ES module.`); } const residual = imports - .map((record) => record.n) + .map((record) => record.specifier) .find((specifier) => specifier !== undefined && reserved.includes(specifier)); if (residual !== undefined) { throw new Error( @@ -396,7 +379,7 @@ const assertExecutableConfig = ( // resolved config without the plugin instance would resolve the // reserved generated paths against the real filesystem. const registryModules = virtualRegistryModules(cwd, entry, meta); - const constructor = virtualModulesPluginConstructor(); + const constructor = rslibVirtualModulesPlugin(); if (config.plugins?.some((plugin) => plugin instanceof constructor) !== true) { throw new Error('Rslib resolved a generated executable environment without its virtual modules.'); } @@ -468,12 +451,6 @@ export const composeEntryLibConfig = ( ]); const enforceInvariants = (config: Rspack.Configuration): Rspack.Configuration => { config.output = { ...config.output, asyncChunks: false }; - if (entry.rscManifest === true) { - config.plugins = [ - ...(config.plugins ?? []), - new rspack.DefinePlugin({ __rspack_rsc_manifest__: JSON.stringify({ clientManifest: {}, cssLinkProps: {}, entryCssFiles: {}, entryJsFiles: [], moduleLoading: { prefix: '' }, serverConsumerModuleMap: {}, serverManifest: {} }) }), - ]; - } if (entry.reactServer === true) { config.resolve = { ...config.resolve, conditionNames: ['react-server', '...'] }; } @@ -500,15 +477,18 @@ export const composeEntryLibConfig = ( }, }; } - if (generatedModules.length > 0) { - // Added after the hatch mutator (this hook is merged last), so a - // consumer cannot strip the generated sources out of the compiler. - const VirtualModulesPlugin = virtualModulesPluginConstructor(); - config.plugins = [ - ...(config.plugins ?? []), - new VirtualModulesPlugin(Object.fromEntries(generatedModules.map((module) => [module.path, module.source]))), - ]; - } + // Framework plugins are added after the hatch mutator (this hook is + // merged last), so a consumer cannot strip the RSC manifest stub or the + // generated sources out of the compiler. + const frameworkPlugins = [ + ...(entry.rscManifest === true + ? [new rspack.DefinePlugin({ __rspack_rsc_manifest__: JSON.stringify({ clientManifest: {}, cssLinkProps: {}, entryCssFiles: {}, entryJsFiles: [], moduleLoading: { prefix: '' }, serverConsumerModuleMap: {}, serverManifest: {} }) })] + : []), + ...(generatedModules.length > 0 + ? [new (rslibVirtualModulesPlugin())(Object.fromEntries(generatedModules.map((module) => [module.path, module.source])))] + : []), + ]; + if (frameworkPlugins.length > 0) config.plugins = [...(config.plugins ?? []), ...frameworkPlugins]; if (virtualSource !== undefined) { // Rslib validates `source.entry` against the real filesystem before // Rspack exists, so the profile keys the entry on the authored program diff --git a/packages/agent-bundle/src/build/validate-artifact-modules.ts b/packages/agent-bundle/src/build/validate-artifact-modules.ts index f29ce57b0..d22e22807 100644 --- a/packages/agent-bundle/src/build/validate-artifact-modules.ts +++ b/packages/agent-bundle/src/build/validate-artifact-modules.ts @@ -3,12 +3,11 @@ import { isBuiltin } from 'node:module'; import { isAbsolute, relative, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; -import { parse as parseJavaScript } from 'acorn'; -import { init, parse } from 'es-module-lexer'; - +import { sha256Hex } from '../core/digest.ts'; import type { Diagnostic } from '../core/diagnostics.ts'; import { artifactDiagnostic as diagnostic, artifactDiagnosticRecoveries } from './artifact-diagnostics.ts'; import type { ArtifactFile } from './emit.ts'; +import { readModuleImports, rememberedModuleImports, type ModuleSyntaxCheck } from './module-imports.ts'; const javaScriptModuleSuffix = /\.(?:m?js)$/u; const generatedJavaScriptRecovery = artifactDiagnosticRecoveries.AB6005; @@ -98,13 +97,19 @@ const resolveJavaScriptImport = async (options: { export const validateJavaScriptModules = async (options: { readonly artifactRoot: string; + /** + * Modules the framework compiled (manifest kind `bundle`), checked at + * `bundleSyntaxCheck`; every other module is parsed in full (`parsed`). + */ + readonly bundledPaths?: ReadonlySet; + /** How a bundled module's syntax is checked; `lexed` unless a caller knows the bundler output may have been rewritten. */ + readonly bundleSyntaxCheck?: ModuleSyntaxCheck; readonly files: readonly ArtifactFile[]; readonly manifestFiles?: ReadonlySet; /** Prebuilt payload files: opaque consumer outputs excluded from graph validation. */ readonly prebuiltPaths?: ReadonlySet; readonly validJson: ReadonlySet; }): Promise => { - await init; const artifactRoot = await realpath(options.artifactRoot); const files = new Map(options.files .filter((file) => options.manifestFiles === undefined || options.manifestFiles.has(file.path)) @@ -120,29 +125,34 @@ export const validateJavaScriptModules = async (options: { return; } visiting.add(path); - let source: string; + const check = options.bundledPaths?.has(path) === true ? options.bundleSyntaxCheck ?? 'lexed' : 'parsed'; + let bytes: Buffer; try { - source = await readFile(resolve(artifactRoot, path), 'utf8'); + bytes = await readFile(resolve(artifactRoot, path)); } catch { diagnostics.push(graphDiagnostic(path, 'cannot be read.')); visiting.delete(path); visited.add(path); return; } - - let imports: ReturnType[0]; - try { - parseJavaScript(source, { ecmaVersion: 'latest', sourceType: 'module' }); - [imports] = parse(source); - } catch { - diagnostics.push(graphDiagnostic(path, 'has invalid syntax.')); - visiting.delete(path); - visited.add(path); - return; + // Keyed by the digest of the bytes just read — not the inspection's — so + // a module rewritten between the two is never answered from the cache, + // while the same bytes scanned earlier in this process are not lexed twice. + const sha256 = sha256Hex(bytes); + let imports = rememberedModuleImports(check, sha256); + if (imports === undefined) { + try { + imports = await readModuleImports(bytes.toString('utf8'), { check, sha256 }); + } catch { + diagnostics.push(graphDiagnostic(path, 'has invalid syntax.')); + visiting.delete(path); + visited.add(path); + return; + } } for (const imported of imports) { - if (imported.d === -2) continue; - if (imported.n === undefined) { + if (imported.kind === 'meta') continue; + if (imported.specifier === undefined) { diagnostics.push(graphDiagnostic(path, 'has a non-literal dynamic import.')); continue; } @@ -150,7 +160,7 @@ export const validateJavaScriptModules = async (options: { artifactRoot, files, importer: path, - specifier: imported.n, + specifier: imported.specifier, validJson: options.validJson, }); if (resolved.diagnostic !== undefined) diagnostics.push(resolved.diagnostic); diff --git a/packages/agent-bundle/src/build/validate-artifact.ts b/packages/agent-bundle/src/build/validate-artifact.ts index 8dead9ac4..354a4cb84 100644 --- a/packages/agent-bundle/src/build/validate-artifact.ts +++ b/packages/agent-bundle/src/build/validate-artifact.ts @@ -30,6 +30,7 @@ import { type ManifestFile, } from './emit.ts'; import { parseArtifactManifest, type ArtifactManifest } from './manifest.ts'; +import type { ModuleSyntaxCheck } from './module-imports.ts'; import type { ValidateArtifactOptions, ValidatedArtifactMcpServerEvidence, @@ -614,6 +615,7 @@ const validateArtifactStructure = (options: { const validateGeneratedFiles = async (options: { readonly artifactRoot: string; + readonly bundleSyntaxCheck?: ModuleSyntaxCheck; readonly files: readonly ArtifactFile[]; readonly manifestFiles?: readonly ManifestFile[]; readonly prebuiltPaths?: ReadonlySet; @@ -653,10 +655,14 @@ const validateGeneratedFiles = async (options: { diagnostics.push(...await validateJavaScriptModules({ artifactRoot: options.artifactRoot, + ...(options.bundleSyntaxCheck === undefined ? {} : { bundleSyntaxCheck: options.bundleSyntaxCheck }), files: options.files, ...(options.manifestFiles === undefined ? {} - : { manifestFiles: new Set(options.manifestFiles.map((file) => file.path)) }), + : { + bundledPaths: new Set(options.manifestFiles.filter((file) => file.kind === 'bundle').map((file) => file.path)), + manifestFiles: new Set(options.manifestFiles.map((file) => file.path)), + }), prebuiltPaths, validJson, })); @@ -664,15 +670,23 @@ const validateGeneratedFiles = async (options: { return Object.freeze(diagnostics); }; +/** + * The pre-manifest content pass `build` runs over a staged tree before it + * writes the manifest. The planned manifest file table, when given, tells + * the JavaScript validator which modules the compiler emitted; without it + * every module is parsed in full. + */ export const validateArtifactFiles = async ( - context: ValidateArtifactOptions, + context: ValidateArtifactOptions & { readonly manifestFiles?: readonly ManifestFile[] }, ): Promise => { const inspection = await inspectArtifact(context); return Object.freeze([ ...filesystemDiagnostics(inspection.filesystem), ...await validateGeneratedFiles({ artifactRoot: context.artifactRoot, + ...(context.bundleSyntaxCheck === undefined ? {} : { bundleSyntaxCheck: context.bundleSyntaxCheck }), files: inspection.files, + ...(context.manifestFiles === undefined ? {} : { manifestFiles: context.manifestFiles }), ...(context.prebuiltPaths === undefined ? {} : { prebuiltPaths: context.prebuiltPaths }), }), ]); @@ -810,6 +824,7 @@ export const validateArtifactWithSnapshot = async ( }), validateGeneratedFiles({ artifactRoot, + ...(context.bundleSyntaxCheck === undefined ? {} : { bundleSyntaxCheck: context.bundleSyntaxCheck }), files: inspection.files, manifestFiles: manifest.files, }), diff --git a/packages/agent-bundle/tests/artifact-validator.test.ts b/packages/agent-bundle/tests/artifact-validator.test.ts index 52eda8115..d643e27ff 100644 --- a/packages/agent-bundle/tests/artifact-validator.test.ts +++ b/packages/agent-bundle/tests/artifact-validator.test.ts @@ -1120,8 +1120,11 @@ it('rejects a canonical manifest that omits an executable file mode', async () = }); it('preserves structural artifact diagnostics after a strict manifest passes', async () => { + // A compiler bundle's syntax is the ESM lexer's to reject: an unterminated + // template is invalid to it, where a bare `export const broken = ;` is not. + const brokenBundle = 'export const broken = `;\n'; const files = [ - { contents: 'export const broken = ;\n', kind: 'bundle' as const, path: 'broken.mjs' }, + { contents: brokenBundle, kind: 'bundle' as const, path: 'broken.mjs' }, { contents: '{', kind: 'generated' as const, path: 'invalid.json' }, { contents: '{"mcpServers":{"local":{"args":["mcp/mcp-local-deadbeef.mjs"]}}}', kind: 'generated' as const, path: 'mcp.json' }, ]; @@ -1130,7 +1133,7 @@ it('preserves structural artifact diagnostics after a strict manifest passes', a try { await writeFile(join(root, 'mcp.json'), '{"mcpServers":{"local":{"args":["mcp/mcp-local-deadbeef.mjs"]}}}\n'); await writeFile(join(root, 'invalid.json'), '{'); - await writeFile(join(root, 'broken.mjs'), 'export const broken = ;\n'); + await writeFile(join(root, 'broken.mjs'), brokenBundle); const diagnostics = await validateArtifact({ artifactRoot: root }); expect(diagnostics).toEqual(expect.arrayContaining([ expect.objectContaining({ code: 'AB6005', generatedPath: 'broken.mjs' }), @@ -1795,7 +1798,7 @@ it('does not repeat JavaScript diagnostics after a validation-side mutation', as const root = await writeArtifact(files, true, [customManifestTarget]); const modulePath = join(root, 'custom', 'scripts', 'mutable.mjs'); const registry = customRegistry(() => { - writeFileSync(modulePath, 'export const broken = ;\n'); + writeFileSync(modulePath, 'export const broken = `;\n'); return []; }); @@ -1808,6 +1811,49 @@ it('does not repeat JavaScript diagnostics after a validation-side mutation', as } }); +/** + * The syntax check a module gets follows who produced it. A module the + * framework compiled (manifest kind `bundle`) is the bundler's own output: + * only the ESM lexer runs over it, so re-parsing megabytes of bundler output + * no longer dominates every build, and a bare `export const broken = ;` — + * which no bundler emits — passes while unterminated input still fails. A + * module the framework did not compile (a copied consumer script, a + * generated installer) is parsed in full and keeps the complete check. + */ +it('parses copied and generated modules in full and trusts compiler bundles to the ESM lexer', async () => { + const brokenStatement = 'export const broken = ;\n'; + const root = await writeArtifact([ + { contents: '{"kind":"custom"}\n', kind: 'generated', path: 'custom/document.json' }, + { contents: brokenStatement, kind: 'copy', path: 'custom/scripts/copied.mjs' }, + { contents: brokenStatement, kind: 'generated', path: 'custom/scripts/generated.mjs' }, + { contents: brokenStatement, kind: 'bundle', path: 'custom/scripts/bundled.mjs' }, + { contents: 'export const unterminated = `;\n', kind: 'bundle', path: 'custom/scripts/unterminated.mjs' }, + { contents: "export { missing } from './missing.mjs';\n", kind: 'bundle', path: 'custom/scripts/dangling.mjs' }, + ], true, [customManifestTarget]); + + try { + const diagnostics = await validateArtifact({ artifactRoot: root, registry: customRegistry() }); + expect(diagnostics.filter((entry) => entry.code === 'AB6005').map((entry) => [entry.generatedPath, entry.message])).toEqual([ + ['custom/scripts/copied.mjs', 'Generated JavaScript import from "custom/scripts/copied.mjs" has invalid syntax.'], + ['custom/scripts/dangling.mjs', 'Generated JavaScript import from "custom/scripts/dangling.mjs" is missing "./missing.mjs".'], + ['custom/scripts/generated.mjs', 'Generated JavaScript import from "custom/scripts/generated.mjs" has invalid syntax.'], + ['custom/scripts/unterminated.mjs', 'Generated JavaScript import from "custom/scripts/unterminated.mjs" has invalid syntax.'], + ]); + // A build whose consumer hatch may have rewritten the emitted assets asks + // for the full parse of bundles too; nothing else changes. + const parsed = await validateArtifact({ artifactRoot: root, bundleSyntaxCheck: 'parsed', registry: customRegistry() }); + expect(parsed.filter((entry) => entry.code === 'AB6005').map((entry) => entry.generatedPath)).toEqual([ + 'custom/scripts/bundled.mjs', + 'custom/scripts/copied.mjs', + 'custom/scripts/dangling.mjs', + 'custom/scripts/generated.mjs', + 'custom/scripts/unterminated.mjs', + ]); + } finally { + await rm(root, { force: true, recursive: true }); + } +}); + it('does not import copied non-JavaScript resources', async () => { const root = await writeArtifact([ { contents: '{"kind":"custom"}\n', kind: 'generated', path: 'custom/document.json' }, diff --git a/packages/agent-bundle/tests/build.test.ts b/packages/agent-bundle/tests/build.test.ts index 121d552f9..4f9401016 100644 --- a/packages/agent-bundle/tests/build.test.ts +++ b/packages/agent-bundle/tests/build.test.ts @@ -1162,6 +1162,34 @@ it('inlines reserved specifiers through exact-match aliases and virtual generate } }, 20_000); +it('parses emitted bundles in full when a tools hatch could have rewritten them', async () => { + // A compiler bundle is trusted to the ESM lexer only while its bytes are the + // bundler's own. A hatch runs after Rspack parsed the source and can rewrite + // the emitted asset — here a raw banner that leaves the lexer satisfied but + // Node unable to start the module — so a hatch build keeps the full parse. + const project = await createProject(); + try { + await expect(build({ + model: modelFor(project), + outputRoot: project.outputRoot, + projectRoot: project.root, + registry: new TargetRegistry().register((await import('../src/adapters/portable.ts')).portableAdapter, { default: true }), + tools: { + rspack: (config, { rspack }) => { + config.plugins = [...(config.plugins ?? []), new rspack.BannerPlugin({ banner: 'export const broken = ;', raw: true })]; + }, + }, + })).rejects.toThrow( + 'Agent Bundle compilation failed with 1 error:\n[AB6005] Generated JavaScript import from "portable/scripts/greeting.mjs" has invalid syntax.', + ); + await expect(readFile(join(project.outputRoot, 'agent-bundle.manifest.json'), 'utf8')).rejects.toMatchObject({ + code: 'ENOENT', + }); + } finally { + await cleanupProject(project); + } +}, 20_000); + it('keeps sibling staged outputs alive under a tools hatch that asks to clean the output root', async () => { const { entry, root } = await reservedSpecifierProject(); try { diff --git a/packages/agent-bundle/tests/module-imports.test.ts b/packages/agent-bundle/tests/module-imports.test.ts new file mode 100644 index 000000000..ebb864b62 --- /dev/null +++ b/packages/agent-bundle/tests/module-imports.test.ts @@ -0,0 +1,53 @@ +import { expect, it } from '@rstest/core'; + +import { sha256Hex } from '../src/core/digest.ts'; +import { readModuleImports, rememberedModuleImports } from '../src/build/module-imports.ts'; + +const source = [ + "import { a } from './a.mjs';", + "export * from './b.mjs';", + "const lazy = () => import('./c.mjs');", + 'const dynamic = (name) => import(name);', + 'const here = import.meta.url;', + 'export { lazy, dynamic, here };', + '', +].join('\n'); + +it('reports every import with its kind and literal specifier', async () => { + expect(await readModuleImports(source, { check: 'lexed' })).toEqual([ + { kind: 'static', specifier: './a.mjs' }, + { kind: 'static', specifier: './b.mjs' }, + { kind: 'dynamic', specifier: './c.mjs' }, + { kind: 'dynamic', specifier: undefined }, + { kind: 'meta', specifier: undefined }, + ]); +}); + +it('remembers imports by digest and check level so the same bytes are lexed once per process', async () => { + const bytes = `${source}// remembered\n`; + const sha256 = sha256Hex(bytes); + expect(rememberedModuleImports('lexed', sha256)).toBeUndefined(); + const imports = await readModuleImports(bytes, { check: 'lexed', sha256 }); + expect(rememberedModuleImports('lexed', sha256)).toBe(imports); + // A full parse is a stronger claim than a lex; each level is remembered on its own. + expect(rememberedModuleImports('parsed', sha256)).toBeUndefined(); + expect(Object.isFrozen(imports)).toBe(true); + // Without a digest nothing is remembered. + await readModuleImports(`${source}// unkeyed\n`, { check: 'lexed' }); + expect(rememberedModuleImports('lexed', sha256Hex(`${source}// unkeyed\n`))).toBeUndefined(); +}); + +it('rejects what each check level rejects and remembers nothing for invalid input', async () => { + const unterminated = 'export const broken = `;\n'; + const badStatement = 'export const broken = ;\n'; + await expect(readModuleImports(unterminated, { check: 'lexed', sha256: sha256Hex(unterminated) })).rejects.toThrow(); + expect(rememberedModuleImports('lexed', sha256Hex(unterminated))).toBeUndefined(); + // The lexer accepts a bare statement error a bundler never emits; the full parse does not. + expect(await readModuleImports(badStatement, { check: 'lexed' })).toEqual([]); + await expect(readModuleImports(badStatement, { check: 'parsed', sha256: sha256Hex(badStatement) })).rejects.toThrow(); + expect(rememberedModuleImports('parsed', sha256Hex(badStatement))).toBeUndefined(); + // A hashbang line is legal ESM input for both levels. + expect(await readModuleImports("#!/usr/bin/env node\nimport './cli.mjs';\n", { check: 'parsed' })).toEqual([ + { kind: 'static', specifier: './cli.mjs' }, + ]); +}); diff --git a/website/docs/en/guide/distribution/validation.mdx b/website/docs/en/guide/distribution/validation.mdx index 3307cdda8..1e4b5a88a 100644 --- a/website/docs/en/guide/distribution/validation.mdx +++ b/website/docs/en/guide/distribution/validation.mdx @@ -26,6 +26,17 @@ hand-edited generated file fails rather than passing because the path still exis files are checked too — a manifest-declared `logo` that is missing from the artifact or escapes the deploy tree reports `AB6025`. +Every emitted JavaScript module is walked as an ES module (`AB6005`): each import must be a +literal specifier that either names a Node built-in (`node:fs`, `fs`) or resolves — as a relative +or `file:` specifier — to a regular file listed in the manifest inside the artifact. No non-literal +dynamic imports, no bare package names, nothing outside the tree. How thoroughly a module's syntax +is checked follows who produced its bytes. A module the framework compiled (manifest kind `bundle`) +is the bundler's own output, so only the ESM lexer runs over it, which rejects unterminated +strings, templates, comments, and regexps and unbalanced braces. A module the framework did not +compile — a copied consumer script, a generated installer — is parsed in full, and so is every +bundle of a build whose [`tools` hatch](../../reference/configuration.mdx#tools) could have rewritten the +emitted assets. Prebuilt payloads (`kind: 'prebuilt'`) stay opaque and hash-locked only. + Every diagnostic is one structured record: a stable `AB` code, a severity, a message, and usually a `sourcePath` and a `recovery` hint. The diagnostic-gated commands — `build`, `prepack`, `validate`, `doctor`, `install`, and `dev` — exit nonzero **only** when an error diagnostic is diff --git a/website/docs/zh/guide/distribution/validation.mdx b/website/docs/zh/guide/distribution/validation.mdx index 95059e63c..a9605bc39 100644 --- a/website/docs/zh/guide/distribution/validation.mdx +++ b/website/docs/zh/guide/distribution/validation.mdx @@ -22,6 +22,14 @@ npx agent-bundle validate --artifact artifact --strict # 已构建字节, 把真实字节与这些摘要比对,因此被手工改过的生成文件会失败,而不会因为路径还在就通过。被引用的文件同样 会被检查——清单声明的 `logo` 若在产物中缺失或逃逸出部署树,会报告 `AB6025`。 +每个输出的 JavaScript 模块都会被当作 ES 模块遍历(`AB6005`):每个 import 必须是字面量说明符,要么指向 +Node 内建模块(`node:fs`、`fs`),要么以相对或 `file:` 说明符解析到产物内、清单中列出的常规文件。不允许 +非字面量的动态 import,不允许裸包名,不允许指向树外。模块语法检查的深度取决于它的字节由谁产出。框架编译的 +模块(清单 kind 为 `bundle`)是打包器自己的输出,因此只由 ESM 词法分析器扫描,它会拒绝未终止的字符串、模板、 +注释与正则以及不配对的花括号。框架没有编译的模块——被复制的消费者脚本、生成的安装器——则会被完整解析;若一次 +构建的 [`tools` 逃生口](../../reference/configuration.mdx#tools)有可能改写了输出资源,该构建的每个 bundle 也会被完整解析。 +预构建载荷(`kind: 'prebuilt'`)保持不透明,只做哈希锁定。 + 每条诊断都是一份结构化记录:稳定的 `AB` 代码、一个严重级别、一条消息,通常还有 `sourcePath` 与一条 `recovery` 提示。由诊断把关的命令——`build`、`prepack`、`validate`、`doctor`、`install` 与 `dev`——只有 存在 error 级诊断时才以非零退出;warning 与 info 绝不会为构建、校验或 dev 重建把关。`eval` 与 `inspect`