diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 474b0cafe..cba49f12c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,10 @@ on: pull_request: push: branches: [main] + # Nightly release-boundary matrix (packed-matrix job): the scaffolder + # template tests beyond the per-PR minimal-template smoke. + schedule: + - cron: '17 6 * * *' workflow_dispatch: permissions: @@ -16,6 +20,7 @@ concurrency: jobs: # Builds and checks every public example through its own toolchain. examples-check: + if: github.event_name != 'schedule' name: Examples check (Node 22.19) runs-on: ubuntu-latest timeout-minutes: 25 @@ -30,6 +35,7 @@ jobs: - run: pnpm examples:check verify: + if: github.event_name != 'schedule' name: Verify (Node ${{ matrix.node-version }}) runs-on: ubuntu-latest timeout-minutes: 45 @@ -56,9 +62,31 @@ jobs: - run: pnpm test release-gates: + if: github.event_name != 'schedule' name: Release gates (Node 22.19) runs-on: ubuntu-latest timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + - uses: pnpm/setup@v2 + with: + cache: true + install: false + runtime: node@22.19.0 + - run: pnpm install --frozen-lockfile + - run: pnpm exec playwright install --with-deps chrome + # Per-PR packed pool: single pack+install proofs plus the + # minimal-template scaffolder smoke. The full template matrix runs in + # the nightly packed-matrix job and in pre-publish `pnpm check:release`. + - run: pnpm check:release:ci + + # Release-boundary scaffolder template matrix (mcp-server, cli-tool) plus + # the full packed pool — the nightly form of pre-publish `check:release`. + packed-matrix: + if: github.event_name == 'schedule' + name: Packed release matrix (Node 22.19) + runs-on: ubuntu-latest + timeout-minutes: 40 steps: - uses: actions/checkout@v7 - uses: pnpm/setup@v2 @@ -75,6 +103,7 @@ jobs: # (hook -> RSC worker -> shared kernel state -> MCP tool lowering) without # any real Claude/Codex host. Real native-host smokes stay skip-gated in # the manually dispatched native-host-smoke workflow on purpose. + if: github.event_name != 'schedule' name: RSC runtime micro-eval (Node 22.19) runs-on: ubuntu-latest timeout-minutes: 15 diff --git a/docs/local-ci.md b/docs/local-ci.md index 0177c4761..7bf1fea6b 100644 --- a/docs/local-ci.md +++ b/docs/local-ci.md @@ -35,6 +35,12 @@ The three hosted Node-22.19 jobs fold into one `gates-node22` worktree because each of their entry scripts starts from `pnpm build` in a fresh install, which one worktree provides just as well as three. +`gates-node22` runs the full `check:release`, a strict superset of the hosted +per-PR release-gates job (`check:release:ci`): it additionally runs the +scaffolder template matrix that the hosted side defers to the nightly +`packed-matrix` job, so local green covers both the per-PR and nightly packed +pools. + All four legs run concurrently. The summary table (leg × step × status × duration × test census) is printed and written to `.worktrees/local-ci/summary.md` (plus `summary.json`); per-step logs land in diff --git a/package.json b/package.json index e6ee0ab5b..df1cf5878 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,8 @@ "eval:spot": "pnpm build && pnpm --filter @agent-bundle/rsc-agent-runtime-demo build && pnpm --filter @agent-bundle/rsc-agent-runtime-demo exec rstest run tests/micro-eval.spot.test.ts --config rstest.config.ts", "check:runtime-topology": "node scripts/rsc-runtime-topology.mjs --root . --output docs/architecture/rsc-runtime-workbench.md --check", "test:examples:browser": "rstest --config rstest.config.ts packages/workbench/tests/examples-real.e2e.test.ts", - "test:packed": "rstest --config rstest.config.ts packages/agent-bundle/tests/release-audit.test.ts packages/agent-bundle/tests/packed-consumer.test.ts packages/agent-bundle/tests/dev-workbench-packaging.test.ts packages/agent-bundle/tests/public-api-packed.test.ts packages/agent-bundle/tests/rsc-runtime-optional-packaging.test.ts packages/agent-bundle/tests/packed-native-smoke.test.ts packages/create-agent-bundle/tests/scaffold-packed.e2e.test.ts packages/workbench/tests/packed-release.e2e.test.ts", + "test:packed": "node scripts/run-packed-tests.mjs", + "test:packed:release": "node scripts/run-packed-tests.mjs --release", "test:packed:native": "rstest --config rstest.config.ts packages/agent-bundle/tests/packed-native-smoke.test.ts", "test:packed:native:claude": "pnpm build && AGENT_BUNDLE_PACKED_NATIVE_CLAUDE_SMOKE=1 pnpm test:packed:native", "test:packed:native:codex": "pnpm build && AGENT_BUNDLE_PACKED_NATIVE_CODEX_SMOKE=1 pnpm test:packed:native", @@ -33,7 +34,8 @@ "preview:publish": "pkg-pr-new publish --previewVersion --peerDeps --no-compact --no-template './packages/agent-bundle' './packages/rsc-runtime' './packages/create-agent-bundle'", "pack:dry-run": "pnpm build && npm pack ./packages/agent-bundle --dry-run --json", "audit:release": "pnpm lint:package && attw --pack --profile esm-only packages/agent-bundle && node scripts/audit-packed-release.mjs", - "check:release": "pnpm pack:dry-run && pnpm audit:release && pnpm test:packed", + "check:release": "pnpm pack:dry-run && pnpm audit:release && pnpm test:packed:release", + "check:release:ci": "pnpm pack:dry-run && pnpm audit:release && pnpm test:packed", "example:hooks": "pnpm build && pnpm --filter @agent-bundle-example/hooks-and-scripts dev", "example:audiobook": "pnpm build && pnpm --filter @agent-bundle-example/audiobook-curator dev", "example:mcp-app": "pnpm build && pnpm --filter @agent-bundle-example/mcp-app dev", diff --git a/packages/agent-bundle/tests/dev-services.test.ts b/packages/agent-bundle/tests/dev-services.test.ts index e16bf7769..5b0073f5d 100644 --- a/packages/agent-bundle/tests/dev-services.test.ts +++ b/packages/agent-bundle/tests/dev-services.test.ts @@ -731,9 +731,9 @@ it('reports snapshot failures as frozen preparation diagnostics', async () => { '', ].join('\n')); const output = join(root, 'snapshot-output'); - const externalOutput = join(root, '..', 'snapshot-output-external'); + // Unique path: a fixed name under tmpdir collides across concurrent runs. + const externalOutput = await mkdtemp(join(tmpdir(), 'agent-bundle-snapshot-output-external-')); try { - await mkdir(externalOutput); await writeFile(join(root, 'agent-bundle.config.ts'), [ "import { symlinkSync } from 'node:fs';", 'export default ({ projectRoot }) => {', diff --git a/packages/agent-bundle/tests/dev-workbench-packaging.test.ts b/packages/agent-bundle/tests/dev-workbench-packaging.test.ts index 52ae47538..326ffde08 100644 --- a/packages/agent-bundle/tests/dev-workbench-packaging.test.ts +++ b/packages/agent-bundle/tests/dev-workbench-packaging.test.ts @@ -7,6 +7,8 @@ import { promisify } from 'node:util'; import { describe, expect, it } from '@rstest/core'; +import { installedEnvironment, npmInstallArguments, sharedPackedTarball } from './support/shared-pack.ts'; + const execFile = promisify(executeFile); const workspaceRoot = process.cwd(); const packageRoot = join(workspaceRoot, 'packages', 'agent-bundle'); @@ -14,16 +16,14 @@ const workbenchRoot = join(workspaceRoot, 'packages', 'workbench'); const appRendererLicense = join('src', 'mcp', 'APP-RENDERER-LICENSE'); let built: Promise | undefined; -const packedEnvironment = (): NodeJS.ProcessEnv => { - const { NODE_PATH: _nodePath, ...environment } = process.env; - return environment; -}; - const buildPackage = async (force = false): Promise => { if (force) { + // The stale-asset pruning test rebuilds on purpose; the prebuilt seam + // never skips it because the rebuild itself is the behavior under test. await execFile('pnpm', ['build'], { cwd: workspaceRoot }); return; } + if (process.env['AGENT_BUNDLE_PACKAGE_PREBUILT'] === '1') return; built ??= execFile('pnpm', ['build'], { cwd: workspaceRoot }).then(() => undefined); await built; }; @@ -71,13 +71,10 @@ it('prunes stale copied workbench assets without removing the package library ou }, 60_000); it('serves prebuilt workbench assets from an installed tarball without the repository source tree', async () => { - await buildPackage(); + const { tarball } = await sharedPackedTarball('agent-bundle'); const consumer = await mkdtemp(join(tmpdir(), 'agent-bundle-workbench-consumer-')); const project = join(consumer, 'project'); try { - const { stdout } = await execFile('npm', ['pack', '--json', '--pack-destination', consumer], { cwd: packageRoot }); - const [packed] = JSON.parse(stdout) as Array<{ readonly filename: string }>; - const tarball = join(consumer, packed.filename); const listing = await execFile('tar', ['-tf', tarball]); expect(listing.stdout).toContain('package/dist/workbench/index.html'); expect(listing.stdout).toContain('package/dist/workbench/THIRD_PARTY_NOTICES'); @@ -86,7 +83,7 @@ it('serves prebuilt workbench assets from an installed tarball without the repos expect(listing.stdout).not.toMatch(/package\/dist\/workbench\/.*-[a-f0-9]{8,}/iu); await writeFile(join(consumer, 'package.json'), '{"type":"module"}\n'); - await execFile('npm', ['install', '--ignore-scripts', '--no-audit', '--no-fund', tarball], { cwd: consumer }); + await execFile('npm', ['install', ...npmInstallArguments, tarball], { cwd: consumer }); await mkdir(join(project, 'skills', 'review'), { recursive: true }); await Promise.all([ writeFile(join(project, 'package.json'), '{"type":"module"}\n'), @@ -113,17 +110,12 @@ it('serves prebuilt workbench assets from an installed tarball without the repos }, 60_000); it('runs the Agent API from an omit-dev installed tarball with its runtime MCP dependencies', async () => { - await buildPackage(); + const { tarball } = await sharedPackedTarball('agent-bundle'); const consumer = await mkdtemp(join(tmpdir(), 'agent-bundle-agent-api-consumer-')); const project = join(consumer, 'project'); try { - const { stdout } = await execFile('npm', ['pack', '--json', '--pack-destination', consumer], { cwd: packageRoot }); - const [packed] = JSON.parse(stdout) as Array<{ readonly filename: string }>; - const tarball = join(consumer, packed.filename); await writeFile(join(consumer, 'package.json'), '{"type":"module"}\n'); - await execFile('npm', [ - 'install', '--omit=dev', '--ignore-scripts', '--no-audit', '--no-fund', tarball, - ], { cwd: consumer }); + await execFile('npm', ['install', '--omit=dev', ...npmInstallArguments, tarball], { cwd: consumer }); await mkdir(join(project, 'skills', 'review'), { recursive: true }); await Promise.all([ writeFile(join(project, 'package.json'), '{"type":"module"}\n'), @@ -153,7 +145,7 @@ it('runs the Agent API from an omit-dev installed tarball with its runtime MCP d ].join('\n'); const result = await execFile(process.execPath, ['--input-type=module', '--eval', script], { cwd: consumer, - env: { ...packedEnvironment(), AGENT_BUNDLE_AGENT_API_TOKEN: 'packed-agent-api-token' }, + env: { ...installedEnvironment(), AGENT_BUNDLE_AGENT_API_TOKEN: 'packed-agent-api-token' }, }); expect(JSON.parse(result.stdout)).toEqual({ runtime: ['function', 'function', 'function'], diff --git a/packages/agent-bundle/tests/packed-consumer.test.ts b/packages/agent-bundle/tests/packed-consumer.test.ts index 476c2798a..10b639724 100644 --- a/packages/agent-bundle/tests/packed-consumer.test.ts +++ b/packages/agent-bundle/tests/packed-consumer.test.ts @@ -20,6 +20,7 @@ import { promisify } from 'node:util'; import { expect, it } from '@rstest/core'; import { sha256Hex } from '../src/core/digest.ts'; +import { installedEnvironment, npmInstallArguments } from './support/shared-pack.ts'; const execFile = promisify(executeFile); const workspaceRoot = process.cwd(); @@ -37,11 +38,6 @@ interface ManifestDigest extends FileDigest { readonly mode?: number; } -const installedEnvironment = (): NodeJS.ProcessEnv => { - const { NODE_PATH: _nodePath, ...environment } = process.env; - return environment; -}; - const artifactDigest = async (root: string): Promise => { const collect = async (directory: string): Promise => { const entries = await readdir(directory, { withFileTypes: true }); @@ -78,6 +74,11 @@ it('uses only an installed tarball after source deletion', async () => { const artifact = join(projectRoot, 'artifact with spaces'); try { + // Deliberately not sharedPackedTarball: this test packs from a deletable + // copy of the package and removes that copy after install, proving the + // tarball's contents hold no path references back to the pack source. The + // shared tarball packs from the live workspace, which survives the run, + // so a leaked path would resolve and pass silently. await cp(packageRoot, packedPackageRoot, { recursive: true }); await execFile(join(workspaceRoot, 'node_modules', '.bin', 'rslib'), [ 'build', '--config', join(packageRoot, 'rslib.config.ts'), '--dist-path', join(packedPackageRoot, 'dist'), @@ -97,10 +98,6 @@ it('uses only an installed tarball after source deletion', async () => { stat(join(projectRoot, 'src', 'shell.sh')).then((metadata) => metadata.mode & 0o777), stat(join(projectRoot, 'src', 'python.py')).then((metadata) => metadata.mode & 0o777), ]); - await execFile('npm', ['install', '--ignore-scripts', '--no-audit', '--no-fund', tarball], { - cwd: projectRoot, - env: installedEnvironment(), - }); await mkdir(scriptProjectRoot, { recursive: true }); await Promise.all([ writeFile(join(scriptProjectRoot, 'package.json'), '{"type":"module"}\n'), @@ -110,10 +107,13 @@ it('uses only an installed tarball after source deletion', async () => { ), writeFile(join(scriptProjectRoot, 'shell.sh'), "printf 'packed script stdout\\n'\nprintf 'packed script stderr\\n' >&2\n"), ]); - await execFile('npm', ['install', '--ignore-scripts', '--no-audit', '--no-fund', tarball], { - cwd: scriptProjectRoot, - env: installedEnvironment(), - }); + // The two consumers are disjoint directories; npm's cache handles the + // concurrent installs (the scaffolder e2e relies on the same property). + await Promise.all([projectRoot, scriptProjectRoot].map(async (consumer) => + execFile('npm', ['install', ...npmInstallArguments, tarball], { + cwd: consumer, + env: installedEnvironment(), + }))); const cli = join(projectRoot, 'node_modules', '.bin', 'agent-bundle'); const installedPackage = await realpath(join(projectRoot, 'node_modules', 'agent-bundle')); @@ -331,7 +331,7 @@ it('uses only an installed tarball after source deletion', async () => { '', ].join('\n')), ]); - await execFile('npm', ['install', '--ignore-scripts', '--no-audit', '--no-fund', tarball], { + await execFile('npm', ['install', ...npmInstallArguments, tarball], { cwd: frameworkRoot, env: installedEnvironment(), }); diff --git a/packages/agent-bundle/tests/public-api-packed.test.ts b/packages/agent-bundle/tests/public-api-packed.test.ts index ed46116b8..5db4ae6bf 100644 --- a/packages/agent-bundle/tests/public-api-packed.test.ts +++ b/packages/agent-bundle/tests/public-api-packed.test.ts @@ -7,6 +7,7 @@ import { promisify } from 'node:util'; import { expect, it } from '@rstest/core'; import { writeFixtureManifest } from './support/manifest.ts'; +import { npmInstallArguments, sharedPackedTarball } from './support/shared-pack.ts'; interface PackageManifest { bin: { @@ -18,14 +19,6 @@ interface PackageManifest { const execFile = promisify(executeFile); const workspaceRoot = process.cwd(); const packageRoot = join(workspaceRoot, 'packages/agent-bundle'); -let buildPromise: Promise | undefined; - -const buildPackage = async (): Promise => { - buildPromise ??= execFile('pnpm', ['build'], { - cwd: workspaceRoot, - }).then(() => undefined); - await buildPromise; -}; const readPackageManifest = async (): Promise => JSON.parse( @@ -58,18 +51,14 @@ const producerFrom = async (output: string): Promise<{ readonly name: string; re }; it('writes the package version as the producer of a packed CLI manifest', async () => { - await buildPackage(); + const { tarball } = await sharedPackedTarball('agent-bundle'); const consumerRoot = await mkdtemp(join(tmpdir(), 'agent-bundle-packed-manifest-')); const manifest = await readPackageManifest(); try { - const { stdout: packedOutput } = await execFile( - 'npm', ['pack', '--json', '--pack-destination', consumerRoot], { cwd: packageRoot }, - ); - const [packed] = JSON.parse(packedOutput) as Array<{ filename: string }>; await writeFile(join(consumerRoot, 'package.json'), '{"type":"module"}\n'); await execFile( - 'npm', ['install', '--ignore-scripts', '--no-audit', '--no-fund', join(consumerRoot, packed.filename)], + 'npm', ['install', ...npmInstallArguments, tarball], { cwd: consumerRoot }, ); @@ -94,22 +83,14 @@ it('writes the package version as the producer of a packed CLI manifest', async }, 30_000); it('imports the externalized config entry from a packed npm consumer', async () => { - await buildPackage(); + const { tarball } = await sharedPackedTarball('agent-bundle'); const consumerRoot = await mkdtemp(join(tmpdir(), 'agent-bundle-consumer-')); try { - const { stdout: packedOutput } = await execFile( - 'npm', - ['pack', '--json', '--pack-destination', consumerRoot], - { cwd: packageRoot }, - ); - const [packed] = JSON.parse(packedOutput) as Array<{ filename: string }>; - const tarball = join(consumerRoot, packed.filename); - await writeFile(join(consumerRoot, 'package.json'), '{"type":"module"}\n'); await execFile( 'npm', - ['install', '--ignore-scripts', '--no-audit', '--no-fund', tarball], + ['install', ...npmInstallArguments, tarball], { cwd: consumerRoot }, ); @@ -161,10 +142,10 @@ it('imports the externalized config entry from a packed npm consumer', async () } finally { await rm(consumerRoot, { force: true, recursive: true }); } -}, 15_000); +}, 30_000); it('invokes a prebuilt MCP server from a clean packed consumer', async () => { - await buildPackage(); + const { tarball } = await sharedPackedTarball('agent-bundle'); const consumerRoot = await mkdtemp(join(tmpdir(), 'agent-bundle-mcp-consumer-')); try { @@ -214,16 +195,10 @@ it('invokes a prebuilt MCP server from a clean packed consumer', async () => { '{"hooks":[]}\n', ); - const { stdout: packedOutput } = await execFile( - 'npm', - ['pack', '--json', '--pack-destination', consumerRoot], - { cwd: packageRoot }, - ); - const [packed] = JSON.parse(packedOutput) as Array<{ filename: string }>; await writeFile(join(consumerRoot, 'package.json'), '{"type":"module"}\n'); await execFile( 'npm', - ['install', '--ignore-scripts', '--no-audit', '--no-fund', join(consumerRoot, packed.filename)], + ['install', ...npmInstallArguments, tarball], { cwd: consumerRoot }, ); const { stdout } = await execFile(process.execPath, [ diff --git a/packages/agent-bundle/tests/release-audit.test.ts b/packages/agent-bundle/tests/release-audit.test.ts index 178505882..d8d8a70ba 100644 --- a/packages/agent-bundle/tests/release-audit.test.ts +++ b/packages/agent-bundle/tests/release-audit.test.ts @@ -6,6 +6,8 @@ import { promisify } from 'node:util'; import { expect, it } from '@rstest/core'; +import { npmInstallArguments, sharedPackedTarball } from './support/shared-pack.ts'; + const execFile = promisify(executeFile); const workspaceRoot = process.cwd(); const packageRoot = join(workspaceRoot, 'packages', 'agent-bundle'); @@ -44,7 +46,12 @@ it('audits an externally installed production tarball and generates its CycloneD const componentReferences = new Set(components.flatMap((component) => ( component['bom-ref'] === undefined ? [] : [component['bom-ref']] ))); - const dependencyReferences = new Set([root?.['bom-ref'], ...componentReferences]); + const rootReference = root?.['bom-ref']; + expect(rootReference).toBeDefined(); + const dependencyReferences = new Set([ + ...(rootReference === undefined ? [] : [rootReference]), + ...componentReferences, + ]); const declaredDependencyReferences = new Set((sbom.dependencies ?? []).flatMap((dependency) => ( dependency.ref === undefined ? [] : [dependency.ref] ))); @@ -58,9 +65,7 @@ it('audits an externally installed production tarball and generates its CycloneD expect(product).toBeDefined(); expect(rootDependencies?.dependsOn).toContain(product?.['bom-ref']); expect(productDependencies?.dependsOn?.length).toBeGreaterThan(0); - expect([...dependencyReferences]).not.toContain(undefined); - expect([...dependencyReferences].filter((reference): reference is string => reference !== undefined) - .every((reference) => declaredDependencyReferences.has(reference))).toBe(true); + expect([...dependencyReferences].every((reference) => declaredDependencyReferences.has(reference))).toBe(true); expect((sbom.dependencies ?? []).every((dependency) => ( dependency.ref !== undefined && dependencyReferences.has(dependency.ref) @@ -81,14 +86,8 @@ it('ships repository and support metadata that matches the verified origin', asy const tarballRoot = await mkdtemp(join(tmpdir(), 'agent-bundle-package-metadata-')); try { - const { stdout } = await execFile('npm', [ - 'pack', - '--json', - '--pack-destination', - tarballRoot, - ], { cwd: packageRoot, env: releaseEnvironment() }); - const [{ filename }] = JSON.parse(stdout) as Array<{ readonly filename: string }>; - await execFile('tar', ['--extract', '--file', join(tarballRoot, filename), '--directory', tarballRoot]); + const { tarball } = await sharedPackedTarball('agent-bundle'); + await execFile('tar', ['--extract', '--file', tarball, '--directory', tarballRoot]); const manifest = JSON.parse(await readFile(join(tarballRoot, 'package', 'package.json'), 'utf8')) as { readonly bugs?: { readonly url?: string }; readonly description?: string; @@ -109,51 +108,24 @@ it('ships repository and support metadata that matches the verified origin', asy }); it('packs generated Workbench legal companion files', async () => { - const tarballRoot = await mkdtemp(join(tmpdir(), 'agent-bundle-release-audit-')); - - try { - await execFile('pnpm', ['build'], { cwd: workspaceRoot, env: releaseEnvironment() }); - const { stdout } = await execFile('npm', [ - 'pack', - '--json', - '--pack-destination', - tarballRoot, - ], { cwd: packageRoot, env: releaseEnvironment() }); - const [{ files }] = JSON.parse(stdout) as Array<{ readonly files: readonly { readonly path: string }[] }>; - const productManifest = await readFile(join(packageRoot, 'package.json'), 'utf8'); + const { packOutput } = await sharedPackedTarball('agent-bundle'); + const productManifest = await readFile(join(packageRoot, 'package.json'), 'utf8'); - expect(files.map((file) => file.path)).toContainEqual( - expect.stringMatching(/^dist\/workbench\/.*\.LICENSE\.txt$/u), - ); - expect(files.some(({ path }) => path.startsWith('examples/'))).toBe(false); - expect(productManifest).not.toContain('workspace:'); - } finally { - await rm(tarballRoot, { force: true, recursive: true }); - } + expect(packOutput.files.map((file) => file.path)).toContainEqual( + expect.stringMatching(/^dist\/workbench\/.*\.LICENSE\.txt$/u), + ); + expect(packOutput.files.some(({ path }) => path.startsWith('examples/'))).toBe(false); + expect(productManifest).not.toContain('workspace:'); }, 120_000); it('installs public entrypoints and an externally resolved CLI for production consumers', async () => { const consumerRoot = await mkdtemp(join(tmpdir(), 'agent-bundle-release-consumer-')); - const tarballRoot = await mkdtemp(join(tmpdir(), 'agent-bundle-release-tarball-')); try { - await execFile('pnpm', ['build'], { cwd: workspaceRoot, env: releaseEnvironment() }); - const { stdout: packed } = await execFile('npm', [ - 'pack', - '--json', - '--pack-destination', - tarballRoot, - ], { cwd: packageRoot, env: releaseEnvironment() }); - const [{ filename }] = JSON.parse(packed) as Array<{ readonly filename: string }>; - const tarball = join(tarballRoot, filename); + const { tarball } = await sharedPackedTarball('agent-bundle'); await writeFile(join(consumerRoot, 'package.json'), '{"private":true,"type":"module"}\n'); await execFile('npm', [ - 'install', - '--omit=dev', - '--ignore-scripts', - '--no-audit', - '--no-fund', - tarball, + 'install', '--omit=dev', ...npmInstallArguments, tarball, ], { cwd: consumerRoot, env: releaseEnvironment() }); const installedPackageRoot = await realpath(join(consumerRoot, 'node_modules', 'agent-bundle')); @@ -221,9 +193,6 @@ it('installs public entrypoints and an externally resolved CLI for production co await rm(join(consumerRoot, 'node_modules', 'commander'), { force: true, recursive: true }); await expect(execFile(cli, ['--help'], { cwd: consumerRoot, env: releaseEnvironment() })).rejects.toThrow(); } finally { - await Promise.all([ - rm(consumerRoot, { force: true, recursive: true }), - rm(tarballRoot, { force: true, recursive: true }), - ]); + await rm(consumerRoot, { force: true, recursive: true }); } }, 120_000); diff --git a/packages/agent-bundle/tests/rsc-runtime-optional-packaging.test.ts b/packages/agent-bundle/tests/rsc-runtime-optional-packaging.test.ts index 9c2a38aa9..ee8477592 100644 --- a/packages/agent-bundle/tests/rsc-runtime-optional-packaging.test.ts +++ b/packages/agent-bundle/tests/rsc-runtime-optional-packaging.test.ts @@ -6,16 +6,11 @@ import { promisify } from 'node:util'; import { describe, expect, it } from '@rstest/core'; +import { npmInstallArguments, sharedPackedTarball } from './support/shared-pack.ts'; + const execFile = promisify(executeFile); const workspaceRoot = process.cwd(); -const packageRoot = join(workspaceRoot, 'packages', 'agent-bundle'); const skillsOnlyFixture = join(workspaceRoot, 'fixtures', 'integration', 'skills-only'); -let built: Promise | undefined; - -const buildPackage = async (): Promise => { - built ??= execFile('npm', ['run', 'build'], { cwd: workspaceRoot }).then(() => undefined); - await built; -}; type InstalledDependencyTree = Readonly<{ readonly dependencies?: Readonly>; @@ -42,19 +37,15 @@ const namedFiles = async (root: string, name: string): Promise { it('runs an ordinary skills-only project from a fresh installed tarball without the RSC runtime', async () => { - await buildPackage(); + const { tarball } = await sharedPackedTarball('agent-bundle'); const consumer = await mkdtemp(join(tmpdir(), 'agent-bundle-rsc-optional-consumer-')); const project = join(consumer, 'project'); const artifact = join(project, '.agent-bundle', 'artifact'); try { - const { stdout } = await execFile('npm', ['pack', '--json', '--pack-destination', consumer], { cwd: packageRoot }); - const [packed] = JSON.parse(stdout) as readonly Readonly<{ readonly filename: string }>[]; - if (packed === undefined) throw new Error('npm pack did not produce an Agent Bundle tarball.'); - const tarball = join(consumer, packed.filename); const tarListing = (await execFile('tar', ['-tf', tarball])).stdout; expect(tarListing).not.toMatch(/examples\/rsc-agent-runtime|react-server-dom-rspack|rsbuild-plugin-rsc/u); - await execFile('npm', ['install', '--ignore-scripts', '--no-audit', '--no-fund', tarball], { cwd: consumer }); + await execFile('npm', ['install', ...npmInstallArguments, tarball], { cwd: consumer }); const dependencyTree = JSON.parse((await execFile('npm', ['ls', '--all', '--json'], { cwd: consumer })).stdout) as InstalledDependencyTree; const installedNames = installedDependencyNames(dependencyTree); for (const name of ['react', 'react-dom', 'react-server-dom-rspack', 'rsbuild-plugin-rsc']) { diff --git a/packages/agent-bundle/tests/support/eval-project.ts b/packages/agent-bundle/tests/support/eval-project.ts index 09e5ba264..fc5c5e873 100644 --- a/packages/agent-bundle/tests/support/eval-project.ts +++ b/packages/agent-bundle/tests/support/eval-project.ts @@ -21,7 +21,11 @@ export interface SeedEvalProjectOptions { } const evalEntryPoint = resolve(process.cwd(), 'packages/agent-bundle/src/eval/index.ts'); -const sourceEntryPoint = resolve(process.cwd(), 'packages/agent-bundle/src/index.ts'); +// defineConfig's defining module, not the package entry: seeded configs load +// through Jiti with the module cache off, so re-exporting src/index.ts made +// every config load re-transpile the whole package graph (~5s per project). +// project-fixture.ts stubs the same way. +const sourceEntryPoint = resolve(process.cwd(), 'packages/agent-bundle/src/core/types.ts'); const graderModule = (expected: string): string => [ "import { readFile } from 'node:fs/promises';", diff --git a/packages/agent-bundle/tests/support/shared-pack.ts b/packages/agent-bundle/tests/support/shared-pack.ts new file mode 100644 index 000000000..124584daf --- /dev/null +++ b/packages/agent-bundle/tests/support/shared-pack.ts @@ -0,0 +1,78 @@ +import { execFile as executeFile } from 'node:child_process'; +import { rmSync } from 'node:fs'; +import { mkdtemp, readFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; + +const execFile = promisify(executeFile); +const workspaceRoot = process.cwd(); + +export interface SharedPackOutput { + readonly filename: string; + readonly files: readonly { readonly path: string }[]; +} + +export interface SharedPack { + /** First `npm pack --json` entry recorded when the tarball was produced. */ + readonly packOutput: SharedPackOutput; + readonly tarball: string; +} + +export type SharedPackPackage = 'agent-bundle' | 'create-agent-bundle'; + +export const installedEnvironment = (): NodeJS.ProcessEnv => { + const { NODE_PATH: _nodePath, ...environment } = process.env; + return environment; +}; + +/** Canonical flags for installing a packed tarball into a consumer fixture. */ +export const npmInstallArguments = ['--ignore-scripts', '--no-audit', '--no-fund'] as const; + +const packs = new Map>(); +let fallbackBuild: Promise | undefined; + +const packOnce = async (packageName: SharedPackPackage): Promise => { + const sharedDirectory = process.env['AGENT_BUNDLE_SHARED_PACK_DIR']; + if (sharedDirectory !== undefined && sharedDirectory.length > 0) { + return JSON.parse(await readFile(join(sharedDirectory, `${packageName}.json`), 'utf8')) as SharedPack; + } + // Ad-hoc single-file runs have no run-level tarball, so build once (unless + // the caller marked the workspace dist prebuilt) and pack into a + // per-process temporary directory that is dropped on exit. The build + // promise is process-wide so concurrent callers share one build, and it + // runs with NODE_ENV=production like the release pipeline the tarball + // stands in for. + if (process.env['AGENT_BUNDLE_PACKAGE_PREBUILT'] !== '1') { + fallbackBuild ??= execFile('pnpm', ['build'], { + cwd: workspaceRoot, + env: { ...installedEnvironment(), NODE_ENV: 'production' }, + }).then(() => undefined); + await fallbackBuild; + } + const destination = await mkdtemp(join(tmpdir(), 'agent-bundle-shared-pack-')); + process.once('exit', () => { + rmSync(destination, { force: true, recursive: true }); + }); + const { stdout } = await execFile('npm', ['pack', '--json', '--pack-destination', destination], { + cwd: join(workspaceRoot, 'packages', packageName), + env: { ...installedEnvironment(), NODE_ENV: 'production' }, + }); + const [packOutput] = JSON.parse(stdout) as [SharedPackOutput]; + return { packOutput, tarball: join(destination, packOutput.filename) }; +}; + +/** + * Run-level release tarball for a public package. `test:packed` builds and + * `npm pack`s each package exactly once per run (scripts/run-packed-tests.mjs) + * and shares the result through AGENT_BUNDLE_SHARED_PACK_DIR, so every + * pack-and-install suite consumes the same tarball a release would publish + * instead of re-packing (and previously rebuilding) per test file. + */ +export const sharedPackedTarball = (packageName: SharedPackPackage): Promise => { + const existing = packs.get(packageName); + if (existing !== undefined) return existing; + const created = packOnce(packageName); + packs.set(packageName, created); + return created; +}; diff --git a/packages/create-agent-bundle/tests/scaffold-packed-matrix.e2e.test.ts b/packages/create-agent-bundle/tests/scaffold-packed-matrix.e2e.test.ts new file mode 100644 index 000000000..7cd602339 --- /dev/null +++ b/packages/create-agent-bundle/tests/scaffold-packed-matrix.e2e.test.ts @@ -0,0 +1,89 @@ +import { execFile as executeFile } from 'node:child_process'; +import { readFile, stat } from 'node:fs/promises'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { promisify } from 'node:util'; + +import { afterAll, expect, it } from '@rstest/core'; + +import { installedEnvironment, npmInstallArguments } from '../../agent-bundle/tests/support/shared-pack.ts'; +import { cleanupScaffoldFixture, expectCleanValidate, npmRun, scaffoldProject } from './support/scaffold-fixture.ts'; + +const execFile = promisify(executeFile); + +afterAll(cleanupScaffoldFixture); + +/** + * Release-boundary template matrix: the remaining scaffolder templates, each + * through scaffold, install, check, and validate. Runs in + * `test:packed:release` (check:release and the nightly CI schedule), not in + * the per-PR packed pool — the per-PR scaffolder proof is the minimal-template + * smoke in scaffold-packed.e2e.test.ts. The template tests run concurrently: + * each scaffolds its own project directory under the shared runner and npm's + * cache tolerates concurrent installs, so the only shared state is the + * memoized fixture promise. + */ +it.concurrent('scaffolds the mcp-server template and serves the conventional entry from the artifact', async () => { + const projectRoot = await scaffoldProject('mcp-server', 'status-plugin', ['--no-install']); + await execFile('npm', ['install', ...npmInstallArguments], { + cwd: projectRoot, + env: installedEnvironment(), + }); + + await npmRun(projectRoot, 'check'); + await expectCleanValidate(projectRoot); + + const artifact = join(projectRoot, 'artifact'); + const manifest = JSON.parse(await readFile(join(artifact, 'portable', 'mcp.json'), 'utf8')) as { + readonly mcpServers: { readonly status: { readonly args: readonly [string, ...string[]] } }; + }; + const entry = join(artifact, 'portable', manifest.mcpServers.status.args[0]); + // The factory export was wrapped in the framework stdio lifecycle shell. + await expect(readFile(entry, 'utf8')).resolves.toContain('stdio heartbeat'); + + const cli = join(projectRoot, 'node_modules', '.bin', 'agent-bundle'); + const { stdout: listed } = await execFile(cli, [ + 'mcp', 'list', '--json', '--root', projectRoot, '--artifact', artifact, '--target', 'portable', '--server', 'status', + ], { cwd: projectRoot, env: installedEnvironment() }); + expect(JSON.parse(listed)).toMatchObject({ tools: [{ name: 'report-status' }] }); + const { stdout: invoked } = await execFile(cli, [ + 'mcp', 'invoke', '--json', '--root', projectRoot, '--artifact', artifact, '--target', 'portable', + '--server', 'status', '--tool', 'report-status', '--input', '{"service":"docs"}', + ], { cwd: projectRoot, env: installedEnvironment() }); + expect(JSON.parse(invoked)).toMatchObject({ + result: { + content: [{ text: 'docs is ready.', type: 'text' }], + structuredContent: { service: 'docs', status: 'healthy' }, + }, + }); +}, 600_000); + +it.concurrent('scaffolds the cli-tool template with a framework-built bin, lib, and artifact script', async () => { + const projectRoot = await scaffoldProject('cli-tool', 'greeter', ['--no-install']); + await execFile('npm', ['install', ...npmInstallArguments], { + cwd: projectRoot, + env: installedEnvironment(), + }); + + await npmRun(projectRoot, 'check'); + await expectCleanValidate(projectRoot); + + // The src/cli.ts convention produced the executable package bin. + const bin = join(projectRoot, 'dist', 'bin', 'greeter.js'); + expect((await stat(bin)).mode & 0o111).not.toBe(0); + expect((await readFile(bin, 'utf8')).startsWith('#!/usr/bin/env node\n')).toBe(true); + await expect(execFile(bin, ['World'], { cwd: projectRoot, env: installedEnvironment() })) + .resolves.toMatchObject({ stdout: 'Hello, World!\n' }); + + // The src/index.ts convention produced the library export with declarations. + const library = await import(pathToFileURL(join(projectRoot, 'dist', 'index.js')).href) as { + readonly greet: (name: string) => { readonly message: string }; + }; + expect(library.greet('World').message).toBe('Hello, World!'); + await expect(readFile(join(projectRoot, 'dist', 'index.d.ts'), 'utf8')).resolves.toContain('Greeting'); + + // The same CLI also shipped inside the host artifact as a script. + await expect(execFile(process.execPath, [ + join(projectRoot, 'artifact', 'portable', 'scripts', 'greeter.mjs'), 'World', + ], { cwd: projectRoot, env: installedEnvironment() })).resolves.toMatchObject({ stdout: 'Hello, World!\n' }); +}, 600_000); diff --git a/packages/create-agent-bundle/tests/scaffold-packed.e2e.test.ts b/packages/create-agent-bundle/tests/scaffold-packed.e2e.test.ts index 1bca9a9e3..936e1f631 100644 --- a/packages/create-agent-bundle/tests/scaffold-packed.e2e.test.ts +++ b/packages/create-agent-bundle/tests/scaffold-packed.e2e.test.ts @@ -1,111 +1,19 @@ -import { execFile as executeFile } from 'node:child_process'; -import { cp, mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; +import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; -import { pathToFileURL } from 'node:url'; -import { promisify } from 'node:util'; import { afterAll, expect, it } from '@rstest/core'; -const execFile = promisify(executeFile); -const workspaceRoot = process.cwd(); +import { cleanupScaffoldFixture, expectCleanValidate, npmRun, scaffoldProject } from './support/scaffold-fixture.ts'; -const installedEnvironment = (): NodeJS.ProcessEnv => { - const { NODE_PATH: _nodePath, ...environment } = process.env; - return environment; -}; - -interface PackedFixture { - readonly frameworkTarball: string; - readonly root: string; - readonly runnerRoot: string; - readonly scaffolderBin: string; -} +afterAll(cleanupScaffoldFixture); /** - * Build and `npm pack` agent-bundle and create-agent-bundle once (the - * packed-consumer mechanism: copy the package, `rslib build --dist-path` - * into the copy, pack the copy), then install the scaffolder tarball into a - * clean runner project. Every template test drives the installed bin and - * pins the framework with `--framework-version file:`, so the run - * never depends on pkg.pr.new. + * Per-PR scaffolder smoke: one template through the full consumer journey — + * installed scaffolder bin, template scaffold, scaffolder-driven npm install, + * project check, clean validate. The mcp-server and cli-tool templates run in + * the release-boundary matrix (scaffold-packed-matrix.e2e.test.ts) via + * `test:packed:release` and the nightly schedule. */ -const packFixture = async (): Promise => { - const root = await mkdtemp(join(tmpdir(), 'create-agent-bundle-e2e-')); - const pack = async (packageName: string): Promise => { - const packageRoot = join(workspaceRoot, 'packages', packageName); - const packedRoot = join(root, `packed-${packageName}`); - await cp(packageRoot, packedRoot, { recursive: true }); - await execFile(join(workspaceRoot, 'node_modules', '.bin', 'rslib'), [ - 'build', '--config', join(packageRoot, 'rslib.config.ts'), '--dist-path', join(packedRoot, 'dist'), - ], { cwd: workspaceRoot, env: installedEnvironment() }); - const { stdout } = await execFile('npm', ['pack', '--json', '--pack-destination', root], { - cwd: packedRoot, - env: installedEnvironment(), - }); - return join(root, (JSON.parse(stdout) as [{ readonly filename: string }])[0].filename); - }; - const frameworkTarball = await pack('agent-bundle'); - const scaffolderTarball = await pack('create-agent-bundle'); - - const runnerRoot = join(root, 'runner'); - await mkdir(runnerRoot, { recursive: true }); - await writeFile(join(runnerRoot, 'package.json'), '{"name":"scaffold-runner","type":"module","private":true}\n'); - await execFile('npm', ['install', '--ignore-scripts', '--no-audit', '--no-fund', scaffolderTarball], { - cwd: runnerRoot, - env: installedEnvironment(), - }); - return { - frameworkTarball, - root, - runnerRoot, - scaffolderBin: join(runnerRoot, 'node_modules', '.bin', 'create-agent-bundle'), - }; -}; - -let fixturePromise: Promise | undefined; -const fixture = (): Promise => { - fixturePromise ??= packFixture(); - return fixturePromise; -}; - -afterAll(async () => { - if (fixturePromise === undefined) return; - const { root } = await fixture(); - await rm(root, { force: true, recursive: true }); -}); - -const scaffoldProject = async ( - template: string, - projectName: string, - extraArguments: readonly string[], -): Promise => { - const { frameworkTarball, runnerRoot, scaffolderBin } = await fixture(); - await execFile(scaffolderBin, [ - projectName, - '--template', template, - '--targets', 'portable,codex,claude', - '--package-manager', 'npm', - '--framework-version', `file:${frameworkTarball}`, - ...extraArguments, - ], { cwd: runnerRoot, env: installedEnvironment() }); - return join(runnerRoot, projectName); -}; - -const npmRun = async (projectRoot: string, script: string): Promise<{ readonly stdout: string }> => - execFile('npm', ['run', script], { cwd: projectRoot, env: installedEnvironment() }); - -/** Zero diagnostics — including the informational AB473x migration nudges. */ -const expectCleanValidate = async (projectRoot: string): Promise => { - const cli = join(projectRoot, 'node_modules', '.bin', 'agent-bundle'); - const { stdout } = await execFile(cli, ['validate', '--json', '--root', projectRoot], { - cwd: projectRoot, - env: installedEnvironment(), - }); - const validated = JSON.parse(stdout) as { readonly diagnostics: readonly unknown[] }; - expect(validated.diagnostics).toEqual([]); -}; - it('scaffolds the minimal template, auto-installs, and passes its own check', async () => { // No --no-install: this run covers the scaffolder-driven `npm install` path. const projectRoot = await scaffoldProject('minimal', 'minimal-project', []); @@ -123,68 +31,3 @@ it('scaffolds the minimal template, auto-installs, and passes its own check', as await expect(readFile(join(projectRoot, 'artifact', 'portable', 'skills', 'getting-started', 'SKILL.md'), 'utf8')) .resolves.toContain('# Getting started'); }, 600_000); - -it('scaffolds the mcp-server template and serves the conventional entry from the artifact', async () => { - const projectRoot = await scaffoldProject('mcp-server', 'status-plugin', ['--no-install']); - await execFile('npm', ['install', '--ignore-scripts', '--no-audit', '--no-fund'], { - cwd: projectRoot, - env: installedEnvironment(), - }); - - await npmRun(projectRoot, 'check'); - await expectCleanValidate(projectRoot); - - const artifact = join(projectRoot, 'artifact'); - const manifest = JSON.parse(await readFile(join(artifact, 'portable', 'mcp.json'), 'utf8')) as { - readonly mcpServers: { readonly status: { readonly args: readonly [string, ...string[]] } }; - }; - const entry = join(artifact, 'portable', manifest.mcpServers.status.args[0]); - // The factory export was wrapped in the framework stdio lifecycle shell. - await expect(readFile(entry, 'utf8')).resolves.toContain('stdio heartbeat'); - - const cli = join(projectRoot, 'node_modules', '.bin', 'agent-bundle'); - const { stdout: listed } = await execFile(cli, [ - 'mcp', 'list', '--json', '--root', projectRoot, '--artifact', artifact, '--target', 'portable', '--server', 'status', - ], { cwd: projectRoot, env: installedEnvironment() }); - expect(JSON.parse(listed)).toMatchObject({ tools: [{ name: 'report-status' }] }); - const { stdout: invoked } = await execFile(cli, [ - 'mcp', 'invoke', '--json', '--root', projectRoot, '--artifact', artifact, '--target', 'portable', - '--server', 'status', '--tool', 'report-status', '--input', '{"service":"docs"}', - ], { cwd: projectRoot, env: installedEnvironment() }); - expect(JSON.parse(invoked)).toMatchObject({ - result: { - content: [{ text: 'docs is ready.', type: 'text' }], - structuredContent: { service: 'docs', status: 'healthy' }, - }, - }); -}, 600_000); - -it('scaffolds the cli-tool template with a framework-built bin, lib, and artifact script', async () => { - const projectRoot = await scaffoldProject('cli-tool', 'greeter', ['--no-install']); - await execFile('npm', ['install', '--ignore-scripts', '--no-audit', '--no-fund'], { - cwd: projectRoot, - env: installedEnvironment(), - }); - - await npmRun(projectRoot, 'check'); - await expectCleanValidate(projectRoot); - - // The src/cli.ts convention produced the executable package bin. - const bin = join(projectRoot, 'dist', 'bin', 'greeter.js'); - expect((await stat(bin)).mode & 0o111).not.toBe(0); - expect((await readFile(bin, 'utf8')).startsWith('#!/usr/bin/env node\n')).toBe(true); - await expect(execFile(bin, ['World'], { cwd: projectRoot, env: installedEnvironment() })) - .resolves.toMatchObject({ stdout: 'Hello, World!\n' }); - - // The src/index.ts convention produced the library export with declarations. - const library = await import(pathToFileURL(join(projectRoot, 'dist', 'index.js')).href) as { - readonly greet: (name: string) => { readonly message: string }; - }; - expect(library.greet('World').message).toBe('Hello, World!'); - await expect(readFile(join(projectRoot, 'dist', 'index.d.ts'), 'utf8')).resolves.toContain('Greeting'); - - // The same CLI also shipped inside the host artifact as a script. - await expect(execFile(process.execPath, [ - join(projectRoot, 'artifact', 'portable', 'scripts', 'greeter.mjs'), 'World', - ], { cwd: projectRoot, env: installedEnvironment() })).resolves.toMatchObject({ stdout: 'Hello, World!\n' }); -}, 600_000); diff --git a/packages/create-agent-bundle/tests/support/scaffold-fixture.ts b/packages/create-agent-bundle/tests/support/scaffold-fixture.ts new file mode 100644 index 000000000..e321895a1 --- /dev/null +++ b/packages/create-agent-bundle/tests/support/scaffold-fixture.ts @@ -0,0 +1,93 @@ +import { execFile as executeFile } from 'node:child_process'; +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; + +import { expect } from '@rstest/core'; + +import { installedEnvironment, npmInstallArguments, sharedPackedTarball } from '../../../agent-bundle/tests/support/shared-pack.ts'; + +const execFile = promisify(executeFile); + +interface PackedFixture { + readonly frameworkTarball: string; + readonly root: string; + readonly runnerRoot: string; + readonly scaffolderBin: string; +} + +/** + * Take the run-level agent-bundle and create-agent-bundle release tarballs + * (packed once per `test:packed` run — see tests/support/shared-pack.ts), + * then install the scaffolder tarball into a clean runner project. Every + * template test drives the installed bin and pins the framework with + * `--framework-version file:`, so the run never depends on + * pkg.pr.new. Each test file gets its own fixture instance (rstest isolates + * files), so register `cleanupScaffoldFixture` in an `afterAll` per file. + */ +const packFixture = async (): Promise => { + const root = await mkdtemp(join(tmpdir(), 'create-agent-bundle-e2e-')); + const [{ tarball: frameworkTarball }, { tarball: scaffolderTarball }] = await Promise.all([ + sharedPackedTarball('agent-bundle'), + sharedPackedTarball('create-agent-bundle'), + ]); + + const runnerRoot = join(root, 'runner'); + await mkdir(runnerRoot, { recursive: true }); + await writeFile(join(runnerRoot, 'package.json'), '{"name":"scaffold-runner","type":"module","private":true}\n'); + await execFile('npm', ['install', ...npmInstallArguments, scaffolderTarball], { + cwd: runnerRoot, + env: installedEnvironment(), + }); + return { + frameworkTarball, + root, + runnerRoot, + scaffolderBin: join(runnerRoot, 'node_modules', '.bin', 'create-agent-bundle'), + }; +}; + +let fixturePromise: Promise | undefined; +const fixture = (): Promise => { + fixturePromise ??= packFixture(); + return fixturePromise; +}; + +export const cleanupScaffoldFixture = async (): Promise => { + if (fixturePromise === undefined) return; + const { root } = await fixturePromise; + await rm(root, { force: true, recursive: true }); +}; + +export const scaffoldProject = async ( + template: string, + projectName: string, + extraArguments: readonly string[], +): Promise => { + const { frameworkTarball, runnerRoot, scaffolderBin } = await fixture(); + await execFile(scaffolderBin, [ + projectName, + '--template', template, + '--targets', 'portable,codex,claude', + '--package-manager', 'npm', + '--framework-version', `file:${frameworkTarball}`, + ...extraArguments, + ], { cwd: runnerRoot, env: installedEnvironment() }); + return join(runnerRoot, projectName); +}; + +export const npmRun = async (projectRoot: string, script: string): Promise => { + await execFile('npm', ['run', script], { cwd: projectRoot, env: installedEnvironment() }); +}; + +/** Zero diagnostics — including the informational AB473x migration nudges. */ +export const expectCleanValidate = async (projectRoot: string): Promise => { + const cli = join(projectRoot, 'node_modules', '.bin', 'agent-bundle'); + const { stdout } = await execFile(cli, ['validate', '--json', '--root', projectRoot], { + cwd: projectRoot, + env: installedEnvironment(), + }); + const validated = JSON.parse(stdout) as { readonly diagnostics: readonly unknown[] }; + expect(validated.diagnostics).toEqual([]); +}; diff --git a/packages/workbench/tests/packed-release.e2e.test.ts b/packages/workbench/tests/packed-release.e2e.test.ts index ccc8e8550..dfe352f39 100644 --- a/packages/workbench/tests/packed-release.e2e.test.ts +++ b/packages/workbench/tests/packed-release.e2e.test.ts @@ -14,18 +14,17 @@ import { validateOutageLedger, type ConsoleErrorRecord, } from './support/packed-outage-ledger.ts'; +import { npmInstallArguments, sharedPackedTarball } from '../../agent-bundle/tests/support/shared-pack.ts'; import { timeScale } from '../../agent-bundle/tests/support/time-scale.ts'; import { availablePort, awaitReady, - buildPackage, closeChild, descendantProcessIds, execFile, firstRecord, installedEnvironment, isWithin, - packageRoot, record, string, workspaceRoot, @@ -36,7 +35,6 @@ import { workbenchUrl } from './support/workbench-e2e.ts'; const fixtureRoot = join(workspaceRoot, 'fixtures', 'integration', 'packed-release'); const browserTimeout = 12_000 * timeScale; -const packedServerStartupBudget = 45_000 * timeScale; const productTemporaryRootPrefixes = [ 'agent-bundle-hook-playground-', 'agent-bundle-mcp-', @@ -71,7 +69,7 @@ const isAppRoute = (url: URL): boolean => e2e('runs every Agent API tool from the installed tarball', { timeout: 360_000 * timeScale }, async ({ page }) => { - await buildPackage(); + const { tarball } = await sharedPackedTarball('agent-bundle'); const consumer = await mkdtemp(join(tmpdir(), 'agent-bundle-packed-release-')); const forbiddenStagedPackage = join(consumer, 'staged-package'); const project = join(consumer, 'project'); @@ -84,14 +82,8 @@ e2e('runs every Agent API tool from the installed tarball', { timeout: 360_000 * let cleanupFailure: AggregateError | undefined; let primaryFailure: Error | undefined; try { - const { stdout } = await execFile('npm', ['pack', '--json', '--pack-destination', consumer], { - cwd: packageRoot, - env: installedEnvironment(), - }); - const [packed] = JSON.parse(stdout) as Array<{ readonly filename: string }>; - const tarball = join(consumer, packed.filename); await writeFile(join(consumer, 'package.json'), '{"type":"module"}\n'); - await execFile('npm', ['install', '--omit=dev', '--ignore-scripts', '--no-audit', '--no-fund', tarball], { + await execFile('npm', ['install', '--omit=dev', ...npmInstallArguments, tarball], { cwd: consumer, env: installedEnvironment(), }); @@ -475,8 +467,7 @@ e2e('runs every Agent API tool from the installed tarball', { timeout: 360_000 * const exportedScriptSection = page.getByRole('heading', { name: 'Exported trace' }).locator('..'); await expect(exportedScriptSection).toContainText(scriptSessionId); await expect(exportedScriptSection).toContainText(scriptCompletedReference); - const scriptSelectedCheckbox = scriptCompletedCheckbox; - await scriptSelectedCheckbox.check(); + await scriptCompletedCheckbox.check(); await expect(page.getByRole('button', { name: 'Promote to draft eval case' })).toBeEnabled({ timeout: browserTimeout }); const promotedScriptResponse = page.waitForResponse((response) => response.url() === `${origin}/api/playground/sessions/${encodeURIComponent(scriptSessionId)}/draft-eval` && @@ -949,9 +940,8 @@ e2e('runs every Agent API tool from the installed tarball', { timeout: 360_000 * const browserMcpSessionBCloseCompletedAt = Date.now(); phase = 'desktop navigation floor'; - await page.setViewportSize({ height: 900, width: 1440 }); - const mobileNavigationRequestIndex = browserRequests.length; - const mobileRoutes: readonly Readonly<{ heading: string; label: string }>[] = [ + const navigationFloorRequestIndex = browserRequests.length; + const navigationRoutes: readonly Readonly<{ heading: string; label: string }>[] = [ { heading: 'Bundle dashboard', label: 'Overview' }, { heading: 'Skills', label: 'Skills' }, { heading: 'Hooks', label: 'Hooks' }, { heading: 'MCP playground', label: 'MCP playground' }, { heading: 'Artifacts', label: 'Artifacts' }, { heading: 'Playground', label: 'Playground' }, { heading: 'Logs', label: 'Logs' }, { heading: 'Evals', label: 'Evals' }, { heading: 'Comparisons', label: 'Comparisons' }, @@ -971,20 +961,20 @@ e2e('runs every Agent API tool from the installed tarball', { timeout: 360_000 * respondedStream?: true; url: string; }>> = []; - let activeMobileRoute: Readonly<{ openedAt: number; urls?: readonly string[] }> | undefined; - const leaveActiveMobileRoute = (leftAt: number): void => { - if (activeMobileRoute === undefined) return; - for (const url of activeMobileRoute.urls ?? []) postRecoveryNavigation.push(Object.freeze({ + let activeNavigationRoute: Readonly<{ openedAt: number; urls?: readonly string[] }> | undefined; + const leaveActiveNavigationRoute = (leftAt: number): void => { + if (activeNavigationRoute === undefined) return; + for (const url of activeNavigationRoute.urls ?? []) postRecoveryNavigation.push(Object.freeze({ leftAt, - openedAt: activeMobileRoute.openedAt, + openedAt: activeNavigationRoute.openedAt, ...(respondedNavigationStreams.has(url) ? { respondedStream: true as const } : {}), url, })); - activeMobileRoute = undefined; + activeNavigationRoute = undefined; }; - for (const route of mobileRoutes) { + for (const route of navigationRoutes) { const openedAt = Date.now(); - leaveActiveMobileRoute(openedAt); + leaveActiveNavigationRoute(openedAt); const logsStreamResponse = route.label === 'Logs' ? page.waitForResponse((response) => { const url = new URL(response.url()); @@ -1010,13 +1000,13 @@ e2e('runs every Agent API tool from the installed tarball', { timeout: 360_000 * respondedNavigationStreams.add(streamUrl); routeUrls.push(streamUrl); } - activeMobileRoute = Object.freeze({ openedAt, urls: Object.freeze(routeUrls) }); + activeNavigationRoute = Object.freeze({ openedAt, urls: Object.freeze(routeUrls) }); } - leaveActiveMobileRoute(Date.now()); + leaveActiveNavigationRoute(Date.now()); await page.getByRole('link', { name: 'Overview', exact: true }).focus(); await page.keyboard.press('Enter'); await expect(page.getByRole('heading', { name: 'Bundle dashboard' })).toBeVisible({ timeout: browserTimeout }); - await waitForBrowserRequestsAfter(mobileNavigationRequestIndex); + await waitForBrowserRequestsAfter(navigationFloorRequestIndex); phase = 'foreground outage ledger quiet fence'; const requestFailuresBeforeQuietFence = browserRequests.filter((request) => request.error !== undefined); diff --git a/packages/workbench/tests/support/packed-release-harness.ts b/packages/workbench/tests/support/packed-release-harness.ts index c1f8cc806..68478f839 100644 --- a/packages/workbench/tests/support/packed-release-harness.ts +++ b/packages/workbench/tests/support/packed-release-harness.ts @@ -1,19 +1,14 @@ -import type { ChildProcess } from 'node:child_process'; +import { execFile as executeFile, type ChildProcess } from 'node:child_process'; import { chmod, mkdir, writeFile } from 'node:fs/promises'; import { createServer } from 'node:net'; -import { join, relative, isAbsolute } from 'node:path'; +import { relative, isAbsolute, join } from 'node:path'; import { promisify } from 'node:util'; -export const execFile = promisify((await import('node:child_process')).execFile); +export { installedEnvironment } from '../../../agent-bundle/tests/support/shared-pack.ts'; + +export const execFile = promisify(executeFile); export const workspaceRoot = process.cwd(); -export const packageRoot = join(workspaceRoot, 'packages', 'agent-bundle'); const packedServerStartupBudget = 45_000; -let builtPackage: Promise | undefined; - -export const installedEnvironment = (): NodeJS.ProcessEnv => { - const { NODE_PATH: _nodePath, ...environment } = process.env; - return environment; -}; export const availablePort = async (): Promise => { const server = createServer(); @@ -30,15 +25,6 @@ export const availablePort = async (): Promise => { return address.port; }; -export const buildPackage = (): Promise => builtPackage ??= (async (): Promise => { - if (process.env['AGENT_BUNDLE_PACKAGE_PREBUILT'] === '1') return; - const { RSTEST: _rstest, ...environment } = process.env; - await execFile('pnpm', ['build'], { - cwd: workspaceRoot, - env: { ...environment, NODE_ENV: 'production' }, - }); -})(); - export const awaitReady = async (origin: string, child: ChildProcess, output: () => string): Promise => { const startedAt = Date.now(); const diagnostics = (): string => diff --git a/rstest.integration-tests.ts b/rstest.integration-tests.ts index 3adb612fa..ab2a6dacb 100644 --- a/rstest.integration-tests.ts +++ b/rstest.integration-tests.ts @@ -66,14 +66,15 @@ export const integrationTestFiles: readonly string[] = [ ]; /** - * Pack-and-install tests: each one runs `npm pack` (and usually a clean - * `npm install` of the tarball), which dominates the serialized integration - * pool. They run through the root `test:packed` / `test:packed:native` - * scripts instead — CI's release-gates job (`check:release`) and the + * Pack-and-install tests: each one consumes the run-level release tarball + * (and usually a clean `npm install` of it), which dominates the serialized + * integration pool. They run through the root `test:packed` / + * `test:packed:native` scripts instead — CI's release-gates job and the * native-host-smoke workflow keep them covered — and stay excluded from the * parallel unit pool. packed-release.e2e lives here (not in the integration - * pool) so `pnpm test` and `check:release` don't each run the same long - * packed-browser suite; `rstest.config.ts` keeps `test:packed` on one worker. + * pool) so `pnpm test` and the release gates don't each run the same long + * packed-browser suite; `rstest.packed.config.ts` keeps `test:packed` on one + * worker. */ export const packedTestFiles: readonly string[] = [ 'packages/agent-bundle/tests/dev-workbench-packaging.test.ts', @@ -86,6 +87,17 @@ export const packedTestFiles: readonly string[] = [ 'packages/workbench/tests/packed-release.e2e.test.ts', ]; +/** + * Release-boundary-only pack-and-install tests: the scaffolder template + * matrix beyond the per-PR minimal-template smoke. Runs through + * `test:packed:release` (check:release and CI's nightly schedule), not on + * every PR — the per-PR release gates keep one full scaffold journey via + * scaffold-packed.e2e.test.ts. + */ +export const packedReleaseOnlyTestFiles: readonly string[] = [ + 'packages/create-agent-bundle/tests/scaffold-packed-matrix.e2e.test.ts', +]; + /** * Checked-in scaffolding templates ship their own test files; they run inside * scaffolded projects (the packed e2e drives them through each project's diff --git a/rstest.packed.config.ts b/rstest.packed.config.ts new file mode 100644 index 000000000..ad15c6c0b --- /dev/null +++ b/rstest.packed.config.ts @@ -0,0 +1,22 @@ +import { defineConfig } from '@rstest/core'; + +import { packedReleaseOnlyTestFiles, packedTestFiles } from './rstest.integration-tests.ts'; +import { withAgentBundleRslibConfig } from './rstest.rslib.ts'; + +/** + * Pack-and-install suites, normally launched through + * `scripts/run-packed-tests.mjs` so every file consumes one shared tarball + * per public package. `--release` (AGENT_BUNDLE_PACKED_RELEASE=1) adds the + * release-boundary-only files — the scaffolder template matrix — on top of + * the per-PR set. The pool stays on one worker: dev-workbench-packaging + * rebuilds the workspace `dist` in place while release-audit's audit script + * packs it, so the files still contend on workspace-shared writes. + */ +export default defineConfig({ + extends: withAgentBundleRslibConfig(), + include: [ + ...packedTestFiles, + ...(process.env['AGENT_BUNDLE_PACKED_RELEASE'] === '1' ? packedReleaseOnlyTestFiles : []), + ], + pool: { maxWorkers: 1 }, +}); diff --git a/scripts/run-packed-tests.mjs b/scripts/run-packed-tests.mjs new file mode 100644 index 000000000..78a0e05b3 --- /dev/null +++ b/scripts/run-packed-tests.mjs @@ -0,0 +1,60 @@ +/** + * Builds once, packs each public package once, and runs the packed pool + * against the shared tarballs (tests/support/shared-pack.ts) with the + * prebuilt seams set instead of every test file rebuilding the workspace for + * itself. Build and pack run with NODE_ENV=production like the release + * pipeline they stand in for; every child inherits the ambient environment + * minus NODE_PATH. `--release` adds the release-boundary-only files (the + * scaffolder template matrix) to the pool; remaining arguments pass through + * to rstest. + */ +import { execFile as executeFile, spawn } from 'node:child_process'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; + +const execFile = promisify(executeFile); +const repositoryRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const { NODE_PATH: _nodePath, ...environment } = process.env; +const releasePool = process.argv.includes('--release'); +const rstestArguments = process.argv.slice(2).filter((argument) => argument !== '--release'); + +const run = (command, args, extraEnvironment = {}) => new Promise((resolvePromise, rejectPromise) => { + const child = spawn(command, args, { + cwd: repositoryRoot, + env: { ...environment, ...extraEnvironment }, + stdio: 'inherit', + }); + child.once('error', rejectPromise); + child.once('exit', (code, signal) => { + resolvePromise(signal === null ? (code ?? 1) : 1); + }); +}); + +const buildExitCode = await run('pnpm', ['build'], { NODE_ENV: 'production' }); +if (buildExitCode !== 0) process.exit(buildExitCode); + +const packDirectory = await mkdtemp(join(tmpdir(), 'agent-bundle-shared-pack-')); +try { + await Promise.all(['agent-bundle', 'create-agent-bundle'].map(async (packageName) => { + const { stdout } = await execFile('npm', ['pack', '--json', '--pack-destination', packDirectory], { + cwd: join(repositoryRoot, 'packages', packageName), + env: { ...environment, NODE_ENV: 'production' }, + }); + const [packOutput] = JSON.parse(stdout); + await writeFile( + join(packDirectory, `${packageName}.json`), + `${JSON.stringify({ packOutput, tarball: join(packDirectory, packOutput.filename) })}\n`, + ); + })); + process.exitCode = await run('pnpm', ['exec', 'rstest', '--config', 'rstest.packed.config.ts', ...rstestArguments], { + AGENT_BUNDLE_PACKAGE_PREBUILT: '1', + ...(releasePool ? { AGENT_BUNDLE_PACKED_RELEASE: '1' } : {}), + AGENT_BUNDLE_SHARED_PACK_DIR: packDirectory, + AGENT_BUNDLE_WORKBENCH_PREBUILT: '1', + }); +} finally { + await rm(packDirectory, { force: true, recursive: true }); +}