From e89a878c535d222e1f5af57842256588c77b46ef Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Wed, 2 Sep 2026 05:47:07 +0000 Subject: [PATCH] test(install): empty-dir packed-tarball host-install proofs for the package-relative installer (#252) --- .github/workflows/native-host-smoke.yml | 3 + package.json | 2 + .../tests/packed-host-install-proof.test.ts | 264 ++++++++++++++++++ .../tests/support/host-install.ts | 86 ++++-- rstest.integration-tests.ts | 1 + 5 files changed, 331 insertions(+), 25 deletions(-) create mode 100644 packages/agent-bundle/tests/packed-host-install-proof.test.ts diff --git a/.github/workflows/native-host-smoke.yml b/.github/workflows/native-host-smoke.yml index 24161c793..c1831e993 100644 --- a/.github/workflows/native-host-smoke.yml +++ b/.github/workflows/native-host-smoke.yml @@ -54,6 +54,9 @@ jobs: - name: ${{ matrix.host }} real host install proof if: inputs.host == 'both' || inputs.host == matrix.host run: pnpm test:host-install + - name: ${{ matrix.host }} packed-tarball host install proof + if: inputs.host == 'both' || inputs.host == matrix.host + run: pnpm test:host-install:packed # Claude only: the session-token proof needs a signed-in `claude -p` turn, # and Codex and Cursor have no equivalent non-interactive surface. - name: ${{ matrix.host }} real session token proof diff --git a/package.json b/package.json index 064e86a7e..882a0e96a 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,8 @@ "test:packed:native:codex": "pnpm build && AGENT_BUNDLE_PACKED_NATIVE_CODEX_SMOKE=1 pnpm test:packed:native", "test:host-install": "rstest --config rstest.config.ts packages/agent-bundle/tests/host-install-proof.test.ts", "test:host-install:build": "pnpm build && pnpm test:host-install", + "test:host-install:packed": "rstest --config rstest.config.ts packages/agent-bundle/tests/packed-host-install-proof.test.ts", + "test:host-install:packed:build": "pnpm build && pnpm test:host-install:packed", "test:host-install:session": "rstest --config rstest.config.ts packages/agent-bundle/tests/host-install-session.test.ts", "test:host-install:session:claude": "pnpm build && AGENT_BUNDLE_HOST_INSTALL_CLAUDE_SESSION=1 pnpm test:host-install:session", "changeset": "changeset", diff --git a/packages/agent-bundle/tests/packed-host-install-proof.test.ts b/packages/agent-bundle/tests/packed-host-install-proof.test.ts new file mode 100644 index 000000000..6e14d8dac --- /dev/null +++ b/packages/agent-bundle/tests/packed-host-install-proof.test.ts @@ -0,0 +1,264 @@ +import { execFile as executeFile, spawnSync } from 'node:child_process'; +import { access, mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { promisify } from 'node:util'; + +import { afterAll, beforeAll, expect, it } from '@rstest/core'; + +import { + buildHostInstallFixture, + disposeHostInstallFixture, + runClaudeHostInstallProof, + runCodexHostInstallProof, + runCursorHostInstallProof, + type BuiltHostInstallFixture, + type HostInstallCommand, +} from './support/host-install.ts'; +import { + installedEnvironment, + npmInstallArguments, + packOutputFromJson, +} from './support/shared-pack.ts'; +import { + HOST_INSTALL_PROOF_LEVEL, + proofLevelLabel, +} from '../src/test/manifest.ts'; + +const execFile = promisify(executeFile); +const proofLabel = proofLevelLabel(HOST_INSTALL_PROOF_LEVEL); +const packageName = 'host-install-proof-fixture'; +const pluginName = 'host-install-proof'; +const claudeMissingEvidence = 'missing evidence: claude binary unavailable on PATH'; +const codexMissingEvidence = 'missing evidence: codex binary unavailable on PATH'; +const claudeAvailable = spawnSync('claude', ['--version'], { + stdio: 'ignore', + timeout: 5_000, + windowsHide: true, +}).status === 0; +const codexAvailable = spawnSync('codex', ['--version'], { + stdio: 'ignore', + timeout: 5_000, + windowsHide: true, +}).status === 0; +const claudePluginIt = claudeAvailable ? it : it.skip; +const codexPluginIt = codexAvailable ? it : it.skip; + +let cleanupRoot: string | undefined; +let sourceFixture: BuiltHostInstallFixture | undefined; +let packedFixture: BuiltHostInstallFixture | undefined; +let packedInstallCommand: HostInstallCommand | undefined; +let fixturePackageVersion: string | undefined; + +beforeAll(async () => { + cleanupRoot = await mkdtemp(join(tmpdir(), 'agent-bundle-packed-host-install-')); + sourceFixture = await buildHostInstallFixture({ + buildCommand: 'prepack', + environment: process.env, + prepareProject: async (projectRoot) => { + const packagePath = join(projectRoot, 'package.json'); + const packageDocument = JSON.parse(await readFile(packagePath, 'utf8')) as Record; + if (typeof packageDocument.version !== 'string') { + throw new TypeError(`[${proofLabel}] host-install fixture package has no string version.`); + } + fixturePackageVersion = packageDocument.version; + delete packageDocument.private; + packageDocument.bin = { [pluginName]: `./dist/bin/${pluginName}.js` }; + packageDocument.files = ['artifact', 'dist']; + await Promise.all([ + writeFile(packagePath, `${JSON.stringify(packageDocument, null, 2)}\n`), + writeFile(join(projectRoot, 'src', 'index.ts'), 'export const fixture = true;\n'), + ]); + const configPath = join(projectRoot, 'agent-bundle.config.ts'); + const config = await readFile(configPath, 'utf8'); + await writeFile(configPath, config.replace( + 'export default {\n', + "export default {\n bin: false,\n lib: { dts: false, entry: './src/index.ts' },\n", + )); + }, + }); + + const projectRoot = dirname(sourceFixture.artifactRoot); + const tarballs = join(cleanupRoot, 'tarballs'); + const consumer = join(cleanupRoot, 'consumer'); + await Promise.all([mkdir(tarballs), mkdir(consumer)]); + expect(await readdir(consumer), proofLabel).toEqual([]); + + const packed = await execFile( + 'npm', + ['pack', '--json', '--ignore-scripts', '--pack-destination', tarballs], + { cwd: projectRoot, env: installedEnvironment() }, + ); + const packOutput = packOutputFromJson(packed.stdout); + const tarball = join(tarballs, packOutput.filename); + await execFile('npm', ['install', ...npmInstallArguments, tarball], { + cwd: consumer, + env: installedEnvironment(), + }); + + const installedPackageRoot = join(consumer, 'node_modules', packageName); + const installedArtifactRoot = join(installedPackageRoot, 'artifact'); + const installedBin = join(consumer, 'node_modules', '.bin', pluginName); + await Promise.all([ + access(installedBin), + access(join(installedArtifactRoot, 'claude')), + access(join(installedArtifactRoot, 'codex')), + access(join(installedArtifactRoot, 'cursor')), + ]); + + await rm(projectRoot, { force: true, recursive: true }); + await expect(stat(projectRoot), proofLabel).rejects.toMatchObject({ code: 'ENOENT' }); + await expect(access(join(projectRoot, 'dist', 'bin', `${pluginName}.js`)), proofLabel) + .rejects.toMatchObject({ code: 'ENOENT' }); + + packedFixture = Object.freeze({ + artifactRoot: installedArtifactRoot, + bundles: Object.freeze({ + claude: join(installedArtifactRoot, 'claude'), + codex: join(installedArtifactRoot, 'codex'), + cursor: join(installedArtifactRoot, 'cursor'), + }), + cli: sourceFixture.cli, + root: cleanupRoot, + }); + packedInstallCommand = Object.freeze({ cwd: consumer, executable: installedBin }); +}, 300_000); + +afterAll(async () => { + await Promise.all([ + cleanupRoot === undefined ? Promise.resolve() : rm(cleanupRoot, { force: true, recursive: true }), + sourceFixture === undefined ? Promise.resolve() : disposeHostInstallFixture(sourceFixture), + ]); +}); + +const builtFixture = (): BuiltHostInstallFixture => { + if (packedFixture === undefined) throw new Error(`[${proofLabel}] packed fixture setup did not complete.`); + return packedFixture; +}; + +const installCommand = (): HostInstallCommand => { + if (packedInstallCommand === undefined) throw new Error(`[${proofLabel}] packed installer setup did not complete.`); + return packedInstallCommand; +}; + +const expectHygienicReport = (report: unknown): void => { + expect(JSON.stringify(report), proofLabel).not.toMatch( + /(?:API_KEY|AUTH_TOKEN|ACCESS_TOKEN|authorization|credential|password|secret|sk-[A-Za-z0-9_-]{16,}|\/home\/|\/Users\/|\/tmp\/|stdout|stderr)/iu, + ); +}; + +claudePluginIt( + claudeAvailable + ? 'installs the packed tarball through Claude and observes the host-owned component inventory' + : `installs the packed tarball through Claude and observes the host-owned component inventory [${claudeMissingEvidence}]`, + async () => { + const report = await runClaudeHostInstallProof(builtFixture(), { + environment: process.env, + installCommand: installCommand(), + }); + + expect(report, proofLabel).toEqual({ + host: 'claude', + install: { state: 'installed', version: '1.0.0' }, + inventory: { hooks: 1, mcpServers: 1, skills: 1 }, + proofLevel: proofLabel, + registration: { + enabled: true, + id: 'host-install-proof@host-install-proof-marketplace', + installPath: 'plugins/cache/host-install-proof-marketplace/host-install-proof/1.0.0', + mcpServers: ['probe'], + scope: 'user', + version: '1.0.0', + }, + skill: 'plugins/cache/host-install-proof-marketplace/host-install-proof/1.0.0/skills/probe/SKILL.md', + status: 'passed', + }); + expect(report.registration.version, proofLabel).toBe(fixturePackageVersion); + expectHygienicReport(report); + }, + 180_000, +); + +codexPluginIt( + codexAvailable + ? 'installs the packed tarball through Codex and observes enabled registration' + : `installs the packed tarball through Codex and observes enabled registration [${codexMissingEvidence}]`, + async () => { + const report = await runCodexHostInstallProof(builtFixture(), { + environment: process.env, + installCommand: installCommand(), + }); + + expect(report, proofLabel).toEqual({ + host: 'codex', + install: { state: 'installed', version: '1.0.0' }, + manifest: { + interfaceCapabilities: ['hooks', 'mcp', 'skills'], + interfaceFields: [ + 'capabilities', + 'category', + 'defaultPrompt', + 'developerName', + 'displayName', + 'longDescription', + 'shortDescription', + ], + path: '.codex-plugin/plugin.json', + }, + proofLevel: proofLabel, + registration: { + cachePath: 'plugins/cache/host-install-proof-marketplace/host-install-proof/1.0.0', + state: 'installed, enabled', + version: '1.0.0', + }, + skill: 'plugins/cache/host-install-proof-marketplace/host-install-proof/1.0.0/skills/probe/SKILL.md', + skillSidecar: { + matchesBuiltArtifact: true, + path: 'skills/probe/agents/openai.yaml', + schema: 'schema-valid', + sections: ['dependencies', 'interface', 'policy'], + }, + status: 'passed', + }); + expect(report.registration.version, proofLabel).toBe(fixturePackageVersion); + expectHygienicReport(report); + }, + 180_000, +); + +it('installs the packed tarball into an isolated Cursor home, validates schemas, and is idempotent', async () => { + const report = await runCursorHostInstallProof(builtFixture(), { + environment: process.env, + installCommand: installCommand(), + }); + + expect(report, proofLabel).toEqual({ + destination: '.cursor/plugins/local/host-install-proof', + documents: { + hooks: 'schema-valid', + mcp: 'schema-valid', + plugin: 'schema-valid', + }, + host: 'cursor', + install: { first: 'installed', second: 'already-installed', version: '1.0.0' }, + logo: { + path: './assets/docs/media/logo.svg', + resolvesInsideDeployTree: true, + }, + pluginRootVariable: { + locations: [ + 'hooks/hooks.json#/hooks/sessionStart/0/command', + 'mcp.json#/mcpServers/probe/args/0', + 'mcp.json#/mcpServers/probe/env/AGENT_BUNDLE_PLUGIN_ROOT', + ], + resolvedAtInstall: false, + sessionEvidence: 'unavailable: Cursor exposes no non-interactive plugin-loading session surface', + spelling: '${CURSOR_PLUGIN_ROOT}', + }, + proofLevel: proofLabel, + skill: '.cursor/plugins/local/host-install-proof/skills/probe/SKILL.md', + status: 'passed', + }); + expect(report.install.version, proofLabel).toBe(fixturePackageVersion); + expectHygienicReport(report); +}, 180_000); diff --git a/packages/agent-bundle/tests/support/host-install.ts b/packages/agent-bundle/tests/support/host-install.ts index 6389fb962..f3f229ace 100644 --- a/packages/agent-bundle/tests/support/host-install.ts +++ b/packages/agent-bundle/tests/support/host-install.ts @@ -94,6 +94,17 @@ export interface BuiltHostInstallTokenFixture extends BuiltFixtureProject { readonly loweredSkillMarkdown: string; } +export interface HostInstallCommand { + readonly cwd?: string; + readonly executable: string; + readonly prefixArguments?: readonly string[]; +} + +interface HostInstallProofOptions { + readonly environment: Readonly; + readonly installCommand?: HostInstallCommand; +} + export interface ClaudeHostInstallReport { readonly host: 'claude'; readonly install: { readonly state: 'installed'; readonly version: '1.0.0' }; @@ -245,6 +256,33 @@ const runNodeCli = ( { ...options, timeout: 180_000 }, ); +const runInstallCommand = ( + fixture: BuiltHostInstallFixture, + host: 'claude' | 'codex' | 'cursor', + bundle: string, + options: HostInstallProofOptions, +): Promise => { + if (options.installCommand === undefined) { + return runNodeCli(fixture, [ + 'install', + host, + '--from', + bundle, + '--json', + ], { cwd: bundle, environment: isolatedEnvironment(options.environment, {}) }); + } + return run(options.installCommand.executable, [ + ...(options.installCommand.prefixArguments ?? []), + 'install', + host, + '--json', + ], { + cwd: options.installCommand.cwd ?? bundle, + environment: isolatedEnvironment(options.environment, {}), + timeout: 180_000, + }); +}; + const parseJson = (text: string, context: string): T => { try { return JSON.parse(text) as T; @@ -305,9 +343,11 @@ const assertInstallResult = ( * separate `packed-stdio` proof level. */ const buildFixtureProject = async (options: { + readonly buildCommand?: 'build' | 'prepack'; readonly bundleNames: readonly string[]; readonly environment: Readonly; readonly fixture: string; + readonly prepareProject?: (projectRoot: string) => Promise; }): Promise => { const root = await mkdtemp(join(tmpdir(), `agent-bundle-${options.fixture}-build-`)); const project = join(root, 'project'); @@ -315,9 +355,10 @@ const buildFixtureProject = async (options: { try { await cp(join(fixturesRoot, options.fixture), project, { recursive: true }); await symlink(join(workspaceRoot, 'node_modules'), join(project, 'node_modules'), 'dir'); + await options.prepareProject?.(project); const result = await run(process.execPath, [ cli, - 'build', + options.buildCommand ?? 'build', '--root', project, '--output', @@ -337,12 +378,16 @@ const buildFixtureProject = async (options: { }; export const buildHostInstallFixture = async (options: { + readonly buildCommand?: 'build' | 'prepack'; readonly environment: Readonly; + readonly prepareProject?: (projectRoot: string) => Promise; }): Promise => { const built = await buildFixtureProject({ + ...(options.buildCommand === undefined ? {} : { buildCommand: options.buildCommand }), bundleNames: ['claude', 'codex', 'cursor'], environment: options.environment, fixture: 'host-install', + ...(options.prepareProject === undefined ? {} : { prepareProject: options.prepareProject }), }); return Object.freeze({ ...built, @@ -382,7 +427,7 @@ export const disposeHostInstallFixture = async (fixture: BuiltFixtureProject): P export const runClaudeHostInstallProof = async ( fixture: BuiltHostInstallFixture, - options: { readonly environment: Readonly }, + options: HostInstallProofOptions, ): Promise => { const root = await mkdtemp(join(tmpdir(), 'agent-bundle-host-install-claude-')); const config = join(root, 'config'); @@ -393,13 +438,10 @@ export const runClaudeHostInstallProof = async ( CLAUDE_CONFIG_DIR: config, HOME: home, }); - const installed = await runNodeCli(fixture, [ - 'install', - 'claude', - '--from', - fixture.bundles.claude, - '--json', - ], { cwd: fixture.bundles.claude, environment }); + const installed = await runInstallCommand(fixture, 'claude', fixture.bundles.claude, { + ...options, + environment, + }); assertProof(installed.exitCode === 0, `Claude public install path failed: ${commandDetail(installed)}`); const installDocument = parseJson(installed.stdout, 'Claude install'); assertInstallResult(installDocument, 'claude', 'installed'); @@ -461,7 +503,7 @@ export const runClaudeHostInstallProof = async ( export const runCodexHostInstallProof = async ( fixture: BuiltHostInstallFixture, - options: { readonly environment: Readonly }, + options: HostInstallProofOptions, ): Promise => { const root = await mkdtemp(join(tmpdir(), 'agent-bundle-host-install-codex-')); const codexHome = join(root, 'codex'); @@ -472,13 +514,10 @@ export const runCodexHostInstallProof = async ( CODEX_HOME: codexHome, HOME: home, }); - const installed = await runNodeCli(fixture, [ - 'install', - 'codex', - '--from', - fixture.bundles.codex, - '--json', - ], { cwd: fixture.bundles.codex, environment }); + const installed = await runInstallCommand(fixture, 'codex', fixture.bundles.codex, { + ...options, + environment, + }); assertProof(installed.exitCode === 0, `Codex public install path failed: ${commandDetail(installed)}`); const installDocument = parseJson(installed.stdout, 'Codex install'); assertInstallResult(installDocument, 'codex', 'installed'); @@ -616,20 +655,17 @@ const assertCursorPluginRootVariable = (input: { export const runCursorHostInstallProof = async ( fixture: BuiltHostInstallFixture, - options: { readonly environment: Readonly }, + options: HostInstallProofOptions, ): Promise => { const home = await mkdtemp(join(tmpdir(), 'agent-bundle-host-install-cursor-')); try { await mkdir(join(home, '.cursor'), { recursive: true }); const environment = isolatedEnvironment(options.environment, { HOME: home }); const install = async (): Promise => { - const result = await runNodeCli(fixture, [ - 'install', - 'cursor', - '--from', - fixture.bundles.cursor, - '--json', - ], { cwd: fixture.bundles.cursor, environment }); + const result = await runInstallCommand(fixture, 'cursor', fixture.bundles.cursor, { + ...options, + environment, + }); assertProof(result.exitCode === 0, `Cursor public install path failed: ${commandDetail(result)}`); return parseJson(result.stdout, 'Cursor install'); }; diff --git a/rstest.integration-tests.ts b/rstest.integration-tests.ts index 5f7dc24fa..609912661 100644 --- a/rstest.integration-tests.ts +++ b/rstest.integration-tests.ts @@ -110,6 +110,7 @@ export const nightlyEvidenceTestFiles: readonly string[] = [ export const packedTestFiles: readonly string[] = [ 'packages/agent-bundle/tests/dev-workbench-packaging.test.ts', 'packages/agent-bundle/tests/packed-consumer.test.ts', + 'packages/agent-bundle/tests/packed-host-install-proof.test.ts', 'packages/agent-bundle/tests/packed-native-smoke.test.ts', 'packages/agent-bundle/tests/packed-stdio-projection.test.ts', 'packages/agent-bundle/tests/public-api-packed.test.ts',