From e6181262bb18c4a036fe880f5eb524595a3c0377 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Tue, 1 Sep 2026 23:19:09 +0000 Subject: [PATCH 1/2] feat(test): host-install proof level with real Claude, Codex, and Cursor install proofs Binary-gated real-host installs into isolated homes through the public install path, observed via each host's own CLI. Fixes emitted 'plugin marketplace add .' to './' (Claude Code 2.1.257 rejects bare '.'). --- .changeset/host-install-proof-level.md | 13 + .github/workflows/native-host-smoke.yml | 3 + ...-09-01-host-install-distribution-design.md | 6 +- package.json | 2 + packages/agent-bundle/README.md | 5 +- .../adapters/capabilities/claude-2.1.250.json | 5 +- .../adapters/capabilities/codex-0.147.0.json | 5 +- packages/agent-bundle/src/adapters/claude.ts | 2 +- packages/agent-bundle/src/adapters/codex.ts | 2 +- packages/agent-bundle/src/install/surface.ts | 4 +- packages/agent-bundle/src/test/index.ts | 7 +- packages/agent-bundle/src/test/manifest.ts | 12 +- .../tests/adapter-metadata.test.ts | 4 +- .../host-install/agent-bundle.config.ts | 18 + .../tests/fixtures/host-install/package.json | 6 + .../host-install/skills/probe/SKILL.md | 8 + .../host-install/src/hooks/session-start.ts | 4 + .../fixtures/host-install/src/mcp/probe.ts | 6 + .../tests/host-install-proof.test.ts | 124 +++++ .../tests/install-surface.test.ts | 4 +- .../tests/support/host-install.ts | 441 ++++++++++++++++++ .../tests/test-harness-manifest.test.ts | 6 + rstest.integration-tests.ts | 1 + 23 files changed, 667 insertions(+), 21 deletions(-) create mode 100644 .changeset/host-install-proof-level.md create mode 100644 packages/agent-bundle/tests/fixtures/host-install/agent-bundle.config.ts create mode 100644 packages/agent-bundle/tests/fixtures/host-install/package.json create mode 100644 packages/agent-bundle/tests/fixtures/host-install/skills/probe/SKILL.md create mode 100644 packages/agent-bundle/tests/fixtures/host-install/src/hooks/session-start.ts create mode 100644 packages/agent-bundle/tests/fixtures/host-install/src/mcp/probe.ts create mode 100644 packages/agent-bundle/tests/host-install-proof.test.ts create mode 100644 packages/agent-bundle/tests/support/host-install.ts diff --git a/.changeset/host-install-proof-level.md b/.changeset/host-install-proof-level.md new file mode 100644 index 000000000..8158d4cb0 --- /dev/null +++ b/.changeset/host-install-proof-level.md @@ -0,0 +1,13 @@ +--- +"agent-bundle": minor +--- + +Add the `host-install` consumer proof level for real public-path installation +into isolated Claude, Codex, and Cursor homes. The source-built proof fixture +exercises Skills, Hooks, and MCP registration without model calls or packed +artifact claims, validates Cursor's emitted documents against the pinned +schemas, and records only path-relative evidence. + +Generated Claude and Codex installation instructions now use +`plugin marketplace add ./`; Claude Code 2.1.257 rejects the previously emitted +bare `.` source. diff --git a/.github/workflows/native-host-smoke.yml b/.github/workflows/native-host-smoke.yml index 9e5223c18..e6d9cd3a1 100644 --- a/.github/workflows/native-host-smoke.yml +++ b/.github/workflows/native-host-smoke.yml @@ -51,3 +51,6 @@ jobs: - name: ${{ matrix.host }} installed-tarball native smoke if: inputs.host == 'both' || inputs.host == matrix.host run: ${{ matrix.packed_command }} + - name: ${{ matrix.host }} real host install proof + if: inputs.host == 'both' || inputs.host == matrix.host + run: pnpm test:host-install diff --git a/docs/superpowers/specs/2026-09-01-host-install-distribution-design.md b/docs/superpowers/specs/2026-09-01-host-install-distribution-design.md index 7b274f5c6..45a1f1c58 100644 --- a/docs/superpowers/specs/2026-09-01-host-install-distribution-design.md +++ b/docs/superpowers/specs/2026-09-01-host-install-distribution-design.md @@ -17,9 +17,9 @@ which real hosts can consume them. Pinned capability tables record an `install` section beside each host's plugin contract: -- Claude: `claude plugin marketplace add .`, then +- Claude: `claude plugin marketplace add ./`, then `claude plugin install @ --scope `. -- Codex: `codex plugin marketplace add .`, then +- Codex: `codex plugin marketplace add ./`, then `codex plugin add @`. - Cursor: no shell install verb; copy a complete plugin to `~/.cursor/plugins/local/`, then reload Cursor. @@ -34,7 +34,7 @@ against a built directory. ## Emitted surface -Every target root contains `INSTALL.md`. Commands use `.` and the real compiled +Every target root contains `INSTALL.md`. Commands use `./` and the real compiled plugin and marketplace names, so a user runs them from that target root without editing placeholders. diff --git a/package.json b/package.json index 5f74fcaf7..0ef1c1efd 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,8 @@ "test:packed:native": "rstest --config rstest.config.ts packages/agent-bundle/tests/packed-native-smoke.test.ts", "test:packed:native:claude": "pnpm build && AGENT_BUNDLE_PACKED_NATIVE_CLAUDE_SMOKE=1 pnpm test:packed:native", "test:packed:native:codex": "pnpm build && AGENT_BUNDLE_PACKED_NATIVE_CODEX_SMOKE=1 pnpm test:packed:native", + "test:host-install": "rstest --config rstest.config.ts packages/agent-bundle/tests/host-install-proof.test.ts", + "test:host-install:build": "pnpm build && pnpm test:host-install", "changeset": "changeset", "version-packages": "changeset version", "release": "pnpm build && changeset publish", diff --git a/packages/agent-bundle/README.md b/packages/agent-bundle/README.md index feca22ea2..f49d557fa 100644 --- a/packages/agent-bundle/README.md +++ b/packages/agent-bundle/README.md @@ -277,6 +277,7 @@ is never a receipt for another. | `cli-dispatch` | `invokeCli`, `cliJson` | an argv vector resolved and run through the routed CLI's own shell, in-process | | `packed-stdio` | `openPackedMcpServer` | a built artifact's generated entry running as a real process over stdio | | `packed-deleted-source` | `removeProjectSource`, `openPackedMcpServer({ deletedSource })` | the packed stdio process still runs after project source and configuration are removed and verified absent | +| `host-install` | repository real-host install proof | a built bundle installed into an isolated real host home through the public install path, with registration observed through the host's own CLI | ```ts import { cliJson, expectEvents, invokeCli, invokeMcpTool } from 'agent-bundle/test'; @@ -303,7 +304,9 @@ once, build once, remove and verify source once, spawn once, and iterate every per-route assertion inside that one session. The deleted-source journey also reads the embedded MCP App resource from the generated server; it does not prove native-host install or dispatch, or an install mode that copies the -artifact elsewhere. +artifact elsewhere. `host-install` is separate real-host process evidence for +built-bundle acceptance and registration, not packed provenance or session +behavior. ## Evaluation diff --git a/packages/agent-bundle/src/adapters/capabilities/claude-2.1.250.json b/packages/agent-bundle/src/adapters/capabilities/claude-2.1.250.json index 492041a2c..e4f56c250 100644 --- a/packages/agent-bundle/src/adapters/capabilities/claude-2.1.250.json +++ b/packages/agent-bundle/src/adapters/capabilities/claude-2.1.250.json @@ -3,9 +3,10 @@ "install": { "evidence": [ "Local marketplaces accept a directory containing .claude-plugin/marketplace.json.", - "claude plugin install accepts plugin@marketplace and user, project, or local scope." + "claude plugin install accepts plugin@marketplace and user, project, or local scope.", + "2026-09-01: Claude Code 2.1.257 rejects a bare `.` marketplace source but accepts `./` and absolute directory paths." ], - "marketplaceAdd": "claude plugin marketplace add .", + "marketplaceAdd": "claude plugin marketplace add ./", "pluginInstall": "claude plugin install @ --scope ", "scopes": ["user", "project", "local"], "source": "https://code.claude.com/docs/en/discover-plugins", diff --git a/packages/agent-bundle/src/adapters/capabilities/codex-0.147.0.json b/packages/agent-bundle/src/adapters/capabilities/codex-0.147.0.json index 54b98a82e..cef4fe49b 100644 --- a/packages/agent-bundle/src/adapters/capabilities/codex-0.147.0.json +++ b/packages/agent-bundle/src/adapters/capabilities/codex-0.147.0.json @@ -3,9 +3,10 @@ "install": { "evidence": [ "codex plugin marketplace add accepts a local marketplace root.", - "codex plugin add installs plugin@marketplace from a configured snapshot." + "codex plugin add installs plugin@marketplace from a configured snapshot.", + "2026-09-01: Codex 0.147.0 accepts both `.` and `./`; emitted instructions use `./` consistently with the Claude Code 2.1.257 requirement." ], - "marketplaceAdd": "codex plugin marketplace add .", + "marketplaceAdd": "codex plugin marketplace add ./", "pluginInstall": "codex plugin add @", "scopes": ["user"], "source": "https://developers.openai.com/codex/cli/reference", diff --git a/packages/agent-bundle/src/adapters/claude.ts b/packages/agent-bundle/src/adapters/claude.ts index bbb8ef069..c84239d8e 100644 --- a/packages/agent-bundle/src/adapters/claude.ts +++ b/packages/agent-bundle/src/adapters/claude.ts @@ -140,7 +140,7 @@ const hookContract = Object.freeze({ const metadata = Object.freeze({ adapterRevision: '1.5.0', capabilityRevision: capabilityTable.observedCliVersion, - capabilitySha256: '553ebbec4bb16b6e075489fab6c31b11ce84870466710da98e2389659842090e', + capabilitySha256: 'd78b76bda7020f7ea64d332c50d73f7ba3213ef69731835d474383ea6ef46612', observedVersion: capabilityTable.observedCliVersion, schemas: schemaDescriptorsFrom(schemaProvenance, schemaProvenance.observedCliVersion), }); diff --git a/packages/agent-bundle/src/adapters/codex.ts b/packages/agent-bundle/src/adapters/codex.ts index fb4e2580d..6e8f6e745 100644 --- a/packages/agent-bundle/src/adapters/codex.ts +++ b/packages/agent-bundle/src/adapters/codex.ts @@ -122,7 +122,7 @@ const hookContract = Object.freeze({ const metadata = Object.freeze({ adapterRevision: '1.2.0', capabilityRevision: capabilityTable.observedCliVersion, - capabilitySha256: 'd944e508941a0660272a253601019957ae94e9140501f0624f85d111e66d9f28', + capabilitySha256: 'bb0a685d680ffd95c468acfcf2fc20dc8fec672ea718f3dd1934fccad3b726c5', observedVersion: capabilityTable.observedCliVersion, schemas: schemaDescriptorsFrom(schemaProvenance, schemaProvenance.observedCliVersion), }); diff --git a/packages/agent-bundle/src/install/surface.ts b/packages/agent-bundle/src/install/surface.ts index 2060cdcac..89ce954ec 100644 --- a/packages/agent-bundle/src/install/surface.ts +++ b/packages/agent-bundle/src/install/surface.ts @@ -27,7 +27,7 @@ const claudeInstructions = (model: NormalizedPlugin): string[] => [ 'Claude Code installs this bundle through its local marketplace contract:', '', '```sh', - 'claude plugin marketplace add .', + 'claude plugin marketplace add ./', `claude plugin install ${model.metadata.name}@${marketplaceName(model)} --scope user`, '```', '', @@ -41,7 +41,7 @@ const codexInstructions = (model: NormalizedPlugin): string[] => [ 'Codex installs this bundle from its local marketplace snapshot:', '', '```sh', - 'codex plugin marketplace add .', + 'codex plugin marketplace add ./', `codex plugin add ${model.metadata.name}@${marketplaceName(model)}`, '```', '', diff --git a/packages/agent-bundle/src/test/index.ts b/packages/agent-bundle/src/test/index.ts index bd44b1d81..3f72e4c44 100644 --- a/packages/agent-bundle/src/test/index.ts +++ b/packages/agent-bundle/src/test/index.ts @@ -1,8 +1,9 @@ /** * `agent-bundle/test` — the consumer test harness helpers. * - * Five Node proof levels ship here, and the browser-safe sixth level ships - * from `agent-bundle/test/browser`. Each helper names the level it supplies, + * Six Node proof levels ship here, and the browser-safe seventh level ships + * from `agent-bundle/test/browser`. The repository's real-host install proof + * uses the same level convention. Each helper names the level it supplies, * stamps it into its provenance, and prints it in every failure: * * | level | helper | what it proves | @@ -13,6 +14,7 @@ * | `packed-stdio` | `openPackedMcpServer` | a built artifact's generated entry running as a real process over stdio | * | `packed-deleted-source` | `removeProjectSource`, `openPackedMcpServer` | the packed stdio process still runs after project source and configuration are removed and verified absent | * | `browser-app` | `mountBrowserApp` (`agent-bundle/test/browser`) | production-compiled MCP App HTML mounted over the product bridge in a real browser page | + * | `host-install` | repository real-host install proof | a built bundle accepted through a real host's public install path in an isolated home, with registration observed by that host | * * A pass at one level is never a receipt for another. The `deletedSource` * option upgrades `openPackedMcpServer` provenance only after every path in a @@ -21,6 +23,7 @@ export { BROWSER_APP_PROOF_LEVEL, CLI_DISPATCH_PROOF_LEVEL, + HOST_INSTALL_PROOF_LEVEL, MCP_IN_MEMORY_PROOF_LEVEL, PACKED_DELETED_SOURCE_PROOF_LEVEL, PACKED_STDIO_PROOF_LEVEL, diff --git a/packages/agent-bundle/src/test/manifest.ts b/packages/agent-bundle/src/test/manifest.ts index d27464779..940a577bc 100644 --- a/packages/agent-bundle/src/test/manifest.ts +++ b/packages/agent-bundle/src/test/manifest.ts @@ -28,15 +28,17 @@ import type { * resolution, argv projection, and exit codes, not a spawned binary. * - `packed-stdio` installs the packed release tarball into a clean consumer, * spawns the generated stdio entry as a real process, and drives it with a - * real MCP client. This is the only level here that is process evidence. + * real MCP client. This is the packed process-and-protocol evidence level. * - `packed-deleted-source` carries the `packed-stdio` proof after project * source and configuration have been removed and verified absent. It proves * that the generated entry is self-contained; it does not prove native-host * install or dispatch, or an install mode that copies the artifact elsewhere. - * * - `browser-app` compiles MCP App HTML through the production Rsbuild * profile and mounts it over the product bridge in a real browser page. It * does not prove host embedding, a packed artifact, or Workbench behavior. + * - `host-install` installs a built bundle into an isolated real host home + * through the public install path and observes registration through the + * host's own CLI. It does not prove session behavior or packed provenance. */ export type AgentTestProofLevel = | 'route-unit' @@ -44,7 +46,8 @@ export type AgentTestProofLevel = | 'cli-dispatch' | 'packed-stdio' | 'packed-deleted-source' - | 'browser-app'; + | 'browser-app' + | 'host-install'; export const ROUTE_UNIT_PROOF_LEVEL = 'route-unit' as const; export const MCP_IN_MEMORY_PROOF_LEVEL = 'mcp-in-memory' as const; @@ -52,6 +55,7 @@ export const CLI_DISPATCH_PROOF_LEVEL = 'cli-dispatch' as const; export const PACKED_STDIO_PROOF_LEVEL = 'packed-stdio' as const; export const PACKED_DELETED_SOURCE_PROOF_LEVEL = 'packed-deleted-source' as const; export const BROWSER_APP_PROOF_LEVEL = 'browser-app' as const; +export const HOST_INSTALL_PROOF_LEVEL = 'host-install' as const; /** * One line per level, printed in every harness failure. A red test has to @@ -72,6 +76,8 @@ export const proofLevelLabel = (level: AgentTestProofLevel): string => { return 'packed-deleted-source (packed tarball installed into a clean consumer, artifact built, project source removed and verified absent, generated stdio entry spawned as a real process; self-contained-artifact evidence)'; case 'browser-app': return 'browser-app (MCP App HTML compiled through the production Rsbuild profile, mounted in a real browser page over the product bridge; NOT host embedding, packed-artifact, or Workbench evidence)'; + case 'host-install': + return 'host-install (built bundle installed into an isolated real host home through the public install path, registration observed via the host\'s own CLI; NOT session-behavior or packed-artifact evidence)'; default: { const exhaustive: never = level; throw new TypeError(`Unknown proof level ${String(exhaustive)}.`); diff --git a/packages/agent-bundle/tests/adapter-metadata.test.ts b/packages/agent-bundle/tests/adapter-metadata.test.ts index 110ebc89b..4e84d35f0 100644 --- a/packages/agent-bundle/tests/adapter-metadata.test.ts +++ b/packages/agent-bundle/tests/adapter-metadata.test.ts @@ -75,7 +75,7 @@ it('records exact immutable metadata for every built-in target', () => { expect(registryMetadata(registry, 'codex')).toEqual({ adapterRevision: '1.2.0', capabilityRevision: '0.147.0', - capabilitySha256: 'd944e508941a0660272a253601019957ae94e9140501f0624f85d111e66d9f28', + capabilitySha256: 'bb0a685d680ffd95c468acfcf2fc20dc8fec672ea718f3dd1934fccad3b726c5', observedVersion: '0.147.0', schemas: [ { @@ -103,7 +103,7 @@ it('records exact immutable metadata for every built-in target', () => { expect(registryMetadata(registry, 'claude')).toEqual({ adapterRevision: '1.5.0', capabilityRevision: '2.1.250', - capabilitySha256: '553ebbec4bb16b6e075489fab6c31b11ce84870466710da98e2389659842090e', + capabilitySha256: 'd78b76bda7020f7ea64d332c50d73f7ba3213ef69731835d474383ea6ef46612', observedVersion: '2.1.250', schemas: [ { diff --git a/packages/agent-bundle/tests/fixtures/host-install/agent-bundle.config.ts b/packages/agent-bundle/tests/fixtures/host-install/agent-bundle.config.ts new file mode 100644 index 000000000..d7534ae49 --- /dev/null +++ b/packages/agent-bundle/tests/fixtures/host-install/agent-bundle.config.ts @@ -0,0 +1,18 @@ +export default { + hooks: { + sessionStart: { handler: './src/hooks/session-start.ts' }, + }, + marketplace: true, + mcp: { + servers: { + probe: {}, + }, + }, + plugin: { + description: 'Proves real host installation of Skills, Hooks, and MCP metadata.', + name: 'host-install-proof', + version: '1.0.0', + }, + skills: ['skills/probe'], + targets: ['claude', 'codex', 'cursor'], +}; diff --git a/packages/agent-bundle/tests/fixtures/host-install/package.json b/packages/agent-bundle/tests/fixtures/host-install/package.json new file mode 100644 index 000000000..e64eb0493 --- /dev/null +++ b/packages/agent-bundle/tests/fixtures/host-install/package.json @@ -0,0 +1,6 @@ +{ + "name": "host-install-proof-fixture", + "private": true, + "type": "module", + "version": "1.0.0" +} diff --git a/packages/agent-bundle/tests/fixtures/host-install/skills/probe/SKILL.md b/packages/agent-bundle/tests/fixtures/host-install/skills/probe/SKILL.md new file mode 100644 index 000000000..4fbf8891d --- /dev/null +++ b/packages/agent-bundle/tests/fixtures/host-install/skills/probe/SKILL.md @@ -0,0 +1,8 @@ +--- +name: probe +description: Confirm that the host-install proof fixture was discovered. +--- + +# Probe + +Report that the host-install proof fixture is available. diff --git a/packages/agent-bundle/tests/fixtures/host-install/src/hooks/session-start.ts b/packages/agent-bundle/tests/fixtures/host-install/src/hooks/session-start.ts new file mode 100644 index 000000000..208bcaba7 --- /dev/null +++ b/packages/agent-bundle/tests/fixtures/host-install/src/hooks/session-start.ts @@ -0,0 +1,4 @@ +export default () => ({ + additionalContext: 'The host-install proof fixture is available.', + outcome: 'continue' as const, +}); diff --git a/packages/agent-bundle/tests/fixtures/host-install/src/mcp/probe.ts b/packages/agent-bundle/tests/fixtures/host-install/src/mcp/probe.ts new file mode 100644 index 000000000..62853ae21 --- /dev/null +++ b/packages/agent-bundle/tests/fixtures/host-install/src/mcp/probe.ts @@ -0,0 +1,6 @@ +import { McpServer } from '@modelcontextprotocol/server'; + +export default () => new McpServer({ + name: 'host-install-proof', + version: '1.0.0', +}); diff --git a/packages/agent-bundle/tests/host-install-proof.test.ts b/packages/agent-bundle/tests/host-install-proof.test.ts new file mode 100644 index 000000000..51c090d19 --- /dev/null +++ b/packages/agent-bundle/tests/host-install-proof.test.ts @@ -0,0 +1,124 @@ +import { spawnSync } from 'node:child_process'; + +import { afterAll, beforeAll, expect, it } from '@rstest/core'; + +import { + buildHostInstallFixture, + disposeHostInstallFixture, + runClaudeHostInstallProof, + runCodexHostInstallProof, + runCursorHostInstallProof, + type BuiltHostInstallFixture, +} from './support/host-install.ts'; +import { + HOST_INSTALL_PROOF_LEVEL, + proofLevelLabel, +} from '../src/test/manifest.ts'; + +const proofLabel = proofLevelLabel(HOST_INSTALL_PROOF_LEVEL); +const claudeMissingEvidence = 'missing evidence: claude binary unavailable on PATH'; +const codexMissingEvidence = 'missing evidence: codex binary unavailable on PATH'; +const claudeAvailable = spawnSync('claude', ['--version'], { + stdio: 'ignore', + timeout: 5_000, + windowsHide: true, +}).status === 0; +const codexAvailable = spawnSync('codex', ['--version'], { + stdio: 'ignore', + timeout: 5_000, + windowsHide: true, +}).status === 0; +const claudePluginIt = claudeAvailable ? it : it.skip; +const codexPluginIt = codexAvailable ? it : it.skip; + +let fixture: BuiltHostInstallFixture | undefined; + +beforeAll(async () => { + fixture = await buildHostInstallFixture({ environment: process.env }); +}, 180_000); + +afterAll(async () => { + if (fixture !== undefined) await disposeHostInstallFixture(fixture); +}); + +const builtFixture = (): BuiltHostInstallFixture => { + if (fixture === undefined) throw new Error(`[${proofLabel}] shared fixture build did not complete.`); + return fixture; +}; + +const expectHygienicReport = (report: unknown): void => { + expect(JSON.stringify(report), proofLabel).not.toMatch( + /(?:API_KEY|AUTH_TOKEN|ACCESS_TOKEN|authorization|credential|password|secret|sk-[A-Za-z0-9_-]{16,}|\/home\/|\/Users\/|\/tmp\/|stdout|stderr)/iu, + ); +}; + +claudePluginIt( + claudeAvailable + ? 'installs through Claude and observes the host-owned component inventory' + : `installs through Claude and observes the host-owned component inventory [${claudeMissingEvidence}]`, + async () => { + const report = await runClaudeHostInstallProof(builtFixture(), { environment: process.env }); + + expect(report, proofLabel).toEqual({ + host: 'claude', + install: { state: 'installed', version: '1.0.0' }, + inventory: { hooks: 1, mcpServers: 1, skills: 1 }, + proofLevel: proofLabel, + registration: { + enabled: true, + id: 'host-install-proof@host-install-proof-marketplace', + installPath: 'plugins/cache/host-install-proof-marketplace/host-install-proof/1.0.0', + mcpServers: ['probe'], + scope: 'user', + version: '1.0.0', + }, + skill: 'plugins/cache/host-install-proof-marketplace/host-install-proof/1.0.0/skills/probe/SKILL.md', + status: 'passed', + }); + expectHygienicReport(report); + }, + 180_000, +); + +codexPluginIt( + codexAvailable + ? 'installs through Codex and observes enabled registration' + : `installs through Codex and observes enabled registration [${codexMissingEvidence}]`, + async () => { + const report = await runCodexHostInstallProof(builtFixture(), { environment: process.env }); + + expect(report, proofLabel).toEqual({ + host: 'codex', + install: { state: 'installed', version: '1.0.0' }, + proofLevel: proofLabel, + registration: { + cachePath: 'plugins/cache/host-install-proof-marketplace/host-install-proof/1.0.0', + state: 'installed, enabled', + version: '1.0.0', + }, + skill: 'plugins/cache/host-install-proof-marketplace/host-install-proof/1.0.0/skills/probe/SKILL.md', + status: 'passed', + }); + expectHygienicReport(report); + }, + 180_000, +); + +it('installs into an isolated Cursor home, validates schemas, and is idempotent', async () => { + const report = await runCursorHostInstallProof(builtFixture(), { environment: process.env }); + + expect(report, proofLabel).toEqual({ + destination: '.cursor/plugins/local/host-install-proof', + documents: { + hooks: 'schema-valid', + mcp: 'schema-valid', + plugin: 'schema-valid', + }, + host: 'cursor', + install: { first: 'installed', second: 'already-installed', version: '1.0.0' }, + proofLevel: proofLabel, + skill: '.cursor/plugins/local/host-install-proof/skills/probe/SKILL.md', + status: 'passed', + }); + expectHygienicReport(report); +}, 180_000); diff --git a/packages/agent-bundle/tests/install-surface.test.ts b/packages/agent-bundle/tests/install-surface.test.ts index d5b4251a2..1a71a241b 100644 --- a/packages/agent-bundle/tests/install-surface.test.ts +++ b/packages/agent-bundle/tests/install-surface.test.ts @@ -53,7 +53,7 @@ it('emits always-installable Claude and Codex local marketplaces with exact comm name: 'install-fixture-marketplace', plugins: [{ name: 'install-fixture', source: './', version: '1.2.3' }], }); - expect(claude.get('INSTALL.md')).toContain('claude plugin marketplace add .'); + expect(claude.get('INSTALL.md')).toContain('claude plugin marketplace add ./'); expect(claude.get('INSTALL.md')).toContain( 'claude plugin install install-fixture@install-fixture-marketplace --scope user', ); @@ -62,7 +62,7 @@ it('emits always-installable Claude and Codex local marketplaces with exact comm name: 'install-fixture-marketplace', plugins: [{ name: 'install-fixture', source: { path: './', source: 'local' } }], }); - expect(codex.get('INSTALL.md')).toContain('codex plugin marketplace add .'); + expect(codex.get('INSTALL.md')).toContain('codex plugin marketplace add ./'); expect(codex.get('INSTALL.md')).toContain( 'codex plugin add install-fixture@install-fixture-marketplace', ); diff --git a/packages/agent-bundle/tests/support/host-install.ts b/packages/agent-bundle/tests/support/host-install.ts new file mode 100644 index 000000000..53915e5f6 --- /dev/null +++ b/packages/agent-bundle/tests/support/host-install.ts @@ -0,0 +1,441 @@ +import { execFile as executeFile } from 'node:child_process'; +import { access, cp, mkdir, mkdtemp, readFile, rm, symlink } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, relative, sep } from 'node:path'; +import { promisify } from 'node:util'; + +import { + cursorHooksValidator, + cursorMcpValidator, + cursorPluginValidator, +} from '../../src/adapters/cursor.ts'; +import { + HOST_INSTALL_PROOF_LEVEL, + proofLevelLabel, +} from '../../src/test/manifest.ts'; +import { packedNativeEnvironment } from './packed-native-smoke.ts'; + +const execFile = promisify(executeFile); +const workspaceRoot = process.cwd(); +const packageRoot = join(workspaceRoot, 'packages', 'agent-bundle'); +const fixtureRoot = join(packageRoot, 'tests', 'fixtures', 'host-install'); +const cli = join(packageRoot, 'dist', 'cli.js'); +const plugin = 'host-install-proof'; +const marketplace = 'host-install-proof-marketplace'; +const version = '1.0.0'; +const proofLevel = proofLevelLabel(HOST_INSTALL_PROOF_LEVEL); + +interface CommandResult { + readonly exitCode: number; + readonly stderr: string; + readonly stdout: string; +} + +interface InstallResult { + readonly bundleRoot?: unknown; + readonly destination?: unknown; + readonly host?: unknown; + readonly marketplace?: unknown; + readonly plugin?: unknown; + readonly state?: unknown; + readonly version?: unknown; +} + +interface ClaudePluginRow { + readonly enabled?: unknown; + readonly id?: unknown; + readonly installPath?: unknown; + readonly mcpServers?: unknown; + readonly scope?: unknown; + readonly version?: unknown; +} + +export interface BuiltHostInstallFixture { + readonly artifactRoot: string; + readonly bundles: Readonly>; + readonly cli: string; + readonly root: string; +} + +export interface ClaudeHostInstallReport { + readonly host: 'claude'; + readonly install: { readonly state: 'installed'; readonly version: '1.0.0' }; + readonly inventory: { readonly hooks: 1; readonly mcpServers: 1; readonly skills: 1 }; + readonly proofLevel: string; + readonly registration: { + readonly enabled: true; + readonly id: string; + readonly installPath: string; + readonly mcpServers: readonly string[]; + readonly scope: 'user'; + readonly version: '1.0.0'; + }; + readonly skill: string; + readonly status: 'passed'; +} + +export interface CodexHostInstallReport { + readonly host: 'codex'; + readonly install: { readonly state: 'installed'; readonly version: '1.0.0' }; + readonly proofLevel: string; + readonly registration: { + readonly cachePath: string; + readonly state: 'installed, enabled'; + readonly version: '1.0.0'; + }; + readonly skill: string; + readonly status: 'passed'; +} + +export interface CursorHostInstallReport { + readonly destination: string; + readonly documents: { + readonly hooks: 'schema-valid'; + readonly mcp: 'schema-valid'; + readonly plugin: 'schema-valid'; + }; + readonly host: 'cursor'; + readonly install: { + readonly first: 'installed'; + readonly second: 'already-installed'; + readonly version: '1.0.0'; + }; + readonly proofLevel: string; + readonly skill: string; + readonly status: 'passed'; +} + +const fail = (message: string): never => { + throw new Error(`[${proofLevel}] ${message}`); +}; + +const assertProof: (condition: unknown, message: string) => asserts condition = (condition, message) => { + if (!condition) fail(message); +}; + +const commandDetail = (result: CommandResult): string => + result.stderr.trim() || result.stdout.trim() || `exit code ${String(result.exitCode)}`; + +const run = async ( + executable: string, + args: readonly string[], + options: { + readonly cwd: string; + readonly environment: NodeJS.ProcessEnv; + readonly timeout?: number; + }, +): Promise => { + try { + const result = await execFile(executable, [...args], { + cwd: options.cwd, + encoding: 'utf8', + env: options.environment, + killSignal: 'SIGTERM', + maxBuffer: 8 * 1024 * 1024, + timeout: options.timeout ?? 120_000, + }); + return Object.freeze({ exitCode: 0, stderr: result.stderr, stdout: result.stdout }); + } catch (error) { + const failed = error as { + readonly code?: number | string; + readonly stderr?: string; + readonly stdout?: string; + }; + return Object.freeze({ + exitCode: typeof failed.code === 'number' ? failed.code : 1, + stderr: typeof failed.stderr === 'string' ? failed.stderr : '', + stdout: typeof failed.stdout === 'string' ? failed.stdout : '', + }); + } +}; + +const runNodeCli = ( + fixture: BuiltHostInstallFixture, + args: readonly string[], + options: { readonly cwd: string; readonly environment: NodeJS.ProcessEnv }, +): Promise => run( + process.execPath, + [fixture.cli, ...args], + { ...options, timeout: 180_000 }, +); + +const parseJson = (text: string, context: string): T => { + try { + return JSON.parse(text) as T; + } catch { + return fail(`${context} did not return JSON.`); + } +}; + +const readJson = async (path: string, context: string): Promise => { + try { + return JSON.parse(await readFile(path, 'utf8')) as unknown; + } catch { + return fail(`${context} was not readable JSON.`); + } +}; + +const normalizedRelative = (root: string, path: string): string => + relative(root, path).split(sep).join('/'); + +const isolatedEnvironment = ( + environment: Readonly, + values: Readonly, +): NodeJS.ProcessEnv => ({ + ...packedNativeEnvironment(environment), + ...values, +}); + +const assertInstallResult = ( + document: InstallResult, + host: 'claude' | 'codex' | 'cursor', + state: 'already-installed' | 'installed', +): void => { + assertProof(document.host === host, `${host} install result did not identify the host.`); + assertProof(document.plugin === plugin, `${host} install result did not identify ${plugin}.`); + assertProof(document.version === version, `${host} install result did not identify version ${version}.`); + assertProof(document.state === state, `${host} install result state was not ${state}.`); + if (host !== 'cursor') { + assertProof(document.marketplace === marketplace, `${host} install result did not identify ${marketplace}.`); + } +}; + +/** + * Builds the fixture once with the workspace's source-built CLI. This proves + * real-host acceptance of that built bundle; packed provenance remains the + * separate `packed-stdio` proof level. + */ +export const buildHostInstallFixture = async (options: { + readonly environment: Readonly; +}): Promise => { + const root = await mkdtemp(join(tmpdir(), 'agent-bundle-host-install-build-')); + const project = join(root, 'project'); + const artifactRoot = join(project, 'artifact'); + const fixture: BuiltHostInstallFixture = Object.freeze({ + artifactRoot, + bundles: Object.freeze({ + claude: join(artifactRoot, 'claude'), + codex: join(artifactRoot, 'codex'), + cursor: join(artifactRoot, 'cursor'), + }), + cli, + root, + }); + try { + await cp(fixtureRoot, project, { recursive: true }); + await symlink(join(workspaceRoot, 'node_modules'), join(project, 'node_modules'), 'dir'); + const result = await run(process.execPath, [ + cli, + 'build', + '--root', + project, + '--output', + artifactRoot, + ], { + cwd: project, + environment: packedNativeEnvironment(options.environment), + timeout: 180_000, + }); + assertProof(result.exitCode === 0, `fixture build failed: ${commandDetail(result)}`); + await Promise.all(Object.values(fixture.bundles).map((bundle) => access(bundle))); + return fixture; + } catch (error) { + await rm(root, { force: true, recursive: true }); + throw error; + } +}; + +export const disposeHostInstallFixture = async (fixture: BuiltHostInstallFixture): Promise => { + await rm(fixture.root, { force: true, recursive: true }); +}; + +export const runClaudeHostInstallProof = async ( + fixture: BuiltHostInstallFixture, + options: { readonly environment: Readonly }, +): Promise => { + const root = await mkdtemp(join(tmpdir(), 'agent-bundle-host-install-claude-')); + const config = join(root, 'config'); + const home = join(root, 'home'); + try { + await Promise.all([mkdir(config, { recursive: true }), mkdir(home, { recursive: true })]); + const environment = isolatedEnvironment(options.environment, { + CLAUDE_CONFIG_DIR: config, + HOME: home, + }); + const installed = await runNodeCli(fixture, [ + 'install', + 'claude', + '--from', + fixture.bundles.claude, + '--json', + ], { cwd: fixture.bundles.claude, environment }); + assertProof(installed.exitCode === 0, `Claude public install path failed: ${commandDetail(installed)}`); + const installDocument = parseJson(installed.stdout, 'Claude install'); + assertInstallResult(installDocument, 'claude', 'installed'); + + const listed = await run('claude', ['plugin', 'list', '--json'], { + cwd: fixture.bundles.claude, + environment, + }); + assertProof(listed.exitCode === 0, `Claude plugin list failed: ${commandDetail(listed)}`); + const listing = parseJson(listed.stdout, 'Claude plugin list'); + assertProof(Array.isArray(listing), 'Claude plugin list was not a JSON array.'); + const id = `${plugin}@${marketplace}`; + const row = listing.find((candidate) => candidate.id === id); + assertProof(row !== undefined, `Claude plugin list did not contain ${id}.`); + assertProof(row.version === version, `Claude registered version was not ${version}.`); + assertProof(row.scope === 'user', 'Claude registered scope was not user.'); + assertProof(row.enabled === true, 'Claude registration was not enabled.'); + assertProof(typeof row.installPath === 'string', 'Claude registration had no installPath.'); + const expectedInstallPath = join(config, 'plugins', 'cache', marketplace, plugin, version); + assertProof(row.installPath === expectedInstallPath, 'Claude installPath was outside the isolated config cache.'); + assertProof( + row.mcpServers !== null && typeof row.mcpServers === 'object' && !Array.isArray(row.mcpServers), + 'Claude registration had no mcpServers object.', + ); + const mcpServers = Object.keys(row.mcpServers as Record).sort(); + assertProof(mcpServers.includes('probe'), 'Claude registration did not expose the probe MCP server.'); + + const details = await run('claude', ['plugin', 'details', id], { + cwd: fixture.bundles.claude, + environment, + }); + assertProof(details.exitCode === 0, `Claude plugin details failed: ${commandDetail(details)}`); + assertProof(/\bSkills\s+\(1\)/iu.test(details.stdout), 'Claude component inventory did not report Skills (1).'); + assertProof(/\bHooks\s+\(1\)/iu.test(details.stdout), 'Claude component inventory did not report Hooks (1).'); + assertProof(/\bMCP servers\s+\(1\)/iu.test(details.stdout), 'Claude component inventory did not report MCP servers (1).'); + + const skillPath = join(expectedInstallPath, 'skills', 'probe', 'SKILL.md'); + await access(skillPath).catch(() => fail('Claude cache did not contain skills/probe/SKILL.md.')); + return Object.freeze({ + host: 'claude', + install: Object.freeze({ state: 'installed', version }), + inventory: Object.freeze({ hooks: 1, mcpServers: 1, skills: 1 }), + proofLevel, + registration: Object.freeze({ + enabled: true, + id, + installPath: normalizedRelative(config, expectedInstallPath), + mcpServers: Object.freeze(mcpServers), + scope: 'user', + version, + }), + skill: normalizedRelative(config, skillPath), + status: 'passed', + }); + } finally { + await rm(root, { force: true, recursive: true }); + } +}; + +export const runCodexHostInstallProof = async ( + fixture: BuiltHostInstallFixture, + options: { readonly environment: Readonly }, +): Promise => { + const root = await mkdtemp(join(tmpdir(), 'agent-bundle-host-install-codex-')); + const codexHome = join(root, 'codex'); + const home = join(root, 'home'); + try { + await Promise.all([mkdir(codexHome, { recursive: true }), mkdir(home, { recursive: true })]); + const environment = isolatedEnvironment(options.environment, { + CODEX_HOME: codexHome, + HOME: home, + }); + const installed = await runNodeCli(fixture, [ + 'install', + 'codex', + '--from', + fixture.bundles.codex, + '--json', + ], { cwd: fixture.bundles.codex, environment }); + assertProof(installed.exitCode === 0, `Codex public install path failed: ${commandDetail(installed)}`); + const installDocument = parseJson(installed.stdout, 'Codex install'); + assertInstallResult(installDocument, 'codex', 'installed'); + + const listed = await run('codex', ['plugin', 'list'], { + cwd: fixture.bundles.codex, + environment, + }); + assertProof(listed.exitCode === 0, `Codex plugin list failed: ${commandDetail(listed)}`); + assertProof(listed.stdout.includes(`${plugin}@${marketplace}`), 'Codex plugin list did not identify the plugin.'); + assertProof(listed.stdout.includes(version), `Codex plugin list did not report version ${version}.`); + assertProof(/installed,\s*enabled/iu.test(listed.stdout), 'Codex plugin list did not report installed, enabled.'); + + const cachePath = join(codexHome, 'plugins', 'cache', marketplace, plugin, version); + const skillPath = join(cachePath, 'skills', 'probe', 'SKILL.md'); + await access(skillPath).catch(() => fail('Codex cache did not contain skills/probe/SKILL.md.')); + return Object.freeze({ + host: 'codex', + install: Object.freeze({ state: 'installed', version }), + proofLevel, + registration: Object.freeze({ + cachePath: normalizedRelative(codexHome, cachePath), + state: 'installed, enabled', + version, + }), + skill: normalizedRelative(codexHome, skillPath), + status: 'passed', + }); + } finally { + await rm(root, { force: true, recursive: true }); + } +}; + +export const runCursorHostInstallProof = async ( + fixture: BuiltHostInstallFixture, + options: { readonly environment: Readonly }, +): Promise => { + const home = await mkdtemp(join(tmpdir(), 'agent-bundle-host-install-cursor-')); + try { + await mkdir(join(home, '.cursor'), { recursive: true }); + const environment = isolatedEnvironment(options.environment, { HOME: home }); + const install = async (): Promise => { + const result = await runNodeCli(fixture, [ + 'install', + 'cursor', + '--from', + fixture.bundles.cursor, + '--json', + ], { cwd: fixture.bundles.cursor, environment }); + assertProof(result.exitCode === 0, `Cursor public install path failed: ${commandDetail(result)}`); + return parseJson(result.stdout, 'Cursor install'); + }; + const first = await install(); + assertInstallResult(first, 'cursor', 'installed'); + const destination = join(home, '.cursor', 'plugins', 'local', plugin); + assertProof(first.destination === destination, 'Cursor install result did not report the isolated destination.'); + + const pluginDocument = await readJson(join(destination, '.cursor-plugin', 'plugin.json'), 'Cursor plugin manifest'); + assertProof(cursorPluginValidator(pluginDocument), `Cursor plugin manifest failed its pinned schema: ${JSON.stringify(cursorPluginValidator.errors)}`); + const hooksDocument = await readJson(join(destination, 'hooks', 'hooks.json'), 'Cursor hooks document'); + assertProof(cursorHooksValidator(hooksDocument), `Cursor hooks document failed its pinned schema: ${JSON.stringify(cursorHooksValidator.errors)}`); + const mcpDocument = await readJson(join(destination, 'mcp.json'), 'Cursor MCP document'); + assertProof(cursorMcpValidator(mcpDocument), `Cursor MCP document failed its pinned schema: ${JSON.stringify(cursorMcpValidator.errors)}`); + + const skillPath = join(destination, 'skills', 'probe', 'SKILL.md'); + await access(skillPath).catch(() => fail('Cursor install did not contain skills/probe/SKILL.md.')); + const second = await install(); + assertInstallResult(second, 'cursor', 'already-installed'); + assertProof(second.destination === destination, 'Cursor idempotent install reported a different destination.'); + + return Object.freeze({ + destination: normalizedRelative(home, destination), + documents: Object.freeze({ + hooks: 'schema-valid', + mcp: 'schema-valid', + plugin: 'schema-valid', + }), + host: 'cursor', + install: Object.freeze({ + first: 'installed', + second: 'already-installed', + version, + }), + proofLevel, + skill: normalizedRelative(home, skillPath), + status: 'passed', + }); + } finally { + await rm(home, { force: true, recursive: true }); + } +}; diff --git a/packages/agent-bundle/tests/test-harness-manifest.test.ts b/packages/agent-bundle/tests/test-harness-manifest.test.ts index 38a9dc359..c4d9cf16d 100644 --- a/packages/agent-bundle/tests/test-harness-manifest.test.ts +++ b/packages/agent-bundle/tests/test-harness-manifest.test.ts @@ -46,6 +46,12 @@ describe('the compiled test manifest', () => { ); }); + it('names host-install as isolated real-host registration without overstating session or package proof', () => { + expect(proofLevelLabel('host-install')).toBe( + 'host-install (built bundle installed into an isolated real host home through the public install path, registration observed via the host\'s own CLI; NOT session-behavior or packed-artifact evidence)', + ); + }); + it('names browser-app as compiled browser evidence without overstating host or artifact proof', () => { expect(proofLevelLabel('browser-app')).toBe( 'browser-app (MCP App HTML compiled through the production Rsbuild profile, mounted in a real browser page over the product bridge; NOT host embedding, packed-artifact, or Workbench evidence)', diff --git a/rstest.integration-tests.ts b/rstest.integration-tests.ts index 9b5efb9b2..15b85cb48 100644 --- a/rstest.integration-tests.ts +++ b/rstest.integration-tests.ts @@ -35,6 +35,7 @@ export const integrationTestFiles: readonly string[] = [ 'packages/agent-bundle/tests/hooks.test.ts', 'packages/agent-bundle/tests/host-adapters.native.test.ts', 'packages/agent-bundle/tests/host-adapters.test.ts', + 'packages/agent-bundle/tests/host-install-proof.test.ts', 'packages/agent-bundle/tests/integration-matrix.test.ts', 'packages/agent-bundle/tests/mcp-session-service.test.ts', 'packages/agent-bundle/tests/mcp.test.ts', From 189f1297d372719ac804690f2f55ce65aabcec0d Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Tue, 1 Sep 2026 23:41:41 +0000 Subject: [PATCH 2/2] feat(test): #108 real-host token proofs riding the host-install level Claude: opt-in real-session proof that $ARGUMENTS, ${CLAUDE_PLUGIN_ROOT}, and ${CLAUDE_SKILL_DIR} resolve in one claude -p turn with the bundle loaded via --plugin-dir (observed 2.1.257; a real turn rewrites .claude.json, so the guard pins settings and installed plugins). Codex: installed cache carries a byte-identical schema-valid agents/openai.yaml sidecar and the manifest interface block. Cursor: installed hooks/mcp documents keep ${CURSOR_PLUGIN_ROOT} unresolved; no non-interactive session surface exists, recorded as the honest ceiling. --- .changeset/host-install-proof-level.md | 12 + .github/workflows/native-host-smoke.yml | 7 + package.json | 2 + .../agent-bundle.config.ts | 12 + .../fixtures/host-install-tokens/package.json | 6 + .../skills/token-probe/SKILL.md | 14 + .../host-install/skills/probe/SKILL.md | 11 + .../tests/host-install-proof.test.ts | 29 ++ .../tests/host-install-session.test.ts | 100 ++++ .../tests/support/host-install.ts | 457 ++++++++++++++++-- .../tests/support/packed-native-smoke.ts | 42 +- rstest.integration-tests.ts | 1 + 12 files changed, 662 insertions(+), 31 deletions(-) create mode 100644 packages/agent-bundle/tests/fixtures/host-install-tokens/agent-bundle.config.ts create mode 100644 packages/agent-bundle/tests/fixtures/host-install-tokens/package.json create mode 100644 packages/agent-bundle/tests/fixtures/host-install-tokens/skills/token-probe/SKILL.md create mode 100644 packages/agent-bundle/tests/host-install-session.test.ts diff --git a/.changeset/host-install-proof-level.md b/.changeset/host-install-proof-level.md index 8158d4cb0..d85da747d 100644 --- a/.changeset/host-install-proof-level.md +++ b/.changeset/host-install-proof-level.md @@ -8,6 +8,18 @@ exercises Skills, Hooks, and MCP registration without model calls or packed artifact claims, validates Cursor's emitted documents against the pinned schemas, and records only path-relative evidence. +The level now also carries the real-host token proofs deferred from the +canonical Skill IR work. The Codex proof asserts the installed cache copy of a +skill's `agents/openai.yaml` sidecar is byte-identical to the built artifact and +valid against the pinned schema, and that the installed `.codex-plugin/plugin.json` +carries its `interface` block. The Cursor proof asserts the installed hooks and +MCP documents keep `${CURSOR_PLUGIN_ROOT}` unresolved, which is the honest +ceiling because Cursor publishes no non-interactive plugin-loading session +surface. An opt-in session-token qualifier +(`AGENT_BUNDLE_HOST_INSTALL_CLAUDE_SESSION=1`) observes `$ARGUMENTS`, +`${CLAUDE_PLUGIN_ROOT}`, and `${CLAUDE_SKILL_DIR}` resolving inside one real +`claude -p` turn with the built bundle loaded inline via `--plugin-dir`. + Generated Claude and Codex installation instructions now use `plugin marketplace add ./`; Claude Code 2.1.257 rejects the previously emitted bare `.` source. diff --git a/.github/workflows/native-host-smoke.yml b/.github/workflows/native-host-smoke.yml index e6d9cd3a1..24161c793 100644 --- a/.github/workflows/native-host-smoke.yml +++ b/.github/workflows/native-host-smoke.yml @@ -54,3 +54,10 @@ jobs: - name: ${{ matrix.host }} real host install proof if: inputs.host == 'both' || inputs.host == matrix.host run: pnpm test:host-install + # Claude only: the session-token proof needs a signed-in `claude -p` turn, + # and Codex and Cursor have no equivalent non-interactive surface. + - name: ${{ matrix.host }} real session token proof + if: matrix.host == 'claude' && (inputs.host == 'both' || inputs.host == matrix.host) + env: + AGENT_BUNDLE_HOST_INSTALL_CLAUDE_SESSION: '1' + run: pnpm test:host-install:session diff --git a/package.json b/package.json index 0ef1c1efd..064e86a7e 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,8 @@ "test:packed:native:codex": "pnpm build && AGENT_BUNDLE_PACKED_NATIVE_CODEX_SMOKE=1 pnpm test:packed:native", "test:host-install": "rstest --config rstest.config.ts packages/agent-bundle/tests/host-install-proof.test.ts", "test:host-install:build": "pnpm build && pnpm test:host-install", + "test:host-install:session": "rstest --config rstest.config.ts packages/agent-bundle/tests/host-install-session.test.ts", + "test:host-install:session:claude": "pnpm build && AGENT_BUNDLE_HOST_INSTALL_CLAUDE_SESSION=1 pnpm test:host-install:session", "changeset": "changeset", "version-packages": "changeset version", "release": "pnpm build && changeset publish", diff --git a/packages/agent-bundle/tests/fixtures/host-install-tokens/agent-bundle.config.ts b/packages/agent-bundle/tests/fixtures/host-install-tokens/agent-bundle.config.ts new file mode 100644 index 000000000..27987c126 --- /dev/null +++ b/packages/agent-bundle/tests/fixtures/host-install-tokens/agent-bundle.config.ts @@ -0,0 +1,12 @@ +// Claude-only on purpose: skill targeting is project-wide, and the arguments +// and skill-root tokens have no Codex or Cursor Skill Markdown equivalent, so +// selecting either host makes this project fail the build with AB3008. +export default { + plugin: { + description: 'Proves canonical Skill tokens resolving in a real Claude session.', + name: 'host-install-token-proof', + version: '1.0.0', + }, + skills: ['skills/token-probe'], + targets: ['claude'], +}; diff --git a/packages/agent-bundle/tests/fixtures/host-install-tokens/package.json b/packages/agent-bundle/tests/fixtures/host-install-tokens/package.json new file mode 100644 index 000000000..64705ed87 --- /dev/null +++ b/packages/agent-bundle/tests/fixtures/host-install-tokens/package.json @@ -0,0 +1,6 @@ +{ + "name": "host-install-token-proof-fixture", + "private": true, + "type": "module", + "version": "1.0.0" +} diff --git a/packages/agent-bundle/tests/fixtures/host-install-tokens/skills/token-probe/SKILL.md b/packages/agent-bundle/tests/fixtures/host-install-tokens/skills/token-probe/SKILL.md new file mode 100644 index 000000000..11f81901f --- /dev/null +++ b/packages/agent-bundle/tests/fixtures/host-install-tokens/skills/token-probe/SKILL.md @@ -0,0 +1,14 @@ +--- +name: token-probe +description: Print the host-resolved token markers. Use when the user asks to run the token probe. +--- + +# Token probe + +Print the following three lines as your entire reply, one per line, verbatim. +The host substitutes the marker values before you see them; never edit, quote, +shorten, or re-derive them, and never add a code fence. + +ARGS_MARKER=agent-bundle:token:arguments +PLUGIN_ROOT_MARKER=agent-bundle:path:plugin-root +SKILL_DIR_MARKER=agent-bundle:token:skill-root diff --git a/packages/agent-bundle/tests/fixtures/host-install/skills/probe/SKILL.md b/packages/agent-bundle/tests/fixtures/host-install/skills/probe/SKILL.md index 4fbf8891d..dc5065fde 100644 --- a/packages/agent-bundle/tests/fixtures/host-install/skills/probe/SKILL.md +++ b/packages/agent-bundle/tests/fixtures/host-install/skills/probe/SKILL.md @@ -1,6 +1,17 @@ --- name: probe description: Confirm that the host-install proof fixture was discovered. +targets: + codex: + interface: + display_name: Host install probe + short_description: Confirm the host-install proof fixture was discovered. + policy: + allow_implicit_invocation: true + dependencies: + tools: + - type: mcp + value: probe --- # Probe diff --git a/packages/agent-bundle/tests/host-install-proof.test.ts b/packages/agent-bundle/tests/host-install-proof.test.ts index 51c090d19..381400b97 100644 --- a/packages/agent-bundle/tests/host-install-proof.test.ts +++ b/packages/agent-bundle/tests/host-install-proof.test.ts @@ -90,6 +90,19 @@ codexPluginIt( expect(report, proofLabel).toEqual({ host: 'codex', install: { state: 'installed', version: '1.0.0' }, + manifest: { + interfaceCapabilities: ['hooks', 'mcp', 'skills'], + interfaceFields: [ + 'capabilities', + 'category', + 'defaultPrompt', + 'developerName', + 'displayName', + 'longDescription', + 'shortDescription', + ], + path: '.codex-plugin/plugin.json', + }, proofLevel: proofLabel, registration: { cachePath: 'plugins/cache/host-install-proof-marketplace/host-install-proof/1.0.0', @@ -97,6 +110,12 @@ codexPluginIt( version: '1.0.0', }, skill: 'plugins/cache/host-install-proof-marketplace/host-install-proof/1.0.0/skills/probe/SKILL.md', + skillSidecar: { + matchesBuiltArtifact: true, + path: 'skills/probe/agents/openai.yaml', + schema: 'schema-valid', + sections: ['dependencies', 'interface', 'policy'], + }, status: 'passed', }); expectHygienicReport(report); @@ -116,6 +135,16 @@ it('installs into an isolated Cursor home, validates schemas, and is idempotent' }, host: 'cursor', install: { first: 'installed', second: 'already-installed', version: '1.0.0' }, + pluginRootVariable: { + locations: [ + 'hooks/hooks.json#/hooks/sessionStart/0/command', + 'mcp.json#/mcpServers/probe/args/0', + 'mcp.json#/mcpServers/probe/env/AGENT_BUNDLE_PLUGIN_ROOT', + ], + resolvedAtInstall: false, + sessionEvidence: 'unavailable: Cursor exposes no non-interactive plugin-loading session surface', + spelling: '${CURSOR_PLUGIN_ROOT}', + }, proofLevel: proofLabel, skill: '.cursor/plugins/local/host-install-proof/skills/probe/SKILL.md', status: 'passed', diff --git a/packages/agent-bundle/tests/host-install-session.test.ts b/packages/agent-bundle/tests/host-install-session.test.ts new file mode 100644 index 000000000..9d1ab54a9 --- /dev/null +++ b/packages/agent-bundle/tests/host-install-session.test.ts @@ -0,0 +1,100 @@ +import { spawnSync } from 'node:child_process'; + +import { afterAll, beforeAll, expect, it } from '@rstest/core'; + +import { + buildHostInstallTokenFixture, + CLAUDE_SESSION_ARGUMENT, + CLAUDE_SESSION_OPT_IN, + disposeHostInstallFixture, + runClaudeTokenSessionProof, + type BuiltHostInstallTokenFixture, +} from './support/host-install.ts'; +import { foreignSkillMarkdownSyntax, skillTokenSpellings } from '../src/skills/tokens.ts'; +import { + HOST_INSTALL_PROOF_LEVEL, + proofLevelLabel, +} from '../src/test/manifest.ts'; + +const proofLabel = proofLevelLabel(HOST_INSTALL_PROOF_LEVEL); +const claudeAvailable = spawnSync('claude', ['--version'], { + stdio: 'ignore', + timeout: 5_000, + windowsHide: true, +}).status === 0; +const sessionOptedIn = process.env[CLAUDE_SESSION_OPT_IN] === '1'; +const sessionMissingEvidence = !claudeAvailable + ? 'missing evidence: claude binary unavailable on PATH' + : `missing evidence: ${CLAUDE_SESSION_OPT_IN}=1 opt-in required for a real model call`; +const sessionIt = claudeAvailable && sessionOptedIn ? it : it.skip; + +let fixture: BuiltHostInstallTokenFixture | undefined; + +beforeAll(async () => { + fixture = await buildHostInstallTokenFixture({ environment: process.env }); +}, 180_000); + +afterAll(async () => { + if (fixture !== undefined) await disposeHostInstallFixture(fixture); +}); + +const builtFixture = (): BuiltHostInstallTokenFixture => { + if (fixture === undefined) throw new Error(`[${proofLabel}] token fixture build did not complete.`); + return fixture; +}; + +it('lowers canonical Skill tokens to Claude spellings without leaking foreign syntax', () => { + const markdown = builtFixture().loweredSkillMarkdown; + + expect(markdown, proofLabel).toContain('ARGS_MARKER=$ARGUMENTS'); + expect(markdown, proofLabel).toContain('PLUGIN_ROOT_MARKER=${CLAUDE_PLUGIN_ROOT}'); + expect(markdown, proofLabel).toContain('SKILL_DIR_MARKER=${CLAUDE_SKILL_DIR}'); + for (const spelling of Object.values(skillTokenSpellings)) { + expect(markdown, proofLabel).not.toContain(spelling); + } + for (const syntax of foreignSkillMarkdownSyntax('claude')) { + expect(markdown, proofLabel).not.toContain(syntax); + } +}); + +sessionIt( + claudeAvailable && sessionOptedIn + ? 'resolves the Claude arguments, plugin-root, and skill-root tokens in a real session' + : `resolves the Claude arguments, plugin-root, and skill-root tokens in a real session [${sessionMissingEvidence}]`, + async () => { + const report = await runClaudeTokenSessionProof(builtFixture(), { environment: process.env }); + + expect(report, proofLabel).toEqual({ + claudeVersion: expect.stringMatching(/^\d+\.\d+\.\d+$/u), + host: 'claude', + invocation: { + attempts: expect.any(Number), + mode: 'inline --plugin-dir session', + model: 'claude-sonnet-4-5', + normalHome: { + sessionBookkeeping: 'rewritten by Claude Code on every real turn', + settingsAndPlugins: 'unchanged', + }, + }, + markers: { + arguments: CLAUDE_SESSION_ARGUMENT, + pluginRoot: '.', + skillRoot: 'skills/token-probe', + }, + proofLevel: proofLabel, + qualifier: expect.stringContaining('session-token'), + resolved: { + arguments: 'substituted', + pluginRoot: 'absolute path that exists and is the loaded bundle root', + skillRoot: 'absolute path that exists and is the loaded skill directory', + }, + status: 'passed', + }); + expect(report.invocation.attempts, proofLabel).toBeLessThanOrEqual(2); + expect(report.qualifier, proofLabel).toContain(report.claudeVersion); + expect(JSON.stringify(report), proofLabel).not.toMatch( + /(?:API_KEY|AUTH_TOKEN|ACCESS_TOKEN|authorization|credential|password|secret|sk-[A-Za-z0-9_-]{16,}|\/home\/|\/Users\/|\/tmp\/|stdout|stderr)/iu, + ); + }, + 660_000, +); diff --git a/packages/agent-bundle/tests/support/host-install.ts b/packages/agent-bundle/tests/support/host-install.ts index 53915e5f6..6166ac24f 100644 --- a/packages/agent-bundle/tests/support/host-install.ts +++ b/packages/agent-bundle/tests/support/host-install.ts @@ -1,29 +1,57 @@ import { execFile as executeFile } from 'node:child_process'; -import { access, cp, mkdir, mkdtemp, readFile, rm, symlink } from 'node:fs/promises'; +import { access, cp, mkdir, mkdtemp, readFile, realpath, rm, symlink } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { join, relative, sep } from 'node:path'; +import { isAbsolute, join, relative, sep } from 'node:path'; import { promisify } from 'node:util'; +import { parse as parseYaml } from 'yaml'; + import { cursorHooksValidator, cursorMcpValidator, cursorPluginValidator, } from '../../src/adapters/cursor.ts'; +import { validateCodexOpenaiYaml } from '../../src/schemas/skill-hosts/contract.ts'; import { HOST_INSTALL_PROOF_LEVEL, proofLevelLabel, } from '../../src/test/manifest.ts'; -import { packedNativeEnvironment } from './packed-native-smoke.ts'; +import { + normalClaudeSettingsAndPluginsUnchanged, + packedNativeEnvironment, +} from './packed-native-smoke.ts'; const execFile = promisify(executeFile); const workspaceRoot = process.cwd(); const packageRoot = join(workspaceRoot, 'packages', 'agent-bundle'); -const fixtureRoot = join(packageRoot, 'tests', 'fixtures', 'host-install'); +const fixturesRoot = join(packageRoot, 'tests', 'fixtures'); const cli = join(packageRoot, 'dist', 'cli.js'); const plugin = 'host-install-proof'; const marketplace = 'host-install-proof-marketplace'; +const tokenPlugin = 'host-install-token-proof'; const version = '1.0.0'; const proofLevel = proofLevelLabel(HOST_INSTALL_PROOF_LEVEL); +const cursorPluginRootVariable = '${CURSOR_PLUGIN_ROOT}'; +const skillSidecarPath = join('skills', 'probe', 'agents', 'openai.yaml'); + +/** Opt-in for the one billable `claude -p` invocation the session proof makes. */ +export const CLAUDE_SESSION_OPT_IN = 'AGENT_BUNDLE_HOST_INSTALL_CLAUDE_SESSION'; + +/** The argument the session prompt passes so `$ARGUMENTS` substitution is observable. */ +export const CLAUDE_SESSION_ARGUMENT = 'alpha-bravo-42'; + +const claudeSessionModel = 'claude-sonnet-4-5'; + +/** + * Session-token evidence is a qualifier on the host-install level, not a level + * of its own: it observes canonical Skill tokens resolving inside one real + * `claude -p` turn against an inline `--plugin-dir` bundle. It proves nothing + * about an installed registration, packed provenance, or any other host. + */ +export const claudeSessionQualifier = (claudeVersion: string): string => + `session-token (canonical Skill tokens observed resolving in one real \`claude -p\` turn ` + + `with the built bundle loaded inline via --plugin-dir; observed Claude Code ${claudeVersion}; ` + + 'NOT installed-registration, packed-artifact, or other-host evidence)'; interface CommandResult { readonly exitCode: number; @@ -50,13 +78,21 @@ interface ClaudePluginRow { readonly version?: unknown; } -export interface BuiltHostInstallFixture { +export interface BuiltFixtureProject { readonly artifactRoot: string; - readonly bundles: Readonly>; readonly cli: string; readonly root: string; } +export interface BuiltHostInstallFixture extends BuiltFixtureProject { + readonly bundles: Readonly>; +} + +export interface BuiltHostInstallTokenFixture extends BuiltFixtureProject { + readonly claudeBundle: string; + readonly loweredSkillMarkdown: string; +} + export interface ClaudeHostInstallReport { readonly host: 'claude'; readonly install: { readonly state: 'installed'; readonly version: '1.0.0' }; @@ -77,6 +113,11 @@ export interface ClaudeHostInstallReport { export interface CodexHostInstallReport { readonly host: 'codex'; readonly install: { readonly state: 'installed'; readonly version: '1.0.0' }; + readonly manifest: { + readonly interfaceCapabilities: readonly string[]; + readonly interfaceFields: readonly string[]; + readonly path: string; + }; readonly proofLevel: string; readonly registration: { readonly cachePath: string; @@ -84,6 +125,12 @@ export interface CodexHostInstallReport { readonly version: '1.0.0'; }; readonly skill: string; + readonly skillSidecar: { + readonly matchesBuiltArtifact: true; + readonly path: string; + readonly schema: 'schema-valid'; + readonly sections: readonly string[]; + }; readonly status: 'passed'; } @@ -100,11 +147,45 @@ export interface CursorHostInstallReport { readonly second: 'already-installed'; readonly version: '1.0.0'; }; + readonly pluginRootVariable: { + readonly locations: readonly string[]; + readonly resolvedAtInstall: false; + readonly sessionEvidence: 'unavailable: Cursor exposes no non-interactive plugin-loading session surface'; + readonly spelling: '${CURSOR_PLUGIN_ROOT}'; + }; readonly proofLevel: string; readonly skill: string; readonly status: 'passed'; } +export interface ClaudeTokenSessionReport { + readonly claudeVersion: string; + readonly host: 'claude'; + readonly invocation: { + readonly attempts: number; + readonly mode: 'inline --plugin-dir session'; + readonly model: string; + readonly normalHome: { + readonly sessionBookkeeping: 'rewritten by Claude Code on every real turn'; + readonly settingsAndPlugins: 'unchanged'; + }; + }; + /** Bundle-relative resolutions of the marker values the real session printed. */ + readonly markers: { + readonly arguments: string; + readonly pluginRoot: string; + readonly skillRoot: string; + }; + readonly proofLevel: string; + readonly qualifier: string; + readonly resolved: { + readonly arguments: 'substituted'; + readonly pluginRoot: 'absolute path that exists and is the loaded bundle root'; + readonly skillRoot: 'absolute path that exists and is the loaded skill directory'; + }; + readonly status: 'passed'; +} + const fail = (message: string): never => { throw new Error(`[${proofLevel}] ${message}`); }; @@ -175,6 +256,19 @@ const readJson = async (path: string, context: string): Promise => { } }; +const readText = async (path: string, context: string): Promise => { + try { + return await readFile(path, 'utf8'); + } catch { + return fail(`${context} was not readable.`); + } +}; + +const record = (value: unknown): Readonly> | undefined => + typeof value === 'object' && value !== null && !Array.isArray(value) + ? (value as Readonly>) + : undefined; + const normalizedRelative = (root: string, path: string): string => relative(root, path).split(sep).join('/'); @@ -201,28 +295,20 @@ const assertInstallResult = ( }; /** - * Builds the fixture once with the workspace's source-built CLI. This proves + * Builds one fixture project with the workspace's source-built CLI. This proves * real-host acceptance of that built bundle; packed provenance remains the * separate `packed-stdio` proof level. */ -export const buildHostInstallFixture = async (options: { +const buildFixtureProject = async (options: { + readonly bundleNames: readonly string[]; readonly environment: Readonly; -}): Promise => { - const root = await mkdtemp(join(tmpdir(), 'agent-bundle-host-install-build-')); + readonly fixture: string; +}): Promise => { + const root = await mkdtemp(join(tmpdir(), `agent-bundle-${options.fixture}-build-`)); const project = join(root, 'project'); const artifactRoot = join(project, 'artifact'); - const fixture: BuiltHostInstallFixture = Object.freeze({ - artifactRoot, - bundles: Object.freeze({ - claude: join(artifactRoot, 'claude'), - codex: join(artifactRoot, 'codex'), - cursor: join(artifactRoot, 'cursor'), - }), - cli, - root, - }); try { - await cp(fixtureRoot, project, { recursive: true }); + await cp(join(fixturesRoot, options.fixture), project, { recursive: true }); await symlink(join(workspaceRoot, 'node_modules'), join(project, 'node_modules'), 'dir'); const result = await run(process.execPath, [ cli, @@ -236,16 +322,56 @@ export const buildHostInstallFixture = async (options: { environment: packedNativeEnvironment(options.environment), timeout: 180_000, }); - assertProof(result.exitCode === 0, `fixture build failed: ${commandDetail(result)}`); - await Promise.all(Object.values(fixture.bundles).map((bundle) => access(bundle))); - return fixture; + assertProof(result.exitCode === 0, `${options.fixture} fixture build failed: ${commandDetail(result)}`); + await Promise.all(options.bundleNames.map((name) => access(join(artifactRoot, name)))); + return Object.freeze({ artifactRoot, cli, root }); } catch (error) { await rm(root, { force: true, recursive: true }); throw error; } }; -export const disposeHostInstallFixture = async (fixture: BuiltHostInstallFixture): Promise => { +export const buildHostInstallFixture = async (options: { + readonly environment: Readonly; +}): Promise => { + const built = await buildFixtureProject({ + bundleNames: ['claude', 'codex', 'cursor'], + environment: options.environment, + fixture: 'host-install', + }); + return Object.freeze({ + ...built, + bundles: Object.freeze({ + claude: join(built.artifactRoot, 'claude'), + codex: join(built.artifactRoot, 'codex'), + cursor: join(built.artifactRoot, 'cursor'), + }), + }); +}; + +/** + * The token fixture is a separate Claude-only project because skill selection + * is project-wide: `targets:` in Skill frontmatter carries per-host extension + * fields, not a host restriction, so the arguments and skill-root tokens would + * raise AB3008 for Codex and Cursor if they shared the host-install fixture. + */ +export const buildHostInstallTokenFixture = async (options: { + readonly environment: Readonly; +}): Promise => { + const built = await buildFixtureProject({ + bundleNames: ['claude'], + environment: options.environment, + fixture: 'host-install-tokens', + }); + const claudeBundle = join(built.artifactRoot, 'claude'); + return Object.freeze({ + ...built, + claudeBundle, + loweredSkillMarkdown: await readFile(join(claudeBundle, 'skills', 'token-probe', 'SKILL.md'), 'utf8'), + }); +}; + +export const disposeHostInstallFixture = async (fixture: BuiltFixtureProject): Promise => { await rm(fixture.root, { force: true, recursive: true }); }; @@ -364,9 +490,56 @@ export const runCodexHostInstallProof = async ( const cachePath = join(codexHome, 'plugins', 'cache', marketplace, plugin, version); const skillPath = join(cachePath, 'skills', 'probe', 'SKILL.md'); await access(skillPath).catch(() => fail('Codex cache did not contain skills/probe/SKILL.md.')); + + const installedSidecar = await readText(join(cachePath, skillSidecarPath), 'Codex installed skill sidecar'); + const builtSidecar = await readText( + join(fixture.bundles.codex, skillSidecarPath), + 'Codex built skill sidecar', + ); + assertProof( + installedSidecar === builtSidecar, + 'Codex install did not copy skills/probe/agents/openai.yaml byte-identically from the built bundle.', + ); + const sidecarDocument = parseYaml(installedSidecar) as unknown; + const sidecar = record(sidecarDocument); + assertProof(sidecar !== undefined, 'Codex installed skill sidecar was not a YAML mapping.'); + const sidecarIssues = validateCodexOpenaiYaml(sidecarDocument); + assertProof( + sidecarIssues.length === 0, + `Codex installed skill sidecar failed its pinned schema: ${JSON.stringify(sidecarIssues)}`, + ); + const sidecarSections = Object.keys(sidecar).sort(); + + const manifestPath = join(cachePath, '.codex-plugin', 'plugin.json'); + const manifest = record(await readJson(manifestPath, 'Codex installed plugin manifest')); + assertProof(manifest !== undefined, 'Codex installed plugin manifest was not a JSON object.'); + const manifestInterface = record(manifest.interface); + assertProof(manifestInterface !== undefined, 'Codex installed plugin manifest carried no interface block.'); + assertProof( + manifestInterface.displayName === plugin, + 'Codex installed plugin manifest interface did not carry the plugin display name.', + ); + const capabilities = manifestInterface.capabilities; + assertProof( + Array.isArray(capabilities) && capabilities.every((entry) => typeof entry === 'string'), + 'Codex installed plugin manifest interface carried no string capability list.', + ); + const interfaceCapabilities = [...capabilities].sort(); + for (const capability of ['hooks', 'mcp', 'skills']) { + assertProof( + interfaceCapabilities.includes(capability), + `Codex installed plugin manifest interface did not advertise ${capability}.`, + ); + } + return Object.freeze({ host: 'codex', install: Object.freeze({ state: 'installed', version }), + manifest: Object.freeze({ + interfaceCapabilities: Object.freeze(interfaceCapabilities), + interfaceFields: Object.freeze(Object.keys(manifestInterface).sort()), + path: normalizedRelative(cachePath, manifestPath), + }), proofLevel, registration: Object.freeze({ cachePath: normalizedRelative(codexHome, cachePath), @@ -374,6 +547,12 @@ export const runCodexHostInstallProof = async ( version, }), skill: normalizedRelative(codexHome, skillPath), + skillSidecar: Object.freeze({ + matchesBuiltArtifact: true, + path: normalizedRelative(cachePath, join(cachePath, skillSidecarPath)), + schema: 'schema-valid', + sections: Object.freeze(sidecarSections), + }), status: 'passed', }); } finally { @@ -381,6 +560,55 @@ export const runCodexHostInstallProof = async ( } }; +/** + * Placement copies the emitted documents; it never resolves the host's own + * plugin-root variable, because resolution belongs to Cursor at load time. + * Cursor publishes no non-interactive plugin-loading session surface, so the + * pinned `${CURSOR_PLUGIN_ROOT}` contract plus schema validity is the honest + * ceiling here — there is no Cursor equivalent of the Claude session proof. + */ +const assertCursorPluginRootVariable = (input: { + readonly destination: string; + readonly hooks: unknown; + readonly hooksText: string; + readonly mcp: unknown; + readonly mcpText: string; +}): readonly string[] => { + const hooks = record(record(input.hooks)?.hooks); + const sessionStart = hooks?.sessionStart; + const firstHook = Array.isArray(sessionStart) ? record(sessionStart[0]) : undefined; + const command = firstHook?.command; + assertProof( + typeof command === 'string' && command.includes(cursorPluginRootVariable), + `Cursor installed hooks document did not carry an unresolved ${cursorPluginRootVariable} command.`, + ); + + const server = record(record(record(input.mcp)?.mcpServers)?.probe); + const args = server?.args; + const firstArgument = Array.isArray(args) ? args[0] : undefined; + assertProof( + typeof firstArgument === 'string' && firstArgument.startsWith(`${cursorPluginRootVariable}/`), + `Cursor installed MCP document did not anchor the probe entry on ${cursorPluginRootVariable}.`, + ); + const anchor = record(server?.env)?.AGENT_BUNDLE_PLUGIN_ROOT; + assertProof( + anchor === cursorPluginRootVariable, + `Cursor installed MCP document did not carry the ${cursorPluginRootVariable} environment anchor.`, + ); + + for (const [name, text] of [['hooks.json', input.hooksText], ['mcp.json', input.mcpText]] as const) { + assertProof( + !text.includes(input.destination), + `Cursor placement resolved ${cursorPluginRootVariable} into an absolute path inside ${name}.`, + ); + } + return Object.freeze([ + 'hooks/hooks.json#/hooks/sessionStart/0/command', + 'mcp.json#/mcpServers/probe/args/0', + 'mcp.json#/mcpServers/probe/env/AGENT_BUNDLE_PLUGIN_ROOT', + ]); +}; + export const runCursorHostInstallProof = async ( fixture: BuiltHostInstallFixture, options: { readonly environment: Readonly }, @@ -407,10 +635,19 @@ export const runCursorHostInstallProof = async ( const pluginDocument = await readJson(join(destination, '.cursor-plugin', 'plugin.json'), 'Cursor plugin manifest'); assertProof(cursorPluginValidator(pluginDocument), `Cursor plugin manifest failed its pinned schema: ${JSON.stringify(cursorPluginValidator.errors)}`); - const hooksDocument = await readJson(join(destination, 'hooks', 'hooks.json'), 'Cursor hooks document'); + const hooksText = await readText(join(destination, 'hooks', 'hooks.json'), 'Cursor hooks document'); + const hooksDocument = parseJson(hooksText, 'Cursor hooks document'); assertProof(cursorHooksValidator(hooksDocument), `Cursor hooks document failed its pinned schema: ${JSON.stringify(cursorHooksValidator.errors)}`); - const mcpDocument = await readJson(join(destination, 'mcp.json'), 'Cursor MCP document'); + const mcpText = await readText(join(destination, 'mcp.json'), 'Cursor MCP document'); + const mcpDocument = parseJson(mcpText, 'Cursor MCP document'); assertProof(cursorMcpValidator(mcpDocument), `Cursor MCP document failed its pinned schema: ${JSON.stringify(cursorMcpValidator.errors)}`); + const pluginRootLocations = assertCursorPluginRootVariable({ + destination, + hooks: hooksDocument, + hooksText, + mcp: mcpDocument, + mcpText, + }); const skillPath = join(destination, 'skills', 'probe', 'SKILL.md'); await access(skillPath).catch(() => fail('Cursor install did not contain skills/probe/SKILL.md.')); @@ -431,6 +668,12 @@ export const runCursorHostInstallProof = async ( second: 'already-installed', version, }), + pluginRootVariable: Object.freeze({ + locations: pluginRootLocations, + resolvedAtInstall: false, + sessionEvidence: 'unavailable: Cursor exposes no non-interactive plugin-loading session surface', + spelling: cursorPluginRootVariable, + }), proofLevel, skill: normalizedRelative(home, skillPath), status: 'passed', @@ -439,3 +682,163 @@ export const runCursorHostInstallProof = async ( await rm(home, { force: true, recursive: true }); } }; + +interface SessionMarkers { + readonly arguments?: string; + readonly pluginRoot?: string; + readonly skillRoot?: string; +} + +interface CompleteSessionMarkers { + readonly arguments: string; + readonly pluginRoot: string; + readonly skillRoot: string; +} + +/** Tolerates surrounding prose: each marker is matched as its own whole line. */ +const markerValue = (text: string, marker: string): string | undefined => { + const matched = new RegExp(`^[^\\S\\n]*${marker}=(.+)$`, 'mu').exec(text)?.[1]?.trim(); + return matched === undefined || matched.length === 0 ? undefined : matched; +}; + +const sessionMarkers = (text: string): SessionMarkers => Object.freeze({ + ...(markerValue(text, 'ARGS_MARKER') === undefined ? {} : { arguments: markerValue(text, 'ARGS_MARKER') }), + ...(markerValue(text, 'PLUGIN_ROOT_MARKER') === undefined + ? {} + : { pluginRoot: markerValue(text, 'PLUGIN_ROOT_MARKER') }), + ...(markerValue(text, 'SKILL_DIR_MARKER') === undefined + ? {} + : { skillRoot: markerValue(text, 'SKILL_DIR_MARKER') }), +}); + +const everyMarkerPresent = (markers: SessionMarkers): boolean => + markers.arguments !== undefined && markers.pluginRoot !== undefined && markers.skillRoot !== undefined; + +const completeMarkers = (markers: SessionMarkers, attempts: number): CompleteSessionMarkers => { + const { arguments: argumentsMarker, pluginRoot, skillRoot } = markers; + if (argumentsMarker === undefined || pluginRoot === undefined || skillRoot === undefined) { + return fail(`Claude session printed no complete marker set after ${String(attempts)} attempt(s).`); + } + return Object.freeze({ arguments: argumentsMarker, pluginRoot, skillRoot }); +}; + +/** + * Resolves a marker the host was expected to substitute, failing on the + * unresolved spelling instead of accepting it as a path. + */ +const assertResolvedPath = async ( + value: string, + spelling: string, + label: string, +): Promise => { + assertProof(!value.includes(spelling), `Claude left ${label} unresolved as ${spelling}.`); + assertProof(isAbsolute(value), `Claude ${label} was not an absolute path.`); + return await realpath(value).catch(() => fail(`Claude ${label} does not exist on disk.`)); +}; + +/** + * One real `claude -p` turn against the normal home (first-party auth lives + * there, so an isolated CLAUDE_CONFIG_DIR has no credentials). The bundle is + * loaded inline with `--plugin-dir`, which registers nothing. + * + * Observed against Claude Code 2.1.257: all three canonical Skill tokens are + * substituted inside the Skill Markdown body — `$ARGUMENTS` from the slash + * invocation, `${CLAUDE_PLUGIN_ROOT}` as the loaded bundle root, and + * `${CLAUDE_SKILL_DIR}` as that bundle's `skills/token-probe` directory. + * + * Also observed at that version: a real turn rewrites Claude Code's own + * `.claude.json` session bookkeeping, so the full `normalClaudeHomeUnchanged` + * digest cannot hold here. The settings and installed-plugin surface is + * unchanged, and that is what this proof guards. + */ +export const runClaudeTokenSessionProof = async ( + fixture: BuiltHostInstallTokenFixture, + options: { readonly environment: Readonly }, +): Promise => { + const environment = packedNativeEnvironment(options.environment); + const versioned = await run('claude', ['--version'], { cwd: fixture.claudeBundle, environment }); + assertProof(versioned.exitCode === 0, `claude --version failed: ${commandDetail(versioned)}`); + const claudeVersion = /(?:^|\s)(\d+\.\d+\.\d+)(?:\s|$)/u.exec(versioned.stdout)?.[1]; + assertProof(claudeVersion !== undefined, 'claude --version did not report a semantic version.'); + + // Model output is nondeterministic, so the skill invocation gets exactly one + // retry with an explicit instruction. Two `-p` calls is the hard ceiling. + const prompts = Object.freeze([ + `/${tokenPlugin}:token-probe ${CLAUDE_SESSION_ARGUMENT}`, + `Run the ${tokenPlugin} plugin's token-probe skill with the arguments ` + + `${CLAUDE_SESSION_ARGUMENT} and reply with its three marker lines verbatim.`, + ]); + let attempts = 0; + let markers: SessionMarkers = {}; + let last: CommandResult | undefined; + const session = async (): Promise => { + for (const prompt of prompts) { + attempts += 1; + last = await run('claude', [ + '--plugin-dir', + fixture.claudeBundle, + '--model', + claudeSessionModel, + '--output-format', + 'text', + '-p', + prompt, + ], { cwd: fixture.claudeBundle, environment, timeout: 300_000 }); + markers = sessionMarkers(last.stdout); + if (last.exitCode === 0 && everyMarkerPresent(markers)) return; + } + }; + const settingsAndPluginsUnchanged = await normalClaudeSettingsAndPluginsUnchanged( + options.environment, + session, + ); + assertProof(last !== undefined, 'The Claude session proof never invoked the host.'); + assertProof(last.exitCode === 0, `Claude session failed: ${commandDetail(last)}`); + assertProof( + settingsAndPluginsUnchanged, + 'The real Claude settings or installed-plugin tree changed while the inline --plugin-dir session ran.', + ); + const { arguments: argumentsMarker, pluginRoot, skillRoot } = completeMarkers(markers, attempts); + assertProof( + argumentsMarker === CLAUDE_SESSION_ARGUMENT, + 'Claude did not substitute the invocation arguments into the arguments token.', + ); + const bundleRoot = await realpath(fixture.claudeBundle); + const resolvedPluginRoot = await assertResolvedPath(pluginRoot, '${CLAUDE_PLUGIN_ROOT}', 'plugin root'); + assertProof( + resolvedPluginRoot === bundleRoot, + 'Claude resolved the plugin-root token outside the loaded bundle.', + ); + const resolvedSkillRoot = await assertResolvedPath(skillRoot, '${CLAUDE_SKILL_DIR}', 'skill root'); + assertProof( + resolvedSkillRoot === join(bundleRoot, 'skills', 'token-probe'), + 'Claude resolved the skill-root token outside the loaded bundle\'s skill directory.', + ); + + return Object.freeze({ + claudeVersion, + host: 'claude', + invocation: Object.freeze({ + attempts, + mode: 'inline --plugin-dir session', + model: claudeSessionModel, + normalHome: Object.freeze({ + sessionBookkeeping: 'rewritten by Claude Code on every real turn', + settingsAndPlugins: 'unchanged', + }), + }), + markers: Object.freeze({ + arguments: argumentsMarker, + pluginRoot: normalizedRelative(bundleRoot, resolvedPluginRoot) || '.', + skillRoot: normalizedRelative(bundleRoot, resolvedSkillRoot), + }), + proofLevel, + qualifier: claudeSessionQualifier(claudeVersion), + resolved: Object.freeze({ + arguments: 'substituted', + pluginRoot: 'absolute path that exists and is the loaded bundle root', + skillRoot: 'absolute path that exists and is the loaded skill directory', + }), + status: 'passed', + }); +}; diff --git a/packages/agent-bundle/tests/support/packed-native-smoke.ts b/packages/agent-bundle/tests/support/packed-native-smoke.ts index c88413503..d16b566ef 100644 --- a/packages/agent-bundle/tests/support/packed-native-smoke.ts +++ b/packages/agent-bundle/tests/support/packed-native-smoke.ts @@ -164,21 +164,35 @@ const digestNormalCodexState = async (codexHome: string) => Object.freeze({ plugins: await digestTree(join(codexHome, 'plugins'), false), }); -const digestNormalClaudeState = async ( +/** + * The Claude state a proof must never mutate: settings and the installed + * plugin tree. Session bookkeeping (`.claude.json`) is deliberately excluded + * so a caller that runs a real turn can still assert this surface. + */ +const digestClaudeSettingsAndPlugins = async ( environment: Readonly, options: Readonly<{ readonly homeDirectory: string }>, ) => { - const customHome = environment.CLAUDE_CONFIG_DIR; - const claudeHome = customHome ?? join(options.homeDirectory, '.claude'); + const claudeHome = environment.CLAUDE_CONFIG_DIR ?? join(options.homeDirectory, '.claude'); return Object.freeze({ config: await digestTree(join(claudeHome, 'config.json'), true), localSettings: await digestTree(join(claudeHome, 'settings.local.json'), true), plugins: await digestTree(join(claudeHome, 'plugins'), true), settings: await digestTree(join(claudeHome, 'settings.json'), true), - state: await digestTree(join(customHome ?? options.homeDirectory, '.claude.json'), true), }); }; +const digestNormalClaudeState = async ( + environment: Readonly, + options: Readonly<{ readonly homeDirectory: string }>, +) => Object.freeze({ + ...(await digestClaudeSettingsAndPlugins(environment, options)), + state: await digestTree( + join(environment.CLAUDE_CONFIG_DIR ?? options.homeDirectory, '.claude.json'), + true, + ), +}); + const sameClaudeState = ( left: Awaited>, right: Awaited>, @@ -198,6 +212,26 @@ export const normalClaudeHomeUnchanged = async ( return sameClaudeState(before, await digestNormalClaudeState(environment, options)); }; +/** + * The guard for an operation that runs a real signed-in turn: Claude Code + * rewrites its own `.claude.json` session bookkeeping on every such turn, so + * `normalClaudeHomeUnchanged` cannot hold. This asserts the surface that must + * still hold — settings and the installed plugin tree. + */ +export const normalClaudeSettingsAndPluginsUnchanged = async ( + environment: Readonly, + operation: () => Promise, + options: Readonly<{ readonly homeDirectory: string }> = { homeDirectory: homedir() }, +): Promise => { + const before = await digestClaudeSettingsAndPlugins(environment, options); + await operation(); + const after = await digestClaudeSettingsAndPlugins(environment, options); + return before.config === after.config + && before.localSettings === after.localSettings + && before.plugins === after.plugins + && before.settings === after.settings; +}; + const summarizeEval = (host: PackedNativeHost, command: CommandResult) => { let document: EvalDocument | undefined; try { diff --git a/rstest.integration-tests.ts b/rstest.integration-tests.ts index 15b85cb48..3314582c8 100644 --- a/rstest.integration-tests.ts +++ b/rstest.integration-tests.ts @@ -36,6 +36,7 @@ export const integrationTestFiles: readonly string[] = [ 'packages/agent-bundle/tests/host-adapters.native.test.ts', 'packages/agent-bundle/tests/host-adapters.test.ts', 'packages/agent-bundle/tests/host-install-proof.test.ts', + 'packages/agent-bundle/tests/host-install-session.test.ts', 'packages/agent-bundle/tests/integration-matrix.test.ts', 'packages/agent-bundle/tests/mcp-session-service.test.ts', 'packages/agent-bundle/tests/mcp.test.ts',