diff --git a/.changeset/claude-host-validation.md b/.changeset/claude-host-validation.md new file mode 100644 index 000000000..ef5c02636 --- /dev/null +++ b/.changeset/claude-host-validation.md @@ -0,0 +1,5 @@ +--- +"agent-bundle": minor +--- + +Validate built Claude bundles with Claude Code's strict plugin developer tools and expose the bounded validator as an opt-in test helper. diff --git a/README.md b/README.md index 9e0445b9e..78dccb8fb 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,13 @@ The same config also owns the npm package build — no second bundler config, bi - `hooks list` / `hooks simulate` — inspect and simulate generated hooks - `eval` — run eval suites against a built artifact +When validating a built `claude` or unified `plugin` target, Agent Bundle uses the installed +Claude Code developer toolchain in addition to its pinned schemas. Use +`agent-bundle validate --artifact dist --strict` in CI; Claude's `--strict` findings remain +warnings locally unless Agent Bundle strict mode is requested. If `claude` is absent, validation +reports an explicit informational skip. For the install-free development loop, run +`claude --plugin-dir dist/claude plugin list --json` after building. + The [package README](packages/agent-bundle/README.md) is the full reference: configuration semantics, the workbench, the optional Agent API, evals, and limitations. ## Examples diff --git a/packages/agent-bundle/README.md b/packages/agent-bundle/README.md index 3d80f26bf..ea9bf8a9a 100644 --- a/packages/agent-bundle/README.md +++ b/packages/agent-bundle/README.md @@ -78,6 +78,26 @@ manifests at files inside those payloads without compiling them. Payload files c `validate --artifact`, `mcp`, and `hooks` work against a built artifact with project sources deleted. +### Validate Claude bundles with Claude Code + +Artifact validation runs `claude plugin validate --strict` for emitted `claude` +and unified `plugin` targets when Claude Code is on `PATH`. Host errors become Agent Bundle +errors; host warnings remain warnings unless `agent-bundle validate --strict` is set. A missing +binary is reported as an explicit informational skip, never as fabricated success. Use +`--no-host-validation` when a deterministic schema-only check is required. + +CI should use strict validation: + +```sh +agent-bundle validate --artifact dist --strict +``` + +During development, load a built target without installing it and verify registration: + +```sh +claude --plugin-dir dist/claude plugin list --json +``` + ## Developer workbench `agent-bundle dev` serves a loopback-only prebuilt workbench. It shows project diff --git a/packages/agent-bundle/src/adapters/capabilities/claude-2.1.250.json b/packages/agent-bundle/src/adapters/capabilities/claude-2.1.250.json index 740d33654..55d11b301 100644 --- a/packages/agent-bundle/src/adapters/capabilities/claude-2.1.250.json +++ b/packages/agent-bundle/src/adapters/capabilities/claude-2.1.250.json @@ -39,6 +39,14 @@ }, "observedCliVersion": "2.1.250", "plugin": { + "devtools": { + "details": true, + "listJson": true, + "pluginDir": true, + "strictValidation": true, + "tag": true, + "validate": true + }, "lsp": { "config": ".lsp.json", "manifestField": "lspServers", @@ -81,7 +89,10 @@ "First-registered-wins collision rule: when more than one enabled server declares the same extension in extensionToLanguage, from one plugin or from different plugins, the first registered handles the extension and the others never start.", "The server binary is never vendored: \"You must install the language server binary separately. LSP plugins configure how Claude Code connects to a language server, but they don't include the server itself.\"", "Placeholder substitution for LSP servers is limited to command, args, env, and workspaceFolder.", - "Codex and Cursor publish no plugin LSP surface at their pinned revisions, so the unified bundle's .lsp.json reaches Claude Code only." + "Codex and Cursor publish no plugin LSP surface at their pinned revisions, so the unified bundle's .lsp.json reaches Claude Code only.", + "Plugin developer tools reference: `claude plugin validate ` checks plugin.json, hooks/hooks.json, and default-directory Skill, agent, and command frontmatter; manifest-less component directories require 2.1.233 or later.", + "`claude plugin validate --strict` promotes tolerated warnings such as unrecognized or near-miss fields and non-object experimental/metadata values to exit failure; the reference recommends strict mode in CI.", + "Development tools include `claude --plugin-dir plugin list --json` for registration proof, `claude plugin details ` for component inventory and host-owned token estimates, `claude plugin tag`, and `claude --debug` for loading diagnostics." ] } } diff --git a/packages/agent-bundle/src/adapters/claude.ts b/packages/agent-bundle/src/adapters/claude.ts index b28aa824d..d4bb783ea 100644 --- a/packages/agent-bundle/src/adapters/claude.ts +++ b/packages/agent-bundle/src/adapters/claude.ts @@ -133,7 +133,7 @@ const hookContract = Object.freeze({ const metadata = Object.freeze({ adapterRevision: '1.2.0', capabilityRevision: capabilityTable.observedCliVersion, - capabilitySha256: '5beb395c075c22a290a70f76b1670fab82b16d933af37cda89da850dbd8d483c', + capabilitySha256: '13bc41224c5343b33d259986a66feb279e15431c5019bb2a1c443eaa60e9a9ea', observedVersion: capabilityTable.observedCliVersion, schemas: schemaDescriptorsFrom(schemaProvenance, schemaProvenance.observedCliVersion), }); diff --git a/packages/agent-bundle/src/adapters/schemas/claude/PROVENANCE.json b/packages/agent-bundle/src/adapters/schemas/claude/PROVENANCE.json index 7053bb386..3f96307d1 100644 --- a/packages/agent-bundle/src/adapters/schemas/claude/PROVENANCE.json +++ b/packages/agent-bundle/src/adapters/schemas/claude/PROVENANCE.json @@ -30,5 +30,15 @@ "url": "https://docs.anthropic.com/en/docs/claude-code/plugins" } }, - "validation": "Pinned JSON Schema snapshots are validated locally with Ajv. Package builds do not download schemas or invoke host-side validators." + "validation": "Pinned JSON Schema snapshots are validated locally with Ajv. Artifact validation can additionally invoke `claude plugin validate --strict`; Agent Bundle preserves host warnings unless its own strict option is enabled, and reports an explicit unavailable diagnostic when the CLI is absent.", + "developerTools": { + "retrievedAt": "2026-09-01", + "source": "https://code.claude.com/docs/en/plugins-reference", + "evidence": [ + "`claude plugin validate ` validates plugin.json, hooks/hooks.json, and frontmatter in default skills, agents, and commands directories; manifest-less directories require Claude Code 2.1.233 or later.", + "`--strict` treats warnings as errors and is recommended for CI, including unrecognized or near-miss fields and non-object experimental/metadata values.", + "`claude --plugin-dir plugin list` includes session-loaded plugins only when the same flag precedes the subcommand; `--json` provides machine-readable registration evidence.", + "`claude plugin details` owns component inventory and token-cost estimation; Agent Bundle does not reimplement its count_tokens or character-fallback costing." + ] + } } diff --git a/packages/agent-bundle/src/api.ts b/packages/agent-bundle/src/api.ts index 453e351eb..406a79d9c 100644 --- a/packages/agent-bundle/src/api.ts +++ b/packages/agent-bundle/src/api.ts @@ -75,11 +75,15 @@ export { } from './build/manifest.ts'; import { composeBundlerInspection, type BundlerInspection } from './build/inspect-bundler.ts'; export type { BundlerInspection, BundlerInspectionEntry } from './build/inspect-bundler.ts'; -import { validateArtifact } from './build/validate-artifact.ts'; +import { validateArtifact, validateArtifactWithSnapshot } from './build/validate-artifact.ts'; import { freezeDiagnostics, hasErrors, DiagnosticError, type Diagnostic } from './core/diagnostics.ts'; export type { Diagnostic, DiagnosticSeverity } from './core/diagnostics.ts'; import type { ProjectContext } from './core/project-context.ts'; import type { NormalizedPlugin } from './core/types.ts'; +import { + validateClaudePlugin, + type ClaudePluginValidationReport, +} from './host-contracts/claude-plugin-validation.ts'; import type { EvalComparison } from './eval/compare.ts'; import { EvalRunStoreError } from './eval/errors.ts'; import { @@ -214,10 +218,15 @@ export interface ProjectOptions { export interface ValidateOptions extends ProjectOptions { readonly artifact?: string; + /** Run installed host developer tools for compatible built targets. */ + readonly hostValidation?: boolean; + /** Promote host-tool warnings to errors. */ + readonly strict?: boolean; } export interface ValidateResult { readonly diagnostics: readonly Diagnostic[]; + readonly hostValidation?: readonly ClaudePluginValidationReport[]; readonly model?: NormalizedPlugin; } @@ -408,6 +417,29 @@ export const validate = async (options: ValidateOptions): Promise target.name === 'claude' || target.name === 'plugin') + .map((target) => validateClaudePlugin({ + pluginDirectory: join(artifact, target.name), + strict: options.strict, + target: target.name, + }))); + return Object.freeze({ + diagnostics: freezeDiagnostics([ + ...validated.diagnostics, + ...reports.flatMap((report) => report.diagnostics), + ]), + ...(reports.length === 0 ? {} : { hostValidation: Object.freeze(reports) }), + }); + } return Object.freeze({ diagnostics: freezeDiagnostics(await validateArtifact({ artifactRoot: artifact, registry: registryFor(options) })), }); diff --git a/packages/agent-bundle/src/build/artifact-diagnostics.ts b/packages/agent-bundle/src/build/artifact-diagnostics.ts index 969482dae..12c07f33d 100644 --- a/packages/agent-bundle/src/build/artifact-diagnostics.ts +++ b/packages/agent-bundle/src/build/artifact-diagnostics.ts @@ -19,7 +19,11 @@ export type ArtifactDiagnosticCode = | 'AB6015' | 'AB6016' | 'AB6017' - | 'AB6018'; + | 'AB6018' + | 'AB6019' + | 'AB6020' + | 'AB6021' + | 'AB6022'; export const artifactDiagnosticRecoveries: Readonly> = Object.freeze({ AB6000: 'Restore a readable artifact root and canonical manifest, then rebuild the artifact.', @@ -41,6 +45,10 @@ export const artifactDiagnosticRecoveries: Readonly --strict`, repair the warning, and rebuild.', + AB6021: 'Run `claude plugin validate --strict`, repair the error, and rebuild.', + AB6022: 'Restore a bounded Claude validator process, then rerun artifact validation.', }); const isArtifactDiagnosticCode = (code: string): code is ArtifactDiagnosticCode => diff --git a/packages/agent-bundle/src/cli.ts b/packages/agent-bundle/src/cli.ts index 955b16c80..330d9f172 100644 --- a/packages/agent-bundle/src/cli.ts +++ b/packages/agent-bundle/src/cli.ts @@ -43,13 +43,17 @@ export interface CliDependencies { /** Injectable only to make foreground shutdown behavior deterministic in tests. */ readonly signals?: CliSignalSource; readonly startDevServer?: typeof startDevServer; + /** Injectable only to verify host-validation CLI policy without an installed host. */ + readonly validate?: typeof validate; } interface SourceCommandOptions { readonly config?: string; + readonly hostValidation?: boolean; readonly json?: boolean; readonly mode?: string; readonly root: string; + readonly strict?: boolean; readonly target?: readonly string[]; } @@ -365,10 +369,19 @@ export const runCli = async ( const validateCommand = configureSourceOptions( program.command('validate').description('Validate project source or one artifact'), - ).option('--artifact ', 'Validate exactly this built artifact'); + ) + .option('--artifact ', 'Validate exactly this built artifact') + .option('--host-validation', 'Run installed host developer tools for compatible built targets', true) + .option('--no-host-validation', 'Skip installed host developer tools') + .option('--strict', 'Promote host-tool warnings to errors'); validateCommand.action(async (options: SourceCommandOptions & { readonly artifact?: string }) => { const { validate } = await import('./api.ts'); - const result = await validate({ ...projectOptions(options), artifact: options.artifact }); + const result = await (dependencies.validate ?? validate)({ + ...projectOptions(options), + artifact: options.artifact, + hostValidation: options.hostValidation, + strict: options.strict, + }); if (result.diagnostics.some((diagnostic) => diagnostic.severity === 'error')) { throw new DiagnosticError(result.diagnostics); } diff --git a/packages/agent-bundle/src/host-contracts/claude-plugin-validation.ts b/packages/agent-bundle/src/host-contracts/claude-plugin-validation.ts new file mode 100644 index 000000000..86149652f --- /dev/null +++ b/packages/agent-bundle/src/host-contracts/claude-plugin-validation.ts @@ -0,0 +1,195 @@ +import { dirname } from 'node:path'; + +import type { Diagnostic, DiagnosticSeverity } from '../core/diagnostics.ts'; +import { freezeDiagnostics } from '../core/diagnostics.ts'; +import { isErrno } from '../core/errors.ts'; +import { + runBoundedChildProcess, + type BoundedChildProcessRequest, + type BoundedChildProcessResult, +} from './process.ts'; + +const maximumOutputBytes = 1024 * 1024; +const validationTimeoutMs = 15_000; +const versionTimeoutMs = 5_000; + +type ClaudePluginTermination = 'output-limit' | 'timed-out'; + +export type ClaudePluginValidationStatus = 'failed' | 'passed' | 'unavailable' | 'warnings'; + +export interface ClaudePluginValidationReport { + readonly diagnostics: readonly Diagnostic[]; + readonly host: 'claude'; + readonly status: ClaudePluginValidationStatus; + readonly target: string; + readonly version?: string; +} + +export type ClaudePluginCommandResult = BoundedChildProcessResult; + +export type ClaudePluginCommandRunner = ( + request: BoundedChildProcessRequest, +) => Promise; + +export interface ValidateClaudePluginOptions { + readonly executable?: string; + readonly pluginDirectory: string; + /** Injectable proof seam. Production always uses the bounded process runner. */ + readonly run?: ClaudePluginCommandRunner; + /** Promote host warnings to Agent Bundle errors. Claude itself always runs with `--strict`. */ + readonly strict?: boolean; + readonly target: string; +} + +const runClaudeCommand: ClaudePluginCommandRunner = (request) => runBoundedChildProcess(request, { + labels: { outputLimit: 'output-limit', timedOut: 'timed-out' }, + maxOutputBytes: maximumOutputBytes, + timeoutMs: request.args[0] === '--version' ? versionTimeoutMs : validationTimeoutMs, + windowsHide: true, +}); + +const versionFrom = (output: string): string | undefined => + /(?:^|\s)(\d+\.\d+\.\d+)(?:\s|$)/u.exec(output)?.[1]; + +const diagnostic = ( + code: 'AB6019' | 'AB6020' | 'AB6021' | 'AB6022', + message: string, + severity: DiagnosticSeverity, + target: string, +): Diagnostic => Object.freeze({ + code, + message, + recovery: code === 'AB6019' + ? 'Install Claude Code and ensure `claude` is on PATH, then rerun artifact validation.' + : 'Run `claude plugin validate --strict`, repair the reported Claude artifact, and rebuild.', + severity, + target, +}); + +const issueLines = (output: string): readonly { readonly message: string; readonly severity: 'error' | 'warning' }[] => { + const issues: { message: string; severity: 'error' | 'warning' }[] = []; + let section: 'error' | 'warning' | undefined; + for (const rawLine of output.split(/\r?\n/u)) { + const line = rawLine.trim(); + if (/Found \d+ warnings?:/u.test(line)) { + section = 'warning'; + continue; + } + if (/Found \d+ errors?:/u.test(line)) { + section = 'error'; + continue; + } + if (!line.startsWith('❯ ') || section === undefined) continue; + issues.push(Object.freeze({ message: line.slice(2).trim(), severity: section })); + } + return Object.freeze(issues); +}; + +export const validateClaudePlugin = async ( + options: ValidateClaudePluginOptions, +): Promise => { + const executable = options.executable ?? 'claude'; + const run = options.run ?? runClaudeCommand; + const cwd = dirname(options.pluginDirectory); + let version: string | undefined; + try { + const probe = await run(Object.freeze({ args: Object.freeze(['--version']), cwd, executable })); + if (probe.exitCode !== 0 || probe.termination !== undefined) { + return Object.freeze({ + diagnostics: freezeDiagnostics([diagnostic( + 'AB6019', + 'The Claude CLI is unavailable for host artifact validation.', + 'info', + options.target, + )]), + host: 'claude', + status: 'unavailable', + target: options.target, + }); + } + version = versionFrom(`${probe.stdout}\n${probe.stderr}`); + } catch (error) { + return Object.freeze({ + diagnostics: freezeDiagnostics([diagnostic( + 'AB6019', + isErrno(error, 'ENOENT') + ? 'The Claude CLI is not installed or is not on PATH; host artifact validation was skipped.' + : 'The Claude CLI could not be started; host artifact validation was skipped.', + 'info', + options.target, + )]), + host: 'claude', + status: 'unavailable', + target: options.target, + }); + } + + let result: ClaudePluginCommandResult; + try { + result = await run(Object.freeze({ + args: Object.freeze(['plugin', 'validate', options.pluginDirectory, '--strict']), + cwd, + executable, + })); + } catch { + return Object.freeze({ + diagnostics: freezeDiagnostics([diagnostic( + 'AB6022', + 'Claude host artifact validation could not be started.', + 'error', + options.target, + )]), + host: 'claude', + status: 'failed', + target: options.target, + ...(version === undefined ? {} : { version }), + }); + } + + if (result.termination !== undefined) { + return Object.freeze({ + diagnostics: freezeDiagnostics([diagnostic( + 'AB6022', + result.termination === 'timed-out' + ? 'Claude host artifact validation timed out.' + : 'Claude host artifact validation exceeded its output limit.', + 'error', + options.target, + )]), + host: 'claude', + status: 'failed', + target: options.target, + ...(version === undefined ? {} : { version }), + }); + } + + const parsed = issueLines(`${result.stdout}\n${result.stderr}`); + const diagnostics = freezeDiagnostics(parsed.map((issue) => diagnostic( + issue.severity === 'warning' ? 'AB6020' : 'AB6021', + `Claude plugin validation: ${issue.message}`, + issue.severity === 'warning' && options.strict !== true ? 'warning' : 'error', + options.target, + ))); + if (result.exitCode !== 0 && diagnostics.length === 0) { + return Object.freeze({ + diagnostics: freezeDiagnostics([diagnostic( + 'AB6022', + 'Claude host artifact validation failed without structured issue output.', + 'error', + options.target, + )]), + host: 'claude', + status: 'failed', + target: options.target, + ...(version === undefined ? {} : { version }), + }); + } + const failed = diagnostics.some((entry) => entry.severity === 'error'); + return Object.freeze({ + diagnostics, + host: 'claude', + status: failed ? 'failed' : diagnostics.length === 0 ? 'passed' : 'warnings', + target: options.target, + ...(version === undefined ? {} : { version }), + }); +}; diff --git a/packages/agent-bundle/src/test/index.ts b/packages/agent-bundle/src/test/index.ts index 5943a1e74..d214a3c45 100644 --- a/packages/agent-bundle/src/test/index.ts +++ b/packages/agent-bundle/src/test/index.ts @@ -81,3 +81,11 @@ export type { RenderableRouteKind, RenderedRouteProvenance, } from './types.ts'; +export { validateClaudePlugin } from '../host-contracts/claude-plugin-validation.ts'; +export type { + ClaudePluginCommandResult, + ClaudePluginCommandRunner, + ClaudePluginValidationReport, + ClaudePluginValidationStatus, + ValidateClaudePluginOptions, +} from '../host-contracts/claude-plugin-validation.ts'; diff --git a/packages/agent-bundle/tests/adapter-metadata.test.ts b/packages/agent-bundle/tests/adapter-metadata.test.ts index 45745a924..ed81f8548 100644 --- a/packages/agent-bundle/tests/adapter-metadata.test.ts +++ b/packages/agent-bundle/tests/adapter-metadata.test.ts @@ -102,7 +102,7 @@ it('records exact immutable metadata for every built-in target', () => { expect(registryMetadata(registry, 'claude')).toEqual({ adapterRevision: '1.2.0', capabilityRevision: '2.1.250', - capabilitySha256: '5beb395c075c22a290a70f76b1670fab82b16d933af37cda89da850dbd8d483c', + capabilitySha256: '13bc41224c5343b33d259986a66feb279e15431c5019bb2a1c443eaa60e9a9ea', observedVersion: '2.1.250', schemas: [ { diff --git a/packages/agent-bundle/tests/artifact-validator.test.ts b/packages/agent-bundle/tests/artifact-validator.test.ts index cdf7d080a..e2972a4df 100644 --- a/packages/agent-bundle/tests/artifact-validator.test.ts +++ b/packages/agent-bundle/tests/artifact-validator.test.ts @@ -1818,7 +1818,8 @@ it('documents recovery for every stable artifact diagnostic code', async () => { expect(Object.keys(artifactDiagnosticRecoveries).sort()).toEqual([ 'AB6000', 'AB6001', 'AB6002', 'AB6003', 'AB6004', 'AB6005', 'AB6006', 'AB6007', 'AB6008', 'AB6009', 'AB6010', 'AB6011', 'AB6012', 'AB6013', - 'AB6014', 'AB6015', 'AB6016', 'AB6017', 'AB6018', + 'AB6014', 'AB6015', 'AB6016', 'AB6017', 'AB6018', 'AB6019', 'AB6020', + 'AB6021', 'AB6022', ]); expect(Object.values(artifactDiagnosticRecoveries).every((recovery) => recovery.trim().length > 0)).toBe(true); expect(artifactDiagnosticRecoveries.AB6015).not.toBe(artifactDiagnosticRecoveries.AB6016); diff --git a/packages/agent-bundle/tests/claude-plugin-validation.test.ts b/packages/agent-bundle/tests/claude-plugin-validation.test.ts new file mode 100644 index 000000000..15c156ddd --- /dev/null +++ b/packages/agent-bundle/tests/claude-plugin-validation.test.ts @@ -0,0 +1,124 @@ +import { expect, it } from '@rstest/core'; + +import { + validateClaudePlugin, + type ClaudePluginCommandRunner, +} from '../src/host-contracts/claude-plugin-validation.ts'; + +const runWith = ( + validation: Readonly<{ exitCode: number; stderr?: string; stdout: string }>, +): { readonly calls: unknown[]; readonly run: ClaudePluginCommandRunner } => { + const calls: unknown[] = []; + return { + calls, + run: async (request) => { + calls.push(request); + return request.args[0] === '--version' + ? { exitCode: 0, signal: null, stderr: '', stdout: '2.1.251 (Claude Code)\n' } + : { exitCode: validation.exitCode, signal: null, stderr: validation.stderr ?? '', stdout: validation.stdout }; + }, + }; +}; + +it('runs the installed Claude validator without shell interpolation', async () => { + const fixture = runWith({ exitCode: 0, stdout: '✔ Validation passed\n' }); + const report = await validateClaudePlugin({ + pluginDirectory: '/tmp/plugin with spaces; echo unsafe', + run: fixture.run, + target: 'claude', + }); + + expect(fixture.calls).toEqual([ + expect.objectContaining({ args: ['--version'], executable: 'claude' }), + expect.objectContaining({ + args: ['plugin', 'validate', '/tmp/plugin with spaces; echo unsafe', '--strict'], + executable: 'claude', + }), + ]); + expect(report).toEqual({ + diagnostics: [], + host: 'claude', + status: 'passed', + target: 'claude', + version: '2.1.251', + }); +}); + +it('keeps host warnings as warnings unless framework strict mode is enabled', async () => { + const output = [ + '⚠ Found 2 warnings:', + '', + " ❯ displayNme: Unknown field 'displayNme' — did you mean 'displayName'?", + ' ❯ author: No author information provided.', + '', + '✘ Validation failed (--strict treats warnings as errors)', + ].join('\n'); + + const normal = await validateClaudePlugin({ + pluginDirectory: '/tmp/plugin', + run: runWith({ exitCode: 1, stdout: output }).run, + target: 'plugin', + }); + expect(normal.status).toBe('warnings'); + expect(normal.diagnostics).toEqual([ + expect.objectContaining({ code: 'AB6020', severity: 'warning', target: 'plugin' }), + expect.objectContaining({ code: 'AB6020', severity: 'warning', target: 'plugin' }), + ]); + + const strict = await validateClaudePlugin({ + pluginDirectory: '/tmp/plugin', + run: runWith({ exitCode: 1, stdout: output }).run, + strict: true, + target: 'plugin', + }); + expect(strict.status).toBe('failed'); + expect(strict.diagnostics.every((diagnostic) => diagnostic.severity === 'error')).toBe(true); +}); + +it('maps Claude validation errors and bounded-process failures to stable diagnostics', async () => { + const invalid = await validateClaudePlugin({ + pluginDirectory: '/tmp/plugin', + run: runWith({ + exitCode: 1, + stdout: '✘ Found 1 error:\n\n ❯ name: Invalid input: expected string, received undefined\n\n✘ Validation failed\n', + }).run, + target: 'claude', + }); + expect(invalid).toMatchObject({ + diagnostics: [expect.objectContaining({ code: 'AB6021', severity: 'error', target: 'claude' })], + status: 'failed', + }); + + const timedOut = await validateClaudePlugin({ + pluginDirectory: '/tmp/plugin', + run: async (request) => request.args[0] === '--version' + ? { exitCode: 0, signal: null, stderr: '', stdout: '2.1.251\n' } + : { exitCode: null, signal: 'SIGTERM', stderr: '', stdout: '', termination: 'timed-out' }, + target: 'claude', + }); + expect(timedOut).toMatchObject({ + diagnostics: [expect.objectContaining({ code: 'AB6022', severity: 'error' })], + status: 'failed', + }); +}); + +it('reports an honest informational skip when Claude is absent', async () => { + const missing = Object.assign(new Error('spawn claude ENOENT'), { code: 'ENOENT' }); + const report = await validateClaudePlugin({ + pluginDirectory: '/tmp/plugin', + run: async () => { throw missing; }, + target: 'claude', + }); + + expect(report).toEqual({ + diagnostics: [expect.objectContaining({ + code: 'AB6019', + message: expect.stringContaining('not installed or is not on PATH'), + severity: 'info', + target: 'claude', + })], + host: 'claude', + status: 'unavailable', + target: 'claude', + }); +}); diff --git a/packages/agent-bundle/tests/cli.test.ts b/packages/agent-bundle/tests/cli.test.ts index 8ce3a9d88..f0fa78e86 100644 --- a/packages/agent-bundle/tests/cli.test.ts +++ b/packages/agent-bundle/tests/cli.test.ts @@ -6,7 +6,7 @@ import { promisify } from 'node:util'; import { expect, it } from '@rstest/core'; -import { runCli as runSourceCli } from '../src/cli.ts'; +import { runCli as runSourceCli, type CliDependencies } from '../src/cli.ts'; import { cachedNpmInstallArguments } from './support/shared-pack.ts'; import { timeScale } from './support/time-scale.ts'; @@ -38,14 +38,17 @@ const runExecutable = async (executable: string, root: string, args: readonly st const runCli = (root: string, args: readonly string[]) => runExecutable(cliPath, root, args); -const runSourceCliWithOutput = async (args: string[]): Promise<{ readonly code: number; readonly stderr: string; readonly stdout: string }> => { +const runSourceCliWithOutput = async ( + args: string[], + dependencies: CliDependencies = {}, +): Promise<{ readonly code: number; readonly stderr: string; readonly stdout: string }> => { const stderr: string[] = []; const stdout: string[] = []; Object.defineProperty(globalThis, '__AGENT_BUNDLE_VERSION__', { configurable: true, value: 'test' }); const code = await runSourceCli(args, { stderr: { write: (chunk: string) => stderr.push(chunk) }, stdout: { write: (chunk: string) => stdout.push(chunk) }, - }); + }, dependencies); return { code, stderr: stderr.join(''), stdout: stdout.join('') }; }; @@ -355,6 +358,37 @@ it('keeps inspect JSON stable and validates only the supplied artifact', async ( } }, 30_000 * timeScale); +it('enables bounded host validation for built artifacts and promotes warnings only under --strict', async () => { + const calls: unknown[] = []; + const validate = async (options: unknown) => { + calls.push(options); + return { + diagnostics: [{ + code: 'AB6020', + message: 'Claude plugin validation warning.', + severity: (options as { strict?: boolean }).strict === true ? 'error' as const : 'warning' as const, + }], + }; + }; + + const normal = await runSourceCliWithOutput([ + 'validate', '--root', '/project', '--artifact', '/artifact', '--json', + ], { validate }); + expect(normal).toMatchObject({ code: 0, stderr: '' }); + expect(JSON.parse(normal.stdout)).toMatchObject({ + diagnostics: [expect.objectContaining({ code: 'AB6020', severity: 'warning' })], + }); + + const strict = await runSourceCliWithOutput([ + 'validate', '--root', '/project', '--artifact', '/artifact', '--strict', '--json', + ], { validate }); + expect(strict.code).toBe(1); + expect(calls).toEqual([ + expect.objectContaining({ artifact: '/artifact', hostValidation: true, strict: undefined }), + expect.objectContaining({ artifact: '/artifact', hostValidation: true, strict: true }), + ]); +}); + it('prints a complete invalid inspection on JSON and human output', async () => { const project = await createCliProject(); try { diff --git a/packages/agent-bundle/tests/packed-consumer.test.ts b/packages/agent-bundle/tests/packed-consumer.test.ts index aec4d5f95..f13868085 100644 --- a/packages/agent-bundle/tests/packed-consumer.test.ts +++ b/packages/agent-bundle/tests/packed-consumer.test.ts @@ -225,7 +225,32 @@ it('uses only an installed tarball after source deletion', async () => { const { stdout: validation } = await runInstalled(cli, projectRoot, [ 'validate', '--json', '--root', projectRoot, '--artifact', artifact, ]); - expect(validation).toBe('{"diagnostics":[]}\n'); + const validationDocument = JSON.parse(validation) as { + readonly diagnostics: readonly { readonly code: string; readonly severity: string }[]; + readonly hostValidation: readonly { + readonly diagnostics: readonly { readonly code: string; readonly severity: string }[]; + readonly host: string; + readonly status: string; + readonly target: string; + }[]; + }; + expect(validationDocument.hostValidation).toHaveLength(1); + expect(validationDocument.hostValidation[0]).toMatchObject({ + host: 'claude', + target: 'claude', + }); + if (validationDocument.hostValidation[0]!.status === 'passed') { + expect(validationDocument.diagnostics).toEqual([]); + expect(validationDocument.hostValidation[0]!.diagnostics).toEqual([]); + } else { + expect(validationDocument.hostValidation[0]).toMatchObject({ + diagnostics: [{ code: 'AB6019', severity: 'info' }], + status: 'unavailable', + }); + expect(validationDocument.diagnostics).toEqual([ + expect.objectContaining({ code: 'AB6019', severity: 'info' }), + ]); + } const bundlePath = join(artifact, 'portable', 'scripts', 'bundle.mjs'); await expect(execFile(process.execPath, [ diff --git a/packages/agent-bundle/tests/packed-native-smoke.test.ts b/packages/agent-bundle/tests/packed-native-smoke.test.ts index ce356c06a..0c6843047 100644 --- a/packages/agent-bundle/tests/packed-native-smoke.test.ts +++ b/packages/agent-bundle/tests/packed-native-smoke.test.ts @@ -1,4 +1,5 @@ import { mkdir, mkdtemp, rm, utimes, writeFile } from 'node:fs/promises'; +import { spawnSync } from 'node:child_process'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -11,6 +12,11 @@ const enabledHosts = [ ...(process.env.AGENT_BUNDLE_PACKED_NATIVE_CODEX_SMOKE === '1' ? ['codex' as const] : []), ]; const nativeIt = enabledHosts.length === 0 ? it.skip : it; +const claudePluginIt = spawnSync('claude', ['--version'], { + stdio: 'ignore', + timeout: 5_000, + windowsHide: true, +}).status === 0 ? it : it.skip; it('requires a host-specific opt-in and keeps the canonical Claude model pinned', async () => { const harness = await loadPackedNativeSmoke(); @@ -162,6 +168,19 @@ it('opaquely detects default ~/.claude.json mutation without extending custom co } }); +claudePluginIt('builds, strictly validates, and smoke-loads a packed Claude artifact', async () => { + const harness = await loadPackedNativeSmoke(); + expect(harness).toBeDefined(); + + await expect(harness!.runPackedClaudePluginProof({ environment: process.env })).resolves.toEqual({ + host: 'claude', + registration: 'observed', + status: 'passed', + strictValidation: 'passed', + version: expect.stringMatching(/^\d+\.\d+\.\d+$/u), + }); +}, 600_000); + nativeIt('runs opted-in authored Eval hosts through one production-only packed installation', async () => { const harness = await loadPackedNativeSmoke(); expect(harness).toBeDefined(); diff --git a/packages/agent-bundle/tests/support/packed-native-smoke.ts b/packages/agent-bundle/tests/support/packed-native-smoke.ts index 00e244612..c88413503 100644 --- a/packages/agent-bundle/tests/support/packed-native-smoke.ts +++ b/packages/agent-bundle/tests/support/packed-native-smoke.ts @@ -17,7 +17,7 @@ import { promisify } from 'node:util'; // The native smoke installs the production closure a real consumer would get, // so it stays on npm's default metadata staleness checks. -import { npmInstallArguments } from './shared-pack.ts'; +import { npmInstallArguments, sharedPackedTarball } from './shared-pack.ts'; const execFile = promisify(executeFile); const workspaceRoot = process.cwd(); @@ -50,6 +50,14 @@ export interface PackedNativeSmokeReport { }; } +export interface PackedClaudePluginProof { + readonly host: 'claude'; + readonly registration: 'observed'; + readonly status: 'passed'; + readonly strictValidation: 'passed'; + readonly version: string; +} + const hostOptIns = Object.freeze({ claude: 'AGENT_BUNDLE_PACKED_NATIVE_CLAUDE_SMOKE', codex: 'AGENT_BUNDLE_PACKED_NATIVE_CODEX_SMOKE', @@ -206,6 +214,85 @@ const summarizeEval = (host: PackedNativeHost, command: CommandResult) => { return Object.freeze({ host, status: passed ? 'passed' as const : 'failed' as const, trials: summary?.trials ?? 0 }); }; +/** + * Packed-artifact proof for Claude's developer tools. It requires only the + * installed binary, never authentication, and retains no plugin-list output. + */ +export const runPackedClaudePluginProof = async (options: { + readonly environment: Readonly; +}): Promise => { + const root = await mkdtemp(join(tmpdir(), 'agent-bundle-packed-claude-plugin-')); + const consumer = join(root, 'consumer'); + const project = join(consumer, 'project'); + const artifact = join(project, 'artifact'); + const environment = packedNativeEnvironment(options.environment); + + try { + await mkdir(consumer, { recursive: true }); + await Promise.all([ + writeFile(join(consumer, 'package.json'), '{"private":true,"type":"module"}\n'), + cp(fixtureRoot, project, { recursive: true }), + ]); + const packed = await sharedPackedTarball('agent-bundle'); + const installed = await run('npm', [ + 'install', + '--omit=dev', + ...npmInstallArguments, + packed.tarball, + ], { cwd: consumer, environment }); + if (installed.exitCode !== 0) throw new Error('packed-claude-proof:install'); + + const cli = await realpath(join(consumer, 'node_modules', 'agent-bundle', 'dist', 'cli.js')); + if (cli.startsWith(workspaceRoot)) throw new Error('Packed Claude plugin proof resolved a workspace-linked binary.'); + const built = await runNodeEntrypoint(cli, [ + 'build', + '--root', + project, + '--output', + artifact, + ], { cwd: project, environment }); + if (built.exitCode !== 0) throw new Error('packed-claude-proof:build'); + + const pluginDirectory = join(artifact, 'claude'); + const version = await run('claude', ['--version'], { cwd: project, environment }); + const versionNumber = /(?:^|\s)(\d+\.\d+\.\d+)(?:\s|$)/u.exec(version.stdout)?.[1]; + if (version.exitCode !== 0 || versionNumber === undefined) { + throw new Error('packed-claude-proof:version'); + } + const validation = await run('claude', ['plugin', 'validate', pluginDirectory, '--strict'], { + cwd: project, + environment, + }); + if (validation.exitCode !== 0) throw new Error('packed-claude-proof:validate'); + const plugins = await run('claude', ['--plugin-dir', pluginDirectory, 'plugin', 'list', '--json'], { + cwd: project, + environment, + }); + let listingDocument: readonly { readonly id?: unknown }[] = []; + try { + listingDocument = JSON.parse(plugins.stdout) as readonly { readonly id?: unknown }[]; + } catch { + listingDocument = []; + } + if ( + plugins.exitCode !== 0 || + !Array.isArray(listingDocument) || + !listingDocument.some((plugin) => plugin.id === 'packed-native-smoke@inline') + ) { + throw new Error('packed-claude-proof:register'); + } + return Object.freeze({ + host: 'claude', + registration: 'observed', + status: 'passed', + strictValidation: 'passed', + version: versionNumber, + }); + } finally { + await rm(root, { force: true, recursive: true }); + } +}; + export const runPackedNativeSmoke = async (options: { readonly environment: Readonly; }): Promise => {