diff --git a/.changeset/codex-host-validation.md b/.changeset/codex-host-validation.md new file mode 100644 index 000000000..f2ad52bb5 --- /dev/null +++ b/.changeset/codex-host-validation.md @@ -0,0 +1,5 @@ +--- +"agent-bundle": minor +--- + +Validate built Codex bundles against vendored pinned schemas, report the host's missing plugin-validation developer tool honestly, and expose bounded app-server schema drift evidence through the public and test APIs. diff --git a/docs/diagnostics.md b/docs/diagnostics.md index c77093a71..3970c3089 100644 --- a/docs/diagnostics.md +++ b/docs/diagnostics.md @@ -34,6 +34,20 @@ gate a build, a validation, or a dev rebuild. | `AB8xxx` | Development server configuration. | | `AB9xxx` | Eval selection, harnesses, and persisted runs. | +## Codex host validation (`AB6030`–`AB6033`) + +Codex 0.147.0 publishes plugin installation commands but no plugin-validation +developer tool. Agent Bundle therefore validates built Codex JSON documents +against its vendored pinned schemas and treats the app-server schema generator +as a separate drift signal, never as a substitute plugin contract. + +| Code | Severity | Meaning | Recovery | +| --- | --- | --- | --- | +| `AB6030` | info | The Codex CLI is unavailable, or the installed Codex release publishes no plugin validation command. | Install Codex and put it on `PATH`; until Codex publishes a validator, use the vendored pinned-schema diagnostics. | +| `AB6031` | info / warning (error in strict mode) | The app-server schema-generation verb is unavailable, or its live output is missing or differs from the pinned generated hook schemas. | Review the attributable host schema source and update the pinned revision only when Codex publishes the matching contract. | +| `AB6032` | error | A required Codex bundle document is missing, unreadable, invalid JSON, or fails its vendored pinned schema. | Repair the named `.codex-plugin/plugin.json`, `hooks/hooks.json`, `.mcp.json`, or marketplace document and rebuild. | +| `AB6033` | error | A bounded Codex version or schema-generation command could not start, failed, timed out, exceeded 1 MiB of output, or produced unreadable output. | Verify `codex --version` and `codex app-server generate-json-schema --out ` complete successfully, then rerun validation. | + ## npm prepack gate (`AB7010`–`AB7013`) | Code | Meaning | diff --git a/packages/agent-bundle/src/adapters/capabilities/codex-0.147.0.json b/packages/agent-bundle/src/adapters/capabilities/codex-0.147.0.json index cef4fe49b..07b17d34d 100644 --- a/packages/agent-bundle/src/adapters/capabilities/codex-0.147.0.json +++ b/packages/agent-bundle/src/adapters/capabilities/codex-0.147.0.json @@ -50,6 +50,57 @@ "marketplace": ".agents/plugins/marketplace.json", "skills": true }, + "validation": { + "observedAt": "2026-09-02", + "versionProbe": { + "command": "codex --version", + "exitCode": 0, + "stdout": "codex-cli 0.147.0" + }, + "pluginDeveloperTools": { + "command": "codex plugin --help", + "exitCode": 0, + "subcommands": ["add", "list", "marketplace", "remove", "help"], + "validator": { + "reason": "Codex 0.147.0 lists plugin installation and marketplace management only; it publishes no plugin validate command.", + "state": "unavailable" + } + }, + "appServerSchemaGeneration": { + "command": "codex app-server generate-json-schema --out ", + "exitCode": 0, + "jsonFileCount": 285, + "stderrBytes": 0, + "stdoutBytes": 0, + "topLevelBundles": { + "codex_app_server_protocol.schemas.json": { + "sha256": "f72b2caa3cbfa4298de9e85c62dda6dfbaf2266ffeb916fed30615ca69ff8c74" + }, + "codex_app_server_protocol.v2.schemas.json": { + "sha256": "f3dec1e031d99a420b137b903f02196d4325eece57620c925bb7130b25f168d2" + } + } + }, + "pinnedGeneratedComparison": { + "liveRevision": "codex-cli 0.147.0 app-server protocol", + "matchingRelativePaths": [], + "missingPinnedRelativePaths": [ + "subagent-start.command.input.schema.json", + "subagent-start.command.output.schema.json", + "subagent-stop.command.input.schema.json", + "subagent-stop.command.output.schema.json" + ], + "pinnedRepositorySha256": { + "subagent-start.command.input.schema.json": "e1cacc5cd92217e96e327cf182038fa93099d194c3107439b4dad4b806d414cc", + "subagent-start.command.output.schema.json": "531f7a457ad8430de82388319ff2bf030fd3a1dc0e9a0d4078447bc30948448b", + "subagent-stop.command.input.schema.json": "27842578768e74fb8bcd86b30156b207011829a81dc01b00cfd55340df8b079f", + "subagent-stop.command.output.schema.json": "a3987dab22b8684ab108bbb76ec5306471d9e01b1b7cac007b1ed90bc9e055cf" + }, + "pinnedRevision": "rust-v0.147.0 hook command schemas", + "reason": "The command emits app-server protocol schemas, not the vendored hook command schemas. There are no matching relative paths to hash-compare, so this output is unpinned for plugin conformance.", + "state": "unavailable" + } + }, "tokens": { "pluginData": false, "pluginRoot": "relative-with-plugin-root-cwd", diff --git a/packages/agent-bundle/src/api.ts b/packages/agent-bundle/src/api.ts index 806c9fd4c..9906cde8d 100644 --- a/packages/agent-bundle/src/api.ts +++ b/packages/agent-bundle/src/api.ts @@ -95,6 +95,10 @@ import { validateClaudePlugin, type ClaudePluginValidationReport, } from './host-contracts/claude-plugin-validation.ts'; +import { + validateCodexPlugin, + type CodexPluginValidationReport, +} from './host-contracts/codex-plugin-validation.ts'; import type { EvalComparison } from './eval/compare.ts'; import { EvalRunStoreError } from './eval/errors.ts'; import { @@ -160,6 +164,8 @@ export type { NativeHost, RedactedEventEnvelope, } from './host-contracts/host-contract.ts'; +export { validateClaudePlugin, validateCodexPlugin }; +export type { ClaudePluginValidationReport, CodexPluginValidationReport }; export { HookService } from './services/hook-service.ts'; export type { HookListOptions, HookSimulationOptions } from './services/hook-service.ts'; @@ -237,7 +243,7 @@ export interface ValidateOptions extends ProjectOptions { export interface ValidateResult { readonly diagnostics: readonly Diagnostic[]; - readonly hostValidation?: readonly ClaudePluginValidationReport[]; + readonly hostValidation?: readonly (ClaudePluginValidationReport | CodexPluginValidationReport)[]; readonly model?: NormalizedPlugin; } @@ -478,12 +484,18 @@ export const validate = async (options: ValidateOptions): Promise target.name === 'claude' || target.name === 'plugin') - .map((target) => validateClaudePlugin({ - pluginDirectory: join(artifact, target.name), - strict: options.strict, - target: target.name, - }))); + .filter((target) => target.name === 'claude' || target.name === 'codex' || target.name === 'plugin') + .map((target) => target.name === 'codex' + ? validateCodexPlugin({ + pluginDirectory: join(artifact, target.name), + strict: options.strict, + target: target.name, + }) + : validateClaudePlugin({ + pluginDirectory: join(artifact, target.name), + strict: options.strict, + target: target.name, + }))); return Object.freeze({ diagnostics: freezeDiagnostics([ ...validated.diagnostics, diff --git a/packages/agent-bundle/src/host-contracts/codex-plugin-validation.ts b/packages/agent-bundle/src/host-contracts/codex-plugin-validation.ts new file mode 100644 index 000000000..910ef6dcb --- /dev/null +++ b/packages/agent-bundle/src/host-contracts/codex-plugin-validation.ts @@ -0,0 +1,410 @@ +import { mkdtemp, readFile, readdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; + +import type { Diagnostic, DiagnosticSeverity } from '../core/diagnostics.ts'; +import { freezeDiagnostics } from '../core/diagnostics.ts'; +import { sha256Hex } from '../core/digest.ts'; +import { isErrno } from '../core/errors.ts'; +import capabilityTable from '../adapters/capabilities/codex-0.147.0.json' with { type: 'json' }; +import hooksSchema from '../adapters/schemas/codex/hooks.schema.json' with { type: 'json' }; +import marketplaceSchema from '../adapters/schemas/codex/marketplace.schema.json' with { type: 'json' }; +import mcpSchema from '../adapters/schemas/codex/mcp.schema.json' with { type: 'json' }; +import pluginSchema from '../adapters/schemas/codex/plugin.schema.json' with { type: 'json' }; +import { + createAdapterValidator, + validateJsonSchemaDocument, + type TargetArtifactDocumentValidator, +} from '../adapters/types.ts'; +import { + runBoundedChildProcess, + type BoundedChildProcessRequest, + type BoundedChildProcessResult, +} from './process.ts'; + +const maximumOutputBytes = 1024 * 1024; +const schemaGenerationTimeoutMs = 15_000; +const versionTimeoutMs = 5_000; +const pinnedRevision = capabilityTable.observedCliVersion; + +const generatedSchemaNames = Object.freeze([ + 'subagent-start.command.input.schema.json', + 'subagent-start.command.output.schema.json', + 'subagent-stop.command.input.schema.json', + 'subagent-stop.command.output.schema.json', +]); + +type CodexPluginTermination = 'output-limit' | 'timed-out'; +type CodexPluginDiagnosticCode = 'AB6030' | 'AB6031' | 'AB6032' | 'AB6033'; + +export type CodexPluginValidationStatus = 'failed' | 'passed' | 'unavailable' | 'warnings'; + +export interface CodexPluginValidationReport { + readonly diagnostics: readonly Diagnostic[]; + readonly host: 'codex'; + readonly status: CodexPluginValidationStatus; + readonly target: string; + readonly version?: string; +} + +export type CodexPluginCommandResult = BoundedChildProcessResult; + +export type CodexPluginCommandRunner = ( + request: BoundedChildProcessRequest, +) => Promise; + +export interface ValidateCodexPluginOptions { + readonly executable?: string; + readonly pluginDirectory: string; + /** Injectable proof seam. Production always uses the bounded process runner. */ + readonly run?: CodexPluginCommandRunner; + /** Promote live-schema drift to an Agent Bundle error. */ + readonly strict?: boolean; + readonly target: string; +} + +interface PinnedDocumentContract { + readonly path: string; + readonly required: boolean; + readonly validate: TargetArtifactDocumentValidator; +} + +const schemaValidator = createAdapterValidator(); +const pinnedDocumentContracts = Object.freeze([ + Object.freeze({ + path: '.codex-plugin/plugin.json', + required: true, + validate: validateJsonSchemaDocument(schemaValidator.compile(pluginSchema)), + }), + Object.freeze({ + path: 'hooks/hooks.json', + required: false, + validate: validateJsonSchemaDocument(schemaValidator.compile(hooksSchema)), + }), + Object.freeze({ + path: '.mcp.json', + required: false, + validate: validateJsonSchemaDocument(schemaValidator.compile(mcpSchema)), + }), + Object.freeze({ + path: '.agents/plugins/marketplace.json', + required: false, + validate: validateJsonSchemaDocument(schemaValidator.compile(marketplaceSchema)), + }), +]); + +const runCodexCommand: CodexPluginCommandRunner = (request) => runBoundedChildProcess(request, { + labels: { outputLimit: 'output-limit', timedOut: 'timed-out' }, + maxOutputBytes: maximumOutputBytes, + timeoutMs: request.args[0] === '--version' ? versionTimeoutMs : schemaGenerationTimeoutMs, + windowsHide: true, +}); + +const versionFrom = (output: string): string | undefined => + /(?:^|\s)(\d+\.\d+\.\d+)(?:\s|$)/u.exec(output)?.[1]; + +const diagnostic = ( + code: CodexPluginDiagnosticCode, + message: string, + severity: DiagnosticSeverity, + target: string, + recovery: string, + generatedPath?: string, +): Diagnostic => Object.freeze({ + code, + ...(generatedPath === undefined ? {} : { generatedPath }), + message, + recovery, + severity, + target, +}); + +const commandFailureMessage = ( + operation: 'schema generation' | 'version probe', + result: CodexPluginCommandResult, +): string => { + if (result.termination === 'timed-out') return `Codex CLI ${operation} timed out.`; + if (result.termination === 'output-limit') return `Codex CLI ${operation} exceeded its output limit.`; + return `Codex CLI ${operation} exited with code ${result.exitCode ?? 'unknown'}.`; +}; + +const validatePinnedDocuments = async ( + pluginDirectory: string, + target: string, +): Promise => { + const diagnostics: Diagnostic[] = []; + for (const contract of pinnedDocumentContracts) { + const path = join(pluginDirectory, contract.path); + let source: string; + try { + source = await readFile(path, 'utf8'); + } catch (error) { + if (isErrno(error, 'ENOENT') && !contract.required) continue; + diagnostics.push(diagnostic( + 'AB6032', + isErrno(error, 'ENOENT') + ? `Required Codex bundle document ${contract.path} is missing.` + : `Codex bundle document ${contract.path} could not be read.`, + 'error', + target, + 'Rebuild the Codex bundle, then repair any document that does not satisfy its vendored pinned schema.', + contract.path, + )); + continue; + } + + let document: unknown; + try { + document = JSON.parse(source); + } catch { + diagnostics.push(diagnostic( + 'AB6032', + `Codex bundle document ${contract.path} is not valid JSON.`, + 'error', + target, + 'Repair the generated JSON document and rebuild the Codex bundle.', + contract.path, + )); + continue; + } + + for (const issue of contract.validate(document)) { + diagnostics.push(diagnostic( + 'AB6032', + `Codex bundle document ${contract.path}${issue.instancePath} ${issue.message}.`, + 'error', + target, + 'Repair the generated Codex document so it satisfies the vendored pinned schema, then rebuild.', + contract.path, + )); + } + } + return freezeDiagnostics(diagnostics); +}; + +const generatedJsonFiles = async (directory: string): Promise => + Object.freeze((await readdir(directory, { recursive: true })) + .filter((path) => path.endsWith('.json')) + .sort()); + +const compareGeneratedSchemas = async ( + liveDirectory: string, + liveVersion: string | undefined, + strict: boolean, + target: string, +): Promise => { + const pinnedDirectory = new URL('../adapters/schemas/codex/generated/', import.meta.url); + const missing: string[] = []; + const changed: string[] = []; + for (const name of generatedSchemaNames) { + try { + const [live, pinned] = await Promise.all([ + readFile(join(liveDirectory, name)), + readFile(new URL(name, pinnedDirectory)), + ]); + if (sha256Hex(live) !== sha256Hex(pinned)) changed.push(name); + } catch (error) { + if (isErrno(error, 'ENOENT')) { + missing.push(name); + continue; + } + throw error; + } + } + if (missing.length === 0 && changed.length === 0) return Object.freeze([]); + + const liveFiles = await generatedJsonFiles(liveDirectory); + const appServerBundlesPresent = [ + 'codex_app_server_protocol.schemas.json', + 'codex_app_server_protocol.v2.schemas.json', + ].every((path) => liveFiles.includes(path)); + if (missing.length === generatedSchemaNames.length && appServerBundlesPresent) { + return freezeDiagnostics([diagnostic( + 'AB6031', + `Codex ${liveVersion ?? 'unknown'} live hook-schema drift is not assessable because the generator emits ` + + `the app-server protocol surface, which is unpinned for plugin hook validation against Codex ${pinnedRevision}.`, + 'info', + target, + 'Retain validation against the vendored pinned hook schemas until Codex publishes a comparable live hook-schema surface.', + )]); + } + const details = [ + ...(missing.length === 0 ? [] : [`missing ${missing.join(', ')}`]), + ...(changed.length === 0 ? [] : [`changed ${changed.join(', ')}`]), + ].join('; '); + return freezeDiagnostics([diagnostic( + 'AB6031', + `The live Codex ${liveVersion ?? 'unknown'} schema set does not match pinned Codex ${pinnedRevision} ` + + `generated hook schemas (${details}); the command emitted ${liveFiles.length} JSON schema files.`, + strict ? 'error' : 'warning', + target, + 'Review the live host schema source and update the pinned revision only from a published, attributable Codex contract.', + )]); +}; + +const schemaVerbUnavailable = (output: string): boolean => + /(?:unrecognized|unknown|invalid) (?:subcommand|command)|no such (?:subcommand|command)/iu.test(output); + +const schemaGenerationDiagnostics = async ( + options: Readonly<{ + readonly cwd: string; + readonly executable: string; + readonly run: CodexPluginCommandRunner; + readonly strict: boolean; + readonly target: string; + readonly version: string | undefined; + }>, +): Promise => { + const outputDirectory = await mkdtemp(join(tmpdir(), 'agent-bundle-codex-schema-')); + try { + let result: CodexPluginCommandResult; + try { + result = await options.run(Object.freeze({ + args: Object.freeze(['app-server', 'generate-json-schema', '--out', outputDirectory]), + cwd: options.cwd, + executable: options.executable, + })); + } catch { + return freezeDiagnostics([diagnostic( + 'AB6033', + 'Codex CLI schema generation could not be started.', + 'error', + options.target, + 'Verify the Codex CLI starts and supports app-server schema generation, then rerun artifact validation.', + )]); + } + + if (result.termination !== undefined) { + return freezeDiagnostics([diagnostic( + 'AB6033', + commandFailureMessage('schema generation', result), + 'error', + options.target, + 'Rerun Codex schema generation within the configured time and output bounds.', + )]); + } + if (result.exitCode !== 0) { + const output = `${result.stdout}\n${result.stderr}`; + if (schemaVerbUnavailable(output)) { + return freezeDiagnostics([diagnostic( + 'AB6031', + `The Codex ${options.version ?? 'unknown'} app-server generate-json-schema verb is unavailable; ` + + `live schema drift could not be checked against pinned Codex ${pinnedRevision}.`, + 'info', + options.target, + 'Use a Codex release that publishes app-server schema generation, or retain validation against the vendored pinned schemas.', + )]); + } + return freezeDiagnostics([diagnostic( + 'AB6033', + commandFailureMessage('schema generation', result), + 'error', + options.target, + 'Run `codex app-server generate-json-schema --out ` successfully, then rerun artifact validation.', + )]); + } + + try { + return await compareGeneratedSchemas( + outputDirectory, + options.version, + options.strict, + options.target, + ); + } catch { + return freezeDiagnostics([diagnostic( + 'AB6033', + 'Codex CLI generated schema output could not be inspected.', + 'error', + options.target, + 'Ensure the generated schema directory is readable, then rerun artifact validation.', + )]); + } + } finally { + await rm(outputDirectory, { force: true, recursive: true }); + } +}; + +export const validateCodexPlugin = async ( + options: ValidateCodexPluginOptions, +): Promise => { + const pluginDirectory = resolve(options.pluginDirectory); + const executable = options.executable ?? 'codex'; + const run = options.run ?? runCodexCommand; + const cwd = dirname(pluginDirectory); + let version: string | undefined; + try { + const probe = await run(Object.freeze({ args: Object.freeze(['--version']), cwd, executable })); + if (probe.exitCode !== 0 || probe.termination !== undefined) { + return Object.freeze({ + diagnostics: freezeDiagnostics([diagnostic( + 'AB6033', + commandFailureMessage('version probe', probe), + 'error', + options.target, + 'Verify the Codex CLI starts and responds to `codex --version`, then rerun artifact validation.', + )]), + host: 'codex', + status: 'failed', + target: options.target, + }); + } + version = versionFrom(`${probe.stdout}\n${probe.stderr}`); + } catch (error) { + if (!isErrno(error, 'ENOENT')) { + return Object.freeze({ + diagnostics: freezeDiagnostics([diagnostic( + 'AB6033', + 'Codex CLI version probe could not be started.', + 'error', + options.target, + 'Verify the Codex executable permissions and runtime dependencies, then rerun artifact validation.', + )]), + host: 'codex', + status: 'failed', + target: options.target, + }); + } + return Object.freeze({ + diagnostics: freezeDiagnostics([diagnostic( + 'AB6030', + 'The Codex CLI is not installed or is not on PATH; host artifact validation was skipped.', + 'info', + options.target, + 'Install Codex and ensure `codex` is on PATH, then rerun artifact validation.', + )]), + host: 'codex', + status: 'unavailable', + target: options.target, + }); + } + + const diagnostics = freezeDiagnostics([ + diagnostic( + 'AB6030', + `Codex ${version ?? 'unknown'} does not publish a plugin validation command; ` + + `bundle documents were checked locally against vendored pinned Codex ${pinnedRevision} schemas.`, + 'info', + options.target, + 'Use the vendored pinned schema diagnostics until Codex publishes a plugin validation developer tool.', + ), + ...await validatePinnedDocuments(pluginDirectory, options.target), + ...await schemaGenerationDiagnostics({ + cwd, + executable, + run, + strict: options.strict === true, + target: options.target, + version, + }), + ]); + const failed = diagnostics.some((entry) => entry.severity === 'error'); + const warnings = diagnostics.some((entry) => entry.severity === 'warning'); + return Object.freeze({ + diagnostics, + host: 'codex', + status: failed ? 'failed' : warnings ? 'warnings' : 'passed', + target: options.target, + ...(version === undefined ? {} : { version }), + }); +}; diff --git a/packages/agent-bundle/src/test/index.ts b/packages/agent-bundle/src/test/index.ts index a8a13f19f..bc0494937 100644 --- a/packages/agent-bundle/src/test/index.ts +++ b/packages/agent-bundle/src/test/index.ts @@ -152,3 +152,11 @@ export type { ClaudePluginValidationStatus, ValidateClaudePluginOptions, } from '../host-contracts/claude-plugin-validation.ts'; +export { validateCodexPlugin } from '../host-contracts/codex-plugin-validation.ts'; +export type { + CodexPluginCommandResult, + CodexPluginCommandRunner, + CodexPluginValidationReport, + CodexPluginValidationStatus, + ValidateCodexPluginOptions, +} from '../host-contracts/codex-plugin-validation.ts'; diff --git a/packages/agent-bundle/tests/codex-plugin-validation.test.ts b/packages/agent-bundle/tests/codex-plugin-validation.test.ts new file mode 100644 index 000000000..8364fdffe --- /dev/null +++ b/packages/agent-bundle/tests/codex-plugin-validation.test.ts @@ -0,0 +1,379 @@ +import { copyFile, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; + +import { expect, it } from '@rstest/core'; + +import { + validateCodexPlugin, + type CodexPluginCommandRunner, +} from '../src/host-contracts/codex-plugin-validation.ts'; + +const generatedSchemaNames = Object.freeze([ + 'subagent-start.command.input.schema.json', + 'subagent-start.command.output.schema.json', + 'subagent-stop.command.input.schema.json', + 'subagent-stop.command.output.schema.json', +]); + +const validDocuments = Object.freeze({ + '.agents/plugins/marketplace.json': { + interface: { displayName: 'Fixture' }, + name: 'fixture-marketplace', + plugins: [{ + category: 'Productivity', + name: 'fixture', + policy: { authentication: 'ON_INSTALL', installation: 'AVAILABLE' }, + source: { path: './', source: 'local' }, + }], + }, + '.codex-plugin/plugin.json': { + author: { name: 'Fixture' }, + description: 'A fixture plugin.', + interface: { + capabilities: ['hooks', 'mcp', 'skills'], + category: 'Productivity', + defaultPrompt: ['Use the fixture.'], + developerName: 'Fixture', + displayName: 'Fixture', + longDescription: 'A fixture plugin.', + shortDescription: 'A fixture plugin.', + }, + hooks: './hooks/hooks.json', + mcpServers: './.mcp.json', + name: 'fixture', + skills: './skills/', + version: '1.0.0', + }, + '.mcp.json': { + mcpServers: { + fixture: { command: 'node', type: 'stdio' }, + }, + }, + 'hooks/hooks.json': { + hooks: { + Stop: [{ + hooks: [{ command: 'node ./hooks/stop.mjs', type: 'command' }], + }], + }, + }, +}); + +const writeBundle = async ( + replacements: Readonly> = {}, +): Promise => { + const directory = await mkdtemp(join(tmpdir(), 'agent-bundle-codex-validation-')); + for (const [relativePath, document] of Object.entries({ ...validDocuments, ...replacements })) { + const path = join(directory, relativePath); + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, `${JSON.stringify(document, null, 2)}\n`, 'utf8'); + } + return directory; +}; + +type GeneratedSchemaBehavior = 'app-server-only' | 'drift' | 'match' | 'missing-verb' | 'output-limit' | 'timed-out'; + +const runWith = ( + behavior: GeneratedSchemaBehavior, +): { readonly calls: unknown[]; readonly run: CodexPluginCommandRunner } => { + const calls: unknown[] = []; + return { + calls, + run: async (request) => { + calls.push(request); + if (request.args[0] === '--version') { + return { exitCode: 0, signal: null, stderr: '', stdout: 'codex-cli 0.147.0\n' }; + } + if (behavior === 'missing-verb') { + return { + exitCode: 2, + signal: null, + stderr: "error: unrecognized subcommand 'generate-json-schema'\n", + stdout: '', + }; + } + if (behavior === 'output-limit') { + return { + exitCode: null, + signal: 'SIGTERM', + stderr: '', + stdout: '', + termination: 'output-limit', + }; + } + if (behavior === 'timed-out') { + return { + exitCode: null, + signal: 'SIGTERM', + stderr: '', + stdout: '', + termination: 'timed-out', + }; + } + const outIndex = request.args.indexOf('--out'); + const outputDirectory = request.args[outIndex + 1]; + if (outputDirectory === undefined) throw new Error('schema output directory was not provided'); + if (behavior === 'app-server-only') { + await writeFile(join(outputDirectory, 'codex_app_server_protocol.schemas.json'), '{}\n', 'utf8'); + await writeFile(join(outputDirectory, 'codex_app_server_protocol.v2.schemas.json'), '{}\n', 'utf8'); + return { exitCode: 0, signal: null, stderr: '', stdout: '' }; + } + const pinnedDirectory = new URL('../src/adapters/schemas/codex/generated/', import.meta.url); + for (const name of generatedSchemaNames) { + await copyFile(new URL(name, pinnedDirectory), join(outputDirectory, name)); + } + if (behavior === 'drift') { + await writeFile(join(outputDirectory, generatedSchemaNames[0]!), '{"type":"null"}\n', 'utf8'); + } + return { exitCode: 0, signal: null, stderr: '', stdout: '' }; + }, + }; +}; + +it('validates Codex bundle documents and matching generated schemas without shell interpolation', async () => { + const pluginDirectory = await writeBundle(); + try { + const fixture = runWith('match'); + const report = await validateCodexPlugin({ + pluginDirectory, + run: fixture.run, + target: 'codex', + }); + + expect(fixture.calls).toEqual([ + expect.objectContaining({ args: ['--version'], executable: 'codex' }), + expect.objectContaining({ + args: ['app-server', 'generate-json-schema', '--out', expect.any(String)], + executable: 'codex', + }), + ]); + expect(report).toEqual({ + diagnostics: [expect.objectContaining({ + code: 'AB6030', + message: expect.stringContaining('does not publish a plugin validation command'), + severity: 'info', + target: 'codex', + })], + host: 'codex', + status: 'passed', + target: 'codex', + version: '0.147.0', + }); + expect(Object.isFrozen(report)).toBe(true); + expect(Object.isFrozen(report.diagnostics)).toBe(true); + } finally { + await rm(pluginDirectory, { force: true, recursive: true }); + } +}); + +it('reports an honest informational skip when Codex is absent', async () => { + const missing = Object.assign(new Error('spawn codex ENOENT'), { code: 'ENOENT' }); + const report = await validateCodexPlugin({ + pluginDirectory: '/tmp/plugin', + run: async () => { throw missing; }, + target: 'codex', + }); + + expect(report).toEqual({ + diagnostics: [expect.objectContaining({ + code: 'AB6030', + message: expect.stringContaining('not installed or is not on PATH'), + severity: 'info', + target: 'codex', + })], + host: 'codex', + status: 'unavailable', + target: 'codex', + }); +}); + +it('fails when the Codex version probe exits nonzero', async () => { + const report = await validateCodexPlugin({ + pluginDirectory: '/tmp/plugin', + run: async () => ({ + exitCode: 2, + signal: null, + stderr: 'version failed', + stdout: '', + }), + target: 'codex', + }); + + expect(report).toMatchObject({ + diagnostics: [expect.objectContaining({ + code: 'AB6033', + message: expect.stringContaining('version probe exited with code 2'), + severity: 'error', + })], + status: 'failed', + }); +}); + +it('reports the missing schema generator verb honestly and still checks pinned documents', async () => { + const pluginDirectory = await writeBundle(); + try { + const report = await validateCodexPlugin({ + pluginDirectory, + run: runWith('missing-verb').run, + target: 'codex', + }); + + expect(report).toMatchObject({ + diagnostics: [ + expect.objectContaining({ code: 'AB6030', severity: 'info' }), + expect.objectContaining({ + code: 'AB6031', + message: expect.stringContaining('generate-json-schema verb is unavailable'), + severity: 'info', + }), + ], + status: 'passed', + version: '0.147.0', + }); + } finally { + await rm(pluginDirectory, { force: true, recursive: true }); + } +}); + +it('warns when live generated schemas drift from the pinned revision', async () => { + const pluginDirectory = await writeBundle(); + try { + const report = await validateCodexPlugin({ + pluginDirectory, + run: runWith('drift').run, + target: 'codex', + }); + + expect(report).toMatchObject({ + diagnostics: [ + expect.objectContaining({ code: 'AB6030', severity: 'info' }), + expect.objectContaining({ + code: 'AB6031', + message: expect.stringContaining('live Codex 0.147.0'), + severity: 'warning', + }), + ], + status: 'warnings', + version: '0.147.0', + }); + } finally { + await rm(pluginDirectory, { force: true, recursive: true }); + } +}); + +it('reports app-server-only schema output as unassessable information even in strict mode', async () => { + const pluginDirectory = await writeBundle(); + try { + const report = await validateCodexPlugin({ + pluginDirectory, + run: runWith('app-server-only').run, + strict: true, + target: 'codex', + }); + + expect(report).toMatchObject({ + diagnostics: [ + expect.objectContaining({ code: 'AB6030', severity: 'info' }), + expect.objectContaining({ + code: 'AB6031', + message: expect.stringContaining( + 'live hook-schema drift is not assessable because the generator emits the app-server protocol surface', + ), + severity: 'info', + }), + ], + status: 'passed', + version: '0.147.0', + }); + } finally { + await rm(pluginDirectory, { force: true, recursive: true }); + } +}); + +it('rejects malformed fixtures for every locally validated Codex schema', async () => { + const malformed = [ + ['.codex-plugin/plugin.json', { ...validDocuments['.codex-plugin/plugin.json'], name: 'Invalid Name' }], + ['hooks/hooks.json', { hooks: { Stop: [{ hooks: [{ command: '', type: 'command' }] }] } }], + ['.mcp.json', { mcpServers: { fixture: { type: 'streamable-http', url: 'not a uri' } } }], + ['.agents/plugins/marketplace.json', { + ...validDocuments['.agents/plugins/marketplace.json'], + plugins: [], + }], + ] as const; + + for (const [relativePath, document] of malformed) { + const pluginDirectory = await writeBundle({ [relativePath]: document }); + try { + const report = await validateCodexPlugin({ + pluginDirectory, + run: runWith('match').run, + target: 'codex', + }); + + expect(report).toMatchObject({ + diagnostics: expect.arrayContaining([expect.objectContaining({ + code: 'AB6032', + generatedPath: relativePath, + severity: 'error', + })]), + status: 'failed', + }); + } finally { + await rm(pluginDirectory, { force: true, recursive: true }); + } + } +}); + +it('maps schema-generation timeout and output-limit terminations to stable failures', async () => { + const pluginDirectory = await writeBundle(); + try { + for (const [behavior, message] of [ + ['timed-out', 'schema generation timed out'], + ['output-limit', 'schema generation exceeded its output limit'], + ] as const) { + const report = await validateCodexPlugin({ + pluginDirectory, + run: runWith(behavior).run, + target: 'codex', + }); + expect(report).toMatchObject({ + diagnostics: expect.arrayContaining([expect.objectContaining({ + code: 'AB6033', + message: expect.stringContaining(message), + severity: 'error', + })]), + status: 'failed', + }); + } + } finally { + await rm(pluginDirectory, { force: true, recursive: true }); + } +}); + +it('fails when the Codex version probe exceeds its output limit or times out', async () => { + for (const [termination, message] of [ + ['output-limit', 'version probe exceeded its output limit'], + ['timed-out', 'version probe timed out'], + ] as const) { + const report = await validateCodexPlugin({ + pluginDirectory: '/tmp/plugin', + run: async () => ({ + exitCode: null, + signal: 'SIGTERM', + stderr: '', + stdout: '', + termination, + }), + target: 'codex', + }); + + expect(report).toMatchObject({ + diagnostics: [expect.objectContaining({ + code: 'AB6033', + message: expect.stringContaining(message), + severity: 'error', + })], + status: 'failed', + }); + } +});