From 113aba92f56253592a6c5865e37b539210575966 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Wed, 2 Sep 2026 03:00:02 +0000 Subject: [PATCH] feat(test): rendered commands at the cli-dispatch proof level (#253) --- .changeset/rendered-cli-dispatch-harness.md | 8 + packages/agent-bundle/README.md | 14 +- .../fixtures/route-harness/src/cli/report.tsx | 51 ++++ packages/agent-bundle/src/test/cli.ts | 243 ++++++++++++------ packages/agent-bundle/src/test/index.ts | 6 +- packages/agent-bundle/src/test/render.ts | 215 +++++++++++++--- .../projection/cli-dispatch-rendered.test.ts | 116 +++++++++ .../tests/projection/cli-dispatch.test.ts | 9 +- .../tests/test-harness-manifest.test.ts | 17 ++ .../templates/cli-tool/README.md | 9 +- 10 files changed, 574 insertions(+), 114 deletions(-) create mode 100644 .changeset/rendered-cli-dispatch-harness.md create mode 100644 packages/agent-bundle/fixtures/route-harness/src/cli/report.tsx create mode 100644 packages/agent-bundle/tests/projection/cli-dispatch-rendered.test.ts diff --git a/.changeset/rendered-cli-dispatch-harness.md b/.changeset/rendered-cli-dispatch-harness.md new file mode 100644 index 000000000..b0a0f1e50 --- /dev/null +++ b/.changeset/rendered-cli-dispatch-harness.md @@ -0,0 +1,8 @@ +--- +"agent-bundle": minor +--- + +Exercise rendered CLI command routes through the public `cli-dispatch` test +harness. `invokeCli` now mirrors the generated executable's render session, +supports explicit TTY projection through `tty`, and exposes `cliNdjson` for +asserting ordered rendered event streams. diff --git a/packages/agent-bundle/README.md b/packages/agent-bundle/README.md index f49d557fa..faba4532a 100644 --- a/packages/agent-bundle/README.md +++ b/packages/agent-bundle/README.md @@ -274,22 +274,30 @@ is never a receipt for another. | --- | --- | --- | | `route-unit` | `renderRoute`, `renderRouteEvents` | a route module renders to the document (and render-event stream) it claims | | `mcp-in-memory` | `openInMemoryMcpServer`, `invokeMcpTool`, `readMcpResource`, `getMcpPrompt`, `listMcpSurface` | the real generated MCP server's protocol contract, over the SDK's in-memory transport | -| `cli-dispatch` | `invokeCli`, `cliJson` | an argv vector resolved and run through the routed CLI's own shell, in-process | +| `cli-dispatch` | `invokeCli`, `cliJson`, `cliNdjson` | a plain or rendered argv vector resolved and run through the routed CLI's own shell, including rendered Markdown, explicit TTY, JSON, and NDJSON modes, in-process | | `packed-stdio` | `openPackedMcpServer` | a built artifact's generated entry running as a real process over stdio | | `packed-deleted-source` | `removeProjectSource`, `openPackedMcpServer({ deletedSource })` | the packed stdio process still runs after project source and configuration are removed and verified absent | | `host-install` | repository real-host install proof | a built bundle installed into an isolated real host home through the public install path, with registration observed through the host's own CLI | ```ts -import { cliJson, expectEvents, invokeCli, invokeMcpTool } from 'agent-bundle/test'; +import { cliJson, cliNdjson, expectEvents, invokeCli, invokeMcpTool } from 'agent-bundle/test'; // mcp-in-memory: the generated server projects the document to protocol content. const call = await invokeMcpTool('summarize', { input: { title: 'Dune' } }); expect(call.structuredContent).toEqual({ chapters: 24 }); -// cli-dispatch: the routed CLI resolves the command, parses argv, and maps the exit code. +// cli-dispatch, plain .ts route: resolve argv, execute, and map the exit code. const run = await invokeCli(['library', 'audit', './books', '--max-files', '8']); expect(run.exitCode).toBe(0); expect(cliJson(run)).toMatchObject({ scanned: 8 }); + +// cli-dispatch, rendered .tsx route: exercise the shell's rendered output modes. +const rendered = await invokeCli(['library', 'report', './books', '--ndjson']); +const events = cliNdjson(rendered); +expect(events.at(-1)?.type).toBe('complete'); + +const tty = await invokeCli(['library', 'report', './books'], { tty: true }); +expect(tty.stdout).toContain('\r\u001B[2K'); ``` `expectEvents` asserts over a render-event stream. `toContainSequence` is diff --git a/packages/agent-bundle/fixtures/route-harness/src/cli/report.tsx b/packages/agent-bundle/fixtures/route-harness/src/cli/report.tsx new file mode 100644 index 000000000..0f5dba58c --- /dev/null +++ b/packages/agent-bundle/fixtures/route-harness/src/cli/report.tsx @@ -0,0 +1,51 @@ +import { Agent, agent } from '@agent-bundle/runtime'; +import type { CliRouteConfig, CliRouteProps } from 'agent-bundle'; +import { z } from 'zod'; + +export const config = { + description: 'Renders a harness report.', + positionals: ['topic'], +} satisfies CliRouteConfig; + +export const inputSchema = z.object({ + mode: z.enum(['success', 'render-error', 'invalid-result', 'wait-for-abort']).default('success'), + topic: z.string().min(1), +}).strict(); + +export const resultSchema = z.object({ + count: z.number().int().nonnegative(), + stateMounted: z.literal(true), + status: z.literal('ready'), + topic: z.string(), +}).strict(); + +export default async function Report({ input, signal }: CliRouteProps) { + const context = await agent(); + await context.progress.report({ completed: 1, message: 'preparing report', total: 2 }); + + if (input.mode === 'render-error') { + throw new Error('report render exploded'); + } + if (input.mode === 'wait-for-abort') { + await new Promise((_resolve, reject) => { + const rejectAborted = () => reject(new DOMException('Report render aborted', 'AbortError')); + if (signal.aborted) { + rejectAborted(); + return; + } + signal.addEventListener('abort', rejectAborted, { once: true }); + }); + } + + await context.progress.report({ completed: 2, message: 'report ready', total: 2 }); + const value = input.mode === 'invalid-result' + ? { count: 'two', stateMounted: context.state !== undefined, status: 'ready', topic: input.topic } + : { count: 2, stateMounted: context.state !== undefined, status: 'ready', topic: input.topic }; + + return ( + + {`# Report: ${input.topic}\n\nGenerated for ${input.topic}.`} + items: 2 + + ); +} diff --git a/packages/agent-bundle/src/test/cli.ts b/packages/agent-bundle/src/test/cli.ts index 29f4a6196..194c5bd15 100644 --- a/packages/agent-bundle/src/test/cli.ts +++ b/packages/agent-bundle/src/test/cli.ts @@ -4,33 +4,38 @@ * `invokeCli` runs one argv vector through the routed CLI's own shell * (`runGeneratedCliEntry`, #102 stage 2) over the compiled command graph the * manifest carries, in this process. Command resolution, the argv projection, - * help, `--version`, and the exit-code mapping are the product's; the only - * thing the harness supplies is the `execute` bridge that runs the matched - * route module — and that mirrors the generated executable's, so a command - * that passes here fails in the same place a shipped binary would. + * help, output-mode selection, and exit-code mapping are the product's. Plain + * commands run through an in-process `execute` bridge; rendered commands run + * through an in-process render session that shares the route-unit harness's + * dispatcher and Flight renderer. * * It does **not** spawn the generated binary: no shebang, no executable bit, - * no process framing. That is the `packed-stdio` level's business. - * - * Rendered (`.tsx`) command routes compile no command until #102 stage 3, so - * this level dispatches plain command routes only; a rendered command is a - * compiler error (`AB4816`) long before it reaches a test. + * worker thread, process framing, or chunk-by-chunk Flight streaming timing. + * The packed CLI route suite owns that evidence. */ import type * as AgentRuntime from '@agent-bundle/runtime'; import { CliInputError, runGeneratedCliEntry } from '../cli-entry.ts'; +import type { CliRenderedEvent } from '../cli-entry.ts'; import type { CompiledCliCommand } from '../routes/types.ts'; import { AgentTestError, captured } from './errors.ts'; import { CLI_DISPATCH_PROOF_LEVEL, type AgentBundleTestManifest } from './manifest.ts'; import { registeredRouteLoader, testManifest } from './registry.ts'; -import type { RenderRouteContext } from './render.ts'; +import { prepareCliRenderHost, type RenderRouteContext } from './render.ts'; import type { AgentRouteModule, RenderedRouteProvenance } from './types.ts'; +export type { CliRenderedEvent }; + export interface InvokeCliOptions { /** Request-scope overrides for the dispatched command, over the runtime's request contract. */ readonly context?: RenderRouteContext; readonly manifest?: AgentBundleTestManifest; readonly signal?: AbortSignal; + /** + * Selects interactive rendered output explicitly. Generated binaries use + * `process.stdout.isTTY`; the in-process harness defaults to piped output. + */ + readonly tty?: boolean; } export interface CliInvocation { @@ -48,9 +53,9 @@ export interface CliInvocation { readonly routeId?: string; /** Everything the shell wrote to its diagnostic stream. */ readonly stderr: string; - /** Everything the shell wrote to its output stream: one canonical JSON line, or help text. */ + /** Everything the shell wrote to stdout, including rendered Markdown, TTY, JSON, or NDJSON output. */ readonly stdout: string; - /** The validated result the command returned; absent unless a command executed. */ + /** The validated plain result or rendered document value; absent unless a command completed validation. */ readonly value?: unknown; } @@ -79,7 +84,7 @@ const noCommands = (manifest: AgentBundleTestManifest): AgentTestError => new Ag ? [] : [`compiler: ${String(manifest.diagnostics.length)} diagnostic(s), first ${manifest.diagnostics[0]!.code}: ${manifest.diagnostics[0]!.message}`]), ], - recovery: 'Add a plain command route under src/cli/ exporting inputSchema, resultSchema, and an async default function.', + recovery: 'Add a command route under src/cli/ exporting inputSchema, resultSchema, and an async default function or component.', }, ); @@ -148,71 +153,115 @@ export const invokeCli = async ( const provenance = provenanceOf(manifest); const runtime = await loadRuntime(); const context = options.context ?? {}; + const signal = options.signal ?? new AbortController().signal; + const renderedCommands = manifest.cliCommands.filter((command) => command.rendered); let executed: CompiledCliCommand | undefined; let value: unknown; let out = ''; let err = ''; - const exitCode = await runGeneratedCliEntry({ - argv, - commands: manifest.cliCommands, - // The bridge the generated executable inlines: the module's own schemas - // stay the validation boundary, an input rejection is a usage failure, - // and the command body runs inside the typed request scope. - execute: async (command, input, execution) => { - executed = command; - const module = await moduleFor(manifest, command.routeId, provenance); - const component = (module as { default?: unknown }).default; - if (typeof component !== 'function') { - throw new AgentTestError('invalid-route-module', `Command route ${command.routeId} must default-export an async function.`, { - details: [`received: default export of type ${typeof component}`], - recovery: 'Export the command function as the module default.', - }); - } - if (module.inputSchema === undefined || module.resultSchema === undefined) { - throw new AgentTestError('invalid-route-module', `Command route ${command.routeId} must export inputSchema and resultSchema.`, { - recovery: 'Export both zod schemas from the command module; the routed CLI validates argv through them.', - }); - } - let parsed: unknown; - try { - parsed = module.inputSchema.parse(input); - } catch (error) { - throw new CliInputError(error instanceof Error ? error.message : String(error)); - } - const root = process.cwd(); - const result = await runtime.runAgentRequest({ - capabilities: { - command: runtime.unavailable(), - filesystem: runtime.unavailable(), - network: runtime.unavailable(), - projectRoot: runtime.available({ root }, 'derived'), - }, - host: runtime.unavailable('unsupported-surface'), - workspace: runtime.available({ root }, 'derived'), - ...context, - invocation: { - kind: 'cli', - operationId: command.routeId, - surface: commandPath(command), - ...context.invocation, - }, - ...(context.progress === undefined ? {} : { progress: context.progress }), - signal: execution.signal, - }, async () => (component as (props: unknown) => Promise)({ - input: parsed, - signal: execution.signal, - })); - value = module.resultSchema.parse(result); - return value; - }, - name: manifest.plugin.name, - version: manifest.plugin.version, - ...(options.signal === undefined ? {} : { signal: options.signal }), - writeErr: (text) => { err += text; }, - writeOut: (text) => { out += text; }, - }); + const renderedModules = new Map(); + for (const command of renderedCommands) { + renderedModules.set(command.routeId, await moduleFor(manifest, command.routeId, provenance)); + } + const firstRendered = renderedCommands[0]; + const firstDescriptor = firstRendered === undefined ? undefined : manifest.routes[firstRendered.routeId]; + const renderHost = firstRendered === undefined + ? undefined + : await prepareCliRenderHost({ + context, + manifest, + modules: renderedModules, + onValidated: (validated) => { value = validated; }, + provenance: { + kind: 'cli', + manifestDigest: manifest.digest, + ...(firstDescriptor === undefined + ? {} + : { + modulePath: firstDescriptor.source, + relativePath: firstDescriptor.relativePath, + }), + projectRoot: manifest.projectRoot, + proofLevel: CLI_DISPATCH_PROOF_LEVEL, + routeId: firstRendered.routeId, + source: 'manifest', + targets: manifest.targets, + }, + signal, + }); + + let exitCode: number; + try { + exitCode = await runGeneratedCliEntry({ + argv, + commands: manifest.cliCommands, + execute: async (command, input, execution) => { + executed = command; + const module = await moduleFor(manifest, command.routeId, provenance); + const component = (module as { default?: unknown }).default; + if (typeof component !== 'function') { + throw new AgentTestError('invalid-route-module', `Command route ${command.routeId} must default-export an async function.`, { + details: [`received: default export of type ${typeof component}`], + recovery: 'Export the command function as the module default.', + }); + } + if (module.inputSchema === undefined || module.resultSchema === undefined) { + throw new AgentTestError('invalid-route-module', `Command route ${command.routeId} must export inputSchema and resultSchema.`, { + recovery: 'Export both zod schemas from the command module; the routed CLI validates argv through them.', + }); + } + let parsed: unknown; + try { + parsed = module.inputSchema.parse(input); + } catch (error) { + throw new CliInputError(error instanceof Error ? error.message : String(error)); + } + const root = process.cwd(); + const result = await runtime.runAgentRequest({ + capabilities: { + command: runtime.unavailable(), + filesystem: runtime.unavailable(), + network: runtime.unavailable(), + projectRoot: runtime.available({ root }, 'derived'), + }, + host: runtime.unavailable('unsupported-surface'), + workspace: runtime.available({ root }, 'derived'), + ...context, + invocation: { + kind: 'cli', + operationId: command.routeId, + surface: commandPath(command), + ...context.invocation, + }, + ...(context.progress === undefined ? {} : { progress: context.progress }), + signal: execution.signal, + }, async () => (component as (props: unknown) => Promise)({ + input: parsed, + signal: execution.signal, + })); + value = module.resultSchema.parse(result); + return value; + }, + isTty: () => options.tty === true, + name: manifest.plugin.name, + ...(renderHost === undefined + ? {} + : { + render: (command, input, execution) => { + executed = command; + return renderHost.render(command, input, execution); + }, + }), + signal, + version: manifest.plugin.version, + writeErr: (text) => { err += text; }, + writeOut: (text) => { out += text; }, + }); + } finally { + await renderHost?.close(); + } return Object.freeze({ argv: Object.freeze([...argv]), @@ -248,3 +297,53 @@ export const cliJson = (invocation: CliInvocation): unknown => { }); } }; + +/** The ordered render events a successful `--ndjson` invocation wrote to stdout. */ +export const cliNdjson = (invocation: CliInvocation): readonly CliRenderedEvent[] => { + try { + const lines = invocation.stdout.endsWith('\n') + ? invocation.stdout.slice(0, -1).split('\n') + : invocation.stdout.split('\n'); + if (lines.length === 0 || lines.some((line) => line.trim() === '')) { + throw new SyntaxError('NDJSON output must contain one non-empty JSON object per line.'); + } + return Object.freeze(lines.map((line) => { + const event = JSON.parse(line) as unknown; + if (typeof event !== 'object' || event === null || Array.isArray(event)) { + throw new SyntaxError('NDJSON output lines must be JSON objects.'); + } + const record = event as Record; + if (!Number.isInteger(record['sequence'])) { + throw new SyntaxError('NDJSON render events must carry an integer sequence.'); + } + switch (record['type']) { + case 'shell': + case 'progress': + case 'replace': + case 'error': + case 'complete': + break; + default: + throw new SyntaxError('NDJSON output contains an unknown render-event type.'); + } + return event as CliRenderedEvent; + })); + } catch (error) { + throw new AgentTestError('projection-failed', 'The dispatched command did not write one JSON object per line to stdout.', { + cause: error, + details: [ + `exit code: ${String(invocation.exitCode)}`, + `stdout: ${captured(invocation.stdout)}`, + ...(invocation.stderr === '' ? [] : [`stderr: ${captured(invocation.stderr)}`]), + ], + provenance: { + ...invocation.provenance, + kind: 'cli', + routeId: invocation.routeId ?? '(no command executed)', + source: 'manifest', + targets: [], + }, + recovery: 'Call cliNdjson() only for a rendered invocation that passed --ndjson and wrote a complete event stream.', + }); + } +}; diff --git a/packages/agent-bundle/src/test/index.ts b/packages/agent-bundle/src/test/index.ts index 36c34ad6c..fd6af45bc 100644 --- a/packages/agent-bundle/src/test/index.ts +++ b/packages/agent-bundle/src/test/index.ts @@ -10,7 +10,7 @@ * | --- | --- | --- | * | `route-unit` | `renderRoute`, `renderRouteEvents`, `createTargetCapabilityFixture`, `projectTargetCapabilities` | the route component and its document through the real Agent renderer; explicit target-capability projection through the real MCP projector, without transport or host proof | * | `mcp-in-memory` | `openInMemoryMcpServer`, `invokeMcpTool`, `readMcpResource`, `getMcpPrompt`, `listMcpSurface` | the real generated MCP server's protocol contract, over the SDK's in-memory transport | - * | `cli-dispatch` | `invokeCli`, `cliJson` | a compiled CLI command dispatched through the routed CLI's own shell, in this process | + * | `cli-dispatch` | `invokeCli`, `cliJson`, `cliNdjson` | a compiled plain or rendered CLI command dispatched through the routed CLI's own shell, including rendered output modes, in this process | * | `packed-stdio` | `openPackedMcpServer` | a built artifact's generated entry running as a real process over stdio | * | `packed-deleted-source` | `removeProjectSource`, `openPackedMcpServer` | the packed stdio process still runs after project source and configuration are removed and verified absent | * | `browser-app` | `mountBrowserApp` (`agent-bundle/test/browser`) | production-compiled MCP App HTML mounted over the product bridge in a real browser page | @@ -92,8 +92,8 @@ export type { McpSurfaceListing, McpToolInvocation, } from './mcp.ts'; -export { cliJson, invokeCli } from './cli.ts'; -export type { CliDispatchProvenance, CliInvocation, InvokeCliOptions } from './cli.ts'; +export { cliJson, cliNdjson, invokeCli } from './cli.ts'; +export type { CliDispatchProvenance, CliInvocation, CliRenderedEvent, InvokeCliOptions } from './cli.ts'; export { openPackedMcpServer, removeProjectSource } from './packed.ts'; export type { DeletedSourceReceipt, diff --git a/packages/agent-bundle/src/test/render.ts b/packages/agent-bundle/src/test/render.ts index f8de90c0b..8a2afbc65 100644 --- a/packages/agent-bundle/src/test/render.ts +++ b/packages/agent-bundle/src/test/render.ts @@ -11,6 +11,7 @@ import type { AgentInvocationInput, AgentProgressReporter, AgentProgressUpdate, + AgentRenderDispatch, AgentRenderEvent, AgentRenderInvocation, AgentRenderLimits, @@ -18,6 +19,13 @@ import type { } from '@agent-bundle/runtime'; import type * as React from 'react'; +import { CliInputError } from '../cli-entry.ts'; +import type { + CliRenderedEvent, + GeneratedCliRenderContext, + GeneratedCliRenderSession, +} from '../cli-entry.ts'; +import type { CompiledCliCommand } from '../routes/types.ts'; import { AgentTestError, captured } from './errors.ts'; import { ROUTE_UNIT_PROOF_LEVEL, type AgentBundleTestManifest } from './manifest.ts'; import { @@ -69,12 +77,14 @@ export interface RenderedRoute { export type RenderRouteTarget = AgentRouteModule | string; interface Renderer { + readonly available: typeof AgentRuntime.available; readonly createAgentRenderDispatcher: typeof AgentRuntime.createAgentRenderDispatcher; readonly createElement: typeof React.createElement; readonly createGeneratedRuntimeState: typeof AgentMount.createGeneratedRuntimeState; readonly createMemoryStateDriver: typeof AgentState.createMemoryStateDriver; readonly renderAgentFlight: typeof AgentFlightServer.renderAgentFlight; readonly runAgentRequest: typeof AgentRuntime.runAgentRequest; + readonly unavailable: typeof AgentRuntime.unavailable; } let rendererPromise: Promise | undefined; @@ -97,22 +107,24 @@ const loadRenderer = async (): Promise => { import('react'), ]); return { + available: runtime.available, createAgentRenderDispatcher: runtime.createAgentRenderDispatcher, createElement: react.createElement, createGeneratedRuntimeState: mount.createGeneratedRuntimeState, createMemoryStateDriver: state.createMemoryStateDriver, renderAgentFlight: flight.renderAgentFlight, runAgentRequest: runtime.runAgentRequest, + unavailable: runtime.unavailable, }; })().catch((error: unknown) => { rendererPromise = undefined; throw new AgentTestError( 'render-failed', - 'Unable to load the Agent renderer for a route-unit render.', + 'Unable to load the Agent renderer for an in-process test render.', { cause: error, details: [`cause: ${error instanceof Error ? error.message : String(error)}`], - recovery: 'Install react and @agent-bundle/runtime, and run the route-unit pool with the react-server condition — agentBundleRstest() from agent-bundle/rstest configures both.', + recovery: 'Install react and @agent-bundle/runtime, and run the cli-dispatch or route-unit pool with the react-server condition — agentBundleRstest() from agent-bundle/rstest configures both.', }, ); }); @@ -434,12 +446,12 @@ const noMountedState: AutoMountedState = Object.freeze({ * a disposable sqlite root so repeated route-unit renders are deterministic. */ const mountManifestState = async ( - resolved: ResolvedTarget, + manifest: AgentBundleTestManifest | undefined, + provenance: RenderedRouteProvenance, context: RenderRouteContext, renderer: Renderer, signal: AbortSignal, ): Promise => { - const manifest = resolved.manifest; const descriptor = manifest?.state; if ( manifest === undefined @@ -452,7 +464,7 @@ const mountManifestState = async ( 'manifest-unavailable', `State ${descriptor.id} is declared but no test-time state module loader is registered for it.`, { - provenance: resolved.provenance, + provenance, recovery: 'Build the Rstest configuration with agentBundleRstest() so the generated setup registers the state loader.', }, ); @@ -504,6 +516,162 @@ const mountManifestState = async ( } }; +const progressFor = ( + collected: AgentProgressUpdate[], + delegated: AgentProgressReporter | undefined, + dispatcher: AgentProgressReporter | undefined, +): AgentProgressReporter => ({ + report: async (update) => { + collected.push(update); + await delegated?.report(update); + await dispatcher?.report(update); + }, +}); + +interface FlightDispatcherOptions { + readonly collected: AgentProgressUpdate[]; + readonly component: (props: never) => unknown; + readonly componentProps: (request: AgentRenderDispatch) => Readonly>; + readonly contextProgress?: AgentProgressReporter; + readonly limits?: Partial; + readonly renderer: Renderer; + readonly requestInit: (request: AgentRenderDispatch) => AgentRequestInit; +} + +const createFlightDispatcher = (options: FlightDispatcherOptions): AgentRuntime.AgentRenderDispatcher => + options.renderer.createAgentRenderDispatcher({ + execute: async (request) => streamOf(await options.renderer.runAgentRequest({ + ...options.requestInit(request), + progress: progressFor(options.collected, options.contextProgress, request.progress), + signal: request.signal, + }, async () => drain(options.renderer.renderAgentFlight( + options.renderer.createElement( + options.component as never, + options.componentProps(request) as never, + ), + { signal: request.signal }, + )))), + }, options.limits === undefined ? {} : { limits: options.limits }); + +export interface PrepareCliRenderHostOptions { + readonly context?: RenderRouteContext; + readonly manifest: AgentBundleTestManifest; + readonly modules: ReadonlyMap; + readonly onValidated: (value: unknown) => void; + readonly provenance: RenderedRouteProvenance; + readonly signal: AbortSignal; +} + +export interface PreparedCliRenderHost { + readonly close: () => Promise; + readonly render: ( + command: CompiledCliCommand, + input: Readonly>, + context: GeneratedCliRenderContext, + ) => GeneratedCliRenderSession; +} + +/** + * Accepts preloaded route modules and prepares the renderer and manifest + * state before the synchronous generated-shell render factory is installed. + */ +export const prepareCliRenderHost = async ( + options: PrepareCliRenderHostOptions, +): Promise => { + const renderer = await loadRenderer(); + const context = options.context ?? {}; + const mounted = await mountManifestState( + options.manifest, + options.provenance, + context, + renderer, + options.signal, + ); + return Object.freeze({ + close: mounted.close, + render: ( + command: CompiledCliCommand, + input: Readonly>, + execution: GeneratedCliRenderContext, + ): GeneratedCliRenderSession => { + const module = options.modules.get(command.routeId); + if (module === undefined) { + throw new AgentTestError( + 'manifest-unavailable', + `Rendered command route ${command.routeId} was not preloaded for CLI dispatch.`, + { + provenance: { ...options.provenance, routeId: command.routeId }, + recovery: 'Build the Rstest configuration with agentBundleRstest() so every rendered command loader is registered.', + }, + ); + } + if (module.inputSchema === undefined || module.resultSchema === undefined) { + throw new AgentTestError( + 'invalid-route-module', + `Rendered command route ${command.routeId} must export inputSchema and resultSchema.`, + { + provenance: { ...options.provenance, routeId: command.routeId }, + recovery: 'Export both zod schemas from the rendered command module.', + }, + ); + } + let parsed: unknown; + try { + parsed = module.inputSchema.parse(input); + } catch (error) { + throw new CliInputError(error instanceof Error ? error.message : String(error)); + } + const commandName = command.path.join(' '); + const invocation = { + kind: 'cli' as const, + props: { args: execution.args, command: commandName }, + }; + const collected: AgentProgressUpdate[] = []; + const dispatcher = createFlightDispatcher({ + collected, + component: componentOf(module, { ...options.provenance, routeId: command.routeId }), + componentProps: (request) => ({ input: parsed, signal: request.signal }), + contextProgress: context.progress, + renderer, + requestInit: (request) => { + const root = process.cwd(); + return { + capabilities: { + command: renderer.unavailable(), + filesystem: renderer.unavailable(), + network: renderer.unavailable(), + projectRoot: renderer.available({ root }, 'derived'), + }, + host: renderer.unavailable('unsupported-surface'), + workspace: renderer.available({ root }, 'derived'), + ...context, + ...mounted.context, + invocation: { + kind: 'cli', + operationId: command.routeId, + surface: commandName, + ...context.invocation, + }, + signal: request.signal, + }; + }, + }); + return Object.freeze({ + close: mounted.close, + events: (): ReadableStream => dispatcher.stream({ + invocation, + signal: execution.signal, + }), + validate: (value: unknown) => { + const validated = module.resultSchema!.parse(value); + options.onValidated(validated); + return validated; + }, + }); + }, + }); +}; + interface PreparedRender { readonly close: () => Promise; readonly collected: readonly AgentProgressUpdate[]; @@ -529,23 +697,15 @@ const prepareRender = async ( const collected: AgentProgressUpdate[] = []; const context = options.context ?? {}; const signal = options.signal ?? new AbortController().signal; - const mounted = await mountManifestState(resolved, context, renderer, signal); - /** - * Every render collects progress for {@link RenderedRoute.progress}, then - * forwards it to the caller's reporter and to the dispatcher's, which is - * what turns an update into a `progress` render event. The collector cannot - * be an either/or fallback: the event-stream entry point always supplies a - * dispatcher reporter, and that would leave the array empty. - */ - const progressFor = (reporter: AgentProgressReporter | undefined): AgentProgressReporter => ({ - report: async (update) => { - collected.push(update); - await context.progress?.report(update); - await reporter?.report(update); - }, - }); - const dispatcher = renderer.createAgentRenderDispatcher({ - execute: async (request) => streamOf(await renderer.runAgentRequest({ + const mounted = await mountManifestState(resolved.manifest, resolved.provenance, context, renderer, signal); + const dispatcher = createFlightDispatcher({ + collected, + component: resolved.component, + componentProps: (request) => componentProps(request.invocation, resolved.kind, options, request.signal), + contextProgress: context.progress, + limits: options.limits, + renderer, + requestInit: (request) => ({ ...context, ...mounted.context, invocation: { @@ -553,16 +713,9 @@ const prepareRender = async ( ...context.invocation, kind: request.invocation.kind, }, - progress: progressFor(request.progress), signal: request.signal, - }, async () => drain(renderer.renderAgentFlight( - renderer.createElement( - resolved.component as never, - componentProps(request.invocation, resolved.kind, options, request.signal) as never, - ), - { signal: request.signal }, - )))), - }, options.limits === undefined ? {} : { limits: options.limits }); + }), + }); return { close: mounted.close, collected, diff --git a/packages/agent-bundle/tests/projection/cli-dispatch-rendered.test.ts b/packages/agent-bundle/tests/projection/cli-dispatch-rendered.test.ts new file mode 100644 index 000000000..de8dbb37c --- /dev/null +++ b/packages/agent-bundle/tests/projection/cli-dispatch-rendered.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, it } from '@rstest/core'; + +import { cliJson, cliNdjson, invokeCli } from '../../src/test/index.ts'; + +describe('rendered commands at the CLI dispatch level', () => { + it('projects a rendered command to final Markdown when stdout is piped', async () => { + const run = await invokeCli(['report', 'books']); + + expect(run.exitCode).toBe(0); + expect(run.stderr).toBe(''); + expect(run.stdout).toBe('# Report: books\n\nGenerated for books.\n\nitems: 2\n'); + expect(run.stdout).not.toContain('preparing report'); + expect(run.value).toEqual({ count: 2, stateMounted: true, status: 'ready', topic: 'books' }); + expect(run.provenance.proofLevel).toBe('cli-dispatch'); + }); + + it('updates rendered progress in place for an explicit TTY', async () => { + const run = await invokeCli(['report', 'books'], { tty: true }); + + expect(run.exitCode).toBe(0); + expect(run.stderr).toBe(''); + expect(run.stdout).toContain('\r\u001B[2Kpreparing report (1/2)'); + expect(run.stdout).toContain('\r\u001B[2Kreport ready (2/2)'); + expect(run.stdout.endsWith('# Report: books\n\nGenerated for books.\n\nitems: 2\n')).toBe(true); + }); + + it('projects a rendered command to one canonical JSON line', async () => { + const run = await invokeCli(['report', 'books', '--json']); + + expect(run.exitCode).toBe(0); + expect(run.stderr).toBe(''); + expect(run.stdout).toBe('{"count":2,"stateMounted":true,"status":"ready","topic":"books"}\n'); + expect(cliJson(run)).toEqual({ count: 2, stateMounted: true, status: 'ready', topic: 'books' }); + }); + + it('returns the pure sequence-numbered CLI event stream as NDJSON', async () => { + const run = await invokeCli(['report', 'books', '--ndjson']); + const events = cliNdjson(run); + const sequences = events.map((event) => event.sequence); + + expect(run.exitCode).toBe(0); + expect(run.stderr).toBe(''); + expect(sequences.every((sequence, index) => index === 0 || sequence > sequences[index - 1]!)).toBe(true); + expect(events.some((event) => event.type === 'progress')).toBe(true); + expect(events.at(-1)).toMatchObject({ + document: { + status: 'success', + value: { count: 2, stateMounted: true, status: 'ready', topic: 'books' }, + }, + type: 'complete', + }); + expect(JSON.stringify(events)).not.toContain('"jsonrpc"'); + expect(run.stdout.trim().split('\n')).toHaveLength(events.length); + }); + + it('reports cancellation through the shell after rendered progress begins', async () => { + const controller = new AbortController(); + const run = await invokeCli(['report', 'books', '--mode', 'wait-for-abort'], { + context: { + progress: { + report: async () => controller.abort(), + }, + }, + signal: controller.signal, + }); + + expect(run.exitCode).toBe(1); + expect(run.stdout).toBe(''); + expect(run.stderr).toContain('Aborted.'); + }); + + it('reports a component render error on stderr', async () => { + const run = await invokeCli(['report', 'books', '--mode', 'render-error']); + + expect(run.exitCode).toBe(1); + expect(run.stdout).toBe(''); + expect(run.stderr).toBe('report render exploded\n'); + }); + + it("reports a rendered resultSchema rejection on stderr", async () => { + const run = await invokeCli(['report', 'books', '--mode', 'invalid-result']); + + expect(run.exitCode).toBe(1); + expect(run.stdout).toBe(''); + expect(run.stderr).toContain('expected number'); + }); + + it('maps a rendered inputSchema rejection to a usage failure', async () => { + const run = await invokeCli(['report', '']); + + expect(run.exitCode).toBe(2); + expect(run.stdout).toBe(''); + expect(run.stderr).toContain("--help' for usage."); + }); + + it('rejects rendered-only and conflicting output flags at the shell boundary', async () => { + const [plainNdjson, conflicting] = await Promise.all([ + invokeCli(['inventory', 'fiction', '--ndjson']), + invokeCli(['report', 'books', '--json', '--ndjson']), + ]); + + expect(plainNdjson.exitCode).toBe(2); + expect(plainNdjson.stdout).toBe(''); + expect(plainNdjson.stderr).toContain('--ndjson requires a rendered command.'); + expect(conflicting.exitCode).toBe(2); + expect(conflicting.stdout).toBe(''); + expect(conflicting.stderr).toContain('Use either --json or --ndjson, not both.'); + }); + + it('rejects Markdown as canonical JSON or NDJSON with honest diagnostics', async () => { + const run = await invokeCli(['report', 'books']); + + expect(() => cliJson(run)).toThrow('did not write one canonical JSON line'); + expect(() => cliNdjson(run)).toThrow('did not write one JSON object per line'); + }); +}); diff --git a/packages/agent-bundle/tests/projection/cli-dispatch.test.ts b/packages/agent-bundle/tests/projection/cli-dispatch.test.ts index 1b8ed71d4..bb164d1b7 100644 --- a/packages/agent-bundle/tests/projection/cli-dispatch.test.ts +++ b/packages/agent-bundle/tests/projection/cli-dispatch.test.ts @@ -11,8 +11,8 @@ import { cliJson, invokeCli } from '../../src/test/cli.ts'; * * What this level does prove is that the product's dispatcher, argv * projection, and exit-code policy agree with the compiled commands: the - * harness contributes only the `execute` bridge, and that mirrors the one the - * generated executable inlines. + * harness supplies the same plain-command execute bridge and rendered-command + * session contract that the generated executable wires around the shell. */ describe('the CLI dispatch level', () => { it('resolves an argv vector to the compiled command and returns its canonical JSON line', async () => { @@ -24,7 +24,10 @@ describe('the CLI dispatch level', () => { expect(run.stderr).toBe(''); expect(cliJson(run)).toEqual({ format: 'json', shelf: 'fiction', titles: ['Piranesi', 'Solaris'] }); expect(run.value).toEqual({ format: 'json', shelf: 'fiction', titles: ['Piranesi', 'Solaris'] }); - expect(run.provenance).toMatchObject({ commands: ['db migrate', 'inventory'], proofLevel: 'cli-dispatch' }); + expect(run.provenance).toMatchObject({ + commands: ['db migrate', 'inventory', 'report'], + proofLevel: 'cli-dispatch', + }); }); it('dispatches a nested command through its compiled path', async () => { diff --git a/packages/agent-bundle/tests/test-harness-manifest.test.ts b/packages/agent-bundle/tests/test-harness-manifest.test.ts index 46a9e3c9f..6cd6e296b 100644 --- a/packages/agent-bundle/tests/test-harness-manifest.test.ts +++ b/packages/agent-bundle/tests/test-harness-manifest.test.ts @@ -66,6 +66,7 @@ describe('the compiled test manifest', () => { 'app:harness/panel', 'cli:db/migrate', 'cli:inventory', + 'cli:report', 'event:tool/after', 'prompt:harness/summarize', 'resource:harness/notes', @@ -133,6 +134,22 @@ describe('the compiled test manifest', () => { rendered: false, routeId: 'cli:inventory', }, + { + aliases: [], + description: 'Renders a harness report.', + exitCode: 'zero', + options: [ + expect.objectContaining({ + choices: ['success', 'render-error', 'invalid-result', 'wait-for-abort'], + defaultValue: 'success', + key: 'mode', + }), + expect.objectContaining({ key: 'topic', positional: 0, required: true }), + ], + path: ['report'], + rendered: true, + routeId: 'cli:report', + }, ]); }); diff --git a/packages/create-agent-bundle/templates/cli-tool/README.md b/packages/create-agent-bundle/templates/cli-tool/README.md index a77f88977..db844a2fa 100644 --- a/packages/create-agent-bundle/templates/cli-tool/README.md +++ b/packages/create-agent-bundle/templates/cli-tool/README.md @@ -43,8 +43,13 @@ vacuous pass is worse than no pool. Adopt the harness when the project grows a routed surface: -- `src/cli/**` command routes make `invokeCli` / `cliJson` (the `cli-dispatch` - level) meaningful — argv resolved and run through the routed CLI's own shell. +- Plain `src/cli/**/*.ts` and rendered `src/cli/**/*.tsx` command routes make + `invokeCli` / `cliJson` (the `cli-dispatch` level) meaningful — argv resolves + and runs through the routed CLI's own shell. Rendered routes can additionally + assert Markdown, explicit TTY, JSON, and NDJSON output; use `cliNdjson` for + the ordered render-event stream. This level remains in-process, so use the + packed CLI route suite for worker-thread, process-framing, executable, and + chunk-by-chunk Flight streaming evidence. This template ships the conventional `src/cli.ts` entry (and a matching `scripts` entry in `agent-bundle.config.ts`). Adding command routes while that file remains triggers `AB4801`. Before creating `src/cli/**` modules,