From a111a573c4140d05f525a9bd76cebc37e7e41b6c Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Tue, 1 Sep 2026 20:11:24 +0000 Subject: [PATCH] feat(test): add the projection-contract proof levels to the consumer harness #103 stage 2. `mcp-in-memory`, `cli-dispatch`, and `packed-stdio` join `route-unit`, each labeled in its provenance and in every failure message. The generated MCP server's warm host, route registration, and projection move into a shared runtime module the generated entry aliases, so the in-memory level exercises the artifact's own code. --- .changeset/generated-route-jsx-runtime.md | 19 + .changeset/projection-proof-levels.md | 45 ++ examples/rsc-agent-runtime/package.json | 3 +- .../rstest.route-unit.config.ts | 10 + .../tests/route-unit/event-route.test.ts | 68 +++ package.json | 5 +- packages/agent-bundle/README.md | 37 ++ .../route-harness/src/cli/db/migrate.ts | 29 ++ .../route-harness/src/cli/inventory.ts | 34 ++ .../route-harness/src/events/tool/after.tsx | 12 +- .../src/mcp/harness/prompts/summarize.tsx | 23 + .../src/mcp/harness/resources/notes.tsx | 10 +- .../src/mcp/harness/tools/catalog.tsx | 34 ++ packages/agent-bundle/rslib.config.ts | 1 + packages/agent-bundle/src/build/entries.ts | 30 +- .../agent-bundle/src/build/entry-shell.ts | 236 +++------ .../agent-bundle/src/build/inspect-bundler.ts | 7 +- packages/agent-bundle/src/build/rslib.ts | 7 + .../agent-bundle/src/mcp-server-runtime.ts | 492 ++++++++++++++++++ packages/agent-bundle/src/test/cli.ts | 250 +++++++++ packages/agent-bundle/src/test/errors.ts | 9 +- packages/agent-bundle/src/test/events.ts | 215 ++++++++ packages/agent-bundle/src/test/index.ts | 62 ++- packages/agent-bundle/src/test/manifest.ts | 93 +++- packages/agent-bundle/src/test/matchers.ts | 17 +- packages/agent-bundle/src/test/mcp.ts | 442 ++++++++++++++++ packages/agent-bundle/src/test/packed.ts | 131 +++++ packages/agent-bundle/src/test/render.ts | 152 +++++- packages/agent-bundle/src/test/types.ts | 13 +- .../agent-bundle/tests/entry-shell.test.ts | 91 +++- .../tests/generated-route-server.test.ts | 10 +- .../tests/inspect-bundler.test.ts | 4 + .../tests/packed-stdio-projection.test.ts | 105 ++++ .../tests/projection/cli-dispatch.test.ts | 87 ++++ .../tests/projection/mcp-in-memory.test.ts | 122 +++++ .../tests/projection/render-events.test.ts | 76 +++ .../tests/route-unit/render-route.test.ts | 32 +- .../tests/test-harness-manifest.test.ts | 116 ++++- rstest.config.ts | 4 +- rstest.integration-tests.ts | 13 + rstest.projection.config.ts | 20 + rstest.unit.config.ts | 2 + 42 files changed, 2892 insertions(+), 276 deletions(-) create mode 100644 .changeset/generated-route-jsx-runtime.md create mode 100644 .changeset/projection-proof-levels.md create mode 100644 examples/rsc-agent-runtime/rstest.route-unit.config.ts create mode 100644 examples/rsc-agent-runtime/tests/route-unit/event-route.test.ts create mode 100644 packages/agent-bundle/fixtures/route-harness/src/cli/db/migrate.ts create mode 100644 packages/agent-bundle/fixtures/route-harness/src/cli/inventory.ts create mode 100644 packages/agent-bundle/fixtures/route-harness/src/mcp/harness/prompts/summarize.tsx create mode 100644 packages/agent-bundle/fixtures/route-harness/src/mcp/harness/tools/catalog.tsx create mode 100644 packages/agent-bundle/src/mcp-server-runtime.ts create mode 100644 packages/agent-bundle/src/test/cli.ts create mode 100644 packages/agent-bundle/src/test/events.ts create mode 100644 packages/agent-bundle/src/test/mcp.ts create mode 100644 packages/agent-bundle/src/test/packed.ts create mode 100644 packages/agent-bundle/tests/packed-stdio-projection.test.ts create mode 100644 packages/agent-bundle/tests/projection/cli-dispatch.test.ts create mode 100644 packages/agent-bundle/tests/projection/mcp-in-memory.test.ts create mode 100644 packages/agent-bundle/tests/projection/render-events.test.ts create mode 100644 rstest.projection.config.ts diff --git a/.changeset/generated-route-jsx-runtime.md b/.changeset/generated-route-jsx-runtime.md new file mode 100644 index 000000000..b5c0825b3 --- /dev/null +++ b/.changeset/generated-route-jsx-runtime.md @@ -0,0 +1,19 @@ +--- +'agent-bundle': patch +--- + +Fix generated executables crashing on any route authored with JSX. + +Route entries were bundled without the React plugin, so Rslib lowered JSX to +the classic `React.createElement` factory — which no generated entry or Flight +worker has in scope. Every documented `.tsx` route (the contract's own example +shape) therefore failed at run time with `React is not defined`, while builds +and route-unit tests stayed green because the test transform selects the +automatic runtime. Route entries now build with the automatic JSX runtime, so +emitted modules import `react/jsx-runtime` themselves — under the +`react-server` condition for worker entries. + +The defect survived because every build-level test authored its routes with an +explicit `createElement` import; the generated-route server test now authors +its tool route as JSX instead, which is what surfaced this from the new +`packed-stdio` proof level. diff --git a/.changeset/projection-proof-levels.md b/.changeset/projection-proof-levels.md new file mode 100644 index 000000000..4a2fb1105 --- /dev/null +++ b/.changeset/projection-proof-levels.md @@ -0,0 +1,45 @@ +--- +'agent-bundle': minor +--- + +Add the projection-contract proof levels to `agent-bundle/test` (#103 stage 2). + +Three levels join `route-unit`, each labeled in its result provenance and in +every failure message, because a pass at one level is never a receipt for +another: + +- `mcp-in-memory` — `openInMemoryMcpServer`, `invokeMcpTool`, `readMcpResource`, + `getMcpPrompt`, and `listMcpSurface` drive the real generated MCP server with + a real MCP client over the SDK's in-memory transport pair. Protocol-contract + proof only: no process, no stdio framing, no packed artifact. +- `cli-dispatch` — `invokeCli` runs an argv vector through the routed CLI's own + shell (#102 stage 2) over the compiled command graph the manifest now + carries, in-process. Command resolution, argv projection, help, `--version`, + and the exit-code policy are the product's; the harness supplies only the + `execute` bridge, and it mirrors the one the generated executable inlines. + `cliJson` reads the canonical stdout line. +- `packed-stdio` — `openPackedMcpServer` spawns a built artifact's generated + stdio entry and connects a real MCP client to it. This is the only level here + that is process evidence. + +`renderRouteEvents` returns the ordered render-event stream alongside the final +document, and `expectEvents` asserts over it. The default matcher +(`toContainSequence`) is sequence-tolerant so a legitimate extra `progress` or +`replace` frame cannot turn a passing render red, while a missing frame, a +reordering, or a regressed ordinal still fails. + +The test manifest gains `cliCommands`, the compiled routed-CLI command graph +from the same compiler pass, so the dispatch level never recompiles it. +`expectDocument` gains `toContainContext` for the context nodes an event route +returns to its host. + +Event routes now render with the props the public contract defines — +`{ canonical, native, signal }`, the same unwrapping the generated Flight +worker performs — instead of the raw invocation payload. A route written +against `AgentEventRouteProps` previously received `undefined` for both. + +Internally, the generated MCP server's warm Flight host, route registration, +and MCP projection move out of the entry template into the shared +`agent-bundle/mcp-server-runtime` module the generated entry aliases, so the +in-memory level exercises the artifact's own code rather than a second copy of +it. Generated-entry behaviour is unchanged. diff --git a/examples/rsc-agent-runtime/package.json b/examples/rsc-agent-runtime/package.json index cbc3ec0e4..297bd0e72 100644 --- a/examples/rsc-agent-runtime/package.json +++ b/examples/rsc-agent-runtime/package.json @@ -5,9 +5,10 @@ "scripts": { "build": "rsbuild build --mode production && agent-bundle build --json --output dist/plugins", "test": "rstest --config rstest.config.ts", + "test:routes": "rstest --config rstest.route-unit.config.ts", "typecheck": "tsc -p tsconfig.json --noEmit", "validate": "agent-bundle validate", - "check": "pnpm validate && pnpm build && pnpm typecheck && pnpm test", + "check": "pnpm validate && pnpm build && pnpm typecheck && pnpm test && pnpm test:routes", "eval:hosts": "node scripts/eval-hosts.mjs", "capture:widget": "node scripts/capture-widget.mjs" }, diff --git a/examples/rsc-agent-runtime/rstest.route-unit.config.ts b/examples/rsc-agent-runtime/rstest.route-unit.config.ts new file mode 100644 index 000000000..623d7d398 --- /dev/null +++ b/examples/rsc-agent-runtime/rstest.route-unit.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from '@rstest/core'; +import { agentBundleRstest } from 'agent-bundle/rstest'; + +/** + * The framework-generated route-unit configuration. One Agent Bundle compiler + * pass runs here — no artifact build — and it supplies the route manifest, the + * TypeScript transform, and the React Server Components conditions the demo's + * event route needs. The example maintains none of that by hand. + */ +export default defineConfig(await agentBundleRstest()); diff --git a/examples/rsc-agent-runtime/tests/route-unit/event-route.test.ts b/examples/rsc-agent-runtime/tests/route-unit/event-route.test.ts new file mode 100644 index 000000000..ffa5040bc --- /dev/null +++ b/examples/rsc-agent-runtime/tests/route-unit/event-route.test.ts @@ -0,0 +1,68 @@ +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { afterEach, beforeEach, expect, it } from '@rstest/core'; +import { expectDocument, renderRoute, testManifest } from 'agent-bundle/test'; + +/** + * The route-unit proof level for the demo's PostToolUse migration: the hook is + * a compiled `src/events/tool/after.tsx` route, and it renders through the same + * renderer and request scope every other route uses. Native wrapper delivery + * and the host response projection are proven by the artifact suites; this is + * not host or process evidence. + */ +const manifest = testManifest(); +const fixture = resolve(import.meta.dirname, '../fixtures/events/claude-post-tool-use.json'); + +let workspace: string; +let previousStateFile: string | undefined; + +beforeEach(async () => { + workspace = await mkdtemp(join(tmpdir(), 'rsc-agent-runtime-event-route-')); + previousStateFile = process.env.AGENT_RUNTIME_STATE_FILE; + process.env.AGENT_RUNTIME_STATE_FILE = join(workspace, 'state.json'); +}); + +afterEach(async () => { + if (previousStateFile === undefined) delete process.env.AGENT_RUNTIME_STATE_FILE; + else process.env.AGENT_RUNTIME_STATE_FILE = previousStateFile; + await rm(workspace, { force: true, recursive: true }); +}); + +it('compiles the PostToolUse hook as a real event route rather than configuration', () => { + expect(manifest.proofLevel).toBe('route-unit'); + expect(manifest.diagnostics.filter((diagnostic) => diagnostic.severity === 'error')).toEqual([]); + expect(manifest.routes['event:tool/after']).toMatchObject({ + kind: 'event-route', + relativePath: 'src/events/tool/after.tsx', + }); +}); + +it('renders a native Claude PostToolUse envelope into the document the host projects from', async () => { + const native = JSON.parse(await readFile(fixture, 'utf8')) as Record; + const rendered = await renderRoute('event:tool/after', { + input: { + canonical: { + event: 'tool/after', + idempotencyKey: 'route-unit-claude-write', + observedAt: '2026-09-01T00:00:00.000Z', + provenance: { + host: 'claude', + hostContractRevision: 'route-unit', + nativeEvent: 'PostToolUse', + source: 'native', + }, + sequence: 1, + }, + native: { ...native, cwd: workspace }, + }, + }); + + expect(rendered.invocation.kind).toBe('event'); + expectDocument(rendered) + .toHaveStatus('success') + .toHaveNodeKinds(['result', 'context']) + .toContainContext('Recorded claude-note.txt from claude. Shared state now contains 1 edit.'); + expect(rendered.provenance).toMatchObject({ kind: 'event-route', proofLevel: 'route-unit' }); +}); diff --git a/package.json b/package.json index 0c25cdcb2..5f74fcaf7 100644 --- a/package.json +++ b/package.json @@ -10,9 +10,10 @@ "scripts": { "build": "pnpm --filter @agent-bundle/runtime build && pnpm --filter agent-bundle build && pnpm --filter create-agent-bundle build", "lint:package": "publint packages/agent-bundle && publint packages/rsc-runtime && publint packages/create-agent-bundle", - "test": "pnpm test:unit && pnpm test:route-unit && pnpm test:integration", + "test": "pnpm test:unit && pnpm test:route-unit && pnpm test:projection && pnpm test:integration", "test:unit": "rstest --config rstest.unit.config.ts", "test:route-unit": "rstest --config rstest.route-unit.config.ts", + "test:projection": "rstest --config rstest.projection.config.ts", "test:integration": "pnpm build && pnpm test:integration:run", "test:integration:run": "AGENT_BUNDLE_WORKBENCH_PREBUILT=1 AGENT_BUNDLE_PACKAGE_PREBUILT=1 rstest --config rstest.integration.config.ts", "test:evidence": "pnpm build && AGENT_BUNDLE_WORKBENCH_PREBUILT=1 AGENT_BUNDLE_PACKAGE_PREBUILT=1 rstest --config rstest.evidence.config.ts", @@ -20,7 +21,7 @@ "lint": "rslint .", "bench:hook-cold-start": "node scripts/measure-hook-cold-start.mjs", "typecheck": "tsc --noEmit && tsc --project packages/workbench/tsconfig.json && tsc --project packages/create-agent-bundle/tsconfig.json", - "check": "pnpm build && pnpm test:unit && pnpm test:route-unit && pnpm test:integration:run && pnpm lint && pnpm typecheck", + "check": "pnpm build && pnpm test:unit && pnpm test:route-unit && pnpm test:projection && pnpm test:integration:run && pnpm lint && pnpm typecheck", "check:local-ci": "node scripts/local-ci.mjs", "docs:runtime-topology": "node scripts/rsc-runtime-topology.mjs --root . --output docs/architecture/rsc-runtime-workbench.md", "eval:spot": "pnpm build && pnpm --filter @agent-bundle/rsc-agent-runtime-demo build && pnpm --filter @agent-bundle/rsc-agent-runtime-demo exec rstest run tests/micro-eval.spot.test.ts --config rstest.config.ts", diff --git a/packages/agent-bundle/README.md b/packages/agent-bundle/README.md index c13b520a8..3d80f26bf 100644 --- a/packages/agent-bundle/README.md +++ b/packages/agent-bundle/README.md @@ -214,6 +214,43 @@ This is the route-unit proof level, and only that: it proves a route module renders to the document it claims. It is not evidence about the MCP transport, a packed artifact, or a browser surface. +### Proof levels + +The levels are separate on purpose. Each helper stamps the level it carried +into its provenance and prints it in every failure, because a pass at one level +is never a receipt for another. + +| level | helpers | what it proves | +| --- | --- | --- | +| `route-unit` | `renderRoute`, `renderRouteEvents` | a route module renders to the document (and render-event stream) it claims | +| `mcp-in-memory` | `openInMemoryMcpServer`, `invokeMcpTool`, `readMcpResource`, `getMcpPrompt`, `listMcpSurface` | the real generated MCP server's protocol contract, over the SDK's in-memory transport | +| `cli-dispatch` | `invokeCli`, `cliJson` | an argv vector resolved and run through the routed CLI's own shell, in-process | +| `packed-stdio` | `openPackedMcpServer` | a built artifact's generated entry running as a real process over stdio | + +```ts +import { cliJson, expectEvents, invokeCli, invokeMcpTool } from 'agent-bundle/test'; + +// mcp-in-memory: the generated server projects the document to protocol content. +const call = await invokeMcpTool('summarize', { input: { title: 'Dune' } }); +expect(call.result.structuredContent).toEqual({ chapters: 24 }); + +// cli-dispatch: the routed CLI resolves the command, parses argv, and maps the exit code. +const run = await invokeCli(['library', 'audit', './books', '--max-files', '8']); +expect(run.exitCode).toBe(0); +expect(cliJson(run)).toMatchObject({ scanned: 8 }); +``` + +`expectEvents` asserts over a render-event stream. `toContainSequence` is +sequence-tolerant — an extra `progress` or `replace` frame is legal and cannot +turn a passing render red — while a missing frame, a reordering, or a regressed +ordinal still fails; `toHaveMonotonicSequence`, `toCompleteOnce`, +`toHaveProgress`, and `toHaveNoErrors` cover the rest of the contract. + +Only `packed-stdio` is process evidence, and it is deliberately expensive: pack +once, install once, spawn once, and iterate every per-route assertion inside +that one session. Browser-App surfaces and deleted-source artifact proofs are +later stages; nothing here stands in for them. + ## Evaluation Eval suites are typed modules discovered by convention: diff --git a/packages/agent-bundle/fixtures/route-harness/src/cli/db/migrate.ts b/packages/agent-bundle/fixtures/route-harness/src/cli/db/migrate.ts new file mode 100644 index 000000000..f6a726b26 --- /dev/null +++ b/packages/agent-bundle/fixtures/route-harness/src/cli/db/migrate.ts @@ -0,0 +1,29 @@ +import type { CliRouteConfig, CliRouteProps } from 'agent-bundle'; +import { z } from 'zod'; + +/** + * Nested one level below the CLI root, so the dispatch level exercises path + * nesting (`db migrate`) rather than a single flat command, and carries the + * `result` exit-code policy so the harness proves that mapping too. + */ +export const config = { + description: 'Applies pending harness migrations.', + exitCode: 'result', +} satisfies CliRouteConfig; + +export const inputSchema = z.object({ + dryRun: z.boolean().default(false), +}).strict(); + +export const resultSchema = z.object({ + applied: z.number().int(), + dryRun: z.boolean(), + exitCode: z.number().int(), +}).strict(); + +export default async function migrate({ input }: CliRouteProps) { + // A dry run reports pending work and exits non-zero without applying it. + return input.dryRun + ? { applied: 0, dryRun: true, exitCode: 3 } + : { applied: 2, dryRun: false, exitCode: 0 }; +} diff --git a/packages/agent-bundle/fixtures/route-harness/src/cli/inventory.ts b/packages/agent-bundle/fixtures/route-harness/src/cli/inventory.ts new file mode 100644 index 000000000..bd49457ed --- /dev/null +++ b/packages/agent-bundle/fixtures/route-harness/src/cli/inventory.ts @@ -0,0 +1,34 @@ +import { agent } from '@agent-bundle/runtime'; +import type { CliRouteConfig, CliRouteProps } from 'agent-bundle'; +import { z } from 'zod'; + +export const config = { + aliases: ['inv'], + description: 'Lists the harness library inventory.', + positionals: ['shelf'], +} satisfies CliRouteConfig; + +export const inputSchema = z.object({ + format: z.enum(['json', 'text']).default('text'), + limit: z.number().int().min(1).max(8).optional(), + shelf: z.string().min(1), +}).strict(); + +export const resultSchema = z.object({ + format: z.string(), + shelf: z.string(), + titles: z.array(z.string()), +}).strict(); + +const shelves: Readonly> = { + fiction: ['Piranesi', 'Solaris'], + history: ['SPQR'], +}; + +export default async function inventory({ input }: CliRouteProps) { + const context = await agent(); + await context.progress.report({ completed: 1, message: 'reading inventory', total: 2 }); + const titles = (shelves[input.shelf] ?? []).slice(0, input.limit ?? 8); + await context.progress.report({ completed: 2, message: 'inventory ready', total: 2 }); + return { format: input.format, shelf: input.shelf, titles: [...titles] }; +} diff --git a/packages/agent-bundle/fixtures/route-harness/src/events/tool/after.tsx b/packages/agent-bundle/fixtures/route-harness/src/events/tool/after.tsx index 33d1b6b1d..4b383c82d 100644 --- a/packages/agent-bundle/fixtures/route-harness/src/events/tool/after.tsx +++ b/packages/agent-bundle/fixtures/route-harness/src/events/tool/after.tsx @@ -1,10 +1,16 @@ import { Agent, agent } from '@agent-bundle/runtime'; +import type { AgentEventRouteProps } from 'agent-bundle'; -export default async function AfterTool({ event, payload }: { readonly event: string; readonly payload: unknown }) { +export default async function AfterTool({ canonical, native }: AgentEventRouteProps) { const context = await agent(); return ( - - {`Observed ${event}.`} + + {`Observed ${canonical.event} from ${canonical.provenance.host}.`} ); } diff --git a/packages/agent-bundle/fixtures/route-harness/src/mcp/harness/prompts/summarize.tsx b/packages/agent-bundle/fixtures/route-harness/src/mcp/harness/prompts/summarize.tsx new file mode 100644 index 000000000..6e2c8d22f --- /dev/null +++ b/packages/agent-bundle/fixtures/route-harness/src/mcp/harness/prompts/summarize.tsx @@ -0,0 +1,23 @@ +import { Agent } from '@agent-bundle/runtime'; +import { z } from 'zod'; + +export const config = { description: 'Summarizes one harness note.', title: 'Summarize' }; + +export const inputSchema = z.object({ note: z.string() }); + +/** The generated server returns a prompt route's result as the protocol's `GetPromptResult`. */ +export const resultSchema = z.object({ + messages: z.array(z.object({ + content: z.object({ text: z.string(), type: z.literal('text') }), + role: z.literal('user'), + })), +}); + +export default async function Summarize({ input }: { readonly input: z.infer }) { + const messages = [{ content: { text: `Summarize ${input.note}`, type: 'text' as const }, role: 'user' as const }]; + return ( + + {`prompt ready for ${input.note}`} + + ); +} diff --git a/packages/agent-bundle/fixtures/route-harness/src/mcp/harness/resources/notes.tsx b/packages/agent-bundle/fixtures/route-harness/src/mcp/harness/resources/notes.tsx index 8336fbd63..e9df810ee 100644 --- a/packages/agent-bundle/fixtures/route-harness/src/mcp/harness/resources/notes.tsx +++ b/packages/agent-bundle/fixtures/route-harness/src/mcp/harness/resources/notes.tsx @@ -5,12 +5,16 @@ export const config = { mimeType: 'text/markdown', title: 'Notes', uri: 'harness export const inputSchema = z.object({ uri: z.string() }); -export const resultSchema = z.object({ uri: z.string() }); +/** The generated server returns a resource route's result as the protocol's `ReadResourceResult`. */ +export const resultSchema = z.object({ + contents: z.array(z.object({ mimeType: z.string(), text: z.string(), uri: z.string() })), +}); export default async function Notes({ input }: { readonly input: z.infer }) { + const text = `# Notes for ${input.uri}`; return ( - - {`# Notes for ${input.uri}`} + + {text} ); } diff --git a/packages/agent-bundle/fixtures/route-harness/src/mcp/harness/tools/catalog.tsx b/packages/agent-bundle/fixtures/route-harness/src/mcp/harness/tools/catalog.tsx new file mode 100644 index 000000000..880d248de --- /dev/null +++ b/packages/agent-bundle/fixtures/route-harness/src/mcp/harness/tools/catalog.tsx @@ -0,0 +1,34 @@ +import { Agent } from '@agent-bundle/runtime'; +import { Suspense } from 'react'; +import { z } from 'zod'; + +export const config = { + description: 'Streams the harness catalog behind one Suspense boundary.', + title: 'Catalog', +}; + +export const inputSchema = z.object({ genre: z.string().optional() }); + +export const resultSchema = z.object({ genre: z.string(), titles: z.array(z.string()) }); + +const titles = ['Piranesi', 'Solaris']; + +/** Resolves after the shell, so the render has a boundary to replace. */ +const Titles = async ({ genre }: { readonly genre: string }) => { + await new Promise((resolve) => { + setTimeout(resolve, 1); + }); + return {`## ${genre}\n\n${titles.map((title) => `- ${title}`).join('\n')}`}; +}; + +export default async function Catalog({ input }: { readonly input: z.infer }) { + const genre = input.genre ?? 'all'; + return ( + + {`catalog: ${genre}`} + }> + + + + ); +} diff --git a/packages/agent-bundle/rslib.config.ts b/packages/agent-bundle/rslib.config.ts index 8cec8328e..1e8780e8d 100644 --- a/packages/agent-bundle/rslib.config.ts +++ b/packages/agent-bundle/rslib.config.ts @@ -38,6 +38,7 @@ export default defineConfig({ index: './src/index.ts', 'mcp-apps': './src/mcp-apps.ts', 'mcp-entry': './src/mcp-entry.ts', + 'mcp-server-runtime': './src/mcp-server-runtime.ts', rstest: './src/rstest/index.ts', test: './src/test/index.ts', }, diff --git a/packages/agent-bundle/src/build/entries.ts b/packages/agent-bundle/src/build/entries.ts index c5f9b4778..5ddbe7f11 100644 --- a/packages/agent-bundle/src/build/entries.ts +++ b/packages/agent-bundle/src/build/entries.ts @@ -1,6 +1,6 @@ import { existsSync } from 'node:fs'; import { readFile, stat } from 'node:fs/promises'; -import { dirname, extname, relative, resolve } from 'node:path'; +import { extname, relative, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { @@ -22,6 +22,8 @@ import { generatedStdioMcpEntrySource, mcpEntryRuntimePath, mcpEntryRuntimeSpecifier, + mcpServerRuntimePath, + mcpServerRuntimeSpecifier, } from './entry-shell.ts'; import type { CompiledMcpApp } from './mcp-apps.ts'; import type { ArtifactOutputKind } from './provenance.ts'; @@ -39,8 +41,17 @@ const eventRuntimeModulePath = (module: 'ipc' | 'project'): string => { throw new Error(`Unable to locate the compiler-owned event ${module} runtime module.`); }; -const eventRuntimeIgnoredRoot = (path: string): string => - resolve(dirname(path), path.replaceAll('\\', '/').includes('/dist/') ? '..' : '../..'); +/** + * The package root owning one compiler-provided runtime module. Provenance + * collects consumer sources, and a runtime module reaches its own siblings + * (`routes/public.ts`, `core/*`) as it is inlined, so the whole owning package + * is what has to be ignored rather than the single aliased file. + */ +const runtimeIgnoredRoot = (path: string): string => { + const normalized = path.replaceAll('\\', '/'); + const marker = normalized.includes('/dist/') ? '/dist/' : '/src/'; + return resolve(normalized.slice(0, normalized.lastIndexOf(marker))); +}; export interface CompiledEntry { readonly name: string; @@ -273,6 +284,9 @@ export const compileMcpEntries = async ( const runtimeShell = entryShells.some((shell) => shell !== undefined) ? mcpEntryRuntimePath() : undefined; const eventIpcRuntime = options.eventHooks.length === 0 ? undefined : eventRuntimeModulePath('ipc'); const eventProjectRuntime = options.eventHooks.length === 0 ? undefined : eventRuntimeModulePath('project'); + const serverRuntime = generatedRouteSources.some((routeSource) => routeSource !== undefined) + ? mcpServerRuntimePath() + : undefined; const mainEntries = compiled.map(({ id, name, source, sourceInputs }, index) => ({ ...(entryShells[index] === undefined || runtimeShell === undefined ? {} @@ -285,6 +299,9 @@ export const compileMcpEntries = async ( [eventIpcRuntimeSpecifier]: eventIpcRuntime, [eventProjectRuntimeSpecifier]: eventProjectRuntime, }), + ...(generatedRouteSources[index] === undefined || serverRuntime === undefined + ? {} + : { [mcpServerRuntimeSpecifier]: serverRuntime }), }, virtualSource: entryShells[index], }), @@ -322,12 +339,13 @@ export const compileMcpEntries = async ( const evidence = await buildWithRslib({ cwd: options.cwd, entries: [...mainEntries, ...workerEntries], - ...([runtimeShell, eventIpcRuntime].filter((path): path is string => path !== undefined).length === 0 + ...([runtimeShell, eventIpcRuntime, serverRuntime].filter((path): path is string => path !== undefined).length === 0 ? {} : { ignoredSourcePaths: [ ...(runtimeShell === undefined ? [] : [runtimeShell]), - ...(eventIpcRuntime === undefined ? [] : [eventRuntimeIgnoredRoot(eventIpcRuntime)]), + ...(eventIpcRuntime === undefined ? [] : [runtimeIgnoredRoot(eventIpcRuntime)]), + ...(serverRuntime === undefined ? [] : [runtimeIgnoredRoot(serverRuntime)]), ], }), logLevel: 'error', @@ -399,7 +417,7 @@ export const compileHooks = async ( ...(eventIpcRuntime === undefined ? {} : { - ignoredSourcePaths: [eventRuntimeIgnoredRoot(eventIpcRuntime)], + ignoredSourcePaths: [runtimeIgnoredRoot(eventIpcRuntime)], }), outputRoot: options.outDir, ...(options.tools === undefined ? {} : { tools: options.tools }), diff --git a/packages/agent-bundle/src/build/entry-shell.ts b/packages/agent-bundle/src/build/entry-shell.ts index fb8e9c5c2..d3c44a5e2 100644 --- a/packages/agent-bundle/src/build/entry-shell.ts +++ b/packages/agent-bundle/src/build/entry-shell.ts @@ -16,24 +16,36 @@ import type { CompiledAgentRoute, CompiledCliCommand } from '../routes/types.ts' export const mcpEntryRuntimeSpecifier = 'agent-bundle/mcp-entry'; /** - * The on-disk location of the `agent-bundle/mcp-entry` runtime module, used - * as a bundler alias so generated entries inline the lifecycle instead of - * leaving an `agent-bundle` import in the emitted artifact (artifacts must - * stay self-contained). From the bundled package this module's URL is - * `dist/.js` with `mcp-entry.js` as a sibling; from checked-out - * sources it is `src/build/entry-shell.ts` with `../mcp-entry.ts`. + * The shared server runtime a generated route entry delegates to: warm Flight + * host, route registration, projection. Aliased rather than imported so the + * emitted artifact stays self-contained, and shared rather than templated so + * `agent-bundle/test`'s in-memory projection level exercises this exact code. */ -export const mcpEntryRuntimePath = (): string => { +export const mcpServerRuntimeSpecifier = 'agent-bundle/mcp-server-runtime'; + +/** + * The on-disk location of one runtime module used as a bundler alias, so + * generated entries inline it instead of leaving an `agent-bundle` import in + * the emitted artifact (artifacts must stay self-contained). From the bundled + * package this module's URL is `dist/.js` with `.js` as a + * sibling; from checked-out sources it is `src/build/entry-shell.ts` with + * `../.ts`. + */ +const runtimeModulePath = (name: string): string => { for (const candidate of [ - new URL('./mcp-entry.js', import.meta.url), - new URL('../mcp-entry.ts', import.meta.url), + new URL(`./${name}.js`, import.meta.url), + new URL(`../${name}.ts`, import.meta.url), ]) { const path = fileURLToPath(candidate); if (existsSync(path)) return path; } - throw new Error('Unable to locate the agent-bundle/mcp-entry runtime module for generated stdio entries.'); + throw new Error(`Unable to locate the agent-bundle/${name} runtime module for generated entries.`); }; +export const mcpEntryRuntimePath = (): string => runtimeModulePath('mcp-entry'); + +export const mcpServerRuntimePath = (): string => runtimeModulePath('mcp-server-runtime'); + /** * The generated stdio MCP entry body for a factory-exporting server module: * the lifecycle installs the console guard before the consumer module @@ -183,13 +195,6 @@ export const generatedRouteArtifactEpoch = (plugin: { const routeProtocolName = (route: CompiledAgentRoute): string => route.id.slice(route.id.lastIndexOf('/') + 1); -const selectedConfig = ( - config: Readonly>, - keys: readonly string[], -): Readonly> => Object.fromEntries( - keys.filter((key) => config[key] !== undefined).map((key) => [key, config[key]]), -); - const executableMcpRoutes = (routes: readonly CompiledAgentRoute[]): readonly CompiledAgentRoute[] => routes.filter((route) => route.kind !== 'app'); @@ -278,51 +283,27 @@ export const generatedRouteFlightWorkerSource = (options: GeneratedRouteFlightWo ].join('\n'); }; -const routeRegistrations = (routes: readonly CompiledAgentRoute[]): readonly string[] => { - const registrations: string[] = []; +/** + * Build-time validation of the MCP route kinds a generated server registers. + * Registration itself is data-driven at runtime (`registerGeneratedRoutes` in + * `agent-bundle/mcp-server-runtime`), but a resource without a static + * `config.uri` and a non-MCP route inside an MCP server are compile-time + * defects: they must fail the build, not the first request. + */ +const assertRegistrableMcpRoutes = (routes: readonly CompiledAgentRoute[]): void => { for (const route of routes) { - const access = `routes[${JSON.stringify(route.id)}]`; switch (route.kind) { - case 'tool': { - const config = selectedConfig(route.config, ['_meta', 'annotations', 'description', 'icons', 'title']); - registrations.push([ - ` server.registerTool(${JSON.stringify(routeProtocolName(route))}, {`, - ` ...${stableJson(config)},`, - ` inputSchema: ${access}.module.inputSchema,`, - ` outputSchema: ${access}.module.resultSchema,`, - ` }, async (input, context) => {`, - ` const rendered = await renderRoute(dispatcher, ${access}, input, context);`, - ' return attachMcpStructuredContent(rendered.toolResult, rendered.result);', - ' });', - ].join('\n')); + case 'tool': + case 'prompt': + case 'app': break; - } case 'resource': { const uri = route.config['uri']; if (typeof uri !== 'string' || uri.trim() === '') { throw new Error(`Generated resource route ${JSON.stringify(route.id)} requires a non-empty static config.uri.`); } - const config = selectedConfig(route.config, ['_meta', 'description', 'icons', 'mimeType', 'title']); - registrations.push([ - ` server.registerResource(${JSON.stringify(routeProtocolName(route))}, ${JSON.stringify(uri)}, ${stableJson(config)}, async (uri, context) => {`, - ` const rendered = await renderRoute(dispatcher, ${access}, { uri: uri.href }, context);`, - ' return rendered.result;', - ' });', - ].join('\n')); - break; - } - case 'prompt': { - const config = selectedConfig(route.config, ['_meta', 'description', 'icons', 'title']); - registrations.push([ - ` server.registerPrompt(${JSON.stringify(routeProtocolName(route))}, {`, - ` ...${stableJson(config)},`, - ` argsSchema: ${access}.module.inputSchema,`, - ` }, async (input, context) => (await renderRoute(dispatcher, ${access}, input, context)).result);`, - ].join('\n')); break; } - case 'app': - break; case 'event-route': case 'cli': case 'script': @@ -333,23 +314,26 @@ const routeRegistrations = (routes: readonly CompiledAgentRoute[]): readonly str } } } - return registrations; }; /** - * The generated MCP entry owns the stream projector and one warm Flight - * worker. The worker is split only to satisfy React's react-server condition; - * it is reused for every request until the MCP server closes. + * The generated MCP entry: the compiled route table, the compiled App + * registry, and one warm Flight worker handed to the shared server runtime. + * The worker is split out only to satisfy React's react-server condition; it + * is reused for every request until the MCP server closes. + * + * Everything below the route table lives in `agent-bundle/mcp-server-runtime`, + * so the in-memory projection proof level registers, renders, and projects + * through the same code this artifact runs. */ export const generatedRouteMcpEntrySource = (options: GeneratedRouteMcpEntryOptions): string => { const routes = executableMcpRoutes(options.routes); + assertRegistrableMcpRoutes(routes); const artifactEpoch = generatedRouteArtifactEpoch(options.plugin); const hasEvents = (options.eventRoutes?.length ?? 0) > 0; return [ ...(hasEvents ? ["import { dirname, resolve } from 'node:path';"] : []), - "import { Worker } from 'node:worker_threads';", - "import { McpServer } from '@modelcontextprotocol/server';", - "import { AgentRuntimeError, agent, attachMcpStructuredContent, available, createAgentRenderDispatcher, createWarmFlightHost, projectMcpRenderStream, runAgentRequest } from '@agent-bundle/runtime';", + `import { createFlightWorkerHost, createGeneratedRouteMcpServer } from ${JSON.stringify(mcpServerRuntimeSpecifier)};`, ...(hasEvents ? [ `import { createEventRuntimeServer } from ${JSON.stringify(eventIpcRuntimeSpecifier)};`, @@ -364,123 +348,31 @@ export const generatedRouteMcpEntrySource = (options: GeneratedRouteMcpEntryOpti ...routeRecords(routes), '});', '', - 'const workerError = (message) => {', - " if (message.code === 'artifact-epoch-mismatch') return new AgentRuntimeError('artifact-epoch-mismatch', message.message, { expectedEpoch: ARTIFACT_EPOCH, receivedEpoch: message.receivedEpoch });", - " if (message.code === 'runtime-unavailable') return new AgentRuntimeError('runtime-unavailable', message.message);", - " if (message.code === 'runtime-restarted') return new AgentRuntimeError('runtime-restarted', message.message);", - ' return new Error(message.message);', - '};', - '', - 'const createWorkerHost = () => {', - ` const worker = new Worker(new URL(${JSON.stringify(`./${options.workerFile}`)}, import.meta.url), { stderr: true, stdout: true });`, - ' worker.stdout?.on(\'data\', (chunk) => process.stderr.write(chunk));', - ' worker.stderr?.on(\'data\', (chunk) => process.stderr.write(chunk));', - ' const pending = new Map();', - ' let sequence = 0;', - ' let exited = false;', - ' const failPending = (error) => { for (const request of pending.values()) request.reject(error); pending.clear(); };', - ' worker.on(\'error\', (error) => { exited = true; failPending(error); });', - ' worker.on(\'exit\', (code) => {', - ' exited = true;', - " failPending(new AgentRuntimeError(code === 0 ? 'runtime-unavailable' : 'runtime-restarted', code === 0 ? 'The MCP render runtime is unavailable' : `The MCP render runtime restarted; worker exited with code ${String(code)}.`));", - ' });', - ' worker.on(\'message\', (message) => {', - ' const request = pending.get(message.id);', - ' if (request === undefined) return;', - " if (message.type === 'progress') { void request.progress?.report(message.update); return; }", - ' pending.delete(message.id);', - ' request.signal.removeEventListener(\'abort\', request.abort);', - " if (message.type === 'error') { request.reject(workerError(message)); return; }", - ' request.resolve(new ReadableStream({ start(controller) { controller.enqueue(message.bytes); controller.close(); } }));', - ' });', - ' const host = Object.freeze({', - ' close: async () => { await worker.terminate(); },', - ' execute: async ({ artifactEpoch, invocation, progress, signal }) => {', - " if (exited) throw new AgentRuntimeError('runtime-unavailable', 'The MCP render runtime is unavailable');", - ' const context = await agent();', - ' const id = ++sequence;', - ' return new Promise((resolve, reject) => {', - " const abort = () => { worker.postMessage({ id, type: 'cancel' }); pending.delete(id); reject(new DOMException('Agent render was aborted', 'AbortError')); };", - ' pending.set(id, { abort, progress, reject, resolve, signal });', - " signal.addEventListener('abort', abort, { once: true });", - ' if (signal.aborted) { abort(); return; }', - ' worker.postMessage({ actor: context.actor, artifactEpoch: artifactEpoch ?? ARTIFACT_EPOCH, id, invocation, session: context.session, type: \'render\' });', - ' });', - ' },', - ' });', - ' return createWarmFlightHost({ artifactEpoch: ARTIFACT_EPOCH, close: host.close, host });', - '};', - '', - 'const requestIdentity = (context) => ({', - ' ...(context.http?.authInfo?.clientId === undefined ? {} : { actor: available({ id: context.http.authInfo.clientId }, \'native\') }),', - " ...(typeof context.sessionId === 'string' && context.sessionId.trim() !== '' ? { session: available({ sessionId: context.sessionId }, 'native') } : {}),", - '});', - '', - 'const mcpProjectorOptions = (context) => {', - ' const progressToken = context.mcpReq._meta?.progressToken;', - ' return {', - ' signal: context.mcpReq.signal,', - ' ...(progressToken === undefined ? {} : {', - ' progressToken,', - " sendProgress: (params) => context.mcpReq.notify({ method: 'notifications/progress', params }),", - ' }),', - ' };', - '};', - '', - 'const renderRoute = async (dispatcher, route, input, context) => runAgentRequest({', - ' ...requestIdentity(context),', - " invocation: { artifactEpoch: ARTIFACT_EPOCH, kind: 'tool', operationId: route.id, surface: route.name },", - ' signal: context.mcpReq.signal,', - '}, async () => {', - ' const projected = await projectMcpRenderStream(dispatcher.stream({', - ' artifactEpoch: ARTIFACT_EPOCH,', - " invocation: { kind: 'tool', props: { input, operationId: route.id } },", - ' signal: context.mcpReq.signal,', - ' }), mcpProjectorOptions(context));', - ' return { document: projected.document, result: route.module.resultSchema.parse(projected.document.value), toolResult: projected.result };', - '});', - '', - 'const createGeneratedRouteServer = async () => {', - ` const server = new McpServer(${stableJson(options.plugin)});`, - ' const workerHost = createWorkerHost();', - ' const dispatcher = createAgentRenderDispatcher(workerHost);', ...(hasEvents ? [ - ` const artifactEpoch = ${JSON.stringify(options.artifactEpoch ?? 'unknown')};`, - ` const target = ${JSON.stringify(options.target ?? 'unknown')};`, - " const endpointId = `${artifactEpoch}:${target}:${dirname(dirname(resolve(process.argv[1])))}`;", - ' const eventRuntime = await createEventRuntimeServer({', - ' artifactEpoch,', - ' endpointId,', - ' handle: async (request) => {', - " const nativeEvent = typeof request.native.hook_event_name === 'string' ? request.native.hook_event_name : request.event;", - ' const controller = new AbortController();', - ' const props = createCanonicalEventProps(request.event, request.native, target, nativeEvent, request.hostContractRevision, controller.signal);', - ' const sessionId = typeof request.native.session_id === \'string\' ? request.native.session_id : typeof request.native.conversation_id === \'string\' ? request.native.conversation_id : undefined;', - ' return runAgentRequest({', - " host: available({ name: target }, 'native'),", - " invocation: { artifactEpoch, hostContractRevision: request.hostContractRevision, kind: 'event', operationId: `event:${request.event}`, surface: request.event },", - " ...(sessionId === undefined ? {} : { session: available({ sessionId }, 'native') }),", - ' signal: controller.signal,', - " ...(typeof request.native.cwd === 'string' ? { workspace: available({ root: request.native.cwd }, 'native') } : {}),", - ' }, async () => {', - " const document = await dispatcher.dispatch({ invocation: { kind: 'event', props: { event: request.event, payload: { canonical: props.canonical, native: props.native } } }, signal: controller.signal });", - ' return projectEventDocument(document, request.event, target, nativeEvent);', - ' });', - ' },', - ' });', + // The endpoint identity is artifact-location dependent, so it stays + // in the artifact rather than the shared runtime. + `const EVENT_ARTIFACT_EPOCH = ${JSON.stringify(options.artifactEpoch ?? 'unknown')};`, + `const EVENT_TARGET = ${JSON.stringify(options.target ?? 'unknown')};`, + 'const events = Object.freeze({', + ' artifactEpoch: EVENT_ARTIFACT_EPOCH,', + ' createCanonicalEventProps,', + ' createEventRuntimeServer,', + ' endpointId: `${EVENT_ARTIFACT_EPOCH}:${EVENT_TARGET}:${dirname(dirname(resolve(process.argv[1])))}`,', + ' projectEventDocument,', + ' target: EVENT_TARGET,', + '});', + '', ] : []), - ...routeRegistrations(routes), - ' for (const app of mcpApps) {', - ' server.registerResource(app.name, app.resourceUri, { ...(app._meta === undefined ? {} : { _meta: app._meta }), mimeType: app.mimeType }, async (uri) => ({ contents: [{ mimeType: app.mimeType, text: app.html, uri: uri.href }] }));', - ' }', - ' const close = server.close.bind(server);', - ` server.close = async () => { ${hasEvents ? 'await eventRuntime.close(); ' : ''}await workerHost.close(); await close(); };`, - ' return server;', - '};', - '', - 'export default createGeneratedRouteServer;', + 'export default async () => createGeneratedRouteMcpServer({', + ' apps: mcpApps,', + ' artifactEpoch: ARTIFACT_EPOCH,', + ...(hasEvents ? [' events,'] : []), + ` host: createFlightWorkerHost(new URL(${JSON.stringify(`./${options.workerFile}`)}, import.meta.url), ARTIFACT_EPOCH),`, + ` plugin: ${stableJson(options.plugin)},`, + ' routes,', + '});', '', ].join('\n'); }; diff --git a/packages/agent-bundle/src/build/inspect-bundler.ts b/packages/agent-bundle/src/build/inspect-bundler.ts index b4670a770..66e3bf6f5 100644 --- a/packages/agent-bundle/src/build/inspect-bundler.ts +++ b/packages/agent-bundle/src/build/inspect-bundler.ts @@ -9,6 +9,8 @@ import { generatedStdioMcpEntrySource, mcpEntryRuntimePath, mcpEntryRuntimeSpecifier, + mcpServerRuntimePath, + mcpServerRuntimeSpecifier, } from './entry-shell.ts'; import { planCompiledMcpEntries } from './entries.ts'; import { composeMcpAppsRsbuildConfig, planCompiledMcpApps } from './mcp-apps.ts'; @@ -175,7 +177,10 @@ const mcpEntryEntries = async ( entry: { ...(wrapped ? { - aliases: { [mcpEntryRuntimeSpecifier]: mcpEntryRuntimePath() }, + aliases: { + [mcpEntryRuntimeSpecifier]: mcpEntryRuntimePath(), + ...(routeSource === undefined ? {} : { [mcpServerRuntimeSpecifier]: mcpServerRuntimePath() }), + }, virtualSource: generatedStdioMcpEntrySource({ entrySource: routeSource === undefined ? entry.source : 'agent-bundle/generated-route-server', serverName, diff --git a/packages/agent-bundle/src/build/rslib.ts b/packages/agent-bundle/src/build/rslib.ts index e00db5c71..bbfdb5143 100644 --- a/packages/agent-bundle/src/build/rslib.ts +++ b/packages/agent-bundle/src/build/rslib.ts @@ -1,6 +1,7 @@ // Rslib re-exports its own Rsbuild/Rspack stack (values and types alike); // installing @rspack/core separately risks version conflicts // (https://rslib.rs/api/javascript-api/core). +import { pluginReact } from '@rsbuild/plugin-react'; import { createRslib, mergeRslibConfig, rspack, type LibConfig, type Rspack } from '@rslib/core'; import { init, parse } from 'es-module-lexer'; import { readFile, realpath } from 'node:fs/promises'; @@ -516,6 +517,12 @@ export const composeEntryLibConfig = ( performance: { buildCache: false, }, + // Routes are authored as TSX, and the bare Rslib transform lowers JSX to + // the classic `React.createElement` factory, which no generated entry has + // in scope. The React plugin selects the automatic runtime, so emitted + // modules import `react/jsx-runtime` themselves — under the react-server + // condition for worker entries, which is what RSC rendering needs. + plugins: [pluginReact({ fastRefresh: false })], // Rsbuild 2.x deprecated performance.chunkSplit 'all-in-one'; the // documented migration is top-level splitChunks: false, which also // guards against the node-target splitting default added in v2.2. diff --git a/packages/agent-bundle/src/mcp-server-runtime.ts b/packages/agent-bundle/src/mcp-server-runtime.ts new file mode 100644 index 000000000..05073c160 --- /dev/null +++ b/packages/agent-bundle/src/mcp-server-runtime.ts @@ -0,0 +1,492 @@ +/** + * The runtime half of a generated MCP server: the warm Flight worker host, + * route registration, request identity, the projected render, and the + * Agent Document → MCP result projection. + * + * The generated stdio entry (`src/build/entry-shell.ts`) used to inline all of + * this as template text, which made it unreachable outside a built artifact. + * It lives here so exactly one implementation serves both the packed server a + * host spawns and the in-memory server the projection proof level connects a + * client to — a test that renders through a second registration or projection + * path proves nothing about the artifact. + * + * The generated entry reaches this module through the `agent-bundle/mcp-server-runtime` + * bundler alias, so artifacts stay self-contained; `agent-bundle/test` imports + * it directly. + */ +import { Worker } from 'node:worker_threads'; + +import { McpServer } from '@modelcontextprotocol/server'; +import { + AgentRuntimeError, + agent, + attachMcpStructuredContent, + available, + createAgentRenderDispatcher, + createWarmFlightHost, + projectMcpRenderStream, + runAgentRequest, +} from '@agent-bundle/runtime'; +import type { createEventRuntimeServer } from './events/ipc.ts'; +import type { createCanonicalEventProps, projectEventDocument } from './events/project.ts'; +import { canonicalAgentEvents, type CanonicalAgentEvent } from './routes/public.ts'; +import type { + AgentActorIdentity, + AgentDocument, + AgentProgressReporter, + AgentRenderDispatch, + AgentRenderDispatcher, + AgentSessionIdentity, + McpProgressNotificationParams, + McpProgressToken, + Observed, + WarmFlightHost, +} from '@agent-bundle/runtime'; + +/** One route the generated server hosts, as the generated module records it. */ +export interface GeneratedRouteRecord { + /** The route module's statically extracted `config` export. */ + readonly config: Readonly>; + readonly id: string; + readonly kind: 'tool' | 'resource' | 'prompt'; + readonly module: { + readonly default: (props: never) => unknown; + readonly inputSchema?: unknown; + readonly resultSchema: { readonly parse: (value: unknown) => unknown }; + }; + /** The protocol name the server registers — the route id's last segment. */ + readonly name: string; +} + +/** One compiled MCP App the generated server serves as a resource. */ +export interface GeneratedMcpAppRecord { + readonly _meta?: Readonly>; + readonly html: string; + readonly mimeType: string; + readonly name: string; + readonly resourceUri: string; +} + +/** + * The subset of the MCP SDK's request context the generated server reads. + * Structural so the harness can build one without importing SDK internals. + */ +export interface GeneratedRouteRequestContext { + readonly http?: { readonly authInfo?: { readonly clientId?: string } }; + readonly mcpReq: { + readonly _meta?: { readonly progressToken?: McpProgressToken }; + readonly notify?: (notification: { + readonly method: 'notifications/progress'; + readonly params: McpProgressNotificationParams; + }) => Promise; + readonly signal: AbortSignal; + }; + readonly sessionId?: string; +} + +export interface RenderedGeneratedRoute { + readonly document: AgentDocument; + /** The document value parsed by the route's own `resultSchema`. */ + readonly result: unknown; + /** The projected `CallToolResult`, before structured content is attached. */ + readonly toolResult: Awaited>['result']; +} + +interface GeneratedRouteIdentity { + readonly actor?: Observed; + readonly session?: Observed; +} + +/** Identity the server derives from the transport's own request context. */ +const requestIdentity = (context: GeneratedRouteRequestContext): GeneratedRouteIdentity => ({ + ...(context.http?.authInfo?.clientId === undefined + ? {} + : { actor: available({ id: context.http.authInfo.clientId }, 'native') }), + ...(typeof context.sessionId === 'string' && context.sessionId.trim() !== '' + ? { session: available({ sessionId: context.sessionId }, 'native') } + : {}), +}); + +/** + * Progress forwarding for one request: the client's own `progressToken` is + * what turns render progress into `notifications/progress`, so a request that + * did not ask for progress gets none. + */ +const projectorOptions = (context: GeneratedRouteRequestContext): { + readonly progressToken?: McpProgressToken; + readonly sendProgress?: (params: McpProgressNotificationParams) => Promise; + readonly signal: AbortSignal; +} => { + const progressToken = context.mcpReq._meta?.progressToken; + const notify = context.mcpReq.notify; + return { + signal: context.mcpReq.signal, + ...(progressToken === undefined || notify === undefined + ? {} + : { + progressToken, + sendProgress: async (params: McpProgressNotificationParams) => + notify({ method: 'notifications/progress', params }), + }), + }; +}; + +/** + * Renders one route inside a request scope, projects its render-event stream + * into an MCP result, and validates the document value against the route's + * own `resultSchema` — exactly what the generated server does per request. + */ +export const renderGeneratedRoute = async ( + dispatcher: AgentRenderDispatcher, + artifactEpoch: string, + route: GeneratedRouteRecord, + input: unknown, + context: GeneratedRouteRequestContext, +): Promise => runAgentRequest({ + ...requestIdentity(context), + invocation: { artifactEpoch, kind: 'tool', operationId: route.id, surface: route.name }, + signal: context.mcpReq.signal, +}, async () => { + const projected = await projectMcpRenderStream(dispatcher.stream({ + artifactEpoch, + invocation: { kind: 'tool', props: { input: input as never, operationId: route.id } }, + signal: context.mcpReq.signal, + }), projectorOptions(context)); + return { + document: projected.document, + result: route.module.resultSchema.parse(projected.document.value), + toolResult: projected.result, + }; +}); + +const selectedConfig = ( + config: Readonly>, + keys: readonly string[], +): Record => Object.fromEntries( + keys.filter((key) => config[key] !== undefined).map((key) => [key, config[key]]), +); + +/** Registers the compiled MCP routes on a server, keyed by route kind. */ +export const registerGeneratedRoutes = ( + server: McpServer, + routes: Readonly>, + dispatcher: AgentRenderDispatcher, + artifactEpoch: string, +): void => { + for (const route of Object.values(routes)) { + switch (route.kind) { + case 'tool': + server.registerTool(route.name, { + ...selectedConfig(route.config, ['_meta', 'annotations', 'description', 'icons', 'title']), + inputSchema: route.module.inputSchema, + outputSchema: route.module.resultSchema, + } as never, (async (input: unknown, context: GeneratedRouteRequestContext) => { + const rendered = await renderGeneratedRoute(dispatcher, artifactEpoch, route, input, context); + return attachMcpStructuredContent(rendered.toolResult, rendered.result); + }) as never); + break; + case 'resource': { + const uri = route.config['uri']; + if (typeof uri !== 'string' || uri.trim() === '') { + throw new Error(`Generated resource route ${JSON.stringify(route.id)} requires a non-empty static config.uri.`); + } + server.registerResource( + route.name, + uri, + selectedConfig(route.config, ['_meta', 'description', 'icons', 'mimeType', 'title']) as never, + (async (resourceUri: URL, context: GeneratedRouteRequestContext) => + (await renderGeneratedRoute(dispatcher, artifactEpoch, route, { uri: resourceUri.href }, context)).result) as never, + ); + break; + } + case 'prompt': + server.registerPrompt(route.name, { + ...selectedConfig(route.config, ['_meta', 'description', 'icons', 'title']), + argsSchema: route.module.inputSchema, + } as never, (async (input: unknown, context: GeneratedRouteRequestContext) => + (await renderGeneratedRoute(dispatcher, artifactEpoch, route, input, context)).result) as never); + break; + default: { + const unreachable: never = route.kind; + throw new TypeError(`Unsupported generated MCP route kind ${String(unreachable)}.`); + } + } + } +}; + +/** Registers compiled MCP App surfaces as inline HTML resources. */ +export const registerGeneratedMcpApps = ( + server: McpServer, + apps: readonly GeneratedMcpAppRecord[], +): void => { + for (const app of apps) { + server.registerResource( + app.name, + app.resourceUri, + { ...(app._meta === undefined ? {} : { _meta: app._meta }), mimeType: app.mimeType } as never, + (async (uri: URL) => ({ + contents: [{ mimeType: app.mimeType, text: app.html, uri: uri.href }], + })) as never, + ); + } +}; + +interface FlightWorkerMessage { + readonly bytes?: Uint8Array; + readonly code?: string; + readonly id: number; + readonly message?: string; + readonly receivedEpoch?: string; + readonly type: string; + readonly update?: never; +} + +/** + * The long-lived react-server worker one generated MCP process renders + * through. The worker exists only to satisfy React's `react-server` + * condition; it is reused for every request until the server closes, and the + * warm host is what turns a dead worker or a stale artifact epoch into a + * typed `AgentRuntimeError` instead of a hung request. + */ +export const createFlightWorkerHost = ( + workerUrl: URL, + artifactEpoch: string, +): WarmFlightHost => { + interface PendingRender { + readonly abort: () => void; + readonly progress?: AgentProgressReporter; + readonly reject: (error: Error) => void; + readonly resolve: (stream: ReadableStream) => void; + readonly signal: AbortSignal; + } + // Generated route modules may write to stdout; stdout is the stdio + // transport's protocol channel, so the worker's own output is rerouted. + const worker = new Worker(workerUrl, { stderr: true, stdout: true }); + worker.stdout.on('data', (chunk: unknown) => process.stderr.write(chunk as Uint8Array)); + worker.stderr.on('data', (chunk: unknown) => process.stderr.write(chunk as Uint8Array)); + const pending = new Map(); + let sequence = 0; + let exited = false; + const failPending = (error: Error): void => { + for (const request of pending.values()) request.reject(error); + pending.clear(); + }; + const workerError = (message: FlightWorkerMessage): Error => { + switch (message.code) { + case 'artifact-epoch-mismatch': + return new AgentRuntimeError('artifact-epoch-mismatch', message.message ?? 'Artifact epoch mismatch', { + expectedEpoch: artifactEpoch, + ...(message.receivedEpoch === undefined ? {} : { receivedEpoch: message.receivedEpoch }), + }); + case 'runtime-unavailable': + case 'runtime-restarted': + return new AgentRuntimeError(message.code, message.message ?? 'The MCP render runtime is unavailable'); + default: + return new Error(message.message); + } + }; + worker.on('error', (error: Error) => { + exited = true; + failPending(error); + }); + worker.on('exit', (code: number) => { + exited = true; + failPending(new AgentRuntimeError( + code === 0 ? 'runtime-unavailable' : 'runtime-restarted', + code === 0 + ? 'The MCP render runtime is unavailable' + : `The MCP render runtime restarted; worker exited with code ${String(code)}.`, + )); + }); + worker.on('message', (message: FlightWorkerMessage) => { + const request = pending.get(message.id); + if (request === undefined) return; + if (message.type === 'progress') { + void request.progress?.report(message.update as never); + return; + } + pending.delete(message.id); + request.signal.removeEventListener('abort', request.abort); + if (message.type === 'error') { + request.reject(workerError(message)); + return; + } + request.resolve(new ReadableStream({ + start(controller) { + controller.enqueue(message.bytes!); + controller.close(); + }, + })); + }); + return createWarmFlightHost({ + artifactEpoch, + close: async (): Promise => { + await worker.terminate(); + }, + host: { + execute: async ({ + artifactEpoch: requestEpoch, + invocation, + progress, + signal, + }: AgentRenderDispatch): Promise> => { + if (exited) { + throw new AgentRuntimeError('runtime-unavailable', 'The MCP render runtime is unavailable'); + } + const context = await agent(); + const id = ++sequence; + return new Promise>((resolve, reject) => { + const abort = (): void => { + worker.postMessage({ id, type: 'cancel' }); + pending.delete(id); + reject(new DOMException('Agent render was aborted', 'AbortError')); + }; + pending.set(id, { abort, ...(progress === undefined ? {} : { progress }), reject, resolve, signal }); + signal.addEventListener('abort', abort, { once: true }); + if (signal.aborted) { + abort(); + return; + } + worker.postMessage({ + actor: context.actor, + artifactEpoch: requestEpoch ?? artifactEpoch, + id, + invocation, + session: context.session, + type: 'render', + }); + }); + }, + }, + }); +}; + +/** The render host a generated server closes over, plus its teardown. */ +export type GeneratedRouteExecutionHost = WarmFlightHost; + +/** + * The shared event runtime an artifact's event routes are served through + * (#97/#180), injected rather than imported: the generated entry reaches the + * IPC and projection modules through bundler aliases that only exist inside a + * built artifact, while this module is also imported directly by the test + * harness. An artifact with no event routes passes nothing. + */ +export interface GeneratedEventRuntimeBinding { + readonly artifactEpoch: string; + readonly createCanonicalEventProps: typeof createCanonicalEventProps; + readonly createEventRuntimeServer: typeof createEventRuntimeServer; + /** Identifies this artifact's socket, so two installs never share a runtime. */ + readonly endpointId: string; + readonly projectEventDocument: typeof projectEventDocument; + readonly target: string; +} + +export interface CreateGeneratedRouteMcpServerOptions { + readonly apps?: readonly GeneratedMcpAppRecord[]; + /** Identity every request carries, so a stale worker fails loudly. */ + readonly artifactEpoch: string; + readonly events?: GeneratedEventRuntimeBinding; + /** Renders one invocation to Flight bytes. Closed when the server closes. */ + readonly host: GeneratedRouteExecutionHost; + readonly plugin: { readonly name: string; readonly version: string }; + readonly routes: Readonly>; +} + +const nativeString = ( + native: Readonly>, + key: string, +): string | undefined => (typeof native[key] === 'string' ? native[key] : undefined); + +/** + * The IPC request carries the event name as a plain string, so it is narrowed + * here rather than asserted: an envelope naming an event this contract does + * not define is a transport failure, not a route render. + */ +const canonicalEvent = (event: string): CanonicalAgentEvent => { + const canonical = canonicalAgentEvents.find((candidate) => candidate === event); + if (canonical === undefined) { + throw new TypeError(`Event runtime received the unknown canonical event ${JSON.stringify(event)}.`); + } + return canonical; +}; + +/** + * The shared event runtime for one artifact: native envelopes arrive over the + * IPC socket, render through the same dispatcher every route uses, and project + * back into the host's own hook response shape. + */ +const startEventRuntime = async ( + events: GeneratedEventRuntimeBinding, + dispatcher: AgentRenderDispatcher, +): Promise<{ readonly close: () => Promise }> => events.createEventRuntimeServer({ + artifactEpoch: events.artifactEpoch, + endpointId: events.endpointId, + handle: async (request) => { + const event = canonicalEvent(request.event); + const nativeEvent = nativeString(request.native, 'hook_event_name') ?? event; + const controller = new AbortController(); + const props = events.createCanonicalEventProps( + event, + request.native, + events.target, + nativeEvent, + request.hostContractRevision, + controller.signal, + ); + const sessionId = nativeString(request.native, 'session_id') + ?? nativeString(request.native, 'conversation_id'); + const workspaceRoot = nativeString(request.native, 'cwd'); + return runAgentRequest({ + host: available({ name: events.target }, 'native'), + invocation: { + artifactEpoch: events.artifactEpoch, + hostContractRevision: request.hostContractRevision, + kind: 'event', + operationId: `event:${event}`, + surface: event, + }, + ...(sessionId === undefined ? {} : { session: available({ sessionId }, 'native') }), + signal: controller.signal, + ...(workspaceRoot === undefined ? {} : { workspace: available({ root: workspaceRoot }, 'native') }), + }, async () => events.projectEventDocument( + await dispatcher.dispatch({ + invocation: { + kind: 'event', + // The event payload crosses the render boundary as data; the route + // props type is what gives it shape on the other side. + props: { event, payload: { canonical: props.canonical, native: props.native } as never }, + }, + signal: controller.signal, + }), + event, + events.target, + nativeEvent, + )); + }, +}); + +/** + * Builds the MCP server a generated artifact serves: the dispatcher over the + * supplied warm render host, every compiled route registered by kind, the + * compiled MCP Apps as inline resources, and — when the artifact has event + * routes — the shared event runtime over that same dispatcher. Closing the + * server closes both. + */ +export const createGeneratedRouteMcpServer = async ( + options: CreateGeneratedRouteMcpServerOptions, +): Promise => { + const server = new McpServer(options.plugin); + const dispatcher = createAgentRenderDispatcher(options.host); + const events = options.events === undefined + ? undefined + : await startEventRuntime(options.events, dispatcher); + registerGeneratedRoutes(server, options.routes, dispatcher, options.artifactEpoch); + registerGeneratedMcpApps(server, options.apps ?? []); + const close = server.close.bind(server); + server.close = async (): Promise => { + await events?.close(); + await options.host.close(); + await close(); + }; + return server; +}; diff --git a/packages/agent-bundle/src/test/cli.ts b/packages/agent-bundle/src/test/cli.ts new file mode 100644 index 000000000..db925db74 --- /dev/null +++ b/packages/agent-bundle/src/test/cli.ts @@ -0,0 +1,250 @@ +/** + * The CLI dispatch proof level. + * + * `invokeCli` runs one argv vector through the routed CLI's own shell + * (`runGeneratedCliEntry`, #102 stage 2) over the compiled command graph the + * manifest carries, in this process. Command resolution, the argv projection, + * help, `--version`, and the exit-code mapping are the product's; the only + * thing the harness supplies is the `execute` bridge that runs the matched + * route module — and that mirrors the generated executable's, so a command + * that passes here fails in the same place a shipped binary would. + * + * It does **not** spawn the generated binary: no shebang, no executable bit, + * no process framing. That is the `packed-stdio` level's business. + * + * Rendered (`.tsx`) command routes compile no command until #102 stage 3, so + * this level dispatches plain command routes only; a rendered command is a + * compiler error (`AB4816`) long before it reaches a test. + */ +import type * as AgentRuntime from '@agent-bundle/runtime'; + +import { CliInputError, runGeneratedCliEntry } from '../cli-entry.ts'; +import type { CompiledCliCommand } from '../routes/types.ts'; +import { AgentTestError, captured } from './errors.ts'; +import { CLI_DISPATCH_PROOF_LEVEL, type AgentBundleTestManifest } from './manifest.ts'; +import { registeredRouteLoader, testManifest } from './registry.ts'; +import type { RenderRouteContext } from './render.ts'; +import type { AgentRouteModule, RenderedRouteProvenance } from './types.ts'; + +export interface InvokeCliOptions { + /** Request-scope overrides for the dispatched command, over the runtime's request contract. */ + readonly context?: RenderRouteContext; + readonly manifest?: AgentBundleTestManifest; + readonly signal?: AbortSignal; +} + +export interface CliInvocation { + /** The argv vector as dispatched, including the command path segments. */ + readonly argv: readonly string[]; + /** The resolved command path (`db migrate`); absent for help, `--version`, and usage failures. */ + readonly command?: string; + /** + * The process exit code the routed shell mapped: 0 success (or the result's + * `exitCode` under `config.exitCode: 'result'`), 1 execution failure, + * 2 usage or input-validation failure. + */ + readonly exitCode: number; + readonly provenance: CliDispatchProvenance; + readonly routeId?: string; + /** Everything the shell wrote to its diagnostic stream. */ + readonly stderr: string; + /** Everything the shell wrote to its output stream: one canonical JSON line, or help text. */ + readonly stdout: string; + /** The validated result the command returned; absent unless a command executed. */ + readonly value?: unknown; +} + +export interface CliDispatchProvenance extends Pick { + readonly proofLevel: typeof CLI_DISPATCH_PROOF_LEVEL; + /** Every command the graph compiled, so a dispatch failure can name the alternatives. */ + readonly commands: readonly string[]; +} + +const commandPath = (command: CompiledCliCommand): string => command.path.join(' '); + +const provenanceOf = (manifest: AgentBundleTestManifest): CliDispatchProvenance => Object.freeze({ + commands: Object.freeze(manifest.cliCommands.map((command) => commandPath(command))), + manifestDigest: manifest.digest, + proofLevel: CLI_DISPATCH_PROOF_LEVEL, + projectRoot: manifest.projectRoot, +}); + +const noCommands = (manifest: AgentBundleTestManifest): AgentTestError => new AgentTestError( + 'command-not-found', + 'This project compiled no routed CLI commands.', + { + details: [ + `project root: ${manifest.projectRoot}`, + ...(manifest.diagnostics.length === 0 + ? [] + : [`compiler: ${String(manifest.diagnostics.length)} diagnostic(s), first ${manifest.diagnostics[0]!.code}: ${manifest.diagnostics[0]!.message}`]), + ], + recovery: 'Add a plain command route under src/cli/ exporting inputSchema, resultSchema, and an async default function.', + }, +); + +interface Runtime { + readonly available: typeof AgentRuntime.available; + readonly runAgentRequest: typeof AgentRuntime.runAgentRequest; + readonly unavailable: typeof AgentRuntime.unavailable; +} + +let runtimePromise: Promise | undefined; + +/** + * The runtime is loaded on first dispatch, not at module scope: + * `@agent-bundle/runtime` is an optional peer, so importing this module must + * stay free for a project that only reads the manifest. + */ +const loadRuntime = async (): Promise => { + runtimePromise ??= import('@agent-bundle/runtime') + .then((runtime) => ({ + available: runtime.available, + runAgentRequest: runtime.runAgentRequest, + unavailable: runtime.unavailable, + })) + .catch((error: unknown) => { + runtimePromise = undefined; + throw new AgentTestError('render-failed', 'Unable to load the Agent runtime for a CLI dispatch.', { + cause: error, + details: [`cause: ${error instanceof Error ? error.message : String(error)}`], + recovery: 'Install @agent-bundle/runtime; routed CLI projects depend on it for the request context.', + }); + }); + return runtimePromise; +}; + +const moduleFor = async ( + manifest: AgentBundleTestManifest, + routeId: string, + provenance: CliDispatchProvenance, +): Promise => { + const loader = registeredRouteLoader(manifest, routeId); + if (loader === undefined) { + throw new AgentTestError( + 'manifest-unavailable', + `Command route ${routeId} is compiled but no test-time module loader is registered for it.`, + { + provenance: { ...provenance, kind: 'cli', routeId, source: 'manifest', targets: [] }, + recovery: 'Build the Rstest configuration with agentBundleRstest() so the generated setup registers route loaders.', + }, + ); + } + return loader(); +}; + +/** + * Dispatches one argv vector through the routed CLI shell in this process and + * returns its exit code, streams, and validated result. + * + * This is the `cli-dispatch` proof level. Nothing is spawned. + */ +export const invokeCli = async ( + argv: readonly string[], + options: InvokeCliOptions = {}, +): Promise => { + const manifest = options.manifest ?? testManifest(); + if (manifest.cliCommands.length === 0) throw noCommands(manifest); + const provenance = provenanceOf(manifest); + const runtime = await loadRuntime(); + const context = options.context ?? {}; + + let executed: CompiledCliCommand | undefined; + let value: unknown; + let out = ''; + let err = ''; + + const exitCode = await runGeneratedCliEntry({ + argv, + commands: manifest.cliCommands, + // The bridge the generated executable inlines: the module's own schemas + // stay the validation boundary, an input rejection is a usage failure, + // and the command body runs inside the typed request scope. + execute: async (command, input, execution) => { + executed = command; + const module = await moduleFor(manifest, command.routeId, provenance); + const component = (module as { default?: unknown }).default; + if (typeof component !== 'function') { + throw new AgentTestError('invalid-route-module', `Command route ${command.routeId} must default-export an async function.`, { + details: [`received: default export of type ${typeof component}`], + recovery: 'Export the command function as the module default.', + }); + } + if (module.inputSchema === undefined || module.resultSchema === undefined) { + throw new AgentTestError('invalid-route-module', `Command route ${command.routeId} must export inputSchema and resultSchema.`, { + recovery: 'Export both zod schemas from the command module; the routed CLI validates argv through them.', + }); + } + let parsed: unknown; + try { + parsed = module.inputSchema.parse(input); + } catch (error) { + throw new CliInputError(error instanceof Error ? error.message : String(error)); + } + const root = manifest.projectRoot; + const result = await runtime.runAgentRequest({ + capabilities: { + command: runtime.unavailable(), + filesystem: runtime.unavailable(), + network: runtime.unavailable(), + projectRoot: runtime.available({ root }, 'derived'), + }, + host: runtime.unavailable('unsupported-surface'), + workspace: runtime.available({ root }, 'derived'), + ...context, + invocation: { + kind: 'cli', + operationId: command.routeId, + surface: commandPath(command), + ...context.invocation, + }, + ...(context.progress === undefined ? {} : { progress: context.progress }), + signal: execution.signal, + }, async () => (component as (props: unknown) => Promise)({ + input: parsed, + signal: execution.signal, + })); + value = module.resultSchema.parse(result); + return value; + }, + name: manifest.plugin.name, + version: manifest.plugin.version, + ...(options.signal === undefined ? {} : { signal: options.signal }), + writeErr: (text) => { err += text; }, + writeOut: (text) => { out += text; }, + }); + + return Object.freeze({ + argv: Object.freeze([...argv]), + ...(executed === undefined ? {} : { command: commandPath(executed), routeId: executed.routeId }), + exitCode, + provenance, + stderr: err, + stdout: out, + ...(value === undefined ? {} : { value }), + }); +}; + +/** The parsed canonical JSON line a successful command wrote to stdout. */ +export const cliJson = (invocation: CliInvocation): unknown => { + try { + return JSON.parse(invocation.stdout) as unknown; + } catch (error) { + throw new AgentTestError('projection-failed', 'The dispatched command did not write one canonical JSON line to stdout.', { + cause: error, + details: [ + `exit code: ${String(invocation.exitCode)}`, + `stdout: ${captured(invocation.stdout)}`, + ...(invocation.stderr === '' ? [] : [`stderr: ${captured(invocation.stderr)}`]), + ], + provenance: { + ...invocation.provenance, + kind: 'cli', + routeId: invocation.routeId ?? '(no command executed)', + source: 'manifest', + targets: [], + }, + recovery: 'Assert stdout only for an invocation that executed a command; help and usage failures write text.', + }); + } +}; diff --git a/packages/agent-bundle/src/test/errors.ts b/packages/agent-bundle/src/test/errors.ts index 94fdf9d79..151d45ff7 100644 --- a/packages/agent-bundle/src/test/errors.ts +++ b/packages/agent-bundle/src/test/errors.ts @@ -1,13 +1,18 @@ +import { proofLevelLabel } from './manifest.ts'; import type { RenderedRouteProvenance } from './types.ts'; export type AgentTestErrorCode = | 'assertion-failed' + | 'command-not-found' | 'invalid-input' | 'invalid-route-module' | 'manifest-unavailable' + | 'packed-unavailable' + | 'projection-failed' | 'render-failed' | 'result-rejected' | 'route-not-found' + | 'server-not-found' | 'unsupported-route-kind'; /** How many characters of a captured value one diagnostic may print. */ @@ -27,7 +32,7 @@ export const captured = (value: unknown): string => { }; const provenanceLines = (provenance: RenderedRouteProvenance): readonly string[] => [ - ` proof level: ${provenance.proofLevel} (renders the route module through the real Agent renderer; not transport, browser, or artifact proof)`, + ` proof level: ${proofLevelLabel(provenance.proofLevel)}`, ` route: ${provenance.routeId} (${provenance.kind})`, ...(provenance.serverId === undefined ? [] : [` server: ${provenance.serverId}`]), ` route source: ${provenance.source === 'manifest' ? 'compiler manifest' : 'module passed to renderRoute'}`, @@ -37,7 +42,7 @@ const provenanceLines = (provenance: RenderedRouteProvenance): readonly string[] : [` module: ${provenance.modulePath}`]), ...(provenance.projectRoot === undefined ? [] : [` project root: ${provenance.projectRoot}`]), ...(provenance.manifestDigest === undefined ? [] : [` manifest: route-graph digest ${provenance.manifestDigest}`]), - ` targets: ${provenance.targets.length === 0 ? 'none selected' : provenance.targets.join(', ')} (route-unit rendering is target-neutral)`, + ` targets: ${provenance.targets.length === 0 ? 'none selected' : provenance.targets.join(', ')}`, ]; /** diff --git a/packages/agent-bundle/src/test/events.ts b/packages/agent-bundle/src/test/events.ts new file mode 100644 index 000000000..472ce8147 --- /dev/null +++ b/packages/agent-bundle/src/test/events.ts @@ -0,0 +1,215 @@ +/** + * Assertions over the ordered render events the runtime emits (#140). + * + * The default matcher is deliberately **sequence-tolerant**. #120 is the + * lesson: a test that pinned the exact frame array failed on a second frame + * that was entirely legitimate — a real changed generation, not a duplicate. + * The correction there was to consume the channel monotonically over its own + * ordinals and bound the count, and that is what these matchers do. + * `toContainSequence` matches an ordered subsequence, so an extra `progress` + * or `replace` frame under load never turns a green test red, while a missing + * frame, a reordering, or a regressed sequence number still fails. + * + * `toHaveTypes` is the exact-array assertion, kept available and deliberately + * not the one the documentation reaches for first. + */ +import type { AgentRenderEvent } from '@agent-bundle/runtime'; + +import { AgentTestError, captured } from './errors.ts'; +import type { RenderedRouteProvenance } from './types.ts'; + +export type AgentRenderEventType = AgentRenderEvent['type']; + +/** Anything that carries render events: the raw array or a harness result. */ +export type RenderEventSubject = + | readonly AgentRenderEvent[] + | { readonly events: readonly AgentRenderEvent[]; readonly provenance?: RenderedRouteProvenance }; + +export interface ProgressExpectation { + /** Every listed substring must appear in some progress message, in order. */ + readonly messages?: readonly string[]; + readonly atLeast?: number; + readonly atMost?: number; +} + +export interface RenderEventAssertions { + /** Asserts a total event count ceiling — the #120 bound that keeps tolerance honest. */ + readonly toBeBoundedBy: (maxEvents: number) => RenderEventAssertions; + /** Asserts exactly one terminal `complete`, and that it is last. */ + readonly toCompleteOnce: () => RenderEventAssertions; + /** Asserts the expected types appear in order, tolerating extra events between them. */ + readonly toContainSequence: (expected: readonly AgentRenderEventType[]) => RenderEventAssertions; + /** Asserts an `error` event, optionally with `code`. */ + readonly toHaveErrorCode: (code?: string) => RenderEventAssertions; + /** Asserts strictly increasing sequence numbers with no repeats or regressions. */ + readonly toHaveMonotonicSequence: () => RenderEventAssertions; + /** Asserts no `error` event was emitted. */ + readonly toHaveNoErrors: () => RenderEventAssertions; + /** Asserts progress reporting: a count window and/or ordered message substrings. */ + readonly toHaveProgress: (expectation?: ProgressExpectation) => RenderEventAssertions; + /** Asserts the exact event-type array. Prefer `toContainSequence` unless the exact frame set is the contract. */ + readonly toHaveTypes: (expected: readonly AgentRenderEventType[]) => RenderEventAssertions; +} + +const eventsOf = (subject: RenderEventSubject): readonly AgentRenderEvent[] => + Array.isArray(subject) ? subject : (subject as { readonly events: readonly AgentRenderEvent[] }).events; + +const provenanceOf = (subject: RenderEventSubject): RenderedRouteProvenance | undefined => + Array.isArray(subject) ? undefined : (subject as { readonly provenance?: RenderedRouteProvenance }).provenance; + +/** The compact rendering every event failure prints: `0:shell 1:progress 2:complete`. */ +const timeline = (events: readonly AgentRenderEvent[]): string => + events.length === 0 + ? 'no events' + : events.map((event) => `${String(event.sequence)}:${event.type}`).join(' '); + +const progressMessages = (events: readonly AgentRenderEvent[]): readonly string[] => + events.flatMap((event) => (event.type === 'progress' && event.message !== undefined ? [event.message] : [])); + +/** + * Index of the first element at or after `from` whose type is `type`, or -1. + * Scanning forward from the previous match is what makes the sequence + * ordered-but-tolerant rather than a set membership check. + */ +const findFrom = ( + events: readonly AgentRenderEvent[], + type: AgentRenderEventType, + from: number, +): number => { + for (let index = from; index < events.length; index += 1) { + if (events[index]!.type === type) return index; + } + return -1; +}; + +export const expectEvents = (subject: RenderEventSubject): RenderEventAssertions => { + const events = eventsOf(subject); + const provenance = provenanceOf(subject); + const fail = (message: string, details: readonly string[]): never => { + throw new AgentTestError('assertion-failed', message, { + details: [...details, `timeline: ${timeline(events)}`], + ...(provenance === undefined ? {} : { provenance }), + }); + }; + + const assertions: RenderEventAssertions = { + toBeBoundedBy(maxEvents) { + if (events.length > maxEvents) { + fail('The render emitted more events than the bound allows.', [ + `expected: at most ${String(maxEvents)} events`, + `received: ${String(events.length)} events`, + ]); + } + return assertions; + }, + toCompleteOnce() { + const completions = events.filter((event) => event.type === 'complete'); + if (completions.length !== 1) { + fail('The render did not emit exactly one terminal complete event.', [ + 'expected: exactly one complete event', + `received: ${String(completions.length)} complete events`, + ]); + } + if (events[events.length - 1]?.type !== 'complete') { + fail('The complete event was not the last event of the render.', [ + 'expected: complete last', + `received: ${String(events[events.length - 1]?.type)} last`, + ]); + } + return assertions; + }, + toContainSequence(expected) { + let cursor = 0; + for (const [position, type] of expected.entries()) { + const found = findFrom(events, type, cursor); + if (found === -1) { + fail('The render events do not contain the expected sequence.', [ + `expected: ${captured(expected)} in order (extra events between them are allowed)`, + `missing: ${JSON.stringify(type)} at expected position ${String(position)}`, + `matched: the first ${String(position)} expected event(s) before running out of stream`, + ]); + } + cursor = found + 1; + } + return assertions; + }, + toHaveErrorCode(code) { + const errors = events.flatMap((event) => (event.type === 'error' ? [event.error] : [])); + const matched = code === undefined ? errors : errors.filter((error) => error.code === code); + if (matched.length === 0) { + fail('The render emitted no matching error event.', [ + `expected: ${code === undefined ? 'an error event' : `an error event with code ${JSON.stringify(code)}`}`, + `received: ${errors.length === 0 ? 'no error events' : captured(errors)}`, + ]); + } + return assertions; + }, + toHaveMonotonicSequence() { + let previous = -1; + for (const event of events) { + if (event.sequence <= previous) { + fail('The render event sequence numbers are not strictly increasing.', [ + 'expected: each event to advance the sequence', + `received: ${String(event.sequence)} after ${String(previous)} on a ${event.type} event`, + ]); + } + previous = event.sequence; + } + return assertions; + }, + toHaveNoErrors() { + const errors = events.flatMap((event) => (event.type === 'error' ? [event.error] : [])); + if (errors.length > 0) { + fail('The render emitted error events.', [ + 'expected: no error events', + `received: ${captured(errors)}`, + ]); + } + return assertions; + }, + toHaveProgress(expectation = {}) { + const progress = events.filter((event) => event.type === 'progress'); + const atLeast = expectation.atLeast ?? (expectation.messages === undefined ? 1 : expectation.messages.length); + if (progress.length < atLeast) { + fail('The render reported fewer progress events than expected.', [ + `expected: at least ${String(atLeast)} progress events`, + `received: ${String(progress.length)}`, + ]); + } + if (expectation.atMost !== undefined && progress.length > expectation.atMost) { + fail('The render reported more progress events than expected.', [ + `expected: at most ${String(expectation.atMost)} progress events`, + `received: ${String(progress.length)}`, + ]); + } + if (expectation.messages !== undefined) { + const messages = progressMessages(events); + let cursor = 0; + for (const needle of expectation.messages) { + const found = messages.findIndex((message, index) => index >= cursor && message.includes(needle)); + if (found === -1) { + fail('The render progress messages do not contain the expected text in order.', [ + `expected: ${captured(expectation.messages)} in order`, + `missing: ${JSON.stringify(needle)}`, + `received: ${messages.length === 0 ? 'no progress messages' : captured(messages)}`, + ]); + } + cursor = found + 1; + } + } + return assertions; + }, + toHaveTypes(expected) { + const received = events.map((event) => event.type); + if (received.length !== expected.length || received.some((type, index) => type !== expected[index])) { + fail('The render event types differ from the expected exact sequence.', [ + `expected: ${captured(expected)} exactly`, + `received: ${captured(received)}`, + 'note: toContainSequence tolerates extra events; prefer it unless the exact frame set is the contract.', + ]); + } + return assertions; + }, + }; + return assertions; +}; diff --git a/packages/agent-bundle/src/test/index.ts b/packages/agent-bundle/src/test/index.ts index d72073568..5943a1e74 100644 --- a/packages/agent-bundle/src/test/index.ts +++ b/packages/agent-bundle/src/test/index.ts @@ -1,32 +1,80 @@ /** * `agent-bundle/test` — the consumer test harness helpers. * - * Stage 1 ships one proof level: `route-unit`. `renderRoute` executes a - * compiled route module through the real Agent renderer and returns the final - * Agent Document; the document matchers assert over the runtime's own document - * contracts. Transport, browser, and packed-artifact levels are separate - * helpers in later stages — nothing here stands in for them. + * Four proof levels ship here, and they are deliberately separate. Each + * helper names the level it supplies, stamps it into its provenance, and + * prints it in every failure: + * + * | level | helper | what it proves | + * | --- | --- | --- | + * | `route-unit` | `renderRoute`, `renderRouteEvents` | the route component and its document, through the real Agent renderer | + * | `mcp-in-memory` | `openInMemoryMcpServer`, `invokeMcpTool`, `readMcpResource`, `getMcpPrompt`, `listMcpSurface` | the real generated MCP server's protocol contract, over the SDK's in-memory transport | + * | `cli-dispatch` | `invokeCli`, `cliJson` | a compiled CLI command dispatched through the routed CLI's own shell, in this process | + * | `packed-stdio` | `openPackedMcpServer` | a built artifact's generated entry running as a real process over stdio | + * + * A pass at one level is never a receipt for another. Browser-App surfaces + * and deleted-source artifact proofs are later stages; nothing here stands in + * for them. */ -export { compileTestManifest, testManifestFromRouteGraph, ROUTE_UNIT_PROOF_LEVEL } from './manifest.ts'; +export { + CLI_DISPATCH_PROOF_LEVEL, + MCP_IN_MEMORY_PROOF_LEVEL, + PACKED_STDIO_PROOF_LEVEL, + ROUTE_UNIT_PROOF_LEVEL, + compileTestManifest, + proofLevelLabel, + testManifestFromRouteGraph, +} from './manifest.ts'; export type { AgentBundleTestManifest, AgentTestProofLevel, CompileTestManifestOptions, + TestManifestPluginIdentity, TestableRouteDescriptor, } from './manifest.ts'; export { AGENT_TEST_REGISTRY_VERSION, registerTestRoutes, testManifest } from './registry.ts'; export type { AgentTestRouteRegistry } from './registry.ts'; export { AgentTestError } from './errors.ts'; export type { AgentTestErrorCode } from './errors.ts'; -export { renderRoute } from './render.ts'; +export { renderRoute, renderRouteEvents } from './render.ts'; export type { RenderRouteContext, RenderRouteOptions, RenderRouteTarget, RenderedRoute, + RenderedRouteEvents, } from './render.ts'; export { expectDocument } from './matchers.ts'; export type { AgentDocumentNodeKind, DocumentAssertions, DocumentSubject } from './matchers.ts'; +export { expectEvents } from './events.ts'; +export type { + AgentRenderEventType, + ProgressExpectation, + RenderEventAssertions, + RenderEventSubject, +} from './events.ts'; +export { + getMcpPrompt, + invokeMcpTool, + listMcpSurface, + openInMemoryMcpServer, + readMcpResource, +} from './mcp.ts'; +export type { + InMemoryMcpSession, + InMemoryMcpSessionOptions, + McpContentBlock, + McpInvocationOptions, + McpProjectionProvenance, + McpPromptResult, + McpResourceRead, + McpSurfaceListing, + McpToolInvocation, +} from './mcp.ts'; +export { cliJson, invokeCli } from './cli.ts'; +export type { CliDispatchProvenance, CliInvocation, InvokeCliOptions } from './cli.ts'; +export { openPackedMcpServer } from './packed.ts'; +export type { PackedMcpProvenance, PackedMcpSession, PackedMcpSessionOptions } from './packed.ts'; export type { AgentRouteModule, AgentRouteModuleLoader, diff --git a/packages/agent-bundle/src/test/manifest.ts b/packages/agent-bundle/src/test/manifest.ts index d65956542..0f557c63a 100644 --- a/packages/agent-bundle/src/test/manifest.ts +++ b/packages/agent-bundle/src/test/manifest.ts @@ -2,19 +2,63 @@ import { resolve } from 'node:path'; import type { Diagnostic } from '../core/diagnostics.ts'; import { deepFreeze } from '../core/freeze.ts'; -import type { CompiledAgentRoute, CompiledRouteGraph, CompiledRouteKind } from '../routes/types.ts'; +import type { + CompiledAgentRoute, + CompiledCliCommand, + CompiledRouteGraph, + CompiledRouteKind, +} from '../routes/types.ts'; /** - * The proof level one harness helper supplies. Stage 1 of the consumer test - * harness ships exactly one level: `route-unit` renders a route module - * through the real Agent renderer and never opens a transport, compiles a - * browser surface, or builds a host artifact. Later stages add their own - * levels beside this one; a route-unit pass is never an artifact, transport, - * or host receipt. + * The proof level one harness helper supplies. The levels are deliberately + * separate and separately named: a pass at one is never a receipt for + * another, and every helper stamps the level it actually carried into its + * provenance and its failure text. + * + * - `route-unit` renders a route module through the real Agent renderer. No + * transport, no browser surface, no host artifact. + * - `mcp-in-memory` runs the real generated MCP server against a real MCP + * client over the SDK's in-memory transport pair. It proves the protocol + * contract — registration, schemas, content projection — and proves + * **nothing** about a process, stdout framing, or a packed artifact. + * - `cli-dispatch` runs an argv vector through the routed CLI's own shell over + * the compiled command graph, in this process. It proves command + * resolution, argv projection, and exit codes, not a spawned binary. + * - `packed-stdio` installs the packed release tarball into a clean consumer, + * spawns the generated stdio entry as a real process, and drives it with a + * real MCP client. This is the only level here that is process evidence. + * + * Browser and deleted-source artifact levels are later stages; nothing here + * stands in for them. */ -export type AgentTestProofLevel = 'route-unit'; +export type AgentTestProofLevel = 'route-unit' | 'mcp-in-memory' | 'cli-dispatch' | 'packed-stdio'; -export const ROUTE_UNIT_PROOF_LEVEL: AgentTestProofLevel = 'route-unit'; +export const ROUTE_UNIT_PROOF_LEVEL = 'route-unit' as const; +export const MCP_IN_MEMORY_PROOF_LEVEL = 'mcp-in-memory' as const; +export const CLI_DISPATCH_PROOF_LEVEL = 'cli-dispatch' as const; +export const PACKED_STDIO_PROOF_LEVEL = 'packed-stdio' as const; + +/** + * One line per level, printed in every harness failure. A red test has to + * say what the passing case would have proven, or the level discipline is + * only a naming convention. + */ +export const proofLevelLabel = (level: AgentTestProofLevel): string => { + switch (level) { + case 'route-unit': + return 'route-unit (real Agent renderer; no transport, browser, or artifact)'; + case 'mcp-in-memory': + return 'mcp-in-memory (real generated MCP server + real client over the SDK in-memory transport; NOT process or packed-artifact evidence)'; + case 'cli-dispatch': + return 'cli-dispatch (argv dispatched through the routed CLI shell in-process; NOT a spawned binary)'; + case 'packed-stdio': + return 'packed-stdio (packed tarball installed into a clean consumer, generated stdio entry spawned as a real process)'; + default: { + const exhaustive: never = level; + throw new TypeError(`Unknown proof level ${String(exhaustive)}.`); + } + } +}; /** One compiled route the harness can address, with the identity its diagnostics report. */ export interface TestableRouteDescriptor { @@ -30,21 +74,36 @@ export interface TestableRouteDescriptor { readonly source: string; } +/** The project identity a generated MCP server advertises on the wire. */ +export interface TestManifestPluginIdentity { + readonly name: string; + readonly version: string; +} + /** - * What the compiler tells the test harness about one project. Stage 1 carries - * the route inventory, the selected targets, and the compiler's own - * diagnostics; projection, browser-App, and artifact descriptors arrive with - * the stages that can honestly prove them. + * What the compiler tells the test harness about one project: the route + * inventory, the project identity its generated servers advertise, the + * selected targets, and the compiler's own diagnostics. Browser-App and + * artifact descriptors arrive with the stages that can honestly prove them. */ export interface AgentBundleTestManifest { + /** + * The collision-checked routed-CLI command graph (#102 stage 2) from the + * same pass, so the CLI dispatch level drives the product's own dispatcher + * over the product's own commands instead of resolving argv itself. Empty + * when the project compiles no `src/cli/**` commands. + */ + readonly cliCommands: readonly CompiledCliCommand[]; /** The absolute config path the compiler pass evaluated. */ readonly configPath?: string; /** Diagnostics from the same pass, so a harness failure can name a compiler cause. */ readonly diagnostics: readonly Diagnostic[]; /** The route graph digest: project-relative route identity, equal on every machine. */ readonly digest: string; + /** Plugin name and version, as the generated MCP server reports them in `initialize`. */ + readonly plugin: TestManifestPluginIdentity; readonly projectRoot: string; - /** The one proof level stage 1 supplies. */ + /** The level the manifest and its registered loaders alone supply; every other level stamps its own. */ readonly proofLevel: AgentTestProofLevel; readonly routes: Readonly>; /** Host targets the project selected. Route-unit rendering is target-neutral; these name the projection surfaces a later proof level owns. */ @@ -81,15 +140,18 @@ export const testManifestFromRouteGraph = (input: { readonly configPath?: string; readonly diagnostics?: readonly Diagnostic[]; readonly graph: CompiledRouteGraph; + readonly plugin?: TestManifestPluginIdentity; readonly projectRoot: string; readonly targets?: readonly string[]; }): AgentBundleTestManifest => { const routes: Record = {}; for (const route of graphRoutes(input.graph)) routes[route.id] = descriptorOf(route); return deepFreeze({ + cliCommands: [...(input.graph.cli?.commands ?? [])], ...(input.configPath === undefined ? {} : { configPath: input.configPath }), diagnostics: [...(input.diagnostics ?? input.graph.diagnostics)], digest: input.graph.digest, + plugin: input.plugin ?? { name: 'unknown', version: '0.0.0' }, projectRoot: input.projectRoot, proofLevel: ROUTE_UNIT_PROOF_LEVEL, routes, @@ -123,6 +185,9 @@ export const compileTestManifest = async ( configPath: prepared.configPath, diagnostics: prepared.diagnostics, graph: prepared.routeGraph ?? emptyCompiledRouteGraph, + ...(prepared.model === undefined + ? {} + : { plugin: { name: prepared.model.metadata.name, version: prepared.model.metadata.version } }), projectRoot: prepared.root, targets: prepared.model?.targets.map((target) => target.name) ?? [], }); diff --git a/packages/agent-bundle/src/test/matchers.ts b/packages/agent-bundle/src/test/matchers.ts index 022b2eb05..e143942d4 100644 --- a/packages/agent-bundle/src/test/matchers.ts +++ b/packages/agent-bundle/src/test/matchers.ts @@ -19,7 +19,10 @@ const provenanceOf = (subject: DocumentSubject): RenderedRouteProvenance | undef const nodes = (node: AgentDocumentNode): readonly AgentDocumentNode[] => node.kind === 'result' ? [node, ...node.children.flatMap((child) => nodes(child))] : [node]; -const textOf = (document: AgentDocument, kind: 'markdown' | 'text'): readonly string[] => +const textOf = ( + document: AgentDocument, + kind: 'context' | 'markdown' | 'text', +): readonly string[] => nodes(document.root).flatMap((node) => (node.kind === kind ? [node.text] : [])); /** @@ -29,6 +32,8 @@ const textOf = (document: AgentDocument, kind: 'markdown' | 'text'): readonly st * that provenance available. */ export interface DocumentAssertions { + /** Asserts a context node contains `text` — the additional context an event route returns to its host. */ + readonly toContainContext: (text: string) => DocumentAssertions; /** Asserts a Markdown node contains `text`. */ readonly toContainMarkdown: (text: string) => DocumentAssertions; /** Asserts a text node contains `text`. */ @@ -56,6 +61,16 @@ export const expectDocument = (subject: DocumentSubject): DocumentAssertions => }); }; const assertions: DocumentAssertions = { + toContainContext(text) { + const found = textOf(document, 'context'); + if (!found.some((value) => value.includes(text))) { + fail('The Agent Document contains no context node with the expected text.', [ + `expected: context containing ${JSON.stringify(text)}`, + `received: ${found.length === 0 ? 'no context nodes' : captured(found)}`, + ]); + } + return assertions; + }, toContainMarkdown(text) { const found = textOf(document, 'markdown'); if (!found.some((value) => value.includes(text))) { diff --git a/packages/agent-bundle/src/test/mcp.ts b/packages/agent-bundle/src/test/mcp.ts new file mode 100644 index 000000000..46b8e57b5 --- /dev/null +++ b/packages/agent-bundle/src/test/mcp.ts @@ -0,0 +1,442 @@ +/** + * The in-memory MCP projection proof level. + * + * A test at this level talks to the **real generated MCP server** — the same + * `agent-bundle/mcp-server-runtime` registration and Agent Document + * projection a built artifact runs — through a **real MCP SDK client** over + * the SDK's in-memory linked transport pair. What differs from a shipped + * server is the transport and the render host: there is no process, no stdio + * framing, and routes render in this process instead of in a spawned Flight + * worker. + * + * That makes this fast protocol-contract proof and nothing else. It is + * labeled `mcp-in-memory` in every result, every provenance record, and every + * failure message, and `invokePackedMcpTool` (`agent-bundle/test`, the + * `packed-stdio` level) is the helper that carries process evidence. + */ +import type { Client } from '@modelcontextprotocol/client'; + +import { AgentTestError, captured } from './errors.ts'; +import { MCP_IN_MEMORY_PROOF_LEVEL, type AgentBundleTestManifest } from './manifest.ts'; +import { registeredRouteLoader, testManifest } from './registry.ts'; +import type { RenderRouteContext } from './render.ts'; +import type { RenderedRouteProvenance, TestableRouteDescriptor } from './types.ts'; + +/** Where an in-memory projection result came from and what it proves. */ +export interface McpProjectionProvenance { + readonly manifestDigest: string; + readonly projectRoot: string; + readonly proofLevel: typeof MCP_IN_MEMORY_PROOF_LEVEL; + /** The MCP server name as the protocol sees it, e.g. `curator`. */ + readonly serverName: string; + /** Compiled route ids this server registered. */ + readonly routeIds: readonly string[]; +} + +export interface McpContentBlock { + readonly data?: string; + readonly mimeType?: string; + readonly name?: string; + readonly text?: string; + readonly type: string; + readonly uri?: string; +} + +export interface McpToolInvocation { + readonly content: readonly McpContentBlock[]; + readonly isError: boolean; + readonly provenance: McpProjectionProvenance; + readonly structuredContent?: unknown; +} + +export interface InMemoryMcpSessionOptions { + /** Request-scoped overrides applied to every route render in this session. */ + readonly context?: RenderRouteContext; + readonly manifest?: AgentBundleTestManifest; + /** MCP server name. Optional when the project compiled exactly one server. */ + readonly server?: string; +} + +export interface InMemoryMcpSession extends AsyncDisposable { + /** The real MCP SDK client, for protocol calls this module does not wrap. */ + readonly client: Client; + readonly close: () => Promise; + readonly provenance: McpProjectionProvenance; +} + +export type McpInvocationOptions = InMemoryMcpSessionOptions & { + readonly input?: unknown; +}; + +const serverRoutes = ( + manifest: AgentBundleTestManifest, + serverName: string, +): readonly TestableRouteDescriptor[] => Object.values(manifest.routes) + .filter((route) => route.serverId === `mcp:${serverName}` && route.kind !== 'app') + .sort((left, right) => left.id.localeCompare(right.id)); + +const compiledServerNames = (manifest: AgentBundleTestManifest): readonly string[] => [ + ...new Set(Object.values(manifest.routes).flatMap((route) => + (route.serverId === undefined ? [] : [route.serverId.replace(/^mcp:/u, '')]))), +].sort((left, right) => left.localeCompare(right)); + +const resolveServerName = ( + manifest: AgentBundleTestManifest, + requested: string | undefined, +): string => { + const names = compiledServerNames(manifest); + if (requested !== undefined) { + if (!names.includes(requested)) { + throw new AgentTestError('server-not-found', `No compiled MCP server is named ${JSON.stringify(requested)}.`, { + details: [ + `project root: ${manifest.projectRoot}`, + `compiled: ${names.length === 0 ? 'this project compiled no MCP servers' : names.join(', ')}`, + ], + recovery: 'Name one of the compiled servers, or add route modules under src/mcp//.', + }); + } + return requested; + } + if (names.length === 1) return names[0]!; + throw new AgentTestError( + 'server-not-found', + names.length === 0 + ? 'This project compiled no MCP servers.' + : 'This project compiled more than one MCP server, so the helper cannot pick one.', + { + details: [ + `project root: ${manifest.projectRoot}`, + `compiled: ${names.length === 0 ? 'none' : names.join(', ')}`, + ], + recovery: 'Pass { server: "" }.', + }, + ); +}; + +const routeProvenance = ( + descriptor: TestableRouteDescriptor, + manifest: AgentBundleTestManifest, +): RenderedRouteProvenance => Object.freeze({ + kind: descriptor.kind as 'prompt' | 'resource' | 'tool', + manifestDigest: manifest.digest, + modulePath: descriptor.source, + projectRoot: manifest.projectRoot, + proofLevel: MCP_IN_MEMORY_PROOF_LEVEL, + relativePath: descriptor.relativePath, + routeId: descriptor.id, + ...(descriptor.serverId === undefined ? {} : { serverId: descriptor.serverId }), + source: 'manifest' as const, + targets: manifest.targets, +}); + +interface ServerRuntime { + readonly createGeneratedRouteMcpServer: typeof import('../mcp-server-runtime.ts').createGeneratedRouteMcpServer; +} + +interface Renderer { + readonly createElement: typeof import('react').createElement; + readonly createWarmFlightHost: typeof import('@agent-bundle/runtime').createWarmFlightHost; + readonly renderAgentFlight: typeof import('@agent-bundle/runtime/flight/server').renderAgentFlight; + readonly runAgentRequest: typeof import('@agent-bundle/runtime').runAgentRequest; +} + +interface Sdk { + readonly Client: typeof import('@modelcontextprotocol/client').Client; + readonly InMemoryTransport: typeof import('@modelcontextprotocol/client').InMemoryTransport; +} + +let dependenciesPromise: Promise | undefined; + +/** + * Loads the server runtime, the renderer, and the MCP SDK on first use. These + * are deliberately not module-scope imports: `@agent-bundle/runtime` and + * `react` are optional peers of this package, and the Flight server entry + * throws on import unless the process enabled the `react-server` condition — + * so importing `agent-bundle/test` for the manifest alone must not require + * either, and a missing condition must fail with the wiring step. + */ +const loadDependencies = async (): Promise => { + dependenciesPromise ??= (async () => { + const [serverRuntime, runtime, flight, react, client] = await Promise.all([ + import('../mcp-server-runtime.ts'), + import('@agent-bundle/runtime'), + import('@agent-bundle/runtime/flight/server'), + import('react'), + import('@modelcontextprotocol/client'), + ]); + return { + Client: client.Client, + InMemoryTransport: client.InMemoryTransport, + createElement: react.createElement, + createGeneratedRouteMcpServer: serverRuntime.createGeneratedRouteMcpServer, + createWarmFlightHost: runtime.createWarmFlightHost, + renderAgentFlight: flight.renderAgentFlight, + runAgentRequest: runtime.runAgentRequest, + }; + })().catch((error: unknown) => { + dependenciesPromise = undefined; + throw new AgentTestError( + 'projection-failed', + 'Unable to load the generated MCP server runtime for an in-memory projection.', + { + cause: error, + details: [`cause: ${error instanceof Error ? error.message : String(error)}`], + recovery: 'Install react and @agent-bundle/runtime, and run the level with the react-server condition — agentBundleRstest() from agent-bundle/rstest configures both.', + }, + ); + }); + return dependenciesPromise; +}; + +const drain = async (stream: ReadableStream): Promise => { + const chunks: Uint8Array[] = []; + const reader = stream.getReader(); + for (;;) { + const next = await reader.read(); + if (next.done) return chunks; + chunks.push(next.value); + } +}; + +const streamOf = (chunks: readonly Uint8Array[]): ReadableStream => + new ReadableStream({ + start(controller) { + for (const chunk of chunks) controller.enqueue(chunk); + controller.close(); + }, + }); + +/** + * Opens a real MCP client against the real generated server for one compiled + * MCP surface, over the SDK's in-memory transport pair. + * + * Compiled MCP Apps are deliberately not registered: their HTML is a browser + * build output, and claiming an App resource without it would be exactly the + * fake this level must not become. App surfaces belong to the browser level. + */ +export const openInMemoryMcpServer = async ( + options: InMemoryMcpSessionOptions = {}, +): Promise => { + const manifest = options.manifest ?? testManifest(); + const serverName = resolveServerName(manifest, options.server); + const descriptors = serverRoutes(manifest, serverName); + const dependencies = await loadDependencies(); + const context = options.context ?? {}; + + const routes: Record>; + id: string; + kind: 'prompt' | 'resource' | 'tool'; + module: { default: (props: never) => unknown; inputSchema?: unknown; resultSchema: { parse: (value: unknown) => unknown } }; + name: string; + }> = {}; + for (const descriptor of descriptors) { + if (descriptor.kind !== 'tool' && descriptor.kind !== 'resource' && descriptor.kind !== 'prompt') continue; + const loader = registeredRouteLoader(manifest, descriptor.id); + if (loader === undefined) { + throw new AgentTestError( + 'manifest-unavailable', + `Route ${descriptor.id} is compiled but no test-time module loader is registered for it.`, + { + provenance: routeProvenance(descriptor, manifest), + recovery: 'Build the Rstest configuration with agentBundleRstest() so the generated setup registers route loaders.', + }, + ); + } + const module = await loader() as { + default: (props: never) => unknown; + inputSchema?: unknown; + resultSchema?: { parse: (value: unknown) => unknown }; + }; + if (module.resultSchema === undefined) { + throw new AgentTestError( + 'invalid-route-module', + `Route ${descriptor.id} exports no resultSchema, which the generated MCP server requires to validate its result.`, + { + provenance: routeProvenance(descriptor, manifest), + recovery: 'Export a resultSchema from the route module; the generated server validates every document value against it.', + }, + ); + } + routes[descriptor.id] = { + config: descriptor.config, + id: descriptor.id, + kind: descriptor.kind, + module: { ...module, resultSchema: module.resultSchema }, + name: descriptor.id.slice(descriptor.id.lastIndexOf('/') + 1), + }; + } + + // The in-process stand-in for the artifact's Flight worker: same request + // scope, same Flight encode, same bytes handed back to the dispatcher, and + // the same warm host wrapper the artifact uses — it simply renders here + // instead of in a spawned thread. + const artifactEpoch = `${manifest.plugin.name}@${manifest.plugin.version}`; + const host = dependencies.createWarmFlightHost({ + artifactEpoch, + host: { + execute: async (request): Promise> => { + // The generated server dispatches every MCP route kind as a tool + // invocation, so the operation id is the compiled route id. + const props = request.invocation.props as { readonly input?: unknown; readonly operationId?: string }; + const route = routes[String(props.operationId)]; + if (route === undefined) { + throw new AgentTestError('route-not-found', `The in-memory server dispatched an unregistered route ${captured(props.operationId)}.`, { + details: [`registered: ${Object.keys(routes).sort().join(', ')}`], + }); + } + return streamOf(await dependencies.runAgentRequest({ + ...context, + invocation: { + kind: 'tool' as const, + operationId: route.id, + surface: route.name, + ...context.invocation, + }, + ...(request.progress === undefined ? {} : { progress: request.progress }), + signal: request.signal, + }, async () => drain(dependencies.renderAgentFlight( + dependencies.createElement(route.module.default as never, { + input: props.input, + signal: request.signal, + } as never), + { signal: request.signal }, + )))); + }, + }, + }); + + const server = await dependencies.createGeneratedRouteMcpServer({ + artifactEpoch, + host, + plugin: manifest.plugin, + routes: routes as never, + }); + const client = new dependencies.Client({ name: 'agent-bundle-in-memory-projection', version: '1.0.0' }); + const [clientTransport, serverTransport] = dependencies.InMemoryTransport.createLinkedPair(); + await Promise.all([server.connect(serverTransport as never), client.connect(clientTransport)]); + + const provenance: McpProjectionProvenance = Object.freeze({ + manifestDigest: manifest.digest, + projectRoot: manifest.projectRoot, + proofLevel: MCP_IN_MEMORY_PROOF_LEVEL, + routeIds: Object.freeze(Object.keys(routes).sort()), + serverName, + }); + let closed = false; + const close = async (): Promise => { + if (closed) return; + closed = true; + await client.close(); + await server.close(); + }; + return Object.freeze({ + client, + close, + provenance, + [Symbol.asyncDispose]: close, + }); +}; + +/** + * Runs `body` against a single-use in-memory session. Every wrapper below + * goes through this so a helper never leaks a connected transport pair. + */ +const withSession = async ( + options: InMemoryMcpSessionOptions, + body: (session: InMemoryMcpSession) => Promise, +): Promise => { + const session = await openInMemoryMcpServer(options); + try { + return await body(session); + } finally { + await session.close(); + } +}; + +const asContentBlocks = (value: unknown): readonly McpContentBlock[] => + Array.isArray(value) ? (value as readonly McpContentBlock[]) : []; + +/** + * Calls one compiled tool through the real protocol and returns the projected + * result. `mcp-in-memory` level: protocol contract proof, not process proof. + */ +export const invokeMcpTool = async ( + tool: string, + options: McpInvocationOptions = {}, +): Promise => withSession(options, async (session) => { + const result = await session.client.callTool({ + arguments: (options.input ?? {}) as Record, + name: tool, + }) as { content?: unknown; isError?: boolean; structuredContent?: unknown }; + return Object.freeze({ + content: asContentBlocks(result.content), + isError: result.isError === true, + provenance: session.provenance, + ...(result.structuredContent === undefined ? {} : { structuredContent: result.structuredContent }), + }); +}); + +export interface McpResourceRead { + readonly contents: readonly Readonly>[]; + readonly provenance: McpProjectionProvenance; +} + +/** Reads one compiled resource route by URI through the real protocol. */ +export const readMcpResource = async ( + uri: string, + options: InMemoryMcpSessionOptions = {}, +): Promise => withSession(options, async (session) => { + const result = await session.client.readResource({ uri }) as { contents?: unknown }; + return Object.freeze({ + contents: Array.isArray(result.contents) ? (result.contents as readonly Readonly>[]) : [], + provenance: session.provenance, + }); +}); + +export interface McpPromptResult { + readonly messages: readonly Readonly>[]; + readonly provenance: McpProjectionProvenance; +} + +/** Gets one compiled prompt route through the real protocol. */ +export const getMcpPrompt = async ( + prompt: string, + options: McpInvocationOptions = {}, +): Promise => withSession(options, async (session) => { + const result = await session.client.getPrompt({ + arguments: (options.input ?? {}) as Record, + name: prompt, + }) as { messages?: unknown }; + return Object.freeze({ + messages: Array.isArray(result.messages) ? (result.messages as readonly Readonly>[]) : [], + provenance: session.provenance, + }); +}); + +export interface McpSurfaceListing { + readonly prompts: readonly string[]; + readonly provenance: McpProjectionProvenance; + readonly resources: readonly string[]; + readonly tools: readonly string[]; +} + +/** + * Lists what the generated server actually registered — the cheapest proof + * that a compiled route reached the protocol at all. + */ +export const listMcpSurface = async ( + options: InMemoryMcpSessionOptions = {}, +): Promise => withSession(options, async (session) => { + const [tools, resources, prompts] = await Promise.all([ + session.client.listTools(), + session.client.listResources(), + session.client.listPrompts(), + ]); + return Object.freeze({ + prompts: Object.freeze(prompts.prompts.map((entry) => entry.name).sort()), + provenance: session.provenance, + resources: Object.freeze(resources.resources.map((entry) => entry.uri).sort()), + tools: Object.freeze(tools.tools.map((entry) => entry.name).sort()), + }); +}); diff --git a/packages/agent-bundle/src/test/packed.ts b/packages/agent-bundle/src/test/packed.ts new file mode 100644 index 000000000..563be55ed --- /dev/null +++ b/packages/agent-bundle/src/test/packed.ts @@ -0,0 +1,131 @@ +/** + * The packed stdio proof level. + * + * This is the only level in `agent-bundle/test` that is process evidence: a + * real MCP client speaks JSON-RPC over stdio to the generated entry running + * as a separate operating-system process, out of a built artifact. Framing, + * the stdout protocol guard, the entry lifecycle, and the warm Flight worker + * are all in the picture here and in none of the other levels. + * + * Packing and installing are deliberately **not** this helper's job. A packed + * journey is the most expensive thing a test suite can do, so the harness + * takes an already-built entry path and spends its cost on assertions + * instead: one artifact, one spawned server, every route asserted inside that + * single session (#103's cost rule). + */ +import type { Client } from '@modelcontextprotocol/client'; + +import { AgentTestError } from './errors.ts'; +import { PACKED_STDIO_PROOF_LEVEL } from './manifest.ts'; + +/** Where a packed session's evidence came from. */ +export interface PackedMcpProvenance { + /** Absolute path of the generated stdio entry that was spawned. */ + readonly entry: string; + readonly pid: number | undefined; + readonly proofLevel: typeof PACKED_STDIO_PROOF_LEVEL; +} + +export interface PackedMcpSessionOptions { + /** Extra argv appended after the entry path. */ + readonly args?: readonly string[]; + /** Working directory for the spawned process; defaults to the entry's directory. */ + readonly cwd?: string; + /** Absolute path of the generated stdio entry (`//mcp/.mjs`). */ + readonly entry: string; + /** Environment for the spawned process; defaults to the current one. */ + readonly env?: Readonly>; + /** Node binary to spawn; defaults to the running one. */ + readonly execPath?: string; + /** Client identity sent in `initialize`. */ + readonly name?: string; +} + +export interface PackedMcpSession extends AsyncDisposable { + readonly client: Client; + readonly close: () => Promise; + readonly provenance: PackedMcpProvenance; + /** Everything the server wrote to stderr, bounded — stdout is the protocol channel. */ + readonly stderr: () => string; +} + +/** Bytes of server stderr one session retains for diagnostics. */ +const maxStderrCharacters = 16_000; + +interface Sdk { + readonly Client: typeof import('@modelcontextprotocol/client').Client; + readonly StdioClientTransport: typeof import('@modelcontextprotocol/client/stdio').StdioClientTransport; +} + +let sdkPromise: Promise | undefined; + +const loadSdk = async (): Promise => { + sdkPromise ??= (async () => { + const [client, stdio] = await Promise.all([ + import('@modelcontextprotocol/client'), + import('@modelcontextprotocol/client/stdio'), + ]); + return { Client: client.Client, StdioClientTransport: stdio.StdioClientTransport }; + })(); + return sdkPromise; +}; + +/** + * Spawns a built stdio MCP entry and connects a real MCP client to it. + * + * The entry must already exist: build the artifact once per suite and reuse + * the session for every assertion. A connect failure reports the captured + * stderr, because a generated entry that dies on startup says why there and + * nowhere else. + */ +export const openPackedMcpServer = async ( + options: PackedMcpSessionOptions, +): Promise => { + const sdk = await loadSdk(); + const client = new sdk.Client({ name: options.name ?? 'agent-bundle-packed-proof', version: '1.0.0' }); + const transport = new sdk.StdioClientTransport({ + args: [options.entry, ...(options.args ?? [])], + command: options.execPath ?? process.execPath, + ...(options.cwd === undefined ? {} : { cwd: options.cwd }), + ...(options.env === undefined ? {} : { env: { ...options.env } }), + stderr: 'pipe', + }); + let captured = ''; + transport.stderr?.on('data', (chunk: unknown) => { + if (captured.length >= maxStderrCharacters) return; + captured = `${captured}${String(chunk)}`.slice(0, maxStderrCharacters); + }); + const stderr = (): string => captured; + try { + await client.connect(transport); + } catch (error) { + throw new AgentTestError('packed-unavailable', 'The packed stdio MCP server did not start.', { + cause: error, + details: [ + `proof level: ${PACKED_STDIO_PROOF_LEVEL}`, + `entry: ${options.entry}`, + `cause: ${error instanceof Error ? error.message : String(error)}`, + `server stderr:${captured === '' ? ' (empty)' : `\n${captured}`}`, + ], + recovery: 'Build the artifact before opening the session, and run the entry manually with `node ` to see its startup output.', + }); + } + const provenance: PackedMcpProvenance = Object.freeze({ + entry: options.entry, + pid: transport.pid ?? undefined, + proofLevel: PACKED_STDIO_PROOF_LEVEL, + }); + let closed = false; + const close = async (): Promise => { + if (closed) return; + closed = true; + await client.close(); + }; + return Object.freeze({ + client, + close, + provenance, + stderr, + [Symbol.asyncDispose]: close, + }); +}; diff --git a/packages/agent-bundle/src/test/render.ts b/packages/agent-bundle/src/test/render.ts index e5e6afe0c..cdd1a92d8 100644 --- a/packages/agent-bundle/src/test/render.ts +++ b/packages/agent-bundle/src/test/render.ts @@ -5,6 +5,7 @@ import type { AgentInvocationInput, AgentProgressReporter, AgentProgressUpdate, + AgentRenderEvent, AgentRenderInvocation, AgentRenderLimits, AgentRequestInit, @@ -167,7 +168,16 @@ const invocationFor = ( case 'tool': return { kind: 'tool', props: { input: (options.input ?? {}) as never, operationId: routeId } }; case 'event-route': - return { kind: 'event', props: { event: routeId, payload: (options.input ?? {}) as never } }; + // The generated server names the canonical event, not the route id, and + // carries the host envelope as `payload`; the harness matches both so a + // route sees the props the artifact would hand it. + return { + kind: 'event', + props: { + event: routeId.startsWith('event:') ? routeId.slice('event:'.length) : routeId, + payload: (options.input ?? {}) as never, + }, + }; case 'cli': return { kind: 'cli', props: { args: cliArguments(options, provenance), command: routeId } }; case 'script': @@ -217,8 +227,15 @@ const componentProps = ( case 'resource': case 'tool': return { input: (invocation.props as { readonly input?: unknown }).input, signal }; + case 'event-route': { + // The public event-route contract is `{ canonical, native, signal }`, + // and the generated Flight worker unwraps the payload into exactly that. + const payload = (invocation.props as { + readonly payload?: { readonly canonical?: unknown; readonly native?: unknown }; + }).payload ?? {}; + return { canonical: payload.canonical, native: payload.native, signal }; + } case 'cli': - case 'event-route': case 'script': return { ...invocation.props, signal }; default: { @@ -378,34 +395,43 @@ const streamOf = (chunks: readonly Uint8Array[]): ReadableStream => }, }); +interface PreparedRender { + readonly collected: readonly AgentProgressUpdate[]; + readonly dispatcher: AgentRuntime.AgentRenderDispatcher; + readonly invocation: AgentRenderInvocation; + readonly resolved: ResolvedTarget; + readonly signal: AbortSignal; +} + /** - * Renders one route through the real Agent renderer and returns its final - * Agent Document. The route component executes inside a real request scope, - * its output is encoded as React Flight, and the runtime's own final-only - * dispatcher decodes it — the harness owns no second rendering path. - * - * This is the route-unit proof level: no transport is opened, no browser - * surface is compiled, and no host artifact is built. + * Resolves the route, loads the real renderer, and wires one dispatcher over + * a request-scoped Flight render of the route component. Both the final-only + * and the event-stream entry points run through this, so neither owns a + * second rendering path. */ -export const renderRoute = async ( +const prepareRender = async ( target: RenderRouteTarget, - options: RenderRouteOptions = {}, -): Promise => { + options: RenderRouteOptions, +): Promise => { const resolved = await resolveTarget(target, options); const renderer = await loadRenderer(); const invocation = invocationFor(resolved.kind, resolved.provenance.routeId, options, resolved.provenance); const collected: AgentProgressUpdate[] = []; const context = options.context ?? {}; - // The result contract exposes the route's request-scoped progress, so the - // harness always records it and then delegates to a caller's reporter. - const reporter = context.progress; - const progress: AgentProgressReporter = { + /** + * Every render collects progress for {@link RenderedRoute.progress}, then + * forwards it to the caller's reporter and to the dispatcher's, which is + * what turns an update into a `progress` render event. The collector cannot + * be an either/or fallback: the event-stream entry point always supplies a + * dispatcher reporter, and that would leave the array empty. + */ + const progressFor = (reporter: AgentProgressReporter | undefined): AgentProgressReporter => ({ report: async (update) => { collected.push(update); + await context.progress?.report(update); await reporter?.report(update); }, - }; - const signal = options.signal ?? new AbortController().signal; + }); const dispatcher = renderer.createAgentRenderDispatcher({ execute: async (request) => streamOf(await renderer.runAgentRequest({ ...context, @@ -414,7 +440,7 @@ export const renderRoute = async ( ...context.invocation, kind: request.invocation.kind, }, - progress, + progress: progressFor(request.progress), signal: request.signal, }, async () => drain(renderer.renderAgentFlight( renderer.createElement( @@ -424,27 +450,105 @@ export const renderRoute = async ( { signal: request.signal }, )))), }, options.limits === undefined ? {} : { limits: options.limits }); + return { + collected, + dispatcher, + invocation, + resolved, + signal: options.signal ?? new AbortController().signal, + }; +}; +const renderFailure = ( + error: unknown, + invocation: AgentRenderInvocation, + provenance: RenderedRouteProvenance, +): AgentTestError => new AgentTestError('render-failed', 'The route render failed.', { + cause: error, + details: [ + `cause: ${error instanceof Error ? `${error.name}: ${error.message}` : String(error)}`, + `invocation: ${invocation.kind}`, + ], + provenance, +}); + +/** + * Renders one route through the real Agent renderer and returns its final + * Agent Document. The route component executes inside a real request scope, + * its output is encoded as React Flight, and the runtime's own final-only + * dispatcher decodes it — the harness owns no second rendering path. + * + * This is the route-unit proof level: no transport is opened, no browser + * surface is compiled, and no host artifact is built. + */ +export const renderRoute = async ( + target: RenderRouteTarget, + options: RenderRouteOptions = {}, +): Promise => { + const { collected, dispatcher, invocation, resolved, signal } = await prepareRender(target, options); let document: AgentDocument; try { document = await dispatcher.dispatch({ invocation, signal }); } catch (error) { - throw new AgentTestError('render-failed', 'The route render failed.', { - cause: error, + throw renderFailure(error, invocation, resolved.provenance); + } + return Object.freeze({ + document, + invocation, + progress: Object.freeze([...collected]), + provenance: resolved.provenance, + ...(resolved.module.resultSchema === undefined + ? {} + : { result: parsedResult(resolved.module.resultSchema, document, resolved.provenance) }), + }); +}; + +export interface RenderedRouteEvents extends RenderedRoute { + /** Every render event the runtime emitted, in the order it emitted them. */ + readonly events: readonly AgentRenderEvent[]; +} + +/** + * Renders one route and collects the ordered render-event stream (#140) + * alongside the final document, for the event matchers in + * `agent-bundle/test`. Same proof level and same renderer as + * {@link renderRoute}: this drains the dispatcher's public event stream + * rather than its final-only entry point. + */ +export const renderRouteEvents = async ( + target: RenderRouteTarget, + options: RenderRouteOptions = {}, +): Promise => { + const { collected, dispatcher, invocation, resolved, signal } = await prepareRender(target, options); + const events: AgentRenderEvent[] = []; + const reader = dispatcher.stream({ invocation, signal }).getReader(); + try { + for (;;) { + const next = await reader.read(); + if (next.done) break; + events.push(next.value); + } + } catch (error) { + throw renderFailure(error, invocation, resolved.provenance); + } + const complete = events.findLast((event) => event.type === 'complete'); + if (complete === undefined) { + throw new AgentTestError('render-failed', 'The route render ended without a complete event.', { details: [ - `cause: ${error instanceof Error ? `${error.name}: ${error.message}` : String(error)}`, `invocation: ${invocation.kind}`, + `events: ${events.length === 0 ? 'none' : events.map((event) => `${String(event.sequence)}:${event.type}`).join(' ')}`, ], provenance: resolved.provenance, }); } return Object.freeze({ - document, + document: complete.document, + events: Object.freeze([...events]), invocation, progress: Object.freeze([...collected]), provenance: resolved.provenance, ...(resolved.module.resultSchema === undefined ? {} - : { result: parsedResult(resolved.module.resultSchema, document, resolved.provenance) }), + : { result: parsedResult(resolved.module.resultSchema, complete.document, resolved.provenance) }), }); }; diff --git a/packages/agent-bundle/src/test/types.ts b/packages/agent-bundle/src/test/types.ts index 51603c300..038966c47 100644 --- a/packages/agent-bundle/src/test/types.ts +++ b/packages/agent-bundle/src/test/types.ts @@ -7,14 +7,21 @@ import type { AgentBundleTestManifest, AgentTestProofLevel, TestableRouteDescrip */ export type RenderableRouteKind = 'cli' | 'event-route' | 'prompt' | 'resource' | 'script' | 'tool'; +/** The structural schema surface the harness needs: parsing, not schema introspection. */ +export interface AgentRouteSchema { + readonly parse: (value: unknown) => unknown; +} + /** * One route module the harness renders: the public route contract's async - * default component, plus the `resultSchema` the generated server validates - * the document value against. + * default component, plus the schemas the generated server and the routed CLI + * validate through — `inputSchema` on the way in (the CLI dispatch level's + * boundary) and `resultSchema` on the document value coming out. */ export interface AgentRouteModule { readonly default: (props: never) => unknown; - readonly resultSchema?: { readonly parse: (value: unknown) => unknown }; + readonly inputSchema?: AgentRouteSchema; + readonly resultSchema?: AgentRouteSchema; } export type AgentRouteModuleLoader = () => Promise; diff --git a/packages/agent-bundle/tests/entry-shell.test.ts b/packages/agent-bundle/tests/entry-shell.test.ts index e3a47e7e0..c886c0b4f 100644 --- a/packages/agent-bundle/tests/entry-shell.test.ts +++ b/packages/agent-bundle/tests/entry-shell.test.ts @@ -1,4 +1,4 @@ -import { access } from 'node:fs/promises'; +import { access, readFile } from 'node:fs/promises'; import { describe, expect, it } from '@rstest/core'; @@ -9,6 +9,8 @@ import { generatedStdioMcpEntrySource, mcpEntryRuntimePath, mcpEntryRuntimeSpecifier, + mcpServerRuntimePath, + mcpServerRuntimeSpecifier, } from '../src/build/entry-shell.ts'; describe('entry export scanning', () => { @@ -133,29 +135,74 @@ it('generates one final-only Flight MCP factory from filesystem routes', () => { workerFile: 'mcp-curator-flight.mjs', }); - expect(source).toContain("from '@agent-bundle/runtime'"); - expect(source).toContain("from 'node:worker_threads'"); - expect(source).toContain('mcp-curator-flight.mjs'); - expect(source).toContain("from '@modelcontextprotocol/server'"); + // The entry is the compiled data — route table, App registry, worker URL, + // artifact epoch — handed to the shared server runtime. Registration, + // projection, and the warm host live in agent-bundle/mcp-server-runtime so + // the in-memory projection proof level exercises this artifact's code + // rather than a second copy of it (#103 stage 2). + expect(source).toContain(`from ${JSON.stringify(mcpServerRuntimeSpecifier)}`); expect(source).toContain("from 'agent-bundle/mcp-apps'"); - expect(source).toContain('createAgentRenderDispatcher'); - expect(source).toContain('createWarmFlightHost'); - expect(source).toContain('{ stderr: true, stdout: true }'); - expect(source).toContain('projectMcpRenderStream'); - expect(source).toContain('attachMcpStructuredContent'); - expect(source).toContain('createEventRuntimeServer'); - expect(source).toContain("kind: 'event'"); - expect(source).toContain('projectEventDocument'); - expect(source).toContain('runAgentRequest'); - expect(source).toContain('notifications/progress'); - expect(source).toContain('ARTIFACT_EPOCH'); - expect(source).toContain('route-fixture@1.2.3'); - expect(source).toContain('server.registerTool("inspect"'); - expect(source).toContain('server.registerResource("catalog", "catalog://books"'); - expect(source).toContain('server.registerPrompt("curate"'); - expect(source).toContain('export default createGeneratedRouteServer'); + expect(source).toContain('import * as route0 from "/project/src/mcp/curator/tools/inspect.tsx"'); + expect(source).toContain('const ARTIFACT_EPOCH = "route-fixture@1.2.3"'); + expect(source).toContain('"tool:curator/inspect": Object.freeze({ config: {"annotations":{"readOnlyHint":true}'); + expect(source).toContain('"resource:curator/catalog"'); + expect(source).toContain('"prompt:curator/curate"'); + expect(source).toContain('createFlightWorkerHost(new URL("./mcp-curator-flight.mjs", import.meta.url), ARTIFACT_EPOCH)'); + expect(source).toContain('artifactEpoch: ARTIFACT_EPOCH'); + expect(source).toContain('plugin: {"name":"route-fixture","version":"1.2.3"}'); + expect(source).toContain('export default async () => createGeneratedRouteMcpServer('); + // The event runtime's modules are aliased into the artifact, so the entry + // imports them and hands them to the shared runtime; the wiring itself is + // not re-templated here. + expect(source).toContain('createEventRuntimeServer,'); + expect(source).toContain('projectEventDocument,'); + expect(source).toContain('endpointId: `${EVENT_ARTIFACT_EPOCH}:${EVENT_TARGET}:'); + expect(source).toContain('events,'); + // Nothing else the shared runtime owns may be re-templated here. + expect(source).not.toContain('server.register'); + expect(source).not.toContain('projectMcpRenderStream'); + expect(source).not.toContain('new Worker('); + expect(source).not.toContain("kind: 'event'"); expect(source).not.toContain('lowerMcpResult'); - expect(source).not.toContain('projectToolResult'); +}); + +it('keeps the generated server behaviour in the shared runtime module the entry aliases', async () => { + const runtime = await readFile(mcpServerRuntimePath(), 'utf8'); + + expect(runtime).toContain('createWarmFlightHost'); + expect(runtime).toContain('projectMcpRenderStream'); + expect(runtime).toContain('attachMcpStructuredContent'); + expect(runtime).toContain('runAgentRequest'); + expect(runtime).toContain('notifications/progress'); + expect(runtime).toContain('{ stderr: true, stdout: true }'); + expect(runtime).toContain('server.registerTool'); + expect(runtime).toContain('server.registerResource'); + expect(runtime).toContain('server.registerPrompt'); + expect(runtime).toContain('createEventRuntimeServer('); + expect(runtime).toContain('projectEventDocument('); +}); + +it('fails the build on an MCP route the generated server cannot register', () => { + const generate = entryShellModule.generatedRouteMcpEntrySource; + const entry = (routes: readonly Readonly>[]): string => generate({ + plugin: { name: 'route-fixture', version: '1.2.3' }, + routes: routes as never, + serverName: 'curator', + workerFile: 'mcp-curator-flight.mjs', + }); + + expect(() => entry([{ + config: {}, + id: 'resource:curator/catalog', + kind: 'resource', + source: '/project/src/mcp/curator/resources/catalog.tsx', + }])).toThrow('non-empty static config.uri'); + expect(() => entry([{ + config: {}, + id: 'cli:migrate', + kind: 'cli', + source: '/project/src/cli/migrate.tsx', + }])).toThrow('non-MCP route'); }); diff --git a/packages/agent-bundle/tests/generated-route-server.test.ts b/packages/agent-bundle/tests/generated-route-server.test.ts index 82a3258d4..08556d4c6 100644 --- a/packages/agent-bundle/tests/generated-route-server.test.ts +++ b/packages/agent-bundle/tests/generated-route-server.test.ts @@ -94,9 +94,11 @@ it('lists and calls a generated filesystem tool through final-only Flight', { re "document.body.textContent = 'Curator dashboard';", '', ].join('\n')), + // Authored as JSX with no React import, the way the documented route + // contract reads: the build has to select the automatic JSX runtime, or + // the emitted module calls a `React` factory that is not in scope. writeProjectFile(root, 'src/mcp/curator/tools/inspect.tsx', [ "import { Agent, agent } from '@agent-bundle/runtime';", - "import { createElement } from 'react';", "import { z } from 'zod';", "export const config = { annotations: { readOnlyHint: true }, description: 'Inspect one source.' };", "export const inputSchema = z.object({ source: z.string() }).strict();", @@ -105,7 +107,11 @@ it('lists and calls a generated filesystem tool through final-only Flight', { re " if (signal.aborted) throw new DOMException('aborted', 'AbortError');", ' const context = await agent();', ' const result = { invocationKind: context.invocation.kind, source: input.source };', - ' return createElement(Agent.Result, { value: result }, createElement(Agent.Markdown, null, `Inspected **${input.source}**.`));', + ' return (', + ' ', + ' {`Inspected **${input.source}**.`}', + ' ', + ' );', '}', '', ].join('\n')), diff --git a/packages/agent-bundle/tests/inspect-bundler.test.ts b/packages/agent-bundle/tests/inspect-bundler.test.ts index 8a8ff754c..c595752e5 100644 --- a/packages/agent-bundle/tests/inspect-bundler.test.ts +++ b/packages/agent-bundle/tests/inspect-bundler.test.ts @@ -98,6 +98,10 @@ it('surfaces every synthesized bundler config with the tools hatch merged over t expect(script.generatedEntry).toContain('process.argv.slice(2)'); expect(script.config).toMatchObject({ id: 'agent-bundle-tool', + // Routes are authored as TSX, so every Rslib entry carries the React + // plugin: without it JSX lowers to a `React` factory that no generated + // executable has in scope. + plugins: [{ name: 'rsbuild:react' }], output: { distPath: { root: '/portable' }, filename: { js: 'scripts/tool.mjs' }, diff --git a/packages/agent-bundle/tests/packed-stdio-projection.test.ts b/packages/agent-bundle/tests/packed-stdio-projection.test.ts new file mode 100644 index 000000000..80bc89795 --- /dev/null +++ b/packages/agent-bundle/tests/packed-stdio-projection.test.ts @@ -0,0 +1,105 @@ +import { execFile as executeFile } from 'node:child_process'; +import { cp, mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { promisify } from 'node:util'; + +import { expect, it } from '@rstest/core'; + +import { openPackedMcpServer } from '../src/test/packed.ts'; +import { cachedNpmInstallArguments, installedEnvironment, sharedPackedTarball } from './support/shared-pack.ts'; + +const execFile = promisify(executeFile); +const fixtureRoot = resolve(import.meta.dirname, '../fixtures/route-harness'); + +interface McpJson { + readonly mcpServers: Readonly>; +} + +/** + * The `packed-stdio` proof level, and the repository's single packed proof + * journey for the consumer test harness (#103 stage 2 cost rule). + * + * One tarball (the run-level shared pack), one install, one artifact build, + * one spawned server. Every per-route assertion runs inside that one client + * session, because a second spawn would double the only cost this level has + * and prove the same thing twice. + * + * This is the only harness level that is process evidence: the generated + * entry runs as a separate operating-system process, out of a built artifact, + * over real stdio framing. The `mcp-in-memory` level (tests/projection/) + * covers the same protocol surface at a fraction of the cost and explicitly + * does not claim any of this. + */ +it('serves every compiled route from a packed install over real stdio', async () => { + const [agentBundle, runtime] = await Promise.all([ + sharedPackedTarball('agent-bundle'), + sharedPackedTarball('runtime'), + ]); + const consumer = await mkdtemp(join(tmpdir(), 'agent-bundle-packed-stdio-')); + const project = join(consumer, 'project'); + const artifact = join(project, 'artifact'); + + try { + await cp(fixtureRoot, project, { recursive: true }); + await execFile('npm', ['install', ...cachedNpmInstallArguments, + agentBundle.tarball, + runtime.tarball, + 'react@19.2.8', + 'react-dom@19.2.8', + 'zod@4.4.3', + ], { cwd: project, env: installedEnvironment() }); + + // The fixture selects `claude`, the only target whose capabilities cover + // its event route; Claude Code reads `.mcp.json` at the plugin root. + const cli = join(project, 'node_modules', '.bin', 'agent-bundle'); + await execFile(cli, ['build', '--root', project, '--output', artifact], { + cwd: project, + env: installedEnvironment(), + }); + + const pluginRoot = join(artifact, 'claude'); + const manifest = JSON.parse(await readFile(join(pluginRoot, '.mcp.json'), 'utf8')) as McpJson; + // Claude Code expands ${CLAUDE_PLUGIN_ROOT} to the installed plugin root + // before it spawns the server; the test stands in for the host there. + const entry = manifest.mcpServers['harness']!.args[0].replaceAll('${CLAUDE_PLUGIN_ROOT}', pluginRoot); + + await using session = await openPackedMcpServer({ cwd: project, entry, env: installedEnvironment() as Record }); + + expect(session.provenance.proofLevel).toBe('packed-stdio'); + expect(session.provenance.pid).toBeGreaterThan(0); + + const tools = await session.client.listTools(); + expect(tools.tools.map((tool) => tool.name).sort()).toEqual(['catalog', 'echo', 'unavailable']); + + // Per-route assertions iterate inside this one session: the packed cost is + // the spawn, not the calls. + await expect(session.client.callTool({ arguments: { message: 'packed' }, name: 'echo' })) + .resolves.toMatchObject({ + content: [{ text: '# Echo\n\npacked', type: 'text' }, { text: expect.stringContaining('workspace:'), type: 'text' }], + structuredContent: { message: 'packed', operationId: 'tool:harness/echo' }, + }); + await expect(session.client.callTool({ arguments: { genre: 'mystery' }, name: 'catalog' })) + .resolves.toMatchObject({ + content: [ + { text: 'catalog: mystery', type: 'text' }, + { text: '## mystery\n\n- Piranesi\n- Solaris', type: 'text' }, + ], + structuredContent: { genre: 'mystery', titles: ['Piranesi', 'Solaris'] }, + }); + await expect(session.client.callTool({ arguments: {}, name: 'unavailable' })) + .resolves.toMatchObject({ isError: true, structuredContent: { available: false } }); + await expect(session.client.readResource({ uri: 'harness://notes' })).resolves.toEqual({ + contents: [{ mimeType: 'text/markdown', text: '# Notes for harness://notes', uri: 'harness://notes' }], + }); + await expect(session.client.getPrompt({ arguments: { note: 'chapter one' }, name: 'summarize' })).resolves.toEqual({ + messages: [{ content: { text: 'Summarize chapter one', type: 'text' }, role: 'user' }], + }); + + // The generated entry keeps stdout for the protocol; anything the routes or + // the warm worker print has to arrive on stderr instead. + expect(session.stderr()).not.toContain('"jsonrpc"'); + } finally { + await rm(consumer, { force: true, recursive: true }); + } +}, 300_000); diff --git a/packages/agent-bundle/tests/projection/cli-dispatch.test.ts b/packages/agent-bundle/tests/projection/cli-dispatch.test.ts new file mode 100644 index 000000000..5f0a7a6de --- /dev/null +++ b/packages/agent-bundle/tests/projection/cli-dispatch.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from '@rstest/core'; + +import { cliJson, invokeCli } from '../../src/test/cli.ts'; + +/** + * The `cli-dispatch` proof level: one argv vector run through the routed CLI's + * own shell (#102 stage 2) over the compiled command graph, in this process. + * No binary is spawned, so a green run here is not evidence that the packaged + * executable starts — `packed-stdio` owns that. + * + * What this level does prove is that the product's dispatcher, argv + * projection, and exit-code policy agree with the compiled commands: the + * harness contributes only the `execute` bridge, and that mirrors the one the + * generated executable inlines. + */ +describe('the CLI dispatch level', () => { + it('resolves an argv vector to the compiled command and returns its canonical JSON line', async () => { + const run = await invokeCli(['inventory', 'fiction', '--format', 'json']); + + expect(run.command).toBe('inventory'); + expect(run.routeId).toBe('cli:inventory'); + expect(run.exitCode).toBe(0); + expect(run.stderr).toBe(''); + expect(cliJson(run)).toEqual({ format: 'json', shelf: 'fiction', titles: ['Piranesi', 'Solaris'] }); + expect(run.value).toEqual({ format: 'json', shelf: 'fiction', titles: ['Piranesi', 'Solaris'] }); + expect(run.provenance).toMatchObject({ commands: ['db migrate', 'inventory'], proofLevel: 'cli-dispatch' }); + }); + + it('dispatches a nested command through its compiled path', async () => { + const run = await invokeCli(['db', 'migrate']); + + expect(run.command).toBe('db migrate'); + expect(run.routeId).toBe('cli:db/migrate'); + expect(cliJson(run)).toEqual({ applied: 2, dryRun: false, exitCode: 0 }); + }); + + it('maps the validated result exit code under the result policy', async () => { + const run = await invokeCli(['db', 'migrate', '--dry-run']); + + expect(run.exitCode).toBe(3); + expect(run.value).toEqual({ applied: 0, dryRun: true, exitCode: 3 }); + }); + + it('projects a schema default and a positional key from the argv the compiler derived', async () => { + const run = await invokeCli(['inv', 'history', '--limit', '1']); + + expect(run.command).toBe('inventory'); + expect(cliJson(run)).toEqual({ format: 'text', shelf: 'history', titles: ['SPQR'] }); + }); + + it('reports a missing required positional as a usage failure without executing the command', async () => { + const run = await invokeCli(['inventory']); + + expect(run.exitCode).toBe(2); + expect(run.stdout).toBe(''); + expect(run.stderr).toContain("--help' for usage."); + expect(run.value).toBeUndefined(); + }); + + it('reports an unknown command as a usage failure that names the argv', async () => { + const run = await invokeCli(['nope']); + + expect(run.exitCode).toBe(2); + expect(run.command).toBeUndefined(); + expect(run.stderr).toContain('Unknown command: nope.'); + }); + + it('answers help and version from the generated shell rather than a command', async () => { + const [help, version] = await Promise.all([invokeCli(['--help']), invokeCli(['--version'])]); + + expect(help.exitCode).toBe(0); + expect(help.command).toBeUndefined(); + expect(help.stdout).toContain('inventory'); + expect(help.stdout).toContain('db'); + expect(version.stdout.trim()).toBe('route-harness 1.0.0'); + }); + + it('runs the command inside a cli request scope', async () => { + const progress: { readonly completed?: number; readonly message?: string }[] = []; + const run = await invokeCli(['inventory', 'fiction'], { + context: { progress: { report: async (update) => void progress.push(update) } }, + }); + + expect(run.exitCode).toBe(0); + expect(progress.map((update) => update.message)).toEqual(['reading inventory', 'inventory ready']); + }); +}); diff --git a/packages/agent-bundle/tests/projection/mcp-in-memory.test.ts b/packages/agent-bundle/tests/projection/mcp-in-memory.test.ts new file mode 100644 index 000000000..24af62a6d --- /dev/null +++ b/packages/agent-bundle/tests/projection/mcp-in-memory.test.ts @@ -0,0 +1,122 @@ +import { describe, expect, it } from '@rstest/core'; + +import { AgentTestError } from '../../src/test/errors.ts'; +import { + getMcpPrompt, + invokeMcpTool, + listMcpSurface, + openInMemoryMcpServer, + readMcpResource, +} from '../../src/test/mcp.ts'; + +/** + * The `mcp-in-memory` proof level: the real generated MCP server, registered + * by the same `mcp-server-runtime` module a built artifact runs, driven by a + * real MCP SDK client over the SDK's in-memory transport pair. + * + * What a green run here does NOT prove: that a process starts, that stdio + * framing is clean, or that a packed tarball contains the entry. Those are + * the `packed-stdio` level's claims (packed-stdio-projection.test.ts). + */ +describe('the in-memory MCP projection level', () => { + it('registers every compiled route kind on the real generated server', async () => { + const surface = await listMcpSurface(); + + expect(surface.tools).toEqual(['catalog', 'echo', 'unavailable']); + expect(surface.prompts).toEqual(['summarize']); + expect(surface.resources).toEqual(['harness://notes']); + expect(surface.provenance).toMatchObject({ + proofLevel: 'mcp-in-memory', + routeIds: [ + 'prompt:harness/summarize', + 'resource:harness/notes', + 'tool:harness/catalog', + 'tool:harness/echo', + 'tool:harness/unavailable', + ], + serverName: 'harness', + }); + }); + + it('projects a rendered Agent Document into the protocol content the server returns', async () => { + const invocation = await invokeMcpTool('echo', { + context: { workspace: { source: 'native', state: 'available', value: { root: '/tmp/harness-library' } } as never }, + input: { message: 'two files ready' }, + }); + + expect(invocation.isError).toBe(false); + expect(invocation.content).toEqual([ + { text: '# Echo\n\ntwo files ready', type: 'text' }, + { text: 'workspace: /tmp/harness-library', type: 'text' }, + ]); + expect(invocation.structuredContent).toEqual({ + message: 'two files ready', + operationId: 'tool:harness/echo', + workspace: '/tmp/harness-library', + }); + expect(invocation.provenance.proofLevel).toBe('mcp-in-memory'); + }); + + it('carries a represented error to the protocol as isError rather than a transport failure', async () => { + const invocation = await invokeMcpTool('unavailable'); + + expect(invocation.isError).toBe(true); + expect(invocation.content).toContainEqual(expect.objectContaining({ type: 'text' })); + expect(invocation.structuredContent).toEqual({ available: false }); + }); + + it('resolves a suspended boundary before the server projects the result', async () => { + const invocation = await invokeMcpTool('catalog', { input: { genre: 'mystery' } }); + + expect(invocation.content).toEqual([ + { text: 'catalog: mystery', type: 'text' }, + { text: '## mystery\n\n- Piranesi\n- Solaris', type: 'text' }, + ]); + expect(invocation.structuredContent).toEqual({ genre: 'mystery', titles: ['Piranesi', 'Solaris'] }); + }); + + it('reads a compiled resource route by its configured URI', async () => { + const read = await readMcpResource('harness://notes'); + + expect(read.contents).toEqual([ + { mimeType: 'text/markdown', text: '# Notes for harness://notes', uri: 'harness://notes' }, + ]); + expect(read.provenance.proofLevel).toBe('mcp-in-memory'); + }); + + it('gets a compiled prompt route with the arguments the client sent', async () => { + const prompt = await getMcpPrompt('summarize', { input: { note: 'chapter one' } }); + + expect(prompt.messages).toEqual([ + { content: { text: 'Summarize chapter one', type: 'text' }, role: 'user' }, + ]); + }); + + it('reuses one connected session for every call a suite makes', async () => { + await using session = await openInMemoryMcpServer(); + + const [first, second] = await Promise.all([ + session.client.callTool({ arguments: { message: 'first' }, name: 'echo' }), + session.client.callTool({ arguments: { message: 'second' }, name: 'echo' }), + ]); + + expect(first).toMatchObject({ structuredContent: { message: 'first' } }); + expect(second).toMatchObject({ structuredContent: { message: 'second' } }); + }); + + it('leaves the browser App surface off the in-memory server', async () => { + const surface = await listMcpSurface(); + + expect(surface.resources).not.toContain('ui://route-harness/panel.html'); + expect(surface.provenance.routeIds).not.toContain('app:harness/panel'); + }); + + it('names the compiled servers when the requested one does not exist', async () => { + const error = await listMcpSurface({ server: 'missing' }).catch((thrown: unknown) => thrown); + + expect(error).toBeInstanceOf(AgentTestError); + expect((error as AgentTestError).code).toBe('server-not-found'); + expect((error as AgentTestError).message).toContain('harness'); + expect((error as AgentTestError).message).toContain('recovery:'); + }); +}); diff --git a/packages/agent-bundle/tests/projection/render-events.test.ts b/packages/agent-bundle/tests/projection/render-events.test.ts new file mode 100644 index 000000000..f7ac4cc4f --- /dev/null +++ b/packages/agent-bundle/tests/projection/render-events.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from '@rstest/core'; + +import { expectDocument } from '../../src/test/matchers.ts'; +import { expectEvents } from '../../src/test/events.ts'; +import { renderRouteEvents } from '../../src/test/render.ts'; + +/** + * Event-stream matchers over the #140 render-event contract, against streams + * the real runtime produced rather than synthetic frames. + * + * These assertions are sequence-tolerant on purpose (#120): a legitimate + * extra `replace` or `progress` frame must never turn a passing render red, + * while a missing frame, a reordering, or a regressed ordinal still fails. + */ +describe('render events from the real runtime', () => { + it('opens with a shell, ends with exactly one complete, and never regresses its ordinals', async () => { + const rendered = await renderRouteEvents('tool:harness/catalog', { input: { genre: 'mystery' } }); + + expectEvents(rendered) + .toContainSequence(['shell', 'complete']) + .toHaveMonotonicSequence() + .toCompleteOnce() + .toHaveNoErrors(); + expect(rendered.events[0]!.type).toBe('shell'); + }); + + it('resolves the suspended boundary into the final document the complete event carries', async () => { + const rendered = await renderRouteEvents('tool:harness/catalog', { input: { genre: 'mystery' } }); + const complete = rendered.events.at(-1); + + if (complete?.type !== 'complete') throw new Error('expected the render to end with a complete event'); + expectDocument(rendered).toHaveStatus('success').toContainMarkdown('- Piranesi'); + expect(complete.document).toEqual(rendered.document); + expect(rendered.result).toEqual({ genre: 'mystery', titles: ['Piranesi', 'Solaris'] }); + }); + + it('reports request-scoped progress on the stream instead of only the final document', async () => { + const rendered = await renderRouteEvents('tool:harness/echo', { input: { message: 'streamed' } }); + + expectEvents(rendered) + .toHaveProgress({ atLeast: 1, messages: ['echoing'] }) + .toContainSequence(['shell', 'progress', 'complete']) + .toHaveMonotonicSequence(); + }); + + it('keeps a represented error inside the document rather than on the event stream', async () => { + const rendered = await renderRouteEvents('tool:harness/unavailable'); + + expectEvents(rendered).toHaveNoErrors().toCompleteOnce(); + expectDocument(rendered).toHaveStatus('represented-error').toHaveError('AB9001'); + }); + + it('renders an event route through the same stream every other route uses', async () => { + const rendered = await renderRouteEvents('event:tool/after', { + input: { + canonical: { + event: 'tool/after', + idempotencyKey: 'projection', + observedAt: '2026-09-01T00:00:00.000Z', + provenance: { + host: 'claude', + hostContractRevision: 'projection', + nativeEvent: 'PostToolUse', + source: 'native', + }, + sequence: 1, + }, + native: { tool_name: 'Write' }, + }, + }); + + expect(rendered.invocation.kind).toBe('event'); + expectEvents(rendered).toContainSequence(['shell', 'complete']).toHaveMonotonicSequence(); + expectDocument(rendered).toContainMarkdown('Observed tool/after from claude.'); + }); +}); diff --git a/packages/agent-bundle/tests/route-unit/render-route.test.ts b/packages/agent-bundle/tests/route-unit/render-route.test.ts index 078ea4947..b0f5759ec 100644 --- a/packages/agent-bundle/tests/route-unit/render-route.test.ts +++ b/packages/agent-bundle/tests/route-unit/render-route.test.ts @@ -100,7 +100,9 @@ describe('renderRoute through the real renderer', () => { props: { input: { uri: 'harness://notes' }, operationId: 'resource:harness/notes' }, }); expectDocument(rendered).toContainMarkdown('# Notes for harness://notes'); - expect(rendered.result).toEqual({ uri: 'harness://notes' }); + expect(rendered.result).toEqual({ + contents: [{ mimeType: 'text/markdown', text: '# Notes for harness://notes', uri: 'harness://notes' }], + }); }); it("validates the document value with the route's own resultSchema", async () => { @@ -113,18 +115,30 @@ describe('renderRoute through the real renderer', () => { }); }); - it('renders an event route with the event invocation the runtime contract defines', async () => { - const rendered = await renderRoute('event:tool/after', { input: { path: 'src/index.ts' } }); + it('renders an event route with the canonical props the runtime contract defines', async () => { + const rendered = await renderRoute('event:tool/after', { + input: { + canonical: { + event: 'tool/after', + idempotencyKey: 'route-unit', + observedAt: '2026-09-01T00:00:00.000Z', + provenance: { + host: 'claude', + hostContractRevision: 'route-unit', + nativeEvent: 'PostToolUse', + source: 'native', + }, + sequence: 1, + }, + native: { tool_name: 'Write' }, + }, + }); expect(rendered.invocation.kind).toBe('event'); expectDocument(rendered) .toHaveStatus('success') - .toContainMarkdown('Observed event:tool/after.') - .toHaveValue({ - event: 'event:tool/after', - invocationKind: 'event', - payload: { path: 'src/index.ts' }, - }); + .toContainMarkdown('Observed tool/after from claude.') + .toHaveValue({ event: 'tool/after', invocationKind: 'event', tool: 'Write' }); }); it('renders a route module handed in directly, without the compiled manifest', async () => { diff --git a/packages/agent-bundle/tests/test-harness-manifest.test.ts b/packages/agent-bundle/tests/test-harness-manifest.test.ts index 0e450b1c0..2f96b2f10 100644 --- a/packages/agent-bundle/tests/test-harness-manifest.test.ts +++ b/packages/agent-bundle/tests/test-harness-manifest.test.ts @@ -14,6 +14,7 @@ import { } from '../src/test/registry.ts'; import { renderRoute } from '../src/test/render.ts'; import { expectDocument } from '../src/test/matchers.ts'; +import { expectEvents } from '../src/test/events.ts'; import { compileRouteGraph } from '../src/routes/graph.ts'; import type { AgentBundleTestManifest } from '../src/test/manifest.ts'; @@ -40,8 +41,12 @@ describe('the compiled test manifest', () => { it('names every conventional route the compiler discovered, with its extracted config', () => { expect(Object.keys(manifest.routes).sort()).toEqual([ 'app:harness/panel', + 'cli:db/migrate', + 'cli:inventory', 'event:tool/after', + 'prompt:harness/summarize', 'resource:harness/notes', + 'tool:harness/catalog', 'tool:harness/echo', 'tool:harness/unavailable', ]); @@ -62,6 +67,31 @@ describe('the compiled test manifest', () => { expect(manifest.targets).toEqual(['claude']); }); + it('carries the compiled command graph the CLI dispatch level dispatches over', () => { + expect(manifest.cliCommands).toEqual([ + { + aliases: [], + description: 'Applies pending harness migrations.', + exitCode: 'result', + options: [expect.objectContaining({ key: 'dryRun', kind: 'boolean', option: 'dry-run' })], + path: ['db', 'migrate'], + routeId: 'cli:db/migrate', + }, + { + aliases: ['inv'], + description: 'Lists the harness library inventory.', + exitCode: 'zero', + options: [ + expect.objectContaining({ choices: ['json', 'text'], defaultValue: 'text', key: 'format' }), + expect.objectContaining({ key: 'limit', kind: 'number' }), + expect.objectContaining({ key: 'shelf', positional: 0, required: true }), + ], + path: ['inventory'], + routeId: 'cli:inventory', + }, + ]); + }); + it('reuses the compiler pass rather than compiling a second route graph', async () => { const graph = await compileRouteGraph(fixtureRoot, { targets: ['claude'] } as never); @@ -197,6 +227,7 @@ describe('document matchers', () => { children: Object.freeze([ Object.freeze({ kind: 'markdown', text: '# Inventory\n\n2 files' }), Object.freeze({ kind: 'text', text: 'workspace: /tmp/library' }), + Object.freeze({ kind: 'context', text: 'Recorded 2 files.' }), ]), kind: 'result', }), @@ -210,8 +241,9 @@ describe('document matchers', () => { .toHaveStatus('success') .toContainMarkdown('2 files') .toContainText('/tmp/library') + .toContainContext('Recorded 2 files.') .toHaveValue({ files: 2 }) - .toHaveNodeKinds(['result', 'markdown', 'text']); + .toHaveNodeKinds(['result', 'markdown', 'text', 'context']); }); it('reports the expected and received value with the route provenance', () => { @@ -266,11 +298,77 @@ describe('document matchers', () => { expect(() => expectDocument(document).toHaveStatus('failed')).toThrow('unexpected status'); expect(() => expectDocument(document).toContainMarkdown('missing')).toThrow('no Markdown node'); expect(() => expectDocument(document).toContainText('missing')).toThrow('no text node'); + expect(() => expectDocument(document).toContainContext('missing')).toThrow('no context node'); expect(() => expectDocument(document).toHaveError()).toThrow('no matching error'); expect(() => expectDocument(document).toHaveNodeKinds(['result'])).toThrow('node kinds differ'); }); }); +describe('render event matchers', () => { + const frame = Object.freeze({ root: { children: [], kind: 'result' }, status: 'success', version: 1 }); + const event = (sequence: number, type: string, extra: Readonly> = {}) => + Object.freeze({ document: frame, sequence, type, ...extra }); + // A shell, two legitimate replaces for the same boundary, and a completion: + // the #120 shape that a pinned exact-array assertion called a regression. + const stream = Object.freeze([ + event(0, 'shell'), + event(1, 'progress', { completed: 1, message: 'reading inventory', total: 2 }), + event(2, 'replace', { boundaryId: 'b:1' }), + event(3, 'replace', { boundaryId: 'b:1' }), + event(4, 'progress', { completed: 2, message: 'inventory ready', total: 2 }), + event(5, 'complete'), + ]) as never; + + it('tolerates extra frames between the events the contract requires', () => { + expectEvents(stream) + .toContainSequence(['shell', 'replace', 'complete']) + .toHaveMonotonicSequence() + .toCompleteOnce() + .toHaveNoErrors() + .toBeBoundedBy(6) + .toHaveProgress({ atMost: 2, messages: ['reading', 'ready'] }); + }); + + it('still fails a missing, reordered, or over-budget frame', () => { + expect(() => expectEvents(stream).toContainSequence(['complete', 'shell'])) + .toThrow('do not contain the expected sequence'); + expect(() => expectEvents(stream).toContainSequence(['shell', 'error'])) + .toThrow('"error"'); + expect(() => expectEvents(stream).toBeBoundedBy(3)).toThrow('more events than the bound'); + expect(() => expectEvents(stream).toHaveProgress({ messages: ['ready', 'reading'] })) + .toThrow('in order'); + }); + + it('rejects a repeated or regressed sequence number', () => { + const regressed = Object.freeze([event(0, 'shell'), event(0, 'complete')]) as never; + + expect(() => expectEvents(regressed).toHaveMonotonicSequence()) + .toThrow('not strictly increasing'); + }); + + it('prints the timeline and the route the events came from', () => { + const subject = { events: stream, provenance: { kind: 'cli', proofLevel: 'cli-dispatch', routeId: 'cli:inventory', source: 'manifest', targets: ['claude'] } } as never; + const error = (() => { + try { + expectEvents(subject).toHaveErrorCode('AB9001'); + return undefined; + } catch (thrown: unknown) { + return thrown as AgentTestError; + } + })(); + + expect(error?.code).toBe('assertion-failed'); + expect(error?.message).toContain('0:shell 1:progress 2:replace 3:replace 4:progress 5:complete'); + expect(error?.message).toContain('cli:inventory'); + expect(error?.message).toContain('cli-dispatch (argv dispatched through the routed CLI shell in-process'); + }); + + it('keeps the exact-array assertion available for a contract that needs it', () => { + expectEvents([event(0, 'shell'), event(1, 'complete')] as never).toHaveTypes(['shell', 'complete']); + expect(() => expectEvents(stream).toHaveTypes(['shell', 'complete'])).toThrow('toContainSequence tolerates'); + }); +}); + describe('the manifest a route-free project compiles', () => { it('reports no routes and no proof beyond the route-unit level', async () => { const empty: AgentBundleTestManifest = await compileTestManifest({ @@ -313,7 +411,21 @@ describe('the harness package boundary', () => { it('keeps every optional peer out of the harness modules at value level', async () => { const sources = await Promise.all( - ['src/rstest/index.ts', 'src/rstest/setup-module.ts', 'src/test/index.ts', 'src/test/manifest.ts', 'src/test/matchers.ts', 'src/test/registry.ts', 'src/test/render.ts', 'src/test/types.ts', 'src/test/errors.ts'] + [ + 'src/rstest/index.ts', + 'src/rstest/setup-module.ts', + 'src/test/cli.ts', + 'src/test/errors.ts', + 'src/test/events.ts', + 'src/test/index.ts', + 'src/test/manifest.ts', + 'src/test/matchers.ts', + 'src/test/mcp.ts', + 'src/test/packed.ts', + 'src/test/registry.ts', + 'src/test/render.ts', + 'src/test/types.ts', + ] .map(async (relativePath) => [relativePath, await readFile(resolve(packageRoot, relativePath), 'utf8')] as const), ); diff --git a/rstest.config.ts b/rstest.config.ts index cf4e2d312..7f63dc431 100644 --- a/rstest.config.ts +++ b/rstest.config.ts @@ -1,6 +1,6 @@ import { defineConfig } from '@rstest/core'; -import { routeUnitTestFiles, templateTestFiles } from './rstest.integration-tests.ts'; +import { projectionTestFiles, routeUnitTestFiles, templateTestFiles } from './rstest.integration-tests.ts'; import { withAgentBundleRslibConfig } from './rstest.rslib.ts'; export default defineConfig({ @@ -8,7 +8,7 @@ export default defineConfig({ include: [ 'packages/**/tests/**/*.test.ts', ], - exclude: [...routeUnitTestFiles, ...templateTestFiles], + exclude: [...projectionTestFiles, ...routeUnitTestFiles, ...templateTestFiles], // The e2e fixtures copy the shared rsc-agent-runtime example dist; build it // once in the orchestrator so parallel workers never race the ensure-build. globalSetup: ['./rstest.integration.setup.ts'], diff --git a/rstest.integration-tests.ts b/rstest.integration-tests.ts index b096c2494..9b5efb9b2 100644 --- a/rstest.integration-tests.ts +++ b/rstest.integration-tests.ts @@ -101,6 +101,7 @@ export const packedTestFiles: readonly string[] = [ 'packages/agent-bundle/tests/dev-workbench-packaging.test.ts', 'packages/agent-bundle/tests/packed-consumer.test.ts', 'packages/agent-bundle/tests/packed-native-smoke.test.ts', + 'packages/agent-bundle/tests/packed-stdio-projection.test.ts', 'packages/agent-bundle/tests/public-api-packed.test.ts', 'packages/agent-bundle/tests/release-audit.test.ts', 'packages/agent-bundle/tests/rsc-runtime-optional-packaging.test.ts', @@ -130,6 +131,18 @@ export const routeUnitTestFiles: readonly string[] = [ 'packages/agent-bundle/tests/route-unit/**', ]; +/** + * In-process projection tests (#103 stage 2): the `mcp-in-memory` and + * `cli-dispatch` proof levels. They render routes, so they need the same + * `react-server` process condition the route-unit level needs, but they get + * their own pool and their own directory so the levels stay visibly separate + * — `pnpm test:projection`. Neither level is process or artifact proof; the + * `packed-stdio` level lives in the packed pool. + */ +export const projectionTestFiles: readonly string[] = [ + 'packages/agent-bundle/tests/projection/**', +]; + /** * Checked-in scaffolding templates ship their own test files; they run inside * scaffolded projects (the packed e2e drives them through each project's diff --git a/rstest.projection.config.ts b/rstest.projection.config.ts new file mode 100644 index 000000000..76d47d9c6 --- /dev/null +++ b/rstest.projection.config.ts @@ -0,0 +1,20 @@ +import { resolve } from 'node:path'; + +import { defineConfig } from '@rstest/core'; + +import { agentBundleRstest } from './packages/agent-bundle/src/rstest/index.ts'; + +/** + * The repository's in-process projection pool (#103 stage 2): the + * `mcp-in-memory` and `cli-dispatch` proof levels, built from the same shipped + * consumer configuration helper the route-unit pool uses. It is a separate run + * from `rstest.route-unit.config.ts` so a route-unit pass and a projection + * pass are separately reported — the levels are separate claims. + * + * Neither level opens a process. The `packed-stdio` level lives in the packed + * pool (`pnpm test:packed`), which owns the run's single build and pack. + */ +export default defineConfig(await agentBundleRstest({ + include: ['packages/agent-bundle/tests/projection/**/*.test.ts'], + root: resolve(import.meta.dirname, 'packages/agent-bundle/fixtures/route-harness'), +})); diff --git a/rstest.unit.config.ts b/rstest.unit.config.ts index 4fb07089b..508f7226f 100644 --- a/rstest.unit.config.ts +++ b/rstest.unit.config.ts @@ -4,6 +4,7 @@ import { integrationTestFiles, nightlyEvidenceTestFiles, packedTestFiles, + projectionTestFiles, routeUnitTestFiles, templateTestFiles, } from './rstest.integration-tests.ts'; @@ -19,6 +20,7 @@ export default defineConfig({ ...integrationTestFiles, ...nightlyEvidenceTestFiles, ...packedTestFiles, + ...projectionTestFiles, ...routeUnitTestFiles, ...templateTestFiles, ],