From 224c7a0ddff4c735ad8fb82fe0405d66cd30b83d Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Tue, 1 Sep 2026 23:18:47 +0000 Subject: [PATCH] feat(test): add target capability fixtures Expose route-unit fixtures that drive the real MCP projector so consumers can prove supported, fallback, and fail-closed rich-content behavior without overstating transport evidence. --- .changeset/target-capability-fixtures.md | 15 ++ packages/agent-bundle/src/test/errors.ts | 1 + packages/agent-bundle/src/test/index.ts | 16 +- packages/agent-bundle/src/test/matchers.ts | 54 +++++ .../src/test/target-capabilities.ts | 205 ++++++++++++++++++ .../target-capabilities/rich-content.ts | 33 +++ .../projection/target-capabilities.test.ts | 114 ++++++++++ .../tests/test-harness-manifest.test.ts | 25 +++ 8 files changed, 461 insertions(+), 2 deletions(-) create mode 100644 .changeset/target-capability-fixtures.md create mode 100644 packages/agent-bundle/src/test/target-capabilities.ts create mode 100644 packages/agent-bundle/tests/fixtures/target-capabilities/rich-content.ts create mode 100644 packages/agent-bundle/tests/projection/target-capabilities.test.ts diff --git a/.changeset/target-capability-fixtures.md b/.changeset/target-capability-fixtures.md new file mode 100644 index 000000000..7840627c1 --- /dev/null +++ b/.changeset/target-capability-fixtures.md @@ -0,0 +1,15 @@ +--- +"agent-bundle": minor +--- + +Add explicit target-capability fixtures to `agent-bundle/test`. + +`createTargetCapabilityFixture()` records support or denial for image, audio, +resource, and progress projection while keeping text as the always-supported +baseline. `projectTargetCapabilities()` projects a real `renderRouteEvents()` +result through the runtime's MCP projector, preserving its rich-content +fallback and fail-closed behavior without claiming transport, packed artifact, +or host proof. + +`expectDocument()` now includes field-aware assertions for image, audio, and +resource nodes. diff --git a/packages/agent-bundle/src/test/errors.ts b/packages/agent-bundle/src/test/errors.ts index 7f964ed1f..1c18481aa 100644 --- a/packages/agent-bundle/src/test/errors.ts +++ b/packages/agent-bundle/src/test/errors.ts @@ -14,6 +14,7 @@ export type AgentTestErrorCode = | 'result-rejected' | 'route-not-found' | 'server-not-found' + | 'unsupported-rich-content' | 'unsupported-route-kind'; /** How many characters of a captured value one diagnostic may print. */ diff --git a/packages/agent-bundle/src/test/index.ts b/packages/agent-bundle/src/test/index.ts index bd44b1d81..d34bc46dd 100644 --- a/packages/agent-bundle/src/test/index.ts +++ b/packages/agent-bundle/src/test/index.ts @@ -7,7 +7,7 @@ * * | level | helper | what it proves | * | --- | --- | --- | - * | `route-unit` | `renderRoute`, `renderRouteEvents` | the route component and its document, through the real Agent renderer | + * | `route-unit` | `renderRoute`, `renderRouteEvents`, `createTargetCapabilityFixture`, `projectTargetCapabilities` | the route component and its document through the real Agent renderer; explicit target-capability projection through the real MCP projector, without transport or host proof | * | `mcp-in-memory` | `openInMemoryMcpServer`, `invokeMcpTool`, `readMcpResource`, `getMcpPrompt`, `listMcpSurface` | the real generated MCP server's protocol contract, over the SDK's in-memory transport | * | `cli-dispatch` | `invokeCli`, `cliJson` | a compiled CLI command dispatched through the routed CLI's own shell, in this process | * | `packed-stdio` | `openPackedMcpServer` | a built artifact's generated entry running as a real process over stdio | @@ -50,7 +50,19 @@ export type { RenderedRouteEvents, } from './render.ts'; export { expectDocument } from './matchers.ts'; -export type { AgentDocumentNodeKind, DocumentAssertions, DocumentSubject } from './matchers.ts'; +export type { + AgentDocumentNodeKind, + DocumentAssertions, + DocumentSubject, + MediaNodeExpectation, + ResourceNodeExpectation, +} from './matchers.ts'; +export { createTargetCapabilityFixture, projectTargetCapabilities } from './target-capabilities.ts'; +export type { + TargetCapabilityFixture, + TargetCapabilityFixtureInput, + TargetCapabilityProjection, +} from './target-capabilities.ts'; export { expectEvents } from './events.ts'; export type { AgentRenderEventType, diff --git a/packages/agent-bundle/src/test/matchers.ts b/packages/agent-bundle/src/test/matchers.ts index e143942d4..cf98d47aa 100644 --- a/packages/agent-bundle/src/test/matchers.ts +++ b/packages/agent-bundle/src/test/matchers.ts @@ -9,6 +9,17 @@ export type AgentDocumentNodeKind = AgentDocumentNode['kind']; export type DocumentSubject = AgentDocument | RenderedRoute; +export interface MediaNodeExpectation { + readonly data?: string; + readonly mimeType?: string; +} + +export interface ResourceNodeExpectation { + readonly mimeType?: string; + readonly name?: string; + readonly uri?: string; +} + const documentOf = (subject: DocumentSubject): AgentDocument => 'document' in subject ? subject.document : subject; @@ -32,10 +43,16 @@ const textOf = ( * that provenance available. */ export interface DocumentAssertions { + /** Asserts an audio node matches every supplied media field. */ + readonly toContainAudio: (expected?: MediaNodeExpectation) => DocumentAssertions; /** Asserts a context node contains `text` — the additional context an event route returns to its host. */ readonly toContainContext: (text: string) => DocumentAssertions; + /** Asserts an image node matches every supplied media field. */ + readonly toContainImage: (expected?: MediaNodeExpectation) => DocumentAssertions; /** Asserts a Markdown node contains `text`. */ readonly toContainMarkdown: (text: string) => DocumentAssertions; + /** Asserts a resource node matches every supplied link field. */ + readonly toContainResource: (expected?: ResourceNodeExpectation) => DocumentAssertions; /** Asserts a text node contains `text`. */ readonly toContainText: (text: string) => DocumentAssertions; /** Asserts the document represents an error node, optionally with `code`. */ @@ -61,6 +78,18 @@ export const expectDocument = (subject: DocumentSubject): DocumentAssertions => }); }; const assertions: DocumentAssertions = { + toContainAudio(expected = {}) { + const found = nodes(document.root).flatMap((node) => (node.kind === 'audio' ? [node] : [])); + if (!found.some((node) => + (expected.data === undefined || node.data === expected.data) + && (expected.mimeType === undefined || node.mimeType === expected.mimeType))) { + fail('The Agent Document contains no audio node matching the expected fields.', [ + `expected: ${captured(expected)}`, + `received: ${found.length === 0 ? 'no audio nodes' : captured(found)}`, + ]); + } + return assertions; + }, toContainContext(text) { const found = textOf(document, 'context'); if (!found.some((value) => value.includes(text))) { @@ -71,6 +100,18 @@ export const expectDocument = (subject: DocumentSubject): DocumentAssertions => } return assertions; }, + toContainImage(expected = {}) { + const found = nodes(document.root).flatMap((node) => (node.kind === 'image' ? [node] : [])); + if (!found.some((node) => + (expected.data === undefined || node.data === expected.data) + && (expected.mimeType === undefined || node.mimeType === expected.mimeType))) { + fail('The Agent Document contains no image node matching the expected fields.', [ + `expected: ${captured(expected)}`, + `received: ${found.length === 0 ? 'no image nodes' : captured(found)}`, + ]); + } + return assertions; + }, toContainMarkdown(text) { const found = textOf(document, 'markdown'); if (!found.some((value) => value.includes(text))) { @@ -81,6 +122,19 @@ export const expectDocument = (subject: DocumentSubject): DocumentAssertions => } return assertions; }, + toContainResource(expected = {}) { + const found = nodes(document.root).flatMap((node) => (node.kind === 'resource' ? [node] : [])); + if (!found.some((node) => + (expected.mimeType === undefined || node.mimeType === expected.mimeType) + && (expected.name === undefined || node.name === expected.name) + && (expected.uri === undefined || node.uri === expected.uri))) { + fail('The Agent Document contains no resource node matching the expected fields.', [ + `expected: ${captured(expected)}`, + `received: ${found.length === 0 ? 'no resource nodes' : captured(found)}`, + ]); + } + return assertions; + }, toContainText(text) { const found = textOf(document, 'text'); if (!found.some((value) => value.includes(text))) { diff --git a/packages/agent-bundle/src/test/target-capabilities.ts b/packages/agent-bundle/src/test/target-capabilities.ts new file mode 100644 index 000000000..c8979298c --- /dev/null +++ b/packages/agent-bundle/src/test/target-capabilities.ts @@ -0,0 +1,205 @@ +import type * as AgentRuntime from '@agent-bundle/runtime'; +import type { + McpProgressNotificationParams, + McpRichContentFallback, + McpRichContentKind, +} from '@agent-bundle/runtime'; + +import { AgentTestError, captured } from './errors.ts'; +import { ROUTE_UNIT_PROOF_LEVEL } from './manifest.ts'; +import type { McpContentBlock } from './mcp.ts'; +import type { RenderedRouteEvents } from './render.ts'; +import type { RenderedRouteProvenance } from './types.ts'; + +const progressToken = 'agent-bundle-target-capability-fixture'; + +/** + * The capabilities one route-unit fixture explicitly advertises. Text is not + * configurable because MCP text is the projector's always-supported baseline. + * Progress is separate from document content: it controls whether the real MCP + * event-stream projector receives a progress token and notification sink. + */ +export interface TargetCapabilityFixtureInput { + readonly audio: boolean; + readonly image: boolean; + readonly progress: boolean; + readonly resource: boolean; + readonly richContentFallback: McpRichContentFallback; +} + +/** + * An explicit target-capability fixture for route-unit projection. + * + * This is projection-layer proof only. It does not open an MCP transport, + * start a packed process, or prove that a host accepts any projected block. + */ +export interface TargetCapabilityFixture extends TargetCapabilityFixtureInput { + readonly proofLevel: typeof ROUTE_UNIT_PROOF_LEVEL; + readonly text: true; +} + +/** The real MCP projector's output plus any progress notifications it emitted. */ +export interface TargetCapabilityProjection { + readonly content: readonly McpContentBlock[]; + readonly isError: boolean; + readonly progress: readonly McpProgressNotificationParams[]; + readonly provenance: RenderedRouteProvenance; + readonly structuredContent?: unknown; +} + +const booleanCapability = ( + input: TargetCapabilityFixtureInput, + key: 'audio' | 'image' | 'progress' | 'resource', +): boolean => { + const value = input[key]; + if (typeof value === 'boolean') return value; + throw new AgentTestError('invalid-input', 'A target-capability fixture must explicitly advertise or deny every capability.', { + details: [ + `capability: ${key}`, + `received: ${captured(value)}`, + ], + recovery: `Pass ${key}: true or ${key}: false to createTargetCapabilityFixture().`, + }); +}; + +/** + * Creates an immutable, explicit fixture. No rich capability defaults to + * supported: callers must choose every boolean and the fallback policy. + */ +export const createTargetCapabilityFixture = ( + input: TargetCapabilityFixtureInput, +): TargetCapabilityFixture => { + if (input.richContentFallback !== 'fail' && input.richContentFallback !== 'text') { + throw new AgentTestError('invalid-input', 'A target-capability fixture requires a real MCP rich-content fallback policy.', { + details: [`received: ${captured(input.richContentFallback)}`], + recovery: 'Pass richContentFallback: "fail" or richContentFallback: "text".', + }); + } + return Object.freeze({ + audio: booleanCapability(input, 'audio'), + image: booleanCapability(input, 'image'), + progress: booleanCapability(input, 'progress'), + proofLevel: ROUTE_UNIT_PROOF_LEVEL, + resource: booleanCapability(input, 'resource'), + richContentFallback: input.richContentFallback, + text: true as const, + }); +}; + +interface Projector { + readonly projectMcpRenderStream: typeof AgentRuntime.projectMcpRenderStream; +} + +let projectorPromise: Promise | undefined; + +/** + * The runtime is an optional peer, so the public `agent-bundle/test` entry must + * remain importable for manifest-only tests without loading it. Projection is + * the point where the peer becomes required, matching the existing render and + * in-memory helpers. + */ +const loadProjector = async (): Promise => { + projectorPromise ??= import('@agent-bundle/runtime') + .then((runtime) => ({ projectMcpRenderStream: runtime.projectMcpRenderStream })) + .catch((error: unknown) => { + projectorPromise = undefined; + throw error; + }); + return projectorPromise; +}; + +const eventStream = ( + rendered: RenderedRouteEvents, +): ReadableStream => new ReadableStream({ + start(controller) { + for (const event of rendered.events) controller.enqueue(event); + controller.close(); + }, +}); + +const richProjectionError = ( + error: unknown, +): { readonly code: 'unsupported-rich-content'; readonly kind?: McpRichContentKind } | undefined => { + if ( + typeof error !== 'object' + || error === null + || !('code' in error) + || error.code !== 'unsupported-rich-content' + ) return undefined; + const kind = 'kind' in error && ( + error.kind === 'audio' + || error.kind === 'image' + || error.kind === 'resource' + ) ? error.kind : undefined; + return { code: 'unsupported-rich-content', ...(kind === undefined ? {} : { kind }) }; +}; + +/** + * Reprojects a real route render-event stream through the runtime's + * `projectMcpRenderStream`, using exactly the capabilities and fallback the + * fixture advertises. + * + * Proof level: `route-unit`. The events came from the real Agent renderer and + * the projection is real, but no MCP transport, packed artifact, or host is + * involved. A denied rich block either becomes the runtime's exact text + * placeholder or fails closed; this helper never marks it as accepted. + */ +export const projectTargetCapabilities = async ( + rendered: RenderedRouteEvents, + fixture: TargetCapabilityFixture, +): Promise => { + const progress: McpProgressNotificationParams[] = []; + try { + const projector = await loadProjector(); + const projected = await projector.projectMcpRenderStream(eventStream(rendered), { + capabilities: { + audio: fixture.audio, + image: fixture.image, + resource: fixture.resource, + }, + richContentFallback: fixture.richContentFallback, + ...(fixture.progress + ? { + progressToken, + sendProgress: async (params: McpProgressNotificationParams) => { + progress.push(params); + }, + } + : {}), + }); + return Object.freeze({ + content: Object.freeze(projected.result.content as readonly McpContentBlock[]), + isError: projected.result.isError === true, + progress: Object.freeze([...progress]), + provenance: rendered.provenance, + ...(projected.result.structuredContent === undefined + ? {} + : { structuredContent: projected.result.structuredContent }), + }); + } catch (error) { + const richError = richProjectionError(error); + if (richError !== undefined) { + throw new AgentTestError( + 'unsupported-rich-content', + 'The target-capability fixture denied rich MCP content and the runtime projector failed closed.', + { + cause: error, + details: [ + `offending kind: ${richError.kind ?? 'unknown'}`, + `fixture: ${captured(fixture)}`, + ], + provenance: rendered.provenance, + recovery: richError.kind === undefined + ? 'Advertise the required rich capability, or choose richContentFallback: "text".' + : `Advertise ${richError.kind}: true, or choose richContentFallback: "text".`, + }, + ); + } + throw new AgentTestError('projection-failed', 'The route-unit target-capability projection failed.', { + cause: error, + details: [`cause: ${error instanceof Error ? error.message : String(error)}`], + provenance: rendered.provenance, + recovery: 'Install @agent-bundle/runtime and project a completed renderRouteEvents() result.', + }); + } +}; diff --git a/packages/agent-bundle/tests/fixtures/target-capabilities/rich-content.ts b/packages/agent-bundle/tests/fixtures/target-capabilities/rich-content.ts new file mode 100644 index 000000000..8f0e099d9 --- /dev/null +++ b/packages/agent-bundle/tests/fixtures/target-capabilities/rich-content.ts @@ -0,0 +1,33 @@ +import { Agent, agent } from '@agent-bundle/runtime'; +import { z } from 'zod'; + +export const imageData = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII='; +export const imageMimeType = 'image/png'; +export const audioData = 'UklGRiQAAABXQVZFZm10IBAAAAABAAEAQB8AAEAfAAABAAgAZGF0YQAAAAA='; +export const audioMimeType = 'audio/wav'; +export const resourceName = 'fixture.bin'; +export const resourceMimeType = 'application/octet-stream'; +export const resourceUri = 'data:application/octet-stream;base64,AAECAwQ='; +export const baselineText = 'Text is the always-supported MCP content baseline.'; + +export const inputSchema = z.object({}); +export const resultSchema = z.object({ fixture: z.literal('target-capabilities') }); + +/** + * One route-unit fixture with every rich Agent Document node. The resource is + * a real binary payload carried by a data URI because Agent.Resource models a + * resource link; it does not pretend that the document contract embeds blobs. + */ +export default async function RichContent() { + const context = await agent(); + await context.progress.report({ completed: 1, message: 'projecting rich content', total: 1 }); + return Agent.Result({ + children: [ + Agent.Text({ children: baselineText }), + Agent.Image({ data: imageData, mimeType: imageMimeType }), + Agent.Audio({ data: audioData, mimeType: audioMimeType }), + Agent.Resource({ mimeType: resourceMimeType, name: resourceName, uri: resourceUri }), + ], + value: { fixture: 'target-capabilities' }, + }); +} diff --git a/packages/agent-bundle/tests/projection/target-capabilities.test.ts b/packages/agent-bundle/tests/projection/target-capabilities.test.ts new file mode 100644 index 000000000..0017b2729 --- /dev/null +++ b/packages/agent-bundle/tests/projection/target-capabilities.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from '@rstest/core'; + +import { + audioData, + audioMimeType, + baselineText, + imageData, + imageMimeType, + resourceMimeType, + resourceName, + resourceUri, +} from '../fixtures/target-capabilities/rich-content.ts'; +import * as RichContentRoute from '../fixtures/target-capabilities/rich-content.ts'; +import { + AgentTestError, + createTargetCapabilityFixture, + expectDocument, + projectTargetCapabilities, + renderRouteEvents, + type RenderedRouteEvents, +} from '../../src/test/index.ts'; + +const renderRichContent = (): Promise => renderRouteEvents(RichContentRoute, { + routeId: 'tool:fixtures/rich-content', +}); + +const fixture = ( + overrides: Partial[0]> = {}, +) => createTargetCapabilityFixture({ + audio: true, + image: true, + progress: true, + resource: true, + richContentFallback: 'fail', + ...overrides, +}); + +describe('route-unit target-capability projection', () => { + it('asserts rich Agent Document nodes without treating them as text', async () => { + const rendered = await renderRichContent(); + + expectDocument(rendered) + .toHaveNodeKinds(['result', 'text', 'image', 'audio', 'resource']) + .toContainText(baselineText) + .toContainImage({ data: imageData, mimeType: imageMimeType }) + .toContainAudio({ data: audioData, mimeType: audioMimeType }) + .toContainResource({ mimeType: resourceMimeType, name: resourceName, uri: resourceUri }); + }); + + it('projects supported rich content and requested progress through the runtime projector', async () => { + const projected = await projectTargetCapabilities(await renderRichContent(), fixture()); + + expect(projected.content).toEqual([ + { text: baselineText, type: 'text' }, + { data: imageData, mimeType: imageMimeType, type: 'image' }, + { data: audioData, mimeType: audioMimeType, type: 'audio' }, + { mimeType: resourceMimeType, name: resourceName, type: 'resource_link', uri: resourceUri }, + ]); + expect(projected.progress).toEqual([{ + message: 'projecting rich content', + progress: 1, + progressToken: 'agent-bundle-target-capability-fixture', + total: 1, + }]); + expect(projected.provenance.proofLevel).toBe('route-unit'); + expect(projected.structuredContent).toEqual({ fixture: 'target-capabilities' }); + }); + + it('uses exact text fallbacks and leaks no denied rich block', async () => { + const projected = await projectTargetCapabilities(await renderRichContent(), fixture({ + audio: false, + image: false, + progress: false, + resource: false, + richContentFallback: 'text', + })); + + expect(projected.content).toEqual([ + { text: baselineText, type: 'text' }, + { text: `[image ${imageMimeType}]`, type: 'text' }, + { text: `[audio ${audioMimeType}]`, type: 'text' }, + { text: `[resource ${resourceName} ${resourceUri}]`, type: 'text' }, + ]); + expect(projected.content.every((block) => block.type === 'text')).toBe(true); + expect(projected.progress).toEqual([]); + }); + + it.each([ + ['image', { image: false }], + ['audio', { audio: false }], + ['resource', { resource: false }], + ] as const)('fails closed and identifies denied %s content', async (kind, denied) => { + const error = await projectTargetCapabilities( + await renderRichContent(), + fixture({ ...denied, progress: false }), + ).catch((thrown: unknown) => thrown); + + expect(error).toBeInstanceOf(AgentTestError); + expect(error).toMatchObject({ code: 'unsupported-rich-content' }); + expect((error as AgentTestError).message).toContain(`offending kind: ${kind}`); + expect((error as AgentTestError).message).toContain('route-unit'); + }); + + it('records text as the immutable baseline capability', () => { + expect(fixture()).toMatchObject({ + audio: true, + image: true, + progress: true, + resource: true, + richContentFallback: 'fail', + text: true, + }); + }); +}); diff --git a/packages/agent-bundle/tests/test-harness-manifest.test.ts b/packages/agent-bundle/tests/test-harness-manifest.test.ts index 38a9dc359..259be6c86 100644 --- a/packages/agent-bundle/tests/test-harness-manifest.test.ts +++ b/packages/agent-bundle/tests/test-harness-manifest.test.ts @@ -413,6 +413,30 @@ describe('document matchers', () => { expect(() => expectDocument(document).toHaveError()).toThrow('no matching error'); expect(() => expectDocument(document).toHaveNodeKinds(['result'])).toThrow('node kinds differ'); }); + + it('matches image, audio, and resource fields without widening text assertions', () => { + const rich = Object.freeze({ + root: Object.freeze({ + children: Object.freeze([ + Object.freeze({ data: 'image-data', kind: 'image', mimeType: 'image/png' }), + Object.freeze({ data: 'audio-data', kind: 'audio', mimeType: 'audio/wav' }), + Object.freeze({ kind: 'resource', mimeType: 'application/octet-stream', name: 'data.bin', uri: 'data:application/octet-stream;base64,AAE=' }), + ]), + kind: 'result', + }), + status: 'success', + version: 1, + }) as never; + + expectDocument(rich) + .toContainImage({ data: 'image-data', mimeType: 'image/png' }) + .toContainAudio({ data: 'audio-data', mimeType: 'audio/wav' }) + .toContainResource({ mimeType: 'application/octet-stream', name: 'data.bin', uri: 'data:application/octet-stream;base64,AAE=' }); + expect(() => expectDocument(rich).toContainText('image-data')).toThrow('no text node'); + expect(() => expectDocument(rich).toContainImage({ mimeType: 'image/jpeg' })).toThrow('no image node matching'); + expect(() => expectDocument(rich).toContainAudio({ data: 'different' })).toThrow('no audio node matching'); + expect(() => expectDocument(rich).toContainResource({ name: 'different.bin' })).toThrow('no resource node matching'); + }); }); describe('render event matchers', () => { @@ -540,6 +564,7 @@ describe('the harness package boundary', () => { 'src/test/packed.ts', 'src/test/registry.ts', 'src/test/render.ts', + 'src/test/target-capabilities.ts', 'src/test/types.ts', ] .map(async (relativePath) => [relativePath, await readFile(resolve(packageRoot, relativePath), 'utf8')] as const),