diff --git a/.changeset/calm-schema-proof.md b/.changeset/calm-schema-proof.md new file mode 100644 index 000000000..e0b2dcfbf --- /dev/null +++ b/.changeset/calm-schema-proof.md @@ -0,0 +1,5 @@ +--- +"agent-bundle": patch +--- + +Expose `resultSchemaState` on `CompiledAgentRoute` and `RouteManifestRoute` so Workbench consumers distinguish absent, unknown, and unprojectable result schemas (#691) diff --git a/packages/agent-bundle/src/api.ts b/packages/agent-bundle/src/api.ts index 070756ff6..89dd7fde4 100644 --- a/packages/agent-bundle/src/api.ts +++ b/packages/agent-bundle/src/api.ts @@ -133,6 +133,7 @@ export type { RouteContract, RouteContractOrigin, RouteProvenance, + RouteResultSchemaState, } from './routes/types.ts'; export type { BuildResult } from './build/build.ts'; export type { PackageBuildResult, PackageOutputFile } from './build/package-build.ts'; diff --git a/packages/agent-bundle/src/contracts/routes.ts b/packages/agent-bundle/src/contracts/routes.ts index ca0c6af25..adf57e8da 100644 --- a/packages/agent-bundle/src/contracts/routes.ts +++ b/packages/agent-bundle/src/contracts/routes.ts @@ -16,6 +16,7 @@ export type { RouteManifestProvenance, RouteManifestProvider, RouteManifestResponse, + RouteManifestResultSchemaState, RouteManifestRoute, RouteManifestServer, RouteManifestServerMode, diff --git a/packages/agent-bundle/src/dev/index.ts b/packages/agent-bundle/src/dev/index.ts index 2b213595c..0a1d0aeff 100644 --- a/packages/agent-bundle/src/dev/index.ts +++ b/packages/agent-bundle/src/dev/index.ts @@ -100,6 +100,7 @@ export type { RouteManifestContract, RouteManifestProvider, RouteManifestResponse, + RouteManifestResultSchemaState, RouteManifestRoute, RouteManifestServer, } from './routes/route-manifest.ts'; diff --git a/packages/agent-bundle/src/dev/routes/application-tree.ts b/packages/agent-bundle/src/dev/routes/application-tree.ts index ad9bd1285..81fc7e7e6 100644 --- a/packages/agent-bundle/src/dev/routes/application-tree.ts +++ b/packages/agent-bundle/src/dev/routes/application-tree.ts @@ -6,6 +6,7 @@ import type { RouteManifestCliCommand, RouteManifestConfigEntry, RouteManifestKind, + RouteManifestResultSchemaState, RouteManifestRoute, } from './route-manifest.ts'; import { @@ -31,6 +32,7 @@ export interface ApplicationLeaf { readonly label: string; readonly preflight?: string; readonly ref: ApplicationNodeRef; + readonly resultSchemaState?: RouteManifestResultSchemaState; readonly routeId?: string; readonly source?: string; } @@ -177,6 +179,7 @@ const leafForRoute = ( label: routeLabel(ref), ...(route.execution?.preflight === undefined ? {} : { preflight: route.execution.preflight }), ref, + ...(route.resultSchemaState === undefined ? {} : { resultSchemaState: route.resultSchemaState }), routeId: route.id, source: route.source, }); diff --git a/packages/agent-bundle/src/dev/routes/route-invocation-result.ts b/packages/agent-bundle/src/dev/routes/route-invocation-result.ts index cdad02cb9..745d55ced 100644 --- a/packages/agent-bundle/src/dev/routes/route-invocation-result.ts +++ b/packages/agent-bundle/src/dev/routes/route-invocation-result.ts @@ -18,7 +18,7 @@ export interface RouteInvocation extends RouteInvocationSummary { readonly events: readonly AgentRenderEvent[]; readonly projection: RouteInvocationProjection; readonly providers: readonly RouteInvocationProvider[]; - /** The document value parsed by the route's own `resultSchema`; absent when the module exports none or rendering failed. */ + /** Structured value recorded by the selected surface; its presence alone proves neither a `resultSchema` declaration nor validation. */ readonly result?: JsonValue; /** Event-kernel phase events emitted by a compiled preflight execution. */ readonly trace?: readonly EventTraceEvent[]; diff --git a/packages/agent-bundle/src/dev/routes/route-manifest.ts b/packages/agent-bundle/src/dev/routes/route-manifest.ts index 193b6295c..3fe7527fd 100644 --- a/packages/agent-bundle/src/dev/routes/route-manifest.ts +++ b/packages/agent-bundle/src/dev/routes/route-manifest.ts @@ -12,6 +12,7 @@ import type { CompiledRouteGraph, CompiledServerMode, CompiledServerSurface, + RouteResultSchemaState, } from '../../routes/types.ts'; import type { ArtifactManifestCliCommand, @@ -64,8 +65,13 @@ export type RouteManifestContract = ArtifactManifestRouteContract; */ export interface RouteManifestRoute extends ArtifactManifestRoute { readonly config: readonly RouteManifestConfigEntry[]; + /** Static declaration/projection evidence; absent only on manifests from older dev servers. */ + readonly resultSchemaState?: RouteManifestResultSchemaState; } +/** Result schemas execute as authored, so a declaration is known without inventing a static schema projection. */ +export type RouteManifestResultSchemaState = RouteResultSchemaState; + /** One MCP server surface with the routes its packaging mode actually compiles. */ export interface RouteManifestServer { readonly id: string; @@ -158,6 +164,7 @@ const configSummary = (config: Readonly>): readonly Rout const manifestRoute = (route: CompiledAgentRoute): RouteManifestRoute => ({ ...artifactRouteFor(route), config: configSummary(route.config), + ...(route.resultSchemaState === undefined ? {} : { resultSchemaState: route.resultSchemaState }), }); const manifestServer = (server: CompiledServerSurface): RouteManifestServer => ({ diff --git a/packages/agent-bundle/src/routes/graph.ts b/packages/agent-bundle/src/routes/graph.ts index f0780d525..1524fc3eb 100644 --- a/packages/agent-bundle/src/routes/graph.ts +++ b/packages/agent-bundle/src/routes/graph.ts @@ -30,6 +30,7 @@ import { } from './config-extract.ts'; import { discoverEventRoutePreflight, + scanRouteModuleExports, type EventRoutePreflightDiscovery, validateEventRouteModuleContract, validateLayoutModuleContract, @@ -67,6 +68,7 @@ import { type CompiledServerSurface, type RouteContract, type RouteInputSchema, + type RouteResultSchemaState, } from './types.ts'; type ProjectIgnoreRules = Awaited>; @@ -573,6 +575,7 @@ const inlineContractIdOf = (route: CompiledAgentRoute): string => const compiledRoute = ( module: DiscoveredRouteModule, config: Readonly>, + resultSchemaState: RouteResultSchemaState, contract?: ContractBinding, preflight?: CompiledEventPreflight, ): CompiledAgentRoute => ({ @@ -584,6 +587,7 @@ const compiledRoute = ( kind: module.kind, ...(preflight === undefined ? {} : { preflight }), provenance: { kind: 'conventional', relativePath: module.relativePath }, + resultSchemaState, ...(module.serverName === undefined ? {} : { serverId: `mcp:${module.serverName}` }), source: module.source, }); @@ -612,6 +616,7 @@ interface ExtractedModuleMetadata { readonly inputSchema?: ExtractedInputSchema; readonly preflight?: CompiledEventPreflight; readonly preflightDiagnostics: readonly Diagnostic[]; + readonly resultSchemaState: RouteResultSchemaState; } const emptyExtractedRouteConfig: ExtractedRouteConfig = deepFreeze({ @@ -627,7 +632,7 @@ const extractedModuleMetadata = ( preflightDiscovery?: EventRoutePreflightDiscovery, ): ExtractedModuleMetadata => { if (moduleText === undefined) { - return { extracted: emptyExtractedRouteConfig, preflightDiagnostics: [] }; + return { extracted: emptyExtractedRouteConfig, preflightDiagnostics: [], resultSchemaState: 'unknown' }; } const extracted = extractRouteConfig(moduleText, module.relativePath, module.source, { projectRoot }); const inputSchema = extractInputSchema(moduleText, module.relativePath, { projectRoot, source: module.source }); @@ -648,6 +653,9 @@ const extractedModuleMetadata = ( ...(inputSchema === undefined ? {} : { inputSchema }), ...(preflight === undefined ? {} : { preflight }), preflightDiagnostics: discovery?.diagnostics ?? [], + resultSchemaState: scanRouteModuleExports(moduleText, module.relativePath, { source: module.source }).named.has('resultSchema') + ? 'unprojectable' + : 'absent', }; }; @@ -1019,6 +1027,7 @@ export const compileRouteGraph = async ( const route = compiledRoute( module, resolved.config, + metadata.resultSchemaState, metadata.inputSchema === undefined ? undefined : contractBindings.get(contractIdOf(metadata.inputSchema.origin)), metadata.preflight, ); diff --git a/packages/agent-bundle/src/routes/index.ts b/packages/agent-bundle/src/routes/index.ts index 568336a49..d51b74770 100644 --- a/packages/agent-bundle/src/routes/index.ts +++ b/packages/agent-bundle/src/routes/index.ts @@ -67,6 +67,7 @@ export type { RouteContract, RouteContractOrigin, RouteProvenance, + RouteResultSchemaState, } from './types.ts'; export { generateRouteTypes, routeTypesRelativePath, writeRouteTypes } from './typegen.ts'; export { diff --git a/packages/agent-bundle/src/routes/types.ts b/packages/agent-bundle/src/routes/types.ts index 48f6e5bf3..4b4533b8f 100644 --- a/packages/agent-bundle/src/routes/types.ts +++ b/packages/agent-bundle/src/routes/types.ts @@ -133,6 +133,12 @@ export interface RouteContract { readonly routes: readonly string[]; } +/** + * Static evidence for a route's `resultSchema`. Result schemas are executed + * as authored and are never reduced to the bounded input-schema projection. + */ +export type RouteResultSchemaState = 'absent' | 'unknown' | 'unprojectable'; + /** One conventional route module compiled into the immutable route graph. */ export interface CompiledAgentRoute { /** Statically extracted from the module's `export const config` declaration; {@link emptyRouteConfig} when absent or rejected. */ @@ -148,6 +154,8 @@ export interface CompiledAgentRoute { /** Static cheap gate; present only on event routes that declare a valid relative default re-export. */ readonly preflight?: CompiledEventPreflight; readonly provenance: RouteProvenance; + /** Omitted only by legacy or manually assembled graphs, where consumers must treat the declaration as unknown. */ + readonly resultSchemaState?: RouteResultSchemaState; /** The owning MCP server id (`mcp:`); MCP route kinds only. */ readonly serverId?: string; /** Absolute route module path. */ diff --git a/packages/agent-bundle/tests/route-manifest-routes.test.ts b/packages/agent-bundle/tests/route-manifest-routes.test.ts index f533b17a3..20b0b0cc1 100644 --- a/packages/agent-bundle/tests/route-manifest-routes.test.ts +++ b/packages/agent-bundle/tests/route-manifest-routes.test.ts @@ -229,6 +229,10 @@ it('projects a compiled graph into the browser manifest with project-relative so expect(route.source.startsWith('/')).toBe(false); expect(route.provenance).toEqual({ kind: 'conventional' }); } + const echo = manifest.servers.flatMap((server) => server.routes) + .find((route) => route.id === 'tool:harness/echo'); + expect(echo?.resultSchemaState).toBe('unprojectable'); + expect(manifest.events[0]?.resultSchemaState).toBe('absent'); expect(manifest.cli?.commands).toContainEqual(expect.objectContaining({ mcp: { confirm: false, server: 'harness', tool: 'echo' }, routeId: 'tool:harness/echo', diff --git a/packages/workbench/src/application/route-inspector.tsx b/packages/workbench/src/application/route-inspector.tsx index 8ca3dcaff..558c35cef 100644 --- a/packages/workbench/src/application/route-inspector.tsx +++ b/packages/workbench/src/application/route-inspector.tsx @@ -107,6 +107,66 @@ const Rows = ({ rows }: { readonly rows: readonly InspectorRow[] }): React.React const Empty = ({ children }: { readonly children: React.ReactNode }): React.ReactNode =>

{children}

; +const resultSchemaRows = ( + leaf: ApplicationLeaf, + invocation: RouteInvocation | undefined, +): readonly InspectorRow[] => { + const structuredResult = row( + 'Structured result', + invocation === undefined + ? 'Unavailable · the route has not run.' + : invocation.result === undefined + ? 'Unavailable · this invocation recorded no structured result.' + : 'Available · open Structured result.', + ); + const state = leaf.resultSchemaState ?? 'unknown'; + switch (state) { + case 'absent': + return [ + row('Declaration', 'Absent · no resultSchema export was observed.'), + row('Static projection', 'Not applicable · no resultSchema is declared.'), + row('Validation / transformation', 'Not applicable · no declared resultSchema can validate or transform this result.'), + structuredResult, + ]; + case 'unknown': + return [ + row('Declaration', 'Unknown · static declaration evidence is unavailable.'), + row('Static projection', 'Unknown · no static result-schema projection evidence is available.'), + row('Validation / transformation', 'Unknown · declaration evidence is unavailable.'), + structuredResult, + ]; + case 'unprojectable': { + const validatesResult = leaf.ref.kind === 'cli' + || leaf.ref.kind === 'prompt' + || leaf.ref.kind === 'resource' + || leaf.ref.kind === 'tool'; + const validation = !validatesResult + ? 'Not applicable · this route execution contract does not apply resultSchema.' + : invocation === undefined + ? 'Not run · invoke the route to observe validation or transformation.' + : invocation.status !== 'succeeded' + ? 'Not recorded · the invocation did not complete with a parsed result.' + : invocation.outcome?.kind !== 'success' + ? 'Not recorded · the invocation completed without a successful outcome.' + : invocation.surface.kind !== 'unit-render' + ? 'Unknown · this invocation surface did not report resultSchema validation or transformation.' + : invocation.result === undefined + ? 'Not recorded · execution returned no parsed resultSchema value.' + : 'Succeeded · execution recorded the value parsed by resultSchema.'; + return [ + row('Declaration', 'Declared · the compiler observed a resultSchema export.'), + row('Static projection', 'Unavailable · resultSchema is not statically projected.'), + row('Validation / transformation', validation), + structuredResult, + ]; + } + default: { + const exhaustive: never = state; + return exhaustive; + } + } +}; + const SourceTab = ({ backendKind, invocation, leaf }: Pick): React.ReactNode => <> The input schema is richer than the statically projectable grammar; the editor accepts raw JSON and the route validates during execution. :
{displayAgentDocumentValue(leaf.inputSchema)}
}

Result schema

- {invocation?.result === undefined - ? The result schema is not projected statically. A route that exports resultSchema shows its parsed value under Structured result after a run. - : This route exports a resultSchema; its parsed value is under Structured result.} + ; const ProvidersTab = ({ invocation }: Pick): React.ReactNode => { diff --git a/packages/workbench/src/routes/route-manifest-client.ts b/packages/workbench/src/routes/route-manifest-client.ts index 547ea9003..668f19911 100644 --- a/packages/workbench/src/routes/route-manifest-client.ts +++ b/packages/workbench/src/routes/route-manifest-client.ts @@ -117,6 +117,7 @@ const routeSchema: z.ZodType = z.strictObject({ inputSchema: inputSchema.optional(), kind: z.enum(['app', 'cli', 'event-route', 'prompt', 'resource', 'script', 'tool']), provenance: z.strictObject({ kind: z.literal('conventional') }), + resultSchemaState: z.enum(['absent', 'unknown', 'unprojectable']).optional(), serverId: z.string().optional(), source: z.string(), }); diff --git a/packages/workbench/tests/audiobook-curator.acceptance.e2e.test.ts b/packages/workbench/tests/audiobook-curator.acceptance.e2e.test.ts index 73d1504a4..b92b3de58 100644 --- a/packages/workbench/tests/audiobook-curator.acceptance.e2e.test.ts +++ b/packages/workbench/tests/audiobook-curator.acceptance.e2e.test.ts @@ -214,6 +214,11 @@ e2e('accepts the audiobook-curator Application workspace at 1440×900', { timeou await page.getByRole('tab', { name: 'Timings' }).click(); const timingRow = page.locator('.inspector-timings li').filter({ hasText: finalTiming.phase }); await expect(timingRow).toContainText(`${String(finalTiming.durationMs)} ms`); + await page.getByRole('tab', { name: 'Schema' }).click(); + const schemaPanel = page.getByRole('tabpanel', { name: 'Schema' }); + await expect(schemaPanel).toContainText('Declared · the compiler observed a resultSchema export.'); + await expect(schemaPanel).toContainText('Unknown · this invocation surface did not report resultSchema validation or transformation.'); + await expect(schemaPanel).toContainText('Available · open Structured result.'); const epochBeforeEdit = await readBuildEpoch(page); const markedSearch = healthySearch.replace( diff --git a/packages/workbench/tests/route-manifest-client.test.ts b/packages/workbench/tests/route-manifest-client.test.ts index 0e85fdab9..9918ff92a 100644 --- a/packages/workbench/tests/route-manifest-client.test.ts +++ b/packages/workbench/tests/route-manifest-client.test.ts @@ -88,6 +88,7 @@ const manifest = { }, kind: 'tool', provenance: { kind: 'conventional' }, + resultSchemaState: 'unprojectable', serverId: 'mcp:library', source: 'src/mcp/library/tools/echo.ts', }], @@ -142,6 +143,7 @@ it('reads the compiled manifest over the shared foreground session', async () => expect(decoded.servers[0]?.routes[0]?.contract).toBe( 'contract:src/lib/protocol-schemas.ts#statusInputSchema', ); + expect(decoded.servers[0]?.routes[0]?.resultSchemaState).toBe('unprojectable'); expect(decoded.events[0]?.execution).toEqual({ fallback: 'none', preflight: 'src/events/tool/after.preflight.ts', diff --git a/packages/workbench/tests/route-workspace.test.ts b/packages/workbench/tests/route-workspace.test.ts index 7adcd50da..069b9a55e 100644 --- a/packages/workbench/tests/route-workspace.test.ts +++ b/packages/workbench/tests/route-workspace.test.ts @@ -3,8 +3,10 @@ import { renderToStaticMarkup } from 'react-dom/server'; import { describe, expect, it } from '@rstest/core'; +import type { RouteInvocation } from '../../agent-bundle/src/contracts/invocations.ts'; import type { TraceEntry } from '../../agent-bundle/src/contracts/trace.ts'; import { appResourceUriFor, appToolCallRequest, catalogToolsFor, orderedToolsForApp } from '../src/application/app-route-workspace.tsx'; +import type { ApplicationLeaf } from '../src/application/application-tree-model.ts'; import { defaultEventHostSelection } from '../src/application/event-route-workspace.tsx'; import { ExecutableRouteWorkspace, resultTabFor } from '../src/application/executable-route-workspace.tsx'; import { idleInvocationState, reduceInvocationState, selectBackend } from '../src/application/invocation-model.ts'; @@ -29,6 +31,20 @@ import { const noop = (): void => undefined; +const schemaLeaf = (resultSchemaState: NonNullable): ApplicationLeaf => + ({ ...toolLeaf, resultSchemaState }); + +const renderSchemaInspector = (leaf: ApplicationLeaf, envelope?: RouteInvocation): string => + renderToStaticMarkup(createElement(RouteInspector, { + backendKind: 'dev-server', + ...(envelope === undefined ? {} : { invocation: envelope }), + leaf, + onTabChange: noop, + onToggle: noop, + open: true, + tab: 'schema', + })); + const controllerWith = (overrides: Partial = {}): RouteInvocationController => ({ backendKind: 'dev-server', cancel: noop, @@ -457,6 +473,102 @@ it('shows an explicit state when a deep-linked invocation is not in this session }); describe('RouteInspector', () => { + it('does not turn a structured result without a schema into declaration or validation evidence', () => { + const markup = renderSchemaInspector(schemaLeaf('absent'), invocation); + + expect(markup).toContain('Absent · no resultSchema export was observed.'); + expect(markup).toContain('Not applicable · no declared resultSchema can validate or transform this result.'); + expect(markup).toContain('Available · open Structured result.'); + expect(markup).not.toContain('This route exports a'); + }); + + it('separates a declared schema, successful validation, and structured-result availability', () => { + const markup = renderSchemaInspector(schemaLeaf('unprojectable'), { + ...invocation, + surface: { kind: 'unit-render' }, + }); + + expect(markup).toContain('Declared · the compiler observed a resultSchema export.'); + expect(markup).toContain('Succeeded · execution recorded the value parsed by resultSchema.'); + expect(markup).toContain('Available · open Structured result.'); + }); + + it('does not report successful validation for a non-success outcome that still carries a result', () => { + const failed: RouteInvocation = { + ...invocation, + outcome: { exitCode: 1, kind: 'process-exit' }, + surface: { args: [], command: 'audible search', kind: 'cli' }, + }; + const markup = renderSchemaInspector(schemaLeaf('unprojectable'), failed); + + expect(markup).toContain('Declared · the compiler observed a resultSchema export.'); + expect(markup).toContain('Not recorded · the invocation completed without a successful outcome.'); + expect(markup).toContain('Available · open Structured result.'); + expect(markup).not.toContain('Succeeded · execution recorded'); + }); + + it('reports a failed unit-render validation without inventing a parsed result', () => { + const { + outcome: _outcome, + result: _result, + ...failedBase + } = invocation; + const markup = renderSchemaInspector(schemaLeaf('unprojectable'), { + ...failedBase, + diagnostics: [{ + code: 'AB8236', + message: "The route's own resultSchema rejected the rendered document value.", + severity: 'error', + }], + status: 'failed', + surface: { kind: 'unit-render' }, + }); + + expect(markup).toContain('Not recorded · the invocation did not complete with a parsed result.'); + expect(markup).toContain('Unavailable · this invocation recorded no structured result.'); + expect(markup).not.toContain('Succeeded · execution recorded'); + }); + + it('keeps validation neutral when a successful production invocation carries a structured result', () => { + const markup = renderSchemaInspector(schemaLeaf('unprojectable'), invocation); + + expect(markup).toContain('Unknown · this invocation surface did not report resultSchema validation or transformation.'); + expect(markup).toContain('Available · open Structured result.'); + expect(markup).not.toContain('Succeeded · execution recorded'); + }); + + it('keeps an absent result separate from a declared schema', () => { + const { result: _result, ...withoutResult } = invocation; + const markup = renderSchemaInspector(schemaLeaf('unprojectable'), { + ...withoutResult, + surface: { kind: 'unit-render' }, + }); + + expect(markup).toContain('Declared · the compiler observed a resultSchema export.'); + expect(markup).toContain('Not recorded · execution returned no parsed resultSchema value.'); + expect(markup).toContain('Unavailable · this invocation recorded no structured result.'); + }); + + it('reports richer unsupported schemas as declared but statically unprojectable', () => { + const markup = renderSchemaInspector( + { ...schemaLeaf('unprojectable'), inputSchema: undefined }, + ); + + expect(markup).toContain('The input schema is richer than the statically projectable grammar'); + expect(markup).toContain('Declared · the compiler observed a resultSchema export.'); + expect(markup).toContain('Unavailable · resultSchema is not statically projected.'); + expect(markup).toContain('Not run · invoke the route to observe validation or transformation.'); + }); + + it('keeps unknown declaration evidence neutral even when a structured result exists', () => { + const markup = renderSchemaInspector(schemaLeaf('unknown'), invocation); + + expect(markup).toContain('Unknown · static declaration evidence is unavailable.'); + expect(markup).toContain('Unknown · declaration evidence is unavailable.'); + expect(markup).toContain('Available · open Structured result.'); + expect(markup).not.toContain('Declared ·'); + }); + it('stays closed by default and opens to the evidence tabs', () => { const closed = renderToStaticMarkup(createElement(RouteInspector, { backendKind: 'dev-server', diff --git a/packages/workbench/tests/support/workspace-fixtures.ts b/packages/workbench/tests/support/workspace-fixtures.ts index 9c6da5229..526c19ca2 100644 --- a/packages/workbench/tests/support/workspace-fixtures.ts +++ b/packages/workbench/tests/support/workspace-fixtures.ts @@ -51,6 +51,7 @@ export const toolLeaf: ApplicationLeaf = Object.freeze({ key: '/routes/mcp/curator/tool/search_audible', label: 'search_audible', ref: Object.freeze({ kind: 'tool' as const, name: 'search_audible', server: 'curator' }), + resultSchemaState: 'unprojectable', routeId: 'tool:curator/search_audible', source: 'src/mcp/curator/tools/search_audible.tsx', });