From f76e6c85b51176088b94607fd6bf1e3ce1d16dee Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Thu, 3 Sep 2026 08:33:36 +0000 Subject: [PATCH 1/3] fix(test): cover MCP App routes in the packed contract matrix without hand-enumerated fixtures (#401) App routes are covered at every boundary that registers app resources (packed-stdio, packed-deleted-source, host-install, dev-epoch) and auto-covered with the new default `apps: 'auto'`; `apps: 'explicit'` restores the fixture requirement. `{ kind: 'resource' }` names a resource/app fixture explicitly (legacy `{}` still accepted) and is a coverage failure on tool/prompt routes. mcp-in-memory keeps apps not-applicable. The cancellation check now tracks whether the abort fired before settlement and reports not-applicable ("invocation completed before abort; use an input that stays in flight") instead of failed. --- .../401-contract-matrix-app-coverage.md | 5 + packages/agent-bundle/README.md | 26 +- .../agent-bundle/src/config/dev-contracts.ts | 5 +- packages/agent-bundle/src/test/contract.ts | 138 +++++++++- packages/agent-bundle/src/test/index.ts | 17 +- .../tests/packed-stdio-projection.test.ts | 8 + .../tests/projection/contract-matrix.test.ts | 244 ++++++++++++++++++ .../tests/support/contract-matrix-fixtures.ts | 5 +- 8 files changed, 427 insertions(+), 21 deletions(-) create mode 100644 .changeset/401-contract-matrix-app-coverage.md diff --git a/.changeset/401-contract-matrix-app-coverage.md b/.changeset/401-contract-matrix-app-coverage.md new file mode 100644 index 000000000..b21844cc1 --- /dev/null +++ b/.changeset/401-contract-matrix-app-coverage.md @@ -0,0 +1,5 @@ +--- +"agent-bundle": minor +--- + +Make the contract matrix's MCP App coverage match its docs (#401). App routes are covered at every boundary that registers app resources (`packed-stdio`, `packed-deleted-source`, `host-install`, `dev-epoch`): `surface-completeness` requires the compiled `ui://` URI and `sweep` reads it. With the new `apps: 'auto'` option (the default) an app route no longer needs a fixture entry — `coverage` passes with a reason naming the auto-covered sweep — and `apps: 'explicit'` restores the requirement. `ContractRouteFixture` gains an optional `kind: 'resource'` discriminator (exported as `ContractResourceFixture`) so `{ kind: 'resource' }` names a resource/app fixture explicitly; legacy `{}` is still accepted, and a resource fixture on a tool or prompt route is a coverage failure. `mcp-in-memory` still reports apps as `not-applicable`. The `cancellation` check now tracks whether the abort fired before the invocation settled: an invocation that completes (or rejects) before `abortAfterMs` is `not-applicable` with "invocation completed before abort; use an input that stays in flight" instead of `failed`; it fails only when the abort was delivered mid-flight and the call still settled without rejecting. diff --git a/packages/agent-bundle/README.md b/packages/agent-bundle/README.md index 881244620..b48325058 100644 --- a/packages/agent-bundle/README.md +++ b/packages/agent-bundle/README.md @@ -487,8 +487,9 @@ the current schema, rejection of negative inputs derived from the advertised `listTools` input JSON Schema, and mid-flight cancellation hygiene. In-memory transport may pass structured values without serialization; the matrix closes that gap with an explicit `JSON.parse(JSON.stringify(...))` round-trip before -validation. MCP Apps are reported as not-applicable for surface registration -because the in-memory level does not register them. +validation. MCP Apps are not registered at this level: every app route reports +`surface-completeness` as `not-applicable` and receives no coverage or sweep +check, and app fixture entries are accepted and ignored. **`runPackedContractMatrix` (`packed-stdio` / `packed-deleted-source`)** runs against an already-open packed session (the single packed journey owns session @@ -502,6 +503,24 @@ version-skew (including their per-lifecycle-phase variants) are reported server validates every tool result through its bundled `resultSchema` before returning; a successful sweep invocation is that evidence. +**MCP App coverage per level.** Fixtures must cover every compiled tool, prompt, +and resource route on the server. App routes are covered at every boundary that +registers app resources — `packed-stdio`, `packed-deleted-source`, +`host-install`, and `dev-epoch` — where `surface-completeness` requires the +compiled `ui://` URI in `listResources` and `sweep` reads that resource. With +the default `apps: 'auto'` an app route needs no fixture entry: `coverage` +passes with a reason naming the auto-covered sweep. An explicit +`{ kind: 'resource' }` (or legacy `{}`) entry is always accepted, and +`apps: 'explicit'` makes a missing app entry a `coverage` failure again. At +`mcp-in-memory` apps are never registered, so `apps` has no effect there. + +The `cancellation` fixture aborts the invocation after `abortAfterMs` +(default 50ms) and requires it to settle rejected. Its input must stay in +flight past that point: when the invocation settles before the abort fires +the check is `not-applicable` ("invocation completed before abort; use an +input that stays in flight"), and it only `fails` when the abort was delivered +mid-flight and the call still settled without rejecting. + Lifecycle fixtures replay `unknown → queued → running → first-progress → repeated-progress → terminal` over the matrix's one open client. The framework validates every phase's @@ -560,11 +579,12 @@ await runContractMatrix({ }); // Packed: pass an already-open session and a manifest compiled before source removal. +// App routes are auto-covered here; `{ kind: 'resource' }` names one explicitly. await runPackedContractMatrix({ eventRuntime: { endpointId: packedEventRuntimeEndpointId }, session: packedSession, manifest: compiledManifest, - fixtures: { /* same shape */ }, + fixtures: { /* same shape, optionally 'app:library/dashboard': { kind: 'resource' } */ }, }); await using installedSession = await openInstalledHostMcpServer({ diff --git a/packages/agent-bundle/src/config/dev-contracts.ts b/packages/agent-bundle/src/config/dev-contracts.ts index 8d4865059..fc53c49e1 100644 --- a/packages/agent-bundle/src/config/dev-contracts.ts +++ b/packages/agent-bundle/src/config/dev-contracts.ts @@ -131,9 +131,12 @@ const validateFixture = (value: unknown, routeId: string): ContractRouteFixture if (!isRecord(value)) return invalid(`Fixture ${JSON.stringify(routeId)} must be an object.`); fields( value, - ['cancellation', 'input', 'inputs', 'lifecycle', 'previousResults', 'resultCompat'], + ['cancellation', 'input', 'inputs', 'kind', 'lifecycle', 'previousResults', 'resultCompat'], `Fixture ${JSON.stringify(routeId)}`, ); + if (value.kind !== undefined && value.kind !== 'resource') { + return invalid(`Fixture ${JSON.stringify(routeId)} kind must be "resource" when provided.`); + } if (value.resultCompat !== undefined && value.resultCompat !== 'additive' && value.resultCompat !== 'closed') { return invalid(`Fixture ${JSON.stringify(routeId)} resultCompat must be "additive" or "closed".`); } diff --git a/packages/agent-bundle/src/test/contract.ts b/packages/agent-bundle/src/test/contract.ts index 01e85453c..0b3ecd410 100644 --- a/packages/agent-bundle/src/test/contract.ts +++ b/packages/agent-bundle/src/test/contract.ts @@ -15,7 +15,8 @@ * behavior, and previous-server payload acceptance. In-memory transport may * pass structured values without serialization; the explicit * `JSON.parse(JSON.stringify(...))` round-trip closes that gap. MCP Apps are - * not registered at this level. + * not registered at this level: every app route reports `surface-completeness` + * as `not-applicable` and receives no coverage, sweep, or other check. * * **`runPackedContractMatrix` (`packed-stdio` / `packed-deleted-source`)** runs * against an already-open packed session. It proves process stdio evidence for @@ -27,6 +28,15 @@ * every tool result through its bundled `resultSchema` before returning; a * successful sweep invocation is that evidence. * + * **MCP App coverage.** At every boundary that registers app resources + * (`packed-stdio`, `packed-deleted-source`, `host-install`, `dev-epoch`) app + * routes ARE part of the matrix: `surface-completeness` requires the compiled + * `ui://` URI in `listResources`, and `sweep` reads that resource. With the + * default `apps: 'auto'` an app route needs no fixture entry — `coverage` + * passes with a reason naming the auto-covered sweep. An explicit + * `{ kind: 'resource' }` (or legacy `{}`) fixture is always accepted; + * `apps: 'explicit'` restores the requirement that every app route be listed. + * * Stateful lifecycle fixtures replay over one open client at every boundary. * Same-store restart callbacks add boundary-local durability evidence; a run * without one reports restart durability as not-applicable. @@ -125,7 +135,22 @@ export interface ContractLifecycleFixture { readonly transitionDriver: () => readonly ContractLifecycleTransition[]; } +/** + * The explicit fixture form for a resource or MCP App route: the sweep reads + * the route's wire URI and no input, policy, or lifecycle applies. Declaring + * it on a tool or prompt route is a coverage failure. + */ +export interface ContractResourceFixture { + readonly kind: 'resource'; +} + export interface ContractRouteFixture { + /** + * `'resource'` marks a resource/MCP App fixture (see `ContractResourceFixture`). + * Omit it for tool and prompt fixtures; a legacy `{}` still covers a + * resource or app route. + */ + readonly kind?: ContractResourceFixture['kind']; /** Valid input for the sweep invocation (tools/prompts; resources need none). */ readonly input?: unknown; /** Additional valid inputs — e.g. one per declared status/discriminant value. */ @@ -139,13 +164,28 @@ export interface ContractRouteFixture { readonly previousResults?: readonly unknown[]; /** * Cancellation case: invocation aborted mid-flight must settle rejected and - * leave the session usable. + * leave the session usable. The input must stay in flight past + * `abortAfterMs` (default 50ms); an invocation that settles before the abort + * fires is reported `not-applicable`, not `failed`. */ readonly cancellation?: { readonly abortAfterMs?: number; readonly input?: unknown }; /** Optional stateful replay over this matrix run's single open client. */ readonly lifecycle?: ContractLifecycleFixture; } +/** + * How MCP App routes are covered at boundaries that register app resources. + * + * - `'auto'` (default): app routes need no fixture entry; `coverage` passes + * and the sweep reads the compiled `ui://` resource. Explicit entries are + * still accepted. + * - `'explicit'`: every compiled app route must have a fixture entry + * (`{ kind: 'resource' }`), matching the rule for every other route kind. + * + * Has no effect at `mcp-in-memory`, where apps are never registered. + */ +export type ContractAppCoverage = 'auto' | 'explicit'; + export type ContractMatrixClient = Pick< Client, 'callTool' | 'getPrompt' | 'listPrompts' | 'listResources' | 'listTools' | 'readResource' @@ -158,8 +198,14 @@ export interface ContractMatrixRestartSession { export interface ContractMatrixOptions extends InMemoryMcpSessionOptions { readonly manifest?: AgentBundleTestManifest; readonly server?: string; - /** Route id -> fixture. Every compiled non-app route on the server must be covered. */ + /** + * Route id -> fixture. Every compiled tool, prompt, and resource route on + * the server must be covered. App routes are not registered at + * `mcp-in-memory`; entries for them are accepted and ignored. + */ readonly fixtures: Readonly>; + /** Accepted for parity with the other entry points; apps are never registered here. */ + readonly apps?: ContractAppCoverage; /** Reopens the same durable store after the matrix closes its initial in-memory session. */ readonly restart?: () => Promise; } @@ -192,8 +238,15 @@ export type ContractEventRuntimeAddress = | { readonly endpoint?: never; readonly endpointId: string }; export interface PackedContractMatrixOptions { + /** App route coverage at this boundary; defaults to `'auto'`. */ + readonly apps?: ContractAppCoverage; /** Read-only status address for the generated event runtime, when this artifact has one. */ readonly eventRuntime?: ContractEventRuntimeAddress; + /** + * Route id -> fixture. Every compiled tool, prompt, and resource route on + * the server must be covered; app routes are auto-covered unless + * `apps: 'explicit'`. + */ readonly fixtures: Readonly>; readonly manifest: AgentBundleTestManifest; readonly server?: string; @@ -217,6 +270,8 @@ export interface DevEpochContractMatrixSession { } export interface DevEpochContractMatrixOptions { + /** App route coverage at this boundary; defaults to `'auto'`. */ + readonly apps?: ContractAppCoverage; readonly fixtures: Readonly>; readonly manifest: AgentBundleTestManifest; readonly server?: string; @@ -225,6 +280,8 @@ export interface DevEpochContractMatrixOptions { } export interface InstalledHostContractMatrixOptions { + /** App route coverage at this boundary; defaults to `'auto'`. */ + readonly apps?: ContractAppCoverage; readonly fixtures: Readonly>; readonly manifest: AgentBundleTestManifest; readonly server?: string; @@ -1027,9 +1084,16 @@ const runCancellation = async ( signal: controller.signal, timeout: settleWithinMs, }); - const timer = setTimeout(() => controller.abort(), abortAfterMs); + // The timer and the settlement race; `abortFired` is read the moment the + // call settles so the verdict reflects whether the abort was ever delivered + // while the invocation was still in flight. + let abortFired = false; + const timer = setTimeout(() => { + abortFired = true; + controller.abort(); + }, abortAfterMs); const result = await Promise.race([ - call.then((settled) => ({ kind: 'settled' as const, settled })), + call.then((settled) => ({ abortedInFlight: abortFired, kind: 'settled' as const, settled })), new Promise<{ kind: 'timeout' }>((resolve) => { setTimeout(() => resolve({ kind: 'timeout' }), settleWithinMs); }), @@ -1038,6 +1102,13 @@ const runCancellation = async ( if (result.kind === 'timeout') { return failed(`callTool did not settle within ${String(settleWithinMs)}ms after abort`); } + if (!result.abortedInFlight) { + return notApplicable( + result.settled.threw + ? `invocation rejected before abort (${result.settled.error instanceof Error ? result.settled.error.message : captured(result.settled.error)}); use an input that stays in flight past ${String(abortAfterMs)}ms.` + : `invocation completed before abort; use an input that stays in flight past ${String(abortAfterMs)}ms.`, + ); + } if (!result.settled.threw) { return failed('aborted callTool settled without throwing or rejecting.'); } @@ -1387,7 +1458,50 @@ const finalizeContractMatrixReport = ( ); }; +const AUTO_APP_FIXTURE: ContractResourceFixture = Object.freeze({ kind: 'resource' }); + +const APP_AUTO_COVERAGE_REASON = + 'app route auto-covered (apps: "auto"): the sweep reads its compiled MCP App resource URI.'; + +interface ResolvedRouteFixture { + readonly coverage: ContractCheckOutcome; + readonly fixture: ContractRouteFixture | undefined; +} + +const resolveRouteFixture = ( + descriptor: TestableRouteDescriptor, + fixture: ContractRouteFixture | undefined, + apps: ContractAppCoverage, +): ResolvedRouteFixture => { + if (fixture === undefined) { + if (descriptor.kind !== 'app') { + return { coverage: failed('compiled route has no fixture entry'), fixture }; + } + switch (apps) { + case 'auto': + return { coverage: passedWithReason(APP_AUTO_COVERAGE_REASON), fixture: AUTO_APP_FIXTURE }; + case 'explicit': + return { + coverage: failed('compiled app route has no fixture entry (apps: "explicit"); add { kind: "resource" } or use apps: "auto"'), + fixture, + }; + default: { + const exhaustive: never = apps; + return { coverage: failed(`unsupported apps coverage mode ${String(exhaustive)}`), fixture: undefined }; + } + } + } + if (fixture.kind === 'resource' && descriptor.kind !== 'resource' && descriptor.kind !== 'app') { + return { + coverage: failed(`fixture kind "resource" declared for a ${descriptor.kind} route; resource fixtures apply to resource and app routes only`), + fixture: undefined, + }; + } + return { coverage: passed(), fixture }; +}; + const executeContractMatrix = async (options: { + readonly apps: ContractAppCoverage; readonly boundary: MatrixBoundaryCapabilities; readonly client: ContractMatrixClient; readonly fixtures: Readonly>; @@ -1395,7 +1509,7 @@ const executeContractMatrix = async (options: { readonly provenance: ContractMatrixProvenance; readonly serverName: string; }): Promise => { - const { boundary, client, fixtures, manifest, provenance, serverName } = options; + const { apps, boundary, client, fixtures, manifest, provenance, serverName } = options; const failures: ContractMatrixFailure[] = []; const matrixChecks: Record = {}; const routeReports: Record = {}; @@ -1443,15 +1557,13 @@ const executeContractMatrix = async (options: { } const checks: Record = {}; - const fixture = fixtures[descriptor.id]; + const { coverage, fixture } = resolveRouteFixture(descriptor, fixtures[descriptor.id], apps); checks[CHECK_COVERAGE] = outcomeFromCheck( failures, descriptor.id, CHECK_COVERAGE, - fixture === undefined - ? failed('compiled route has no fixture entry') - : passed(), + coverage, ); checks[CHECK_SURFACE] = outcomeFromCheck( @@ -1731,6 +1843,7 @@ export const runContractMatrix = async ( }, }); return await executeContractMatrix({ + apps: options.apps ?? 'auto', boundary, client: session.client, fixtures: options.fixtures, @@ -1753,6 +1866,7 @@ export const runDevEpochContractMatrix = async ( ): Promise => { const serverName = resolveServerName(options.manifest, options.server); return executeContractMatrix({ + apps: options.apps ?? 'auto', boundary: DEV_EPOCH_BOUNDARY, client: options.session.client, fixtures: options.fixtures, @@ -1765,12 +1879,15 @@ export const runDevEpochContractMatrix = async ( /** * Runs the contract matrix against an already-open packed stdio session. * Never opens or closes the session; stamps the session's own proof level. + * Compiled MCP App routes are covered here (surface + `ui://` sweep) and + * auto-covered without a fixture entry unless `apps: 'explicit'`. */ export const runPackedContractMatrix = async ( options: PackedContractMatrixOptions, ): Promise => { const serverName = resolveServerName(options.manifest, options.server); return executeContractMatrix({ + apps: options.apps ?? 'auto', boundary: packedBoundaryFromSession( options.session, options.eventRuntime, @@ -1794,6 +1911,7 @@ export const runInstalledHostContractMatrix = async ( ): Promise => { const serverName = resolveServerName(options.manifest, options.server); const matrix = await executeContractMatrix({ + apps: options.apps ?? 'auto', boundary: installedHostBoundaryFromSession(options.session), client: options.session.client, fixtures: options.fixtures, diff --git a/packages/agent-bundle/src/test/index.ts b/packages/agent-bundle/src/test/index.ts index 0209d9b26..d6d94fb30 100644 --- a/packages/agent-bundle/src/test/index.ts +++ b/packages/agent-bundle/src/test/index.ts @@ -9,14 +9,19 @@ * | level | helper | what it proves | * | --- | --- | --- | * | `route-unit` | `renderRoute`, `renderRouteEvents`, `createTargetCapabilityFixture`, `projectTargetCapabilities` | the route component and its document through the real Agent renderer; explicit target-capability projection through the real MCP projector, without transport or host proof | - * | `mcp-in-memory` | `openInMemoryMcpServer`, `invokeMcpTool`, `readMcpResource`, `getMcpPrompt`, `listMcpSurface`, `runContractMatrix` | the real generated MCP server's protocol contract, over the SDK's in-memory transport | - * | `dev-epoch` | `runDevEpochContractMatrix` | an epoch-pinned generated stdio process opened through the Workbench session service | + * | `mcp-in-memory` | `openInMemoryMcpServer`, `invokeMcpTool`, `readMcpResource`, `getMcpPrompt`, `listMcpSurface`, `runContractMatrix` | the real generated MCP server's protocol contract, over the SDK's in-memory transport; MCP App routes are not registered and report `not-applicable` | + * | `dev-epoch` | `runDevEpochContractMatrix` | an epoch-pinned generated stdio process opened through the Workbench session service; MCP App routes are covered (surface + `ui://` sweep) and auto-covered without a fixture | * | `cli-dispatch` | `invokeCli`, `cliJson`, `cliNdjson` | a compiled plain or rendered CLI command dispatched through the routed CLI's own shell, including rendered output modes, in this process | - * | `packed-stdio` | `openPackedMcpServer`, `runPackedContractMatrix` | a built artifact's generated entry running as a real process over stdio | - * | `packed-deleted-source` | `removeProjectSource`, `openPackedMcpServer({ deletedSource })`, `runPackedContractMatrix` | the packed stdio process still runs after project source and configuration are removed and verified absent | + * | `packed-stdio` | `openPackedMcpServer`, `runPackedContractMatrix` | a built artifact's generated entry running as a real process over stdio; MCP App routes are covered (surface + `ui://` sweep) and auto-covered without a fixture | + * | `packed-deleted-source` | `removeProjectSource`, `openPackedMcpServer({ deletedSource })`, `runPackedContractMatrix` | the packed stdio process still runs after project source and configuration are removed and verified absent; MCP App routes are covered as at `packed-stdio` | * | `browser-app` | `mountBrowserApp` (`agent-bundle/test/browser`) | production-compiled MCP App HTML mounted over the product bridge in a real browser page | * | `simulated` | `openInstalledHostMcpServer` without `sessionEvidence` | an emitted bundle staged directly into an isolated host-shaped root and spawned without a host-owned install | - * | `host-install` | `openInstalledHostMcpServer`, `runInstalledHostContractMatrix` | a built bundle staged into an isolated host root, discovered in the host's emitted format, and spawned from the installed layout | + * | `host-install` | `openInstalledHostMcpServer`, `runInstalledHostContractMatrix` | a built bundle staged into an isolated host root, discovered in the host's emitted format, and spawned from the installed layout; MCP App routes are covered as at `packed-stdio` | + * + * Contract-matrix fixtures cover every compiled tool, prompt, and resource + * route. App routes are auto-covered (`apps: 'auto'`, the default) wherever + * they are registered; `{ kind: 'resource' }` names one explicitly and + * `apps: 'explicit'` requires that for every app route. * * A pass at one level is never a receipt for another. The `deletedSource` * option upgrades `openPackedMcpServer` provenance only after every path in a @@ -97,9 +102,11 @@ export { runPackedContractMatrix, } from './contract.ts'; export type { + ContractAppCoverage, ContractCheckOutcome, ContractCheckStatus, ContractEventRuntimeAddress, + ContractResourceFixture, ContractLifecycleFixture, ContractLifecyclePhase, ContractLifecycleTransition, diff --git a/packages/agent-bundle/tests/packed-stdio-projection.test.ts b/packages/agent-bundle/tests/packed-stdio-projection.test.ts index 0662f2e44..4cadf0ace 100644 --- a/packages/agent-bundle/tests/packed-stdio-projection.test.ts +++ b/packages/agent-bundle/tests/packed-stdio-projection.test.ts @@ -220,9 +220,17 @@ it('serves compiled routes and durable state across packed process restarts', as session: firstSession, }); expect(matrixReport.provenance.proofLevel).toBe('packed-deleted-source'); + // No fixture names the app route: the packed level auto-covers it (#401). + expect(matrixReport.routes['app:harness/panel']?.checks.coverage).toEqual({ + reason: expect.stringContaining('auto-covered'), + status: 'passed', + }); expect(matrixReport.routes['app:harness/panel']?.checks['surface-completeness']).toEqual({ status: 'passed', }); + expect(matrixReport.routes['app:harness/panel']?.checks.sweep).toEqual({ + status: 'passed', + }); expect(matrixReport.routes['tool:harness/lifecycle']?.checks['restart-durability']).toEqual({ status: 'passed', }); diff --git a/packages/agent-bundle/tests/projection/contract-matrix.test.ts b/packages/agent-bundle/tests/projection/contract-matrix.test.ts index 7914a0a76..34520377d 100644 --- a/packages/agent-bundle/tests/projection/contract-matrix.test.ts +++ b/packages/agent-bundle/tests/projection/contract-matrix.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { describe, expect, it } from '@rstest/core'; +import type { Client } from '@modelcontextprotocol/client'; import { createSqliteStateDriver } from '@agent-bundle/runtime/state/sqlite'; import stateDefinition from '../../fixtures/route-harness/src/state.ts'; @@ -12,6 +13,7 @@ import { compileTestManifest, MCP_IN_MEMORY_PROOF_LEVEL, proofLevelLabel, + type AgentBundleTestManifest, } from '../../src/test/manifest.ts'; import { runContractMatrix, @@ -60,6 +62,108 @@ const withStatefulMatrix = async ( } }; +const PANEL_URI = 'ui://harness/panel'; +const PANEL_MIME = 'text/html;profile=mcp-app'; + +/** + * Stands in for the packed server's inline MCP App registry: the in-memory + * level never registers app resources, so a packed-shaped session over it + * needs the compiled `ui://` surface added on the client side. Every other + * call, and the progress-handler map the lifecycle replay composes, delegates + * to the real client. + */ +const withAppSurface = (client: Client): Client => { + const panelListing = { mimeType: PANEL_MIME, name: 'panel', uri: PANEL_URI }; + const wrapper = { + _notificationHandlers: (client as unknown as { readonly _notificationHandlers: unknown }) + ._notificationHandlers, + callTool: (...arguments_: Parameters) => client.callTool(...arguments_), + getPrompt: (...arguments_: Parameters) => client.getPrompt(...arguments_), + listPrompts: (...arguments_: Parameters) => client.listPrompts(...arguments_), + listResources: async (...arguments_: Parameters) => { + const listed = await client.listResources(...arguments_); + return { ...listed, resources: [...listed.resources, panelListing] }; + }, + listTools: (...arguments_: Parameters) => client.listTools(...arguments_), + readResource: async (...arguments_: Parameters) => { + const [params] = arguments_; + if (params.uri !== PANEL_URI) return client.readResource(...arguments_); + return { contents: [{ mimeType: PANEL_MIME, text: 'route-harness panel', uri: PANEL_URI }] }; + }, + }; + return wrapper as unknown as Client; +}; + +/** Drops the abort signal from `callTool` for one tool so the abort is never delivered to the server. */ +const ignoringAbortFor = (toolName: string) => (client: Client): Client => { + const wrapper = { + _notificationHandlers: (client as unknown as { readonly _notificationHandlers: unknown }) + ._notificationHandlers, + callTool: (...arguments_: Parameters) => { + const [params, options] = arguments_; + if (params.name !== toolName || options === undefined) return client.callTool(...arguments_); + const withoutSignal = { ...options }; + delete withoutSignal.signal; + return client.callTool(params, withoutSignal); + }, + getPrompt: (...arguments_: Parameters) => client.getPrompt(...arguments_), + listPrompts: (...arguments_: Parameters) => client.listPrompts(...arguments_), + listResources: (...arguments_: Parameters) => client.listResources(...arguments_), + listTools: (...arguments_: Parameters) => client.listTools(...arguments_), + readResource: (...arguments_: Parameters) => client.readResource(...arguments_), + }; + return wrapper as unknown as Client; +}; + +/** + * Opens the route-harness server in memory and presents it as a packed-stdio + * session so the shared matrix runs with `registersAppResources: true`. + */ +const withPackedShapedSession = async ( + options: { + readonly decorate?: (client: Client) => Client; + readonly entry: string; + readonly includeApps: boolean; + }, + body: (session: PackedMcpSession, manifest: AgentBundleTestManifest) => Promise, +): Promise => { + const root = await mkdtemp(join(tmpdir(), 'agent-bundle-contract-packed-shaped-')); + const compiledManifest = await compileTestManifest({ root: fixtureRoot }); + const manifest = options.includeApps + ? compiledManifest + : Object.freeze({ + ...compiledManifest, + apps: Object.freeze({}), + routes: Object.freeze(Object.fromEntries( + Object.entries(compiledManifest.routes).filter(([, route]) => route.kind !== 'app'), + )), + }); + const session = await openInMemoryMcpServer({ + manifest, + state: { + definition: stateDefinition, + driver: createSqliteStateDriver({ root }), + }, + }); + const packedSession: PackedMcpSession = Object.freeze({ + client: options.decorate === undefined ? session.client : options.decorate(session.client), + close: session.close, + provenance: Object.freeze({ + entry: options.entry, + pid: undefined, + proofLevel: 'packed-stdio' as const, + }), + stderr: () => '', + [Symbol.asyncDispose]: session[Symbol.asyncDispose], + }); + try { + return await body(packedSession, manifest); + } finally { + await session.close(); + await rm(root, { force: true, recursive: true }); + } +}; + describe('the generated-plugin contract matrix', () => { it('passes every check for the route-harness server at mcp-in-memory', async () => { const report = await withStatefulMatrix(routeHarnessContractFixtures(), (options) => @@ -481,6 +585,146 @@ describe('the generated-plugin contract matrix', () => { expect((error as AgentTestError).message).toContain(proofLabel); }, 30_000); + it('auto-covers a compiled app route at the packed level without a fixture entry', async () => { + const fixtures = routeHarnessContractFixtures(); + expect(fixtures['app:harness/panel']).toBeUndefined(); + + const report = await withPackedShapedSession( + { decorate: withAppSurface, entry: 'in-memory app auto-coverage fixture', includeApps: true }, + (session, manifest) => runPackedContractMatrix({ fixtures, manifest, server: 'harness', session }), + ); + + expect(report.provenance.proofLevel).toBe('packed-stdio'); + expect(report.routes['app:harness/panel']?.checks).toMatchObject({ + coverage: { reason: expect.stringContaining('auto-covered'), status: 'passed' }, + 'surface-completeness': { status: 'passed' }, + sweep: { status: 'passed' }, + }); + expect(report.routes['app:harness/panel']?.checks.cancellation).toEqual({ + reason: 'applies to tool routes only.', + status: 'not-applicable', + }); + }, 30_000); + + it('accepts an explicit { kind: "resource" } fixture for app and resource routes at the packed level', async () => { + const fixtures = { + ...routeHarnessContractFixtures(), + 'app:harness/panel': { kind: 'resource' as const }, + 'resource:harness/notes': { kind: 'resource' as const }, + }; + + const report = await withPackedShapedSession( + { decorate: withAppSurface, entry: 'in-memory explicit resource fixture', includeApps: true }, + (session, manifest) => runPackedContractMatrix({ + apps: 'explicit', + fixtures, + manifest, + server: 'harness', + session, + }), + ); + + expect(report.routes['app:harness/panel']?.checks).toMatchObject({ + coverage: { status: 'passed' }, + 'surface-completeness': { status: 'passed' }, + sweep: { status: 'passed' }, + }); + expect(report.routes['resource:harness/notes']?.checks).toMatchObject({ + coverage: { status: 'passed' }, + sweep: { status: 'passed' }, + }); + }, 30_000); + + it('requires an app fixture entry only when apps: "explicit" is requested', async () => { + const error = await withPackedShapedSession( + { decorate: withAppSurface, entry: 'in-memory explicit app coverage fixture', includeApps: true }, + (session, manifest) => runPackedContractMatrix({ + apps: 'explicit', + fixtures: routeHarnessContractFixtures(), + manifest, + server: 'harness', + session, + }).catch((thrown: unknown) => thrown), + ); + + expect(error).toBeInstanceOf(AgentTestError); + expect((error as AgentTestError).code).toBe('contract-violation'); + expect((error as AgentTestError).message).toContain('app:harness/panel / coverage'); + expect((error as AgentTestError).message).toContain('apps: "explicit"'); + expect((error as AgentTestError).message).toContain(proofLevelLabel('packed-stdio')); + }, 30_000); + + it('keeps app routes not-applicable at mcp-in-memory regardless of app fixtures', async () => { + const report = await withStatefulMatrix({ + ...routeHarnessContractFixtures(), + 'app:harness/panel': { kind: 'resource' as const }, + 'resource:harness/notes': { kind: 'resource' as const }, + }, (options) => runContractMatrix({ ...options, apps: 'explicit' })); + + expect(report.routes['app:harness/panel']?.checks).toEqual({ + 'surface-completeness': { + reason: 'MCP Apps are not registered by the in-memory projection level.', + status: 'not-applicable', + }, + }); + expect(report.routes['resource:harness/notes']?.checks).toMatchObject({ + coverage: { status: 'passed' }, + sweep: { status: 'passed' }, + }); + }, 30_000); + + it('rejects a { kind: "resource" } fixture declared for a tool route', async () => { + const error = await withStatefulMatrix({ + ...routeHarnessContractFixtures(), + 'tool:harness/echo': { kind: 'resource' as const }, + }, (options) => runContractMatrix(options).catch((thrown: unknown) => thrown)); + + expect(error).toBeInstanceOf(AgentTestError); + expect((error as AgentTestError).code).toBe('contract-violation'); + expect((error as AgentTestError).message).toContain('tool:harness/echo / coverage'); + expect((error as AgentTestError).message).toContain('resource fixtures apply to resource and app routes only'); + }, 30_000); + + it('reports cancellation as not-applicable when the invocation settles before the abort fires', async () => { + const report = await withStatefulMatrix({ + ...routeHarnessContractFixtures(), + 'tool:harness/wait': { + cancellation: { abortAfterMs: 1_500, input: { holdMs: 1 } }, + input: { holdMs: 1 }, + resultCompat: 'additive' as const, + }, + }, (options) => runContractMatrix(options)); + + expect(report.routes['tool:harness/wait']?.checks.cancellation).toEqual({ + reason: expect.stringContaining('invocation completed before abort; use an input that stays in flight'), + status: 'not-applicable', + }); + }, 30_000); + + it('fails cancellation when the abort is delivered in flight and the call still settles without rejecting', async () => { + const error = await withPackedShapedSession( + { decorate: ignoringAbortFor('wait'), entry: 'in-memory abort-ignoring fixture', includeApps: false }, + (session, manifest) => runPackedContractMatrix({ + fixtures: { + ...routeHarnessContractFixtures(), + 'tool:harness/wait': { + cancellation: { abortAfterMs: 50, input: { holdMs: 400 } }, + input: { holdMs: 1 }, + resultCompat: 'additive' as const, + }, + }, + manifest, + server: 'harness', + session, + }).catch((thrown: unknown) => thrown), + ); + + expect(error).toBeInstanceOf(AgentTestError); + expect((error as AgentTestError).code).toBe('contract-violation'); + expect((error as AgentTestError).message).toContain('tool:harness/wait / cancellation'); + expect((error as AgentTestError).message).toContain('aborted callTool settled without throwing or rejecting'); + }, 30_000); + it('aggregates missing resultCompat on a tool with the proof-level label', async () => { const fixtures = { ...routeHarnessContractFixtures(), diff --git a/packages/agent-bundle/tests/support/contract-matrix-fixtures.ts b/packages/agent-bundle/tests/support/contract-matrix-fixtures.ts index 9a3d6c11d..34a456dc0 100644 --- a/packages/agent-bundle/tests/support/contract-matrix-fixtures.ts +++ b/packages/agent-bundle/tests/support/contract-matrix-fixtures.ts @@ -111,11 +111,12 @@ export const routeHarnessContractFixtures = (): Record => ({ ...routeHarnessContractFixtures(), - 'app:harness/panel': {}, 'tool:harness/journal': { resultCompat: 'closed' }, 'tool:harness/lifecycle': lifecycleFixture(1), }); From 15e98112aef72d4269da376def1df5e920de5fa0 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Thu, 3 Sep 2026 08:50:18 +0000 Subject: [PATCH 2/3] chore(changeset): patch bump and user-facing summary for #401 (#417) --- .changeset/401-contract-matrix-app-coverage.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/401-contract-matrix-app-coverage.md b/.changeset/401-contract-matrix-app-coverage.md index b21844cc1..f72ddecaa 100644 --- a/.changeset/401-contract-matrix-app-coverage.md +++ b/.changeset/401-contract-matrix-app-coverage.md @@ -1,5 +1,5 @@ --- -"agent-bundle": minor +"agent-bundle": patch --- -Make the contract matrix's MCP App coverage match its docs (#401). App routes are covered at every boundary that registers app resources (`packed-stdio`, `packed-deleted-source`, `host-install`, `dev-epoch`): `surface-completeness` requires the compiled `ui://` URI and `sweep` reads it. With the new `apps: 'auto'` option (the default) an app route no longer needs a fixture entry — `coverage` passes with a reason naming the auto-covered sweep — and `apps: 'explicit'` restores the requirement. `ContractRouteFixture` gains an optional `kind: 'resource'` discriminator (exported as `ContractResourceFixture`) so `{ kind: 'resource' }` names a resource/app fixture explicitly; legacy `{}` is still accepted, and a resource fixture on a tool or prompt route is a coverage failure. `mcp-in-memory` still reports apps as `not-applicable`. The `cancellation` check now tracks whether the abort fired before the invocation settled: an invocation that completes (or rejects) before `abortAfterMs` is `not-applicable` with "invocation completed before abort; use an input that stays in flight" instead of `failed`; it fails only when the abort was delivered mid-flight and the call still settled without rejecting. +Stop requiring hand-enumerated MCP App fixtures in `runPackedContractMatrix`, `runInstalledHostContractMatrix`, and `runDevEpochContractMatrix`: app routes are auto-covered at those levels with the new `apps: 'auto'` default (`coverage` passes with a reason naming the `ui://` resource sweep; `apps: 'explicit'` restores the fixture requirement), declare a resource or app fixture as `{ kind: 'resource' }` (`ContractResourceFixture`; legacy `{}` still accepted), keep apps `not-applicable` at `mcp-in-memory`, and report the `cancellation` check as `not-applicable` ("invocation completed before abort; use an input that stays in flight") instead of a `contract-violation` when the aborted call settled before `abortAfterMs` elapsed. The `agent-bundle/test` docs now state per level whether apps are covered. No diagnostic codes change. (#417) From 49cd737bd63aeb1fb8294fac86b6cf7ebbce4c60 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Thu, 3 Sep 2026 09:09:38 +0000 Subject: [PATCH 3/3] fix(test): record a rejected app resource read as a sweep failure in the contract matrix (#417) --- packages/agent-bundle/src/test/contract.ts | 7 +++++- .../tests/projection/contract-matrix.test.ts | 25 ++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/packages/agent-bundle/src/test/contract.ts b/packages/agent-bundle/src/test/contract.ts index 0b3ecd410..01803dd13 100644 --- a/packages/agent-bundle/src/test/contract.ts +++ b/packages/agent-bundle/src/test/contract.ts @@ -918,7 +918,12 @@ const runSweep = async ( if (uri === undefined) { return failed(`${descriptor.kind} route config exports no uri to read`); } - const read = await client.readResource({ uri }) as { contents?: unknown }; + let read: { contents?: unknown }; + try { + read = await client.readResource({ uri }) as { contents?: unknown }; + } catch (error) { + return failed(`readResource threw for ${JSON.stringify(uri)}: ${error instanceof Error ? error.message : captured(error)}`); + } const contents = Array.isArray(read.contents) ? read.contents : []; return contents.length === 0 ? failed(`readResource returned no contents for ${JSON.stringify(uri)}`) diff --git a/packages/agent-bundle/tests/projection/contract-matrix.test.ts b/packages/agent-bundle/tests/projection/contract-matrix.test.ts index 34520377d..5c0c3ea93 100644 --- a/packages/agent-bundle/tests/projection/contract-matrix.test.ts +++ b/packages/agent-bundle/tests/projection/contract-matrix.test.ts @@ -72,7 +72,11 @@ const PANEL_MIME = 'text/html;profile=mcp-app'; * call, and the progress-handler map the lifecycle replay composes, delegates * to the real client. */ -const withAppSurface = (client: Client): Client => { +const withAppSurface = (client: Client): Client => appSurface(client, 'serves'); + +const withRejectingAppSurface = (client: Client): Client => appSurface(client, 'rejects'); + +const appSurface = (client: Client, read: 'rejects' | 'serves'): Client => { const panelListing = { mimeType: PANEL_MIME, name: 'panel', uri: PANEL_URI }; const wrapper = { _notificationHandlers: (client as unknown as { readonly _notificationHandlers: unknown }) @@ -88,6 +92,7 @@ const withAppSurface = (client: Client): Client => { readResource: async (...arguments_: Parameters) => { const [params] = arguments_; if (params.uri !== PANEL_URI) return client.readResource(...arguments_); + if (read === 'rejects') throw new Error('panel resource handler exploded'); return { contents: [{ mimeType: PANEL_MIME, text: 'route-harness panel', uri: PANEL_URI }] }; }, }; @@ -635,6 +640,24 @@ describe('the generated-plugin contract matrix', () => { }); }, 30_000); + it('records a rejected auto-covered app read as a sweep failure inside the aggregated violation', async () => { + const error = await withPackedShapedSession( + { decorate: withRejectingAppSurface, entry: 'in-memory rejecting app read fixture', includeApps: true }, + (session, manifest) => runPackedContractMatrix({ + fixtures: routeHarnessContractFixtures(), + manifest, + server: 'harness', + session, + }).catch((thrown: unknown) => thrown), + ); + + expect(error).toBeInstanceOf(AgentTestError); + expect((error as AgentTestError).code).toBe('contract-violation'); + expect((error as AgentTestError).message).toContain('app:harness/panel / sweep'); + expect((error as AgentTestError).message).toContain('readResource threw for "ui://harness/panel"'); + expect((error as AgentTestError).message).toContain('panel resource handler exploded'); + }, 30_000); + it('requires an app fixture entry only when apps: "explicit" is requested', async () => { const error = await withPackedShapedSession( { decorate: withAppSurface, entry: 'in-memory explicit app coverage fixture', includeApps: true },