From 36f52c5b8360aa7bc3585820babda9000f69d0eb Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Wed, 2 Sep 2026 23:25:12 +0000 Subject: [PATCH 1/2] feat(test): assert warm runtime identity in contract matrix Pin packed and installed-host event sequences to one warm runtime instance and match stateful lifecycle fixtures to the compiled state catalog. --- .changeset/warm-runtime-contract-identity.md | 5 + packages/agent-bundle/README.md | 19 +- packages/agent-bundle/src/test/contract.ts | 176 ++++++++++++++- packages/agent-bundle/src/test/index.ts | 1 + packages/agent-bundle/src/test/installed.ts | 29 ++- .../tests/host-install-proof.test.ts | 6 + .../tests/packed-stdio-projection.test.ts | 17 +- .../tests/projection/contract-matrix.test.ts | 200 ++++++++++++++++++ .../tests/support/contract-matrix-fixtures.ts | 4 + 9 files changed, 438 insertions(+), 19 deletions(-) create mode 100644 .changeset/warm-runtime-contract-identity.md diff --git a/.changeset/warm-runtime-contract-identity.md b/.changeset/warm-runtime-contract-identity.md new file mode 100644 index 000000000..029dc2180 --- /dev/null +++ b/.changeset/warm-runtime-contract-identity.md @@ -0,0 +1,5 @@ +--- +'agent-bundle': patch +--- + +Assert stable warm-runtime identity and compiled state catalogs at packed and installed-host contract-matrix boundaries. diff --git a/packages/agent-bundle/README.md b/packages/agent-bundle/README.md index ba88a52c6..60b2dda62 100644 --- a/packages/agent-bundle/README.md +++ b/packages/agent-bundle/README.md @@ -428,7 +428,9 @@ idempotent commit replay, and typed budget rejection. A caller-supplied same-store `restart` callback adds durability evidence at that boundary; without one the check is honestly `not-applicable`. Packed callers should wire that callback into the existing packed journey's restart rather than creating -a second pack/build/install path. +a second pack/build/install path. A lifecycle fixture's optional +`state.catalog` assertion pins its declared id and lifetime to the compiler +manifest used by that same mounted-state replay. **`runInstalledHostContractMatrix` (`host-install`)** runs against an already-open session from `openInstalledHostMcpServer`. The opener reads the @@ -440,12 +442,16 @@ running-process versions separately and fails closed when any value is missing or differs. Metadata records the host binary version when observed, adapter revision, manifest/schema digest, and framework version. Module-backed checks remain honestly not-applicable because loading project modules would cross back -into the source/build tree. +into the source/build tree. When the compiled manifest contains event routes, +the packed and installed-host boundaries sample the read-only event-runtime +status before and throughout sequential matrix events. The +`runtime-instance-identity` check fails if the warm `instanceId` changes, the +artifact epoch drifts, or availability degrades to `runtime-restarted` / +`runtime-unavailable`. -No matrix boundary proves browser App HTML, artifact-rebuild replay, -state-lifetime catalog identity, or running-process identity beyond what the -live MCP session reports; deeper runtime-instance introspection depends on -#269. +No matrix boundary proves browser App HTML or artifact-rebuild replay. +In-memory runs and compiled artifacts without event routes report runtime +identity as honestly `not-applicable`. When the advertised input schema declares `additionalProperties: false`, plain `z.object` tool routes may still strip unknown keys without a protocol failure. @@ -472,6 +478,7 @@ await runContractMatrix({ // Packed: pass an already-open session and a manifest compiled before source removal. await runPackedContractMatrix({ + eventRuntime: { endpointId: packedEventRuntimeEndpointId }, session: packedSession, manifest: compiledManifest, fixtures: { /* same shape */ }, diff --git a/packages/agent-bundle/src/test/contract.ts b/packages/agent-bundle/src/test/contract.ts index 165ec750d..0e911cb57 100644 --- a/packages/agent-bundle/src/test/contract.ts +++ b/packages/agent-bundle/src/test/contract.ts @@ -35,11 +35,18 @@ * from a clean installed layout. It carries static layout checks and the * source/artifact/installed/running version quadruple from `installed.ts`. * - * No boundary here proves browser App HTML, state-lifetime catalog identity, - * or runtime-instance identity beyond the live MCP initialize result (#269). + * Packed and installed-host runs with event runtimes sample the pinned status + * IPC before and throughout the sequential matrix events. They fail if the + * warm instance changes or degrades. Stateful lifecycle fixtures may also pin + * their mounted-state declaration against the compiler manifest catalog. + * Boundaries without an event runtime report identity as not-applicable. */ import type { Client } from '@modelcontextprotocol/client'; +import { + requestEventRuntimeStatus, + type EventRuntimeStatusResult, +} from '../events/ipc.ts'; import { AgentTestError, captured } from './errors.ts'; import { MCP_IN_MEMORY_PROOF_LEVEL, @@ -90,6 +97,10 @@ export interface ContractLifecycleFixture { readonly input: unknown; readonly revisionPath: readonly string[]; }; + readonly catalog?: { + readonly id: string; + readonly lifetime: NonNullable['lifetime']; + }; readonly durability?: { readonly expectedStructuredContent: unknown; readonly input: unknown; @@ -164,11 +175,18 @@ export type ContractMatrixProvenance = | PackedMcpProvenance; export interface ContractMatrixReport { + readonly checks: Readonly>; readonly provenance: ContractMatrixProvenance; readonly routes: Readonly>; } +export type ContractEventRuntimeAddress = + | { readonly endpoint: string; readonly endpointId?: never } + | { readonly endpoint?: never; readonly endpointId: string }; + export interface PackedContractMatrixOptions { + /** Read-only status address for the generated event runtime, when this artifact has one. */ + readonly eventRuntime?: ContractEventRuntimeAddress; readonly fixtures: Readonly>; readonly manifest: AgentBundleTestManifest; readonly server?: string; @@ -199,6 +217,8 @@ export interface InstalledHostContractMatrixReport { interface MatrixBoundaryCapabilities { readonly canLoadRouteModules: boolean; + readonly eventRuntime?: ContractEventRuntimeAddress; + readonly eventRuntimeNotApplicableReason: string; readonly moduleSchemaNotApplicableReason: string; readonly proofLevel: AgentTestProofLevel; readonly registersAppResources: boolean; @@ -214,6 +234,7 @@ const INSTALLED_HOST_MODULE_SCHEMA_NOT_APPLICABLE_REASON = const IN_MEMORY_BOUNDARY: MatrixBoundaryCapabilities = Object.freeze({ canLoadRouteModules: true, + eventRuntimeNotApplicableReason: 'the mcp-in-memory boundary has no generated event runtime.', moduleSchemaNotApplicableReason: '', proofLevel: MCP_IN_MEMORY_PROOF_LEVEL, registersAppResources: false, @@ -222,10 +243,14 @@ const IN_MEMORY_BOUNDARY: MatrixBoundaryCapabilities = Object.freeze({ const packedBoundaryFromSession = ( session: PackedMcpSession, + eventRuntime: ContractEventRuntimeAddress | undefined, restart: (() => Promise) | undefined, ): MatrixBoundaryCapabilities => Object.freeze({ canLoadRouteModules: false, + ...(eventRuntime === undefined ? {} : { eventRuntime }), + eventRuntimeNotApplicableReason: + 'this packed matrix run was not supplied the generated event runtime endpoint.', moduleSchemaNotApplicableReason: PACKED_MODULE_SCHEMA_NOT_APPLICABLE_REASON, proofLevel: session.provenance.proofLevel, registersAppResources: true, @@ -238,6 +263,11 @@ const installedHostBoundaryFromSession = ( ): MatrixBoundaryCapabilities => Object.freeze({ canLoadRouteModules: false, + ...(session.eventRuntimeEndpoint === undefined + ? {} + : { eventRuntime: { endpoint: session.eventRuntimeEndpoint } }), + eventRuntimeNotApplicableReason: + 'the installed-host session exposed no generated event runtime endpoint.', moduleSchemaNotApplicableReason: INSTALLED_HOST_MODULE_SCHEMA_NOT_APPLICABLE_REASON, proofLevel: session.provenance.proofLevel, registersAppResources: true, @@ -262,7 +292,9 @@ const CHECK_STATE_JOURNAL = 'state-journal'; const CHECK_STATE_NOTICE = 'state-notice'; const CHECK_STATE_IDEMPOTENCY = 'state-idempotency'; const CHECK_STATE_BUDGET = 'state-budget'; +const CHECK_STATE_CATALOG = 'state-catalog'; const CHECK_RESTART_DURABILITY = 'restart-durability'; +const CHECK_RUNTIME_INSTANCE_IDENTITY = 'runtime-instance-identity'; interface MatrixFailure { readonly check: string; @@ -363,6 +395,79 @@ const passedWithReason = (reason: string): ContractCheckOutcome => ({ reason, st const failed = (reason: string): ContractCheckOutcome => ({ reason, status: 'failed' }); const notApplicable = (reason: string): ContractCheckOutcome => ({ reason, status: 'not-applicable' }); +interface RuntimeIdentityTracker { + readonly observe: (label: string) => Promise; + readonly outcome: () => ContractCheckOutcome; +} + +const readRuntimeStatus = async ( + address: ContractEventRuntimeAddress, +): Promise => address.endpoint === undefined + ? requestEventRuntimeStatus({ endpointId: address.endpointId, timeoutMs: 1_000 }) + : requestEventRuntimeStatus({ endpoint: address.endpoint, timeoutMs: 1_000 }); + +const createRuntimeIdentityTracker = ( + boundary: MatrixBoundaryCapabilities, + manifest: AgentBundleTestManifest, +): RuntimeIdentityTracker => { + const hasEventRoutes = Object.values(manifest.routes) + .some((route) => route.kind === 'event-route'); + if (!hasEventRoutes) { + const outcome = notApplicable( + 'the compiled manifest declares no event routes, so this boundary has no event runtime.', + ); + return Object.freeze({ observe: async () => undefined, outcome: () => outcome }); + } + if (boundary.eventRuntime === undefined) { + const outcome = notApplicable(boundary.eventRuntimeNotApplicableReason); + return Object.freeze({ observe: async () => undefined, outcome: () => outcome }); + } + + const expectedArtifactEpoch = `${manifest.plugin.name}@${manifest.plugin.version}`; + let first: Extract | undefined; + let failure: string | undefined; + let observations = 0; + return Object.freeze({ + observe: async (label: string): Promise => { + if (failure !== undefined) return; + let status: EventRuntimeStatusResult; + try { + status = await readRuntimeStatus(boundary.eventRuntime!); + } catch (error) { + failure = `${label} status request failed: ${error instanceof Error ? error.message : captured(error)}`; + return; + } + observations += 1; + if (status.status !== 'available') { + failure = `${label} event runtime status was ${status.status}`; + return; + } + if (status.availability !== 'available') { + failure = `${label} event runtime availability was ${status.availability} for instance ${JSON.stringify(status.instanceId)}`; + return; + } + if (status.artifactEpoch !== expectedArtifactEpoch) { + failure = `${label} event runtime artifact epoch ${JSON.stringify(status.artifactEpoch)} did not match compiled epoch ${JSON.stringify(expectedArtifactEpoch)}`; + return; + } + if (first === undefined) { + first = status; + return; + } + if (status.instanceId !== first.instanceId) { + failure = `${label} event runtime instance changed from ${JSON.stringify(first.instanceId)} to ${JSON.stringify(status.instanceId)}`; + } + }, + outcome: (): ContractCheckOutcome => { + if (failure !== undefined) return failed(failure); + if (first === undefined || observations < 2) { + return failed('event runtime identity was not observed before and after the matrix event sequence'); + } + return passed(); + }, + }); +}; + const recordFailure = ( failures: MatrixFailure[], routeId: string, @@ -934,6 +1039,7 @@ const executeLifecycleTransitions = async ( client: Client, descriptor: TestableRouteDescriptor, lifecycle: ContractLifecycleFixture, + runtimeIdentity: RuntimeIdentityTracker, ): Promise => { let transitions: readonly ContractLifecycleTransition[]; try { @@ -991,6 +1097,7 @@ const executeLifecycleTransitions = async ( ); settled = true; byPhase.set(transition.phase, { liveProgress, result, transition }); + await runtimeIdentity.observe(`${descriptor.id}/${transition.phase}`); } } finally { if (callerHandler === undefined) { @@ -1102,6 +1209,27 @@ const checkLifecyclePath = ( : failed(`${phase} structuredContent path ${path.join('.')} expected ${captured(expected)}; received ${captured(actual)}`); }; +const checkStateCatalog = ( + manifest: AgentBundleTestManifest, + lifecycle: ContractLifecycleFixture, +): ContractCheckOutcome => { + const expected = lifecycle.state?.catalog; + if (expected === undefined) { + return notApplicable('no lifecycle state catalog assertion declared.'); + } + const actual = manifest.state; + if (actual === undefined) { + return failed(`lifecycle fixture declares state ${JSON.stringify(expected.id)}, but the compiled manifest declares no state`); + } + if (actual.id !== expected.id || actual.lifetime !== expected.lifetime) { + return failed( + `compiled state catalog was ${JSON.stringify(actual.id)} (${actual.lifetime}); ` + + `the mounted-state lifecycle fixture declares ${JSON.stringify(expected.id)} (${expected.lifetime})`, + ); + } + return passed(); +}; + const runStateIdempotency = async ( client: Client, descriptor: TestableRouteDescriptor, @@ -1162,10 +1290,12 @@ const matrixRouteDescriptors = ( const finalizeContractMatrixReport = ( failures: MatrixFailure[], boundary: MatrixBoundaryCapabilities, + checks: Readonly>, provenance: ContractMatrixProvenance, routeReports: Record, ): ContractMatrixReport => { const report: ContractMatrixReport = Object.freeze({ + checks: Object.freeze(checks), provenance, routes: Object.freeze(routeReports), }); @@ -1194,7 +1324,9 @@ const executeContractMatrix = async (options: { }): Promise => { const { boundary, client, fixtures, manifest, provenance, serverName } = options; const failures: MatrixFailure[] = []; + const matrixChecks: Record = {}; const routeReports: Record = {}; + const runtimeIdentity = createRuntimeIdentityTracker(boundary, manifest); const moduleSchemaNotApplicable = (): ContractCheckOutcome => notApplicable(boundary.moduleSchemaNotApplicableReason); @@ -1217,6 +1349,7 @@ const executeContractMatrix = async (options: { } } + await runtimeIdentity.observe('before matrix events'); const surface = await listLiveSurface(client); const invocationCache = new Map(); const durabilityChecks: Array<{ @@ -1232,6 +1365,7 @@ const executeContractMatrix = async (options: { [CHECK_SURFACE]: notApplicable('MCP Apps are not registered by the in-memory projection level.'), }, }; + await runtimeIdentity.observe(`after ${descriptor.id}`); continue; } @@ -1256,6 +1390,7 @@ const executeContractMatrix = async (options: { if (fixture === undefined) { routeReports[descriptor.id] = { checks }; + await runtimeIdentity.observe(`after ${descriptor.id}`); continue; } @@ -1330,10 +1465,16 @@ const executeContractMatrix = async (options: { checks[CHECK_STATE_NOTICE] = notApplicable(reason); checks[CHECK_STATE_IDEMPOTENCY] = notApplicable(reason); checks[CHECK_STATE_BUDGET] = notApplicable(reason); + checks[CHECK_STATE_CATALOG] = notApplicable(reason); checks[CHECK_RESTART_DURABILITY] = notApplicable(reason); } else { const lifecycle = fixture.lifecycle; - const evidence = await executeLifecycleTransitions(client, descriptor, lifecycle); + const evidence = await executeLifecycleTransitions( + client, + descriptor, + lifecycle, + runtimeIdentity, + ); checks[CHECK_LIFECYCLE_REPLAY] = outcomeFromCheck( failures, descriptor.id, @@ -1400,6 +1541,12 @@ const executeContractMatrix = async (options: { ? notApplicable('no lifecycle budget assertion declared.') : await runStateBudget(client, descriptor, evidence, budget), ); + checks[CHECK_STATE_CATALOG] = outcomeFromCheck( + failures, + descriptor.id, + CHECK_STATE_CATALOG, + checkStateCatalog(manifest, lifecycle), + ); const durability = lifecycle.state?.durability; if (durability === undefined) { checks[CHECK_RESTART_DURABILITY] = notApplicable( @@ -1423,12 +1570,21 @@ const executeContractMatrix = async (options: { checks[CHECK_STATE_NOTICE] = notApplicable('applies to tool routes only.'); checks[CHECK_STATE_IDEMPOTENCY] = notApplicable('applies to tool routes only.'); checks[CHECK_STATE_BUDGET] = notApplicable('applies to tool routes only.'); + checks[CHECK_STATE_CATALOG] = notApplicable('applies to tool routes only.'); checks[CHECK_RESTART_DURABILITY] = notApplicable('applies to tool routes only.'); } routeReports[descriptor.id] = { checks }; + await runtimeIdentity.observe(`after ${descriptor.id}`); } + matrixChecks[CHECK_RUNTIME_INSTANCE_IDENTITY] = outcomeFromCheck( + failures, + 'boundary', + CHECK_RUNTIME_INSTANCE_IDENTITY, + runtimeIdentity.outcome(), + ); + if (durabilityChecks.length > 0) { if (boundary.restart === undefined) { for (const pending of durabilityChecks) { @@ -1470,7 +1626,13 @@ const executeContractMatrix = async (options: { } } - return finalizeContractMatrixReport(failures, boundary, provenance, routeReports); + return finalizeContractMatrixReport( + failures, + boundary, + matrixChecks, + provenance, + routeReports, + ); }; /** @@ -1517,7 +1679,11 @@ export const runPackedContractMatrix = async ( ): Promise => { const serverName = resolveServerName(options.manifest, options.server); return executeContractMatrix({ - boundary: packedBoundaryFromSession(options.session, options.restart), + boundary: packedBoundaryFromSession( + options.session, + options.eventRuntime, + options.restart, + ), client: options.session.client, fixtures: options.fixtures, manifest: options.manifest, diff --git a/packages/agent-bundle/src/test/index.ts b/packages/agent-bundle/src/test/index.ts index f8112e7de..ac25b1e51 100644 --- a/packages/agent-bundle/src/test/index.ts +++ b/packages/agent-bundle/src/test/index.ts @@ -92,6 +92,7 @@ export { export type { ContractCheckOutcome, ContractCheckStatus, + ContractEventRuntimeAddress, ContractLifecycleFixture, ContractLifecyclePhase, ContractLifecycleTransition, diff --git a/packages/agent-bundle/src/test/installed.ts b/packages/agent-bundle/src/test/installed.ts index e0d8083eb..6daea9113 100644 --- a/packages/agent-bundle/src/test/installed.ts +++ b/packages/agent-bundle/src/test/installed.ts @@ -1,5 +1,5 @@ import { lstat, readFile } from 'node:fs/promises'; -import { isAbsolute, join, relative, resolve, sep } from 'node:path'; +import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; import { Client } from '@modelcontextprotocol/client'; import { StdioClientTransport } from '@modelcontextprotocol/client/stdio'; @@ -8,6 +8,7 @@ import { artifactManifestName } from '../build/emit.ts'; import { parseArtifactHookIndex, type ArtifactHook } from '../build/hook-index.ts'; import { parseArtifactManifest } from '../build/manifest.ts'; import { digest, sha256Hex } from '../core/digest.ts'; +import { eventRuntimeEndpoint } from '../events/ipc.ts'; import { resolveBundleRoot } from '../install/doctor.ts'; import type { InstallHost } from '../install/install.ts'; import { AgentTestError } from './errors.ts'; @@ -71,6 +72,8 @@ export interface InstalledHostMcpProvenance { export interface InstalledHostMcpSession extends AsyncDisposable { readonly client: Client; readonly close: () => Promise; + /** Raw read-only status socket for the installed generated event runtime. */ + readonly eventRuntimeEndpoint?: string; readonly observation: InstalledHostObservation; readonly provenance: InstalledHostMcpProvenance; readonly stderr: () => string; @@ -260,8 +263,8 @@ const outcomes = (failures: readonly Failure[]): Readonly expandHostPath(argument, options.host, installedRoot)); const entryArgument = args.find((argument) => /\.mjs$/u.test(argument)); - if (entryArgument === undefined || await fileHash(resolvedCommandPath(entryArgument, cwd)) === undefined) { + const resolvedEntry = entryArgument === undefined + ? undefined + : resolvedCommandPath(entryArgument, cwd); + if (resolvedEntry === undefined || await fileHash(resolvedEntry) === undefined) { failures.push({ check: 'mcp-command', reason: 'installed MCP entry argument did not resolve to an installed file' }); } const declaredEnvironment = record(discovered.server.env); @@ -421,7 +427,21 @@ export const openInstalledHostMcpServer = async ( ...expandedDeclaredEnvironment, }; + const eventRuntimeEndpointPath = artifactManifest === undefined || resolvedEntry === undefined + ? undefined + : eventRuntimeEndpoint( + `${artifactManifest.project.revision}:${options.host}:${dirname(dirname(resolvedEntry))}`, + ); + if (eventRuntimeEndpointPath === undefined && failures.length === 0) { + failures.push({ check: 'mcp-command', reason: 'installed event runtime endpoint could not be derived' }); + } if (failures.length > 0) throw installedFailure(failures, proofLevel); + if (eventRuntimeEndpointPath === undefined) { + throw installedFailure( + [{ check: 'mcp-command', reason: 'installed event runtime endpoint could not be derived' }], + proofLevel, + ); + } const client = new Client({ name: 'agent-bundle-installed-host-proof', version: '1.0.0' }); const transport = new StdioClientTransport({ args: [...args], @@ -506,6 +526,7 @@ export const openInstalledHostMcpServer = async ( return Object.freeze({ client, close, + eventRuntimeEndpoint: eventRuntimeEndpointPath, observation, provenance, stderr: () => captured, diff --git a/packages/agent-bundle/tests/host-install-proof.test.ts b/packages/agent-bundle/tests/host-install-proof.test.ts index cded0fec6..65eb2043f 100644 --- a/packages/agent-bundle/tests/host-install-proof.test.ts +++ b/packages/agent-bundle/tests/host-install-proof.test.ts @@ -127,6 +127,12 @@ it('stages a clean adapter-simulated host and runs the shared matrix from its in }, host: 'claude', matrix: { + checks: { + 'runtime-instance-identity': { + reason: 'the compiled manifest declares no event routes, so this boundary has no event runtime.', + status: 'not-applicable', + }, + }, provenance: { host: 'claude', proofLevel: 'simulated' }, routes: { 'tool:probe/echo': { diff --git a/packages/agent-bundle/tests/packed-stdio-projection.test.ts b/packages/agent-bundle/tests/packed-stdio-projection.test.ts index 0d95b5fec..0662f2e44 100644 --- a/packages/agent-bundle/tests/packed-stdio-projection.test.ts +++ b/packages/agent-bundle/tests/packed-stdio-projection.test.ts @@ -95,6 +95,11 @@ it('serves compiled routes and durable state across packed process restarts', as expect(workerSource).toContain('AGENT_BUNDLE_PLUGIN_ROOT'); expect(workerSource).toMatch(/new URL\(["']\.\.["'], import\.meta\.url\)/u); const harnessManifest = await compileTestManifest({ root: project }); + const artifactManifest = JSON.parse( + await readFile(join(artifact, 'agent-bundle.manifest.json'), 'utf8'), + ) as { readonly project: { readonly revision: string } }; + const eventRuntimeEndpointId = + `${artifactManifest.project.revision}:claude:${dirname(dirname(resolve(entry)))}`; const deletedSource = await removeProjectSource({ projectRoot: project }); const firstSession = await openPackedMcpServer({ @@ -198,6 +203,7 @@ it('serves compiled routes and durable state across packed process restarts', as messages: [{ content: { text: 'Summarize chapter one', type: 'text' }, role: 'user' }], }); const matrixReport = await runPackedContractMatrix({ + eventRuntime: { endpointId: eventRuntimeEndpointId }, fixtures: routeHarnessPackedContractFixtures(), manifest: harnessManifest, restart: async () => { @@ -220,6 +226,12 @@ it('serves compiled routes and durable state across packed process restarts', as expect(matrixReport.routes['tool:harness/lifecycle']?.checks['restart-durability']).toEqual({ status: 'passed', }); + expect(matrixReport.checks['runtime-instance-identity']).toEqual({ + status: 'passed', + }); + expect(matrixReport.routes['tool:harness/lifecycle']?.checks['state-catalog']).toEqual({ + status: 'passed', + }); expect(matrixReport.routes['tool:harness/lifecycle']?.checks['lifecycle-serialized-round-trip']).toEqual({ reason: expect.stringContaining('packed sessions cannot load project route modules'), status: 'not-applicable', @@ -240,14 +252,11 @@ it('serves compiled routes and durable state across packed process restarts', as .resolves.toMatchObject({ structuredContent: { entries: [{ note: 'packed durable proof' }], revision: 6 }, }); - const artifactManifest = JSON.parse( - await readFile(join(artifact, 'agent-bundle.manifest.json'), 'utf8'), - ) as { readonly project: { readonly revision: string } }; let eventResponse: unknown; try { eventResponse = await requestEventRuntime({ artifactEpoch: artifactManifest.project.revision, - endpointId: `${artifactManifest.project.revision}:claude:${dirname(dirname(resolve(entry)))}`, + endpointId: eventRuntimeEndpointId, event: 'tool/after', hostContractRevision: 'packed-proof', native: { diff --git a/packages/agent-bundle/tests/projection/contract-matrix.test.ts b/packages/agent-bundle/tests/projection/contract-matrix.test.ts index ea9456205..3690b995b 100644 --- a/packages/agent-bundle/tests/projection/contract-matrix.test.ts +++ b/packages/agent-bundle/tests/projection/contract-matrix.test.ts @@ -6,6 +6,7 @@ import { describe, expect, it } from '@rstest/core'; import { createSqliteStateDriver } from '@agent-bundle/runtime/state/sqlite'; import stateDefinition from '../../fixtures/route-harness/src/state.ts'; +import { createEventRuntimeServer } from '../../src/events/ipc.ts'; import { AgentTestError } from '../../src/test/errors.ts'; import { compileTestManifest, @@ -14,9 +15,11 @@ import { } from '../../src/test/manifest.ts'; import { runContractMatrix, + runInstalledHostContractMatrix, runPackedContractMatrix, type ContractMatrixOptions, } from '../../src/test/contract.ts'; +import type { InstalledHostMcpSession } from '../../src/test/installed.ts'; import { openInMemoryMcpServer, type InMemoryMcpSession } from '../../src/test/mcp.ts'; import type { PackedMcpSession } from '../../src/test/packed.ts'; import { @@ -71,6 +74,11 @@ describe('the generated-plugin contract matrix', () => { }); expect(report.routes['tool:harness/lifecycle']?.checks['state-idempotency']).toEqual({ status: 'passed' }); expect(report.routes['tool:harness/lifecycle']?.checks['state-budget']).toEqual({ status: 'passed' }); + expect(report.routes['tool:harness/lifecycle']?.checks['state-catalog']).toEqual({ status: 'passed' }); + expect(report.checks['runtime-instance-identity']).toEqual({ + reason: 'the mcp-in-memory boundary has no generated event runtime.', + status: 'not-applicable', + }); expect(report.routes['tool:harness/lifecycle']?.checks['restart-durability']).toEqual({ status: 'passed' }); expect(report.routes['app:harness/panel']?.checks['surface-completeness']).toEqual({ reason: 'MCP Apps are not registered by the in-memory projection level.', @@ -93,8 +101,48 @@ describe('the generated-plugin contract matrix', () => { expect((error as AgentTestError).message).toContain(proofLabel); }, 30_000); + it('rejects a lifecycle fixture whose mounted-state declaration drifts from the manifest catalog', async () => { + const fixtures = routeHarnessContractFixtures(); + const lifecycleFixture = fixtures['tool:harness/lifecycle']; + if (lifecycleFixture?.lifecycle?.state === undefined) { + throw new TypeError('Lifecycle state fixture is unavailable.'); + } + const error = await withStatefulMatrix({ + ...fixtures, + 'tool:harness/lifecycle': { + ...lifecycleFixture, + lifecycle: { + ...lifecycleFixture.lifecycle, + state: { + ...lifecycleFixture.lifecycle.state, + catalog: { + id: 'route-harness/not-the-mounted-store', + lifetime: 'workspace-durable', + }, + }, + }, + }, + }, (options) => runContractMatrix(options).catch((thrown: unknown) => thrown)); + + expect(error).toBeInstanceOf(AgentTestError); + expect((error as AgentTestError).message).toContain('state-catalog'); + expect((error as AgentTestError).message).toContain('route-harness/journal'); + expect((error as AgentTestError).message).toContain('route-harness/not-the-mounted-store'); + }, 30_000); + it('composes and restores a caller-owned progress handler after lifecycle replay', async () => { const root = await mkdtemp(join(tmpdir(), 'agent-bundle-contract-handler-')); + const runtime = await createEventRuntimeServer({ + artifactEpoch: 'route-harness@1.0.0', + endpointId: `contract-matrix-progress:${root}`, + handle: async () => undefined, + status: () => ({ + artifactEpoch: 'route-harness@1.0.0', + availability: 'available', + instanceId: 'runtime-instance-a', + pid: process.pid, + }), + }); const compiledManifest = await compileTestManifest({ root: fixtureRoot }); const manifest = Object.freeze({ ...compiledManifest, @@ -134,6 +182,7 @@ describe('the generated-plugin contract matrix', () => { const callerHandler = notificationHandlers.get('notifications/progress'); try { const report = await runPackedContractMatrix({ + eventRuntime: { endpoint: runtime.endpoint }, fixtures: routeHarnessContractFixtures(), manifest, server: 'harness', @@ -143,10 +192,161 @@ describe('the generated-plugin contract matrix', () => { expect(report.routes['tool:harness/lifecycle']?.checks['lifecycle-replay']).toEqual({ status: 'passed', }); + expect(report.checks['runtime-instance-identity']).toEqual({ + status: 'passed', + }); expect(callerProgress).toBeGreaterThan(0); expect(notificationHandlers.get('notifications/progress')).toBe(callerHandler); } finally { await session.close(); + await runtime.close(); + await rm(root, { force: true, recursive: true }); + } + }, 30_000); + + it('fails when the packed event runtime instance changes during sequential matrix events', async () => { + const root = await mkdtemp(join(tmpdir(), 'agent-bundle-contract-identity-drift-')); + let statusCalls = 0; + const runtime = await createEventRuntimeServer({ + artifactEpoch: 'route-harness@1.0.0', + endpointId: `contract-matrix-identity-drift:${root}`, + handle: async () => undefined, + status: () => ({ + artifactEpoch: 'route-harness@1.0.0', + availability: 'available', + instanceId: statusCalls++ === 0 ? 'runtime-instance-a' : 'runtime-instance-b', + pid: process.pid, + }), + }); + const compiledManifest = await compileTestManifest({ root: fixtureRoot }); + const manifest = Object.freeze({ + ...compiledManifest, + apps: Object.freeze({}), + routes: Object.freeze(Object.fromEntries( + Object.entries(compiledManifest.routes).filter(([, route]) => route.kind !== 'app'), + )), + }); + const session = await openInMemoryMcpServer({ + manifest, + state: { + definition: stateDefinition, + driver: createSqliteStateDriver({ root }), + }, + }); + const packedSession: PackedMcpSession = Object.freeze({ + client: session.client, + close: session.close, + provenance: Object.freeze({ + entry: 'in-memory runtime-identity regression fixture', + pid: undefined, + proofLevel: 'packed-stdio' as const, + }), + stderr: () => '', + [Symbol.asyncDispose]: session[Symbol.asyncDispose], + }); + try { + const error = await runPackedContractMatrix({ + eventRuntime: { endpoint: runtime.endpoint }, + fixtures: routeHarnessContractFixtures(), + manifest, + server: 'harness', + session: packedSession, + }).catch((thrown: unknown) => thrown); + + expect(error).toBeInstanceOf(AgentTestError); + expect((error as AgentTestError).message).toContain('runtime-instance-identity'); + expect((error as AgentTestError).message).toContain('runtime-instance-a'); + expect((error as AgentTestError).message).toContain('runtime-instance-b'); + } finally { + await session.close(); + await runtime.close(); + await rm(root, { force: true, recursive: true }); + } + }, 30_000); + + it('reads one warm runtime identity across installed-host matrix events', async () => { + const root = await mkdtemp(join(tmpdir(), 'agent-bundle-installed-identity-')); + const runtime = await createEventRuntimeServer({ + artifactEpoch: 'route-harness@1.0.0', + endpointId: `installed-contract-matrix:${root}`, + handle: async () => undefined, + status: () => ({ + artifactEpoch: 'route-harness@1.0.0', + availability: 'available', + instanceId: 'installed-runtime-instance-a', + pid: process.pid, + }), + }); + const compiledManifest = await compileTestManifest({ root: fixtureRoot }); + const manifest = Object.freeze({ + ...compiledManifest, + apps: Object.freeze({}), + routes: Object.freeze(Object.fromEntries( + Object.entries(compiledManifest.routes).filter(([, route]) => route.kind !== 'app'), + )), + }); + const session = await openInMemoryMcpServer({ + manifest, + state: { + definition: stateDefinition, + driver: createSqliteStateDriver({ root }), + }, + }); + const checks = Object.freeze({ + 'component-paths': Object.freeze({ status: 'passed' as const }), + 'hook-commands': Object.freeze({ status: 'passed' as const }), + 'manifest-schema': Object.freeze({ status: 'passed' as const }), + 'mcp-command': Object.freeze({ status: 'passed' as const }), + resources: Object.freeze({ status: 'passed' as const }), + 'version-digests': Object.freeze({ status: 'passed' as const }), + 'version-quadruple': Object.freeze({ status: 'passed' as const }), + }); + const installedSession: InstalledHostMcpSession = Object.freeze({ + client: session.client, + close: session.close, + eventRuntimeEndpoint: runtime.endpoint, + observation: Object.freeze({ + checks, + host: 'claude' as const, + metadata: Object.freeze({ + adapterRevision: 'test', + frameworkVersion: '0.1.0', + hostBinaryVersion: Object.freeze({ reason: 'test fixture', status: 'unavailable' as const }), + manifestSchemaDigest: '0'.repeat(64), + }), + proofLevel: 'host-install test fixture', + sessionEvidence: 'installed-host runtime-identity regression fixture', + versions: Object.freeze({ + builtArtifact: '1.0.0', + installedArtifact: '1.0.0', + runningProcess: '1.0.0', + source: '1.0.0', + }), + }), + provenance: Object.freeze({ + entry: 'mcp/harness.mjs', + host: 'claude' as const, + pid: process.pid, + proofLevel: 'host-install' as const, + }), + stderr: () => '', + [Symbol.asyncDispose]: session[Symbol.asyncDispose], + }); + try { + const report = await runInstalledHostContractMatrix({ + fixtures: routeHarnessContractFixtures(), + manifest, + server: 'harness', + session: installedSession, + }); + + expect(report.matrix.checks['runtime-instance-identity']).toEqual({ status: 'passed' }); + expect(report.matrix.routes['tool:harness/lifecycle']?.checks['state-catalog']).toEqual({ + status: 'passed', + }); + } finally { + await session.close(); + await runtime.close(); await rm(root, { force: true, recursive: true }); } }, 30_000); diff --git a/packages/agent-bundle/tests/support/contract-matrix-fixtures.ts b/packages/agent-bundle/tests/support/contract-matrix-fixtures.ts index ecf3f6411..9a3d6c11d 100644 --- a/packages/agent-bundle/tests/support/contract-matrix-fixtures.ts +++ b/packages/agent-bundle/tests/support/contract-matrix-fixtures.ts @@ -27,6 +27,10 @@ const lifecycleFixture = (revisionOffset = 0): ContractRouteFixture => ({ input: { action: 'exceed-budget', payload: 'x'.repeat(512) }, revisionPath: ['revision'], }, + catalog: { + id: 'route-harness/journal', + lifetime: 'workspace-durable', + }, durability: { expectedStructuredContent: { history: lifecyclePhases.slice(1), From 28124cea99fc64e1e0feecdde888d12152d3c274 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Wed, 2 Sep 2026 23:46:32 +0000 Subject: [PATCH 2/2] fix(test): scope runtime identity to event owner Record the generated server that owns event routes so multi-server matrices do not probe another server's runtime. --- packages/agent-bundle/src/test/contract.ts | 17 ++++++- packages/agent-bundle/src/test/manifest.ts | 6 +++ .../tests/projection/contract-matrix.test.ts | 48 +++++++++++++++++++ 3 files changed, 70 insertions(+), 1 deletion(-) diff --git a/packages/agent-bundle/src/test/contract.ts b/packages/agent-bundle/src/test/contract.ts index 0e911cb57..22f3d3bd9 100644 --- a/packages/agent-bundle/src/test/contract.ts +++ b/packages/agent-bundle/src/test/contract.ts @@ -409,6 +409,7 @@ const readRuntimeStatus = async ( const createRuntimeIdentityTracker = ( boundary: MatrixBoundaryCapabilities, manifest: AgentBundleTestManifest, + serverName: string, ): RuntimeIdentityTracker => { const hasEventRoutes = Object.values(manifest.routes) .some((route) => route.kind === 'event-route'); @@ -418,6 +419,20 @@ const createRuntimeIdentityTracker = ( ); return Object.freeze({ observe: async () => undefined, outcome: () => outcome }); } + const serverId = `mcp:${serverName}`; + if (manifest.eventRuntimeServerId === undefined) { + const outcome = notApplicable( + 'the compiled manifest declares event routes, but no generated MCP server owns their event runtime.', + ); + return Object.freeze({ observe: async () => undefined, outcome: () => outcome }); + } + if (manifest.eventRuntimeServerId !== serverId) { + const outcome = notApplicable( + `compiled server ${JSON.stringify(serverId)} does not own the event runtime; ` + + `owner is ${JSON.stringify(manifest.eventRuntimeServerId)}.`, + ); + return Object.freeze({ observe: async () => undefined, outcome: () => outcome }); + } if (boundary.eventRuntime === undefined) { const outcome = notApplicable(boundary.eventRuntimeNotApplicableReason); return Object.freeze({ observe: async () => undefined, outcome: () => outcome }); @@ -1326,7 +1341,7 @@ const executeContractMatrix = async (options: { const failures: MatrixFailure[] = []; const matrixChecks: Record = {}; const routeReports: Record = {}; - const runtimeIdentity = createRuntimeIdentityTracker(boundary, manifest); + const runtimeIdentity = createRuntimeIdentityTracker(boundary, manifest, serverName); const moduleSchemaNotApplicable = (): ContractCheckOutcome => notApplicable(boundary.moduleSchemaNotApplicableReason); diff --git a/packages/agent-bundle/src/test/manifest.ts b/packages/agent-bundle/src/test/manifest.ts index 2bab8cca2..e99a6423b 100644 --- a/packages/agent-bundle/src/test/manifest.ts +++ b/packages/agent-bundle/src/test/manifest.ts @@ -163,6 +163,8 @@ export interface AgentBundleTestManifest { readonly diagnostics: readonly Diagnostic[]; /** The route graph digest: project-relative route identity, equal on every machine. */ readonly digest: string; + /** Generated MCP server that owns the shared event runtime, when event routes and a generated server coexist. */ + readonly eventRuntimeServerId?: string; /** Plugin name and version, as the generated MCP server reports them in `initialize`. */ readonly plugin: TestManifestPluginIdentity; readonly projectRoot: string; @@ -258,12 +260,16 @@ export const testManifestFromRouteGraph = (input: { }): AgentBundleTestManifest => { const routes: Record = {}; for (const route of graphRoutes(input.graph)) routes[route.id] = descriptorOf(route); + const eventRuntimeServerId = input.graph.events.length === 0 + ? undefined + : input.graph.servers.find((server) => server.mode === 'generated')?.id; return deepFreeze({ apps: appDescriptors(input.apps ?? [], input.projectRoot), cliCommands: [...(input.graph.cli?.commands ?? [])], ...(input.configPath === undefined ? {} : { configPath: input.configPath }), diagnostics: [...(input.diagnostics ?? input.graph.diagnostics)], digest: input.graph.digest, + ...(eventRuntimeServerId === undefined ? {} : { eventRuntimeServerId }), plugin: input.plugin ?? { name: 'unknown', version: '0.0.0' }, projectRoot: input.projectRoot, proofLevel: ROUTE_UNIT_PROOF_LEVEL, diff --git a/packages/agent-bundle/tests/projection/contract-matrix.test.ts b/packages/agent-bundle/tests/projection/contract-matrix.test.ts index 3690b995b..7914a0a76 100644 --- a/packages/agent-bundle/tests/projection/contract-matrix.test.ts +++ b/packages/agent-bundle/tests/projection/contract-matrix.test.ts @@ -264,6 +264,54 @@ describe('the generated-plugin contract matrix', () => { } }, 30_000); + it('does not require identity from a generated server that does not own the event runtime', async () => { + const root = await mkdtemp(join(tmpdir(), 'agent-bundle-contract-non-owner-')); + const compiledManifest = await compileTestManifest({ root: fixtureRoot }); + const manifest = Object.freeze({ + ...compiledManifest, + apps: Object.freeze({}), + eventRuntimeServerId: 'mcp:not-harness', + routes: Object.freeze(Object.fromEntries( + Object.entries(compiledManifest.routes).filter(([, route]) => route.kind !== 'app'), + )), + }); + const session = await openInMemoryMcpServer({ + manifest, + state: { + definition: stateDefinition, + driver: createSqliteStateDriver({ root }), + }, + }); + const packedSession: PackedMcpSession = Object.freeze({ + client: session.client, + close: session.close, + provenance: Object.freeze({ + entry: 'in-memory non-owning-server regression fixture', + pid: undefined, + proofLevel: 'packed-stdio' as const, + }), + stderr: () => '', + [Symbol.asyncDispose]: session[Symbol.asyncDispose], + }); + try { + const report = await runPackedContractMatrix({ + eventRuntime: { endpoint: join(root, 'must-not-be-read.sock') }, + fixtures: routeHarnessContractFixtures(), + manifest, + server: 'harness', + session: packedSession, + }); + + expect(report.checks['runtime-instance-identity']).toEqual({ + reason: 'compiled server "mcp:harness" does not own the event runtime; owner is "mcp:not-harness".', + status: 'not-applicable', + }); + } finally { + await session.close(); + await rm(root, { force: true, recursive: true }); + } + }, 30_000); + it('reads one warm runtime identity across installed-host matrix events', async () => { const root = await mkdtemp(join(tmpdir(), 'agent-bundle-installed-identity-')); const runtime = await createEventRuntimeServer({