From 907bfa990f0e9a53f8cdc883ef5d14cd2573858f Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Wed, 2 Sep 2026 23:23:44 +0000 Subject: [PATCH] test(mcp): add opt-in conformance lane (#133) Exercise generated MCP artifacts with the official active suite while keeping the costly networked lane off default CI paths. --- .changeset/official-mcp-conformance.md | 9 + .github/workflows/mcp-conformance.yml | 36 ++ .gitignore | 1 + docs/mcp-conformance.md | 45 ++ package.json | 2 + .../src/mcp/harness/tools/unavailable.tsx | 5 +- .../mcp-conformance-expected-failures.yml | 47 ++ .../tests/mcp-conformance.test.ts | 19 + .../tests/support/mcp-conformance.ts | 415 ++++++++++++++++++ .../tests/test-harness-manifest.test.ts | 2 +- pnpm-lock.yaml | 182 ++++++++ rstest.config.ts | 14 +- rstest.integration-tests.ts | 9 + rstest.mcp-conformance.config.ts | 12 + rstest.unit.config.ts | 2 + 15 files changed, 796 insertions(+), 4 deletions(-) create mode 100644 .changeset/official-mcp-conformance.md create mode 100644 .github/workflows/mcp-conformance.yml create mode 100644 docs/mcp-conformance.md create mode 100644 packages/agent-bundle/tests/fixtures/mcp-conformance-expected-failures.yml create mode 100644 packages/agent-bundle/tests/mcp-conformance.test.ts create mode 100644 packages/agent-bundle/tests/support/mcp-conformance.ts create mode 100644 rstest.mcp-conformance.config.ts diff --git a/.changeset/official-mcp-conformance.md b/.changeset/official-mcp-conformance.md new file mode 100644 index 000000000..b19df59a9 --- /dev/null +++ b/.changeset/official-mcp-conformance.md @@ -0,0 +1,9 @@ +--- +"agent-bundle": minor +--- + +Add an opt-in MCP conformance lane that builds a generated route server, +adapts its stdio transport to loopback Streamable HTTP, and runs the official +`@modelcontextprotocol/conformance` active suite for specification +`2025-11-25`. The manually dispatched workflow preserves per-scenario runner +artifacts and keeps known gaps in a stale-detecting expected-failure baseline. diff --git a/.github/workflows/mcp-conformance.yml b/.github/workflows/mcp-conformance.yml new file mode 100644 index 000000000..a2aa6ed65 --- /dev/null +++ b/.github/workflows/mcp-conformance.yml @@ -0,0 +1,36 @@ +name: MCP conformance + +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: mcp-conformance-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + mcp-conformance: + name: MCP conformance (Node 22.19) + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + - uses: pnpm/setup@v2 + with: + cache: true + install: false + runtime: node@22.19.0 + - run: pnpm install --frozen-lockfile + - run: pnpm test:mcp-conformance + env: + AGENT_BUNDLE_MCP_CONFORMANCE_OUTPUT: artifacts/mcp-conformance + - name: Preserve MCP conformance results + if: always() + uses: actions/upload-artifact@v7 + with: + if-no-files-found: warn + name: mcp-conformance-results + path: artifacts/mcp-conformance + retention-days: 14 diff --git a/.gitignore b/.gitignore index e6b9ec671..ab57b83ed 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ dist/ .agent-bundle/ .worktrees/ coverage/ +artifacts/ *.log examples/audiobook-curator/artifact/ diff --git a/docs/mcp-conformance.md b/docs/mcp-conformance.md new file mode 100644 index 000000000..553ef4050 --- /dev/null +++ b/docs/mcp-conformance.md @@ -0,0 +1,45 @@ +# MCP conformance evidence + +The MCP conformance lane is intentionally opt-in. Run it locally with: + +```sh +pnpm test:mcp-conformance +``` + +CI exposes the same command through the manually dispatched `MCP conformance` +workflow. The lane uses one Node version and one existing fixture, and uploads +the official runner's `checks.json` artifacts from `artifacts/mcp-conformance`. +It is not part of pull request, push, scheduled, or default Rstest execution. + +## Transport and fixture + +Generated agent-bundle MCP artifacts are managed stdio executables; they do +not expose an HTTP transport seam. The harness therefore builds the existing +`packages/agent-bundle/fixtures/route-harness` generated-route fixture through +the public build pipeline, starts the generated stdio artifact, and forwards +raw JSON-RPC frames through the official SDK's +`NodeStreamableHTTPServerTransport`. The adapter owns only loopback transport, +health checking, bounded runner execution, and teardown. Route registration, +Flight rendering, schemas, and responses remain the generated artifact's +logic. + +The copied fixture is narrowed to its MCP routes before building. Its state, +event, and CLI sources are omitted because conformance needs one generated MCP +surface and must not add the packed fixture's separate npm-install journey. + +## Recorded run: 2026-09-02 + +- Runner: `@modelcontextprotocol/conformance@0.1.16` +- Suite: `server --suite active` +- Specification: `2025-11-25` +- Scenarios: 30 total; 7 passed, 23 expected failures, 0 skipped +- Unexpected failures: 0 + +The dated expected-failure baseline is +`packages/agent-bundle/tests/fixtures/mcp-conformance-expected-failures.yml`. +It records three unsupported generated-server operation families +(`completion/complete`, `logging/setLevel`, and resource subscriptions) plus +the canonical content-specific tools, resources, and prompts that the reused +route harness does not provide. The official runner rejects both unexpected +failures and stale baseline entries, so newly fixed scenarios must be removed +from the baseline. diff --git a/package.json b/package.json index eaf992893..cf47c0a63 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "test:integration": "pnpm build && pnpm test:integration:run", "test:integration:run": "AGENT_BUNDLE_WORKBENCH_PREBUILT=1 AGENT_BUNDLE_PACKAGE_PREBUILT=1 rstest --config rstest.integration.config.ts", "test:evidence": "pnpm build && AGENT_BUNDLE_WORKBENCH_PREBUILT=1 AGENT_BUNDLE_PACKAGE_PREBUILT=1 rstest --config rstest.evidence.config.ts", + "test:mcp-conformance": "pnpm build && AGENT_BUNDLE_MCP_CONFORMANCE=1 rstest --config rstest.mcp-conformance.config.ts", "test:watch": "rstest --config rstest.config.ts --watch", "lint": "rslint .", "bench:hook-cold-start": "node scripts/measure-hook-cold-start.mjs", @@ -57,6 +58,7 @@ "@changesets/cli": "3.0.1", "@effect/language-service": "0.87.2", "@effect/tsgo": "0.39.0", + "@modelcontextprotocol/conformance": "0.1.16", "@modelcontextprotocol/server": "2.0.0", "@rsbuild/core": "2.2.1", "@rsbuild/plugin-react": "2.1.0", diff --git a/packages/agent-bundle/fixtures/route-harness/src/mcp/harness/tools/unavailable.tsx b/packages/agent-bundle/fixtures/route-harness/src/mcp/harness/tools/unavailable.tsx index 048483f1a..72c1b1b48 100644 --- a/packages/agent-bundle/fixtures/route-harness/src/mcp/harness/tools/unavailable.tsx +++ b/packages/agent-bundle/fixtures/route-harness/src/mcp/harness/tools/unavailable.tsx @@ -1,7 +1,10 @@ import { Agent } from '@agent-bundle/runtime'; import { z } from 'zod'; -export const config = { title: 'Unavailable' }; +export const config = { + description: 'Returns a typed unavailable result for projection checks.', + title: 'Unavailable', +}; export const inputSchema = z.object({}); diff --git a/packages/agent-bundle/tests/fixtures/mcp-conformance-expected-failures.yml b/packages/agent-bundle/tests/fixtures/mcp-conformance-expected-failures.yml new file mode 100644 index 000000000..2d671ce57 --- /dev/null +++ b/packages/agent-bundle/tests/fixtures/mcp-conformance-expected-failures.yml @@ -0,0 +1,47 @@ +server: + # 2026-09-02: Generated route servers do not register the optional completion/complete operation. + - completion-complete + # 2026-09-02: Generated route servers do not register the optional logging/setLevel operation. + - logging-set-level + # 2026-09-02: route-harness has no generated test_elicitation_sep1034_defaults tool. + - elicitation-sep1034-defaults + # 2026-09-02: route-harness has no generated test_elicitation_sep1330_enums tool. + - elicitation-sep1330-enums + # 2026-09-02: route-harness has no generated test_prompt_with_embedded_resource prompt. + - prompts-get-embedded-resource + # 2026-09-02: route-harness has no generated test_simple_prompt prompt. + - prompts-get-simple + # 2026-09-02: route-harness has no generated test_prompt_with_arguments prompt. + - prompts-get-with-args + # 2026-09-02: route-harness has no generated test_prompt_with_image prompt. + - prompts-get-with-image + # 2026-09-02: route-harness has no generated test://static-binary resource. + - resources-read-binary + # 2026-09-02: route-harness has no generated test://static-text resource. + - resources-read-text + # 2026-09-02: Generated static resources do not register the optional resources/subscribe operation. + - resources-subscribe + # 2026-09-02: route-harness has no generated test://template/{id}/data resource template. + - resources-templates-read + # 2026-09-02: Generated static resources do not register the optional resources/unsubscribe operation. + - resources-unsubscribe + # 2026-09-02: route-harness has no generated test_audio_content tool. + - tools-call-audio + # 2026-09-02: route-harness has no generated test_elicitation tool. + - tools-call-elicitation + # 2026-09-02: route-harness has no generated test_embedded_resource tool. + - tools-call-embedded-resource + # 2026-09-02: route-harness has no generated test_error_handling tool. + - tools-call-error + # 2026-09-02: route-harness has no generated test_image_content tool. + - tools-call-image + # 2026-09-02: route-harness has no generated test_multiple_content_types tool. + - tools-call-mixed-content + # 2026-09-02: route-harness has no generated test_sampling tool. + - tools-call-sampling + # 2026-09-02: route-harness has no generated test_simple_text tool. + - tools-call-simple-text + # 2026-09-02: route-harness has no generated tool that emits SDK logging notifications. + - tools-call-with-logging + # 2026-09-02: route-harness has no generated test_tool_with_progress tool. + - tools-call-with-progress diff --git a/packages/agent-bundle/tests/mcp-conformance.test.ts b/packages/agent-bundle/tests/mcp-conformance.test.ts new file mode 100644 index 000000000..52f95f79d --- /dev/null +++ b/packages/agent-bundle/tests/mcp-conformance.test.ts @@ -0,0 +1,19 @@ +import { expect, it } from '@rstest/core'; + +import { runMcpConformance } from './support/mcp-conformance.ts'; + +const enabled = process.env['AGENT_BUNDLE_MCP_CONFORMANCE'] === '1'; + +it.skipIf(!enabled)( + 'passes the official active MCP server conformance suite against a generated route server', + async () => { + const report = await runMcpConformance(); + + expect(report.runnerVersion).toBe('0.1.16'); + expect(report.specVersion).toBe('2025-11-25'); + expect(report.failed).toBe(report.expectedFailures.length); + expect(report.passed).toBeGreaterThan(0); + expect(report.skipped).toBe(0); + }, + 180_000, +); diff --git a/packages/agent-bundle/tests/support/mcp-conformance.ts b/packages/agent-bundle/tests/support/mcp-conformance.ts new file mode 100644 index 000000000..81385ff34 --- /dev/null +++ b/packages/agent-bundle/tests/support/mcp-conformance.ts @@ -0,0 +1,415 @@ +import { StdioClientTransport } from '@modelcontextprotocol/client/stdio'; +import { + localhostHostValidation, + localhostOriginValidation, + NodeStreamableHTTPServerTransport, +} from '@modelcontextprotocol/node'; +import { + SUPPORTED_PROTOCOL_VERSIONS, + type JSONRPCMessage, + type RequestId, +} from '@modelcontextprotocol/server'; +import { parse } from 'yaml'; +import { randomUUID } from 'node:crypto'; +import { cp, mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises'; +import { + createServer, + type IncomingMessage, + type Server, + type ServerResponse, +} from 'node:http'; +import { createRequire } from 'node:module'; +import type { AddressInfo } from 'node:net'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; + +import { build } from '../../src/api.ts'; +import { runBoundedChildProcess } from '../../src/host-contracts/process.ts'; + +const runnerVersion = '0.1.16'; +const specVersion = '2025-11-25'; +const healthTimeoutMs = 10_000; +const runnerTimeoutMs = 120_000; +const runnerOutputLimitBytes = 4 * 1024 * 1024; +const scenarioSummary = /^[✓✗] ([^:\n]+): (\d+) passed, (\d+) failed(?:, \d+ warnings)?$/gmu; +const suiteSummary = /Running active suite \((\d+) scenarios\)/u; + +const workspaceRoot = resolve(import.meta.dirname, '../../../..'); +const fixtureRoot = resolve(import.meta.dirname, '../../fixtures/route-harness'); +const packageNodeModules = resolve(import.meta.dirname, '../../node_modules'); +const defaultOutputRoot = resolve(workspaceRoot, 'artifacts/mcp-conformance'); +const expectedFailuresPath = resolve( + import.meta.dirname, + '../fixtures/mcp-conformance-expected-failures.yml', +); +const conformanceEntry = createRequire(import.meta.url) + .resolve('@modelcontextprotocol/conformance/dist/index.js'); + +export interface McpConformanceReport { + readonly expectedFailures: readonly string[]; + readonly failed: number; + readonly passed: number; + readonly runnerVersion: string; + readonly skipped: number; + readonly specVersion: string; +} + +interface GeneratedMcpBridge { + readonly close: () => Promise; + readonly diagnostics: () => string; + readonly url: string; +} + +const requestId = (message: JSONRPCMessage): RequestId | undefined => + 'id' in message ? message.id : undefined; + +const isRequest = (message: JSONRPCMessage): boolean => + 'method' in message && 'id' in message; + +const isResponse = (message: JSONRPCMessage): boolean => + 'id' in message && !('method' in message); + +const sessionHeader = (request: IncomingMessage): string | undefined => { + const value = request.headers['mcp-session-id']; + return Array.isArray(value) ? value[0] : value; +}; + +const readJsonBody = async (request: IncomingMessage): Promise => { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + const text = Buffer.concat(chunks).toString('utf8'); + return text === '' ? undefined : JSON.parse(text) as unknown; +}; + +const isInitialize = (body: unknown): boolean => + typeof body === 'object' && + body !== null && + 'method' in body && + body.method === 'initialize'; + +const closeHttpServer = (server: Server): Promise => + new Promise((resolvePromise, rejectPromise) => { + server.close((error) => { + if (error === undefined) resolvePromise(); + else rejectPromise(error); + }); + }); + +const waitForHealth = async (url: string): Promise => { + const signal = AbortSignal.timeout(healthTimeoutMs); + let lastFailure: unknown; + while (!signal.aborted) { + try { + const response = await fetch(url, { signal }); + if (response.ok) return; + lastFailure = new Error(`Health check returned HTTP ${String(response.status)}.`); + } catch (error) { + lastFailure = error; + } + await new Promise((resolvePromise) => setTimeout(resolvePromise, 50)); + } + throw new Error( + `Generated MCP bridge did not become healthy within ${String(healthTimeoutMs)}ms.`, + { cause: lastFailure }, + ); +}; + +/** + * The generated artifact is a managed stdio executable and intentionally has + * no HTTP entry. This bridge leaves that artifact untouched: it forwards raw + * JSON-RPC between the generated stdio process and the official SDK's + * Streamable HTTP transport, owning only transport adaptation and lifecycle. + */ +const startGeneratedMcpBridge = async (options: { + readonly cwd: string; + readonly entry: string; + readonly pluginRoot: string; +}): Promise => { + const stdio = new StdioClientTransport({ + args: [options.entry], + command: process.execPath, + cwd: options.cwd, + env: { + ...process.env, + AGENT_BUNDLE_PLUGIN_ROOT: options.pluginRoot, + } as Record, + stderr: 'pipe', + }); + + let diagnostics = ''; + stdio.stderr?.on('data', (chunk) => { + if (diagnostics.length < runnerOutputLimitBytes) diagnostics += String(chunk); + }); + + const activeRequests = new Map(); + const sessions = new Map(); + const transports = new Set(); + let transportFailure: unknown; + const recordFailure = (error: unknown): void => { + transportFailure ??= error; + }; + stdio.onerror = recordFailure; + stdio.onmessage = (message) => { + const id = requestId(message); + const directTransport = id !== undefined && isResponse(message) + ? activeRequests.get(id) + : undefined; + const onlyActive = activeRequests.size === 1 + ? activeRequests.entries().next().value + : undefined; + const transport = directTransport ?? onlyActive?.[1]; + const relatedRequestId = directTransport !== undefined + ? id + : onlyActive?.[0]; + if (transport === undefined) { + recordFailure(new Error('Generated MCP server emitted a message with no active HTTP request.')); + return; + } + void transport.send( + message, + relatedRequestId === undefined ? undefined : { relatedRequestId }, + ).then(() => { + if (id !== undefined && isResponse(message)) activeRequests.delete(id); + }, recordFailure); + }; + + const createHttpTransport = async (): Promise => { + const transport = new NodeStreamableHTTPServerTransport({ + onsessioninitialized: (sessionId) => { + sessions.set(sessionId, transport); + }, + sessionIdGenerator: randomUUID, + }); + transport.setSupportedProtocolVersions([...SUPPORTED_PROTOCOL_VERSIONS]); + transport.onerror = recordFailure; + transport.onclose = () => { + transports.delete(transport); + const id = transport.sessionId; + if (id !== undefined) sessions.delete(id); + for (const [request, owner] of activeRequests) { + if (owner === transport) activeRequests.delete(request); + } + }; + transport.onmessage = (message) => { + const id = requestId(message); + if (id !== undefined && isRequest(message)) activeRequests.set(id, transport); + void stdio.send(message).catch(recordFailure); + }; + transports.add(transport); + await transport.start(); + return transport; + }; + + const handleMcpRequest = async ( + request: IncomingMessage, + response: ServerResponse, + ): Promise => { + const body = request.method === 'POST' ? await readJsonBody(request) : undefined; + const sessionId = sessionHeader(request); + const transport = sessionId === undefined + ? (isInitialize(body) ? await createHttpTransport() : undefined) + : sessions.get(sessionId); + if (transport === undefined) { + response.writeHead(400, { 'content-type': 'application/json' }); + response.end(JSON.stringify({ + error: { code: -32_000, message: 'Invalid or missing MCP session ID.' }, + id: null, + jsonrpc: '2.0', + })); + return; + } + await transport.handleRequest(request, response, body); + }; + + await stdio.start(); + const validateHost = localhostHostValidation(); + const validateOrigin = localhostOriginValidation(); + let healthy = true; + const server = createServer((request, response) => { + if (request.method === 'GET' && request.url === '/health') { + if (healthy && transportFailure === undefined) response.writeHead(200).end('ok'); + else response.writeHead(503).end('unhealthy'); + return; + } + if (request.url !== '/mcp') { + response.writeHead(404).end(); + return; + } + if (!validateHost(request, response) || !validateOrigin(request, response)) return; + void handleMcpRequest(request, response).catch((error: unknown) => { + recordFailure(error); + if (!response.headersSent) response.writeHead(500); + if (!response.writableEnded) response.end(); + }); + }); + await new Promise((resolvePromise, rejectPromise) => { + server.once('error', rejectPromise); + server.listen({ host: '127.0.0.1', port: 0 }, () => { + server.off('error', rejectPromise); + resolvePromise(); + }); + }); + const address = server.address() as AddressInfo; + const origin = `http://127.0.0.1:${String(address.port)}`; + await waitForHealth(`${origin}/health`); + + let closed = false; + return Object.freeze({ + close: async () => { + if (closed) return; + closed = true; + healthy = false; + await Promise.allSettled([ + closeHttpServer(server), + ...[...transports].map((transport) => transport.close()), + stdio.close(), + ]); + }, + diagnostics: () => diagnostics, + url: `${origin}/mcp`, + }); +}; + +const readExpectedFailures = async (): Promise => { + const document = parse(await readFile(expectedFailuresPath, 'utf8')) as { + readonly server?: unknown; + }; + if ( + !Array.isArray(document.server) || + !document.server.every((entry): entry is string => typeof entry === 'string') + ) { + throw new TypeError('MCP conformance expected failures must be a YAML server string list.'); + } + return Object.freeze([...document.server]); +}; + +const parseReport = ( + output: string, + expectedFailures: readonly string[], +): McpConformanceReport => { + const normalized = output; + let passed = 0; + let failed = 0; + for (const match of normalized.matchAll(scenarioSummary)) { + if (Number(match[3]) === 0) passed += 1; + else failed += 1; + } + if (passed + failed === 0) { + throw new Error(`Official MCP conformance output contained no scenario summaries.\n${normalized}`); + } + const total = Number(suiteSummary.exec(normalized)?.[1] ?? passed + failed); + return Object.freeze({ + expectedFailures, + failed, + passed, + runnerVersion, + skipped: Math.max(0, total - passed - failed), + specVersion, + }); +}; + +export const runMcpConformance = async (): Promise => { + const fixture = await mkdtemp(join(tmpdir(), 'agent-bundle-mcp-conformance-')); + const project = join(fixture, 'route-harness'); + const artifact = join(project, 'artifact'); + const outputRoot = resolve( + process.env['AGENT_BUNDLE_MCP_CONFORMANCE_OUTPUT'] ?? defaultOutputRoot, + ); + await rm(outputRoot, { force: true, recursive: true }); + await mkdir(dirname(outputRoot), { recursive: true }); + const expectedFailures = await readExpectedFailures(); + + let bridge: GeneratedMcpBridge | undefined; + try { + await cp(fixtureRoot, project, { recursive: true }); + // Persistent state injects the runtime's generated notice-inbox source, + // which only the packed fixture's npm install places beneath this copied + // root. Conformance needs one generated MCP surface, not another install + // journey, so omit unrelated state, event, and CLI surfaces and narrow the + // copied fixture config to its generated MCP routes. + await Promise.all([ + rm(join(project, 'src/cli'), { force: true, recursive: true }), + rm(join(project, 'src/events'), { force: true, recursive: true }), + rm(join(project, 'src/state.ts'), { force: true }), + writeFile(join(project, 'agent-bundle.config.ts'), [ + 'export default {', + " plugin: { name: 'route-harness', version: '1.0.0' },", + ' routes: { mcpCommands: true },', + " targets: ['claude'],", + '};', + '', + ].join('\n')), + writeFile(join(project, 'package.json'), JSON.stringify({ + dependencies: { + '@agent-bundle/runtime': 'workspace:*', + '@modelcontextprotocol/server': '2.0.0', + react: '19.2.8', + zod: '4.4.3', + }, + name: 'route-harness-conformance', + private: true, + type: 'module', + version: '1.0.0', + })), + ]); + await symlink(packageNodeModules, join(project, 'node_modules'), 'dir'); + const compiled = await build({ output: artifact, root: project, targets: ['claude'] }); + const entry = compiled.build.compiledMcpEntries.find((candidate) => candidate.id === 'mcp:harness'); + if (entry === undefined) { + throw new Error( + `Route harness build produced no generated MCP server (model: ${compiled.model.mcpServers.map((candidate) => candidate.name).join(', ')}; compiled: ${compiled.build.compiledMcpEntries.map((candidate) => candidate.name).join(', ')}).`, + ); + } + + bridge = await startGeneratedMcpBridge({ + cwd: project, + entry: entry.output, + pluginRoot: join(artifact, 'claude'), + }); + const result = await runBoundedChildProcess({ + args: [ + conformanceEntry, + 'server', + '--url', + bridge.url, + '--suite', + 'active', + '--spec-version', + specVersion, + '--output-dir', + outputRoot, + '--expected-failures', + expectedFailuresPath, + ], + cwd: workspaceRoot, + executable: process.execPath, + }, { + forceFinishMs: 5_000, + labels: { + outputLimit: 'output-limit', + timedOut: 'timed-out', + }, + maxOutputBytes: runnerOutputLimitBytes, + outputBudget: 'combined', + timeoutMs: runnerTimeoutMs, + }); + const output = [result.stdout, result.stderr].filter(Boolean).join('\n'); + const report = parseReport(output, expectedFailures); + if (result.exitCode !== 0 || result.termination !== undefined) { + throw new Error( + [ + `Official MCP conformance failed (exit ${String(result.exitCode)}, termination ${String(result.termination ?? 'none')}).`, + output, + bridge.diagnostics() === '' ? '' : `Generated server stderr:\n${bridge.diagnostics()}`, + ].filter(Boolean).join('\n\n'), + ); + } + console.info( + `MCP conformance ${runnerVersion} / spec ${specVersion}: ${String(report.passed)} passed, ${String(report.failed)} failed, ${String(report.skipped)} skipped.`, + ); + return report; + } finally { + await bridge?.close(); + await rm(fixture, { force: true, recursive: true }); + } +}; diff --git a/packages/agent-bundle/tests/test-harness-manifest.test.ts b/packages/agent-bundle/tests/test-harness-manifest.test.ts index 2c9174b84..d181fc360 100644 --- a/packages/agent-bundle/tests/test-harness-manifest.test.ts +++ b/packages/agent-bundle/tests/test-harness-manifest.test.ts @@ -198,7 +198,7 @@ describe('the compiled test manifest', () => { projected('publish-notice', 'Publishes a durable notice for a later session event.', true), projected('strict-report', 'Returns a closed-object report that rejects unknown serialized keys.', true), projected('ticket', 'Returns a cargo-conductor-shaped ticket status with optional diagnostics fields.', true), - projected('unavailable', undefined, true), + projected('unavailable', 'Returns a typed unavailable result for projection checks.', true), projected('wait', 'Waits until aborted or holdMs elapses, for cancellation contract proof.', true), ]); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1fb8d377f..094830f13 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -23,6 +23,9 @@ importers: '@effect/tsgo': specifier: 0.39.0 version: 0.39.0 + '@modelcontextprotocol/conformance': + specifier: 0.1.16 + version: 0.1.16 '@modelcontextprotocol/server': specifier: 2.0.0 version: 2.0.0 @@ -672,6 +675,10 @@ packages: resolution: {integrity: sha512-8f1OghQ2rjzIOfqgUCP+8GiUWqRs89njoWLNqAe8kWmDePv3s1fZXseej+QXemssEuuOvLLmLO/kqM3IQHtISw==} engines: {node: '>=20'} + '@modelcontextprotocol/conformance@0.1.16': + resolution: {integrity: sha512-GI7qiN0r39/MH2srVUR3AXaEN0YLCro20lIBbnvc1frBhszenxvUifBuTzxeVQVagILfBzCIcnungUOma8OrgA==} + hasBin: true + '@modelcontextprotocol/core@2.0.0': resolution: {integrity: sha512-pJCEwGG7Lfr/+PQp9ZTwKXNeO5wzbfKL7H3MYpCorM4oFBoQrdjnBgEoqG+RjhsvS1FKrDbKux+M1HhlnGWqcA==} engines: {node: '>=20'} @@ -762,6 +769,64 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} + '@octokit/auth-token@6.0.0': + resolution: {integrity: sha512-P4YJBPdPSpWTQ1NU4XYdvHvXJJDxM6YwpS0FZHRgP7YFkdVxsWcpWGy/NVqlAA7PcPCnMacXlRm1y2PFZRWL/w==} + engines: {node: '>= 20'} + + '@octokit/core@7.0.8': + resolution: {integrity: sha512-L7y8eYc+AwxGr2PWI4WFt1VG4TiJ66c26BD16mXpYIlXxG0SMigM1+m4aTSlYyBr5BlQsGAlz8uDCoZN4SEMcg==} + engines: {node: '>= 20'} + + '@octokit/endpoint@11.0.5': + resolution: {integrity: sha512-iXa654H3yFafF/ieHkukfbgWo2rmXD2ceD0ZOtrPhw1bc3FDch1d9N/TNs0FQ1/cIbwb7kspUX8jzIs8nzb9DQ==} + engines: {node: '>= 20'} + + '@octokit/graphql@9.0.5': + resolution: {integrity: sha512-bt/hm03LeU6Vy7FwTrkkC9p3XGT/lBwClglMqxBSe5/q0E5CdJTXeAqEI0vlw89/LF/G6tryTIH8HirZ3prMVg==} + engines: {node: '>= 20'} + + '@octokit/openapi-types@27.0.0': + resolution: {integrity: sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==} + + '@octokit/openapi-types@29.0.1': + resolution: {integrity: sha512-9qWOMFNxxLokERcms42rU0PTLqQmVs7g5E41TI4mCOxmpFayD1rfC7XxOL55cG9MBZLFlC31BrR37myMKardwg==} + + '@octokit/plugin-paginate-rest@14.0.0': + resolution: {integrity: sha512-fNVRE7ufJiAA3XUrha2omTA39M6IXIc6GIZLvlbsm8QOQCYvpq/LkMNGyFlB1d8hTDzsAXa3OKtybdMAYsV/fw==} + engines: {node: '>= 20'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/plugin-request-log@6.0.0': + resolution: {integrity: sha512-UkOzeEN3W91/eBq9sPZNQ7sUBvYCqYbrrD8gTbBuGtHEuycE4/awMXcYvx6sVYo7LypPhmQwwpUe4Yyu4QZN5Q==} + engines: {node: '>= 20'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/plugin-rest-endpoint-methods@17.0.0': + resolution: {integrity: sha512-B5yCyIlOJFPqUUeiD0cnBJwWJO8lkJs5d8+ze9QDP6SvfiXSz1BF+91+0MeI1d2yxgOhU/O+CvtiZ9jSkHhFAw==} + engines: {node: '>= 20'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/request-error@7.1.2': + resolution: {integrity: sha512-XZRuT3xZ84D3gYErI1DZvhJ33dCWVV6uzBtWkaBB4TvA/L6eOeTZodxLFVB44bBEEo3vEx7y00UfX1tBLrtLRg==} + engines: {node: '>= 20'} + + '@octokit/request@10.0.16': + resolution: {integrity: sha512-A0zWGjHzISIb+9ccG8s0dq7LKO5zVpJLRICjgUb+sJxEWqn8RUHB1rD3AE51+PECvXHIxqZ1VVvs4fHTSD9nUQ==} + engines: {node: '>= 20'} + + '@octokit/rest@22.0.1': + resolution: {integrity: sha512-Jzbhzl3CEexhnivb1iQ0KJ7s5vvjMWcmRtq5aUsKmKDrRW6z3r84ngmiFKFvpZjpiU/9/S6ITPFRpn5s/3uQJw==} + engines: {node: '>= 20'} + + '@octokit/types@16.0.0': + resolution: {integrity: sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==} + + '@octokit/types@18.0.0': + resolution: {integrity: sha512-l6bAF43PNxkJp6g+W4PjoUSSkxHomXw2nOum5CTftJz1NlV3vu93NImgOYtLf6CbBUb5j+fiuzW0PPQ5JTSvZA==} + '@pnpm/deps.graph-sequencer@1100.0.1': resolution: {integrity: sha512-pOr5+q1fLYKwFN3LAJuGZEnfXDcQ73zqgDHMtGy+K+uIoUqyY+6MeDCWFwfu+4EFuq76I5EPFofoNAI+Bmmq4A==} engines: {node: '>=22.13'} @@ -1407,6 +1472,9 @@ packages: bail@2.0.2: resolution: {integrity: sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==} + before-after-hook@4.0.0: + resolution: {integrity: sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ==} + body-parser@2.3.0: resolution: {integrity: sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==} engines: {node: '>=18'} @@ -1495,6 +1563,10 @@ packages: resolution: {integrity: sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==} engines: {node: '>=14'} + commander@14.0.3: + resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==} + engines: {node: '>=20'} + commander@15.0.0: resolution: {integrity: sha512-z67u4ZhzCL/Tydu1lJARtEZYWbWaN7oYLHbsuzocr6y4N6WZAagG3RQ4FW61V1/0+jImpj293XfrcYnd1qxtPg==} engines: {node: '>=22.12.0'} @@ -1511,6 +1583,10 @@ packages: resolution: {integrity: sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==} engines: {node: '>=18'} + content-type@3.0.0: + resolution: {integrity: sha512-AIi5H6p0xk5uknXcN3/rmhP8jgp69OfSe/JuKiQAFprJ7UGw7mwj7m4XcmDzlrnJDG+cGpphAINGdU3g3g7kDw==} + engines: {node: '>=22'} + convert-source-map@2.0.0: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} @@ -1906,6 +1982,9 @@ packages: json-schema-typed@8.0.2: resolution: {integrity: sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==} + json-with-bigint@3.5.12: + resolution: {integrity: sha512-uwbF/wSSuOgC7qqlq27Xp5B6a2MHVug3t0idZdTqu0JnlFvgJuH7ju+KAk/J06C7GfhoYy2gnb9wz2INqcne7w==} + jsonc-parser@3.3.1: resolution: {integrity: sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==} @@ -2527,6 +2606,10 @@ packages: undici-types@8.3.0: resolution: {integrity: sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==} + undici@7.29.0: + resolution: {integrity: sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==} + engines: {node: '>=20.18.1'} + unicode-emoji-modifier-base@1.0.0: resolution: {integrity: sha512-yLSH4py7oFH3oG/9K+XWrz1pSi3dfUrWEnInbxMfArOfc1+33BlGPQtLsOYwvdMy11AwUBetYuaRxSPqgkq+8g==} engines: {node: '>=4'} @@ -2557,6 +2640,9 @@ packages: unist-util-visit@5.1.0: resolution: {integrity: sha512-m+vIdyeCOpdr/QeQCu2EzxX/ohgS8KbnPDgFni4dQsfSCtpz8UqDyY5GjRru8PDKuYn7Fq19j1CQ+nJSsGKOzg==} + universal-user-agent@7.0.3: + resolution: {integrity: sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A==} + unpipe@1.0.0: resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} engines: {node: '>= 0.8'} @@ -2921,6 +3007,21 @@ snapshots: pkce-challenge: 5.0.1 zod: 4.4.3 + '@modelcontextprotocol/conformance@0.1.16': + dependencies: + '@modelcontextprotocol/sdk': 1.30.0(supports-color@7.2.0)(zod@4.4.3) + '@octokit/rest': 22.0.1 + commander: 14.0.3 + eventsource-parser: 3.1.1 + express: 5.2.1(supports-color@7.2.0) + jose: 6.2.10 + undici: 7.29.0 + yaml: 2.9.0 + zod: 4.4.3 + transitivePeerDependencies: + - '@cfworker/json-schema' + - supports-color + '@modelcontextprotocol/core@2.0.0': dependencies: zod: 4.4.3 @@ -3005,6 +3106,75 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.20.1 + '@octokit/auth-token@6.0.0': {} + + '@octokit/core@7.0.8': + dependencies: + '@octokit/auth-token': 6.0.0 + '@octokit/graphql': 9.0.5 + '@octokit/request': 10.0.16 + '@octokit/request-error': 7.1.2 + '@octokit/types': 18.0.0 + before-after-hook: 4.0.0 + universal-user-agent: 7.0.3 + + '@octokit/endpoint@11.0.5': + dependencies: + '@octokit/types': 18.0.0 + universal-user-agent: 7.0.3 + + '@octokit/graphql@9.0.5': + dependencies: + '@octokit/request': 10.0.16 + '@octokit/types': 18.0.0 + universal-user-agent: 7.0.3 + + '@octokit/openapi-types@27.0.0': {} + + '@octokit/openapi-types@29.0.1': {} + + '@octokit/plugin-paginate-rest@14.0.0(@octokit/core@7.0.8)': + dependencies: + '@octokit/core': 7.0.8 + '@octokit/types': 16.0.0 + + '@octokit/plugin-request-log@6.0.0(@octokit/core@7.0.8)': + dependencies: + '@octokit/core': 7.0.8 + + '@octokit/plugin-rest-endpoint-methods@17.0.0(@octokit/core@7.0.8)': + dependencies: + '@octokit/core': 7.0.8 + '@octokit/types': 16.0.0 + + '@octokit/request-error@7.1.2': + dependencies: + '@octokit/types': 18.0.0 + + '@octokit/request@10.0.16': + dependencies: + '@octokit/endpoint': 11.0.5 + '@octokit/request-error': 7.1.2 + '@octokit/types': 18.0.0 + content-type: 3.0.0 + json-with-bigint: 3.5.12 + universal-user-agent: 7.0.3 + + '@octokit/rest@22.0.1': + dependencies: + '@octokit/core': 7.0.8 + '@octokit/plugin-paginate-rest': 14.0.0(@octokit/core@7.0.8) + '@octokit/plugin-request-log': 6.0.0(@octokit/core@7.0.8) + '@octokit/plugin-rest-endpoint-methods': 17.0.0(@octokit/core@7.0.8) + + '@octokit/types@16.0.0': + dependencies: + '@octokit/openapi-types': 27.0.0 + + '@octokit/types@18.0.0': + dependencies: + '@octokit/openapi-types': 29.0.1 + '@pnpm/deps.graph-sequencer@1100.0.1': {} '@polka/url@1.0.0-next.29': {} @@ -3513,6 +3683,8 @@ snapshots: bail@2.0.2: {} + before-after-hook@4.0.0: {} + body-parser@2.3.0(supports-color@7.2.0): dependencies: bytes: 3.1.2 @@ -3605,6 +3777,8 @@ snapshots: commander@10.0.1: {} + commander@14.0.3: {} + commander@15.0.0: {} content-disposition@1.1.0: {} @@ -3613,6 +3787,8 @@ snapshots: content-type@2.1.0: {} + content-type@3.0.0: {} + convert-source-map@2.0.0: {} cookie-signature@1.2.2: {} @@ -4004,6 +4180,8 @@ snapshots: json-schema-typed@8.0.2: {} + json-with-bigint@3.5.12: {} + jsonc-parser@3.3.1: {} launch-editor@2.14.1: @@ -4883,6 +5061,8 @@ snapshots: undici-types@8.3.0: {} + undici@7.29.0: {} + unicode-emoji-modifier-base@1.0.0: {} unicorn-magic@0.3.0: {} @@ -4922,6 +5102,8 @@ snapshots: unist-util-is: 6.0.1 unist-util-visit-parents: 6.0.2 + universal-user-agent@7.0.3: {} + unpipe@1.0.0: {} validate-npm-package-name@5.0.1: {} diff --git a/rstest.config.ts b/rstest.config.ts index 7f63dc431..7356b7b09 100644 --- a/rstest.config.ts +++ b/rstest.config.ts @@ -1,6 +1,11 @@ import { defineConfig } from '@rstest/core'; -import { projectionTestFiles, routeUnitTestFiles, templateTestFiles } from './rstest.integration-tests.ts'; +import { + mcpConformanceTestFiles, + projectionTestFiles, + routeUnitTestFiles, + templateTestFiles, +} from './rstest.integration-tests.ts'; import { withAgentBundleRslibConfig } from './rstest.rslib.ts'; export default defineConfig({ @@ -8,7 +13,12 @@ export default defineConfig({ include: [ 'packages/**/tests/**/*.test.ts', ], - exclude: [...projectionTestFiles, ...routeUnitTestFiles, ...templateTestFiles], + exclude: [ + ...mcpConformanceTestFiles, + ...projectionTestFiles, + ...routeUnitTestFiles, + ...templateTestFiles, + ], // The e2e fixtures copy the shared rsc-agent-runtime example dist; build it // once in the orchestrator so parallel workers never race the ensure-build. globalSetup: ['./rstest.integration.setup.ts'], diff --git a/rstest.integration-tests.ts b/rstest.integration-tests.ts index e14705b5c..ec68b8932 100644 --- a/rstest.integration-tests.ts +++ b/rstest.integration-tests.ts @@ -100,6 +100,15 @@ export const nightlyEvidenceTestFiles: readonly string[] = [ 'packages/workbench/tests/runtime-playground-capture.test.ts', ]; +/** + * Official MCP server conformance runs only through the manually dispatched + * lane. It builds one generated fixture, opens loopback HTTP, and invokes the + * external runner, so no default Rstest pool may collect it. + */ +export const mcpConformanceTestFiles: readonly string[] = [ + 'packages/agent-bundle/tests/mcp-conformance.test.ts', +]; + /** * Pack-and-install tests: each one consumes the run-level release tarball * (and usually a clean `npm install` of it), which dominates the serialized diff --git a/rstest.mcp-conformance.config.ts b/rstest.mcp-conformance.config.ts new file mode 100644 index 000000000..9c0755858 --- /dev/null +++ b/rstest.mcp-conformance.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from '@rstest/core'; + +import { mcpConformanceTestFiles } from './rstest.integration-tests.ts'; +import { withAgentBundleRslibConfig } from './rstest.rslib.ts'; + +export default defineConfig({ + extends: withAgentBundleRslibConfig(), + include: [...mcpConformanceTestFiles], + pool: { maxWorkers: 1 }, + setupFiles: ['./rstest.setup.ts'], + testTimeout: 180_000, +}); diff --git a/rstest.unit.config.ts b/rstest.unit.config.ts index 0eae76bea..c474463a9 100644 --- a/rstest.unit.config.ts +++ b/rstest.unit.config.ts @@ -2,6 +2,7 @@ import { defineConfig } from '@rstest/core'; import { integrationTestFiles, + mcpConformanceTestFiles, nightlyEvidenceTestFiles, packedReleaseOnlyTestFiles, packedTestFiles, @@ -19,6 +20,7 @@ export default defineConfig({ ], exclude: [ ...integrationTestFiles, + ...mcpConformanceTestFiles, ...nightlyEvidenceTestFiles, // Packs and installs like packedTestFiles, and is release-boundary-only: // `test:packed:release` owns it, not the build-free per-PR pool.