test(mcp): opt-in official conformance lane against a generated route server (#133)

.changeset/official-mcp-conformance.md

@@ -0,0 +1,9 @@
+---
+"agent-bundle": minor
+---
+
+Add an opt-in MCP conformance lane that builds a generated route server,
+adapts its stdio transport to loopback Streamable HTTP, and runs the official
+`@modelcontextprotocol/conformance` active suite for specification
+`2025-11-25`. The manually dispatched workflow preserves per-scenario runner
+artifacts and keeps known gaps in a stale-detecting expected-failure baseline.

.github/workflows/mcp-conformance.yml

@@ -0,0 +1,36 @@
+name: MCP conformance
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: mcp-conformance-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  mcp-conformance:
+    name: MCP conformance (Node 22.19)
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v7
+      - uses: pnpm/setup@v2
+        with:
+          cache: true
+          install: false
+          runtime: node@22.19.0
+      - run: pnpm install --frozen-lockfile
+      - run: pnpm test:mcp-conformance
+        env:
+          AGENT_BUNDLE_MCP_CONFORMANCE_OUTPUT: artifacts/mcp-conformance
+      - name: Preserve MCP conformance results
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          if-no-files-found: warn
+          name: mcp-conformance-results
+          path: artifacts/mcp-conformance
+          retention-days: 14

.gitignore

@@ -3,6 +3,7 @@ dist/
 .agent-bundle/
 .worktrees/
 coverage/
+artifacts/
 *.log
 examples/audiobook-curator/artifact/
 

docs/mcp-conformance.md

@@ -0,0 +1,45 @@
+# MCP conformance evidence
+
+The MCP conformance lane is intentionally opt-in. Run it locally with:
+
+```sh
+pnpm test:mcp-conformance
+```
+
+CI exposes the same command through the manually dispatched `MCP conformance`
+workflow. The lane uses one Node version and one existing fixture, and uploads
+the official runner's `checks.json` artifacts from `artifacts/mcp-conformance`.
+It is not part of pull request, push, scheduled, or default Rstest execution.
+
+## Transport and fixture
+
+Generated agent-bundle MCP artifacts are managed stdio executables; they do
+not expose an HTTP transport seam. The harness therefore builds the existing
+`packages/agent-bundle/fixtures/route-harness` generated-route fixture through
+the public build pipeline, starts the generated stdio artifact, and forwards
+raw JSON-RPC frames through the official SDK's
+`NodeStreamableHTTPServerTransport`. The adapter owns only loopback transport,
+health checking, bounded runner execution, and teardown. Route registration,
+Flight rendering, schemas, and responses remain the generated artifact's
+logic.
+
+The copied fixture is narrowed to its MCP routes before building. Its state,
+event, and CLI sources are omitted because conformance needs one generated MCP
+surface and must not add the packed fixture's separate npm-install journey.
+
+## Recorded run: 2026-09-02
+
+- Runner: `@modelcontextprotocol/conformance@0.1.16`
+- Suite: `server --suite active`
+- Specification: `2025-11-25`
+- Scenarios: 30 total; 7 passed, 23 expected failures, 0 skipped
+- Unexpected failures: 0
+
+The dated expected-failure baseline is
+`packages/agent-bundle/tests/fixtures/mcp-conformance-expected-failures.yml`.
+It records three unsupported generated-server operation families
+(`completion/complete`, `logging/setLevel`, and resource subscriptions) plus
+the canonical content-specific tools, resources, and prompts that the reused
+route harness does not provide. The official runner rejects both unexpected
+failures and stale baseline entries, so newly fixed scenarios must be removed
+from the baseline.

package.json

@@ -17,6 +17,7 @@
     "test:integration": "pnpm build && pnpm test:integration:run",
     "test:integration:run": "AGENT_BUNDLE_WORKBENCH_PREBUILT=1 AGENT_BUNDLE_PACKAGE_PREBUILT=1 rstest --config rstest.integration.config.ts",
     "test:evidence": "pnpm build && AGENT_BUNDLE_WORKBENCH_PREBUILT=1 AGENT_BUNDLE_PACKAGE_PREBUILT=1 rstest --config rstest.evidence.config.ts",
+    "test:mcp-conformance": "pnpm build && AGENT_BUNDLE_MCP_CONFORMANCE=1 rstest --config rstest.mcp-conformance.config.ts",
     "test:watch": "rstest --config rstest.config.ts --watch",
     "lint": "rslint .",
     "bench:hook-cold-start": "node scripts/measure-hook-cold-start.mjs",
@@ -57,6 +58,7 @@
     "@changesets/cli": "3.0.1",
     "@effect/language-service": "0.87.2",
     "@effect/tsgo": "0.39.0",
+    "@modelcontextprotocol/conformance": "0.1.16",
     "@modelcontextprotocol/server": "2.0.0",
     "@rsbuild/core": "2.2.1",
     "@rsbuild/plugin-react": "2.1.0",

packages/agent-bundle/fixtures/route-harness/src/mcp/harness/tools/unavailable.tsx

@@ -1,7 +1,10 @@
 import { Agent } from '@agent-bundle/runtime';
 import { z } from 'zod';
 
-export const config = { title: 'Unavailable' };
+export const config = {
+  description: 'Returns a typed unavailable result for projection checks.',
+  title: 'Unavailable',
+};
 
 export const inputSchema = z.object({});
 

packages/agent-bundle/tests/fixtures/mcp-conformance-expected-failures.yml

@@ -0,0 +1,47 @@
+server:
+  # 2026-09-02: Generated route servers do not register the optional completion/complete operation.
+  - completion-complete
+  # 2026-09-02: Generated route servers do not register the optional logging/setLevel operation.
+  - logging-set-level
+  # 2026-09-02: route-harness has no generated test_elicitation_sep1034_defaults tool.
+  - elicitation-sep1034-defaults
+  # 2026-09-02: route-harness has no generated test_elicitation_sep1330_enums tool.
+  - elicitation-sep1330-enums
+  # 2026-09-02: route-harness has no generated test_prompt_with_embedded_resource prompt.
+  - prompts-get-embedded-resource
+  # 2026-09-02: route-harness has no generated test_simple_prompt prompt.
+  - prompts-get-simple
+  # 2026-09-02: route-harness has no generated test_prompt_with_arguments prompt.
+  - prompts-get-with-args
+  # 2026-09-02: route-harness has no generated test_prompt_with_image prompt.
+  - prompts-get-with-image
+  # 2026-09-02: route-harness has no generated test://static-binary resource.
+  - resources-read-binary
+  # 2026-09-02: route-harness has no generated test://static-text resource.
+  - resources-read-text
+  # 2026-09-02: Generated static resources do not register the optional resources/subscribe operation.
+  - resources-subscribe
+  # 2026-09-02: route-harness has no generated test://template/{id}/data resource template.
+  - resources-templates-read
+  # 2026-09-02: Generated static resources do not register the optional resources/unsubscribe operation.
+  - resources-unsubscribe
+  # 2026-09-02: route-harness has no generated test_audio_content tool.
+  - tools-call-audio
+  # 2026-09-02: route-harness has no generated test_elicitation tool.
+  - tools-call-elicitation
+  # 2026-09-02: route-harness has no generated test_embedded_resource tool.
+  - tools-call-embedded-resource
+  # 2026-09-02: route-harness has no generated test_error_handling tool.
+  - tools-call-error
+  # 2026-09-02: route-harness has no generated test_image_content tool.
+  - tools-call-image
+  # 2026-09-02: route-harness has no generated test_multiple_content_types tool.
+  - tools-call-mixed-content
+  # 2026-09-02: route-harness has no generated test_sampling tool.
+  - tools-call-sampling
+  # 2026-09-02: route-harness has no generated test_simple_text tool.
+  - tools-call-simple-text
+  # 2026-09-02: route-harness has no generated tool that emits SDK logging notifications.
+  - tools-call-with-logging
+  # 2026-09-02: route-harness has no generated test_tool_with_progress tool.
+  - tools-call-with-progress

packages/agent-bundle/tests/mcp-conformance.test.ts

@@ -0,0 +1,19 @@
+import { expect, it } from '@rstest/core';
+
+import { runMcpConformance } from './support/mcp-conformance.ts';
+
+const enabled = process.env['AGENT_BUNDLE_MCP_CONFORMANCE'] === '1';
+
+it.skipIf(!enabled)(
+  'passes the official active MCP server conformance suite against a generated route server',
+  async () => {
+    const report = await runMcpConformance();
+
+    expect(report.runnerVersion).toBe('0.1.16');
+    expect(report.specVersion).toBe('2025-11-25');
+    expect(report.failed).toBe(report.expectedFailures.length);
+    expect(report.passed).toBeGreaterThan(0);
+    expect(report.skipped).toBe(0);
+  },
+  180_000,
+);