Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/conformance.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ jobs:
- run: pnpm install
- run: pnpm run build:all
- run: pnpm run test:conformance:client:all
- run: pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:client:2026

server-conformance:
runs-on: ubuntu-latest
Expand All @@ -48,3 +49,4 @@ jobs:
- run: pnpm run build:all
- run: pnpm run test:conformance:server
- run: pnpm run test:conformance:server:draft
- run: pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server:2026
2 changes: 1 addition & 1 deletion packages/client/test/client/probeClassifier.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,7 @@ describe('row: plain-text/unparseable 400, code 0, empty body, 406, any unrecogn
});

describe('row: -32001 / -32003 are NEVER probe-recognized → fall into unrecognized → legacy', () => {
test('-32001 (session-404 overload on deployed servers; ladder cell underived pending conformance #336)', () => {
test('-32001 (session-404 overload on deployed servers; the spec-assigned HeaderMismatch code is still never probe evidence)', () => {
expect(classify({ kind: 'rpc-error', code: -32_001, message: 'Session not found' })).toEqual({ kind: 'legacy' });
expect(classify({ kind: 'http-error', status: 404, body: httpErrorBody(-32_001, 'Session not found') })).toEqual({
kind: 'legacy'
Expand Down
10 changes: 5 additions & 5 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

106 changes: 106 additions & 0 deletions test/conformance/expected-failures.2026-07-28.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
# Expected failures for the carried-forward x 2026-07-28 legs
# (`test:conformance:client:2026` and `test:conformance:server:2026`, both
# `--suite all --spec-version 2026-07-28`).
#
# This baseline is separate from expected-failures.yaml because entries are
# keyed by scenario name only: a scenario that passes at its default version
# in the 2025 legs but fails when forced to 2026-07-28 (or vice versa) cannot
# be expressed in a shared file (the passing leg would flag the entry as
# stale). Like expected-failures.yaml, this single file covers both
# directions: the client 2026 leg reads the `client:` section and the server
# 2026 leg reads the `server:` section. Both burn down independently of the
# 2025 legs.
#
# Baseline established against the published @modelcontextprotocol/conformance
# release pinned in package.json. Newer conformance releases are adopted by
# deliberately bumping the pin and reconciling this file in the same change.
#
# Entries are grouped by what unblocks them. As each gap closes the
# corresponding scenarios start passing and MUST be removed from this list
# (the runner fails on stale entries), so the baseline burns down per
# milestone.

client:
# --- SEP-837 (application_type during DCR) ---
# The sep-837-application-type-present check only fires on draft-version
# runs; the client omits application_type during Dynamic Client
# Registration, so every auth scenario that reaches DCR fails it on this
# leg (the same scenarios pass at their default version in the 2025 legs).
- auth/metadata-default
- auth/metadata-var1
- auth/metadata-var2
- auth/metadata-var3
- auth/scope-from-www-authenticate
- auth/scope-from-scopes-supported
- auth/scope-omitted-when-undefined
- auth/token-endpoint-auth-basic
- auth/token-endpoint-auth-post
- auth/token-endpoint-auth-none
- auth/offline-access-not-supported

# --- Auth scenarios cut short by the 2026 connection lifecycle ---
# The fixture's auth flow drives the 2025 stateful lifecycle; the
# 2026-mode mock rejects the MCP POST (-32001, missing
# MCP-Protocol-Version header) before the scope-escalation behaviour these
# scenarios measure, so no authorization requests are observed. Unblocks
# when the auth fixture flow speaks the 2026 per-request lifecycle.
- auth/scope-step-up
- auth/scope-retry-limit

# --- Same gaps as the 2025 baseline (fail identically when forced to 2026-07-28) ---
# SEP-2322 (multi-round-trip requests): client does not echo requestState /
# handle IncompleteResult yet.
- sep-2322-client-request-state
# SEP-2243 (HTTP standardization): no fixture handler / client header support yet.
- http-custom-headers
- http-invalid-tool-headers
# SEP-2106 (JSON Schema $ref handling): no fixture handler for the scenario yet.
- json-schema-ref-no-deref
# SEP-2468 (authorization response iss parameter): not implemented in the client.
- auth/iss-supported
- auth/iss-not-advertised
- auth/iss-supported-missing
- auth/iss-wrong-issuer
- auth/iss-unexpected
- auth/iss-normalized
- auth/metadata-issuer-mismatch
# SEP-2352 (authorization server migration): client does not re-register
# when PRM authorization_servers changes.
- auth/authorization-server-migration

server:
# --- Carried-forward scenarios (also run by the 2025 legs) ---
# Pre-existing fixture/baseline bug: the fixture tool's schema is a plain
# Zod object with none of the JSON Schema 2020-12 keywords the scenario
# checks; it fails identically at 2025 in `--suite all` (not a 2026-path
# regression).
- json-schema-2020-12
# SEP-2164: server returns -32002 without the requested URI in error.data
# (WARNING-only; the expected-failures evaluator counts WARNINGs as
# failures). Same failure as in the 2025 baseline.
- sep-2164-resource-not-found

# --- Draft scenarios (same failures and reasons as the `--suite draft` leg) ---
# SEP-2243 (HTTP header standardization): the reject cells the SDK does
# answer now use -32001 (HeaderMismatch), but missing-header enforcement
# (Mcp-Method, Mcp-Name) and the Mcp-Name cross-check are not implemented,
# so those reject cells are still accepted with 200.
- http-header-validation
# SEP-2322 (multi-round-trip requests / IncompleteResult): not implemented
# in the SDK, so the fixture does not register the scenarios' diagnostic
# test_input_required_result_* tools.
- input-required-result-basic-elicitation
- input-required-result-basic-sampling
- input-required-result-basic-list-roots
- input-required-result-request-state
- input-required-result-multiple-input-requests
- input-required-result-multi-round
- input-required-result-non-tool-request
- input-required-result-result-type
- input-required-result-tampered-state
- input-required-result-capability-check
# SEP-2322 SHOULD-level behaviours (re-request missing inputResponses,
# ignore unrecognized inputResponses keys): WARNING-only, but the
# expected-failures evaluator counts WARNINGs as failures.
- input-required-result-missing-input-response
- input-required-result-ignore-extra-params
24 changes: 5 additions & 19 deletions test/conformance/expected-failures.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,9 @@
# CI exits 0 if only these fail, exits 1 on unexpected failures or stale entries.
#
# Baseline established against the published @modelcontextprotocol/conformance
# release pinned in package.json (0.2.0-alpha.3). Newer conformance releases
# release pinned in package.json (0.2.0-alpha.4). Newer conformance releases
# are adopted by deliberately bumping the package.json pin and reconciling
# this file in the same change. 0.2.0-alpha.3 fixes the draft wire version
# (2026-07-28). Several auth scenarios in this baseline (auth/iss-*,
# auth/authorization-server-migration, auth/enterprise-managed-authorization)
# are still not shipped in the published release — the runner reports them
# unknown/failed; their entries below cover them either way.
# this file in the same change.
#
# NOTE: the SDK's modern-path rejection codes are aligned with what this
# referee asserts: header/body mismatches answer -32001 (HeaderMismatch) and a
Expand All @@ -22,9 +18,6 @@

client:
# --- Draft-spec scenarios (in `--suite draft`, also part of `--suite all`) ---
# SEP-2575 (request metadata / _meta envelope): client does not populate the
# _meta envelope or the MCP-Protocol-Version header semantics yet.
- request-metadata
# SEP-2322 (multi-round-trip requests): client does not echo requestState /
# handle IncompleteResult yet.
- sep-2322-client-request-state
Expand Down Expand Up @@ -59,12 +52,9 @@ client:

server:
# --- Draft-spec scenarios (in `--suite draft`; the default `active` suite is green) ---
# SEP-2575 (stateless HTTP / _meta envelope): server has no stateless mode,
# _meta-derived capabilities, error-code mappings, or server/discover yet.
- server-stateless
# SEP-2322 (multi-round-trip requests / IncompleteResult): not implemented;
# most scenarios currently fail early with "Session ID required" because the
# fixture only runs in stateful mode.
# SEP-2322 (multi-round-trip requests / IncompleteResult): not implemented
# in the SDK, so the fixture does not register the scenarios' diagnostic
# test_input_required_result_* tools.
- input-required-result-basic-elicitation
- input-required-result-basic-sampling
- input-required-result-basic-list-roots
Expand All @@ -75,15 +65,11 @@ server:
- input-required-result-result-type
- input-required-result-tampered-state
- input-required-result-capability-check
# SEP-2549 (caching): no ttlMs/cacheScope support; scenario also hits the
# stateful-mode "Session ID required" error.
- caching
# SEP-2243 (HTTP header standardization): the reject cells the SDK does
# answer now use -32001 (HeaderMismatch), but missing-header enforcement
# (Mcp-Method, Mcp-Name) and the Mcp-Name cross-check are not implemented,
# so those reject cells are still accepted with 200.
- http-header-validation
- http-custom-header-server-validation
# WARNING-only entries: these scenarios emit no FAILURE checks, only SHOULD-level
# WARNINGs, but the expected-failures evaluator counts WARNINGs as failures.
# SEP-2164: server returns -32002 without the requested URI in error.data.
Expand Down
4 changes: 3 additions & 1 deletion test/conformance/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,17 @@
"client": "tsx scripts/cli.ts client",
"test:conformance:client": "conformance client --command 'node --import tsx ./src/everythingClient.ts' --suite core --expected-failures ./expected-failures.yaml",
"test:conformance:client:all": "conformance client --command 'node --import tsx ./src/everythingClient.ts' --suite all --expected-failures ./expected-failures.yaml",
"test:conformance:client:2026": "conformance client --command 'node --import tsx ./src/everythingClient.ts' --suite all --spec-version 2026-07-28 --expected-failures ./expected-failures.2026-07-28.yaml",
"test:conformance:client:run": "node --import tsx ./src/everythingClient.ts",
"test:conformance:server": "scripts/run-server-conformance.sh --expected-failures ./expected-failures.yaml",
"test:conformance:server:draft": "scripts/run-server-conformance.sh --suite draft --expected-failures ./expected-failures.yaml",
"test:conformance:server:all": "scripts/run-server-conformance.sh --suite all --expected-failures ./expected-failures.yaml",
"test:conformance:server:2026": "scripts/run-server-conformance.sh --suite all --spec-version 2026-07-28 --expected-failures ./expected-failures.2026-07-28.yaml",
"test:conformance:server:run": "node --import tsx ./src/everythingServer.ts",
"test:conformance:all": "pnpm run test:conformance:client:all && pnpm run test:conformance:server:all"
},
"devDependencies": {
"@modelcontextprotocol/conformance": "0.2.0-alpha.3",
"@modelcontextprotocol/conformance": "0.2.0-alpha.4",
"@modelcontextprotocol/client": "workspace:^",
"@modelcontextprotocol/server": "workspace:^",
"@modelcontextprotocol/core": "workspace:^",
Expand Down
91 changes: 91 additions & 0 deletions test/conformance/src/everythingClient.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,12 @@

import {
Client,
CLIENT_CAPABILITIES_META_KEY,
CLIENT_INFO_META_KEY,
ClientCredentialsProvider,
CrossAppAccessProvider,
PrivateKeyJwtProvider,
PROTOCOL_VERSION_META_KEY,
requestJwtAuthorizationGrant,
StreamableHTTPClientTransport
} from '@modelcontextprotocol/client';
Expand Down Expand Up @@ -96,6 +99,38 @@ function registerScenarios(names: string[], handler: ScenarioHandler): void {
}
}

// ============================================================================
// 2026-07-28 (modern era) helpers
// ============================================================================

/**
* Spec versions whose wire lifecycle is the 2026-07-28 per-request envelope
* (no `initialize` handshake). The conformance runner passes the resolved
* spec version of the current scenario run via the
* MCP_CONFORMANCE_PROTOCOL_VERSION environment variable; when it names a
* modern version, version-spanning scenarios (e.g. tools_call) must speak the
* modern lifecycle instead of the 2025 stateful one.
*/
const MODERN_SPEC_VERSIONS = new Set(['2026-07-28']);

function isModernConformanceRun(): boolean {
const version = process.env.MCP_CONFORMANCE_PROTOCOL_VERSION;
return version !== undefined && MODERN_SPEC_VERSIONS.has(version);
}

/**
* The per-request `_meta` envelope every 2026-era request carries on the wire.
* Automatic envelope emission is not implemented in the client yet (it is a
* client-side follow-up), so modern-era requests attach it explicitly.
*/
function modernEnvelope(clientInfo: { name: string; version: string }, capabilities: object, protocolVersion: string | undefined) {
return {
[PROTOCOL_VERSION_META_KEY]: protocolVersion ?? '2026-07-28',
[CLIENT_INFO_META_KEY]: clientInfo,
[CLIENT_CAPABILITIES_META_KEY]: capabilities
};
}

// ============================================================================
// Basic scenarios (initialize, tools_call)
// ============================================================================
Expand All @@ -117,6 +152,10 @@ async function runBasicClient(serverUrl: string): Promise<void> {

// tools_call scenario needs to actually call a tool
async function runToolsCallClient(serverUrl: string): Promise<void> {
if (isModernConformanceRun()) {
return runToolsCallModernClient(serverUrl);
}

const client = new Client({ name: 'test-client', version: '1.0.0' }, { capabilities: {} });

const transport = new StreamableHTTPClientTransport(new URL(serverUrl));
Expand All @@ -141,8 +180,60 @@ async function runToolsCallClient(serverUrl: string): Promise<void> {
logger.debug('Connection closed successfully');
}

// tools_call under a 2026-07-28 run: negotiate the modern era via
// server/discover (versionNegotiation), then drive the same tool flow with
// the per-request _meta envelope attached to every request.
async function runToolsCallModernClient(serverUrl: string): Promise<void> {
const clientInfo = { name: 'test-client', version: '1.0.0' };
const client = new Client(clientInfo, { capabilities: {}, versionNegotiation: { mode: 'auto' } });

const transport = new StreamableHTTPClientTransport(new URL(serverUrl));

await client.connect(transport);
logger.debug('Negotiated protocol version:', client.getNegotiatedProtocolVersion());

const envelope = modernEnvelope(clientInfo, {}, client.getNegotiatedProtocolVersion());
const tools = await client.request({ method: 'tools/list', params: { _meta: envelope } });
logger.debug('Successfully listed tools');

// Call the add_numbers tool
const addTool = tools.tools.find(t => t.name === 'add_numbers');
if (addTool) {
const result = await client.request({
method: 'tools/call',
params: { name: 'add_numbers', arguments: { a: 5, b: 3 }, _meta: envelope }
});
logger.debug('Tool call result:', JSON.stringify(result, null, 2));
}

await client.close();
logger.debug('Connection closed successfully');
}

// request-metadata scenario (SEP-2575): every request must carry the
// MCP-Protocol-Version header and the per-request _meta envelope, and the
// client must retry with a supported version when its first choice is
// rejected with -32004. The version-negotiation probe (server/discover plus
// the corrective continuation) is exactly that mechanism.
async function runRequestMetadataClient(serverUrl: string): Promise<void> {
const clientInfo = { name: 'test-client', version: '1.0.0' };
const client = new Client(clientInfo, {
capabilities: { roots: { listChanged: true }, sampling: {}, elicitation: {} },
versionNegotiation: { mode: 'auto' }
});

const transport = new StreamableHTTPClientTransport(new URL(serverUrl));

await client.connect(transport);
logger.debug('Negotiated protocol version:', client.getNegotiatedProtocolVersion());

await client.close();
logger.debug('Connection closed successfully');
}

registerScenario('initialize', runBasicClient);
registerScenario('tools_call', runToolsCallClient);
registerScenario('request-metadata', runRequestMetadataClient);

// ============================================================================
// Auth scenarios - well-behaved client
Expand Down
Loading
Loading