diff --git a/packages/agent-bundle/src/contracts/eval.ts b/packages/agent-bundle/src/contracts/eval.ts index 0736a7627..dc678889d 100644 --- a/packages/agent-bundle/src/contracts/eval.ts +++ b/packages/agent-bundle/src/contracts/eval.ts @@ -9,6 +9,8 @@ export { provenanceIdentifierPattern, semanticGraderIdentityPattern, } from '../eval/provenance.ts'; +export { EVAL_HARNESS_NAMES } from '../eval/harness-names.ts'; +export type { EvalHarnessName } from '../eval/harness-names.ts'; export type { EvalCaseSummary, EvalRunEventsReplay, diff --git a/packages/agent-bundle/src/contracts/mcp-session.ts b/packages/agent-bundle/src/contracts/mcp-session.ts index 6cc51ba54..d5346bc0c 100644 --- a/packages/agent-bundle/src/contracts/mcp-session.ts +++ b/packages/agent-bundle/src/contracts/mcp-session.ts @@ -1,6 +1,7 @@ /** - * Browser-consumable contract surface for MCP session inspection. Type-only: - * the session service runs on the server. + * Browser-consumable contract surface for MCP session inspection. The target + * vocabulary is the only runtime export; the session service itself runs on + * the server. */ export type { McpSessionBinding, @@ -9,3 +10,11 @@ export type { McpSessionTraceEntry, McpSessionTraceReplayGap, } from '../dev/mcp-session/mcp-session-protocol.ts'; + +/** The host targets a Workbench MCP session may bind. */ +export const MCP_SESSION_TARGETS = Object.freeze(['claude', 'codex', 'cursor', 'portable'] as const); + +export type McpSessionTarget = (typeof MCP_SESSION_TARGETS)[number]; + +export const isMcpSessionTarget = (value: unknown): value is McpSessionTarget => + (MCP_SESSION_TARGETS as readonly unknown[]).includes(value); diff --git a/packages/agent-bundle/src/contracts/playground.ts b/packages/agent-bundle/src/contracts/playground.ts index 4f4a1c52e..ff4d8e78a 100644 --- a/packages/agent-bundle/src/contracts/playground.ts +++ b/packages/agent-bundle/src/contracts/playground.ts @@ -1,10 +1,13 @@ /** * Browser-consumable contract surface for Playground sessions, operation - * requests, and native host catalogs. Type-only: the playground services - * behind these shapes touch Node builtins. + * requests, and native host catalogs. The host vocabulary is the only + * runtime export; everything else is type-only because the playground + * services behind these shapes touch Node builtins. */ +export { NATIVE_HOSTS, NATIVE_HOST_LABELS } from '../host-contracts/native-hosts.ts'; export type { PlaygroundOperationRequest, PlaygroundRun } from '../dev/playground/playground-contract.ts'; -export type { NativePlaygroundCatalog, NativePlaygroundHost } from '../dev/playground/native-playground-service.ts'; +export type { NativePlaygroundCatalog } from '../dev/playground/native-playground-service.ts'; +export type { NativePlaygroundHost } from '../dev/playground/native-playground-types.ts'; export type { DraftEvalCase, PlaygroundEpochIdentity, diff --git a/packages/agent-bundle/src/dev/mcp-app-runtime-binding-service.ts b/packages/agent-bundle/src/dev/mcp-app-runtime-binding-service.ts index 7161918c5..8f8b420d8 100644 --- a/packages/agent-bundle/src/dev/mcp-app-runtime-binding-service.ts +++ b/packages/agent-bundle/src/dev/mcp-app-runtime-binding-service.ts @@ -1,6 +1,7 @@ import { randomUUID } from 'node:crypto'; import { cloneMcpAppFiniteJson, type McpAppJsonValue } from './mcp-app-metadata.ts'; +import { MCP_APP_PROFILE_DESCRIPTORS, type McpAppProfileId } from './mcp-app-profile-descriptors.ts'; import type { DevRuntimeMcpSessionView } from './runtime-provider.ts'; import type { DevRuntimeMcpAppRunBinding, @@ -10,7 +11,7 @@ import type { RuntimeVector, } from './runtime-protocol.ts'; -export type McpAppProfileId = 'portable' | 'chatgpt' | 'claude'; +export type { McpAppProfileId } from './mcp-app-profile-descriptors.ts'; export interface McpAppStableSessionIdentity { readonly definitionDigest: string; @@ -99,12 +100,6 @@ interface McpAppRuntimeBinding { unsubscribe: () => void; } -const PROFILE_VERSIONS: Readonly> = Object.freeze({ - chatgpt: 'agent-bundle:chatgpt-sim:1', - claude: 'agent-bundle:claude-sim:1', - portable: 'agent-bundle:mcp-apps:2026-01-26', -}); - const isRecord = (value: unknown): value is Readonly> => typeof value === 'object' && value !== null && !Array.isArray(value) && Object.getPrototypeOf(value) === Object.prototype; @@ -255,7 +250,7 @@ export class McpAppRuntimeBindingService { evidence: 'simulated' as const, id: randomUUID(), profileId, - profileVersion: PROFILE_VERSIONS[profileId], + profileVersion: MCP_APP_PROFILE_DESCRIPTORS[profileId].version, runVector: publicVector(privateRunVector), }); entry = { diff --git a/packages/agent-bundle/src/dev/mcp-apps/mcp-app-binding-service.ts b/packages/agent-bundle/src/dev/mcp-apps/mcp-app-binding-service.ts index 66cd9db2c..17f1ef1f6 100644 --- a/packages/agent-bundle/src/dev/mcp-apps/mcp-app-binding-service.ts +++ b/packages/agent-bundle/src/dev/mcp-apps/mcp-app-binding-service.ts @@ -1,5 +1,7 @@ import { randomUUID } from 'node:crypto'; +import { MCP_APP_PROFILE_DESCRIPTORS, type McpAppProfileId } from '../mcp-app-profile-descriptors.ts'; + export type McpAppJsonValue = | null | boolean @@ -8,7 +10,7 @@ export type McpAppJsonValue = | readonly McpAppJsonValue[] | { readonly [key: string]: McpAppJsonValue }; -export type McpAppPreviewProfile = 'chatgpt' | 'claude' | 'portable'; +export type McpAppPreviewProfile = McpAppProfileId; export interface McpAppToolDefinition { readonly _meta?: { readonly [key: string]: McpAppJsonValue }; @@ -143,7 +145,7 @@ const requireNonempty = (value: string, label: string): string => { }; const requireProfile = (profile: McpAppPreviewProfile): McpAppPreviewProfile => { - if (profile === 'chatgpt' || profile === 'claude' || profile === 'portable') return profile; + if (Object.hasOwn(MCP_APP_PROFILE_DESCRIPTORS, profile)) return profile; throw new Error(`Unsupported MCP App preview profile ${JSON.stringify(profile)}.`); }; diff --git a/packages/agent-bundle/src/dev/playground/native-playground-catalog.ts b/packages/agent-bundle/src/dev/playground/native-playground-catalog.ts index b8b44a1e3..a05dcc212 100644 --- a/packages/agent-bundle/src/dev/playground/native-playground-catalog.ts +++ b/packages/agent-bundle/src/dev/playground/native-playground-catalog.ts @@ -4,6 +4,7 @@ import { digest } from '../../core/digest.ts'; import { isInsideOrEqual } from '../../core/paths.ts'; import { hasExactOwnKeys, isJsonRecord as isRecord, type JsonValue } from '../../core/strict-json.ts'; import type { EvalFixturePlan } from '../../eval/fixtures.ts'; +import { NATIVE_HOSTS } from '../../host-contracts/native-hosts.ts'; import { normalizeEvalCase } from '../../eval/suite.ts'; import type { EvalCase } from '../../eval/types.ts'; import type { ArtifactEpoch } from '../types.ts'; @@ -53,7 +54,7 @@ export interface PersistedCatalogSnapshot { readonly selections: readonly PersistedCatalogSelection[]; } -export const nativePlaygroundHosts = new Set(['claude', 'codex']); +export const nativePlaygroundHosts = new Set(NATIVE_HOSTS); export const maximumCatalogSelections = 256; export const maximumCatalogSnapshotBytes = 8 * 1_024 * 1_024; const maximumCatalogSnapshotNodes = 65_536; diff --git a/packages/agent-bundle/src/dev/playground/native-playground-service.ts b/packages/agent-bundle/src/dev/playground/native-playground-service.ts index 218685ca2..353876104 100644 --- a/packages/agent-bundle/src/dev/playground/native-playground-service.ts +++ b/packages/agent-bundle/src/dev/playground/native-playground-service.ts @@ -17,14 +17,16 @@ import type { EvalTrialRecord, EvalTrialWriter } from '../../eval/run-store.ts'; import { normalizeEvalCase } from '../../eval/suite.ts'; import type { EvalCase } from '../../eval/types.ts'; import type { NativeClaudeProcessRunner } from '../../host-contracts/native-claude-contract.ts'; +import { NATIVE_HOSTS } from '../../host-contracts/native-hosts.ts'; import type { PlaygroundEventInput, PlaygroundJsonObject } from './playground-store.ts'; +import type { NativePlaygroundHost } from './native-playground-types.ts'; import { safeDevWireText } from '../logs/dev-log-service.ts'; import type { ArtifactEpoch } from '../types.ts'; import { workspaceDiff, type WorkspaceDiff } from '../../eval/workspace-diff.ts'; import { isErrno } from '../../core/errors.ts'; import { isInsideOrEqual } from '../../core/paths.ts'; -export type NativePlaygroundHost = 'claude' | 'codex'; +export type { NativePlaygroundHost } from './native-playground-types.ts'; /** The exact browser shape accepted only after route-level strict decoding. */ export interface NativePlaygroundRequest { @@ -198,7 +200,7 @@ class DiscardingTrialWriter implements EvalTrialWriter { } } -const nativeHosts = new Set(['claude', 'codex']); +const nativeHosts = new Set(NATIVE_HOSTS); const catalogDurabilityPlatformKey = Symbol.for('agent-bundle.native-playground-service.catalog-durability-platform'); const maximumCatalogSelections = 256; const maximumCatalogSnapshotBytes = 8 * 1_024 * 1_024; diff --git a/packages/agent-bundle/src/dev/playground/native-playground-types.ts b/packages/agent-bundle/src/dev/playground/native-playground-types.ts index fd510165b..18d6b7f3c 100644 --- a/packages/agent-bundle/src/dev/playground/native-playground-types.ts +++ b/packages/agent-bundle/src/dev/playground/native-playground-types.ts @@ -8,8 +8,9 @@ import type { EvalCase } from '../../eval/types.ts'; import type { NativeClaudeProcessRunner } from '../../host-contracts/native-claude-contract.ts'; import type { PlaygroundEventInput, PlaygroundJsonObject } from './playground-store.ts'; import type { ArtifactEpoch } from '../types.ts'; +import type { NativeHost } from '../../host-contracts/native-hosts.ts'; -export type NativePlaygroundHost = 'claude' | 'codex'; +export type NativePlaygroundHost = NativeHost; /** The exact browser shape accepted only after route-level strict decoding. */ export interface NativePlaygroundRequest { diff --git a/packages/agent-bundle/src/dev/playground/playground-contract.ts b/packages/agent-bundle/src/dev/playground/playground-contract.ts index 685fe0b1a..90b791ad7 100644 --- a/packages/agent-bundle/src/dev/playground/playground-contract.ts +++ b/packages/agent-bundle/src/dev/playground/playground-contract.ts @@ -1,3 +1,4 @@ +import type { NativePlaygroundHost } from './native-playground-types.ts'; import type { PlaygroundJsonObject, PlaygroundSession } from './playground-store.ts'; /** The only operation shapes a browser may request from Playground. */ @@ -9,7 +10,7 @@ export type PlaygroundOperationRequest = readonly caseId: string; readonly epochId?: string; readonly fixtureId: string; - readonly host: 'claude' | 'codex'; + readonly host: NativePlaygroundHost; readonly modelPinId: string; readonly operation: 'native.prompt'; readonly prompt: string; diff --git a/packages/agent-bundle/src/eval/harness-names.ts b/packages/agent-bundle/src/eval/harness-names.ts new file mode 100644 index 000000000..dabeb21d0 --- /dev/null +++ b/packages/agent-bundle/src/eval/harness-names.ts @@ -0,0 +1,6 @@ +import { NATIVE_HOSTS } from '../host-contracts/native-hosts.ts'; + +/** Stable harness selector ids shared by the CLI, Agent API, and Workbench. */ +export const EVAL_HARNESS_NAMES = Object.freeze([...NATIVE_HOSTS, 'deterministic'] as const); + +export type EvalHarnessName = (typeof EVAL_HARNESS_NAMES)[number]; diff --git a/packages/agent-bundle/src/eval/harness.ts b/packages/agent-bundle/src/eval/harness.ts index 94c37039d..9f419fa75 100644 --- a/packages/agent-bundle/src/eval/harness.ts +++ b/packages/agent-bundle/src/eval/harness.ts @@ -9,6 +9,7 @@ import { redactEvalCredentialText, withoutEvalCredentialEnvironment } from './cr import { harnessError } from './errors.ts'; import { materializeEvalFixture, type EvalFixturePlan } from './fixtures.ts'; import { graderFailureFor, outcomeGraderSpecs, runEvalGraders, type EvalGraderSpec } from './graders.ts'; +import type { EvalHarnessName } from './harness-names.ts'; import type { PreparedEvalArtifact } from './artifact.ts'; import type { EvalRunWriter, EvalTrialRecord } from './run-store.ts'; import type { @@ -91,7 +92,7 @@ const harnessKinds: Readonly> = Object.freez claude: 'native-claude', codex: 'native-codex', deterministic: 'deterministic', -}); +} satisfies Record); /** Case-qualified trial ids keep records and raw evidence unique across one multi-case run. */ export const evalTrialId = (caseId: string, host: string, trialIndex: number): string => diff --git a/packages/agent-bundle/src/host-contracts/host-contract.ts b/packages/agent-bundle/src/host-contracts/host-contract.ts index b2641c82a..ffdaa4aa9 100644 --- a/packages/agent-bundle/src/host-contracts/host-contract.ts +++ b/packages/agent-bundle/src/host-contracts/host-contract.ts @@ -4,9 +4,10 @@ import { promisify } from 'node:util'; import { isRecord } from '../core/strict-json.ts'; import { escapeRegExp } from '../core/strings.ts'; import { isMissingExecutableError } from './native-host-spine.ts'; +import type { NativeHost } from './native-hosts.ts'; // Pure parsing and opt-in probing contract for subscription-backed native host CLIs. -export type NativeHost = 'claude' | 'codex'; +export type { NativeHost } from './native-hosts.ts'; export type HostContractStatus = 'changed' | 'compatible' | 'incompatible' | 'missing' | 'skipped'; diff --git a/packages/agent-bundle/src/host-contracts/native-hosts.ts b/packages/agent-bundle/src/host-contracts/native-hosts.ts new file mode 100644 index 000000000..63490385b --- /dev/null +++ b/packages/agent-bundle/src/host-contracts/native-hosts.ts @@ -0,0 +1,13 @@ +/** + * Browser-safe vocabulary for the subscription-backed native host CLIs Agent + * Bundle can drive. Every server-side host set, request union, and Workbench + * selector derives from this list so adding a native host is one edit. + */ +export const NATIVE_HOSTS = Object.freeze(['claude', 'codex'] as const); + +export type NativeHost = (typeof NATIVE_HOSTS)[number]; + +export const NATIVE_HOST_LABELS: Readonly> = Object.freeze({ + claude: 'Claude', + codex: 'Codex', +}); diff --git a/packages/workbench/src/evals/eval-client.ts b/packages/workbench/src/evals/eval-client.ts index 24a59291f..2e5c603dd 100644 --- a/packages/workbench/src/evals/eval-client.ts +++ b/packages/workbench/src/evals/eval-client.ts @@ -7,7 +7,7 @@ import type { EvalSuiteListing, } from '../../../agent-bundle/src/contracts/eval.ts'; import { parseJsonWithoutDuplicateKeys, snapshotStrictJsonValue, type JsonValue } from '../../../agent-bundle/src/contracts/strict-json.ts'; -import type { EvalRunEvent, EvalRunRecord } from '../../../agent-bundle/src/contracts/eval.ts'; +import type { EvalHarnessName, EvalRunEvent, EvalRunRecord } from '../../../agent-bundle/src/contracts/eval.ts'; import { awaitWithAbort, type ForegroundRequestAuthority } from '../mcp/mcp-route-client.ts'; import { nonnegativeIntegerSchema, @@ -20,7 +20,7 @@ export interface EvalClientOptions { readonly foreground: ForegroundRequestAuthority; } -export type EvalHarness = 'claude' | 'codex' | 'deterministic'; +export type EvalHarness = EvalHarnessName; /** Exactly what a browser may choose: authored suites, authored cases, and a trial count. */ export interface EvalRunStart extends EvalRunSelection { diff --git a/packages/workbench/src/mcp/mcp-app-client.ts b/packages/workbench/src/mcp/mcp-app-client.ts index dbb954ec4..9d9bc93fd 100644 --- a/packages/workbench/src/mcp/mcp-app-client.ts +++ b/packages/workbench/src/mcp/mcp-app-client.ts @@ -3,7 +3,8 @@ import { isCallToolResult } from '@modelcontextprotocol/client'; import type { ProjectClient } from '../project-client.ts'; import type { ProjectEventMessage } from '../../../agent-bundle/src/contracts/project.ts'; import { validateMcpAppUiUri } from '../../../agent-bundle/src/contracts/mcp-apps.ts'; -import { runtimeAppMessageLimits } from '../../../agent-bundle/src/contracts/mcp-apps.ts'; +import { MCP_APP_PROFILE_DESCRIPTORS, runtimeAppMessageLimits } from '../../../agent-bundle/src/contracts/mcp-apps.ts'; +import type { McpAppProfileId } from '../../../agent-bundle/src/contracts/mcp-apps.ts'; import type { CreateMcpAppPreviewRequest as RuntimeCreateRequest, McpAppBindingOperation, @@ -32,7 +33,7 @@ export interface McpAppJsonObject { export type McpAppJsonValue = McpAppJsonArray | McpAppJsonObject | McpAppJsonPrimitive; -export type McpAppPreviewProfile = 'chatgpt' | 'claude' | 'portable'; +export type McpAppPreviewProfile = McpAppProfileId; export type McpAppBridgeLifecycle = 'created' | 'initializing' | 'initialized' | 'closing' | 'closed'; export type McpAppRequestId = string | number | null; @@ -548,13 +549,13 @@ const runtimeConfigExtensions = (value: unknown): unknown => { return Object.freeze({ entries: Object.freeze(entries), sourceRevision }); }; -const runtimeDescriptor = (value: unknown, profileId: 'portable' | 'chatgpt' | 'claude', profileVersion: string): unknown => { +const runtimeDescriptor = (value: unknown, profileId: McpAppProfileId, profileVersion: string): unknown => { const record = runtimeRecord(value, ['claimsRealHostParity', 'evidence', 'id', 'label', 'version']); - const labels: Readonly> = { chatgpt: 'ChatGPT Simulation', claude: 'Claude Simulation', portable: 'Portable MCP Apps' }; - if (record.claimsRealHostParity !== false || record.evidence !== 'simulated' || record.id !== profileId || record.label !== labels[profileId] || record.version !== profileVersion) { + const label = MCP_APP_PROFILE_DESCRIPTORS[profileId].label; + if (record.claimsRealHostParity !== false || record.evidence !== 'simulated' || record.id !== profileId || record.label !== label || record.version !== profileVersion) { runtimeInvalid('Runtime MCP App route returned an invalid profile descriptor.'); } - return Object.freeze({ claimsRealHostParity: false, evidence: 'simulated', id: profileId, label: labels[profileId], version: profileVersion }); + return Object.freeze({ claimsRealHostParity: false, evidence: 'simulated', id: profileId, label, version: profileVersion }); }; const runtimeResource = (value: unknown): unknown => { diff --git a/packages/workbench/src/mcp/mcp-route-client.ts b/packages/workbench/src/mcp/mcp-route-client.ts index 59b959e90..a26a15fa7 100644 --- a/packages/workbench/src/mcp/mcp-route-client.ts +++ b/packages/workbench/src/mcp/mcp-route-client.ts @@ -8,8 +8,9 @@ import type { RuntimeVector, } from '../../../agent-bundle/src/contracts/runtime.ts'; import type { JsonObject } from '../../../agent-bundle/src/contracts/runtime.ts'; +import { isMcpSessionTarget, type McpSessionTarget } from '../../../agent-bundle/src/contracts/mcp-session.ts'; -export type McpRouteTarget = 'claude' | 'codex' | 'cursor' | 'portable'; +export type McpRouteTarget = McpSessionTarget; export interface McpRouteSessionBinding { readonly epochId: string; @@ -108,8 +109,7 @@ interface Diagnostic { const isRecord = (value: unknown): value is Record => typeof value === 'object' && value !== null && !Array.isArray(value); -const isTarget = (value: unknown): value is McpRouteTarget => - value === 'claude' || value === 'codex' || value === 'cursor' || value === 'portable'; +const isTarget = isMcpSessionTarget; const detachedJson = (value: unknown, ancestors = new WeakSet()): unknown => { if (value === null || typeof value === 'string' || typeof value === 'boolean') return value; diff --git a/packages/workbench/src/mcp/mcp-session-controller.ts b/packages/workbench/src/mcp/mcp-session-controller.ts index 0189e2b5c..ca80a476f 100644 --- a/packages/workbench/src/mcp/mcp-session-controller.ts +++ b/packages/workbench/src/mcp/mcp-session-controller.ts @@ -1,5 +1,6 @@ import { Client, type JSONRPCMessage, type Transport, type TransportSendOptions } from '@modelcontextprotocol/client'; +import { isMcpSessionTarget } from '../../../agent-bundle/src/contracts/mcp-session.ts'; import type { McpSessionBinding, McpSessionInspectorConfig, @@ -257,7 +258,7 @@ const artifactBindingSnapshot = (value: unknown): McpRouteSessionBinding | undef if ( typeof epochId.value !== 'string' || epochId.value.length === 0 || typeof serverName.value !== 'string' || serverName.value.length === 0 || - (target.value !== 'claude' && target.value !== 'codex' && target.value !== 'cursor' && target.value !== 'portable') + !isMcpSessionTarget(target.value) ) return undefined; return Object.freeze({ epochId: epochId.value, serverName: serverName.value, target: target.value }); } catch { diff --git a/packages/workbench/src/playground/playground-client.ts b/packages/workbench/src/playground/playground-client.ts index 3e8bc37ad..9bab22b92 100644 --- a/packages/workbench/src/playground/playground-client.ts +++ b/packages/workbench/src/playground/playground-client.ts @@ -1,5 +1,6 @@ import { z } from 'zod'; +import { NATIVE_HOSTS } from '../../../agent-bundle/src/contracts/playground.ts'; import type { DraftEvalCase, PlaygroundExport, @@ -136,7 +137,7 @@ const nativeCatalogJsonLimits: DetachedJsonLimits = Object.freeze({ maxValues: 4_096, }); const textSchema = z.string().min(1).max(maximumNativeCatalogStringLength); -const nativeHostSchema = z.enum(['claude', 'codex']); +const nativeHostSchema = z.enum(NATIVE_HOSTS); const nativeCatalogItemSchema = z.strictObject({ id: textSchema, label: textSchema }); const nativeModelPinSchema = nativeCatalogItemSchema.extend({ host: nativeHostSchema }); const nativeCatalogSelectionSchema = z.strictObject({ diff --git a/packages/workbench/src/playground/playground-page.tsx b/packages/workbench/src/playground/playground-page.tsx index e4aef3a66..778ce5bbd 100644 --- a/packages/workbench/src/playground/playground-page.tsx +++ b/packages/workbench/src/playground/playground-page.tsx @@ -13,6 +13,7 @@ import type { PlaygroundTraceEvent, } from '../../../agent-bundle/src/contracts/playground.ts'; import type { PlaygroundOperationRequest, PlaygroundRun } from '../../../agent-bundle/src/contracts/playground.ts'; +import { NATIVE_HOST_LABELS } from '../../../agent-bundle/src/contracts/playground.ts'; import type { NativePlaygroundCatalog, NativePlaygroundHost } from '../../../agent-bundle/src/contracts/playground.ts'; import { canonicalHookInputFor } from '../hooks/hooks-page.tsx'; @@ -595,7 +596,7 @@ export const PlaygroundNativePromptControls = ({