Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions packages/agent-bundle/src/contracts/eval.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ export {
provenanceIdentifierPattern,
semanticGraderIdentityPattern,
} from '../eval/provenance.ts';
export { EVAL_HARNESS_NAMES } from '../eval/harness-names.ts';
export type { EvalHarnessName } from '../eval/harness-names.ts';
export type {
EvalCaseSummary,
EvalRunEventsReplay,
Expand Down
13 changes: 11 additions & 2 deletions packages/agent-bundle/src/contracts/mcp-session.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
/**
* Browser-consumable contract surface for MCP session inspection. Type-only:
* the session service runs on the server.
* Browser-consumable contract surface for MCP session inspection. The target
* vocabulary is the only runtime export; the session service itself runs on
* the server.
*/
export type {
McpSessionBinding,
Expand All @@ -9,3 +10,11 @@ export type {
McpSessionTraceEntry,
McpSessionTraceReplayGap,
} from '../dev/mcp-session/mcp-session-protocol.ts';

/** The host targets a Workbench MCP session may bind. */
export const MCP_SESSION_TARGETS = Object.freeze(['claude', 'codex', 'cursor', 'portable'] as const);

export type McpSessionTarget = (typeof MCP_SESSION_TARGETS)[number];

export const isMcpSessionTarget = (value: unknown): value is McpSessionTarget =>
(MCP_SESSION_TARGETS as readonly unknown[]).includes(value);
9 changes: 6 additions & 3 deletions packages/agent-bundle/src/contracts/playground.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
/**
* Browser-consumable contract surface for Playground sessions, operation
* requests, and native host catalogs. Type-only: the playground services
* behind these shapes touch Node builtins.
* requests, and native host catalogs. The host vocabulary is the only
* runtime export; everything else is type-only because the playground
* services behind these shapes touch Node builtins.
*/
export { NATIVE_HOSTS, NATIVE_HOST_LABELS } from '../host-contracts/native-hosts.ts';
export type { PlaygroundOperationRequest, PlaygroundRun } from '../dev/playground/playground-contract.ts';
export type { NativePlaygroundCatalog, NativePlaygroundHost } from '../dev/playground/native-playground-service.ts';
export type { NativePlaygroundCatalog } from '../dev/playground/native-playground-service.ts';
export type { NativePlaygroundHost } from '../dev/playground/native-playground-types.ts';
export type {
DraftEvalCase,
PlaygroundEpochIdentity,
Expand Down
11 changes: 3 additions & 8 deletions packages/agent-bundle/src/dev/mcp-app-runtime-binding-service.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { randomUUID } from 'node:crypto';

import { cloneMcpAppFiniteJson, type McpAppJsonValue } from './mcp-app-metadata.ts';
import { MCP_APP_PROFILE_DESCRIPTORS, type McpAppProfileId } from './mcp-app-profile-descriptors.ts';
import type { DevRuntimeMcpSessionView } from './runtime-provider.ts';
import type {
DevRuntimeMcpAppRunBinding,
Expand All @@ -10,7 +11,7 @@ import type {
RuntimeVector,
} from './runtime-protocol.ts';

export type McpAppProfileId = 'portable' | 'chatgpt' | 'claude';
export type { McpAppProfileId } from './mcp-app-profile-descriptors.ts';

export interface McpAppStableSessionIdentity {
readonly definitionDigest: string;
Expand Down Expand Up @@ -99,12 +100,6 @@ interface McpAppRuntimeBinding {
unsubscribe: () => void;
}

const PROFILE_VERSIONS: Readonly<Record<McpAppProfileId, string>> = Object.freeze({
chatgpt: 'agent-bundle:chatgpt-sim:1',
claude: 'agent-bundle:claude-sim:1',
portable: 'agent-bundle:mcp-apps:2026-01-26',
});

const isRecord = (value: unknown): value is Readonly<Record<string, unknown>> =>
typeof value === 'object' && value !== null && !Array.isArray(value) && Object.getPrototypeOf(value) === Object.prototype;

Expand Down Expand Up @@ -255,7 +250,7 @@ export class McpAppRuntimeBindingService {
evidence: 'simulated' as const,
id: randomUUID(),
profileId,
profileVersion: PROFILE_VERSIONS[profileId],
profileVersion: MCP_APP_PROFILE_DESCRIPTORS[profileId].version,
runVector: publicVector(privateRunVector),
});
entry = {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import { randomUUID } from 'node:crypto';

import { MCP_APP_PROFILE_DESCRIPTORS, type McpAppProfileId } from '../mcp-app-profile-descriptors.ts';

export type McpAppJsonValue =
| null
| boolean
Expand All @@ -8,7 +10,7 @@ export type McpAppJsonValue =
| readonly McpAppJsonValue[]
| { readonly [key: string]: McpAppJsonValue };

export type McpAppPreviewProfile = 'chatgpt' | 'claude' | 'portable';
export type McpAppPreviewProfile = McpAppProfileId;

export interface McpAppToolDefinition {
readonly _meta?: { readonly [key: string]: McpAppJsonValue };
Expand Down Expand Up @@ -143,7 +145,7 @@ const requireNonempty = (value: string, label: string): string => {
};

const requireProfile = (profile: McpAppPreviewProfile): McpAppPreviewProfile => {
if (profile === 'chatgpt' || profile === 'claude' || profile === 'portable') return profile;
if (Object.hasOwn(MCP_APP_PROFILE_DESCRIPTORS, profile)) return profile;
throw new Error(`Unsupported MCP App preview profile ${JSON.stringify(profile)}.`);
};

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { digest } from '../../core/digest.ts';
import { isInsideOrEqual } from '../../core/paths.ts';
import { hasExactOwnKeys, isJsonRecord as isRecord, type JsonValue } from '../../core/strict-json.ts';
import type { EvalFixturePlan } from '../../eval/fixtures.ts';
import { NATIVE_HOSTS } from '../../host-contracts/native-hosts.ts';
import { normalizeEvalCase } from '../../eval/suite.ts';
import type { EvalCase } from '../../eval/types.ts';
import type { ArtifactEpoch } from '../types.ts';
Expand Down Expand Up @@ -53,7 +54,7 @@ export interface PersistedCatalogSnapshot {
readonly selections: readonly PersistedCatalogSelection[];
}

export const nativePlaygroundHosts = new Set<NativePlaygroundHost>(['claude', 'codex']);
export const nativePlaygroundHosts = new Set<NativePlaygroundHost>(NATIVE_HOSTS);
export const maximumCatalogSelections = 256;
export const maximumCatalogSnapshotBytes = 8 * 1_024 * 1_024;
const maximumCatalogSnapshotNodes = 65_536;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,16 @@ import type { EvalTrialRecord, EvalTrialWriter } from '../../eval/run-store.ts';
import { normalizeEvalCase } from '../../eval/suite.ts';
import type { EvalCase } from '../../eval/types.ts';
import type { NativeClaudeProcessRunner } from '../../host-contracts/native-claude-contract.ts';
import { NATIVE_HOSTS } from '../../host-contracts/native-hosts.ts';
import type { PlaygroundEventInput, PlaygroundJsonObject } from './playground-store.ts';
import type { NativePlaygroundHost } from './native-playground-types.ts';
import { safeDevWireText } from '../logs/dev-log-service.ts';
import type { ArtifactEpoch } from '../types.ts';
import { workspaceDiff, type WorkspaceDiff } from '../../eval/workspace-diff.ts';
import { isErrno } from '../../core/errors.ts';
import { isInsideOrEqual } from '../../core/paths.ts';

export type NativePlaygroundHost = 'claude' | 'codex';
export type { NativePlaygroundHost } from './native-playground-types.ts';

/** The exact browser shape accepted only after route-level strict decoding. */
export interface NativePlaygroundRequest {
Expand Down Expand Up @@ -198,7 +200,7 @@ class DiscardingTrialWriter implements EvalTrialWriter {
}
}

const nativeHosts = new Set<NativePlaygroundHost>(['claude', 'codex']);
const nativeHosts = new Set<NativePlaygroundHost>(NATIVE_HOSTS);
const catalogDurabilityPlatformKey = Symbol.for('agent-bundle.native-playground-service.catalog-durability-platform');
const maximumCatalogSelections = 256;
const maximumCatalogSnapshotBytes = 8 * 1_024 * 1_024;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@ import type { EvalCase } from '../../eval/types.ts';
import type { NativeClaudeProcessRunner } from '../../host-contracts/native-claude-contract.ts';
import type { PlaygroundEventInput, PlaygroundJsonObject } from './playground-store.ts';
import type { ArtifactEpoch } from '../types.ts';
import type { NativeHost } from '../../host-contracts/native-hosts.ts';

export type NativePlaygroundHost = 'claude' | 'codex';
export type NativePlaygroundHost = NativeHost;

/** The exact browser shape accepted only after route-level strict decoding. */
export interface NativePlaygroundRequest {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import type { NativePlaygroundHost } from './native-playground-types.ts';
import type { PlaygroundJsonObject, PlaygroundSession } from './playground-store.ts';

/** The only operation shapes a browser may request from Playground. */
Expand All @@ -9,7 +10,7 @@ export type PlaygroundOperationRequest =
readonly caseId: string;
readonly epochId?: string;
readonly fixtureId: string;
readonly host: 'claude' | 'codex';
readonly host: NativePlaygroundHost;
readonly modelPinId: string;
readonly operation: 'native.prompt';
readonly prompt: string;
Expand Down
6 changes: 6 additions & 0 deletions packages/agent-bundle/src/eval/harness-names.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
import { NATIVE_HOSTS } from '../host-contracts/native-hosts.ts';

/** Stable harness selector ids shared by the CLI, Agent API, and Workbench. */
export const EVAL_HARNESS_NAMES = Object.freeze([...NATIVE_HOSTS, 'deterministic'] as const);

export type EvalHarnessName = (typeof EVAL_HARNESS_NAMES)[number];
3 changes: 2 additions & 1 deletion packages/agent-bundle/src/eval/harness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { redactEvalCredentialText, withoutEvalCredentialEnvironment } from './cr
import { harnessError } from './errors.ts';
import { materializeEvalFixture, type EvalFixturePlan } from './fixtures.ts';
import { graderFailureFor, outcomeGraderSpecs, runEvalGraders, type EvalGraderSpec } from './graders.ts';
import type { EvalHarnessName } from './harness-names.ts';
import type { PreparedEvalArtifact } from './artifact.ts';
import type { EvalRunWriter, EvalTrialRecord } from './run-store.ts';
import type {
Expand Down Expand Up @@ -91,7 +92,7 @@ const harnessKinds: Readonly<Record<string, EvalHarness['kind']>> = Object.freez
claude: 'native-claude',
codex: 'native-codex',
deterministic: 'deterministic',
});
} satisfies Record<EvalHarnessName, EvalHarness['kind']>);

/** Case-qualified trial ids keep records and raw evidence unique across one multi-case run. */
export const evalTrialId = (caseId: string, host: string, trialIndex: number): string =>
Expand Down
3 changes: 2 additions & 1 deletion packages/agent-bundle/src/host-contracts/host-contract.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@ import { promisify } from 'node:util';
import { isRecord } from '../core/strict-json.ts';
import { escapeRegExp } from '../core/strings.ts';
import { isMissingExecutableError } from './native-host-spine.ts';
import type { NativeHost } from './native-hosts.ts';

// Pure parsing and opt-in probing contract for subscription-backed native host CLIs.
export type NativeHost = 'claude' | 'codex';
export type { NativeHost } from './native-hosts.ts';

export type HostContractStatus = 'changed' | 'compatible' | 'incompatible' | 'missing' | 'skipped';

Expand Down
13 changes: 13 additions & 0 deletions packages/agent-bundle/src/host-contracts/native-hosts.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
/**
* Browser-safe vocabulary for the subscription-backed native host CLIs Agent
* Bundle can drive. Every server-side host set, request union, and Workbench
* selector derives from this list so adding a native host is one edit.
*/
export const NATIVE_HOSTS = Object.freeze(['claude', 'codex'] as const);
Comment on lines +3 to +6

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Derive the remaining playground host gates from this list

When a third native host is added here, the catalog and browser schema will accept and display it, but playground-model.ts:182, playground-routes.ts:190, and the service/catalog decoders still explicitly allow only Claude and Codex. The Workbench will consequently clear the advertised selection or the route will reject the generated request, so this is not yet the promised single edit point; those runtime gates should use the shared vocabulary or a shared predicate too.

Useful? React with 👍 / 👎.


export type NativeHost = (typeof NATIVE_HOSTS)[number];

export const NATIVE_HOST_LABELS: Readonly<Record<NativeHost, string>> = Object.freeze({
claude: 'Claude',
codex: 'Codex',
});
4 changes: 2 additions & 2 deletions packages/workbench/src/evals/eval-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import type {
EvalSuiteListing,
} from '../../../agent-bundle/src/contracts/eval.ts';
import { parseJsonWithoutDuplicateKeys, snapshotStrictJsonValue, type JsonValue } from '../../../agent-bundle/src/contracts/strict-json.ts';
import type { EvalRunEvent, EvalRunRecord } from '../../../agent-bundle/src/contracts/eval.ts';
import type { EvalHarnessName, EvalRunEvent, EvalRunRecord } from '../../../agent-bundle/src/contracts/eval.ts';
import { awaitWithAbort, type ForegroundRequestAuthority } from '../mcp/mcp-route-client.ts';
import {
nonnegativeIntegerSchema,
Expand All @@ -20,7 +20,7 @@ export interface EvalClientOptions {
readonly foreground: ForegroundRequestAuthority;
}

export type EvalHarness = 'claude' | 'codex' | 'deterministic';
export type EvalHarness = EvalHarnessName;

/** Exactly what a browser may choose: authored suites, authored cases, and a trial count. */
export interface EvalRunStart extends EvalRunSelection {
Expand Down
13 changes: 7 additions & 6 deletions packages/workbench/src/mcp/mcp-app-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ import { isCallToolResult } from '@modelcontextprotocol/client';
import type { ProjectClient } from '../project-client.ts';
import type { ProjectEventMessage } from '../../../agent-bundle/src/contracts/project.ts';
import { validateMcpAppUiUri } from '../../../agent-bundle/src/contracts/mcp-apps.ts';
import { runtimeAppMessageLimits } from '../../../agent-bundle/src/contracts/mcp-apps.ts';
import { MCP_APP_PROFILE_DESCRIPTORS, runtimeAppMessageLimits } from '../../../agent-bundle/src/contracts/mcp-apps.ts';
import type { McpAppProfileId } from '../../../agent-bundle/src/contracts/mcp-apps.ts';
import type {
CreateMcpAppPreviewRequest as RuntimeCreateRequest,
McpAppBindingOperation,
Expand Down Expand Up @@ -32,7 +33,7 @@ export interface McpAppJsonObject {

export type McpAppJsonValue = McpAppJsonArray | McpAppJsonObject | McpAppJsonPrimitive;

export type McpAppPreviewProfile = 'chatgpt' | 'claude' | 'portable';
export type McpAppPreviewProfile = McpAppProfileId;
export type McpAppBridgeLifecycle = 'created' | 'initializing' | 'initialized' | 'closing' | 'closed';
export type McpAppRequestId = string | number | null;

Expand Down Expand Up @@ -548,13 +549,13 @@ const runtimeConfigExtensions = (value: unknown): unknown => {
return Object.freeze({ entries: Object.freeze(entries), sourceRevision });
};

const runtimeDescriptor = (value: unknown, profileId: 'portable' | 'chatgpt' | 'claude', profileVersion: string): unknown => {
const runtimeDescriptor = (value: unknown, profileId: McpAppProfileId, profileVersion: string): unknown => {
const record = runtimeRecord(value, ['claimsRealHostParity', 'evidence', 'id', 'label', 'version']);
const labels: Readonly<Record<typeof profileId, string>> = { chatgpt: 'ChatGPT Simulation', claude: 'Claude Simulation', portable: 'Portable MCP Apps' };
if (record.claimsRealHostParity !== false || record.evidence !== 'simulated' || record.id !== profileId || record.label !== labels[profileId] || record.version !== profileVersion) {
const label = MCP_APP_PROFILE_DESCRIPTORS[profileId].label;
if (record.claimsRealHostParity !== false || record.evidence !== 'simulated' || record.id !== profileId || record.label !== label || record.version !== profileVersion) {
runtimeInvalid('Runtime MCP App route returned an invalid profile descriptor.');
}
return Object.freeze({ claimsRealHostParity: false, evidence: 'simulated', id: profileId, label: labels[profileId], version: profileVersion });
return Object.freeze({ claimsRealHostParity: false, evidence: 'simulated', id: profileId, label, version: profileVersion });
};

const runtimeResource = (value: unknown): unknown => {
Expand Down
6 changes: 3 additions & 3 deletions packages/workbench/src/mcp/mcp-route-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@ import type {
RuntimeVector,
} from '../../../agent-bundle/src/contracts/runtime.ts';
import type { JsonObject } from '../../../agent-bundle/src/contracts/runtime.ts';
import { isMcpSessionTarget, type McpSessionTarget } from '../../../agent-bundle/src/contracts/mcp-session.ts';

export type McpRouteTarget = 'claude' | 'codex' | 'cursor' | 'portable';
export type McpRouteTarget = McpSessionTarget;

export interface McpRouteSessionBinding {
readonly epochId: string;
Expand Down Expand Up @@ -108,8 +109,7 @@ interface Diagnostic {
const isRecord = (value: unknown): value is Record<string, unknown> =>
typeof value === 'object' && value !== null && !Array.isArray(value);

const isTarget = (value: unknown): value is McpRouteTarget =>
value === 'claude' || value === 'codex' || value === 'cursor' || value === 'portable';
const isTarget = isMcpSessionTarget;

const detachedJson = (value: unknown, ancestors = new WeakSet<object>()): unknown => {
if (value === null || typeof value === 'string' || typeof value === 'boolean') return value;
Expand Down
3 changes: 2 additions & 1 deletion packages/workbench/src/mcp/mcp-session-controller.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { Client, type JSONRPCMessage, type Transport, type TransportSendOptions } from '@modelcontextprotocol/client';

import { isMcpSessionTarget } from '../../../agent-bundle/src/contracts/mcp-session.ts';
import type {
McpSessionBinding,
McpSessionInspectorConfig,
Expand Down Expand Up @@ -257,7 +258,7 @@ const artifactBindingSnapshot = (value: unknown): McpRouteSessionBinding | undef
if (
typeof epochId.value !== 'string' || epochId.value.length === 0 ||
typeof serverName.value !== 'string' || serverName.value.length === 0 ||
(target.value !== 'claude' && target.value !== 'codex' && target.value !== 'cursor' && target.value !== 'portable')
!isMcpSessionTarget(target.value)
) return undefined;
return Object.freeze({ epochId: epochId.value, serverName: serverName.value, target: target.value });
} catch {
Expand Down
3 changes: 2 additions & 1 deletion packages/workbench/src/playground/playground-client.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { z } from 'zod';

import { NATIVE_HOSTS } from '../../../agent-bundle/src/contracts/playground.ts';
import type {
DraftEvalCase,
PlaygroundExport,
Expand Down Expand Up @@ -136,7 +137,7 @@ const nativeCatalogJsonLimits: DetachedJsonLimits = Object.freeze({
maxValues: 4_096,
});
const textSchema = z.string().min(1).max(maximumNativeCatalogStringLength);
const nativeHostSchema = z.enum(['claude', 'codex']);
const nativeHostSchema = z.enum(NATIVE_HOSTS);
const nativeCatalogItemSchema = z.strictObject({ id: textSchema, label: textSchema });
const nativeModelPinSchema = nativeCatalogItemSchema.extend({ host: nativeHostSchema });
const nativeCatalogSelectionSchema = z.strictObject({
Expand Down
3 changes: 2 additions & 1 deletion packages/workbench/src/playground/playground-page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import type {
PlaygroundTraceEvent,
} from '../../../agent-bundle/src/contracts/playground.ts';
import type { PlaygroundOperationRequest, PlaygroundRun } from '../../../agent-bundle/src/contracts/playground.ts';
import { NATIVE_HOST_LABELS } from '../../../agent-bundle/src/contracts/playground.ts';
import type { NativePlaygroundCatalog, NativePlaygroundHost } from '../../../agent-bundle/src/contracts/playground.ts';

import { canonicalHookInputFor } from '../hooks/hooks-page.tsx';
Expand Down Expand Up @@ -595,7 +596,7 @@ export const PlaygroundNativePromptControls = ({
<label htmlFor="playground-native-host">Host
<select disabled={catalogDisabled || hosts.length === 0} id="playground-native-host" onChange={(event) => onHostChange(event.currentTarget.value as '' | NativePlaygroundHost)} value={selection.host}>
<option value="">Select a native host</option>
{hosts.map((host) => <option key={host} value={host}>{host === 'claude' ? 'Claude' : 'Codex'}</option>)}
{hosts.map((host) => <option key={host} value={host}>{NATIVE_HOST_LABELS[host]}</option>)}
</select>
</label>
<label htmlFor="playground-native-case">Case
Expand Down
Loading