From 321b27216e561561e1e022a7b4d973e2182ee480 Mon Sep 17 00:00:00 2001 From: kjgbot Date: Wed, 2 Sep 2026 18:01:40 +0200 Subject: [PATCH 01/15] feat(sdk): add declared agent model contract Session-Id: 01a062b7-2aef-7772-8225-7cb00ee311dd --- docs/SURFACE.md | 42 ++++++-- sdk/src/cli.ts | 3 +- sdk/src/cli/check.ts | 57 +++++++++-- sdk/src/compile.ts | 31 +++++- sdk/src/failure-kinds.ts | 2 + sdk/src/index.ts | 2 + sdk/src/model-name.ts | 20 ++++ sdk/src/preflight.ts | 59 ++++++++++++ sdk/src/spec.ts | 15 +++ sdk/src/unknown-keys.ts | 52 ++++++++++ sdk/src/validate.ts | 102 +++++++++----------- sdk/tests/cli.test.ts | 105 +++++++++++++++++++- sdk/tests/live-kernel.test.ts | 22 ++++- sdk/tests/model-selection.test.ts | 155 ++++++++++++++++++++++++++++++ sdk/tests/preflight.test.ts | 20 +++- testdata/flows.json | 8 +- 16 files changed, 614 insertions(+), 81 deletions(-) create mode 100644 sdk/src/model-name.ts create mode 100644 sdk/src/unknown-keys.ts create mode 100644 sdk/tests/model-selection.test.ts diff --git a/docs/SURFACE.md b/docs/SURFACE.md index 4ea7f313..87267aff 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -66,18 +66,42 @@ No process runs between events: the handler wakes, executes to its next await, p - `f.mcp` — one line to declare (`tools: { mcp: [stripe] }`), one call to use (`f.mcp.stripe.create_invoice({...})`). Preflight connects to every declared server before the run starts. 4. **`{{prev}}` / return-value chaining.** Output flows downward implicitly; naming steps is for reaching back, not bookkeeping. 5. **Headers are optional escalation.** identity, memory, budget, tools appear only when used. The empty header is the common case. -6. **Agent definitions come in three sizes** — and a reusable agent *is* a flow: +6. **Agent definitions escalate by composition** — and a reusable agent *is* a flow: ```yaml - agent: Review this diff for security issues. # 1. anonymous agents: - reviewer: claude # 2. named — name: cli - auditor: { cli: claude, memory: true, tools: { mcp: [semgrep] }, workspace: readonly } # 3. escalated + reviewer: { cli: claude, model: claude-sonnet-4-6 } # 2. named — explicit and reusable ``` - Defining your team's reviewer = writing `reviewer.flow.ts` (identity + memory + body); other flows compose it with `use:` / `f.agent(reviewer, task)`. Persona import is flow composition, not a special mechanism. + The declarative named-agent schema in this slice is exactly `{ cli, model }`; + unknown fields fail closed. Defining a richer team reviewer means writing + `reviewer.flow.ts` (identity + memory + body); other flows compose it with + `use:` / `f.agent(reviewer, task)`. Persona import is flow composition, not + a special mechanism. + + In the canonical declarative YAML/JSON dialect, `agents:` is a top-level + map and an agent step selects one with `agent: reviewer`. Each named + declaration requires both `cli` and `model`. Compilation lowers them into + the existing per-step `cli` and `model` fields and removes both the selector + and map before the kernel boundary. Explicit step values win independently: + step `cli`/`model` → named declaration → the existing flow/project CLI + default. Model has no flow/project default. An inline step that selects no + named declaration keeps the existing optional-model behavior; the worker + explicitly removes ambient `RELAYFLOW_MODEL` when it is absent. **Anonymous resolution law:** `f.agent\`task\`` with no name is the *default agent*, resolved (never guessed) in order: step options → flow header → project config (`flows.json`) → platform default. *The platform-default rung is declared but not yet implemented: no platform default is provisioned as of gate 1, so a flow that reaches this rung refuses with `cli_unresolved` rather than guessing. `flows check` never invents an implicit default.* `flows check` prints each resolved step CLI and its declaration source, validates it before submission, and refuses a missing or unauthenticated resolution before the checked flow is submitted, never at minute 27. Gate 1 does not make this guarantee for callers that bypass `flows check`: the journal client's direct `run.start` path does not invoke surface preflight. - **Preflightable-CLI contract:** to be checkable, a declared `cli` must answer ` auth status` — exit `0` for authenticated, non-zero for not. `flows check` resolves the binary (a path is taken relative to the file that declares it — the flow for a step/flow-level `cli`, the project config for a `flows.json` default — while a bare name resolves via `PATH`) and runs that probe once per resolved `(cli, source)`: a path that does not resolve as an executable is `cli_missing`, and non-zero is `cli_unauthenticated`. A probe process that cannot be started, is terminated by a signal, or exceeds the 10-second auth-probe timeout is `probe_failed`; the diagnostic carries that classified cause without exposing raw process errors. The probe executes the flow-declared CLI with the checking process's complete caller environment inherited. This is the whole contract — preflight never sends a prompt, never spends a token, and never invokes any other subcommand. A non-zero refusal names the exact `auth status` probe and tells the operator to authenticate the CLI or implement the probe to return exit `0`; health is never assumed. + **Preflightable-CLI contract:** to be checkable, a declared `cli` must answer ` auth status` — exit `0` for authenticated, non-zero for not. When a step declares a model, the same probe runs with that exact value in `RELAYFLOW_MODEL`; exit `0` means the current credential can use that exact model. If the scoped probe fails, an unscoped probe distinguishes `model_unavailable` from `cli_unauthenticated`. `flows check` resolves the binary (a path is taken relative to the file that declares it — the flow for a step/flow-level `cli`, the project config for a `flows.json` default — while a bare name resolves via `PATH`) and runs that probe once per resolved `(cli, source, model)`: a path that does not resolve as an executable is `cli_missing`. A probe process that cannot be started, is terminated by a signal, or exceeds the 10-second auth-probe timeout is `probe_failed`; the diagnostic carries that classified cause without exposing raw process errors. The probe inherits the caller environment except that `RELAYFLOW_MODEL` is always removed and then set only from the compiled step. Preflight never invokes an undeclared model or guesses from host state. + + **Deterministic model registry:** model existence is not inferred from a + regex or provider prefix. The nearest `flows.json` owns an exact, + case-sensitive `models` allowlist. `flows check` first refuses a declared + model absent from that list as `model_unknown`, without starting the CLI; + only an allowlisted value reaches the live model-scoped probe above. The + registry is author-owned project configuration, reviewed and versioned with + the project. Updating it is an explicit file change made only after the + project verifies access to the added model. No remote catalog is fetched, + so a checkout plus its nearest config reproduces typo decisions offline. + Runtime access remains a live fact and is re-probed on every check call. **Accepted deterministic-command limitation (Codex P1):** `flows check` warns with `command_unresolved`, rather than refusing, when a deterministic @@ -88,7 +112,13 @@ No process runs between events: the handler wakes, executes to its next await, p deterministic-command preflight gap.” Consequently, `cli_missing` applies to declared `llm` and `agent` CLIs, not deterministic command words. - **Project-config discovery:** starting in the flow file's directory, `flows check` walks parent directories through the filesystem root and selects the first readable `flows.json`. That nearest file is the whole project config; it is not merged with outer files. Its schema is `{ "cli"?: , "executors"?: [] }`; unknown keys fail closed as `config_invalid`. A nearer config therefore defines a self-contained nested project boundary and prevents accidental inheritance of outer credentials or executors. The selected path is printed with project-level resolutions and named in an unresolved-CLI refusal; if it declares no `cli`, outer configs remain shadowed. At gate 1, a trigger executor is considered registered only when its name is present in this author-written `executors` array; `flows check` does not yet contact a registry, broker, or RelayCron, and absence is `no_executor`. + **Project-config discovery:** starting in the flow file's directory, `flows check` walks parent directories through the filesystem root and selects the first readable `flows.json`. That nearest file is the whole project config; it is not merged with outer files. Its schema is `{ "cli"?: , "executors"?: [], "models"?: [] }`; unknown keys, malformed model entries, and duplicates fail closed as `config_invalid`. A nearer config therefore defines a self-contained nested project boundary and prevents accidental inheritance of outer credentials, executors, or model approvals. The selected path is printed with project-level resolutions and named in refusals; if it declares no `cli` or models, outer configs remain shadowed. At gate 1, a trigger executor is considered registered only when its name is present in this author-written `executors` array; `flows check` does not yet contact a registry, broker, or RelayCron, and absence is `no_executor`. + + Implementation status for issue #132: this named-agent contract currently + ships in the canonical declarative YAML/JSON compiler. Matching + `FlowHeader.agents` TypeScript types depend on the separately reviewed, + unmerged `@relayflows/surface` package in PR #134 and are a follow-on after + that package lands; this compiler slice does not duplicate that package. 7. **Two dialects, one journal.** Declarative YAML — data, fully preflightable, sage's compile target, gate 9's self-authoring output. Imperative TS — journal-memoized function, maximum ergonomics. YAML is canonical; TS is the power tool. TS preflights its declared surface (agents, helpers, tools, identity), not arbitrary control flow — declared honestly per covenant 2. The authoring surface deliberately narrows `steps: []`: `flows check` refuses diff --git a/sdk/src/cli.ts b/sdk/src/cli.ts index 20231854..6c393ea1 100644 --- a/sdk/src/cli.ts +++ b/sdk/src/cli.ts @@ -172,7 +172,8 @@ function emitCheckReport(report: CheckReport, json: boolean, io: CliIo): void { const config = resolution.source === 'project' && report.projectConfigPath !== undefined ? ` (${report.projectConfigPath})` : ''; - io.stdout(`RESOLVED step "${resolution.stepId}" cli "${resolution.cli}" from ${resolution.source}${config}`); + const model = resolution.model === undefined ? '' : ` model "${resolution.model}"`; + io.stdout(`RESOLVED step "${resolution.stepId}" cli "${resolution.cli}"${model} from ${resolution.source}${config}`); } if (report.ok) io.stdout(`CHECK PASSED ${report.path ?? ''}`.trimEnd()); } diff --git a/sdk/src/cli/check.ts b/sdk/src/cli/check.ts index 7d14e9df..384570cf 100644 --- a/sdk/src/cli/check.ts +++ b/sdk/src/cli/check.ts @@ -4,12 +4,14 @@ import { spawnSync } from 'node:child_process'; import { parse as parseYaml } from 'yaml'; import { CompileError, compileSpec, kernelToAuthoring } from '../compile.js'; import { MODEL_ENV } from '../worker.js'; +import { modelNameError } from '../model-name.js'; import type { FlowSpec } from '../spec.js'; import type { CheckFailureKind } from '../failure-kinds.js'; import { preflight, CliProbeError, type CliResolution, + type CliProbeResult, type PreflightDiagnostic, type PreflightProbes, } from '../preflight.js'; @@ -17,6 +19,7 @@ import { interface ProjectConfig { cli?: string; executors: string[]; + models: string[]; directory: string; path?: string; } @@ -57,6 +60,8 @@ export function checkFlow(path: string): CheckExecution { projectCli: config.cli, projectConfigPath: config.path, projectSearchStart: dirname(absolutePath), + models: config.models, + ...(config.path !== undefined ? { modelRegistryPath: config.path } : {}), probes, }); return { @@ -123,15 +128,15 @@ function readFlow(path: string): FlowSpec { function readProjectConfig(start: string): ProjectConfig { const configPath = findConfig(start); - if (configPath === undefined) return { executors: [], directory: start }; + if (configPath === undefined) return { executors: [], models: [], directory: start }; let value: unknown; try { value = JSON.parse(readFileSync(configPath, 'utf8')); } catch { throw new CheckFailure('config_invalid', `Project config "${configPath}" is not valid JSON.`); } - if (!isObject(value) || Object.keys(value).some((key) => !['cli', 'executors'].includes(key))) { - throw new CheckFailure('config_invalid', `Project config "${configPath}" expects only cli and executors.`); + if (!isObject(value) || Object.keys(value).some((key) => !['cli', 'executors', 'models'].includes(key))) { + throw new CheckFailure('config_invalid', `Project config "${configPath}" expects only cli, executors, and models.`); } if (value['cli'] !== undefined && !isNonEmptyString(value['cli'])) { throw new CheckFailure('config_invalid', `Project config "${configPath}" has an invalid cli.`); @@ -139,9 +144,24 @@ function readProjectConfig(start: string): ProjectConfig { if (value['executors'] !== undefined && (!Array.isArray(value['executors']) || !value['executors'].every(isNonEmptyString))) { throw new CheckFailure('config_invalid', `Project config "${configPath}" has invalid executors.`); } + if (value['models'] !== undefined) { + if (!Array.isArray(value['models'])) { + throw new CheckFailure('config_invalid', `Project config "${configPath}" has invalid models; expected an exact string allowlist.`); + } + for (const [index, model] of value['models'].entries()) { + const problem = modelNameError(model); + if (problem !== undefined) { + throw new CheckFailure('config_invalid', `Project config "${configPath}" models[${index}]: ${problem}.`); + } + } + if (new Set(value['models']).size !== value['models'].length) { + throw new CheckFailure('config_invalid', `Project config "${configPath}" has duplicate models.`); + } + } return { ...(value['cli'] !== undefined ? { cli: value['cli'] as string } : {}), executors: (value['executors'] as string[] | undefined) ?? [], + models: (value['models'] as string[] | undefined) ?? [], directory: dirname(configPath), path: configPath, }; @@ -175,13 +195,32 @@ function probeCli( cli: string, directory: string, model?: string, -): { exists: boolean; authenticated: boolean } { +): CliProbeResult { const executable = resolveExecutable(cli, directory); if (executable === undefined) return { exists: false, authenticated: false }; - // Hand the declared model to the probe the same way the worker hands it to - // the real invocation, and unset it otherwise — a leaked RELAYFLOW_MODEL - // from the checking shell would make preflight validate a model the run - // will never use. + if (model === undefined) { + return { exists: true, authenticated: runAuthProbe(executable, directory) === 0 }; + } + + // A successful scoped probe proves both auth and exact-model access in one + // round trip. On failure, repeat without a model solely to distinguish an + // authentication failure from a typed model_unavailable refusal. + const scopedStatus = runAuthProbe(executable, directory, model); + if (scopedStatus === 0) { + return { exists: true, authenticated: true, modelAvailable: true }; + } + const authStatus = runAuthProbe(executable, directory); + return { + exists: true, + authenticated: authStatus === 0, + modelAvailable: false, + }; +} + +function runAuthProbe(executable: string, directory: string, model?: string): number | null { + // Hand the declared model to the probe exactly as the worker hands it to the + // real invocation, and explicitly unset it otherwise. The CLI contract says + // exit 0 only when that exact model is usable by the current credential. const env = { ...process.env }; delete env[MODEL_ENV]; if (model !== undefined) env[MODEL_ENV] = model; @@ -193,7 +232,7 @@ function probeCli( }); const failure = classifySpawnFailure(result.error, result.signal, 10_000); if (failure !== undefined) throw failure; - return { exists: true, authenticated: result.status === 0 }; + return result.status; } function resolveExecutable(command: string, directory: string): string | undefined { diff --git a/sdk/src/compile.ts b/sdk/src/compile.ts index aeecde36..48ffa45f 100644 --- a/sdk/src/compile.ts +++ b/sdk/src/compile.ts @@ -25,6 +25,7 @@ import type { KernelStepSpec, KernelVerificationSpec, LlmStepSpec, + NamedAgentSpec, StepSpec, StepType, } from './spec.js'; @@ -67,7 +68,7 @@ export function compileSpec(spec: unknown): FlowSpec { if (!validation.ok) throw new CompileError(validation.errors); const input = spec as FlowSpec; - const steps = input.steps.map(compileStep); + const steps = input.steps.map((step) => compileStep(resolveNamedAgent(step, input.agents))); const flow: FlowSpec = { version: input.version, ...(input.name !== undefined ? { name: input.name } : {}), @@ -130,6 +131,32 @@ function compileStep(step: StepSpec): StepSpec { } } +/** + * Resolve declarative named-agent sugar before normalization or kernel + * lowering. Explicit step fields win independently, so an author may override + * only the CLI or only the model. The selector and declaration map never + * cross the journal boundary. + */ +function resolveNamedAgent( + step: StepSpec, + agents: Record | undefined, +): StepSpec { + if (step.type !== 'agent' || step.agent === undefined) return step; + const declaration = agents !== undefined && Object.hasOwn(agents, step.agent) + ? agents[step.agent] + : undefined; + if (declaration === undefined) { + throw new CompileError([ + `step "${step.id}": unknown named agent "${step.agent}"`, + ]); + } + return { + ...step, + ...(step.cli === undefined ? { cli: declaration.cli } : {}), + ...(step.model === undefined ? { model: declaration.model } : {}), + }; +} + // Kernel defaults, materialized at compile time so the emitted spec is // byte-identical to the kernel's own serialization of it (spec.rs defaults). const KERNEL_RETRY_DEFAULTS = { @@ -151,7 +178,7 @@ export function toKernelSpec(flow: FlowSpec): KernelRunSpec { ...(flow.description !== undefined ? { description: flow.description } : {}), ...(flow.cli !== undefined ? { cli: flow.cli } : {}), ...(flow.triggers?.length ? { triggers: flow.triggers } : {}), - steps: flow.steps.map(toKernelStep), + steps: flow.steps.map((step) => toKernelStep(resolveNamedAgent(step, flow.agents))), ...(flow.budget !== undefined ? { budget: { diff --git a/sdk/src/failure-kinds.ts b/sdk/src/failure-kinds.ts index cabec045..a669624a 100644 --- a/sdk/src/failure-kinds.ts +++ b/sdk/src/failure-kinds.ts @@ -4,6 +4,8 @@ export const PREFLIGHT_FAILURE_KINDS = [ 'cli_unauthenticated', 'cli_unresolved', 'command_missing', + 'model_unavailable', + 'model_unknown', 'no_executor', 'probe_failed', ] as const; diff --git a/sdk/src/index.ts b/sdk/src/index.ts index f6ba9445..af44bc20 100644 --- a/sdk/src/index.ts +++ b/sdk/src/index.ts @@ -24,6 +24,7 @@ export type { KernelTriggerSpec, KernelVerificationSpec, LlmStepSpec, + NamedAgentSpec, OutputContainsGate, PermissionsSpec, RecoveryMode, @@ -53,6 +54,7 @@ export { preflight, type CliResolution, type CliResolutionSource, + type CliProbeResult, type PreflightDiagnostic, type PreflightOptions, type PreflightProbes, diff --git a/sdk/src/model-name.ts b/sdk/src/model-name.ts new file mode 100644 index 00000000..9abba364 --- /dev/null +++ b/sdk/src/model-name.ts @@ -0,0 +1,20 @@ +/** + * Validate model declaration syntax only. Existence is proven separately by + * the exact project allowlist and the model-scoped CLI preflight; this helper + * deliberately does not infer providers or accept names by pattern. + */ +export function modelNameError(value: unknown): string | undefined { + if (typeof value !== 'string' || value.length === 0) { + return 'expected a non-empty string'; + } + if (value !== value.trim()) { + return 'expected a trimmed string'; + } + for (const character of value) { + const code = character.charCodeAt(0); + if (code < 0x20 || code === 0x7f) { + return 'must not contain control characters'; + } + } + return undefined; +} diff --git a/sdk/src/preflight.ts b/sdk/src/preflight.ts index ee980a69..35bb03a9 100644 --- a/sdk/src/preflight.ts +++ b/sdk/src/preflight.ts @@ -17,6 +17,8 @@ export interface CliResolution { export interface CliProbeResult { exists: boolean; authenticated: boolean; + /** Exact declared model passed the CLI's model-scoped readiness probe. */ + modelAvailable?: boolean; } export type CliProbeFailureDetail = @@ -56,6 +58,9 @@ export interface PreflightOptions { projectCli?: string; projectConfigPath?: string; projectSearchStart?: string; + /** Exact, project-owned model allowlist from the nearest flows.json. */ + models?: readonly string[]; + modelRegistryPath?: string; probes: PreflightProbes; } @@ -65,6 +70,7 @@ export interface PreflightRefusal { message: string; stepId?: string; cli?: string; + model?: string; triggerId?: string; executor?: string; detail?: CliProbeFailureDetail; @@ -94,6 +100,8 @@ export function preflight(flow: FlowSpec, options: PreflightOptions): PreflightR warnOnUnprovableEffects(step, options.probes, diagnostics); if (step.type === 'deterministic') continue; + const declaredModel = step.model; + const modelUnknown = declaredModel !== undefined && !isKnownModel(declaredModel, options.models); const resolution = resolveCli(step, flow, options.projectCli); if (resolution === undefined) { diagnostics.push({ @@ -102,9 +110,34 @@ export function preflight(flow: FlowSpec, options: PreflightOptions): PreflightR stepId: step.id, message: unresolvedCliMessage(step.id, options), }); + if (modelUnknown && declaredModel !== undefined) { + diagnostics.push({ + severity: 'refusal', + kind: 'model_unknown', + stepId: step.id, + model: declaredModel, + message: unknownModelMessage(step.id, declaredModel, undefined, options.modelRegistryPath), + }); + } continue; } resolutions.push(resolution); + if (modelUnknown && resolution.model !== undefined) { + diagnostics.push({ + severity: 'refusal', + kind: 'model_unknown', + stepId: resolution.stepId, + cli: resolution.cli, + model: resolution.model, + message: unknownModelMessage( + resolution.stepId, + resolution.model, + resolution.cli, + options.modelRegistryPath, + ), + }); + continue; + } probeResolvedCli(resolution, options.probes, cliProbeResults, diagnostics); } @@ -119,6 +152,23 @@ export function preflight(flow: FlowSpec, options: PreflightOptions): PreflightR }; } +function isKnownModel(model: string, models: readonly string[] | undefined): boolean { + return models?.includes(model) === true; +} + +function unknownModelMessage( + stepId: string, + model: string, + cli: string | undefined, + registryPath: string | undefined, +): string { + const source = registryPath === undefined + ? 'the nearest project config (no model registry was found)' + : `project model registry "${registryPath}"`; + const cliContext = cli === undefined ? '' : ` for CLI "${cli}"`; + return `Step "${stepId}" declares model "${model}"${cliContext}, but it is not listed in ${source}; add the exact model only after verifying that project is allowed to use it.`; +} + function unresolvedCliMessage(stepId: string, options: PreflightOptions): string { const context = options.projectConfigPath !== undefined ? ` Nearest project config "${options.projectConfigPath}" declares no cli; outer configs are shadowed.` @@ -195,6 +245,15 @@ function probeResolvedCli( cli: resolution.cli, message: `Step "${resolution.stepId}" declares CLI "${resolution.cli}", but "${resolution.cli} auth status" exited non-zero; authenticate it or implement that probe to return exit 0 when authenticated.`, }); + } else if (resolution.model !== undefined && result.modelAvailable !== true) { + diagnostics.push({ + severity: 'refusal', + kind: 'model_unavailable', + stepId: resolution.stepId, + cli: resolution.cli, + model: resolution.model, + message: `Step "${resolution.stepId}" declares model "${resolution.model}" for CLI "${resolution.cli}", but its model-scoped "${resolution.cli} auth status" probe exited non-zero; verify the model name and this credential's access.`, + }); } } diff --git a/sdk/src/spec.ts b/sdk/src/spec.ts index 613293ef..fc9c5838 100644 --- a/sdk/src/spec.ts +++ b/sdk/src/spec.ts @@ -136,6 +136,8 @@ export interface LlmStepSpec extends BaseStepSpec { export interface AgentStepSpec extends BaseStepSpec { type: 'agent'; instruction: string; + /** Named authoring declaration selected from `FlowSpec.agents`. Compiled away. */ + agent?: string; /** Inert preflight declaration; overrides the flow/project CLI default. */ cli?: string; /** @@ -153,6 +155,17 @@ export interface AgentStepSpec extends BaseStepSpec { export type StepSpec = DeterministicStepSpec | LlmStepSpec | AgentStepSpec; +/** + * Reusable authoring declaration for an agent CLI/model pair. Both fields are + * required so selecting a named agent can never inherit a host model. The + * compiler lowers these values into the selected `AgentStepSpec`; the kernel + * never receives this map or a new step field. + */ +export interface NamedAgentSpec { + cli: string; + model: string; +} + /** Inert gate-1 trigger declaration. Matching and dispatch belong to gate 2. */ export interface TriggerSpec { id: string; @@ -173,6 +186,8 @@ export interface FlowSpec { description?: string; /** Inert preflight default for llm/agent steps that do not declare a CLI. */ cli?: string; + /** Named authoring declarations. Compiled into agent steps, never journaled as a new primitive. */ + agents?: Record; /** Declarations checked by preflight; gate 1 never dispatches them. */ triggers?: TriggerSpec[]; steps: StepSpec[]; diff --git a/sdk/src/unknown-keys.ts b/sdk/src/unknown-keys.ts new file mode 100644 index 00000000..a660ebe1 --- /dev/null +++ b/sdk/src/unknown-keys.ts @@ -0,0 +1,52 @@ +/** + * Produce author-facing diagnostics for keys outside a closed schema. A key + * that differs only in casing/separators always matches; otherwise a small + * edit distance catches plain misspellings. + */ +export function unknownKeyErrors( + object: Record, + allowed: readonly string[], + at: string, +): string[] { + const errors: string[] = []; + for (const key of Object.keys(object)) { + if (allowed.includes(key)) continue; + const suggestion = nearestKey(key, allowed); + errors.push( + suggestion !== null + ? `${at}: unknown key "${key}" — did you mean "${suggestion}"?` + : `${at}: unknown key "${key}" (expected one of ${allowed.join(' | ')})`, + ); + } + return errors; +} + +function nearestKey(key: string, allowed: readonly string[]): string | null { + const normalize = (value: string): string => value.toLowerCase().replace(/[_-]/g, ''); + let best: string | null = null; + let bestDistance = Number.POSITIVE_INFINITY; + for (const candidate of allowed) { + if (normalize(candidate) === normalize(key)) return candidate; + const distance = levenshtein(key.toLowerCase(), candidate.toLowerCase()); + if (distance < bestDistance) { + bestDistance = distance; + best = candidate; + } + } + return best !== null && bestDistance <= 3 && bestDistance < best.length ? best : null; +} + +function levenshtein(a: string, b: string): number { + let previous: number[] = Array.from({ length: b.length + 1 }, (_, i) => i); + for (let i = 1; i <= a.length; i++) { + const current: number[] = [i]; + for (let j = 1; j <= b.length; j++) { + const deletion = (previous[j] ?? 0) + 1; + const insertion = (current[j - 1] ?? 0) + 1; + const substitution = (previous[j - 1] ?? 0) + (a[i - 1] === b[j - 1] ? 0 : 1); + current[j] = Math.min(deletion, insertion, substitution); + } + previous = current; + } + return previous[b.length] ?? 0; +} diff --git a/sdk/src/validate.ts b/sdk/src/validate.ts index dbe0ff9f..f0403042 100644 --- a/sdk/src/validate.ts +++ b/sdk/src/validate.ts @@ -9,6 +9,7 @@ import type { DeterministicStepSpec, FlowSpec, LlmStepSpec, + NamedAgentSpec, PermissionsSpec, RecoveryMode, StepSpec, @@ -17,6 +18,8 @@ import type { VerificationSpec, } from './spec.js'; import { SPEC_SCHEMA_VERSION } from './spec.js'; +import { modelNameError } from './model-name.js'; +import { unknownKeyErrors } from './unknown-keys.js'; export interface ValidationResult { ok: boolean; @@ -41,13 +44,14 @@ const DECIMAL_RE = /^\d+(\.\d+)?$/; // unknown keys (AGENTS.md rule 4; RFC covenant 2): a typo'd key like // `depends_on` must be an error naming the nearest valid key, never a // silently discarded field — silently dropping `dependsOn` loses ordering. -const ROOT_KEYS = ['version', 'name', 'description', 'cli', 'triggers', 'steps', 'budget'] as const; +const ROOT_KEYS = ['version', 'name', 'description', 'cli', 'agents', 'triggers', 'steps', 'budget'] as const; +const AGENT_DECLARATION_KEYS = ['cli', 'model'] as const; const BUDGET_KEYS = ['maxTokensIn', 'maxTokensOut', 'maxDollars'] as const; const STEP_COMMON_KEYS = ['id', 'type', 'dependsOn', 'verification', 'maxIterations', 'timeoutMs'] as const; const STEP_TYPE_KEYS: Record = { deterministic: ['command'], llm: ['prompt', 'model', 'cli'], - agent: ['instruction', 'cli', 'model', 'surfaces', 'recoveryMode', 'permissions'], + agent: ['instruction', 'agent', 'cli', 'model', 'surfaces', 'recoveryMode', 'permissions'], }; const VERIFICATION_KEYS: Record = { exit_code: ['type', 'expect'], @@ -73,6 +77,7 @@ const TRIGGER_KEYS = [ class Validator { private errors: string[] = []; private ids = new Set(); + private agentNames = new Set(); fail(msg: string): void { this.errors.push(msg); @@ -84,15 +89,7 @@ class Validator { * `unknown key "depends_on" — did you mean "dependsOn"?`. */ private checkKeys(obj: Record, allowed: readonly string[], at: string): void { - for (const key of Object.keys(obj)) { - if (allowed.includes(key)) continue; - const suggestion = nearestKey(key, allowed); - this.fail( - suggestion !== null - ? `${at}: unknown key "${key}" — did you mean "${suggestion}"?` - : `${at}: unknown key "${key}" (expected one of ${allowed.join(' | ')})`, - ); - } + for (const error of unknownKeyErrors(obj, allowed, at)) this.fail(error); } result(): ValidationResult { @@ -125,6 +122,8 @@ class Validator { this.fail('spec.cli: expected a non-empty string'); } + if (s['agents'] !== undefined) this.validateAgents(s['agents']); + if (s['triggers'] !== undefined) this.validateTriggers(s['triggers']); if (s['budget'] !== undefined) this.validateBudget(s['budget']); @@ -144,6 +143,31 @@ class Validator { return this.result(); } + private validateAgents(value: unknown): void { + if (!isObject(value)) { + this.fail('spec.agents: expected a map of named { cli, model } declarations'); + return; + } + for (const [name, raw] of Object.entries(value)) { + const at = `spec.agents.${name}`; + if (!isNonEmptyString(name) || name !== name.trim()) { + this.fail('spec.agents: agent names must be non-empty trimmed strings'); + continue; + } + this.agentNames.add(name); + if (!isObject(raw)) { + this.fail(`${at}: expected an object with cli and model`); + continue; + } + this.checkKeys(raw, AGENT_DECLARATION_KEYS, at); + const declaration = raw as unknown as NamedAgentSpec; + if (!isNonEmptyString(declaration.cli) || declaration.cli !== declaration.cli.trim()) { + this.fail(`${at}.cli: expected a non-empty trimmed string`); + } + this.validateModel(declaration.model, at, true); + } + } + private validateBudget(b: unknown): void { if (!isObject(b)) { this.fail('spec.budget: expected an object'); @@ -292,9 +316,7 @@ class Validator { if (!isNonEmptyString(st.prompt)) { this.fail(`${at}.prompt: expected a non-empty string`); } - if (st.model !== undefined && typeof st.model !== 'string') { - this.fail(`${at}.model: expected a string`); - } + this.validateModel(st.model, at); this.validateCli(st.cli, at); } @@ -302,6 +324,13 @@ class Validator { if (!isNonEmptyString(st.instruction)) { this.fail(`${at}.instruction: expected a non-empty string`); } + if (st.agent !== undefined) { + if (!isNonEmptyString(st.agent)) { + this.fail(`${at}.agent: expected a non-empty named agent`); + } else if (!this.agentNames.has(st.agent)) { + this.fail(`${at}.agent: unknown named agent "${st.agent}"`); + } + } if (st.recoveryMode !== undefined && !RECOVERY_MODES.has(st.recoveryMode)) { this.fail(`${at}.recoveryMode: expected reset | inspect | manual`); } @@ -317,14 +346,14 @@ class Validator { } } - private validateModel(model: unknown, at: string): void { + private validateModel(model: unknown, at: string, required = false): void { // Rejecting the empty string matters: it would reach the CLI as // RELAYFLOW_MODEL='', which reads as "declared, and declared as // nothing" — the CLI cannot tell it from a real value and would // pass an empty --model. Absent and empty must not look alike. - if (model !== undefined && !isNonEmptyString(model)) { - this.fail(`${at}.model: expected a non-empty string`); - } + if (model === undefined && !required) return; + const problem = modelNameError(model); + if (problem !== undefined) this.fail(`${at}.model: ${problem}`); } private validateSurfaces(surfaces: AgentStepSpec['surfaces'], at: string): void { @@ -419,43 +448,6 @@ export function validateSpec(spec: unknown): ValidationResult { return new Validator().run(spec); } -// --- unknown-key suggestions ------------------------------------------------ - -/** - * The nearest valid key for a typo, or null when nothing is close. A key that - * differs only in casing/separators (`depends_on` -> `dependsOn`) always - * matches; otherwise small edit distances catch plain misspellings. - */ -function nearestKey(key: string, allowed: readonly string[]): string | null { - const normalize = (value: string): string => value.toLowerCase().replace(/[_-]/g, ''); - let best: string | null = null; - let bestDistance = Number.POSITIVE_INFINITY; - for (const candidate of allowed) { - if (normalize(candidate) === normalize(key)) return candidate; - const distance = levenshtein(key.toLowerCase(), candidate.toLowerCase()); - if (distance < bestDistance) { - bestDistance = distance; - best = candidate; - } - } - return best !== null && bestDistance <= 3 && bestDistance < best.length ? best : null; -} - -function levenshtein(a: string, b: string): number { - let previous: number[] = Array.from({ length: b.length + 1 }, (_, i) => i); - for (let i = 1; i <= a.length; i++) { - const current: number[] = [i]; - for (let j = 1; j <= b.length; j++) { - const deletion = (previous[j] ?? 0) + 1; - const insertion = (current[j - 1] ?? 0) + 1; - const substitution = (previous[j - 1] ?? 0) + (a[i - 1] === b[j - 1] ? 0 : 1); - current[j] = Math.min(deletion, insertion, substitution); - } - previous = current; - } - return previous[b.length] ?? 0; -} - // --- predicates ------------------------------------------------------------- function isObject(v: unknown): v is Record { diff --git a/sdk/tests/cli.test.ts b/sdk/tests/cli.test.ts index ca55d559..33755fbe 100644 --- a/sdk/tests/cli.test.ts +++ b/sdk/tests/cli.test.ts @@ -1,4 +1,4 @@ -import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { once } from 'node:events'; import type { Server } from 'node:net'; import { tmpdir } from 'node:os'; @@ -23,6 +23,12 @@ import { const TESTDATA = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'testdata'); const PREFLIGHT = join(TESTDATA, 'preflight'); const LADDER = ['hello-deterministic', 'hello-llm', 'hello-agent'] as const; +const TEST_MODELS = [ + 'claude-haiku-4-5-20251001', + 'claude-sonnet-5', + 'deterministic-test-stub', + 'test-model-v1', +] as const; const temporaryDirectories: string[] = []; const loopbackServers: Server[] = []; const KERNEL_RETRY = { @@ -55,10 +61,41 @@ async function run(path: string, json = false): Promise<{ code: number; stdout: function temporaryProject(prefix = 'flows-check-'): string { const directory = mkdtempSync(join(tmpdir(), prefix)); temporaryDirectories.push(directory); - writeFileSync(join(directory, 'flows.json'), JSON.stringify({ executors: [] })); + writeFileSync(join(directory, 'flows.json'), JSON.stringify({ executors: [], models: TEST_MODELS })); return directory; } +function namedAgentProject(model: string, allowedModels: string[]): { + directory: string; + flowPath: string; + probeLog: string; +} { + const directory = temporaryProject('flows-model-'); + const cliPath = join(directory, 'model-cli'); + const probeLog = `${cliPath}.log`; + writeFileSync(cliPath, `#!/bin/sh +test "$1 $2" = "auth status" || exit 9 +printf '%s\n' "\${RELAYFLOW_MODEL-UNSET}" >> "$0.log" +test "\${RELAYFLOW_MODEL-UNSET}" = "UNSET" -o "\${RELAYFLOW_MODEL-UNSET}" = "available-model" +`); + chmodSync(cliPath, 0o755); + writeFileSync(join(directory, 'flows.json'), JSON.stringify({ executors: [], models: allowedModels })); + const flowPath = join(directory, 'named-agent.flow.yaml'); + writeFileSync(flowPath, ` +version: '0.1.0' +agents: + reviewer: + cli: ./model-cli + model: ${model} +steps: + - id: review + type: agent + agent: reviewer + instruction: Review the change. +`); + return { directory, flowPath, probeLog }; +} + /** * RFC-0001 §96 makes *the ladder flows* the subject of the refusal clause, so * the fault is induced on the canonical flows themselves rather than on a @@ -94,6 +131,70 @@ async function startCliLoopback(dataDir: string, handlers: LoopbackHandlers): Pr } describe('flows check CLI', () => { + it('accepts an exact allowlisted named-agent model and probes that model', async () => { + const fixture = namedAgentProject('available-model', ['available-model']); + const result = await run(fixture.flowPath); + + expect(result.code).toBe(0); + expect(result.stdout.join('\n')).toContain('model "available-model"'); + expect(readFileSync(fixture.probeLog, 'utf8')).toBe('available-model\n'); + }); + + it('checks the same named-agent contract from declarative JSON', async () => { + const fixture = namedAgentProject('available-model', ['available-model']); + const jsonPath = join(fixture.directory, 'named-agent.flow.json'); + writeFileSync(jsonPath, JSON.stringify(parseYaml(readFileSync(fixture.flowPath, 'utf8')))); + + const result = await run(jsonPath); + expect(result.code).toBe(0); + expect(result.stdout.join('\n')).toContain('model "available-model"'); + }); + + it('refuses a typo model before probing or contacting relayflowd', async () => { + const fixture = namedAgentProject('available-modle', ['available-model']); + const checked = await run(fixture.flowPath); + + expect(checked.code).toBe(2); + expect(checked.stderr.join('\n')).toContain('REFUSED [model_unknown]'); + expect(checked.stderr.join('\n')).toContain('available-modle'); + expect(existsSync(fixture.probeLog)).toBe(false); + + const output = capture(); + const runCode = await runCli([ + 'run', '--data-dir', join(fixture.directory, 'no-daemon'), fixture.flowPath, + ], output.io); + expect(runCode).toBe(2); + expect(output.stderr.join('\n')).toContain('REFUSED [model_unknown]'); + expect(output.stderr.join('\n')).not.toContain('daemon_unreachable'); + expect(existsSync(fixture.probeLog)).toBe(false); + }); + + it('distinguishes an allowlisted but inaccessible model from broken auth', async () => { + const fixture = namedAgentProject('denied-model', ['denied-model']); + const result = await run(fixture.flowPath); + + expect(result.code).toBe(2); + expect(result.stderr.join('\n')).toContain('REFUSED [model_unavailable]'); + expect(result.stderr.join('\n')).toContain('denied-model'); + expect(readFileSync(fixture.probeLog, 'utf8')).toBe('denied-model\nUNSET\n'); + }); + + it.each([ + [{ models: ['available-model', 'available-model'] }, 'duplicate models'], + [{ models: [' '] }, 'models[0]: expected a trimmed string'], + [{ models: 'available-model' }, 'expected an exact string allowlist'], + ] as const)('refuses malformed project model registry %j', async (config, expected) => { + const directory = temporaryProject('flows-model-config-'); + writeFileSync(join(directory, 'flows.json'), JSON.stringify(config)); + const path = join(directory, 'flow.yaml'); + writeFileSync(path, "version: '0.1.0'\nsteps:\n - id: ready\n type: deterministic\n command: printf ready\n"); + + const result = await run(path); + expect(result.code).toBe(2); + expect(result.stderr.join('\n')).toContain('REFUSED [config_invalid]'); + expect(result.stderr.join('\n')).toContain(expected); + }); + it('explains kernel-dialect routing and names the offending mixed-dialect key', async () => { const directory = temporaryProject(); const path = join(directory, 'mixed.flow.yaml'); diff --git a/sdk/tests/live-kernel.test.ts b/sdk/tests/live-kernel.test.ts index 5ae7a9b0..e7c78153 100644 --- a/sdk/tests/live-kernel.test.ts +++ b/sdk/tests/live-kernel.test.ts @@ -704,21 +704,35 @@ steps: }); await worker.attach(); - const started = await client.runStart(toKernelSpec(compileYaml(` + const compiled = compileYaml(` version: '0.1.0' +agents: + model-probe: + cli: ${JSON.stringify(cli)} + model: declared-model-xyz steps: - id: probe type: agent - cli: ${JSON.stringify(cli)} - model: declared-model-xyz + agent: model-probe instruction: Report the model env var. -`))); +`); + expect(compiled).not.toHaveProperty('agents'); + expect(compiled.steps[0]).toMatchObject({ + type: 'agent', + cli, + model: 'declared-model-xyz', + }); + const started = await client.runStart(toKernelSpec(compiled)); expect(await waitForStep(client, started.run_id, 'probe', 'done')).toMatchObject({ type: 'agent', state: 'done', }); const entries = (await client.journalRead(started.run_id)).entries; + const spawned = entries.find( + (entry) => (entry as { entry_type: string }).entry_type === 'run.spawned', + ) as { payload: { spec: { steps: Array<{ model?: string }> } } } | undefined; + expect(spawned?.payload.spec.steps[0]?.model).toBe('declared-model-xyz'); const completed = entries.find( (entry) => (entry as { entry_type: string; step_id?: string }).entry_type === 'step.completed' && (entry as { step_id?: string }).step_id === 'probe', diff --git a/sdk/tests/model-selection.test.ts b/sdk/tests/model-selection.test.ts new file mode 100644 index 00000000..92c779bc --- /dev/null +++ b/sdk/tests/model-selection.test.ts @@ -0,0 +1,155 @@ +import { describe, expect, it } from 'vitest'; +import { CompileError, compileSpec, compileYaml, toKernelSpec } from '../src/compile.js'; +import type { AgentStepSpec } from '../src/spec.js'; + +describe('named agent declarations', () => { + it('lowers a selected agent CLI and model into the existing agent step', () => { + const flow = compileYaml(` +version: '0.1.0' +agents: + reviewer: + cli: claude + model: claude-sonnet-4-6 +steps: + - id: review + type: agent + agent: reviewer + instruction: Review the change. +`); + + expect(flow).not.toHaveProperty('agents'); + expect(flow.steps[0] as AgentStepSpec).toMatchObject({ + id: 'review', + type: 'agent', + cli: 'claude', + model: 'claude-sonnet-4-6', + }); + expect(toKernelSpec(flow).steps[0]).toMatchObject({ + type: 'agent', + cli: 'claude', + model: 'claude-sonnet-4-6', + }); + }); + + it('applies independent precedence: step override > named agent > flow CLI', () => { + const flow = compileYaml(` +version: '0.1.0' +cli: flow-cli +agents: + reviewer: + cli: named-cli + model: named-model +steps: + - id: named + type: agent + agent: reviewer + instruction: Named values. + - id: cli-override + type: agent + agent: reviewer + cli: step-cli + instruction: Override only CLI. + - id: model-override + type: agent + agent: reviewer + model: step-model + instruction: Override only model. + - id: anonymous + type: agent + instruction: Preserve existing anonymous resolution. +`); + + expect(flow.steps).toEqual(expect.arrayContaining([ + expect.objectContaining({ id: 'named', cli: 'named-cli', model: 'named-model' }), + expect.objectContaining({ id: 'cli-override', cli: 'step-cli', model: 'named-model' }), + expect.objectContaining({ id: 'model-override', cli: 'named-cli', model: 'step-model' }), + ])); + expect(flow.steps.find((step) => step.id === 'anonymous')).not.toHaveProperty('cli'); + expect(flow.steps.find((step) => step.id === 'anonymous')).not.toHaveProperty('model'); + expect(flow.cli).toBe('flow-cli'); + }); + + it('lowers a raw typed FlowSpec passed directly to the kernel mapper', () => { + const kernel = toKernelSpec({ + version: '0.1.0', + agents: { reviewer: { cli: 'claude', model: 'declared-model' } }, + steps: [{ + id: 'review', + type: 'agent', + agent: 'reviewer', + instruction: 'Review.', + }], + }); + + expect(kernel).not.toHaveProperty('agents'); + expect(kernel.steps[0]).toMatchObject({ cli: 'claude', model: 'declared-model' }); + }); + + it('refuses an unknown named agent at the authoring boundary', () => { + expect(() => compileYaml(` +version: '0.1.0' +agents: + reviewer: { cli: claude, model: declared-model } +steps: + - id: review + type: agent + agent: reviwer + instruction: Review. +`)).toThrow('spec.steps[0].agent: unknown named agent "reviwer"'); + }); + + it.each([ + ['', 'expected a non-empty string'], + [' declared-model', 'expected a trimmed string'], + ['declared-model ', 'expected a trimmed string'], + ['declared\\tmodel', 'must not contain control characters'], + ])('refuses malformed model %j with an author-facing field error', (model, expected) => { + const value = model === 'declared\\tmodel' ? 'declared\tmodel' : model; + expect(() => compileSpec({ + version: '0.1.0', + steps: [{ id: 'review', type: 'agent', instruction: 'Review.', model: value }], + })).toThrow(`spec.steps[0].model: ${expected}`); + }); + + it('requires both CLI and model on every named declaration', () => { + expect(() => compileSpec({ + version: '0.1.0', + agents: { reviewer: { cli: 'claude' } }, + steps: [{ id: 'review', type: 'agent', agent: 'reviewer', instruction: 'Review.' }], + })).toThrow('spec.agents.reviewer.model: expected a non-empty string'); + }); + + it('refuses a model typo inside a named declaration instead of dropping it', () => { + expect(() => compileYaml(` +version: '0.1.0' +agents: + reviewer: + cli: claude + modle: claude-sonnet-4-6 +steps: + - id: review + type: agent + agent: reviewer + instruction: Review the change. +`)).toThrow(CompileError); + + try { + compileYaml(` +version: '0.1.0' +agents: + reviewer: + cli: claude + modle: claude-sonnet-4-6 +steps: + - id: review + type: agent + agent: reviewer + instruction: Review the change. +`); + } catch (error) { + expect((error as CompileError).errors.join('\n')).toContain( + 'spec.agents.reviewer: unknown key "modle" — did you mean "model"?', + ); + } + }); +}); diff --git a/sdk/tests/preflight.test.ts b/sdk/tests/preflight.test.ts index d55fe50f..8034c1a1 100644 --- a/sdk/tests/preflight.test.ts +++ b/sdk/tests/preflight.test.ts @@ -17,7 +17,7 @@ function flow(step: FlowSpec['steps'][number]): FlowSpec { function probes(overrides: Partial = {}): PreflightProbes { return { - cli: (): CliProbeResult => ({ exists: true, authenticated: true }), + cli: (): CliProbeResult => ({ exists: true, authenticated: true, modelAvailable: true }), executor: () => true, command: () => true, ...overrides, @@ -224,6 +224,8 @@ describe('preflight: CLI resolution and refusal predicates', () => { preflight(flow({ id: 'a', type: 'llm', prompt: 'p', cli: 'x' }), { probes: probes({ cli: () => ({ exists: true, authenticated: false }) }) }), preflight(flow({ id: 'a', type: 'llm', prompt: 'p' }), { probes: probes() }), preflight(flow({ id: 'a', type: 'deterministic', command: './missing' }), { probes: probes({ command: () => false }) }), + preflight(flow({ id: 'a', type: 'agent', instruction: 'i', cli: 'x', model: 'typo-model' }), { models: ['known-model'], probes: probes() }), + preflight(flow({ id: 'a', type: 'agent', instruction: 'i', cli: 'x', model: 'known-model' }), { models: ['known-model'], probes: probes({ cli: () => ({ exists: true, authenticated: true, modelAvailable: false }) }) }), preflight({ ...flow({ id: 'a', type: 'deterministic', command: 'x' }), triggers: [{ id: 't', executor: 'e' }] }, { probes: probes({ executor: () => false, command: () => false }) }), preflight(flow({ id: 'a', type: 'llm', prompt: 'p', cli: 'x' }), { probes: probes({ cli: () => { throw new Error('raw secret'); } }) }), ]; @@ -234,4 +236,20 @@ describe('preflight: CLI resolution and refusal predicates', () => { expect(new Set(refusalKinds)).toEqual(new Set(PREFLIGHT_FAILURE_KINDS)); expect(JSON.stringify(scenarios)).not.toContain('raw secret'); }); + + it('reports an unknown model even when the same step has no resolvable CLI', () => { + const result = preflight( + flow({ id: 'a', type: 'agent', instruction: 'i', model: 'typo-model' }), + { models: ['known-model'], probes: probes() }, + ); + + expect(result.diagnostics.map((diagnostic) => diagnostic.kind)).toEqual([ + 'cli_unresolved', + 'model_unknown', + ]); + expect(result.diagnostics[1]).toMatchObject({ + stepId: 'a', + model: 'typo-model', + }); + }); }); diff --git a/testdata/flows.json b/testdata/flows.json index 4e7062d1..3396475c 100644 --- a/testdata/flows.json +++ b/testdata/flows.json @@ -1,4 +1,10 @@ { "cli": "./preflight/authenticated-cli", - "executors": ["agent-worker"] + "executors": ["agent-worker"], + "models": [ + "claude-haiku-4-5-20251001", + "claude-sonnet-5", + "deterministic-test-stub", + "test-model-v1" + ] } From 78efc0d15f32246332f9cf64ffba7f838573d02e Mon Sep 17 00:00:00 2001 From: kjgbot Date: Wed, 2 Sep 2026 18:49:05 +0200 Subject: [PATCH 02/15] docs(review): record PR 136 fresh review Session-Id: 01a062b7-2aef-7772-8225-7cb00ee311dd --- ops/reviews/20260902-1710-pr136-history.md | 449 ++++++++++++++++++ .../20260902-1710-pr136-maintainability.md | 357 ++++++++++++++ ops/reviews/20260902-1710-pr136-structure.md | 261 ++++++++++ 3 files changed, 1067 insertions(+) create mode 100644 ops/reviews/20260902-1710-pr136-history.md create mode 100644 ops/reviews/20260902-1710-pr136-maintainability.md create mode 100644 ops/reviews/20260902-1710-pr136-structure.md diff --git a/ops/reviews/20260902-1710-pr136-history.md b/ops/reviews/20260902-1710-pr136-history.md new file mode 100644 index 00000000..4ca6d20e --- /dev/null +++ b/ops/reviews/20260902-1710-pr136-history.md @@ -0,0 +1,449 @@ +# PR #136 — history and regression-fit review + +- **Lens:** history/regression fit, prior model failures, v1/default compatibility, + issue #132 scope, and the deferred TypeScript `FlowHeader` +- **PR:** #136 — `feat(sdk): declare agent CLI and model with fail-closed checks` +- **Exact head reviewed:** `321b27216e561561e1e022a7b4d973e2182ee480` +- **Merged main / merge base:** `a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2` +- **Constitution read in full:** `AGENTS.md` and + `docs/RFC-0001-everything-is-a-relayflow.md` +- **Issue and PR record read in full:** issue #132, PR #136 body/history/diff, + and the relevant dependency statement in open PR #134 +- **Mode:** assessment only; no product code was edited, committed, pushed, or + merged + +## Verdict + +**PASS — no blocking findings at this exact head.** + +The one-commit change fits the repository's history: PR #130 already added the +inert agent-step `model` field, carried it through the existing kernel dialect, +journaled it in `run.spawned.payload.spec`, handed it to the worker as +`RELAYFLOW_MODEL`, and explicitly removed ambient parent values when no model +was declared. PR #136 does not add kernel vocabulary. It adds declarative +YAML/JSON named-agent sugar and a fail-closed, project-owned exact model +registry above those existing fields. + +The change rejects misspelled declaration keys, malformed model strings, +models absent from the exact case-sensitive allowlist, inaccessible allowlisted +models, and unknown named-agent selectors. The compiler independently applies +step overrides over named declarations; the established flow/project fallback +remains CLI-only. The selector and declaration map disappear before the kernel +boundary, while the chosen `cli` and `model` remain on the existing agent step +and therefore in the journaled spec. + +Issue #132 is **not complete**: the requested TypeScript +`FlowHeader.agents`/model declaration still does not exist. That is not hidden. +PR #136 says it ships only the canonical declarative YAML/JSON contract and +uses `Refs #132`, not a closing claim. `docs/SURFACE.md` repeats the same +implementation-status boundary, and open PR #134 expressly lists model headers +as deferred. This head is therefore honest and mergeable as a partial issue +slice, but it must not be credited as completing issue #132's model-header item +or the issue's broader done criteria. + +## 1. Exact review boundary and commit story + +Literal commands: + +```sh +git rev-parse 'a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2^{commit}' +git rev-parse '321b27216e561561e1e022a7b4d973e2182ee480^{commit}' +git merge-base a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2 321b27216e561561e1e022a7b4d973e2182ee480 +git status --porcelain=v1 +git log --format='%H %P%n%s%n%b' a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..321b27216e561561e1e022a7b4d973e2182ee480 +``` + +Captured output: + +```text +a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2 +321b27216e561561e1e022a7b4d973e2182ee480 +a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2 +321b27216e561561e1e022a7b4d973e2182ee480 a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2 +feat(sdk): add declared agent model contract +Session-Id: 01a062b7-2aef-7772-8225-7cb00ee311dd +``` + +`git status --porcelain=v1` produced no output. The commit is directly based on +the required merged-main commit. + +Literal commands: + +```sh +git diff --stat a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..321b27216e561561e1e022a7b4d973e2182ee480 +git diff --name-status a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..321b27216e561561e1e022a7b4d973e2182ee480 +``` + +Captured output: + +```text + docs/SURFACE.md | 42 +++++++++-- + sdk/src/cli.ts | 3 +- + sdk/src/cli/check.ts | 57 +++++++++++--- + sdk/src/compile.ts | 31 +++++++- + sdk/src/failure-kinds.ts | 2 + + sdk/src/index.ts | 2 + + sdk/src/model-name.ts | 20 +++++ + sdk/src/preflight.ts | 59 +++++++++++++++ + sdk/src/spec.ts | 15 ++++ + sdk/src/unknown-keys.ts | 52 +++++++++++++ + sdk/src/validate.ts | 102 ++++++++++++------------- + sdk/tests/cli.test.ts | 105 +++++++++++++++++++++++++- + sdk/tests/live-kernel.test.ts | 22 +++++- + sdk/tests/model-selection.test.ts | 155 ++++++++++++++++++++++++++++++++++++++ + sdk/tests/preflight.test.ts | 20 ++++- + testdata/flows.json | 8 +- + 16 files changed, 614 insertions(+), 81 deletions(-) +M docs/SURFACE.md +M sdk/src/cli.ts +M sdk/src/cli/check.ts +M sdk/src/compile.ts +M sdk/src/failure-kinds.ts +M sdk/src/index.ts +A sdk/src/model-name.ts +M sdk/src/preflight.ts +M sdk/src/spec.ts +A sdk/src/unknown-keys.ts +M sdk/src/validate.ts +M sdk/tests/cli.test.ts +M sdk/tests/live-kernel.test.ts +A sdk/tests/model-selection.test.ts +M sdk/tests/preflight.test.ts +M testdata/flows.json +``` + +There is no kernel, protocol, v1 runtime, regression declaration, workflow, or +gate-definition edit. The added modules are small and single-purpose +(`model-name.ts`, 20 lines; `unknown-keys.ts`, 52 lines), and no changed source +file crosses the repository's 500-line smell threshold. + +Remote identity was rechecked after testing: + +```sh +gh pr view 136 --repo AgentWorkforce/flows --json number,state,baseRefOid,headRefOid,title --jq . +``` + +```text +{"baseRefOid":"a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2","headRefOid":"321b27216e561561e1e022a7b4d973e2182ee480","number":136,"state":"OPEN","title":"feat(sdk): declare agent CLI and model with fail-closed checks"} +``` + +## 2. History fit: closes the real ambient-model failure above the existing boundary + +The relevant source history contains exactly two commits touching +`RELAYFLOW_MODEL`: + +```sh +git log --all --format='%H %s' -S'RELAYFLOW_MODEL' -- sdk/src testdata docs +``` + +```text +321b27216e561561e1e022a7b4d973e2182ee480 feat(sdk): add declared agent model contract +51415d9c65ef5c727c560c700f63932893a1e224 feat(gate2): real Claude analyzer for hn-monitor, with a declared model (#130) +82be45ff91c1e77db8422b72324fba7ecf7fdda7 feat(gate2): real Claude analyzer for hn-monitor, with a declared model +``` + +The merged PR #130 commit records why the existing field was added: a host +model alias (`fable`) broke four real paths and made the executed model +unrecoverable from the journal. At current head, `sdk/src/worker.ts:200-206` +still deletes inherited `RELAYFLOW_MODEL` and sets it only from the dispatched +step. `kernel/relayflowd-core/src/spec.rs:282-291` still carries `cli` and +`model` as inert fields on `StepKind::Agent`. PR #136 correctly builds on that +history instead of adding provider logic or a second model field. + +The live regression for both halves passed: + +```sh +RELAYFLOWD_BIN=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd ./node_modules/.bin/vitest run tests/live-kernel.test.ts -t 'AgentWorker passes a declared model|AgentWorker leaves RELAYFLOW_MODEL UNSET' --reporter=verbose +``` + +```text + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd +LIVE_KERNEL flows=/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk/dist/cli.js + + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker passes a declared model to the CLI as RELAYFLOW_MODEL + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker leaves RELAYFLOW_MODEL UNSET when the step declares no model + + Test Files 1 passed (1) + Tests 2 passed | 15 skipped (17) + Start at 18:19:20 + Duration 2.06s (transform 305ms, setup 0ms, collect 475ms, tests 511ms, environment 1ms, prepare 177ms) +``` + +The first test asserts both the `run.spawned.payload.spec.steps[0].model` value +and the subprocess-observed value. The second contaminates the parent +environment first and asserts that an inline agent with no model sees the +variable as absent. This directly preserves the prior bug fix. + +## 3. Exact, fail-closed typo and model checks + +The two layers have separate jobs: + +- compile-time validation closes object schemas and model syntax; +- preflight requires an exact, case-sensitive membership in the nearest + `flows.json` `models` array before any CLI probe, then asks the declared CLI + whether the allowlisted model is available to the current credential. + +There is no regex, prefix, remote-catalog guess, warning fallback, or ambient +model lookup. An unknown value produces typed `model_unknown`; an allowlisted +value whose scoped probe fails while unscoped auth succeeds produces typed +`model_unavailable`; inability to prove the probe is typed `probe_failed`. + +Literal adversarial command against the built SDK: + +```sh +node --input-type=module -e 'import { compileSpec, preflight } from "./dist/index.js"; const typo={version:"0.1.0",agents:{reviewer:{cli:"claude",modle:"exact-model"}},steps:[{id:"review",type:"agent",agent:"reviewer",instruction:"review"}]}; try { compileSpec(typo); } catch (error) { console.log("TYPO_REFUSED="+error.errors.join(" | ")); } const compiled=compileSpec({version:"0.1.0",agents:{reviewer:{cli:"claude",model:"exact-modle"}},steps:[{id:"review",type:"agent",agent:"reviewer",instruction:"review"}]}); let calls=[]; const stub={executor:()=>true,command:()=>true,cli:(cli,source,model)=>{calls.push([cli,source,model]);return {exists:true,authenticated:true,modelAvailable:true};}}; const unknown=preflight(compiled,{models:["exact-model"],modelRegistryPath:"/project/flows.json",probes:stub}); console.log("UNKNOWN="+JSON.stringify({ok:unknown.ok,kinds:unknown.diagnostics.map(d=>d.kind),probeCalls:calls})); calls=[]; const exact=preflight({...compiled,steps:compiled.steps.map(s=>({...s,model:"exact-model"}))},{models:["exact-model"],modelRegistryPath:"/project/flows.json",probes:stub}); console.log("EXACT="+JSON.stringify({ok:exact.ok,kinds:exact.diagnostics.map(d=>d.kind),probeCalls:calls}));' +``` + +Captured output: + +```text +TYPO_REFUSED=spec.agents.reviewer: unknown key "modle" — did you mean "model"? | spec.agents.reviewer.model: expected a non-empty string +UNKNOWN={"ok":false,"kinds":["model_unknown"],"probeCalls":[]} +EXACT={"ok":true,"kinds":[],"probeCalls":[["claude","step","exact-model"]]} +``` + +That demonstrates the important ordering: a typo in a declaration is refused +at compilation; a syntactically valid but absent model is refused without any +probe call; only the exact allowlisted value reaches the CLI with that value. + +The real filesystem/CLI test matrix also passed: + +```sh +./node_modules/.bin/vitest run tests/cli.test.ts -t 'named-agent|typo model|inaccessible model|malformed project model registry' --reporter=verbose +``` + +```text + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + + ✓ tests/cli.test.ts > flows check CLI > accepts an exact allowlisted named-agent model and probes that model 337ms + ✓ tests/cli.test.ts > flows check CLI > checks the same named-agent contract from declarative JSON + ✓ tests/cli.test.ts > flows check CLI > refuses a typo model before probing or contacting relayflowd + ✓ tests/cli.test.ts > flows check CLI > distinguishes an allowlisted but inaccessible model from broken auth + ✓ tests/cli.test.ts > flows check CLI > refuses malformed project model registry {"models":["available-model","available-model"]} + ✓ tests/cli.test.ts > flows check CLI > refuses malformed project model registry {"models":[" "]} + ✓ tests/cli.test.ts > flows check CLI > refuses malformed project model registry {"models":"available-model"} + + Test Files 1 passed (1) + Tests 7 passed | 50 skipped (57) + Start at 18:19:10 + Duration 2.09s (transform 334ms, setup 0ms, collect 470ms, tests 610ms, environment 0ms, prepare 195ms) +``` + +## 4. Deterministic precedence and existing journal lowering + +`compileSpec` resolves a selected declaration before normalizing the step. Its +two conditional assignments make CLI and model precedence independent: + +```text +step cli/model > named declaration cli/model > existing flow/project CLI +model has no flow/project default +``` + +The compiler omits `FlowSpec.agents` from its normalized result and omits the +step's `agent` selector. `toKernelSpec` then writes the resolved values into the +existing `KernelAgentStep.cli` and `.model`. No `agents` collection or selector +crosses the journal protocol boundary. + +Literal built-SDK exercise, with the command itself selecting only the +precedence and boundary fields under review: + +```sh +node --input-type=module -e 'import { compileYaml, toKernelSpec } from "./dist/index.js"; const flow=compileYaml(`version: "0.1.0"\ncli: flow-cli\nagents:\n reviewer: { cli: named-cli, model: named-model }\nsteps:\n - { id: named, type: agent, agent: reviewer, instruction: named }\n - { id: cli-override, type: agent, agent: reviewer, cli: step-cli, instruction: cli }\n - { id: model-override, type: agent, agent: reviewer, model: step-model, instruction: model }\n - { id: inline, type: agent, instruction: inline }\n`); const pick=s=>({id:s.id,cli:s.cli??null,model:s.model??null,agent:s.agent??null}); console.log(JSON.stringify({authoringHasAgents:Object.hasOwn(flow,"agents"),authoring:flow.steps.map(pick),kernel:toKernelSpec(flow).steps.map(pick)}));' && node dist/cli.js check ../testdata/hello-agent.flow.yaml +``` + +Captured output: + +```text +{"authoringHasAgents":false,"authoring":[{"id":"named","cli":"named-cli","model":"named-model","agent":null},{"id":"cli-override","cli":"step-cli","model":"named-model","agent":null},{"id":"model-override","cli":"named-cli","model":"step-model","agent":null},{"id":"inline","cli":null,"model":null,"agent":null}],"kernel":[{"id":"named","cli":"named-cli","model":"named-model","agent":null},{"id":"cli-override","cli":"step-cli","model":"named-model","agent":null},{"id":"model-override","cli":"named-cli","model":"step-model","agent":null},{"id":"inline","cli":null,"model":null,"agent":null}]} +WARNING [unprovable_effects] Step "greet" command "printf" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "finish" command "printf" resolves, but its effects cannot be proven before execution. +RESOLVED step "edit" cli "./preflight/authenticated-cli" model "test-model-v1" from project (/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/testdata/flows.json) +CHECK PASSED ../testdata/hello-agent.flow.yaml +``` + +The checked canonical `hello-agent` remains the pre-existing inline-model +form, so this also exercises compatibility without selecting a named +declaration. + +## 5. v1/default and inline compatibility + +The PR does not touch any v1/default runtime path and does not change the +authoring schema version (`0.1.0`). It is therefore appropriately described as +not deprecating v1. That is an absence-of-change result, not a claim that this +review executed an external v1 repository. + +Within this repository's v2 SDK, compatibility has positive evidence: + +- inline agent `cli`/`model` still compiles and passes `flows check`; +- an inline agent with no model still reaches the worker with + `RELAYFLOW_MODEL` absent even if the parent environment is polluted; +- anonymous inline agents retain no step CLI/model at compile time, allowing + the established flow/project CLI fallback; +- deterministic and LLM/agent ladder tests remain green; +- kernel vocabulary remains the same three step types and the kernel tree is + unchanged by this PR. + +The full serial SDK/live-kernel suite passed at the exact head. The live Claude +analyzer was available and actually ran; the opt-in skip environment variable +was set by the command but not exercised. + +```sh +RELAYFLOWD_BIN=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 ./node_modules/.bin/vitest run --reporter=dot --maxWorkers=1 --minWorkers=1 +``` + +```text + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd +LIVE_KERNEL flows=/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk/dist/cli.js + +stdout | tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +LIVE_ANALYZER ready: claude -p --model claude-haiku-4-5-20251001 round-trip OK + +stdout | tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +LIVE_ANALYZER analysis: {"reasoning":"This story directly demonstrates an AI agent performing autonomous software development tasks including opening and reviewing pull requests, which is a core example of practical AI agents and automation in development workflows.","relevance_score":10,"story_title":"Show HN: an agent that opens and reviews its own pull requests [wake-nonce-7f3a91c4]"} + +stdout | tests/live-kernel.test.ts > surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once +LIVE_KERNEL kill -9 pid=47892 run=01M1HEH3Y4A25XB2EETNQXCZK2 while step=two state=Running + + ✓ tests/live-kernel.test.ts (17 tests) 56833ms + ✓ built flows CLI against live relayflowd > runs rung (a), parks rung (b), and keeps JSON report-shaped 1898ms + ✓ built flows CLI against live relayflowd > allows a deterministic run to exceed the bounded request timeout 32742ms + ✓ built flows CLI against live relayflowd > follows a live worker dispatch through flows run 827ms + ✓ built flows CLI against live relayflowd > can always get a parked run to a late-attaching worker 5568ms + ✓ built flows CLI against live relayflowd > reports a real manual-recovery NeedsHuman state as parked 440ms + ✓ built flows CLI against live relayflowd > AgentWorker exposes wake_context to the CLI via RELAYFLOW_WAKE_CONTEXT env var (real analyzer prerequisite) 338ms + ✓ built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI 11558ms + ✓ built flows CLI against live relayflowd > preflights before journaling and names an unreachable socket 560ms + ✓ surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once 1220ms + ✓ tests/cli.test.ts (57 tests) 2023ms + ✓ tests/journal-client.test.ts (13 tests) 195ms + ✓ tests/validate.test.ts (36 tests) 187ms + ✓ tests/cli-hn-monitor.test.ts (16 tests) 88ms + ✓ tests/preflight.test.ts (15 tests) 13ms + ✓ tests/backlog-picker.test.ts (14 tests) 295ms +SKIPPED_UNACTIONABLE=0 +SKIPPED_UNACTIONABLE=0 +NO_ACTIONABLE_BACKLOG_ENTRY scanned=0 +node:fs:539 + return binding.readFileUtf8(path, stringToFlags(options.flag)); + ^ + +Error: ENOENT: no such file or directory, open '.relayflow/backlog-picker-entry.json' + at Object.readFileSync (node:fs:539:20) + at [eval]:1:478 + at runScriptInThisContext (node:internal/vm:219:10) + at node:internal/process/execution:483:12 + at [eval]-wrapper:6:24 + at runScriptInContext (node:internal/process/execution:481:60) + at evalFunction (node:internal/process/execution:315:30) + at evalTypeScript (node:internal/process/execution:327:3) + at node:internal/main/eval_string:71:3 { + errno: -2, + code: 'ENOENT', + syscall: 'open', + path: '.relayflow/backlog-picker-entry.json' +} + +Node.js v26.7.0 +SKIPPED_UNACTIONABLE=0 + ✓ tests/backlog-picker-flow.test.ts (6 tests) 1519ms +SKIPPED_UNACTIONABLE=0 +NO_ACTIONABLE_BACKLOG_ENTRY scanned=1 Scoped but unverifiable[missing_definition_of_done] + ✓ tests/work-package-consumer.test.ts (13 tests) 788ms + ✓ tests/model-selection.test.ts (10 tests) 51ms + ✓ tests/deterministic-llm.test.ts (5 tests) 49ms + ✓ tests/bin.test.ts (7 tests) 1743ms + ✓ tests/hn-poller.test.ts (6 tests) 22ms + ✓ tests/dir-watcher-poller.test.ts (6 tests) 16ms + ✓ tests/hello-deterministic.test.ts (5 tests) 41ms + ✓ tests/work-package-validator.test.ts (7 tests) 26ms + ✓ tests/spec-parity.test.ts (15 tests) 287ms + ✓ tests/parse-json-output.test.ts (7 tests) 7ms + + Test Files 18 passed (18) + Tests 255 passed (255) + Start at 18:15:46 + Duration 83.12s (transform 2.37s, setup 0ms, collect 4.25s, tests 64.18s, environment 8ms, prepare 4.54s) +``` + +The printed `ENOENT` is expected stderr from a negative backlog-picker child +case; Vitest completed with exit 0 and all 255 tests passed. + +Focused typecheck and compiler/preflight/CLI suites also passed: + +```sh +./node_modules/.bin/tsc --noEmit && ./node_modules/.bin/vitest run tests/model-selection.test.ts tests/validate.test.ts tests/spec-parity.test.ts tests/preflight.test.ts tests/cli.test.ts --reporter=dot +``` + +```text + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + + ✓ tests/preflight.test.ts (15 tests) 23ms + ✓ tests/validate.test.ts (36 tests) 51ms + ✓ tests/model-selection.test.ts (10 tests) 49ms + ✓ tests/spec-parity.test.ts (15 tests) 147ms + ✓ tests/cli.test.ts (57 tests) 1881ms + + Test Files 5 passed (5) + Tests 133 passed (133) + Start at 18:15:06 + Duration 6.03s (transform 1.47s, setup 0ms, collect 4.29s, tests 2.15s, environment 3ms, prepare 7.46s) +``` + +## 6. TypeScript `FlowHeader` scope is incomplete but honestly stated + +Literal issue/PR evidence: + +```sh +gh issue view 132 --repo AgentWorkforce/flows --json state,body --jq '.state, (.body | split("## Done when")[0] | split("4. **`agents:` header with `model`.**")[1] | split("5. **Parallel dispatch")[0])' +gh pr view 136 --repo AgentWorkforce/flows --json body --jq '.body | split("## TypeScript surface dependency")[1] | split("## Literal RED")[0]' +gh pr view 134 --repo AgentWorkforce/flows --json state,headRefOid,body --jq '.state, .headRefOid, (.body | split("This remains issue #132 slice 2 foundation work.")[1] | split("v1 remains")[0])' +``` + +Captured output: + +```text +OPEN + Law 6 shows `{ cli, memory, tools, workspace }`; `AgentStepSpec.model` already exists and the research shim surfaces it as `RELAYFLOW_MODEL`. Add `model` to the header so the choice is journaled, never inherited from the host. + +This PR ships the canonical declarative YAML/JSON compiler contract. Matching `FlowHeader.agents` TypeScript types remain a follow-on after the separately reviewed, currently unmerged `@relayflows/surface` package in PR #134 lands (or in that repair lane). This PR does not duplicate that package. + +OPEN +7266c5134c01151c28c7cc412380b2c0ee6b3dfe + The issue's “ship `@relayflows/surface`” item stays open until there is a release/publication path and the intended consumers migrate. Direct `.flow.ts` CLI execution/input, `flow.on(...)`, research/sales migration, typed outputs, model headers, parallel scheduling, and gate-language decisions are not claimed here. +``` + +At merged main, the only `FlowHeader` is the explicitly declaration-only shim +in `regressions/surface.d.ts`, and it has identity/memory/budget/tools/workspace +but no `agents`. The open PR #134 package has the same omission and itself says +model headers are deferred. PR #136 cannot truthfully be called the complete +cross-dialect declaration requested by issue #132, but neither its body nor its +docs do that. The correct ledger after merge would be: YAML/JSON named-agent +model contract complete; TypeScript header/runtime lowering still open. + +## 7. Hygiene and current first-party check + +```sh +gh pr checks 136 --repo AgentWorkforce/flows +git status --short --branch +git diff --check a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..321b27216e561561e1e022a7b4d973e2182ee480 +printf 'diff_check_exit=%s\n' "$?" +``` + +```text +CodeRabbit pass 0 Review rate limited +linux-x64-artifact pass 3m30s https://github.com/AgentWorkforce/flows/actions/runs/33652459985/job/100322629144 +## feat/v2-declared-model...origin/feat/v2-declared-model +diff_check_exit=0 +``` + +Per RFC-0001, the rate-limited external reviewer is not review signal. The +first-party artifact check is green. The worktree remained clean before this +report was added, and the diff has no whitespace errors. + +REVIEW_PASSED diff --git a/ops/reviews/20260902-1710-pr136-maintainability.md b/ops/reviews/20260902-1710-pr136-maintainability.md new file mode 100644 index 00000000..1a069c42 --- /dev/null +++ b/ops/reviews/20260902-1710-pr136-maintainability.md @@ -0,0 +1,357 @@ +# PR #136 — maintainability / API / type-honesty review + +- **PR:** #136 — `feat(sdk): declare agent CLI and model with fail-closed checks` +- **Exact head reviewed:** `321b27216e561561e1e022a7b4d973e2182ee480` +- **Base reviewed:** merged `main` at `a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2` +- **Lens:** maintainability, API/type honesty, fail-closed taxonomy, load-bearing tests, + exact model-registry/offline semantics, and real CLI auth preflight +- **Constitution read in full:** `AGENTS.md` and + `docs/RFC-0001-everything-is-a-relayflow.md` +- **Issue/PR material read:** issue #132, the complete one-commit diff/history, and the + complete PR body +- **Mode:** assessment only. I did not edit product code, commit, push, merge, or + self-remove. The only non-report files created were disposable fixtures under + `/tmp/pr136-real-cli.u3fo9m/`. + +## Verdict + +**FAIL.** The authoring/compiler and journal lowering are sound, but the advertised +`{ cli, model }` contract is not true for the real CLIs named by the surface. A bare +`claude` declaration passes `flows check` for an allowlisted impossible model, while a +logged-in `codex` is refused as unauthenticated because `codex auth status` is not a +real Codex command. At execution the generic worker passes the model only through the +project-specific `RELAYFLOW_MODEL` environment convention and never supplies the real +CLI model flag. This can silently execute a different host-default model after a green +preflight, which is the exact ambient-model failure this PR says it closes. + +## Scope confirmation + +```text +$ git rev-parse HEAD +321b27216e561561e1e022a7b4d973e2182ee480 +$ git rev-parse a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2 +a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2 +$ git log --format='%H %s' a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..HEAD +321b27216e561561e1e022a7b4d973e2182ee480 feat(sdk): add declared agent model contract +$ git diff --check a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..HEAD; echo "EXIT=$?" +EXIT=0 +$ gh pr view 136 --repo AgentWorkforce/flows --json headRefOid,baseRefOid,state,title +{"baseRefOid":"a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2","headRefOid":"321b27216e561561e1e022a7b4d973e2182ee480","state":"OPEN","title":"feat(sdk): declare agent CLI and model with fail-closed checks"} +``` + +## Findings + +### F1 — HIGH — the declared model does not control the advertised bare CLI + +The surface's canonical named-agent example is literally +`{ cli: claude, model: claude-sonnet-4-6 }` (`docs/SURFACE.md:69-74`). The type says the +declared model is the model the CLI “should use” (`sdk/src/spec.ts:143-150`). However, +the generic worker does only this: + +- remove ambient `RELAYFLOW_MODEL`; +- set `RELAYFLOW_MODEL` to the journaled model; +- spawn `cli` with only `[instruction]` (`sdk/src/worker.ts:200-206,236`). + +The installed real CLIs expose model selection as command-line options, not as this +Relayflows-private environment contract: + +```text +$ claude --help | rg -n -- '--model|RELAYFLOW_MODEL' +104: --model Model for the current session. Provide +$ codex --help | rg -n -- '--model|RELAYFLOW_MODEL|login' +12: login Manage login +76: -m, --model +``` + +No `RELAYFLOW_MODEL` support is advertised. The repo's one real Claude runtime works +only because `testdata/preflight/analyze-story-claude-cli` is a bespoke adapter that +reads `RELAYFLOW_MODEL` and translates it to `claude -p --model MODEL`. The generic +worker does not do that, and the PR's live lowering test uses +`testdata/preflight/echo-model-cli`, another bespoke fixture +(`sdk/tests/live-kernel.test.ts:691-741`). + +Consequences: + +1. `cli: claude` can be preflight-green but execute the host-selected model rather than + the journaled model. +2. `cli: codex` is not invoked in Codex's non-interactive `exec` shape and receives no + `-m/--model` argument. +3. The journal remains internally honest about what was *declared*, but it is not proof + of what provider/model actually ran. + +This violates RFC covenant 2 and the PR outcome “the choice is journaled, never +inherited from the host.” Fix this at a typed CLI-adapter boundary: each supported CLI +needs an explicit auth probe and invocation mapping (including model argument), or the +surface must require and identify a conforming wrapper rather than advertising raw +`claude`/`codex` executables. A private env convention alone is not a portable CLI/model +contract. + +### F2 — HIGH — the real auth/model probe produces both a false pass and a false refusal + +`probeCli` treats exit 0 from ` auth status` with `RELAYFLOW_MODEL` set as proof +that the exact model is usable (`sdk/src/cli/check.ts:194-217`). `runAuthProbe` invokes +that same command for every CLI (`sdk/src/cli/check.ts:220-235`). This assumption is +not true for the actual CLIs named by the API. + +Direct executable evidence: + +```text +$ env -u RELAYFLOW_MODEL claude auth status >/dev/null 2>&1; echo "CLAUDE_UNSCOPED_EXIT=$?" +CLAUDE_UNSCOPED_EXIT=0 +$ RELAYFLOW_MODEL=definitely-not-a-real-model claude auth status >/dev/null 2>&1; echo "CLAUDE_IMPOSSIBLE_MODEL_EXIT=$?" +CLAUDE_IMPOSSIBLE_MODEL_EXIT=0 +$ env -u RELAYFLOW_MODEL codex auth status >/dev/null 2>&1; echo "CODEX_UNSCOPED_EXIT=$?" +CODEX_UNSCOPED_EXIT=2 +``` + +The end-to-end `flows check` counterexample used the following exact allowlist and +flows: + +```text +$ sed -n '1,80p' /tmp/pr136-real-cli.u3fo9m/flows.json /tmp/pr136-real-cli.u3fo9m/claude.flow.yaml /tmp/pr136-real-cli.u3fo9m/codex.flow.yaml +{"models":["definitely-not-a-real-model"]} +version: '0.1.0' +agents: + reviewer: + cli: claude + model: definitely-not-a-real-model +steps: + - id: review + type: agent + agent: reviewer + instruction: Review. +version: '0.1.0' +agents: + reviewer: + cli: codex + model: definitely-not-a-real-model +steps: + - id: review + type: agent + agent: reviewer + instruction: Review. +$ node sdk/dist/cli.js check /tmp/pr136-real-cli.u3fo9m/claude.flow.yaml; echo "CLAUDE_CHECK_EXIT=$?" +RESOLVED step "review" cli "claude" model "definitely-not-a-real-model" from step +CHECK PASSED /tmp/pr136-real-cli.u3fo9m/claude.flow.yaml +CLAUDE_CHECK_EXIT=0 +$ node sdk/dist/cli.js check /tmp/pr136-real-cli.u3fo9m/codex.flow.yaml; echo "CODEX_CHECK_EXIT=$?" +REFUSED [cli_unauthenticated] Step "review" declares CLI "codex", but "codex auth status" exited non-zero; authenticate it or implement that probe to return exit 0 when authenticated. +RESOLVED step "review" cli "codex" model "definitely-not-a-real-model" from step +CODEX_CHECK_EXIT=2 +``` + +The Claude result is a fail-open false positive for exact model access. The Codex +result uses the wrong taxonomy: an unsupported probe verb is reported as +`cli_unauthenticated`, telling an already-authenticated operator to authenticate. A +nonzero status cannot distinguish “credential rejected” from “this CLI has no such +probe,” so the taxonomy is typed but not truthful. + +The new integration tests cannot catch either defect because +`namedAgentProject()` creates a shell fixture specifically programmed to accept +`auth status` and interpret `RELAYFLOW_MODEL` (`sdk/tests/cli.test.ts:68-96`). Those +tests are valuable protocol tests, but they are not real-CLI compatibility tests. +The custom analyzer wrapper is real and model-scoped, but it does not prove the +documented bare-CLI example. + +## Contracts that PASS at this head + +### Compiler/type boundary and deterministic precedence + +- Runtime validation requires exact named declaration keys `{ cli, model }`, requires + both fields, rejects malformed/empty/untrimmed/control-character models, and rejects + unknown agent selectors. Unknown-key diagnostics use author vocabulary and suggestions. +- `compileSpec` resolves named declarations before normalization, then drops the + `agents` map and `agent` selector. Explicit step `cli` and `model` win independently + over the named declaration; anonymous steps remain unresolved for the existing + flow/project CLI precedence (`sdk/src/compile.ts:66-83,134-157`). +- The nearest `flows.json` owns one closed, exact, case-sensitive `models` array. + Missing allowlist membership refuses as `model_unknown` before a CLI or daemon is + contacted. Malformed entries and duplicates refuse as `config_invalid` + (`sdk/src/cli/check.ts:129-167`; `sdk/src/preflight.ts:94-141,155-169`). This is a + deterministic offline approval list. It is not a provider catalog, which is acceptable + only if the live adapter probe is repaired per F2. + +Focused command and complete captured output: + +```text +$ ./node_modules/.bin/tsc --noEmit +$ ./node_modules/.bin/vitest run tests/model-selection.test.ts tests/validate.test.ts tests/spec-parity.test.ts tests/preflight.test.ts tests/cli.test.ts --reporter=dot + + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + + ✓ tests/preflight.test.ts (15 tests) 35ms + ✓ tests/validate.test.ts (36 tests) 98ms + ✓ tests/model-selection.test.ts (10 tests) 104ms + ✓ tests/spec-parity.test.ts (15 tests) 234ms + ✓ tests/cli.test.ts (57 tests) 1956ms + + Test Files 5 passed (5) + Tests 133 passed (133) + Start at 18:14:15 + Duration 4.51s (transform 1.38s, setup 0ms, collect 3.29s, tests 2.43s, environment 5ms, prepare 2.74s) +``` + +`tsc --noEmit` emitted no output and the chained command continued into Vitest, so its +exit was zero. + +### Existing journal fields, not new kernel vocabulary + +The named sugar lowers to the existing agent step `cli` and `model` fields. The focused +live test asserts the compiled shape, `run.spawned.payload.spec.steps[0].model`, and the +worker-observed model after a real relayflowd boundary. Captured execution: + +```text +$ RELAYFLOWD_BIN=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 ./node_modules/.bin/vitest run tests/live-kernel.test.ts -t 'AgentWorker passes a declared model to the CLI as RELAYFLOW_MODEL' --reporter=verbose --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd +LIVE_KERNEL flows=/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk/dist/cli.js + + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker passes a declared model to the CLI as RELAYFLOW_MODEL + + Test Files 1 passed (1) + Tests 1 passed | 16 skipped (17) + Start at 18:18:02 + Duration 1.42s (transform 354ms, setup 0ms, collect 512ms, tests 304ms, environment 0ms, prepare 163ms) +``` + +This proves lowering/journaling/env transport. It does not prove a real bare CLI honors +the env value; that distinction is the substance of F1. + +### Existing inline/default behavior + +The three canonical ladder flows still pass `flows check`; the agent and llm retain +their pre-existing inline `model` plus project-CLI resolution and deterministic flows +remain unaffected: + +```text +$ node sdk/dist/cli.js check testdata/hello-agent.flow.yaml; echo "EXIT=$?" +WARNING [unprovable_effects] Step "greet" command "printf" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "finish" command "printf" resolves, but its effects cannot be proven before execution. +RESOLVED step "edit" cli "./preflight/authenticated-cli" model "test-model-v1" from project (/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/testdata/flows.json) +CHECK PASSED testdata/hello-agent.flow.yaml +EXIT=0 +$ node sdk/dist/cli.js check testdata/hello-llm.flow.yaml; echo "EXIT=$?" +WARNING [unprovable_effects] Step "greet" command "printf" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "finish" command "printf" resolves, but its effects cannot be proven before execution. +RESOLVED step "answer" cli "./preflight/authenticated-cli" model "deterministic-test-stub" from project (/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/testdata/flows.json) +CHECK PASSED testdata/hello-llm.flow.yaml +EXIT=0 +$ node sdk/dist/cli.js check testdata/hello-deterministic.flow.yaml; echo "EXIT=$?" +WARNING [unprovable_effects] Step "greet" command "echo" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "shout" command "echo" resolves, but its effects cannot be proven before execution. +CHECK PASSED testdata/hello-deterministic.flow.yaml +EXIT=0 +``` + +### TypeScript `FlowHeader` scope is honest + +The issue asks for `agents:` with model on the TypeScript header, but this branch has no +canonical `FlowHeader` implementation to amend. The PR does **not** claim otherwise: +its body calls this the YAML/JSON compiler contract and explicitly defers +`FlowHeader.agents` until the separately reviewed, unmerged surface package lands. +`docs/SURFACE.md:117-121` repeats that limitation. Exporting `NamedAgentSpec` and adding +`FlowSpec.agents` is truthful for the declarative SDK shape; it is not represented as +completion of issue #132's TypeScript header item. + +## Whole-suite evidence + +The full serial SDK suite passed at the exact head, including the custom real-Claude +analyzer, live journal/kernel tests, and crash/resume. Its success does not close F1/F2 +because none of its real-CLI cases exercises the documented bare `cli: claude` contract; +the real analyzer uses the translating wrapper described above. + +```text +$ RELAYFLOWD_BIN=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 ./node_modules/.bin/vitest run --reporter=dot --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd +LIVE_KERNEL flows=/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk/dist/cli.js + +stdout | tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +LIVE_ANALYZER ready: claude -p --model claude-haiku-4-5-20251001 round-trip OK + +stdout | tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +LIVE_ANALYZER analysis: {"reasoning":"This story is highly relevant to AI agents and automation as it demonstrates a concrete implementation of an autonomous agent performing core software development tasks (opening and reviewing pull requests). The agent demonstrates self-directed capability and workflow automation, which are central themes in agent development and align directly with autonomous systems design.","relevance_score":9,"story_title":"Show HN: an agent that opens and reviews its own pull requests [wake-nonce-7f3a91c4]"} + +stdout | tests/live-kernel.test.ts > surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once +LIVE_KERNEL kill -9 pid=48527 run=01M1HEHN0P6HWSQZ865SAFQ5DY while step=two state=Running + + ✓ tests/live-kernel.test.ts (17 tests) 63060ms + ✓ tests/cli.test.ts (57 tests) 2292ms + ✓ tests/journal-client.test.ts (13 tests) 152ms + ✓ tests/validate.test.ts (36 tests) 49ms + ✓ tests/cli-hn-monitor.test.ts (16 tests) 109ms + ✓ tests/preflight.test.ts (15 tests) 22ms + ✓ tests/backlog-picker.test.ts (14 tests) 241ms +SKIPPED_UNACTIONABLE=0 +SKIPPED_UNACTIONABLE=0 +NO_ACTIONABLE_BACKLOG_ENTRY scanned=0 +node:fs:539 + return binding.readFileUtf8(path, stringToFlags(options.flag)); + ^ + +Error: ENOENT: no such file or directory, open '.relayflow/backlog-picker-entry.json' + at Object.readFileSync (node:fs:539:20) + at [eval]:1:478 + at runScriptInThisContext (node:internal/vm:219:10) + at node:internal/process/execution:483:12 + at [eval]-wrapper:6:24 + at runScriptInContext (node:internal/process/execution:481:60) + at evalFunction (node:internal/process/execution:315:30) + at evalTypeScript (node:internal/process/execution:327:3) + at node:internal/main/eval_string:71:3 { + errno: -2, + code: 'ENOENT', + syscall: 'open', + path: '.relayflow/backlog-picker-entry.json' +} + +Node.js v26.7.0 +SKIPPED_UNACTIONABLE=0 + ✓ tests/backlog-picker-flow.test.ts (6 tests) 1494ms +SKIPPED_UNACTIONABLE=0 +NO_ACTIONABLE_BACKLOG_ENTRY scanned=1 Scoped but unverifiable[missing_definition_of_done] + ✓ tests/work-package-consumer.test.ts (13 tests) 640ms + ✓ tests/model-selection.test.ts (10 tests) 44ms + ✓ tests/deterministic-llm.test.ts (5 tests) 31ms + ✓ tests/bin.test.ts (7 tests) 2176ms + ✓ tests/hn-poller.test.ts (6 tests) 55ms + ✓ tests/dir-watcher-poller.test.ts (6 tests) 77ms + ✓ tests/hello-deterministic.test.ts (5 tests) 71ms + ✓ tests/work-package-validator.test.ts (7 tests) 16ms + ✓ tests/spec-parity.test.ts (15 tests) 173ms + ✓ tests/parse-json-output.test.ts (7 tests) 5ms + + Test Files 18 passed (18) + Tests 255 passed (255) + Start at 18:15:57 + Duration 89.79s (transform 2.86s, setup 0ms, collect 5.25s, tests 70.71s, environment 10ms, prepare 4.00s) +``` + +The ENOENT stack is emitted by an expected negative-path child exercised by +`backlog-picker-flow.test.ts`; the Vitest process exited zero and reported all 255 tests +passing. + +## Required repair and regression gates + +1. Establish a typed adapter/runner contract per supported CLI. For the docs' raw + `claude` example, invoke non-interactively with the declared `--model`; for Codex, + use its actual login/status and `exec --model` shapes. If raw provider CLIs are not + supported, refuse them with an honest kind and document that `cli` must be a + conforming Relayflows adapter. +2. Make scoped readiness actually exercise or query the exact model. Unsupported probe + verbs must be `probe_failed`/unsupported-contract (or a new closed kind), not + `cli_unauthenticated`. +3. Add executable tests against the supported real CLI adapters. Keep the current fake + probe tests for deterministic taxonomy, but do not present them as provider auth/model + evidence. +4. Retain the current compiler/lowering, allowlist, precedence, inline/unset, and + `run.spawned` assertions; those parts are good and should not be rewritten to fix the + adapter boundary. + +REVIEW_FAILED diff --git a/ops/reviews/20260902-1710-pr136-structure.md b/ops/reviews/20260902-1710-pr136-structure.md new file mode 100644 index 00000000..aecf69ad --- /dev/null +++ b/ops/reviews/20260902-1710-pr136-structure.md @@ -0,0 +1,261 @@ +# PR #136 independent structure / RFC review + +- Reviewed head: `321b27216e561561e1e022a7b4d973e2182ee480` +- Compared with merged `main`: `a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2` +- Lens: RFC boundary structure and model-registry trust boundary +- Verdict: **FAIL** — one blocking P1 finding + +## Finding + +### P1 — a registry-invalid named-agent model can be silently erased before preflight + +`validateAgents` validates only model *syntax* (`sdk/src/validate.ts:145-169`). +`compileSpec` then resolves models used by steps and constructs a new flow without +the `agents` map (`sdk/src/compile.ts:66-83`). `preflight` can consequently check +only effective `step.model` values (`sdk/src/preflight.ts:99-141`). + +This means a model absent from the nearest `flows.json` passes whenever its named +declaration is unused, and it also passes when a selecting step overrides that +model with an allowlisted value. The declaration containing the unknown model is +dropped. That contradicts the PR's unqualified claims that it "fails closed on +unknown model names" and that "a model absent from that registry produces +`model_unknown` before any subprocess or submission." It also undercuts issue +#132's closed-schema/typo-lint goal: a typo in a declared reusable agent is +accepted rather than named at authoring time. + +This does not cause the currently effective step to execute the unknown model; +the defect is at the declared-config trust boundary. It is nevertheless +blocking because exact, fail-closed model lint is the central contract of this +slice, and `flows check` reports these invalid declarations as acceptable. + +Literal reproduction (both cases incorrectly report `ok: true`): + +```text +$ cd sdk && node --input-type=module -e 'import {compileSpec,preflight} from "./dist/index.js"; const probes={cli:()=>({exists:true,authenticated:true,modelAvailable:true}),executor:()=>true,command:()=>true}; for (const [label,input] of [["unused",{version:"0.1.0",agents:{reviewer:{cli:"claude",model:"typo-model"}},steps:[{id:"ready",type:"deterministic",command:"printf ready"}]}],["overridden",{version:"0.1.0",agents:{reviewer:{cli:"claude",model:"typo-model"}},steps:[{id:"review",type:"agent",agent:"reviewer",model:"known-model",instruction:"Review"}]}]]) { const compiled=compileSpec(input); const result=preflight(compiled,{models:["known-model"],probes}); console.log(JSON.stringify({label,compiled,result})); }' +{"label":"unused","compiled":{"version":"0.1.0","steps":[{"id":"ready","type":"deterministic","maxIterations":1,"command":"printf ready","verification":{"type":"exit_code"}}]},"result":{"ok":true,"resolutions":[],"diagnostics":[{"severity":"warning","kind":"unprovable_effects","stepId":"ready","message":"Step \"ready\" command \"printf\" resolves, but its effects cannot be proven before execution."}]}} +{"label":"overridden","compiled":{"version":"0.1.0","steps":[{"id":"review","type":"agent","maxIterations":1,"instruction":"Review","cli":"claude","model":"known-model","recoveryMode":"reset"}]},"result":{"ok":true,"resolutions":[{"stepId":"review","cli":"claude","source":"step","model":"known-model"}],"diagnostics":[]}} +``` + +Required repair: validate every value in the author-declared `agents` model map +against the selected project registry before the map can be erased, including +unused declarations and declarations whose value is shadowed by a step override. +Keep this validation at the TypeScript authoring/CLI edge; do not move the model +registry into the Rust kernel. + +## What otherwise holds + +### Exact effective-model check is fail-closed + +For an effective step model, allowlist membership uses exact case-sensitive +`includes`; an unknown value stops before the CLI probe. Literal adversarial +check: + +```text +$ cd sdk && node --input-type=module -e 'import {compileSpec,preflight} from "./dist/index.js"; let calls=0; const flow=compileSpec({version:"0.1.0",steps:[{id:"review",type:"agent",instruction:"Review",cli:"claude",model:"Model-A"}]}); const result=preflight(flow,{models:["model-a"],modelRegistryPath:"/project/flows.json",probes:{cli:()=>{calls+=1;return {exists:true,authenticated:true,modelAvailable:true}},executor:()=>true,command:()=>true}}); console.log(JSON.stringify({calls,result},null,2));' +{ + "calls": 0, + "result": { + "ok": false, + "resolutions": [ + { + "stepId": "review", + "cli": "claude", + "source": "step", + "model": "Model-A" + } + ], + "diagnostics": [ + { + "severity": "refusal", + "kind": "model_unknown", + "stepId": "review", + "cli": "claude", + "model": "Model-A", + "message": "Step \"review\" declares model \"Model-A\" for CLI \"claude\", but it is not listed in project model registry \"/project/flows.json\"; add the exact model only after verifying that project is allowed to use it." + } + ] + } +} +``` + +Malformed effective models (empty, surrounding whitespace, C0/DEL controls), +unknown fields, unknown selected agent names, malformed/duplicate registries, +model-scoped probe failures, and missing `modelAvailable: true` all refuse. The +new `model_unknown` and `model_unavailable` kinds remain in the SDK's closed +preflight taxonomy. + +### Lowering, journal boundary, and precedence are structurally correct + +- `resolveNamedAgent` applies step `cli`/`model` independently over the named + declaration. The resulting step then uses the existing flow/project CLI + fallback; no model fallback was added. +- `agents` and `agent` do not appear in `KernelRunSpec`; only the existing + `KernelAgentStep.cli` and `.model` fields cross the boundary. +- There is no kernel diff. The Rust kernel remains model-registry and provider + unaware, as RFC-0001 requires. +- The live test reads `run.spawned.payload.spec.steps[0].model` and verifies the + same model reaches `RELAYFLOW_MODEL`; the unset case remains pinned against + ambient parent leakage. +- Anonymous/inline steps and flow/project CLI resolution remain exercised by + the existing suites. Inline model declarations now correctly require the same + exact project allowlist as effective named selections. + +Literal boundary checks: + +```text +$ git diff --quiet a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2...321b27216e561561e1e022a7b4d973e2182ee480 -- kernel; printf 'kernel_diff_exit=%s\n' "$?" +kernel_diff_exit=0 +$ git diff --quiet a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2...321b27216e561561e1e022a7b4d973e2182ee480 -- regressions/surface.d.ts; printf 'flow_header_diff_exit=%s\n' "$?" +flow_header_diff_exit=0 +``` + +Literal live lowering/worker verification: + +```text +$ cd sdk && RELAYFLOWD_BIN=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 ./node_modules/.bin/vitest run tests/live-kernel.test.ts -t 'AgentWorker passes a declared model to the CLI as RELAYFLOW_MODEL|AgentWorker leaves RELAYFLOW_MODEL UNSET when the step declares no model' --reporter=verbose + + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd +LIVE_KERNEL flows=/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk/dist/cli.js + + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker passes a declared model to the CLI as RELAYFLOW_MODEL + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker leaves RELAYFLOW_MODEL UNSET when the step declares no model + + Test Files 1 passed (1) + Tests 2 passed | 15 skipped (17) + Start at 18:13:48 + Duration 2.09s (transform 271ms, setup 0ms, collect 398ms, tests 513ms, environment 0ms, prepare 222ms) +``` + +### The absent TypeScript header is disclosed honestly + +The repository's only current `FlowHeader` is the declaration-only shim at +`regressions/surface.d.ts:22-28`, and it has no `agents` field. The PR does not +change that file. Both the PR body and `docs/SURFACE.md:117-121` expressly limit +this slice to canonical declarative YAML/JSON and identify +`FlowHeader.agents` as follow-on work after the separately reviewed, unmerged +surface package. That is an honest partial delivery against issue #132, not a +claim that the TypeScript surface has shipped here. + +Literal source/claim evidence: + +```text +$ rg -n "interface FlowHeader|type FlowHeader|FlowHeader" . --glob '!sdk/node_modules/**' --glob '!target/**' +./regressions/surface.d.ts:22: export interface FlowHeader { +./regressions/surface.d.ts:127: header: FlowHeader, +./ops/reviews/20260827-2305-pr8-maintainability.md:51:This is load-bearing — it says the SDK-side CLI holds covenant 2 but the kernel does not. Nothing on the code side signals the split. `JournalClient.runStart` (re-exported from `sdk/src/index.ts`) has no comment naming preflight; `RunSpec::parse` in `kernel/relayflowd-core/src/spec.rs:53-62` doesn't say its callers skip surface preflight either. A stranger writing a new caller of `JournalClient`, or a new server entry point on the kernel side, learns nothing about the asymmetry from the code. +./docs/SURFACE.md:119: `FlowHeader.agents` TypeScript types depend on the separately reviewed, +``` + +```text +$ gh pr view 136 --repo AgentWorkforce/flows --json body --jq '.body | split("\n") | map(select(test("TypeScript surface|FlowHeader"))) | .[]' +## TypeScript surface dependency +This PR ships the canonical declarative YAML/JSON compiler contract. Matching `FlowHeader.agents` TypeScript types remain a follow-on after the separately reviewed, currently unmerged `@relayflows/surface` package in PR #134 lands (or in that repair lane). This PR does not duplicate that package. +``` + +## Scope and history evidence + +```text +$ git rev-parse HEAD +321b27216e561561e1e022a7b4d973e2182ee480 +$ git rev-parse main +a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2 +$ git log --oneline --decorate --graph a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..321b27216e561561e1e022a7b4d973e2182ee480 +* 321b272 (HEAD -> feat/v2-declared-model, origin/pr-136, origin/feat/v2-declared-model) feat(sdk): add declared agent model contract +``` + +```text +$ gh issue view 132 --repo AgentWorkforce/flows --json body --jq '.body | split("\n") | map(select(test("agents:` header|AgentStepSpec.model|journaled|mechanism against optional-field sprawl|unknown"))) | .[]' +4. **`agents:` header with `model`.** Law 6 shows `{ cli, memory, tools, workspace }`; `AgentStepSpec.model` already exists and the research shim surfaces it as `RELAYFLOW_MODEL`. Add `model` to the header so the choice is journaled, never inherited from the host. +7. **A mechanism against optional-field sprawl.** Decision 13 (closed kernel vocabulary) is a rule. v1 grew to ~25 optional fields on one step type under the same intent. Consider a lint in `flows check` that refuses a step spec with fields outside its verb's schema, so the SDK cannot regrow v1's shape. +``` + +## Verification evidence + +TypeScript check: + +```text +$ cd sdk && ./node_modules/.bin/tsc --noEmit; task_tsc_status=$?; printf 'tsc_exit=%s\n' "$task_tsc_status"; exit "$task_tsc_status" +tsc_exit=0 +``` + +Full direct-binary serial suite: + +```text +$ cd sdk && RELAYFLOWD_BIN=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 ./node_modules/.bin/vitest run --reporter=dot --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd +LIVE_KERNEL flows=/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk/dist/cli.js + +stdout | tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +LIVE_ANALYZER ready: claude -p --model claude-haiku-4-5-20251001 round-trip OK + +stdout | tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +LIVE_ANALYZER analysis: {"reasoning":"This story directly describes an AI agent autonomously performing software development tasks—opening and reviewing pull requests—which is a core example of practical AI agent automation in a development workflow.","relevance_score":10,"story_title":"Show HN: an agent that opens and reviews its own pull requests [wake-nonce-7f3a91c4]"} + +stdout | tests/live-kernel.test.ts > surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once +LIVE_KERNEL kill -9 pid=50561 run=01M1HEK49V51DCNEC7ARGAWDWK while step=two state=Running + + ✓ tests/live-kernel.test.ts (17 tests) 76662ms + ✓ tests/cli.test.ts (57 tests) 2133ms + ✓ tests/journal-client.test.ts (13 tests) 143ms + ✓ tests/validate.test.ts (36 tests) 40ms + ✓ tests/cli-hn-monitor.test.ts (16 tests) 107ms + ✓ tests/preflight.test.ts (15 tests) 10ms + ✓ tests/backlog-picker.test.ts (14 tests) 234ms +SKIPPED_UNACTIONABLE=0 +SKIPPED_UNACTIONABLE=0 +NO_ACTIONABLE_BACKLOG_ENTRY scanned=0 +node:fs:539 + return binding.readFileUtf8(path, stringToFlags(options.flag)); + ^ + +Error: ENOENT: no such file or directory, open '.relayflow/backlog-picker-entry.json' + at Object.readFileSync (node:fs:539:20) + at [eval]:1:478 + at runScriptInThisContext (node:internal/vm:219:10) + at node:internal/process/execution:483:12 + at [eval]-wrapper:6:24 + at runScriptInContext (node:internal/process/execution:481:60) + at evalFunction (node:internal/process/execution:315:30) + at evalTypeScript (node:internal/process/execution:327:3) + at node:internal/main/eval_string:71:3 { + errno: -2, + code: 'ENOENT', + syscall: 'open', + path: '.relayflow/backlog-picker-entry.json' +} + +Node.js v26.7.0 +SKIPPED_UNACTIONABLE=0 + ✓ tests/backlog-picker-flow.test.ts (6 tests) 1668ms +SKIPPED_UNACTIONABLE=0 +NO_ACTIONABLE_BACKLOG_ENTRY scanned=1 Scoped but unverifiable[missing_definition_of_done] + ✓ tests/work-package-consumer.test.ts (13 tests) 767ms + ✓ tests/model-selection.test.ts (10 tests) 62ms + ✓ tests/deterministic-llm.test.ts (5 tests) 56ms + ✓ tests/bin.test.ts (7 tests) 1841ms + ✓ tests/hn-poller.test.ts (6 tests) 21ms + ✓ tests/dir-watcher-poller.test.ts (6 tests) 14ms + ✓ tests/hello-deterministic.test.ts (5 tests) 48ms + ✓ tests/work-package-validator.test.ts (7 tests) 25ms + ✓ tests/spec-parity.test.ts (15 tests) 90ms + ✓ tests/parse-json-output.test.ts (7 tests) 7ms + + Test Files 18 passed (18) + Tests 255 passed (255) + Start at 18:16:32 + Duration 95.70s (transform 780ms, setup 0ms, collect 2.21s, tests 83.93s, environment 8ms, prepare 2.42s) +``` + +The ENOENT text is intentional stderr exercised inside the passing +`backlog-picker-flow` cases; the Vitest process completed successfully and the +suite summary is green. No product code, gate, commit, push, or merge was +changed by this review. + +REVIEW_FAILED From 4888d1572ed047c5161042614ac72068d047783a Mon Sep 17 00:00:00 2001 From: kjgbot Date: Wed, 2 Sep 2026 18:53:05 +0200 Subject: [PATCH 03/15] fix(sdk): make model adapters fail closed Session-Id: 01a062b7-2aef-7772-8225-7cb00ee311dd --- docs/SURFACE.md | 46 ++++++- sdk/src/cli-adapter.ts | 118 ++++++++++++++++++ sdk/src/cli/check.ts | 76 ++++++++--- sdk/src/compile.ts | 2 + sdk/src/failure-kinds.ts | 1 + sdk/src/preflight.ts | 50 +++++++- sdk/src/spec.ts | 9 +- sdk/src/worker.ts | 13 +- sdk/tests/cli-adapter.test.ts | 62 +++++++++ sdk/tests/cli.test.ts | 116 ++++++++++++++++- sdk/tests/live-kernel.test.ts | 57 ++++++++- sdk/tests/model-selection.test.ts | 10 +- sdk/tests/preflight.test.ts | 38 ++++++ sdk/tests/real-cli-adapters.test.ts | 49 ++++++++ testdata/preflight/analyze-story-claude-cli | 13 +- .../preflight/analyze-story-echo-wake-cli | 4 + .../analyze-story-missing-fields-cli | 4 + testdata/preflight/analyze-story-stub-cli | 4 + .../preflight/analyze-story-text-only-cli | 4 + testdata/preflight/authenticated-cli | 4 + testdata/preflight/counting-cli | 4 + testdata/preflight/echo-model-cli | 4 + testdata/preflight/signal-probe-cli | 4 + testdata/preflight/unauthenticated-cli | 4 + testdata/preflight/wake-context-probe-cli | 4 + 25 files changed, 651 insertions(+), 49 deletions(-) create mode 100644 sdk/src/cli-adapter.ts create mode 100644 sdk/tests/cli-adapter.test.ts create mode 100644 sdk/tests/real-cli-adapters.test.ts diff --git a/docs/SURFACE.md b/docs/SURFACE.md index 87267aff..6bf38443 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -81,21 +81,55 @@ No process runs between events: the handler wakes, executes to its next await, p In the canonical declarative YAML/JSON dialect, `agents:` is a top-level map and an agent step selects one with `agent: reviewer`. Each named declaration requires both `cli` and `model`. Compilation lowers them into - the existing per-step `cli` and `model` fields and removes both the selector - and map before the kernel boundary. Explicit step values win independently: + the existing per-step `cli` and `model` fields. The validated selector and + map remain authoring metadata through `flows check`, so unused and + step-shadowed declarations are linted too; both are removed at the kernel + boundary. Explicit step values win independently: step `cli`/`model` → named declaration → the existing flow/project CLI default. Model has no flow/project default. An inline step that selects no - named declaration keeps the existing optional-model behavior; the worker - explicitly removes ambient `RELAYFLOW_MODEL` when it is absent. + named declaration keeps the existing optional-model behavior. The worker + explicitly removes ambient `RELAYFLOW_MODEL`; raw provider adapters use a + model flag, while an identified wrapper receives the variable only when the + step declares a model. **Anonymous resolution law:** `f.agent\`task\`` with no name is the *default agent*, resolved (never guessed) in order: step options → flow header → project config (`flows.json`) → platform default. *The platform-default rung is declared but not yet implemented: no platform default is provisioned as of gate 1, so a flow that reaches this rung refuses with `cli_unresolved` rather than guessing. `flows check` never invents an implicit default.* `flows check` prints each resolved step CLI and its declaration source, validates it before submission, and refuses a missing or unauthenticated resolution before the checked flow is submitted, never at minute 27. Gate 1 does not make this guarantee for callers that bypass `flows check`: the journal client's direct `run.start` path does not invoke surface preflight. - **Preflightable-CLI contract:** to be checkable, a declared `cli` must answer ` auth status` — exit `0` for authenticated, non-zero for not. When a step declares a model, the same probe runs with that exact value in `RELAYFLOW_MODEL`; exit `0` means the current credential can use that exact model. If the scoped probe fails, an unscoped probe distinguishes `model_unavailable` from `cli_unauthenticated`. `flows check` resolves the binary (a path is taken relative to the file that declares it — the flow for a step/flow-level `cli`, the project config for a `flows.json` default — while a bare name resolves via `PATH`) and runs that probe once per resolved `(cli, source, model)`: a path that does not resolve as an executable is `cli_missing`. A probe process that cannot be started, is terminated by a signal, or exceeds the 10-second auth-probe timeout is `probe_failed`; the diagnostic carries that classified cause without exposing raw process errors. The probe inherits the caller environment except that `RELAYFLOW_MODEL` is always removed and then set only from the compiled step. Preflight never invokes an undeclared model or guesses from host state. + **Typed CLI-adapter contract:** `flows check` and `AgentWorker` share one + closed adapter table. A resolved executable whose basename is `claude` uses + `claude auth status`, probes the exact model with a real noninteractive + `claude -p --model ` round trip, and executes with that same model + flag. A basename of `codex` uses `codex login status`, probes with + `codex exec --model ` in an ephemeral read-only session, and executes + noninteractively with `codex exec --model `. Model-scoped probes may + contact the provider and have a 60-second timeout; this cost is the + honest price of proving current credential/model access rather than + accepting an unrelated auth command as model proof. + + Every other executable is a custom Relayflows wrapper and must first answer + ` --relayflows-adapter-v1` with exactly + `relayflows-agent-cli-v1`. Only an identified wrapper uses the established + ` auth status` plus exact `RELAYFLOW_MODEL` scoped-probe/execution + protocol. A missing or wrong identification is `cli_unsupported`, never + mislabeled as `cli_unauthenticated`. If a model-scoped probe fails, the + adapter's real unscoped authentication command distinguishes + `model_unavailable` from `cli_unauthenticated`. + + `flows check` resolves the binary (a path is relative to the declaring flow + or project config; a bare name resolves via `PATH`) and caches each resolved + `(cli, source, model)` probe. A missing executable is `cli_missing`. A probe + that cannot start, is signaled, or exceeds its adapter timeout is + `probe_failed`, with a classified diagnostic rather than a raw process + error. Every subprocess starts with ambient `RELAYFLOW_MODEL` removed; + provider adapters pass only the declared flag, and wrapper adapters set the + private variable only from the compiled step. Preflight never invokes an + undeclared model or guesses from host state. **Deterministic model registry:** model existence is not inferred from a regex or provider prefix. The nearest `flows.json` owns an exact, case-sensitive `models` allowlist. `flows check` first refuses a declared - model absent from that list as `model_unknown`, without starting the CLI; + model absent from that list as `model_unknown`, without starting the CLI. + This includes every named declaration, even when unused or shadowed by a + step override; only an allowlisted value reaches the live model-scoped probe above. The registry is author-owned project configuration, reviewed and versioned with the project. Updating it is an explicit file change made only after the diff --git a/sdk/src/cli-adapter.ts b/sdk/src/cli-adapter.ts new file mode 100644 index 00000000..cda07368 --- /dev/null +++ b/sdk/src/cli-adapter.ts @@ -0,0 +1,118 @@ +import { basename } from 'node:path'; + +export type CliAdapterKind = 'claude' | 'codex' | 'relayflows-wrapper-v1'; + +export interface CliInvocation { + args: string[]; + timeoutMs: number; + /** Set only for the explicit wrapper protocol; raw providers receive a model flag. */ + modelEnv?: string; +} + +export interface CliAdapterIdentification { + invocation: CliInvocation; + expectedStdout?: string; +} + +export const WRAPPER_IDENTIFY_ARG = '--relayflows-adapter-v1'; +export const WRAPPER_IDENTIFY_TOKEN = 'relayflows-agent-cli-v1'; + +const MODEL_PROBE_PROMPT = 'Reply with exactly RELAYFLOWS_MODEL_READY and nothing else.'; + +/** Select a closed adapter from the resolved executable's basename. */ +export function cliAdapterKind(executable: string): CliAdapterKind { + const name = basename(executable).replace(/\.exe$/i, ''); + if (name === 'claude') return 'claude'; + if (name === 'codex') return 'codex'; + return 'relayflows-wrapper-v1'; +} + +/** Prove the adapter command shape before classifying an auth failure. */ +export function adapterIdentification(kind: CliAdapterKind): CliAdapterIdentification { + if (kind === 'claude') { + return { invocation: { args: ['auth', 'status', '--help'], timeoutMs: 10_000 } }; + } + if (kind === 'codex') { + return { invocation: { args: ['login', 'status', '--help'], timeoutMs: 10_000 } }; + } + return { + invocation: { args: [WRAPPER_IDENTIFY_ARG], timeoutMs: 10_000 }, + expectedStdout: WRAPPER_IDENTIFY_TOKEN, + }; +} + +export function authenticationProbe(kind: CliAdapterKind): CliInvocation { + if (kind === 'codex') return { args: ['login', 'status'], timeoutMs: 10_000 }; + return { args: ['auth', 'status'], timeoutMs: 10_000 }; +} + +/** + * A provider model probe is a real, noninteractive model round trip. The + * wrapper protocol keeps its established auth-status shape and receives the + * exact model through its explicitly identified environment contract. + */ +export function modelReadinessProbe(kind: CliAdapterKind, model: string): CliInvocation { + if (kind === 'claude') { + return { + args: [ + '-p', '--model', model, '--tools', '', '--no-session-persistence', + MODEL_PROBE_PROMPT, + ], + timeoutMs: 60_000, + }; + } + if (kind === 'codex') { + return { + args: [ + 'exec', '--ephemeral', '--sandbox', 'read-only', '--skip-git-repo-check', + '--model', model, MODEL_PROBE_PROMPT, + ], + timeoutMs: 60_000, + }; + } + return { + args: ['auth', 'status'], + timeoutMs: 60_000, + modelEnv: model, + }; +} + +/** Build the actual worker argv; this is shared contract, not probe-only lore. */ +export function agentExecution( + kind: CliAdapterKind, + instruction: string, + model?: string, +): CliInvocation { + if (kind === 'claude') { + return { + args: ['-p', ...(model === undefined ? [] : ['--model', model]), instruction], + timeoutMs: 0, + }; + } + if (kind === 'codex') { + return { + args: [ + 'exec', '--ephemeral', + ...(model === undefined ? [] : ['--model', model]), + instruction, + ], + timeoutMs: 0, + }; + } + return { + args: [instruction], + timeoutMs: 0, + ...(model === undefined ? {} : { modelEnv: model }), + }; +} + +export function displayInvocation(cli: string, invocation: CliInvocation): string { + const command = [cli, ...invocation.args].map(shellDisplayWord).join(' '); + return invocation.modelEnv === undefined + ? command + : `RELAYFLOW_MODEL=${shellDisplayWord(invocation.modelEnv)} ${command}`; +} + +function shellDisplayWord(word: string): string { + return /^[A-Za-z0-9_./:-]+$/.test(word) ? word : JSON.stringify(word); +} diff --git a/sdk/src/cli/check.ts b/sdk/src/cli/check.ts index 384570cf..50959782 100644 --- a/sdk/src/cli/check.ts +++ b/sdk/src/cli/check.ts @@ -3,6 +3,14 @@ import { dirname, isAbsolute, join, parse as parsePath, resolve } from 'node:pat import { spawnSync } from 'node:child_process'; import { parse as parseYaml } from 'yaml'; import { CompileError, compileSpec, kernelToAuthoring } from '../compile.js'; +import { + adapterIdentification, + authenticationProbe, + cliAdapterKind, + displayInvocation, + modelReadinessProbe, + type CliInvocation, +} from '../cli-adapter.js'; import { MODEL_ENV } from '../worker.js'; import { modelNameError } from '../model-name.js'; import type { FlowSpec } from '../spec.js'; @@ -198,41 +206,71 @@ function probeCli( ): CliProbeResult { const executable = resolveExecutable(cli, directory); if (executable === undefined) return { exists: false, authenticated: false }; + const kind = cliAdapterKind(executable); + const identification = adapterIdentification(kind); + const identified = runProbe(executable, directory, identification.invocation); + if ( + identified.status !== 0 + || (identification.expectedStdout !== undefined + && identified.stdout.trim() !== identification.expectedStdout) + ) { + return { exists: true, supported: false, authenticated: false }; + } + const auth = authenticationProbe(kind); + const authCommand = displayInvocation(cli, auth); if (model === undefined) { - return { exists: true, authenticated: runAuthProbe(executable, directory) === 0 }; + return { + exists: true, + supported: true, + authenticated: runProbe(executable, directory, auth).status === 0, + authCommand, + }; } - // A successful scoped probe proves both auth and exact-model access in one - // round trip. On failure, repeat without a model solely to distinguish an - // authentication failure from a typed model_unavailable refusal. - const scopedStatus = runAuthProbe(executable, directory, model); - if (scopedStatus === 0) { - return { exists: true, authenticated: true, modelAvailable: true }; + const scoped = modelReadinessProbe(kind, model); + const modelCommand = displayInvocation(cli, scoped); + // A successful real provider round trip (or identified wrapper probe) + // proves both auth and exact-model access. On failure, run the adapter's + // actual auth command solely to classify auth vs model access truthfully. + if (runProbe(executable, directory, scoped).status === 0) { + return { + exists: true, + supported: true, + authenticated: true, + modelAvailable: true, + authCommand, + modelCommand, + }; } - const authStatus = runAuthProbe(executable, directory); + const authStatus = runProbe(executable, directory, auth).status; return { exists: true, + supported: true, authenticated: authStatus === 0, modelAvailable: false, + authCommand, + modelCommand, }; } -function runAuthProbe(executable: string, directory: string, model?: string): number | null { - // Hand the declared model to the probe exactly as the worker hands it to the - // real invocation, and explicitly unset it otherwise. The CLI contract says - // exit 0 only when that exact model is usable by the current credential. +function runProbe( + executable: string, + directory: string, + invocation: CliInvocation, +): { status: number | null; stdout: string } { const env = { ...process.env }; delete env[MODEL_ENV]; - if (model !== undefined) env[MODEL_ENV] = model; - const result = spawnSync(executable, ['auth', 'status'], { + if (invocation.modelEnv !== undefined) env[MODEL_ENV] = invocation.modelEnv; + const result = spawnSync(executable, invocation.args, { cwd: directory, - stdio: 'ignore', - timeout: 10_000, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + timeout: invocation.timeoutMs, env, }); - const failure = classifySpawnFailure(result.error, result.signal, 10_000); + const failure = classifySpawnFailure(result.error, result.signal, invocation.timeoutMs); if (failure !== undefined) throw failure; - return result.status; + return { status: result.status, stdout: result.stdout }; } function resolveExecutable(command: string, directory: string): string | undefined { @@ -254,7 +292,7 @@ function resolveExecutable(command: string, directory: string): string | undefin function classifySpawnFailure( error: Error | undefined, signal: NodeJS.Signals | null, - timeoutMs: 5_000 | 10_000, + timeoutMs: number, ): CliProbeError | undefined { if (error !== undefined) { const detail = (error as NodeJS.ErrnoException).code === 'ETIMEDOUT' diff --git a/sdk/src/compile.ts b/sdk/src/compile.ts index 48ffa45f..6ad43c1c 100644 --- a/sdk/src/compile.ts +++ b/sdk/src/compile.ts @@ -74,6 +74,7 @@ export function compileSpec(spec: unknown): FlowSpec { ...(input.name !== undefined ? { name: input.name } : {}), ...(input.description !== undefined ? { description: input.description } : {}), ...(input.cli !== undefined ? { cli: input.cli } : {}), + ...(input.agents !== undefined ? { agents: input.agents } : {}), // The kernel omits an empty trigger list when serializing RunSpec. Normalize // it here so the authoring shape and boundary shape retain one hashable form. ...(input.triggers?.length ? { triggers: input.triggers } : {}), @@ -118,6 +119,7 @@ function compileStep(step: StepSpec): StepSpec { ...base, type: 'agent', instruction: s.instruction, + ...(s.agent !== undefined ? { agent: s.agent } : {}), ...(s.cli !== undefined ? { cli: s.cli } : {}), ...(s.model !== undefined ? { model: s.model } : {}), recoveryMode, diff --git a/sdk/src/failure-kinds.ts b/sdk/src/failure-kinds.ts index a669624a..312ae655 100644 --- a/sdk/src/failure-kinds.ts +++ b/sdk/src/failure-kinds.ts @@ -3,6 +3,7 @@ export const PREFLIGHT_FAILURE_KINDS = [ 'cli_missing', 'cli_unauthenticated', 'cli_unresolved', + 'cli_unsupported', 'command_missing', 'model_unavailable', 'model_unknown', diff --git a/sdk/src/preflight.ts b/sdk/src/preflight.ts index 35bb03a9..ddc73bb5 100644 --- a/sdk/src/preflight.ts +++ b/sdk/src/preflight.ts @@ -17,14 +17,17 @@ export interface CliResolution { export interface CliProbeResult { exists: boolean; authenticated: boolean; + /** False when a custom executable did not identify as a wrapper adapter. */ + supported?: boolean; /** Exact declared model passed the CLI's model-scoped readiness probe. */ modelAvailable?: boolean; + authCommand?: string; + modelCommand?: string; } export type CliProbeFailureDetail = | 'spawn_failed' - | 'timeout:5000ms' - | 'timeout:10000ms' + | `timeout:${number}ms` | `signal:${string}`; export class CliProbeError extends Error { @@ -70,6 +73,7 @@ export interface PreflightRefusal { message: string; stepId?: string; cli?: string; + agent?: string; model?: string; triggerId?: string; executor?: string; @@ -96,6 +100,23 @@ export function preflight(flow: FlowSpec, options: PreflightOptions): PreflightR const resolutions: CliResolution[] = []; const cliProbeResults = new Map(); + // Named declarations remain in the normalized authoring object until this + // boundary so even unused or step-shadowed models are checked. Return before + // any environment probe; toKernelSpec erases the map and selector only after + // this authoring preflight has had the chance to fail closed. + for (const [agent, declaration] of Object.entries(flow.agents ?? {})) { + if (isKnownModel(declaration.model, options.models)) continue; + diagnostics.push({ + severity: 'refusal', + kind: 'model_unknown', + agent, + cli: declaration.cli, + model: declaration.model, + message: unknownNamedAgentModelMessage(agent, declaration.cli, declaration.model, options.modelRegistryPath), + }); + } + if (diagnostics.length > 0) return { ok: false, resolutions, diagnostics }; + for (const step of flow.steps) { warnOnUnprovableEffects(step, options.probes, diagnostics); if (step.type === 'deterministic') continue; @@ -152,6 +173,18 @@ export function preflight(flow: FlowSpec, options: PreflightOptions): PreflightR }; } +function unknownNamedAgentModelMessage( + agent: string, + cli: string, + model: string, + registryPath: string | undefined, +): string { + const source = registryPath === undefined + ? 'the nearest project config (no model registry was found)' + : `project model registry "${registryPath}"`; + return `Named agent "${agent}" declares model "${model}" for CLI "${cli}", but it is not listed in ${source}; add the exact model only after verifying that project is allowed to use it.`; +} + function isKnownModel(model: string, models: readonly string[] | undefined): boolean { return models?.includes(model) === true; } @@ -237,13 +270,22 @@ function probeResolvedCli( cli: resolution.cli, message: `Step "${resolution.stepId}" declares CLI "${resolution.cli}", but it does not resolve as an executable.`, }); + } else if (result.supported === false) { + diagnostics.push({ + severity: 'refusal', + kind: 'cli_unsupported', + stepId: resolution.stepId, + cli: resolution.cli, + message: `Step "${resolution.stepId}" declares CLI "${resolution.cli}", but it is neither a supported raw Claude/Codex executable nor a conforming Relayflows wrapper; custom wrappers must identify with the relayflows-agent-cli-v1 contract.`, + }); } else if (!result.authenticated) { + const command = result.authCommand ?? `${resolution.cli} auth status`; diagnostics.push({ severity: 'refusal', kind: 'cli_unauthenticated', stepId: resolution.stepId, cli: resolution.cli, - message: `Step "${resolution.stepId}" declares CLI "${resolution.cli}", but "${resolution.cli} auth status" exited non-zero; authenticate it or implement that probe to return exit 0 when authenticated.`, + message: `Step "${resolution.stepId}" declares CLI "${resolution.cli}", but "${command}" exited non-zero; authenticate it or repair that adapter's authentication probe.`, }); } else if (resolution.model !== undefined && result.modelAvailable !== true) { diagnostics.push({ @@ -252,7 +294,7 @@ function probeResolvedCli( stepId: resolution.stepId, cli: resolution.cli, model: resolution.model, - message: `Step "${resolution.stepId}" declares model "${resolution.model}" for CLI "${resolution.cli}", but its model-scoped "${resolution.cli} auth status" probe exited non-zero; verify the model name and this credential's access.`, + message: `Step "${resolution.stepId}" declares model "${resolution.model}" for CLI "${resolution.cli}", but its model-scoped "${result.modelCommand ?? `${resolution.cli} auth status`}" probe exited non-zero; verify the model name and this credential's access.`, }); } } diff --git a/sdk/src/spec.ts b/sdk/src/spec.ts index fc9c5838..5c35631f 100644 --- a/sdk/src/spec.ts +++ b/sdk/src/spec.ts @@ -141,11 +141,10 @@ export interface AgentStepSpec extends BaseStepSpec { /** Inert preflight declaration; overrides the flow/project CLI default. */ cli?: string; /** - * Model the declared CLI should use, surfaced to it as `RELAYFLOW_MODEL`. - * Declared here so the choice is journaled with the step instead of being - * ambient host state — a CLI that inherits whatever the machine happens to - * pin produces runs whose model cannot be recovered from the journal, and - * fails outright on a host pinning something it cannot resolve. + * Model the declared CLI must use. Raw Claude/Codex adapters receive their + * real model flag; an identified Relayflows wrapper receives + * `RELAYFLOW_MODEL`. Declared here so the choice is journaled with the step + * instead of being ambient host state. */ model?: string; surfaces?: AgentSurfaces; diff --git a/sdk/src/worker.ts b/sdk/src/worker.ts index 77f583cf..3707138c 100644 --- a/sdk/src/worker.ts +++ b/sdk/src/worker.ts @@ -3,6 +3,7 @@ import { EventEmitter } from 'node:events'; import type { JournalClient } from './journal-client.js'; import type { Pins, StepDispatchEvent } from './protocol.js'; import type { KernelAgentStep } from './spec.js'; +import { agentExecution, cliAdapterKind } from './cli-adapter.js'; export interface AgentWorkerOptions { workerId: string; @@ -167,10 +168,11 @@ export const WAKE_CONTEXT_ENV = 'RELAYFLOW_WAKE_CONTEXT'; /** * Environment variable AgentWorker sets when the dispatched agent step * DECLARED a `model`. Same contract as {@link WAKE_CONTEXT_ENV}: when the - * step declares no model the variable is not merely empty, it is ABSENT, + * wrapper step declares no model the variable is not merely empty, it is ABSENT, * so a CLI can tell "the flow author chose nothing" from "the flow author - * chose something". A CLI that finds it unset is free to apply its own - * default; one that finds it set must not override it. + * chose something". Raw Claude/Codex adapters use their real model flags + * instead; only an explicitly identified Relayflows wrapper receives this + * private environment contract. * * This exists because a CLI inheriting whatever model the host happens to * pin produces two failures: runs whose model cannot be recovered from the @@ -187,6 +189,7 @@ function runCli( ): Promise { return new Promise((resolve) => { const env: NodeJS.ProcessEnv = { ...process.env }; + const invocation = agentExecution(cliAdapterKind(cli), instruction, model); // Explicit unset. Without this, a parent process (wrapper // script, systemd unit, docker env, or a prior test) that // already had RELAYFLOW_WAKE_CONTEXT set would leak into @@ -203,7 +206,7 @@ function runCli( // no model look like one that did, silently pinning the run to whatever // the launching shell happened to export. delete env[MODEL_ENV]; - if (model !== undefined) env[MODEL_ENV] = model; + if (invocation.modelEnv !== undefined) env[MODEL_ENV] = invocation.modelEnv; if (wakeContext !== undefined) { // `execve` caps argv + envp at ARG_MAX (macOS ~256 KB, Linux // ~2 MB). A wake_context that packs a rich payload could @@ -233,7 +236,7 @@ function runCli( return; } } - const child = spawn(cli, [instruction], { stdio: ['ignore', 'pipe', 'pipe'], env }); + const child = spawn(cli, invocation.args, { stdio: ['ignore', 'pipe', 'pipe'], env }); const stdout: Buffer[] = []; const stderr: Buffer[] = []; child.stdout.on('data', (chunk: Buffer) => stdout.push(chunk)); diff --git a/sdk/tests/cli-adapter.test.ts b/sdk/tests/cli-adapter.test.ts new file mode 100644 index 00000000..c33b066a --- /dev/null +++ b/sdk/tests/cli-adapter.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from 'vitest'; +import { + agentExecution, + adapterIdentification, + authenticationProbe, + cliAdapterKind, + modelReadinessProbe, + WRAPPER_IDENTIFY_ARG, +} from '../src/cli-adapter.js'; + +describe('typed CLI adapters', () => { + it('maps raw Claude to real auth, noninteractive, and model flag shapes', () => { + const kind = cliAdapterKind('/usr/local/bin/claude'); + + expect(kind).toBe('claude'); + expect(adapterIdentification(kind).invocation.args).toEqual(['auth', 'status', '--help']); + expect(authenticationProbe(kind).args).toEqual(['auth', 'status']); + const readiness = modelReadinessProbe(kind, 'claude-model'); + expect(readiness).toMatchObject({ + args: expect.arrayContaining(['-p', '--model', 'claude-model']), + }); + expect(readiness).not.toHaveProperty('modelEnv'); + expect(agentExecution(kind, 'Review.', 'claude-model')).toEqual({ + args: ['-p', '--model', 'claude-model', 'Review.'], + timeoutMs: 0, + }); + }); + + it('maps raw Codex to login status and noninteractive exec --model', () => { + const kind = cliAdapterKind('/opt/bin/codex'); + + expect(kind).toBe('codex'); + expect(adapterIdentification(kind).invocation.args).toEqual(['login', 'status', '--help']); + expect(authenticationProbe(kind).args).toEqual(['login', 'status']); + const readiness = modelReadinessProbe(kind, 'gpt-model'); + expect(readiness).toMatchObject({ + args: expect.arrayContaining(['exec', '--ephemeral', '--model', 'gpt-model']), + }); + expect(readiness).not.toHaveProperty('modelEnv'); + expect(agentExecution(kind, 'Review.', 'gpt-model')).toEqual({ + args: ['exec', '--ephemeral', '--model', 'gpt-model', 'Review.'], + timeoutMs: 0, + }); + }); + + it('requires custom executables to identify before using the wrapper env protocol', () => { + const kind = cliAdapterKind('/project/bin/team-reviewer'); + + expect(kind).toBe('relayflows-wrapper-v1'); + expect(adapterIdentification(kind).invocation.args).toEqual([WRAPPER_IDENTIFY_ARG]); + expect(authenticationProbe(kind).args).toEqual(['auth', 'status']); + expect(modelReadinessProbe(kind, 'team-model')).toMatchObject({ + args: ['auth', 'status'], + modelEnv: 'team-model', + }); + expect(agentExecution(kind, 'Review.', 'team-model')).toEqual({ + args: ['Review.'], + timeoutMs: 0, + modelEnv: 'team-model', + }); + }); +}); diff --git a/sdk/tests/cli.test.ts b/sdk/tests/cli.test.ts index 33755fbe..3d7a126d 100644 --- a/sdk/tests/cli.test.ts +++ b/sdk/tests/cli.test.ts @@ -74,6 +74,10 @@ function namedAgentProject(model: string, allowedModels: string[]): { const cliPath = join(directory, 'model-cli'); const probeLog = `${cliPath}.log`; writeFileSync(cliPath, `#!/bin/sh +if [ "\${1-}" = "--relayflows-adapter-v1" ]; then + printf '%s\\n' 'relayflows-agent-cli-v1' + exit 0 +fi test "$1 $2" = "auth status" || exit 9 printf '%s\n' "\${RELAYFLOW_MODEL-UNSET}" >> "$0.log" test "\${RELAYFLOW_MODEL-UNSET}" = "UNSET" -o "\${RELAYFLOW_MODEL-UNSET}" = "available-model" @@ -96,6 +100,13 @@ steps: return { directory, flowPath, probeLog }; } +function executableFixture(directory: string, name: string, body: string): string { + const path = join(directory, name); + writeFileSync(path, `#!/bin/sh\n${body}\n`); + chmodSync(path, 0o755); + return path; +} + /** * RFC-0001 §96 makes *the ladder flows* the subject of the refusal clause, so * the fault is induced on the canonical flows themselves rather than on a @@ -131,6 +142,109 @@ async function startCliLoopback(dataDir: string, handlers: LoopbackHandlers): Pr } describe('flows check CLI', () => { + it.each(['unused', 'shadowed'] as const)( + 'refuses an unknown %s named-agent declaration before compilation erases it', + async (variant) => { + const directory = temporaryProject('flows-declared-model-'); + const probeLog = join(directory, 'probe.log'); + const cli = executableFixture(directory, 'model-cli', `printf '%s\\n' "$*" >> ${JSON.stringify(probeLog)}\nexit 0`); + writeFileSync(join(directory, 'flows.json'), JSON.stringify({ models: ['known-model'] })); + const steps = variant === 'unused' + ? ' - id: ready\n type: deterministic\n command: printf ready' + : ' - id: review\n type: agent\n agent: reviewer\n model: known-model\n instruction: Review.'; + const path = join(directory, `${variant}.flow.yaml`); + writeFileSync(path, `version: '0.1.0' +agents: + reviewer: + cli: ${JSON.stringify(cli)} + model: typo-model +steps: +${steps} +`); + + const result = await run(path); + expect(result.code).toBe(2); + expect(result.stderr.join('\n')).toContain('REFUSED [model_unknown]'); + expect(result.stderr.join('\n')).toContain('Named agent "reviewer"'); + expect(result.stderr.join('\n')).toContain('typo-model'); + expect(existsSync(probeLog)).toBe(false); + }, + ); + + it('uses the raw Claude adapter model flag instead of accepting auth status as model proof', async () => { + const directory = temporaryProject('flows-claude-adapter-'); + const log = join(directory, 'claude.log'); + const cli = executableFixture(directory, 'claude', `printf '%s|MODEL_ENV=%s\\n' "$*" "\${RELAYFLOW_MODEL-UNSET}" >> ${JSON.stringify(log)} +if [ "$1 $2" = "auth status" ]; then exit 0; fi +if [ "$1 $2 $3" = "-p --model available-model" ]; then exit 0; fi +exit 7`); + writeFileSync(join(directory, 'flows.json'), JSON.stringify({ models: ['available-model'] })); + const path = join(directory, 'claude.flow.yaml'); + writeFileSync(path, `version: '0.1.0' +agents: + reviewer: { cli: ${JSON.stringify(cli)}, model: available-model } +steps: + - id: review + type: agent + agent: reviewer + instruction: Review. +`); + + const result = await run(path); + expect(result.code).toBe(0); + expect(readFileSync(log, 'utf8')).toContain('-p --model available-model'); + expect(readFileSync(log, 'utf8')).toContain('MODEL_ENV=UNSET'); + expect(readFileSync(log, 'utf8')).toContain('auth status --help|MODEL_ENV=UNSET'); + }); + + it('uses Codex login status and reports a rejected model as unavailable, not unauthenticated', async () => { + const directory = temporaryProject('flows-codex-adapter-'); + const log = join(directory, 'codex.log'); + const cli = executableFixture(directory, 'codex', `printf '%s|MODEL_ENV=%s\\n' "$*" "\${RELAYFLOW_MODEL-UNSET}" >> ${JSON.stringify(log)} +if [ "$1 $2" = "login status" ]; then exit 0; fi +if [ "$1" = "exec" ]; then exit 8; fi +exit 9`); + writeFileSync(join(directory, 'flows.json'), JSON.stringify({ models: ['denied-model'] })); + const path = join(directory, 'codex.flow.yaml'); + writeFileSync(path, `version: '0.1.0' +agents: + reviewer: { cli: ${JSON.stringify(cli)}, model: denied-model } +steps: + - id: review + type: agent + agent: reviewer + instruction: Review. +`); + + const result = await run(path); + expect(result.code).toBe(2); + expect(result.stderr.join('\n')).toContain('REFUSED [model_unavailable]'); + expect(result.stderr.join('\n')).not.toContain('cli_unauthenticated'); + expect(readFileSync(log, 'utf8')).toContain('exec --ephemeral'); + expect(readFileSync(log, 'utf8')).toContain('--model denied-model'); + expect(readFileSync(log, 'utf8')).toContain('login status|MODEL_ENV=UNSET'); + expect(readFileSync(log, 'utf8')).toContain('login status --help|MODEL_ENV=UNSET'); + expect(readFileSync(log, 'utf8')).not.toContain('auth status'); + }); + + it('refuses a nonconforming custom wrapper without calling it an authentication failure', async () => { + const directory = temporaryProject('flows-wrapper-adapter-'); + const cli = executableFixture(directory, 'not-an-adapter', 'exit 0'); + const path = join(directory, 'wrapper.flow.yaml'); + writeFileSync(path, `version: '0.1.0' +steps: + - id: review + type: agent + cli: ${JSON.stringify(cli)} + instruction: Review. +`); + + const result = await run(path); + expect(result.code).toBe(2); + expect(result.stderr.join('\n')).toContain('REFUSED [cli_unsupported]'); + expect(result.stderr.join('\n')).not.toContain('cli_unauthenticated'); + }); + it('accepts an exact allowlisted named-agent model and probes that model', async () => { const fixture = namedAgentProject('available-model', ['available-model']); const result = await run(fixture.flowPath); @@ -423,7 +537,7 @@ steps: mkdirSync(flowDirectory); writeFileSync(join(directory, 'flows.json'), JSON.stringify({ cli: './authenticated-cli', executors: [] })); const cli = join(directory, 'authenticated-cli'); - writeFileSync(cli, '#!/bin/sh\n[ "$1 $2" = "auth status" ]\n'); + writeFileSync(cli, '#!/bin/sh\nif [ "${1-}" = "--relayflows-adapter-v1" ]; then echo relayflows-agent-cli-v1; exit 0; fi\n[ "$1 $2" = "auth status" ]\n'); chmodSync(cli, 0o755); const flow = join(flowDirectory, 'project-cli.flow.yaml'); writeFileSync(flow, "version: '0.1.0'\nsteps:\n - id: answer\n type: llm\n prompt: answer\n"); diff --git a/sdk/tests/live-kernel.test.ts b/sdk/tests/live-kernel.test.ts index e7c78153..8179e4a1 100644 --- a/sdk/tests/live-kernel.test.ts +++ b/sdk/tests/live-kernel.test.ts @@ -688,7 +688,7 @@ steps: await worker.close(); }, 30_000); - it('AgentWorker passes a declared model to the CLI as RELAYFLOW_MODEL', async () => { + it('AgentWorker passes a declared model to an identified wrapper as RELAYFLOW_MODEL', async () => { // The whole point of declaring `model` on the step is that the CLI // stops inheriting whatever the host pinned. This proves the declared // value survives the full boundary: SDK compile → kernel parse → @@ -716,13 +716,16 @@ steps: agent: model-probe instruction: Report the model env var. `); - expect(compiled).not.toHaveProperty('agents'); + expect(compiled).toHaveProperty('agents.model-probe.model', 'declared-model-xyz'); expect(compiled.steps[0]).toMatchObject({ type: 'agent', cli, model: 'declared-model-xyz', }); - const started = await client.runStart(toKernelSpec(compiled)); + const kernel = toKernelSpec(compiled); + expect(kernel).not.toHaveProperty('agents'); + expect(kernel.steps[0]).not.toHaveProperty('agent'); + const started = await client.runStart(kernel); expect(await waitForStep(client, started.run_id, 'probe', 'done')).toMatchObject({ type: 'agent', @@ -743,6 +746,50 @@ steps: await worker.close(); }, 30_000); + it.each([ + ['claude', '-p --model declared-model-xyz'], + ['codex', 'exec --ephemeral --model declared-model-xyz'], + ] as const)('AgentWorker executes the raw %s adapter with its real model flag', async (name, prefix) => { + const dataDir = temporaryDirectory(`flows-live-${name}-adapter-`); + await startDaemon(dataDir); + const cli = join(dataDir, name); + writeFileSync(cli, `#!/bin/sh +case "$*" in + ${JSON.stringify(`${prefix} `)}*) ;; + *) printf '%s\\n' "unexpected argv: $*" >&2; exit 9 ;; +esac +test "\${RELAYFLOW_MODEL+x}" != x || exit 8 +printf '%s' '{"adapter":"${name}","model_flag":"declared-model-xyz"}' +`); + chmodSync(cli, 0o755); + const client = await connectClient(dataDir); + await client.hello(`live-${name}-adapter`); + const worker = new AgentWorker(client, { + workerId: `live-${name}-adapter-worker`, + pins: { workspace: [{ surface: 'repo', revision_id: 'rev-a' }], streams: [] }, + }); + await worker.attach(); + + const started = await client.runStart(toKernelSpec(compileYaml(` +version: '0.1.0' +steps: + - id: probe + type: agent + cli: ${JSON.stringify(cli)} + model: declared-model-xyz + instruction: Report the adapter. +`))); + + expect(await waitForStep(client, started.run_id, 'probe', 'done')).toMatchObject({ state: 'done' }); + const completed = (await client.journalRead(started.run_id)).entries.find( + (entry) => (entry as { entry_type: string; step_id?: string }).entry_type === 'step.completed' + && (entry as { step_id?: string }).step_id === 'probe', + ) as { payload: { output: { adapter: string; model_flag: string } } } | undefined; + expect(completed?.payload.output).toEqual({ adapter: name, model_flag: 'declared-model-xyz' }); + + await worker.close(); + }, 30_000); + it('AgentWorker leaves RELAYFLOW_MODEL UNSET when the step declares no model', async () => { // Absence must stay absence, so a CLI can apply its own default and a // reader can tell "the author chose nothing" from "the author chose". @@ -1255,6 +1302,10 @@ async function waitForStep( */ function probeAnalyzer(cli: string): { ready: boolean; detail: string } { if (!existsSync(cli)) return { ready: false, detail: `analyzer CLI does not exist: ${cli}` }; + const identified = spawnSync(cli, ['--relayflows-adapter-v1'], { encoding: 'utf8', timeout: 10_000 }); + if (identified.error !== undefined || identified.status !== 0 || identified.stdout.trim() !== 'relayflows-agent-cli-v1') { + return { ready: false, detail: `"${cli}" does not identify as relayflows-agent-cli-v1` }; + } const probe = spawnSync(cli, ['auth', 'status'], { encoding: 'utf8', timeout: 30_000 }); if (probe.error !== undefined) { return { ready: false, detail: `"${cli} auth status" could not run: ${probe.error.message}` }; diff --git a/sdk/tests/model-selection.test.ts b/sdk/tests/model-selection.test.ts index 92c779bc..ee3afea2 100644 --- a/sdk/tests/model-selection.test.ts +++ b/sdk/tests/model-selection.test.ts @@ -17,14 +17,20 @@ steps: instruction: Review the change. `); - expect(flow).not.toHaveProperty('agents'); + expect(flow.agents).toEqual({ + reviewer: { cli: 'claude', model: 'claude-sonnet-4-6' }, + }); + expect(flow.steps[0]).toHaveProperty('agent', 'reviewer'); expect(flow.steps[0] as AgentStepSpec).toMatchObject({ id: 'review', type: 'agent', cli: 'claude', model: 'claude-sonnet-4-6', }); - expect(toKernelSpec(flow).steps[0]).toMatchObject({ + const kernel = toKernelSpec(flow); + expect(kernel).not.toHaveProperty('agents'); + expect(kernel.steps[0]).not.toHaveProperty('agent'); + expect(kernel.steps[0]).toMatchObject({ type: 'agent', cli: 'claude', model: 'claude-sonnet-4-6', diff --git a/sdk/tests/preflight.test.ts b/sdk/tests/preflight.test.ts index 8034c1a1..1fe689e2 100644 --- a/sdk/tests/preflight.test.ts +++ b/sdk/tests/preflight.test.ts @@ -10,6 +10,7 @@ import { type PreflightProbes, } from '../src/preflight.js'; import type { FlowSpec } from '../src/spec.js'; +import { compileSpec, toKernelSpec } from '../src/compile.js'; function flow(step: FlowSpec['steps'][number]): FlowSpec { return { version: '0.1.0', name: 'test', steps: [step] }; @@ -222,6 +223,7 @@ describe('preflight: CLI resolution and refusal predicates', () => { const scenarios = [ preflight(flow({ id: 'a', type: 'llm', prompt: 'p', cli: 'x' }), { probes: probes({ cli: () => ({ exists: false, authenticated: false }) }) }), preflight(flow({ id: 'a', type: 'llm', prompt: 'p', cli: 'x' }), { probes: probes({ cli: () => ({ exists: true, authenticated: false }) }) }), + preflight(flow({ id: 'a', type: 'llm', prompt: 'p', cli: 'x' }), { probes: probes({ cli: () => ({ exists: true, supported: false, authenticated: false }) }) }), preflight(flow({ id: 'a', type: 'llm', prompt: 'p' }), { probes: probes() }), preflight(flow({ id: 'a', type: 'deterministic', command: './missing' }), { probes: probes({ command: () => false }) }), preflight(flow({ id: 'a', type: 'agent', instruction: 'i', cli: 'x', model: 'typo-model' }), { models: ['known-model'], probes: probes() }), @@ -252,4 +254,40 @@ describe('preflight: CLI resolution and refusal predicates', () => { model: 'typo-model', }); }); + + it.each(['unused', 'shadowed'] as const)( + 'checks an unknown %s named declaration before authoring metadata is erased', + (variant) => { + let probeCalls = 0; + const compiled = compileSpec({ + version: '0.1.0', + agents: { reviewer: { cli: 'claude', model: 'typo-model' } }, + steps: variant === 'unused' + ? [{ id: 'ready', type: 'deterministic', command: 'printf ready' }] + : [{ + id: 'review', + type: 'agent', + agent: 'reviewer', + model: 'known-model', + instruction: 'Review.', + }], + }); + + const result = preflight(compiled, { + models: ['known-model'], + probes: probes({ cli: () => { + probeCalls += 1; + return { exists: true, authenticated: true, modelAvailable: true }; + } }), + }); + + expect(compiled.agents?.reviewer?.model).toBe('typo-model'); + expect(result.ok).toBe(false); + expect(result.diagnostics).toEqual([ + expect.objectContaining({ kind: 'model_unknown', agent: 'reviewer', model: 'typo-model' }), + ]); + expect(probeCalls).toBe(0); + expect(toKernelSpec(compiled)).not.toHaveProperty('agents'); + }, + ); }); diff --git a/sdk/tests/real-cli-adapters.test.ts b/sdk/tests/real-cli-adapters.test.ts new file mode 100644 index 00000000..c4dc0e9d --- /dev/null +++ b/sdk/tests/real-cli-adapters.test.ts @@ -0,0 +1,49 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, describe, expect, it } from 'vitest'; +import { checkFlow } from '../src/cli/check.js'; + +const RUN_REAL = process.env['RELAYFLOWS_REAL_CLI_ADAPTERS'] === '1'; +const directories: string[] = []; + +afterAll(() => { + for (const directory of directories) rmSync(directory, { recursive: true, force: true }); +}); + +function realFlow(cli: 'claude' | 'codex', model: string): string { + const directory = mkdtempSync(join(tmpdir(), `flows-real-${cli}-`)); + directories.push(directory); + writeFileSync(join(directory, 'flows.json'), JSON.stringify({ models: [model] })); + const path = join(directory, `${cli}.flow.yaml`); + writeFileSync(path, `version: '0.1.0' +agents: + reviewer: { cli: ${cli}, model: ${model} } +steps: + - id: review + type: agent + agent: reviewer + instruction: Review. +`); + return path; +} + +describe.runIf(RUN_REAL)('installed raw CLI adapters', () => { + it('round-trips the exact declared Claude model and refuses an impossible one', () => { + const available = process.env['RELAYFLOWS_REAL_CLAUDE_MODEL'] ?? 'claude-haiku-4-5-20251001'; + expect(checkFlow(realFlow('claude', available)).report.ok).toBe(true); + + const impossible = 'relayflows-definitely-not-a-real-claude-model'; + const refused = checkFlow(realFlow('claude', impossible)).report; + expect(refused.ok).toBe(false); + expect(refused.diagnostics).toContainEqual(expect.objectContaining({ kind: 'model_unavailable' })); + }, 130_000); + + it('uses Codex login status and classifies an impossible model as unavailable', () => { + const impossible = 'relayflows-definitely-not-a-real-codex-model'; + const refused = checkFlow(realFlow('codex', impossible)).report; + expect(refused.ok).toBe(false); + expect(refused.diagnostics).toContainEqual(expect.objectContaining({ kind: 'model_unavailable' })); + expect(refused.diagnostics).not.toContainEqual(expect.objectContaining({ kind: 'cli_unauthenticated' })); + }, 70_000); +}); diff --git a/testdata/preflight/analyze-story-claude-cli b/testdata/preflight/analyze-story-claude-cli index 50f71e7b..a3c786f1 100755 --- a/testdata/preflight/analyze-story-claude-cli +++ b/testdata/preflight/analyze-story-claude-cli @@ -12,13 +12,18 @@ // prep step chmods `testdata/preflight/*-cli` — so the Node floor is // the tradeoff for matching it. // -// Two invocation shapes: -// `auth status` → repo-wide preflight probe (sdk/src/preflight.ts). -// Exit 0 only when a real round-trip succeeds. -// `` → analyze; one JSON object on stdout, exit 0. +// Three invocation shapes: +// `--relayflows-adapter-v1` → identify the explicit wrapper contract. +// `auth status` → model-scoped preflight round trip. +// `` → analyze; one JSON object on stdout, exit 0. import { spawnSync } from 'node:child_process'; +if (process.argv[2] === '--relayflows-adapter-v1') { + process.stdout.write('relayflows-agent-cli-v1\n'); + process.exit(0); +} + // Say so plainly. Without this, Node 20 fails on the ESM/top-level-await // above with a parse error that names a syntax position, not the cause. const major = Number(process.versions.node.split('.')[0]); diff --git a/testdata/preflight/analyze-story-echo-wake-cli b/testdata/preflight/analyze-story-echo-wake-cli index c8d13e21..58ad6421 100755 --- a/testdata/preflight/analyze-story-echo-wake-cli +++ b/testdata/preflight/analyze-story-echo-wake-cli @@ -9,6 +9,10 @@ // // Node-native so the test never needs to skip on missing tooling // (jq is not a stock macOS dep) — the SDK ships Node already. +if (process.argv[2] === '--relayflows-adapter-v1') { + process.stdout.write('relayflows-agent-cli-v1\n'); + process.exit(0); +} const raw = process.env.RELAYFLOW_WAKE_CONTEXT; if (!raw) { process.stderr.write('wake-context env var RELAYFLOW_WAKE_CONTEXT was empty or unset\n'); diff --git a/testdata/preflight/analyze-story-missing-fields-cli b/testdata/preflight/analyze-story-missing-fields-cli index 9b1349d1..69b5ae4b 100755 --- a/testdata/preflight/analyze-story-missing-fields-cli +++ b/testdata/preflight/analyze-story-missing-fields-cli @@ -10,4 +10,8 @@ # required schema field. The positive test above catches the # wrapper-not-promoted mutation.) set -eu +if [ "${1-}" = "--relayflows-adapter-v1" ]; then + printf '%s\n' 'relayflows-agent-cli-v1' + exit 0 +fi printf '%s' '{"story_title":"partial"}' diff --git a/testdata/preflight/analyze-story-stub-cli b/testdata/preflight/analyze-story-stub-cli index 723e6010..1992983b 100755 --- a/testdata/preflight/analyze-story-stub-cli +++ b/testdata/preflight/analyze-story-stub-cli @@ -7,4 +7,8 @@ # would read `$1` (the instruction) and — once wake_context injection # lands — the triggering-event JSON, then invoke a real LLM. set -eu +if [ "${1-}" = "--relayflows-adapter-v1" ]; then + printf '%s\n' 'relayflows-agent-cli-v1' + exit 0 +fi printf '%s' '{"story_title":"stub","relevance_score":5,"reasoning":"stub agent runtime — deterministic output for gate-2 clause-2 demo"}' diff --git a/testdata/preflight/analyze-story-text-only-cli b/testdata/preflight/analyze-story-text-only-cli index d3da1422..2a89e8b8 100755 --- a/testdata/preflight/analyze-story-text-only-cli +++ b/testdata/preflight/analyze-story-text-only-cli @@ -5,4 +5,8 @@ # tools (progress bars, chatty CLIs) still round-trip usefully # without the JSON promotion path silently discarding stdout/stderr. set -eu +if [ "${1-}" = "--relayflows-adapter-v1" ]; then + printf '%s\n' 'relayflows-agent-cli-v1' + exit 0 +fi printf '%s' 'looked at the story, seemed fine to me' diff --git a/testdata/preflight/authenticated-cli b/testdata/preflight/authenticated-cli index 15dc7aac..5a4e0175 100755 --- a/testdata/preflight/authenticated-cli +++ b/testdata/preflight/authenticated-cli @@ -1,3 +1,7 @@ #!/bin/sh # Deterministic preflight fixture: the auth probe is healthy. +if [ "${1-}" = "--relayflows-adapter-v1" ]; then + printf '%s\n' 'relayflows-agent-cli-v1' + exit 0 +fi test "$1" = "auth" && test "$2" = "status" diff --git a/testdata/preflight/counting-cli b/testdata/preflight/counting-cli index 06c367d0..4a49dee5 100755 --- a/testdata/preflight/counting-cli +++ b/testdata/preflight/counting-cli @@ -1,5 +1,9 @@ #!/bin/sh # Deterministic preflight fixture: record each auth probe in the caller's log. +if [ "${1-}" = "--relayflows-adapter-v1" ]; then + printf '%s\n' 'relayflows-agent-cli-v1' + exit 0 +fi test "$1" = "auth" && test "$2" = "status" || exit 1 test -n "$PREFLIGHT_PROBE_LOG" || exit 1 printf 'auth status\n' >> "$PREFLIGHT_PROBE_LOG" diff --git a/testdata/preflight/echo-model-cli b/testdata/preflight/echo-model-cli index 35f236d4..b5db5edd 100755 --- a/testdata/preflight/echo-model-cli +++ b/testdata/preflight/echo-model-cli @@ -9,6 +9,10 @@ // string as its own answer — if `RELAYFLOW_MODEL=''` ever reaches a CLI, // that is a bug the test must be able to see rather than silently read as // "unset". +if (process.argv[2] === '--relayflows-adapter-v1') { + process.stdout.write('relayflows-agent-cli-v1\n'); + process.exit(0); +} const raw = process.env.RELAYFLOW_MODEL; const declared = raw === undefined ? 'UNSET' : raw === '' ? 'EMPTY' : raw; process.stdout.write(JSON.stringify({ diff --git a/testdata/preflight/signal-probe-cli b/testdata/preflight/signal-probe-cli index c627cbdc..a1d595cc 100755 --- a/testdata/preflight/signal-probe-cli +++ b/testdata/preflight/signal-probe-cli @@ -1,4 +1,8 @@ #!/bin/sh # Deterministic preflight fixture: the auth probe is terminated by a signal. +if [ "${1-}" = "--relayflows-adapter-v1" ]; then + printf '%s\n' 'relayflows-agent-cli-v1' + exit 0 +fi test "$1" = "auth" && test "$2" = "status" || exit 0 kill -SEGV "$$" diff --git a/testdata/preflight/unauthenticated-cli b/testdata/preflight/unauthenticated-cli index e91ecae2..82dbb291 100755 --- a/testdata/preflight/unauthenticated-cli +++ b/testdata/preflight/unauthenticated-cli @@ -1,3 +1,7 @@ #!/bin/sh # Deterministic preflight fixture: installed, but auth is unhealthy. +if [ "${1-}" = "--relayflows-adapter-v1" ]; then + printf '%s\n' 'relayflows-agent-cli-v1' + exit 0 +fi exit 1 diff --git a/testdata/preflight/wake-context-probe-cli b/testdata/preflight/wake-context-probe-cli index baefd27d..d1c51ecd 100755 --- a/testdata/preflight/wake-context-probe-cli +++ b/testdata/preflight/wake-context-probe-cli @@ -4,5 +4,9 @@ // wake-context-absent test to pin the undefined-vs-null // invariant AgentWorker documents: a run started without a // triggering event must dispatch with no env var set. +if (process.argv[2] === '--relayflows-adapter-v1') { + process.stdout.write('relayflows-agent-cli-v1\n'); + process.exit(0); +} const present = process.env.RELAYFLOW_WAKE_CONTEXT !== undefined; process.stdout.write(JSON.stringify({ env_present: present })); From 060e8b971a04f27785d88e1520aa7c1eb3fd3fc5 Mon Sep 17 00:00:00 2001 From: kjgbot Date: Wed, 2 Sep 2026 19:46:22 +0200 Subject: [PATCH 04/15] fix(sdk): bind declared model execution checks Session-Id: 01a062b7-2aef-7772-8225-7cb00ee311dd --- docs/SURFACE.md | 18 +- ops/reviews/20260902-1905-pr136-history.md | 652 ++++++++++++++++++ .../20260902-1905-pr136-maintainability.md | 387 +++++++++++ ops/reviews/20260902-1905-pr136-structure.md | 294 ++++++++ ops/reviews/20260902-1945-pr136-repair.md | 118 ++++ sdk/src/cli-adapter.ts | 2 +- sdk/src/cli/check.ts | 2 +- sdk/src/preflight.ts | 88 +-- sdk/src/worker-cli.ts | 110 +++ sdk/src/worker.ts | 119 +--- sdk/tests/cli-adapter.test.ts | 12 +- sdk/tests/live-kernel.test.ts | 77 ++- sdk/tests/preflight.test.ts | 67 +- sdk/tests/real-cli-adapters.test.ts | 19 + 14 files changed, 1792 insertions(+), 173 deletions(-) create mode 100644 ops/reviews/20260902-1905-pr136-history.md create mode 100644 ops/reviews/20260902-1905-pr136-maintainability.md create mode 100644 ops/reviews/20260902-1905-pr136-structure.md create mode 100644 ops/reviews/20260902-1945-pr136-repair.md create mode 100644 sdk/src/worker-cli.ts diff --git a/docs/SURFACE.md b/docs/SURFACE.md index 6bf38443..d1a495f9 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -99,8 +99,10 @@ No process runs between events: the handler wakes, executes to its next await, p `claude auth status`, probes the exact model with a real noninteractive `claude -p --model ` round trip, and executes with that same model flag. A basename of `codex` uses `codex login status`, probes with - `codex exec --model ` in an ephemeral read-only session, and executes - noninteractively with `codex exec --model `. Model-scoped probes may + `codex exec --skip-git-repo-check --model ` in an ephemeral read-only + session, and executes noninteractively with the same Git/cwd flag. A Git + checkout is not a Relayflow execution prerequisite, so readiness and worker + execution both support non-Git working directories. Model-scoped probes may contact the provider and have a 60-second timeout; this cost is the honest price of proving current credential/model access rather than accepting an unrelated auth command as model proof. @@ -112,7 +114,11 @@ No process runs between events: the handler wakes, executes to its next await, p protocol. A missing or wrong identification is `cli_unsupported`, never mislabeled as `cli_unauthenticated`. If a model-scoped probe fails, the adapter's real unscoped authentication command distinguishes - `model_unavailable` from `cli_unauthenticated`. + `model_unavailable` from `cli_unauthenticated`. The worker repeats the exact + wrapper identification immediately before execution, with the private model + and wake-context variables absent. A direct journal submission or an + executable replaced after preflight therefore completes `worker_error` + without receiving `RELAYFLOW_MODEL` unless the current binary identifies. `flows check` resolves the binary (a path is relative to the declaring flow or project config; a bare name resolves via `PATH`) and caches each resolved @@ -128,8 +134,10 @@ No process runs between events: the handler wakes, executes to its next await, p regex or provider prefix. The nearest `flows.json` owns an exact, case-sensitive `models` allowlist. `flows check` first refuses a declared model absent from that list as `model_unknown`, without starting the CLI. - This includes every named declaration, even when unused or shadowed by a - step override; + One pure first pass collects every unknown named and inline declaration + before any CLI, command, executor, or daemon probe, independent of step + order. This includes every named declaration, even when unused or shadowed + by a step override; only an allowlisted value reaches the live model-scoped probe above. The registry is author-owned project configuration, reviewed and versioned with the project. Updating it is an explicit file change made only after the diff --git a/ops/reviews/20260902-1905-pr136-history.md b/ops/reviews/20260902-1905-pr136-history.md new file mode 100644 index 00000000..1f4e1285 --- /dev/null +++ b/ops/reviews/20260902-1905-pr136-history.md @@ -0,0 +1,652 @@ +# PR #136 exact-head history / regression review + +- Lens: prior blocker closure, regression fit, preflight/lowering bypasses, + v1/default compatibility, and real Claude/Codex adapter behavior +- Exact head: 4888d1572ed047c5161042614ac72068d047783a +- Merged main/base: a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2 +- Mode: independent assessment only; no product code was edited +- Constitution: AGENTS.md and docs/RFC-0001-everything-is-a-relayflow.md + were read in full before assessment + +## Verdict + +**FAIL.** One high-severity execution bypass remains. The Codex model-readiness +probe deliberately adds --skip-git-repo-check, but the exact-head Codex worker +invocation omits it. A v0.1.0 flow using a real installed Codex and an available +allowlisted model therefore passes flows check from a non-Git working directory, +then the exact invocation produced by agentExecution fails immediately because +Codex refuses that directory. The control command succeeds when the missing +flag is added. + +The previous structure blocker is fixed: compileSpec preserves the named-agent +map and selected agent through authoring preflight, including unused and +step-shadowed invalid declarations, and only toKernelSpec removes the authoring +sugar. The old auth/model-classification blocker is also fixed for both raw +providers, and Claude execution is real and green. The remaining Codex mismatch +means the former “real CLI execution” blocker is only partially closed. + +## Finding + +### [HIGH] A green Codex preflight can still become an immediate worker refusal outside a Git repository + +At exact-head sdk/src/cli-adapter.ts:64-71, the Codex readiness probe uses: + + exec --ephemeral --sandbox read-only --skip-git-repo-check --model ... + +At exact-head sdk/src/cli-adapter.ts:92-99, agentExecution uses: + + exec --ephemeral --model ... + +AgentWorker launches that second argv without setting cwd, so it inherits the +worker process directory. The surface does not require that directory to be a +trusted Git repository. This is not hypothetical: the same declared CLI/model +and same non-Git directory pass exact-head preflight and fail the exact +worker-side invocation. + +Literal command (run from a pristine git archive of the exact head): + +~~~sh +node dist/cli.js check --json /tmp/pr136-real-codex-worker.BQK070/flow.yaml +check_status=$? +echo "non_git_codex_preflight_exit=$check_status" +PR136_CODEX_CWD=/tmp/pr136-real-codex-worker.BQK070 node --input-type=module <<'EOF' +import { spawnSync } from 'node:child_process'; +import { agentExecution } from './dist/cli-adapter.js'; +const invocation = agentExecution('codex', 'Reply with exactly RELAYFLOWS_NON_GIT_READY.', 'gpt-5.6-sol'); +console.log(JSON.stringify({cwd:process.env.PR136_CODEX_CWD,args:invocation.args})); +const result = spawnSync('codex', invocation.args, { + cwd:process.env.PR136_CODEX_CWD, + encoding:'utf8', + timeout:120000, + stdio:['ignore','pipe','pipe'] +}); +console.log(JSON.stringify({ + status:result.status, + signal:result.signal, + error:result.error?.message??null, + stdout:(result.stdout??'').trim(), + stderr:(result.stderr??'').trim() +})); +process.exit(result.status ?? 1); +EOF +worker_status=$? +echo "non_git_codex_worker_invocation_exit=$worker_status" +test "$check_status" -eq 0 -a "$worker_status" -ne 0 +evidence_status=$? +echo "preflight_green_worker_red_evidence_exit=$evidence_status" +exit "$evidence_status" +~~~ + +Captured output: + +~~~text +{"ok":true,"path":"/tmp/pr136-real-codex-worker.BQK070/flow.yaml","projectConfigPath":"/tmp/pr136-real-codex-worker.BQK070/flows.json","resolutions":[{"stepId":"review","cli":"codex","source":"step","model":"gpt-5.6-sol"}],"diagnostics":[]} +non_git_codex_preflight_exit=0 +{"cwd":"/tmp/pr136-real-codex-worker.BQK070","args":["exec","--ephemeral","--model","gpt-5.6-sol","Reply with exactly RELAYFLOWS_NON_GIT_READY."]} +{"status":1,"signal":null,"error":null,"stdout":"","stderr":"Reading additional input from stdin...\nNot inside a trusted directory and --skip-git-repo-check was not specified."} +non_git_codex_worker_invocation_exit=1 +preflight_green_worker_red_evidence_exit=0 +~~~ + +Control command: + +~~~sh +review_tmp=$(mktemp -d /tmp/pr136-real-codex-control.XXXXXX) +(cd "$review_tmp" && codex exec --ephemeral --skip-git-repo-check --model gpt-5.6-sol 'Reply with exactly RELAYFLOWS_NON_GIT_READY.') +review_status=$? +echo "real_codex_non_git_with_skip_exit=$review_status" +exit "$review_status" +~~~ + +Captured output: + +~~~text +Reading additional input from stdin... +OpenAI Codex v0.152.1 +-------- +workdir: /private/tmp/pr136-real-codex-control.j6so4N +model: gpt-5.6-sol +provider: openai +approval: never +sandbox: danger-full-access +reasoning effort: high +reasoning summaries: none +session id: 01a0632d-32ed-7401-ae2b-603bc07cfdaf +-------- +user +Reply with exactly RELAYFLOWS_NON_GIT_READY. +codex +RELAYFLOWS_NON_GIT_READY +tokens used +8,268 +RELAYFLOWS_NON_GIT_READY +real_codex_non_git_with_skip_exit=0 +~~~ + +Required repair: make the real Codex execution argv valid in the same directory +class accepted by preflight (for example, carry --skip-git-repo-check into +agentExecution), and add a real installed-Codex positive execution regression +from a non-Git temporary directory. The current exact-head opt-in suite checks +Codex auth and an impossible model, but never executes an available Codex model. + +## Exact review boundary and history + +Literal command: + +~~~sh +git rev-parse HEAD +git rev-parse main +git diff --name-only a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..4888d1572ed047c5161042614ac72068d047783a +~~~ + +Captured output (git status --short was empty at the start of review): + +~~~text +4888d1572ed047c5161042614ac72068d047783a +a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2 +docs/SURFACE.md +ops/reviews/20260902-1710-pr136-history.md +ops/reviews/20260902-1710-pr136-maintainability.md +ops/reviews/20260902-1710-pr136-structure.md +sdk/src/cli-adapter.ts +sdk/src/cli.ts +sdk/src/cli/check.ts +sdk/src/compile.ts +sdk/src/failure-kinds.ts +sdk/src/index.ts +sdk/src/model-name.ts +sdk/src/preflight.ts +sdk/src/spec.ts +sdk/src/unknown-keys.ts +sdk/src/validate.ts +sdk/src/worker.ts +sdk/tests/cli-adapter.test.ts +sdk/tests/cli.test.ts +sdk/tests/live-kernel.test.ts +sdk/tests/model-selection.test.ts +sdk/tests/preflight.test.ts +sdk/tests/real-cli-adapters.test.ts +testdata/flows.json +testdata/preflight/analyze-story-claude-cli +testdata/preflight/analyze-story-echo-wake-cli +testdata/preflight/analyze-story-missing-fields-cli +testdata/preflight/analyze-story-stub-cli +testdata/preflight/analyze-story-text-only-cli +testdata/preflight/authenticated-cli +testdata/preflight/counting-cli +testdata/preflight/echo-model-cli +testdata/preflight/signal-probe-cli +testdata/preflight/unauthenticated-cli +testdata/preflight/wake-context-probe-cli +~~~ + +Literal remote command: + +~~~sh +gh pr view 136 --json number,state,title,baseRefName,baseRefOid,headRefName,headRefOid,mergeable,statusCheckRollup,commits --jq '{number,state,title,baseRefName,baseRefOid,headRefName,headRefOid,mergeable,checks:[.statusCheckRollup[]|{name:.name,status:.status,conclusion:.conclusion}],commits:[.commits[]|{oid:.oid,messageHeadline:.messageHeadline}]}' +~~~ + +Captured output: + +~~~text +{"baseRefName":"main","baseRefOid":"a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2","checks":[{"conclusion":"SUCCESS","name":"linux-x64-artifact","status":"COMPLETED"},{"conclusion":null,"name":null,"status":null}],"commits":[{"messageHeadline":"feat(sdk): add declared agent model contract","oid":"321b27216e561561e1e022a7b4d973e2182ee480"},{"messageHeadline":"docs(review): record PR 136 fresh review","oid":"78efc0d15f32246332f9cf64ffba7f838573d02e"},{"messageHeadline":"fix(sdk): make model adapters fail closed","oid":"4888d1572ed047c5161042614ac72068d047783a"}],"headRefName":"feat/v2-declared-model","headRefOid":"4888d1572ed047c5161042614ac72068d047783a","mergeable":"MERGEABLE","number":136,"state":"OPEN","title":"feat(sdk): declare agent CLI and model with fail-closed checks"} +~~~ + +The head is exactly the assigned commit on the assigned base. The repair commit +touches SDK/docs/tests/test fixtures, not kernel/. The remote Linux artifact +check is green. + +## Historical blocker reproduction: declaration erasure is fixed + +I built commit 78efc0d15f32246332f9cf64ffba7f838573d02e (whose +product tree is the pre-repair implementation) in a temporary archive and ran +the same exercise later run at the assigned head. + +Literal old-head command: + +~~~sh +review_tmp=$(mktemp -d /tmp/flows-pr136-old-head.XXXXXX) +git archive 78efc0d15f32246332f9cf64ffba7f838573d02e | tar -x -C "$review_tmp" +ln -s /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk/node_modules "$review_tmp/sdk/node_modules" +cd "$review_tmp/sdk" +./node_modules/.bin/tsc && node scripts/make-cli-executable.mjs +echo "old_head_temp=$review_tmp" +echo "old_head_build_exit=$?" +node --input-type=module <<'BLOCKERS' +import { compileSpec, preflight, toKernelSpec } from './dist/index.js'; +const cases = [ + ['unused', { version: '0.1.0', agents: { reviewer: { cli: 'claude', model: 'typo-model' } }, steps: [{ id: 'noop', type: 'deterministic', command: 'true' }] }], + ['shadowed', { version: '0.1.0', agents: { reviewer: { cli: 'claude', model: 'typo-model' } }, steps: [{ id: 'review', type: 'agent', agent: 'reviewer', cli: 'claude', model: 'known-model', instruction: 'review' }] }], +]; +for (const [label, input] of cases) { + let probeCalls = 0; + const compiled = compileSpec(input); + const result = preflight(compiled, { models: ['known-model'], probes: { + cli: () => { probeCalls++; return { exists: true, authenticated: true, modelAvailable: true }; }, + executor: () => true, + command: () => true + } }); + const kernel = toKernelSpec(compiled); + console.log(JSON.stringify({ + label, + authoringAgent: compiled.agents?.reviewer ?? null, + selector: compiled.steps.find(step => step.type === 'agent')?.agent ?? null, + effectiveModel: compiled.steps.find(step => step.type === 'agent')?.model ?? null, + preflightOk: result.ok, + kinds: result.diagnostics.map(d => d.kind), + probeCalls, + kernelHasAgents: Object.hasOwn(kernel, 'agents'), + kernelHasSelector: kernel.steps.some(step => Object.hasOwn(step, 'agent')) + })); +} +BLOCKERS +echo "old_head_blocker_script_exit=$?" +~~~ + +Captured output: + +~~~text +old_head_temp=/tmp/flows-pr136-old-head.Pnk0pu +old_head_build_exit=0 +{"label":"unused","authoringAgent":null,"selector":null,"effectiveModel":null,"preflightOk":true,"kinds":["unprovable_effects"],"probeCalls":0,"kernelHasAgents":false,"kernelHasSelector":false} +{"label":"shadowed","authoringAgent":null,"selector":null,"effectiveModel":"known-model","preflightOk":true,"kinds":[],"probeCalls":1,"kernelHasAgents":false,"kernelHasSelector":false} +old_head_blocker_script_exit=0 +~~~ + +Literal exact-head command used the same BLOCKERS program after building the +assigned head. + +Captured output: + +~~~text +{"label":"unused","authoringAgent":{"cli":"claude","model":"typo-model"},"selector":null,"effectiveModel":null,"preflightOk":false,"kinds":["model_unknown"],"probeCalls":0,"kernelHasAgents":false,"kernelHasSelector":false} +{"label":"shadowed","authoringAgent":{"cli":"claude","model":"typo-model"},"selector":"reviewer","effectiveModel":"known-model","preflightOk":false,"kinds":["model_unknown"],"probeCalls":0,"kernelHasAgents":false,"kernelHasSelector":false} +exact_head_blocker_script_exit=0 +~~~ + +This closes the prior structure P1: bad declarations survive compile, fail +before all probes, and disappear only from the kernel dialect. + +The pre-repair raw-Claude probe false-positive also reproduces literally: + +~~~sh +chmod +x ./claude +node dist/cli.js check --json ./old-flow.yaml +review_status=$? +echo "old_head_raw_cli_check_exit=$review_status" +echo "old_head_raw_cli_argv:" +sed -n '1,20p' ./old-claude.log +exit "$review_status" +~~~ + +Captured output: + +~~~text +{"ok":true,"path":"./old-flow.yaml","projectConfigPath":"/private/tmp/flows-pr136-old-head.Pnk0pu/sdk/flows.json","resolutions":[{"stepId":"review","cli":"./claude","source":"step","model":"impossible-model"}],"diagnostics":[]} +old_head_raw_cli_check_exit=0 +old_head_raw_cli_argv: +auth status|MODEL_ENV=impossible-model +~~~ + +At exact head the blocker-focused deterministic suite now rejects this family +of false positives and wrong classifications: + +~~~sh +./node_modules/.bin/vitest run tests/cli.test.ts -t 'before compilation erases|raw Claude adapter|raw Codex adapter|nonconforming custom wrapper|Codex login status' --reporter=verbose --maxWorkers=1 --minWorkers=1 +review_status=$? +echo "focused_blockers_exit=$review_status" +exit "$review_status" +~~~ + +Captured output: + +~~~text + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + + ✓ tests/cli.test.ts > flows check CLI > refuses an unknown unused named-agent declaration before compilation erases it + ✓ tests/cli.test.ts > flows check CLI > refuses an unknown shadowed named-agent declaration before compilation erases it + ✓ tests/cli.test.ts > flows check CLI > uses the raw Claude adapter model flag instead of accepting auth status as model proof 308ms + ✓ tests/cli.test.ts > flows check CLI > uses Codex login status and reports a rejected model as unavailable, not unauthenticated 303ms + ✓ tests/cli.test.ts > flows check CLI > refuses a nonconforming custom wrapper without calling it an authentication failure + + Test Files 1 passed (1) + Tests 5 passed | 57 skipped (62) + Start at 19:13:55 + Duration 2.63s (transform 684ms, setup 0ms, collect 980ms, tests 791ms, environment 1ms, prepare 228ms) + +focused_blockers_exit=0 +~~~ + +## Valid v0.1.0 shapes, precedence, and lowering + +Literal exact-head command: + +~~~sh +node --input-type=module <<'EOF' +import { compileYaml, preflight, toKernelSpec } from './dist/index.js'; +const flow = compileYaml("version: '0.1.0'\ncli: flow-cli\nagents:\n reviewer: { cli: named-cli, model: named-model }\nsteps:\n - { id: named, type: agent, agent: reviewer, instruction: named }\n - { id: cli-override, type: agent, agent: reviewer, cli: step-cli, instruction: cli }\n - { id: model-override, type: agent, agent: reviewer, model: step-model, instruction: model }\n - { id: inline, type: agent, instruction: inline }\n"); +const result = preflight(flow, { + projectCli: 'project-cli', + models: ['named-model','step-model'], + probes: { + cli: () => ({exists:true,supported:true,authenticated:true,modelAvailable:true}), + executor:()=>true, + command:()=>true + } +}); +const pick = s => ({id:s.id,agent:s.agent??null,cli:s.cli??null,model:s.model??null}); +console.log(JSON.stringify({ + version:flow.version, + agents:flow.agents, + authoring:flow.steps.map(pick), + preflightOk:result.ok, + resolutions:result.resolutions, + kernel:toKernelSpec(flow).steps.map(pick), + kernelHasAgents:Object.hasOwn(toKernelSpec(flow),'agents') +})); +EOF +review_status=$? +echo "valid_shapes_and_lowering_exit=$review_status" +~~~ + +Captured output: + +~~~text +{"version":"0.1.0","agents":{"reviewer":{"cli":"named-cli","model":"named-model"}},"authoring":[{"id":"named","agent":"reviewer","cli":"named-cli","model":"named-model"},{"id":"cli-override","agent":"reviewer","cli":"step-cli","model":"named-model"},{"id":"model-override","agent":"reviewer","cli":"named-cli","model":"step-model"},{"id":"inline","agent":null,"cli":null,"model":null}],"preflightOk":true,"resolutions":[{"stepId":"named","cli":"named-cli","source":"step","model":"named-model"},{"stepId":"cli-override","cli":"step-cli","source":"step","model":"named-model"},{"stepId":"model-override","cli":"named-cli","source":"step","model":"step-model"},{"stepId":"inline","cli":"flow-cli","source":"flow"}],"kernel":[{"id":"named","agent":null,"cli":"named-cli","model":"named-model"},{"id":"cli-override","agent":null,"cli":"step-cli","model":"named-model"},{"id":"model-override","agent":null,"cli":"named-cli","model":"step-model"},{"id":"inline","agent":null,"cli":null,"model":null}],"kernelHasAgents":false} +valid_shapes_and_lowering_exit=0 +~~~ + +The precedence remains independent per field: step value, then named +declaration, with the established flow/project fallback applying only to CLI. +Inline model/no-model forms remain valid. The selected values reach the +existing kernel step fields; authoring map/selector do not. + +## v1/default compatibility and scope honesty + +This PR does not change kernel/, regressions/surface.d.ts, the schema version, +or a v1/default runtime path. That is an absence-of-change result, not a claim +that an external v1 repository was executed. The canonical inline +deterministic, LLM, and agent flows remain green. + +Literal command: + +~~~sh +for flow in ../testdata/hello-deterministic.flow.yaml ../testdata/hello-llm.flow.yaml ../testdata/hello-agent.flow.yaml; do + echo "COMMAND: node dist/cli.js check --json $flow" + node dist/cli.js check --json "$flow" + echo "exit=$?" +done +git diff --quiet a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..4888d1572ed047c5161042614ac72068d047783a -- ../kernel ../regressions/surface.d.ts +echo "v1_kernel_surface_diff_exit=$?" +~~~ + +Captured output: + +~~~text +COMMAND: node dist/cli.js check --json ../testdata/hello-deterministic.flow.yaml +WARNING [unprovable_effects] Step "greet" command "echo" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "shout" command "echo" resolves, but its effects cannot be proven before execution. +{"ok":true,"path":"../testdata/hello-deterministic.flow.yaml","projectConfigPath":"/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/testdata/flows.json","resolutions":[],"diagnostics":[{"severity":"warning","kind":"unprovable_effects","stepId":"greet","message":"Step \"greet\" command \"echo\" resolves, but its effects cannot be proven before execution."},{"severity":"warning","kind":"unprovable_effects","stepId":"shout","message":"Step \"shout\" command \"echo\" resolves, but its effects cannot be proven before execution."}]} +exit=0 +COMMAND: node dist/cli.js check --json ../testdata/hello-llm.flow.yaml +WARNING [unprovable_effects] Step "greet" command "printf" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "finish" command "printf" resolves, but its effects cannot be proven before execution. +{"ok":true,"path":"../testdata/hello-llm.flow.yaml","projectConfigPath":"/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/testdata/flows.json","resolutions":[{"stepId":"answer","cli":"./preflight/authenticated-cli","source":"project","model":"deterministic-test-stub"}],"diagnostics":[{"severity":"warning","kind":"unprovable_effects","stepId":"greet","message":"Step \"greet\" command \"printf\" resolves, but its effects cannot be proven before execution."},{"severity":"warning","kind":"unprovable_effects","stepId":"finish","message":"Step \"finish\" command \"printf\" resolves, but its effects cannot be proven before execution."}]} +exit=0 +COMMAND: node dist/cli.js check --json ../testdata/hello-agent.flow.yaml +WARNING [unprovable_effects] Step "greet" command "printf" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "finish" command "printf" resolves, but its effects cannot be proven before execution. +{"ok":true,"path":"../testdata/hello-agent.flow.yaml","projectConfigPath":"/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/testdata/flows.json","resolutions":[{"stepId":"edit","cli":"./preflight/authenticated-cli","source":"project","model":"test-model-v1"}],"diagnostics":[{"severity":"warning","kind":"unprovable_effects","stepId":"greet","message":"Step \"greet\" command \"printf\" resolves, but its effects cannot be proven before execution."},{"severity":"warning","kind":"unprovable_effects","stepId":"finish","message":"Step \"finish\" command \"printf\" resolves, but its effects cannot be proven before execution."}]} +exit=0 +v1_kernel_surface_diff_exit=0 +~~~ + +Issue #132 remains open. PR #136 and docs/SURFACE.md explicitly limit this +slice to canonical YAML/JSON and state that TypeScript FlowHeader.agents remains +a follow-on; no completion of that broader issue is claimed. + +## Real provider evidence + +Installed provider command/auth shapes: + +~~~sh +command -v claude +claude --version +claude auth status >/dev/null 2>&1 +echo "claude_auth_exit=$?" +claude auth status --help >/dev/null 2>&1 +echo "claude_identify_shape_exit=$?" +command -v codex +codex --version +codex login status >/dev/null 2>&1 +echo "codex_login_exit=$?" +codex login status --help >/dev/null 2>&1 +echo "codex_identify_shape_exit=$?" +~~~ + +Captured output: + +~~~text +/opt/homebrew/bin/claude +2.1.153 (Claude Code) +claude_auth_exit=0 +claude_identify_shape_exit=0 +/opt/homebrew/bin/codex +codex-cli 0.152.1 +codex_login_exit=0 +codex_identify_shape_exit=0 +~~~ + +Real Claude execution using the exact-head agentExecution mapping succeeds +from a non-Git directory: + +~~~text +{"cwd":"/tmp/pr136-real-claude-worker.E8rKcH","args":["-p","--model","claude-haiku-4-5-20251001","Reply with exactly RELAYFLOWS_NON_GIT_READY."]} +{"status":0,"signal":null,"error":null,"stdout":"RELAYFLOWS_NON_GIT_READY","stderr":""} +real_claude_non_git_execution_exit=0 +~~~ + +The exact-head opt-in provider suite was run from the pristine archive. One +Claude readiness attempt returned false during transient provider activity; +the same direct check then returned ok, and the immediate suite rerun passed +both provider tests. This is recorded rather than hidden. + +First run: + +~~~text + RUN v2.1.9 /private/tmp/flows-pr136-exact-head.KcqGSx/sdk + + × tests/real-cli-adapters.test.ts > installed raw CLI adapters > round-trips the exact declared Claude model and refuses an impossible one 10382ms + → expected false to be true // Object.is equality + ✓ tests/real-cli-adapters.test.ts > installed raw CLI adapters > uses Codex login status and classifies an impossible model as unavailable 16547ms + + Test Files 1 failed (1) + Tests 1 failed | 1 passed (2) +pristine_exact_head_real_adapters_exit=1 +~~~ + +Direct retry of the positive Claude check: + +~~~text +{ + "ok": true, + "path": "/tmp/pr136-real-claude-debug.kfTOQk/flow.yaml", + "projectConfigPath": "/tmp/pr136-real-claude-debug.kfTOQk/flows.json", + "resolutions": [ + { + "stepId": "review", + "cli": "claude", + "source": "step", + "model": "claude-haiku-4-5-20251001" + } + ], + "diagnostics": [] +} +debug_exit=0 +~~~ + +Immediate suite rerun: + +~~~text + RUN v2.1.9 /private/tmp/flows-pr136-exact-head.KcqGSx/sdk + + ✓ tests/real-cli-adapters.test.ts > installed raw CLI adapters > round-trips the exact declared Claude model and refuses an impossible one 15700ms + ✓ tests/real-cli-adapters.test.ts > installed raw CLI adapters > uses Codex login status and classifies an impossible model as unavailable 10014ms + + Test Files 1 passed (1) + Tests 2 passed (2) + Start at 19:29:30 + Duration 28.78s (transform 774ms, setup 0ms, collect 1.14s, tests 25.72s, environment 1ms, prepare 552ms) + +pristine_exact_head_real_adapters_rerun_exit=0 +~~~ + +These tests prove real Claude positive/negative readiness and truthful Codex +auth/impossible-model classification. They do not prove real positive Codex +execution; the finding above supplies that missing exercise and it is red. + +## Pristine exact-head complete suite + +Because other reviewers began changing the shared worktree during this +assessment, final verification used a pristine archive of the assigned commit: + +~~~sh +exact_tmp=$(mktemp -d /tmp/flows-pr136-exact-head.XXXXXX) +git archive 4888d1572ed047c5161042614ac72068d047783a | tar -x -C "$exact_tmp" +ln -s /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk/node_modules "$exact_tmp/sdk/node_modules" +cd "$exact_tmp/sdk" +./node_modules/.bin/tsc && node scripts/make-cli-executable.mjs +echo "exact_head_temp=$exact_tmp" +echo "exact_head_build_exit=$?" +git --git-dir=/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/.git diff --check a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..4888d1572ed047c5161042614ac72068d047783a +echo "exact_head_diff_check_exit=$?" +~~~ + +Captured output: + +~~~text +exact_head_temp=/tmp/flows-pr136-exact-head.KcqGSx +exact_head_build_exit=0 +exact_head_diff_check_exit=0 +~~~ + +Literal full-suite command: + +~~~sh +RELAYFLOWD_BIN=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 ./node_modules/.bin/vitest run --reporter=dot --maxWorkers=1 --minWorkers=1 +review_status=$? +echo "pristine_exact_head_full_suite_exit=$review_status" +exit "$review_status" +~~~ + +Captured output: + +~~~text + RUN v2.1.9 /private/tmp/flows-pr136-exact-head.KcqGSx/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd +LIVE_KERNEL flows=/private/tmp/flows-pr136-exact-head.KcqGSx/sdk/dist/cli.js + +stdout | tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +LIVE_ANALYZER ready: claude -p --model claude-haiku-4-5-20251001 round-trip OK + +stdout | tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +LIVE_ANALYZER analysis: {"reasoning":"This story is directly relevant to AI agents and automation as it describes an autonomous agent capable of creating and self-reviewing pull requests, demonstrating practical automation of core software development workflows.","relevance_score":9,"story_title":"Show HN: an agent that opens and reviews its own pull requests [wake-nonce-7f3a91c4]"} + +stdout | tests/live-kernel.test.ts > surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once +LIVE_KERNEL kill -9 pid=26694 run=01M1HJKHZ3YBNR8EBRM91GJ9AC while step=two state=Running + + ✓ tests/live-kernel.test.ts (19 tests) 60214ms + ✓ tests/cli.test.ts (62 tests) 3212ms + ✓ tests/journal-client.test.ts (13 tests) 162ms + ✓ tests/preflight.test.ts (17 tests) 33ms + ✓ tests/validate.test.ts (36 tests) 57ms + ✓ tests/cli-hn-monitor.test.ts (16 tests) 138ms + ✓ tests/backlog-picker.test.ts (14 tests) 359ms +SKIPPED_UNACTIONABLE=0 +SKIPPED_UNACTIONABLE=0 +NO_ACTIONABLE_BACKLOG_ENTRY scanned=0 +node:fs:539 + return binding.readFileUtf8(path, stringToFlags(options.flag)); + ^ + +Error: ENOENT: no such file or directory, open '.relayflow/backlog-picker-entry.json' + at Object.readFileSync (node:fs:539:20) + at [eval]:1:478 + at runScriptInThisContext (node:internal/vm:219:10) + at node:internal/process/execution:483:12 + at [eval]-wrapper:6:24 + at runScriptInContext (node:internal/process/execution:481:60) + at evalFunction (node:internal/process/execution:315:30) + at evalTypeScript (node:internal/process/execution:327:3) + at node:internal/main/eval_string:71:3 { + errno: -2, + code: 'ENOENT', + syscall: 'open', + path: '.relayflow/backlog-picker-entry.json' +} + +Node.js v26.7.0 +SKIPPED_UNACTIONABLE=0 + ✓ tests/backlog-picker-flow.test.ts (6 tests) 1555ms +SKIPPED_UNACTIONABLE=0 +NO_ACTIONABLE_BACKLOG_ENTRY scanned=1 Scoped but unverifiable[missing_definition_of_done] + ✓ tests/work-package-consumer.test.ts (13 tests) 663ms + ✓ tests/model-selection.test.ts (10 tests) 31ms + ✓ tests/deterministic-llm.test.ts (5 tests) 32ms + ✓ tests/bin.test.ts (7 tests) 2506ms + ✓ tests/hn-poller.test.ts (6 tests) 20ms + ✓ tests/dir-watcher-poller.test.ts (6 tests) 15ms + ✓ tests/hello-deterministic.test.ts (5 tests) 30ms + ✓ tests/work-package-validator.test.ts (7 tests) 9ms + ✓ tests/spec-parity.test.ts (15 tests) 263ms + ✓ tests/parse-json-output.test.ts (7 tests) 4ms + ✓ tests/cli-adapter.test.ts (3 tests) 5ms + ↓ tests/real-cli-adapters.test.ts (2 tests | 2 skipped) + + Test Files 19 passed | 1 skipped (20) + Tests 267 passed | 2 skipped (269) + Start at 19:26:56 + Duration 91.09s (transform 2.38s, setup 0ms, collect 6.66s, tests 69.31s, environment 7ms, prepare 3.91s) + +pristine_exact_head_full_suite_exit=0 +~~~ + +The ENOENT stack is output from an expected negative-path child; Vitest exited +zero. This full suite is green but does not cover successful real Codex worker +execution. + +## Environment limitation + +A fresh kernel build could not start because this machine's cargo mise shim is +broken. No kernel file differs in this PR, and the pristine SDK/live-kernel +suite above used the existing executable kernel. This is not represented as a +fresh Rust build. + +Literal command: + +~~~sh +sh ../ops/cargo.sh build +build_status=$? +echo "kernel_build_exit=$build_status" +exit "$build_status" +~~~ + +Captured output: + +~~~text +mise ERROR cargo is not a valid shim. This likely means you uninstalled a tool and the shim does not point to anything. Run `mise use ` to reinstall the tool. +mise ERROR Run with --verbose or MISE_VERBOSE=1 for more information +kernel_build_exit=1 +~~~ + +## Repair gate + +Do not merge exact head 4888d1572ed047c5161042614ac72068d047783a. +Add the missing non-Git execution support to the Codex worker adapter and pin a +positive installed-Codex execution from a non-Git directory. Re-run the +focused adapter tests, the opt-in real provider suite, and the complete SDK +suite from the repaired exact head. + +REVIEW_FAILED diff --git a/ops/reviews/20260902-1905-pr136-maintainability.md b/ops/reviews/20260902-1905-pr136-maintainability.md new file mode 100644 index 00000000..30b2b338 --- /dev/null +++ b/ops/reviews/20260902-1905-pr136-maintainability.md @@ -0,0 +1,387 @@ +# PR #136 fresh exact-head maintainability/adversarial review + +- **Exact head:** `4888d1572ed047c5161042614ac72068d047783a` +- **Merged main:** `a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2` +- **Lens:** typo/unknown-model ordering, auth-vs-model classification, wrapper identity, installed Claude/Codex argv, false-positive preflight, and load-bearing test quality. +- **Constitution read in full:** `AGENTS.md`, `docs/RFC-0001-everything-is-a-relayflow.md` +- **Mode:** independent assessment only; no product code, merge state, or release state was changed. + +## Verdict + +**REVIEW_FAILED.** The repair correctly introduces typed Claude/Codex adapters, fixes the old auth-vs-model misclassification, and passes both focused deterministic tests and opt-in tests against the installed providers. Two blocking fail-closed mismatches remain. First, only named declarations are globally scanned for unknown models; a typo in a later inline step allows earlier real provider probes to run before the flow is refused. Second, Codex readiness explicitly bypasses the Git-repository check while actual worker execution does not, so preflight can prove a command shape that the worker immediately rejects from a legal non-Git workspace. The wrapper execution path also does not carry or re-establish the identity proof on which its private environment contract depends. + +## Scope and history + +```text +$ git status --short --branch +$ git rev-parse HEAD +$ git merge-base a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2 4888d1572ed047c5161042614ac72068d047783a +$ git log --format='%H %s' --reverse a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..4888d1572ed047c5161042614ac72068d047783a +$ git diff --stat a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..4888d1572ed047c5161042614ac72068d047783a | tail -5 +## feat/v2-declared-model...origin/feat/v2-declared-model +4888d1572ed047c5161042614ac72068d047783a +a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2 +321b27216e561561e1e022a7b4d973e2182ee480 feat(sdk): add declared agent model contract +78efc0d15f32246332f9cf64ffba7f838573d02e docs(review): record PR 136 fresh review +4888d1572ed047c5161042614ac72068d047783a fix(sdk): make model adapters fail closed + testdata/preflight/echo-model-cli | 4 + + testdata/preflight/signal-probe-cli | 4 + + testdata/preflight/unauthenticated-cli | 4 + + testdata/preflight/wake-context-probe-cli | 4 + + 34 files changed, 2308 insertions(+), 106 deletions(-) +``` + +## F1 — P1: a later inline model typo permits earlier provider calls + +The repair pre-scans every named declaration and returns before probes, including unused and shadowed declarations. Inline `step.model` values are still checked inside the same loop that probes each step. Therefore the result depends on author order: a valid first step performs its model round trip before an unknown inline model on a later step is discovered. + +```text +$ nl -ba sdk/src/preflight.ts | sed -n '103,126p;145,161p' | sed -E '/^[[:space:]]*[0-9]+[[:space:]]*$/d' + 103 // Named declarations remain in the normalized authoring object until this + 104 // boundary so even unused or step-shadowed models are checked. Return before + 105 // any environment probe; toKernelSpec erases the map and selector only after + 106 // this authoring preflight has had the chance to fail closed. + 107 for (const [agent, declaration] of Object.entries(flow.agents ?? {})) { + 108 if (isKnownModel(declaration.model, options.models)) continue; + 109 diagnostics.push({ + 110 severity: 'refusal', + 111 kind: 'model_unknown', + 112 agent, + 113 cli: declaration.cli, + 114 model: declaration.model, + 115 message: unknownNamedAgentModelMessage(agent, declaration.cli, declaration.model, options.modelRegistryPath), + 116 }); + 117 } + 118 if (diagnostics.length > 0) return { ok: false, resolutions, diagnostics }; + 120 for (const step of flow.steps) { + 121 warnOnUnprovableEffects(step, options.probes, diagnostics); + 122 if (step.type === 'deterministic') continue; + 124 const declaredModel = step.model; + 125 const modelUnknown = declaredModel !== undefined && !isKnownModel(declaredModel, options.models); + 126 const resolution = resolveCli(step, flow, options.projectCli); + 145 resolutions.push(resolution); + 146 if (modelUnknown && resolution.model !== undefined) { + 147 diagnostics.push({ + 148 severity: 'refusal', + 149 kind: 'model_unknown', + 150 stepId: resolution.stepId, + 151 cli: resolution.cli, + 152 model: resolution.model, + 153 message: unknownModelMessage( + 154 resolution.stepId, + 155 resolution.model, + 156 resolution.cli, + 157 options.modelRegistryPath, + 158 ), + 159 }); + 160 continue; + 161 } +``` + +Independent reproduction. The `calls` array proves the first CLI/model was probed even though the second step makes the spec deterministically invalid: + +```text +$ node --input-type=module -e 'import { preflight } from "./dist/preflight.js"; const calls=[]; const result=preflight({version:"0.1.0",steps:[{id:"first",type:"agent",cli:"claude",model:"known-model",instruction:"First"},{id:"typo",type:"agent",cli:"claude",model:"known-modle",instruction:"Second"}]},{models:["known-model"],modelRegistryPath:"/project/flows.json",probes:{cli(...args){calls.push(args);return {exists:true,supported:true,authenticated:true,modelAvailable:true}},executor(){return true},command(){return true}}}); console.log(JSON.stringify({calls,result},null,2));' +{ + "calls": [ + [ + "claude", + "step", + "known-model" + ] + ], + "result": { + "ok": false, + "resolutions": [ + { + "stepId": "first", + "cli": "claude", + "source": "step", + "model": "known-model" + }, + { + "stepId": "typo", + "cli": "claude", + "source": "step", + "model": "known-modle" + } + ], + "diagnostics": [ + { + "severity": "refusal", + "kind": "model_unknown", + "stepId": "typo", + "cli": "claude", + "model": "known-modle", + "message": "Step \"typo\" declares model \"known-modle\" for CLI \"claude\", but it is not listed in project model registry \"/project/flows.json\"; add the exact model only after verifying that project is allowed to use it." + } + ] + } +} +``` + +For raw providers, that probe is a real model call rather than a side-effect-free registry lookup. This violates the PR's outcome that unknown models refuse before any CLI call and makes a deterministic typo capable of spending time/tokens before refusal. + +Required repair evidence: collect all named and inline model declarations, validate their exact allowlist membership in one pure first pass, and return all `model_unknown` diagnostics before command, CLI, executor, or daemon probes. Add a two-step regression whose first valid model probe throws if called and whose second inline model is unknown; assert zero probe calls in both step orders. + +## F2 — P1: Codex readiness and worker argv disagree on the Git trust prerequisite + +The readiness probe uses `--skip-git-repo-check`; the actual worker invocation omits it. `AgentWorker` also supplies no `cwd`, so the child inherits the worker process's directory. Agent steps are legal with relayfile surfaces or no worktree declaration, so a Git repository is not a stated execution prerequisite. + +```text +$ nl -ba sdk/src/cli-adapter.ts | sed -n '64,70p;92,100p' + 64 if (kind === 'codex') { + 65 return { + 66 args: [ + 67 'exec', '--ephemeral', '--sandbox', 'read-only', '--skip-git-repo-check', + 68 '--model', model, MODEL_PROBE_PROMPT, + 69 ], + 70 timeoutMs: 60_000, + 92 if (kind === 'codex') { + 93 return { + 94 args: [ + 95 'exec', '--ephemeral', + 96 ...(model === undefined ? [] : ['--model', model]), + 97 instruction, + 98 ], + 99 timeoutMs: 0, + 100 }; +``` + +```text +$ nl -ba sdk/src/worker.ts | sed -n '190,193p;238,240p' + 190 return new Promise((resolve) => { + 191 const env: NodeJS.ProcessEnv = { ...process.env }; + 192 const invocation = agentExecution(cliAdapterKind(cli), instruction, model); + 193 // Explicit unset. Without this, a parent process (wrapper + 238 } + 239 const child = spawn(cli, invocation.args, { stdio: ['ignore', 'pipe', 'pipe'], env }); + 240 const stdout: Buffer[] = []; +``` + +The installed Codex CLI accepts the readiness argv from this exact head and reaches model validation outside a repository: + +```text +$ codex exec --ephemeral --sandbox read-only --skip-git-repo-check --model relayflows-definitely-not-a-real-codex-model 'Reply with exactly RELAYFLOWS_MODEL_READY and nothing else.'; codex_status=$?; printf 'codex_exit=%s\n' "$codex_status"; exit 0 +Reading additional input from stdin... +OpenAI Codex v0.152.1 +-------- +workdir: /Users/khaliqgant/AgentWorkforce/flows-132-model-wt +model: relayflows-definitely-not-a-real-codex-model +provider: openai +approval: never +sandbox: read-only +reasoning effort: high +reasoning summaries: none +session id: 01a0631b-0159-7fe3-8fbd-6ef1c3fcafd3 +-------- +user +Reply with exactly RELAYFLOWS_MODEL_READY and nothing else. +warning: Model metadata for `relayflows-definitely-not-a-real-codex-model` not found. Defaulting to fallback metadata; this can degrade performance and cause issues. +ERROR: {"type":"error","status":400,"error":{"type":"invalid_request_error","message":"The 'relayflows-definitely-not-a-real-codex-model' model is not supported when using Codex with a ChatGPT account."}} +ERROR: {"type":"error","status":400,"error":{"type":"invalid_request_error","message":"The 'relayflows-definitely-not-a-real-codex-model' model is not supported when using Codex with a ChatGPT account."}} +codex_exit=1 +``` + +The actual worker argv from a non-Git directory fails before attempting the declared model. This command was run with `/tmp` as its working directory: + +```text +$ codex exec --ephemeral --model relayflows-definitely-not-a-real-codex-model 'Reply with exactly RELAYFLOWS_MODEL_READY and nothing else.'; codex_status=$?; printf 'codex_worker_argv_exit=%s\n' "$codex_status"; exit 0 +Reading additional input from stdin... +Not inside a trusted directory and --skip-git-repo-check was not specified. +codex_worker_argv_exit=1 +``` + +This is a preflight false positive for runnability, independent of model existence: with an accessible allowlisted model, the readiness call can exit zero because it bypasses the check, while execution still exits one solely because the worker cwd is not a Git repository. + +Required repair evidence: make readiness and execution share all runnability-relevant Codex flags and cwd assumptions. Add an end-to-end test using the installed Codex CLI from a non-Git directory with a known accessible model, or make a Git worktree a validated declared prerequisite and refuse before the provider call. The deterministic fake CLI test must assert the complete argv, not only the prefix through `--model`. + +## F3 — P2: wrapper identity is checked during `flows check` but is not an execution invariant + +`cliAdapterKind` labels every executable not literally named `claude` or `codex` as `relayflows-wrapper-v1`. Identification and execution are separate functions; `agentExecution` receives only the enum, not an identity proof. Consequently every custom executable gets `RELAYFLOW_MODEL` at worker execution even when the current executable never returned the required token. + +```text +$ node --input-type=module -e 'import { adapterIdentification, agentExecution, cliAdapterKind } from "./dist/cli-adapter.js"; for (const cli of ["/tmp/not-an-adapter","/tmp/team-reviewer","/opt/homebrew/bin/claude","/opt/homebrew/bin/codex"]) { const kind=cliAdapterKind(cli); console.log(JSON.stringify({cli,kind,identify:adapterIdentification(kind),execute:agentExecution(kind,"Review.","declared-model")})); }' +{"cli":"/tmp/not-an-adapter","kind":"relayflows-wrapper-v1","identify":{"invocation":{"args":["--relayflows-adapter-v1"],"timeoutMs":10000},"expectedStdout":"relayflows-agent-cli-v1"},"execute":{"args":["Review."],"timeoutMs":0,"modelEnv":"declared-model"}} +{"cli":"/tmp/team-reviewer","kind":"relayflows-wrapper-v1","identify":{"invocation":{"args":["--relayflows-adapter-v1"],"timeoutMs":10000},"expectedStdout":"relayflows-agent-cli-v1"},"execute":{"args":["Review."],"timeoutMs":0,"modelEnv":"declared-model"}} +{"cli":"/opt/homebrew/bin/claude","kind":"claude","identify":{"invocation":{"args":["auth","status","--help"],"timeoutMs":10000}},"execute":{"args":["-p","--model","declared-model","Review."],"timeoutMs":0}} +{"cli":"/opt/homebrew/bin/codex","kind":"codex","identify":{"invocation":{"args":["login","status","--help"],"timeoutMs":10000}},"execute":{"args":["exec","--ephemeral","--model","declared-model","Review."],"timeoutMs":0}} +``` + +The live test named “identified wrapper” bypasses `checkFlow` and submits the kernel spec directly. Nothing in lines 691–747 invokes `--relayflows-adapter-v1`; the worker trusts the basename classification. The test therefore cannot fail if execution loses its identity prerequisite: + +```text +$ nl -ba sdk/tests/live-kernel.test.ts | sed -n '691,729p' | sed -E '/^[[:space:]]*[0-9]+[[:space:]]*$/d' + 691 it('AgentWorker passes a declared model to an identified wrapper as RELAYFLOW_MODEL', async () => { + 692 // The whole point of declaring `model` on the step is that the CLI + 693 // stops inheriting whatever the host pinned. This proves the declared + 694 // value survives the full boundary: SDK compile → kernel parse → + 695 // dispatch → AgentWorker → subprocess env. + 696 const dataDir = temporaryDirectory('flows-live-model-set-'); + 697 await startDaemon(dataDir); + 698 const cli = join(TESTDATA, 'preflight', 'echo-model-cli'); + 699 const client = await connectClient(dataDir); + 700 await client.hello('live-model-set'); + 701 const worker = new AgentWorker(client, { + 702 workerId: 'live-model-set-worker', + 703 pins: { workspace: [{ surface: 'repo', revision_id: 'rev-a' }], streams: [] }, + 704 }); + 705 await worker.attach(); + 707 const compiled = compileYaml(` + 708 version: '0.1.0' + 709 agents: + 710 model-probe: + 711 cli: ${JSON.stringify(cli)} + 712 model: declared-model-xyz + 713 steps: + 714 - id: probe + 715 type: agent + 716 agent: model-probe + 717 instruction: Report the model env var. + 718 `); + 719 expect(compiled).toHaveProperty('agents.model-probe.model', 'declared-model-xyz'); + 720 expect(compiled.steps[0]).toMatchObject({ + 721 type: 'agent', + 722 cli, + 723 model: 'declared-model-xyz', + 724 }); + 725 const kernel = toKernelSpec(compiled); + 726 expect(kernel).not.toHaveProperty('agents'); + 727 expect(kernel.steps[0]).not.toHaveProperty('agent'); + 728 const started = await client.runStart(kernel); + 729 +``` + +Normal `flows run` performs preflight before submission, so this is not a claim that its unchanged executable always bypasses identification. The gap is that the proof is neither journaled nor bound to the executable that a potentially remote/later worker resolves; direct journal-protocol submission and executable replacement between preflight and dispatch both bypass it. That contradicts the worker comment that only an explicitly identified wrapper receives the private environment contract. + +Required repair evidence: make adapter identity part of the immutable compiled/runtime contract, or re-identify custom wrappers in `AgentWorker` before sending `RELAYFLOW_MODEL`. Add a negative journal-to-worker test with a nonconforming custom executable and assert it completes `worker_error` without running the instruction or receiving the private model variable. + +## Repaired behavior that passes + +The installed binaries expose the selected adapter shapes and both are authenticated: + +```text +$ command -v claude; command -v codex; claude --version; codex --version +/opt/homebrew/bin/claude +/opt/homebrew/bin/codex +2.1.153 (Claude Code) +codex-cli 0.152.1 +$ claude auth status >/dev/null 2>&1; printf 'claude_auth_exit=%s\n' "$?"; codex login status >/dev/null 2>&1; printf 'codex_auth_exit=%s\n' "$?" +claude_auth_exit=0 +codex_auth_exit=0 +``` + +The repaired preflight taxonomy distinguishes unsupported adapter, authentication failure, inaccessible model, and ready state: + +```text +$ node --input-type=module -e 'import { preflight } from "./dist/preflight.js"; const flow={version:"0.1.0",steps:[{id:"review",type:"agent",cli:"tool",model:"known",instruction:"Review"}]}; for (const [label,answer] of [["unsupported",{exists:true,supported:false,authenticated:false}],["unauthenticated",{exists:true,supported:true,authenticated:false,modelAvailable:false,authCommand:"tool login status"}],["model-unavailable",{exists:true,supported:true,authenticated:true,modelAvailable:false,modelCommand:"tool exec --model known"}],["ready",{exists:true,supported:true,authenticated:true,modelAvailable:true}]]) { const result=preflight(flow,{models:["known"],probes:{cli(){return answer},executor(){return true},command(){return true}}}); console.log(label, JSON.stringify({ok:result.ok,kinds:result.diagnostics.map(d=>d.kind)})); }' +unsupported {"ok":false,"kinds":["cli_unsupported"]} +unauthenticated {"ok":false,"kinds":["cli_unauthenticated"]} +model-unavailable {"ok":false,"kinds":["model_unavailable"]} +ready {"ok":true,"kinds":[]} +``` + +An exact model typo is refused with a useful path, value, registry path, and security-conscious remediation before that declaration's own probe. It deliberately does not auto-suggest changing to a nearby allowed model, which is reasonable for an authorization allowlist: + +```text +$ node --input-type=module -e 'import { preflight } from "./dist/preflight.js"; const result=preflight({version:"0.1.0",agents:{reviewer:{cli:"claude",model:"claude-sonnet-4-5"}},steps:[{id:"review",type:"agent",agent:"reviewer",instruction:"Review",cli:"claude",model:"claude-sonnet-4-5"}]},{models:["claude-sonnet-4-6"],modelRegistryPath:"/project/flows.json",probes:{cli(){throw new Error("PROBE_CALLED")},executor(){throw new Error("PROBE_CALLED")},command(){throw new Error("PROBE_CALLED")}}}); console.log(JSON.stringify(result,null,2));' +{ + "ok": false, + "resolutions": [], + "diagnostics": [ + { + "severity": "refusal", + "kind": "model_unknown", + "agent": "reviewer", + "cli": "claude", + "model": "claude-sonnet-4-5", + "message": "Named agent \"reviewer\" declares model \"claude-sonnet-4-5\" for CLI \"claude\", but it is not listed in project model registry \"/project/flows.json\"; add the exact model only after verifying that project is allowed to use it." + } + ] +} +``` + +## Verification evidence + +Focused TypeScript and deterministic adapter/model/CLI suites: + +```text +$ ./node_modules/.bin/tsc --noEmit && ./node_modules/.bin/vitest run tests/cli-adapter.test.ts tests/model-selection.test.ts tests/validate.test.ts tests/spec-parity.test.ts tests/preflight.test.ts tests/cli.test.ts tests/bin.test.ts --reporter=dot --maxWorkers=1 --minWorkers=1; review_status=$?; echo "exit_code=$review_status"; exit "$review_status" + + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + + ✓ tests/cli.test.ts (62 tests) 2997ms + ✓ flows check CLI > uses the raw Claude adapter model flag instead of accepting auth status as model proof 347ms + ✓ tests/preflight.test.ts (17 tests) 26ms + ✓ tests/validate.test.ts (36 tests) 47ms + ✓ tests/model-selection.test.ts (10 tests) 74ms + ✓ tests/bin.test.ts (7 tests) 2126ms + ✓ built flows binary > classifies a signal-terminated auth probe as probe_failed 389ms + ✓ built flows binary > does not describe a present non-executable CLI as missing 479ms + ✓ built flows binary > runs one auth probe for three steps sharing a flow CLI 553ms + ✓ tests/spec-parity.test.ts (15 tests) 131ms + ✓ tests/cli-adapter.test.ts (3 tests) 11ms + + Test Files 7 passed (7) + Tests 150 passed (150) + Start at 19:13:36 + Duration 10.66s (transform 476ms, setup 0ms, collect 1.15s, tests 5.41s, environment 3ms, prepare 1.08s) + +exit_code=0 +``` + +Opt-in real-provider preflight suite against the installed CLIs: + +```text +$ RELAYFLOWS_REAL_CLI_ADAPTERS=1 ./node_modules/.bin/vitest run tests/real-cli-adapters.test.ts --reporter=verbose --maxWorkers=1 --minWorkers=1; review_status=$?; echo "exit_code=$review_status"; exit "$review_status" + + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + + ✓ tests/real-cli-adapters.test.ts > installed raw CLI adapters > round-trips the exact declared Claude model and refuses an impossible one 22299ms + ✓ tests/real-cli-adapters.test.ts > installed raw CLI adapters > uses Codex login status and classifies an impossible model as unavailable 10792ms + + Test Files 1 passed (1) + Tests 2 passed (2) + Start at 19:10:42 + Duration 35.94s (transform 1.09s, setup 0ms, collect 1.72s, tests 33.10s, environment 0ms, prepare 455ms) + +exit_code=0 +``` + +Focused live journal/worker tests against the disclosed existing relayflowd binary: + +```text +$ test -x /Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd && echo relayflowd_fixture=executable +$ RELAYFLOWD_BIN=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 ./node_modules/.bin/vitest run tests/live-kernel.test.ts -t 'AgentWorker (passes a declared model to an identified wrapper|executes the raw)' --reporter=verbose --maxWorkers=1 --minWorkers=1; review_status=$?; echo "exit_code=$review_status"; exit "$review_status" +relayflowd_fixture=executable + + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd +LIVE_KERNEL flows=/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk/dist/cli.js + + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker passes a declared model to an identified wrapper as RELAYFLOW_MODEL 932ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker executes the raw claude adapter with its real model flag + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker executes the raw codex adapter with its real model flag + + Test Files 1 passed (1) + Tests 3 passed | 16 skipped (19) + Start at 19:14:47 + Duration 5.84s (transform 1.29s, setup 0ms, collect 1.81s, tests 1.46s, environment 8ms, prepare 532ms) + +exit_code=0 +``` + +Those green tests pin the repaired happy paths but miss F1's cross-step ordering, F2's real worker cwd, and F3's negative wrapper execution. I did not mutate product code, so I make no mutation-verification claim. I also do not claim a fresh full SDK suite; the focused and real-provider commands above are the complete verification claim for this review. + +The exact commit diff is whitespace-clean: + +```text +$ git diff --check a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..4888d1572ed047c5161042614ac72068d047783a; review_status=$?; echo "exit_code=$review_status"; exit "$review_status" +exit_code=0 +``` + +No merge, release, or self-removal was performed. + +REVIEW_FAILED diff --git a/ops/reviews/20260902-1905-pr136-structure.md b/ops/reviews/20260902-1905-pr136-structure.md new file mode 100644 index 00000000..2d9d7728 --- /dev/null +++ b/ops/reviews/20260902-1905-pr136-structure.md @@ -0,0 +1,294 @@ +# PR #136 exact-head structure/security review + +Date: 2026-09-02 + +Reviewer: independent Codex structure/security reviewer + +Scope: assessment only of `AgentWorkforce/flows` PR #136 at exact head +`4888d1572ed047c5161042614ac72068d047783a` against merged `main` +`a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2`. The requested lenses were typed +adapter boundaries, environment scrubbing, allowlist enforcement for unused and +shadowed declarations, kernel erasure at the correct boundary, and fail-closed +diagnostics. I read `AGENTS.md` and +`docs/RFC-0001-everything-is-a-relayflow.md` in full before assessing the diff +and history. I did not edit product code. + +## Verdict + +No blocking finding. The repair commit closes the previously reported erasure +and adapter-contract gaps without moving provider/model policy into the kernel. + +## Findings + +None. + +## Architecture and security assessment + +- The adapter vocabulary is closed and typed at the SDK surface: + `claude | codex | relayflows-wrapper-v1`. Identification, authentication, + model-readiness, and execution argv are centralized in + `sdk/src/cli-adapter.ts`; both preflight and the worker consume that table. + Raw Claude/Codex adapters carry the declared model in provider-native argv. + A custom executable must identify as the versioned wrapper contract before + it may receive `RELAYFLOW_MODEL`. +- Both probe and worker environments begin from a copy of the parent environment + and delete `RELAYFLOW_MODEL`. It is reintroduced only for a successfully + identified wrapper invocation with a declared model. Raw provider adapters + receive no model environment variable, including when the parent process is + polluted. +- Compilation retains the named-agent map and per-step selector in the SDK + authoring shape. Preflight checks every named declaration, including unused + declarations and declarations whose model is shadowed by a step override, + against the exact case-sensitive project allowlist before any executable + probe. An invalid declaration therefore produces typed `model_unknown` + diagnostics instead of being erased or causing provider work. +- `toKernelSpec` is the erasure boundary: it resolves the selected named agent + into the existing per-step `cli`/`model` fields and does not emit the authoring + map or selector. `flows run` performs `checkFlow` before connecting and before + calling this lowering function. The PR has no diff under `kernel/` or + `sdk/src/protocol.ts`; the kernel remains unaware of provider registries and + named authoring declarations, consistent with RFC-0001. +- Unsupported custom-wrapper identification fails as typed `cli_unsupported`; + auth and model-readiness probes retain distinct typed diagnostics. Diagnostic + commands are rendered from the same invocation definitions used for execution, + so the displayed remediation no longer claims a generic, false provider + command. + +## Exact revision and boundary evidence + +Command (repository root): + +```text +git status --short --branch --untracked-files=no +git rev-parse HEAD +git merge-base origin/main HEAD +git log --format='%H %s' origin/main..HEAD +git diff --name-only origin/main...HEAD -- kernel sdk/src/protocol.ts +gh pr view 136 --json number,state,baseRefOid,headRefOid,headRefName,title +``` + +Captured output: + +```text +## feat/v2-declared-model...origin/feat/v2-declared-model +4888d1572ed047c5161042614ac72068d047783a +a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2 +4888d1572ed047c5161042614ac72068d047783a fix(sdk): make model adapters fail closed +78efc0d15f32246332f9cf64ffba7f838573d02e docs(review): record PR 136 fresh review +321b27216e561561e1e022a7b4d973e2182ee480 feat(sdk): add declared agent model contract +{"baseRefOid":"a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2","headRefName":"feat/v2-declared-model","headRefOid":"4888d1572ed047c5161042614ac72068d047783a","number":136,"state":"OPEN","title":"feat(sdk): declare agent CLI and model with fail-closed checks"} +exit_code=0 +``` + +The blank position between the commit list and JSON is the literal empty output +from the path-restricted diff, i.e. no changes in `kernel/` or +`sdk/src/protocol.ts`. + +Command (repository root): + +```text +rg -n "export type CliAdapterKind|function cliAdapterKind|RELAYFLOW_MODEL|flow\.agents|model_unknown|toKernelSpec|agentExecution" sdk/src/cli-adapter.ts sdk/src/cli/check.ts sdk/src/compile.ts sdk/src/preflight.ts sdk/src/worker.ts +rg -n -i "unused|shadowed" sdk/tests/preflight.test.ts sdk/tests/cli.test.ts sdk/tests/model-selection.test.ts +``` + +Captured output: + +```text +sdk/src/worker.ts:6:import { agentExecution, cliAdapterKind } from './cli-adapter.js'; +sdk/src/worker.ts:182:export const MODEL_ENV = 'RELAYFLOW_MODEL'; +sdk/src/worker.ts:192: const invocation = agentExecution(cliAdapterKind(cli), instruction, model); +sdk/src/worker.ts:205: // RELAYFLOW_MODEL from a parent process would make a step that declared +sdk/src/cli-adapter.ts:3:export type CliAdapterKind = 'claude' | 'codex' | 'relayflows-wrapper-v1'; +sdk/src/cli-adapter.ts:23:export function cliAdapterKind(executable: string): CliAdapterKind { +sdk/src/cli-adapter.ts:81:export function agentExecution( +sdk/src/cli-adapter.ts:113: : `RELAYFLOW_MODEL=${shellDisplayWord(invocation.modelEnv)} ${command}`; +sdk/src/preflight.ts:105: // any environment probe; toKernelSpec erases the map and selector only after +sdk/src/preflight.ts:107: for (const [agent, declaration] of Object.entries(flow.agents ?? {})) { +sdk/src/preflight.ts:111: kind: 'model_unknown', +sdk/src/preflight.ts:137: kind: 'model_unknown', +sdk/src/preflight.ts:149: kind: 'model_unknown', +sdk/src/compile.ts:7:// toKernelSpec — authoring shape -> the ONE boundary dialect +sdk/src/compile.ts:59: return canonicalize(toKernelSpec(compileYaml(yaml))); +sdk/src/compile.ts:176:export function toKernelSpec(flow: FlowSpec): KernelRunSpec { +sdk/src/compile.ts:183: steps: flow.steps.map((step) => toKernelStep(resolveNamedAgent(step, flow.agents))), +sdk/src/compile.ts:198: * This is the inverse of `toKernelSpec` over specs this compiler emits. +sdk/src/compile.ts:435: const kernelSpec = toKernelSpec(spec); +sdk/tests/preflight.test.ts:258: it.each(['unused', 'shadowed'] as const)( +sdk/tests/preflight.test.ts:265: steps: variant === 'unused' +sdk/tests/cli.test.ts:145: it.each(['unused', 'shadowed'] as const)( +sdk/tests/cli.test.ts:152: const steps = variant === 'unused' +sdk/tests/cli.test.ts:559: const path = join(flowDirectory, 'shadowed.flow.yaml'); +sdk/tests/cli.test.ts:565: expect(result.stderr.join('\n')).toContain('outer configs are shadowed'); +exit_code=0 +``` + +## Independent verification + +Command (working directory `sdk/`): + +```text +./node_modules/.bin/tsc --noEmit && ./node_modules/.bin/vitest run tests/cli-adapter.test.ts tests/model-selection.test.ts tests/preflight.test.ts tests/cli.test.ts tests/validate.test.ts tests/spec-parity.test.ts --reporter=dot && git diff --check +``` + +Captured output (`tsc` and `git diff --check` were silent; the chained command +exited 0): + +```text + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + + ✓ tests/cli-adapter.test.ts (3 tests) 103ms + ✓ tests/model-selection.test.ts (10 tests) 1322ms + ✓ named agent declarations > lowers a selected agent CLI and model into the existing agent step 690ms + ✓ tests/preflight.test.ts (17 tests) 826ms + ✓ tests/validate.test.ts (36 tests) 1569ms + ✓ compile: surfaces validation failures as CompileError > throws CompileError with the offending messages for a malformed YAML spec 535ms + ✓ tests/spec-parity.test.ts (15 tests) 2618ms + ✓ spec parity: one dialect at the SDK<->kernel boundary > compiles hello-deterministic to the pinned canonical JSON 696ms + ✓ spec parity: one dialect at the SDK<->kernel boundary > compiles hello-agent to the pinned canonical JSON 312ms + ✓ spec parity: one dialect at the SDK<->kernel boundary > normalizes empty triggers exactly as kernel serialization does 463ms + ✓ tests/cli.test.ts (62 tests) 22017ms + ✓ flows check CLI > refuses an unknown unused named-agent declaration before compilation erases it 548ms + ✓ flows check CLI > uses the raw Claude adapter model flag instead of accepting auth status as model proof 805ms + ✓ flows check CLI > uses Codex login status and reports a rejected model as unavailable, not unauthenticated 510ms + ✓ flows check CLI > checks the same named-agent contract from declarative JSON 350ms + ✓ flows check CLI > distinguishes an allowlisted but inaccessible model from broken auth 306ms + ✓ flows check CLI > passes all three canonical ladder flows and prints their resolved CLI 1242ms + ✓ flows check CLI > passes relocated ladder flow hello-deterministic when no fault is induced 532ms + ✓ flows check CLI > passes relocated ladder flow hello-llm when no fault is induced 442ms + ✓ flows check CLI > passes relocated ladder flow hello-agent when no fault is induced 428ms + ✓ flows check CLI > refuses ladder flow hello-deterministic with no_executor under an induced fault 371ms + ✓ flows check CLI > refuses ladder flow hello-llm with cli_missing under an induced fault 354ms + ✓ flows check CLI > refuses ladder flow hello-llm with cli_unauthenticated under an induced fault 631ms + ✓ flows check CLI > refuses ladder flow hello-llm with no_executor under an induced fault 608ms + ✓ flows check CLI > refuses ladder flow hello-agent with cli_unauthenticated under an induced fault 492ms + ✓ flows check CLI > refuses ladder flow hello-agent with cli_unresolved under an induced fault 389ms + ✓ flows check CLI > refuses ladder flow hello-agent with no_executor under an induced fault 458ms + ✓ flows check CLI > accepts cli-declared.flow.yaml without refusing 410ms + ✓ flows check CLI > checks the compiled kernel-dialect canonical spec as well as YAML 466ms + ✓ flows check CLI > recognizes kernel dialect from retry alone 385ms + ✓ flows check CLI > recognizes kernel dialect from recovery_mode alone 390ms + ✓ flows check CLI > recognizes kernel dialect from verification.output_contains alone 335ms + ✓ flows check CLI > recognizes kernel dialect from permissions.file_globs alone 487ms + ✓ flows check CLI > loads the final CLI resolution source from the nearest flows.json 368ms + ✓ flows check CLI > resolves a project CLI path relative to the flows.json that declares it 485ms + ✓ flows check CLI > maps every input refusal path to its declared kind without raw exceptions 309ms + ✓ flows run/resume CLI over the journal protocol > parses run options, submits the kernel dialect, and exits 0 on success 995ms + ✓ flows run/resume CLI over the journal protocol > exits 1 and emits the declared completionReason for a failed run 581ms + ✓ flows run/resume CLI over the journal protocol > exits 3 and names the parked llm step 684ms + ✓ flows run/resume CLI over the journal protocol > reports a needs_human agent step as parked for human recovery 778ms + ✓ flows run/resume CLI over the journal protocol > classifies a typed hello refusal as a protocol error, not an unreachable daemon 733ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 791ms + ✓ flows run/resume CLI over the journal protocol > bounds a worker wait by its lease and reports what it is waiting for 536ms + ✓ flows run/resume CLI over the journal protocol > allows a caller to cancel a worker-lease wait 623ms + + Test Files 6 passed (6) + Tests 143 passed (143) + Start at 19:17:04 + Duration 40.89s (transform 12.55s, setup 0ms, collect 34.69s, tests 28.45s, environment 38ms, prepare 20.35s) + +exit_code=0 +``` + +Command (working directory `sdk/`), exercising the built SDK worker against the +live local kernel: + +```text +test -x /Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd && RELAYFLOWD_BIN=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd ./node_modules/.bin/vitest run tests/live-kernel.test.ts -t 'AgentWorker passes a declared model to an identified wrapper as RELAYFLOW_MODEL|AgentWorker executes the raw|AgentWorker leaves RELAYFLOW_MODEL UNSET when the step declares no model' --reporter=verbose --maxWorkers=1 --minWorkers=1 +``` + +Captured output: + +```text + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd +LIVE_KERNEL flows=/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk/dist/cli.js + + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker passes a declared model to an identified wrapper as RELAYFLOW_MODEL 1813ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker executes the raw claude adapter with its real model flag 796ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker executes the raw codex adapter with its real model flag + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker leaves RELAYFLOW_MODEL UNSET when the step declares no model 1150ms + + Test Files 1 passed (1) + Tests 4 passed | 15 skipped (19) + Start at 19:18:56 + Duration 13.54s (transform 3.16s, setup 0ms, collect 4.46s, tests 4.06s, environment 19ms, prepare 1.14s) + +exit_code=0 +``` + +Command (working directory `sdk/`), with a deliberately contaminated parent +environment: + +```text +RELAYFLOW_MODEL=ambient-should-not-leak ./node_modules/.bin/vitest run tests/cli.test.ts -t 'raw Claude adapter|Codex login status' --reporter=verbose && RELAYFLOW_MODEL=ambient-should-not-leak RELAYFLOWD_BIN=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd ./node_modules/.bin/vitest run tests/live-kernel.test.ts -t 'AgentWorker executes the raw' --reporter=verbose --maxWorkers=1 --minWorkers=1 +``` + +Captured output: + +```text + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + + ✓ tests/cli.test.ts > flows check CLI > uses the raw Claude adapter model flag instead of accepting auth status as model proof 535ms + ✓ tests/cli.test.ts > flows check CLI > uses Codex login status and reports a rejected model as unavailable, not unauthenticated 339ms + + Test Files 1 passed (1) + Tests 2 passed | 60 skipped (62) + Start at 19:19:28 + Duration 13.62s (transform 3.07s, setup 0ms, collect 4.65s, tests 881ms, environment 21ms, prepare 1.98s) + + + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd +LIVE_KERNEL flows=/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk/dist/cli.js + + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker executes the raw claude adapter with its real model flag + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker executes the raw claude adapter with its real model flag 636ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker executes the raw codex adapter with its real model flag 412ms + + Test Files 1 passed (1) + Tests 2 passed | 17 skipped (19) + Start at 19:19:48 + Duration 8.73s (transform 2.19s, setup 0ms, collect 3.69s, tests 1.06s, environment 0ms, prepare 1.19s) + +exit_code=0 +``` + +Command (working directory `sdk/`), isolating the previous unused/shadowed +declaration failure mode: + +```text +./node_modules/.bin/vitest run tests/preflight.test.ts tests/cli.test.ts -t 'unknown unused|unknown shadowed' --reporter=verbose +``` + +Captured output: + +```text + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + + ✓ tests/preflight.test.ts > preflight: CLI resolution and refusal predicates > checks an unknown unused named declaration before authoring metadata is erased + ✓ tests/preflight.test.ts > preflight: CLI resolution and refusal predicates > checks an unknown shadowed named declaration before authoring metadata is erased + ✓ tests/cli.test.ts > flows check CLI > refuses an unknown unused named-agent declaration before compilation erases it + ✓ tests/cli.test.ts > flows check CLI > refuses an unknown shadowed named-agent declaration before compilation erases it + + Test Files 2 passed (2) + Tests 4 passed | 75 skipped (79) + Start at 19:20:07 + Duration 6.52s (transform 1.84s, setup 0ms, collect 4.04s, tests 292ms, environment 1ms, prepare 2.07s) + +exit_code=0 +``` + +## Scope limits + +I ran TypeScript checking, the focused deterministic SDK suites, focused live +kernel worker tests, explicit polluted-environment checks, and the exact prior +unused/shadowed regression cases. I did not claim a full repository test run. I +also did not run the opt-in real-provider readiness tests because those execute +actual model requests and may consume provider quota; the deterministic adapter +fixtures and live-kernel worker path were exercised instead. + +REVIEW_PASSED diff --git a/ops/reviews/20260902-1945-pr136-repair.md b/ops/reviews/20260902-1945-pr136-repair.md new file mode 100644 index 00000000..1f53c898 --- /dev/null +++ b/ops/reviews/20260902-1945-pr136-repair.md @@ -0,0 +1,118 @@ +# PR #136 19:05 review repair evidence + +- Repair base: `4888d1572ed047c5161042614ac72068d047783a` +- Reports read completely before finalization: + `20260902-1905-pr136-maintainability.md`, + `20260902-1905-pr136-structure.md`, and + `20260902-1905-pr136-history.md`. +- Scope: F1 pure all-model validation, F2 Codex non-Git argv parity, and F3 + wrapper identity at worker execution. No merge or release action was taken. + +## Literal RED + +Before the repair, the new deterministic regressions failed because a valid +step was probed before a later typo and because worker Codex argv omitted the +Git/cwd flag: + +```text +$ ./node_modules/.bin/vitest run tests/preflight.test.ts tests/cli-adapter.test.ts --reporter=verbose --maxWorkers=1 --minWorkers=1 + +FAIL tests/preflight.test.ts > ... validates every inline model before every probe: valid first +Received: [{"kind":"probe_failed","stepId":"valid",...},{"kind":"model_unknown","stepId":"typo",...}] + +FAIL tests/preflight.test.ts > ... validates every inline model before every probe: typo first +Received: [{"kind":"model_unknown","stepId":"typo",...},{"kind":"probe_failed","stepId":"valid",...}] + +FAIL tests/cli-adapter.test.ts > ... maps raw Codex to login status and noninteractive exec --model +- Expected: ["exec","--ephemeral","--skip-git-repo-check","--model","gpt-model","Review."] ++ Received: ["exec","--ephemeral","--model","gpt-model","Review."] + +Test Files 2 failed (2) +Tests 4 failed | 18 passed (22) +``` + +The new journal-to-worker negative was also literally red: the untrusted +custom executable ran successfully instead of reaching `worker_error`. + +```text +$ RELAYFLOWD_BIN=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 ./node_modules/.bin/vitest run tests/live-kernel.test.ts -t 'nonconforming journal-submitted wrapper' --reporter=verbose --maxWorkers=1 --minWorkers=1 + +FAIL tests/live-kernel.test.ts > ... AgentWorker refuses a nonconforming journal-submitted wrapper before exposing RELAYFLOW_MODEL +Error: step probe did not reach failed within 5000ms + +Test Files 1 failed (1) +Tests 1 failed | 19 skipped (20) +exit_code=1 +``` + +## Literal focused GREEN + +```text +$ ./node_modules/.bin/tsc --noEmit && ./node_modules/.bin/vitest run tests/preflight.test.ts tests/cli-adapter.test.ts tests/model-selection.test.ts tests/validate.test.ts tests/spec-parity.test.ts tests/cli.test.ts tests/bin.test.ts --reporter=dot --maxWorkers=1 --minWorkers=1 + +Test Files 7 passed (7) +Tests 153 passed (153) +Duration 13.48s +exit_code=0 +``` + +The pure validation result is independent of author order and performs no +environment probe: + +```text +{"order":["valid","typo"],"calls":[],"ok":false,"diagnostics":[{"kind":"model_unknown","stepId":"typo","model":"known-modle"}]} +{"order":["typo","valid"],"calls":[],"ok":false,"diagnostics":[{"kind":"model_unknown","stepId":"typo","model":"known-modle"}]} +exit_code=0 +``` + +The focused live journal/worker cases include a wrapper that passes preflight, +is replaced, and is then submitted directly to the journal. The replacement's +evidence is asserted as identity argv only with `model: null`, and the journal +completion is asserted as `worker_error`. + +```text +$ RELAYFLOWD_BIN=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 ./node_modules/.bin/vitest run tests/live-kernel.test.ts -t 'AgentWorker (passes a declared model to an identified wrapper|refuses a nonconforming journal-submitted wrapper|executes the raw|leaves RELAYFLOW_MODEL UNSET)' --reporter=verbose --maxWorkers=1 --minWorkers=1 + +PASS ... passes a declared model to an identified wrapper as RELAYFLOW_MODEL +PASS ... refuses a nonconforming journal-submitted wrapper before exposing RELAYFLOW_MODEL +PASS ... executes the raw claude adapter with its real model flag +PASS ... executes the raw codex adapter with its real model flag +PASS ... leaves RELAYFLOW_MODEL UNSET when the step declares no model + +Test Files 1 passed (1) +Tests 5 passed | 15 skipped (20) +exit_code=0 +``` + +## Literal installed-provider GREEN + +This runs the exact worker Codex argv in a fresh `mkdtemp` directory, which is +not a Git repository: + +```text +$ RELAYFLOWS_REAL_CLI_ADAPTERS=1 ./node_modules/.bin/vitest run tests/real-cli-adapters.test.ts --reporter=verbose --maxWorkers=1 --minWorkers=1 + +PASS ... round-trips the exact declared Claude model and refuses an impossible one 16783ms +PASS ... uses Codex login status and classifies an impossible model as unavailable 5792ms +PASS ... executes the declared Codex model from a real non-Git directory 6659ms + +Test Files 1 passed (1) +Tests 3 passed (3) +Duration 32.53s +exit_code=0 +``` + +## Literal full-suite GREEN + +```text +$ RELAYFLOWD_BIN=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 ./node_modules/.bin/vitest run --reporter=dot --maxWorkers=1 --minWorkers=1 + +Test Files 19 passed | 1 skipped (20) +Tests 271 passed | 3 skipped (274) +Duration 114.32s +exit_code=0 +``` + +The skipped file is the same opt-in real-provider suite run separately above. +The expected backlog-picker negative-path child prints an ENOENT stack during +the full run; Vitest exits zero and all asserted suites are green. diff --git a/sdk/src/cli-adapter.ts b/sdk/src/cli-adapter.ts index cda07368..72ca6a74 100644 --- a/sdk/src/cli-adapter.ts +++ b/sdk/src/cli-adapter.ts @@ -92,7 +92,7 @@ export function agentExecution( if (kind === 'codex') { return { args: [ - 'exec', '--ephemeral', + 'exec', '--ephemeral', '--skip-git-repo-check', ...(model === undefined ? [] : ['--model', model]), instruction, ], diff --git a/sdk/src/cli/check.ts b/sdk/src/cli/check.ts index 50959782..3192e237 100644 --- a/sdk/src/cli/check.ts +++ b/sdk/src/cli/check.ts @@ -11,7 +11,7 @@ import { modelReadinessProbe, type CliInvocation, } from '../cli-adapter.js'; -import { MODEL_ENV } from '../worker.js'; +import { MODEL_ENV } from '../worker-cli.js'; import { modelNameError } from '../model-name.js'; import type { FlowSpec } from '../spec.js'; import type { CheckFailureKind } from '../failure-kinds.js'; diff --git a/sdk/src/preflight.ts b/sdk/src/preflight.ts index ddc73bb5..1a013d01 100644 --- a/sdk/src/preflight.ts +++ b/sdk/src/preflight.ts @@ -100,29 +100,13 @@ export function preflight(flow: FlowSpec, options: PreflightOptions): PreflightR const resolutions: CliResolution[] = []; const cliProbeResults = new Map(); - // Named declarations remain in the normalized authoring object until this - // boundary so even unused or step-shadowed models are checked. Return before - // any environment probe; toKernelSpec erases the map and selector only after - // this authoring preflight has had the chance to fail closed. - for (const [agent, declaration] of Object.entries(flow.agents ?? {})) { - if (isKnownModel(declaration.model, options.models)) continue; - diagnostics.push({ - severity: 'refusal', - kind: 'model_unknown', - agent, - cli: declaration.cli, - model: declaration.model, - message: unknownNamedAgentModelMessage(agent, declaration.cli, declaration.model, options.modelRegistryPath), - }); - } + diagnostics.push(...unknownModelDiagnostics(flow, options)); if (diagnostics.length > 0) return { ok: false, resolutions, diagnostics }; for (const step of flow.steps) { warnOnUnprovableEffects(step, options.probes, diagnostics); if (step.type === 'deterministic') continue; - const declaredModel = step.model; - const modelUnknown = declaredModel !== undefined && !isKnownModel(declaredModel, options.models); const resolution = resolveCli(step, flow, options.projectCli); if (resolution === undefined) { diagnostics.push({ @@ -131,34 +115,9 @@ export function preflight(flow: FlowSpec, options: PreflightOptions): PreflightR stepId: step.id, message: unresolvedCliMessage(step.id, options), }); - if (modelUnknown && declaredModel !== undefined) { - diagnostics.push({ - severity: 'refusal', - kind: 'model_unknown', - stepId: step.id, - model: declaredModel, - message: unknownModelMessage(step.id, declaredModel, undefined, options.modelRegistryPath), - }); - } continue; } resolutions.push(resolution); - if (modelUnknown && resolution.model !== undefined) { - diagnostics.push({ - severity: 'refusal', - kind: 'model_unknown', - stepId: resolution.stepId, - cli: resolution.cli, - model: resolution.model, - message: unknownModelMessage( - resolution.stepId, - resolution.model, - resolution.cli, - options.modelRegistryPath, - ), - }); - continue; - } probeResolvedCli(resolution, options.probes, cliProbeResults, diagnostics); } @@ -173,6 +132,51 @@ export function preflight(flow: FlowSpec, options: PreflightOptions): PreflightR }; } +/** Pure authoring validation: no executable, command, trigger, or daemon probe. */ +function unknownModelDiagnostics( + flow: FlowSpec, + options: PreflightOptions, +): PreflightRefusal[] { + const diagnostics: PreflightRefusal[] = []; + + // Named declarations remain in the normalized authoring object until this + // boundary so even unused or step-shadowed models are checked. toKernelSpec + // erases the map and selector only after this pass has had a chance to fail. + for (const [agent, declaration] of Object.entries(flow.agents ?? {})) { + if (isKnownModel(declaration.model, options.models)) continue; + diagnostics.push({ + severity: 'refusal', + kind: 'model_unknown', + agent, + cli: declaration.cli, + model: declaration.model, + message: unknownNamedAgentModelMessage(agent, declaration.cli, declaration.model, options.modelRegistryPath), + }); + } + + for (const step of flow.steps) { + if (step.type === 'deterministic' || step.model === undefined) continue; + const selected = step.type === 'agent' && step.agent !== undefined + ? flow.agents?.[step.agent] + : undefined; + // compileSpec copies a selected declaration onto the step. The declaration + // was already checked above; only a different value is an inline override. + if (selected?.model === step.model) continue; + if (isKnownModel(step.model, options.models)) continue; + const resolution = resolveCli(step, flow, options.projectCli); + diagnostics.push({ + severity: 'refusal', + kind: 'model_unknown', + stepId: step.id, + ...(resolution === undefined ? {} : { cli: resolution.cli }), + model: step.model, + message: unknownModelMessage(step.id, step.model, resolution?.cli, options.modelRegistryPath), + }); + } + + return diagnostics; +} + function unknownNamedAgentModelMessage( agent: string, cli: string, diff --git a/sdk/src/worker-cli.ts b/sdk/src/worker-cli.ts new file mode 100644 index 00000000..f3a67a08 --- /dev/null +++ b/sdk/src/worker-cli.ts @@ -0,0 +1,110 @@ +import { spawn } from 'node:child_process'; +import { + adapterIdentification, + agentExecution, + cliAdapterKind, + WRAPPER_IDENTIFY_TOKEN, + type CliInvocation, +} from './cli-adapter.js'; + +/** Present only when a dispatched agent step carries a journaled wake context. */ +export const WAKE_CONTEXT_ENV = 'RELAYFLOW_WAKE_CONTEXT'; + +/** + * Present only for a declared model sent to a currently identified custom + * wrapper. Raw Claude/Codex adapters receive provider-native model flags. + */ +export const MODEL_ENV = 'RELAYFLOW_MODEL'; + +export interface WorkerCliResult { + exit_code: number | null; + stdout_tail: string; + stderr_tail: string; +} + +export async function runAgentCli( + cli: string, + instruction: string, + wakeContext: unknown, + model?: string, +): Promise { + const kind = cliAdapterKind(cli); + const invocation = agentExecution(kind, instruction, model); + const env: NodeJS.ProcessEnv = { ...process.env }; + delete env[WAKE_CONTEXT_ENV]; + delete env[MODEL_ENV]; + + if (wakeContext !== undefined) { + try { + env[WAKE_CONTEXT_ENV] = JSON.stringify(wakeContext); + } catch (error) { + return { + exit_code: null, + stdout_tail: '', + stderr_tail: `wake_context could not be JSON-serialized for the CLI: ${String(error)}`, + }; + } + } + + if (kind === 'relayflows-wrapper-v1') { + const identity = adapterIdentification(kind); + const identityEnv = { ...env }; + delete identityEnv[WAKE_CONTEXT_ENV]; + const identified = await spawnInvocation(cli, identity.invocation, identityEnv); + if ( + identified.exit_code !== 0 + || identified.stdout_tail.trim() !== identity.expectedStdout + ) { + return { + exit_code: null, + stdout_tail: '', + stderr_tail: `CLI "${cli}" did not identify as ${WRAPPER_IDENTIFY_TOKEN} at worker execution.`, + }; + } + } + + if (invocation.modelEnv !== undefined) env[MODEL_ENV] = invocation.modelEnv; + return spawnInvocation(cli, invocation, env); +} + +function spawnInvocation( + cli: string, + invocation: CliInvocation, + env: NodeJS.ProcessEnv, +): Promise { + return new Promise((resolve) => { + const child = spawn(cli, invocation.args, { stdio: ['ignore', 'pipe', 'pipe'], env }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + let settled = false; + let timer: NodeJS.Timeout | undefined; + const finish = (result: WorkerCliResult): void => { + if (settled) return; + settled = true; + if (timer !== undefined) clearTimeout(timer); + resolve(result); + }; + child.stdout.on('data', (chunk: Buffer) => stdout.push(chunk)); + child.stderr.on('data', (chunk: Buffer) => stderr.push(chunk)); + child.once('error', (error) => finish({ + exit_code: null, + stdout_tail: Buffer.concat(stdout).toString('utf8'), + stderr_tail: error.message, + })); + child.once('close', (code) => finish({ + exit_code: code, + stdout_tail: Buffer.concat(stdout).toString('utf8'), + stderr_tail: Buffer.concat(stderr).toString('utf8'), + })); + if (invocation.timeoutMs > 0) { + timer = setTimeout(() => { + child.kill('SIGTERM'); + finish({ + exit_code: null, + stdout_tail: Buffer.concat(stdout).toString('utf8'), + stderr_tail: `CLI invocation timed out after ${invocation.timeoutMs}ms.`, + }); + }, invocation.timeoutMs); + } + }); +} diff --git a/sdk/src/worker.ts b/sdk/src/worker.ts index 3707138c..6dece42c 100644 --- a/sdk/src/worker.ts +++ b/sdk/src/worker.ts @@ -1,21 +1,16 @@ -import { spawn } from 'node:child_process'; import { EventEmitter } from 'node:events'; import type { JournalClient } from './journal-client.js'; import type { Pins, StepDispatchEvent } from './protocol.js'; import type { KernelAgentStep } from './spec.js'; -import { agentExecution, cliAdapterKind } from './cli-adapter.js'; +import { runAgentCli } from './worker-cli.js'; + +export { MODEL_ENV, WAKE_CONTEXT_ENV } from './worker-cli.js'; export interface AgentWorkerOptions { workerId: string; pins: Pins; } -interface CliResult { - exit_code: number | null; - stdout_tail: string; - stderr_tail: string; -} - /** * Executes dispatched agent steps using their declared CLI. * @@ -90,7 +85,7 @@ export class AgentWorker extends EventEmitter { private async execute(dispatch: StepDispatchEvent): Promise { const spec = dispatch.spec as Partial; const result = typeof spec.cli === 'string' && typeof spec.instruction === 'string' - ? await runCli(spec.cli, spec.instruction, dispatch.wake_context, spec.model) + ? await runAgentCli(spec.cli, spec.instruction, dispatch.wake_context, spec.model) : { exit_code: null, stdout_tail: '', stderr_tail: 'agent step has no declared CLI' }; const completionReason = result.exit_code === 0 ? 'success' : 'worker_error'; @@ -147,109 +142,3 @@ export function parseJsonOutput(stdout: string): Record | null } return parsed as Record; } - -/** - * Environment variable name AgentWorker sets when dispatching an - * agent step whose kernel dispatch carried a `wake_context` (the - * payload assembled at subscription.matched, containing the - * triggering event). A real analyzer reads this to see which HN - * story / webhook / trigger woke it — an env var is a stable, - * language-agnostic surface that works with any CLI shape, without - * changing the `spawn(cli, [instruction])` argv contract every - * existing agent CLI already depends on. - * - * Not set when `wake_context` is absent (e.g. a directly-started - * run, no trigger fired) — the variable simply won't exist. That is - * DELIBERATE, so a CLI that reads `RELAYFLOW_WAKE_CONTEXT` can - * distinguish "no wake context available" from "wake context = null". - */ -export const WAKE_CONTEXT_ENV = 'RELAYFLOW_WAKE_CONTEXT'; - -/** - * Environment variable AgentWorker sets when the dispatched agent step - * DECLARED a `model`. Same contract as {@link WAKE_CONTEXT_ENV}: when the - * wrapper step declares no model the variable is not merely empty, it is ABSENT, - * so a CLI can tell "the flow author chose nothing" from "the flow author - * chose something". Raw Claude/Codex adapters use their real model flags - * instead; only an explicitly identified Relayflows wrapper receives this - * private environment contract. - * - * This exists because a CLI inheriting whatever model the host happens to - * pin produces two failures: runs whose model cannot be recovered from the - * journal, and hard failure on a host pinning an alias the CLI cannot - * resolve. Declaring it on the step makes the choice portable and recorded. - */ -export const MODEL_ENV = 'RELAYFLOW_MODEL'; - -function runCli( - cli: string, - instruction: string, - wakeContext: unknown, - model?: string, -): Promise { - return new Promise((resolve) => { - const env: NodeJS.ProcessEnv = { ...process.env }; - const invocation = agentExecution(cliAdapterKind(cli), instruction, model); - // Explicit unset. Without this, a parent process (wrapper - // script, systemd unit, docker env, or a prior test) that - // already had RELAYFLOW_WAKE_CONTEXT set would leak into - // this subprocess even on a run with no wake context — which - // would defeat the WAKE_CONTEXT_ENV doc guarantee that CLIs - // can key on absence to distinguish "no wake context - // available" from "wake context = null". Unset first, then - // set only if we have context. Deleting a key from - // ProcessEnv drops it from the child's environ; a subsequent - // conditional assign is the source of truth. - delete env[WAKE_CONTEXT_ENV]; - // Same explicit-unset reasoning as WAKE_CONTEXT_ENV below: an inherited - // RELAYFLOW_MODEL from a parent process would make a step that declared - // no model look like one that did, silently pinning the run to whatever - // the launching shell happened to export. - delete env[MODEL_ENV]; - if (invocation.modelEnv !== undefined) env[MODEL_ENV] = invocation.modelEnv; - if (wakeContext !== undefined) { - // `execve` caps argv + envp at ARG_MAX (macOS ~256 KB, Linux - // ~2 MB). A wake_context that packs a rich payload could - // exceed that and make spawn fail with an opaque E2BIG. - // Truncation would be worse than a loud failure — the CLI - // needs the intact context to analyze the event correctly — - // so we let spawn's error surface naturally. - // - // `JSON.stringify` can throw synchronously (on a cycle or a - // non-serializable value like BigInt). Values that arrived - // through the wire protocol are already JSON-clean by - // construction, but this worker also runs inside test rigs - // and future callers may construct `wake_context` in-process. - // Catching the throw here converts a would-be silent - // lease-expiration (Promise executor throw → no `resolve`, - // no stepComplete written) into a clean `worker_error` - // completion the kernel journals normally. Fail-closed per - // AGENTS.md. - try { - env[WAKE_CONTEXT_ENV] = JSON.stringify(wakeContext); - } catch (error) { - resolve({ - exit_code: null, - stdout_tail: '', - stderr_tail: `wake_context could not be JSON-serialized for the CLI: ${String(error)}`, - }); - return; - } - } - const child = spawn(cli, invocation.args, { stdio: ['ignore', 'pipe', 'pipe'], env }); - const stdout: Buffer[] = []; - const stderr: Buffer[] = []; - child.stdout.on('data', (chunk: Buffer) => stdout.push(chunk)); - child.stderr.on('data', (chunk: Buffer) => stderr.push(chunk)); - child.once('error', (error) => resolve({ - exit_code: null, - stdout_tail: Buffer.concat(stdout).toString('utf8'), - stderr_tail: error.message, - })); - child.once('close', (code) => resolve({ - exit_code: code, - stdout_tail: Buffer.concat(stdout).toString('utf8'), - stderr_tail: Buffer.concat(stderr).toString('utf8'), - })); - }); -} diff --git a/sdk/tests/cli-adapter.test.ts b/sdk/tests/cli-adapter.test.ts index c33b066a..9196881e 100644 --- a/sdk/tests/cli-adapter.test.ts +++ b/sdk/tests/cli-adapter.test.ts @@ -32,13 +32,15 @@ describe('typed CLI adapters', () => { expect(kind).toBe('codex'); expect(adapterIdentification(kind).invocation.args).toEqual(['login', 'status', '--help']); expect(authenticationProbe(kind).args).toEqual(['login', 'status']); - const readiness = modelReadinessProbe(kind, 'gpt-model'); - expect(readiness).toMatchObject({ - args: expect.arrayContaining(['exec', '--ephemeral', '--model', 'gpt-model']), + expect(modelReadinessProbe(kind, 'gpt-model')).toEqual({ + args: [ + 'exec', '--ephemeral', '--sandbox', 'read-only', '--skip-git-repo-check', + '--model', 'gpt-model', 'Reply with exactly RELAYFLOWS_MODEL_READY and nothing else.', + ], + timeoutMs: 60_000, }); - expect(readiness).not.toHaveProperty('modelEnv'); expect(agentExecution(kind, 'Review.', 'gpt-model')).toEqual({ - args: ['exec', '--ephemeral', '--model', 'gpt-model', 'Review.'], + args: ['exec', '--ephemeral', '--skip-git-repo-check', '--model', 'gpt-model', 'Review.'], timeoutMs: 0, }); }); diff --git a/sdk/tests/live-kernel.test.ts b/sdk/tests/live-kernel.test.ts index 8179e4a1..2f6d606c 100644 --- a/sdk/tests/live-kernel.test.ts +++ b/sdk/tests/live-kernel.test.ts @@ -7,6 +7,7 @@ import { mkdtempSync, readdirSync, rmSync, + statSync, writeFileSync, readFileSync, } from 'node:fs'; @@ -16,6 +17,7 @@ import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; import { fileURLToPath } from 'node:url'; import { afterEach, beforeAll, describe, expect, it } from 'vitest'; import { compileYaml, toKernelSpec } from '../src/compile.js'; +import { checkFlow } from '../src/cli/check.js'; import { JournalClient } from '../src/journal-client.js'; import type { StepDispatchEvent } from '../src/protocol.js'; import { AgentWorker } from '../src/worker.js'; @@ -148,7 +150,7 @@ steps: await expect(liveClient.runResume('absent-run')).rejects.toMatchObject({ code: 'run_not_found', }); - }); + }, 30_000); it('allows a deterministic run to exceed the bounded request timeout', async () => { const dataDir = temporaryDirectory('flows-live-long-run-'); @@ -746,9 +748,80 @@ steps: await worker.close(); }, 30_000); + it('AgentWorker refuses a nonconforming journal-submitted wrapper before exposing RELAYFLOW_MODEL', async () => { + const dataDir = temporaryDirectory('flows-live-wrapper-identity-'); + await startDaemon(dataDir); + const cli = join(dataDir, 'not-a-relayflows-wrapper'); + const evidence = join(dataDir, 'wrapper-evidence.json'); + writeFileSync(cli, `#!/bin/sh +case "$1 $2" in + "--relayflows-adapter-v1 ") printf '%s\\n' relayflows-agent-cli-v1 ;; + "auth status") exit 0 ;; + *) exit 9 ;; +esac +`); + chmodSync(cli, 0o755); + writeFileSync(join(dataDir, 'flows.json'), JSON.stringify({ models: ['declared-model-xyz'] })); + const flowSource = ` +version: '0.1.0' +steps: + - id: probe + type: agent + cli: ${JSON.stringify(cli)} + model: declared-model-xyz + instruction: This instruction must not execute. +`; + const flowPath = join(dataDir, 'replacement.flow.yaml'); + writeFileSync(flowPath, flowSource); + expect(checkFlow(flowPath).report.ok).toBe(true); + + // Replace the previously identified executable before dispatch. The + // worker must bind trust to what it executes now, not an earlier check. + writeFileSync(cli, `#!/usr/bin/env node +const fs = require('node:fs'); +fs.writeFileSync(${JSON.stringify(evidence)}, JSON.stringify({ argv: process.argv.slice(2), model: process.env.RELAYFLOW_MODEL ?? null })); +if (process.argv[2] === '--relayflows-adapter-v1') { + process.stdout.write('not-the-required-token\\n'); + process.exit(0); +} +process.stdout.write('{"must_not":"execute"}'); +`); + chmodSync(cli, 0o755); + const client = await connectClient(dataDir); + await client.hello('live-wrapper-identity'); + const worker = new AgentWorker(client, { + workerId: 'live-wrapper-identity-worker', + pins: { workspace: [{ surface: 'repo', revision_id: 'rev-a' }], streams: [] }, + }); + await worker.attach(); + + // Submit the compiled kernel object directly, as journal clients can. + const started = await client.runStart(toKernelSpec(compileYaml(flowSource))); + + expect(await waitForStep(client, started.run_id, 'probe', 'done')).toMatchObject({ + type: 'agent', + state: 'done', + }); + expect(JSON.parse(readFileSync(evidence, 'utf8'))).toEqual({ + argv: ['--relayflows-adapter-v1'], + model: null, + }); + const completed = (await client.journalRead(started.run_id)).entries.find( + (entry) => (entry as { entry_type: string; step_id?: string }).entry_type === 'step.completed' + && (entry as { step_id?: string }).step_id === 'probe', + ); + expect(completed).toMatchObject({ + payload: { + completionReason: 'worker_error', + }, + }); + + await worker.close(); + }, 30_000); + it.each([ ['claude', '-p --model declared-model-xyz'], - ['codex', 'exec --ephemeral --model declared-model-xyz'], + ['codex', 'exec --ephemeral --skip-git-repo-check --model declared-model-xyz'], ] as const)('AgentWorker executes the raw %s adapter with its real model flag', async (name, prefix) => { const dataDir = temporaryDirectory(`flows-live-${name}-adapter-`); await startDaemon(dataDir); diff --git a/sdk/tests/preflight.test.ts b/sdk/tests/preflight.test.ts index 1fe689e2..d86a71e3 100644 --- a/sdk/tests/preflight.test.ts +++ b/sdk/tests/preflight.test.ts @@ -246,15 +246,78 @@ describe('preflight: CLI resolution and refusal predicates', () => { ); expect(result.diagnostics.map((diagnostic) => diagnostic.kind)).toEqual([ - 'cli_unresolved', 'model_unknown', ]); - expect(result.diagnostics[1]).toMatchObject({ + expect(result.diagnostics[0]).toMatchObject({ stepId: 'a', model: 'typo-model', }); }); + it.each([ + ['valid first', ['valid', 'typo']], + ['typo first', ['typo', 'valid']], + ] as const)('validates every inline model before every probe: %s', (_label, order) => { + const calls: string[] = []; + const steps: Record<(typeof order)[number], FlowSpec['steps'][number]> = { + valid: { id: 'valid', type: 'agent', cli: 'claude', model: 'known-model', instruction: 'Valid.' }, + typo: { id: 'typo', type: 'agent', cli: 'claude', model: 'known-modle', instruction: 'Typo.' }, + }; + + const result = preflight({ + version: '0.1.0', + steps: [ + { id: 'deterministic', type: 'deterministic', command: './must-not-probe' }, + ...order.map((id) => steps[id]), + ], + triggers: [{ id: 'trigger', executor: 'must-not-probe' }], + }, { + models: ['known-model'], + probes: { + cli: () => { calls.push('cli'); throw new Error('PROBE_CALLED'); }, + command: () => { calls.push('command'); throw new Error('PROBE_CALLED'); }, + executor: () => { calls.push('executor'); throw new Error('PROBE_CALLED'); }, + }, + }); + + expect(result.ok).toBe(false); + expect(result.diagnostics).toEqual([ + expect.objectContaining({ kind: 'model_unknown', stepId: 'typo', model: 'known-modle' }), + ]); + expect(calls).toEqual([]); + }); + + it('returns every named and inline unknown-model diagnostic in the pure first pass', () => { + let probeCalls = 0; + const result = preflight(compileSpec({ + version: '0.1.0', + agents: { + unused: { cli: 'claude', model: 'unknown-named' }, + }, + steps: [{ + id: 'inline', + type: 'agent', + cli: 'codex', + model: 'unknown-inline', + instruction: 'Review.', + }], + }), { + models: ['known-model'], + probes: probes({ + cli: () => { + probeCalls += 1; + return { exists: true, authenticated: true, modelAvailable: true }; + }, + }), + }); + + expect(result.diagnostics).toEqual([ + expect.objectContaining({ kind: 'model_unknown', agent: 'unused', model: 'unknown-named' }), + expect.objectContaining({ kind: 'model_unknown', stepId: 'inline', model: 'unknown-inline' }), + ]); + expect(probeCalls).toBe(0); + }); + it.each(['unused', 'shadowed'] as const)( 'checks an unknown %s named declaration before authoring metadata is erased', (variant) => { diff --git a/sdk/tests/real-cli-adapters.test.ts b/sdk/tests/real-cli-adapters.test.ts index c4dc0e9d..7e47c1f3 100644 --- a/sdk/tests/real-cli-adapters.test.ts +++ b/sdk/tests/real-cli-adapters.test.ts @@ -1,8 +1,10 @@ +import { spawnSync } from 'node:child_process'; import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterAll, describe, expect, it } from 'vitest'; import { checkFlow } from '../src/cli/check.js'; +import { agentExecution } from '../src/cli-adapter.js'; const RUN_REAL = process.env['RELAYFLOWS_REAL_CLI_ADAPTERS'] === '1'; const directories: string[] = []; @@ -46,4 +48,21 @@ describe.runIf(RUN_REAL)('installed raw CLI adapters', () => { expect(refused.diagnostics).toContainEqual(expect.objectContaining({ kind: 'model_unavailable' })); expect(refused.diagnostics).not.toContainEqual(expect.objectContaining({ kind: 'cli_unauthenticated' })); }, 70_000); + + it('executes the declared Codex model from a real non-Git directory', () => { + const directory = mkdtempSync(join(tmpdir(), 'flows-real-codex-worker-')); + directories.push(directory); + const model = process.env['RELAYFLOWS_REAL_CODEX_MODEL'] ?? 'gpt-5.6-sol'; + const invocation = agentExecution('codex', 'Reply with exactly RELAYFLOWS_NON_GIT_READY.', model); + const result = spawnSync('codex', invocation.args, { + cwd: directory, + encoding: 'utf8', + timeout: 120_000, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + expect(result.error).toBeUndefined(); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain('RELAYFLOWS_NON_GIT_READY'); + }, 130_000); }); From fccc52abbef194d5536e96f4008ab5445467365b Mon Sep 17 00:00:00 2001 From: kjgbot Date: Wed, 2 Sep 2026 20:29:01 +0200 Subject: [PATCH 05/15] docs(review): record PR 136 integration review Session-Id: 01a062b7-2aef-7772-8225-7cb00ee311dd --- ops/reviews/20260902-1950-pr136-history.md | 422 ++++++++++++++++++ ops/reviews/20260902-1950-pr136-structure.md | 443 +++++++++++++++++++ 2 files changed, 865 insertions(+) create mode 100644 ops/reviews/20260902-1950-pr136-history.md create mode 100644 ops/reviews/20260902-1950-pr136-structure.md diff --git a/ops/reviews/20260902-1950-pr136-history.md b/ops/reviews/20260902-1950-pr136-history.md new file mode 100644 index 00000000..935c56cf --- /dev/null +++ b/ops/reviews/20260902-1950-pr136-history.md @@ -0,0 +1,422 @@ +# PR #136 fresh exact-head history / integration review + +Date: 2026-09-02 + +- PR: #136, `feat(sdk): declare agent CLI and model with fail-closed checks` +- Exact head: `060e8b971a04f27785d88e1520aa7c1eb3fd3fc5` +- Base: merged `main` at `a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2` +- Lens: complete prior-blocker history, model-validation ordering, authoring-to-journal lowering, real provider behavior, worker trust boundary, non-Git execution, v1 compatibility, and exact CI +- Constitution read: `AGENTS.md` and `docs/RFC-0001-everything-is-a-relayflow.md` +- Scope: assessment only; no product code, gate, commit, push, merge, or release action + +## Verdict + +PASS. I found no remaining blocking defect at the assigned head. The repair history closes all six concrete blocker classes raised by the 17:10 and 19:05 reviews: + +1. raw Claude/Codex now receive provider-native argv and the declared model; +2. identification, authentication, and model readiness are distinct and truthfully classified; +3. unused and shadowed named declarations survive compilation until authoring preflight; +4. every named and inline model is checked in one pure first pass before any executable, deterministic-command, trigger-executor, or daemon probe, independent of step order; +5. Codex readiness and execution both work outside Git repositories; +6. custom wrappers are re-identified by the worker immediately before execution, before `RELAYFLOW_MODEL` is exposed, including a direct-journal executable-swap case. + +The exact-head local deterministic suites, live journal/worker tests, installed-provider tests, and full serial SDK suite all pass. GitHub's exact-head `linux-x64-artifact` job is also green. The GitHub workflow is artifact/smoke CI, not the full SDK regression suite; I do not attribute the local 274-test result to CI. + +## Prior-blocker ledger + +| Prior report | Blocker | Exact-head assessment | +| --- | --- | --- | +| 17:10 maintainability F1 | Declared model did not control bare Claude/Codex execution | PASS: shared adapter table emits Claude `-p --model` and Codex `exec --ephemeral --skip-git-repo-check --model`; raw providers receive no private model env. | +| 17:10 maintainability F2 | Generic `auth status` caused Claude false pass and Codex false refusal | PASS: provider-specific identification/auth commands and real model round trips; installed CLIs returned successful identification/auth, available models ran, impossible models were `model_unavailable`. | +| 17:10 structure P1 | Unknown unused/shadowed named model erased before preflight | PASS: normalized authoring object retains `agents` and selectors; the pure preflight pass rejects named declarations before lowering. | +| 19:05 maintainability F1 | A valid earlier step could probe before a later inline typo | PASS: both step orders produce the same named+inline `model_unknown` diagnostics with zero calls across all injected probe kinds. | +| 19:05 history / maintainability F2 | Codex readiness passed outside Git but worker argv failed there | PASS: `--skip-git-repo-check` is shared by readiness and execution; installed Codex executed the declared model from a fresh non-Git directory. | +| 19:05 maintainability F3 | Custom-wrapper identity was only a preflight fact | PASS: live test replaces a preflight-green wrapper, submits the kernel object directly to the journal, observes only worker identity argv with no model env, and journals `worker_error`. | + +The 19:05 structure review reported no independent blocker. The 19:45 repair report targeted the three later findings; the evidence below independently reruns those contracts at the current head rather than accepting that report as proof. + +## Exact boundary and history + +Literal command and captured output: + +```text +$ pwd && git status --short && git branch --show-current && git rev-parse HEAD && git rev-parse origin/main && git merge-base origin/main HEAD && git log --oneline --decorate --graph origin/main..HEAD +/Users/khaliqgant/AgentWorkforce/flows-132-model-wt +feat/v2-declared-model +060e8b971a04f27785d88e1520aa7c1eb3fd3fc5 +a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2 +a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2 +* 060e8b9 (HEAD -> feat/v2-declared-model, origin/feat/v2-declared-model) fix(sdk): bind declared model execution checks +* 4888d15 (origin/pr-136) fix(sdk): make model adapters fail closed +* 78efc0d docs(review): record PR 136 fresh review +* 321b272 feat(sdk): add declared agent model contract +``` + +The empty line after `pwd` was an empty initial `git status --short`: the worktree was clean before review. The four-commit story is coherent: authoring contract, persisted first reviews, adapter/preflight repair, then ordering/Codex/worker-boundary repair. No kernel change is present; the registry and provider logic remain in the TypeScript SDK. + +Literal hygiene/boundary command and captured output: + +```text +$ git diff --check a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..HEAD; print "diff_check_exit=$?"; git diff --quiet a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..HEAD -- kernel regressions/surface.d.ts; print "kernel_surface_diff_exit=$?"; git status --short +diff_check_exit=0 +kernel_surface_diff_exit=0 +``` + +## Model validation is globally first and order-independent + +`compileSpec()` validates and normalizes the authoring dialect while retaining the `agents` map and `agent` selectors. `preflight()` begins with `unknownModelDiagnostics()` and returns immediately if that pure scan finds anything. Only the following loop can invoke deterministic-command, CLI/model, or trigger-executor probes. `runFlow()` calls `checkFlow()` before it creates/connects the journal client and calls `toKernelSpec()` only after that check returns a flow. + +I exercised an unknown unused named declaration plus an unknown inline model, a valid model, a deterministic command, and a trigger executor. Both step orders return both model diagnostics and make no probe call. + +Literal command: + +```sh +node --input-type=module <<'EOF' +import { compileSpec, preflight, toKernelSpec } from './dist/index.js'; +for (const order of [['valid', 'typo'], ['typo', 'valid']]) { + const calls = []; + const byId = { + valid: { id: 'valid', type: 'agent', cli: 'claude', model: 'known-model', instruction: 'Valid.' }, + typo: { id: 'typo', type: 'agent', cli: 'codex', model: 'unknown-inline', instruction: 'Typo.' }, + }; + const flow = compileSpec({ + version: '0.1.0', + agents: { unused: { cli: 'claude', model: 'unknown-named' } }, + triggers: [{ id: 'trigger', executor: 'must-not-probe' }], + steps: [ + { id: 'deterministic', type: 'deterministic', command: './must-not-probe' }, + ...order.map((id) => byId[id]), + ], + }); + const result = preflight(flow, { + models: ['known-model'], + probes: { + cli: () => { calls.push('cli'); throw new Error('PROBE_CALLED'); }, + command: () => { calls.push('command'); throw new Error('PROBE_CALLED'); }, + executor: () => { calls.push('executor'); throw new Error('PROBE_CALLED'); }, + }, + }); + const kernel = toKernelSpec(flow); + console.log(JSON.stringify({ + order, + calls, + ok: result.ok, + diagnostics: result.diagnostics.map(({ kind, agent, stepId, model }) => ({ kind, agent: agent ?? null, stepId: stepId ?? null, model: model ?? null })), + authoringHasAgents: Object.hasOwn(flow, 'agents'), + authoringHasSelector: flow.steps.some((step) => Object.hasOwn(step, 'agent')), + kernelHasAgents: Object.hasOwn(kernel, 'agents'), + kernelHasSelector: kernel.steps.some((step) => Object.hasOwn(step, 'agent')), + })); +} +EOF +``` + +Captured output: + +```text +{"order":["valid","typo"],"calls":[],"ok":false,"diagnostics":[{"kind":"model_unknown","agent":"unused","stepId":null,"model":"unknown-named"},{"kind":"model_unknown","agent":null,"stepId":"typo","model":"unknown-inline"}],"authoringHasAgents":true,"authoringHasSelector":false,"kernelHasAgents":false,"kernelHasSelector":false} +{"order":["typo","valid"],"calls":[],"ok":false,"diagnostics":[{"kind":"model_unknown","agent":"unused","stepId":null,"model":"unknown-named"},{"kind":"model_unknown","agent":null,"stepId":"typo","model":"unknown-inline"}],"authoringHasAgents":true,"authoringHasSelector":false,"kernelHasAgents":false,"kernelHasSelector":false} +``` + +The named declaration in that probe is intentionally unused, so no step selector exists. The next independent probe covers selected named agents, independent field precedence, successful preflight, and erasure timing. + +Literal command: + +```sh +node --input-type=module <<'EOF' +import { compileYaml, preflight, toKernelSpec } from './dist/index.js'; +const flow = compileYaml(` +version: '0.1.0' +cli: flow-cli +agents: + reviewer: { cli: named-cli, model: named-model } +steps: + - { id: named, type: agent, agent: reviewer, instruction: named } + - { id: cli-override, type: agent, agent: reviewer, cli: step-cli, instruction: cli } + - { id: model-override, type: agent, agent: reviewer, model: step-model, instruction: model } + - { id: inline, type: agent, instruction: inline } +`); +const authoringBefore = { hasAgents: Object.hasOwn(flow, 'agents'), selectors: flow.steps.map((step) => step.type === 'agent' ? step.agent ?? null : null) }; +const calls = []; +const result = preflight(flow, { + projectCli: 'project-cli', + models: ['named-model', 'step-model'], + probes: { + cli: (cli, source, model) => { calls.push({ cli, source, model: model ?? null }); return { exists: true, supported: true, authenticated: true, modelAvailable: model === undefined ? undefined : true }; }, + command: () => true, + executor: () => true, + }, +}); +const authoringAfter = { hasAgents: Object.hasOwn(flow, 'agents'), selectors: flow.steps.map((step) => step.type === 'agent' ? step.agent ?? null : null) }; +const kernel = toKernelSpec(flow); +console.log(JSON.stringify({ authoringBefore, result, calls, authoringAfter, kernelHasAgents: Object.hasOwn(kernel, 'agents'), kernelSteps: kernel.steps.map((step) => ({ id: step.id, cli: step.cli ?? null, model: step.model ?? null, selector: Object.hasOwn(step, 'agent') ? step.agent : null })) })); +EOF +``` + +Captured output: + +```text +{"authoringBefore":{"hasAgents":true,"selectors":["reviewer","reviewer","reviewer",null]},"result":{"ok":true,"resolutions":[{"stepId":"named","cli":"named-cli","source":"step","model":"named-model"},{"stepId":"cli-override","cli":"step-cli","source":"step","model":"named-model"},{"stepId":"model-override","cli":"named-cli","source":"step","model":"step-model"},{"stepId":"inline","cli":"flow-cli","source":"flow"}],"diagnostics":[]},"calls":[{"cli":"named-cli","source":"step","model":"named-model"},{"cli":"step-cli","source":"step","model":"named-model"},{"cli":"named-cli","source":"step","model":"step-model"},{"cli":"flow-cli","source":"flow","model":null}],"authoringAfter":{"hasAgents":true,"selectors":["reviewer","reviewer","reviewer",null]},"kernelHasAgents":false,"kernelSteps":[{"id":"named","cli":"named-cli","model":"named-model","selector":null},{"id":"cli-override","cli":"step-cli","model":"named-model","selector":null},{"id":"model-override","cli":"named-cli","model":"step-model","selector":null},{"id":"inline","cli":null,"model":null,"selector":null}]} +``` + +This pins the intended precedence separately for each field: step override, then named declaration, then existing flow/project fallback for CLI only. The journal boundary carries only the already-existing `cli` / `model` step fields; `agents` and `agent` are absent. + +## Provider-native behavior and non-Git execution + +The shared adapter table maps: + +- Claude identification to `auth status --help`, auth to `auth status`, readiness to a real `-p --model MODEL` round trip, and execution to `-p --model MODEL INSTRUCTION`; +- Codex identification to `login status --help`, auth to `login status`, readiness to read-only `exec --ephemeral --skip-git-repo-check --model MODEL`, and execution to the same non-Git-capable prefix without the readiness-only sandbox restriction; +- custom wrappers to the exact `relayflows-agent-cli-v1` token, `auth status`, and the private model env only after identification. + +Ambient `RELAYFLOW_MODEL` is deleted for all child processes. Raw providers receive their model only through the provider-native flag. The installed commands identify and authenticate with the adapter's exact shapes: + +```text +$ node --input-type=module <<'EOF' +import { spawnSync } from 'node:child_process'; +for (const [cli, versionArgs, identifyArgs, authArgs] of [ + ['claude', ['--version'], ['auth', 'status', '--help'], ['auth', 'status']], + ['codex', ['--version'], ['login', 'status', '--help'], ['login', 'status']], +]) { + const version = spawnSync(cli, versionArgs, { encoding: 'utf8' }); + const identified = spawnSync(cli, identifyArgs, { encoding: 'utf8', stdio: ['ignore', 'ignore', 'ignore'] }); + const authenticated = spawnSync(cli, authArgs, { encoding: 'utf8', stdio: ['ignore', 'ignore', 'ignore'] }); + console.log(JSON.stringify({ cli, version: (version.stdout || version.stderr).trim(), identificationExit: identified.status, authExit: authenticated.status })); +} +EOF +{"cli":"claude","version":"2.1.153 (Claude Code)","identificationExit":0,"authExit":0} +{"cli":"codex","version":"codex-cli 0.152.1","identificationExit":0,"authExit":0} +``` + +The opt-in installed-provider suite proves positive Claude readiness, negative Claude classification, truthful Codex auth/negative-model classification, and positive Codex execution from a fresh non-Git directory: + +```text +$ RELAYFLOWS_REAL_CLI_ADAPTERS=1 ./node_modules/.bin/vitest run tests/real-cli-adapters.test.ts --reporter=verbose --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + + ✓ tests/real-cli-adapters.test.ts > installed raw CLI adapters > round-trips the exact declared Claude model and refuses an impossible one 13837ms + ✓ tests/real-cli-adapters.test.ts > installed raw CLI adapters > uses Codex login status and classifies an impossible model as unavailable 6775ms + ✓ tests/real-cli-adapters.test.ts > installed raw CLI adapters > executes the declared Codex model from a real non-Git directory 9886ms + + Test Files 1 passed (1) + Tests 3 passed (3) + Start at 19:53:36 + Duration 31.83s (transform 317ms, setup 0ms, collect 469ms, tests 30.50s, environment 1ms, prepare 237ms) +``` + +I also directly executed the exact worker argv for both providers. Claude command: + +```sh +node --input-type=module <<'EOF' +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { agentExecution } from './dist/cli-adapter.js'; +const directory = mkdtempSync(join(tmpdir(), 'pr136-real-claude-worker-')); +try { + const invocation = agentExecution('claude', 'Reply with exactly RELAYFLOWS_CLAUDE_EXEC_READY.', 'claude-haiku-4-5-20251001'); + const result = spawnSync('claude', invocation.args, { cwd: directory, encoding: 'utf8', timeout: 120000, stdio: ['ignore', 'pipe', 'pipe'] }); + console.log(JSON.stringify({ cwdIsGit: spawnSync('git', ['rev-parse', '--is-inside-work-tree'], { cwd: directory, stdio: 'ignore' }).status === 0, args: invocation.args, status: result.status, signal: result.signal, error: result.error?.message ?? null, stdout: result.stdout.trim(), stderr: result.stderr.trim() })); + process.exitCode = result.status ?? 1; +} finally { + rmSync(directory, { recursive: true, force: true }); +} +EOF +``` + +Captured output: + +```text +{"cwdIsGit":false,"args":["-p","--model","claude-haiku-4-5-20251001","Reply with exactly RELAYFLOWS_CLAUDE_EXEC_READY."],"status":0,"signal":null,"error":null,"stdout":"RELAYFLOWS_CLAUDE_EXEC_READY","stderr":""} +``` + +Codex command: + +```sh +node --input-type=module <<'EOF' +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { agentExecution } from './dist/cli-adapter.js'; +const directory = mkdtempSync(join(tmpdir(), 'pr136-real-codex-worker-')); +try { + const invocation = agentExecution('codex', 'Reply with exactly RELAYFLOWS_CODEX_EXEC_READY.', 'gpt-5.6-sol'); + const result = spawnSync('codex', invocation.args, { cwd: directory, encoding: 'utf8', timeout: 120000, stdio: ['ignore', 'pipe', 'pipe'] }); + console.log(JSON.stringify({ cwdIsGit: spawnSync('git', ['rev-parse', '--is-inside-work-tree'], { cwd: directory, stdio: 'ignore' }).status === 0, args: invocation.args, status: result.status, signal: result.signal, error: result.error?.message ?? null, stdoutHasToken: result.stdout.includes('RELAYFLOWS_CODEX_EXEC_READY'), stderrHasModel: result.stderr.includes('model: gpt-5.6-sol') })); + process.exitCode = result.status ?? 1; +} finally { + rmSync(directory, { recursive: true, force: true }); +} +EOF +``` + +Captured output: + +```text +{"cwdIsGit":false,"args":["exec","--ephemeral","--skip-git-repo-check","--model","gpt-5.6-sol","Reply with exactly RELAYFLOWS_CODEX_EXEC_READY."],"status":0,"signal":null,"error":null,"stdoutHasToken":true,"stderrHasModel":true} +``` + +Both commands used `agentExecution()` from the exact-head built SDK and fresh `mkdtemp` directories; `cwdIsGit:false` is checked by `git rev-parse --is-inside-work-tree` in each command. + +## Worker-side wrapper trust and direct-journal refusal + +`runAgentCli()` deletes inherited model/wake env first. For a custom executable, it runs `--relayflows-adapter-v1` in an env with both the private model value and wake context absent, requires exit 0 and exact stdout `relayflows-agent-cli-v1`, then and only then adds a declared model and executes the instruction. Raw Claude/Codex skip the private env entirely. + +The live test is load-bearing: its first wrapper passes real `checkFlow()` preflight, the executable is replaced, and the compiled kernel object is submitted directly with `JournalClient.runStart()`. The worker sees the replacement, invokes only its identity route, exposes `model:null`, never invokes the instruction route, and completes the journal step with `completionReason: worker_error`. + +Literal command and captured output: + +```text +$ RELAYFLOWD_BIN=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 ./node_modules/.bin/vitest run tests/live-kernel.test.ts -t 'AgentWorker (passes a declared model to an identified wrapper|refuses a nonconforming journal-submitted wrapper|executes the raw|leaves RELAYFLOW_MODEL UNSET)' --reporter=verbose --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd +LIVE_KERNEL flows=/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk/dist/cli.js + + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker passes a declared model to an identified wrapper as RELAYFLOW_MODEL 1511ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker refuses a nonconforming journal-submitted wrapper before exposing RELAYFLOW_MODEL 973ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker executes the raw claude adapter with its real model flag 389ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker executes the raw codex adapter with its real model flag 1093ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker leaves RELAYFLOW_MODEL UNSET when the step declares no model 454ms + + Test Files 1 passed (1) + Tests 5 passed | 15 skipped (20) + Start at 19:53:21 + Duration 8.85s (transform 1.50s, setup 0ms, collect 1.99s, tests 4.43s, environment 0ms, prepare 569ms) +``` + +## Deterministic and full regression suites + +TypeScript checking completed with no output and exit 0: + +```text +$ ./node_modules/.bin/tsc --noEmit +[no output; exit 0] +``` + +Focused exact-head result (unmodified captured summary lines): + +```text +$ ./node_modules/.bin/vitest run tests/preflight.test.ts tests/cli-adapter.test.ts tests/model-selection.test.ts tests/validate.test.ts tests/spec-parity.test.ts tests/cli.test.ts tests/bin.test.ts --reporter=dot --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + + ✓ tests/cli.test.ts (62 tests) 7566ms + ✓ tests/preflight.test.ts (20 tests) 22ms + ✓ tests/validate.test.ts (36 tests) 73ms + ✓ tests/model-selection.test.ts (10 tests) 251ms + ✓ tests/bin.test.ts (7 tests) 4426ms + ✓ tests/spec-parity.test.ts (15 tests) 813ms + ✓ tests/cli-adapter.test.ts (3 tests) 48ms + + Test Files 7 passed (7) + Tests 153 passed (153) + Start at 19:51:40 + Duration 40.10s (transform 5.77s, setup 0ms, collect 9.55s, tests 13.20s, environment 4ms, prepare 6.41s) +``` + +Full serial suite result, with the real-provider file intentionally skipped because it was separately run above (unmodified captured start, live evidence, per-file result, and final summary lines): + +```text +$ RELAYFLOWD_BIN=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 ./node_modules/.bin/vitest run --reporter=dot --maxWorkers=1 --minWorkers=1 + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd +LIVE_KERNEL flows=/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk/dist/cli.js + +stdout | tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +LIVE_ANALYZER ready: claude -p --model claude-haiku-4-5-20251001 round-trip OK + +stdout | tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +LIVE_ANALYZER analysis: {"reasoning":"This story directly describes an AI agent performing autonomous software development tasks, including opening and reviewing pull requests, which is a core application of agentic AI and automation in engineering workflows.","relevance_score":9,"story_title":"Show HN: an agent that opens and reviews its own pull requests [wake-nonce-7f3a91c4]"} + +stdout | tests/live-kernel.test.ts > surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once +LIVE_KERNEL kill -9 pid=8817 run=01M1HM81WTS26D06Y2E6E2F4MH while step=two state=Running + + ✓ tests/live-kernel.test.ts (20 tests) 65445ms + ✓ tests/cli.test.ts (62 tests) 3273ms + ✓ tests/journal-client.test.ts (13 tests) 135ms + ✓ tests/preflight.test.ts (20 tests) 16ms + ✓ tests/validate.test.ts (36 tests) 16ms + ✓ tests/cli-hn-monitor.test.ts (16 tests) 86ms + ✓ tests/backlog-picker.test.ts (14 tests) 227ms + ✓ tests/backlog-picker-flow.test.ts (6 tests) 1448ms + ✓ tests/work-package-consumer.test.ts (13 tests) 570ms + ✓ tests/model-selection.test.ts (10 tests) 52ms + ✓ tests/deterministic-llm.test.ts (5 tests) 60ms + ✓ tests/bin.test.ts (7 tests) 1716ms + ✓ tests/hn-poller.test.ts (6 tests) 37ms + ✓ tests/dir-watcher-poller.test.ts (6 tests) 13ms + ✓ tests/hello-deterministic.test.ts (5 tests) 67ms + ✓ tests/work-package-validator.test.ts (7 tests) 20ms + ✓ tests/spec-parity.test.ts (15 tests) 116ms + ↓ tests/real-cli-adapters.test.ts (3 tests | 3 skipped) + ✓ tests/parse-json-output.test.ts (7 tests) 8ms + ✓ tests/cli-adapter.test.ts (3 tests) 13ms + + Test Files 19 passed | 1 skipped (20) + Tests 271 passed | 3 skipped (274) + Start at 19:55:31 + Duration 87.10s (transform 1.10s, setup 0ms, collect 3.08s, tests 73.32s, environment 8ms, prepare 2.82s) +``` + +The suite also emitted the expected backlog-picker negative-path child `ENOENT` stack while its parent assertions passed; that literal child-process stack falls between the captured result lines above and does not change Vitest's exit 0. + +## v1/default compatibility and issue scope + +The PR changes neither `kernel/` nor the declaration-only `regressions/surface.d.ts`, and keeps schema version `0.1.0`. The canonical deterministic, LLM, and agent YAML flows all remain valid through the built CLI: + +```text +$ for flow in ../testdata/hello-deterministic.flow.yaml ../testdata/hello-llm.flow.yaml ../testdata/hello-agent.flow.yaml; do node dist/cli.js check --json "$flow"; print "exit=$? flow=$flow"; done +WARNING [unprovable_effects] Step "greet" command "echo" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "shout" command "echo" resolves, but its effects cannot be proven before execution. +{"ok":true,"path":"../testdata/hello-deterministic.flow.yaml","projectConfigPath":"/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/testdata/flows.json","resolutions":[],"diagnostics":[{"severity":"warning","kind":"unprovable_effects","stepId":"greet","message":"Step \"greet\" command \"echo\" resolves, but its effects cannot be proven before execution."},{"severity":"warning","kind":"unprovable_effects","stepId":"shout","message":"Step \"shout\" command \"echo\" resolves, but its effects cannot be proven before execution."}]} +exit=0 flow=../testdata/hello-deterministic.flow.yaml +WARNING [unprovable_effects] Step "greet" command "printf" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "finish" command "printf" resolves, but its effects cannot be proven before execution. +{"ok":true,"path":"../testdata/hello-llm.flow.yaml","projectConfigPath":"/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/testdata/flows.json","resolutions":[{"stepId":"answer","cli":"./preflight/authenticated-cli","source":"project","model":"deterministic-test-stub"}],"diagnostics":[{"severity":"warning","kind":"unprovable_effects","stepId":"greet","message":"Step \"greet\" command \"printf\" resolves, but its effects cannot be proven before execution."},{"severity":"warning","kind":"unprovable_effects","stepId":"finish","message":"Step \"finish\" command \"printf\" resolves, but its effects cannot be proven before execution."}]} +exit=0 flow=../testdata/hello-llm.flow.yaml +WARNING [unprovable_effects] Step "greet" command "printf" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "finish" command "printf" resolves, but its effects cannot be proven before execution. +{"ok":true,"path":"../testdata/hello-agent.flow.yaml","projectConfigPath":"/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/testdata/flows.json","resolutions":[{"stepId":"edit","cli":"./preflight/authenticated-cli","source":"project","model":"test-model-v1"}],"diagnostics":[{"severity":"warning","kind":"unprovable_effects","stepId":"greet","message":"Step \"greet\" command \"printf\" resolves, but its effects cannot be proven before execution."},{"severity":"warning","kind":"unprovable_effects","stepId":"finish","message":"Step \"finish\" command \"printf\" resolves, but its effects cannot be proven before execution."}]} +exit=0 flow=../testdata/hello-agent.flow.yaml +``` + +Issue #132 remains open. This PR implements its declarative YAML/JSON `agents: { cli, model }` slice and honestly says that matching TypeScript `FlowHeader.agents` types remain follow-on work after the separately reviewed surface package. It does not claim the issue's integrated done condition or duplicate the unmerged surface package. + +## Exact GitHub CI + +Current PR state and rollup: + +```text +$ gh pr view 136 --json headRefOid,mergeable,mergeStateStatus,reviewDecision,statusCheckRollup --jq '{headRefOid,mergeable,mergeStateStatus,reviewDecision,checks:[.statusCheckRollup[]|{name:(.name // .context),status,conclusion,detailsUrl}]}' +{"checks":[{"conclusion":"SUCCESS","detailsUrl":"https://github.com/AgentWorkforce/flows/actions/runs/33663176803/job/100358405871","name":"linux-x64-artifact","status":"COMPLETED"},{"conclusion":null,"detailsUrl":null,"name":"CodeRabbit","status":null}],"headRefOid":"060e8b971a04f27785d88e1520aa7c1eb3fd3fc5","mergeStateStatus":"CLEAN","mergeable":"MERGEABLE","reviewDecision":""} +``` + +Exact run/job metadata: + +```text +$ gh run view 33663176803 --json headSha,status,conclusion,url,jobs --jq '{headSha,status,conclusion,url,jobs:[.jobs[]|{name,status,conclusion,steps:[.steps[]|{name,number,status,conclusion}]}]}' +{"conclusion":"success","headSha":"060e8b971a04f27785d88e1520aa7c1eb3fd3fc5","jobs":[{"conclusion":"success","name":"linux-x64-artifact","status":"completed","steps":[{"conclusion":"success","name":"Set up job","number":1,"status":"completed"},{"conclusion":"success","name":"Run actions/checkout@v4","number":2,"status":"completed"},{"conclusion":"success","name":"Run actions/setup-node@v4","number":3,"status":"completed"},{"conclusion":"success","name":"Run oven-sh/setup-bun@v2","number":4,"status":"completed"},{"conclusion":"success","name":"Run dtolnay/rust-toolchain@stable","number":5,"status":"completed"},{"conclusion":"success","name":"Test artifact contract","number":6,"status":"completed"},{"conclusion":"success","name":"Build relayflowd","number":7,"status":"completed"},{"conclusion":"success","name":"Build standalone flows CLI","number":8,"status":"completed"},{"conclusion":"success","name":"Assemble artifact and smoke verifier path","number":9,"status":"completed"},{"conclusion":"success","name":"Smoke exact Linux artifact","number":10,"status":"completed"},{"conclusion":"success","name":"Run actions/upload-artifact@v4","number":11,"status":"completed"},{"conclusion":"success","name":"Post Run oven-sh/setup-bun@v2","number":20,"status":"completed"},{"conclusion":"success","name":"Post Run actions/setup-node@v4","number":21,"status":"completed"},{"conclusion":"success","name":"Post Run actions/checkout@v4","number":22,"status":"completed"},{"conclusion":"success","name":"Complete job","number":23,"status":"completed"}]}],"status":"completed","url":"https://github.com/AgentWorkforce/flows/actions/runs/33663176803"} +``` + +The uploaded artifact is named for the exact head, and its smoke step ran the packaged `relayflowd --help` plus a packaged `flows check --json testdata/hello-deterministic.flow.yaml` that returned `ok:true`. CodeRabbit has no conclusion and was rate-limited; per RFC §2 rule 7 it is not review signal. + +Literal exact-run log filter and captured output: + +```text +$ gh run view 33663176803 --job 100358405871 --log | rg 'relayflow-v2-linux-x64-060e8b971a04f27785d88e1520aa7c1eb3fd3fc5|\{\"ok\":true,\"path\":\"testdata/hello-deterministic.flow.yaml\"' +linux-x64-artifact Smoke exact Linux artifact 2026-09-02T17:49:52.6983089Z {"ok":true,"path":"testdata/hello-deterministic.flow.yaml","projectConfigPath":"/home/runner/work/flows/flows/testdata/flows.json","resolutions":[],"diagnostics":[{"severity":"warning","kind":"unprovable_effects","stepId":"greet","message":"Step \"greet\" command \"echo\" resolves, but its effects cannot be proven before execution."},{"severity":"warning","kind":"unprovable_effects","stepId":"shout","message":"Step \"shout\" command \"echo\" resolves, but its effects cannot be proven before execution."}]} +linux-x64-artifact Run actions/upload-artifact@v4 2026-09-02T17:49:52.7315504Z name: relayflow-v2-linux-x64-060e8b971a04f27785d88e1520aa7c1eb3fd3fc5 +linux-x64-artifact Run actions/upload-artifact@v4 2026-09-02T17:49:56.2509346Z Artifact relayflow-v2-linux-x64-060e8b971a04f27785d88e1520aa7c1eb3fd3fc5.zip successfully finalized. Artifact ID 9859609595 +linux-x64-artifact Run actions/upload-artifact@v4 2026-09-02T17:49:56.2528263Z Artifact relayflow-v2-linux-x64-060e8b971a04f27785d88e1520aa7c1eb3fd3fc5 has been successfully uploaded! Final size is 40560111 bytes. Artifact ID is 9859609595 +``` + +Only this report is staged by this review. + +REVIEW_PASSED diff --git a/ops/reviews/20260902-1950-pr136-structure.md b/ops/reviews/20260902-1950-pr136-structure.md new file mode 100644 index 00000000..d626f29a --- /dev/null +++ b/ops/reviews/20260902-1950-pr136-structure.md @@ -0,0 +1,443 @@ +# PR #136 fresh exact-head structure / API / RFC review + +Date: 2026-09-02 + +Scope: independent assessment only of `AgentWorkforce/flows` PR #136 at exact +head `060e8b971a04f27785d88e1520aa7c1eb3fd3fc5` against merge base +`a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2`. I read `AGENTS.md`, +`docs/RFC-0001-everything-is-a-relayflow.md`, `docs/SURFACE.md`, issue #132, +the PR body/history, the complete changed product surface, and the relevant +tests. I also inspected composition with current PR #134 head +`5092b76decca1530aaf6be0f81897945f9143ce5` and PR #138 head +`22c7d31fbd46db20d95f6fcabe000f36ece7ec52`. No product code was edited. + +## Verdict + +FAIL. The exact head has two runtime/API blockers and does not currently +compose with the validation lane that is supposed to close its exported +boundary. The adapter table and kernel boundary themselves are well-shaped, +and the full deterministic/live SDK suite is green, but those tests do not +exercise the failing public and path-provenance cases below. + +## Blocking findings + +### P1 — the exported `preflight(FlowSpec, ...)` does not accept the new valid named-agent authoring shape + +The PR adds `FlowSpec.agents`, `AgentStepSpec.agent`, and publicly exports +`preflight(flow: FlowSpec, ...)`. A value that `validateSpec` accepts as a valid +`FlowSpec` is nevertheless refused as `cli_unresolved` when handed directly to +that API. `preflight` only understands the hidden post-`compileSpec` shape in +which named CLI/model values have already been copied onto the step. There is +no distinct type or API contract separating those two shapes. + +Running `compileSpec` first makes the same input pass, but introduces a second +API lie: the resolution source is reported as `step`, even though the value was +declared under `agents.reviewer`. This contradicts the documented precedence +`step -> named declaration -> flow/project` and the promise that `flows check` +prints the declaration source. The selector and declaration map are still +present at this point, so the true source is recoverable. + +This is also not validation-first at the exported boundary. A malformed +cross-verb object can cause an environment probe, pass `preflight`, and then +have its invalid field silently removed by `toKernelSpec`. PR #138 is intended +to close that older generic hole, but the exact #136 head does not include the +guard and its new named-agent API depends on the same ambiguous raw/compiled +`FlowSpec` type. + +Required repair: make the public preflight boundary accept and normalize the +documented authoring `FlowSpec` itself, or introduce a distinct validated / +normalized type that cannot be confused with raw authoring input. Validation +must run before every environment probe; named CLI/model resolution must occur +exactly once while retaining a truthful `named` source; lowering must refuse +rather than strip invalid input. + +Literal reproduction (`sdk/`; the preceding `tsc` build was silent and exited +zero): + +```text +$ ./node_modules/.bin/tsc +$ node --input-type=module <<'NODE' +import { compileSpec, preflight, validateSpec } from './dist/index.js'; +const authored = { + version: '0.1.0', + agents: { reviewer: { cli: 'wrapper', model: 'allowed-model' } }, + steps: [{ id: 'review', type: 'agent', agent: 'reviewer', instruction: 'review' }], +}; +const calls = []; +const options = { + models: ['allowed-model'], + probes: { + cli(cli, source, model) { calls.push([cli, source, model ?? null]); return { exists: true, supported: true, authenticated: true, modelAvailable: true }; }, + executor() { return true; }, command() { return true; }, + }, +}; +console.log(JSON.stringify({ validation: validateSpec(authored), direct: preflight(authored, options), directCalls: calls })); +calls.length = 0; +const compiled = compileSpec(authored); +console.log(JSON.stringify({ compiledStep: compiled.steps[0], compiled: preflight(compiled, options), compiledCalls: calls })); +NODE +{"validation":{"ok":true,"errors":[]},"direct":{"ok":false,"resolutions":[],"diagnostics":[{"severity":"refusal","kind":"cli_unresolved","stepId":"review","message":"Step \"review\" has no CLI at step, flow, or project level."}]},"directCalls":[]} +{"compiledStep":{"id":"review","type":"agent","maxIterations":1,"instruction":"review","agent":"reviewer","cli":"wrapper","model":"allowed-model","recoveryMode":"reset"},"compiled":{"ok":true,"resolutions":[{"stepId":"review","cli":"wrapper","source":"step","model":"allowed-model"}],"diagnostics":[]},"compiledCalls":[["wrapper","step","allowed-model"]]} +``` + +Literal generic validation-order reproduction (`sdk/`): + +```text +$ node --input-type=module <<'NODE' +import { preflight, toKernelSpec, validateSpec } from './dist/index.js'; +const malformed = { + version: '0.1.0', + steps: [{ id: 'agent', type: 'agent', instruction: 'work', cli: 'fake-wrapper', command: 'must-not-cross-verbs' }], +}; +const calls = []; +const probes = { + cli(cli, source, model) { calls.push(['cli', cli, source, model ?? null]); return { exists: true, supported: true, authenticated: true }; }, + executor(trigger) { calls.push(['executor', trigger.id]); return true; }, + command(binary) { calls.push(['command', binary]); return true; }, +}; +console.log(JSON.stringify({ validation: validateSpec(malformed), preflight: preflight(malformed, { probes, models: [] }), calls, kernel: toKernelSpec(malformed) })); +NODE +{"validation":{"ok":false,"errors":["spec.steps[0]: unknown key \"command\" (expected one of id | type | dependsOn | verification | maxIterations | timeoutMs | instruction | agent | cli | model | surfaces | recoveryMode | permissions)"]},"preflight":{"ok":true,"resolutions":[{"stepId":"agent","cli":"fake-wrapper","source":"step"}],"diagnostics":[]},"calls":[["cli","fake-wrapper","step",null]],"kernel":{"version":"0.1.0","steps":[{"id":"agent","depends_on":[],"max_iterations":1,"retry":{"initial_backoff_ms":100,"max_backoff_ms":60000,"multiplier":2,"jitter_percent":20},"verification":{},"type":"agent","instruction":"work","cli":"fake-wrapper","recovery_mode":"reset"}]}} +``` + +### P1 — preflight proves one relative executable, but the generic worker invokes the unchanged path from another cwd + +`checkFlow` resolves a relative CLI against the declaring flow or project +config directory and probes that resolved executable. The normalized/kernel +step retains the original relative string. `runAgentCli` then calls +`spawn(cli, ...)` without a cwd or resolved executable, so `AgentWorker` uses +the worker process cwd instead. The shared adapter argv table therefore does +not bind preflight to the executable actually invoked. + +This is a direct RFC covenant-2 violation: a check-green flow can fail later as +`worker_error` for a fact claimed proven at minute zero. It also weakens the +closed wrapper identity contract: the worker may identify a different path (or +no path) than preflight identified. The repository already contains a special +`resolveSpecCliPaths` workaround for hn-monitor whose comment describes this +exact failure, but generic `flows run` / `AgentWorker` does not use it. + +Required repair: carry executable provenance through the checked submission so +worker execution uses the same resolved identity, without putting provider +logic in the kernel. For gate-1 local paths, canonicalizing a declared relative +wrapper before submission is sufficient; the bundle design must later replace +host paths with bundle-owned executable identity. Add an end-to-end test with +the flow file and worker in different directories. + +Literal reproduction (`sdk/`): + +```text +$ ./node_modules/.bin/tsc +$ node --input-type=module <<'NODE' +import { checkFlow } from './dist/cli/check.js'; +import { runAgentCli } from './dist/worker-cli.js'; +const checked = checkFlow('../testdata/preflight/cli-declared.flow.yaml'); +const cli = checked.flow?.steps[0]?.cli; +console.log(JSON.stringify({ cwd: process.cwd(), checkOk: checked.report.ok, resolution: checked.report.resolutions[0], compiledCli: cli })); +console.log(JSON.stringify(await runAgentCli(cli, 'do work', undefined))); +NODE +{"cwd":"/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk","checkOk":true,"resolution":{"stepId":"answer","cli":"./authenticated-cli","source":"step"},"compiledCli":"./authenticated-cli"} +{"exit_code":null,"stdout_tail":"","stderr_tail":"CLI \"./authenticated-cli\" did not identify as relayflows-agent-cli-v1 at worker execution."} +``` + +### P1 stack blocker — current #136 and #138 heads conflict and #138's schema omits both new authoring fields + +PR #138 is the lane that adds validation at public `preflight` and +`toKernelSpec`, which #136 needs for a genuinely validation-first boundary. +The exact current heads do not merge: `sdk/src/validate.ts` has a content +conflict. This is not only mechanical. #138's centralized descriptors omit +`agents` from root fields and omit the `agent` selector from agent-step fields. +A resolution that merely takes #138's descriptor refactor would reject #136's +valid authoring contract, and its new `toKernelSpec(validateSpec(flow))` guard +would make named-agent lowering unusable. + +Required repair: rebase/reconcile the two lanes explicitly. The composed +descriptor must include the closed `{cli, model}` declaration map and agent +selector, retain #136's model-name validation and pure unknown-model pass, and +retain #138's validation-before-probe / validation-before-lowering behavior. +Run both suites plus direct public-boundary and named-agent lowering tests on +the composed tree. + +Literal composition evidence (repository root): + +```text +$ set +e +$ git merge-tree --write-tree refs/remotes/review/pr138 HEAD +d436ad3a3f5e2a6c54c09ce9e304ef9a80ffa152 +100644 dbe0ff9fb42371e43e97378f5fd6aec3c2c33e93 1 sdk/src/validate.ts +100644 740e379b5587d8a08df5250f76f40b1a825b33ad 2 sdk/src/validate.ts +100644 f0403042f0fd7dbceb25e9f38b0d4b5cdf044437 3 sdk/src/validate.ts + +Auto-merging sdk/src/compile.ts +Auto-merging sdk/src/failure-kinds.ts +Auto-merging sdk/src/index.ts +Auto-merging sdk/src/preflight.ts +Auto-merging sdk/src/validate.ts +CONFLICT (content): Merge conflict in sdk/src/validate.ts +Auto-merging sdk/tests/preflight.test.ts +$ printf 'pr138_merge_exit=%s\n' "$?" +pr138_merge_exit=1 +$ printf 'pr138_agent_fields='; git show refs/remotes/review/pr138:sdk/src/step-fields.ts | rg 'agent:' +pr138_agent_fields= agent: ['instruction', 'cli', 'model', 'surfaces', 'recoveryMode', 'permissions'], +$ printf 'pr138_root_keys='; git show refs/remotes/review/pr138:sdk/src/validate.ts | rg 'const ROOT_KEYS' +pr138_root_keys=const ROOT_KEYS = ['version', 'name', 'description', 'cli', 'triggers', 'steps', 'budget'] as const; +``` + +## Non-blocking architecture assessment + +- The compiler-side precedence is otherwise exact and independent: explicit + step CLI/model override the corresponding named value; named CLI precedes + flow/project CLI; model has no flow/project default. Selected values lower + into the existing journaled step fields. +- The named declaration shape is closed to exactly `{cli, model}` and malformed + models are rejected. Unknown allowlist entries are collected in a pure pass + before command, CLI, executor, or daemon probes in the supported + `checkFlow -> compileSpec -> preflight` path. +- The adapter vocabulary is closed in the SDK as `claude | codex | + relayflows-wrapper-v1`. Raw Claude/Codex use native model flags; custom + wrappers must identify with the exact versioned token before receiving the + private model environment variable. The worker repeats wrapper identity + immediately before execution with model and wake-context absent. Installed + provider tests confirm the expected model and non-Git argv paths. +- Provider/model policy remains outside the Rust kernel. There is no diff under + `kernel/` or `sdk/src/protocol.ts`; named authoring metadata is erased at + `toKernelSpec` and only existing per-step `cli`/`model` fields cross the + journal boundary. +- Changed production modules remain below the repository's 500-line smell + threshold. The largest are `validate.ts` (467), `compile.ts` (439), and + `preflight.ts` (424); the new adapter and worker modules are 118 and 110 + lines. I found no unreferenced new production module or exported constant. +- No v1/default runtime, regression flow, or kernel file changes in this diff. + Inline steps without a model preserve optional-model behavior, and the full + SDK/live-kernel suite is green. This is compatibility evidence, not a claim + that the external v1 repository was re-executed. + +## #134 composition and authoring/kernel boundary + +The #134 and #136 heads merge mechanically. The resulting tree preserves the +correct separation: TypeScript surface definitions stay outside the kernel, +and #136's declarative metadata lowers to existing kernel fields. However, the +combined tree still has no `FlowHeader.agents`; #134's `FlowHeader` contains +only identity, memory, budget, tools, and workspace. Thus issue #132 item 4 is +not complete for the TypeScript-first authoring surface. The PR body and docs +state this limitation honestly, so I treat it as a dependency/scope limit, not +an additional standalone blocker or a v1 compatibility regression. + +Literal evidence: + +```text +$ MERGED_134=$(git merge-tree --write-tree refs/remotes/review/pr134 HEAD) +$ printf 'pr134_merged_tree=%s\n' "$MERGED_134" +pr134_merged_tree=a30be7f1cabed1465b990d3d6026feda6e7628c2 +$ git show "$MERGED_134":surface/src/flow.ts | sed -n '1,28p' +import type { Ctx } from "./context.js"; + +/** Optional escalation header; the empty header is the common case. */ +export interface FlowHeader { + identity?: string; + memory?: { script?: boolean; agent?: boolean }; + budget?: string; + tools?: { relayfile?: string[]; mcp?: string[] }; + workspace?: string; +} + +export type FlowBody = (f: Ctx) => Promise; + +export interface ReadonlyFlowHeader { + readonly identity?: string; + readonly memory?: Readonly<{ script?: boolean; agent?: boolean }>; + readonly budget?: string; + readonly tools?: Readonly<{ + relayfile?: readonly string[]; + mcp?: readonly string[]; + }>; + readonly workspace?: string; +} +``` + +## Exact revision, history, diff, and CI + +Command (repository root): + +```text +$ git fetch origin main pull/136/head:refs/remotes/review/pr136 pull/134/head:refs/remotes/review/pr134 pull/138/head:refs/remotes/review/pr138 +$ git rev-parse HEAD +060e8b971a04f27785d88e1520aa7c1eb3fd3fc5 +$ git rev-parse refs/remotes/review/pr136 +060e8b971a04f27785d88e1520aa7c1eb3fd3fc5 +$ git rev-parse origin/main +a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2 +$ git merge-base origin/main HEAD +a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2 +$ git log --oneline --decorate --reverse a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..HEAD +321b272 feat(sdk): add declared agent model contract +78efc0d docs(review): record PR 136 fresh review +4888d15 (origin/pr-136) fix(sdk): make model adapters fail closed +060e8b9 (HEAD -> feat/v2-declared-model, review/pr136, origin/feat/v2-declared-model) fix(sdk): bind declared model execution checks +``` + +Command and captured output: + +```text +$ git diff --stat a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..HEAD + docs/SURFACE.md | 84 ++- + ops/reviews/20260902-1710-pr136-history.md | 449 ++++++++++++++ + ops/reviews/20260902-1710-pr136-maintainability.md | 357 +++++++++++ + ops/reviews/20260902-1710-pr136-structure.md | 261 +++++++++ + ops/reviews/20260902-1905-pr136-history.md | 652 +++++++++++++++++++++ + ops/reviews/20260902-1905-pr136-maintainability.md | 387 ++++++++++++ + ops/reviews/20260902-1905-pr136-structure.md | 294 ++++++++++ + ops/reviews/20260902-1945-pr136-repair.md | 118 ++++ + sdk/src/cli-adapter.ts | 118 ++++ + sdk/src/cli.ts | 3 +- + sdk/src/cli/check.ts | 109 +++- + sdk/src/compile.ts | 33 +- + sdk/src/failure-kinds.ts | 3 + + sdk/src/index.ts | 2 + + sdk/src/model-name.ts | 20 + + sdk/src/preflight.ts | 111 +++- + sdk/src/spec.ts | 24 +- + sdk/src/unknown-keys.ts | 52 ++ + sdk/src/validate.ts | 102 ++-- + sdk/src/worker-cli.ts | 110 ++++ + sdk/src/worker.ts | 116 +--- + sdk/tests/cli-adapter.test.ts | 64 ++ + sdk/tests/cli.test.ts | 221 ++++++- + sdk/tests/live-kernel.test.ts | 150 ++++- + sdk/tests/model-selection.test.ts | 161 +++++ + sdk/tests/preflight.test.ts | 121 +++- + sdk/tests/real-cli-adapters.test.ts | 68 +++ + testdata/flows.json | 8 +- + testdata/preflight/analyze-story-claude-cli | 13 +- + testdata/preflight/analyze-story-echo-wake-cli | 4 + + testdata/preflight/analyze-story-missing-fields-cli | 4 + + testdata/preflight/analyze-story-stub-cli | 4 + + testdata/preflight/analyze-story-text-only-cli | 4 + + testdata/preflight/authenticated-cli | 4 + + testdata/preflight/counting-cli | 4 + + testdata/preflight/echo-model-cli | 4 + + testdata/preflight/signal-probe-cli | 4 + + testdata/preflight/unauthenticated-cli | 4 + + testdata/preflight/wake-context-probe-cli | 4 + + 39 files changed, 4036 insertions(+), 215 deletions(-) +$ git diff --check a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2..HEAD +# no output +$ git diff --name-only origin/main...HEAD -- kernel sdk/src/protocol.ts +# no output +``` + +Current GitHub status at the reviewed head: + +```text +$ gh pr checks 136 --repo AgentWorkforce/flows +CodeRabbit pass 0 Review rate limited +linux-x64-artifact pass 3m6s https://github.com/AgentWorkforce/flows/actions/runs/33663176803/job/100358405871 +$ gh pr view 136 --repo AgentWorkforce/flows --json headRefOid,mergeStateStatus,reviewDecision,statusCheckRollup +{"headRefOid":"060e8b971a04f27785d88e1520aa7c1eb3fd3fc5","mergeStateStatus":"CLEAN","reviewDecision":"","statusCheckRollup":[{"__typename":"CheckRun","completedAt":"2026-09-02T17:49:59Z","conclusion":"SUCCESS","detailsUrl":"https://github.com/AgentWorkforce/flows/actions/runs/33663176803/job/100358405871","name":"linux-x64-artifact","startedAt":"2026-09-02T17:46:53Z","status":"COMPLETED","workflowName":"Relayflow v2 Cloud runtime artifact"},{"__typename":"StatusContext","context":"CodeRabbit","startedAt":"2026-09-02T17:46:51Z","state":"SUCCESS","targetUrl":""}]} +``` + +## Verification + +Full serial SDK/live-kernel suite (`sdk/`): + +```text +$ RELAYFLOWD_BIN=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 ./node_modules/.bin/vitest run --reporter=dot --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd +LIVE_KERNEL flows=/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk/dist/cli.js + +stdout | tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +LIVE_ANALYZER ready: claude -p --model claude-haiku-4-5-20251001 round-trip OK +LIVE_ANALYZER analysis: {"reasoning":"This story directly demonstrates an AI agent autonomously managing the software development lifecycle by opening and reviewing pull requests, which is a foundational use case for AI agents and automation in code development workflows.","relevance_score":10,"story_title":"Show HN: an agent that opens and reviews its own pull requests [wake-nonce-7f3a91c4]"} + +stdout | tests/live-kernel.test.ts > surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once +LIVE_KERNEL kill -9 pid=7817 run=01M1HM7F21X9XSR109252ZPGA8 while step=two state=Running + + ✓ tests/live-kernel.test.ts (20 tests) 69392ms + ✓ tests/cli.test.ts (62 tests) 4027ms + ✓ tests/journal-client.test.ts (13 tests) 179ms + ✓ tests/preflight.test.ts (20 tests) 86ms + ✓ tests/validate.test.ts (36 tests) 126ms + ✓ tests/cli-hn-monitor.test.ts (16 tests) 144ms + ✓ tests/backlog-picker.test.ts (14 tests) 205ms +SKIPPED_UNACTIONABLE=0 +SKIPPED_UNACTIONABLE=0 +NO_ACTIONABLE_BACKLOG_ENTRY scanned=0 +node:fs:539 + return binding.readFileUtf8(path, stringToFlags(options.flag)); + ^ + +Error: ENOENT: no such file or directory, open '.relayflow/backlog-picker-entry.json' + at Object.readFileSync (node:fs:539:20) + at [eval]:1:478 + at runScriptInThisContext (node:internal/vm:219:10) + at node:internal/process/execution:483:12 + at [eval]-wrapper:6:24 + at runScriptInContext (node:internal/process/execution:481:60) + at evalFunction (node:internal/process/execution:315:30) + at evalTypeScript (node:internal/process/execution:327:3) + at node:internal/main/eval_string:71:3 { + errno: -2, + code: 'ENOENT', + syscall: 'open', + path: '.relayflow/backlog-picker-entry.json' +} + +Node.js v26.7.0 +SKIPPED_UNACTIONABLE=0 + ✓ tests/backlog-picker-flow.test.ts (6 tests) 1401ms +SKIPPED_UNACTIONABLE=0 +NO_ACTIONABLE_BACKLOG_ENTRY scanned=1 Scoped but unverifiable[missing_definition_of_done] + ✓ tests/work-package-consumer.test.ts (13 tests) 608ms + ✓ tests/model-selection.test.ts (10 tests) 145ms + ✓ tests/deterministic-llm.test.ts (5 tests) 48ms + ✓ tests/bin.test.ts (7 tests) 2079ms + ✓ tests/hn-poller.test.ts (6 tests) 13ms + ✓ tests/dir-watcher-poller.test.ts (6 tests) 11ms + ✓ tests/hello-deterministic.test.ts (5 tests) 48ms + ✓ tests/work-package-validator.test.ts (7 tests) 17ms + ✓ tests/spec-parity.test.ts (15 tests) 113ms + ↓ tests/real-cli-adapters.test.ts (3 tests | 3 skipped) + ✓ tests/parse-json-output.test.ts (7 tests) 9ms + ✓ tests/cli-adapter.test.ts (3 tests) 11ms + + Test Files 19 passed | 1 skipped (20) + Tests 271 passed | 3 skipped (274) + Start at 19:55:07 + Duration 95.62s (transform 1.13s, setup 0ms, collect 3.14s, tests 78.66s, environment 11ms, prepare 4.18s) + +exit_code=0 +``` + +The ENOENT stack is the suite's expected backlog-picker negative-path child; +Vitest exited zero. The opt-in provider file skipped above was run separately: + +```text +$ RELAYFLOWS_REAL_CLI_ADAPTERS=1 ./node_modules/.bin/vitest run tests/real-cli-adapters.test.ts --reporter=verbose --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + + ✓ tests/real-cli-adapters.test.ts > installed raw CLI adapters > round-trips the exact declared Claude model and refuses an impossible one 8380ms + ✓ tests/real-cli-adapters.test.ts > installed raw CLI adapters > uses Codex login status and classifies an impossible model as unavailable 5124ms + ✓ tests/real-cli-adapters.test.ts > installed raw CLI adapters > executes the declared Codex model from a real non-Git directory 5395ms + + Test Files 1 passed (1) + Tests 3 passed (3) + Start at 19:56:50 + Duration 19.79s (transform 241ms, setup 0ms, collect 343ms, tests 18.91s, environment 0ms, prepare 175ms) + +exit_code=0 +``` + +Final worktree check before staging the assigned report: + +```text +$ git status --short --branch --untracked-files=all +## feat/v2-declared-model...origin/feat/v2-declared-model +$ git diff HEAD --check +# no output +``` + +REVIEW_FAILED From 30f23248f897b5f8fc7f87ca23c7e37ede6a653f Mon Sep 17 00:00:00 2001 From: kjgbot Date: Wed, 2 Sep 2026 20:29:10 +0200 Subject: [PATCH 06/15] test(sdk): cover public named-agent boundaries Session-Id: 01a062b7-2aef-7772-8225-7cb00ee311dd --- sdk/tests/cli.test.ts | 35 +++++++++++++++++++++++ sdk/tests/preflight.test.ts | 56 +++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) diff --git a/sdk/tests/cli.test.ts b/sdk/tests/cli.test.ts index 3d7a126d..6471dee0 100644 --- a/sdk/tests/cli.test.ts +++ b/sdk/tests/cli.test.ts @@ -7,7 +7,9 @@ import { fileURLToPath } from 'node:url'; import { parse as parseYaml, stringify as stringifyYaml } from 'yaml'; import { afterEach, describe, expect, it } from 'vitest'; import { runCli, type CheckReport, type CliIo } from '../src/cli.js'; +import { checkFlow } from '../src/cli/check.js'; import { runFlow } from '../src/cli/run.js'; +import { runAgentCli } from '../src/worker-cli.js'; import { CHECK_INPUT_FAILURE_KINDS, isCheckFailureKind, @@ -142,6 +144,39 @@ async function startCliLoopback(dataDir: string, handlers: LoopbackHandlers): Pr } describe('flows check CLI', () => { + it('binds a checked relative wrapper to the flow directory for worker execution', async () => { + const directory = temporaryProject('flows-relative-worker-'); + const wrapper = join(directory, 'wrapper'); + writeFileSync(wrapper, `#!/bin/sh +if [ "\${1-}" = "--relayflows-adapter-v1" ]; then + printf '%s\\n' relayflows-agent-cli-v1 + exit 0 +fi +if [ "\${1-} \${2-}" = "auth status" ]; then exit 0; fi +printf '%s' '{"executed":true}' +`); + chmodSync(wrapper, 0o755); + const path = join(directory, 'relative.flow.yaml'); + writeFileSync(path, ` +version: '0.1.0' +steps: + - id: work + type: agent + cli: ./wrapper + instruction: Work from another directory. +`); + + const checked = checkFlow(path); + expect(checked.report.ok).toBe(true); + expect(checked.flow?.steps[0]).toHaveProperty('cli', wrapper); + const result = await runAgentCli( + (checked.flow?.steps[0] as { cli?: string } | undefined)?.cli ?? '', + 'Work from another directory.', + undefined, + ); + expect(result).toMatchObject({ exit_code: 0, stdout_tail: '{"executed":true}' }); + }); + it.each(['unused', 'shadowed'] as const)( 'refuses an unknown %s named-agent declaration before compilation erases it', async (variant) => { diff --git a/sdk/tests/preflight.test.ts b/sdk/tests/preflight.test.ts index d86a71e3..17212e97 100644 --- a/sdk/tests/preflight.test.ts +++ b/sdk/tests/preflight.test.ts @@ -26,6 +26,62 @@ function probes(overrides: Partial = {}): PreflightProbes { } describe('preflight: CLI resolution and refusal predicates', () => { + it('validates and resolves a raw named-agent authoring spec at the public boundary', () => { + const calls: Array<[string, string, string | undefined]> = []; + const authored: FlowSpec = { + version: '0.1.0', + agents: { reviewer: { cli: 'wrapper', model: 'allowed-model' } }, + steps: [{ id: 'review', type: 'agent', agent: 'reviewer', instruction: 'Review.' }], + }; + + const result = preflight(authored, { + models: ['allowed-model'], + probes: probes({ + cli: (cli, source, model) => { + calls.push([cli, source, model]); + return { exists: true, supported: true, authenticated: true, modelAvailable: true }; + }, + }), + }); + + expect(result.ok).toBe(true); + expect(result.resolutions).toEqual([{ + stepId: 'review', + cli: 'wrapper', + source: 'named', + model: 'allowed-model', + }]); + expect(calls).toEqual([['wrapper', 'named', 'allowed-model']]); + }); + + it('refuses malformed raw input before any public preflight probe', () => { + const calls: string[] = []; + const result = preflight({ + version: '0.1.0', + steps: [{ + id: 'agent', + type: 'agent', + instruction: 'Work.', + cli: 'wrapper', + command: 'must-not-cross-verbs', + }], + } as never, { + probes: { + cli: () => { calls.push('cli'); return { exists: true, authenticated: true }; }, + command: () => { calls.push('command'); return true; }, + executor: () => { calls.push('executor'); return true; }, + }, + }); + + expect(result).toMatchObject({ + ok: false, + resolutions: [], + diagnostics: [{ severity: 'refusal', kind: 'invalid_spec' }], + }); + expect(result.diagnostics[0]?.message).toContain('unknown key "command"'); + expect(calls).toEqual([]); + }); + it('resolves step, then flow, then project without guessing a platform default', () => { const seen: string[] = []; const check = (spec: FlowSpec, projectCli?: string) => preflight(spec, { From e50d977bf8f6bb2a6498e4739f6e2d80029ccd87 Mon Sep 17 00:00:00 2001 From: kjgbot Date: Wed, 2 Sep 2026 18:24:13 +0200 Subject: [PATCH 07/15] fix(sdk): refuse fields outside step verb schemas Session-Id: 01a062df-b92c-7a13-815f-147a82a4fc51 Session-Id: 01a062df-b92c-7a13-815f-147a82a4fc51 Session-Id: 01a062b7-2aef-7772-8225-7cb00ee311dd --- sdk/src/compile.ts | 2 + sdk/src/failure-kinds.ts | 16 ++- sdk/src/index.ts | 2 + sdk/src/preflight.ts | 16 +++ sdk/tests/preflight.test.ts | 4 + sdk/tests/verb-field-lint.test.ts | 226 ++++++++++++++++++++++++++++++ 6 files changed, 262 insertions(+), 4 deletions(-) create mode 100644 sdk/tests/verb-field-lint.test.ts diff --git a/sdk/src/compile.ts b/sdk/src/compile.ts index 6ad43c1c..867ea134 100644 --- a/sdk/src/compile.ts +++ b/sdk/src/compile.ts @@ -174,6 +174,8 @@ const KERNEL_RETRY_DEFAULTS = { * sugar that the dialect cannot carry is a `CompileError`, never a silent drop. */ export function toKernelSpec(flow: FlowSpec): KernelRunSpec { + const validation = validateSpec(flow); + if (!validation.ok) throw new CompileError(validation.errors); return { version: flow.version, ...(flow.name !== undefined ? { name: flow.name } : {}), diff --git a/sdk/src/failure-kinds.ts b/sdk/src/failure-kinds.ts index 312ae655..ef36508e 100644 --- a/sdk/src/failure-kinds.ts +++ b/sdk/src/failure-kinds.ts @@ -1,5 +1,7 @@ -/** Closed refusal taxonomy for `flows check` (RFC covenant 2). */ -export const PREFLIGHT_FAILURE_KINDS = [ +const SHARED_SPEC_FAILURE_KINDS = ['invalid_spec'] as const; + +/** Environment refusal kinds produced after spec validation succeeds. */ +const PREFLIGHT_ENVIRONMENT_FAILURE_KINDS = [ 'cli_missing', 'cli_unauthenticated', 'cli_unresolved', @@ -11,17 +13,23 @@ export const PREFLIGHT_FAILURE_KINDS = [ 'probe_failed', ] as const; +/** Closed refusal taxonomy for public preflight (RFC covenant 2). */ +export const PREFLIGHT_FAILURE_KINDS = [ + ...SHARED_SPEC_FAILURE_KINDS, + ...PREFLIGHT_ENVIRONMENT_FAILURE_KINDS, +] as const; + /** Input/command refusals emitted before the pure preflight predicates run. */ export const CHECK_INPUT_FAILURE_KINDS = [ 'config_invalid', 'input_unreadable', 'invalid_invocation', - 'invalid_spec', + ...SHARED_SPEC_FAILURE_KINDS, ] as const; export const CHECK_FAILURE_KINDS = [ ...CHECK_INPUT_FAILURE_KINDS, - ...PREFLIGHT_FAILURE_KINDS, + ...PREFLIGHT_ENVIRONMENT_FAILURE_KINDS, ] as const; /** diff --git a/sdk/src/index.ts b/sdk/src/index.ts index af44bc20..6f42de6b 100644 --- a/sdk/src/index.ts +++ b/sdk/src/index.ts @@ -58,7 +58,9 @@ export { type PreflightDiagnostic, type PreflightOptions, type PreflightProbes, + type PreflightRefusal, type PreflightResult, + type PreflightWarning, } from './preflight.js'; export { CHECK_FAILURE_KINDS, diff --git a/sdk/src/preflight.ts b/sdk/src/preflight.ts index 1a013d01..c6b7bc89 100644 --- a/sdk/src/preflight.ts +++ b/sdk/src/preflight.ts @@ -3,6 +3,7 @@ import type { PreflightFailureKind, PreflightWarningKind, } from './failure-kinds.js'; +import { validateSpec } from './validate.js'; export type CliResolutionSource = 'step' | 'flow' | 'project'; @@ -78,6 +79,8 @@ export interface PreflightRefusal { triggerId?: string; executor?: string; detail?: CliProbeFailureDetail; + /** Author-facing validation errors when kind is `invalid_spec`. */ + errors?: string[]; } export interface PreflightWarning { @@ -96,6 +99,19 @@ export interface PreflightResult { } export function preflight(flow: FlowSpec, options: PreflightOptions): PreflightResult { + const validation = validateSpec(flow); + if (!validation.ok) { + return { + ok: false, + resolutions: [], + diagnostics: [{ + severity: 'refusal', + kind: 'invalid_spec', + message: `Relayflow spec is invalid: ${validation.errors.join('; ')}`, + errors: validation.errors, + }], + }; + } const diagnostics: PreflightDiagnostic[] = []; const resolutions: CliResolution[] = []; const cliProbeResults = new Map(); diff --git a/sdk/tests/preflight.test.ts b/sdk/tests/preflight.test.ts index 17212e97..27a6698f 100644 --- a/sdk/tests/preflight.test.ts +++ b/sdk/tests/preflight.test.ts @@ -277,6 +277,10 @@ describe('preflight: CLI resolution and refusal predicates', () => { // not read as coming from this test alone. it('reaches every declared refusal kind, with the converse held by the type', () => { const scenarios = [ + preflight({ + version: '0.1.0', + steps: [{ id: 'a', type: 'deterministic', command: 'x', prompt: 'cross-verb' }], + }, { probes: probes() }), preflight(flow({ id: 'a', type: 'llm', prompt: 'p', cli: 'x' }), { probes: probes({ cli: () => ({ exists: false, authenticated: false }) }) }), preflight(flow({ id: 'a', type: 'llm', prompt: 'p', cli: 'x' }), { probes: probes({ cli: () => ({ exists: true, authenticated: false }) }) }), preflight(flow({ id: 'a', type: 'llm', prompt: 'p', cli: 'x' }), { probes: probes({ cli: () => ({ exists: true, supported: false, authenticated: false }) }) }), diff --git a/sdk/tests/verb-field-lint.test.ts b/sdk/tests/verb-field-lint.test.ts new file mode 100644 index 00000000..f185b899 --- /dev/null +++ b/sdk/tests/verb-field-lint.test.ts @@ -0,0 +1,226 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { stringify as stringifyYaml } from 'yaml'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + CompileError, + compileSpec, + compileYaml, + toKernelSpec, +} from '../src/compile.js'; +import { runCli, type CliIo } from '../src/cli.js'; +import { PREFLIGHT_FAILURE_KINDS } from '../src/failure-kinds.js'; +import { + preflight, + type PreflightProbes, +} from '../src/preflight.js'; +import type { FlowSpec } from '../src/spec.js'; +import { validateSpec } from '../src/validate.js'; + +type RawSpec = Record & { + steps: Array>; +}; + +const INVALID_STEP_FIELDS = [ + { + label: 'deterministic typo', + step: { id: 'work', type: 'deterministic', command: 'printf ok', commnad: 'printf wrong' }, + unknown: 'commnad', + suggestion: 'command', + }, + { + label: 'deterministic cross-verb field', + step: { id: 'work', type: 'deterministic', command: 'printf ok', prompt: 'not deterministic' }, + unknown: 'prompt', + }, + { + label: 'llm typo', + step: { id: 'work', type: 'llm', prompt: 'answer', cli: 'test-cli', promt: 'misspelled' }, + unknown: 'promt', + suggestion: 'prompt', + }, + { + label: 'llm cross-verb field', + step: { id: 'work', type: 'llm', prompt: 'answer', cli: 'test-cli', instruction: 'not llm' }, + unknown: 'instruction', + }, + { + label: 'agent typo', + step: { id: 'work', type: 'agent', instruction: 'act', cli: 'test-cli', instructon: 'misspelled' }, + unknown: 'instructon', + suggestion: 'instruction', + }, + { + label: 'agent cross-verb field', + step: { id: 'work', type: 'agent', instruction: 'act', cli: 'test-cli', command: 'not agent' }, + unknown: 'command', + }, +] as const; + +const temporaryDirectories: string[] = []; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function specWith(step: Record): RawSpec { + return { version: '0.1.0', name: 'field-lint', steps: [step] }; +} + +function expectedUnknownField(case_: (typeof INVALID_STEP_FIELDS)[number]): string { + return case_.suggestion === undefined + ? `spec.steps[0]: unknown key "${case_.unknown}"` + : `spec.steps[0]: unknown key "${case_.unknown}" — did you mean "${case_.suggestion}"?`; +} + +function probes(onProbe: () => void): PreflightProbes { + return { + cli: () => { + onProbe(); + return { exists: true, authenticated: true }; + }, + executor: () => { + onProbe(); + return true; + }, + command: () => { + onProbe(); + return true; + }, + }; +} + +describe('closed per-verb step fields', () => { + it.each(INVALID_STEP_FIELDS)('$label is rejected by every public compiler/validator path', (case_) => { + const raw = specWith({ ...case_.step }); + const expected = expectedUnknownField(case_); + + const validation = validateSpec(raw); + expect(validation.ok).toBe(false); + expect(validation.errors.join('\n')).toContain(expected); + + expect(() => compileSpec(raw)).toThrow(CompileError); + expect(() => compileSpec(raw)).toThrow(expected); + expect(() => compileYaml(stringifyYaml(raw))).toThrow(CompileError); + expect(() => compileYaml(stringifyYaml(raw))).toThrow(expected); + + // toKernelSpec is exported and callable directly by JavaScript consumers. + // It must not silently strip fields just because TypeScript callers would + // normally have passed through compileSpec first. + expect(() => toKernelSpec(raw as never)).toThrow(CompileError); + expect(() => toKernelSpec(raw as never)).toThrow(expected); + }); + + it.each(INVALID_STEP_FIELDS)('$label is a typed direct-preflight refusal before any probe', (case_) => { + const raw = specWith({ ...case_.step }); + let probeCount = 0; + + const result = preflight(raw as never, { + probes: probes(() => { probeCount += 1; }), + }); + + expect(result).toEqual({ + ok: false, + resolutions: [], + diagnostics: [{ + severity: 'refusal', + kind: 'invalid_spec', + message: expect.stringContaining(expectedUnknownField(case_)), + errors: expect.arrayContaining([expect.stringContaining(expectedUnknownField(case_))]), + }], + }); + expect(probeCount).toBe(0); + expect(PREFLIGHT_FAILURE_KINDS).toContain('invalid_spec'); + }); + + it.each(INVALID_STEP_FIELDS)('$label is refused by flows check with the typed invalid_spec kind', async (case_) => { + const directory = mkdtempSync(join(tmpdir(), 'flows-field-lint-')); + temporaryDirectories.push(directory); + writeFileSync(join(directory, 'flows.json'), JSON.stringify({ executors: [] })); + const path = join(directory, 'invalid.flow.yaml'); + writeFileSync(path, stringifyYaml(specWith({ ...case_.step }))); + const stdout: string[] = []; + const stderr: string[] = []; + const io: CliIo = { + stdout: (line) => stdout.push(line), + stderr: (line) => stderr.push(line), + }; + + const exitCode = await runCli(['check', path], io); + + expect(exitCode).toBe(2); + expect(stdout).toEqual([]); + expect(stderr.join('\n')).toContain('REFUSED [invalid_spec]'); + expect(stderr.join('\n')).toContain(expectedUnknownField(case_)); + }); + + it('preserves a valid v0.1.0 ladder with every declared per-verb field', () => { + const valid: FlowSpec = { + version: '0.1.0', + name: 'valid-v1', + cli: 'flow-cli', + steps: [ + { + id: 'prepare', + type: 'deterministic', + command: 'printf ready', + dependsOn: [], + verification: { type: 'exit_code' }, + maxIterations: 2, + timeoutMs: 5_000, + }, + { + id: 'answer', + type: 'llm', + prompt: 'answer', + model: 'project-model', + cli: 'llm-cli', + dependsOn: ['prepare'], + verification: { type: 'output_contains', value: 'done' }, + maxIterations: 2, + }, + { + id: 'act', + type: 'agent', + instruction: 'act', + model: 'project-model', + cli: 'agent-cli', + dependsOn: ['answer'], + verification: { type: 'json_schema', schema: { type: 'object' } }, + maxIterations: 2, + surfaces: { + workspace: [{ surface: 'worktree' }], + streams: [{ stream: 'updates' }], + external: ['/github/pulls/create.json'], + }, + recoveryMode: 'inspect', + permissions: { + fileGlobs: ['src/**'], + networkAllowlist: ['example.com'], + accessPreset: 'readwrite', + }, + }, + ], + budget: { maxTokensIn: 100, maxTokensOut: 50, maxDollars: '1.50' }, + }; + + expect(validateSpec(valid)).toEqual({ ok: true, errors: [] }); + expect(compileSpec(valid).steps).toHaveLength(3); + expect(compileYaml(stringifyYaml(valid)).steps).toHaveLength(3); + expect(toKernelSpec(valid).steps.map((step) => step.type)).toEqual([ + 'deterministic', + 'llm', + 'agent', + ]); + + const result = preflight(valid, { probes: probes(() => {}) }); + expect(result.ok).toBe(true); + expect(result.resolutions).toEqual([ + { stepId: 'answer', cli: 'llm-cli', source: 'step', model: 'project-model' }, + { stepId: 'act', cli: 'agent-cli', source: 'step', model: 'project-model' }, + ]); + }); +}); From 866d39978bb10fa8be007728f42075ae35a4823c Mon Sep 17 00:00:00 2001 From: kjgbot Date: Wed, 2 Sep 2026 19:28:32 +0200 Subject: [PATCH 08/15] fix(sdk): harden malformed step validation Session-Id: 01a062df-b92c-7a13-815f-147a82a4fc51 Session-Id: 01a062b7-2aef-7772-8225-7cb00ee311dd --- sdk/src/step-dependencies.ts | 63 +++++++++++ sdk/src/step-fields.ts | 37 +++++++ sdk/src/validate.ts | 60 ++--------- sdk/tests/verb-field-lint.test.ts | 171 +++++++++++++++++++++++++++--- 4 files changed, 265 insertions(+), 66 deletions(-) create mode 100644 sdk/src/step-dependencies.ts create mode 100644 sdk/src/step-fields.ts diff --git a/sdk/src/step-dependencies.ts b/sdk/src/step-dependencies.ts new file mode 100644 index 00000000..4dbfb825 --- /dev/null +++ b/sdk/src/step-dependencies.ts @@ -0,0 +1,63 @@ +/** Return author-facing dependency errors without assuming parsed step shapes. */ +export function stepDependencyErrors( + steps: readonly unknown[], + knownIds: ReadonlySet, +): string[] { + const errors: string[] = []; + const adjacency = new Map(); + + for (const value of steps) { + // The main validator reports the shape error. This pass must not replace + // that typed result by dereferencing or iterating a malformed value. + if (!isObject(value) || !isNonEmptyString(value['id'])) continue; + const rawDependencies = value['dependsOn']; + if ( + rawDependencies !== undefined + && (!Array.isArray(rawDependencies) || !rawDependencies.every(isNonEmptyString)) + ) { + continue; + } + + const id = value['id']; + const dependencies = (rawDependencies ?? []) as string[]; + for (const dependency of dependencies) { + if (!knownIds.has(dependency)) { + errors.push(`spec.steps: step "${id}" dependsOn unknown step "${dependency}"`); + } + } + adjacency.set(id, dependencies); + } + + const WHITE = 0, GRAY = 1, BLACK = 2; + const color = new Map(); + for (const id of adjacency.keys()) color.set(id, WHITE); + const stack: string[] = []; + const visit = (id: string): void => { + color.set(id, GRAY); + stack.push(id); + for (const dependency of adjacency.get(id) ?? []) { + const dependencyColor = color.get(dependency); + if (dependencyColor === GRAY) { + errors.push( + `spec.steps: dependency cycle detected at "${dependency}" (path: ${[...stack].join(' -> ')} -> ${dependency})`, + ); + } else if (dependencyColor === WHITE) { + visit(dependency); + } + } + stack.pop(); + color.set(id, BLACK); + }; + for (const id of adjacency.keys()) { + if (color.get(id) === WHITE) visit(id); + } + return errors; +} + +function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isNonEmptyString(value: unknown): value is string { + return typeof value === 'string' && value.length > 0; +} diff --git a/sdk/src/step-fields.ts b/sdk/src/step-fields.ts new file mode 100644 index 00000000..0c8f88b2 --- /dev/null +++ b/sdk/src/step-fields.ts @@ -0,0 +1,37 @@ +import type { StepType } from './spec.js'; + +/** Closed top-level authoring schema, including named agent declarations. */ +export const FLOW_FIELDS = [ + 'version', + 'name', + 'description', + 'cli', + 'agents', + 'triggers', + 'steps', + 'budget', +] as const; + +/** Closed named-agent declaration schema. */ +export const AGENT_DECLARATION_FIELDS = ['cli', 'model'] as const; + +/** Fields shared by every authoring step, regardless of its verb. */ +export const STEP_COMMON_FIELDS = [ + 'id', + 'type', + 'dependsOn', + 'verification', + 'maxIterations', + 'timeoutMs', +] as const; + +/** + * Closed verb-specific authoring schema (RFC-0001 decision 13). Validation + * consumes this descriptor directly so a new optional field cannot bypass the + * per-verb boundary through a second, drifting allowlist. + */ +export const STEP_FIELDS_BY_TYPE = { + deterministic: ['command'], + llm: ['prompt', 'model', 'cli'], + agent: ['instruction', 'agent', 'cli', 'model', 'surfaces', 'recoveryMode', 'permissions'], +} as const satisfies Record; diff --git a/sdk/src/validate.ts b/sdk/src/validate.ts index f0403042..11797d6e 100644 --- a/sdk/src/validate.ts +++ b/sdk/src/validate.ts @@ -12,7 +12,6 @@ import type { NamedAgentSpec, PermissionsSpec, RecoveryMode, - StepSpec, StepType, TriggerSpec, VerificationSpec, @@ -20,6 +19,13 @@ import type { import { SPEC_SCHEMA_VERSION } from './spec.js'; import { modelNameError } from './model-name.js'; import { unknownKeyErrors } from './unknown-keys.js'; +import { stepDependencyErrors } from './step-dependencies.js'; +import { + AGENT_DECLARATION_FIELDS, + FLOW_FIELDS, + STEP_COMMON_FIELDS, + STEP_FIELDS_BY_TYPE, +} from './step-fields.js'; export interface ValidationResult { ok: boolean; @@ -44,15 +50,7 @@ const DECIMAL_RE = /^\d+(\.\d+)?$/; // unknown keys (AGENTS.md rule 4; RFC covenant 2): a typo'd key like // `depends_on` must be an error naming the nearest valid key, never a // silently discarded field — silently dropping `dependsOn` loses ordering. -const ROOT_KEYS = ['version', 'name', 'description', 'cli', 'agents', 'triggers', 'steps', 'budget'] as const; -const AGENT_DECLARATION_KEYS = ['cli', 'model'] as const; const BUDGET_KEYS = ['maxTokensIn', 'maxTokensOut', 'maxDollars'] as const; -const STEP_COMMON_KEYS = ['id', 'type', 'dependsOn', 'verification', 'maxIterations', 'timeoutMs'] as const; -const STEP_TYPE_KEYS: Record = { - deterministic: ['command'], - llm: ['prompt', 'model', 'cli'], - agent: ['instruction', 'agent', 'cli', 'model', 'surfaces', 'recoveryMode', 'permissions'], -}; const VERIFICATION_KEYS: Record = { exit_code: ['type', 'expect'], output_contains: ['type', 'value'], @@ -102,7 +100,7 @@ class Validator { return this.result(); } const s = spec as Record; - this.checkKeys(s, ROOT_KEYS, 'spec'); + this.checkKeys(s, FLOW_FIELDS, 'spec'); if (!isNonEmptyString(s['version'])) { this.fail(`spec.version: expected supported version "${SPEC_SCHEMA_VERSION}"`); @@ -139,7 +137,7 @@ class Validator { } // Dependents must reference real step ids and form a DAG (no cycles). - this.validateDeps(steps as StepSpec[]); + for (const error of stepDependencyErrors(steps, this.ids)) this.fail(error); return this.result(); } @@ -159,7 +157,7 @@ class Validator { this.fail(`${at}: expected an object with cli and model`); continue; } - this.checkKeys(raw, AGENT_DECLARATION_KEYS, at); + this.checkKeys(raw, AGENT_DECLARATION_FIELDS, at); const declaration = raw as unknown as NamedAgentSpec; if (!isNonEmptyString(declaration.cli) || declaration.cli !== declaration.cli.trim()) { this.fail(`${at}.cli: expected a non-empty trimmed string`); @@ -242,7 +240,7 @@ class Validator { return; } const type = st['type'] as StepType; - this.checkKeys(st, [...STEP_COMMON_KEYS, ...STEP_TYPE_KEYS[type]], at); + this.checkKeys(st, [...STEP_COMMON_FIELDS, ...STEP_FIELDS_BY_TYPE[type]], at); if (st['dependsOn'] !== undefined) { if (!Array.isArray(st['dependsOn']) || !(st['dependsOn'] as unknown[]).every(isNonEmptyString)) { @@ -405,42 +403,6 @@ class Validator { } } - private validateDeps(steps: StepSpec[]): void { - const known = this.ids; - const adj = new Map(); - for (const step of steps) { - const deps = step.dependsOn ?? []; - for (const d of deps) { - if (!known.has(d)) { - this.fail(`spec.steps: step "${step.id}" dependsOn unknown step "${d}"`); - } - } - adj.set(step.id, deps); - } - // Cycle detection (DFS, WHITE/GRAY/BLACK). - const WHITE = 0, GRAY = 1, BLACK = 2; - const color = new Map(); - for (const id of adj.keys()) color.set(id, WHITE); - const stack: string[] = []; - const dfs = (id: string): void => { - color.set(id, GRAY); - stack.push(id); - const deps = adj.get(id) ?? []; - for (const d of deps) { - const c = color.get(d); - if (c === GRAY) { - this.fail(`spec.steps: dependency cycle detected at "${d}" (path: ${[...stack].join(' -> ')} -> ${d})`); - } else if (c === WHITE) { - dfs(d); - } - } - stack.pop(); - color.set(id, BLACK); - }; - for (const id of adj.keys()) { - if (color.get(id) === WHITE) dfs(id); - } - } } /** Validate a parsed spec object. Returns `{ok, errors}`; never throws. */ diff --git a/sdk/tests/verb-field-lint.test.ts b/sdk/tests/verb-field-lint.test.ts index f185b899..8b5b440d 100644 --- a/sdk/tests/verb-field-lint.test.ts +++ b/sdk/tests/verb-field-lint.test.ts @@ -15,46 +15,90 @@ import { preflight, type PreflightProbes, } from '../src/preflight.js'; -import type { FlowSpec } from '../src/spec.js'; +import type { FlowSpec, StepType } from '../src/spec.js'; +import { + AGENT_DECLARATION_FIELDS, + FLOW_FIELDS, + STEP_FIELDS_BY_TYPE, +} from '../src/step-fields.js'; import { validateSpec } from '../src/validate.js'; type RawSpec = Record & { - steps: Array>; + steps: unknown[]; }; -const INVALID_STEP_FIELDS = [ +interface InvalidFieldCase { + label: string; + step: Record; + unknown: string; + suggestion?: string; +} + +const TYPO_STEP_FIELDS = [ { label: 'deterministic typo', step: { id: 'work', type: 'deterministic', command: 'printf ok', commnad: 'printf wrong' }, unknown: 'commnad', suggestion: 'command', }, - { - label: 'deterministic cross-verb field', - step: { id: 'work', type: 'deterministic', command: 'printf ok', prompt: 'not deterministic' }, - unknown: 'prompt', - }, { label: 'llm typo', step: { id: 'work', type: 'llm', prompt: 'answer', cli: 'test-cli', promt: 'misspelled' }, unknown: 'promt', suggestion: 'prompt', }, - { - label: 'llm cross-verb field', - step: { id: 'work', type: 'llm', prompt: 'answer', cli: 'test-cli', instruction: 'not llm' }, - unknown: 'instruction', - }, { label: 'agent typo', step: { id: 'work', type: 'agent', instruction: 'act', cli: 'test-cli', instructon: 'misspelled' }, unknown: 'instructon', suggestion: 'instruction', }, +] as const satisfies readonly InvalidFieldCase[]; + +const VALID_STEP_BY_TYPE: Record> = { + deterministic: { id: 'work', type: 'deterministic', command: 'printf ok' }, + llm: { id: 'work', type: 'llm', prompt: 'answer', cli: 'test-cli' }, + agent: { id: 'work', type: 'agent', instruction: 'act', cli: 'test-cli' }, +}; + +const VERB_FIELD_VALUES: Record = { + agent: 'reviewer', + command: 'printf foreign', + prompt: 'foreign prompt', + model: 'foreign-model', + cli: 'foreign-cli', + instruction: 'foreign instruction', + surfaces: { workspace: [{ surface: 'foreign-worktree' }] }, + recoveryMode: 'reset', + permissions: { accessPreset: 'readonly' }, +}; + +const ALL_VERB_FIELDS = [...new Set(Object.values(STEP_FIELDS_BY_TYPE).flat())]; +const CROSS_VERB_STEP_FIELDS: InvalidFieldCase[] = ( + Object.entries(STEP_FIELDS_BY_TYPE) as Array<[StepType, readonly string[]]> +).flatMap(([type, allowed]) => ALL_VERB_FIELDS + .filter((field) => !allowed.includes(field)) + .map((field) => ({ + label: `${type} foreign ${field}`, + step: { ...VALID_STEP_BY_TYPE[type], [field]: VERB_FIELD_VALUES[field] }, + unknown: field, + }))); + +const INVALID_STEP_FIELDS: readonly InvalidFieldCase[] = [ + ...TYPO_STEP_FIELDS, + ...CROSS_VERB_STEP_FIELDS, +]; + +const MALFORMED_STEP_SHAPES = [ { - label: 'agent cross-verb field', - step: { id: 'work', type: 'agent', instruction: 'act', cli: 'test-cli', command: 'not agent' }, - unknown: 'command', + label: 'null step', + step: null, + expected: 'spec.steps[0]: expected an object', + }, + { + label: 'non-array dependsOn', + step: { id: 'work', type: 'deterministic', command: 'printf ok', dependsOn: 'earlier' }, + expected: 'spec.steps[0].dependsOn: expected an array of step ids', }, ] as const; @@ -70,7 +114,11 @@ function specWith(step: Record): RawSpec { return { version: '0.1.0', name: 'field-lint', steps: [step] }; } -function expectedUnknownField(case_: (typeof INVALID_STEP_FIELDS)[number]): string { +function malformedSpecWith(step: unknown): RawSpec { + return { version: '0.1.0', name: 'malformed-shape', steps: [step] } as RawSpec; +} + +function expectedUnknownField(case_: InvalidFieldCase): string { return case_.suggestion === undefined ? `spec.steps[0]: unknown key "${case_.unknown}"` : `spec.steps[0]: unknown key "${case_.unknown}" — did you mean "${case_.suggestion}"?`; @@ -94,6 +142,93 @@ function probes(onProbe: () => void): PreflightProbes { } describe('closed per-verb step fields', () => { + it('pins the per-verb descriptor and generates every foreign-field pair from it', () => { + expect(FLOW_FIELDS).toEqual([ + 'version', 'name', 'description', 'cli', 'agents', 'triggers', 'steps', 'budget', + ]); + expect(AGENT_DECLARATION_FIELDS).toEqual(['cli', 'model']); + expect(STEP_FIELDS_BY_TYPE).toEqual({ + deterministic: ['command'], + llm: ['prompt', 'model', 'cli'], + agent: ['instruction', 'agent', 'cli', 'model', 'surfaces', 'recoveryMode', 'permissions'], + }); + expect(CROSS_VERB_STEP_FIELDS.map(({ label }) => label).sort()).toEqual([ + 'agent foreign command', + 'agent foreign prompt', + 'deterministic foreign agent', + 'deterministic foreign cli', + 'deterministic foreign instruction', + 'deterministic foreign model', + 'deterministic foreign permissions', + 'deterministic foreign prompt', + 'deterministic foreign recoveryMode', + 'deterministic foreign surfaces', + 'llm foreign agent', + 'llm foreign command', + 'llm foreign instruction', + 'llm foreign permissions', + 'llm foreign recoveryMode', + 'llm foreign surfaces', + ]); + }); + + it.each(MALFORMED_STEP_SHAPES)('$label is a typed validation/compiler/preflight failure', (case_) => { + const raw = malformedSpecWith(case_.step); + let validation: ReturnType | undefined; + + expect(() => { validation = validateSpec(raw); }).not.toThrow(); + expect(validation).toEqual({ + ok: false, + errors: expect.arrayContaining([case_.expected]), + }); + + for (const compile of [ + () => compileSpec(raw), + () => compileYaml(stringifyYaml(raw)), + () => toKernelSpec(raw as never), + ]) { + expect(compile).toThrow(CompileError); + expect(compile).toThrow(case_.expected); + } + + let probeCount = 0; + const result = preflight(raw as never, { + probes: probes(() => { probeCount += 1; }), + }); + expect(result).toEqual({ + ok: false, + resolutions: [], + diagnostics: [{ + severity: 'refusal', + kind: 'invalid_spec', + message: expect.stringContaining(case_.expected), + errors: expect.arrayContaining([case_.expected]), + }], + }); + expect(probeCount).toBe(0); + }); + + it.each(MALFORMED_STEP_SHAPES)('$label is refused by flows check without a raw exception', async (case_) => { + const directory = mkdtempSync(join(tmpdir(), 'flows-malformed-shape-')); + temporaryDirectories.push(directory); + writeFileSync(join(directory, 'flows.json'), JSON.stringify({ executors: [] })); + const path = join(directory, 'malformed.flow.yaml'); + writeFileSync(path, stringifyYaml(malformedSpecWith(case_.step))); + const stdout: string[] = []; + const stderr: string[] = []; + + const exitCode = await runCli(['check', path], { + stdout: (line) => stdout.push(line), + stderr: (line) => stderr.push(line), + }); + + expect(exitCode).toBe(2); + expect(stdout).toEqual([]); + expect(stderr.join('\n')).toContain('REFUSED [invalid_spec]'); + expect(stderr.join('\n')).toContain(case_.expected); + expect(stderr.join('\n')).not.toContain('TypeError'); + }); + it.each(INVALID_STEP_FIELDS)('$label is rejected by every public compiler/validator path', (case_) => { const raw = specWith({ ...case_.step }); const expected = expectedUnknownField(case_); @@ -162,6 +297,7 @@ describe('closed per-verb step fields', () => { version: '0.1.0', name: 'valid-v1', cli: 'flow-cli', + agents: { reviewer: { cli: 'named-cli', model: 'named-model' } }, steps: [ { id: 'prepare', @@ -185,6 +321,7 @@ describe('closed per-verb step fields', () => { { id: 'act', type: 'agent', + agent: 'reviewer', instruction: 'act', model: 'project-model', cli: 'agent-cli', From 62a647fcae07edf7427e3cd2dcb4a618c0842dc3 Mon Sep 17 00:00:00 2001 From: kjgbot Date: Wed, 2 Sep 2026 20:45:54 +0200 Subject: [PATCH 09/15] fix(sdk): bind named agent preflight provenance Session-Id: 01a062b7-2aef-7772-8225-7cb00ee311dd --- docs/SURFACE.md | 3 + ops/reviews/20260902-2045-pr136-repair.md | 90 +++++++++++++++++++++++ sdk/src/cli/check.ts | 44 +++++++++-- sdk/src/compile.ts | 9 ++- sdk/src/preflight.ts | 24 +++--- sdk/src/spec.ts | 4 +- sdk/tests/cli.test.ts | 1 + sdk/tests/live-kernel.test.ts | 33 ++++++--- sdk/tests/model-selection.test.ts | 16 +++- sdk/tests/verb-field-lint.test.ts | 7 +- 10 files changed, 192 insertions(+), 39 deletions(-) create mode 100644 ops/reviews/20260902-2045-pr136-repair.md diff --git a/docs/SURFACE.md b/docs/SURFACE.md index d1a495f9..9dd92696 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -123,6 +123,9 @@ No process runs between events: the handler wakes, executes to its next await, p `flows check` resolves the binary (a path is relative to the declaring flow or project config; a bare name resolves via `PATH`) and caches each resolved `(cli, source, model)` probe. A missing executable is `cli_missing`. A probe + that succeeds for a relative path binds its canonical absolute executable + into the checked step before journal submission, so a worker running from a + different directory identifies and executes the same binary. A probe that cannot start, is signaled, or exceeds its adapter timeout is `probe_failed`, with a classified diagnostic rather than a raw process error. Every subprocess starts with ambient `RELAYFLOW_MODEL` removed; diff --git a/ops/reviews/20260902-2045-pr136-repair.md b/ops/reviews/20260902-2045-pr136-repair.md new file mode 100644 index 00000000..b45908b3 --- /dev/null +++ b/ops/reviews/20260902-2045-pr136-repair.md @@ -0,0 +1,90 @@ +# PR #136 raw-preflight / executable-identity / PR #138 composition repair + +Date: 2026-09-02 + +Starting revision: `060e8b971a04f27785d88e1520aa7c1eb3fd3fc5` + +Scope: repair the three P1 findings in +`20260902-1950-pr136-structure.md` without changing kernel vocabulary or review +gates. The PR #138 descriptor and validation commits were composed explicitly; +the resolved descriptors include root `agents`, the closed `{ cli, model }` +declaration, and the agent-step `agent` selector. + +## Red first + +The focused boundary tests were committed before the repair as `30f2324`. The +literal failing summary was: + +```text +Test Files 2 failed (2) +Tests 3 failed | 82 skipped (85) +``` + +The three failures respectively demonstrated that a checked relative wrapper was not bound +to the flow directory, a valid raw named-agent spec was refused by public +preflight, and a malformed raw cross-verb spec probed/passed instead of failing +closed. + +## Focused green + +Command (`sdk/`): + +```sh +./node_modules/.bin/tsc --noEmit && ./node_modules/.bin/vitest run tests/preflight.test.ts tests/cli.test.ts tests/model-selection.test.ts tests/verb-field-lint.test.ts --reporter=dot --maxWorkers=1 --minWorkers=1 +``` + +Literal result: + +```text +Test Files 4 passed (4) + Tests 158 passed (158) +Duration 8.03s +``` + +The live check-to-worker test uses a named `./echo-model-cli` in the flow +directory while the SDK worker runs elsewhere. `checkFlow` returns the exact +absolute executable, kernel lowering journals that path and the named model, +and the real `AgentWorker` executes it successfully: + +```text +✓ built flows CLI against live relayflowd > AgentWorker passes a declared model to an identified wrapper as RELAYFLOW_MODEL +Test Files 1 passed (1) +Tests 1 passed | 19 skipped (20) +``` + +## Full deterministic and live regression suite + +Command (`sdk/`): + +```sh +RELAYFLOWD_BIN=/Users/khaliqgant/.relayflows-toolchain/target/gate-contract/debug/relayflowd RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 ./node_modules/.bin/vitest run --reporter=dot --maxWorkers=1 --minWorkers=1 +``` + +Literal result: + +```text +Test Files 20 passed | 1 skipped (21) + Tests 337 passed | 3 skipped (340) +Duration 121.09s +``` + +The three intentional skips are the opt-in installed-provider cases. They were +run separately against the installed CLIs: + +```sh +RELAYFLOWS_REAL_CLI_ADAPTERS=1 ./node_modules/.bin/vitest run tests/real-cli-adapters.test.ts --reporter=verbose --maxWorkers=1 --minWorkers=1 +``` + +Literal result: + +```text +✓ installed raw CLI adapters > round-trips the exact declared Claude model and refuses an impossible one +✓ installed raw CLI adapters > uses Codex login status and classifies an impossible model as unavailable +✓ installed raw CLI adapters > executes the declared Codex model from a real non-Git directory +Test Files 1 passed (1) +Tests 3 passed (3) +Duration 29.19s +``` + +`git diff --check` and `./node_modules/.bin/tsc --noEmit` both exited zero with +no output. diff --git a/sdk/src/cli/check.ts b/sdk/src/cli/check.ts index 3192e237..8c644929 100644 --- a/sdk/src/cli/check.ts +++ b/sdk/src/cli/check.ts @@ -57,14 +57,14 @@ class CheckFailure extends Error { } } -/** Compile and preflight one working-tree spec without starting a run. */ +/** Validate and preflight one working-tree spec without starting a run. */ export function checkFlow(path: string): CheckExecution { const absolutePath = resolve(path); try { - const flow = readFlow(absolutePath); + const authoring = readFlow(absolutePath); const config = readProjectConfig(dirname(absolutePath)); const probes = systemProbes(dirname(absolutePath), config); - const result = preflight(flow, { + const result = preflight(authoring, { projectCli: config.cli, projectConfigPath: config.path, projectSearchStart: dirname(absolutePath), @@ -72,6 +72,14 @@ export function checkFlow(path: string): CheckExecution { ...(config.path !== undefined ? { modelRegistryPath: config.path } : {}), probes, }); + const flow = result.ok + ? bindResolvedCliPaths( + compileSpec(authoring), + result.resolutions, + dirname(absolutePath), + config.directory, + ) + : undefined; return { report: { ok: result.ok, @@ -80,7 +88,7 @@ export function checkFlow(path: string): CheckExecution { resolutions: result.resolutions, diagnostics: result.diagnostics, }, - ...(result.ok ? { flow } : {}), + ...(flow !== undefined ? { flow } : {}), }; } catch (error) { const failure = error instanceof CheckFailure @@ -120,7 +128,9 @@ function readFlow(path: string): FlowSpec { try { const marker = kernelDialectMarker(parsed); const authoring = marker === undefined ? parsed : kernelToAuthoring(parsed); - return compileSpec(authoring); + // Public preflight owns the first validation pass. Returning raw authoring + // here preserves named-agent provenance until every declaration is checked. + return authoring as FlowSpec; } catch (error) { if (error instanceof CheckFailure) throw error; if (error instanceof CompileError) { @@ -199,6 +209,30 @@ function systemProbes(flowDirectory: string, config: ProjectConfig): PreflightPr }; } +function bindResolvedCliPaths( + flow: FlowSpec, + resolutions: readonly CliResolution[], + flowDirectory: string, + configDirectory: string, +): FlowSpec { + const byStep = new Map(resolutions.map((resolution) => [resolution.stepId, resolution])); + return { + ...flow, + steps: flow.steps.map((step) => { + if (step.type === 'deterministic') return step; + const resolution = byStep.get(step.id); + if (resolution === undefined) return step; + const directory = resolution.source === 'project' ? configDirectory : flowDirectory; + return { ...step, cli: canonicalCli(resolution.cli, directory) }; + }), + }; +} + +function canonicalCli(cli: string, directory: string): string { + if (isAbsolute(cli) || (!cli.includes('/') && !cli.includes('\\'))) return cli; + return resolve(directory, cli); +} + function probeCli( cli: string, directory: string, diff --git a/sdk/src/compile.ts b/sdk/src/compile.ts index 867ea134..85ba25c5 100644 --- a/sdk/src/compile.ts +++ b/sdk/src/compile.ts @@ -68,7 +68,10 @@ export function compileSpec(spec: unknown): FlowSpec { if (!validation.ok) throw new CompileError(validation.errors); const input = spec as FlowSpec; - const steps = input.steps.map((step) => compileStep(resolveNamedAgent(step, input.agents))); + // Preserve named declarations and selectors through authoring normalization. + // They are resolved exactly once at the kernel boundary, after public + // preflight has validated every declaration with truthful provenance. + const steps = input.steps.map(compileStep); const flow: FlowSpec = { version: input.version, ...(input.name !== undefined ? { name: input.name } : {}), @@ -134,8 +137,8 @@ function compileStep(step: StepSpec): StepSpec { } /** - * Resolve declarative named-agent sugar before normalization or kernel - * lowering. Explicit step fields win independently, so an author may override + * Resolve declarative named-agent sugar at kernel lowering. Explicit step + * fields win independently, so an author may override * only the CLI or only the model. The selector and declaration map never * cross the journal boundary. */ diff --git a/sdk/src/preflight.ts b/sdk/src/preflight.ts index c6b7bc89..1dce1cf9 100644 --- a/sdk/src/preflight.ts +++ b/sdk/src/preflight.ts @@ -5,13 +5,13 @@ import type { } from './failure-kinds.js'; import { validateSpec } from './validate.js'; -export type CliResolutionSource = 'step' | 'flow' | 'project'; +export type CliResolutionSource = 'step' | 'named' | 'flow' | 'project'; export interface CliResolution { stepId: string; cli: string; source: CliResolutionSource; - /** Model the step declared, probed together with the CLI. */ + /** Model the step or selected named agent declared, probed with the CLI. */ model?: string; } @@ -172,12 +172,6 @@ function unknownModelDiagnostics( for (const step of flow.steps) { if (step.type === 'deterministic' || step.model === undefined) continue; - const selected = step.type === 'agent' && step.agent !== undefined - ? flow.agents?.[step.agent] - : undefined; - // compileSpec copies a selected declaration onto the step. The declaration - // was already checked above; only a different value is an inline override. - if (selected?.model === step.model) continue; if (isKnownModel(step.model, options.models)) continue; const resolution = resolveCli(step, flow, options.projectCli); diagnostics.push({ @@ -236,11 +230,15 @@ function resolveCli( flow: FlowSpec, projectCli: string | undefined, ): CliResolution | undefined { - // Model is a step-level declaration only — there is deliberately no flow or - // project default. A CLI inheriting a model from two levels up is the - // ambient-state problem this field exists to remove. - const model = step.model !== undefined ? { model: step.model } : {}; + const named = step.type === 'agent' && step.agent !== undefined + ? flow.agents?.[step.agent] + : undefined; + // Model comes only from the step or its explicitly selected declaration. + // There is deliberately no flow/project or host default. + const effectiveModel = step.model ?? named?.model; + const model = effectiveModel !== undefined ? { model: effectiveModel } : {}; if (step.cli !== undefined) return { stepId: step.id, cli: step.cli, source: 'step', ...model }; + if (named !== undefined) return { stepId: step.id, cli: named.cli, source: 'named', ...model }; if (flow.cli !== undefined) return { stepId: step.id, cli: flow.cli, source: 'flow', ...model }; if (projectCli !== undefined) return { stepId: step.id, cli: projectCli, source: 'project', ...model }; return undefined; @@ -253,7 +251,7 @@ function probeResolvedCli( diagnostics: PreflightDiagnostic[], ): void { // Source is load-bearing: the same relative CLI string resolves from the - // flow directory for step/flow declarations and the config directory for + // flow directory for step/named/flow declarations and the config directory for // project declarations. // Model is part of the key: the same CLI probed with two different models // is two different questions, and caching on the CLI alone would let a diff --git a/sdk/src/spec.ts b/sdk/src/spec.ts index 5c35631f..d7faf1b8 100644 --- a/sdk/src/spec.ts +++ b/sdk/src/spec.ts @@ -157,8 +157,8 @@ export type StepSpec = DeterministicStepSpec | LlmStepSpec | AgentStepSpec; /** * Reusable authoring declaration for an agent CLI/model pair. Both fields are * required so selecting a named agent can never inherit a host model. The - * compiler lowers these values into the selected `AgentStepSpec`; the kernel - * never receives this map or a new step field. + * compiler lowers these values into the selected kernel agent step at the + * journal boundary; the kernel never receives this map or a new step field. */ export interface NamedAgentSpec { cli: string; diff --git a/sdk/tests/cli.test.ts b/sdk/tests/cli.test.ts index 6471dee0..42641563 100644 --- a/sdk/tests/cli.test.ts +++ b/sdk/tests/cli.test.ts @@ -582,6 +582,7 @@ steps: expect(result.stdout.join('\n')).toContain( `RESOLVED step "answer" cli "./authenticated-cli" from project (${join(directory, 'flows.json')})`, ); + expect(checkFlow(flow).flow?.steps[0]).toHaveProperty('cli', cli); }); it('uses the nearest flows.json as a whole project boundary and names it on refusal', async () => { diff --git a/sdk/tests/live-kernel.test.ts b/sdk/tests/live-kernel.test.ts index 2f6d606c..fc436eba 100644 --- a/sdk/tests/live-kernel.test.ts +++ b/sdk/tests/live-kernel.test.ts @@ -697,20 +697,16 @@ steps: // dispatch → AgentWorker → subprocess env. const dataDir = temporaryDirectory('flows-live-model-set-'); await startDaemon(dataDir); - const cli = join(TESTDATA, 'preflight', 'echo-model-cli'); - const client = await connectClient(dataDir); - await client.hello('live-model-set'); - const worker = new AgentWorker(client, { - workerId: 'live-model-set-worker', - pins: { workspace: [{ surface: 'repo', revision_id: 'rev-a' }], streams: [] }, - }); - await worker.attach(); - - const compiled = compileYaml(` + const cli = join(dataDir, 'echo-model-cli'); + writeFileSync(cli, readFileSync(join(TESTDATA, 'preflight', 'echo-model-cli'))); + chmodSync(cli, 0o755); + writeFileSync(join(dataDir, 'flows.json'), JSON.stringify({ models: ['declared-model-xyz'] })); + const flowPath = join(dataDir, 'relative-wrapper.flow.yaml'); + writeFileSync(flowPath, ` version: '0.1.0' agents: model-probe: - cli: ${JSON.stringify(cli)} + cli: ./echo-model-cli model: declared-model-xyz steps: - id: probe @@ -718,15 +714,28 @@ steps: agent: model-probe instruction: Report the model env var. `); + const client = await connectClient(dataDir); + await client.hello('live-model-set'); + const worker = new AgentWorker(client, { + workerId: 'live-model-set-worker', + pins: { workspace: [{ surface: 'repo', revision_id: 'rev-a' }], streams: [] }, + }); + await worker.attach(); + + const checked = checkFlow(flowPath); + expect(checked.report.ok).toBe(true); + const compiled = checked.flow!; expect(compiled).toHaveProperty('agents.model-probe.model', 'declared-model-xyz'); expect(compiled.steps[0]).toMatchObject({ type: 'agent', + agent: 'model-probe', cli, - model: 'declared-model-xyz', }); + expect(compiled.steps[0]).not.toHaveProperty('model'); const kernel = toKernelSpec(compiled); expect(kernel).not.toHaveProperty('agents'); expect(kernel.steps[0]).not.toHaveProperty('agent'); + expect(kernel.steps[0]).toMatchObject({ cli, model: 'declared-model-xyz' }); const started = await client.runStart(kernel); expect(await waitForStep(client, started.run_id, 'probe', 'done')).toMatchObject({ diff --git a/sdk/tests/model-selection.test.ts b/sdk/tests/model-selection.test.ts index ee3afea2..991e9016 100644 --- a/sdk/tests/model-selection.test.ts +++ b/sdk/tests/model-selection.test.ts @@ -24,9 +24,9 @@ steps: expect(flow.steps[0] as AgentStepSpec).toMatchObject({ id: 'review', type: 'agent', - cli: 'claude', - model: 'claude-sonnet-4-6', }); + expect(flow.steps[0]).not.toHaveProperty('cli'); + expect(flow.steps[0]).not.toHaveProperty('model'); const kernel = toKernelSpec(flow); expect(kernel).not.toHaveProperty('agents'); expect(kernel.steps[0]).not.toHaveProperty('agent'); @@ -65,7 +65,18 @@ steps: instruction: Preserve existing anonymous resolution. `); + // Normalization preserves what the author wrote so validation/preflight + // can distinguish declarations from overrides. Precedence is materialized + // only at the journal boundary. expect(flow.steps).toEqual(expect.arrayContaining([ + expect.objectContaining({ id: 'named', agent: 'reviewer' }), + expect.objectContaining({ id: 'cli-override', cli: 'step-cli' }), + expect.objectContaining({ id: 'model-override', model: 'step-model' }), + ])); + expect(flow.steps.find((step) => step.id === 'named')).not.toHaveProperty('cli'); + expect(flow.steps.find((step) => step.id === 'named')).not.toHaveProperty('model'); + const kernel = toKernelSpec(flow); + expect(kernel.steps).toEqual(expect.arrayContaining([ expect.objectContaining({ id: 'named', cli: 'named-cli', model: 'named-model' }), expect.objectContaining({ id: 'cli-override', cli: 'step-cli', model: 'named-model' }), expect.objectContaining({ id: 'model-override', cli: 'named-cli', model: 'step-model' }), @@ -73,6 +84,7 @@ steps: expect(flow.steps.find((step) => step.id === 'anonymous')).not.toHaveProperty('cli'); expect(flow.steps.find((step) => step.id === 'anonymous')).not.toHaveProperty('model'); expect(flow.cli).toBe('flow-cli'); + expect(kernel).toHaveProperty('cli', 'flow-cli'); }); it('lowers a raw typed FlowSpec passed directly to the kernel mapper', () => { diff --git a/sdk/tests/verb-field-lint.test.ts b/sdk/tests/verb-field-lint.test.ts index 8b5b440d..15c547cf 100644 --- a/sdk/tests/verb-field-lint.test.ts +++ b/sdk/tests/verb-field-lint.test.ts @@ -128,7 +128,7 @@ function probes(onProbe: () => void): PreflightProbes { return { cli: () => { onProbe(); - return { exists: true, authenticated: true }; + return { exists: true, authenticated: true, modelAvailable: true }; }, executor: () => { onProbe(); @@ -353,7 +353,10 @@ describe('closed per-verb step fields', () => { 'agent', ]); - const result = preflight(valid, { probes: probes(() => {}) }); + const result = preflight(valid, { + models: ['named-model', 'project-model'], + probes: probes(() => {}), + }); expect(result.ok).toBe(true); expect(result.resolutions).toEqual([ { stepId: 'answer', cli: 'llm-cli', source: 'step', model: 'project-model' }, From 5a0a1d3b210671af14c0e8b480691db5c79f20c8 Mon Sep 17 00:00:00 2001 From: kjgbot Date: Wed, 2 Sep 2026 21:07:24 +0200 Subject: [PATCH 10/15] docs(review): record PR 136 identity review Session-Id: 01a062b7-2aef-7772-8225-7cb00ee311dd --- ops/reviews/20260902-2048-pr136-history.md | 418 +++++++++++++ .../20260902-2048-pr136-maintainability.md | 573 ++++++++++++++++++ ops/reviews/20260902-2048-pr136-structure.md | 399 ++++++++++++ 3 files changed, 1390 insertions(+) create mode 100644 ops/reviews/20260902-2048-pr136-history.md create mode 100644 ops/reviews/20260902-2048-pr136-maintainability.md create mode 100644 ops/reviews/20260902-2048-pr136-structure.md diff --git a/ops/reviews/20260902-2048-pr136-history.md b/ops/reviews/20260902-2048-pr136-history.md new file mode 100644 index 00000000..326e8811 --- /dev/null +++ b/ops/reviews/20260902-2048-pr136-history.md @@ -0,0 +1,418 @@ +# PR #136 exact-head history / integration review + +Date: 2026-09-02 + +- PR: #136, `feat(sdk): declare agent CLI and model with fail-closed checks` +- Exact head: `62a647fcae07edf7427e3cd2dcb4a618c0842dc3` +- Base: merged `origin/main` at `a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2` +- Constitution read: `AGENTS.md` and `docs/RFC-0001-everything-is-a-relayflow.md`, in full +- Lens: history fit, raw/named model provenance, validation-before-probe ordering, + relative-wrapper identity across cwd changes, installed Claude/Codex argv, + non-Git Codex execution, execution-time wrapper replacement, v1/default + compatibility, and literal verification evidence +- Scope: assessment only; no product code, gate, merge, push, or commit action + +## Verdict + +PASS. I found no blocking history or integration defect at the assigned exact +head. + +The final commit fixes the provenance regression present in the earlier review: +`compileSpec()` now retains named declarations and selectors through public +preflight, `preflight()` reports a selected declaration as `source: "named"`, +and named sugar is resolved only at `toKernelSpec()`. The checked flow then +binds a relative resolved executable to an absolute path before it can be run +from another cwd. Validation and the global model allowlist scan both finish +before any CLI, deterministic-command, trigger-executor, or daemon probe. + +The adapter/worker boundary also holds: raw Claude and Codex receive the exact +declared model through provider-native argv, custom wrappers receive the private +model environment only after exact v1 identification, and a wrapper replaced +after successful preflight is re-identified and refused at worker execution. +Installed Claude and Codex completed the exact generated argv from fresh +non-Git directories; Codex included `--skip-git-repo-check`. + +The PR leaves the kernel, workflows, and the existing v1/default surface +declaration untouched. The canonical inline/default ladder still checks, the +full serial SDK/live-kernel suite passes, and the separately opted-in installed +provider suite passes. The TypeScript `FlowHeader.agents` follow-on remains +outside this PR and is stated as such in the PR body; this review does not treat +that unclaimed follow-on as a defect. + +## Exact boundary and commit story + +Literal command and captured output: + +```text +$ git status --short --branch && git rev-parse HEAD && git rev-parse origin/main && git merge-base HEAD origin/main && git log --reverse --oneline origin/main..HEAD +## feat/v2-declared-model...origin/feat/v2-declared-model +62a647fcae07edf7427e3cd2dcb4a618c0842dc3 +a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2 +a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2 +321b272 feat(sdk): add declared agent model contract +78efc0d docs(review): record PR 136 fresh review +4888d15 fix(sdk): make model adapters fail closed +060e8b9 fix(sdk): bind declared model execution checks +fccc52a docs(review): record PR 136 integration review +30f2324 test(sdk): cover public named-agent boundaries +e50d977 fix(sdk): refuse fields outside step verb schemas +866d399 fix(sdk): harden malformed step validation +62a647f fix(sdk): bind named agent preflight provenance +``` + +The branch is directly based on the fetched merged main. The history is +coherent: declaration, fresh-review repair, provider execution repair, +integration review, public-boundary coverage, closed-schema repair, +malformed-input hardening, then the final named-provenance/bound-executable +repair. The earlier review documents are historical evidence rather than +squashed-away claims, and each blocking class has a corresponding later code +or test commit. + +Remote PR identity and first-party check were re-read at review time: + +```text +$ gh pr view 136 --repo AgentWorkforce/flows --json number,state,title,baseRefOid,headRefOid,headRefName,commits,statusCheckRollup --jq '{number,state,title,baseRefOid,headRefOid,headRefName,commits:[.commits[].oid],checks:[.statusCheckRollup[]|{name,status,conclusion}]}' +{"baseRefOid":"a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2","checks":[{"conclusion":"SUCCESS","name":"linux-x64-artifact","status":"COMPLETED"},{"conclusion":null,"name":null,"status":null}],"commits":["321b27216e561561e1e022a7b4d973e2182ee480","78efc0d15f32246332f9cf64ffba7f838573d02e","4888d1572ed047c5161042614ac72068d047783a","060e8b971a04f27785d88e1520aa7c1eb3fd3fc5","fccc52abbef194d5536e96f4008ab5445467365b","30f23248f897b5f8fc7f87ca23c7e37ede6a653f","e50d977bf8f6bb2a6498e4739f6e2d80029ccd87","866d39978bb10fa8be007728f42075ae35a4823c","62a647fcae07edf7427e3cd2dcb4a618c0842dc3"],"headRefName":"feat/v2-declared-model","headRefOid":"62a647fcae07edf7427e3cd2dcb4a618c0842dc3","number":136,"state":"OPEN","title":"feat(sdk): declare agent CLI and model with fail-closed checks"} +``` + +The null unnamed status-rollup entry is not review signal. The named +first-party `linux-x64-artifact` check is successful. + +Relevant merged history already carried the per-step model through the kernel +and worker; this PR extends that existing boundary instead of creating a second +kernel concept: + +```text +$ git log --all --format='%H %s' -S'RELAYFLOW_MODEL' -- sdk/src testdata docs | head -n 30 +060e8b971a04f27785d88e1520aa7c1eb3fd3fc5 fix(sdk): bind declared model execution checks +4888d1572ed047c5161042614ac72068d047783a fix(sdk): make model adapters fail closed +321b27216e561561e1e022a7b4d973e2182ee480 feat(sdk): add declared agent model contract +51415d9c65ef5c727c560c700f63932893a1e224 feat(gate2): real Claude analyzer for hn-monitor, with a declared model (#130) +82be45ff91c1e77db8422b72324fba7ecf7fdda7 feat(gate2): real Claude analyzer for hn-monitor, with a declared model +``` + +## Independent provenance, ordering, and cwd probe + +This built-SDK probe covers both model origins, the authoring-to-journal +erasure point, a global named+inline unknown-model scan before every injected +probe kind, and checked relative-wrapper execution after changing cwd to `/`. +The temporary wrapper directory printed below existed for the invocation and +was removed by its `finally` block. + +Literal command: + +```sh +node --input-type=module <<'EOF' +import { mkdtempSync, writeFileSync, chmodSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { compileSpec, preflight, toKernelSpec } from './dist/index.js'; +import { checkFlow } from './dist/cli/check.js'; +import { runAgentCli } from './dist/worker-cli.js'; + +const calls = []; +const authored = compileSpec({ + version: '0.1.0', + agents: { reviewer: { cli: 'named-wrapper', model: 'named-model' } }, + steps: [ + { id: 'named', type: 'agent', agent: 'reviewer', instruction: 'named' }, + { id: 'inline', type: 'agent', cli: 'raw-wrapper', model: 'raw-model', instruction: 'inline' }, + ], +}); +const result = preflight(authored, { + models: ['named-model', 'raw-model'], + probes: { + cli: (cli, source, model) => { calls.push({ cli, source, model }); return { exists: true, supported: true, authenticated: true, modelAvailable: true }; }, + command: () => true, + executor: () => true, + }, +}); +console.log('PROVENANCE=' + JSON.stringify({ result, calls, authoring: authored.steps, kernel: toKernelSpec(authored).steps.map(({ id, type, cli, model }) => ({ id, type, cli, model })) })); + +const blockedCalls = []; +const blocked = preflight({ + version: '0.1.0', + agents: { unused: { cli: 'must-not-probe', model: 'unknown-named' } }, + triggers: [{ id: 'wake', executor: 'must-not-probe' }], + steps: [ + { id: 'deterministic', type: 'deterministic', command: './must-not-probe' }, + { id: 'known', type: 'agent', cli: 'must-not-probe', model: 'known-model', instruction: 'known' }, + { id: 'typo', type: 'agent', cli: 'must-not-probe', model: 'unknown-inline', instruction: 'typo' }, + ], +}, { + models: ['known-model'], + probes: { + cli: () => { blockedCalls.push('cli'); throw new Error('PROBE_CALLED'); }, + command: () => { blockedCalls.push('command'); throw new Error('PROBE_CALLED'); }, + executor: () => { blockedCalls.push('executor'); throw new Error('PROBE_CALLED'); }, + }, +}); +console.log('VALIDATE_BEFORE_PROBE=' + JSON.stringify({ ok: blocked.ok, kinds: blocked.diagnostics.map(d => d.kind), models: blocked.diagnostics.map(d => d.model ?? null), calls: blockedCalls })); + +const directory = mkdtempSync(join(tmpdir(), 'pr136-relative-wrapper-')); +try { + const wrapper = join(directory, 'wrapper'); + writeFileSync(wrapper, `#!/bin/sh\nif [ "\${1-}" = "--relayflows-adapter-v1" ]; then printf '%s\\n' relayflows-agent-cli-v1; exit 0; fi\nif [ "\${1-} \${2-}" = "auth status" ]; then exit 0; fi\nprintf '%s' '{"cwd_independent":true}'\n`); + chmodSync(wrapper, 0o755); + writeFileSync(join(directory, 'flows.json'), JSON.stringify({ models: ['wrapper-model'] })); + const flowPath = join(directory, 'relative.flow.yaml'); + writeFileSync(flowPath, `version: '0.1.0'\nagents:\n reviewer: { cli: ./wrapper, model: wrapper-model }\nsteps:\n - id: work\n type: agent\n agent: reviewer\n instruction: work\n`); + const checked = checkFlow(flowPath); + const checkedCli = checked.flow?.steps[0]?.cli; + process.chdir('/'); + const execution = await runAgentCli(checkedCli, 'work', undefined, 'wrapper-model'); + console.log('RELATIVE_WRAPPER=' + JSON.stringify({ checkOk: checked.report.ok, source: checked.report.resolutions[0]?.source, checkedCli, isAbsolute: checkedCli?.startsWith('/'), execution })); +} finally { + rmSync(directory, { recursive: true, force: true }); +} +EOF +``` + +Captured output: + +```text +PROVENANCE={"result":{"ok":true,"resolutions":[{"stepId":"named","cli":"named-wrapper","source":"named","model":"named-model"},{"stepId":"inline","cli":"raw-wrapper","source":"step","model":"raw-model"}],"diagnostics":[]},"calls":[{"cli":"named-wrapper","source":"named","model":"named-model"},{"cli":"raw-wrapper","source":"step","model":"raw-model"}],"authoring":[{"id":"named","type":"agent","maxIterations":1,"instruction":"named","agent":"reviewer","recoveryMode":"reset"},{"id":"inline","type":"agent","maxIterations":1,"instruction":"inline","cli":"raw-wrapper","model":"raw-model","recoveryMode":"reset"}],"kernel":[{"id":"named","type":"agent","cli":"named-wrapper","model":"named-model"},{"id":"inline","type":"agent","cli":"raw-wrapper","model":"raw-model"}]} +VALIDATE_BEFORE_PROBE={"ok":false,"kinds":["model_unknown","model_unknown"],"models":["unknown-named","unknown-inline"],"calls":[]} +RELATIVE_WRAPPER={"checkOk":true,"source":"named","checkedCli":"/var/folders/yv/nbp9l2c55wlbj1x0gml37s7c0000gn/T/pr136-relative-wrapper-iKHKTw/wrapper","isAbsolute":true,"execution":{"exit_code":0,"stdout_tail":"{\"cwd_independent\":true}","stderr_tail":""}} +``` + +This is the expected provenance boundary: the normalized authoring value keeps +`agents` and the selected `agent`; preflight distinguishes `named` from `step`; +the kernel gets only per-step `cli` and `model`. Unknown named and inline models +produce both diagnostics with zero probe calls. + +## Worker trust boundary and journal integration + +The live tests use the actual daemon and worker. The replacement case first +passes `checkFlow()`, replaces the executable, submits a kernel object directly +to the journal, then asserts the worker invoked only +`--relayflows-adapter-v1`, exposed no model, and journaled `worker_error`. +The same command covers exact raw-provider flags and the no-model compatibility +case. + +Literal command and captured output: + +```text +$ RELAYFLOWD_BIN=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 ./node_modules/.bin/vitest run tests/live-kernel.test.ts -t 'AgentWorker (passes a declared model to an identified wrapper|refuses a nonconforming journal-submitted wrapper|executes the raw|leaves RELAYFLOW_MODEL UNSET)' --reporter=verbose --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd +LIVE_KERNEL flows=/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk/dist/cli.js + + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker passes a declared model to an identified wrapper as RELAYFLOW_MODEL 502ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker refuses a nonconforming journal-submitted wrapper before exposing RELAYFLOW_MODEL 342ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker executes the raw claude adapter with its real model flag + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker executes the raw codex adapter with its real model flag 324ms + ✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker leaves RELAYFLOW_MODEL UNSET when the step declares no model + + Test Files 1 passed (1) + Tests 5 passed | 15 skipped (20) + Start at 20:53:21 + Duration 1.80s (transform 83ms, setup 0ms, collect 120ms, tests 1.58s, environment 0ms, prepare 26ms) +``` + +## Installed Claude/Codex exact argv + +Installed adapter identification/authentication used the provider-specific +commands, not a generic wrapper assumption. + +```text +$ node --input-type=module <<'EOF' +import { spawnSync } from 'node:child_process'; +for (const [cli, versionArgs, identifyArgs, authArgs] of [ + ['claude', ['--version'], ['auth', 'status', '--help'], ['auth', 'status']], + ['codex', ['--version'], ['login', 'status', '--help'], ['login', 'status']], +]) { + const version = spawnSync(cli, versionArgs, { encoding: 'utf8' }); + const identified = spawnSync(cli, identifyArgs, { encoding: 'utf8', stdio: ['ignore', 'ignore', 'ignore'] }); + const authenticated = spawnSync(cli, authArgs, { encoding: 'utf8', stdio: ['ignore', 'ignore', 'ignore'] }); + console.log(JSON.stringify({ cli, version: (version.stdout || version.stderr).trim(), identificationArgs: identifyArgs, identificationExit: identified.status, authArgs, authExit: authenticated.status })); +} +EOF +{"cli":"claude","version":"2.1.153 (Claude Code)","identificationArgs":["auth","status","--help"],"identificationExit":0,"authArgs":["auth","status"],"authExit":0} +{"cli":"codex","version":"codex-cli 0.152.1","identificationArgs":["login","status","--help"],"identificationExit":0,"authArgs":["login","status"],"authExit":0} +``` + +The opt-in installed-provider suite exercised positive Claude readiness, +impossible Claude refusal, truthful Codex auth/model classification, and real +Codex execution from a new non-Git directory. + +```text +$ RELAYFLOWS_REAL_CLI_ADAPTERS=1 ./node_modules/.bin/vitest run tests/real-cli-adapters.test.ts --reporter=verbose --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + + ✓ tests/real-cli-adapters.test.ts > installed raw CLI adapters > round-trips the exact declared Claude model and refuses an impossible one 5259ms + ✓ tests/real-cli-adapters.test.ts > installed raw CLI adapters > uses Codex login status and classifies an impossible model as unavailable 4329ms + ✓ tests/real-cli-adapters.test.ts > installed raw CLI adapters > executes the declared Codex model from a real non-Git directory 21546ms + + Test Files 1 passed (1) + Tests 3 passed (3) + Start at 20:53:37 + Duration 31.32s (transform 47ms, setup 0ms, collect 68ms, tests 31.14s, environment 0ms, prepare 24ms) +``` + +I additionally executed the worker-generated argv for both installed providers +and printed the exact argument vector. Both temporary cwd values were verified +as outside a Git worktree; neither provider received the wrapper-private model +environment. + +Literal command: + +```sh +node --input-type=module <<'EOF' +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { agentExecution } from './dist/cli-adapter.js'; +for (const [cli, model, token] of [ + ['claude', 'claude-haiku-4-5-20251001', 'RELAYFLOWS_CLAUDE_EXEC_READY'], + ['codex', 'gpt-5.6-sol', 'RELAYFLOWS_CODEX_EXEC_READY'], +]) { + const directory = mkdtempSync(join(tmpdir(), `pr136-real-${cli}-worker-`)); + try { + const invocation = agentExecution(cli, `Reply with exactly ${token}.`, model); + const result = spawnSync(cli, invocation.args, { cwd: directory, encoding: 'utf8', timeout: 120000, stdio: ['ignore', 'pipe', 'pipe'] }); + const cwdIsGit = spawnSync('git', ['rev-parse', '--is-inside-work-tree'], { cwd: directory, stdio: 'ignore' }).status === 0; + console.log(JSON.stringify({ cli, cwdIsGit, args: invocation.args, modelEnv: invocation.modelEnv ?? null, status: result.status, signal: result.signal, error: result.error?.message ?? null, stdoutHasToken: result.stdout.includes(token) })); + if (result.status !== 0 || !result.stdout.includes(token)) process.exitCode = 1; + } finally { + rmSync(directory, { recursive: true, force: true }); + } +} +EOF +``` + +Captured output: + +```text +{"cli":"claude","cwdIsGit":false,"args":["-p","--model","claude-haiku-4-5-20251001","Reply with exactly RELAYFLOWS_CLAUDE_EXEC_READY."],"modelEnv":null,"status":0,"signal":null,"error":null,"stdoutHasToken":true} +{"cli":"codex","cwdIsGit":false,"args":["exec","--ephemeral","--skip-git-repo-check","--model","gpt-5.6-sol","Reply with exactly RELAYFLOWS_CODEX_EXEC_READY."],"modelEnv":null,"status":0,"signal":null,"error":null,"stdoutHasToken":true} +``` + +## v1/default compatibility and boundary hygiene + +No kernel, workflow, or existing v1 declaration file changes are in the PR, +and the diff has no whitespace error: + +```text +$ git diff --check origin/main...HEAD; diff_check=$?; git diff --quiet origin/main...HEAD -- kernel regressions/surface.d.ts workflows; v1_surface_diff=$?; printf 'diff_check_exit=%s\nv1_kernel_surface_workflow_diff_exit=%s\n' "$diff_check" "$v1_surface_diff" +diff_check_exit=0 +v1_kernel_surface_workflow_diff_exit=0 +``` + +The existing canonical inline/default agent flow still checks through the +nearest project config with its pre-existing model: + +```text +$ node dist/cli.js check --json ../testdata/hello-agent.flow.yaml +WARNING [unprovable_effects] Step "greet" command "printf" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "finish" command "printf" resolves, but its effects cannot be proven before execution. +{"ok":true,"path":"../testdata/hello-agent.flow.yaml","projectConfigPath":"/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/testdata/flows.json","resolutions":[{"stepId":"edit","cli":"./preflight/authenticated-cli","source":"project","model":"test-model-v1"}],"diagnostics":[{"severity":"warning","kind":"unprovable_effects","stepId":"greet","message":"Step \"greet\" command \"printf\" resolves, but its effects cannot be proven before execution."},{"severity":"warning","kind":"unprovable_effects","stepId":"finish","message":"Step \"finish\" command \"printf\" resolves, but its effects cannot be proven before execution."}]} +``` + +The no-model worker case in the live command above deliberately pollutes the +parent with `RELAYFLOW_MODEL` and proves that a legacy/default step still sees +the variable absent. This is positive compatibility evidence inside this repo; +it is not a claim that an external v1 repository was executed. + +## Typecheck, build, and complete regression result + +The SDK typecheck and direct build completed with explicit statuses: + +```text +$ ./node_modules/.bin/tsc --noEmit; typecheck_status=$?; ./node_modules/.bin/tsc; build_status=$?; node scripts/make-cli-executable.mjs; cli_status=$?; printf 'typecheck_exit=%s\ntsc_build_exit=%s\nmake_cli_executable_exit=%s\n' "$typecheck_status" "$build_status" "$cli_status" +typecheck_exit=0 +tsc_build_exit=0 +make_cli_executable_exit=0 +``` + +Full serial SDK/live-kernel command and captured output: + +```text +$ RELAYFLOWD_BIN=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 ./node_modules/.bin/vitest run --reporter=dot --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/Users/khaliqgant/.relayflows-toolchain/target/1914866954/debug/relayflowd +LIVE_KERNEL flows=/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk/dist/cli.js + +stdout | tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +LIVE_ANALYZER ready: claude -p --model claude-haiku-4-5-20251001 round-trip OK + +stdout | tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +LIVE_ANALYZER analysis: {"reasoning":"This story directly describes an AI agent performing autonomous software development tasks—opening and reviewing pull requests—which is a core application of AI agents and automation in development workflows.","relevance_score":10,"story_title":"Show HN: an agent that opens and reviews its own pull requests [wake-nonce-7f3a91c4]"} + +stdout | tests/live-kernel.test.ts > surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once +LIVE_KERNEL kill -9 pid=48162 run=01M1HQMBDYPEHKMHB873CERHKJ while step=two state=Running + + ✓ tests/live-kernel.test.ts (20 tests) 49860ms + ✓ built flows CLI against live relayflowd > runs rung (a), parks rung (b), and keeps JSON report-shaped 407ms + ✓ built flows CLI against live relayflowd > allows a deterministic run to exceed the bounded request timeout 32398ms + ✓ built flows CLI against live relayflowd > can always get a parked run to a late-attaching worker 5553ms + ✓ built flows CLI against live relayflowd > AgentWorker passes a declared model to an identified wrapper as RELAYFLOW_MODEL 332ms + ✓ built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI 8867ms + ✓ surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once 402ms + ✓ tests/cli.test.ts (63 tests) 1913ms + ✓ tests/preflight.test.ts (22 tests) 5ms + ✓ tests/journal-client.test.ts (13 tests) 63ms + ✓ tests/validate.test.ts (36 tests) 7ms + ✓ tests/cli-hn-monitor.test.ts (16 tests) 48ms + ✓ tests/verb-field-lint.test.ts (63 tests) 33ms + ✓ tests/backlog-picker.test.ts (14 tests) 36ms +SKIPPED_UNACTIONABLE=0 +SKIPPED_UNACTIONABLE=0 +NO_ACTIONABLE_BACKLOG_ENTRY scanned=0 +node:fs:539 + return binding.readFileUtf8(path, stringToFlags(options.flag)); + ^ + +Error: ENOENT: no such file or directory, open '.relayflow/backlog-picker-entry.json' + at Object.readFileSync (node:fs:539:20) + at [eval]:1:478 + at runScriptInThisContext (node:internal/vm:219:10) + at node:internal/process/execution:483:12 + at [eval]-wrapper:6:24 + at runScriptInContext (node:internal/process/execution:481:60) + at evalFunction (node:internal/process/execution:315:30) + at evalTypeScript (node:internal/process/execution:327:3) + at node:internal/main/eval_string:71:3 { + errno: -2, + code: 'ENOENT', + syscall: 'open', + path: '.relayflow/backlog-picker-entry.json' +} + +Node.js v26.7.0 +SKIPPED_UNACTIONABLE=0 + ✓ tests/backlog-picker-flow.test.ts (6 tests) 262ms +SKIPPED_UNACTIONABLE=0 +NO_ACTIONABLE_BACKLOG_ENTRY scanned=1 Scoped but unverifiable[missing_definition_of_done] + ✓ tests/work-package-consumer.test.ts (13 tests) 105ms + ✓ tests/model-selection.test.ts (10 tests) 7ms + ✓ tests/deterministic-llm.test.ts (5 tests) 7ms + ✓ tests/bin.test.ts (7 tests) 298ms + ✓ tests/hn-poller.test.ts (6 tests) 3ms + ✓ tests/dir-watcher-poller.test.ts (6 tests) 2ms + ✓ tests/hello-deterministic.test.ts (5 tests) 7ms + ✓ tests/work-package-validator.test.ts (7 tests) 3ms + ✓ tests/spec-parity.test.ts (15 tests) 16ms + ↓ tests/real-cli-adapters.test.ts (3 tests | 3 skipped) + ✓ tests/parse-json-output.test.ts (7 tests) 1ms + ✓ tests/cli-adapter.test.ts (3 tests) 2ms + + Test Files 20 passed | 1 skipped (21) + Tests 337 passed | 3 skipped (340) + Start at 20:54:56 + Duration 55.06s (transform 212ms, setup 0ms, collect 484ms, tests 52.68s, environment 1ms, prepare 417ms) +``` + +The printed `ENOENT` is expected stderr from a negative backlog-picker child +case; Vitest itself returned exit 0. The three skipped installed-provider tests +are opt-in and passed separately in the literal provider run above. + +REVIEW_PASSED diff --git a/ops/reviews/20260902-2048-pr136-maintainability.md b/ops/reviews/20260902-2048-pr136-maintainability.md new file mode 100644 index 00000000..d2888157 --- /dev/null +++ b/ops/reviews/20260902-2048-pr136-maintainability.md @@ -0,0 +1,573 @@ +# PR #136 fresh adversarial / maintainability review + +Date: 2026-09-02 + +- PR: `AgentWorkforce/flows#136` +- Exact head: `62a647fcae07edf7427e3cd2dcb4a618c0842dc3` +- Merged main / merge base: `a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2` +- Lens: later-step model ordering, unused and shadowed declarations, wrapper + symlink/swap and cwd identity, installed Codex outside Git, refusal before + run effects, type/schema drift, module size, and load-bearing tests +- Constitution read fully: `AGENTS.md` and + `docs/RFC-0001-everything-is-a-relayflow.md` +- Scope: assessment only. No product or gate change is retained; no commit, + push, merge, or release action was performed. + +## Verdict + +**FAIL — one P1 execution-identity race remains.** + +The static preflight half is now strong. Validation and all unknown-model +collection happen before environment probes; unused and step-shadowed named +declarations remain visible; relative wrappers are bound to an absolute path; +and the real Codex adapter executes its selected model from a non-Git working +directory. The descriptor, TypeScript declarations, validator, preflight, and +kernel lowering agree on the new named-agent fields. + +The worker-side wrapper boundary is nevertheless fail-open under an executable +swap between its identification subprocess and its execution subprocess. +`runAgentCli` identifies `cli` in one `spawn`, then starts the instruction by +resolving the same pathname in a second `spawn`. A symlink can be atomically +retargeted after the first child returns the required token. The second target +does not have to implement or pass the identification protocol: it receives +the instruction, declared model, and wake context and can exit zero. The full +checked path reproduces this while `checkFlow` is green. + +## Blocking finding + +### P1 — wrapper identification and execution are two pathname resolutions, so a post-identification symlink swap bypasses the trust boundary + +The relevant sequence is `sdk/src/worker-cli.ts:49-67`: identify the custom +wrapper with `spawnInvocation(cli, identity.invocation, identityEnv)`, accept +the token, then add `RELAYFLOW_MODEL` and call a second +`spawnInvocation(cli, invocation, env)`. The second `spawn` at line 76 resolves +the pathname again. Absolute-path binding fixes cwd ambiguity but does not pin +the object named by that path. + +Independent full-path reproduction (`sdk/`), including preflight and its +absolute CLI binding: + +```sh +node --input-type=module <<'NODE' +import { mkdtempSync, writeFileSync, chmodSync, symlinkSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { checkFlow } from './dist/cli/check.js'; +import { runAgentCli } from './dist/worker-cli.js'; + +const directory = mkdtempSync(join(tmpdir(), 'pr136-checked-symlink-race-')); +try { + const good = join(directory, 'good-wrapper'); + const replacement = join(directory, 'replacement-wrapper'); + const link = join(directory, 'declared-wrapper'); + const counter = join(directory, 'identity-count'); + const evidence = join(directory, 'replacement-evidence.json'); + writeFileSync(replacement, `#!/usr/bin/env node\nconst fs = require('node:fs');\nfs.writeFileSync(${JSON.stringify(evidence)}, JSON.stringify({ argv: process.argv.slice(2), model: process.env.RELAYFLOW_MODEL ?? null, wake: process.env.RELAYFLOW_WAKE_CONTEXT ?? null }));\nprocess.stdout.write('{"replacement_executed":true}');\n`); + chmodSync(replacement, 0o755); + writeFileSync(good, `#!/usr/bin/env node\nconst fs = require('node:fs');\nif (process.argv[2] === '--relayflows-adapter-v1') {\n let count = 1; try { count = Number(fs.readFileSync(${JSON.stringify(counter)}, 'utf8')) + 1; } catch {}\n fs.writeFileSync(${JSON.stringify(counter)}, String(count));\n if (count === 2) {\n const temp = ${JSON.stringify(link)} + '.next';\n fs.symlinkSync(${JSON.stringify(replacement)}, temp);\n fs.renameSync(temp, ${JSON.stringify(link)});\n }\n process.stdout.write('relayflows-agent-cli-v1\\n');\n process.exit(0);\n}\nif (process.argv[2] === 'auth' && process.argv[3] === 'status') process.exit(0);\nprocess.exit(91);\n`); + chmodSync(good, 0o755); + symlinkSync(good, link); + writeFileSync(join(directory, 'flows.json'), JSON.stringify({ models: ['declared-sensitive-model'] })); + const flowPath = join(directory, 'flow.yaml'); + writeFileSync(flowPath, `version: '0.1.0'\nsteps:\n - id: race\n type: agent\n cli: ./declared-wrapper\n model: declared-sensitive-model\n instruction: MUST_NOT_REACH_REPLACEMENT\n`); + const checked = checkFlow(flowPath); + const boundCli = checked.flow?.steps[0]?.cli; + const result = await runAgentCli(boundCli ?? '', 'MUST_NOT_REACH_REPLACEMENT', { event: 'private' }, 'declared-sensitive-model'); + console.log(JSON.stringify({ + checkOk: checked.report.ok, + resolution: checked.report.resolutions[0], + boundCli, + identityCount: readFileSync(counter, 'utf8'), + result, + replacementEvidence: JSON.parse(readFileSync(evidence, 'utf8')), + })); +} finally { + rmSync(directory, { recursive: true, force: true }); +} +NODE +``` + +Captured output: + +```text +{"checkOk":true,"resolution":{"stepId":"race","cli":"./declared-wrapper","source":"step","model":"declared-sensitive-model"},"boundCli":"/var/folders/yv/nbp9l2c55wlbj1x0gml37s7c0000gn/T/pr136-checked-symlink-race-ynGYkU/declared-wrapper","identityCount":"2","result":{"exit_code":0,"stdout_tail":"{\"replacement_executed\":true}","stderr_tail":""},"replacementEvidence":{"argv":["MUST_NOT_REACH_REPLACEMENT"],"model":"declared-sensitive-model","wake":"{\"event\":\"private\"}"}} +``` + +This violates the fail-closed promise that the wrapper identified immediately +before execution is the process receiving the private execution inputs. It is +also a check/runtime identity gap even though cwd identity is now fixed. + +Required repair: make identification and secret-bearing execution one process +identity. A robust protocol can start the wrapper without private inputs, +validate its token, and only then send model, wake context, and instruction to +that same child over stdin/IPC. Merely comparing `realpath`, inode, or a digest +and then doing another pathname-based spawn leaves another check/use race. +Alternatively execute a sealed, already-open bundle-owned artifact through a +platform primitive that preserves the opened object identity. Add a +deterministic test whose identification child atomically retargets a symlink; +the replacement must not execute and must receive neither private environment +variable. + +## Static refusal and provenance checks + +### Later-step typo plus unused/shadowed named declarations causes zero probes + +Independent negative test (`sdk/`): + +```sh +node --input-type=module <<'NODE' +import { preflight } from './dist/preflight.js'; +const calls = []; +const result = preflight({ + version: '0.1.0', + agents: { + unused: { cli: 'must-not-run', model: 'unused-typo' }, + shadowed: { cli: 'must-not-run', model: 'shadowed-typo' }, + }, + triggers: [{ id: 'trigger', executor: 'must-not-probe' }], + steps: [ + { id: 'valid-first', type: 'agent', cli: 'must-not-run', model: 'known', instruction: 'valid' }, + { id: 'shadowed-step', type: 'agent', agent: 'shadowed', cli: 'must-not-run', model: 'known', instruction: 'shadow' }, + { id: 'later-typo', type: 'agent', cli: 'must-not-run', model: 'known-modle', instruction: 'typo' }, + ], +}, { + models: ['known'], + probes: { + cli: (...args) => { calls.push(['cli', ...args]); throw new Error('must not probe'); }, + command: (...args) => { calls.push(['command', ...args]); throw new Error('must not probe'); }, + executor: (...args) => { calls.push(['executor', ...args]); throw new Error('must not probe'); }, + }, +}); +console.log(JSON.stringify({ ok: result.ok, calls, diagnostics: result.diagnostics.map(({kind, agent, stepId, model}) => ({kind, agent: agent ?? null, stepId: stepId ?? null, model: model ?? null})) })); +NODE +``` + +Captured output: + +```text +{"ok":false,"calls":[],"diagnostics":[{"kind":"model_unknown","agent":"unused","stepId":null,"model":"unused-typo"},{"kind":"model_unknown","agent":"shadowed","stepId":null,"model":"shadowed-typo"},{"kind":"model_unknown","agent":null,"stepId":"later-typo","model":"known-modle"}]} +``` + +Run-surface command (`sdk/`): + +```sh +node --input-type=module <<'NODE' +import { mkdtempSync, mkdirSync, writeFileSync, chmodSync, existsSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createServer } from 'node:net'; +import { runFlow } from './dist/cli/run.js'; +const directory = mkdtempSync(join(tmpdir(), 'pr136-no-effects-')); +try { + const log = join(directory, 'probe.log'); + const cli = join(directory, 'wrapper'); + writeFileSync(cli, `#!/bin/sh\nprintf called >> ${JSON.stringify(log)}\nexit 0\n`); + chmodSync(cli, 0o755); + writeFileSync(join(directory, 'flows.json'), JSON.stringify({ models: ['known'] })); + const flow = join(directory, 'flow.yaml'); + writeFileSync(flow, `version: '0.1.0'\nsteps:\n - { id: first, type: agent, cli: ${JSON.stringify(cli)}, model: known, instruction: valid }\n - { id: later, type: agent, cli: ${JSON.stringify(cli)}, model: impossible-typo, instruction: impossible }\n`); + const dataDir = join(directory, 'daemon'); + mkdirSync(dataDir); + let connections = 0; + const server = createServer((socket) => { connections += 1; socket.destroy(); }); + await new Promise((resolve, reject) => { server.once('error', reject); server.listen(join(dataDir, 'relayflowd.sock'), resolve); }); + const result = await runFlow(flow, dataDir); + await new Promise((resolve) => server.close(resolve)); + console.log(JSON.stringify({ exitCode: result.exitCode, diagnostics: result.report.diagnostics.map(({kind}) => kind), probeLogExists: existsSync(log), daemonConnections: connections })); +} finally { + rmSync(directory, { recursive: true, force: true }); +} +NODE +``` + +Captured output: + +```text +{"exitCode":2,"diagnostics":["model_unknown"],"probeLogExists":false,"daemonConnections":0} +``` + +Thus static refusal precedes +CLI, command, executor, and journal-client effects independent of step order. + +### Mutation verification: the pure first-pass return is load-bearing + +I temporarily replaced the return after `unknownModelDiagnostics` with a +comment, ran the focused ordering/declaration tests, restored the line with +`apply_patch`, and reran. Before and after SHA-256 was +`7d4a8823f09fa43d2397ead103d1748ff6940738fa00d85630c59292f0e04c7c`. + +Mutated command: + +```sh +./node_modules/.bin/vitest run tests/preflight.test.ts -t 'validates every inline model before every probe|returns every named and inline unknown-model diagnostic|checks an unknown .* named declaration' --reporter=verbose --maxWorkers=1 --minWorkers=1 +``` + +Captured mutated output: + +```text + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + + × tests/preflight.test.ts > preflight: CLI resolution and refusal predicates > validates every inline model before every probe: valid first + → expected [ { severity: 'refusal', …(5) }, …(4) ] to deeply equal [ ObjectContaining{…} ] + × tests/preflight.test.ts > preflight: CLI resolution and refusal predicates > validates every inline model before every probe: typo first + → expected [ { severity: 'refusal', …(5) }, …(4) ] to deeply equal [ ObjectContaining{…} ] + × tests/preflight.test.ts > preflight: CLI resolution and refusal predicates > returns every named and inline unknown-model diagnostic in the pure first pass + → expected 1 to be +0 // Object.is equality + × tests/preflight.test.ts > preflight: CLI resolution and refusal predicates > checks an unknown unused named declaration before authoring metadata is erased + → expected [ { severity: 'refusal', …(5) }, …(1) ] to deeply equal [ ObjectContaining{…} ] + × tests/preflight.test.ts > preflight: CLI resolution and refusal predicates > checks an unknown shadowed named declaration before authoring metadata is erased + → expected 1 to be +0 // Object.is equality + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 5 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/preflight.test.ts > preflight: CLI resolution and refusal predicates > validates every inline model before every probe: valid first + FAIL tests/preflight.test.ts > preflight: CLI resolution and refusal predicates > validates every inline model before every probe: typo first +AssertionError: expected [ { severity: 'refusal', …(5) }, …(4) ] to deeply equal [ ObjectContaining{…} ] + +- Expected ++ Received + + Array [ + ObjectContaining { + "kind": "model_unknown", + "model": "known-modle", + "stepId": "typo", + }, ++ Object { ++ "kind": "command_unprovable", ++ "message": "Step \"deterministic\" command \"./must-not-probe\" could not be probed, so its presence is unproven before execution.", ++ "severity": "warning", ++ "stepId": "deterministic", ++ }, ++ Object { ++ "cli": "claude", ++ "kind": "probe_failed", ++ "message": "Could not verify CLI \"claude\" for step \"valid\".", ++ "severity": "refusal", ++ "stepId": "valid", ++ }, ++ Object { ++ "cli": "claude", ++ "kind": "probe_failed", ++ "message": "Could not verify CLI \"claude\" for step \"typo\".", ++ "severity": "refusal", ++ "stepId": "typo", ++ }, ++ Object { ++ "executor": "must-not-probe", ++ "kind": "probe_failed", ++ "message": "Could not verify executor \"must-not-probe\" for trigger \"trigger\".", ++ "severity": "refusal", ++ "triggerId": "trigger", ++ }, + ] + + ❯ tests/preflight.test.ts:344:32 + + FAIL tests/preflight.test.ts > preflight: CLI resolution and refusal predicates > returns every named and inline unknown-model diagnostic in the pure first pass +AssertionError: expected 1 to be +0 // Object.is equality + +- Expected ++ Received + +- 0 ++ 1 + + ❯ tests/preflight.test.ts:378:24 + + FAIL tests/preflight.test.ts > preflight: CLI resolution and refusal predicates > checks an unknown unused named declaration before authoring metadata is erased +AssertionError: expected [ { severity: 'refusal', …(5) }, …(1) ] to deeply equal [ ObjectContaining{…} ] + +- Expected ++ Received + + Array [ + ObjectContaining { + "agent": "reviewer", + "kind": "model_unknown", + "model": "typo-model", + }, ++ Object { ++ "kind": "unprovable_effects", ++ "message": "Step \"ready\" command \"printf\" resolves, but its effects cannot be proven before execution.", ++ "severity": "warning", ++ "stepId": "ready", ++ }, + ] + + ❯ tests/preflight.test.ts:409:34 + + FAIL tests/preflight.test.ts > preflight: CLI resolution and refusal predicates > checks an unknown shadowed named declaration before authoring metadata is erased +AssertionError: expected 1 to be +0 // Object.is equality + +- Expected ++ Received + +- 0 ++ 1 + + ❯ tests/preflight.test.ts:412:26 + + Test Files 1 failed (1) + Tests 5 failed | 17 skipped (22) + Start at 20:53:17 + Duration 224ms +``` + +The detailed assertion output showed command, CLI, and executor probe calls +after the mutation, including `probeCalls` changing from `0` to `1`. + +Restored command and captured output: + +```text + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + + ✓ tests/preflight.test.ts > preflight: CLI resolution and refusal predicates > validates every inline model before every probe: valid first + ✓ tests/preflight.test.ts > preflight: CLI resolution and refusal predicates > validates every inline model before every probe: typo first + ✓ tests/preflight.test.ts > preflight: CLI resolution and refusal predicates > returns every named and inline unknown-model diagnostic in the pure first pass + ✓ tests/preflight.test.ts > preflight: CLI resolution and refusal predicates > checks an unknown unused named declaration before authoring metadata is erased + ✓ tests/preflight.test.ts > preflight: CLI resolution and refusal predicates > checks an unknown shadowed named declaration before authoring metadata is erased + + Test Files 1 passed (1) + Tests 5 passed | 17 skipped (22) + Duration 200ms + +7d4a8823f09fa43d2397ead103d1748ff6940738fa00d85630c59292f0e04c7c src/preflight.ts +``` + +### Mutation verification: the existing execution-time identity test is load-bearing but does not cover the two-spawn race + +I changed only the wrapper-identification conditional to false, ran its live +test, restored it byte-for-byte, and reran. Before and after SHA-256 was +`fe4a23734ce43250e3446842413b610d54e3e1035138c1f952a261d650af093f`. + +Mutated output: + +```text +FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker refuses a nonconforming journal-submitted wrapper before exposing RELAYFLOW_MODEL + +- Expected ++ Received + + Object { + "argv": Array [ +- "--relayflows-adapter-v1", ++ "This instruction must not execute.", + ], +- "model": null, ++ "model": "declared-model-xyz", + } + +Test Files 1 failed (1) +Tests 1 failed | 19 skipped (20) +``` + +Restored output: + +```text +✓ tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker refuses a nonconforming journal-submitted wrapper before exposing RELAYFLOW_MODEL 392ms + +Test Files 1 passed (1) +Tests 1 passed | 19 skipped (20) + +fe4a23734ce43250e3446842413b610d54e3e1035138c1f952a261d650af093f src/worker-cli.ts +``` + +The current test proves replacement *before* dispatch is caught. The blocking +reproduction changes the path after that second identification subprocess has +already succeeded, which is why the suite remains green. + +## Real-provider and regression evidence + +Installed provider suite (`sdk/`): + +```sh +RELAYFLOWS_REAL_CLI_ADAPTERS=1 ./node_modules/.bin/vitest run tests/real-cli-adapters.test.ts --reporter=verbose --maxWorkers=1 --minWorkers=1 +``` + +Captured output: + +```text + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + +✓ installed raw CLI adapters > round-trips the exact declared Claude model and refuses an impossible one 5669ms +✓ installed raw CLI adapters > uses Codex login status and classifies an impossible model as unavailable 7860ms +✓ installed raw CLI adapters > executes the declared Codex model from a real non-Git directory 6878ms + +Test Files 1 passed (1) + Tests 3 passed (3) + Start at 20:53:58 + Duration 20.61s (transform 49ms, setup 0ms, collect 90ms, tests 20.41s, environment 0ms, prepare 30ms) +``` + +Focused typecheck and model/schema suites (`sdk/`): + +```sh +./node_modules/.bin/tsc --noEmit && ./node_modules/.bin/vitest run tests/preflight.test.ts tests/cli.test.ts tests/model-selection.test.ts tests/verb-field-lint.test.ts tests/cli-adapter.test.ts --reporter=dot --maxWorkers=1 --minWorkers=1 +``` + +Captured output: + +```text + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + +✓ tests/cli.test.ts (63 tests) 6845ms + ✓ flows check CLI > binds a checked relative wrapper to the flow directory for worker execution 2205ms + ✓ flows check CLI > uses the raw Claude adapter model flag instead of accepting auth status as model proof 910ms + ✓ flows check CLI > uses Codex login status and reports a rejected model as unavailable, not unauthenticated 727ms + ✓ flows check CLI > refuses a nonconforming custom wrapper without calling it an authentication failure 332ms + ✓ flows check CLI > accepts an exact allowlisted named-agent model and probes that model 593ms + ✓ flows check CLI > checks the same named-agent contract from declarative JSON 573ms + ✓ flows check CLI > distinguishes an allowlisted but inaccessible model from broken auth 837ms +✓ tests/preflight.test.ts (22 tests) 5ms +✓ tests/verb-field-lint.test.ts (63 tests) 36ms +✓ tests/model-selection.test.ts (10 tests) 8ms +✓ tests/cli-adapter.test.ts (3 tests) 4ms + +Test Files 5 passed (5) + Tests 161 passed (161) + Start at 20:56:09 + Duration 7.62s (transform 106ms, setup 0ms, collect 200ms, tests 6.90s, environment 0ms, prepare 137ms) +``` + +Full serial SDK/live-kernel suite (`sdk/`): + +```sh +RELAYFLOWD_BIN=/Users/khaliqgant/.relayflows-toolchain/target/gate-contract/debug/relayflowd RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 ./node_modules/.bin/vitest run --reporter=dot --maxWorkers=1 --minWorkers=1 +``` + +Captured output: + +```text + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/Users/khaliqgant/.relayflows-toolchain/target/gate-contract/debug/relayflowd +LIVE_KERNEL flows=/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk/dist/cli.js + +stdout | tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +LIVE_ANALYZER ready: claude -p --model claude-haiku-4-5-20251001 round-trip OK + +stdout | tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +LIVE_ANALYZER analysis: {"reasoning":"This story is highly relevant to AI agents and automation as it directly describes an autonomous agent system automating core software development workflows, specifically the opening and review of pull requests, which represents a sophisticated practical application of agent technology.","relevance_score":9,"story_title":"Show HN: an agent that opens and reviews its own pull requests [wake-nonce-7f3a91c4]"} + +stdout | tests/live-kernel.test.ts > surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once +LIVE_KERNEL kill -9 pid=46875 run=01M1HQKTF8N4VC4HEHQEHX5YS8 while step=two state=Running + +✓ tests/live-kernel.test.ts (20 tests) 49305ms + ✓ built flows CLI against live relayflowd > runs rung (a), parks rung (b), and keeps JSON report-shaped 462ms + ✓ built flows CLI against live relayflowd > allows a deterministic run to exceed the bounded request timeout 32411ms + ✓ built flows CLI against live relayflowd > can always get a parked run to a late-attaching worker 5594ms + ✓ built flows CLI against live relayflowd > AgentWorker passes a declared model to an identified wrapper as RELAYFLOW_MODEL 312ms + ✓ built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI 8185ms + ✓ surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once 379ms +✓ tests/cli.test.ts (63 tests) 2085ms + ✓ flows check CLI > uses the raw Claude adapter model flag instead of accepting auth status as model proof 304ms + ✓ flows check CLI > uses Codex login status and reports a rejected model as unavailable, not unauthenticated 557ms +✓ tests/preflight.test.ts (22 tests) 5ms +✓ tests/journal-client.test.ts (13 tests) 65ms +✓ tests/validate.test.ts (36 tests) 7ms +✓ tests/cli-hn-monitor.test.ts (16 tests) 53ms +✓ tests/verb-field-lint.test.ts (63 tests) 35ms +✓ tests/backlog-picker.test.ts (14 tests) 38ms +SKIPPED_UNACTIONABLE=0 +SKIPPED_UNACTIONABLE=0 +NO_ACTIONABLE_BACKLOG_ENTRY scanned=0 +node:fs:539 + return binding.readFileUtf8(path, stringToFlags(options.flag)); + ^ + +Error: ENOENT: no such file or directory, open '.relayflow/backlog-picker-entry.json' + at Object.readFileSync (node:fs:539:20) + at [eval]:1:478 + at runScriptInThisContext (node:internal/vm:219:10) + at node:internal/process/execution:483:12 + at [eval]-wrapper:6:24 + at runScriptInContext (node:internal/process/execution:481:60) + at evalFunction (node:internal/process/execution:315:30) + at evalTypeScript (node:internal/process/execution:327:3) + at node:internal/main/eval_string:71:3 { + errno: -2, + code: 'ENOENT', + syscall: 'open', + path: '.relayflow/backlog-picker-entry.json' +} + +Node.js v26.7.0 +SKIPPED_UNACTIONABLE=0 +✓ tests/backlog-picker-flow.test.ts (6 tests) 289ms +SKIPPED_UNACTIONABLE=0 +NO_ACTIONABLE_BACKLOG_ENTRY scanned=1 Scoped but unverifiable[missing_definition_of_done] +✓ tests/work-package-consumer.test.ts (13 tests) 115ms +✓ tests/model-selection.test.ts (10 tests) 8ms +✓ tests/deterministic-llm.test.ts (5 tests) 7ms +✓ tests/bin.test.ts (7 tests) 318ms +✓ tests/hn-poller.test.ts (6 tests) 3ms +✓ tests/dir-watcher-poller.test.ts (6 tests) 3ms +✓ tests/hello-deterministic.test.ts (5 tests) 7ms +✓ tests/work-package-validator.test.ts (7 tests) 3ms +✓ tests/spec-parity.test.ts (15 tests) 16ms +↓ tests/real-cli-adapters.test.ts (3 tests | 3 skipped) +✓ tests/parse-json-output.test.ts (7 tests) 1ms +✓ tests/cli-adapter.test.ts (3 tests) 2ms + +Test Files 20 passed | 1 skipped (21) + Tests 337 passed | 3 skipped (340) + Start at 20:54:40 + Duration 54.70s (transform 148ms, setup 0ms, collect 437ms, tests 52.36s, environment 1ms, prepare 430ms) +``` + +The three skips are exactly the opt-in installed-provider file, which was run +separately above and passed all three cases. + +## Type/schema and module-size assessment + +No new type/schema drift found. `FlowSpec.agents`, `AgentStepSpec.agent`, and +the optional step models align with `FLOW_FIELDS`, +`AGENT_DECLARATION_FIELDS = ['cli', 'model']`, and +`STEP_FIELDS_BY_TYPE.agent`. `validateSpec`, public `preflight`, +`compileSpec`, and `toKernelSpec` exercise the same descriptors; the generated +foreign-field matrix tests every cross-verb pair. Named metadata survives to +preflight and is erased only when selected values lower into the existing +kernel `cli`/`model` fields. + +Literal size comparison: + +```text +sdk/src/compile.ts base=410 head=444 delta=34 +sdk/src/preflight.ts base=319 head=438 delta=119 +sdk/src/validate.ts base=475 head=429 delta=-46 +sdk/src/cli/check.ts base=263 head=374 delta=111 +sdk/src/cli-adapter.ts base=0 head=118 delta=118 +sdk/src/worker-cli.ts base=0 head=110 delta=110 +``` + +No production module crosses 500 lines, and the adapter/worker split is good. +`compile.ts`, `preflight.ts`, and `validate.ts` are nevertheless all within 71 +lines of the repository's design-smell threshold. This PR improved validation +cohesion by extracting `step-fields.ts`, `unknown-keys.ts`, and +`step-dependencies.ts`; that is enough to avoid a separate blocker here. +`cli/check.ts` now owns input/dialect parsing, config discovery, resolution, +probing, and path binding and grew by 111 lines. The next feature in that area +should split those concerns rather than pushing it toward another large +runner-shaped module. + +## Exact-head and restoration hygiene + +Literal final command and output before writing this report: + +```text +$ git rev-parse HEAD +62a647fcae07edf7427e3cd2dcb4a618c0842dc3 +$ git merge-base main HEAD +a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2 +$ git status --short +$ git diff --check main...HEAD +$ printf 'diff_check_exit=%s\n' "$?" +diff_check_exit=0 +``` + +Both mutated product files matched their pre-mutation SHA-256 values before +this report was created. Only this review file is intended to be staged. + +REVIEW_FAILED diff --git a/ops/reviews/20260902-2048-pr136-structure.md b/ops/reviews/20260902-2048-pr136-structure.md new file mode 100644 index 00000000..93643ffb --- /dev/null +++ b/ops/reviews/20260902-2048-pr136-structure.md @@ -0,0 +1,399 @@ +# PR #136 independent structure / RFC review + +Date: 2026-09-02 + +Verdict: **CHANGES REQUESTED — merge blocked by one P1 preflight-ordering defect.** + +Reviewed exact head `62a647fcae07edf7427e3cd2dcb4a618c0842dc3` +against merged `main` `a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2`. +This was an assessment-only pass. No product code, gate, merge, push, or +mutation-verification action was performed. + +## Finding + +### P1 — a later static `cli_unresolved` refusal does not prevent earlier live probes + +`preflight` correctly performs whole-spec schema validation and whole-spec +model-registry validation before probing (`sdk/src/preflight.ts:101-120`). It +does **not** perform every static refusal pass before probing. The step loop +calls the deterministic-command probe at line 123 and the CLI/model probe at +line 137, while discovering `cli_unresolved` only as each step is reached at +lines 126-134. + +Consequently, ordering the statically unresolved step after a valid agent or +deterministic step executes a probe before returning the refusal. A declared +model probe is a real provider round trip, so this can spend latency/provider +capacity on a flow that was already provably unable to start. That violates the +requested zero-probe-before-all-static-refusals boundary and RFC covenant 2's +submit-time proof discipline. + +Required repair: split preflight into pure and effectful phases. After schema +and model-registry validation, resolve every llm/agent CLI and collect every +`cli_unresolved` diagnostic for the complete flow. If any static refusal +exists, return with zero calls to `probes.cli`, `probes.command`, or +`probes.executor`. Only then run command, CLI/model, and executor probes. +Pin both step orders and include a deterministic step and trigger in the +no-probe assertion so a future per-kind repair cannot reopen the boundary. + +Literal reproduction from `sdk/`: + +```sh +node --input-type=module <<'NODE' +import { preflight } from './dist/preflight.js'; + +for (const [label, steps] of [ + ['cli-before-unresolved', [ + { id: 'probed', type: 'agent', cli: 'wrapper', instruction: 'work' }, + { id: 'static-refusal', type: 'agent', instruction: 'missing cli' }, + ]], + ['command-before-unresolved', [ + { id: 'probed', type: 'deterministic', command: 'printf ok' }, + { id: 'static-refusal', type: 'llm', prompt: 'missing cli' }, + ]], +]) { + const calls = []; + const result = preflight({ version: '0.1.0', steps }, { + probes: { + cli: () => { calls.push('cli'); return { exists: true, supported: true, authenticated: true }; }, + command: () => { calls.push('command'); return true; }, + executor: () => { calls.push('executor'); return true; }, + }, + }); + console.log(JSON.stringify({ label, ok: result.ok, calls, kinds: result.diagnostics.map(({ kind }) => kind) })); +} +NODE +``` + +Captured output: + +```text +{"label":"cli-before-unresolved","ok":false,"calls":["cli"],"kinds":["cli_unresolved"]} +{"label":"command-before-unresolved","ok":false,"calls":["command"],"kinds":["unprovable_effects","cli_unresolved"]} +``` + +## Requested seam assessment + +- **Agents-map retention and precedence: sound.** `compileSpec` preserves the + top-level map and step selector (`sdk/src/compile.ts:66-87,118-131`). Public + preflight sees the unlowered declaration, including unused and shadowed + declarations. Kernel lowering independently applies explicit step CLI/model + over the corresponding named value (`sdk/src/compile.ts:145-162`); named CLI + then precedes flow/project CLI in preflight (`sdk/src/preflight.ts:228-244`). + There is deliberately no flow/project/ambient model default. The `agents` + map and `agent` selector do not cross the journal boundary. + +- **PR #138 descriptor composition: sound.** The composed centralized + descriptor includes root `agents`, the closed named declaration + `{ cli, model }`, and agent-step `agent` + (`sdk/src/step-fields.ts:3-16,33-37`). `validateSpec`, direct + `toKernelSpec`, and public `preflight` consume the composed validation path; + malformed cross-verb fields fail before probes. + +- **Canonical relative-wrapper identity boundary: sound.** `checkFlow` probes + the resolved executable, then binds relative step/named/flow declarations to + the flow directory and project declarations to the selected config directory + (`sdk/src/cli/check.ts:60-82,204-233`). The absolute executable is lowered + into the existing step field and therefore journaled. `AgentWorker` uses that + exact path and re-runs wrapper identification immediately before execution, + with wake context and `RELAYFLOW_MODEL` absent from identification + (`sdk/src/worker-cli.ts:25-67`). The live-kernel suite confirms the selected + model is present in `run.spawned` and reaches the wrapper. + +- **Model-registry typo diagnostics: sound for the declared contract.** The + nearest project `models` array is an exact, case-sensitive allowlist; + malformed/duplicate registries are `config_invalid`. Every named + declaration, including unused and model-shadowed entries, and every inline + model is checked in the pure first pass. `model_unknown` names the agent or + step, model, CLI when available, and registry path, and the typo paths call + no probes. A malformed declaration key such as `modle` is an author-facing + `invalid_spec` diagnostic with `did you mean "model"?`. + +- **Kernel journal boundary and v1 compatibility: sound.** The diff contains + zero kernel files. Merged `main` already carries optional `model` on the + existing kernel `agent` step; this PR adds authoring sugar and lowers it into + that existing field. `agents` and `agent` never enter the kernel spec. Both + base and head retain `JOURNAL_VERSION = 1`; an absent model remains valid by + the existing serde default. Kernel spec parity and journal tests pass, + including rung-C specs both with and without an agent model. + +## Revision and kernel-boundary evidence + +Command (repository root): + +```sh +printf 'kernel_diff_files='; git diff --name-only main...62a647fcae07edf7427e3cd2dcb4a618c0842dc3 -- kernel | wc -l | tr -d ' ' +printf 'main_journal_version='; git show main:kernel/relayflowd-core/src/lib.rs | rg -o 'JOURNAL_VERSION: u32 = [0-9]+' +printf 'head_journal_version='; git show 62a647fcae07edf7427e3cd2dcb4a618c0842dc3:kernel/relayflowd-core/src/lib.rs | rg -o 'JOURNAL_VERSION: u32 = [0-9]+' +printf 'head='; git rev-parse HEAD +printf 'main='; git rev-parse main +``` + +Captured output: + +```text +kernel_diff_files=0 +main_journal_version=JOURNAL_VERSION: u32 = 1 +head_journal_version=JOURNAL_VERSION: u32 = 1 +head=62a647fcae07edf7427e3cd2dcb4a618c0842dc3 +main=a0d42ffbdc7fb60b42c0b5bea4f58408249b08a2 +``` + +The existing optional agent-model field is identical on merged main and the +reviewed head. Command (repository root): + +```sh +git show main:kernel/relayflowd-core/src/spec.rs | sed -n '280,295p' +git show 62a647fcae07edf7427e3cd2dcb4a618c0842dc3:kernel/relayflowd-core/src/spec.rs | sed -n '280,295p' +``` + +Captured output: + +```text + cli: Option, + }, + Agent { + instruction: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + cli: Option, + /// Model the declared CLI should use. The kernel never calls a model + /// and never interprets this — it is carried and journaled so the + /// choice is part of the run's record rather than ambient host state, + /// then handed to the worker, which surfaces it to the CLI. + #[serde(default, skip_serializing_if = "Option::is_none")] + model: Option, + #[serde(default)] + recovery_mode: RecoveryMode, + /// Declared mutable surfaces (RFC Appendix A rule 1) — names only. + /// Revision/offset *pins* are runtime facts journaled per attempt + cli: Option, + }, + Agent { + instruction: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + cli: Option, + /// Model the declared CLI should use. The kernel never calls a model + /// and never interprets this — it is carried and journaled so the + /// choice is part of the run's record rather than ambient host state, + /// then handed to the worker, which surfaces it to the CLI. + #[serde(default, skip_serializing_if = "Option::is_none")] + model: Option, + #[serde(default)] + recovery_mode: RecoveryMode, + /// Declared mutable surfaces (RFC Appendix A rule 1) — names only. + /// Revision/offset *pins* are runtime facts journaled per attempt +``` + +## Focused public-boundary evidence + +Command (`sdk/`): + +```sh +./node_modules/.bin/vitest run tests/model-selection.test.ts --reporter=verbose --maxWorkers=1 --minWorkers=1 +./node_modules/.bin/vitest run tests/cli.test.ts -t 'binds a checked relative wrapper|refuses a typo model' --reporter=verbose --maxWorkers=1 --minWorkers=1 +./node_modules/.bin/vitest run tests/preflight.test.ts -t 'validates every inline model before every probe|checks an unknown .* named declaration|validates and resolves a raw named-agent' --reporter=verbose --maxWorkers=1 --minWorkers=1 +./node_modules/.bin/vitest run tests/verb-field-lint.test.ts -t 'pins the per-verb descriptor' --reporter=verbose --maxWorkers=1 --minWorkers=1 +``` + +Captured output: + +```text + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + + ✓ tests/model-selection.test.ts > named agent declarations > lowers a selected agent CLI and model into the existing agent step + ✓ tests/model-selection.test.ts > named agent declarations > applies independent precedence: step override > named agent > flow CLI + ✓ tests/model-selection.test.ts > named agent declarations > lowers a raw typed FlowSpec passed directly to the kernel mapper + ✓ tests/model-selection.test.ts > named agent declarations > refuses an unknown named agent at the authoring boundary + ✓ tests/model-selection.test.ts > named agent declarations > refuses malformed model "" with an author-facing field error + ✓ tests/model-selection.test.ts > named agent declarations > refuses malformed model " declared-model" with an author-facing field error + ✓ tests/model-selection.test.ts > named agent declarations > refuses malformed model "declared-model " with an author-facing field error + ✓ tests/model-selection.test.ts > named agent declarations > refuses malformed model "declared\\tmodel" with an author-facing field error + ✓ tests/model-selection.test.ts > named agent declarations > requires both CLI and model on every named declaration + ✓ tests/model-selection.test.ts > named agent declarations > refuses a model typo inside a named declaration instead of dropping it + + Test Files 1 passed (1) + Tests 10 passed (10) + Start at 20:56:48 + Duration 175ms (transform 41ms, setup 0ms, collect 64ms, tests 8ms, environment 0ms, prepare 31ms) + + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + + ✓ tests/cli.test.ts > flows check CLI > binds a checked relative wrapper to the flow directory for worker execution + ✓ tests/cli.test.ts > flows check CLI > refuses a typo model before probing or contacting relayflowd + + Test Files 1 passed (1) + Tests 2 passed | 61 skipped (63) + Start at 20:56:48 + Duration 340ms (transform 74ms, setup 0ms, collect 102ms, tests 144ms, environment 0ms, prepare 23ms) + + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + + ✓ tests/preflight.test.ts > preflight: CLI resolution and refusal predicates > validates and resolves a raw named-agent authoring spec at the public boundary + ✓ tests/preflight.test.ts > preflight: CLI resolution and refusal predicates > validates every inline model before every probe: valid first + ✓ tests/preflight.test.ts > preflight: CLI resolution and refusal predicates > validates every inline model before every probe: typo first + ✓ tests/preflight.test.ts > preflight: CLI resolution and refusal predicates > checks an unknown unused named declaration before authoring metadata is erased + ✓ tests/preflight.test.ts > preflight: CLI resolution and refusal predicates > checks an unknown shadowed named declaration before authoring metadata is erased + + Test Files 1 passed (1) + Tests 5 passed | 17 skipped (22) + Start at 20:56:49 + Duration 158ms (transform 45ms, setup 0ms, collect 62ms, tests 3ms, environment 0ms, prepare 23ms) + + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + + ✓ tests/verb-field-lint.test.ts > closed per-verb step fields > pins the per-verb descriptor and generates every foreign-field pair from it + + Test Files 1 passed (1) + Tests 1 passed | 62 skipped (63) + Start at 20:56:49 + Duration 185ms (transform 62ms, setup 0ms, collect 88ms, tests 1ms, environment 0ms, prepare 24ms) +``` + +## Full SDK and live-kernel regression evidence + +Command (`sdk/`): + +```sh +RELAYFLOWD_BIN=/Users/khaliqgant/.relayflows-toolchain/target/gate-contract/debug/relayflowd RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 ./node_modules/.bin/vitest run --reporter=dot --maxWorkers=1 --minWorkers=1 +``` + +Captured output (the `ENOENT` text is emitted by an exercised backlog-picker +fixture; Vitest records that file as passing and the command exits successfully): + +```text + RUN v2.1.9 /Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/Users/khaliqgant/.relayflows-toolchain/target/gate-contract/debug/relayflowd +LIVE_KERNEL flows=/Users/khaliqgant/AgentWorkforce/flows-132-model-wt/sdk/dist/cli.js + +stdout | tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +LIVE_ANALYZER ready: claude -p --model claude-haiku-4-5-20251001 round-trip OK + +stdout | tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +LIVE_ANALYZER analysis: {"reasoning":"This story is directly relevant to AI agents and automation, as it demonstrates an autonomous agent performing software development workflows (opening and reviewing pull requests) without human intervention, which is a core application of AI agent technology.","relevance_score":9,"story_title":"Show HN: an agent that opens and reviews its own pull requests [wake-nonce-7f3a91c4]"} + +stdout | tests/live-kernel.test.ts > surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once +LIVE_KERNEL kill -9 pid=46765 run=01M1HQKS5C6G2NTMM2CZEHB4WT while step=two state=Running + + ✓ tests/live-kernel.test.ts (20 tests) 51151ms + ✓ built flows CLI against live relayflowd > runs rung (a), parks rung (b), and keeps JSON report-shaped 525ms + ✓ built flows CLI against live relayflowd > allows a deterministic run to exceed the bounded request timeout 32407ms + ✓ built flows CLI against live relayflowd > runs an agent CLI end to end through the SDK worker 470ms + ✓ built flows CLI against live relayflowd > can always get a parked run to a late-attaching worker 5586ms + ✓ built flows CLI against live relayflowd > AgentWorker passes a declared model to an identified wrapper as RELAYFLOW_MODEL 359ms + ✓ built flows CLI against live relayflowd > AgentWorker refuses a nonconforming journal-submitted wrapper before exposing RELAYFLOW_MODEL 349ms + ✓ built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI 9487ms + ✓ surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once 408ms + ✓ tests/cli.test.ts (63 tests) 2404ms + ✓ flows check CLI > uses the raw Claude adapter model flag instead of accepting auth status as model proof 370ms + ✓ flows check CLI > resolves a project CLI path relative to the flows.json that declares it 383ms + ✓ tests/preflight.test.ts (22 tests) 5ms + ✓ tests/journal-client.test.ts (13 tests) 64ms + ✓ tests/validate.test.ts (36 tests) 7ms + ✓ tests/cli-hn-monitor.test.ts (16 tests) 48ms + ✓ tests/verb-field-lint.test.ts (63 tests) 34ms + ✓ tests/backlog-picker.test.ts (14 tests) 38ms +SKIPPED_UNACTIONABLE=0 +SKIPPED_UNACTIONABLE=0 +NO_ACTIONABLE_BACKLOG_ENTRY scanned=0 +node:fs:539 + return binding.readFileUtf8(path, stringToFlags(options.flag)); + ^ + +Error: ENOENT: no such file or directory, open '.relayflow/backlog-picker-entry.json' + at Object.readFileSync (node:fs:539:20) + at [eval]:1:478 + at runScriptInThisContext (node:internal/vm:219:10) + at node:internal/process/execution:483:12 + at [eval]-wrapper:6:24 + at runScriptInContext (node:internal/process/execution:481:60) + at evalFunction (node:internal/process/execution:315:30) + at evalTypeScript (node:internal/process/execution:327:3) + at node:internal/main/eval_string:71:3 { + errno: -2, + code: 'ENOENT', + syscall: 'open', + path: '.relayflow/backlog-picker-entry.json' +} + +Node.js v26.7.0 +SKIPPED_UNACTIONABLE=0 + ✓ tests/backlog-picker-flow.test.ts (6 tests) 292ms +SKIPPED_UNACTIONABLE=0 +NO_ACTIONABLE_BACKLOG_ENTRY scanned=1 Scoped but unverifiable[missing_definition_of_done] + ✓ tests/work-package-consumer.test.ts (13 tests) 116ms + ✓ tests/model-selection.test.ts (10 tests) 7ms + ✓ tests/deterministic-llm.test.ts (5 tests) 7ms + ✓ tests/bin.test.ts (7 tests) 348ms + ✓ tests/hn-poller.test.ts (6 tests) 3ms + ✓ tests/dir-watcher-poller.test.ts (6 tests) 2ms + ✓ tests/hello-deterministic.test.ts (5 tests) 8ms + ✓ tests/work-package-validator.test.ts (7 tests) 3ms + ✓ tests/spec-parity.test.ts (15 tests) 17ms + ↓ tests/real-cli-adapters.test.ts (3 tests | 3 skipped) + ✓ tests/parse-json-output.test.ts (7 tests) 1ms + ✓ tests/cli-adapter.test.ts (3 tests) 2ms + + Test Files 20 passed | 1 skipped (21) + Tests 337 passed | 3 skipped (340) + Start at 20:54:36 + Duration 56.91s (transform 178ms, setup 0ms, collect 468ms, tests 54.56s, environment 1ms, prepare 482ms) +``` + +## Kernel spec/journal evidence + +Command (`kernel/`): + +```sh +sh ../ops/cargo.sh test -p relayflowd-core --test spec_parity -- --nocapture +sh ../ops/cargo.sh test -p relayflowd-journal --lib -- --nocapture +``` + +Captured output: + +```text + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.29s + Running tests/spec_parity.rs (/Users/khaliqgant/.relayflows-toolchain/target/3390792696/debug/deps/spec_parity-25916cc116a7848d) + +running 5 tests +test the_kernel_parses_the_event_triggered_spec_and_stamps_the_same_hash ... ok +test the_kernel_parses_the_deterministic_rung_and_stamps_the_same_hash ... ok +test the_kernel_parses_the_sdk_compiled_spec_and_stamps_the_same_hash ... ok +test the_kernel_parses_the_rung_b_spec_and_stamps_the_same_hash ... ok +test the_kernel_parses_the_rung_c_agent_spec_and_stamps_the_same_hash ... ok + +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.06s + Running unittests src/lib.rs (/Users/khaliqgant/.relayflows-toolchain/target/3390792696/debug/deps/relayflowd_journal-b98f2a8a28c047d6) + +running 17 tests +test subscriptions::tests::sweep_ignores_subscriptions_whose_silence_is_still_within_budget ... ok +test registry::tests::registry_is_a_rebuildable_run_locator ... ok +test subscriptions::tests::last_run_for_subscription_returns_none_before_first_arrival ... ok +test subscriptions::tests::prune_sweep_claims_deletes_only_rows_older_than_cutoff ... ok +test subscriptions::tests::sweep_marks_row_stale_when_silence_exceeds_budget ... ok +test subscriptions::tests::last_run_for_subscription_returns_the_lex_greatest_ulid_regardless_of_insertion ... ok +test subscriptions::tests::sweep_does_not_re_emit_the_same_stale_row_on_a_later_tick ... ok +test subscriptions::tests::detect_without_latch_stays_available_for_the_next_sweep ... ok +test subscriptions::tests::latch_is_a_no_op_if_a_fresh_event_arrived_between_detect_and_latch ... ok +test subscriptions::tests::sweep_election_gives_the_first_caller_the_result_and_second_gets_empty ... ok +test subscriptions::tests::upsert_after_stale_re_arms_and_next_silence_can_re_emit ... ok +test subscriptions::tests::upsert_is_idempotent_across_bumps_and_preserves_event_type_updates ... ok +test tests::failed_commit_is_returned_not_swallowed ... ok +test tests::rollover_is_atomic_scaffolding_for_epoch_resume ... ok +test tests::append_is_durable_and_monotonic_after_reopen ... ok +test tests::effects_are_deduplicated_at_the_journal_boundary ... ok +test tests::an_unconfirmed_election_is_reclaimed_by_the_next_attempt_not_treated_as_done ... ok + +test result: ok. 17 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s +``` + +## Final assessment + +The named-agent/model feature is correctly composed with #138, stays on the +SDK side of the journal protocol, preserves v1 compatibility, binds checked +relative wrappers to the executable the worker later receives, and produces +useful typo diagnostics. The preflight phase boundary is nevertheless not yet +fail-closed for the whole flow: static CLI-resolution refusals can occur after +effectful probes. Repair and pin that ordering before merge. From deb99e6360377c67962b13cc0d9b1249550a8783 Mon Sep 17 00:00:00 2001 From: kjgbot Date: Wed, 2 Sep 2026 21:07:25 +0200 Subject: [PATCH 11/15] test(sdk): reproduce static and wrapper identity races Session-Id: 01a062b7-2aef-7772-8225-7cb00ee311dd --- sdk/tests/preflight.test.ts | 37 ++++++++++++++++ sdk/tests/worker-cli.test.ts | 83 ++++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 sdk/tests/worker-cli.test.ts diff --git a/sdk/tests/preflight.test.ts b/sdk/tests/preflight.test.ts index 27a6698f..692fac9a 100644 --- a/sdk/tests/preflight.test.ts +++ b/sdk/tests/preflight.test.ts @@ -95,6 +95,43 @@ describe('preflight: CLI resolution and refusal predicates', () => { expect(seen).toEqual(['step-cli', 'flow-cli', 'project-cli']); }); + it.each(['unresolved first', 'unresolved last'] as const)( + 'collects every static CLI refusal before every probe: %s', + (order) => { + const calls: string[] = []; + const unresolved: FlowSpec['steps'][number] = { + id: 'unresolved', + type: 'agent', + instruction: 'No CLI is declared.', + }; + const resolvable: FlowSpec['steps'][number] = { + id: 'resolvable', + type: 'agent', + cli: 'must-not-probe', + instruction: 'Would otherwise probe.', + }; + const result = preflight({ + version: '0.1.0', + steps: order === 'unresolved first' + ? [unresolved, resolvable, { id: 'command', type: 'deterministic', command: 'printf ready' }] + : [{ id: 'command', type: 'deterministic', command: 'printf ready' }, resolvable, unresolved], + triggers: [{ id: 'trigger', executor: 'must-not-probe' }], + }, { + probes: { + cli: () => { calls.push('cli'); return { exists: true, authenticated: true }; }, + command: () => { calls.push('command'); return true; }, + executor: () => { calls.push('executor'); return true; }, + }, + }); + + expect(result.ok).toBe(false); + expect(result.diagnostics).toEqual([ + expect.objectContaining({ kind: 'cli_unresolved', stepId: 'unresolved' }), + ]); + expect(calls).toEqual([]); + }, + ); + it('keeps missing and unauthenticated CLIs distinct and names the step and CLI', () => { const missing = preflight(flow({ id: 'missing-step', type: 'llm', prompt: 'p', cli: 'absent' }), { probes: probes({ cli: () => ({ exists: false, authenticated: false }) }), diff --git a/sdk/tests/worker-cli.test.ts b/sdk/tests/worker-cli.test.ts new file mode 100644 index 00000000..57bc8fab --- /dev/null +++ b/sdk/tests/worker-cli.test.ts @@ -0,0 +1,83 @@ +import { + chmodSync, + existsSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { runAgentCli } from '../src/worker-cli.js'; + +const directories: string[] = []; + +afterEach(() => { + for (const directory of directories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +describe('custom wrapper execution identity', () => { + it('keeps identification and private execution on one process when an absolute symlink retargets', async () => { + const directory = mkdtempSync(join(tmpdir(), 'flows-wrapper-symlink-')); + directories.push(directory); + const trusted = join(directory, 'trusted-wrapper'); + const replacement = join(directory, 'replacement-wrapper'); + const declared = join(directory, 'declared-wrapper'); + const evidence = join(directory, 'replacement-evidence.json'); + + writeFileSync(replacement, `#!/usr/bin/env node +const fs = require('node:fs'); +fs.writeFileSync(${JSON.stringify(evidence)}, JSON.stringify({ + argv: process.argv.slice(2), + model: process.env.RELAYFLOW_MODEL ?? null, + wake: process.env.RELAYFLOW_WAKE_CONTEXT ?? null, +})); +process.stdout.write('{"replacement":true}'); +`); + chmodSync(replacement, 0o755); + writeFileSync(trusted, `#!/usr/bin/env node +const fs = require('node:fs'); +if (process.argv[2] !== '--relayflows-adapter-v1') process.exit(90); +process.stdout.write('relayflows-agent-cli-v1\\n'); +const next = ${JSON.stringify(declared)} + '.next'; +fs.symlinkSync(${JSON.stringify(replacement)}, next); +fs.renameSync(next, ${JSON.stringify(declared)}); +let input = ''; +process.stdin.setEncoding('utf8'); +process.stdin.on('data', chunk => { input += chunk; }); +process.stdin.on('end', () => { + if (input.trim() === '') process.exit(0); + const request = JSON.parse(input); + process.stdout.write('relayflows-agent-cli-v1-execute\\n'); + process.stdout.write(JSON.stringify({ + trusted: true, + instruction: request.instruction, + model: request.model, + wakeContext: request.wakeContext, + })); +}); +`); + chmodSync(trusted, 0o755); + symlinkSync(trusted, declared); + + const result = await runAgentCli( + declared, + 'MUST_STAY_WITH_IDENTIFIED_PROCESS', + { private: 'wake' }, + 'private-model', + ); + + expect(result).toMatchObject({ exit_code: 0 }); + expect(JSON.parse(result.stdout_tail)).toEqual({ + trusted: true, + instruction: 'MUST_STAY_WITH_IDENTIFIED_PROCESS', + model: 'private-model', + wakeContext: { private: 'wake' }, + }); + expect(existsSync(evidence) ? readFileSync(evidence, 'utf8') : undefined).toBeUndefined(); + }); +}); From 3f129ba1ef955252c1a71efff0473384b6a47370 Mon Sep 17 00:00:00 2001 From: kjgbot Date: Wed, 2 Sep 2026 21:25:17 +0200 Subject: [PATCH 12/15] fix(sdk): seal wrapper execution handshake Session-Id: 01a062b7-2aef-7772-8225-7cb00ee311dd --- docs/SURFACE.md | 35 ++-- sdk/src/cli-adapter.ts | 9 +- sdk/src/preflight.ts | 20 ++- sdk/src/spec.ts | 4 +- sdk/src/worker-cli.ts | 162 +++++++++++++++--- sdk/tests/cli-adapter.test.ts | 10 +- sdk/tests/cli.test.ts | 20 ++- sdk/tests/live-kernel.test.ts | 18 +- sdk/tests/preflight.test.ts | 3 +- sdk/tests/worker-cli.test.ts | 18 ++ testdata/preflight/analyze-story-claude-cli | 46 +++-- .../preflight/analyze-story-echo-wake-cli | 9 +- .../analyze-story-missing-fields-cli | 23 +-- testdata/preflight/analyze-story-stub-cli | 24 ++- .../preflight/analyze-story-text-only-cli | 18 +- testdata/preflight/echo-model-cli | 11 +- testdata/preflight/wake-context-probe-cli | 7 +- testdata/preflight/wrapper-session.mjs | 30 ++++ 18 files changed, 321 insertions(+), 146 deletions(-) create mode 100644 testdata/preflight/wrapper-session.mjs diff --git a/docs/SURFACE.md b/docs/SURFACE.md index 9dd92696..ef926932 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -89,8 +89,8 @@ No process runs between events: the handler wakes, executes to its next await, p default. Model has no flow/project default. An inline step that selects no named declaration keeps the existing optional-model behavior. The worker explicitly removes ambient `RELAYFLOW_MODEL`; raw provider adapters use a - model flag, while an identified wrapper receives the variable only when the - step declares a model. + model flag, while at worker execution a custom wrapper receives the model + only inside its identified same-process session when the step declares one. **Anonymous resolution law:** `f.agent\`task\`` with no name is the *default agent*, resolved (never guessed) in order: step options → flow header → project config (`flows.json`) → platform default. *The platform-default rung is declared but not yet implemented: no platform default is provisioned as of gate 1, so a flow that reaches this rung refuses with `cli_unresolved` rather than guessing. `flows check` never invents an implicit default.* `flows check` prints each resolved step CLI and its declaration source, validates it before submission, and refuses a missing or unauthenticated resolution before the checked flow is submitted, never at minute 27. Gate 1 does not make this guarantee for callers that bypass `flows check`: the journal client's direct `run.start` path does not invoke surface preflight. @@ -110,15 +110,20 @@ No process runs between events: the handler wakes, executes to its next await, p Every other executable is a custom Relayflows wrapper and must first answer ` --relayflows-adapter-v1` with exactly `relayflows-agent-cli-v1`. Only an identified wrapper uses the established - ` auth status` plus exact `RELAYFLOW_MODEL` scoped-probe/execution - protocol. A missing or wrong identification is `cli_unsupported`, never + ` auth status` plus an exact-model scoped readiness probe. A missing or + wrong identification is `cli_unsupported`, never mislabeled as `cli_unauthenticated`. If a model-scoped probe fails, the adapter's real unscoped authentication command distinguishes - `model_unavailable` from `cli_unauthenticated`. The worker repeats the exact - wrapper identification immediately before execution, with the private model - and wake-context variables absent. A direct journal submission or an - executable replaced after preflight therefore completes `worker_error` - without receiving `RELAYFLOW_MODEL` unless the current binary identifies. + `model_unavailable` from `cli_unauthenticated`. At execution the worker + starts one wrapper process with only `--relayflows-adapter-v1` and a scrubbed + environment, waits for the exact identity token, then sends one JSON line + containing instruction plus any declared model/wake context over that + child's stdin. The same child must acknowledge with + `relayflows-agent-cli-v1-execute` before its remaining stdout is treated as + agent output. There is no second pathname resolution: replacing or + retargeting the declared executable after identification cannot receive the + private request. A direct journal submission, nonconforming wrapper, or + process that exits after identifying completes `worker_error`. `flows check` resolves the binary (a path is relative to the declaring flow or project config; a bare name resolves via `PATH`) and caches each resolved @@ -128,16 +133,18 @@ No process runs between events: the handler wakes, executes to its next await, p different directory identifies and executes the same binary. A probe that cannot start, is signaled, or exceeds its adapter timeout is `probe_failed`, with a classified diagnostic rather than a raw process - error. Every subprocess starts with ambient `RELAYFLOW_MODEL` removed; - provider adapters pass only the declared flag, and wrapper adapters set the - private variable only from the compiled step. Preflight never invokes an - undeclared model or guesses from host state. + error. Every subprocess starts with ambient `RELAYFLOW_MODEL` removed. + Provider adapters pass only the declared flag; wrapper readiness receives + only an allowlisted declared model, while worker instruction/model/wake + values travel only in the post-identification session request. Preflight + never invokes an undeclared model or guesses from host state. **Deterministic model registry:** model existence is not inferred from a regex or provider prefix. The nearest `flows.json` owns an exact, case-sensitive `models` allowlist. `flows check` first refuses a declared model absent from that list as `model_unknown`, without starting the CLI. - One pure first pass collects every unknown named and inline declaration + One pure first pass collects every unknown named/inline model and every + unresolved step CLI before any CLI, command, executor, or daemon probe, independent of step order. This includes every named declaration, even when unused or shadowed by a step override; diff --git a/sdk/src/cli-adapter.ts b/sdk/src/cli-adapter.ts index 72ca6a74..33affac0 100644 --- a/sdk/src/cli-adapter.ts +++ b/sdk/src/cli-adapter.ts @@ -5,7 +5,7 @@ export type CliAdapterKind = 'claude' | 'codex' | 'relayflows-wrapper-v1'; export interface CliInvocation { args: string[]; timeoutMs: number; - /** Set only for the explicit wrapper protocol; raw providers receive a model flag. */ + /** Set only for wrapper readiness probes; raw providers receive a model flag. */ modelEnv?: string; } @@ -16,6 +16,7 @@ export interface CliAdapterIdentification { export const WRAPPER_IDENTIFY_ARG = '--relayflows-adapter-v1'; export const WRAPPER_IDENTIFY_TOKEN = 'relayflows-agent-cli-v1'; +export const WRAPPER_EXECUTE_TOKEN = 'relayflows-agent-cli-v1-execute'; const MODEL_PROBE_PROMPT = 'Reply with exactly RELAYFLOWS_MODEL_READY and nothing else.'; @@ -99,11 +100,7 @@ export function agentExecution( timeoutMs: 0, }; } - return { - args: [instruction], - timeoutMs: 0, - ...(model === undefined ? {} : { modelEnv: model }), - }; + throw new Error('custom wrapper execution requires the runAgentCli same-process session'); } export function displayInvocation(cli: string, invocation: CliInvocation): string { diff --git a/sdk/src/preflight.ts b/sdk/src/preflight.ts index 1dce1cf9..4195f340 100644 --- a/sdk/src/preflight.ts +++ b/sdk/src/preflight.ts @@ -114,15 +114,15 @@ export function preflight(flow: FlowSpec, options: PreflightOptions): PreflightR } const diagnostics: PreflightDiagnostic[] = []; const resolutions: CliResolution[] = []; + const resolutionByStep = new Map(); const cliProbeResults = new Map(); diagnostics.push(...unknownModelDiagnostics(flow, options)); - if (diagnostics.length > 0) return { ok: false, resolutions, diagnostics }; - + // Resolve the complete flow before touching any environment fact. A later + // statically unresolved CLI makes the whole submission impossible, so no + // earlier command, provider/model, or trigger probe may run first. for (const step of flow.steps) { - warnOnUnprovableEffects(step, options.probes, diagnostics); if (step.type === 'deterministic') continue; - const resolution = resolveCli(step, flow, options.projectCli); if (resolution === undefined) { diagnostics.push({ @@ -131,9 +131,17 @@ export function preflight(flow: FlowSpec, options: PreflightOptions): PreflightR stepId: step.id, message: unresolvedCliMessage(step.id, options), }); - continue; + } else { + resolutions.push(resolution); + resolutionByStep.set(step.id, resolution); } - resolutions.push(resolution); + } + if (diagnostics.length > 0) return { ok: false, resolutions, diagnostics }; + + for (const step of flow.steps) { + warnOnUnprovableEffects(step, options.probes, diagnostics); + if (step.type === 'deterministic') continue; + const resolution = resolutionByStep.get(step.id)!; probeResolvedCli(resolution, options.probes, cliProbeResults, diagnostics); } diff --git a/sdk/src/spec.ts b/sdk/src/spec.ts index d7faf1b8..4f1485b7 100644 --- a/sdk/src/spec.ts +++ b/sdk/src/spec.ts @@ -142,8 +142,8 @@ export interface AgentStepSpec extends BaseStepSpec { cli?: string; /** * Model the declared CLI must use. Raw Claude/Codex adapters receive their - * real model flag; an identified Relayflows wrapper receives - * `RELAYFLOW_MODEL`. Declared here so the choice is journaled with the step + * real model flag; an identified Relayflows wrapper receives it in its + * same-process execution request. Declared here so the choice is journaled with the step * instead of being ambient host state. */ model?: string; diff --git a/sdk/src/worker-cli.ts b/sdk/src/worker-cli.ts index f3a67a08..552d2659 100644 --- a/sdk/src/worker-cli.ts +++ b/sdk/src/worker-cli.ts @@ -1,8 +1,9 @@ import { spawn } from 'node:child_process'; import { - adapterIdentification, agentExecution, cliAdapterKind, + WRAPPER_EXECUTE_TOKEN, + WRAPPER_IDENTIFY_ARG, WRAPPER_IDENTIFY_TOKEN, type CliInvocation, } from './cli-adapter.js'; @@ -11,8 +12,9 @@ import { export const WAKE_CONTEXT_ENV = 'RELAYFLOW_WAKE_CONTEXT'; /** - * Present only for a declared model sent to a currently identified custom - * wrapper. Raw Claude/Codex adapters receive provider-native model flags. + * Private wrapper model variable name. Ambient values are scrubbed; a wrapper + * may set it inside the already-identified process from the session request. + * Raw Claude/Codex adapters receive provider-native model flags. */ export const MODEL_ENV = 'RELAYFLOW_MODEL'; @@ -29,11 +31,16 @@ export async function runAgentCli( model?: string, ): Promise { const kind = cliAdapterKind(cli); - const invocation = agentExecution(kind, instruction, model); const env: NodeJS.ProcessEnv = { ...process.env }; delete env[WAKE_CONTEXT_ENV]; delete env[MODEL_ENV]; + if (kind === 'relayflows-wrapper-v1') { + return runWrapperSession(cli, instruction, wakeContext, model, env); + } + + const invocation = agentExecution(kind, instruction, model); + if (wakeContext !== undefined) { try { env[WAKE_CONTEXT_ENV] = JSON.stringify(wakeContext); @@ -46,27 +53,140 @@ export async function runAgentCli( } } - if (kind === 'relayflows-wrapper-v1') { - const identity = adapterIdentification(kind); - const identityEnv = { ...env }; - delete identityEnv[WAKE_CONTEXT_ENV]; - const identified = await spawnInvocation(cli, identity.invocation, identityEnv); - if ( - identified.exit_code !== 0 - || identified.stdout_tail.trim() !== identity.expectedStdout - ) { - return { - exit_code: null, - stdout_tail: '', - stderr_tail: `CLI "${cli}" did not identify as ${WRAPPER_IDENTIFY_TOKEN} at worker execution.`, - }; - } - } - if (invocation.modelEnv !== undefined) env[MODEL_ENV] = invocation.modelEnv; return spawnInvocation(cli, invocation, env); } +/** + * Custom wrappers identify and execute within one child process. Private + * values are withheld from argv/env and sent over stdin only after that exact + * process emits the identity token. A second acknowledgement proves it parsed + * the request; both protocol lines are removed from the agent output. + */ +function runWrapperSession( + cli: string, + instruction: string, + wakeContext: unknown, + model: string | undefined, + env: NodeJS.ProcessEnv, +): Promise { + let request: string; + try { + request = JSON.stringify({ + protocol: WRAPPER_IDENTIFY_TOKEN, + instruction, + ...(model !== undefined ? { model } : {}), + ...(wakeContext !== undefined ? { wakeContext } : {}), + }); + } catch (error) { + return Promise.resolve({ + exit_code: null, + stdout_tail: '', + stderr_tail: `wake_context could not be JSON-serialized for the CLI: ${String(error)}`, + }); + } + + return new Promise((resolve) => { + const child = spawn(cli, [WRAPPER_IDENTIFY_ARG], { + stdio: ['pipe', 'pipe', 'pipe'], + env, + }); + const output: string[] = []; + const stderr: Buffer[] = []; + let pending = ''; + let phase: 'identity' | 'ack' | 'execute' = 'identity'; + let protocolError: string | undefined; + let settled = false; + const timer = setTimeout(() => { + protocolError = `CLI "${cli}" did not identify as ${WRAPPER_IDENTIFY_TOKEN} within 10000ms.`; + child.kill('SIGTERM'); + }, 10_000); + + const finish = (result: WorkerCliResult): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve(result); + }; + const failProtocol = (message: string): void => { + if (protocolError !== undefined) return; + protocolError = message; + child.kill('SIGTERM'); + }; + const handshakeComplete = (): boolean => phase === 'execute'; + const acceptLine = (line: string): void => { + const normalized = line.endsWith('\r') ? line.slice(0, -1) : line; + if (phase === 'identity') { + if (normalized !== WRAPPER_IDENTIFY_TOKEN) { + failProtocol(`CLI "${cli}" did not identify as ${WRAPPER_IDENTIFY_TOKEN} at worker execution.`); + return; + } + phase = 'ack'; + child.stdin.end(`${request}\n`); + return; + } + if (phase === 'ack') { + if (normalized !== WRAPPER_EXECUTE_TOKEN) { + failProtocol(`CLI "${cli}" did not accept the ${WRAPPER_IDENTIFY_TOKEN} same-process execution request.`); + return; + } + phase = 'execute'; + clearTimeout(timer); + } + }; + + child.stdout.setEncoding('utf8'); + child.stdout.on('data', (chunk: string) => { + if (phase === 'execute') { + output.push(chunk); + return; + } + pending += chunk; + if (pending.length > 8_192) { + failProtocol(`CLI "${cli}" exceeded the wrapper handshake limit.`); + return; + } + while (!handshakeComplete()) { + const newline = pending.indexOf('\n'); + if (newline < 0) return; + const line = pending.slice(0, newline); + pending = pending.slice(newline + 1); + acceptLine(line); + if (protocolError !== undefined) return; + } + if (pending.length > 0) { + output.push(pending); + pending = ''; + } + }); + child.stderr.on('data', (chunk: Buffer) => stderr.push(chunk)); + child.stdin.on('error', () => { + // A child that exits before acknowledging is classified on close. + }); + child.once('error', (error) => finish({ + exit_code: null, + stdout_tail: '', + stderr_tail: error.message, + })); + child.once('close', (code) => { + if (protocolError !== undefined || phase !== 'execute') { + finish({ + exit_code: null, + stdout_tail: '', + stderr_tail: protocolError + ?? `CLI "${cli}" exited before completing the ${WRAPPER_IDENTIFY_TOKEN} same-process handshake.`, + }); + return; + } + finish({ + exit_code: code, + stdout_tail: output.join(''), + stderr_tail: Buffer.concat(stderr).toString('utf8'), + }); + }); + }); +} + function spawnInvocation( cli: string, invocation: CliInvocation, diff --git a/sdk/tests/cli-adapter.test.ts b/sdk/tests/cli-adapter.test.ts index 9196881e..573ea481 100644 --- a/sdk/tests/cli-adapter.test.ts +++ b/sdk/tests/cli-adapter.test.ts @@ -45,7 +45,7 @@ describe('typed CLI adapters', () => { }); }); - it('requires custom executables to identify before using the wrapper env protocol', () => { + it('requires custom executables to use the same-process wrapper session', () => { const kind = cliAdapterKind('/project/bin/team-reviewer'); expect(kind).toBe('relayflows-wrapper-v1'); @@ -55,10 +55,8 @@ describe('typed CLI adapters', () => { args: ['auth', 'status'], modelEnv: 'team-model', }); - expect(agentExecution(kind, 'Review.', 'team-model')).toEqual({ - args: ['Review.'], - timeoutMs: 0, - modelEnv: 'team-model', - }); + expect(() => agentExecution(kind, 'Review.', 'team-model')).toThrow( + 'custom wrapper execution requires the runAgentCli same-process session', + ); }); }); diff --git a/sdk/tests/cli.test.ts b/sdk/tests/cli.test.ts index 42641563..04f46b8c 100644 --- a/sdk/tests/cli.test.ts +++ b/sdk/tests/cli.test.ts @@ -147,13 +147,19 @@ describe('flows check CLI', () => { it('binds a checked relative wrapper to the flow directory for worker execution', async () => { const directory = temporaryProject('flows-relative-worker-'); const wrapper = join(directory, 'wrapper'); - writeFileSync(wrapper, `#!/bin/sh -if [ "\${1-}" = "--relayflows-adapter-v1" ]; then - printf '%s\\n' relayflows-agent-cli-v1 - exit 0 -fi -if [ "\${1-} \${2-}" = "auth status" ]; then exit 0; fi -printf '%s' '{"executed":true}' + writeFileSync(wrapper, `#!/usr/bin/env node +if (process.argv[2] === 'auth' && process.argv[3] === 'status') process.exit(0); +if (process.argv[2] !== '--relayflows-adapter-v1') process.exit(9); +process.stdout.write('relayflows-agent-cli-v1\\n'); +let input = ''; +process.stdin.setEncoding('utf8'); +process.stdin.on('data', chunk => { input += chunk; }); +process.stdin.on('end', () => { + if (input.trim() === '') process.exit(0); + JSON.parse(input); + process.stdout.write('relayflows-agent-cli-v1-execute\\n'); + process.stdout.write('{"executed":true}'); +}); `); chmodSync(wrapper, 0o755); const path = join(directory, 'relative.flow.yaml'); diff --git a/sdk/tests/live-kernel.test.ts b/sdk/tests/live-kernel.test.ts index fc436eba..cc7325f3 100644 --- a/sdk/tests/live-kernel.test.ts +++ b/sdk/tests/live-kernel.test.ts @@ -210,7 +210,19 @@ steps: const directory = temporaryDirectory('flows-live-agent-worker-'); const dataDir = join(directory, 'data'); const cli = join(directory, 'agent-cli'); - writeFileSync(cli, '#!/bin/sh\nprintf \'handled: %s\' "$1"\n'); + writeFileSync(cli, `#!/usr/bin/env node +if (process.argv[2] !== '--relayflows-adapter-v1') process.exit(9); +process.stdout.write('relayflows-agent-cli-v1\\n'); +let input = ''; +process.stdin.setEncoding('utf8'); +process.stdin.on('data', chunk => { input += chunk; }); +process.stdin.on('end', () => { + if (input.trim() === '') process.exit(0); + const request = JSON.parse(input); + process.stdout.write('relayflows-agent-cli-v1-execute\\n'); + process.stdout.write('handled: ' + request.instruction); +}); +`); chmodSync(cli, 0o755); await startDaemon(dataDir); @@ -699,6 +711,10 @@ steps: await startDaemon(dataDir); const cli = join(dataDir, 'echo-model-cli'); writeFileSync(cli, readFileSync(join(TESTDATA, 'preflight', 'echo-model-cli'))); + writeFileSync( + join(dataDir, 'wrapper-session.mjs'), + readFileSync(join(TESTDATA, 'preflight', 'wrapper-session.mjs')), + ); chmodSync(cli, 0o755); writeFileSync(join(dataDir, 'flows.json'), JSON.stringify({ models: ['declared-model-xyz'] })); const flowPath = join(dataDir, 'relative-wrapper.flow.yaml'); diff --git a/sdk/tests/preflight.test.ts b/sdk/tests/preflight.test.ts index 692fac9a..d64bd441 100644 --- a/sdk/tests/preflight.test.ts +++ b/sdk/tests/preflight.test.ts @@ -336,7 +336,7 @@ describe('preflight: CLI resolution and refusal predicates', () => { expect(JSON.stringify(scenarios)).not.toContain('raw secret'); }); - it('reports an unknown model even when the same step has no resolvable CLI', () => { + it('reports every model and CLI static refusal on the same step', () => { const result = preflight( flow({ id: 'a', type: 'agent', instruction: 'i', model: 'typo-model' }), { models: ['known-model'], probes: probes() }, @@ -344,6 +344,7 @@ describe('preflight: CLI resolution and refusal predicates', () => { expect(result.diagnostics.map((diagnostic) => diagnostic.kind)).toEqual([ 'model_unknown', + 'cli_unresolved', ]); expect(result.diagnostics[0]).toMatchObject({ stepId: 'a', diff --git a/sdk/tests/worker-cli.test.ts b/sdk/tests/worker-cli.test.ts index 57bc8fab..4989bda7 100644 --- a/sdk/tests/worker-cli.test.ts +++ b/sdk/tests/worker-cli.test.ts @@ -3,6 +3,7 @@ import { existsSync, mkdtempSync, readFileSync, + realpathSync, rmSync, symlinkSync, writeFileSync, @@ -42,6 +43,8 @@ process.stdout.write('{"replacement":true}'); writeFileSync(trusted, `#!/usr/bin/env node const fs = require('node:fs'); if (process.argv[2] !== '--relayflows-adapter-v1') process.exit(90); +const ambientModel = process.env.RELAYFLOW_MODEL ?? null; +const ambientWake = process.env.RELAYFLOW_WAKE_CONTEXT ?? null; process.stdout.write('relayflows-agent-cli-v1\\n'); const next = ${JSON.stringify(declared)} + '.next'; fs.symlinkSync(${JSON.stringify(replacement)}, next); @@ -58,18 +61,29 @@ process.stdin.on('end', () => { instruction: request.instruction, model: request.model, wakeContext: request.wakeContext, + ambientModel, + ambientWake, + argv: process.argv.slice(2), })); }); `); chmodSync(trusted, 0o755); symlinkSync(trusted, declared); + const priorModel = process.env.RELAYFLOW_MODEL; + const priorWake = process.env.RELAYFLOW_WAKE_CONTEXT; + process.env.RELAYFLOW_MODEL = 'ambient-model-must-not-cross'; + process.env.RELAYFLOW_WAKE_CONTEXT = 'ambient-wake-must-not-cross'; const result = await runAgentCli( declared, 'MUST_STAY_WITH_IDENTIFIED_PROCESS', { private: 'wake' }, 'private-model', ); + if (priorModel === undefined) delete process.env.RELAYFLOW_MODEL; + else process.env.RELAYFLOW_MODEL = priorModel; + if (priorWake === undefined) delete process.env.RELAYFLOW_WAKE_CONTEXT; + else process.env.RELAYFLOW_WAKE_CONTEXT = priorWake; expect(result).toMatchObject({ exit_code: 0 }); expect(JSON.parse(result.stdout_tail)).toEqual({ @@ -77,7 +91,11 @@ process.stdin.on('end', () => { instruction: 'MUST_STAY_WITH_IDENTIFIED_PROCESS', model: 'private-model', wakeContext: { private: 'wake' }, + ambientModel: null, + ambientWake: null, + argv: ['--relayflows-adapter-v1'], }); + expect(realpathSync(declared)).toBe(realpathSync(replacement)); expect(existsSync(evidence) ? readFileSync(evidence, 'utf8') : undefined).toBeUndefined(); }); }); diff --git a/testdata/preflight/analyze-story-claude-cli b/testdata/preflight/analyze-story-claude-cli index a3c786f1..c5695482 100755 --- a/testdata/preflight/analyze-story-claude-cli +++ b/testdata/preflight/analyze-story-claude-cli @@ -12,17 +12,13 @@ // prep step chmods `testdata/preflight/*-cli` — so the Node floor is // the tradeoff for matching it. // -// Three invocation shapes: -// `--relayflows-adapter-v1` → identify the explicit wrapper contract. -// `auth status` → model-scoped preflight round trip. -// `` → analyze; one JSON object on stdout, exit 0. +// Two invocation shapes: `auth status` performs the model-scoped preflight; +// `--relayflows-adapter-v1` opens the same-process worker session. import { spawnSync } from 'node:child_process'; +import { receiveWrapperRequest } from './wrapper-session.mjs'; -if (process.argv[2] === '--relayflows-adapter-v1') { - process.stdout.write('relayflows-agent-cli-v1\n'); - process.exit(0); -} +const wrapperRequest = await receiveWrapperRequest(); // Say so plainly. Without this, Node 20 fails on the ESM/top-level-await // above with a parse error that names a syntax position, not the cause. @@ -38,11 +34,10 @@ if (Number.isFinite(major) && major < 22) { // keeps this CLI independent of whatever the host has pinned. // // Precedence: the step's DECLARED model wins, because that is the choice -// recorded in the journal. AgentWorker passes it as RELAYFLOW_MODEL and -// leaves the variable UNSET when the step declared nothing, so absence is -// distinguishable here rather than arriving as an empty string. The -// operator override and the built-in default apply only to that absence. -const MODEL = process.env.RELAYFLOW_MODEL +// recorded in the journal. AgentWorker sends it only after this exact process +// identifies. The operator override and built-in default apply to absence. +const MODEL = wrapperRequest?.model + ?? process.env.RELAYFLOW_MODEL ?? process.env.RELAYFLOWS_ANALYZER_MODEL ?? 'claude-haiku-4-5-20251001'; @@ -75,20 +70,23 @@ if (process.argv[2] === 'auth' && process.argv[3] === 'status') { process.exit(0); } -const instruction = process.argv[2]; +const instruction = wrapperRequest?.instruction ?? process.argv[2]; if (!instruction) fail('no instruction argument supplied'); -// AgentWorker sets RELAYFLOW_WAKE_CONTEXT only when the run carries -// wake context (sdk/src/worker.ts). Absence is meaningful, so it is an -// error here rather than a silently-empty analysis. -const raw = process.env.RELAYFLOW_WAKE_CONTEXT; -if (!raw) fail('RELAYFLOW_WAKE_CONTEXT is unset — no triggering story to analyze'); - +// The same-process wrapper helper projects the request's wake context into +// RELAYFLOW_WAKE_CONTEXT only after identification. Absence is meaningful, so +// it is an error here rather than a silently-empty analysis. let story; -try { - story = JSON.parse(raw)?.triggering_event?.payload; -} catch (error) { - fail(`RELAYFLOW_WAKE_CONTEXT is not valid JSON: ${String(error)}`); +if (wrapperRequest?.wakeContext !== undefined) { + story = wrapperRequest.wakeContext?.triggering_event?.payload; +} else { + const raw = process.env.RELAYFLOW_WAKE_CONTEXT; + if (!raw) fail('wrapper wake context is absent — no triggering story to analyze'); + try { + story = JSON.parse(raw)?.triggering_event?.payload; + } catch (error) { + fail(`RELAYFLOW_WAKE_CONTEXT is not valid JSON: ${String(error)}`); + } } if (!story || story.id === undefined) fail('wake context has no triggering_event.payload.id'); diff --git a/testdata/preflight/analyze-story-echo-wake-cli b/testdata/preflight/analyze-story-echo-wake-cli index 58ad6421..a04e6873 100755 --- a/testdata/preflight/analyze-story-echo-wake-cli +++ b/testdata/preflight/analyze-story-echo-wake-cli @@ -9,12 +9,11 @@ // // Node-native so the test never needs to skip on missing tooling // (jq is not a stock macOS dep) — the SDK ships Node already. -if (process.argv[2] === '--relayflows-adapter-v1') { - process.stdout.write('relayflows-agent-cli-v1\n'); - process.exit(0); -} +import { receiveWrapperRequest } from './wrapper-session.mjs'; + +await receiveWrapperRequest(); const raw = process.env.RELAYFLOW_WAKE_CONTEXT; -if (!raw) { +if (raw === undefined) { process.stderr.write('wake-context env var RELAYFLOW_WAKE_CONTEXT was empty or unset\n'); process.exit(1); } diff --git a/testdata/preflight/analyze-story-missing-fields-cli b/testdata/preflight/analyze-story-missing-fields-cli index 69b5ae4b..e6c1c23f 100755 --- a/testdata/preflight/analyze-story-missing-fields-cli +++ b/testdata/preflight/analyze-story-missing-fields-cli @@ -1,17 +1,6 @@ -#!/bin/sh -# Stub agent-runtime CLI that emits JSON MISSING required schema -# fields (only story_title, no relevance_score/reasoning). Used by -# the negative gate-2-clause-2 test: pins that the json_schema gate -# is LIVE over the promoted payload — a mutation that removed the -# schema check would let this stub's exit-0 slide through to -# `success`, and the negative test would fail. (A separate mutation -# that reverted the wrapper promotion would ALSO make the run fail, -# but for a different reason — the wrapper's shape lacks every -# required schema field. The positive test above catches the -# wrapper-not-promoted mutation.) -set -eu -if [ "${1-}" = "--relayflows-adapter-v1" ]; then - printf '%s\n' 'relayflows-agent-cli-v1' - exit 0 -fi -printf '%s' '{"story_title":"partial"}' +#!/usr/bin/env node +// Negative schema-gate wrapper fixture: intentionally omits required fields. +import { receiveWrapperRequest } from './wrapper-session.mjs'; + +await receiveWrapperRequest(); +process.stdout.write(JSON.stringify({ story_title: 'partial' })); diff --git a/testdata/preflight/analyze-story-stub-cli b/testdata/preflight/analyze-story-stub-cli index 1992983b..f5e009c0 100755 --- a/testdata/preflight/analyze-story-stub-cli +++ b/testdata/preflight/analyze-story-stub-cli @@ -1,14 +1,10 @@ -#!/bin/sh -# Stub agent-runtime CLI for the hn-monitor gate-2-clause-2 demo. -# Emits a deterministic JSON payload that satisfies the -# `analyze-story` step's `json_schema` verification gate. -# Not a real analyzer — its purpose is to prove the dispatch → -# agent-CLI → stepComplete pipeline works end-to-end. A real analyzer -# would read `$1` (the instruction) and — once wake_context injection -# lands — the triggering-event JSON, then invoke a real LLM. -set -eu -if [ "${1-}" = "--relayflows-adapter-v1" ]; then - printf '%s\n' 'relayflows-agent-cli-v1' - exit 0 -fi -printf '%s' '{"story_title":"stub","relevance_score":5,"reasoning":"stub agent runtime — deterministic output for gate-2 clause-2 demo"}' +#!/usr/bin/env node +// Deterministic gate-2 wrapper fixture. +import { receiveWrapperRequest } from './wrapper-session.mjs'; + +await receiveWrapperRequest(); +process.stdout.write(JSON.stringify({ + story_title: 'stub', + relevance_score: 5, + reasoning: 'stub agent runtime — deterministic output for gate-2 clause-2 demo', +})); diff --git a/testdata/preflight/analyze-story-text-only-cli b/testdata/preflight/analyze-story-text-only-cli index 2a89e8b8..3d344134 100755 --- a/testdata/preflight/analyze-story-text-only-cli +++ b/testdata/preflight/analyze-story-text-only-cli @@ -1,12 +1,6 @@ -#!/bin/sh -# Text-emitting stub agent-runtime CLI (no JSON in stdout). Used by -# the text-fallback test: proves AgentWorker keeps the CliResult -# wrapper as `output` when stdout is not JSON, so text-emitting -# tools (progress bars, chatty CLIs) still round-trip usefully -# without the JSON promotion path silently discarding stdout/stderr. -set -eu -if [ "${1-}" = "--relayflows-adapter-v1" ]; then - printf '%s\n' 'relayflows-agent-cli-v1' - exit 0 -fi -printf '%s' 'looked at the story, seemed fine to me' +#!/usr/bin/env node +// Text-only wrapper fixture for the CliResult fallback. +import { receiveWrapperRequest } from './wrapper-session.mjs'; + +await receiveWrapperRequest(); +process.stdout.write('looked at the story, seemed fine to me'); diff --git a/testdata/preflight/echo-model-cli b/testdata/preflight/echo-model-cli index b5db5edd..24653da0 100755 --- a/testdata/preflight/echo-model-cli +++ b/testdata/preflight/echo-model-cli @@ -1,6 +1,6 @@ #!/usr/bin/env node -// Stub agent-runtime CLI that reports what AgentWorker did with -// $RELAYFLOW_MODEL, for the set/unset pair in live-kernel.test.ts. +// Stub agent-runtime CLI that reports the model AgentWorker delivered through +// the identified same-process wrapper session. // // The distinction under test is three-way, not two-way: a step that // DECLARED a model, a step that declared none, and the failure mode where @@ -9,10 +9,9 @@ // string as its own answer — if `RELAYFLOW_MODEL=''` ever reaches a CLI, // that is a bug the test must be able to see rather than silently read as // "unset". -if (process.argv[2] === '--relayflows-adapter-v1') { - process.stdout.write('relayflows-agent-cli-v1\n'); - process.exit(0); -} +import { receiveWrapperRequest } from './wrapper-session.mjs'; + +await receiveWrapperRequest(); const raw = process.env.RELAYFLOW_MODEL; const declared = raw === undefined ? 'UNSET' : raw === '' ? 'EMPTY' : raw; process.stdout.write(JSON.stringify({ diff --git a/testdata/preflight/wake-context-probe-cli b/testdata/preflight/wake-context-probe-cli index d1c51ecd..fb431e7b 100755 --- a/testdata/preflight/wake-context-probe-cli +++ b/testdata/preflight/wake-context-probe-cli @@ -4,9 +4,8 @@ // wake-context-absent test to pin the undefined-vs-null // invariant AgentWorker documents: a run started without a // triggering event must dispatch with no env var set. -if (process.argv[2] === '--relayflows-adapter-v1') { - process.stdout.write('relayflows-agent-cli-v1\n'); - process.exit(0); -} +import { receiveWrapperRequest } from './wrapper-session.mjs'; + +await receiveWrapperRequest(); const present = process.env.RELAYFLOW_WAKE_CONTEXT !== undefined; process.stdout.write(JSON.stringify({ env_present: present })); diff --git a/testdata/preflight/wrapper-session.mjs b/testdata/preflight/wrapper-session.mjs new file mode 100644 index 00000000..4baa8363 --- /dev/null +++ b/testdata/preflight/wrapper-session.mjs @@ -0,0 +1,30 @@ +const IDENTIFY_ARG = '--relayflows-adapter-v1'; +const IDENTIFY_TOKEN = 'relayflows-agent-cli-v1'; +const EXECUTE_TOKEN = 'relayflows-agent-cli-v1-execute'; + +/** + * Enter the wrapper-v1 same-process session when invoked with IDENTIFY_ARG. + * An identify-only preflight closes stdin and exits after the first token. A + * worker sends one JSON request; only then do we acknowledge and return it. + */ +export async function receiveWrapperRequest() { + if (process.argv[2] !== IDENTIFY_ARG) return undefined; + delete process.env.RELAYFLOW_MODEL; + delete process.env.RELAYFLOW_WAKE_CONTEXT; + process.stdout.write(`${IDENTIFY_TOKEN}\n`); + let input = ''; + process.stdin.setEncoding('utf8'); + for await (const chunk of process.stdin) input += chunk; + if (input.trim() === '') process.exit(0); + const request = JSON.parse(input); + if (request.protocol !== IDENTIFY_TOKEN || typeof request.instruction !== 'string') { + process.stderr.write('invalid relayflows wrapper session request\n'); + process.exit(2); + } + if (request.model !== undefined) process.env.RELAYFLOW_MODEL = request.model; + if (request.wakeContext !== undefined) { + process.env.RELAYFLOW_WAKE_CONTEXT = JSON.stringify(request.wakeContext); + } + process.stdout.write(`${EXECUTE_TOKEN}\n`); + return request; +} From ade4b67b185f3a1957f1723237d74b3b09cc9af9 Mon Sep 17 00:00:00 2001 From: kjgbot Date: Wed, 2 Sep 2026 22:10:39 +0200 Subject: [PATCH 13/15] test(sdk): reproduce wrapper execution boundary gaps Session-Id: 01a062b7-2aef-7772-8225-7cb00ee311dd --- sdk/tests/worker-cli.test.ts | 218 +++++++++++++++++++++++++++-------- 1 file changed, 167 insertions(+), 51 deletions(-) diff --git a/sdk/tests/worker-cli.test.ts b/sdk/tests/worker-cli.test.ts index 4989bda7..f7c0db44 100644 --- a/sdk/tests/worker-cli.test.ts +++ b/sdk/tests/worker-cli.test.ts @@ -2,7 +2,6 @@ import { chmodSync, existsSync, mkdtempSync, - readFileSync, realpathSync, rmSync, symlinkSync, @@ -21,30 +20,114 @@ afterEach(() => { } }); +function makeDirectory(): string { + const directory = mkdtempSync(join(tmpdir(), 'flows-wrapper-')); + directories.push(directory); + return directory; +} + +function makeWrapper(directory: string, name: string, source: string): string { + const wrapper = join(directory, name); + writeFileSync(wrapper, `#!/usr/bin/env node\n${source}`); + chmodSync(wrapper, 0o755); + return wrapper; +} + +async function withEnvironment( + values: Record, + operation: () => Promise, +): Promise { + const prior = new Map(); + for (const [name, value] of Object.entries(values)) { + prior.set(name, process.env[name]); + process.env[name] = value; + } + try { + return await operation(); + } finally { + for (const [name, value] of prior) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + } +} + describe('custom wrapper execution identity', () => { - it('keeps identification and private execution on one process when an absolute symlink retargets', async () => { - const directory = mkdtempSync(join(tmpdir(), 'flows-wrapper-symlink-')); - directories.push(directory); - const trusted = join(directory, 'trusted-wrapper'); - const replacement = join(directory, 'replacement-wrapper'); - const declared = join(directory, 'declared-wrapper'); - const evidence = join(directory, 'replacement-evidence.json'); + it('passes an explicit safe environment at identification and execution', async () => { + const directory = makeDirectory(); + const wrapper = makeWrapper(directory, 'environment-wrapper', ` +const secretNames = [ + 'RELAYFLOW_MODEL', + 'RELAYFLOW_WAKE_CONTEXT', + 'RELAYFLOWS_TEST_SECRET', + 'AWS_SECRET_ACCESS_KEY', + 'GITHUB_TOKEN', +]; +const snapshot = () => Object.fromEntries(secretNames.map(name => [name, process.env[name] ?? null])); +const identificationEnvironment = snapshot(); +process.stdout.write('relayflows-agent-cli-v1\\n'); +let input = ''; +process.stdin.setEncoding('utf8'); +process.stdin.on('data', chunk => { input += chunk; }); +process.stdin.on('end', () => { + const request = JSON.parse(input); + process.stdout.write('relayflows-agent-cli-v1-execute\\n'); + const executionEnvironment = snapshot(); + process.stdout.write(JSON.stringify({ + identificationEnvironment, + executionEnvironment, + pathPresent: typeof process.env.PATH === 'string' && process.env.PATH.length > 0, + instruction: request.instruction, + model: request.model, + wakeContext: request.wakeContext, + })); +}); +`); + + const result = await withEnvironment({ + RELAYFLOW_MODEL: 'ambient-model', + RELAYFLOW_WAKE_CONTEXT: 'ambient-wake', + RELAYFLOWS_TEST_SECRET: 'private-test-secret', + AWS_SECRET_ACCESS_KEY: 'private-aws-secret', + GITHUB_TOKEN: 'private-github-token', + }, () => runAgentCli(wrapper, 'private instruction', { private: 'wake' }, 'private-model')); + + expect(result).toMatchObject({ exit_code: 0, stderr_tail: '' }); + expect(JSON.parse(result.stdout_tail)).toEqual({ + identificationEnvironment: { + RELAYFLOW_MODEL: null, + RELAYFLOW_WAKE_CONTEXT: null, + RELAYFLOWS_TEST_SECRET: null, + AWS_SECRET_ACCESS_KEY: null, + GITHUB_TOKEN: null, + }, + executionEnvironment: { + RELAYFLOW_MODEL: null, + RELAYFLOW_WAKE_CONTEXT: null, + RELAYFLOWS_TEST_SECRET: null, + AWS_SECRET_ACCESS_KEY: null, + GITHUB_TOKEN: null, + }, + pathPresent: true, + instruction: 'private instruction', + model: 'private-model', + wakeContext: { private: 'wake' }, + }); + }); - writeFileSync(replacement, `#!/usr/bin/env node + it('refuses a wrapper symlink retarget before delivering private values', async () => { + const directory = makeDirectory(); + const declared = join(directory, 'declared-wrapper'); + const requestEvidence = join(directory, 'trusted-request-evidence.json'); + const replacementEvidence = join(directory, 'replacement-evidence.json'); + const replacement = makeWrapper(directory, 'replacement-wrapper', ` const fs = require('node:fs'); -fs.writeFileSync(${JSON.stringify(evidence)}, JSON.stringify({ - argv: process.argv.slice(2), - model: process.env.RELAYFLOW_MODEL ?? null, - wake: process.env.RELAYFLOW_WAKE_CONTEXT ?? null, -})); +fs.writeFileSync(${JSON.stringify(replacementEvidence)}, JSON.stringify(process.argv.slice(2))); process.stdout.write('{"replacement":true}'); `); - chmodSync(replacement, 0o755); - writeFileSync(trusted, `#!/usr/bin/env node + const trusted = makeWrapper(directory, 'trusted-wrapper', ` const fs = require('node:fs'); if (process.argv[2] !== '--relayflows-adapter-v1') process.exit(90); -const ambientModel = process.env.RELAYFLOW_MODEL ?? null; -const ambientWake = process.env.RELAYFLOW_WAKE_CONTEXT ?? null; process.stdout.write('relayflows-agent-cli-v1\\n'); const next = ${JSON.stringify(declared)} + '.next'; fs.symlinkSync(${JSON.stringify(replacement)}, next); @@ -53,49 +136,82 @@ let input = ''; process.stdin.setEncoding('utf8'); process.stdin.on('data', chunk => { input += chunk; }); process.stdin.on('end', () => { - if (input.trim() === '') process.exit(0); - const request = JSON.parse(input); - process.stdout.write('relayflows-agent-cli-v1-execute\\n'); - process.stdout.write(JSON.stringify({ - trusted: true, - instruction: request.instruction, - model: request.model, - wakeContext: request.wakeContext, - ambientModel, - ambientWake, - argv: process.argv.slice(2), - })); + if (input.trim() !== '') { + fs.writeFileSync(${JSON.stringify(requestEvidence)}, input); + process.stdout.write('relayflows-agent-cli-v1-execute\\n'); + } }); `); - chmodSync(trusted, 0o755); symlinkSync(trusted, declared); - const priorModel = process.env.RELAYFLOW_MODEL; - const priorWake = process.env.RELAYFLOW_WAKE_CONTEXT; - process.env.RELAYFLOW_MODEL = 'ambient-model-must-not-cross'; - process.env.RELAYFLOW_WAKE_CONTEXT = 'ambient-wake-must-not-cross'; const result = await runAgentCli( declared, - 'MUST_STAY_WITH_IDENTIFIED_PROCESS', + 'MUST_NOT_CROSS_RETARGET', { private: 'wake' }, 'private-model', ); - if (priorModel === undefined) delete process.env.RELAYFLOW_MODEL; - else process.env.RELAYFLOW_MODEL = priorModel; - if (priorWake === undefined) delete process.env.RELAYFLOW_WAKE_CONTEXT; - else process.env.RELAYFLOW_WAKE_CONTEXT = priorWake; - expect(result).toMatchObject({ exit_code: 0 }); - expect(JSON.parse(result.stdout_tail)).toEqual({ - trusted: true, - instruction: 'MUST_STAY_WITH_IDENTIFIED_PROCESS', - model: 'private-model', - wakeContext: { private: 'wake' }, - ambientModel: null, - ambientWake: null, - argv: ['--relayflows-adapter-v1'], - }); + expect(result.exit_code).toBeNull(); + expect(result.stderr_tail).toMatch(/identity changed/i); expect(realpathSync(declared)).toBe(realpathSync(replacement)); - expect(existsSync(evidence) ? readFileSync(evidence, 'utf8') : undefined).toBeUndefined(); + expect(existsSync(requestEvidence)).toBe(false); + expect(existsSync(replacementEvidence)).toBe(false); + }); + + it('bounds wrapper execution after acknowledgement', async () => { + const directory = makeDirectory(); + const wrapper = makeWrapper(directory, 'slow-wrapper', ` +process.stdout.write('relayflows-agent-cli-v1\\n'); +process.stdin.resume(); +process.stdin.on('end', () => { + process.stdout.write('relayflows-agent-cli-v1-execute\\n'); + setTimeout(() => process.exit(0), 250); +}); +`); + + const result = await runAgentCli(wrapper, 'instruction', undefined, undefined, { + executionTimeoutMs: 50, + }); + + expect(result.exit_code).toBeNull(); + expect(result.stderr_tail).toMatch(/timed out after 50ms/i); + }); + + it('bounds captured wrapper output', async () => { + const directory = makeDirectory(); + const wrapper = makeWrapper(directory, 'noisy-wrapper', ` +process.stdout.write('relayflows-agent-cli-v1\\n'); +process.stdin.resume(); +process.stdin.on('end', () => { + process.stdout.write('relayflows-agent-cli-v1-execute\\n'); + process.stdout.write('x'.repeat(256)); +}); +`); + + const result = await runAgentCli(wrapper, 'instruction', undefined, undefined, { + maxOutputBytes: 128, + }); + + expect(result.exit_code).toBeNull(); + expect(result.stderr_tail).toMatch(/output limit of 128 bytes/i); + }); + + it('refuses a duplicate execute protocol frame', async () => { + const directory = makeDirectory(); + const wrapper = makeWrapper(directory, 'duplicate-frame-wrapper', ` +process.stdout.write('relayflows-agent-cli-v1\\n'); +process.stdin.resume(); +process.stdin.on('end', () => { + process.stdout.write('relayflows-agent-cli-v1-execute\\n'); + process.stdout.write('relayflows-agent-cli-v1-execute\\n'); + process.stdout.write('{"mustNotBeAccepted":true}'); +}); +`); + + const result = await runAgentCli(wrapper, 'instruction', undefined); + + expect(result.exit_code).toBeNull(); + expect(result.stdout_tail).toBe(''); + expect(result.stderr_tail).toMatch(/duplicate execute protocol frame/i); }); }); From 3fcf2dcbfcb056060b8f91e2ab6decfea5b34ba1 Mon Sep 17 00:00:00 2001 From: kjgbot Date: Wed, 2 Sep 2026 22:19:03 +0200 Subject: [PATCH 14/15] fix(sdk): harden custom wrapper execution boundary Session-Id: 01a062b7-2aef-7772-8225-7cb00ee311dd --- sdk/src/worker-cli.ts | 154 +++------------------ sdk/src/wrapper-runtime.ts | 93 +++++++++++++ sdk/src/wrapper-session.ts | 253 +++++++++++++++++++++++++++++++++++ sdk/tests/worker-cli.test.ts | 5 +- 4 files changed, 367 insertions(+), 138 deletions(-) create mode 100644 sdk/src/wrapper-runtime.ts create mode 100644 sdk/src/wrapper-session.ts diff --git a/sdk/src/worker-cli.ts b/sdk/src/worker-cli.ts index 552d2659..70cbd935 100644 --- a/sdk/src/worker-cli.ts +++ b/sdk/src/worker-cli.ts @@ -2,11 +2,13 @@ import { spawn } from 'node:child_process'; import { agentExecution, cliAdapterKind, - WRAPPER_EXECUTE_TOKEN, - WRAPPER_IDENTIFY_ARG, - WRAPPER_IDENTIFY_TOKEN, type CliInvocation, } from './cli-adapter.js'; +import { + runWrapperSession, + type WrapperSessionLimits, +} from './wrapper-session.js'; +import { wrapperEnvironment } from './wrapper-runtime.js'; /** Present only when a dispatched agent step carries a journaled wake context. */ export const WAKE_CONTEXT_ENV = 'RELAYFLOW_WAKE_CONTEXT'; @@ -29,16 +31,24 @@ export async function runAgentCli( instruction: string, wakeContext: unknown, model?: string, + wrapperLimits?: Partial, ): Promise { const kind = cliAdapterKind(cli); - const env: NodeJS.ProcessEnv = { ...process.env }; - delete env[WAKE_CONTEXT_ENV]; - delete env[MODEL_ENV]; if (kind === 'relayflows-wrapper-v1') { - return runWrapperSession(cli, instruction, wakeContext, model, env); + return runWrapperSession( + cli, + instruction, + wakeContext, + model, + wrapperEnvironment(process.env), + wrapperLimits, + ); } + const env: NodeJS.ProcessEnv = { ...process.env }; + delete env[WAKE_CONTEXT_ENV]; + delete env[MODEL_ENV]; const invocation = agentExecution(kind, instruction, model); if (wakeContext !== undefined) { @@ -57,136 +67,6 @@ export async function runAgentCli( return spawnInvocation(cli, invocation, env); } -/** - * Custom wrappers identify and execute within one child process. Private - * values are withheld from argv/env and sent over stdin only after that exact - * process emits the identity token. A second acknowledgement proves it parsed - * the request; both protocol lines are removed from the agent output. - */ -function runWrapperSession( - cli: string, - instruction: string, - wakeContext: unknown, - model: string | undefined, - env: NodeJS.ProcessEnv, -): Promise { - let request: string; - try { - request = JSON.stringify({ - protocol: WRAPPER_IDENTIFY_TOKEN, - instruction, - ...(model !== undefined ? { model } : {}), - ...(wakeContext !== undefined ? { wakeContext } : {}), - }); - } catch (error) { - return Promise.resolve({ - exit_code: null, - stdout_tail: '', - stderr_tail: `wake_context could not be JSON-serialized for the CLI: ${String(error)}`, - }); - } - - return new Promise((resolve) => { - const child = spawn(cli, [WRAPPER_IDENTIFY_ARG], { - stdio: ['pipe', 'pipe', 'pipe'], - env, - }); - const output: string[] = []; - const stderr: Buffer[] = []; - let pending = ''; - let phase: 'identity' | 'ack' | 'execute' = 'identity'; - let protocolError: string | undefined; - let settled = false; - const timer = setTimeout(() => { - protocolError = `CLI "${cli}" did not identify as ${WRAPPER_IDENTIFY_TOKEN} within 10000ms.`; - child.kill('SIGTERM'); - }, 10_000); - - const finish = (result: WorkerCliResult): void => { - if (settled) return; - settled = true; - clearTimeout(timer); - resolve(result); - }; - const failProtocol = (message: string): void => { - if (protocolError !== undefined) return; - protocolError = message; - child.kill('SIGTERM'); - }; - const handshakeComplete = (): boolean => phase === 'execute'; - const acceptLine = (line: string): void => { - const normalized = line.endsWith('\r') ? line.slice(0, -1) : line; - if (phase === 'identity') { - if (normalized !== WRAPPER_IDENTIFY_TOKEN) { - failProtocol(`CLI "${cli}" did not identify as ${WRAPPER_IDENTIFY_TOKEN} at worker execution.`); - return; - } - phase = 'ack'; - child.stdin.end(`${request}\n`); - return; - } - if (phase === 'ack') { - if (normalized !== WRAPPER_EXECUTE_TOKEN) { - failProtocol(`CLI "${cli}" did not accept the ${WRAPPER_IDENTIFY_TOKEN} same-process execution request.`); - return; - } - phase = 'execute'; - clearTimeout(timer); - } - }; - - child.stdout.setEncoding('utf8'); - child.stdout.on('data', (chunk: string) => { - if (phase === 'execute') { - output.push(chunk); - return; - } - pending += chunk; - if (pending.length > 8_192) { - failProtocol(`CLI "${cli}" exceeded the wrapper handshake limit.`); - return; - } - while (!handshakeComplete()) { - const newline = pending.indexOf('\n'); - if (newline < 0) return; - const line = pending.slice(0, newline); - pending = pending.slice(newline + 1); - acceptLine(line); - if (protocolError !== undefined) return; - } - if (pending.length > 0) { - output.push(pending); - pending = ''; - } - }); - child.stderr.on('data', (chunk: Buffer) => stderr.push(chunk)); - child.stdin.on('error', () => { - // A child that exits before acknowledging is classified on close. - }); - child.once('error', (error) => finish({ - exit_code: null, - stdout_tail: '', - stderr_tail: error.message, - })); - child.once('close', (code) => { - if (protocolError !== undefined || phase !== 'execute') { - finish({ - exit_code: null, - stdout_tail: '', - stderr_tail: protocolError - ?? `CLI "${cli}" exited before completing the ${WRAPPER_IDENTIFY_TOKEN} same-process handshake.`, - }); - return; - } - finish({ - exit_code: code, - stdout_tail: output.join(''), - stderr_tail: Buffer.concat(stderr).toString('utf8'), - }); - }); - }); -} - function spawnInvocation( cli: string, invocation: CliInvocation, diff --git a/sdk/src/wrapper-runtime.ts b/sdk/src/wrapper-runtime.ts new file mode 100644 index 00000000..7d671647 --- /dev/null +++ b/sdk/src/wrapper-runtime.ts @@ -0,0 +1,93 @@ +import { accessSync, constants, realpathSync, statSync } from 'node:fs'; +import { delimiter, isAbsolute, join, resolve } from 'node:path'; + +const WRAPPER_ENV_ALLOWLIST = [ + 'PATH', + 'HOME', + 'TMPDIR', + 'TMP', + 'TEMP', + 'SHELL', + 'LANG', + 'LC_ALL', + 'LC_CTYPE', + 'TZ', + 'USER', + 'LOGNAME', + 'SystemRoot', + 'ComSpec', + 'PATHEXT', + 'WINDIR', +] as const; + +export interface WrapperIdentity { + executable: string; + fingerprint: string; +} + +/** Build a wrapper environment from a closed list; ambient credentials never cross. */ +export function wrapperEnvironment(source: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = {}; + for (const name of WRAPPER_ENV_ALLOWLIST) { + const value = source[name]; + if (value !== undefined) env[name] = value; + } + return env; +} + +/** Resolve and fingerprint the exact executable that will enter the private protocol. */ +export function captureWrapperIdentity( + cli: string, + env: NodeJS.ProcessEnv, +): WrapperIdentity { + const executable = realpathSync(resolveWrapperExecutable(cli, env)); + accessSync(executable, constants.X_OK); + const stat = statSync(executable, { bigint: true }); + if (!stat.isFile()) throw new Error('resolved wrapper is not a regular file'); + return { + executable, + fingerprint: [ + stat.dev, + stat.ino, + stat.mode, + stat.size, + stat.mtimeNs, + stat.ctimeNs, + ].join(':'), + }; +} + +export function sameWrapperIdentity( + cli: string, + env: NodeJS.ProcessEnv, + expected: WrapperIdentity, +): boolean { + try { + const current = captureWrapperIdentity(cli, env); + return current.executable === expected.executable + && current.fingerprint === expected.fingerprint; + } catch { + return false; + } +} + +function resolveWrapperExecutable(cli: string, env: NodeJS.ProcessEnv): string { + if (isAbsolute(cli) || cli.includes('/') || cli.includes('\\')) return resolve(cli); + const path = env.PATH; + if (path === undefined) throw new Error('PATH is unavailable while resolving the wrapper'); + const extensions = process.platform === 'win32' + ? (env.PATHEXT ?? '.EXE;.CMD;.BAT;.COM').split(';') + : ['']; + for (const directory of path.split(delimiter)) { + for (const extension of extensions) { + const candidate = join(directory || '.', `${cli}${extension}`); + try { + accessSync(candidate, constants.X_OK); + return candidate; + } catch { + // Continue through the deterministic PATH candidate list. + } + } + } + throw new Error(`wrapper executable ${JSON.stringify(cli)} was not found on PATH`); +} diff --git a/sdk/src/wrapper-session.ts b/sdk/src/wrapper-session.ts new file mode 100644 index 00000000..d9fd4d7f --- /dev/null +++ b/sdk/src/wrapper-session.ts @@ -0,0 +1,253 @@ +import { spawn } from 'node:child_process'; +import { + WRAPPER_EXECUTE_TOKEN, + WRAPPER_IDENTIFY_ARG, + WRAPPER_IDENTIFY_TOKEN, +} from './cli-adapter.js'; +import { + captureWrapperIdentity, + sameWrapperIdentity, + type WrapperIdentity, +} from './wrapper-runtime.js'; + +export interface WrapperSessionLimits { + handshakeTimeoutMs: number; + executionTimeoutMs: number; + maxOutputBytes: number; +} + +export interface WrapperSessionResult { + exit_code: number | null; + stdout_tail: string; + stderr_tail: string; +} + +const DEFAULT_LIMITS: WrapperSessionLimits = { + handshakeTimeoutMs: 10_000, + executionTimeoutMs: 300_000, + maxOutputBytes: 1_048_576, +}; +const HANDSHAKE_OUTPUT_LIMIT = 8_192; +const FORCE_KILL_DELAY_MS = 1_000; + +/** + * Identify and execute a custom wrapper in one pinned process. No private + * request is written until its declared path still names the captured file. + */ +export function runWrapperSession( + cli: string, + instruction: string, + wakeContext: unknown, + model: string | undefined, + env: NodeJS.ProcessEnv, + overrides: Partial = {}, +): Promise { + const limits = sessionLimits(overrides); + let request: string; + try { + request = JSON.stringify({ + protocol: WRAPPER_IDENTIFY_TOKEN, + instruction, + ...(model !== undefined ? { model } : {}), + ...(wakeContext !== undefined ? { wakeContext } : {}), + }); + } catch (error) { + return Promise.resolve(failure( + `wake_context could not be JSON-serialized for the CLI: ${String(error)}`, + )); + } + let identity: WrapperIdentity; + try { + identity = captureWrapperIdentity(cli, env); + } catch (error) { + return Promise.resolve(failure( + `CLI ${JSON.stringify(cli)} wrapper identity could not be pinned: ${String(error)}`, + )); + } + + return executePinnedWrapper(cli, identity, request, env, limits); +} + +function executePinnedWrapper( + cli: string, + identity: WrapperIdentity, + request: string, + env: NodeJS.ProcessEnv, + limits: WrapperSessionLimits, +): Promise { + return new Promise((resolve) => { + const child = spawn(identity.executable, [WRAPPER_IDENTIFY_ARG], { + stdio: ['pipe', 'pipe', 'pipe'], + env, + }); + const stdout: string[] = []; + const stderr: Buffer[] = []; + let handshakePending = ''; + let executionPending = ''; + let capturedBytes = 0; + let phase: 'identity' | 'ack' | 'execute' = 'identity'; + let protocolError: string | undefined; + let settled = false; + let lifecycleTimer: NodeJS.Timeout | undefined; + let killTimer: NodeJS.Timeout | undefined; + + const clearTimers = (): void => { + if (lifecycleTimer !== undefined) clearTimeout(lifecycleTimer); + if (killTimer !== undefined) clearTimeout(killTimer); + }; + const finish = (result: WrapperSessionResult): void => { + if (settled) return; + settled = true; + clearTimers(); + resolve(result); + }; + const terminate = (message: string): void => { + if (protocolError !== undefined) return; + protocolError = message; + if (lifecycleTimer !== undefined) clearTimeout(lifecycleTimer); + child.kill('SIGTERM'); + killTimer = setTimeout(() => child.kill('SIGKILL'), FORCE_KILL_DELAY_MS); + killTimer.unref(); + }; + const startExecutionTimer = (): void => { + if (lifecycleTimer !== undefined) clearTimeout(lifecycleTimer); + lifecycleTimer = setTimeout(() => terminate( + `CLI ${JSON.stringify(cli)} execution timed out after ${limits.executionTimeoutMs}ms.`, + ), limits.executionTimeoutMs); + }; + const exceedsOutputLimit = (additionalBytes: number): boolean => { + const pendingBytes = Buffer.byteLength(executionPending); + if (capturedBytes + pendingBytes + additionalBytes <= limits.maxOutputBytes) return false; + terminate( + `CLI ${JSON.stringify(cli)} exceeded the captured output limit of ${limits.maxOutputBytes} bytes.`, + ); + return true; + }; + const appendStdout = (value: string): void => { + const bytes = Buffer.byteLength(value); + capturedBytes += bytes; + stdout.push(value); + }; + const acceptHandshakeLine = (line: string): void => { + const normalized = normalizeLine(line); + if (phase === 'identity') { + if (normalized !== WRAPPER_IDENTIFY_TOKEN) { + terminate(`CLI ${JSON.stringify(cli)} did not identify as ${WRAPPER_IDENTIFY_TOKEN} at worker execution.`); + return; + } + if (!sameWrapperIdentity(cli, env, identity)) { + terminate(`CLI ${JSON.stringify(cli)} wrapper identity changed before private request delivery.`); + return; + } + phase = 'ack'; + child.stdin.end(`${request}\n`); + return; + } + if (phase === 'ack') { + if (normalized !== WRAPPER_EXECUTE_TOKEN) { + terminate(`CLI ${JSON.stringify(cli)} did not accept the ${WRAPPER_IDENTIFY_TOKEN} same-process execution request.`); + return; + } + phase = 'execute'; + startExecutionTimer(); + } + }; + const acceptExecutionData = (chunk: string): void => { + executionPending += chunk; + if (exceedsOutputLimit(0)) return; + while (true) { + const newline = executionPending.indexOf('\n'); + if (newline < 0) return; + const line = executionPending.slice(0, newline); + executionPending = executionPending.slice(newline + 1); + if (normalizeLine(line) === WRAPPER_EXECUTE_TOKEN) { + terminate(`CLI ${JSON.stringify(cli)} emitted a duplicate execute protocol frame.`); + return; + } + appendStdout(`${line}\n`); + } + }; + const handshakeComplete = (): boolean => phase === 'execute'; + + lifecycleTimer = setTimeout(() => terminate( + `CLI ${JSON.stringify(cli)} did not identify as ${WRAPPER_IDENTIFY_TOKEN} within ${limits.handshakeTimeoutMs}ms.`, + ), limits.handshakeTimeoutMs); + + child.stdout.setEncoding('utf8'); + child.stdout.on('data', (chunk: string) => { + if (protocolError !== undefined) return; + if (phase === 'execute') { + acceptExecutionData(chunk); + return; + } + handshakePending += chunk; + if (Buffer.byteLength(handshakePending) > HANDSHAKE_OUTPUT_LIMIT) { + terminate(`CLI ${JSON.stringify(cli)} exceeded the wrapper handshake limit.`); + return; + } + while (!handshakeComplete()) { + const newline = handshakePending.indexOf('\n'); + if (newline < 0) return; + const line = handshakePending.slice(0, newline); + handshakePending = handshakePending.slice(newline + 1); + acceptHandshakeLine(line); + if (protocolError !== undefined) return; + } + if (handshakePending.length > 0) { + const remainder = handshakePending; + handshakePending = ''; + acceptExecutionData(remainder); + } + }); + child.stderr.on('data', (chunk: Buffer) => { + if (protocolError !== undefined || exceedsOutputLimit(chunk.byteLength)) return; + capturedBytes += chunk.byteLength; + stderr.push(chunk); + }); + child.stdin.on('error', () => { + // A child that closes stdin before acknowledgement is classified on close. + }); + child.once('error', (error) => finish(failure(error.message))); + child.once('close', (code) => { + if (protocolError === undefined && phase === 'execute' && executionPending.length > 0) { + if (normalizeLine(executionPending) === WRAPPER_EXECUTE_TOKEN) { + protocolError = `CLI ${JSON.stringify(cli)} emitted a duplicate execute protocol frame.`; + } else if (!exceedsOutputLimit(0)) { + appendStdout(executionPending); + } + } + if (protocolError !== undefined || phase !== 'execute') { + finish(failure( + protocolError + ?? `CLI ${JSON.stringify(cli)} exited before completing the ${WRAPPER_IDENTIFY_TOKEN} same-process handshake.`, + )); + return; + } + finish({ + exit_code: code, + stdout_tail: stdout.join(''), + stderr_tail: Buffer.concat(stderr).toString('utf8'), + }); + }); + }); +} + +function sessionLimits(overrides: Partial): WrapperSessionLimits { + return { + handshakeTimeoutMs: positiveLimit(overrides.handshakeTimeoutMs, DEFAULT_LIMITS.handshakeTimeoutMs), + executionTimeoutMs: positiveLimit(overrides.executionTimeoutMs, DEFAULT_LIMITS.executionTimeoutMs), + maxOutputBytes: positiveLimit(overrides.maxOutputBytes, DEFAULT_LIMITS.maxOutputBytes), + }; +} + +function positiveLimit(value: number | undefined, fallback: number): number { + return value !== undefined && Number.isSafeInteger(value) && value > 0 ? value : fallback; +} + +function normalizeLine(line: string): string { + return line.endsWith('\r') ? line.slice(0, -1) : line; +} + +function failure(stderr_tail: string): WrapperSessionResult { + return { exit_code: null, stdout_tail: '', stderr_tail }; +} diff --git a/sdk/tests/worker-cli.test.ts b/sdk/tests/worker-cli.test.ts index f7c0db44..c2a109b0 100644 --- a/sdk/tests/worker-cli.test.ts +++ b/sdk/tests/worker-cli.test.ts @@ -128,10 +128,12 @@ process.stdout.write('{"replacement":true}'); const trusted = makeWrapper(directory, 'trusted-wrapper', ` const fs = require('node:fs'); if (process.argv[2] !== '--relayflows-adapter-v1') process.exit(90); +process.stdout.cork(); process.stdout.write('relayflows-agent-cli-v1\\n'); const next = ${JSON.stringify(declared)} + '.next'; fs.symlinkSync(${JSON.stringify(replacement)}, next); fs.renameSync(next, ${JSON.stringify(declared)}); +process.stdout.uncork(); let input = ''; process.stdin.setEncoding('utf8'); process.stdin.on('data', chunk => { input += chunk; }); @@ -184,7 +186,8 @@ process.stdout.write('relayflows-agent-cli-v1\\n'); process.stdin.resume(); process.stdin.on('end', () => { process.stdout.write('relayflows-agent-cli-v1-execute\\n'); - process.stdout.write('x'.repeat(256)); + process.stdout.write('x'.repeat(80)); + process.stderr.write('y'.repeat(80)); }); `); From 193dd5f05fd86944386bae92c5f5890217b37b70 Mon Sep 17 00:00:00 2001 From: kjgbot Date: Wed, 2 Sep 2026 22:28:52 +0200 Subject: [PATCH 15/15] feat(sdk): add structured headless agent adapters --- docs/SURFACE.md | 21 ++-- sdk/src/cli-adapter.ts | 60 ++++++++---- sdk/src/cli/check.ts | 40 +++++--- sdk/src/headless-adapter.ts | 150 +++++++++++++++++++++++++++++ sdk/src/index.ts | 7 ++ sdk/src/worker-cli.ts | 44 ++++++++- sdk/src/worker.ts | 29 +++--- sdk/tests/cli-adapter.test.ts | 26 ++++- sdk/tests/cli.test.ts | 6 +- sdk/tests/headless-adapter.test.ts | 129 +++++++++++++++++++++++++ sdk/tests/live-kernel.test.ts | 21 +++- 11 files changed, 469 insertions(+), 64 deletions(-) create mode 100644 sdk/src/headless-adapter.ts create mode 100644 sdk/tests/headless-adapter.test.ts diff --git a/docs/SURFACE.md b/docs/SURFACE.md index ef926932..816d6632 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -97,12 +97,16 @@ No process runs between events: the handler wakes, executes to its next await, p **Typed CLI-adapter contract:** `flows check` and `AgentWorker` share one closed adapter table. A resolved executable whose basename is `claude` uses `claude auth status`, probes the exact model with a real noninteractive - `claude -p --model ` round trip, and executes with that same model - flag. A basename of `codex` uses `codex login status`, probes with - `codex exec --skip-git-repo-check --model ` in an ephemeral read-only - session, and executes noninteractively with the same Git/cwd flag. A Git - checkout is not a Relayflow execution prerequisite, so readiness and worker - execution both support non-Git working directories. Model-scoped probes may + `claude -p --output-format stream-json --verbose --model ` round + trip, and executes with that same structured mode and model flag. A basename + of `codex` uses `codex login status`, probes with + `codex exec --json --skip-git-repo-check --model -` in an ephemeral + read-only session, and executes noninteractively with the same Git/cwd flag. + A basename of `grok` uses `grok auth status` and its JSON prompt-file mode. + Instructions travel by stdin (Claude/Codex) or a private prompt file (Grok), + never argv. A Git checkout is not a Relayflow execution prerequisite, so + readiness and worker execution both support non-Git working directories. + Model-scoped probes may contact the provider and have a 60-second timeout; this cost is the honest price of proving current credential/model access rather than accepting an unrelated auth command as model proof. @@ -134,7 +138,10 @@ No process runs between events: the handler wakes, executes to its next await, p that cannot start, is signaled, or exceeds its adapter timeout is `probe_failed`, with a classified diagnostic rather than a raw process error. Every subprocess starts with ambient `RELAYFLOW_MODEL` removed. - Provider adapters pass only the declared flag; wrapper readiness receives + Provider adapters pass only the declared flag and journal their structured + trajectory tail, usage line, session id, and available subagent evidence; + a zero-exit provider response with no readable final message is + `worker_error`. Wrapper readiness receives only an allowlisted declared model, while worker instruction/model/wake values travel only in the post-identification session request. Preflight never invokes an undeclared model or guesses from host state. diff --git a/sdk/src/cli-adapter.ts b/sdk/src/cli-adapter.ts index 33affac0..45288296 100644 --- a/sdk/src/cli-adapter.ts +++ b/sdk/src/cli-adapter.ts @@ -1,10 +1,14 @@ import { basename } from 'node:path'; -export type CliAdapterKind = 'claude' | 'codex' | 'relayflows-wrapper-v1'; +export type CliAdapterKind = 'claude' | 'codex' | 'grok' | 'relayflows-wrapper-v1'; export interface CliInvocation { args: string[]; timeoutMs: number; + /** Prompt delivered over stdin, never appended to argv. */ + stdin?: string; + /** Prompt content written to a private temporary file before spawning. */ + promptFile?: string; /** Set only for wrapper readiness probes; raw providers receive a model flag. */ modelEnv?: string; } @@ -17,6 +21,8 @@ export interface CliAdapterIdentification { export const WRAPPER_IDENTIFY_ARG = '--relayflows-adapter-v1'; export const WRAPPER_IDENTIFY_TOKEN = 'relayflows-agent-cli-v1'; export const WRAPPER_EXECUTE_TOKEN = 'relayflows-agent-cli-v1-execute'; +/** Replaced with a private temporary pathname immediately before spawning Grok. */ +export const HEADLESS_PROMPT_FILE = '__relayflows_prompt_file__'; const MODEL_PROBE_PROMPT = 'Reply with exactly RELAYFLOWS_MODEL_READY and nothing else.'; @@ -25,6 +31,7 @@ export function cliAdapterKind(executable: string): CliAdapterKind { const name = basename(executable).replace(/\.exe$/i, ''); if (name === 'claude') return 'claude'; if (name === 'codex') return 'codex'; + if (name === 'grok') return 'grok'; return 'relayflows-wrapper-v1'; } @@ -36,6 +43,9 @@ export function adapterIdentification(kind: CliAdapterKind): CliAdapterIdentific if (kind === 'codex') { return { invocation: { args: ['login', 'status', '--help'], timeoutMs: 10_000 } }; } + if (kind === 'grok') { + return { invocation: { args: ['auth', 'status', '--help'], timeoutMs: 10_000 } }; + } return { invocation: { args: [WRAPPER_IDENTIFY_ARG], timeoutMs: 10_000 }, expectedStdout: WRAPPER_IDENTIFY_TOKEN, @@ -53,21 +63,20 @@ export function authenticationProbe(kind: CliAdapterKind): CliInvocation { * exact model through its explicitly identified environment contract. */ export function modelReadinessProbe(kind: CliAdapterKind, model: string): CliInvocation { - if (kind === 'claude') { + if (kind !== 'relayflows-wrapper-v1') { + const invocation = agentExecution(kind, MODEL_PROBE_PROMPT, model); return { - args: [ - '-p', '--model', model, '--tools', '', '--no-session-persistence', - MODEL_PROBE_PROMPT, - ], - timeoutMs: 60_000, - }; - } - if (kind === 'codex') { - return { - args: [ - 'exec', '--ephemeral', '--sandbox', 'read-only', '--skip-git-repo-check', - '--model', model, MODEL_PROBE_PROMPT, - ], + ...invocation, + // Claude's readiness call is deliberately side-effect-free. Codex has + // the same read-only sandbox rail as its historical probe. + args: kind === 'claude' + ? [...invocation.args, '--tools', '', '--no-session-persistence'] + : kind === 'codex' + ? [ + 'exec', '--json', '--ephemeral', '--sandbox', 'read-only', '--skip-git-repo-check', + ...(model === undefined ? [] : ['--model', model]), '-', + ] + : invocation.args, timeoutMs: 60_000, }; } @@ -86,18 +95,33 @@ export function agentExecution( ): CliInvocation { if (kind === 'claude') { return { - args: ['-p', ...(model === undefined ? [] : ['--model', model]), instruction], + args: [ + '-p', '--output-format', 'stream-json', '--verbose', + ...(model === undefined ? [] : ['--model', model]), + ], timeoutMs: 0, + stdin: instruction, }; } if (kind === 'codex') { return { args: [ - 'exec', '--ephemeral', '--skip-git-repo-check', + 'exec', '--json', '--ephemeral', '--skip-git-repo-check', + ...(model === undefined ? [] : ['--model', model]), + '-', + ], + timeoutMs: 0, + stdin: instruction, + }; + } + if (kind === 'grok') { + return { + args: [ + '--prompt-file', HEADLESS_PROMPT_FILE, '--output-format', 'json', ...(model === undefined ? [] : ['--model', model]), - instruction, ], timeoutMs: 0, + promptFile: instruction, }; } throw new Error('custom wrapper execution requires the runAgentCli same-process session'); diff --git a/sdk/src/cli/check.ts b/sdk/src/cli/check.ts index 8c644929..ba9dd851 100644 --- a/sdk/src/cli/check.ts +++ b/sdk/src/cli/check.ts @@ -1,4 +1,5 @@ -import { accessSync, constants, readFileSync } from 'node:fs'; +import { accessSync, constants, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; import { dirname, isAbsolute, join, parse as parsePath, resolve } from 'node:path'; import { spawnSync } from 'node:child_process'; import { parse as parseYaml } from 'yaml'; @@ -8,6 +9,7 @@ import { authenticationProbe, cliAdapterKind, displayInvocation, + HEADLESS_PROMPT_FILE, modelReadinessProbe, type CliInvocation, } from '../cli-adapter.js'; @@ -295,16 +297,32 @@ function runProbe( const env = { ...process.env }; delete env[MODEL_ENV]; if (invocation.modelEnv !== undefined) env[MODEL_ENV] = invocation.modelEnv; - const result = spawnSync(executable, invocation.args, { - cwd: directory, - encoding: 'utf8', - stdio: ['ignore', 'pipe', 'ignore'], - timeout: invocation.timeoutMs, - env, - }); - const failure = classifySpawnFailure(result.error, result.signal, invocation.timeoutMs); - if (failure !== undefined) throw failure; - return { status: result.status, stdout: result.stdout }; + let promptDirectory: string | undefined; + try { + const args = invocation.promptFile === undefined + ? invocation.args + : (() => { + promptDirectory = mkdtempSync(join(tmpdir(), 'relayflows-probe-prompt-')); + const promptPath = join(promptDirectory, 'prompt'); + writeFileSync(promptPath, invocation.promptFile, { mode: 0o600 }); + return invocation.args.map((arg, index) => ( + invocation.args[index - 1] === '--prompt-file' && arg === HEADLESS_PROMPT_FILE ? promptPath : arg + )); + })(); + const result = spawnSync(executable, args, { + cwd: directory, + encoding: 'utf8', + input: invocation.stdin, + stdio: ['pipe', 'pipe', 'ignore'], + timeout: invocation.timeoutMs, + env, + }); + const failure = classifySpawnFailure(result.error, result.signal, invocation.timeoutMs); + if (failure !== undefined) throw failure; + return { status: result.status, stdout: result.stdout }; + } finally { + if (promptDirectory !== undefined) rmSync(promptDirectory, { recursive: true, force: true }); + } } function resolveExecutable(command: string, directory: string): string | undefined { diff --git a/sdk/src/headless-adapter.ts b/sdk/src/headless-adapter.ts new file mode 100644 index 00000000..7d5cbaee --- /dev/null +++ b/sdk/src/headless-adapter.ts @@ -0,0 +1,150 @@ +import type { CliAdapterKind } from './cli-adapter.js'; + +/** Tokens and money reported by a provider, normalized for the journal boundary. */ +export interface HeadlessUsage { + tokens_in: number; + tokens_out: number; + dollars: string; +} + +/** Provider evidence that belongs to an agent step, not to a wrapper script. */ +export interface HeadlessResult { + finalText: string; + trajectory: unknown[]; + usage?: HeadlessUsage; + sessionId?: string; + subagents?: unknown; +} + +/** Closed SDK-owned provider contract; custom wrappers keep their v1 session. */ +export interface HeadlessAdapter { + kind: 'claude' | 'codex' | 'grok'; + parse(stdout: string): HeadlessResult; +} + +const HEADLESS_ADAPTERS: ReadonlyMap = new Map< + HeadlessAdapter['kind'], HeadlessAdapter +>([ + ['claude', { kind: 'claude', parse: parseClaude }], + ['codex', { kind: 'codex', parse: parseCodex }], + ['grok', { kind: 'grok', parse: parseGrok }], +]); + +export function headlessAdapter(kind: CliAdapterKind): HeadlessAdapter | undefined { + return kind === 'relayflows-wrapper-v1' ? undefined : HEADLESS_ADAPTERS.get(kind); +} + +/** Parse a successful provider's structured response. Empty finals are failures. */ +export function parseHeadlessOutput(kind: CliAdapterKind, stdout: string): HeadlessResult { + const adapter = headlessAdapter(kind); + if (adapter === undefined) throw new Error('custom wrapper output is not a provider headless stream'); + return adapter.parse(stdout); +} + +function parseClaude(stdout: string): HeadlessResult { + const events = parseJsonLines(stdout, 'Claude'); + const result = [...events].reverse().find(isClaudeResult); + if (result === undefined || typeof result.result !== 'string' || result.result.trim() === '') { + throw new Error('Claude completed without a readable final result message.'); + } + if (result.is_error === true) throw new Error('Claude reported an error result.'); + return { + finalText: result.result, + trajectory: events, + ...(usageFrom(result.usage, result.total_cost_usd)), + ...(stringField(result, 'session_id', 'Claude') === undefined + ? {} : { sessionId: stringField(result, 'session_id', 'Claude') }), + ...(result.subagent_stats === undefined ? {} : { subagents: result.subagent_stats }), + }; +} + +function parseCodex(stdout: string): HeadlessResult { + const events = parseJsonLines(stdout, 'Codex'); + const thread = events.find((event) => event.type === 'thread.started'); + const messages = events.filter(isCodexAgentMessage); + const final = messages.at(-1); + if (final === undefined) throw new Error('Codex completed without a readable final agent message.'); + const completed = [...events].reverse().find((event) => event.type === 'turn.completed'); + return { + finalText: final.item.text, + trajectory: events, + ...(completed === undefined ? {} : usageFrom(completed.usage, undefined)), + ...(thread !== undefined && typeof thread.thread_id === 'string' ? { sessionId: thread.thread_id } : {}), + }; +} + +function parseGrok(stdout: string): HeadlessResult { + const value = parseJson(stdout.trim(), 'Grok'); + if (typeof value.text !== 'string' || value.text.trim() === '') { + throw new Error('Grok completed without a readable final text message.'); + } + return { + finalText: value.text, + trajectory: [value], + ...(usageFrom(value.usage, value.total_cost_usd)), + ...(typeof value.sessionId === 'string' ? { sessionId: value.sessionId } : {}), + ...(value.subagents === undefined ? {} : { subagents: value.subagents }), + }; +} + +function parseJsonLines(stdout: string, provider: string): Record[] { + const lines = stdout.split(/\r?\n/).filter((line) => line.trim() !== ''); + if (lines.length === 0) throw new Error(`${provider} produced no structured events.`); + return lines.map((line, index) => parseJson(line, `${provider} event ${index + 1}`)); +} + +function parseJson(source: string, label: string): Record { + let parsed: unknown; + try { + parsed = JSON.parse(source); + } catch { + throw new Error(`${label} emitted malformed structured JSON.`); + } + if (!isRecord(parsed)) throw new Error(`${label} emitted a non-object structured event.`); + return parsed; +} + +function isClaudeResult(value: Record): value is Record & { result: string } { + return value.type === 'result'; +} + +function isCodexAgentMessage(value: Record): value is Record & { item: { text: string } } { + return value.type === 'item.completed' + && isRecord(value.item) + && value.item.type === 'agent_message' + && typeof value.item.text === 'string' + && value.item.text.trim() !== ''; +} + +function usageFrom(value: unknown, dollars: unknown): { usage?: HeadlessUsage } { + if (!isRecord(value)) return {}; + const tokensIn = numberField(value, 'input_tokens') ?? numberField(value, 'inputTokens'); + const tokensOut = numberField(value, 'output_tokens') ?? numberField(value, 'outputTokens'); + if (tokensIn === undefined || tokensOut === undefined) return {}; + return { + usage: { + tokens_in: tokensIn, + tokens_out: tokensOut, + dollars: decimalString(dollars) ?? '0', + }, + }; +} + +function numberField(value: Record, key: string): number | undefined { + const number = value[key]; + return typeof number === 'number' && Number.isSafeInteger(number) && number >= 0 ? number : undefined; +} + +function decimalString(value: unknown): string | undefined { + if (typeof value === 'string' && /^(?:0|[1-9]\d*)(?:\.\d+)?$/.test(value)) return value; + if (typeof value === 'number' && Number.isFinite(value) && value >= 0) return String(value); + return undefined; +} + +function stringField(value: Record, key: string, _provider: string): string | undefined { + return typeof value[key] === 'string' ? value[key] : undefined; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} diff --git a/sdk/src/index.ts b/sdk/src/index.ts index 6f42de6b..32dc0bd2 100644 --- a/sdk/src/index.ts +++ b/sdk/src/index.ts @@ -117,6 +117,13 @@ export { JOURNAL_WRITE_FAILED, PROTOCOL_VERSION } from './protocol.js'; export { JournalClient, type JournalClientOptions } from './journal-client.js'; export { AgentWorker, type AgentWorkerOptions } from './worker.js'; +export { + headlessAdapter, + parseHeadlessOutput, + type HeadlessAdapter, + type HeadlessResult, + type HeadlessUsage, +} from './headless-adapter.js'; export { validateWorkPackage, diff --git a/sdk/src/worker-cli.ts b/sdk/src/worker-cli.ts index 70cbd935..f2933374 100644 --- a/sdk/src/worker-cli.ts +++ b/sdk/src/worker-cli.ts @@ -1,9 +1,14 @@ import { spawn } from 'node:child_process'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { agentExecution, cliAdapterKind, + HEADLESS_PROMPT_FILE, type CliInvocation, } from './cli-adapter.js'; +import { parseHeadlessOutput, type HeadlessResult } from './headless-adapter.js'; import { runWrapperSession, type WrapperSessionLimits, @@ -24,6 +29,8 @@ export interface WorkerCliResult { exit_code: number | null; stdout_tail: string; stderr_tail: string; + /** Present only for supported raw provider CLIs with a validated final event. */ + headless?: HeadlessResult; } export async function runAgentCli( @@ -64,7 +71,17 @@ export async function runAgentCli( } if (invocation.modelEnv !== undefined) env[MODEL_ENV] = invocation.modelEnv; - return spawnInvocation(cli, invocation, env); + const raw = await spawnWithPromptFile(cli, invocation, env); + if (raw.exit_code !== 0) return raw; + try { + return { ...raw, headless: parseHeadlessOutput(kind, raw.stdout_tail) }; + } catch (error) { + return { + exit_code: null, + stdout_tail: raw.stdout_tail, + stderr_tail: `${raw.stderr_tail}${raw.stderr_tail === '' ? '' : '\n'}headless adapter: ${String(error)}`, + }; + } } function spawnInvocation( @@ -73,7 +90,7 @@ function spawnInvocation( env: NodeJS.ProcessEnv, ): Promise { return new Promise((resolve) => { - const child = spawn(cli, invocation.args, { stdio: ['ignore', 'pipe', 'pipe'], env }); + const child = spawn(cli, invocation.args, { stdio: ['pipe', 'pipe', 'pipe'], env }); const stdout: Buffer[] = []; const stderr: Buffer[] = []; let settled = false; @@ -96,6 +113,7 @@ function spawnInvocation( stdout_tail: Buffer.concat(stdout).toString('utf8'), stderr_tail: Buffer.concat(stderr).toString('utf8'), })); + child.stdin.end(invocation.stdin); if (invocation.timeoutMs > 0) { timer = setTimeout(() => { child.kill('SIGTERM'); @@ -108,3 +126,25 @@ function spawnInvocation( } }); } + +/** Writes Grok's private prompt file only for the lifetime of its child. */ +async function spawnWithPromptFile( + cli: string, + invocation: CliInvocation, + env: NodeJS.ProcessEnv, +): Promise { + if (invocation.promptFile === undefined) return spawnInvocation(cli, invocation, env); + const directory = await mkdtemp(join(tmpdir(), 'relayflows-agent-prompt-')); + const path = join(directory, 'prompt'); + try { + await writeFile(path, invocation.promptFile, { mode: 0o600 }); + return await spawnInvocation(cli, { + ...invocation, + args: invocation.args.map((arg, index) => ( + invocation.args[index - 1] === '--prompt-file' && arg === HEADLESS_PROMPT_FILE ? path : arg + )), + }, env); + } finally { + await rm(directory, { recursive: true, force: true }); + } +} diff --git a/sdk/src/worker.ts b/sdk/src/worker.ts index 6dece42c..cb696aed 100644 --- a/sdk/src/worker.ts +++ b/sdk/src/worker.ts @@ -89,20 +89,13 @@ export class AgentWorker extends EventEmitter { : { exit_code: null, stdout_tail: '', stderr_tail: 'agent step has no declared CLI' }; const completionReason = result.exit_code === 0 ? 'success' : 'worker_error'; - // Output shape: if the CLI's stdout parses as JSON, promote THAT - // as the step's `output` value so `json_schema` verification - // validates the analysis payload, not a wrapper around stdout. - // On the JSON path the CliResult (exit_code / stdout_tail / - // stderr_tail) is DISCARDED from `output` — the schema author - // wrote a shape for the analysis, not for the process wrapper. - // Non-JSON stdout falls back to the wrapper so text-emitting - // tools still round-trip usefully. - // - // Implicit contract: CLIs signal errors via non-zero exit, not by - // emitting an error JSON with exit 0. `completionReason` is - // derived from exit code, so a CLI that exits 0 while emitting - // `{"error":...}` will report success with an error payload. - const output = parseJsonOutput(result.stdout_tail) ?? result; + // Structured providers expose a final message separately from their + // trajectory. Promote JSON in that message so json_schema gates judge the + // agent's answer, never the CLI's event envelope. A provider parser + // rejects a zero-exit stream without a final message before this point. + // Custom wrappers retain their established raw-stdout behavior. + const agentText = result.headless?.finalText ?? result.stdout_tail; + const output = parseJsonOutput(agentText) ?? (result.headless === undefined ? result : agentText); await this.client.stepComplete( dispatch.run_id, @@ -112,6 +105,14 @@ export class AgentWorker extends EventEmitter { completionReason, { output, + ...(result.headless?.usage === undefined ? {} : { usage: result.headless.usage }), + ...(result.headless === undefined ? {} : { + trajectory_tail: { + events: result.headless.trajectory, + ...(result.headless.sessionId === undefined ? {} : { sessionId: result.headless.sessionId }), + ...(result.headless.subagents === undefined ? {} : { subagents: result.headless.subagents }), + }, + }), started_pins: dispatch.pins, end_pins: dispatch.pins, }, diff --git a/sdk/tests/cli-adapter.test.ts b/sdk/tests/cli-adapter.test.ts index 573ea481..996e6e38 100644 --- a/sdk/tests/cli-adapter.test.ts +++ b/sdk/tests/cli-adapter.test.ts @@ -4,6 +4,7 @@ import { adapterIdentification, authenticationProbe, cliAdapterKind, + HEADLESS_PROMPT_FILE, modelReadinessProbe, WRAPPER_IDENTIFY_ARG, } from '../src/cli-adapter.js'; @@ -19,10 +20,12 @@ describe('typed CLI adapters', () => { expect(readiness).toMatchObject({ args: expect.arrayContaining(['-p', '--model', 'claude-model']), }); + expect(readiness.stdin).toContain('RELAYFLOWS_MODEL_READY'); expect(readiness).not.toHaveProperty('modelEnv'); expect(agentExecution(kind, 'Review.', 'claude-model')).toEqual({ - args: ['-p', '--model', 'claude-model', 'Review.'], + args: ['-p', '--output-format', 'stream-json', '--verbose', '--model', 'claude-model'], timeoutMs: 0, + stdin: 'Review.', }); }); @@ -34,14 +37,16 @@ describe('typed CLI adapters', () => { expect(authenticationProbe(kind).args).toEqual(['login', 'status']); expect(modelReadinessProbe(kind, 'gpt-model')).toEqual({ args: [ - 'exec', '--ephemeral', '--sandbox', 'read-only', '--skip-git-repo-check', - '--model', 'gpt-model', 'Reply with exactly RELAYFLOWS_MODEL_READY and nothing else.', + 'exec', '--json', '--ephemeral', '--sandbox', 'read-only', '--skip-git-repo-check', + '--model', 'gpt-model', '-', ], timeoutMs: 60_000, + stdin: 'Reply with exactly RELAYFLOWS_MODEL_READY and nothing else.', }); expect(agentExecution(kind, 'Review.', 'gpt-model')).toEqual({ - args: ['exec', '--ephemeral', '--skip-git-repo-check', '--model', 'gpt-model', 'Review.'], + args: ['exec', '--json', '--ephemeral', '--skip-git-repo-check', '--model', 'gpt-model', '-'], timeoutMs: 0, + stdin: 'Review.', }); }); @@ -59,4 +64,17 @@ describe('typed CLI adapters', () => { 'custom wrapper execution requires the runAgentCli same-process session', ); }); + + it('maps raw Grok to its prompt-file structured mode', () => { + const kind = cliAdapterKind('/opt/bin/grok'); + + expect(kind).toBe('grok'); + expect(adapterIdentification(kind).invocation.args).toEqual(['auth', 'status', '--help']); + expect(authenticationProbe(kind).args).toEqual(['auth', 'status']); + expect(agentExecution(kind, 'Review.', 'grok-model')).toEqual({ + args: ['--prompt-file', HEADLESS_PROMPT_FILE, '--output-format', 'json', '--model', 'grok-model'], + timeoutMs: 0, + promptFile: 'Review.', + }); + }); }); diff --git a/sdk/tests/cli.test.ts b/sdk/tests/cli.test.ts index 04f46b8c..bb094028 100644 --- a/sdk/tests/cli.test.ts +++ b/sdk/tests/cli.test.ts @@ -217,7 +217,7 @@ ${steps} const log = join(directory, 'claude.log'); const cli = executableFixture(directory, 'claude', `printf '%s|MODEL_ENV=%s\\n' "$*" "\${RELAYFLOW_MODEL-UNSET}" >> ${JSON.stringify(log)} if [ "$1 $2" = "auth status" ]; then exit 0; fi -if [ "$1 $2 $3" = "-p --model available-model" ]; then exit 0; fi +if [ "$1 $2 $3 $4 $5 $6" = "-p --output-format stream-json --verbose --model available-model" ]; then exit 0; fi exit 7`); writeFileSync(join(directory, 'flows.json'), JSON.stringify({ models: ['available-model'] })); const path = join(directory, 'claude.flow.yaml'); @@ -233,7 +233,7 @@ steps: const result = await run(path); expect(result.code).toBe(0); - expect(readFileSync(log, 'utf8')).toContain('-p --model available-model'); + expect(readFileSync(log, 'utf8')).toContain('-p --output-format stream-json --verbose --model available-model'); expect(readFileSync(log, 'utf8')).toContain('MODEL_ENV=UNSET'); expect(readFileSync(log, 'utf8')).toContain('auth status --help|MODEL_ENV=UNSET'); }); @@ -261,7 +261,7 @@ steps: expect(result.code).toBe(2); expect(result.stderr.join('\n')).toContain('REFUSED [model_unavailable]'); expect(result.stderr.join('\n')).not.toContain('cli_unauthenticated'); - expect(readFileSync(log, 'utf8')).toContain('exec --ephemeral'); + expect(readFileSync(log, 'utf8')).toContain('exec --json --ephemeral'); expect(readFileSync(log, 'utf8')).toContain('--model denied-model'); expect(readFileSync(log, 'utf8')).toContain('login status|MODEL_ENV=UNSET'); expect(readFileSync(log, 'utf8')).toContain('login status --help|MODEL_ENV=UNSET'); diff --git a/sdk/tests/headless-adapter.test.ts b/sdk/tests/headless-adapter.test.ts new file mode 100644 index 00000000..5c15a412 --- /dev/null +++ b/sdk/tests/headless-adapter.test.ts @@ -0,0 +1,129 @@ +import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { agentExecution, HEADLESS_PROMPT_FILE } from '../src/cli-adapter.js'; +import { parseHeadlessOutput } from '../src/headless-adapter.js'; +import { runAgentCli } from '../src/worker-cli.js'; + +const directories: string[] = []; + +afterEach(() => { + for (const directory of directories.splice(0)) rmSync(directory, { recursive: true, force: true }); +}); + +describe('SDK-owned provider headless adapters', () => { + it('keeps a huge Claude instruction off argv, parses trajectory/usage/session/subagents', async () => { + const directory = temporaryDirectory(); + const evidence = join(directory, 'evidence.json'); + const claude = executable(directory, 'claude', ` +let prompt = ''; +process.stdin.setEncoding('utf8'); +process.stdin.on('data', chunk => { prompt += chunk; }); +process.stdin.on('end', () => { + require('node:fs').writeFileSync(${JSON.stringify(evidence)}, JSON.stringify({ argv: process.argv.slice(2), prompt })); + console.log(JSON.stringify({ type: 'assistant', message: { content: [] } })); + console.log(JSON.stringify({ type: 'result', result: 'finished', session_id: 'session-1', usage: { input_tokens: 12, output_tokens: 4 }, total_cost_usd: 0.003, subagent_stats: { spawned: 2 } })); +}); +`); + const prompt = 'x'.repeat(1_000_000); + + const result = await runAgentCli(claude, prompt, undefined, 'claude-test-model'); + + expect(result.exit_code).toBe(0); + expect(result.headless).toEqual({ + finalText: 'finished', + trajectory: [ + { type: 'assistant', message: { content: [] } }, + expect.objectContaining({ type: 'result', result: 'finished' }), + ], + usage: { tokens_in: 12, tokens_out: 4, dollars: '0.003' }, + sessionId: 'session-1', + subagents: { spawned: 2 }, + }); + const received = JSON.parse(readFileSync(evidence, 'utf8')) as { argv: string[]; prompt: string }; + expect(received.argv).toEqual([ + '-p', '--output-format', 'stream-json', '--verbose', '--model', 'claude-test-model', + ]); + expect(received.argv.join(' ')).not.toContain(prompt.slice(0, 100)); + expect(received.prompt).toHaveLength(prompt.length); + }); + + it('fails closed when Claude exits zero with malformed structured output or no final message', async () => { + const directory = temporaryDirectory(); + const malformed = executable(directory, 'claude', 'console.log("not-json")'); + const malformedResult = await runAgentCli(malformed, 'do work', undefined); + expect(malformedResult).toMatchObject({ exit_code: null }); + expect(malformedResult.stderr_tail).toContain('malformed structured JSON'); + + // The basename selects the provider. A separate directory avoids changing + // the malformed fixture the prior assertion is evidence for. + const providerDirectory = temporaryDirectory(); + const provider = executable(providerDirectory, 'claude', 'console.log(JSON.stringify({ type: "assistant" }))'); + const noFinalResult = await runAgentCli(provider, 'do work', undefined); + expect(noFinalResult).toMatchObject({ exit_code: null }); + expect(noFinalResult.stderr_tail).toContain('without a readable final'); + }); + + it('reports a crashed provider process as worker failure material instead of throwing', async () => { + const directory = temporaryDirectory(); + const claude = executable(directory, 'claude', 'process.kill(process.pid, "SIGKILL")'); + + const result = await runAgentCli(claude, 'do work', undefined); + + expect(result.exit_code).toBeNull(); + expect(result.headless).toBeUndefined(); + }); + + it('uses Codex stdin with the non-Git rail and accepts its final agent message', () => { + const invocation = agentExecution('codex', 'never on argv', 'gpt-test-model'); + expect(invocation).toEqual({ + args: ['exec', '--json', '--ephemeral', '--skip-git-repo-check', '--model', 'gpt-test-model', '-'], + timeoutMs: 0, + stdin: 'never on argv', + }); + expect(parseHeadlessOutput('codex', [ + JSON.stringify({ type: 'thread.started', thread_id: 'thread-1' }), + JSON.stringify({ type: 'item.completed', item: { type: 'agent_message', text: 'final' } }), + JSON.stringify({ type: 'turn.completed', usage: { input_tokens: 7, output_tokens: 3 } }), + ].join('\n'))).toEqual({ + finalText: 'final', + trajectory: expect.any(Array), + usage: { tokens_in: 7, tokens_out: 3, dollars: '0' }, + sessionId: 'thread-1', + }); + }); + + it('uses a private Grok prompt file and deletes it after its zero-exit result', async () => { + const directory = temporaryDirectory(); + const evidence = join(directory, 'grok-evidence.json'); + const grok = executable(directory, 'grok', ` +const fs = require('node:fs'); +const index = process.argv.indexOf('--prompt-file'); +const path = process.argv[index + 1]; +fs.writeFileSync(${JSON.stringify(evidence)}, JSON.stringify({ argv: process.argv.slice(2), path, prompt: fs.readFileSync(path, 'utf8') })); +console.log(JSON.stringify({ text: 'grok final', sessionId: 'grok-session', usage: { input_tokens: 5, output_tokens: 2 }, total_cost_usd: '0.004' })); +`); + + const result = await runAgentCli(grok, 'private instruction', undefined, HEADLESS_PROMPT_FILE); + + expect(result).toMatchObject({ exit_code: 0, headless: { finalText: 'grok final', sessionId: 'grok-session' } }); + const received = JSON.parse(readFileSync(evidence, 'utf8')) as { argv: string[]; path: string; prompt: string }; + expect(received.argv).toEqual(['--prompt-file', received.path, '--output-format', 'json', '--model', HEADLESS_PROMPT_FILE]); + expect(received.prompt).toBe('private instruction'); + expect(() => readFileSync(received.path, 'utf8')).toThrow(); + }); +}); + +function temporaryDirectory(): string { + const directory = mkdtempSync(join(tmpdir(), 'flows-headless-adapter-')); + directories.push(directory); + return directory; +} + +function executable(directory: string, name: string, body: string): string { + const path = join(directory, name); + writeFileSync(path, `#!/usr/bin/env node\n${body}`); + chmodSync(path, 0o755); + return path; +} diff --git a/sdk/tests/live-kernel.test.ts b/sdk/tests/live-kernel.test.ts index cc7325f3..de4a778d 100644 --- a/sdk/tests/live-kernel.test.ts +++ b/sdk/tests/live-kernel.test.ts @@ -845,19 +845,22 @@ process.stdout.write('{"must_not":"execute"}'); }, 30_000); it.each([ - ['claude', '-p --model declared-model-xyz'], - ['codex', 'exec --ephemeral --skip-git-repo-check --model declared-model-xyz'], + ['claude', '-p --output-format stream-json --verbose --model declared-model-xyz'], + ['codex', 'exec --json --ephemeral --skip-git-repo-check --model declared-model-xyz -'], ] as const)('AgentWorker executes the raw %s adapter with its real model flag', async (name, prefix) => { const dataDir = temporaryDirectory(`flows-live-${name}-adapter-`); await startDaemon(dataDir); const cli = join(dataDir, name); writeFileSync(cli, `#!/bin/sh case "$*" in - ${JSON.stringify(`${prefix} `)}*) ;; + ${JSON.stringify(prefix)}*) ;; *) printf '%s\\n' "unexpected argv: $*" >&2; exit 9 ;; esac test "\${RELAYFLOW_MODEL+x}" != x || exit 8 -printf '%s' '{"adapter":"${name}","model_flag":"declared-model-xyz"}' +case ${JSON.stringify(name)} in + claude) printf '%s\\n' '{"type":"result","result":"{\\"adapter\\":\\"claude\\",\\"model_flag\\":\\"declared-model-xyz\\"}","session_id":"claude-session","subagent_stats":{"spawned":2},"usage":{"input_tokens":1,"output_tokens":1},"total_cost_usd":"0.001"}' ;; + codex) printf '%s\\n' '{"type":"item.completed","item":{"type":"agent_message","text":"{\\"adapter\\":\\"codex\\",\\"model_flag\\":\\"declared-model-xyz\\"}"}}' ;; +esac `); chmodSync(cli, 0o755); const client = await connectClient(dataDir); @@ -882,8 +885,16 @@ steps: const completed = (await client.journalRead(started.run_id)).entries.find( (entry) => (entry as { entry_type: string; step_id?: string }).entry_type === 'step.completed' && (entry as { step_id?: string }).step_id === 'probe', - ) as { payload: { output: { adapter: string; model_flag: string } } } | undefined; + ) as { payload: { output: { adapter: string; model_flag: string }; budget?: unknown; trajectory_tail?: unknown } } | undefined; expect(completed?.payload.output).toEqual({ adapter: name, model_flag: 'declared-model-xyz' }); + if (name === 'claude') { + expect(completed?.payload.budget).toEqual({ tokens_in: 1, tokens_out: 1, dollars: '0.001' }); + expect(completed?.payload.trajectory_tail).toMatchObject({ + sessionId: 'claude-session', + subagents: { spawned: 2 }, + events: [expect.objectContaining({ type: 'result' })], + }); + } await worker.close(); }, 30_000);