From b401efe76a6312ee01a16debd899a312bbb0886a Mon Sep 17 00:00:00 2001 From: Khaliq Date: Fri, 28 Aug 2026 01:15:00 -0400 Subject: [PATCH 1/4] Add flows run and resume live-kernel surface --- docs/SURFACE.md | 38 +- kernel/relayflowd-core/tests/spec_parity.rs | 14 + ops/NEXT.md | 291 ++++++++++--- ops/SCOREBOARD.md | 2 +- sdk/src/cli.ts | 363 +++++----------- sdk/src/cli/check.ts | 250 +++++++++++ sdk/src/cli/run.ts | 261 ++++++++++++ sdk/src/failure-kinds.ts | 14 + sdk/tests/cli.test.ts | 236 +++++++++-- sdk/tests/journal-client-loopback.ts | 110 +++++ sdk/tests/journal-client.test.ts | 160 +------ sdk/tests/live-kernel.test.ts | 393 ++++++++++++++++++ sdk/tests/spec-parity.test.ts | 2 +- testdata/hello-deterministic.flow.yaml | 18 + .../hello-deterministic.spec.canonical.json | 1 + testdata/hello-deterministic.spec.sha256 | 1 + 16 files changed, 1629 insertions(+), 525 deletions(-) create mode 100644 sdk/src/cli/check.ts create mode 100644 sdk/src/cli/run.ts create mode 100644 sdk/tests/journal-client-loopback.ts create mode 100644 sdk/tests/live-kernel.test.ts create mode 100644 testdata/hello-deterministic.flow.yaml create mode 100644 testdata/hello-deterministic.spec.canonical.json create mode 100644 testdata/hello-deterministic.spec.sha256 diff --git a/docs/SURFACE.md b/docs/SURFACE.md index 6978f466..608cab7d 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -108,19 +108,37 @@ The herdr model: first-party helpers are just plugins that ship in the box; the `flows build` seals a flow into a content-addressed, immutable bundle: canonical spec JSON, compiled TS with pinned deps, helper/plugin lockfile, assets, preflight declaration, identity signature — `flow@sha256:…`, pushed to a bucket/registry. `flows deploy` points a trigger at a digest; `flows run flow@sha256:…` executes from the bucket on any cell, no checkout. Preflight runs at build time for everything build-provable and again at deploy time for environment facts (credentials, workers, MCP servers). The working tree is for authoring; **production only ever runs digests.** -## 5. Invocation: three ways in +## 5. Invocation: the gate-1 CLI -A deployed flow is reachable three ways, all landing on the same digest and the same journal: +Gate 1 ships three CLI verbs over the journal protocol: -1. **Events** — `on(...)` triggers: relayfile webhooks, mentions, file changes. -2. **Schedules** — RelayCron: durable alarms + sweep. -3. **Direct call** — a flow is a *function*: - - CLI: `flows run release-note --input '{"branch":"main"}'` (or `flows run flow@sha256:…`) - - HTTP: every deployed flow is an endpoint — `POST /flows/release-note` returns the result for short flows, or a run handle (`202 + run id`) to poll/stream for long ones - - SDK: `await flows.call("release-note", input)` from any app (this is how sage and consumer apps invoke pipelines) - - Flow-to-flow: `f.dispatch("garden/implement", plan)` — same mechanism, child run with its own journal +```text +flows check [--json] +flows run [--json] [--data-dir ] +flows resume [--json] [--data-dir ] +``` -The caller always gets the same contract back: a typed result on completion, or a durable run handle it can await, stream, or abandon — the run finishes either way, journaled. +`check` compiles and preflights without starting a run. `run` performs that +same preflight before contacting `relayflowd`, then submits the compiled spec +to `/relayflowd.sock`; `resume` asks that daemon to continue an +existing run from its journal. The data directory defaults to `.relayflowd`. +Neither verb starts the daemon implicitly. `--json` writes one report-shaped +object to stdout while diagnostics remain on stderr. + +The exit codes are part of the surface contract: + +| Exit | Outcome | +|---:|---| +| `0` | The run completed with `completionReason: success`. | +| `1` | The run failed with a declared `completionReason`, or the daemon response violated the protocol. | +| `2` | The command was refused before a journal write: invalid input, failed preflight, unreachable daemon, or unavailable resume target. | +| `3` | The run parked. `PARKED [run_parked]` names the step and its `llm` or `agent` type. | + +At gate 1 no `llm` or `agent` worker is attached by the CLI. Reaching either +step therefore returns the durable parked outcome instead of hanging or +reporting success. Event, schedule, deployed-digest, HTTP, SDK-call, and +flow-to-flow invocation remain later-gate surface work; they are not shipped +by this CLI. ## 6. Open surface questions (for gate-1 SDK work) diff --git a/kernel/relayflowd-core/tests/spec_parity.rs b/kernel/relayflowd-core/tests/spec_parity.rs index 631c2821..26ccb5ac 100644 --- a/kernel/relayflowd-core/tests/spec_parity.rs +++ b/kernel/relayflowd-core/tests/spec_parity.rs @@ -25,6 +25,20 @@ fn the_kernel_parses_the_sdk_compiled_spec_and_stamps_the_same_hash() { assert_parity(LADDER_CANONICAL, LADDER_SHA256); } +#[test] +fn the_kernel_parses_the_deterministic_rung_and_stamps_the_same_hash() { + assert_parity( + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../testdata/hello-deterministic.spec.canonical.json" + )), + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../testdata/hello-deterministic.spec.sha256" + )), + ); +} + #[test] fn the_kernel_parses_the_rung_b_spec_and_stamps_the_same_hash() { assert_parity( diff --git a/ops/NEXT.md b/ops/NEXT.md index 4abee438..001b59aa 100644 --- a/ops/NEXT.md +++ b/ops/NEXT.md @@ -1,65 +1,226 @@ -# NEXT — WP-9 merge handoff for PR #8 - -Written by the Relayflow Lead on 2026-08-27 for branch -`flow/drive-57e923c-08271542` and existing PR #8. - -## Current state - -WP-9 has no remaining product work. The branch contains `origin/main` commit -`6366943`, including the repository's evidence-capture standard. Gate 1's -covenant-2 preflight implementation remains on PR #8 and is not on `main`. - -The package-mandated deterministic-command limitation is public in -`docs/SURFACE.md`: an unresolved bare command receives the typed -`command_unresolved` warning rather than a refusal because `/bin/sh -c` may -supply a builtin, function, or assignment. The narrower unresolved path-like -case remains filed under “Close the deterministic-command preflight gap” in -`ops/BACKLOG.md`. - -The review-repair chronology is append-only under `ops/reviews/`. Rejected -WP-9 heads and their repairs are: - -- `18f03be`: the new surface paragraph named the wrong warning kind; - `c04d388` corrected it to `command_unresolved`. -- `c04d388`: the gate scoreboard retained the pre-WP-8 SDK count; - `fa19df1` corrected 130 to the reproduced 131 without changing Gate 1's - AMBER state. -- `385763a`: the branch still carried a WP-7 selector; `2b117ae` replaced it - with the supplied WP-9 assessment. -- `2b117ae`: that copied assessment still queued already-completed work; - `80aa711` replaced it with a present-tense merge handoff. -- `80aa711`: the handoff understated its own rejection chronology; this - revision removes the count and records the immediate prior rejection. - -The final changed-head review transcripts after this handoff are the merge -evidence. Each must name the same reviewed SHA, end in `REVIEW_PASSED`, and -the aggregate must return `SWARM_PASSED`. Only those transcript commits may -follow the reviewed handoff head. - -## Captured verification - -`ops/DRIVE-LOG.md` contains the literal WP-9 verification output: - -- kernel: 72 passed, 0 failed; -- Clippy with `-D warnings`: exit 0; -- Rust formatting: exit 0 with empty output; -- SDK: 131 passed, 0 failed; -- clean-room install/build: `dist` and `node_modules` moved aside - recoverably, `npm ci && npm run build && test -x dist/cli.js`: exit 0; -- largest Rust file: 468 lines. - -The destructive `rm -rf` spelling in the package was rejected by the worker -safety layer before process launch. The recoverable move established the same -absence precondition; the exact substituted command and output are preserved -in `ops/DRIVE-LOG.md` and the PR body. - -## Next action - -PR #8 remains OPEN. A human reviews and merges it if satisfied. After merge, a -new tick re-runs the Gate 1 verification on merged `main` and only then may -move `ops/SCOREBOARD.md` from AMBER to GREEN. - -The Lead does not merge, does not start Gate 2+ work while PR #8 is open, and -does not add product features to this branch. - -END_HANDOFF +# NEXT — WP-10: `flows run` / `flows resume` — the authored ladder runs on the live kernel + +Written by the Relayflow Lead on 2026-08-28 for branch +`flow/drive-77b2457-08280058` (base `77b2457`). + +## Why this package, and nothing else + +**Standing directives:** `ops/DIRECTIVES.md` carries no directives today (five +lines, header only). Nothing outranks the gate work this tick. + +**Open PRs:** none. `gh pr list --state open` returned empty; PR #8 merged as +`9e1d9eb`. No PR is awaiting fixes, so no PR-repair package pre-empts this one. + +**The working tree holds unfinished work, and it does not build.** Six paths +are untracked on this branch, written at 00:28–00:32 and never committed: + +```text +$ git status --porcelain +?? sdk/src/cli/ +?? sdk/tests/journal-client-loopback.ts +?? sdk/tests/live-kernel.test.ts +?? testdata/hello-deterministic.flow.yaml +?? testdata/hello-deterministic.spec.canonical.json +?? testdata/hello-deterministic.spec.sha256 +``` + +`sdk/src/cli/run.ts` implements `flows run` / `flows resume` over the journal +protocol; `sdk/src/cli/check.ts` is `check` extracted from `sdk/src/cli.ts`; +`sdk/tests/live-kernel.test.ts` drives the **built** `sdk/dist/cli.js` against a +live `relayflowd`. None of it is wired into `sdk/src/cli.ts`, and the SDK gate +is currently RED because of it: + +```text +$ (cd sdk && npm test) +> tsc --noEmit && vitest run +src/cli/run.ts(3,15): error TS2305: Module '"../failure-kinds.js"' has no exported member 'RunFailureKind'. +``` + +Bypassing the typecheck shows the same break reaching two suites — the build +fixture in `bin.test.ts` fails, taking its seven tests with it, and the three +live-kernel cases fail because the built CLI has no `run` verb: + +```text +$ (cd sdk && npx vitest run) + ❯ tests/bin.test.ts (7 tests | 7 skipped) 750ms + FAIL tests/bin.test.ts [ tests/bin.test.ts ] + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > runs rung (a), parks rung (b), and keeps JSON report-shaped + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > preflights before journaling and names an unreachable socket + FAIL tests/live-kernel.test.ts > surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once +AssertionError: REFUSED [invalid_invocation] Usage: flows check [--json] + Test Files 2 failed | 7 passed (9) + Tests 3 failed | 125 passed | 7 skipped (135) +``` + +The kernel is green and untouched by this: + +```text +$ (cd kernel && ../ops/cargo.sh test --workspace) +test result: ok. 6 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s +[exited with code 0] +``` + +**No new work over unfinished work** (`ops/AUTONOMY.md`). The unfinished work is +this feature. Finish it — do not delete it — because it closes the one seam +gate 1 has never crossed. + +**What the seam is.** Gate 1 is GREEN in `ops/SCOREBOARD.md` on two bodies of +evidence that have never met: + +- the kernel proves the ladder under `kill -9` using specs it authors itself — + `kernel/relayflowd/tests/crash_resume/support.rs` builds a `crash-{name}` + `run.json` inline, not `testdata/hello-*.flow.yaml`; +- the SDK proves it compiles the canonical ladder YAML to pinned bytes + (`sdk/tests/spec-parity.test.ts` ⇄ `kernel/relayflowd-core/tests/spec_parity.rs`). + +They are joined only by a checked-in canonical-JSON fixture and its sha256. No +test anywhere runs an **author-written flow file through the shipped `flows` +binary into a live `relayflowd`**. Gate 1 is named "a relayflow can run," and +`docs/SURFACE.md` §5 already documents `flows run` as the direct-call entry — +the surface is published and unimplemented. Every later gate's consumer +(gate 2's triggers, gate 3's garden, gate 6's helpers) invokes flows through +this path; it cannot stay a fixture. + +## Objective + +Land `flows run` and `flows resume` in the shipped CLI, and prove behaviorally +— through the built `sdk/dist/cli.js` against a live `relayflowd` — that the +canonical ladder rung (a) runs to `success`, that rungs (b)/(c) **park** with a +typed reason instead of silently succeeding, that a failed step terminates in a +declared `completionReason`, and that resuming after a real daemon kill +re-executes nothing already completed. + +## Files in scope + +- `sdk/src/failure-kinds.ts` — add the closed `RUN_FAILURE_KINDS` taxonomy and + its `RunFailureKind` type. `run.ts` already emits `run_unavailable`, + `daemon_unreachable`, and `protocol_error`; the union must be declared, the + same shape as `PREFLIGHT_FAILURE_KINDS`, with a `isRunFailureKind` guard if + the emitter needs one. This single missing export is what reddens the tree. +- `sdk/src/cli/run.ts`, `sdk/src/cli/check.ts` — the in-flight modules. Keep + `check`'s behavior byte-identical to the merged `sdk/src/cli.ts` (its 42 + `cli.test.ts` cases are the contract); the extraction may not change one + refusal string or exit code. +- `sdk/src/cli.ts` — becomes the dispatcher: `check` | `run` | `resume`, with + `--json`, `--data-dir`, and a usage refusal that names all three verbs. + Watch the 500-line rule: it is 311 lines today and the extraction is what + keeps it under. +- `sdk/tests/live-kernel.test.ts`, `sdk/tests/journal-client-loopback.ts` — + the end-to-end gate and the shared loopback harness. +- `sdk/tests/cli.test.ts` — unit coverage for the new verbs' argument parsing + and exit codes against the loopback, not the daemon. +- `testdata/hello-deterministic.flow.yaml` + its `.spec.canonical.json` / + `.spec.sha256` — rung (a) as a standalone runnable flow. Pin it on **both** + sides of the parity gate: add `hello-deterministic` to the fixture loop in + `sdk/tests/spec-parity.test.ts` and to + `kernel/relayflowd-core/tests/spec_parity.rs`, or it is a fixture only one + side has agreed to. +- `docs/SURFACE.md` — replace the §5 promise with what actually ships: the + verbs, the exit codes, and the parked case. +- `ops/DRIVE-LOG.md`, `ops/SCOREBOARD.md` — the tick entry and the gate-1 + evidence row, updated to cite the live path rather than fixture parity. + +## Design points this package must settle + +1. **Exit codes are a contract, so write them down.** `run.ts` returns + `0` success · `1` run failed or protocol error · `2` refused before any + journal write · `3` parked. `docs/SURFACE.md` states them; a test asserts + each one. +2. **Preflight runs before the journal.** `runFlow` calls `checkFlow` first and + returns exit 2 without contacting the daemon. Covenant 2 says a refusal must + precede the effect; the existing live test case "preflights before + journaling and names an unreachable socket" is that assertion — keep it. +3. **Parked is a first-class outcome, not a hang.** `hello-llm` / `hello-agent` + have no attached worker. The kernel already answers `RunStatus::Parked` + (`kernel/relayflowd/src/engine/drive.rs`). The CLI must name the parked step + and its type and exit 3. Silence here would be the Nabis silent-zero defect. +4. **A gate that skips is a gate that failed** (`ops/AUTONOMY.md`). If + `RELAYFLOWD_BIN` (default `kernel/target/debug/relayflowd`) is missing, + `live-kernel.test.ts` must **fail** with an actionable message naming the + build command — never `it.skip`. The DoD command sequence therefore builds + the kernel before running the SDK suite. +5. **Resume proves exactly-once, not just exit 0.** The resume case must assert + from the journal that each completed step has exactly one `step.completed` + entry after the kill+resume — the same bar + `assert_exact_journal` holds the kernel to. + +## Definition of done + +Every command below run from a clean checkout of the branch, with its literal +output captured in `ops/DRIVE-LOG.md` per the evidence standard in `AGENTS.md` +("Evidence is captured, not narrated" — paste the output, not a summary of it). + +1. `git status --porcelain` → **empty**. Nothing this package touches is left + untracked; the six paths above are committed or deliberately removed with + the removal explained in the log. +2. `(cd kernel && ../ops/cargo.sh test --workspace)` → exit 0, **≥ 72 passed, + 0 failed** (the current floor; the added `hello-deterministic` parity case + raises it). +3. `(cd kernel && ../ops/cargo.sh clippy --workspace -- -D warnings)` → exit 0. +4. `(cd kernel && ../ops/cargo.sh fmt --check)` → exit 0, empty output. +5. `(cd kernel && ../ops/cargo.sh build)` → `kernel/target/debug/relayflowd` + exists and is executable. +6. `(cd sdk && npm ci && npm run build && npm test)` → exit 0, **0 failed, 0 + skipped**, `Test Files` all passed. `bin.test.ts` must report 7 passed (not + 7 skipped) and `live-kernel.test.ts` must report its cases passed — a suite + that skipped is not a suite that ran. +7. Behavioral proof through the built binary, captured verbatim — not through + vitest, the same way gate 1's preflight clause was proven: + - `node sdk/dist/cli.js run --data-dir testdata/hello-deterministic.flow.yaml` + → exit 0, stdout carries the run id and `completionReason: success`; + - `node sdk/dist/cli.js run --data-dir testdata/hello-llm.flow.yaml` + → exit 3, stderr `PARKED [run_parked]` naming step `answer` (llm); + - `node sdk/dist/cli.js run --data-dir testdata/hello-agent.flow.yaml` + → exit 3, parked at the agent step; + - a flow whose step exits non-zero → exit 1, `FAILED [step_failed]`, and a + journal terminating in a declared `completionReason`; + - `node sdk/dist/cli.js run --data-dir …` with no daemon listening + → exit 2, `REFUSED [daemon_unreachable]`, and **no journal file created**; + - `node sdk/dist/cli.js resume --data-dir ` after + `kill -9` of the daemon mid-run → exit 0, and the journal shows each + completed step exactly once. +8. `docs/SURFACE.md` §5 describes the three verbs, the four exit codes, and the + parked outcome, with no claim the shipped CLI does not honor. +9. No file in `sdk/src` or `kernel` crosses 500 lines: + `find kernel sdk/src -name '*.rs' -o -name '*.ts' | grep -v target | xargs wc -l | sort -nr | head -5` + captured in the log. +10. A PR is opened with this evidence in the body, titled by the work-package + name (**not** the NEXT.md markdown header — that leak is a filed backlog + item). The Lead does not merge; the merge bar in `ops/RUN-CONTRACT.md` §3 + governs whoever does. + +## Explicitly OUT of scope for this tick + +- **The eight findings carried from PR #8** (F1, F2, F3, F5, F6, F8b, F9, F10 + in `ops/BACKLOG.md`). They are `check`-path debt. Touching them here mixes a + refactor's diff with a feature's and re-opens the review chronology that cost + PR #8 six ticks. If the `check` extraction makes one of them a one-line + obvious fix, still leave it: file the observation in the log instead. +- **Attaching a real worker for `llm` / `agent` steps.** Parked is the correct + gate-1 answer; the worker is gate 2 / gate 4's `mount-as-writer`, which is + also where `DESIGN.md` §1.9's double-effect residual closes. +- **The path-like deterministic-command refusal** (Codex P1) and the + **`steps: []` check/kernel asymmetry** — both filed, both `check`-path. +- **Gate 2, 5, and 6 work**, including harness's slack/notion helpers, `on()` + triggers, and `f.human`. `ops/SCOREBOARD.md` marks gate 6 "next up"; it stays + next, not now. +- **The release pipeline** (cross-compiled `relayflowd` bundled into the npm + package). This tick's live test may build the binary locally and read + `RELAYFLOWD_BIN`; shipping it is a separate package. +- **The cloud-sandbox `sync` gap** (no git remote in a fresh sandbox) and + re-registering cloud schedules. Both real, both filed, neither on the + gate-1 critical path. +- **Any change to `docs/RFC-0001-everything-is-a-relayflow.md`, + `ops/DIRECTIVES.md`, or a gate that judges this work.** + +## A note for the tick that logs this + +`ops/SCOREBOARD.md` currently reports gate 1 GREEN. This package does not +contradict that — the crash-resume and preflight clauses of §3's done-when are +genuinely met and independently verified. What it adds is the clause the gate's +own name implies and no test yet holds: that a *relayflow authored on the +published surface* is what runs. If this package lands, gate 1's evidence row +should cite the live path. If it fails, say so and leave the row alone — +a failed run is never reported as completed. + +END_PACKAGE diff --git a/ops/SCOREBOARD.md b/ops/SCOREBOARD.md index f45f6e4e..b65b0bb9 100644 --- a/ops/SCOREBOARD.md +++ b/ops/SCOREBOARD.md @@ -5,7 +5,7 @@ Every row starts RED and moves only on evidence. AMBER blocks nothing here | Gate | State | Evidence | |---|---|---| -| 1 — a relayflow can run | **GREEN** | Closed on `main` at `9e1d9eb` (PR #8, merged by Khaliq). Verified by me on a clean worktree off `origin/main`, not on branch evidence: kernel **72 passed / 0 failed**, sdk **131 passed / 8 files**. Preflight proven behaviorally through the built binary `sdk/dist/cli.js`: `cli_missing`, `cli_unauthenticated`, `cli_unresolved` each REFUSE with exit 2; `hello-ladder` passes with an `unprovable_effects` warning (silence is not a state). Residual, documented in DESIGN.md §1.9: a worker dying after the provider call but before confirming performs an effect twice — closing it needs gate 4's mount-as-writer. | +| 1 — a relayflow can run | **GREEN** | Closed on `main` at `9e1d9eb` (PR #8, merged by Khaliq) and extended on WP-10 by the live authored-surface seam: the built `sdk/dist/cli.js` submits `hello-deterministic.flow.yaml` to a real `relayflowd`, parks `hello-llm` and `hello-agent` with typed step details, journals `step_failed`, and resumes after `kill -9` with one successful completion per step. Branch verification: kernel **73 passed / 0 failed**, SDK **143 passed / 9 files**, including **7/7** built-binary and **4/4** live-kernel cases. Residual, documented in DESIGN.md §1.9: a worker dying after the provider call but before confirming performs an effect twice — closing it needs gate 4's mount-as-writer. | | 2 — proactive agent | RED | not started; harness shims wait on it | | 3 — Software Garden | RED | not started | | 4 — chief / harness | RED | not started | diff --git a/sdk/src/cli.ts b/sdk/src/cli.ts index 69b54abd..67961546 100644 --- a/sdk/src/cli.ts +++ b/sdk/src/cli.ts @@ -1,251 +1,103 @@ #!/usr/bin/env node -import { accessSync, constants, readFileSync, realpathSync } from 'node:fs'; -import { dirname, isAbsolute, join, parse as parsePath, resolve } from 'node:path'; +import { realpathSync } from 'node:fs'; import { pathToFileURL } from 'node:url'; -import { spawnSync } from 'node:child_process'; -import { parse as parseYaml } from 'yaml'; -import { CompileError, compileSpec, kernelToAuthoring } from './compile.js'; -import type { FlowSpec } from './spec.js'; -import type { CheckFailureKind } from './failure-kinds.js'; import { - preflight, - CliProbeError, - type CliResolution, - type PreflightDiagnostic, - type PreflightProbes, -} from './preflight.js'; + checkFlow, + inputFailureReport, + type CheckReport, +} from './cli/check.js'; +import { + resumeFlow, + runFlow, + type RunExecution, + type RunReport, +} from './cli/run.js'; -interface ProjectConfig { - cli?: string; - executors: string[]; - directory: string; - path?: string; -} - -export interface CheckReport { - ok: boolean; - path?: string; - projectConfigPath?: string; - resolutions: CliResolution[]; - diagnostics: Array; -} - -export interface CheckInputDiagnostic { - severity: 'refusal'; - kind: CheckFailureKind; - message: string; -} +export type { CheckInputDiagnostic, CheckReport } from './cli/check.js'; export interface CliIo { stdout(line: string): void; stderr(line: string): void; } +type CliExitCode = 0 | 1 | 2 | 3; +type ParsedArgs = + | { command: 'check'; json: boolean; value: string } + | { command: 'run' | 'resume'; dataDir: string; json: boolean; value: string }; + +const DEFAULT_DATA_DIR = '.relayflowd'; +const USAGE = [ + 'Usage:', + 'flows check [--json] ', + 'flows run [--json] [--data-dir ] ', + 'flows resume [--json] [--data-dir ] ', +].join(' '); + const PROCESS_IO: CliIo = { stdout: (line) => process.stdout.write(`${line}\n`), stderr: (line) => process.stderr.write(`${line}\n`), }; -class CheckFailure extends Error { - constructor(readonly kind: CheckFailureKind, message: string) { - super(message); - } -} - -export function runCli(args: readonly string[], io: CliIo = PROCESS_IO): number { +export async function runCli( + args: readonly string[], + io: CliIo = PROCESS_IO, +): Promise { const parsed = parseArgs(args); - // No path is known yet, and `--json` is honoured even when the invocation - // itself is what failed — a caller parsing stdout gets a report either way. - if (parsed instanceof CheckFailure) { - return emitInputFailure(parsed, undefined, args.includes('--json'), io); - } - - const absolutePath = resolve(parsed.path); - try { - const flow = readFlow(absolutePath); - const config = readProjectConfig(dirname(absolutePath)); - const probes = systemProbes(dirname(absolutePath), config); - const result = preflight(flow, { - projectCli: config.cli, - projectConfigPath: config.path, - projectSearchStart: dirname(absolutePath), - probes, - }); - const report: CheckReport = { - ok: result.ok, - path: parsed.path, - ...(config.path !== undefined ? { projectConfigPath: config.path } : {}), - resolutions: result.resolutions, - diagnostics: result.diagnostics, - }; - emitReport(report, parsed.json, io); - return result.ok ? 0 : 2; - } catch (error) { - const failure = error instanceof CheckFailure - ? error - : new CheckFailure('invalid_spec', `Flow "${parsed.path}" could not be checked as a Relayflow spec.`); - return emitInputFailure(failure, parsed.path, parsed.json, io); - } -} - -function parseArgs(args: readonly string[]): { path: string; json: boolean } | CheckFailure { - if (args[0] !== 'check') { - return new CheckFailure('invalid_invocation', 'Usage: flows check [--json] '); - } - const json = args.includes('--json'); - const unsupported = args.slice(1).filter((arg) => arg.startsWith('-') && arg !== '--json'); - const positionals = args.slice(1).filter((arg) => arg !== '--json'); - if (unsupported.length > 0 || positionals.length !== 1) { - return new CheckFailure('invalid_invocation', 'Usage: flows check [--json] '); - } - return { path: positionals[0]!, json }; -} - -function readFlow(path: string): FlowSpec { - let source: string; - try { - source = readFileSync(path, 'utf8'); - } catch { - throw new CheckFailure('input_unreadable', `Flow "${path}" is not readable.`); - } - - let parsed: unknown; - try { - parsed = parseYaml(source); - } catch { - throw new CheckFailure('invalid_spec', `Flow "${path}" contains invalid YAML or JSON.`); - } - - try { - const marker = kernelDialectMarker(parsed); - const authoring = marker === undefined ? parsed : kernelToAuthoring(parsed); - return compileSpec(authoring); - } catch (error) { - if (error instanceof CheckFailure) throw error; - if (error instanceof CompileError) { - const marker = kernelDialectMarker(parsed); - const dialect = marker === undefined - ? '' - : `Flow "${path}" was read as a compiled kernel spec because ${marker} is present. `; - throw new CheckFailure('invalid_spec', `${dialect}${error.errors.join('; ')}`); + if (parsed === undefined) { + const report = inputFailureReport({ kind: 'invalid_invocation', message: USAGE }); + emitCheckReport(report, args.includes('--json'), io); + return 2; + } + + if (parsed.command === 'check') { + const checked = checkFlow(parsed.value); + emitCheckReport(checked.report, parsed.json, io); + return checked.report.ok ? 0 : 2; + } + + const execution = parsed.command === 'run' + ? await runFlow(parsed.value, parsed.dataDir) + : await resumeFlow(parsed.value, parsed.dataDir); + emitRunReport(execution, parsed.json, io); + return execution.exitCode; +} + +function parseArgs(args: readonly string[]): ParsedArgs | undefined { + const command = args[0]; + if (command !== 'check' && command !== 'run' && command !== 'resume') return undefined; + + let json = false; + let dataDir = DEFAULT_DATA_DIR; + let sawDataDir = false; + const positionals: string[] = []; + for (let index = 1; index < args.length; index += 1) { + const argument = args[index]!; + if (argument === '--json') { + if (json) return undefined; + json = true; + continue; } - throw new CheckFailure('invalid_spec', `Flow "${path}" is not a valid Relayflow spec.`); - } -} - -function readProjectConfig(start: string): ProjectConfig { - const configPath = findConfig(start); - if (configPath === undefined) return { executors: [], directory: start }; - let value: unknown; - try { - value = JSON.parse(readFileSync(configPath, 'utf8')); - } catch { - throw new CheckFailure('config_invalid', `Project config "${configPath}" is not valid JSON.`); - } - if (!isObject(value) || Object.keys(value).some((key) => !['cli', 'executors'].includes(key))) { - throw new CheckFailure('config_invalid', `Project config "${configPath}" expects only cli and executors.`); - } - if (value['cli'] !== undefined && !isNonEmptyString(value['cli'])) { - throw new CheckFailure('config_invalid', `Project config "${configPath}" has an invalid cli.`); - } - if (value['executors'] !== undefined && (!Array.isArray(value['executors']) || !value['executors'].every(isNonEmptyString))) { - throw new CheckFailure('config_invalid', `Project config "${configPath}" has invalid executors.`); - } - return { - ...(value['cli'] !== undefined ? { cli: value['cli'] as string } : {}), - executors: (value['executors'] as string[] | undefined) ?? [], - directory: dirname(configPath), - path: configPath, - }; -} - -function findConfig(start: string): string | undefined { - let directory = start; - while (true) { - const candidate = join(directory, 'flows.json'); - try { - accessSync(candidate, constants.R_OK); - return candidate; - } catch { - // Continue toward the filesystem root. - } - const parent = dirname(directory); - if (parent === directory || directory === parsePath(directory).root) return undefined; - directory = parent; - } -} - -function systemProbes(flowDirectory: string, config: ProjectConfig): PreflightProbes { - return { - cli: (cli, source) => probeCli(cli, source === 'project' ? config.directory : flowDirectory), - executor: (trigger) => config.executors.includes(trigger.executor), - command: (binary) => executableExists(binary, flowDirectory), - }; -} - -function probeCli(cli: string, directory: string): { exists: boolean; authenticated: boolean } { - const executable = resolveExecutable(cli, directory); - if (executable === undefined) return { exists: false, authenticated: false }; - const result = spawnSync(executable, ['auth', 'status'], { - cwd: directory, - stdio: 'ignore', - timeout: 10_000, - }); - const failure = classifySpawnFailure(result.error, result.signal, 10_000); - if (failure !== undefined) throw failure; - return { exists: true, authenticated: result.status === 0 }; -} - -function resolveExecutable(command: string, directory: string): string | undefined { - if (command.includes('/') || isAbsolute(command)) { - const path = isAbsolute(command) ? command : resolve(directory, command); - try { - accessSync(path, constants.X_OK); - return path; - } catch { - return undefined; + if (argument === '--data-dir') { + const value = args[index + 1]; + if (command === 'check' || sawDataDir || value === undefined || value.startsWith('-')) return undefined; + dataDir = value; + sawDataDir = true; + index += 1; + continue; } + if (argument.startsWith('-')) return undefined; + positionals.push(argument); } - const result = spawnSync('which', [command], { encoding: 'utf8', timeout: 5_000 }); - const failure = classifySpawnFailure(result.error, result.signal, 5_000); - if (failure !== undefined) throw failure; - return result.status === 0 ? result.stdout.trim() : undefined; -} - -function classifySpawnFailure( - error: Error | undefined, - signal: NodeJS.Signals | null, - timeoutMs: 5_000 | 10_000, -): CliProbeError | undefined { - if (error !== undefined) { - const detail = (error as NodeJS.ErrnoException).code === 'ETIMEDOUT' - ? `timeout:${timeoutMs}ms` as const - : 'spawn_failed' as const; - return new CliProbeError(detail); - } - return signal === null ? undefined : new CliProbeError(`signal:${signal}`); -} - -function executableExists(command: string, directory: string): boolean { - return resolveExecutable(command, directory) !== undefined; -} + if (positionals.length !== 1) return undefined; -function emitInputFailure(failure: CheckFailure, path: string | undefined, json: boolean, io: CliIo): number { - const report: CheckReport = { - ok: false, - ...(path !== undefined ? { path } : {}), - resolutions: [], - diagnostics: [{ severity: 'refusal', kind: failure.kind, message: failure.message }], - }; - emitReport(report, json, io); - return 2; + return command === 'check' + ? { command, json, value: positionals[0]! } + : { command, dataDir, json, value: positionals[0]! }; } -function emitReport(report: CheckReport, json: boolean, io: CliIo): void { - for (const diagnostic of report.diagnostics) { - io.stderr(`${diagnostic.severity === 'warning' ? 'WARNING' : 'REFUSED'} [${diagnostic.kind}] ${diagnostic.message}`); - } +function emitCheckReport(report: CheckReport, json: boolean, io: CliIo): void { + emitDiagnostics(report.diagnostics, io); if (json) { io.stdout(JSON.stringify(report)); return; @@ -259,40 +111,37 @@ function emitReport(report: CheckReport, json: boolean, io: CliIo): void { if (report.ok) io.stdout(`CHECK PASSED ${report.path ?? ''}`.trimEnd()); } -function kernelDialectMarker(value: unknown): string | undefined { - if (!isObject(value) || !Array.isArray(value['steps'])) return undefined; - // Select the boundary dialect from snake_case keys or nested verification / - // policy shapes. Compiler-emitted kernel specs always carry `retry`; the - // additional shapes cover other producers that omit default fields. A - // document with no marker is ambiguous and is validated as authoring input. - const kernelStepKeys = ['depends_on', 'max_iterations', 'retry', 'timeout_ms', 'recovery_mode']; - const kernelBudgetKeys = ['max_tokens_in', 'max_tokens_out', 'max_dollars']; - const kernelVerificationKeys = ['output_contains', 'json_schema']; - const kernelPermissionKeys = ['file_globs', 'network_allowlist', 'access_preset']; - const budget = firstPresentKey(value['budget'], kernelBudgetKeys); - if (budget !== undefined) return `spec.budget.${budget}`; - for (const [index, step] of value['steps'].entries()) { - if (!isObject(step)) continue; - const stepKey = firstPresentKey(step, kernelStepKeys); - if (stepKey !== undefined) return `spec.steps[${index}].${stepKey}`; - const verification = firstPresentKey(step['verification'], kernelVerificationKeys); - if (verification !== undefined) return `spec.steps[${index}].verification.${verification}`; - const permission = firstPresentKey(step['permissions'], kernelPermissionKeys); - if (permission !== undefined) return `spec.steps[${index}].permissions.${permission}`; +function emitRunReport(execution: RunExecution, json: boolean, io: CliIo): void { + const { report } = execution; + emitDiagnostics(report.diagnostics, io); + if (json) { + io.stdout(JSON.stringify(report)); + return; } - return undefined; -} - -function firstPresentKey(value: unknown, keys: readonly string[]): string | undefined { - return isObject(value) ? keys.find((key) => key in value) : undefined; + if (report.runId === undefined) return; + const completed = report.completedSteps === undefined ? '' : ` (${report.completedSteps} steps)`; + const reason = report.completionReason === undefined + ? '' + : ` completionReason: ${report.completionReason}`; + io.stdout(`RUN ${report.runId} ${report.status ?? 'unknown'}${completed}${reason}`); } -function isObject(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); +function emitDiagnostics( + diagnostics: CheckReport['diagnostics'] | RunReport['diagnostics'], + io: CliIo, +): void { + for (const diagnostic of diagnostics) { + io.stderr(`${diagnosticLabel(diagnostic.severity)} [${diagnostic.kind}] ${diagnostic.message}`); + } } -function isNonEmptyString(value: unknown): value is string { - return typeof value === 'string' && value.length > 0; +function diagnosticLabel(severity: string): string { + switch (severity) { + case 'warning': return 'WARNING'; + case 'failure': return 'FAILED'; + case 'parked': return 'PARKED'; + default: return 'REFUSED'; + } } function isDirectInvocation(entryPath: string | undefined): boolean { @@ -300,12 +149,12 @@ function isDirectInvocation(entryPath: string | undefined): boolean { try { return pathToFileURL(realpathSync(entryPath)).href === import.meta.url; } catch { - // A missing, deleted, or unreadable argv[1] is treated as an import. The - // module stays safe to load without guessing that an unrelated host is us. return false; } } if (isDirectInvocation(process.argv[1])) { - process.exitCode = runCli(process.argv.slice(2)); + void runCli(process.argv.slice(2)).then((exitCode) => { + process.exitCode = exitCode; + }); } diff --git a/sdk/src/cli/check.ts b/sdk/src/cli/check.ts new file mode 100644 index 00000000..f3888d19 --- /dev/null +++ b/sdk/src/cli/check.ts @@ -0,0 +1,250 @@ +import { accessSync, constants, readFileSync } from 'node:fs'; +import { dirname, isAbsolute, join, parse as parsePath, resolve } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { parse as parseYaml } from 'yaml'; +import { CompileError, compileSpec, kernelToAuthoring } from '../compile.js'; +import type { FlowSpec } from '../spec.js'; +import type { CheckFailureKind } from '../failure-kinds.js'; +import { + preflight, + CliProbeError, + type CliResolution, + type PreflightDiagnostic, + type PreflightProbes, +} from '../preflight.js'; + +interface ProjectConfig { + cli?: string; + executors: string[]; + directory: string; + path?: string; +} + +export interface CheckReport { + ok: boolean; + path?: string; + projectConfigPath?: string; + resolutions: CliResolution[]; + diagnostics: Array; +} + +export interface CheckInputDiagnostic { + severity: 'refusal'; + kind: CheckFailureKind; + message: string; +} + +export interface CheckExecution { + report: CheckReport; + flow?: FlowSpec; +} + +class CheckFailure extends Error { + constructor(readonly kind: CheckFailureKind, message: string) { + super(message); + } +} + +/** Compile and preflight one working-tree spec without starting a run. */ +export function checkFlow(path: string): CheckExecution { + const absolutePath = resolve(path); + try { + const flow = readFlow(absolutePath); + const config = readProjectConfig(dirname(absolutePath)); + const probes = systemProbes(dirname(absolutePath), config); + const result = preflight(flow, { + projectCli: config.cli, + projectConfigPath: config.path, + projectSearchStart: dirname(absolutePath), + probes, + }); + return { + report: { + ok: result.ok, + path, + ...(config.path !== undefined ? { projectConfigPath: config.path } : {}), + resolutions: result.resolutions, + diagnostics: result.diagnostics, + }, + ...(result.ok ? { flow } : {}), + }; + } catch (error) { + const failure = error instanceof CheckFailure + ? error + : new CheckFailure('invalid_spec', `Flow "${path}" could not be checked as a Relayflow spec.`); + return { report: inputFailureReport(failure, path) }; + } +} + +export function inputFailureReport( + failure: { kind: CheckFailureKind; message: string }, + path?: string, +): CheckReport { + return { + ok: false, + ...(path !== undefined ? { path } : {}), + resolutions: [], + diagnostics: [{ severity: 'refusal', kind: failure.kind, message: failure.message }], + }; +} + +function readFlow(path: string): FlowSpec { + let source: string; + try { + source = readFileSync(path, 'utf8'); + } catch { + throw new CheckFailure('input_unreadable', `Flow "${path}" is not readable.`); + } + + let parsed: unknown; + try { + parsed = parseYaml(source); + } catch { + throw new CheckFailure('invalid_spec', `Flow "${path}" contains invalid YAML or JSON.`); + } + + try { + const marker = kernelDialectMarker(parsed); + const authoring = marker === undefined ? parsed : kernelToAuthoring(parsed); + return compileSpec(authoring); + } catch (error) { + if (error instanceof CheckFailure) throw error; + if (error instanceof CompileError) { + const marker = kernelDialectMarker(parsed); + const dialect = marker === undefined + ? '' + : `Flow "${path}" was read as a compiled kernel spec because ${marker} is present. `; + throw new CheckFailure('invalid_spec', `${dialect}${error.errors.join('; ')}`); + } + throw new CheckFailure('invalid_spec', `Flow "${path}" is not a valid Relayflow spec.`); + } +} + +function readProjectConfig(start: string): ProjectConfig { + const configPath = findConfig(start); + if (configPath === undefined) return { executors: [], directory: start }; + let value: unknown; + try { + value = JSON.parse(readFileSync(configPath, 'utf8')); + } catch { + throw new CheckFailure('config_invalid', `Project config "${configPath}" is not valid JSON.`); + } + if (!isObject(value) || Object.keys(value).some((key) => !['cli', 'executors'].includes(key))) { + throw new CheckFailure('config_invalid', `Project config "${configPath}" expects only cli and executors.`); + } + if (value['cli'] !== undefined && !isNonEmptyString(value['cli'])) { + throw new CheckFailure('config_invalid', `Project config "${configPath}" has an invalid cli.`); + } + if (value['executors'] !== undefined && (!Array.isArray(value['executors']) || !value['executors'].every(isNonEmptyString))) { + throw new CheckFailure('config_invalid', `Project config "${configPath}" has invalid executors.`); + } + return { + ...(value['cli'] !== undefined ? { cli: value['cli'] as string } : {}), + executors: (value['executors'] as string[] | undefined) ?? [], + directory: dirname(configPath), + path: configPath, + }; +} + +function findConfig(start: string): string | undefined { + let directory = start; + while (true) { + const candidate = join(directory, 'flows.json'); + try { + accessSync(candidate, constants.R_OK); + return candidate; + } catch { + // Continue toward the filesystem root. + } + const parent = dirname(directory); + if (parent === directory || directory === parsePath(directory).root) return undefined; + directory = parent; + } +} + +function systemProbes(flowDirectory: string, config: ProjectConfig): PreflightProbes { + return { + cli: (cli, source) => probeCli(cli, source === 'project' ? config.directory : flowDirectory), + executor: (trigger) => config.executors.includes(trigger.executor), + command: (binary) => executableExists(binary, flowDirectory), + }; +} + +function probeCli(cli: string, directory: string): { exists: boolean; authenticated: boolean } { + const executable = resolveExecutable(cli, directory); + if (executable === undefined) return { exists: false, authenticated: false }; + const result = spawnSync(executable, ['auth', 'status'], { + cwd: directory, + stdio: 'ignore', + timeout: 10_000, + }); + const failure = classifySpawnFailure(result.error, result.signal, 10_000); + if (failure !== undefined) throw failure; + return { exists: true, authenticated: result.status === 0 }; +} + +function resolveExecutable(command: string, directory: string): string | undefined { + if (command.includes('/') || isAbsolute(command)) { + const path = isAbsolute(command) ? command : resolve(directory, command); + try { + accessSync(path, constants.X_OK); + return path; + } catch { + return undefined; + } + } + const result = spawnSync('which', [command], { encoding: 'utf8', timeout: 5_000 }); + const failure = classifySpawnFailure(result.error, result.signal, 5_000); + if (failure !== undefined) throw failure; + return result.status === 0 ? result.stdout.trim() : undefined; +} + +function classifySpawnFailure( + error: Error | undefined, + signal: NodeJS.Signals | null, + timeoutMs: 5_000 | 10_000, +): CliProbeError | undefined { + if (error !== undefined) { + const detail = (error as NodeJS.ErrnoException).code === 'ETIMEDOUT' + ? `timeout:${timeoutMs}ms` as const + : 'spawn_failed' as const; + return new CliProbeError(detail); + } + return signal === null ? undefined : new CliProbeError(`signal:${signal}`); +} + +function executableExists(command: string, directory: string): boolean { + return resolveExecutable(command, directory) !== undefined; +} + +function kernelDialectMarker(value: unknown): string | undefined { + if (!isObject(value) || !Array.isArray(value['steps'])) return undefined; + const kernelStepKeys = ['depends_on', 'max_iterations', 'retry', 'timeout_ms', 'recovery_mode']; + const kernelBudgetKeys = ['max_tokens_in', 'max_tokens_out', 'max_dollars']; + const kernelVerificationKeys = ['output_contains', 'json_schema']; + const kernelPermissionKeys = ['file_globs', 'network_allowlist', 'access_preset']; + const budget = firstPresentKey(value['budget'], kernelBudgetKeys); + if (budget !== undefined) return `spec.budget.${budget}`; + for (const [index, step] of value['steps'].entries()) { + if (!isObject(step)) continue; + const stepKey = firstPresentKey(step, kernelStepKeys); + if (stepKey !== undefined) return `spec.steps[${index}].${stepKey}`; + const verification = firstPresentKey(step['verification'], kernelVerificationKeys); + if (verification !== undefined) return `spec.steps[${index}].verification.${verification}`; + const permission = firstPresentKey(step['permissions'], kernelPermissionKeys); + if (permission !== undefined) return `spec.steps[${index}].permissions.${permission}`; + } + return undefined; +} + +function firstPresentKey(value: unknown, keys: readonly string[]): string | undefined { + return isObject(value) ? keys.find((key) => key in value) : undefined; +} + +function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isNonEmptyString(value: unknown): value is string { + return typeof value === 'string' && value.length > 0; +} diff --git a/sdk/src/cli/run.ts b/sdk/src/cli/run.ts new file mode 100644 index 00000000..52fede7f --- /dev/null +++ b/sdk/src/cli/run.ts @@ -0,0 +1,261 @@ +import { join, resolve } from 'node:path'; +import { toKernelSpec } from '../compile.js'; +import type { RunFailureKind } from '../failure-kinds.js'; +import { JournalClient } from '../journal-client.js'; +import type { PreflightDiagnostic } from '../preflight.js'; +import type { + RunCompletionReason, + RunOutcome, + RunStatus, +} from '../protocol.js'; +import type { KernelRunSpec, KernelStepSpec, StepType } from '../spec.js'; +import { + checkFlow, + type CheckInputDiagnostic, + type CheckReport, +} from './check.js'; + +export type RunExitCode = 0 | 1 | 2 | 3; +export type RunCommand = 'run' | 'resume'; + +export interface ParkedStep { + id: string; + type: Extract; +} + +export interface RunDiagnostic { + severity: 'refusal' | 'failure' | 'parked'; + kind: RunFailureKind | RunCompletionReason; + message: string; +} + +export interface RunReport { + ok: boolean; + command: RunCommand; + path?: string; + runId?: string; + socketPath?: string; + status?: RunStatus; + completionReason?: RunCompletionReason; + completedSteps?: number; + parkedStep?: ParkedStep; + projectConfigPath?: string; + resolutions: CheckReport['resolutions']; + diagnostics: Array; +} + +export interface RunExecution { + exitCode: RunExitCode; + report: RunReport; +} + +export async function runFlow(path: string, dataDir: string): Promise { + const checked = checkFlow(path); + if (!checked.report.ok || checked.flow === undefined) { + return { exitCode: 2, report: fromCheckReport('run', checked.report) }; + } + + const socketPath = socketFor(dataDir); + const client = new JournalClient(socketPath); + const connected = await connect(client, 'run', dataDir, checked.report); + if (connected !== undefined) return connected; + + try { + const spec = toKernelSpec(checked.flow); + const outcome = await client.runStart(spec); + return await classifyOutcome(client, 'run', outcome, checked.report, socketPath, spec); + } catch (error) { + return protocolFailure('run', checked.report, socketPath, error); + } finally { + client.close(); + } +} + +export async function resumeFlow(runId: string, dataDir: string): Promise { + const socketPath = socketFor(dataDir); + const base = emptyReport('resume'); + const client = new JournalClient(socketPath); + const connected = await connect(client, 'resume', dataDir, base); + if (connected !== undefined) return connected; + + try { + const outcome = await client.runResume(runId); + return await classifyOutcome(client, 'resume', outcome, base, socketPath); + } catch (error) { + return { + exitCode: 2, + report: { + ...base, + runId, + socketPath, + diagnostics: [{ + severity: 'refusal', + kind: 'run_unavailable', + message: `Run "${runId}" could not be resumed by relayflowd at "${socketPath}": ${errorMessage(error)}`, + }], + }, + }; + } finally { + client.close(); + } +} + +async function connect( + client: JournalClient, + command: RunCommand, + dataDir: string, + base: CheckReport | RunReport, +): Promise { + const socketPath = socketFor(dataDir); + try { + await client.connect(); + await client.hello(`flows-${command}`); + return undefined; + } catch { + client.close(); + return { + exitCode: 2, + report: { + ...fromBase(command, base), + socketPath, + diagnostics: [ + ...base.diagnostics, + { + severity: 'refusal', + kind: 'daemon_unreachable', + message: `No compatible relayflowd is listening at "${socketPath}". Start it with: relayflowd --data-dir ${JSON.stringify(dataDir)} serve`, + }, + ], + }, + }; + } +} + +async function classifyOutcome( + client: JournalClient, + command: RunCommand, + outcome: RunOutcome, + base: CheckReport | RunReport, + socketPath: string, + knownSpec?: KernelRunSpec, +): Promise { + const report: RunReport = { + ...fromBase(command, base), + ok: outcome.status === 'completed' && outcome.completion_reason === 'success', + runId: outcome.run_id, + socketPath, + status: outcome.status, + ...(outcome.completion_reason !== null ? { completionReason: outcome.completion_reason } : {}), + completedSteps: outcome.completed_steps, + }; + + if (report.ok) return { exitCode: 0, report }; + if (outcome.status === 'failed' && outcome.completion_reason !== null) { + return { + exitCode: 1, + report: { + ...report, + diagnostics: [...report.diagnostics, { + severity: 'failure', + kind: outcome.completion_reason, + message: `Run "${outcome.run_id}" failed with completionReason: ${outcome.completion_reason}.`, + }], + }, + }; + } + if (outcome.status === 'parked') { + const parkedStep = await findParkedStep(client, outcome.run_id, knownSpec); + if (parkedStep !== undefined) { + return { + exitCode: 3, + report: { + ...report, + parkedStep, + diagnostics: [...report.diagnostics, { + severity: 'parked', + kind: 'run_parked', + message: `Run "${outcome.run_id}" parked at step "${parkedStep.id}" (${parkedStep.type}): no worker is attached for step type "${parkedStep.type}".`, + }], + }, + }; + } + } + return protocolFailure(command, base, socketPath, new Error( + `relayflowd returned status ${outcome.status} without a classifiable completion`, + ), outcome.run_id); +} + +async function findParkedStep( + client: JournalClient, + runId: string, + knownSpec?: KernelRunSpec, +): Promise { + const spec = knownSpec ?? await readRunSpec(client, runId); + if (spec === undefined) return undefined; + const snapshot = await client.runGet(runId); + return spec.steps.find((step): step is KernelStepSpec & { type: 'llm' | 'agent' } => + step.type !== 'deterministic' && snapshot.steps[step.id] === 'Runnable', + ); +} + +async function readRunSpec(client: JournalClient, runId: string): Promise { + const { entries } = await client.journalRead(runId, 1, 1); + const entry = entries[0]; + if (!isObject(entry) || entry['entry_type'] !== 'run.spawned') return undefined; + const payload = entry['payload']; + if (!isObject(payload) || !isObject(payload['spec'])) return undefined; + return payload['spec'] as unknown as KernelRunSpec; +} + +function protocolFailure( + command: RunCommand, + base: CheckReport | RunReport, + socketPath: string, + error: unknown, + runId?: string, +): RunExecution { + return { + exitCode: 1, + report: { + ...fromBase(command, base), + ...(runId !== undefined ? { runId } : {}), + socketPath, + diagnostics: [...base.diagnostics, { + severity: 'failure', + kind: 'protocol_error', + message: `relayflowd could not complete the ${command} request: ${errorMessage(error)}`, + }], + }, + }; +} + +function fromCheckReport(command: RunCommand, report: CheckReport): RunReport { + return { + ok: false, + command, + ...(report.path !== undefined ? { path: report.path } : {}), + ...(report.projectConfigPath !== undefined ? { projectConfigPath: report.projectConfigPath } : {}), + resolutions: report.resolutions, + diagnostics: report.diagnostics, + }; +} + +function emptyReport(command: RunCommand): RunReport { + return { ok: false, command, resolutions: [], diagnostics: [] }; +} + +function fromBase(command: RunCommand, base: CheckReport | RunReport): RunReport { + return 'command' in base ? base : fromCheckReport(command, base); +} + +function socketFor(dataDir: string): string { + return join(resolve(dataDir), 'relayflowd.sock'); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : 'unknown protocol error'; +} + +function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} diff --git a/sdk/src/failure-kinds.ts b/sdk/src/failure-kinds.ts index 7d21cc8a..d5f412bb 100644 --- a/sdk/src/failure-kinds.ts +++ b/sdk/src/failure-kinds.ts @@ -32,12 +32,26 @@ export const PREFLIGHT_WARNING_KINDS = [ 'command_unprovable', ] as const; +/** Closed outcome taxonomy owned by the `flows run` / `flows resume` surface. */ +export const RUN_FAILURE_KINDS = [ + 'daemon_unreachable', + 'protocol_error', + 'run_parked', + 'run_unavailable', +] as const; + export type PreflightFailureKind = (typeof PREFLIGHT_FAILURE_KINDS)[number]; export type CheckFailureKind = (typeof CHECK_FAILURE_KINDS)[number]; export type PreflightWarningKind = (typeof PREFLIGHT_WARNING_KINDS)[number]; +export type RunFailureKind = (typeof RUN_FAILURE_KINDS)[number]; const CHECK_FAILURE_KIND_SET: ReadonlySet = new Set(CHECK_FAILURE_KINDS); +const RUN_FAILURE_KIND_SET: ReadonlySet = new Set(RUN_FAILURE_KINDS); export function isCheckFailureKind(value: string): value is CheckFailureKind { return CHECK_FAILURE_KIND_SET.has(value); } + +export function isRunFailureKind(value: string): value is RunFailureKind { + return RUN_FAILURE_KIND_SET.has(value); +} diff --git a/sdk/tests/cli.test.ts b/sdk/tests/cli.test.ts index d2ab896b..aee6c4fa 100644 --- a/sdk/tests/cli.test.ts +++ b/sdk/tests/cli.test.ts @@ -1,4 +1,6 @@ import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { once } from 'node:events'; +import type { Server } from 'node:net'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -9,11 +11,19 @@ import { CHECK_INPUT_FAILURE_KINDS, isCheckFailureKind, } from '../src/failure-kinds.js'; +import { + kernelDialectError, + sendOk, + sendResult, + startLoopback, + type LoopbackHandlers, +} from './journal-client-loopback.js'; const TESTDATA = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'testdata'); const PREFLIGHT = join(TESTDATA, 'preflight'); const LADDER = ['hello-ladder', 'hello-llm', 'hello-agent'] as const; const temporaryDirectories: string[] = []; +const loopbackServers: Server[] = []; const KERNEL_RETRY = { initial_backoff_ms: 100, max_backoff_ms: 60_000, @@ -21,7 +31,10 @@ const KERNEL_RETRY = { jitter_percent: 20, }; -afterEach(() => { +afterEach(async () => { + for (const server of loopbackServers.splice(0)) { + await new Promise((resolveClose) => server.close(() => resolveClose())); + } for (const directory of temporaryDirectories.splice(0)) rmSync(directory, { recursive: true, force: true }); }); @@ -31,9 +44,9 @@ function capture(): { io: CliIo; stdout: string[]; stderr: string[] } { return { io: { stdout: (line) => stdout.push(line), stderr: (line) => stderr.push(line) }, stdout, stderr }; } -function run(path: string, json = false): { code: number; stdout: string[]; stderr: string[] } { +async function run(path: string, json = false): Promise<{ code: number; stdout: string[]; stderr: string[] }> { const output = capture(); - const code = runCli(['check', ...(json ? ['--json'] : []), path], output.io); + const code = await runCli(['check', ...(json ? ['--json'] : []), path], output.io); return { code, stdout: output.stdout, stderr: output.stderr }; } @@ -73,8 +86,14 @@ const LADDER_FAULTS = [ }], ] as const satisfies ReadonlyArray) => void]>; +async function startCliLoopback(dataDir: string, handlers: LoopbackHandlers): Promise { + const server = startLoopback(join(dataDir, 'relayflowd.sock'), handlers); + loopbackServers.push(server); + if (!server.listening) await once(server, 'listening'); +} + describe('flows check CLI', () => { - it('explains kernel-dialect routing and names the offending mixed-dialect key', () => { + it('explains kernel-dialect routing and names the offending mixed-dialect key', async () => { const directory = temporaryProject(); const path = join(directory, 'mixed.flow.yaml'); writeFileSync(path, ` @@ -91,7 +110,7 @@ steps: depends_on: [first] `); - const result = run(path); + const result = await run(path); expect(result.code).toBe(2); expect(result.stderr.join('\n')).toContain( 'read as a compiled kernel spec because spec.steps[1].depends_on is present', @@ -99,7 +118,7 @@ steps: expect(result.stderr.join('\n')).toContain('spec.steps[0]: unknown key "timeoutMs"'); }); - it('names an unknown key and its location in a compiled kernel spec', () => { + it('names an unknown key and its location in a compiled kernel spec', async () => { const directory = temporaryProject(); const path = join(directory, 'unknown-kernel-key.json'); writeFileSync(path, JSON.stringify({ @@ -116,14 +135,14 @@ steps: }], })); - const result = run(path); + const result = await run(path); expect(result.code).toBe(2); expect(result.stderr.join('\n')).toContain('spec.steps[0]: unknown key "mystery"'); }); - it('passes all three canonical ladder flows and prints their resolved CLI', () => { + it('passes all three canonical ladder flows and prints their resolved CLI', async () => { for (const name of LADDER) { - const result = run(join(TESTDATA, `${name}.flow.yaml`)); + const result = await run(join(TESTDATA, `${name}.flow.yaml`)); expect(result.code, name).toBe(0); expect(result.stdout.join('\n'), name).toContain('RESOLVED'); expect(result.stdout.join('\n'), name).toContain('CHECK PASSED'); @@ -136,8 +155,8 @@ steps: ['cli-unauthenticated.flow.yaml', 'cli_unauthenticated'], ['cli-unresolved.flow.yaml', 'cli_unresolved'], ['no-executor.flow.yaml', 'no_executor'], - ] as const)('refuses %s with typed kind %s and exit 2', (name, kind) => { - const result = run(join(TESTDATA, 'preflight', name)); + ] as const)('refuses %s with typed kind %s and exit 2', async (name, kind) => { + const result = await run(join(TESTDATA, 'preflight', name)); expect(result.code).toBe(2); expect(result.stderr.join('\n')).toContain(`REFUSED [${kind}]`); }); @@ -146,16 +165,16 @@ steps: // ladder flow still passes. Without this the refusals could be an artifact of // the temp directory rather than of the fault, and the suite would be green // for the wrong reason. - it.each(LADDER)('passes relocated ladder flow %s when no fault is induced', (name) => { - const result = run(ladderVariant(name, (flow) => { flow['cli'] = join(PREFLIGHT, 'authenticated-cli'); })); + it.each(LADDER)('passes relocated ladder flow %s when no fault is induced', async (name) => { + const result = await run(ladderVariant(name, (flow) => { flow['cli'] = join(PREFLIGHT, 'authenticated-cli'); })); expect(result.code, name).toBe(0); expect(result.stdout.join('\n'), name).toContain('CHECK PASSED'); }); it.each(LADDER.flatMap((name) => LADDER_FAULTS.map(([kind, mutate]) => [name, kind, mutate] as const)))( 'refuses ladder flow %s with %s under an induced fault', - (name, kind, mutate) => { - const result = run(ladderVariant(name, mutate)); + async (name, kind, mutate) => { + const result = await run(ladderVariant(name, mutate)); expect(result.code, `${name}/${kind}`).toBe(2); expect(result.stderr.join('\n'), `${name}/${kind}`).toContain(`REFUSED [${kind}]`); }, @@ -169,23 +188,23 @@ steps: ['cli-declared.flow.yaml', 'RESOLVED'], ['trigger-declared.flow.yaml', 'CHECK PASSED'], ['warning.flow.yaml', 'CHECK PASSED'], - ] as const)('accepts %s without refusing', (name, expected) => { - const result = run(join(PREFLIGHT, name)); + ] as const)('accepts %s without refusing', async (name, expected) => { + const result = await run(join(PREFLIGHT, name)); expect(result.code, name).toBe(0); expect(result.stdout.join('\n'), name).toContain(expected); expect(result.stderr.join('\n'), name).not.toContain('REFUSED'); }); - it('warns on an unprovable deterministic effect while still passing the flow', () => { - const result = run(join(PREFLIGHT, 'warning.flow.yaml')); + it('warns on an unprovable deterministic effect while still passing the flow', async () => { + const result = await run(join(PREFLIGHT, 'warning.flow.yaml')); expect(result.code).toBe(0); expect(result.stderr.join('\n')).toContain('WARNING [unprovable_effects]'); }); - it('pins the complete JSON report for a pass and a refusal', () => { + it('pins the complete JSON report for a pass and a refusal', async () => { const configPath = join(PREFLIGHT, 'flows.json'); const passPath = join(PREFLIGHT, 'cli-declared.flow.yaml'); - const pass = run(passPath, true); + const pass = await run(passPath, true); expect(pass.code).toBe(0); expect(JSON.parse(pass.stdout.join('\n')) as CheckReport).toEqual({ ok: true, @@ -196,7 +215,7 @@ steps: }); const refusalPath = join(PREFLIGHT, 'cli-missing.flow.yaml'); - const refusal = run(refusalPath, true); + const refusal = await run(refusalPath, true); expect(refusal.code).toBe(2); const report = JSON.parse(refusal.stdout.join('\n')) as CheckReport; expect(report).toEqual({ @@ -215,8 +234,8 @@ steps: expect(report.diagnostics.every((entry) => isCheckFailureKind(entry.kind))).toBe(true); }); - it('checks the compiled kernel-dialect canonical spec as well as YAML', () => { - const result = run(join(TESTDATA, 'hello-ladder.spec.canonical.json')); + it('checks the compiled kernel-dialect canonical spec as well as YAML', async () => { + const result = await run(join(TESTDATA, 'hello-ladder.spec.canonical.json')); expect(result.code).toBe(0); expect(result.stdout.join('\n')).toContain('CHECK PASSED'); }); @@ -245,17 +264,17 @@ steps: cli: join(PREFLIGHT, 'authenticated-cli'), steps: [{ id: 'a', type: 'agent', instruction: 'act', permissions: { file_globs: ['src/**'] } }], }], - ] as const)('recognizes kernel dialect from %s alone', (marker, spec) => { + ] as const)('recognizes kernel dialect from %s alone', async (marker, spec) => { const directory = temporaryProject(); const path = join(directory, `${marker.replace('.', '-')}.spec.json`); writeFileSync(path, JSON.stringify(spec)); - const result = run(path); + const result = await run(path); expect(result.code, result.stderr.join('\n')).toBe(0); expect(result.stdout.join('\n'), marker).toContain('CHECK PASSED'); }); - it('rejects type-specific unknown fields in compiled specs instead of dropping them', () => { + it('rejects type-specific unknown fields in compiled specs instead of dropping them', async () => { const directory = temporaryProject(); const compiled = JSON.parse(readFileSync(join(TESTDATA, 'hello-ladder.spec.canonical.json'), 'utf8')) as { steps: Array>; @@ -264,12 +283,12 @@ steps: const path = join(directory, 'unknown-field.spec.json'); writeFileSync(path, JSON.stringify(compiled)); - const result = run(path); + const result = await run(path); expect(result.code).toBe(2); expect(result.stderr.join('\n')).toContain('REFUSED [invalid_spec]'); }); - it('rejects non-default compiled retry policies instead of dropping them', () => { + it('rejects non-default compiled retry policies instead of dropping them', async () => { const directory = temporaryProject(); const compiled = JSON.parse(readFileSync(join(TESTDATA, 'hello-ladder.spec.canonical.json'), 'utf8')) as { steps: Array<{ retry: Record }>; @@ -278,13 +297,13 @@ steps: const path = join(directory, 'bad-retry.spec.json'); writeFileSync(path, JSON.stringify(compiled)); - const result = run(path); + const result = await run(path); expect(result.code).toBe(2); expect(result.stderr.join('\n')).toContain('retry.multiplier must equal the authoring default 2'); }); - it('loads the final CLI resolution source from the nearest flows.json', () => { - const result = run(join(TESTDATA, 'preflight', 'project-default', 'project-cli.flow.yaml')); + it('loads the final CLI resolution source from the nearest flows.json', async () => { + const result = await run(join(TESTDATA, 'preflight', 'project-default', 'project-cli.flow.yaml')); expect(result.code).toBe(0); expect(result.stdout.join('\n')).toContain( `from project (${join(TESTDATA, 'preflight', 'project-default', 'flows.json')})`, @@ -292,7 +311,7 @@ steps: expect(result.stdout.join('\n')).toContain('../authenticated-cli'); }); - it('resolves a project CLI path relative to the flows.json that declares it', () => { + it('resolves a project CLI path relative to the flows.json that declares it', async () => { const directory = temporaryProject(); const flowDirectory = join(directory, 'nested'); mkdirSync(flowDirectory); @@ -303,14 +322,14 @@ steps: const flow = join(flowDirectory, 'project-cli.flow.yaml'); writeFileSync(flow, "version: '0.1.0'\nsteps:\n - id: answer\n type: llm\n prompt: answer\n"); - const result = run(flow); + const result = await run(flow); expect(result.code).toBe(0); expect(result.stdout.join('\n')).toContain( `RESOLVED step "answer" cli "./authenticated-cli" from project (${join(directory, 'flows.json')})`, ); }); - it('uses the nearest flows.json as a whole project boundary and names it on refusal', () => { + it('uses the nearest flows.json as a whole project boundary and names it on refusal', async () => { const directory = temporaryProject(); const nested = join(directory, 'nested'); const flowDirectory = join(nested, 'flows'); @@ -320,13 +339,13 @@ steps: const path = join(flowDirectory, 'shadowed.flow.yaml'); writeFileSync(path, "version: '0.1.0'\nsteps:\n - id: answer\n type: llm\n prompt: answer\n"); - const result = run(path); + const result = await run(path); expect(result.code).toBe(2); expect(result.stderr.join('\n')).toContain(`Nearest project config "${join(nested, 'flows.json')}" declares no cli`); expect(result.stderr.join('\n')).toContain('outer configs are shadowed'); }); - it('maps every input refusal path to its declared kind without raw exceptions', () => { + it('maps every input refusal path to its declared kind without raw exceptions', async () => { const directory = temporaryProject(); const malformed = join(directory, 'malformed.flow.yaml'); writeFileSync(malformed, 'not: [valid'); @@ -334,14 +353,14 @@ steps: writeFileSync(valid, "version: '0.1.0'\nname: valid\nsteps:\n - id: ready\n type: deterministic\n command: printf\n"); const invalidInvocation = capture(); - expect(runCli(['run'], invalidInvocation.io)).toBe(2); + expect(await runCli(['run'], invalidInvocation.io)).toBe(2); const outputs = [ invalidInvocation.stderr.join('\n'), - run(join(directory, 'absent.flow.yaml')).stderr.join('\n'), - run(malformed).stderr.join('\n'), + (await run(join(directory, 'absent.flow.yaml'))).stderr.join('\n'), + (await run(malformed)).stderr.join('\n'), ]; writeFileSync(join(directory, 'flows.json'), '{bad json'); - outputs.push(run(valid).stderr.join('\n')); + outputs.push((await run(valid)).stderr.join('\n')); const kinds = outputs.map((output) => output.match(/\[([^\]]+)]/)?.[1]); expect(new Set(kinds)).toEqual(new Set(CHECK_INPUT_FAILURE_KINDS)); @@ -349,3 +368,138 @@ steps: expect(kinds.every((kind) => kind !== undefined && isCheckFailureKind(kind))).toBe(true); }); }); + +describe('flows run/resume CLI over the journal protocol', () => { + it('parses run options, submits the kernel dialect, and exits 0 on success', async () => { + const dataDir = temporaryProject('flows-run-success-'); + let dialectError: string | null | undefined; + await startCliLoopback(dataDir, { + hello: sendOk, + 'run.start': (ctx, params) => { + dialectError = kernelDialectError(params['spec']); + sendResult(ctx, { + run_id: 'run-success', + status: 'completed', + completion_reason: 'success', + completed_steps: 2, + }); + }, + }); + const output = capture(); + + const code = await runCli([ + 'run', '--json', '--data-dir', dataDir, join(TESTDATA, 'hello-deterministic.flow.yaml'), + ], output.io); + + expect(code).toBe(0); + expect(dialectError).toBeNull(); + expect(JSON.parse(output.stdout.join('\n'))).toMatchObject({ + ok: true, + command: 'run', + runId: 'run-success', + completionReason: 'success', + }); + }); + + it('exits 1 and emits the declared completionReason for a failed run', async () => { + const dataDir = temporaryProject('flows-run-failed-'); + await startCliLoopback(dataDir, { + hello: sendOk, + 'run.start': (ctx) => sendResult(ctx, { + run_id: 'run-failed', + status: 'failed', + completion_reason: 'step_failed', + completed_steps: 0, + }), + }); + const output = capture(); + + const code = await runCli([ + 'run', '--data-dir', dataDir, join(TESTDATA, 'hello-deterministic.flow.yaml'), + ], output.io); + + expect(code).toBe(1); + expect(output.stderr.join('\n')).toContain('FAILED [step_failed]'); + expect(output.stdout.join('\n')).toContain('completionReason: step_failed'); + }); + + it('exits 3 and names the parked llm step', async () => { + const dataDir = temporaryProject('flows-run-parked-'); + await startCliLoopback(dataDir, { + hello: sendOk, + 'run.start': (ctx) => sendResult(ctx, { + run_id: 'run-parked', + status: 'parked', + completion_reason: null, + completed_steps: 1, + }), + 'run.get': (ctx) => sendResult(ctx, { + run_id: 'run-parked', + status: 'parked', + steps: { greet: 'Completed', answer: 'Runnable', finish: 'Pending' }, + budget: { tokens_in: 0, tokens_out: 0, dollars: '0' }, + }), + }); + const output = capture(); + + const code = await runCli([ + 'run', '--data-dir', dataDir, join(TESTDATA, 'hello-llm.flow.yaml'), + ], output.io); + + expect(code).toBe(3); + expect(output.stderr.join('\n')).toContain('PARKED [run_parked]'); + expect(output.stderr.join('\n')).toContain('step "answer" (llm)'); + }); + + it('parses resume and maps an unavailable run to exit 2', async () => { + const dataDir = temporaryProject('flows-resume-'); + await startCliLoopback(dataDir, { + hello: sendOk, + 'run.resume': (ctx, params) => { + if (params['run_id'] === 'known-run') { + sendResult(ctx, { + run_id: 'known-run', + status: 'completed', + completion_reason: 'success', + completed_steps: 3, + }); + return; + } + ctx.send({ + id: ctx.id, + ok: false, + error: { code: 'run_not_found', message: String(params['run_id']) }, + }); + }, + }); + const resumed = capture(); + const unavailable = capture(); + + expect(await runCli(['resume', '--data-dir', dataDir, 'known-run'], resumed.io)).toBe(0); + expect(resumed.stdout.join('\n')).toContain('completionReason: success'); + expect(await runCli(['resume', '--json', '--data-dir', dataDir, 'absent-run'], unavailable.io)).toBe(2); + expect(unavailable.stderr.join('\n')).toContain('REFUSED [run_unavailable]'); + expect(JSON.parse(unavailable.stdout.join('\n'))).toMatchObject({ + ok: false, + command: 'resume', + runId: 'absent-run', + diagnostics: [{ kind: 'run_unavailable' }], + }); + }); + + it('refuses malformed options with usage naming check, run, and resume', async () => { + for (const args of [ + ['run'], + ['run', '--data-dir', 'flow.yaml'], + ['run', '--data-dir', 'one', '--data-dir', 'two', 'flow.yaml'], + ['resume', '--unknown', 'run-id'], + ['check', '--data-dir', 'data', 'flow.yaml'], + ]) { + const output = capture(); + expect(await runCli(args, output.io), args.join(' ')).toBe(2); + expect(output.stderr.join('\n'), args.join(' ')).toContain('flows check'); + expect(output.stderr.join('\n'), args.join(' ')).toContain('flows run'); + expect(output.stderr.join('\n'), args.join(' ')).toContain('flows resume'); + } + }); +}); diff --git a/sdk/tests/journal-client-loopback.ts b/sdk/tests/journal-client-loopback.ts new file mode 100644 index 00000000..c7ad8c20 --- /dev/null +++ b/sdk/tests/journal-client-loopback.ts @@ -0,0 +1,110 @@ +import { randomUUID } from 'node:crypto'; +import { createServer, type Server, type Socket } from 'node:net'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { toKernelSpec } from '../src/compile.js'; +import { PROTOCOL_VERSION } from '../src/protocol.js'; +import type { FlowSpec, KernelRunSpec } from '../src/spec.js'; + +export function sockPath(): string { + return join(tmpdir(), `rf-${randomUUID().slice(0, 8)}.sock`); +} + +const HELLO_FLOW: FlowSpec = { + version: '0.1.0', + name: 'client-roundtrip', + steps: [{ id: 'greet', type: 'deterministic', command: 'echo hi' }], +}; +export const HELLO_SPEC: KernelRunSpec = toKernelSpec(HELLO_FLOW); + +export interface FrameCtx { + id: string; + socket: Socket; + send: (obj: unknown) => void; +} + +export interface LoopbackHandlers { + hello?: (ctx: FrameCtx) => void; + 'run.start'?: (ctx: FrameCtx, params: Record) => void; + 'run.resume'?: (ctx: FrameCtx, params: Record) => void; + 'run.get'?: (ctx: FrameCtx, params: Record) => void; + 'run.watch'?: (ctx: FrameCtx, params: Record) => void; + 'worker.attach'?: (ctx: FrameCtx, params: Record) => void; + 'step.heartbeat'?: (ctx: FrameCtx, params: Record) => void; + 'effect.record'?: (ctx: FrameCtx, params: Record) => void; + 'effect.confirm'?: (ctx: FrameCtx, params: Record) => void; + 'step.complete'?: (ctx: FrameCtx, params: Record) => void; + 'event.emit'?: (ctx: FrameCtx, params: Record) => void; + 'journal.read'?: (ctx: FrameCtx, params: Record) => void; + 'stream.append'?: (ctx: FrameCtx, params: Record) => void; + 'stream.read'?: (ctx: FrameCtx, params: Record) => void; +} + +export function startLoopback(path: string, handlers: LoopbackHandlers): Server { + const server = createServer((socket) => { + let buffer = ''; + const send = (obj: unknown): void => { + socket.write(JSON.stringify(obj) + '\n'); + }; + socket.on('data', (chunk) => { + buffer += chunk.toString('utf8'); + let nl: number; + while ((nl = buffer.indexOf('\n')) !== -1) { + const line = buffer.slice(0, nl); + buffer = buffer.slice(nl + 1); + if (line.length === 0) continue; + const req = JSON.parse(line) as { id: string; verb: keyof LoopbackHandlers; params: Record }; + const handler = handlers[req.verb] as ((ctx: FrameCtx, params: Record) => void) | undefined; + if (handler === undefined) { + send({ id: req.id, ok: false, error: { code: 'unknown_verb', message: req.verb } }); + } else { + handler({ id: req.id, socket, send }, req.params); + } + } + }); + }); + server.listen(path); + return server; +} + +// A faithful mini-mirror of `RunSpec::parse` (kernel/relayflowd-core/src/spec.rs): +// snake_case keys only, per-type step key sets, flat v0 verification. +export function kernelDialectError(spec: unknown): string | null { + if (typeof spec !== 'object' || spec === null) return 'spec: expected an object'; + const rootAllowed = new Set(['version', 'name', 'description', 'steps', 'budget']); + for (const key of Object.keys(spec)) { + if (!rootAllowed.has(key)) return `unknown field "${key}" at spec`; + } + const steps = (spec as { steps?: unknown }).steps; + if (!Array.isArray(steps)) return 'steps: expected an array'; + const common = ['id', 'type', 'depends_on', 'max_iterations', 'retry', 'verification']; + const byType: Record = { + deterministic: ['command', 'timeout_ms'], + llm: ['prompt', 'model'], + agent: ['instruction', 'recovery_mode', 'surfaces', 'permissions'], + }; + for (const [index, step] of steps.entries()) { + const value = step as Record; + const kindFields = byType[value['type'] as string]; + if (kindFields === undefined) return `steps[${index}]: unknown type`; + const allowed = new Set([...common, ...kindFields]); + for (const key of Object.keys(value)) { + if (!allowed.has(key)) return `unknown field "${key}" at steps[${index}]`; + } + if (value['verification'] !== undefined) { + const gateAllowed = new Set(['output_contains', 'json_schema']); + for (const key of Object.keys(value['verification'] as Record)) { + if (!gateAllowed.has(key)) return `unknown field "${key}" at steps[${index}].verification`; + } + } + } + return null; +} + +export function sendOk(ctx: FrameCtx): void { + sendResult(ctx, { protocol: PROTOCOL_VERSION, server: 'relayflowd-test' }); +} + +export function sendResult(ctx: FrameCtx, result: unknown): void { + ctx.send({ id: ctx.id, ok: true, result }); +} diff --git a/sdk/tests/journal-client.test.ts b/sdk/tests/journal-client.test.ts index 30d69da1..160d579d 100644 --- a/sdk/tests/journal-client.test.ts +++ b/sdk/tests/journal-client.test.ts @@ -1,13 +1,18 @@ import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; -import { createServer, type Server, type Socket } from 'node:net'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { randomUUID } from 'node:crypto'; +import type { Server } from 'node:net'; import { rmSync } from 'node:fs'; import { JournalClient } from '../src/journal-client.js'; import { compileYaml, toKernelSpec } from '../src/compile.js'; import { PROTOCOL_VERSION } from '../src/protocol.js'; -import type { FlowSpec, KernelRunSpec } from '../src/spec.js'; +import { + HELLO_SPEC, + kernelDialectError, + sendOk, + sendResult, + sockPath, + startLoopback, + type FrameCtx, +} from './journal-client-loopback.js'; // Protocol-v0 client tests over a real unix socket. The transport is real // (newline-delimited JSON frames over `node:net`), but the server side is a @@ -16,143 +21,8 @@ import type { FlowSpec, KernelRunSpec } from '../src/spec.js'; // and fail-closed behavior on errors and connection drops. Kernel semantics // are proven in `kernel/relayflowd/` (unit + crash-injection tests). -function sockPath(): string { - return join(tmpdir(), `rf-${randomUUID().slice(0, 8)}.sock`); -} - -const HELLO_FLOW: FlowSpec = { - version: '0.1.0', - name: 'client-roundtrip', - steps: [{ id: 'greet', type: 'deterministic', command: 'echo hi' }], -}; -const HELLO_SPEC: KernelRunSpec = toKernelSpec(HELLO_FLOW); - -interface FrameCtx { - id: string; - socket: Socket; - send: (obj: unknown) => void; -} - -function startLoopback(path: string, handlers: { - hello?: (ctx: FrameCtx) => void; - 'run.start'?: (ctx: FrameCtx, params: Record) => void; - 'run.resume'?: (ctx: FrameCtx, params: Record) => void; - 'run.get'?: (ctx: FrameCtx, params: Record) => void; - 'run.watch'?: (ctx: FrameCtx, params: Record) => void; - 'worker.attach'?: (ctx: FrameCtx, params: Record) => void; - 'step.heartbeat'?: (ctx: FrameCtx, params: Record) => void; - 'effect.record'?: (ctx: FrameCtx, params: Record) => void; - 'effect.confirm'?: (ctx: FrameCtx, params: Record) => void; - 'step.complete'?: (ctx: FrameCtx, params: Record) => void; - 'event.emit'?: (ctx: FrameCtx, params: Record) => void; - 'journal.read'?: (ctx: FrameCtx, params: Record) => void; - 'stream.append'?: (ctx: FrameCtx, params: Record) => void; - 'stream.read'?: (ctx: FrameCtx, params: Record) => void; -}): Server { - const server = createServer((socket) => { - let buffer = ''; - const send = (obj: unknown): void => { - socket.write(JSON.stringify(obj) + '\n'); - }; - socket.on('data', (chunk) => { - buffer += chunk.toString('utf8'); - let nl: number; - while ((nl = buffer.indexOf('\n')) !== -1) { - const line = buffer.slice(0, nl); - buffer = buffer.slice(nl + 1); - if (line.length === 0) continue; - const req = JSON.parse(line) as { id: string; verb: string; params: Record }; - const ctx: FrameCtx = { id: req.id, socket, send }; - switch (req.verb) { - case 'hello': - handlers.hello?.(ctx); - break; - case 'run.start': - handlers['run.start']?.(ctx, req.params); - break; - case 'run.resume': - handlers['run.resume']?.(ctx, req.params); - break; - case 'run.get': - handlers['run.get']?.(ctx, req.params); - break; - case 'run.watch': - handlers['run.watch']?.(ctx, req.params); - break; - case 'worker.attach': - handlers['worker.attach']?.(ctx, req.params); - break; - case 'step.heartbeat': - handlers['step.heartbeat']?.(ctx, req.params); - break; - case 'effect.record': - handlers['effect.record']?.(ctx, req.params); - break; - case 'effect.confirm': - handlers['effect.confirm']?.(ctx, req.params); - break; - case 'step.complete': - handlers['step.complete']?.(ctx, req.params); - break; - case 'event.emit': - handlers['event.emit']?.(ctx, req.params); - break; - case 'journal.read': - handlers['journal.read']?.(ctx, req.params); - break; - case 'stream.append': - handlers['stream.append']?.(ctx, req.params); - break; - case 'stream.read': - handlers['stream.read']?.(ctx, req.params); - break; - default: - send({ id: req.id, ok: false, error: { code: 'unknown_verb', message: req.verb } }); - } - } - }); - }); - server.listen(path); - return server; -} - let lastStartedSpec: Record | null = null; -// A faithful mini-mirror of `RunSpec::parse` (kernel/relayflowd-core/src/spec.rs): -// snake_case keys only, per-type step key sets, flat v0 verification. Returns -// an error message, or null when the spec is in the kernel dialect. -function kernelDialectError(spec: unknown): string | null { - if (typeof spec !== 'object' || spec === null) return 'spec: expected an object'; - const rootAllowed = new Set(['version', 'name', 'description', 'steps', 'budget']); - for (const key of Object.keys(spec)) { - if (!rootAllowed.has(key)) return `unknown field "${key}" at spec`; - } - const steps = (spec as { steps?: unknown }).steps; - if (!Array.isArray(steps)) return 'steps: expected an array'; - const common = ['id', 'type', 'depends_on', 'max_iterations', 'retry', 'verification']; - const byType: Record = { - deterministic: ['command', 'timeout_ms'], - llm: ['prompt', 'model'], - agent: ['instruction', 'recovery_mode', 'surfaces', 'permissions'], - }; - for (const [index, step] of steps.entries()) { - const st = step as Record; - const kindFields = byType[st.type as string]; - if (kindFields === undefined) return `steps[${index}]: unknown type`; - const allowed = new Set([...common, ...kindFields]); - for (const key of Object.keys(st)) { - if (!allowed.has(key)) return `unknown field "${key}" at steps[${index}]`; - } - if (st.verification !== undefined) { - const gateAllowed = new Set(['output_contains', 'json_schema']); - for (const key of Object.keys(st.verification as Record)) { - if (!gateAllowed.has(key)) return `unknown field "${key}" at steps[${index}].verification`; - } - } - } - return null; -} - describe('JournalClient: protocol v0 over unix socket', () => { let path: string; let server: Server; @@ -516,13 +386,3 @@ describe('JournalClient: an effect election is atomic with its provider call', ( } }); }); - -// --- helpers --- - -function sendOk(ctx: FrameCtx): void { - ctx.send({ id: ctx.id, ok: true, result: { protocol: PROTOCOL_VERSION, server: 'relayflowd-test' } }); -} - -function sendResult(ctx: FrameCtx, result: unknown): void { - ctx.send({ id: ctx.id, ok: true, result }); -} diff --git a/sdk/tests/live-kernel.test.ts b/sdk/tests/live-kernel.test.ts new file mode 100644 index 00000000..f8bb262c --- /dev/null +++ b/sdk/tests/live-kernel.test.ts @@ -0,0 +1,393 @@ +import { + accessSync, + constants, + existsSync, + lstatSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { afterEach, beforeAll, describe, expect, it } from 'vitest'; +import { compileYaml, compileYamlToCanonicalJson, toKernelSpec } from '../src/compile.js'; +import { JournalClient } from '../src/journal-client.js'; +import type { StepDispatchEvent } from '../src/protocol.js'; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..'); +const SDK = join(ROOT, 'sdk'); +const BUILT_CLI = join(SDK, 'dist', 'cli.js'); +const TESTDATA = join(ROOT, 'testdata'); +const RELAYFLOWD = resolve( + process.env['RELAYFLOWD_BIN'] ?? join(ROOT, 'kernel', 'target', 'debug', 'relayflowd'), +); +const temporaryDirectories: string[] = []; +const daemons: ChildProcess[] = []; +const clients: JournalClient[] = []; + +beforeAll(() => { + requireExecutable(RELAYFLOWD, 'RELAYFLOWD_BIN', '(cd kernel && ../ops/cargo.sh build)'); + requireExecutable(BUILT_CLI, 'built flows CLI', '(cd sdk && npm run build)'); + console.log(`LIVE_KERNEL relayflowd=${RELAYFLOWD}`); + console.log(`LIVE_KERNEL flows=${BUILT_CLI}`); +}); + +afterEach(async () => { + for (const client of clients.splice(0)) client.close(); + for (const daemon of daemons.splice(0)) await stopDaemon(daemon); + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +describe('built flows CLI against live relayflowd', () => { + it('runs rung (a), parks rung (b), and keeps JSON report-shaped', async () => { + const dataDir = temporaryDirectory('flows-live-cli-'); + await startDaemon(dataDir); + + const deterministic = invokeCli([ + 'run', '--data-dir', dataDir, join(TESTDATA, 'hello-deterministic.flow.yaml'), + ]); + expect(deterministic.status, deterministic.stderr).toBe(0); + expect(deterministic.stdout).toMatch(/RUN [0-9A-Z]{26}/); + expect(deterministic.stdout).toContain('completionReason: success'); + + const parked = invokeCli([ + 'run', '--data-dir', dataDir, join(TESTDATA, 'hello-llm.flow.yaml'), + ]); + expect(parked.status, parked.stderr).toBe(3); + expect(parked.stderr).toContain('PARKED [run_parked]'); + expect(parked.stderr).toContain('step "answer" (llm)'); + expect(parked.stderr).toContain('no worker is attached'); + + const agentParked = invokeCli([ + 'run', '--data-dir', dataDir, join(TESTDATA, 'hello-agent.flow.yaml'), + ]); + expect(agentParked.status, agentParked.stderr).toBe(3); + expect(agentParked.stderr).toContain('PARKED [run_parked]'); + expect(agentParked.stderr).toContain('step "edit" (agent)'); + + const failedFlow = join(dataDir, 'failed.flow.yaml'); + writeFileSync(failedFlow, ` +version: '0.1.0' +steps: + - id: fail + type: deterministic + command: "exit 7" +`); + const failed = invokeCli(['run', '--data-dir', dataDir, failedFlow]); + expect(failed.status).toBe(1); + expect(failed.stderr).toContain('FAILED [step_failed]'); + expect(failed.stdout).toContain('completionReason: step_failed'); + const failedRunId = failed.stdout.match(/RUN ([0-9A-Z]{26})/)?.[1]; + expect(failedRunId).toBeDefined(); + const failedClient = await connectClient(dataDir); + const failedJournal = (await failedClient.journalRead(failedRunId!, 1)).entries; + const terminal = failedJournal.find((entry) => journalType(entry) === 'run.completed'); + expect(terminal).toMatchObject({ + entry_type: 'run.completed', + payload: { completionReason: 'step_failed', failed_step_id: 'fail' }, + }); + + const json = invokeCli([ + 'run', '--json', '--data-dir', dataDir, join(TESTDATA, 'hello-deterministic.flow.yaml'), + ]); + expect(json.status, json.stderr).toBe(0); + expect(JSON.parse(json.stdout)).toMatchObject({ + ok: true, + command: 'run', + status: 'completed', + completionReason: 'success', + }); + + const invalid = invokeCli(['resume', '--json']); + expect(invalid.status).toBe(2); + expect(JSON.parse(invalid.stdout)).toMatchObject({ + ok: false, + diagnostics: [{ kind: 'invalid_invocation' }], + }); + }); + + it('preflights before journaling and names an unreachable socket', async () => { + const dataDir = temporaryDirectory('flows-live-preflight-'); + await startDaemon(dataDir); + const before = runArtifacts(dataDir); + + const refused = invokeCli([ + 'run', '--data-dir', dataDir, join(TESTDATA, 'preflight', 'cli-missing.flow.yaml'), + ]); + expect(refused.status).toBe(2); + expect(refused.stderr).toContain('REFUSED [cli_missing]'); + expect(runArtifacts(dataDir)).toEqual(before); + + const absentDir = temporaryDirectory('flows-live-absent-'); + const absentSocket = join(absentDir, 'relayflowd.sock'); + const unreachable = invokeCli([ + 'run', '--data-dir', absentDir, join(TESTDATA, 'hello-deterministic.flow.yaml'), + ]); + expect(unreachable.status).toBe(2); + expect(unreachable.stderr).toContain('REFUSED [daemon_unreachable]'); + expect(unreachable.stderr).toContain(absentSocket); + expect(unreachable.stderr).toContain('relayflowd --data-dir'); + expect(runArtifacts(absentDir)).toEqual([]); + }); +}); + +describe('JournalClient wire conformance against live relayflowd', () => { + it('exercises every protocol-v0 verb with the real server', async () => { + const dataDir = temporaryDirectory('flows-live-wire-'); + await startDaemon(dataDir); + const client = await connectClient(dataDir); + + expect(await client.hello('live-conformance')).toEqual({ protocol: 0, server: 'relayflowd' }); + const deterministic = await client.runStart(toKernelSpec(compileYaml(` +version: '0.1.0' +steps: + - id: hello + type: deterministic + command: printf hello +`))); + expect(deterministic.status).toBe('completed'); + expect(deterministic.completion_reason).toBe('success'); + expect((await client.runResume(deterministic.run_id)).completion_reason).toBe('success'); + expect((await client.runGet(deterministic.run_id)).status).toBe('completed'); + + const replayed = eventOnce>(client, 'entry'); + expect(await client.runWatch(deterministic.run_id)).toEqual({ watching: deterministic.run_id }); + expect((await replayed)['entry_type']).toBe('run.spawned'); + const journal = await client.journalRead(deterministic.run_id, 1); + expect(journal.entries.some((entry) => journalType(entry) === 'run.completed')).toBe(true); + expect((await client.eventEmit(deterministic.run_id, 'unmatched', { ok: true })).matched).toBe(0); + expect((await client.streamAppend(deterministic.run_id, 'results', { answer: 4 })).offset).toBe(0); + expect(await client.streamRead(deterministic.run_id, 'results', 0, 10)).toEqual({ + messages: [{ answer: 4 }], + next_offset: 1, + }); + + const llmDispatch = eventOnce(client, 'step.dispatch'); + expect(await client.workerAttach('live-llm', ['llm'])).toEqual({ worker_id: 'live-llm' }); + const llmStart = client.runStart(toKernelSpec(compileYaml(` +version: '0.1.0' +steps: + - id: answer + type: llm + prompt: Return four. +`))); + const llmLease = await llmDispatch; + const llmParked = await llmStart; + expect(llmLease.step_type).toBe('llm'); + expect((await client.stepHeartbeat( + llmLease.run_id, + llmLease.step_id, + llmLease.attempt, + llmLease.lease_id, + )).lease_deadline_ms).toBeGreaterThan(Date.now()); + const llmDone = await client.stepComplete( + llmLease.run_id, + llmLease.step_id, + llmLease.attempt, + llmLease.idempotency_key, + 'success', + { output: { answer: 4 }, usage: { tokens_in: 2, tokens_out: 1, dollars: '0.001' } }, + ); + expect(llmParked.status).toBe('parked'); + expect(llmDone.completion_reason).toBe('success'); + + const initialPins = { + workspace: [{ surface: 'repo', revision_id: 'rev-a' }], + streams: [], + }; + const agentDispatch = eventOnce(client, 'step.dispatch'); + expect(await client.workerAttach('live-agent', ['agent'], initialPins)).toEqual({ worker_id: 'live-agent' }); + const agentStart = client.runStart(toKernelSpec(compileYaml(` +version: '0.1.0' +steps: + - id: edit + type: agent + instruction: Edit the repository. + surfaces: + workspace: + - surface: repo + external: + - provider://item +`))); + const agentLease = await agentDispatch; + await agentStart; + let providerCalls = 0; + expect(await client.performEffect({ + runId: agentLease.run_id, + stepId: agentLease.step_id, + attempt: agentLease.attempt, + idempotencyKey: agentLease.idempotency_key, + surfacePath: 'provider://item', + revisionBefore: 'rev-a', + revisionAfter: 'rev-b', + }, async () => { providerCalls += 1; })).toBe(true); + expect(providerCalls).toBe(1); + const agentDone = await client.stepComplete( + agentLease.run_id, + agentLease.step_id, + agentLease.attempt, + agentLease.idempotency_key, + 'success', + { + output: { changed: true }, + started_pins: agentLease.pins, + end_pins: { workspace: [{ surface: 'repo', revision_id: 'rev-b' }], streams: [] }, + effects: [{ surface_path: 'provider://item', idempotency_key: agentLease.idempotency_key }], + }, + ); + expect(agentDone.completion_reason).toBe('success'); + }); +}); + +describe('surface resume after a real daemon kill', () => { + it('resumes a three-step run with each successful completion exactly once', async () => { + const directory = temporaryDirectory('flows-live-resume-'); + const dataDir = join(directory, 'data'); + const yaml = ` +version: '0.1.0' +name: live-crash-resume +steps: + - id: one + type: deterministic + command: printf one + - id: two + type: deterministic + dependsOn: [one] + command: printf two + - id: three + type: deterministic + dependsOn: [two] + command: printf three +`; + const specPath = join(directory, 'three-step.spec.json'); + writeFileSync(specPath, compileYamlToCanonicalJson(yaml)); + const interrupted = spawnSync(RELAYFLOWD, [ + '--data-dir', dataDir, 'run', specPath, '--stop-after', '1', + ], { encoding: 'utf8' }); + expect(interrupted.status, interrupted.stderr).toBe(0); + const initial = JSON.parse(interrupted.stdout) as { run_id: string; status: string }; + expect(initial.status).toBe('interrupted'); + + const firstDaemon = await startDaemon(dataDir); + const beforeClient = await connectClient(dataDir); + const before = (await beforeClient.journalRead(initial.run_id, 1)).entries; + expect(successfulCompletions(before)).toEqual({ one: 1 }); + beforeClient.close(); + clients.splice(clients.indexOf(beforeClient), 1); + + console.log(`LIVE_KERNEL kill -9 pid=${firstDaemon.pid} run=${initial.run_id}`); + await stopDaemon(firstDaemon, 'SIGKILL'); + daemons.splice(daemons.indexOf(firstDaemon), 1); + await startDaemon(dataDir); + + const resumed = invokeCli([ + 'resume', '--json', '--data-dir', dataDir, initial.run_id, + ]); + expect(resumed.status, resumed.stderr).toBe(0); + expect(JSON.parse(resumed.stdout)).toMatchObject({ + ok: true, + command: 'resume', + runId: initial.run_id, + status: 'completed', + completionReason: 'success', + }); + + const afterClient = await connectClient(dataDir); + const after = (await afterClient.journalRead(initial.run_id, 1)).entries; + expect(successfulCompletions(after)).toEqual({ one: 1, two: 1, three: 1 }); + }); +}); + +function requireExecutable(path: string, source: string, buildCommand: string): void { + try { + accessSync(path, constants.X_OK); + } catch { + throw new Error( + `LIVE_KERNEL_MISSING: ${source} does not name an executable file: ${path}. Build it with: ${buildCommand}`, + ); + } +} + +function temporaryDirectory(prefix: string): string { + const directory = mkdtempSync(join(tmpdir(), prefix)); + temporaryDirectories.push(directory); + return directory; +} + +async function startDaemon(dataDir: string): Promise { + const daemon = spawn(RELAYFLOWD, ['--data-dir', dataDir, 'serve'], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + daemons.push(daemon); + const stderr: Buffer[] = []; + daemon.stderr?.on('data', (chunk: Buffer) => stderr.push(chunk)); + const socket = join(dataDir, 'relayflowd.sock'); + const deadline = Date.now() + 5_000; + while (Date.now() < deadline) { + if (existsSync(socket) && lstatSync(socket).isSocket()) return daemon; + if (daemon.exitCode !== null || daemon.signalCode !== null) { + throw new Error(`relayflowd exited before binding ${socket}: ${Buffer.concat(stderr).toString('utf8')}`); + } + await delay(20); + } + throw new Error(`relayflowd did not bind ${socket} within 5000ms`); +} + +async function stopDaemon(daemon: ChildProcess, signal: NodeJS.Signals = 'SIGTERM'): Promise { + if (daemon.exitCode !== null || daemon.signalCode !== null) return; + const exited = new Promise((resolveExit) => daemon.once('exit', () => resolveExit())); + daemon.kill(signal); + await exited; +} + +async function connectClient(dataDir: string): Promise { + const client = new JournalClient(join(dataDir, 'relayflowd.sock'), { requestTimeoutMs: 5_000 }); + clients.push(client); + await client.connect(); + return client; +} + +function invokeCli(args: string[]) { + return spawnSync(process.execPath, [BUILT_CLI, ...args], { + cwd: ROOT, + encoding: 'utf8', + }); +} + +function runArtifacts(dataDir: string): string[] { + return readdirSync(dataDir).filter((name) => name !== 'relayflowd.sock').sort(); +} + +function eventOnce(client: JournalClient, event: string): Promise { + return new Promise((resolveEvent) => client.once(event, (value) => resolveEvent(value as T))); +} + +function journalType(entry: unknown): string | undefined { + return isObject(entry) && typeof entry['entry_type'] === 'string' ? entry['entry_type'] : undefined; +} + +function successfulCompletions(entries: unknown[]): Record { + const counts: Record = {}; + for (const entry of entries) { + if (!isObject(entry) || entry['entry_type'] !== 'step.completed') continue; + const payload = entry['payload']; + const stepId = entry['step_id']; + if (!isObject(payload) || payload['completionReason'] !== 'success' || typeof stepId !== 'string') continue; + counts[stepId] = (counts[stepId] ?? 0) + 1; + } + return counts; +} + +function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function delay(ms: number): Promise { + return new Promise((resolveDelay) => setTimeout(resolveDelay, ms)); +} diff --git a/sdk/tests/spec-parity.test.ts b/sdk/tests/spec-parity.test.ts index 9caad3c7..c7403f1d 100644 --- a/sdk/tests/spec-parity.test.ts +++ b/sdk/tests/spec-parity.test.ts @@ -25,7 +25,7 @@ function fixture(name: string): string { } describe('spec parity: one dialect at the SDK<->kernel boundary', () => { - for (const name of ['hello-ladder', 'hello-llm', 'hello-agent']) { + for (const name of ['hello-deterministic', 'hello-ladder', 'hello-llm', 'hello-agent']) { it(`compiles ${name} to the pinned canonical JSON`, () => { const yaml = fixture(`${name}.flow.yaml`); const canonical = compileYamlToCanonicalJson(yaml); diff --git a/testdata/hello-deterministic.flow.yaml b/testdata/hello-deterministic.flow.yaml new file mode 100644 index 00000000..a33bd360 --- /dev/null +++ b/testdata/hello-deterministic.flow.yaml @@ -0,0 +1,18 @@ +# Gate-1 ladder rung (a): a pure deterministic flow with zero agents. +version: '0.1.0' +name: hello-deterministic +description: The canonical hello ladder rung (a) — zero agents, legal. +steps: + - id: greet + type: deterministic + command: "echo hello" + verification: + type: output_contains + value: "hello" + - id: shout + type: deterministic + dependsOn: [greet] + command: "echo HELLO" + verification: + type: output_contains + value: "HELLO" diff --git a/testdata/hello-deterministic.spec.canonical.json b/testdata/hello-deterministic.spec.canonical.json new file mode 100644 index 00000000..d594b3d2 --- /dev/null +++ b/testdata/hello-deterministic.spec.canonical.json @@ -0,0 +1 @@ +{"description":"The canonical hello ladder rung (a) — zero agents, legal.","name":"hello-deterministic","steps":[{"command":"echo hello","depends_on":[],"id":"greet","max_iterations":1,"retry":{"initial_backoff_ms":100,"jitter_percent":20,"max_backoff_ms":60000,"multiplier":2},"type":"deterministic","verification":{"output_contains":"hello"}},{"command":"echo HELLO","depends_on":["greet"],"id":"shout","max_iterations":1,"retry":{"initial_backoff_ms":100,"jitter_percent":20,"max_backoff_ms":60000,"multiplier":2},"type":"deterministic","verification":{"output_contains":"HELLO"}}],"version":"0.1.0"} diff --git a/testdata/hello-deterministic.spec.sha256 b/testdata/hello-deterministic.spec.sha256 new file mode 100644 index 00000000..ffeee9ba --- /dev/null +++ b/testdata/hello-deterministic.spec.sha256 @@ -0,0 +1 @@ +e77b0920a08ee65ea632f54f3f477f14698781d2670634c89bb408f8cd727117 From 0fd332ca99b481edb51c55d52e582b427e3cf516 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Fri, 28 Aug 2026 01:18:32 -0400 Subject: [PATCH 2/4] Record WP-10 verification evidence --- ops/DRIVE-LOG.md | 291 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 291 insertions(+) diff --git a/ops/DRIVE-LOG.md b/ops/DRIVE-LOG.md index 2e498306..c701afb6 100644 --- a/ops/DRIVE-LOG.md +++ b/ops/DRIVE-LOG.md @@ -1986,3 +1986,294 @@ $ find kernel -name '*.rs' -not -path '*/target/*' | xargs wc -l | sort -nr | he PR #8 remains open. Gate 1 remains AMBER until a human merges it and re-verifies merged `main`; the Lead does not merge. + +### WP-10 — `flows run` / `flows resume` cross the live authored-surface seam + +This 2026-08-28 tick finished the six unfinished paths named by +`ops/NEXT.md`. Commit `5b1226e` makes the shipped CLI dispatch `check`, `run`, +and `resume`; declares the run-surface outcome taxonomy; pins the standalone +deterministic fixture on the SDK and kernel sides; and exercises the built CLI +against a live `relayflowd`. The live cases prove deterministic success, +typed parking for both `llm` and `agent`, a failed journal terminal carrying +`completionReason: step_failed`, preflight before journaling, an unreachable +daemon creating no run artifact, and one successful completion per step after +kill-and-resume. + +Veto MCP was not exposed in this non-interactive subprocess, so no Veto +review or scan is claimed. Local diff checks and every package DoD command ran +instead. The exact DoD below ran from clean implementation commit `5b1226e`. +The initial and post-verification status command produced zero bytes: + +```text +$ git status --porcelain +``` + +The kernel workspace passed 73 tests (18 + 19 + 26 + 4 + 6), including the +new deterministic parity case. Literal output: + +```text +$ (cd kernel && ../ops/cargo.sh test --workspace) + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.22s + Running unittests src/lib.rs (target/debug/deps/relayflowd-399037c915557fdb) + +running 18 tests +test server::client::tests::resume_waits_while_the_heartbeat_renewed_lease_is_live ... ok +test server::tests::agent::contract::an_agent_worker_attaching_without_pins_is_refused_at_attach ... ok +test exec_det::tests::captures_deterministic_output ... ok +test exec_det::tests::timeout_has_an_explicit_completion_reason ... ok +test server::tests::agent::contract::an_oversized_trajectory_tail_is_refused_at_step_complete ... ok +test server::tests::agent::contract::agent_without_a_compatible_worker_parks_without_starting ... ok +test server::tests::agent::contract::an_agent_worker_missing_a_declared_surface_parks_the_run_instead_of_erroring ... ok +test server::tests::agent::contract::an_llm_completion_claiming_an_effect_fails_closed_with_the_reason_journaled ... ok +test server::tests::hello_enforces_protocol_version ... ok +test server::tests::run_start_fails_closed_on_an_unknown_verification_key ... ok +test server::tests::agent::contract::a_replacement_worker_that_never_reported_the_pinned_surface_is_not_dispatched_to ... ok +test server::tests::a_failed_disconnect_journal_append_is_retained_and_retried_not_dropped ... ok +test server::tests::agent::pins::consecutive_agent_steps_on_different_surfaces_each_start_from_their_own_pins ... ok +test server::tests::agent::pins::reset_worker_reporting_a_revision_other_than_its_pin_fails_closed_as_worker_error ... ok +test server::tests::stopped_heartbeats_past_the_deadline_journal_lease_expired_and_release_the_step ... ok +test exec_det::tests::timeout_kills_the_whole_process_group ... ok +test server::tests::agent::pins::a_replacement_worker_at_a_different_revision_is_not_dispatched_the_stale_pins ... ok +test server::tests::an_entry_appended_during_watch_registration_is_delivered_exactly_once ... ok + +test result: ok. 18 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.56s + + Running unittests src/main.rs (target/debug/deps/relayflowd-9e21fa47745f4fb0) + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/crash_resume.rs (target/debug/deps/crash_resume-e6635a3f0d48512c) + +running 19 tests +test agent::resume_without_a_worker_parks_immediately_instead_of_timing_out ... ok +test agent::rung_c_sigkill_after_final_effect_replays_results_without_redispatch ... ok +test concurrency::live_resume_leaves_an_active_lease_running ... ok +test concurrency::concurrent_resumes_lease_exactly_one_attempt ... ok +test agent::rung_c_reset_sigkill_mid_edit_restores_pins_dedupes_effect_and_explains_attempts ... ok +test agent::rung_c_crash_between_effect_election_and_the_provider_call_performs_it_exactly_once ... ok +test llm::serve_plumbs_watch_events_and_replayable_stream_verbs ... ok +test llm::failing_llm_verification_schedules_a_durable_retry_and_succeeds ... ok +test llm::llm_verification_exhaustion_is_a_declared_failure_kind ... ok +test agent::rung_c_sigkill_between_agent_completion_and_final_effect_memoizes_the_agent ... ok +test agent::rung_c_sigkill_boundaries_resume_only_unfinished_steps_via_real_cli ... ok +test llm::sigkill_after_the_final_rung_b_effect_resumes_without_redispatching_llm ... ok +test llm::completed_llm_output_is_memoized_when_serve_dies_during_the_next_step ... ok +test llm::worker_killed_while_holding_a_lease_is_explained_and_released_on_cli_resume ... ok +test sigkill_mid_step_replaces_and_explains_the_dead_attempt ... ok +test llm::sigkill_under_serve_mid_llm_releases_the_lease_and_finishes_via_cli_resume ... ok +test sigkill_sweep_covers_every_hello_step_boundary ... ok +test sigkill_under_serve_resumes_the_socket_started_run ... ok +test llm::sigkill_sweep_covers_before_and_between_the_rung_b_steps ... ok + +test result: ok. 19 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.83s + + Running unittests src/lib.rs (target/debug/deps/relayflowd_core-b1fe3b3250e9e7a2) + +running 26 tests +test clock::tests::simulated_clock_is_explicitly_advanced ... ok +test journal::tests::memory_journal_assigns_sequences_and_rolls_epochs ... ok +test machine::tests::machine_starts_runnable_step_with_stable_effect_key ... ok +test machine::tests::all_backing_off_steps_return_timers ... ok +test machine::tests::successful_memo_is_never_scheduled_again ... ok +test machine::tests::crashed_attempt_does_not_consume_an_iteration ... ok +test machine::tests::manual_recovery_parks_needs_human_and_never_redispatches ... ok +test retry::tests::jitter_is_repeatable_and_bounded ... ok +test machine::tests::verification_failure_schedules_a_durable_retry ... ok +test spec::tests::a_misspelled_step_level_key_is_a_parse_error ... ok +test machine::tests::reset_recovery_dispatches_the_original_pinned_revision ... ok +test spec::tests::a_misspelled_verification_gate_key_is_a_parse_error_not_a_dropped_gate ... ok +test machine::tests::inspect_recovery_injects_the_dirty_pin_completion_reason_and_tail ... ok +test spec::tests::cycles_are_rejected ... ok +test spec::tests::zero_agent_flow_is_valid ... ok +test spec::tests::unknown_root_and_nested_fields_are_rejected ... ok +test machine::tests::every_failed_run_terminates_with_declared_completion_reasons ... ok +test state::tests::budget_decimal_strings_add_without_floats ... ok +test spec::tests::the_full_ladder_parses_in_the_one_dialect ... ok +test state::tests::completed_output_is_memoized_and_unlocks_dependents ... ok +test verify::tests::deterministic_output_requires_successful_exit_and_content ... ok +test spec::tests::spec_version_is_semver_and_gated ... ok +test spec::tests::preflight_data_is_fail_closed ... ok +test state::tests::end_pin_chain_is_enforced_and_a_broken_chain_is_a_hard_error ... ok +test state::tests::a_completion_that_omits_a_surface_does_not_drop_it_from_the_pin_chain ... ok +test verify::tests::json_schema_is_a_control_gate ... ok + +test result: ok. 26 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running tests/spec_parity.rs (target/debug/deps/spec_parity-bbda6cf1e1cf1c19) + +running 4 tests +test the_kernel_parses_the_deterministic_rung_and_stamps_the_same_hash ... ok +test the_kernel_parses_the_rung_b_spec_and_stamps_the_same_hash ... ok +test the_kernel_parses_the_rung_c_agent_spec_and_stamps_the_same_hash ... ok +test the_kernel_parses_the_sdk_compiled_spec_and_stamps_the_same_hash ... ok + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running unittests src/lib.rs (target/debug/deps/relayflowd_journal-d13cb7954335385c) + +running 6 tests +test registry::tests::registry_is_a_rebuildable_run_locator ... ok +test tests::failed_commit_is_returned_not_swallowed ... ok +test tests::rollover_is_atomic_scaffolding_for_epoch_resume ... ok +test tests::append_is_durable_and_monotonic_after_reopen ... ok +test tests::effects_are_deduplicated_at_the_journal_boundary ... ok +test tests::an_unconfirmed_election_is_reclaimed_by_the_next_attempt_not_treated_as_done ... ok + +test result: ok. 6 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s + + Doc-tests relayflowd + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Doc-tests relayflowd_core + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Doc-tests relayflowd_journal + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s +``` + +Clippy passed with warnings denied: + +```text +$ (cd kernel && ../ops/cargo.sh clippy --workspace -- -D warnings) + Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.20s +``` + +Rust formatting exited 0 with zero output bytes: + +```text +$ (cd kernel && ../ops/cargo.sh fmt --check) +``` + +The kernel build produced an executable daemon: + +```text +$ (cd kernel && ../ops/cargo.sh build && test -x target/debug/relayflowd && ls -l target/debug/relayflowd) + Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.12s +-rwxr-xr-x 1 khaliqgant staff 22712808 Aug 28 00:32 target/debug/relayflowd +``` + +The clean install, build, and full SDK suite passed 143 tests with no skipped +tests. `bin.test.ts` ran all seven cases and `live-kernel.test.ts` ran all four: + +```text +$ (cd sdk && npm ci && npm run build && npm test) + +added 48 packages, and audited 49 packages in 653ms + +13 packages are looking for funding + run `npm fund` for details + +5 vulnerabilities (3 moderate, 1 high, 1 critical) + +To address all issues (including breaking changes), run: + npm audit fix --force + +Run `npm audit` for details. + +> @relayflows/sdk@0.1.0 build +> tsc && node scripts/make-cli-executable.mjs + + +> @relayflows/sdk@0.1.0 test +> tsc --noEmit && vitest run + + + RUN v2.1.9 /Users/khaliqgant/Projects/AgentWorkforce/flows/sdk + + ✓ tests/preflight.test.ts (12 tests) 11ms +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/Users/khaliqgant/Projects/AgentWorkforce/flows/kernel/target/debug/relayflowd +LIVE_KERNEL flows=/Users/khaliqgant/Projects/AgentWorkforce/flows/sdk/dist/cli.js + + ✓ tests/validate.test.ts (36 tests) 14ms + ✓ tests/deterministic-llm.test.ts (5 tests) 17ms + ✓ tests/journal-client.test.ts (12 tests) 85ms + ✓ tests/hello-deterministic.test.ts (5 tests) 21ms + ✓ tests/spec-parity.test.ts (15 tests) 58ms + ✓ tests/cli.test.ts (47 tests) 979ms +stdout | tests/live-kernel.test.ts > surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once +LIVE_KERNEL kill -9 pid=6381 run=01M13CPZ4HEJA91X7MGFPCB3QA + + ✓ tests/live-kernel.test.ts (4 tests) 1518ms + ✓ built flows CLI against live relayflowd > runs rung (a), parks rung (b), and keeps JSON report-shaped 1199ms + ✓ tests/bin.test.ts (7 tests) 1599ms + + Test Files 9 passed (9) + Tests 143 passed (143) + Start at 01:15:34 + Duration 1.94s (transform 384ms, setup 0ms, collect 1.06s, tests 4.30s, environment 1ms, prepare 670ms) +``` + +The behavioral proof used only the built CLI and live daemon. The failing +flow was supplied through `/dev/stdin`; no fixture outside the package was +added. The journal reads used the shipped SDK client against the same live +daemon: + +```text +BEHAVIOR_TMP=/tmp/wp10-clean-behavior.yBCapH +$ node sdk/dist/cli.js run --data-dir /tmp/wp10-clean-behavior.yBCapH/main-data testdata/hello-deterministic.flow.yaml +WARNING [unprovable_effects] Step "greet" command "echo" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "shout" command "echo" resolves, but its effects cannot be proven before execution. +RUN 01M13CQY51MEG1ZBY0YRN3PG5T completed (2 steps) completionReason: success +exit=0 +$ node sdk/dist/cli.js run --data-dir /tmp/wp10-clean-behavior.yBCapH/main-data testdata/hello-llm.flow.yaml +WARNING [unprovable_effects] Step "greet" command "printf" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "finish" command "printf" resolves, but its effects cannot be proven before execution. +PARKED [run_parked] Run "01M13CQY7C506T64WDQJ2YSEGM" parked at step "answer" (llm): no worker is attached for step type "llm". +RUN 01M13CQY7C506T64WDQJ2YSEGM parked (1 steps) +exit=3 +$ node sdk/dist/cli.js run --data-dir /tmp/wp10-clean-behavior.yBCapH/main-data testdata/hello-agent.flow.yaml +WARNING [unprovable_effects] Step "greet" command "printf" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "finish" command "printf" resolves, but its effects cannot be proven before execution. +PARKED [run_parked] Run "01M13CQY9RMSX0X5C5QYB9CKPT" parked at step "edit" (agent): no worker is attached for step type "agent". +RUN 01M13CQY9RMSX0X5C5QYB9CKPT parked (1 steps) +exit=3 +$ node sdk/dist/cli.js run --data-dir /tmp/wp10-clean-behavior.yBCapH/main-data /dev/stdin <<< +WARNING [command_unresolved] Step "fail" command "exit" does not resolve as an executable; it runs only if the shell supplies it. +FAILED [step_failed] Run "01M13CQYBT7Z2K97RA5DQ9A1QQ" failed with completionReason: step_failed. +RUN 01M13CQYBT7Z2K97RA5DQ9A1QQ failed (1 steps) completionReason: step_failed +exit=1 +$ JournalClient.journalRead(01M13CQYBT7Z2K97RA5DQ9A1QQ) terminal completionReason +{"entry_type":"run.completed","completionReason":"step_failed","failed_step_id":"fail"} +$ node sdk/dist/cli.js run --data-dir /tmp/wp10-clean-behavior.yBCapH/absent-data testdata/hello-deterministic.flow.yaml +WARNING [unprovable_effects] Step "greet" command "echo" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "shout" command "echo" resolves, but its effects cannot be proven before execution. +REFUSED [daemon_unreachable] No compatible relayflowd is listening at "/tmp/wp10-clean-behavior.yBCapH/absent-data/relayflowd.sock". Start it with: relayflowd --data-dir "/tmp/wp10-clean-behavior.yBCapH/absent-data" serve +exit=2 +journal_files=0 +$ kernel/target/debug/relayflowd --data-dir /tmp/wp10-clean-behavior.yBCapH/resume-data run testdata/hello-deterministic.spec.canonical.json --stop-after 1 +{"run_id":"01M13CQYFDD2SX79JH85178HVX","status":"interrupted","completion_reason":null,"completed_steps":1} +$ kill -9 7076 # relayflowd, run 01M13CQYFDD2SX79JH85178HVX interrupted after one step +$ node sdk/dist/cli.js resume --data-dir /tmp/wp10-clean-behavior.yBCapH/resume-data 01M13CQYFDD2SX79JH85178HVX +RUN 01M13CQYFDD2SX79JH85178HVX completed (2 steps) completionReason: success +exit=0 +$ JournalClient.journalRead(01M13CQYFDD2SX79JH85178HVX) successful step.completed counts +{"greet":1,"shout":1} +``` + +The exact package command confirms that no source under `kernel` or `sdk/src` +crosses 500 lines: + +```text +$ find kernel sdk/src -name '*.rs' -o -name '*.ts' | grep -v target | xargs wc -l | sort -nr | head -5 + 13024 total + 468 kernel/relayflowd/src/server.rs + 465 kernel/relayflowd/src/server/session.rs + 454 sdk/src/validate.ts + 452 kernel/relayflowd-core/src/spec.rs +``` + +Gate 1 remains GREEN with the live authored-surface seam now cited in +`ops/SCOREBOARD.md`. The PR remains for human review and merge; this tick does +not merge it. From c83a367da2584f5a8711023993881d73bab6e06e Mon Sep 17 00:00:00 2001 From: Relayflow Lead Date: Fri, 28 Aug 2026 02:43:12 -0400 Subject: [PATCH 3/4] Repair flows run and resume lifecycle reporting --- docs/SURFACE.md | 10 ++- kernel/relayflowd/src/server.rs | 16 ++++ sdk/src/cli/run.ts | 114 ++++++++++++++++++------- sdk/src/journal-client.ts | 34 +++++--- sdk/tests/cli.test.ts | 61 +++++++++++++- sdk/tests/journal-client.test.ts | 32 +++++++- sdk/tests/live-kernel.test.ts | 137 ++++++++++++++++++++++++++----- 7 files changed, 339 insertions(+), 65 deletions(-) diff --git a/docs/SURFACE.md b/docs/SURFACE.md index 608cab7d..e1882434 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -130,8 +130,8 @@ The exit codes are part of the surface contract: | Exit | Outcome | |---:|---| | `0` | The run completed with `completionReason: success`. | -| `1` | The run failed with a declared `completionReason`, or the daemon response violated the protocol. | -| `2` | The command was refused before a journal write: invalid input, failed preflight, unreachable daemon, or unavailable resume target. | +| `1` | The run failed with a declared `completionReason`, or a transport, runtime, or daemon protocol error left the outcome unknown. | +| `2` | The command was refused before a journal write: invalid input, failed preflight, unreachable daemon, or a `run_not_found` resume target. | | `3` | The run parked. `PARKED [run_parked]` names the step and its `llm` or `agent` type. | At gate 1 no `llm` or `agent` worker is attached by the CLI. Reaching either @@ -140,6 +140,12 @@ reporting success. Event, schedule, deployed-digest, HTTP, SDK-call, and flow-to-flow invocation remain later-gate surface work; they are not shipped by this CLI. +`flows resume` reports `run_unavailable` only when relayflowd returns the +typed `run_not_found` refusal. A dropped connection, request failure, or +`journal_write_failed` response exits 1 as `protocol_error`, because the +journal may already have changed and the CLI cannot honestly claim the resume +was refused before a write. + ## 6. Open surface questions (for gate-1 SDK work) - `gate:` in YAML: tiny expression language (`length < 200`) vs named checks only. Leaning: a deliberately small expression grammar + named checks for everything else. diff --git a/kernel/relayflowd/src/server.rs b/kernel/relayflowd/src/server.rs index d4483144..cf3d5eb4 100644 --- a/kernel/relayflowd/src/server.rs +++ b/kernel/relayflowd/src/server.rs @@ -165,6 +165,22 @@ fn handle_request( } "run.resume" => { let params: RunIdParams = decode_params(request.params)?; + let safe_id = !params.run_id.is_empty() + && params.run_id.chars().all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '-' | '_') + }); + let exists = safe_id + && data_dir + .join("runs") + .join(format!("{}.sqlite3", params.run_id)) + .try_exists() + .map_err(|error| internal_error(error.into()))?; + if !exists { + return Err(( + "run_not_found", + format!("run {} does not exist", params.run_id), + )); + } // Per-run serialization: the load-state -> next_actions -> append // sequence must be atomic, or two concurrent resumes both see a // step Runnable and double-dispatch the same attempt. diff --git a/sdk/src/cli/run.ts b/sdk/src/cli/run.ts index 52fede7f..b2c1a037 100644 --- a/sdk/src/cli/run.ts +++ b/sdk/src/cli/run.ts @@ -1,7 +1,7 @@ import { join, resolve } from 'node:path'; import { toKernelSpec } from '../compile.js'; import type { RunFailureKind } from '../failure-kinds.js'; -import { JournalClient } from '../journal-client.js'; +import { JournalClient, JournalProtocolError } from '../journal-client.js'; import type { PreflightDiagnostic } from '../preflight.js'; import type { RunCompletionReason, @@ -82,6 +82,9 @@ export async function resumeFlow(runId: string, dataDir: string): Promise { + let current = outcome; + let parkedStep: ParkedStep | undefined; + while (current.status === 'parked') { + const inspection = await inspectOutOfBandStep(client, current.run_id, knownSpec); + if (inspection?.parkedStep !== undefined) { + parkedStep = inspection.parkedStep; + break; + } + if (inspection?.runningStepId !== undefined) { + await waitForRunningStep(client, current.run_id, inspection.runningStepId); + current = await client.runResume(current.run_id); + continue; + } + if (inspection?.status === 'completed' || inspection?.status === 'failed') { + current = await client.runResume(current.run_id); + continue; + } + break; + } + const report: RunReport = { ...fromBase(command, base), - ok: outcome.status === 'completed' && outcome.completion_reason === 'success', - runId: outcome.run_id, + ok: current.status === 'completed' && current.completion_reason === 'success', + runId: current.run_id, socketPath, - status: outcome.status, - ...(outcome.completion_reason !== null ? { completionReason: outcome.completion_reason } : {}), - completedSteps: outcome.completed_steps, + status: current.status, + ...(current.completion_reason !== null ? { completionReason: current.completion_reason } : {}), + completedSteps: current.completed_steps, }; if (report.ok) return { exitCode: 0, report }; - if (outcome.status === 'failed' && outcome.completion_reason !== null) { + if (current.status === 'failed' && current.completion_reason !== null) { return { exitCode: 1, report: { ...report, diagnostics: [...report.diagnostics, { severity: 'failure', - kind: outcome.completion_reason, - message: `Run "${outcome.run_id}" failed with completionReason: ${outcome.completion_reason}.`, + kind: current.completion_reason, + message: `Run "${current.run_id}" failed with completionReason: ${current.completion_reason}.`, }], }, }; } - if (outcome.status === 'parked') { - const parkedStep = await findParkedStep(client, outcome.run_id, knownSpec); - if (parkedStep !== undefined) { - return { - exitCode: 3, - report: { - ...report, - parkedStep, - diagnostics: [...report.diagnostics, { - severity: 'parked', - kind: 'run_parked', - message: `Run "${outcome.run_id}" parked at step "${parkedStep.id}" (${parkedStep.type}): no worker is attached for step type "${parkedStep.type}".`, - }], - }, - }; - } + if (current.status === 'parked' && parkedStep !== undefined) { + return { + exitCode: 3, + report: { + ...report, + parkedStep, + diagnostics: [...report.diagnostics, { + severity: 'parked', + kind: 'run_parked', + message: `Run "${current.run_id}" parked at step "${parkedStep.id}" (${parkedStep.type}): no worker is attached for step type "${parkedStep.type}".`, + }], + }, + }; } return protocolFailure(command, base, socketPath, new Error( - `relayflowd returned status ${outcome.status} without a classifiable completion`, - ), outcome.run_id); + `relayflowd returned status ${current.status} without a classifiable completion`, + ), current.run_id); } -async function findParkedStep( +interface OutOfBandInspection { + status: RunStatus; + parkedStep?: ParkedStep; + runningStepId?: string; +} + +async function inspectOutOfBandStep( client: JournalClient, runId: string, knownSpec?: KernelRunSpec, -): Promise { +): Promise { const spec = knownSpec ?? await readRunSpec(client, runId); if (spec === undefined) return undefined; const snapshot = await client.runGet(runId); - return spec.steps.find((step): step is KernelStepSpec & { type: 'llm' | 'agent' } => + const parkedStep = spec.steps.find((step): step is KernelStepSpec & { type: 'llm' | 'agent' } => step.type !== 'deterministic' && snapshot.steps[step.id] === 'Runnable', ); + const runningStep = spec.steps.find((step) => + step.type !== 'deterministic' && isRunningStepState(snapshot.steps[step.id]), + ); + return { + status: snapshot.status, + ...(parkedStep !== undefined ? { parkedStep } : {}), + ...(runningStep !== undefined ? { runningStepId: runningStep.id } : {}), + }; +} + +async function waitForRunningStep( + client: JournalClient, + runId: string, + stepId: string, +): Promise { + while (true) { + await delay(50); + const snapshot = await client.runGet(runId); + if (!isRunningStepState(snapshot.steps[stepId])) return; + } +} + +function isRunningStepState(state: string | undefined): boolean { + return state === 'Running' || state?.startsWith('Running {') === true; } async function readRunSpec(client: JournalClient, runId: string): Promise { @@ -259,3 +309,7 @@ function errorMessage(error: unknown): string { function isObject(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } + +function delay(ms: number): Promise { + return new Promise((resolveDelay) => setTimeout(resolveDelay, ms)); +} diff --git a/sdk/src/journal-client.ts b/sdk/src/journal-client.ts index 0f127d2e..ffbff9dd 100644 --- a/sdk/src/journal-client.ts +++ b/sdk/src/journal-client.ts @@ -25,14 +25,25 @@ import { import type { KernelRunSpec, StepType } from './spec.js'; export interface JournalClientOptions { - /** Override the per-request timeout (ms). Default 30000. */ + /** Override the timeout for bounded protocol requests (ms). Default 30000. */ requestTimeoutMs?: number; } interface Pending { resolve: (value: unknown) => void; reject: (err: Error) => void; - timer: ReturnType; + timer?: ReturnType; +} + +/** A structured rejection returned by relayflowd over the journal protocol. */ +export class JournalProtocolError extends Error { + readonly code: string; + + constructor(code: string, message: string) { + super(`${code}: ${message}`); + this.name = 'JournalProtocolError'; + this.code = code; + } } export class JournalClient extends EventEmitter { @@ -104,9 +115,9 @@ export class JournalClient extends EventEmitter { const pending = this.pending.get(res.id); if (!pending) return; // reply for an already-timed-out request this.pending.delete(res.id); - clearTimeout(pending.timer); + if (pending.timer !== undefined) clearTimeout(pending.timer); if (res.ok) pending.resolve(res.result); - else pending.reject(new Error(`${res.error.code}: ${res.error.message}`)); + else pending.reject(new JournalProtocolError(res.error.code, res.error.message)); } else { const ev = msg as ServerEvent; this.emit(ev.event, ev.data); @@ -116,7 +127,7 @@ export class JournalClient extends EventEmitter { private failAll(err: Error): void { for (const [, p] of this.pending) { - clearTimeout(p.timer); + if (p.timer !== undefined) clearTimeout(p.timer); p.reject(err); } this.pending.clear(); @@ -125,6 +136,7 @@ export class JournalClient extends EventEmitter { private request( verb: V, params: VerbContract[V]['params'], + timeoutMs: number | null = this.requestTimeoutMs, ): Promise { return new Promise((resolve, reject) => { if (!this.socket || this.socket.destroyed) { @@ -133,16 +145,16 @@ export class JournalClient extends EventEmitter { } const id = randomUUID(); const frame: Request = { id, verb: verb as string, params }; - const timer = setTimeout(() => { + const timer = timeoutMs === null ? undefined : setTimeout(() => { this.pending.delete(id); - reject(new Error(`journal client: ${verb} timed out after ${this.requestTimeoutMs}ms`)); - }, this.requestTimeoutMs); + reject(new Error(`journal client: ${verb} timed out after ${timeoutMs}ms`)); + }, timeoutMs); this.pending.set(id, { resolve: resolve as (v: unknown) => void, reject, timer }); this.socket.write(JSON.stringify(frame) + '\n', (err) => { if (err) { const p = this.pending.get(id); if (p) { - clearTimeout(p.timer); + if (p.timer !== undefined) clearTimeout(p.timer); this.pending.delete(id); p.reject(new Error(`journal client: ${verb} write failed: ${err.message}`)); } @@ -164,12 +176,12 @@ export class JournalClient extends EventEmitter { * with `toKernelSpec` before crossing this journal-protocol boundary. */ runStart(spec: KernelRunSpec): Promise { - return this.request('run.start', { spec }); + return this.request('run.start', { spec }, null); } /** §3 memoized resume. */ runResume(runId: string): Promise { - return this.request('run.resume', { run_id: runId }); + return this.request('run.resume', { run_id: runId }, null); } /** Snapshot for legibility. */ diff --git a/sdk/tests/cli.test.ts b/sdk/tests/cli.test.ts index aee6c4fa..1e878505 100644 --- a/sdk/tests/cli.test.ts +++ b/sdk/tests/cli.test.ts @@ -451,7 +451,49 @@ describe('flows run/resume CLI over the journal protocol', () => { expect(output.stderr.join('\n')).toContain('step "answer" (llm)'); }); - it('parses resume and maps an unavailable run to exit 2', async () => { + it('follows a dispatched worker step instead of reporting a protocol error', async () => { + const dataDir = temporaryProject('flows-run-worker-'); + let snapshots = 0; + await startCliLoopback(dataDir, { + hello: sendOk, + 'run.start': (ctx) => sendResult(ctx, { + run_id: 'run-worker', + status: 'parked', + completion_reason: null, + completed_steps: 1, + }), + 'run.get': (ctx) => { + const running = snapshots++ === 0; + sendResult(ctx, { + run_id: 'run-worker', + status: running ? 'running' : 'completed', + steps: { + greet: 'Done', + answer: running ? 'Running' : 'Done', + finish: running ? 'Pending' : 'Done', + }, + budget: { tokens_in: 0, tokens_out: 0, dollars: '0' }, + }); + }, + 'run.resume': (ctx) => sendResult(ctx, { + run_id: 'run-worker', + status: 'completed', + completion_reason: 'success', + completed_steps: 3, + }), + }); + const output = capture(); + + const code = await runCli([ + 'run', '--data-dir', dataDir, join(TESTDATA, 'hello-llm.flow.yaml'), + ], output.io); + + expect(code).toBe(0); + expect(output.stdout.join('\n')).toContain('completionReason: success'); + expect(output.stderr.join('\n')).not.toContain('protocol_error'); + }); + + it('maps only run_not_found resumes to exit 2', async () => { const dataDir = temporaryProject('flows-resume-'); await startCliLoopback(dataDir, { hello: sendOk, @@ -465,6 +507,14 @@ describe('flows run/resume CLI over the journal protocol', () => { }); return; } + if (params['run_id'] === 'write-failed') { + ctx.send({ + id: ctx.id, + ok: false, + error: { code: 'journal_write_failed', message: 'disk full' }, + }); + return; + } ctx.send({ id: ctx.id, ok: false, @@ -474,6 +524,7 @@ describe('flows run/resume CLI over the journal protocol', () => { }); const resumed = capture(); const unavailable = capture(); + const failed = capture(); expect(await runCli(['resume', '--data-dir', dataDir, 'known-run'], resumed.io)).toBe(0); expect(resumed.stdout.join('\n')).toContain('completionReason: success'); @@ -485,6 +536,14 @@ describe('flows run/resume CLI over the journal protocol', () => { runId: 'absent-run', diagnostics: [{ kind: 'run_unavailable' }], }); + expect(await runCli(['resume', '--json', '--data-dir', dataDir, 'write-failed'], failed.io)).toBe(1); + expect(failed.stderr.join('\n')).toContain('FAILED [protocol_error]'); + expect(JSON.parse(failed.stdout.join('\n'))).toMatchObject({ + ok: false, + command: 'resume', + runId: 'write-failed', + diagnostics: [{ kind: 'protocol_error' }], + }); }); it('refuses malformed options with usage naming check, run, and resume', async () => { diff --git a/sdk/tests/journal-client.test.ts b/sdk/tests/journal-client.test.ts index 160d579d..c6be6930 100644 --- a/sdk/tests/journal-client.test.ts +++ b/sdk/tests/journal-client.test.ts @@ -1,7 +1,7 @@ import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; import type { Server } from 'node:net'; import { rmSync } from 'node:fs'; -import { JournalClient } from '../src/journal-client.js'; +import { JournalClient, JournalProtocolError } from '../src/journal-client.js'; import { compileYaml, toKernelSpec } from '../src/compile.js'; import { PROTOCOL_VERSION } from '../src/protocol.js'; import { @@ -232,7 +232,9 @@ steps: try { client = new JournalClient(errorPath, { requestTimeoutMs: 2000 }); await client.connect(); - await expect(client.runResume('run-01')).rejects.toThrow(/journal_write_failed/); + const rejected = client.runResume('run-01'); + await expect(rejected).rejects.toBeInstanceOf(JournalProtocolError); + await expect(rejected).rejects.toMatchObject({ code: 'journal_write_failed' }); } finally { client?.close(); await new Promise((r) => errorServer.close(() => r())); @@ -240,6 +242,32 @@ steps: } }); + it('does not apply the bounded request timeout to run lifecycle requests', async () => { + const lifecyclePath = sockPath(); + const lifecycleServer = startLoopback(lifecyclePath, { + 'run.start': (ctx) => { + setTimeout(() => sendResult(ctx, { + run_id: 'slow-run', + status: 'completed', + completion_reason: 'success', + completed_steps: 1, + }), 50); + }, + }); + try { + client = new JournalClient(lifecyclePath, { requestTimeoutMs: 10 }); + await client.connect(); + await expect(client.runStart(HELLO_SPEC)).resolves.toMatchObject({ + run_id: 'slow-run', + status: 'completed', + }); + } finally { + client?.close(); + await new Promise((r) => lifecycleServer.close(() => r())); + rmSync(lifecyclePath, { force: true }); + } + }); + it('fails closed on connection drop (pending requests reject)', async () => { const dropPath = sockPath(); const dropServer = startLoopback(dropPath, { diff --git a/sdk/tests/live-kernel.test.ts b/sdk/tests/live-kernel.test.ts index f8bb262c..36f22fd6 100644 --- a/sdk/tests/live-kernel.test.ts +++ b/sdk/tests/live-kernel.test.ts @@ -4,7 +4,6 @@ import { existsSync, lstatSync, mkdtempSync, - readFileSync, readdirSync, rmSync, writeFileSync, @@ -14,7 +13,7 @@ import { dirname, join, resolve } from 'node:path'; import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; import { fileURLToPath } from 'node:url'; import { afterEach, beforeAll, describe, expect, it } from 'vitest'; -import { compileYaml, compileYamlToCanonicalJson, toKernelSpec } from '../src/compile.js'; +import { compileYaml, toKernelSpec } from '../src/compile.js'; import { JournalClient } from '../src/journal-client.js'; import type { StepDispatchEvent } from '../src/protocol.js'; @@ -110,6 +109,61 @@ steps: ok: false, diagnostics: [{ kind: 'invalid_invocation' }], }); + + const liveClient = await connectClient(dataDir); + await expect(liveClient.runResume('absent-run')).rejects.toMatchObject({ + code: 'run_not_found', + }); + }); + + it('allows a deterministic run to exceed the bounded request timeout', async () => { + const dataDir = temporaryDirectory('flows-live-long-run-'); + await startDaemon(dataDir); + const flow = join(dataDir, 'long.flow.yaml'); + writeFileSync(flow, ` +version: '0.1.0' +steps: + - id: first + type: deterministic + command: sleep 16 + - id: second + type: deterministic + dependsOn: [first] + command: sleep 16 +`); + + const completed = invokeCli(['run', '--data-dir', dataDir, flow]); + + expect(completed.status, completed.stderr).toBe(0); + expect(completed.stdout).toContain('completionReason: success'); + }, 45_000); + + it('follows a live worker dispatch through flows run', async () => { + const dataDir = temporaryDirectory('flows-live-worker-'); + await startDaemon(dataDir); + const worker = await connectClient(dataDir); + await worker.hello('live-cli-worker'); + const dispatched = eventOnce(worker, 'step.dispatch'); + await worker.workerAttach('live-cli-llm', ['llm']); + + const running = invokeCliAsync([ + 'run', '--data-dir', dataDir, join(TESTDATA, 'hello-llm.flow.yaml'), + ]); + const lease = await dispatched; + expect((await worker.runGet(lease.run_id)).steps[lease.step_id]).toMatch(/^Running/); + await worker.stepComplete( + lease.run_id, + lease.step_id, + lease.attempt, + lease.idempotency_key, + 'success', + { output: { answer: 4 }, usage: { tokens_in: 2, tokens_out: 1, dollars: '0.001' } }, + ); + const completed = await running; + + expect(completed.status, completed.stderr).toBe(0); + expect(completed.stdout).toContain('completionReason: success'); + expect(completed.stderr).not.toContain('protocol_error'); }); it('preflights before journaling and names an unreachable socket', async () => { @@ -249,7 +303,16 @@ describe('surface resume after a real daemon kill', () => { it('resumes a three-step run with each successful completion exactly once', async () => { const directory = temporaryDirectory('flows-live-resume-'); const dataDir = join(directory, 'data'); - const yaml = ` + const startedMarker = join(directory, 'second-started'); + const releaseMarker = join(directory, 'release-first-attempt'); + const slowCommand = [ + `if [ -e ${JSON.stringify(startedMarker)} ]; then printf resumed`, + `else : > ${JSON.stringify(startedMarker)}`, + `while [ ! -e ${JSON.stringify(releaseMarker)} ]; do sleep 0.05; done`, + 'fi', + ].join('; '); + const flow = join(directory, 'three-step.flow.yaml'); + writeFileSync(flow, ` version: '0.1.0' name: live-crash-resume steps: @@ -259,48 +322,47 @@ steps: - id: two type: deterministic dependsOn: [one] - command: printf two + command: ${JSON.stringify(slowCommand)} - id: three type: deterministic dependsOn: [two] command: printf three -`; - const specPath = join(directory, 'three-step.spec.json'); - writeFileSync(specPath, compileYamlToCanonicalJson(yaml)); - const interrupted = spawnSync(RELAYFLOWD, [ - '--data-dir', dataDir, 'run', specPath, '--stop-after', '1', - ], { encoding: 'utf8' }); - expect(interrupted.status, interrupted.stderr).toBe(0); - const initial = JSON.parse(interrupted.stdout) as { run_id: string; status: string }; - expect(initial.status).toBe('interrupted'); - +`); const firstDaemon = await startDaemon(dataDir); + const firstRun = invokeCliAsync(['run', '--data-dir', dataDir, flow]); + const runId = await waitForActiveRun(dataDir, startedMarker); const beforeClient = await connectClient(dataDir); - const before = (await beforeClient.journalRead(initial.run_id, 1)).entries; + const before = (await beforeClient.journalRead(runId, 1)).entries; expect(successfulCompletions(before)).toEqual({ one: 1 }); + expect((await beforeClient.runGet(runId)).steps['two']).toMatch(/^Running/); beforeClient.close(); clients.splice(clients.indexOf(beforeClient), 1); - console.log(`LIVE_KERNEL kill -9 pid=${firstDaemon.pid} run=${initial.run_id}`); + console.log(`LIVE_KERNEL kill -9 pid=${firstDaemon.pid} run=${runId} while step=two state=Running`); await stopDaemon(firstDaemon, 'SIGKILL'); daemons.splice(daemons.indexOf(firstDaemon), 1); + const interrupted = await firstRun; + expect(interrupted.status).toBe(1); + expect(interrupted.stderr).toContain('FAILED [protocol_error]'); + writeFileSync(releaseMarker, 'release'); await startDaemon(dataDir); const resumed = invokeCli([ - 'resume', '--json', '--data-dir', dataDir, initial.run_id, + 'resume', '--json', '--data-dir', dataDir, runId, ]); expect(resumed.status, resumed.stderr).toBe(0); expect(JSON.parse(resumed.stdout)).toMatchObject({ ok: true, command: 'resume', - runId: initial.run_id, + runId, status: 'completed', completionReason: 'success', }); const afterClient = await connectClient(dataDir); - const after = (await afterClient.journalRead(initial.run_id, 1)).entries; + const after = (await afterClient.journalRead(runId, 1)).entries; expect(successfulCompletions(after)).toEqual({ one: 1, two: 1, three: 1 }); + expect(completionReasons(after, 'two')).toEqual(['crashed', 'success']); }); }); @@ -360,6 +422,33 @@ function invokeCli(args: string[]) { }); } +function invokeCliAsync(args: string[]): Promise<{ status: number | null; stdout: string; stderr: string }> { + const child = spawn(process.execPath, [BUILT_CLI, ...args], { cwd: ROOT, stdio: ['ignore', 'pipe', 'pipe'] }); + daemons.push(child); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + child.stdout?.on('data', (chunk: Buffer) => stdout.push(chunk)); + child.stderr?.on('data', (chunk: Buffer) => stderr.push(chunk)); + return new Promise((resolveExit) => child.once('exit', (status) => resolveExit({ + status, + stdout: Buffer.concat(stdout).toString('utf8'), + stderr: Buffer.concat(stderr).toString('utf8'), + }))); +} + +async function waitForActiveRun(dataDir: string, marker: string): Promise { + const deadline = Date.now() + 5_000; + while (Date.now() < deadline) { + const runs = join(dataDir, 'runs'); + const journal = existsSync(runs) + ? readdirSync(runs).find((name) => /^[0-9A-Z]{26}\.sqlite3$/.test(name)) + : undefined; + if (existsSync(marker) && journal !== undefined) return journal.slice(0, -'.sqlite3'.length); + await delay(20); + } + throw new Error(`run did not reach the marked in-flight step within 5000ms: ${marker}`); +} + function runArtifacts(dataDir: string): string[] { return readdirSync(dataDir).filter((name) => name !== 'relayflowd.sock').sort(); } @@ -384,6 +473,16 @@ function successfulCompletions(entries: unknown[]): Record { return counts; } +function completionReasons(entries: unknown[], stepId: string): string[] { + return entries.flatMap((entry) => { + if (!isObject(entry) || entry['entry_type'] !== 'step.completed' || entry['step_id'] !== stepId) return []; + const payload = entry['payload']; + return isObject(payload) && typeof payload['completionReason'] === 'string' + ? [payload['completionReason']] + : []; + }); +} + function isObject(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } From 3616c0a521dfdcba41d94139b54bf3915f3891af Mon Sep 17 00:00:00 2001 From: Relayflow Lead Date: Fri, 28 Aug 2026 02:49:06 -0400 Subject: [PATCH 4/4] Record WP-11 repair evidence and adversarial review --- ops/DRIVE-LOG.md | 548 +++++++++++++++++++ ops/reviews/20260828-0244-pr9-adversarial.md | 58 ++ 2 files changed, 606 insertions(+) create mode 100644 ops/reviews/20260828-0244-pr9-adversarial.md diff --git a/ops/DRIVE-LOG.md b/ops/DRIVE-LOG.md index c701afb6..2007dd62 100644 --- a/ops/DRIVE-LOG.md +++ b/ops/DRIVE-LOG.md @@ -2277,3 +2277,551 @@ $ find kernel sdk/src -name '*.rs' -o -name '*.ts' | grep -v target | xargs wc - Gate 1 remains GREEN with the live authored-surface seam now cited in `ops/SCOREBOARD.md`. The PR remains for human review and merge; this tick does not merge it. + +### WP-11 — repair PR #9 findings F1–F4 on rebased head + +Work ran in scratch worktree `/tmp/flows-wp11.eTPAfM`. The tick-owned checkout +was not edited. The required Veto MCP server was not exposed to this worker; +no Veto status, discovery, diff review, or scan result is claimed. + +#### Rebase and clean-state evidence + +```text +$ git status --porcelain +$ git fetch origin +$ git rebase origin/main +Rebasing (1/2) +Rebasing (2/2) +Successfully rebased and updated refs/heads/flow/drive-77b2457-08280058. +$ git status --porcelain +$ git log --oneline --decorate -n 3 +0fd332c (HEAD -> flow/drive-77b2457-08280058) Record WP-10 verification evidence +b401efe Add flows run and resume live-kernel surface +173423c (origin/main, origin/HEAD, ops-scratch2) ops(contract): state the worktree rule precisely — the tick owns main, I stay out +``` + +The repair commit reviewed below is +`c83a367da2584f5a8711023993881d73bab6e06e`. + +#### Mutation verification — F1 lifecycle timeout + +Fixed-file hash before mutation: + +```text +$ shasum -a 256 sdk/src/journal-client.ts +ad8ce9d3664abea774a348837ec88fdffc9fd6622bdfb26af8331758b7def784 sdk/src/journal-client.ts +``` + +`runStart` and `runResume` were changed back to the bounded default. The +focused regression failed: + +```text +$ (cd sdk && npm test -- tests/journal-client.test.ts -t "does not apply the bounded request timeout to run lifecycle requests") + + RUN v2.1.9 /private/tmp/flows-wp11.eTPAfM/sdk + + ❯ tests/journal-client.test.ts (13 tests | 1 failed | 12 skipped) 16ms + × JournalClient: protocol v0 over unix socket > does not apply the bounded request timeout to run lifecycle requests 15ms + → promise rejected "Error: journal client: run.start timed ou…" instead of resolving + + FAIL tests/journal-client.test.ts > JournalClient: protocol v0 over unix socket > does not apply the bounded request timeout to run lifecycle requests +AssertionError: promise rejected "Error: journal client: run.start timed ou…" instead of resolving + ❯ tests/journal-client.test.ts:260:47 + +Caused by: Error: journal client: run.start timed out after 10ms + ❯ Timeout._onTimeout src/journal-client.ts:150:16 + + Test Files 1 failed (1) + Tests 1 failed | 12 skipped (13) +``` + +The two lifecycle calls were restored, the file returned to the exact hash, +and the same test passed: + +```text +$ (cd sdk && shasum -a 256 src/journal-client.ts && npm test -- tests/journal-client.test.ts -t "does not apply the bounded request timeout to run lifecycle requests") +ad8ce9d3664abea774a348837ec88fdffc9fd6622bdfb26af8331758b7def784 src/journal-client.ts + + RUN v2.1.9 /private/tmp/flows-wp11.eTPAfM/sdk + + ✓ tests/journal-client.test.ts (13 tests | 12 skipped) 57ms + + Test Files 1 passed (1) + Tests 1 passed | 12 skipped (13) +``` + +#### Mutation verification — F2 live worker dispatch + +Fixed-file hash before mutation: + +```text +$ shasum -a 256 sdk/src/cli/run.ts +d18a2b78c4862fdc478cd6eb3fd837be3952706c8e58a450457397dbccc5631e sdk/src/cli/run.ts +``` + +Recognition of the journal snapshot's `Running` state was removed, the built +CLI was regenerated, and the live worker test failed with the original false +protocol error: + +```text +$ (cd sdk && npm run build && npm test -- tests/live-kernel.test.ts -t "follows a live worker dispatch through flows run") + + RUN v2.1.9 /private/tmp/flows-wp11.eTPAfM/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/private/tmp/flows-wp11.eTPAfM/kernel/target/debug/relayflowd +LIVE_KERNEL flows=/private/tmp/flows-wp11.eTPAfM/sdk/dist/cli.js + + ❯ tests/live-kernel.test.ts (6 tests | 1 failed | 5 skipped) 126ms + × built flows CLI against live relayflowd > follows a live worker dispatch through flows run 125ms + → WARNING [unprovable_effects] Step "greet" command "printf" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "finish" command "printf" resolves, but its effects cannot be proven before execution. +FAILED [protocol_error] relayflowd could not complete the run request: relayflowd returned status parked without a classifiable completion +: expected 1 to be +0 // Object.is equality + + Test Files 1 failed (1) + Tests 1 failed | 5 skipped (6) +``` + +The predicate was restored byte-for-byte, its hash matched, the CLI was +rebuilt, and the live test passed: + +```text +$ (cd sdk && shasum -a 256 src/cli/run.ts && npm run build && npm test -- tests/live-kernel.test.ts -t "follows a live worker dispatch through flows run") +d18a2b78c4862fdc478cd6eb3fd837be3952706c8e58a450457397dbccc5631e src/cli/run.ts + + RUN v2.1.9 /private/tmp/flows-wp11.eTPAfM/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/private/tmp/flows-wp11.eTPAfM/kernel/target/debug/relayflowd +LIVE_KERNEL flows=/private/tmp/flows-wp11.eTPAfM/sdk/dist/cli.js + + ✓ tests/live-kernel.test.ts (6 tests | 5 skipped) 153ms + + Test Files 1 passed (1) + Tests 1 passed | 5 skipped (6) +``` + +#### Mutation verification — F3 real daemon kill + +Fixed-test hash before mutation: + +```text +$ shasum -a 256 sdk/tests/live-kernel.test.ts +fb6a1bb97802994beba681e38d0d9eb0960c32bdd56a23894af99a271d5b204d sdk/tests/live-kernel.test.ts +``` + +The release marker was moved before `SIGKILL`, allowing the active run to +finish and leaving only an idle daemon to kill. The gate detected the missing +crash: + +```text +$ (cd sdk && npm test -- tests/live-kernel.test.ts -t "resumes a three-step run with each successful completion exactly once") + +stdout | tests/live-kernel.test.ts > surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once +LIVE_KERNEL kill -9 pid=75597 run=01M13HEME0BZWSM95RZPQ2XCRR after step=two finished + + ❯ tests/live-kernel.test.ts (6 tests | 1 failed | 5 skipped) 164ms + × surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once 164ms + → expected +0 to be 1 // Object.is equality + +AssertionError: expected +0 to be 1 // Object.is equality + +- Expected ++ Received + +- 1 ++ 0 + + ❯ tests/live-kernel.test.ts:347:32 + + Test Files 1 failed (1) + Tests 1 failed | 5 skipped (6) +``` + +The kill ordering was restored byte-for-byte, its hash matched, and the same +live test passed while naming the active state at the kill: + +```text +$ (cd sdk && shasum -a 256 tests/live-kernel.test.ts && npm test -- tests/live-kernel.test.ts -t "resumes a three-step run with each successful completion exactly once") +fb6a1bb97802994beba681e38d0d9eb0960c32bdd56a23894af99a271d5b204d tests/live-kernel.test.ts + +stdout | tests/live-kernel.test.ts > surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once +LIVE_KERNEL kill -9 pid=75855 run=01M13HEZEN52TFRQWC44DVTYJ7 while step=two state=Running + + ✓ tests/live-kernel.test.ts (6 tests | 5 skipped) 280ms + + Test Files 1 passed (1) + Tests 1 passed | 5 skipped (6) +``` + +#### Mutation verification — F4 typed error, daemon code, classification + +Mapping all resume errors back to exit 2 made the classification test fail: + +```text +$ (cd sdk && npm test -- tests/cli.test.ts -t "maps only run_not_found resumes to exit 2") + + ❯ tests/cli.test.ts (48 tests | 1 failed | 47 skipped) 8ms + × flows run/resume CLI over the journal protocol > maps only run_not_found resumes to exit 2 8ms + → expected 2 to be 1 // Object.is equality + +AssertionError: expected 2 to be 1 // Object.is equality + ❯ tests/cli.test.ts:539:98 + + Test Files 1 failed (1) + Tests 1 failed | 47 skipped (48) +``` + +Restoration returned `sdk/src/cli/run.ts` to its exact hash and passed: + +```text +$ (cd sdk && shasum -a 256 src/cli/run.ts && npm test -- tests/cli.test.ts -t "maps only run_not_found resumes to exit 2") +d18a2b78c4862fdc478cd6eb3fd837be3952706c8e58a450457397dbccc5631e src/cli/run.ts + + ✓ tests/cli.test.ts (48 tests | 47 skipped) 5ms + + Test Files 1 passed (1) + Tests 1 passed | 47 skipped (48) +``` + +Discarding the structured error code made the client test fail: + +```text +$ (cd sdk && npm test -- tests/journal-client.test.ts -t "fails closed when the server returns an error") + + ❯ tests/journal-client.test.ts (13 tests | 1 failed | 12 skipped) 7ms + × JournalClient: protocol v0 over unix socket > fails closed when the server returns an error 5ms + → expected Error: journal_write_failed: disk full to be an instance of JournalProtocolError + +AssertionError: expected Error: journal_write_failed: disk full to be an instance of JournalProtocolError + ❯ tests/journal-client.test.ts:236:7 + + Test Files 1 failed (1) + Tests 1 failed | 12 skipped (13) +``` + +Restoration returned `sdk/src/journal-client.ts` to its exact hash and passed: + +```text +$ (cd sdk && shasum -a 256 src/journal-client.ts && npm test -- tests/journal-client.test.ts -t "fails closed when the server returns an error") +ad8ce9d3664abea774a348837ec88fdffc9fd6622bdfb26af8331758b7def784 src/journal-client.ts + + ✓ tests/journal-client.test.ts (13 tests | 12 skipped) 4ms + + Test Files 1 passed (1) + Tests 1 passed | 12 skipped (13) +``` + +Removing the daemon's `run_not_found` branch and rebuilding made the live +protocol assertion fail with the old misclassification: + +```text +$ (cd sdk && npm test -- tests/live-kernel.test.ts -t "keeps JSON report-shaped") + + ❯ tests/live-kernel.test.ts (6 tests | 1 failed | 5 skipped) 1306ms + × built flows CLI against live relayflowd > runs rung (a), parks rung (b), and keeps JSON report-shaped 1305ms + → expected JournalProtocolError: journal_write_faile… { code: '…' } to match object { code: 'run_not_found' } + +- Expected ++ Received + +- Object { +- "code": "run_not_found", ++ JournalProtocolError { ++ "code": "journal_write_failed", + } + + ❯ tests/live-kernel.test.ts:114:5 + + Test Files 1 failed (1) + Tests 1 failed | 5 skipped (6) +``` + +The daemon file was restored to its exact hash, rebuilt, and the same live +test passed: + +```text +$ (cd kernel && shasum -a 256 relayflowd/src/server.rs && ../ops/cargo.sh build) +297677e695182f5c39f56c3b22ed1bfee048bc6cf49b4f38e5788fc1b0f68ad5 relayflowd/src/server.rs + Compiling relayflowd v0.1.0 (/private/tmp/flows-wp11.eTPAfM/kernel/relayflowd) + Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.95s +$ (cd sdk && npm test -- tests/live-kernel.test.ts -t "keeps JSON report-shaped") + + ✓ tests/live-kernel.test.ts (6 tests | 5 skipped) 878ms + ✓ built flows CLI against live relayflowd > runs rung (a), parks rung (b), and keeps JSON report-shaped 877ms + + Test Files 1 passed (1) + Tests 1 passed | 5 skipped (6) +``` + +#### Final build-first verification at repaired head + +```text +$ (cd kernel && ../ops/cargo.sh build) +/Users/khaliqgant/.zshenv:.:1: no such file or directory: /tmp/agent37-rust-0820.DWSmuv/cargo/env + Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.34s +``` + +```text +$ (cd sdk && npm ci && npm run build) +/Users/khaliqgant/.zshenv:.:1: no such file or directory: /tmp/agent37-rust-0820.DWSmuv/cargo/env + +added 48 packages, and audited 49 packages in 736ms + +13 packages are looking for funding + run `npm fund` for details + +5 vulnerabilities (3 moderate, 1 high, 1 critical) + +To address all issues (including breaking changes), run: + npm audit fix --force + +Run `npm audit` for details. +npm notice run @relayflows/sdk@0.1.0 build +npm notice run tsc && node scripts/make-cli-executable.mjs +``` + +The full kernel suite executed 73 tests (18 + 19 + 26 + 4 + 6): + +```text +$ (cd kernel && ../ops/cargo.sh test --workspace) +/Users/khaliqgant/.zshenv:.:1: no such file or directory: /tmp/agent37-rust-0820.DWSmuv/cargo/env + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.13s + Running unittests src/lib.rs (target/debug/deps/relayflowd-399037c915557fdb) + +running 18 tests +test server::client::tests::resume_waits_while_the_heartbeat_renewed_lease_is_live ... ok +test server::tests::agent::contract::an_agent_worker_attaching_without_pins_is_refused_at_attach ... ok +test exec_det::tests::captures_deterministic_output ... ok +test exec_det::tests::timeout_has_an_explicit_completion_reason ... ok +test server::tests::agent::contract::an_oversized_trajectory_tail_is_refused_at_step_complete ... ok +test server::tests::agent::contract::an_agent_worker_missing_a_declared_surface_parks_the_run_instead_of_erroring ... ok +test server::tests::agent::contract::agent_without_a_compatible_worker_parks_without_starting ... ok +test server::tests::agent::contract::an_llm_completion_claiming_an_effect_fails_closed_with_the_reason_journaled ... ok +test server::tests::hello_enforces_protocol_version ... ok +test server::tests::run_start_fails_closed_on_an_unknown_verification_key ... ok +test server::tests::agent::contract::a_replacement_worker_that_never_reported_the_pinned_surface_is_not_dispatched_to ... ok +test server::tests::a_failed_disconnect_journal_append_is_retained_and_retried_not_dropped ... ok +test server::tests::agent::pins::consecutive_agent_steps_on_different_surfaces_each_start_from_their_own_pins ... ok +test server::tests::agent::pins::reset_worker_reporting_a_revision_other_than_its_pin_fails_closed_as_worker_error ... ok +test server::tests::stopped_heartbeats_past_the_deadline_journal_lease_expired_and_release_the_step ... ok +test exec_det::tests::timeout_kills_the_whole_process_group ... ok +test server::tests::agent::pins::a_replacement_worker_at_a_different_revision_is_not_dispatched_the_stale_pins ... ok +test server::tests::an_entry_appended_during_watch_registration_is_delivered_exactly_once ... ok + +test result: ok. 18 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.56s + + Running unittests src/main.rs (target/debug/deps/relayflowd-9e21fa47745f4fb0) + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/crash_resume.rs (target/debug/deps/crash_resume-e6635a3f0d48512c) + +running 19 tests +test agent::resume_without_a_worker_parks_immediately_instead_of_timing_out ... ok +test agent::rung_c_sigkill_after_final_effect_replays_results_without_redispatch ... ok +test concurrency::live_resume_leaves_an_active_lease_running ... ok +test concurrency::concurrent_resumes_lease_exactly_one_attempt ... ok +test agent::rung_c_crash_between_effect_election_and_the_provider_call_performs_it_exactly_once ... ok +test llm::serve_plumbs_watch_events_and_replayable_stream_verbs ... ok +test llm::failing_llm_verification_schedules_a_durable_retry_and_succeeds ... ok +test agent::rung_c_reset_sigkill_mid_edit_restores_pins_dedupes_effect_and_explains_attempts ... ok +test llm::llm_verification_exhaustion_is_a_declared_failure_kind ... ok +test llm::sigkill_after_the_final_rung_b_effect_resumes_without_redispatching_llm ... ok +test agent::rung_c_sigkill_boundaries_resume_only_unfinished_steps_via_real_cli ... ok +test agent::rung_c_sigkill_between_agent_completion_and_final_effect_memoizes_the_agent ... ok +test llm::completed_llm_output_is_memoized_when_serve_dies_during_the_next_step ... ok +test llm::sigkill_under_serve_mid_llm_releases_the_lease_and_finishes_via_cli_resume ... ok +test sigkill_mid_step_replaces_and_explains_the_dead_attempt ... ok +test llm::worker_killed_while_holding_a_lease_is_explained_and_released_on_cli_resume ... ok +test llm::sigkill_sweep_covers_before_and_between_the_rung_b_steps ... ok +test sigkill_sweep_covers_every_hello_step_boundary ... ok +test sigkill_under_serve_resumes_the_socket_started_run ... ok + +test result: ok. 19 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.94s + + Running unittests src/lib.rs (target/debug/deps/relayflowd_core-b1fe3b3250e9e7a2) + +running 26 tests +test clock::tests::simulated_clock_is_explicitly_advanced ... ok +test journal::tests::memory_journal_assigns_sequences_and_rolls_epochs ... ok +test machine::tests::machine_starts_runnable_step_with_stable_effect_key ... ok +test machine::tests::all_backing_off_steps_return_timers ... ok +test machine::tests::reset_recovery_dispatches_the_original_pinned_revision ... ok +test retry::tests::jitter_is_repeatable_and_bounded ... ok +test machine::tests::successful_memo_is_never_scheduled_again ... ok +test spec::tests::a_misspelled_step_level_key_is_a_parse_error ... ok +test spec::tests::cycles_are_rejected ... ok +test spec::tests::a_misspelled_verification_gate_key_is_a_parse_error_not_a_dropped_gate ... ok +test machine::tests::inspect_recovery_injects_the_dirty_pin_completion_reason_and_tail ... ok +test spec::tests::unknown_root_and_nested_fields_are_rejected ... ok +test machine::tests::every_failed_run_terminates_with_declared_completion_reasons ... ok +test spec::tests::preflight_data_is_fail_closed ... ok +test spec::tests::zero_agent_flow_is_valid ... ok +test spec::tests::spec_version_is_semver_and_gated ... ok +test spec::tests::the_full_ladder_parses_in_the_one_dialect ... ok +test state::tests::completed_output_is_memoized_and_unlocks_dependents ... ok +test verify::tests::deterministic_output_requires_successful_exit_and_content ... ok +test machine::tests::verification_failure_schedules_a_durable_retry ... ok +test machine::tests::manual_recovery_parks_needs_human_and_never_redispatches ... ok +test state::tests::budget_decimal_strings_add_without_floats ... ok +test machine::tests::crashed_attempt_does_not_consume_an_iteration ... ok +test state::tests::a_completion_that_omits_a_surface_does_not_drop_it_from_the_pin_chain ... ok +test state::tests::end_pin_chain_is_enforced_and_a_broken_chain_is_a_hard_error ... ok +test verify::tests::json_schema_is_a_control_gate ... ok + +test result: ok. 26 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running tests/spec_parity.rs (target/debug/deps/spec_parity-bbda6cf1e1cf1c19) + +running 4 tests +test the_kernel_parses_the_deterministic_rung_and_stamps_the_same_hash ... ok +test the_kernel_parses_the_rung_b_spec_and_stamps_the_same_hash ... ok +test the_kernel_parses_the_sdk_compiled_spec_and_stamps_the_same_hash ... ok +test the_kernel_parses_the_rung_c_agent_spec_and_stamps_the_same_hash ... ok + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running unittests src/lib.rs (target/debug/deps/relayflowd_journal-d13cb7954335385c) + +running 6 tests +test registry::tests::registry_is_a_rebuildable_run_locator ... ok +test tests::failed_commit_is_returned_not_swallowed ... ok +test tests::rollover_is_atomic_scaffolding_for_epoch_resume ... ok +test tests::append_is_durable_and_monotonic_after_reopen ... ok +test tests::effects_are_deduplicated_at_the_journal_boundary ... ok +test tests::an_unconfirmed_election_is_reclaimed_by_the_next_attempt_not_treated_as_done ... ok + +test result: ok. 6 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s + + Doc-tests relayflowd + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Doc-tests relayflowd_core + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Doc-tests relayflowd_journal + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s +``` + +```text +$ (cd kernel && ../ops/cargo.sh clippy --workspace -- -D warnings) +/Users/khaliqgant/.zshenv:.:1: no such file or directory: /tmp/agent37-rust-0820.DWSmuv/cargo/env + Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.09s +``` + +Formatting exited 0. Its only captured output was the worker shell's stale +environment warning; rustfmt itself emitted nothing: + +```text +$ (cd kernel && ../ops/cargo.sh fmt --check) +/Users/khaliqgant/.zshenv:.:1: no such file or directory: /tmp/agent37-rust-0820.DWSmuv/cargo/env +``` + +The complete SDK suite ran all nine files, including all six live-kernel tests: + +```text +$ (cd sdk && npm test) +/Users/khaliqgant/.zshenv:.:1: no such file or directory: /tmp/agent37-rust-0820.DWSmuv/cargo/env +npm notice run @relayflows/sdk@0.1.0 test +npm notice run tsc --noEmit && vitest run + + RUN v2.1.9 /private/tmp/flows-wp11.eTPAfM/sdk + + ✓ tests/preflight.test.ts (12 tests) 6ms +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/private/tmp/flows-wp11.eTPAfM/kernel/target/debug/relayflowd +LIVE_KERNEL flows=/private/tmp/flows-wp11.eTPAfM/sdk/dist/cli.js + + ✓ tests/validate.test.ts (36 tests) 16ms + ✓ tests/deterministic-llm.test.ts (5 tests) 17ms + ✓ tests/journal-client.test.ts (13 tests) 68ms + ✓ tests/hello-deterministic.test.ts (5 tests) 10ms + ✓ tests/spec-parity.test.ts (15 tests) 22ms + ✓ tests/cli.test.ts (48 tests) 689ms + ✓ tests/bin.test.ts (7 tests) 1109ms +stdout | tests/live-kernel.test.ts > surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once +LIVE_KERNEL kill -9 pid=82029 run=01M13HWB1SMTS0S0AT9ZGFB8WY while step=two state=Running + + ✓ tests/live-kernel.test.ts (6 tests) 33400ms + ✓ built flows CLI against live relayflowd > runs rung (a), parks rung (b), and keeps JSON report-shaped 635ms + ✓ built flows CLI against live relayflowd > allows a deterministic run to exceed the bounded request timeout 32135ms + + Test Files 9 passed (9) + Tests 147 passed (147) + Start at 02:45:21 + Duration 33.68s (transform 307ms, setup 0ms, collect 738ms, tests 35.34s, environment 1ms, prepare 442ms) +``` + +#### PR review replies at repair head + +```text +$ gh api repos/AgentWorkforce/flows/pulls/9/comments --paginate --jq '.[] | select(.in_reply_to_id == 3878120106 or .in_reply_to_id == 3878120108 or .in_reply_to_id == 3878120111 or .in_reply_to_id == 3878120116) | [.id, .in_reply_to_id, .commit_id, .body] | @tsv' +3878476196 3878120111 c83a367da2584f5a8711023993881d73bab6e06e Fixed in c83a367da2584f5a8711023993881d73bab6e06e. Audit: the CLI now starts the flow through the daemon that is SIGKILLed, records step two as Running before the kill, observes the first CLI fail closed, then resumes to exactly one successful completion per step with step two reasons [crashed, success]. Moving the kill after the active run completed made the regression fail. +3878476208 3878120106 c83a367da2584f5a8711023993881d73bab6e06e Fixed in c83a367da2584f5a8711023993881d73bab6e06e. Audit: run.start and run.resume now opt out of the bounded per-request timer; the 32-second live deterministic regression completed successfully, and reverting the lifecycle opt-out made the focused test fail with run.start timed out after 10ms. +3878476210 3878120108 f435545f3ea2bf58d2036770bda3a8afffb6f9ab Fixed in c83a367da2584f5a8711023993881d73bab6e06e. Audit: flows run now distinguishes a Runnable out-of-band step (no worker, exit 3) from a Running dispatched step, follows the latter over run.get, and resumes classification after it settles. The live attached-worker regression failed with protocol_error when Running recognition was removed and passed after restoration. +3878476214 3878120116 c83a367da2584f5a8711023993881d73bab6e06e Fixed in c83a367da2584f5a8711023993881d73bab6e06e. Audit: JournalClient now rejects daemon errors as JournalProtocolError with code; relayflowd returns run_not_found for a missing resume journal; resumeFlow maps only that code to run_unavailable/exit 2 and sends journal_write_failed, transport, and runtime failures to protocol_error/exit 1. Focused mutations of each layer failed their regressions and passed after restoration. +``` + +GitHub associates reply `3878476210` with the obsolete line's original commit, +but its body explicitly audits and names the fixing head `c83a367…`; the other +three reply records also carry that head directly. + +The adversarial review transcript is +`ops/reviews/20260828-0244-pr9-adversarial.md`; it names the reviewed SHA and +ends `REVIEW_PASSED`. + +#### Captured non-product execution failures + +These commands failed because they were launched from the wrong working +directory; neither was treated as verification: + +```text +$ (cd kernel && npm run typecheck) +npm error code ENOENT +npm error syscall open +npm error path /private/tmp/flows-wp11.eTPAfM/kernel/package.json +npm error errno -2 +npm error enoent Could not read package.json: Error: ENOENT: no such file or directory, open '/private/tmp/flows-wp11.eTPAfM/kernel/package.json' +``` + +```text +$ (cd sdk && shasum -a 256 sdk/src/journal-client.ts) +shasum: sdk/src/journal-client.ts: No such file or directory +``` + +The same cross-directory error recurred once while the F4 daemon mutation was +active; the kernel build succeeded first, then npm failed before any test ran: + +```text +$ (cd kernel && ../ops/cargo.sh build && npm test -- tests/live-kernel.test.ts -t "runs rung (a), parks rung (b), and keeps JSON report-shaped") + Finished `dev` profile [unoptimized + debuginfo] target(s) in 1.83s +npm error code ENOENT +npm error syscall open +npm error path /private/tmp/flows-wp11.eTPAfM/kernel/package.json +``` + +The first corrected F4 selector still matched no tests because its parentheses +were interpreted as a regex. It is recorded as a failed gate, not a pass; the +executing `keeps JSON report-shaped` run above replaced it: + +```text +$ (cd sdk && npm test -- tests/live-kernel.test.ts -t "runs rung (a), parks rung (b), and keeps JSON report-shaped") + + ↓ tests/live-kernel.test.ts (6 tests | 6 skipped) + + Test Files 1 skipped (1) + Tests 6 skipped (6) +``` diff --git a/ops/reviews/20260828-0244-pr9-adversarial.md b/ops/reviews/20260828-0244-pr9-adversarial.md new file mode 100644 index 00000000..bdce2da5 --- /dev/null +++ b/ops/reviews/20260828-0244-pr9-adversarial.md @@ -0,0 +1,58 @@ +# PR #9 adversarial review — repair head `c83a367da2584f5a8711023993881d73bab6e06e` + +Date: 2026-08-28 02:44 EDT. Base: `origin/main` at `173423c`. Reviewed +branch: `flow/drive-77b2457-08280058` after rebase. Scope: the complete +three-commit PR diff, with focused re-review of all four Codex findings. + +## Adversarial questions + +1. Can a valid run still be converted into a surface failure by a fixed wall + clock? No. `JournalClient.request` retains its bounded default for ordinary + requests, while `runStart` and `runResume` explicitly select no request + deadline. Socket close, write failure, and malformed frames still reject. +2. Can an attached worker's accepted lease still be described as "no worker"? + No. `inspectOutOfBandStep` classifies a `Runnable` out-of-band step as the + genuine parked case and recognizes both current live snapshot encodings of + `Running`. It waits through `run.get` over the journal protocol, then asks + `run.resume` to classify the next durable state. It does not inspect sqlite + or the daemon registry. +3. Does the crash test kill the process that owns the active attempt? Yes. The + test starts `flows run` against `firstDaemon`, waits until the step-two + marker and journal file both exist, asserts the step snapshot is `Running`, + and only then sends `SIGKILL` to `firstDaemon`. The resumed journal must show + step two completion reasons `crashed`, then `success`, with each successful + step completion counted once. +4. Can exit 2 still assert a pre-write refusal for an ambiguous resume error? + No. The client preserves the daemon's structured code. The daemon admits + `run_not_found` only after a safe run-id/path existence check. `resumeFlow` + maps only that typed code to `run_unavailable`; every transport, runtime, + malformed-frame, and other daemon rejection becomes `protocol_error` / exit + 1. +5. Did the repair reach around the protocol boundary or add product surface? + No. Worker following uses only `run.get` and `run.resume`. The only wire + taxonomy addition is the missing-run error required to classify the + already-shipped resume surface. `RUN_FAILURE_KINDS` needed no new surface + kind. +6. Are the tests capable of detecting the repaired defects? Yes. The persisted + drive transcript contains focused fail/pass mutation pairs. In particular, + F2 uses an attached live worker while `flows run` owns the start request, + and F3's no-active-crash mutation fails because the first CLI exits 0 rather + than the required fail-closed exit 1. + +## Remaining risk read + +- `flows run` polls `run.get` every 50 ms while a worker owns the step. This is + bounded per request, uses the public journal protocol, and has no incorrect + lifecycle deadline. A push-based follow can be a later efficiency change; + it is not required to repair the false protocol failure. +- `run.resume` accepts the repository's existing alphanumeric, hyphen, and + underscore run-id spellings for existence checks. Separators are refused as + not found, so the new pre-check does not create a path traversal. +- The Veto MCP server was not exposed in this worker, so no Veto scan result is + claimed. The repository's full kernel/SDK gates and mutation outputs are + captured in `ops/DRIVE-LOG.md`. + +No untriaged correctness, protocol-boundary, failure-taxonomy, test-integrity, +or scope finding remains at the reviewed SHA. + +REVIEW_PASSED