From 9cab5175107584f2dc5bbd8036f4b8c80183b4d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 30 Jun 2026 15:38:10 +0200 Subject: [PATCH] ci: automate iOS runner request-count gate for the Apple runner unwind (Phase 3 step c prep) Replaces the manual "run with --debug, hand-count the runner phases" check with an automated, committed assertion so the Phase 3 step (c) runner relocation (and future runner refactors) can prove byte-identical runner request behavior. - src/daemon/runner-request-count.ts: pure, unit-testable counter. Parses the daemon --debug diagnostics ndjson and counts the iOS-runner round-trip phases, plus baseline parse/compare logic. Owns RUNNER_ROUND_TRIP_PHASES as the single source of truth, now imported by request-router.ts (was a local const) so the in-process cost graft and the external counter never drift. - src/daemon/__tests__/runner-request-count.test.ts: 13 unit tests over synthetic ndjson fixtures (tolerant parse, counting, baseline parse/compare). Run in the normal unit suite; no hardware. - scripts/runner-request-count/: assertion harness (run.ts) + committed baseline (expected-counts.json). Drives the existing smoke-ios replay scenario with --debug in an isolated --state-dir, counts runner round-trips from daemon.log, and asserts against the baseline. --update regenerates the baseline. Infra hiccups are inconclusive (don't fail); only a real count drift fails. - .github/workflows/ios.yml: new "Assert iOS runner request count" step in the smoke-ios job, reusing the booted simulator. - package.json: `validate:runner-count` script. .fallowrc.json: harness entry. The baseline ships unarmed (established=false); the harness records observed counts (printed + uploaded as a test/artifacts artifact) without failing, so the maintainer arms it once from a real CI run. --- .fallowrc.json | 1 + .github/workflows/ios.yml | 10 + package.json | 1 + .../runner-request-count/expected-counts.json | 10 + scripts/runner-request-count/run.ts | 352 ++++++++++++++++++ .../__tests__/runner-request-count.test.ts | 191 ++++++++++ src/daemon/request-router.ts | 13 +- src/daemon/runner-request-count.ts | 220 +++++++++++ 8 files changed, 789 insertions(+), 9 deletions(-) create mode 100644 scripts/runner-request-count/expected-counts.json create mode 100644 scripts/runner-request-count/run.ts create mode 100644 src/daemon/__tests__/runner-request-count.test.ts create mode 100644 src/daemon/runner-request-count.ts diff --git a/.fallowrc.json b/.fallowrc.json index bea408f128..666aa8af3b 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -17,6 +17,7 @@ "src/daemon.ts", "src/utils/png-worker.ts", "scripts/patch-xcuitest-runner-icon.ts", + "scripts/runner-request-count/run.ts", "src/utils/update-check-entry.ts", "test/scripts/metro-prepare-packaged-smoke.mjs", "test/integration/*.test.ts", diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 25cbf56445..9af5192eaa 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -66,6 +66,16 @@ jobs: run: | node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator/01-settings.ad --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --retries 2 --artifacts-dir test/artifacts/replays-ios-simulator-smoke --report-junit test/artifacts/replays-ios-simulator-smoke.junit.xml + # Gate for runner refactors (Phase 3 step c): assert the iOS runner request + # count is unchanged vs. the committed baseline. Runs in its own isolated + # daemon (temp --state-dir) so it never contends with the smoke daemon's + # lease; `clean:daemon` first releases the shared daemon's UDID lease. A + # real count drift fails loudly; an infra hiccup is tolerated (see harness). + - name: Assert iOS runner request count + run: | + pnpm clean:daemon + node --experimental-strip-types scripts/runner-request-count/run.ts --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --prepare-timeout-ms "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --artifacts-dir test/artifacts/runner-request-count + - name: Run iOS physical device smoke replay if: env.IOS_UDID != '' env: diff --git a/package.json b/package.json index d259f73e6c..508968b43a 100644 --- a/package.json +++ b/package.json @@ -98,6 +98,7 @@ "perf": "node --experimental-strip-types scripts/perf/run.ts", "perf:ios": "node --experimental-strip-types scripts/perf/run.ts --platform ios", "perf:android": "node --experimental-strip-types scripts/perf/run.ts --platform android", + "validate:runner-count": "node --experimental-strip-types scripts/runner-request-count/run.ts", "lint": "oxlint . --deny-warnings", "format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills package.json tsconfig.json tsconfig.lib.json rslib.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'", "format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills package.json tsconfig.json tsconfig.lib.json rslib.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'", diff --git a/scripts/runner-request-count/expected-counts.json b/scripts/runner-request-count/expected-counts.json new file mode 100644 index 0000000000..d514114604 --- /dev/null +++ b/scripts/runner-request-count/expected-counts.json @@ -0,0 +1,10 @@ +{ + "$comment": "Expected iOS runner request-count baseline for the smoke-ios scenario. This gate proves a runner refactor (e.g. Phase 3 step c) does not add/drop runner round-trips. `established: false` means the gate is NOT armed yet: the harness records observed counts instead of asserting. To arm/regenerate, run on a host with a booted iOS simulator: `node --experimental-strip-types scripts/runner-request-count/run.ts --udid --update` (or read the values from the smoke-ios CI step log / uploaded artifact and edit this file), then commit it. Counts: runnerRoundTrips = ios_runner_command_send + ios_runner_readiness_preflight.", + "scenario": "test/integration/replays/ios/simulator/01-settings.ad", + "established": false, + "runnerRoundTrips": 0, + "byPhase": { + "ios_runner_command_send": 0, + "ios_runner_readiness_preflight": 0 + } +} diff --git a/scripts/runner-request-count/run.ts b/scripts/runner-request-count/run.ts new file mode 100644 index 0000000000..ad6dee341c --- /dev/null +++ b/scripts/runner-request-count/run.ts @@ -0,0 +1,352 @@ +#!/usr/bin/env node +/** + * iOS runner request-count gate. + * + * Drives a small, representative iOS replay scenario against a booted simulator + * with `--debug`, reads the per-request diagnostics ndjson the daemon appends to + * `/daemon.log`, counts the iOS-runner round-trip phases, and asserts + * the total is unchanged versus the committed baseline + * (`scripts/runner-request-count/expected-counts.json`). This proves a runner + * refactor (e.g. Phase 3 step c — relocating the shared Apple XCTest runner) + * adds or drops zero runner requests. + * + * Counting/assertion logic is the pure, unit-tested module + * `src/daemon/runner-request-count.ts`; this script is only orchestration + I/O. + * + * Usage: + * node --experimental-strip-types scripts/runner-request-count/run.ts \ + * --udid [--scenario ] [--artifacts-dir ] \ + * [--prepare-timeout-ms ] [--state-dir ] [--keep] [--strict] [--update] + * + * Modes: + * (default) assert observed counts == committed baseline (skips when unarmed). + * --update record observed counts into the committed baseline (arm/regenerate). + * + * Robustness: an infra hiccup (scenario fails to run, or zero round-trips + * captured) is reported as INCONCLUSIVE and does NOT fail the build unless + * `--strict` is passed; only a real count drift against an armed baseline fails. + * + * The CLI invocation defaults to running from source + * (`node --experimental-strip-types src/bin.ts`), matching the iOS workflow. + * Override with AGENT_DEVICE_RUNNER_COUNT_CLI (e.g. the built dist binary path). + */ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { runCmdSync } from '../../src/utils/exec.ts'; +import { + buildRunnerRequestCountBaseline, + compareRunnerCounts, + countRunnerRequests, + parseRunnerRequestCountBaseline, + RUNNER_ROUND_TRIP_PHASES, + type RunnerRequestCountBaseline, + type RunnerRequestCounts, +} from '../../src/daemon/runner-request-count.ts'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = path.resolve(HERE, '..', '..'); +const BASELINE_PATH = path.join(HERE, 'expected-counts.json'); +const DEFAULT_SCENARIO = 'test/integration/replays/ios/simulator/01-settings.ad'; +const DEFAULT_PREPARE_TIMEOUT_MS = 420_000; +const SCENARIO_TIMEOUT_MS = 600_000; +const MAX_BUFFER = 64 * 1024 * 1024; + +type HarnessConfig = { + mode: 'assert' | 'update'; + udid?: string; + scenario: string; + stateDir?: string; + artifactsDir?: string; + prepareTimeoutMs: number; + strict: boolean; + keep: boolean; +}; + +function log(msg: string): void { + process.stderr.write(`[runner-count] ${msg}\n`); +} + +function readValue(argv: string[], index: number, flag: string): string { + const value = argv[index]; + if (value === undefined) throw new Error(`Missing value for ${flag}`); + return value; +} + +function envPrepareTimeoutMs(): number { + const raw = process.env.AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS?.trim(); + const parsed = raw ? Number(raw) : NaN; + return Number.isInteger(parsed) && parsed > 0 ? parsed : DEFAULT_PREPARE_TIMEOUT_MS; +} + +// Uncovered CLI arg parser: fallow's CRAP score is inflated for scripts (no test +// coverage feeds the audit), so suppress the complexity finding here. +// fallow-ignore-next-line complexity +function parseArgs(argv: string[]): HarnessConfig { + const cfg: HarnessConfig = { + mode: 'assert', + scenario: DEFAULT_SCENARIO, + prepareTimeoutMs: envPrepareTimeoutMs(), + strict: false, + keep: false, + }; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]!; + if (a === '--update' || a === '--save') cfg.mode = 'update'; + else if (a === '--strict') cfg.strict = true; + else if (a === '--keep') cfg.keep = true; + else if (a === '--help' || a === '-h') { + process.stdout.write(HELP); + process.exit(0); + } else i = applyValueFlag(cfg, a, argv, i); + } + return cfg; +} + +// Apply a flag that consumes the next argv token; returns the advanced index. +// fallow-ignore-next-line complexity +function applyValueFlag(cfg: HarnessConfig, flag: string, argv: string[], i: number): number { + const value = readValue(argv, i + 1, flag); + switch (flag) { + case '--udid': + cfg.udid = value; + break; + case '--scenario': + cfg.scenario = value; + break; + case '--state-dir': + cfg.stateDir = path.resolve(value); + break; + case '--artifacts-dir': + cfg.artifactsDir = path.resolve(value); + break; + case '--prepare-timeout-ms': + cfg.prepareTimeoutMs = Number(value); + break; + default: + throw new Error(`Unknown flag: ${flag}`); + } + return i + 1; +} + +const HELP = `iOS runner request-count gate + + --udid Simulator UDID to drive (required for a real run). + --scenario Replay scenario (default: ${DEFAULT_SCENARIO}). + --artifacts-dir Where to write the replay + observed-count artifacts. + --state-dir Reuse an existing daemon state dir instead of a temp one. + --prepare-timeout-ms Runner prepare timeout (default ${DEFAULT_PREPARE_TIMEOUT_MS}). + --update | --save Record observed counts into the committed baseline. + --strict Fail (not warn) on inconclusive/infra outcomes. + --keep Keep the temp state dir after running. + -h, --help Show this help. +`; + +function cliArgv(): string[] { + const override = process.env.AGENT_DEVICE_RUNNER_COUNT_CLI?.trim(); + if (override) return override.split(/\s+/); + return ['--experimental-strip-types', path.join(REPO_ROOT, 'src', 'bin.ts')]; +} + +function runCli(args: string[], timeoutMs: number): { exitCode: number; stderr: string } { + const full = [...cliArgv(), ...args]; + try { + const result = runCmdSync(process.execPath, full, { + cwd: REPO_ROOT, + maxBuffer: MAX_BUFFER, + allowFailure: true, + timeoutMs, + }); + return { exitCode: result.exitCode, stderr: result.stderr }; + } catch (error) { + return { exitCode: -1, stderr: error instanceof Error ? error.message : String(error) }; + } +} + +function loadBaseline(): RunnerRequestCountBaseline { + const raw = fs.readFileSync(BASELINE_PATH, 'utf8'); + return parseRunnerRequestCountBaseline(JSON.parse(raw) as unknown); +} + +function writeBaseline(scenario: string, counts: RunnerRequestCounts): void { + const baseline = buildRunnerRequestCountBaseline(scenario, counts); + const doc = { + $comment: + 'Expected iOS runner request-count baseline for the smoke-ios scenario. ' + + 'Regenerate with: node --experimental-strip-types scripts/runner-request-count/run.ts --udid --update. ' + + 'runnerRoundTrips = ios_runner_command_send + ios_runner_readiness_preflight.', + ...baseline, + }; + fs.writeFileSync(BASELINE_PATH, `${JSON.stringify(doc, null, 2)}\n`); + log(`baseline updated: ${BASELINE_PATH}`); +} + +function recordObserved(cfg: HarnessConfig, counts: RunnerRequestCounts): void { + const doc = buildRunnerRequestCountBaseline(cfg.scenario, counts); + process.stdout.write(`${JSON.stringify(doc, null, 2)}\n`); + if (!cfg.artifactsDir) return; + try { + fs.mkdirSync(cfg.artifactsDir, { recursive: true }); + fs.writeFileSync( + path.join(cfg.artifactsDir, 'expected-counts.observed.json'), + `${JSON.stringify(doc, null, 2)}\n`, + ); + } catch (error) { + log(`warning: could not write observed-count artifact: ${String(error)}`); + } +} + +function describeCounts(counts: RunnerRequestCounts): string { + const phases = RUNNER_ROUND_TRIP_PHASES.map((p) => `${p}=${counts.byPhase[p]}`).join(', '); + return `runnerRoundTrips=${counts.runnerRoundTrips} (${phases})`; +} + +function inconclusive(cfg: HarnessConfig, reason: string): number { + log(`INCONCLUSIVE: ${reason}`); + if (cfg.strict) { + log('exiting non-zero because --strict was set'); + return 1; + } + log('treating as an infra hiccup (not a count drift); not failing the build'); + return 0; +} + +// Warm the runner WITHOUT --debug so prepare diagnostics never pollute the count. +function prepareRunner(cfg: HarnessConfig, udid: string, stateDir: string): boolean { + log('preparing iOS runner (no --debug)…'); + const prepare = runCli( + [ + 'prepare', + 'ios-runner', + '--platform', + 'ios', + '--udid', + udid, + '--timeout', + String(cfg.prepareTimeoutMs), + '--json', + '--state-dir', + stateDir, + ], + cfg.prepareTimeoutMs + 120_000, + ); + return prepare.exitCode === 0; +} + +// Drive the scenario with --debug (single attempt for a deterministic count) and +// return the run's exit code + the round-trip counts read from the daemon log. +function runScenario( + cfg: HarnessConfig, + udid: string, + stateDir: string, +): { exitCode: number; observed: RunnerRequestCounts } { + // Truncate daemon.log so we count ONLY the scenario's --debug round-trips. + const logPath = path.join(stateDir, 'daemon.log'); + try { + fs.writeFileSync(logPath, ''); + } catch { + /* fresh state dir may not have a log yet; the daemon recreates it */ + } + + log('running scenario with --debug (single attempt)…'); + const args = [ + 'test', + cfg.scenario, + '--udid', + udid, + '--debug', + '--retries', + '0', + '--json', + '--state-dir', + stateDir, + ]; + if (cfg.artifactsDir) args.push('--artifacts-dir', path.join(cfg.artifactsDir, 'replay')); + const exitCode = runCli(args, SCENARIO_TIMEOUT_MS).exitCode; + + let logText = ''; + try { + logText = fs.readFileSync(logPath, 'utf8'); + } catch { + /* no log => zero counts, handled as inconclusive downstream */ + } + return { exitCode, observed: countRunnerRequests(logText) }; +} + +// Assert the observed counts against the committed baseline. Infra hiccups +// (failed run / zero captures) are inconclusive; only a real drift fails. +// fallow-ignore-next-line complexity +function assertObserved( + cfg: HarnessConfig, + exitCode: number, + observed: RunnerRequestCounts, +): number { + if (exitCode !== 0) { + return inconclusive(cfg, `scenario run failed (exit ${exitCode}); likely simulator/infra`); + } + if (observed.runnerRoundTrips === 0) { + return inconclusive( + cfg, + 'scenario passed but zero runner round-trips were captured (likely a capture/infra issue, not a real drift)', + ); + } + const comparison = compareRunnerCounts(loadBaseline(), observed); + if (comparison.status === 'unarmed') { + log('GATE NOT ARMED: committed baseline has established=false.'); + log('Arm it by committing the observed counts above (or re-run with --update).'); + return 0; + } + if (comparison.status === 'match') { + log(`MATCH: ${describeCounts(observed)} == committed baseline. No runner request drift.`); + return 0; + } + log('MISMATCH: iOS runner request count drifted from the committed baseline:'); + for (const diff of comparison.differences) { + log(` ${diff.key}: expected ${diff.expected}, got ${diff.actual}`); + } + log('If this drift is intentional, regenerate the baseline with --update and commit it.'); + return 1; +} + +function runGate(cfg: HarnessConfig, stateDir: string): number { + if (!cfg.udid) return inconclusive(cfg, 'no --udid provided; cannot drive a simulator scenario'); + if (!prepareRunner(cfg, cfg.udid, stateDir)) { + return inconclusive(cfg, 'prepare ios-runner failed'); + } + const { exitCode, observed } = runScenario(cfg, cfg.udid, stateDir); + log(`observed: ${describeCounts(observed)}`); + recordObserved(cfg, observed); + if (cfg.mode === 'update') { + writeBaseline(cfg.scenario, observed); + log('OK: baseline recorded (update mode)'); + return 0; + } + return assertObserved(cfg, exitCode, observed); +} + +function teardown(cfg: HarnessConfig, stateDir: string, createdStateDir: boolean): void { + runCli(['close', '--shutdown', '--state-dir', stateDir], 60_000); + if (!createdStateDir || cfg.keep) return; + try { + fs.rmSync(stateDir, { recursive: true, force: true }); + } catch { + /* best-effort */ + } +} + +function main(): number { + const cfg = parseArgs(process.argv.slice(2)); + const createdStateDir = !cfg.stateDir; + const stateDir = + cfg.stateDir ?? fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-runner-count-')); + log(`scenario: ${cfg.scenario}`); + log(`state-dir: ${stateDir}${createdStateDir ? ' (temp)' : ''}`); + try { + return runGate(cfg, stateDir); + } finally { + teardown(cfg, stateDir, createdStateDir); + } +} + +process.exit(main()); diff --git a/src/daemon/__tests__/runner-request-count.test.ts b/src/daemon/__tests__/runner-request-count.test.ts new file mode 100644 index 0000000000..5ee913002f --- /dev/null +++ b/src/daemon/__tests__/runner-request-count.test.ts @@ -0,0 +1,191 @@ +import { test, expect } from 'vitest'; +import { + buildRunnerRequestCountBaseline, + compareRunnerCounts, + countRunnerRequests, + emptyRunnerRequestCounts, + parseDiagnosticNdjson, + parseRunnerRequestCountBaseline, + RUNNER_ROUND_TRIP_PHASES, + type RunnerRequestCountBaseline, +} from '../runner-request-count.ts'; + +// A representative daemon `--debug` daemon.log capture: plain (non-JSON) daemon +// log lines interleaved with diagnostic ndjson, plus a stderr-prefixed line, a +// blank line, and a malformed JSON line — every one of which the tolerant parser +// must skip without throwing. +function ndjsonLine(phase: string, extra: Record = {}): string { + return JSON.stringify({ + ts: '2026-06-30T00:00:00.000Z', + level: 'info', + phase, + session: 'gate', + requestId: 'req-1', + command: 'click', + durationMs: 12, + ...extra, + }); +} + +const SAMPLE_LOG = [ + '[daemon] started, pid 4242', + ndjsonLine('ios_runner_readiness_preflight'), + ndjsonLine('ios_runner_command_send', { command: 'open' }), + '', + ndjsonLine('ios_runner_command_send', { command: 'click' }), + // Not a round-trip — the daemon excludes these explicitly. + ndjsonLine('ios_runner_readiness_preflight_skipped'), + // Unrelated phase from another subsystem. + ndjsonLine('android_adb_shell'), + '{ this is not valid json', + `[agent-device][diag] ${ndjsonLine('ios_runner_command_send', { command: 'back' })}`, + '[daemon] request complete', +].join('\n'); + +test('parseDiagnosticNdjson skips non-JSON, blank, malformed, and phaseless lines', () => { + const events = parseDiagnosticNdjson(SAMPLE_LOG); + // 5 well-formed diagnostic events (4 runner phases + 1 unrelated + 1 skipped = 6), + // including the stderr-prefixed one; the plain log lines and bad JSON are dropped. + expect(events.map((e) => e.phase)).toEqual([ + 'ios_runner_readiness_preflight', + 'ios_runner_command_send', + 'ios_runner_command_send', + 'ios_runner_readiness_preflight_skipped', + 'android_adb_shell', + 'ios_runner_command_send', + ]); +}); + +test('parseDiagnosticNdjson strips the stderr diagnostic prefix', () => { + const events = parseDiagnosticNdjson( + `[agent-device][diag] ${ndjsonLine('ios_runner_command_send')}`, + ); + expect(events).toHaveLength(1); + expect(events[0]?.phase).toBe('ios_runner_command_send'); + expect(events[0]?.command).toBe('click'); +}); + +test('countRunnerRequests counts only the two round-trip phases (from text)', () => { + const counts = countRunnerRequests(SAMPLE_LOG); + expect(counts.runnerRoundTrips).toBe(4); + expect(counts.byPhase).toEqual({ + ios_runner_command_send: 3, + ios_runner_readiness_preflight: 1, + }); +}); + +test('countRunnerRequests matches the in-process counting semantics (3 round-trips)', () => { + // Mirrors request-router-cost.test.ts: 1 preflight + 2 command_send + a skipped + // marker + an unrelated phase => 3 runner round-trips. + const events = parseDiagnosticNdjson( + [ + ndjsonLine('ios_runner_readiness_preflight'), + ndjsonLine('ios_runner_command_send'), + ndjsonLine('ios_runner_command_send'), + ndjsonLine('ios_runner_readiness_preflight_skipped'), + ndjsonLine('snapshot_capture'), + ].join('\n'), + ); + expect(countRunnerRequests(events).runnerRoundTrips).toBe(3); +}); + +test('countRunnerRequests on empty input is zeroed', () => { + expect(countRunnerRequests('')).toEqual(emptyRunnerRequestCounts()); + expect(emptyRunnerRequestCounts().runnerRoundTrips).toBe(0); +}); + +test('RUNNER_ROUND_TRIP_PHASES is the documented pair', () => { + expect([...RUNNER_ROUND_TRIP_PHASES]).toEqual([ + 'ios_runner_command_send', + 'ios_runner_readiness_preflight', + ]); +}); + +// --- baseline parse + compare ------------------------------------------------ + +const ARMED_BASELINE: RunnerRequestCountBaseline = { + scenario: 'test/integration/replays/ios/simulator/01-settings.ad', + established: true, + runnerRoundTrips: 4, + byPhase: { ios_runner_command_send: 3, ios_runner_readiness_preflight: 1 }, +}; + +test('parseRunnerRequestCountBaseline validates and ignores unknown keys', () => { + const parsed = parseRunnerRequestCountBaseline({ + $comment: 'regenerate with --update', + scenario: ARMED_BASELINE.scenario, + established: true, + runnerRoundTrips: 4, + byPhase: { ios_runner_command_send: 3, ios_runner_readiness_preflight: 1 }, + }); + expect(parsed).toEqual(ARMED_BASELINE); +}); + +test('parseRunnerRequestCountBaseline treats missing/false established as unarmed', () => { + const parsed = parseRunnerRequestCountBaseline({ + scenario: ARMED_BASELINE.scenario, + runnerRoundTrips: 0, + byPhase: { ios_runner_command_send: 0, ios_runner_readiness_preflight: 0 }, + }); + expect(parsed.established).toBe(false); +}); + +test('parseRunnerRequestCountBaseline rejects malformed payloads', () => { + expect(() => parseRunnerRequestCountBaseline(null)).toThrow(/must be a JSON object/); + expect(() => parseRunnerRequestCountBaseline({ byPhase: {} })).toThrow(/scenario/); + expect(() => parseRunnerRequestCountBaseline({ scenario: 'x', runnerRoundTrips: 1 })).toThrow( + /byPhase/, + ); + expect(() => + parseRunnerRequestCountBaseline({ + scenario: 'x', + runnerRoundTrips: -1, + byPhase: { ios_runner_command_send: 0, ios_runner_readiness_preflight: 0 }, + }), + ).toThrow(/non-negative integer/); +}); + +test('compareRunnerCounts skips assertion when the baseline is unarmed', () => { + const unarmed = parseRunnerRequestCountBaseline({ + scenario: ARMED_BASELINE.scenario, + established: false, + runnerRoundTrips: 0, + byPhase: { ios_runner_command_send: 0, ios_runner_readiness_preflight: 0 }, + }); + expect(compareRunnerCounts(unarmed, countRunnerRequests(SAMPLE_LOG))).toEqual({ + status: 'unarmed', + }); +}); + +test('compareRunnerCounts matches identical counts', () => { + expect(compareRunnerCounts(ARMED_BASELINE, countRunnerRequests(SAMPLE_LOG))).toEqual({ + status: 'match', + }); +}); + +test('compareRunnerCounts reports per-key differences on drift', () => { + // Drop one command_send (a runner refactor that removed a request). + const drifted = countRunnerRequests( + [ + ndjsonLine('ios_runner_readiness_preflight'), + ndjsonLine('ios_runner_command_send'), + ndjsonLine('ios_runner_command_send'), + ].join('\n'), + ); + const result = compareRunnerCounts(ARMED_BASELINE, drifted); + expect(result).toEqual({ + status: 'mismatch', + differences: [ + { key: 'runnerRoundTrips', expected: 4, actual: 3 }, + { key: 'ios_runner_command_send', expected: 3, actual: 2 }, + ], + }); +}); + +test('buildRunnerRequestCountBaseline arms a baseline from observed counts', () => { + const baseline = buildRunnerRequestCountBaseline( + ARMED_BASELINE.scenario, + countRunnerRequests(SAMPLE_LOG), + ); + expect(baseline).toEqual(ARMED_BASELINE); +}); diff --git a/src/daemon/request-router.ts b/src/daemon/request-router.ts index 66a4d2ac1f..7fa16bec03 100644 --- a/src/daemon/request-router.ts +++ b/src/daemon/request-router.ts @@ -40,20 +40,15 @@ import { import { canRunReplayScopedAction } from './daemon-command-registry.ts'; import { createAgentBrowserWebProvider } from '../platforms/web/agent-browser-provider.ts'; import type { LeaseLifecycleProvider } from './handlers/lease.ts'; +// Single source of truth for which diagnostic phases count as a real iOS-runner +// round-trip — shared with the external ndjson counter used by the +// runner-request-count CI gate (`scripts/runner-request-count/`). +import { RUNNER_ROUND_TRIP_PHASES } from './runner-request-count.ts'; // --------------------------------------------------------------------------- // Request handler API // --------------------------------------------------------------------------- -// Diagnostic phases emitted once per real iOS-runner round-trip. `..._command_send` -// is the command itself; `..._readiness_preflight` is the pre-command uptime probe -// (a real network round-trip). The `..._skipped` / `..._recovered` markers do NOT -// hit the runner and are intentionally excluded. -const RUNNER_ROUND_TRIP_PHASES = [ - 'ios_runner_command_send', - 'ios_runner_readiness_preflight', -] as const; - export type RequestRouterDeps = { logPath: string; stateDir?: string; diff --git a/src/daemon/runner-request-count.ts b/src/daemon/runner-request-count.ts new file mode 100644 index 0000000000..c221d21f95 --- /dev/null +++ b/src/daemon/runner-request-count.ts @@ -0,0 +1,220 @@ +/** + * Hardware-free counter for "iOS runner requests" in the daemon `--debug` + * diagnostics ndjson stream. + * + * The daemon emits one diagnostic event per real iOS-runner round-trip + * (`emitDiagnostic` in `../utils/diagnostics.ts`). When a request is in debug + * mode those events are streamed as one JSON object per line into the daemon + * log (`/daemon.log`). This module parses that stream and counts the + * round-trip phases, so the runner request count can be asserted in CI without + * re-implementing the hand-counting an operator used to do by reading the + * ndjson by eye. + * + * `RUNNER_ROUND_TRIP_PHASES` is the single source of truth shared by the + * in-process cost graft (`request-router.ts` `buildResponseCost`) and this + * external ndjson counter, so the two can never drift on which phases count. + */ + +// Diagnostic phases emitted once per real iOS-runner round-trip. `..._command_send` +// is the command itself; `..._readiness_preflight` is the pre-command uptime probe +// (a real network round-trip). The `..._skipped` / `..._recovered` markers do NOT +// hit the runner and are intentionally excluded. +export const RUNNER_ROUND_TRIP_PHASES = [ + 'ios_runner_command_send', + 'ios_runner_readiness_preflight', +] as const; + +export type RunnerRoundTripPhase = (typeof RUNNER_ROUND_TRIP_PHASES)[number]; + +/** + * A single parsed line of the daemon `--debug` diagnostics ndjson stream. Only + * the fields the counter and its drift reporting need are retained; the full + * record carries more (ts/level/requestId/session/durationMs). + */ +export type ParsedDiagnosticEvent = { + phase: string; + command?: string; +}; + +export type RunnerRequestCounts = { + runnerRoundTrips: number; + byPhase: Record; +}; + +// The stderr fallback path in `emitDiagnostic` prefixes each ndjson line with +// this tag. The daemon-log path does not, but we tolerate both so the counter +// works against captured stderr too. +const STDERR_DIAGNOSTIC_PREFIX = '[agent-device][diag] '; + +/** + * Tolerant ndjson parser: the daemon log interleaves plain log text with the + * diagnostic ndjson lines, so non-JSON lines, blank lines, malformed JSON, and + * objects without a `phase` are skipped rather than throwing. + */ +export function parseDiagnosticNdjson(text: string): ParsedDiagnosticEvent[] { + const events: ParsedDiagnosticEvent[] = []; + for (const rawLine of text.split(/\r?\n/)) { + let line = rawLine.trim(); + if (line.length === 0) continue; + if (line.startsWith(STDERR_DIAGNOSTIC_PREFIX)) { + line = line.slice(STDERR_DIAGNOSTIC_PREFIX.length).trim(); + } + // Fast-skip plain daemon log lines that are not JSON objects. + if (!line.startsWith('{')) continue; + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch { + continue; + } + const event = toDiagnosticEvent(parsed); + if (event) events.push(event); + } + return events; +} + +function toDiagnosticEvent(value: unknown): ParsedDiagnosticEvent | null { + if (!value || typeof value !== 'object' || Array.isArray(value)) return null; + const record = value as Record; + const phase = record.phase; + if (typeof phase !== 'string') return null; + const command = record.command; + return typeof command === 'string' ? { phase, command } : { phase }; +} + +export function emptyRunnerRequestCounts(): RunnerRequestCounts { + return { + runnerRoundTrips: 0, + byPhase: { ios_runner_command_send: 0, ios_runner_readiness_preflight: 0 }, + }; +} + +/** + * Count iOS-runner round-trips the way the daemon itself does: tally events + * whose phase is one of `RUNNER_ROUND_TRIP_PHASES`. Accepts raw ndjson text or + * already-parsed events. + */ +export function countRunnerRequests( + input: string | readonly ParsedDiagnosticEvent[], +): RunnerRequestCounts { + const events = typeof input === 'string' ? parseDiagnosticNdjson(input) : input; + const counts = emptyRunnerRequestCounts(); + for (const event of events) { + if (isRunnerRoundTripPhase(event.phase)) { + counts.byPhase[event.phase] += 1; + counts.runnerRoundTrips += 1; + } + } + return counts; +} + +function isRunnerRoundTripPhase(phase: string): phase is RunnerRoundTripPhase { + return (RUNNER_ROUND_TRIP_PHASES as readonly string[]).includes(phase); +} + +// --------------------------------------------------------------------------- +// Committed baseline + assertion logic (pure, so the CI harness only does I/O) +// --------------------------------------------------------------------------- + +/** + * The committed expected-count baseline. `established: false` means the gate has + * not been armed yet (no real simulator run has recorded the counts), so the + * harness records the observed counts instead of asserting. + */ +export type RunnerRequestCountBaseline = RunnerRequestCounts & { + scenario: string; + established: boolean; +}; + +export type RunnerCountDifference = { + key: 'runnerRoundTrips' | RunnerRoundTripPhase; + expected: number; + actual: number; +}; + +export type RunnerCountComparison = + | { status: 'unarmed' } + | { status: 'match' } + | { status: 'mismatch'; differences: RunnerCountDifference[] }; + +export function buildRunnerRequestCountBaseline( + scenario: string, + counts: RunnerRequestCounts, +): RunnerRequestCountBaseline { + return { + scenario, + established: true, + runnerRoundTrips: counts.runnerRoundTrips, + byPhase: { ...counts.byPhase }, + }; +} + +/** + * Validate an untrusted baseline payload (read from disk) into a typed baseline. + * Unknown keys (e.g. a documentation `$comment`) are ignored. + */ +export function parseRunnerRequestCountBaseline(value: unknown): RunnerRequestCountBaseline { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('Runner request-count baseline must be a JSON object.'); + } + const record = value as Record; + const scenario = record.scenario; + if (typeof scenario !== 'string' || scenario.length === 0) { + throw new Error('Runner request-count baseline is missing a "scenario" string.'); + } + const byPhaseRaw = record.byPhase; + if (!byPhaseRaw || typeof byPhaseRaw !== 'object' || Array.isArray(byPhaseRaw)) { + throw new Error('Runner request-count baseline is missing a "byPhase" object.'); + } + const byPhaseRecord = byPhaseRaw as Record; + const byPhase = emptyRunnerRequestCounts().byPhase; + for (const phase of RUNNER_ROUND_TRIP_PHASES) { + byPhase[phase] = asCount(byPhaseRecord[phase], `byPhase.${phase}`); + } + return { + scenario, + established: record.established === true, + runnerRoundTrips: asCount(record.runnerRoundTrips, 'runnerRoundTrips'), + byPhase, + }; +} + +function asCount(value: unknown, field: string): number { + if (typeof value !== 'number' || !Number.isInteger(value) || value < 0) { + throw new Error( + `Runner request-count baseline field "${field}" must be a non-negative integer.`, + ); + } + return value; +} + +/** + * Compare observed counts against the committed baseline. Returns `unarmed` + * when the baseline has not been established yet (the caller should record, not + * fail), `match` when every count is identical, or `mismatch` with the exact + * per-key differences a runner refactor introduced. + */ +export function compareRunnerCounts( + baseline: RunnerRequestCountBaseline, + observed: RunnerRequestCounts, +): RunnerCountComparison { + if (!baseline.established) return { status: 'unarmed' }; + const differences: RunnerCountDifference[] = []; + if (baseline.runnerRoundTrips !== observed.runnerRoundTrips) { + differences.push({ + key: 'runnerRoundTrips', + expected: baseline.runnerRoundTrips, + actual: observed.runnerRoundTrips, + }); + } + for (const phase of RUNNER_ROUND_TRIP_PHASES) { + if (baseline.byPhase[phase] !== observed.byPhase[phase]) { + differences.push({ + key: phase, + expected: baseline.byPhase[phase], + actual: observed.byPhase[phase], + }); + } + } + return differences.length === 0 ? { status: 'match' } : { status: 'mismatch', differences }; +}