ci: automate iOS runner request-count gate for the Apple runner unwind (Phase 3 step c prep)

.fallowrc.json

@@ -17,6 +17,7 @@
     "src/daemon.ts",
     "src/utils/png-worker.ts",
     "scripts/patch-xcuitest-runner-icon.ts",
+    "scripts/runner-request-count/run.ts",
     "src/utils/update-check-entry.ts",
     "test/scripts/metro-prepare-packaged-smoke.mjs",
     "test/integration/*.test.ts",

.github/workflows/ios.yml

@@ -66,6 +66,16 @@ jobs:
         run: |
           node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator/01-settings.ad --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --retries 2 --artifacts-dir test/artifacts/replays-ios-simulator-smoke --report-junit test/artifacts/replays-ios-simulator-smoke.junit.xml
 
+      # Gate for runner refactors (Phase 3 step c): assert the iOS runner request
+      # count is unchanged vs. the committed baseline. Runs in its own isolated
+      # daemon (temp --state-dir) so it never contends with the smoke daemon's
+      # lease; `clean:daemon` first releases the shared daemon's UDID lease. A
+      # real count drift fails loudly; an infra hiccup is tolerated (see harness).
+      - name: Assert iOS runner request count
+        run: |
+          pnpm clean:daemon
+          node --experimental-strip-types scripts/runner-request-count/run.ts --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --prepare-timeout-ms "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --artifacts-dir test/artifacts/runner-request-count
+
       - name: Run iOS physical device smoke replay
         if: env.IOS_UDID != ''
         env:

package.json

@@ -98,6 +98,7 @@
     "perf": "node --experimental-strip-types scripts/perf/run.ts",
     "perf:ios": "node --experimental-strip-types scripts/perf/run.ts --platform ios",
     "perf:android": "node --experimental-strip-types scripts/perf/run.ts --platform android",
+    "validate:runner-count": "node --experimental-strip-types scripts/runner-request-count/run.ts",
     "lint": "oxlint . --deny-warnings",
     "format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills package.json tsconfig.json tsconfig.lib.json rslib.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
     "format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills package.json tsconfig.json tsconfig.lib.json rslib.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",

scripts/runner-request-count/expected-counts.json

@@ -0,0 +1,10 @@
+{
+  "$comment": "Expected iOS runner request-count baseline for the smoke-ios scenario. This gate proves a runner refactor (e.g. Phase 3 step c) does not add/drop runner round-trips. `established: false` means the gate is NOT armed yet: the harness records observed counts instead of asserting. To arm/regenerate, run on a host with a booted iOS simulator: `node --experimental-strip-types scripts/runner-request-count/run.ts --udid <UDID> --update` (or read the values from the smoke-ios CI step log / uploaded artifact and edit this file), then commit it. Counts: runnerRoundTrips = ios_runner_command_send + ios_runner_readiness_preflight.",
+  "scenario": "test/integration/replays/ios/simulator/01-settings.ad",
+  "established": false,
+  "runnerRoundTrips": 0,
+  "byPhase": {
+    "ios_runner_command_send": 0,
+    "ios_runner_readiness_preflight": 0
+  }
+}

scripts/runner-request-count/run.ts

@@ -0,0 +1,352 @@
+#!/usr/bin/env node
+/**
+ * iOS runner request-count gate.
+ *
+ * Drives a small, representative iOS replay scenario against a booted simulator
+ * with `--debug`, reads the per-request diagnostics ndjson the daemon appends to
+ * `<state-dir>/daemon.log`, counts the iOS-runner round-trip phases, and asserts
+ * the total is unchanged versus the committed baseline
+ * (`scripts/runner-request-count/expected-counts.json`). This proves a runner
+ * refactor (e.g. Phase 3 step c — relocating the shared Apple XCTest runner)
+ * adds or drops zero runner requests.
+ *
+ * Counting/assertion logic is the pure, unit-tested module
+ * `src/daemon/runner-request-count.ts`; this script is only orchestration + I/O.
+ *
+ * Usage:
+ *   node --experimental-strip-types scripts/runner-request-count/run.ts \
+ *     --udid <UDID> [--scenario <path>] [--artifacts-dir <dir>] \
+ *     [--prepare-timeout-ms <ms>] [--state-dir <dir>] [--keep] [--strict] [--update]
+ *
+ * Modes:
+ *   (default)  assert observed counts == committed baseline (skips when unarmed).
+ *   --update   record observed counts into the committed baseline (arm/regenerate).
+ *
+ * Robustness: an infra hiccup (scenario fails to run, or zero round-trips
+ * captured) is reported as INCONCLUSIVE and does NOT fail the build unless
+ * `--strict` is passed; only a real count drift against an armed baseline fails.
+ *
+ * The CLI invocation defaults to running from source
+ * (`node --experimental-strip-types src/bin.ts`), matching the iOS workflow.
+ * Override with AGENT_DEVICE_RUNNER_COUNT_CLI (e.g. the built dist binary path).
+ */
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { runCmdSync } from '../../src/utils/exec.ts';
+import {
+  buildRunnerRequestCountBaseline,
+  compareRunnerCounts,
+  countRunnerRequests,
+  parseRunnerRequestCountBaseline,
+  RUNNER_ROUND_TRIP_PHASES,
+  type RunnerRequestCountBaseline,
+  type RunnerRequestCounts,
+} from '../../src/daemon/runner-request-count.ts';
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const REPO_ROOT = path.resolve(HERE, '..', '..');
+const BASELINE_PATH = path.join(HERE, 'expected-counts.json');
+const DEFAULT_SCENARIO = 'test/integration/replays/ios/simulator/01-settings.ad';
+const DEFAULT_PREPARE_TIMEOUT_MS = 420_000;
+const SCENARIO_TIMEOUT_MS = 600_000;
+const MAX_BUFFER = 64 * 1024 * 1024;
+
+type HarnessConfig = {
+  mode: 'assert' | 'update';
+  udid?: string;
+  scenario: string;
+  stateDir?: string;
+  artifactsDir?: string;
+  prepareTimeoutMs: number;
+  strict: boolean;
+  keep: boolean;
+};
+
+function log(msg: string): void {
+  process.stderr.write(`[runner-count] ${msg}\n`);
+}
+
+function readValue(argv: string[], index: number, flag: string): string {
+  const value = argv[index];
+  if (value === undefined) throw new Error(`Missing value for ${flag}`);
+  return value;
+}
+
+function envPrepareTimeoutMs(): number {
+  const raw = process.env.AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS?.trim();
+  const parsed = raw ? Number(raw) : NaN;
+  return Number.isInteger(parsed) && parsed > 0 ? parsed : DEFAULT_PREPARE_TIMEOUT_MS;
+}
+
+// Uncovered CLI arg parser: fallow's CRAP score is inflated for scripts (no test
+// coverage feeds the audit), so suppress the complexity finding here.
+// fallow-ignore-next-line complexity
+function parseArgs(argv: string[]): HarnessConfig {
+  const cfg: HarnessConfig = {
+    mode: 'assert',
+    scenario: DEFAULT_SCENARIO,
+    prepareTimeoutMs: envPrepareTimeoutMs(),
+    strict: false,
+    keep: false,
+  };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i]!;
+    if (a === '--update' || a === '--save') cfg.mode = 'update';
+    else if (a === '--strict') cfg.strict = true;
+    else if (a === '--keep') cfg.keep = true;
+    else if (a === '--help' || a === '-h') {
+      process.stdout.write(HELP);
+      process.exit(0);
+    } else i = applyValueFlag(cfg, a, argv, i);
+  }
+  return cfg;
+}
+
+// Apply a flag that consumes the next argv token; returns the advanced index.
+// fallow-ignore-next-line complexity
+function applyValueFlag(cfg: HarnessConfig, flag: string, argv: string[], i: number): number {
+  const value = readValue(argv, i + 1, flag);
+  switch (flag) {
+    case '--udid':
+      cfg.udid = value;
+      break;
+    case '--scenario':
+      cfg.scenario = value;
+      break;
+    case '--state-dir':
+      cfg.stateDir = path.resolve(value);
+      break;
+    case '--artifacts-dir':
+      cfg.artifactsDir = path.resolve(value);
+      break;
+    case '--prepare-timeout-ms':
+      cfg.prepareTimeoutMs = Number(value);
+      break;
+    default:
+      throw new Error(`Unknown flag: ${flag}`);
+  }
+  return i + 1;
+}
+
+const HELP = `iOS runner request-count gate
+
+  --udid <UDID>            Simulator UDID to drive (required for a real run).
+  --scenario <path>        Replay scenario (default: ${DEFAULT_SCENARIO}).
+  --artifacts-dir <dir>    Where to write the replay + observed-count artifacts.
+  --state-dir <dir>        Reuse an existing daemon state dir instead of a temp one.
+  --prepare-timeout-ms <n> Runner prepare timeout (default ${DEFAULT_PREPARE_TIMEOUT_MS}).
+  --update | --save        Record observed counts into the committed baseline.
+  --strict                 Fail (not warn) on inconclusive/infra outcomes.
+  --keep                   Keep the temp state dir after running.
+  -h, --help               Show this help.
+`;
+
+function cliArgv(): string[] {
+  const override = process.env.AGENT_DEVICE_RUNNER_COUNT_CLI?.trim();
+  if (override) return override.split(/\s+/);
+  return ['--experimental-strip-types', path.join(REPO_ROOT, 'src', 'bin.ts')];
+}
+
+function runCli(args: string[], timeoutMs: number): { exitCode: number; stderr: string } {
+  const full = [...cliArgv(), ...args];
+  try {
+    const result = runCmdSync(process.execPath, full, {
+      cwd: REPO_ROOT,
+      maxBuffer: MAX_BUFFER,
+      allowFailure: true,
+      timeoutMs,
+    });
+    return { exitCode: result.exitCode, stderr: result.stderr };
+  } catch (error) {
+    return { exitCode: -1, stderr: error instanceof Error ? error.message : String(error) };
+  }
+}
+
+function loadBaseline(): RunnerRequestCountBaseline {
+  const raw = fs.readFileSync(BASELINE_PATH, 'utf8');
+  return parseRunnerRequestCountBaseline(JSON.parse(raw) as unknown);
+}
+
+function writeBaseline(scenario: string, counts: RunnerRequestCounts): void {
+  const baseline = buildRunnerRequestCountBaseline(scenario, counts);
+  const doc = {
+    $comment:
+      'Expected iOS runner request-count baseline for the smoke-ios scenario. ' +
+      'Regenerate with: node --experimental-strip-types scripts/runner-request-count/run.ts --udid <UDID> --update. ' +
+      'runnerRoundTrips = ios_runner_command_send + ios_runner_readiness_preflight.',
+    ...baseline,
+  };
+  fs.writeFileSync(BASELINE_PATH, `${JSON.stringify(doc, null, 2)}\n`);
+  log(`baseline updated: ${BASELINE_PATH}`);
+}
+
+function recordObserved(cfg: HarnessConfig, counts: RunnerRequestCounts): void {
+  const doc = buildRunnerRequestCountBaseline(cfg.scenario, counts);
+  process.stdout.write(`${JSON.stringify(doc, null, 2)}\n`);
+  if (!cfg.artifactsDir) return;
+  try {
+    fs.mkdirSync(cfg.artifactsDir, { recursive: true });
+    fs.writeFileSync(
+      path.join(cfg.artifactsDir, 'expected-counts.observed.json'),
+      `${JSON.stringify(doc, null, 2)}\n`,
+    );
+  } catch (error) {
+    log(`warning: could not write observed-count artifact: ${String(error)}`);
+  }
+}
+
+function describeCounts(counts: RunnerRequestCounts): string {
+  const phases = RUNNER_ROUND_TRIP_PHASES.map((p) => `${p}=${counts.byPhase[p]}`).join(', ');
+  return `runnerRoundTrips=${counts.runnerRoundTrips} (${phases})`;
+}
+
+function inconclusive(cfg: HarnessConfig, reason: string): number {
+  log(`INCONCLUSIVE: ${reason}`);
+  if (cfg.strict) {
+    log('exiting non-zero because --strict was set');
+    return 1;
+  }
+  log('treating as an infra hiccup (not a count drift); not failing the build');
+  return 0;
+}
+
+// Warm the runner WITHOUT --debug so prepare diagnostics never pollute the count.
+function prepareRunner(cfg: HarnessConfig, udid: string, stateDir: string): boolean {
+  log('preparing iOS runner (no --debug)…');
+  const prepare = runCli(
+    [
+      'prepare',
+      'ios-runner',
+      '--platform',
+      'ios',
+      '--udid',
+      udid,
+      '--timeout',
+      String(cfg.prepareTimeoutMs),
+      '--json',
+      '--state-dir',
+      stateDir,
+    ],
+    cfg.prepareTimeoutMs + 120_000,
+  );
+  return prepare.exitCode === 0;
+}
+
+// Drive the scenario with --debug (single attempt for a deterministic count) and
+// return the run's exit code + the round-trip counts read from the daemon log.
+function runScenario(
+  cfg: HarnessConfig,
+  udid: string,
+  stateDir: string,
+): { exitCode: number; observed: RunnerRequestCounts } {
+  // Truncate daemon.log so we count ONLY the scenario's --debug round-trips.
+  const logPath = path.join(stateDir, 'daemon.log');
+  try {
+    fs.writeFileSync(logPath, '');
+  } catch {
+    /* fresh state dir may not have a log yet; the daemon recreates it */
+  }
+
+  log('running scenario with --debug (single attempt)…');
+  const args = [
+    'test',
+    cfg.scenario,
+    '--udid',
+    udid,
+    '--debug',
+    '--retries',
+    '0',
+    '--json',
+    '--state-dir',
+    stateDir,
+  ];
+  if (cfg.artifactsDir) args.push('--artifacts-dir', path.join(cfg.artifactsDir, 'replay'));
+  const exitCode = runCli(args, SCENARIO_TIMEOUT_MS).exitCode;
+
+  let logText = '';
+  try {
+    logText = fs.readFileSync(logPath, 'utf8');
+  } catch {
+    /* no log => zero counts, handled as inconclusive downstream */
+  }
+  return { exitCode, observed: countRunnerRequests(logText) };
+}
+
+// Assert the observed counts against the committed baseline. Infra hiccups
+// (failed run / zero captures) are inconclusive; only a real drift fails.
+// fallow-ignore-next-line complexity
+function assertObserved(
+  cfg: HarnessConfig,
+  exitCode: number,
+  observed: RunnerRequestCounts,
+): number {
+  if (exitCode !== 0) {
+    return inconclusive(cfg, `scenario run failed (exit ${exitCode}); likely simulator/infra`);
+  }
+  if (observed.runnerRoundTrips === 0) {
+    return inconclusive(
+      cfg,
+      'scenario passed but zero runner round-trips were captured (likely a capture/infra issue, not a real drift)',
+    );
+  }
+  const comparison = compareRunnerCounts(loadBaseline(), observed);
+  if (comparison.status === 'unarmed') {
+    log('GATE NOT ARMED: committed baseline has established=false.');
+    log('Arm it by committing the observed counts above (or re-run with --update).');
+    return 0;
+  }
+  if (comparison.status === 'match') {
+    log(`MATCH: ${describeCounts(observed)} == committed baseline. No runner request drift.`);
+    return 0;
+  }
+  log('MISMATCH: iOS runner request count drifted from the committed baseline:');
+  for (const diff of comparison.differences) {
+    log(`  ${diff.key}: expected ${diff.expected}, got ${diff.actual}`);
+  }
+  log('If this drift is intentional, regenerate the baseline with --update and commit it.');
+  return 1;
+}
+
+function runGate(cfg: HarnessConfig, stateDir: string): number {
+  if (!cfg.udid) return inconclusive(cfg, 'no --udid provided; cannot drive a simulator scenario');
+  if (!prepareRunner(cfg, cfg.udid, stateDir)) {
+    return inconclusive(cfg, 'prepare ios-runner failed');
+  }
+  const { exitCode, observed } = runScenario(cfg, cfg.udid, stateDir);
+  log(`observed: ${describeCounts(observed)}`);
+  recordObserved(cfg, observed);
+  if (cfg.mode === 'update') {
+    writeBaseline(cfg.scenario, observed);
+    log('OK: baseline recorded (update mode)');
+    return 0;
+  }
+  return assertObserved(cfg, exitCode, observed);
+}
+
+function teardown(cfg: HarnessConfig, stateDir: string, createdStateDir: boolean): void {
+  runCli(['close', '--shutdown', '--state-dir', stateDir], 60_000);
+  if (!createdStateDir || cfg.keep) return;
+  try {
+    fs.rmSync(stateDir, { recursive: true, force: true });
+  } catch {
+    /* best-effort */
+  }
+}
+
+function main(): number {
+  const cfg = parseArgs(process.argv.slice(2));
+  const createdStateDir = !cfg.stateDir;
+  const stateDir =
+    cfg.stateDir ?? fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-runner-count-'));
+  log(`scenario: ${cfg.scenario}`);
+  log(`state-dir: ${stateDir}${createdStateDir ? ' (temp)' : ''}`);
+  try {
+    return runGate(cfg, stateDir);
+  } finally {
+    teardown(cfg, stateDir, createdStateDir);
+  }
+}
+
+process.exit(main());