ci: extract docs-only classifier tests and retry stale-diagnostic e2e flakes

.github/workflows/ci.yml

@@ -27,6 +27,10 @@ jobs:
   # packages contain compiled SKILL.md artifacts, and package markdown affects
   # npm pack audits. Classification fails open so uncertain PRs run every heavy
   # job; pushes to main never skip any job based on changed paths.
+  # Path rules and fail-open listing checks live in
+  # scripts/classify-docs-only.mjs (unit-tested). This job sparse-checkouts
+  # only that script so the vendored Effect subtree never lands on the
+  # classify critical path.
   changes:
     if: github.event_name == 'pull_request'
     name: Detect changed paths
@@ -37,6 +41,14 @@ jobs:
     outputs:
       docs_only: ${{ steps.classify.outputs.docs_only }}
     steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 1
+          filter: blob:none
+          persist-credentials: false
+          sparse-checkout: |
+            scripts/classify-docs-only.mjs
+          sparse-checkout-cone-mode: false
       - name: Classify changed files
         id: classify
         env:
@@ -52,12 +64,7 @@ jobs:
               --jq '.changed_files'
           )"; then
             echo "Could not read the PR changed-files count; failing open so heavy jobs run."
-            echo "docs_only=false" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          if [[ ! "$changed_files" =~ ^[0-9]+$ ]]; then
-            echo "Invalid PR changed-files count '$changed_files'; failing open so heavy jobs run."
-            echo "docs_only=false" >> "$GITHUB_OUTPUT"
+            node scripts/classify-docs-only.mjs --listing-error
             exit 0
           fi
 
@@ -66,54 +73,13 @@ jobs:
             --jq '.[] | [.filename, (.previous_filename // "")] | @tsv' \
             > "$files_file"; then
             echo "Could not list changed files; failing open so heavy jobs run."
-            echo "docs_only=false" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-
-          mapfile -t entries < "$files_file"
-          if (( ${#entries[@]} == 0 )); then
-            echo "No changed files returned; failing open so heavy jobs run."
-            echo "docs_only=false" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          if (( ${#entries[@]} != changed_files )); then
-            echo "Listed ${#entries[@]} of $changed_files changed files; failing open so heavy jobs run."
-            echo "docs_only=false" >> "$GITHUB_OUTPUT"
+            node scripts/classify-docs-only.mjs --listing-error
             exit 0
           fi
 
-          paths=()
-          for entry in "${entries[@]}"; do
-            IFS=$'\t' read -r filename previous_filename <<< "$entry"
-            paths+=("$filename")
-            if [[ -n "$previous_filename" ]]; then
-              paths+=("$previous_filename")
-            fi
-          done
-
-          docs_only=true
-          for f in "${paths[@]}"; do
-            case "$f" in
-              docs/*|agent-patterns/*) ;;
-              .changeset/*.md) ;;
-              */*)
-                echo "'$f' is nested outside the docs allowlist; heavy jobs will run."
-                docs_only=false
-                break
-                ;;
-              *.md) ;;
-              *)
-                echo "'$f' is not a docs-only path; heavy jobs will run."
-                docs_only=false
-                break
-                ;;
-            esac
-          done
-
-          if [[ "$docs_only" == "true" ]]; then
-            echo "All ${#entries[@]} changed files are docs-only; heavy jobs will be skipped."
-          fi
-          echo "docs_only=$docs_only" >> "$GITHUB_OUTPUT"
+          node scripts/classify-docs-only.mjs \
+            --changed-files-count "$changed_files" \
+            --listing "$files_file"
 
   # Builds and checks every public example through its own toolchain.
   examples-check:

docs/local-ci.md

@@ -24,7 +24,9 @@ examples/release/micro-eval gates, so it is a fast signal, not a merge gate.
 Docs-only PRs skip the hosted Verify, examples, release-gates, and micro-eval
 jobs. Docs-only means changes under `docs/` or `agent-patterns/`, changeset
 markdown (`.changeset/*.md`), or top-level markdown. Nested markdown elsewhere
-is treated as code. Pushes to `main` never use this skip.
+is treated as code. Pushes to `main` never use this skip. The allowlist and
+fail-open listing checks are implemented by `scripts/classify-docs-only.mjs`
+and covered by `packages/agent-bundle/tests/classify-docs-only.test.ts`.
 
 ## What it runs
 

packages/agent-bundle/tests/classify-docs-only.test.ts

@@ -0,0 +1,163 @@
+import { execFile as executeFile } from 'node:child_process';
+import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { promisify } from 'node:util';
+
+import { expect, it } from '@rstest/core';
+
+import {
+  classifyDocsOnlyListing,
+  isDocsOnlyPath,
+  parseGhFilesListing,
+} from '../../../scripts/classify-docs-only.mjs';
+
+const execFile = promisify(executeFile);
+const scriptPath = join(dirname(fileURLToPath(import.meta.url)), '../../../scripts/classify-docs-only.mjs');
+
+const docsOnlyPaths = [
+  'docs/local-ci.md',
+  'docs/architecture/nested.md',
+  'agent-patterns/effect-stream.md',
+  'agent-patterns/nested/guide.md',
+  '.changeset/trim-pr-matrix.md',
+  '.changeset/nested/still-markdown.md',
+  'README.md',
+  'AGENTS.md',
+] as const;
+
+const codePaths = [
+  'packages/agent-bundle/README.md',
+  'examples/skills-starter/skills/release-review/SKILL.md',
+  '.changeset/config.json',
+  'docs',
+  '.github/workflows/ci.yml',
+  'package.json',
+  'scripts/classify-docs-only.mjs',
+] as const;
+
+it('treats the documented allowlist as docs-only and everything else as code', () => {
+  for (const path of docsOnlyPaths) {
+    expect(isDocsOnlyPath(path), path).toBe(true);
+  }
+  for (const path of codePaths) {
+    expect(isDocsOnlyPath(path), path).toBe(false);
+  }
+});
+
+it('fails open when the listing is missing, invalid, empty, or truncated', () => {
+  expect(classifyDocsOnlyListing({
+    changedFilesCount: '1',
+    entries: [{ filename: 'README.md', previousFilename: '' }],
+    listingOk: false,
+  })).toMatchObject({ docsOnly: false, reason: 'listing-error' });
+
+  expect(classifyDocsOnlyListing({
+    changedFilesCount: 'abc',
+    entries: [{ filename: 'README.md', previousFilename: '' }],
+    listingOk: true,
+  })).toMatchObject({ docsOnly: false, reason: 'invalid-count' });
+
+  expect(classifyDocsOnlyListing({
+    changedFilesCount: '1',
+    entries: [],
+    listingOk: true,
+  })).toMatchObject({ docsOnly: false, reason: 'empty-listing' });
+
+  expect(classifyDocsOnlyListing({
+    changedFilesCount: '3',
+    entries: [
+      { filename: 'README.md', previousFilename: '' },
+      { filename: 'docs/local-ci.md', previousFilename: '' },
+    ],
+    listingOk: true,
+  })).toMatchObject({ docsOnly: false, reason: 'truncated-listing' });
+});
+
+it('classifies mixed, nested-markdown, and rename pairs from a GitHub files listing', () => {
+  expect(classifyDocsOnlyListing({
+    changedFilesCount: '3',
+    entries: parseGhFilesListing([
+      'docs/local-ci.md\t',
+      'agent-patterns/effect-stream.md\t',
+      'README.md\t',
+    ].join('\n')),
+    listingOk: true,
+  })).toMatchObject({ docsOnly: true, reason: 'docs-only' });
+
+  expect(classifyDocsOnlyListing({
+    changedFilesCount: '2',
+    entries: parseGhFilesListing([
+      'docs/local-ci.md\t',
+      'packages/agent-bundle/README.md\t',
+    ].join('\n')),
+    listingOk: true,
+  })).toMatchObject({ docsOnly: false, reason: 'non-docs-path', path: 'packages/agent-bundle/README.md' });
+
+  expect(classifyDocsOnlyListing({
+    changedFilesCount: '1',
+    entries: parseGhFilesListing('docs/moved.md\tdocs/old.md\n'),
+    listingOk: true,
+  })).toMatchObject({ docsOnly: true, reason: 'docs-only' });
+
+  expect(classifyDocsOnlyListing({
+    changedFilesCount: '1',
+    entries: parseGhFilesListing('docs/from-code.md\tsrc/from-code.ts\n'),
+    listingOk: true,
+  })).toMatchObject({ docsOnly: false, reason: 'non-docs-path', path: 'src/from-code.ts' });
+
+  expect(classifyDocsOnlyListing({
+    changedFilesCount: '1',
+    entries: parseGhFilesListing('src/into-code.ts\tdocs/into-code.md\n'),
+    listingOk: true,
+  })).toMatchObject({ docsOnly: false, reason: 'non-docs-path', path: 'src/into-code.ts' });
+});
+
+it('writes docs_only to GITHUB_OUTPUT and always exits 0', async () => {
+  const fixtureRoot = await mkdtemp(join(tmpdir(), 'agent-bundle-docs-only-'));
+  const listingPath = join(fixtureRoot, 'files.tsv');
+  const outputPath = join(fixtureRoot, 'github-output');
+  await writeFile(listingPath, 'README.md\t\n', 'utf8');
+  await writeFile(outputPath, '', 'utf8');
+
+  try {
+    const docsOnly = await execFile(process.execPath, [
+      scriptPath,
+      '--changed-files-count',
+      '1',
+      '--listing',
+      listingPath,
+    ], {
+      env: { ...process.env, GITHUB_OUTPUT: outputPath },
+    });
+    expect(docsOnly.stdout).toContain('docs-only');
+    expect(await readFile(outputPath, 'utf8')).toBe('docs_only=true\n');
+
+    await writeFile(listingPath, 'packages/foo/README.md\t\n', 'utf8');
+    await writeFile(outputPath, '', 'utf8');
+    const code = await execFile(process.execPath, [
+      scriptPath,
+      '--changed-files-count',
+      '1',
+      '--listing',
+      listingPath,
+    ], {
+      env: { ...process.env, GITHUB_OUTPUT: outputPath },
+    });
+    expect(code.stdout).toContain('non-docs-path');
+    expect(await readFile(outputPath, 'utf8')).toBe('docs_only=false\n');
+
+    await writeFile(outputPath, '', 'utf8');
+    const failedListing = await execFile(process.execPath, [
+      scriptPath,
+      '--listing-error',
+    ], {
+      env: { ...process.env, GITHUB_OUTPUT: outputPath },
+    });
+    expect(failedListing.stdout).toContain('listing-error');
+    expect(await readFile(outputPath, 'utf8')).toBe('docs_only=false\n');
+  } finally {
+    await rm(fixtureRoot, { force: true, recursive: true });
+  }
+});

packages/workbench/tests/examples-real.e2e.test.ts

@@ -108,7 +108,7 @@ e2e('drives the populated Skills Starter in real Chrome', { timeout: 90_000 }, a
   }
 });
 
-e2e('reveals, retains, repairs, and removes capabilities without reloading Chrome', { timeout: 120_000 }, async ({ page }) => {
+e2e('reveals, retains, repairs, and removes capabilities without reloading Chrome', { retry: 2, timeout: 120_000 }, async ({ page }) => {
   await buildWorkbench();
   const project = await copyExample('skills-starter');
   const configPath = join(project.root, 'agent-bundle.config.ts');
@@ -177,7 +177,7 @@ e2e('reveals, retains, repairs, and removes capabilities without reloading Chrom
   }
 });
 
-e2e('drives Hooks, scripts, logs, diagnostics, and repair in real Chrome', { timeout: 150_000 }, async ({ page }) => {
+e2e('drives Hooks, scripts, logs, diagnostics, and repair in real Chrome', { retry: 2, timeout: 150_000 }, async ({ page }) => {
   await buildWorkbench();
   const project = await copyExample('hooks-and-scripts');
   const hookSource = join(project.root, 'src', 'hooks', 'session-start.ts');
@@ -570,7 +570,7 @@ e2e('drives every populated MCP App workflow surface in real Chrome', { timeout:
   }
 });
 
-e2e('renders the flagship compiled route catalog by server and kind in real Chrome', { timeout: 150_000 }, async ({ page }) => {
+e2e('renders the flagship compiled route catalog by server and kind in real Chrome', { retry: 2, timeout: 150_000 }, async ({ page }) => {
   await buildWorkbench();
   const project = await copyExample('audiobook-curator');
   const conversionSource = join(project.root, 'src', 'conversion.ts');