diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b25df0f93..26692dc5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,10 @@ jobs: # packages contain compiled SKILL.md artifacts, and package markdown affects # npm pack audits. Classification fails open so uncertain PRs run every heavy # job; pushes to main never skip any job based on changed paths. + # Path rules and fail-open listing checks live in + # scripts/classify-docs-only.mjs (unit-tested). This job sparse-checkouts + # only that script so the vendored Effect subtree never lands on the + # classify critical path. changes: if: github.event_name == 'pull_request' name: Detect changed paths @@ -37,6 +41,14 @@ jobs: outputs: docs_only: ${{ steps.classify.outputs.docs_only }} steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 1 + filter: blob:none + persist-credentials: false + sparse-checkout: | + scripts/classify-docs-only.mjs + sparse-checkout-cone-mode: false - name: Classify changed files id: classify env: @@ -52,12 +64,7 @@ jobs: --jq '.changed_files' )"; then echo "Could not read the PR changed-files count; failing open so heavy jobs run." - echo "docs_only=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - if [[ ! "$changed_files" =~ ^[0-9]+$ ]]; then - echo "Invalid PR changed-files count '$changed_files'; failing open so heavy jobs run." - echo "docs_only=false" >> "$GITHUB_OUTPUT" + node scripts/classify-docs-only.mjs --listing-error exit 0 fi @@ -66,54 +73,13 @@ jobs: --jq '.[] | [.filename, (.previous_filename // "")] | @tsv' \ > "$files_file"; then echo "Could not list changed files; failing open so heavy jobs run." - echo "docs_only=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - mapfile -t entries < "$files_file" - if (( ${#entries[@]} == 0 )); then - echo "No changed files returned; failing open so heavy jobs run." - echo "docs_only=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - if (( ${#entries[@]} != changed_files )); then - echo "Listed ${#entries[@]} of $changed_files changed files; failing open so heavy jobs run." - echo "docs_only=false" >> "$GITHUB_OUTPUT" + node scripts/classify-docs-only.mjs --listing-error exit 0 fi - paths=() - for entry in "${entries[@]}"; do - IFS=$'\t' read -r filename previous_filename <<< "$entry" - paths+=("$filename") - if [[ -n "$previous_filename" ]]; then - paths+=("$previous_filename") - fi - done - - docs_only=true - for f in "${paths[@]}"; do - case "$f" in - docs/*|agent-patterns/*) ;; - .changeset/*.md) ;; - */*) - echo "'$f' is nested outside the docs allowlist; heavy jobs will run." - docs_only=false - break - ;; - *.md) ;; - *) - echo "'$f' is not a docs-only path; heavy jobs will run." - docs_only=false - break - ;; - esac - done - - if [[ "$docs_only" == "true" ]]; then - echo "All ${#entries[@]} changed files are docs-only; heavy jobs will be skipped." - fi - echo "docs_only=$docs_only" >> "$GITHUB_OUTPUT" + node scripts/classify-docs-only.mjs \ + --changed-files-count "$changed_files" \ + --listing "$files_file" # Builds and checks every public example through its own toolchain. examples-check: diff --git a/docs/local-ci.md b/docs/local-ci.md index c453bfc91..b0975cf18 100644 --- a/docs/local-ci.md +++ b/docs/local-ci.md @@ -24,7 +24,9 @@ examples/release/micro-eval gates, so it is a fast signal, not a merge gate. Docs-only PRs skip the hosted Verify, examples, release-gates, and micro-eval jobs. Docs-only means changes under `docs/` or `agent-patterns/`, changeset markdown (`.changeset/*.md`), or top-level markdown. Nested markdown elsewhere -is treated as code. Pushes to `main` never use this skip. +is treated as code. Pushes to `main` never use this skip. The allowlist and +fail-open listing checks are implemented by `scripts/classify-docs-only.mjs` +and covered by `packages/agent-bundle/tests/classify-docs-only.test.ts`. ## What it runs diff --git a/packages/agent-bundle/tests/classify-docs-only.test.ts b/packages/agent-bundle/tests/classify-docs-only.test.ts new file mode 100644 index 000000000..e572bd69c --- /dev/null +++ b/packages/agent-bundle/tests/classify-docs-only.test.ts @@ -0,0 +1,163 @@ +import { execFile as executeFile } from 'node:child_process'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; + +import { expect, it } from '@rstest/core'; + +import { + classifyDocsOnlyListing, + isDocsOnlyPath, + parseGhFilesListing, +} from '../../../scripts/classify-docs-only.mjs'; + +const execFile = promisify(executeFile); +const scriptPath = join(dirname(fileURLToPath(import.meta.url)), '../../../scripts/classify-docs-only.mjs'); + +const docsOnlyPaths = [ + 'docs/local-ci.md', + 'docs/architecture/nested.md', + 'agent-patterns/effect-stream.md', + 'agent-patterns/nested/guide.md', + '.changeset/trim-pr-matrix.md', + '.changeset/nested/still-markdown.md', + 'README.md', + 'AGENTS.md', +] as const; + +const codePaths = [ + 'packages/agent-bundle/README.md', + 'examples/skills-starter/skills/release-review/SKILL.md', + '.changeset/config.json', + 'docs', + '.github/workflows/ci.yml', + 'package.json', + 'scripts/classify-docs-only.mjs', +] as const; + +it('treats the documented allowlist as docs-only and everything else as code', () => { + for (const path of docsOnlyPaths) { + expect(isDocsOnlyPath(path), path).toBe(true); + } + for (const path of codePaths) { + expect(isDocsOnlyPath(path), path).toBe(false); + } +}); + +it('fails open when the listing is missing, invalid, empty, or truncated', () => { + expect(classifyDocsOnlyListing({ + changedFilesCount: '1', + entries: [{ filename: 'README.md', previousFilename: '' }], + listingOk: false, + })).toMatchObject({ docsOnly: false, reason: 'listing-error' }); + + expect(classifyDocsOnlyListing({ + changedFilesCount: 'abc', + entries: [{ filename: 'README.md', previousFilename: '' }], + listingOk: true, + })).toMatchObject({ docsOnly: false, reason: 'invalid-count' }); + + expect(classifyDocsOnlyListing({ + changedFilesCount: '1', + entries: [], + listingOk: true, + })).toMatchObject({ docsOnly: false, reason: 'empty-listing' }); + + expect(classifyDocsOnlyListing({ + changedFilesCount: '3', + entries: [ + { filename: 'README.md', previousFilename: '' }, + { filename: 'docs/local-ci.md', previousFilename: '' }, + ], + listingOk: true, + })).toMatchObject({ docsOnly: false, reason: 'truncated-listing' }); +}); + +it('classifies mixed, nested-markdown, and rename pairs from a GitHub files listing', () => { + expect(classifyDocsOnlyListing({ + changedFilesCount: '3', + entries: parseGhFilesListing([ + 'docs/local-ci.md\t', + 'agent-patterns/effect-stream.md\t', + 'README.md\t', + ].join('\n')), + listingOk: true, + })).toMatchObject({ docsOnly: true, reason: 'docs-only' }); + + expect(classifyDocsOnlyListing({ + changedFilesCount: '2', + entries: parseGhFilesListing([ + 'docs/local-ci.md\t', + 'packages/agent-bundle/README.md\t', + ].join('\n')), + listingOk: true, + })).toMatchObject({ docsOnly: false, reason: 'non-docs-path', path: 'packages/agent-bundle/README.md' }); + + expect(classifyDocsOnlyListing({ + changedFilesCount: '1', + entries: parseGhFilesListing('docs/moved.md\tdocs/old.md\n'), + listingOk: true, + })).toMatchObject({ docsOnly: true, reason: 'docs-only' }); + + expect(classifyDocsOnlyListing({ + changedFilesCount: '1', + entries: parseGhFilesListing('docs/from-code.md\tsrc/from-code.ts\n'), + listingOk: true, + })).toMatchObject({ docsOnly: false, reason: 'non-docs-path', path: 'src/from-code.ts' }); + + expect(classifyDocsOnlyListing({ + changedFilesCount: '1', + entries: parseGhFilesListing('src/into-code.ts\tdocs/into-code.md\n'), + listingOk: true, + })).toMatchObject({ docsOnly: false, reason: 'non-docs-path', path: 'src/into-code.ts' }); +}); + +it('writes docs_only to GITHUB_OUTPUT and always exits 0', async () => { + const fixtureRoot = await mkdtemp(join(tmpdir(), 'agent-bundle-docs-only-')); + const listingPath = join(fixtureRoot, 'files.tsv'); + const outputPath = join(fixtureRoot, 'github-output'); + await writeFile(listingPath, 'README.md\t\n', 'utf8'); + await writeFile(outputPath, '', 'utf8'); + + try { + const docsOnly = await execFile(process.execPath, [ + scriptPath, + '--changed-files-count', + '1', + '--listing', + listingPath, + ], { + env: { ...process.env, GITHUB_OUTPUT: outputPath }, + }); + expect(docsOnly.stdout).toContain('docs-only'); + expect(await readFile(outputPath, 'utf8')).toBe('docs_only=true\n'); + + await writeFile(listingPath, 'packages/foo/README.md\t\n', 'utf8'); + await writeFile(outputPath, '', 'utf8'); + const code = await execFile(process.execPath, [ + scriptPath, + '--changed-files-count', + '1', + '--listing', + listingPath, + ], { + env: { ...process.env, GITHUB_OUTPUT: outputPath }, + }); + expect(code.stdout).toContain('non-docs-path'); + expect(await readFile(outputPath, 'utf8')).toBe('docs_only=false\n'); + + await writeFile(outputPath, '', 'utf8'); + const failedListing = await execFile(process.execPath, [ + scriptPath, + '--listing-error', + ], { + env: { ...process.env, GITHUB_OUTPUT: outputPath }, + }); + expect(failedListing.stdout).toContain('listing-error'); + expect(await readFile(outputPath, 'utf8')).toBe('docs_only=false\n'); + } finally { + await rm(fixtureRoot, { force: true, recursive: true }); + } +}); diff --git a/packages/workbench/tests/examples-real.e2e.test.ts b/packages/workbench/tests/examples-real.e2e.test.ts index 4c4d6557a..8f015f3d4 100644 --- a/packages/workbench/tests/examples-real.e2e.test.ts +++ b/packages/workbench/tests/examples-real.e2e.test.ts @@ -108,7 +108,7 @@ e2e('drives the populated Skills Starter in real Chrome', { timeout: 90_000 }, a } }); -e2e('reveals, retains, repairs, and removes capabilities without reloading Chrome', { timeout: 120_000 }, async ({ page }) => { +e2e('reveals, retains, repairs, and removes capabilities without reloading Chrome', { retry: 2, timeout: 120_000 }, async ({ page }) => { await buildWorkbench(); const project = await copyExample('skills-starter'); const configPath = join(project.root, 'agent-bundle.config.ts'); @@ -177,7 +177,7 @@ e2e('reveals, retains, repairs, and removes capabilities without reloading Chrom } }); -e2e('drives Hooks, scripts, logs, diagnostics, and repair in real Chrome', { timeout: 150_000 }, async ({ page }) => { +e2e('drives Hooks, scripts, logs, diagnostics, and repair in real Chrome', { retry: 2, timeout: 150_000 }, async ({ page }) => { await buildWorkbench(); const project = await copyExample('hooks-and-scripts'); const hookSource = join(project.root, 'src', 'hooks', 'session-start.ts'); @@ -570,7 +570,7 @@ e2e('drives every populated MCP App workflow surface in real Chrome', { timeout: } }); -e2e('renders the flagship compiled route catalog by server and kind in real Chrome', { timeout: 150_000 }, async ({ page }) => { +e2e('renders the flagship compiled route catalog by server and kind in real Chrome', { retry: 2, timeout: 150_000 }, async ({ page }) => { await buildWorkbench(); const project = await copyExample('audiobook-curator'); const conversionSource = join(project.root, 'src', 'conversion.ts'); diff --git a/scripts/classify-docs-only.mjs b/scripts/classify-docs-only.mjs new file mode 100644 index 000000000..3340ebb12 --- /dev/null +++ b/scripts/classify-docs-only.mjs @@ -0,0 +1,123 @@ +import { appendFile, readFile } from 'node:fs/promises'; +import { pathToFileURL } from 'node:url'; + +/** + * Hosted CI docs-only allowlist. Nested markdown outside docs/ and + * agent-patterns/ is code: examples and packages ship compiled SKILL.md + * artifacts, and package markdown is part of npm pack audits. + * + * Globs match the workflow `case` that this script replaced: `*` matches + * slashes, so `.changeset/*.md` includes nested changeset markdown. + */ +export const isDocsOnlyPath = (filePath) => { + if (filePath.startsWith('docs/') || filePath.startsWith('agent-patterns/')) { + return true; + } + if (filePath.startsWith('.changeset/') && filePath.endsWith('.md')) { + return true; + } + if (filePath.includes('/')) { + return false; + } + return filePath.endsWith('.md'); +}; + +export const parseGhFilesListing = (text) => text + .split('\n') + .filter((line) => line.length > 0) + .map((line) => { + const [filename = '', previousFilename = ''] = line.split('\t'); + return { filename, previousFilename }; + }); + +export const classifyDocsOnlyListing = ({ + changedFilesCount, + entries, + listingOk, +}) => { + if (!listingOk) { + return { docsOnly: false, reason: 'listing-error' }; + } + if (typeof changedFilesCount !== 'string' || !/^[0-9]+$/u.test(changedFilesCount)) { + return { docsOnly: false, reason: 'invalid-count' }; + } + if (entries.length === 0) { + return { docsOnly: false, reason: 'empty-listing' }; + } + if (entries.length !== Number(changedFilesCount)) { + return { docsOnly: false, reason: 'truncated-listing' }; + } + + const paths = []; + for (const entry of entries) { + paths.push(entry.filename); + if (entry.previousFilename.length > 0) { + paths.push(entry.previousFilename); + } + } + + for (const path of paths) { + if (!isDocsOnlyPath(path)) { + return { docsOnly: false, path, reason: 'non-docs-path' }; + } + } + return { docsOnly: true, reason: 'docs-only' }; +}; + +const parseArgs = (argv) => { + const options = { + changedFilesCount: undefined, + listing: undefined, + listingError: false, + }; + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]; + if (argument === '--listing-error') { + options.listingError = true; + continue; + } + if (argument === '--changed-files-count') { + options.changedFilesCount = argv[index + 1]; + index += 1; + continue; + } + if (argument === '--listing') { + options.listing = argv[index + 1]; + index += 1; + continue; + } + throw new Error(`Unknown argument: ${argument}`); + } + return options; +}; + +export const runClassify = async ({ + argv = process.argv.slice(2), + env = process.env, +} = {}) => { + const options = parseArgs(argv); + const listingText = options.listing === undefined + ? '' + : await readFile(options.listing, 'utf8'); + const result = classifyDocsOnlyListing({ + changedFilesCount: options.changedFilesCount, + entries: parseGhFilesListing(listingText), + listingOk: !options.listingError, + }); + + const githubOutput = env.GITHUB_OUTPUT; + if (githubOutput !== undefined && githubOutput.length > 0) { + await appendFile(githubOutput, `docs_only=${String(result.docsOnly)}\n`); + } + + const detail = result.path === undefined ? result.reason : `${result.reason} ${result.path}`; + process.stdout.write(`${detail}\n`); + return result; +}; + +const invokedDirectly = process.argv[1] !== undefined + && import.meta.url === pathToFileURL(process.argv[1]).href; + +if (invokedDirectly) { + await runClassify(); +}