diff --git a/.fallowrc.json b/.fallowrc.json index 23a5a9c1dd..66428e28fc 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -64,7 +64,7 @@ }, { "comment": "Tool config default exports, loaded by the tool rather than imported.", - "file": "{tsdown.config.ts,vitest.mutation.config.ts,website/rspress.config.ts,test/skillgym/skillgym.config.ts,test/skillgym/suites/*.ts}", + "file": "{tsdown.config.ts,vitest.mutation.config.ts,website/rspress.config.ts}", "exports": ["default"] }, { @@ -88,7 +88,7 @@ "exports": ["ownedTestFiles"] }, { - "comment": "Help-benchmark conformance seams: helpTopicIds feeds the topic-coverage gate and PRIVATE_AX_RECOVERY_SAMPLE feeds the skillgym suite; both consumers are test-tree files outside --production analysis.", + "comment": "Help-benchmark conformance seams: helpTopicIds feeds the topic-coverage gate and PRIVATE_AX_RECOVERY_SAMPLE feeds the bench sample producers; both consumers are test-tree files outside --production analysis.", "file": "{src/cli/parser/cli-help.ts,scripts/help-conformance-sample-outputs.mjs}", "exports": ["helpTopicIds", "PRIVATE_AX_RECOVERY_SAMPLE"] }, diff --git a/.gitignore b/.gitignore index 40ae8f1d57..d0af94b2fa 100644 --- a/.gitignore +++ b/.gitignore @@ -35,7 +35,6 @@ xcuserdata/ *.app *.xctestrun *.xcarchive -.skillgym-results/ android/snapshot-helper/build/ android/snapshot-helper/dist/ android/ime-helper/build/ diff --git a/.oxfmtrc.json b/.oxfmtrc.json index 0ae6cd6e0f..268d7fc3bd 100644 --- a/.oxfmtrc.json +++ b/.oxfmtrc.json @@ -7,7 +7,6 @@ "ignorePatterns": [ "dist/**", "node_modules/**", - "**/.skillgym-results/**", "**/*.md", "scripts/maestro-conformance/corpus/**", "fallow-baselines/**", diff --git a/AGENTS.md b/AGENTS.md index 3bee709457..6e1e28c922 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -233,10 +233,8 @@ connect errors, retry policy, or command typing, start in layer after writing is how the same contract ends up duplicated across two of them. `docs/agents/cli-flags.md` walks the layers for the flag case. - Decide docs impact with the change, not after. For behavior/CLI-surface changes: update - help/metadata, README or `website/docs/**` when user-facing, and a SkillGym case in - `test/skillgym/suites/agent-device-smoke-suite.ts` when command-planning guidance changes. -- Keep SkillGym cases behavioral and command-planning oriented: assert the user-visible contract and - expected command family, forbid known bad patterns, avoid brittle exact output. + help/metadata, README or `website/docs/**` when user-facing, and a help-conformance bench case + (`scripts/help-conformance-*.mjs`) when command-planning guidance changes. - State in the final summary whether docs/skills were updated, and why not if they weren't. When guidance conflicts, Hard Rules win, then scope, then testing, then style. diff --git a/docs/agents/pull-requests.md b/docs/agents/pull-requests.md index 1af3b1116f..e2f75eede9 100644 --- a/docs/agents/pull-requests.md +++ b/docs/agents/pull-requests.md @@ -14,7 +14,7 @@ observes the artifact/state the feature claims to handle. If live verification is blocked, state the blocker and the exact command/device needed, and downgrade the PR to residual risk rather than calling it ready. -- Command-surface changes preserve CLI, Node.js, daemon, MCP, help, docs, and SkillGym coverage +- Command-surface changes preserve CLI, Node.js, daemon, MCP, help, and docs coverage where that surface is affected, without duplicating command contracts across layers. - Runtime output stays agent-friendly: compact defaults, top offenders first for diagnostics/perf, bounded arrays in JSON, artifact paths for large raw data, progressive lookup for deeper detail. diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 0ccd648ebe..a35e808d47 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -18,9 +18,8 @@ The mapping it encodes, for when you need to run a gate directly or reason about | iOS runner / Swift | `pnpm build:xcuitest` | | CLI help/guidance (`src/cli/parser/cli-help.ts`, `src/cli-schema/`) | `pnpm exec vitest run src/cli/parser/__tests__ src/cli-schema/command-schema-guards.test.ts scripts/__tests__` — the `scripts/__tests__` gates enforce help-topic benchmark coverage and pin the bench's quoted CLI samples to the real renderers | | Help benchmark cases (`scripts/help-conformance-*.mjs`) | `pnpm exec vitest run scripts/__tests__` (deterministic gates); model-backed: `pnpm bench:help-conformance` (paid LLM calls, local only) | -| SkillGym prompts/assertions | `pnpm test:skillgym:case ` (broad: `pnpm test:skillgym`, filter with `-- --tag fixture-smoke` or `-- --tag skill-guidance`) — agentic routing + local-help-consumption proof only; command-planning knowledge checks belong in the help bench | | `.ad` grammar (`src/replay/script.ts`, gesture arity, replay vars) | `pnpm exec vitest run --project unit-core test/replay-compat` — the frozen replay-compat corpus asserts which released script surfaces still parse; a flipped verdict is edited in `test/replay-compat/manifest.ts`, never in the script. Adding or re-pinning a corpus entry also runs `pnpm check:replay-compat`, which re-derives each entry from its release tag in git history | -| Anything in `src/`, `test/`, `skills/` | `pnpm format` | +| Anything in `src/`, `test/` | `pnpm format` (`skills/` is Markdown-only guidance: oxfmt ignores `**/*.md`, and the affected-check selector classifies it docs-only) | | A decision kernel or its tests (`src/kernel/errors.ts`, `src/daemon/ref-frame.ts`, `src/commands/interaction/runtime/settle.ts`, `src/utils/scroll-edge-state.ts`, `src/selectors/`) | `pnpm mutation:affected --base origin/main` (minutes; GitHub runs it per PR — see the mutation ratchet section) | Two traps worth naming: @@ -88,8 +87,8 @@ hand-maintained path map: import ownership. Dynamic-import relationships remain outside Vitest's analysis; GitHub's authoritative full suites still cover that boundary. - **Non-Vitest suites** retain explicit ownership. Root - `test/integration/*.ts` files use the Node integration lane, SkillGym owns its - harness and skill guidance, and platform/build tools keep their native gates. + `test/integration/*.ts` files use the Node integration lane, and + platform/build tools keep their native gates. - **Always-on gates** (`lint`, `typecheck`, `layering`, `fallow`, `format`) fire for their input categories and are never silently skipped. Platform source also selects the provider-integration and coverage gates required by the @@ -99,9 +98,6 @@ hand-maintained path map: - A **small explicit build-ownership layer** covers the paths whose owning build cannot be derived: Swift runner, Android helpers, macOS helper, MCP metadata, and the public package surface (itself derived from `package.json` `exports`). -- **SkillGym ownership** covers skill guidance (`skills/`) and the SkillGym - harness (`test/skillgym/`) — those changes select the (local-only) SkillGym - suite, and their Markdown is treated as skill/harness input, not inert docs. Changed-file discovery folds working-tree state into the local plan: in the default local mode (`--head HEAD`) it unions the committed `base..HEAD` diff with diff --git a/examples/README.md b/examples/README.md index 271fa4bfd3..7a78f3b5cc 100644 --- a/examples/README.md +++ b/examples/README.md @@ -33,6 +33,5 @@ way a Node consumer or the agent-device-cloud bridge would. Each file: ## test-app/ -[`test-app/`](./test-app) is the Expo dogfood fixture used for `agent-device` and `skillgym` -experiments (see its own [README](./test-app/README.md)) — it is a test fixture, not an SDK usage -example. +[`test-app/`](./test-app) is the Expo dogfood fixture used for `agent-device` experiments (see +its own [README](./test-app/README.md)) — it is a test fixture, not an SDK usage example. diff --git a/examples/test-app/README.md b/examples/test-app/README.md index 4138169448..8b20852a4e 100644 --- a/examples/test-app/README.md +++ b/examples/test-app/README.md @@ -1,13 +1,12 @@ # Agent Device Tester -`Agent Device Tester` is a minimal Expo Router fixture app for `agent-device` and `skillgym` experiments. +`Agent Device Tester` is a minimal Expo Router fixture app for `agent-device` experiments. It is intentionally small, but each surface is dense with durable accessibility targets so a few screens cover a large share of the workflows we care about. ## Why this app exists - It gives `agent-device` a stable React Native target that we control. -- It makes `skillgym` prompts concrete: the agent can inspect real app files instead of answering against an imagined UI. - It keeps the number of screens low while still covering roughly 50 practical interaction and verification cases. ## Screens diff --git a/examples/test-app/src/screens/FormScreen.tsx b/examples/test-app/src/screens/FormScreen.tsx index 4cec8023b5..42802194c8 100644 --- a/examples/test-app/src/screens/FormScreen.tsx +++ b/examples/test-app/src/screens/FormScreen.tsx @@ -140,7 +140,7 @@ export function FormScreen(props: FormScreenProps) { title="Android IME capture" testID="android-ime-capture-fixture" > - {/* SkillGym fixture: static diagnostic copy, not live state. */} + {/* Fixture: static diagnostic copy, not live state. */} = 14'} - ansi-regex@6.2.2: - resolution: {integrity: sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==} - engines: {node: '>=12'} - ansis@4.3.1: resolution: {integrity: sha512-BJ8/l4R5LRE7hW9WdSuGYrLSHi2ynxeFpDFbH0K/CgNeY/tyhk+vO6TYxXC5r5CpUhNVX310xzPsN/H9lCdfOA==} engines: {node: '>=14'} @@ -1724,10 +1714,6 @@ packages: chardet@2.2.0: resolution: {integrity: sha512-rddelWYNPRrXq6PtNEN2S3f6t9ILzvqaN5pVgi4kqt9jHQaXIial9PznB5iSPVlQSLNaaH22ItWz3EJtQ10+OA==} - cli-spinners@3.4.0: - resolution: {integrity: sha512-bXfOC4QcT1tKXGorxL3wbJm6XJPDqEnij2gQ2m7ESQuE+/z9YFIWnl/5RpTiKWbMq3EVKR4fRLJGn6DVfu0mpw==} - engines: {node: '>=18.20'} - cli-width@4.1.0: resolution: {integrity: sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==} engines: {node: '>= 12'} @@ -1742,10 +1728,6 @@ packages: comma-separated-tokens@2.0.3: resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} - commander@13.1.0: - resolution: {integrity: sha512-/rFeCpNJQbhSZjGVwO9RFV3xPqbnERS8MmIQzCtD/zl6gpJuV/bMLuN92oG3F7d8oDEHHRrujSXNUr8fpjntKw==} - engines: {node: '>=18'} - commander@14.0.3: resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==} engines: {node: '>=20'} @@ -2444,10 +2426,6 @@ packages: resolution: {integrity: sha512-dkEJPVvun4FryqBmZ5KhDo0K9iDXAwn08tMLDinNdRBNPcYEDiWYysLcc6k3mjTMlbP9KyylvRpd4wFtwrT9rw==} engines: {node: ^20.17.0 || >=22.9.0} - nano-spawn@2.1.0: - resolution: {integrity: sha512-yTW+2okrElHiH4fsiz/+/zc0EDo9BDDoC3iKk8dpv1GeRc9nUWzUZHx6TofMWErchhUQR8hY9/Eu1Uja9x1nqA==} - engines: {node: '>=20.17'} - nanoid@3.3.12: resolution: {integrity: sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} @@ -2783,11 +2761,6 @@ packages: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} - skillgym@0.9.1: - resolution: {integrity: sha512-vEFioLv7JQHWhPW7Cs2u7WIGl7GhE4D3K+IOAUEnuIewIdjAsmjnZXK0BS5oyY9TvKJECZh6a0XY46+GNaptpw==} - engines: {node: '>=22.18.0'} - hasBin: true - source-map-js@1.2.1: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} @@ -2805,17 +2778,9 @@ packages: std-env@4.0.0: resolution: {integrity: sha512-zUMPtQ/HBY3/50VbpkupYHbRroTRZJPRLvreamgErJVys0ceuzMkD44J/QjqhHjOzK42GQ3QZIeFG1OYfOtKqQ==} - string-width@7.2.0: - resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==} - engines: {node: '>=18'} - stringify-entities@4.0.4: resolution: {integrity: sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg==} - strip-ansi@7.2.0: - resolution: {integrity: sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==} - engines: {node: '>=12'} - strip-final-newline@4.0.0: resolution: {integrity: sha512-aulFJcD6YK8V1G7iRB5tigAP4TsHBZZrOV8pjV++zdUwmeV8uzbY7yn6h9MswN62adStNZFuCIx4haBnRuMDaw==} engines: {node: '>=18'} @@ -3140,20 +3105,20 @@ snapshots: '@babel/compat-data@7.29.7': {} - '@babel/core@7.29.7': + '@babel/core@7.29.7(supports-color@7.2.0)': dependencies: '@babel/code-frame': 7.29.7 '@babel/generator': 7.29.7 '@babel/helper-compilation-targets': 7.29.7 - '@babel/helper-module-transforms': 7.29.7(@babel/core@7.29.7) + '@babel/helper-module-transforms': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0) '@babel/helpers': 7.29.7 '@babel/parser': 7.29.7 '@babel/template': 7.29.7 - '@babel/traverse': 7.29.7 + '@babel/traverse': 7.29.7(supports-color@7.2.0) '@babel/types': 7.29.7 '@jridgewell/remapping': 2.3.5 convert-source-map: 2.0.0 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -3180,41 +3145,41 @@ snapshots: lru-cache: 5.1.1 semver: 6.3.1 - '@babel/helper-create-class-features-plugin@7.29.7(@babel/core@7.29.7)': + '@babel/helper-create-class-features-plugin@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.7 + '@babel/core': 7.29.7(supports-color@7.2.0) '@babel/helper-annotate-as-pure': 7.29.7 - '@babel/helper-member-expression-to-functions': 7.29.7 + '@babel/helper-member-expression-to-functions': 7.29.7(supports-color@7.2.0) '@babel/helper-optimise-call-expression': 7.29.7 - '@babel/helper-replace-supers': 7.29.7(@babel/core@7.29.7) - '@babel/helper-skip-transparent-expression-wrappers': 7.29.7 - '@babel/traverse': 7.29.7 + '@babel/helper-replace-supers': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0) + '@babel/helper-skip-transparent-expression-wrappers': 7.29.7(supports-color@7.2.0) + '@babel/traverse': 7.29.7(supports-color@7.2.0) semver: 6.3.1 transitivePeerDependencies: - supports-color '@babel/helper-globals@7.29.7': {} - '@babel/helper-member-expression-to-functions@7.29.7': + '@babel/helper-member-expression-to-functions@7.29.7(supports-color@7.2.0)': dependencies: - '@babel/traverse': 7.29.7 + '@babel/traverse': 7.29.7(supports-color@7.2.0) '@babel/types': 7.29.7 transitivePeerDependencies: - supports-color - '@babel/helper-module-imports@7.29.7': + '@babel/helper-module-imports@7.29.7(supports-color@7.2.0)': dependencies: - '@babel/traverse': 7.29.7 + '@babel/traverse': 7.29.7(supports-color@7.2.0) '@babel/types': 7.29.7 transitivePeerDependencies: - supports-color - '@babel/helper-module-transforms@7.29.7(@babel/core@7.29.7)': + '@babel/helper-module-transforms@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.7 - '@babel/helper-module-imports': 7.29.7 + '@babel/core': 7.29.7(supports-color@7.2.0) + '@babel/helper-module-imports': 7.29.7(supports-color@7.2.0) '@babel/helper-validator-identifier': 7.29.7 - '@babel/traverse': 7.29.7 + '@babel/traverse': 7.29.7(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -3224,18 +3189,18 @@ snapshots: '@babel/helper-plugin-utils@7.29.7': {} - '@babel/helper-replace-supers@7.29.7(@babel/core@7.29.7)': + '@babel/helper-replace-supers@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.7 - '@babel/helper-member-expression-to-functions': 7.29.7 + '@babel/core': 7.29.7(supports-color@7.2.0) + '@babel/helper-member-expression-to-functions': 7.29.7(supports-color@7.2.0) '@babel/helper-optimise-call-expression': 7.29.7 - '@babel/traverse': 7.29.7 + '@babel/traverse': 7.29.7(supports-color@7.2.0) transitivePeerDependencies: - supports-color - '@babel/helper-skip-transparent-expression-wrappers@7.29.7': + '@babel/helper-skip-transparent-expression-wrappers@7.29.7(supports-color@7.2.0)': dependencies: - '@babel/traverse': 7.29.7 + '@babel/traverse': 7.29.7(supports-color@7.2.0) '@babel/types': 7.29.7 transitivePeerDependencies: - supports-color @@ -3263,73 +3228,73 @@ snapshots: dependencies: '@babel/types': 7.29.7 - '@babel/plugin-proposal-decorators@7.29.7(@babel/core@7.29.7)': + '@babel/plugin-proposal-decorators@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.7 - '@babel/helper-create-class-features-plugin': 7.29.7(@babel/core@7.29.7) + '@babel/core': 7.29.7(supports-color@7.2.0) + '@babel/helper-create-class-features-plugin': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.29.7 - '@babel/plugin-syntax-decorators': 7.29.7(@babel/core@7.29.7) + '@babel/plugin-syntax-decorators': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0)) transitivePeerDependencies: - supports-color - '@babel/plugin-syntax-decorators@7.29.7(@babel/core@7.29.7)': + '@babel/plugin-syntax-decorators@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))': dependencies: - '@babel/core': 7.29.7 + '@babel/core': 7.29.7(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.29.7 - '@babel/plugin-syntax-jsx@7.29.7(@babel/core@7.29.7)': + '@babel/plugin-syntax-jsx@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))': dependencies: - '@babel/core': 7.29.7 + '@babel/core': 7.29.7(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.29.7 - '@babel/plugin-syntax-typescript@7.29.7(@babel/core@7.29.7)': + '@babel/plugin-syntax-typescript@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))': dependencies: - '@babel/core': 7.29.7 + '@babel/core': 7.29.7(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.29.7 - '@babel/plugin-transform-destructuring@7.29.7(@babel/core@7.29.7)': + '@babel/plugin-transform-destructuring@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.7 + '@babel/core': 7.29.7(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.29.7 - '@babel/traverse': 7.29.7 + '@babel/traverse': 7.29.7(supports-color@7.2.0) transitivePeerDependencies: - supports-color - '@babel/plugin-transform-explicit-resource-management@7.29.7(@babel/core@7.29.7)': + '@babel/plugin-transform-explicit-resource-management@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.7 + '@babel/core': 7.29.7(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.29.7 - '@babel/plugin-transform-destructuring': 7.29.7(@babel/core@7.29.7) + '@babel/plugin-transform-destructuring': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0) transitivePeerDependencies: - supports-color - '@babel/plugin-transform-modules-commonjs@7.29.7(@babel/core@7.29.7)': + '@babel/plugin-transform-modules-commonjs@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.7 - '@babel/helper-module-transforms': 7.29.7(@babel/core@7.29.7) + '@babel/core': 7.29.7(supports-color@7.2.0) + '@babel/helper-module-transforms': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.29.7 transitivePeerDependencies: - supports-color - '@babel/plugin-transform-typescript@7.29.7(@babel/core@7.29.7)': + '@babel/plugin-transform-typescript@7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.7 + '@babel/core': 7.29.7(supports-color@7.2.0) '@babel/helper-annotate-as-pure': 7.29.7 - '@babel/helper-create-class-features-plugin': 7.29.7(@babel/core@7.29.7) + '@babel/helper-create-class-features-plugin': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.29.7 - '@babel/helper-skip-transparent-expression-wrappers': 7.29.7 - '@babel/plugin-syntax-typescript': 7.29.7(@babel/core@7.29.7) + '@babel/helper-skip-transparent-expression-wrappers': 7.29.7(supports-color@7.2.0) + '@babel/plugin-syntax-typescript': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0)) transitivePeerDependencies: - supports-color - '@babel/preset-typescript@7.28.5(@babel/core@7.29.7)': + '@babel/preset-typescript@7.28.5(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.7 + '@babel/core': 7.29.7(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.29.7 '@babel/helper-validator-option': 7.29.7 - '@babel/plugin-syntax-jsx': 7.29.7(@babel/core@7.29.7) - '@babel/plugin-transform-modules-commonjs': 7.29.7(@babel/core@7.29.7) - '@babel/plugin-transform-typescript': 7.29.7(@babel/core@7.29.7) + '@babel/plugin-syntax-jsx': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0)) + '@babel/plugin-transform-modules-commonjs': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0) + '@babel/plugin-transform-typescript': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -3339,7 +3304,7 @@ snapshots: '@babel/parser': 7.29.7 '@babel/types': 7.29.7 - '@babel/traverse@7.29.7': + '@babel/traverse@7.29.7(supports-color@7.2.0)': dependencies: '@babel/code-frame': 7.29.7 '@babel/generator': 7.29.7 @@ -3347,7 +3312,7 @@ snapshots: '@babel/parser': 7.29.7 '@babel/template': 7.29.7 '@babel/types': 7.29.7 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -3363,11 +3328,11 @@ snapshots: '@bcoe/v8-coverage@1.0.2': {} - '@callstack/rspress-preset@0.6.6(@rsbuild/core@2.0.11)(@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2))(react-dom@19.2.7(react@19.2.7))(react@19.2.7)': + '@callstack/rspress-preset@0.6.6(@rsbuild/core@2.0.11)(@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(supports-color@7.2.0))(react-dom@19.2.7(react@19.2.7))(react@19.2.7)': dependencies: - '@callstack/rspress-theme': 0.6.6(@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2))(react-dom@19.2.7(react@19.2.7))(react@19.2.7) - '@rspress/core': 2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2) - '@rspress/plugin-sitemap': 2.0.8(@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2)) + '@callstack/rspress-theme': 0.6.6(@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(supports-color@7.2.0))(react-dom@19.2.7(react@19.2.7))(react@19.2.7) + '@rspress/core': 2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(supports-color@7.2.0) + '@rspress/plugin-sitemap': 2.0.8(@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(supports-color@7.2.0)) '@vercel/analytics': 2.0.1(react@19.2.7) rsbuild-plugin-open-graph: 1.1.2(@rsbuild/core@2.0.11) zod: 4.3.6 @@ -3383,9 +3348,9 @@ snapshots: - vue - vue-router - '@callstack/rspress-theme@0.6.6(@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2))(react-dom@19.2.7(react@19.2.7))(react@19.2.7)': + '@callstack/rspress-theme@0.6.6(@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(supports-color@7.2.0))(react-dom@19.2.7(react@19.2.7))(react@19.2.7)': dependencies: - '@rspress/core': 2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2) + '@rspress/core': 2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(supports-color@7.2.0) react: 19.2.7 react-dom: 19.2.7(react@19.2.7) @@ -3587,10 +3552,10 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 - '@limrun/api@0.24.5': + '@limrun/api@0.24.5(supports-color@7.2.0)': dependencies: eventsource-client: 1.2.0 - https-proxy-agent: 7.0.6 + https-proxy-agent: 7.0.6(supports-color@7.2.0) ignore: 7.0.6 proxy-from-env: 2.1.0 undici: 7.28.0 @@ -3600,7 +3565,7 @@ snapshots: - supports-color - utf-8-validate - '@mdx-js/mdx@3.1.1': + '@mdx-js/mdx@3.1.1(supports-color@7.2.0)': dependencies: '@types/estree': 1.0.8 '@types/estree-jsx': 1.0.5 @@ -3612,14 +3577,14 @@ snapshots: estree-util-is-identifier-name: 3.0.0 estree-util-scope: 1.0.0 estree-walker: 3.0.3 - hast-util-to-jsx-runtime: 2.3.6 + hast-util-to-jsx-runtime: 2.3.6(supports-color@7.2.0) markdown-extensions: 2.0.0 recma-build-jsx: 1.0.0 recma-jsx: 1.0.1(acorn@8.16.0) recma-stringify: 1.0.0 - rehype-recma: 1.0.0 - remark-mdx: 3.1.1 - remark-parse: 11.0.0 + rehype-recma: 1.0.0(supports-color@7.2.0) + remark-mdx: 3.1.1(supports-color@7.2.0) + remark-parse: 11.0.0(supports-color@7.2.0) remark-rehype: 11.1.2 source-map: 0.7.6 unified: 11.0.5 @@ -4018,9 +3983,9 @@ snapshots: optionalDependencies: '@rspack/core': 2.0.6(@swc/helpers@0.5.23) - '@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2)': + '@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@mdx-js/mdx': 3.1.1 + '@mdx-js/mdx': 3.1.1(supports-color@7.2.0) '@mdx-js/react': 3.1.1(@types/react@19.2.13)(react@19.2.7) '@rsbuild/core': 2.0.11 '@rsbuild/plugin-react': 2.0.0(@rsbuild/core@2.0.11)(@rspack/core@2.0.6(@swc/helpers@0.5.23)) @@ -4033,9 +3998,9 @@ snapshots: copy-to-clipboard: 3.3.3 flexsearch: 0.8.212 hast-util-heading-rank: 3.0.0 - hast-util-to-jsx-runtime: 2.3.6 - mdast-util-mdx: 3.0.0 - mdast-util-mdxjs-esm: 2.0.1 + hast-util-to-jsx-runtime: 2.3.6(supports-color@7.2.0) + mdast-util-mdx: 3.0.0(supports-color@7.2.0) + mdast-util-mdxjs-esm: 2.0.1(supports-color@7.2.0) medium-zoom: 1.1.0 nprogress: 0.2.0 react: 19.2.7 @@ -4046,11 +4011,11 @@ snapshots: react-router-dom: 7.18.1(react-dom@19.2.7(react@19.2.7))(react@19.2.7) rehype-external-links: 3.0.0 rehype-raw: 7.0.0 - remark-cjk-friendly: 2.0.1(@types/mdast@4.0.4)(micromark-util-types@2.0.2)(micromark@4.0.2)(unified@11.0.5) - remark-cjk-friendly-gfm-strikethrough: 2.0.1(@types/mdast@4.0.4)(micromark-util-types@2.0.2)(micromark@4.0.2)(unified@11.0.5) - remark-gfm: 4.0.1 - remark-mdx: 3.1.1 - remark-parse: 11.0.0 + remark-cjk-friendly: 2.0.1(@types/mdast@4.0.4)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(unified@11.0.5) + remark-cjk-friendly-gfm-strikethrough: 2.0.1(@types/mdast@4.0.4)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(unified@11.0.5) + remark-gfm: 4.0.1(supports-color@7.2.0) + remark-mdx: 3.1.1(supports-color@7.2.0) + remark-parse: 11.0.0(supports-color@7.2.0) remark-stringify: 11.0.0 scroll-into-view-if-needed: 3.1.0 shiki: 4.0.2 @@ -4068,9 +4033,9 @@ snapshots: - micromark-util-types - supports-color - '@rspress/plugin-sitemap@2.0.8(@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2))': + '@rspress/plugin-sitemap@2.0.8(@rspress/core@2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(supports-color@7.2.0))': dependencies: - '@rspress/core': 2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2) + '@rspress/core': 2.0.12(@rspack/core@2.0.6(@swc/helpers@0.5.23))(@types/mdast@4.0.4)(@types/react@19.2.13)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(supports-color@7.2.0) '@rspress/shared@2.0.12': dependencies: @@ -4143,11 +4108,11 @@ snapshots: tslib: 2.8.1 typed-inject: 5.0.0 - '@stryker-mutator/core@9.6.1(@types/node@22.19.21)': + '@stryker-mutator/core@9.6.1(@types/node@22.19.21)(supports-color@7.2.0)': dependencies: '@inquirer/prompts': 8.5.2(@types/node@22.19.21) '@stryker-mutator/api': 9.6.1 - '@stryker-mutator/instrumenter': 9.6.1 + '@stryker-mutator/instrumenter': 9.6.1(supports-color@7.2.0) '@stryker-mutator/util': 9.6.1 ajv: 8.18.0 chalk: 5.6.2 @@ -4175,14 +4140,14 @@ snapshots: - '@types/node' - supports-color - '@stryker-mutator/instrumenter@9.6.1': + '@stryker-mutator/instrumenter@9.6.1(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.7 + '@babel/core': 7.29.7(supports-color@7.2.0) '@babel/generator': 7.29.7 '@babel/parser': 7.29.3 - '@babel/plugin-proposal-decorators': 7.29.7(@babel/core@7.29.7) - '@babel/plugin-transform-explicit-resource-management': 7.29.7(@babel/core@7.29.7) - '@babel/preset-typescript': 7.28.5(@babel/core@7.29.7) + '@babel/plugin-proposal-decorators': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0) + '@babel/plugin-transform-explicit-resource-management': 7.29.7(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0) + '@babel/preset-typescript': 7.28.5(@babel/core@7.29.7(supports-color@7.2.0))(supports-color@7.2.0) '@stryker-mutator/api': 9.6.1 '@stryker-mutator/util': 9.6.1 angular-html-parser: 10.4.0 @@ -4194,10 +4159,10 @@ snapshots: '@stryker-mutator/util@9.6.1': {} - '@stryker-mutator/vitest-runner@9.6.1(@stryker-mutator/core@9.6.1(@types/node@22.19.21))(vitest@4.1.8)': + '@stryker-mutator/vitest-runner@9.6.1(@stryker-mutator/core@9.6.1(@types/node@22.19.21)(supports-color@7.2.0))(vitest@4.1.8)': dependencies: '@stryker-mutator/api': 9.6.1 - '@stryker-mutator/core': 9.6.1(@types/node@22.19.21) + '@stryker-mutator/core': 9.6.1(@types/node@22.19.21)(supports-color@7.2.0) '@stryker-mutator/util': 9.6.1 semver: 7.8.5 tslib: 2.8.1 @@ -4245,10 +4210,6 @@ snapshots: dependencies: undici-types: 6.21.0 - '@types/node@22.19.7': - dependencies: - undici-types: 6.21.0 - '@types/react@19.2.13': dependencies: csstype: 3.2.3 @@ -4468,8 +4429,6 @@ snapshots: angular-html-parser@10.4.0: {} - ansi-regex@6.2.2: {} - ansis@4.3.1: {} assertion-error@2.0.1: {} @@ -4532,8 +4491,6 @@ snapshots: chardet@2.2.0: {} - cli-spinners@3.4.0: {} - cli-width@4.1.0: {} clsx@2.1.1: {} @@ -4542,8 +4499,6 @@ snapshots: comma-separated-tokens@2.0.3: {} - commander@13.1.0: {} - commander@14.0.3: {} compute-scroll-into-view@3.1.1: {} @@ -4564,9 +4519,11 @@ snapshots: csstype@3.2.3: {} - debug@4.4.3: + debug@4.4.3(supports-color@7.2.0): dependencies: ms: 2.1.3 + optionalDependencies: + supports-color: 7.2.0 decode-named-character-reference@1.3.0: dependencies: @@ -4817,7 +4774,7 @@ snapshots: web-namespaces: 2.0.1 zwitch: 2.0.4 - hast-util-to-estree@3.1.3: + hast-util-to-estree@3.1.3(supports-color@7.2.0): dependencies: '@types/estree': 1.0.8 '@types/estree-jsx': 1.0.5 @@ -4827,9 +4784,9 @@ snapshots: estree-util-attach-comments: 3.0.0 estree-util-is-identifier-name: 3.0.0 hast-util-whitespace: 3.0.0 - mdast-util-mdx-expression: 2.0.1 - mdast-util-mdx-jsx: 3.2.0 - mdast-util-mdxjs-esm: 2.0.1 + mdast-util-mdx-expression: 2.0.1(supports-color@7.2.0) + mdast-util-mdx-jsx: 3.2.0(supports-color@7.2.0) + mdast-util-mdxjs-esm: 2.0.1(supports-color@7.2.0) property-information: 7.1.0 space-separated-tokens: 2.0.2 style-to-js: 1.1.21 @@ -4852,7 +4809,7 @@ snapshots: stringify-entities: 4.0.4 zwitch: 2.0.4 - hast-util-to-jsx-runtime@2.3.6: + hast-util-to-jsx-runtime@2.3.6(supports-color@7.2.0): dependencies: '@types/estree': 1.0.8 '@types/hast': 3.0.4 @@ -4861,9 +4818,9 @@ snapshots: devlop: 1.1.0 estree-util-is-identifier-name: 3.0.0 hast-util-whitespace: 3.0.0 - mdast-util-mdx-expression: 2.0.1 - mdast-util-mdx-jsx: 3.2.0 - mdast-util-mdxjs-esm: 2.0.1 + mdast-util-mdx-expression: 2.0.1(supports-color@7.2.0) + mdast-util-mdx-jsx: 3.2.0(supports-color@7.2.0) + mdast-util-mdxjs-esm: 2.0.1(supports-color@7.2.0) property-information: 7.1.0 space-separated-tokens: 2.0.2 style-to-js: 1.1.21 @@ -4906,10 +4863,10 @@ snapshots: html-void-elements@3.0.0: {} - https-proxy-agent@7.0.6: + https-proxy-agent@7.0.6(supports-color@7.2.0): dependencies: agent-base: 7.1.4 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -5059,14 +5016,14 @@ snapshots: unist-util-is: 6.0.1 unist-util-visit-parents: 6.0.2 - mdast-util-from-markdown@2.0.3: + mdast-util-from-markdown@2.0.3(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 '@types/unist': 3.0.3 decode-named-character-reference: 1.3.0 devlop: 1.1.0 mdast-util-to-string: 4.0.0 - micromark: 4.0.2 + micromark: 4.0.2(supports-color@7.2.0) micromark-util-decode-numeric-character-reference: 2.0.2 micromark-util-decode-string: 2.0.1 micromark-util-normalize-identifier: 2.0.1 @@ -5084,67 +5041,67 @@ snapshots: mdast-util-find-and-replace: 3.0.2 micromark-util-character: 2.1.1 - mdast-util-gfm-footnote@2.1.0: + mdast-util-gfm-footnote@2.1.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 micromark-util-normalize-identifier: 2.0.1 transitivePeerDependencies: - supports-color - mdast-util-gfm-strikethrough@2.0.0: + mdast-util-gfm-strikethrough@2.0.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-gfm-table@2.0.0: + mdast-util-gfm-table@2.0.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 devlop: 1.1.0 markdown-table: 3.0.4 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-gfm-task-list-item@2.0.0: + mdast-util-gfm-task-list-item@2.0.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-gfm@3.1.0: + mdast-util-gfm@3.1.0(supports-color@7.2.0): dependencies: - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-gfm-autolink-literal: 2.0.1 - mdast-util-gfm-footnote: 2.1.0 - mdast-util-gfm-strikethrough: 2.0.0 - mdast-util-gfm-table: 2.0.0 - mdast-util-gfm-task-list-item: 2.0.0 + mdast-util-gfm-footnote: 2.1.0(supports-color@7.2.0) + mdast-util-gfm-strikethrough: 2.0.0(supports-color@7.2.0) + mdast-util-gfm-table: 2.0.0(supports-color@7.2.0) + mdast-util-gfm-task-list-item: 2.0.0(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-mdx-expression@2.0.1: + mdast-util-mdx-expression@2.0.1(supports-color@7.2.0): dependencies: '@types/estree-jsx': 1.0.5 '@types/hast': 3.0.4 '@types/mdast': 4.0.4 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-mdx-jsx@3.2.0: + mdast-util-mdx-jsx@3.2.0(supports-color@7.2.0): dependencies: '@types/estree-jsx': 1.0.5 '@types/hast': 3.0.4 @@ -5152,7 +5109,7 @@ snapshots: '@types/unist': 3.0.3 ccount: 2.0.1 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 parse-entities: 4.0.2 stringify-entities: 4.0.4 @@ -5161,23 +5118,23 @@ snapshots: transitivePeerDependencies: - supports-color - mdast-util-mdx@3.0.0: + mdast-util-mdx@3.0.0(supports-color@7.2.0): dependencies: - mdast-util-from-markdown: 2.0.3 - mdast-util-mdx-expression: 2.0.1 - mdast-util-mdx-jsx: 3.2.0 - mdast-util-mdxjs-esm: 2.0.1 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) + mdast-util-mdx-expression: 2.0.1(supports-color@7.2.0) + mdast-util-mdx-jsx: 3.2.0(supports-color@7.2.0) + mdast-util-mdxjs-esm: 2.0.1(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-mdxjs-esm@2.0.1: + mdast-util-mdxjs-esm@2.0.1(supports-color@7.2.0): dependencies: '@types/estree-jsx': 1.0.5 '@types/hast': 3.0.4 '@types/mdast': 4.0.4 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color @@ -5236,11 +5193,11 @@ snapshots: micromark-util-symbol: 2.0.1 micromark-util-types: 2.0.2 - micromark-extension-cjk-friendly-gfm-strikethrough@2.0.1(micromark-util-types@2.0.2)(micromark@4.0.2): + micromark-extension-cjk-friendly-gfm-strikethrough@2.0.1(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0)): dependencies: devlop: 1.1.0 get-east-asian-width: 1.5.0 - micromark: 4.0.2 + micromark: 4.0.2(supports-color@7.2.0) micromark-extension-cjk-friendly-util: 3.0.1(micromark-util-types@2.0.2) micromark-util-character: 2.1.1 micromark-util-chunked: 2.0.1 @@ -5257,10 +5214,10 @@ snapshots: optionalDependencies: micromark-util-types: 2.0.2 - micromark-extension-cjk-friendly@2.0.1(micromark-util-types@2.0.2)(micromark@4.0.2): + micromark-extension-cjk-friendly@2.0.1(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0)): dependencies: devlop: 1.1.0 - micromark: 4.0.2 + micromark: 4.0.2(supports-color@7.2.0) micromark-extension-cjk-friendly-util: 3.0.1(micromark-util-types@2.0.2) micromark-util-chunked: 2.0.1 micromark-util-resolve-all: 2.0.1 @@ -5491,10 +5448,10 @@ snapshots: micromark-util-types@2.0.2: {} - micromark@4.0.2: + micromark@4.0.2(supports-color@7.2.0): dependencies: '@types/debug': 4.1.13 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) decode-named-character-reference: 1.3.0 devlop: 1.1.0 micromark-core-commonmark: 2.0.3 @@ -5535,8 +5492,6 @@ snapshots: mute-stream@3.0.0: {} - nano-spawn@2.1.0: {} - nanoid@3.3.12: {} node-releases@2.0.51: {} @@ -5774,17 +5729,17 @@ snapshots: hast-util-raw: 9.1.0 vfile: 6.0.3 - rehype-recma@1.0.0: + rehype-recma@1.0.0(supports-color@7.2.0): dependencies: '@types/estree': 1.0.8 '@types/hast': 3.0.4 - hast-util-to-estree: 3.1.3 + hast-util-to-estree: 3.1.3(supports-color@7.2.0) transitivePeerDependencies: - supports-color - remark-cjk-friendly-gfm-strikethrough@2.0.1(@types/mdast@4.0.4)(micromark-util-types@2.0.2)(micromark@4.0.2)(unified@11.0.5): + remark-cjk-friendly-gfm-strikethrough@2.0.1(@types/mdast@4.0.4)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(unified@11.0.5): dependencies: - micromark-extension-cjk-friendly-gfm-strikethrough: 2.0.1(micromark-util-types@2.0.2)(micromark@4.0.2) + micromark-extension-cjk-friendly-gfm-strikethrough: 2.0.1(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0)) unified: 11.0.5 optionalDependencies: '@types/mdast': 4.0.4 @@ -5792,9 +5747,9 @@ snapshots: - micromark - micromark-util-types - remark-cjk-friendly@2.0.1(@types/mdast@4.0.4)(micromark-util-types@2.0.2)(micromark@4.0.2)(unified@11.0.5): + remark-cjk-friendly@2.0.1(@types/mdast@4.0.4)(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0))(unified@11.0.5): dependencies: - micromark-extension-cjk-friendly: 2.0.1(micromark-util-types@2.0.2)(micromark@4.0.2) + micromark-extension-cjk-friendly: 2.0.1(micromark-util-types@2.0.2)(micromark@4.0.2(supports-color@7.2.0)) unified: 11.0.5 optionalDependencies: '@types/mdast': 4.0.4 @@ -5802,28 +5757,28 @@ snapshots: - micromark - micromark-util-types - remark-gfm@4.0.1: + remark-gfm@4.0.1(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 - mdast-util-gfm: 3.1.0 + mdast-util-gfm: 3.1.0(supports-color@7.2.0) micromark-extension-gfm: 3.0.0 - remark-parse: 11.0.0 + remark-parse: 11.0.0(supports-color@7.2.0) remark-stringify: 11.0.0 unified: 11.0.5 transitivePeerDependencies: - supports-color - remark-mdx@3.1.1: + remark-mdx@3.1.1(supports-color@7.2.0): dependencies: - mdast-util-mdx: 3.0.0 + mdast-util-mdx: 3.0.0(supports-color@7.2.0) micromark-extension-mdxjs: 3.0.0 transitivePeerDependencies: - supports-color - remark-parse@11.0.0: + remark-parse@11.0.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) micromark-util-types: 2.0.2 unified: 11.0.5 transitivePeerDependencies: @@ -5976,16 +5931,6 @@ snapshots: signal-exit@4.1.0: {} - skillgym@0.9.1: - dependencies: - '@types/node': 22.19.7 - cli-spinners: 3.4.0 - commander: 13.1.0 - nano-spawn: 2.1.0 - picocolors: 1.1.1 - pretty-ms: 9.3.0 - string-width: 7.2.0 - source-map-js@1.2.1: {} source-map@0.7.6: {} @@ -5996,21 +5941,11 @@ snapshots: std-env@4.0.0: {} - string-width@7.2.0: - dependencies: - emoji-regex: 10.6.0 - get-east-asian-width: 1.5.0 - strip-ansi: 7.2.0 - stringify-entities@4.0.4: dependencies: character-entities-html4: 2.1.0 character-entities-legacy: 3.0.0 - strip-ansi@7.2.0: - dependencies: - ansi-regex: 6.2.2 - strip-final-newline@4.0.0: {} style-to-js@1.1.21: diff --git a/scripts/check-affected/checks.ts b/scripts/check-affected/checks.ts index 3130cfd9cc..a26ef0ce19 100644 --- a/scripts/check-affected/checks.ts +++ b/scripts/check-affected/checks.ts @@ -141,17 +141,6 @@ export const CHECK_CATALOG: readonly CheckSpec[] = [ ciJobs: ['Web Platform Smoke'], localRunnable: false, }, - { - id: 'skillgym', - label: 'SkillGym command-planning suite', - kind: { type: 'script', script: 'test:skillgym' }, - // No GitHub workflow runs SkillGym; per the Testing Matrix in - // docs/agents/testing.md it is a local-only gate (`pnpm test:skillgym`). - // Keep it locally runnable rather than claiming a CI job that does not - // exist and silently skipping it. - ciJobs: [], - localRunnable: true, - }, { id: 'replay-compat', label: 'Replay-compat corpus provenance (released blobs)', diff --git a/scripts/check-affected/model.test.ts b/scripts/check-affected/model.test.ts index 584e50990e..9b75a13701 100644 --- a/scripts/check-affected/model.test.ts +++ b/scripts/check-affected/model.test.ts @@ -127,17 +127,11 @@ test('a replay-compat manifest edit selects the provenance verifier', () => { assert.ok(result.checks.includes('replay-compat')); }); -test('skills guidance change selects format + skillgym, not docs-only', () => { +test('skills guidance change is docs-only', () => { const result = plan(['skills/agent-device/SKILL.md']); assert.equal(result.failOpen, false); - assert.equal(result.docsOnlyPaths.length, 0); - assert.ok(result.checks.includes('skillgym'), 'skills change must select the SkillGym suite'); - assert.ok(result.checks.includes('format'), 'skills change must still run format'); -}); - -test('SkillGym harness change selects the skillgym suite', () => { - const result = ids(['test/skillgym/suites/agent-device-smoke-suite.ts']); - assert.ok(result.includes('skillgym')); + assert.deepEqual(result.docsOnlyPaths, ['skills/agent-device/SKILL.md']); + assert.deepEqual(result.checks, []); }); test('workflow/tooling and selector-owning changes fail open', () => { @@ -187,7 +181,6 @@ test('every catalog command resolves against package scripts', () => { 'build:android-snapshot-helper': 'x', 'build:macos-helper': 'x', 'test:smoke:web': 'x', - 'test:skillgym': 'x', 'check:replay-compat': 'x', }; for (const spec of CHECK_CATALOG) { @@ -272,9 +265,3 @@ test('every catalog CI job maps to a real workflow job (no fabricated checks)', } } }); - -test('skillgym is a local-only gate (locally runnable, claims no CI job)', () => { - const skillgym = CHECK_CATALOG.find((spec) => spec.id === 'skillgym')!; - assert.equal(skillgym.localRunnable, true); - assert.deepEqual([...skillgym.ciJobs], []); -}); diff --git a/scripts/check-affected/model.ts b/scripts/check-affected/model.ts index 5f519d2351..b216c92988 100644 --- a/scripts/check-affected/model.ts +++ b/scripts/check-affected/model.ts @@ -11,10 +11,7 @@ // categories, so they are never silently skipped (issue constraint); // - a small explicit build-ownership layer covers Swift, Android helpers, // the macOS helper, MCP metadata, and the public package surface — the -// only paths whose owning build the sources of truth cannot derive; -// - SkillGym owns skill guidance (`skills/`) and its harness -// (`test/skillgym/`): those changes select the SkillGym suite, and their -// Markdown is skill/harness input, not inert docs. +// only paths whose owning build the sources of truth cannot derive. // // Anything the model cannot confidently classify fails open to the full check // set: unknown paths, workflow/tooling, the selector's own sources, and @@ -40,7 +37,6 @@ export type CheckId = | 'android-helpers' | 'macos-helper' | 'web-smoke' - | 'skillgym' | 'replay-compat'; // The complete local check universe. A fail-open plan selects all of these; @@ -63,7 +59,6 @@ export const ALL_CHECKS: readonly CheckId[] = [ 'android-helpers', 'macos-helper', 'web-smoke', - 'skillgym', 'replay-compat', ]; @@ -127,10 +122,8 @@ function isWorkflowTooling(file: string): boolean { } function isDocs(file: string): boolean { - // skills/ and the SkillGym harness are validated by the SkillGym suite (and - // formatting), not treated as inert docs — even their Markdown is skill - // guidance or harness input, so let them flow to ownership rules instead. - if (file.startsWith('skills/') || file.startsWith('test/skillgym/')) return false; + // skills/ Markdown is agent guidance prose with no owning suite (the + // SkillGym harness was removed), so it classifies as docs like the rest. return ( file.startsWith('docs/') || file.startsWith('website/') || @@ -153,7 +146,6 @@ type FileFacts = { isTs: boolean; underSrc: boolean; underTest: boolean; - underSkills: boolean; isSrcProd: boolean; }; @@ -163,9 +155,9 @@ function reason(check: CheckId, file: string, rule: string, detail: string): Sel return { check, path: file, rule, detail }; } -const formatGate: OwnershipRule = ({ file, underSrc, underTest, underSkills }) => - underSrc || underTest || underSkills - ? [reason('format', file, 'gate:format', 'oxfmt covers src/, test/, and skills/')] +const formatGate: OwnershipRule = ({ file, underSrc, underTest }) => + underSrc || underTest + ? [reason('format', file, 'gate:format', 'oxfmt covers src/ and test/')] : []; const staticTsGates: OwnershipRule = ({ file, isTs, underSrc, underTest }) => @@ -211,10 +203,7 @@ function isNodeIntegrationPath(file: string): boolean { } const vitestRelatedOwnership: OwnershipRule = ({ file, isTs, underSrc, underTest }) => - isTs && - (underSrc || underTest) && - !isNodeIntegrationPath(file) && - !file.startsWith('test/skillgym/') + isTs && (underSrc || underTest) && !isNodeIntegrationPath(file) ? [ reason( 'vitest-related', @@ -258,21 +247,6 @@ const replayCompatOwnership: OwnershipRule = ({ file }) => { return selections; }; -// SkillGym validates skill guidance (`skills/`) and owns its harness -// (`test/skillgym/`); the Testing Matrix in docs/agents/testing.md routes -// skill-prompt/assertion changes here. -const skillgymOwnership: OwnershipRule = ({ file, underSkills }) => - underSkills || file.startsWith('test/skillgym/') - ? [ - reason( - 'skillgym', - file, - 'own:skillgym', - 'SkillGym suite validates skill guidance and its harness', - ), - ] - : []; - const BUILD_OWNERSHIP: ReadonlyArray<{ check: CheckId; rule: string; @@ -325,7 +299,6 @@ const OWNERSHIP_RULES: readonly OwnershipRule[] = [ vitestRelatedOwnership, nodeIntegrationOwnership, replayCompatOwnership, - skillgymOwnership, buildOwnership, ]; @@ -337,7 +310,6 @@ function fileFacts(file: string): FileFacts { isTs, underSrc, underTest: file.startsWith('test/'), - underSkills: file.startsWith('skills/'), isSrcProd: underSrc && isTs && !isTestPath(file), }; } diff --git a/scripts/check-affected/run.test.ts b/scripts/check-affected/run.test.ts index d69ba6f59e..02a63f490c 100644 --- a/scripts/check-affected/run.test.ts +++ b/scripts/check-affected/run.test.ts @@ -100,7 +100,6 @@ const ALL_SCRIPTS: Record = { 'test:integration:provider': 'x', 'test:integration:node': 'x', 'test:integration:progress:check': 'x', - 'test:skillgym': 'x', 'check:replay-compat': 'x', }; diff --git a/scripts/layering/check.ts b/scripts/layering/check.ts index 241ab642c2..1676f5c952 100644 --- a/scripts/layering/check.ts +++ b/scripts/layering/check.ts @@ -261,8 +261,6 @@ function checkTypeInversions(edges: readonly ResolvedImportEdge[]): Violation[] // moves here bring it to 102. Measured on the rebased tree — an earlier 87 was taken against an // older main and was stale rather than violated, which is exactly the kind of drift a ratchet // measured at merge time prevents. -// Exported so the repo-health snapshot (scripts/repo-health) can record the R9 ratchet value as -// observatory data without re-deriving it. The gate remains the authority; the snapshot follows. export const TYPE_CYCLE_BASELINE = 102; function checkTypeCycleGrowth(actual: number): Violation[] { diff --git a/scripts/repo-health/README.md b/scripts/repo-health/README.md deleted file mode 100644 index 1f41a1bbfb..0000000000 --- a/scripts/repo-health/README.md +++ /dev/null @@ -1,136 +0,0 @@ -# Repo-health snapshot - -```sh -pnpm repo-health # human-readable summary -pnpm repo-health --json # the raw JSON snapshot on stdout -pnpm repo-health --out p # also write the JSON snapshot to a file -``` - -One JSON snapshot aggregating the signals this repo already computes across 24+ CI checks, so an -agent can query the repo's state in one read instead of scraping job logs. It is **data, not a -dashboard** (#1409's lesson): there is no rendering, and it draws no conclusions for you. - -It reuses each analyzer rather than reimplementing any metric, runs from a clean checkout in well -under two minutes, and needs no network — everything comes from the working tree, `git`, and the -artifacts other gates already write. - -## What it does NOT do - -- **No history / trends / PR comments.** That is the follow-up quality-delta issue (#1424); this - snapshot is the input it persists. -- **No gating on component metrics.** Instability / abstractness / main-sequence distance are - observatory data to locate concrete high-fan-in modules worth pinning harder (CONTEXT.md - "Principles and their gates"). They must **never** become CI thresholds. -- **The one thing that can fail the command** is the depgraph-vs-layering R6 consistency - assertion — see below. - -## The R6 consistency assertion (the only wired-into-CI part) - -The dependency graph the snapshot builds must reproduce the layering gate's -`TYPE_INVERSION_BASELINE` (`scripts/layering/check.ts`), pair for pair (#1410). `pnpm repo-health` -exits non-zero and prints the difference when they diverge. The **identical** assertion is already -wired into CI by the **Layering Guard** job, which runs `scripts/depgraph/model.test.ts` — so no -new CI job or cost is added here. The gate stays the authority: a mismatch means the tree or the -baseline changed, never the snapshot. - -## Schema (`schemaVersion: 1`) - -Field names are the stable **trend/delta contract** consumed by #1424. Any change bumps -`schemaVersion`; #1424 refuses to diff across schema versions without a migration note. - -```jsonc -{ - "schemaVersion": 1, - "provenance": { - "schemaVersion": 1, - "commit": "", - "ref": "", - "node": "v22.x", - "generatedAt": "", - // Content hash per analyzer/config, standing in for a version: a source change is a version bump. - "tool": { "depgraph": "…", "layering": "…", "fallow": "…", "coverage": "…", "size": "…", - "slowTest": "…", "bench": "…", "skillgym": "…" }, - "inputs": { - "sourceFiles": 932, // production files fed to the graph build - "lockfile": { "path": "pnpm-lock.yaml", "sha256": "…" }, - // coverage and size are read artifacts this command does NOT produce. Their freshness relative - // to `commit` is proven ONLY from a producing commit the artifact stamps in its own bytes — - // never mtime (a copy/restore/CI-cache download resets it) nor an enumerated input list (never - // provably complete). status: "fresh" = stamped commit == HEAD; "stale" = stamped commit != - // HEAD; "unknown" = no stamp. Today neither producer stamps a commit, so both report "unknown" - // (honest — cannot be proven current); if one starts emitting `commit`, it is verified. The - // bytes are still hashed so #1424 keys history on the exact metrics. A consumer must treat - // "unknown"/"stale" alike as not-current, never as `commit`. - "coverageSummary": { "path": "coverage/coverage-summary.json", "sha256": "…", - "producerCommit": null, "status": "unknown" } | null, - "sizeReport": { "path": ".tmp/size-report.json", "sha256": "…", - "producerCommit": null, "status": "unknown" } | null - } - }, - "metrics": { - "depgraph": { // from scripts/depgraph (reuses the layering edge model) - "files": 932, "edges": 4791, - "valueCycles": 0, "typeCycles": 7, "dynamicCycles": 1, - "redundantEdges": 1366, // value edges also reachable at distance >= 2 (NOT removable) - "backEdges": 0, // R5 ranked-spine value back-edges (gate keeps this at 0) - "typeInversionEdges": 7 // R6 type-only spine inversions, summed - }, - "layering": { // ratchet values from scripts/layering/check.ts - "typeInversionBaseline": { "commands -> client": 3, "…": 0 }, - "typeInversionTotal": 7, // R6 ratchet (may only shrink) - "typeCycleBaseline": 102 // R9 largest-type-cycle ceiling (growth-only) - }, - "coverage": { // from coverage/coverage-summary.json (json-summary reporter) - "available": true, - "lines": { "total": …, "covered": …, "pct": … }, - "statements": …, "functions": …, "branches": … - }, // { "available": false } when the artifact is absent - "size": { // from .tmp/size-report.json (scripts/size-report.mjs) - "available": true, - "jsRawBytes": …, "jsGzipBytes": …, "npmTarballBytes": …, "npmUnpackedBytes": … - }, // { "available": false } when the artifact is absent - "fallow": { // from fallow-baselines/*.json + .fallowrc.json - "deadCodeFindings": 0, "healthFindings": 161, - "suppressions": { "ignoredExports": …, "ignoredDependencies": …, "total": 19 } - }, - "slowTest": { // ratchet state from scripts/vitest-slow-test-reporter.ts - "unitBudgetMs": 2500, "integrationBudgetMs": 15000, "enforceFactor": 2 - }, - "bench": { "cases": 19, "topics": 10 }, // scripts/help-conformance-cases.mjs registry - "skillgym": { "cases": 5 }, // test/skillgym/suites/agent-device-smoke-suite.ts - "components": { // OBSERVATORY ONLY — never a CI threshold - "byZone": [ - { "zone": "kernel", "rank": 0, "classification": "ranked", - "files": …, "loc": …, - "afferent": 764, "efferent": 0, // cross-zone couplings (Ca / Ce) - "instability": 0.0, // I = Ce / (Ca + Ce) - "abstractness": 0.36, // A ≈ type-only share of afferent edges (approx.) - "distance": 0.64 } // |A + I - 1| - ] - }, - "mainSequence": { // OBSERVATORY ONLY - "concreteHighFanIn": [ // concrete (A < 0.5), fan-in >= 10, most-depended-on first - { "file": "utils/exec.ts", "zone": "utils", "fanIn": 81, "fanOut": 3, - "instability": 0.04, "abstractness": 0.17, "distance": 0.79, "concrete": true } - ] - } - }, - "consistency": { "r6": { "ok": true, "expected": { … }, "actual": { … } } } -} -``` - -### Coverage and size availability - -`coverage` and `size` are read from the artifacts the coverage and size gates already produce -(`coverage/coverage-summary.json` via the vitest `json-summary` reporter; `.tmp/size-report.json` -via `pnpm size:markdown`). Recomputing either would blow the two-minute / no-network budget, so a -clean checkout that has run neither gets `{ "available": false }` with the producing command -named in the summary. Run `pnpm test:coverage` and/or `pnpm size:markdown` first (or point #1424's -history job at a run that already did) to populate them. - -### Abstractness is a first approximation - -Per the issue, `abstractness` approximates a component's type-only export share by the share of -its **incoming edges that are type-only**. A module most of whose dependents import it for values -(e.g. `src/utils/exec.ts`, `src/daemon/ref-frame.ts`) reads as concrete; when its fan-in is also -high, it is a foundation worth pinning harder. This is a locator, not a verdict. diff --git a/scripts/repo-health/components.ts b/scripts/repo-health/components.ts deleted file mode 100644 index e90e084efd..0000000000 --- a/scripts/repo-health/components.ts +++ /dev/null @@ -1,132 +0,0 @@ -// Component metrics for the repo-health snapshot (#1423): instability, abstractness, and -// main-sequence distance derived from the dependency graph's fan-in/fan-out. -// -// These are OBSERVATORY DATA ONLY. They locate concrete, high-fan-in modules worth pinning harder; -// they must NEVER become CI thresholds (CONTEXT.md "Principles and their gates"). Kept in their own -// module so that constraint — and the approximations these metrics make — stays visible instead of -// blending into the analyzer adapters that feed the gate-adjacent numbers. - -import { zoneRank } from '../layering/model.ts'; -import type { GraphData } from '../depgraph/model.ts'; - -export type ZoneComponent = { - zone: string; - rank: number | null; - classification: string; - files: number; - loc: number; - /** Afferent couplings: cross-zone edges INTO the zone (how much depends on it). */ - afferent: number; - /** Efferent couplings: cross-zone edges OUT of the zone (how much it depends on). */ - efferent: number; - /** Instability I = efferent / (afferent + efferent); 0 when the zone has no cross-zone edges. */ - instability: number; - /** - * Abstractness A, approximated by the type-only share of afferent edges (first approximation - * per the issue). 0 when nothing depends on the zone. - */ - abstractness: number; - /** Distance from the main sequence, |A + I - 1|. Lower is "more balanced". */ - distance: number; -}; - -function round(value: number): number { - return Math.round(value * 1000) / 1000; -} - -function zoneMapOf(graph: GraphData): Map { - return new Map(graph.nodes.map((node) => [node.id, node.zone])); -} - -/** - * Per-zone instability / abstractness / main-sequence distance from the graph's cross-zone edges. - * Observatory data — never a CI threshold. - */ -export function zoneComponents(graph: GraphData): ZoneComponent[] { - const zoneOf = zoneMapOf(graph); - const afferent = new Map(); - const efferent = new Map(); - const typeAfferent = new Map(); - const bump = (map: Map, key: string): void => - map.set(key, (map.get(key) ?? 0) + 1); - - for (const edge of graph.edges) { - const from = zoneOf.get(edge.from); - const to = zoneOf.get(edge.to); - if (from === undefined || to === undefined || from === to) continue; - bump(efferent, from); - bump(afferent, to); - if (edge.kind === 'type') bump(typeAfferent, to); - } - - return graph.zones - .map((zone) => { - const ca = afferent.get(zone.id) ?? 0; - const ce = efferent.get(zone.id) ?? 0; - const instability = ca + ce === 0 ? 0 : ce / (ca + ce); - const abstractness = ca === 0 ? 0 : (typeAfferent.get(zone.id) ?? 0) / ca; - return { - zone: zone.id, - rank: zoneRank(zone.id), - classification: zone.classification, - files: zone.files, - loc: zone.loc, - afferent: ca, - efferent: ce, - instability: round(instability), - abstractness: round(abstractness), - distance: round(Math.abs(abstractness + instability - 1)), - }; - }) - .sort((left, right) => right.afferent - left.afferent); -} - -export type MainSequenceModule = { - file: string; - zone: string; - fanIn: number; - fanOut: number; - instability: number; - abstractness: number; - distance: number; - concrete: boolean; -}; - -/** - * The main-sequence report: concrete, high-fan-in modules worth pinning harder. Per-file - * abstractness is approximated by the module's type-only dependent share; a module most of whose - * dependents import it for VALUES (e.g. src/utils/exec.ts, src/daemon/ref-frame.ts) is concrete - * and, when its fan-in is high, a foundation the tree leans on. Observatory data only. - */ -export function mainSequenceModules( - graph: GraphData, - options: { minFanIn?: number; limit?: number } = {}, -): MainSequenceModule[] { - const minFanIn = options.minFanIn ?? 10; - const limit = options.limit ?? 40; - const typeIn = new Map(); - for (const edge of graph.edges) { - if (edge.kind === 'type') typeIn.set(edge.to, (typeIn.get(edge.to) ?? 0) + 1); - } - - return graph.nodes - .filter((node) => node.fanIn >= minFanIn) - .map((node) => { - const abstractness = node.fanIn === 0 ? 0 : (typeIn.get(node.id) ?? 0) / node.fanIn; - const total = node.fanIn + node.fanOut; - const instability = total === 0 ? 0 : node.fanOut / total; - return { - file: node.id.replace(/^src\//, ''), - zone: node.zone, - fanIn: node.fanIn, - fanOut: node.fanOut, - instability: round(instability), - abstractness: round(abstractness), - distance: round(Math.abs(abstractness + instability - 1)), - concrete: abstractness < 0.5, - }; - }) - .filter((module) => module.concrete) - .sort((left, right) => right.fanIn - left.fanIn) - .slice(0, limit); -} diff --git a/scripts/repo-health/model.test.ts b/scripts/repo-health/model.test.ts deleted file mode 100644 index dea1a43ab1..0000000000 --- a/scripts/repo-health/model.test.ts +++ /dev/null @@ -1,215 +0,0 @@ -import { readFileSync } from 'node:fs'; -import { describe, expect, test } from 'vitest'; -import { listSourceFiles, TYPE_INVERSION_BASELINE } from '../layering/check.ts'; -import { resolveImportEdges } from '../layering/model.ts'; -import { buildGraph, type GraphData } from '../depgraph/model.ts'; -import { mainSequenceModules, zoneComponents } from './components.ts'; -import { - collectArtifactProvenance, - benchMetrics, - buildSnapshot, - coverageMetrics, - depgraphMetrics, - fallowMetrics, - layeringRatchets, - r6Consistency, - ratchetTotal, - sizeMetrics, - SNAPSHOT_SCHEMA_VERSION, - type Provenance, -} from './model.ts'; - -/** Build the report's graph over the real production tree, the way the command does. */ -function realGraph(): GraphData { - const files = listSourceFiles(); - const sources = new Map(files.map((file) => [file, readFileSync(file, 'utf8')])); - return buildGraph(sources, resolveImportEdges(sources)); -} - -/** - * A tiny hand-made graph: two zones depend on kernel for VALUES and one for TYPES, so kernel is - * concrete (abstractness 1/3) with fan-in 3 — a miniature of the real exec.ts/ref-frame.ts shape. - */ -function fixtureGraph(): GraphData { - const sources = new Map([ - ['src/kernel/errors.ts', 'export const fail = 1;\nexport type Err = string;\n'], - ['src/core/run.ts', "import { fail } from '../kernel/errors.ts';\nexport const run = 1;\n"], - ['src/core/wrap.ts', "import { fail } from '../kernel/errors.ts';\nexport const wrap = 1;\n"], - [ - 'src/commands/tap.ts', - [ - "import { run } from '../core/run.ts';", - "import type { Err } from '../kernel/errors.ts';", - ].join('\n'), - ], - ]); - return buildGraph(sources, resolveImportEdges(sources)); -} - -test('depgraphMetrics reuses the graph rather than recomputing any number', () => { - const metrics = depgraphMetrics(fixtureGraph()); - expect(metrics.files).toBe(4); - expect(metrics.edges).toBe(4); - expect(metrics.valueCycles).toBe(0); - // commands -> kernel is a type-only import; the ranked spine outranks kernel so it is an R6 - // inversion, not a value/back edge. - expect(metrics.backEdges).toBe(0); -}); - -test('ratchetTotal sums a per-pair ratchet map', () => { - expect(ratchetTotal({ a: 3, b: 1, c: 2 })).toBe(6); -}); - -test('layeringRatchets reports the gate ratchets without re-deriving them', () => { - const ratchets = layeringRatchets(); - expect(ratchets.typeInversionBaseline).toBe(TYPE_INVERSION_BASELINE); - expect(ratchets.typeInversionTotal).toBe(ratchetTotal(TYPE_INVERSION_BASELINE)); - expect(ratchets.typeCycleBaseline).toBeGreaterThan(0); -}); - -describe('component metrics are derived from fan-in/fan-out', () => { - test('zoneComponents computes instability, abstractness and main-sequence distance', () => { - const byZone = zoneComponents(fixtureGraph()); - const kernel = byZone.find((zone) => zone.zone === 'kernel')!; - // Nothing in the fixture leaves kernel, and three files depend on it (one type-only). - expect(kernel.efferent).toBe(0); - expect(kernel.afferent).toBe(3); - expect(kernel.instability).toBe(0); - expect(kernel.abstractness).toBe(0.333); - }); - - test('mainSequenceModules surfaces concrete high-fan-in modules only', () => { - const modules = mainSequenceModules(fixtureGraph(), { minFanIn: 1 }); - const kernel = modules.find((module) => module.file === 'kernel/errors.ts')!; - expect(kernel.fanIn).toBe(3); - // Most dependents import it for values, so abstractness is 1/3 and it is concrete. - expect(kernel.abstractness).toBe(0.333); - expect(kernel.concrete).toBe(true); - }); -}); - -test('r6Consistency passes when the graph reproduces the baseline and fails on drift', () => { - const passing = r6Consistency(TYPE_INVERSION_BASELINE); - expect(passing.ok).toBe(true); - - const drifted = r6Consistency({ ...TYPE_INVERSION_BASELINE, 'commands -> client': 99 }); - expect(drifted.ok).toBe(false); - expect(drifted.expected).toEqual(passing.expected); -}); - -test('fallowMetrics counts findings from the baselines and suppressions from the config', () => { - const metrics = fallowMetrics( - { unused_exports: [{}, {}], circular_dependencies: [], unused_files: [{}] }, - { - finding_counts: { - 'src/a.ts': { complexity_critical: { count: 1 }, crap_moderate: { count: 2 } }, - 'src/b.ts': { crap_moderate: { count: 1 } }, - }, - }, - { - ignoreExports: [{ exports: ['x', 'y'] }, { exports: ['z'] }], - ignoreDependencies: ['@theme'], - }, - ); - expect(metrics.deadCodeFindings).toBe(3); - expect(metrics.healthFindings).toBe(4); - expect(metrics.suppressions).toEqual({ ignoredExports: 3, ignoredDependencies: 1, total: 4 }); -}); - -test('coverage and size degrade to unavailable when the artifact is absent', () => { - expect(coverageMetrics(null)).toEqual({ available: false }); - expect(sizeMetrics(null)).toEqual({ available: false }); - expect( - coverageMetrics({ total: { lines: { total: 10, covered: 8, pct: 80 } } as never }).available, - ).toBe(true); -}); - -// Drives the real collector (parse → read stamped commit → hash → status), so removing the -// commit-verification breaks these — unlike a fabricated status. Freshness is proven ONLY from a -// commit the artifact stamps, never mtime or a producer-input list. -test('collectArtifactProvenance proves freshness only from a commit the artifact stamps', () => { - const path = 'coverage/coverage-summary.json'; - - // A real coverage-summary shape stamps no commit — we cannot prove it matches HEAD, so `unknown`. - const coverageShaped = JSON.stringify({ total: { lines: { total: 10, covered: 8, pct: 80 } } }); - const unknown = collectArtifactProvenance(path, coverageShaped, 'abc123'); - expect(unknown?.status).toBe('unknown'); - expect(unknown?.producerCommit).toBeNull(); - expect(unknown?.sha256).toMatch(/^[0-9a-f]{12}$/); // bytes still hashed for #1424 - - // An artifact that stamps the current commit is proven current. - expect( - collectArtifactProvenance(path, JSON.stringify({ commit: 'abc123', total: {} }), 'abc123') - ?.status, - ).toBe('fresh'); - - // A stamped commit that differs is stale — metrics predate HEAD. - const stale = collectArtifactProvenance(path, JSON.stringify({ commit: 'old999' }), 'abc123'); - expect(stale?.status).toBe('stale'); - expect(stale?.producerCommit).toBe('old999'); - - // Absent artifact degrades cleanly to null (coverage/size become { available: false }). - expect(collectArtifactProvenance(path, null, 'abc123')).toBeNull(); -}); - -test('benchMetrics counts cases and distinct topics from the case registry', () => { - expect(benchMetrics([{ docs: ['a', 'b'] }, { docs: ['b'] }, { docs: ['c'] }])).toEqual({ - cases: 3, - topics: 3, - }); -}); - -const PROVENANCE: Provenance = { - schemaVersion: SNAPSHOT_SCHEMA_VERSION, - commit: 'abc', - ref: 'main', - node: 'v22', - generatedAt: '2026-07-28T00:00:00.000Z', - tool: {}, - inputs: { sourceFiles: 3, lockfile: null, coverageSummary: null, sizeReport: null }, -}; - -test('buildSnapshot assembles every metric family and pins the schema version', () => { - const snapshot = buildSnapshot({ - provenance: PROVENANCE, - graph: fixtureGraph(), - fallow: { deadCode: {}, health: {}, config: {} }, - coverage: null, - size: null, - slowTest: { unitBudgetMs: 2500, integrationBudgetMs: 15000, enforceFactor: 2 }, - benchCases: [{ docs: ['a'] }], - skillgymCaseCount: 5, - }); - expect(snapshot.schemaVersion).toBe(SNAPSHOT_SCHEMA_VERSION); - expect(Object.keys(snapshot.metrics).sort()).toEqual([ - 'bench', - 'components', - 'coverage', - 'depgraph', - 'fallow', - 'layering', - 'mainSequence', - 'size', - 'skillgym', - 'slowTest', - ]); - expect(snapshot.metrics.skillgym.cases).toBe(5); -}); - -// The acceptance spot-check (issue #1423): the main-sequence report must locate the concrete, -// high-fan-in foundations. src/utils/exec.ts (the process-exec seam) and src/daemon/ref-frame.ts -// both have many value dependents, so both must appear as concrete. Guards the metric against a -// sign flip or an abstractness inversion that would make the report point at the wrong modules. -test('the real-tree main-sequence report surfaces exec.ts and ref-frame.ts as concrete', () => { - const graph = realGraph(); - const concrete = new Map(mainSequenceModules(graph).map((module) => [module.file, module])); - for (const file of ['utils/exec.ts', 'daemon/ref-frame.ts']) { - const module = concrete.get(file); - expect(module, `${file} should surface as a concrete high-fan-in module`).toBeDefined(); - expect(module!.concrete).toBe(true); - expect(module!.fanIn).toBeGreaterThanOrEqual(10); - } - - // And the graph the report builds must reproduce the gate's R6 baseline over the real tree. - expect(r6Consistency(graph.typeInversions).ok).toBe(true); -}); diff --git a/scripts/repo-health/model.ts b/scripts/repo-health/model.ts deleted file mode 100644 index 9246d2ecb3..0000000000 --- a/scripts/repo-health/model.ts +++ /dev/null @@ -1,376 +0,0 @@ -// Repo-health snapshot model — pure aggregation over the analyzers this repo already runs. -// -// The snapshot is DATA for agents to query (issue #1423, Track C of #1412), not a dashboard and -// not a new gate. Every metric here is REUSED from an existing analyzer rather than recomputed: -// the dependency graph comes from scripts/depgraph (which itself reuses the layering gate's edge -// model), the R6/R9 ratchet values come from scripts/layering/check.ts, fallow counts come from -// the checked-in baselines, coverage/size come from the artifacts their own gates already write, -// the slow-test ratchet comes from the vitest reporter, and the bench/skillgym counts come from -// their case registries. -// -// Field names are the trend/delta contract consumed by the follow-up quality-delta comment -// (#1424): they must stay stable, and any change bumps SNAPSHOT_SCHEMA_VERSION so a consumer -// refuses to diff across schema versions without a migration note. -// -// The component metrics (instability / abstractness / main-sequence distance) are OBSERVATORY -// DATA ONLY. They locate concrete, high-fan-in modules worth pinning harder; they must NEVER -// become CI thresholds (CONTEXT.md "Principles and their gates"). The single thing wired into CI -// is the depgraph-vs-layering R6 consistency assertion, which lives in the Layering Guard job. - -import { createHash } from 'node:crypto'; -import { TYPE_CYCLE_BASELINE, TYPE_INVERSION_BASELINE } from '../layering/check.ts'; -import type { EdgeKind, GraphData } from '../depgraph/model.ts'; -import { - mainSequenceModules, - zoneComponents, - type MainSequenceModule, - type ZoneComponent, -} from './components.ts'; - -export const SNAPSHOT_SCHEMA_VERSION = 1; - -/** Total across a per-pair ratchet map, e.g. TYPE_INVERSION_BASELINE. */ -export function ratchetTotal(byPair: Readonly>): number { - return Object.values(byPair).reduce((sum, count) => sum + count, 0); -} - -export type DepgraphMetrics = { - files: number; - edges: number; - valueCycles: number; - typeCycles: number; - dynamicCycles: number; - /** Value edges whose target is ALSO reachable at distance >= 2. Reachability, not removability. */ - redundantEdges: number; - /** R5 ranked-spine back-edges over value edges (the gate keeps this at zero). */ - backEdges: number; - /** R6 type-only spine-inversion edges, summed across zone pairs. */ - typeInversionEdges: number; -}; - -export function depgraphMetrics(graph: GraphData): DepgraphMetrics { - const cyclesByKind = (kind: EdgeKind): number => - graph.cycles.filter((cycle) => cycle.kind === kind).length; - return { - files: graph.nodes.length, - edges: graph.edges.length, - valueCycles: cyclesByKind('value'), - typeCycles: cyclesByKind('type'), - dynamicCycles: cyclesByKind('dynamic'), - redundantEdges: graph.edges.filter((edge) => edge.transitivelyReachable).length, - backEdges: graph.edges.filter((edge) => edge.backEdge !== null).length, - typeInversionEdges: ratchetTotal(graph.typeInversions), - }; -} - -export type LayeringRatchets = { - /** R6 baseline, per zone pair, and its total. The counts may only shrink (gate authority). */ - typeInversionBaseline: Readonly>; - typeInversionTotal: number; - /** R9 largest-type-cycle ceiling (growth-only ratchet). */ - typeCycleBaseline: number; -}; - -export function layeringRatchets(): LayeringRatchets { - return { - typeInversionBaseline: TYPE_INVERSION_BASELINE, - typeInversionTotal: ratchetTotal(TYPE_INVERSION_BASELINE), - typeCycleBaseline: TYPE_CYCLE_BASELINE, - }; -} - -/** - * The depgraph-vs-layering R6 cross-check (#1410): the graph the report builds over the real tree - * must reproduce the gate's TYPE_INVERSION_BASELINE, pair for pair. `pnpm repo-health` fails when - * this is false, and the identical assertion is wired into the Layering Guard CI job via - * scripts/depgraph/model.test.ts. The gate stays the authority; a mismatch means the tree or the - * baseline changed, never this check. - */ -export type R6Consistency = { - ok: boolean; - expected: Readonly>; - actual: Readonly>; -}; - -function sortedEntries(byPair: Readonly>): [string, number][] { - return Object.entries(byPair).sort(([left], [right]) => left.localeCompare(right)); -} - -export function r6Consistency( - graphTypeInversions: Readonly>, -): R6Consistency { - const expected = Object.fromEntries(sortedEntries(TYPE_INVERSION_BASELINE)); - const actual = Object.fromEntries(sortedEntries(graphTypeInversions)); - return { - ok: JSON.stringify(actual) === JSON.stringify(expected), - expected, - actual, - }; -} - -// ---- Fallow ----------------------------------------------------------------------------------- - -/** The fallow dead-code baseline: arrays of findings keyed by category. */ -export type FallowDeadCodeBaseline = Record; -/** The fallow health baseline: per-file maps of finding kind -> { count }. */ -export type FallowHealthBaseline = { - finding_counts?: Record>; -}; -export type FallowConfig = { - ignoreExports?: { exports: string[] }[]; - ignoreDependencies?: string[]; -}; - -export type FallowMetrics = { - /** Total dead-code findings recorded in the dead-code baseline. */ - deadCodeFindings: number; - /** Total complexity/health findings recorded in the health baseline. */ - healthFindings: number; - suppressions: { - /** Distinct export entries suppressed via `ignoreExports` in .fallowrc.json. */ - ignoredExports: number; - ignoredDependencies: number; - total: number; - }; -}; - -export function fallowMetrics( - deadCode: FallowDeadCodeBaseline, - health: FallowHealthBaseline, - config: FallowConfig, -): FallowMetrics { - const deadCodeFindings = Object.values(deadCode).reduce( - (sum, list) => sum + (Array.isArray(list) ? list.length : 0), - 0, - ); - let healthFindings = 0; - for (const perKind of Object.values(health.finding_counts ?? {})) { - for (const finding of Object.values(perKind)) healthFindings += finding.count; - } - const ignoredExports = (config.ignoreExports ?? []).reduce( - (sum, entry) => sum + (entry.exports?.length ?? 0), - 0, - ); - const ignoredDependencies = (config.ignoreDependencies ?? []).length; - return { - deadCodeFindings, - healthFindings, - suppressions: { - ignoredExports, - ignoredDependencies, - total: ignoredExports + ignoredDependencies, - }, - }; -} - -// ---- Coverage & size (read from the artifacts their own gates already produce) ---------------- - -type CoverageTotalEntry = { total: number; covered: number; pct: number }; -export type CoverageSummary = { - total?: Record<'lines' | 'statements' | 'functions' | 'branches', CoverageTotalEntry>; -}; -export type CoverageMetrics = { - available: boolean; - lines?: CoverageTotalEntry; - statements?: CoverageTotalEntry; - functions?: CoverageTotalEntry; - branches?: CoverageTotalEntry; -}; - -export function coverageMetrics(summary: CoverageSummary | null): CoverageMetrics { - if (!summary?.total) return { available: false }; - const { lines, statements, functions, branches } = summary.total; - return { available: true, lines, statements, functions, branches }; -} - -export type SizeReport = { - js?: { rawBytes: number; gzipBytes: number }; - npmPack?: { tarballBytes: number; unpackedBytes: number }; -}; -export type SizeMetrics = { - available: boolean; - jsRawBytes?: number; - jsGzipBytes?: number; - npmTarballBytes?: number; - npmUnpackedBytes?: number; -}; - -export function sizeMetrics(report: SizeReport | null): SizeMetrics { - if (!report?.js || !report.npmPack) return { available: false }; - return { - available: true, - jsRawBytes: report.js.rawBytes, - jsGzipBytes: report.js.gzipBytes, - npmTarballBytes: report.npmPack.tarballBytes, - npmUnpackedBytes: report.npmPack.unpackedBytes, - }; -} - -// ---- Slow-test / bench / skillgym (registry-derived counts) ----------------------------------- - -export type SlowTestRatchet = { - unitBudgetMs: number; - integrationBudgetMs: number; - enforceFactor: number; -}; - -export type BenchMetrics = { - /** Help-conformance bench cases (scripts/help-conformance-cases.mjs). */ - cases: number; - /** Distinct help topics the cases exercise. */ - topics: number; -}; - -export function benchMetrics(cases: readonly { docs: readonly string[] }[]): BenchMetrics { - return { - cases: cases.length, - topics: new Set(cases.flatMap((testCase) => testCase.docs)).size, - }; -} - -// ---- Provenance ------------------------------------------------------------------------------- - -/** - * Freshness of a READ artifact relative to the snapshot's `commit`. `fresh` requires proof; anything - * unproven is `unknown`, never silently trusted. - */ -export type ArtifactStatus = 'fresh' | 'stale' | 'unknown'; - -/** - * Provenance for an analyzer artifact this snapshot READ but did not itself produce (coverage, - * size). Its freshness relative to `commit` is proven ONLY from a producing commit the artifact - * stamps in its own bytes. We deliberately reject the two signals that cannot prove it: mtime (a - * copy, restore, or CI cache download resets it) and an enumerated producer-input list (never - * provably complete — the last review showed it silently omitted inputs). So a stamped commit equal - * to `commit` is `fresh`; a different stamped commit is `stale`; NO stamp is `unknown`. Today - * neither producer stamps a commit, so both report `unknown` — honest, since we cannot prove they - * match; if a producer starts emitting one, it is verified automatically. `sha256` still records the - * bytes so #1424 keys history on the exact metrics read. A consumer must treat `stale` and `unknown` - * alike as not-current, never as `commit`. - */ -export type ArtifactProvenance = { - path: string; - sha256: string; - /** The producing commit read from the artifact's own bytes, or null when it stamps none. */ - producerCommit: string | null; - status: ArtifactStatus; -}; - -/** The producing commit an artifact stamps in its own JSON, if any (`producerCommit` or `commit`). */ -function readStampedCommit(raw: string): string | null { - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch { - return null; - } - if (parsed === null || typeof parsed !== 'object') return null; - const record = parsed as Record; - for (const key of ['producerCommit', 'commit']) { - const value = record[key]; - if (typeof value === 'string' && value.length > 0) return value; - } - return null; -} - -/** - * The whole read-artifact collector: hash the bytes, read any producing commit the artifact stamps, - * and derive freshness against `currentCommit`. Returns null when the artifact is absent. This is - * the function the runner wires to disk, so a test that drives it with real artifact bytes exercises - * the actual freshness path end-to-end (not a fabricated status). - */ -export function collectArtifactProvenance( - path: string, - raw: string | null, - currentCommit: string, -): ArtifactProvenance | null { - if (raw === null) return null; - const producerCommit = readStampedCommit(raw); - const status: ArtifactStatus = - producerCommit === null ? 'unknown' : producerCommit === currentCommit ? 'fresh' : 'stale'; - return { - path, - sha256: createHash('sha256').update(raw).digest('hex').slice(0, 12), - producerCommit, - status, - }; -} - -/** - * What produced the snapshot and from which inputs. Mandatory in v1 (issue #1423 amendment) so - * #1424 can persist history and refuse to diff across schema versions. `tool` holds a content - * hash per analyzer (its own source/config, standing in for a version); `inputs` records the - * lockfile and the read-only artifacts each family of metrics was computed from, each hashed and - * given a proven freshness `status` so a false commit-indexed history entry cannot slip through. - */ -export type Provenance = { - schemaVersion: number; - commit: string; - ref: string | null; - node: string; - generatedAt: string; - tool: Record; - inputs: { - /** Production source files fed to the graph build. */ - sourceFiles: number; - lockfile: { path: string; sha256: string } | null; - /** Read artifacts, hashed with a proven freshness status; null when absent (degrade cleanly). */ - coverageSummary: ArtifactProvenance | null; - sizeReport: ArtifactProvenance | null; - }; -}; - -// ---- Full snapshot ---------------------------------------------------------------------------- - -export type RepoHealthSnapshot = { - schemaVersion: number; - provenance: Provenance; - /** - * Component metrics are observatory data — they locate concrete high-fan-in modules and must - * NEVER become CI thresholds. Only `consistency.r6` gates anything. - */ - metrics: { - depgraph: DepgraphMetrics; - layering: LayeringRatchets; - coverage: CoverageMetrics; - size: SizeMetrics; - fallow: FallowMetrics; - slowTest: SlowTestRatchet; - bench: BenchMetrics; - skillgym: { cases: number }; - components: { byZone: ZoneComponent[] }; - mainSequence: { concreteHighFanIn: MainSequenceModule[] }; - }; - consistency: { r6: R6Consistency }; -}; - -export type SnapshotInputs = { - provenance: Provenance; - graph: GraphData; - fallow: { deadCode: FallowDeadCodeBaseline; health: FallowHealthBaseline; config: FallowConfig }; - coverage: CoverageSummary | null; - size: SizeReport | null; - slowTest: SlowTestRatchet; - benchCases: readonly { docs: readonly string[] }[]; - skillgymCaseCount: number; -}; - -export function buildSnapshot(inputs: SnapshotInputs): RepoHealthSnapshot { - return { - schemaVersion: SNAPSHOT_SCHEMA_VERSION, - provenance: inputs.provenance, - metrics: { - depgraph: depgraphMetrics(inputs.graph), - layering: layeringRatchets(), - coverage: coverageMetrics(inputs.coverage), - size: sizeMetrics(inputs.size), - fallow: fallowMetrics(inputs.fallow.deadCode, inputs.fallow.health, inputs.fallow.config), - slowTest: inputs.slowTest, - bench: benchMetrics(inputs.benchCases), - skillgym: { cases: inputs.skillgymCaseCount }, - components: { byZone: zoneComponents(inputs.graph) }, - mainSequence: { concreteHighFanIn: mainSequenceModules(inputs.graph) }, - }, - consistency: { r6: r6Consistency(inputs.graph.typeInversions) }, - }; -} diff --git a/scripts/repo-health/run.test.ts b/scripts/repo-health/run.test.ts deleted file mode 100644 index bcaa4c3c60..0000000000 --- a/scripts/repo-health/run.test.ts +++ /dev/null @@ -1,70 +0,0 @@ -// End-to-end coverage of the `pnpm repo-health` CLI. It spawns the command as a subprocess (so it -// exercises argument handling and the file it writes over the real tree), which is why this lives -// in the `subprocess-stub` vitest project rather than `unit-core` — see #1412 coordination rule 4. - -import { spawnSync } from 'node:child_process'; -import { mkdtempSync, readFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { expect, test } from 'vitest'; -import type { RepoHealthSnapshot } from './model.ts'; - -function runRepoHealth(args: readonly string[]): { - status: number | null; - stdout: string; - stderr: string; -} { - const result = spawnSync( - process.execPath, - ['--experimental-strip-types', 'scripts/repo-health/run.ts', ...args], - { encoding: 'utf8', maxBuffer: 1024 * 1024 * 32 }, - ); - return { status: result.status, stdout: result.stdout ?? '', stderr: result.stderr ?? '' }; -} - -test('the default run prints a summary and exits zero over the real tree', () => { - const { status, stdout } = runRepoHealth([]); - expect(status, stdout).toBe(0); - expect(stdout).toContain('Repo health snapshot'); - expect(stdout).toContain('graph reproduces baseline: yes'); - expect(stdout).toContain('observatory only'); -}); - -test('--json emits a schema-versioned snapshot with every metric family', () => { - const { status, stdout } = runRepoHealth(['--json']); - expect(status, stdout).toBe(0); - - const snapshot = JSON.parse(stdout) as RepoHealthSnapshot; - expect(snapshot.schemaVersion).toBe(1); - - // Provenance is mandatory in v1 (issue amendment): schemaVersion, commit, per-analyzer tool - // hashes, and input provenance must all be present so #1424 can persist history. - expect(snapshot.provenance.commit).toMatch(/^[0-9a-f]{7,40}$/); - expect(Object.keys(snapshot.provenance.tool)).toContain('depgraph'); - expect(snapshot.provenance.inputs.sourceFiles).toBeGreaterThan(0); - - // Field names are the #1424 delta contract — pin the ones the comment consumes. - expect(snapshot.metrics.depgraph.redundantEdges).toBeGreaterThan(0); - expect(snapshot.metrics.layering.typeInversionTotal).toBeGreaterThanOrEqual(0); - expect(snapshot.metrics.slowTest.unitBudgetMs).toBeGreaterThan(0); - expect(snapshot.metrics.fallow.suppressions.total).toBeGreaterThanOrEqual(0); - expect(snapshot.metrics.bench.cases).toBeGreaterThan(0); - expect(snapshot.metrics.skillgym.cases).toBeGreaterThan(0); - - // The consistency assertion must pass on a clean tree. - expect(snapshot.consistency.r6.ok).toBe(true); - - // The main-sequence spot-check the acceptance names. - const concrete = snapshot.metrics.mainSequence.concreteHighFanIn.map((module) => module.file); - expect(concrete).toContain('utils/exec.ts'); - expect(concrete).toContain('daemon/ref-frame.ts'); -}); - -test('--out writes the same JSON snapshot to a file', () => { - const out = join(mkdtempSync(join(tmpdir(), 'repo-health-')), 'snapshot.json'); - const { status } = runRepoHealth(['--out', out]); - expect(status).toBe(0); - const written = JSON.parse(readFileSync(out, 'utf8')) as RepoHealthSnapshot; - expect(written.schemaVersion).toBe(1); - expect(written.consistency.r6.ok).toBe(true); -}); diff --git a/scripts/repo-health/run.ts b/scripts/repo-health/run.ts deleted file mode 100644 index 0a2ff9528d..0000000000 --- a/scripts/repo-health/run.ts +++ /dev/null @@ -1,264 +0,0 @@ -// Repo-health snapshot command (issue #1423, Track C of #1412): -// -// pnpm repo-health # human-readable summary -// pnpm repo-health --json # the raw JSON snapshot on stdout -// pnpm repo-health --out p # also write the JSON snapshot to a file -// -// One JSON snapshot aggregating the signals this repo already computes, by REUSING each analyzer -// rather than reimplementing any metric (see model.ts for which analyzer owns each number). It is -// DATA for agents to query, not a dashboard: no rendering, no history (that is #1424), and the -// component metrics are observatory-only and must never gate. -// -// It runs from a clean checkout in well under two minutes with no network: everything is read -// from the working tree, git, and the artifacts other gates already write. Coverage and package -// size are read from coverage/coverage-summary.json and .tmp/size-report.json when present and -// degrade to `available: false` (with the producing command) when they are not, so a fresh -// checkout still produces a complete, honest snapshot. Those two artifacts carry no producing-commit -// stamp today, so provenance hashes them and reports freshness as `unknown` unless the artifact -// itself stamps a commit equal to HEAD (see collectArtifactProvenance in model.ts) — we never infer -// freshness from mtime or a producer-input list, neither of which can prove it. A consumer like -// #1424 must treat `unknown`/`stale` as not-current, never as HEAD. -// -// The one thing that can FAIL the command is the depgraph-vs-layering R6 consistency assertion: -// the graph built here must reproduce scripts/layering/check.ts's TYPE_INVERSION_BASELINE. The -// identical assertion is wired into CI by the Layering Guard job (scripts/depgraph/model.test.ts). - -import { createHash } from 'node:crypto'; -import fs from 'node:fs'; -import path from 'node:path'; -import { runCmdSync } from '../../src/utils/exec.ts'; -import { listSourceFiles } from '../layering/check.ts'; -import { resolveImportEdges } from '../layering/model.ts'; -import { buildGraph, type GraphData } from '../depgraph/model.ts'; -import { SLOW_TEST_RATCHET } from '../vitest-slow-test-budgets.ts'; -import { parseScriptArgs } from '../lib/cli-args.ts'; -import { runAsMain } from '../lib/run-as-main.ts'; -import { CASES } from '../help-conformance-cases.mjs'; -import skillgymSuite from '../../test/skillgym/suites/agent-device-smoke-suite.ts'; -import { - buildSnapshot, - collectArtifactProvenance, - SNAPSHOT_SCHEMA_VERSION, - type ArtifactProvenance, - type FallowConfig, - type FallowDeadCodeBaseline, - type FallowHealthBaseline, - type Provenance, - type RepoHealthSnapshot, -} from './model.ts'; - -const USAGE = 'Usage: pnpm repo-health [--json] [--out ]\n'; - -const COVERAGE_SUMMARY_PATH = 'coverage/coverage-summary.json'; -const SIZE_REPORT_PATH = '.tmp/size-report.json'; -const LOCKFILE_PATH = 'pnpm-lock.yaml'; - -const repoRoot = runCmdSync('git', ['rev-parse', '--show-toplevel']).stdout.trim(); - -function absolute(relativePath: string): string { - return path.join(repoRoot, relativePath); -} - -function readIfExists(relativePath: string): string | null { - const abs = absolute(relativePath); - return fs.existsSync(abs) ? fs.readFileSync(abs, 'utf8') : null; -} - -function readJsonIfExists(relativePath: string): T | null { - const raw = readIfExists(relativePath); - return raw === null ? null : (JSON.parse(raw) as T); -} - -/** A read artifact's raw bytes (for content-hash provenance) and its parsed form (for metrics). */ -function readArtifact(relativePath: string): { raw: string | null; parsed: T | null } { - const raw = readIfExists(relativePath); - return { raw, parsed: raw === null ? null : (JSON.parse(raw) as T) }; -} - -function shortSha(raw: string): string { - return createHash('sha256').update(raw).digest('hex').slice(0, 12); -} - -/** Short content hash standing in for an analyzer's version — a source change is a version bump. */ -function contentHash(relativePath: string): string { - const raw = readIfExists(relativePath); - return raw === null ? 'absent' : shortSha(raw); -} - -function lockfileProvenance(): { path: string; sha256: string } | null { - const raw = readIfExists(LOCKFILE_PATH); - return raw === null ? null : { path: LOCKFILE_PATH, sha256: shortSha(raw) }; -} - -function gitOutput(args: string[]): string | null { - const result = runCmdSync('git', args, { cwd: repoRoot, allowFailure: true }); - return result.exitCode === 0 ? result.stdout.trim() : null; -} - -function buildGraphFromTree(files: readonly string[]): GraphData { - const sources = new Map(files.map((file) => [file, fs.readFileSync(absolute(file), 'utf8')])); - return buildGraph(sources, resolveImportEdges(sources)); -} - -function currentCommit(): string { - return gitOutput(['rev-parse', 'HEAD']) ?? 'unknown'; -} - -function currentRef(): string | null { - return process.env.GITHUB_REF_NAME ?? gitOutput(['rev-parse', '--abbrev-ref', 'HEAD']); -} - -// A content hash per analyzer, standing in for a version: a change to the analyzer's own source -// is a version bump the snapshot records without a real semver to point at. -function toolHashes(): Record { - return { - depgraph: contentHash('scripts/depgraph/model.ts'), - layering: contentHash('scripts/layering/check.ts'), - fallow: contentHash('.fallowrc.json'), - coverage: contentHash('vitest.config.ts'), - size: contentHash('scripts/size-report.mjs'), - slowTest: contentHash('scripts/vitest-slow-test-budgets.ts'), - bench: contentHash('scripts/help-conformance-cases.mjs'), - skillgym: contentHash('test/skillgym/suites/agent-device-smoke-suite.ts'), - }; -} - -function collectProvenance( - graph: GraphData, - commit: string, - coverage: ArtifactProvenance | null, - size: ArtifactProvenance | null, -): Provenance { - return { - schemaVersion: SNAPSHOT_SCHEMA_VERSION, - commit, - ref: currentRef(), - node: process.version, - generatedAt: new Date().toISOString(), - tool: toolHashes(), - inputs: { - sourceFiles: graph.nodes.length, - lockfile: lockfileProvenance(), - coverageSummary: coverage, - sizeReport: size, - }, - }; -} - -function collectSnapshot(): RepoHealthSnapshot { - const graph = buildGraphFromTree(listSourceFiles()); - const commit = currentCommit(); - const coverage = readArtifact(COVERAGE_SUMMARY_PATH); - const size = readArtifact(SIZE_REPORT_PATH); - return buildSnapshot({ - provenance: collectProvenance( - graph, - commit, - collectArtifactProvenance(COVERAGE_SUMMARY_PATH, coverage.raw, commit), - collectArtifactProvenance(SIZE_REPORT_PATH, size.raw, commit), - ), - graph, - fallow: { - deadCode: readJsonIfExists('fallow-baselines/dead-code.json') ?? {}, - health: readJsonIfExists('fallow-baselines/health.json') ?? {}, - config: readJsonIfExists('.fallowrc.json') ?? {}, - }, - coverage: coverage.parsed, - size: size.parsed, - slowTest: SLOW_TEST_RATCHET, - benchCases: CASES, - skillgymCaseCount: skillgymSuite.length, - }); -} - -function pct(entry: { covered: number; total: number; pct: number } | undefined): string { - return entry - ? `${entry.pct.toFixed(2)}% (${entry.covered}/${entry.total})` - : 'n/a (run pnpm test:coverage)'; -} - -function bytes(value: number | undefined): string { - if (value === undefined) return 'n/a (run pnpm size:markdown)'; - if (value < 1000) return `${value} B`; - if (value < 1_000_000) return `${(value / 1000).toFixed(1)} kB`; - return `${(value / 1_000_000).toFixed(2)} MB`; -} - -function staleMark(artifact: ArtifactProvenance | null): string { - if (artifact === null || artifact.status === 'fresh') return ''; - return artifact.status === 'stale' - ? ' [STALE — artifact stamps a different commit]' - : ' [UNKNOWN — artifact stamps no commit; may not reflect HEAD]'; -} - -function renderSummary(snapshot: RepoHealthSnapshot): string { - const { metrics, provenance, consistency } = snapshot; - const lines: string[] = [ - `Repo health snapshot (schemaVersion ${snapshot.schemaVersion}) @ ${provenance.commit.slice(0, 12)}`, - ` depgraph: ${metrics.depgraph.files} files, ${metrics.depgraph.edges} edges, ` + - `${metrics.depgraph.redundantEdges} redundant; cycles value=${metrics.depgraph.valueCycles} ` + - `type=${metrics.depgraph.typeCycles} dynamic=${metrics.depgraph.dynamicCycles}`, - ` layering: R6 type-inversions=${metrics.layering.typeInversionTotal} ` + - `(graph reproduces baseline: ${consistency.r6.ok ? 'yes' : 'NO'}); R9 type-cycle baseline=${metrics.layering.typeCycleBaseline}`, - ` coverage: lines ${pct(metrics.coverage.lines)}; statements ${pct(metrics.coverage.statements)}` + - staleMark(provenance.inputs.coverageSummary), - ` size: js raw ${bytes(metrics.size.jsRawBytes)}, gzip ${bytes(metrics.size.jsGzipBytes)}; ` + - `npm tarball ${bytes(metrics.size.npmTarballBytes)}` + - staleMark(provenance.inputs.sizeReport), - ` fallow: dead-code findings=${metrics.fallow.deadCodeFindings}, health findings=${metrics.fallow.healthFindings}, ` + - `suppressions=${metrics.fallow.suppressions.total}`, - ` slow-test: unit ${metrics.slowTest.unitBudgetMs}ms / integration ${metrics.slowTest.integrationBudgetMs}ms (enforce ${metrics.slowTest.enforceFactor}x)`, - ` bench: ${metrics.bench.cases} cases across ${metrics.bench.topics} help topics; skillgym ${metrics.skillgym.cases} cases`, - '', - ' Component metrics (observatory only — never a CI threshold):', - ...metrics.components.byZone - .slice(0, 8) - .map( - (zone) => - ` ${zone.zone.padEnd(16)} Ca=${String(zone.afferent).padStart(4)} Ce=${String(zone.efferent).padStart(4)} ` + - `I=${zone.instability.toFixed(2)} A=${zone.abstractness.toFixed(2)} D=${zone.distance.toFixed(2)}`, - ), - ' Concrete high-fan-in modules worth pinning harder:', - ...metrics.mainSequence.concreteHighFanIn - .slice(0, 8) - .map( - (module) => - ` ${module.file.padEnd(40)} fanIn=${String(module.fanIn).padStart(3)} A=${module.abstractness.toFixed(2)}`, - ), - ]; - return `${lines.join('\n')}\n`; -} - -function run(argv: readonly string[]): number { - const values = parseScriptArgs(argv, USAGE, { - json: { type: 'boolean', default: false }, - out: { type: 'string' }, - }); - - const snapshot = collectSnapshot(); - const serialized = `${JSON.stringify(snapshot, null, 2)}\n`; - - if (typeof values.out === 'string') { - const outPath = path.resolve(values.out); - fs.mkdirSync(path.dirname(outPath), { recursive: true }); - fs.writeFileSync(outPath, serialized); - } - - process.stdout.write(values.json ? serialized : renderSummary(snapshot)); - - const { r6 } = snapshot.consistency; - if (!r6.ok) { - process.stderr.write( - 'repo-health: the dependency graph disagrees with the layering gate about R6 type-only ' + - 'spine inversions.\n' + - ` gate baseline (TYPE_INVERSION_BASELINE): ${JSON.stringify(r6.expected)}\n` + - ` graph over the current tree: ${JSON.stringify(r6.actual)}\n` + - 'The gate is the authority — fix the inverting edge, or update TYPE_INVERSION_BASELINE in ' + - 'scripts/layering/check.ts. See scripts/depgraph/README.md.\n', - ); - return 1; - } - return 0; -} - -runAsMain(import.meta.url, 'repo-health', run); diff --git a/scripts/vitest-slow-test-budgets.ts b/scripts/vitest-slow-test-budgets.ts deleted file mode 100644 index e1d42efdb4..0000000000 --- a/scripts/vitest-slow-test-budgets.ts +++ /dev/null @@ -1,21 +0,0 @@ -// The slow-test ratchet's budgets, as a single source of truth (see docs/agents/testing.md -// "Speed rules"). Kept in a data-only module so consumers that only need the numbers — the -// repo-health snapshot (scripts/repo-health), which records them as observatory data — can import -// them without pulling the vitest reporter (a config-loaded module) into their dependency graph. - -// Unit tests must not wait real time; integration scenarios drive a real daemon request path and -// get more room. -export const UNIT_BUDGET_MS = 2_500; -export const INTEGRATION_BUDGET_MS = 15_000; - -// Enforcement fires at 2x budget: host load legitimately stretches a borderline test by tens of -// percent, and a wall-clock gate that flakes under contention trains people to ignore it. Between -// budget and 2x budget the gate reports without failing. -export const ENFORCE_FACTOR = 2; - -/** The ratchet as one record, for callers that report it rather than enforce it. */ -export const SLOW_TEST_RATCHET = { - unitBudgetMs: UNIT_BUDGET_MS, - integrationBudgetMs: INTEGRATION_BUDGET_MS, - enforceFactor: ENFORCE_FACTOR, -} as const; diff --git a/scripts/vitest-slow-test-reporter.ts b/scripts/vitest-slow-test-reporter.ts index 6963d812b1..53f657576c 100644 --- a/scripts/vitest-slow-test-reporter.ts +++ b/scripts/vitest-slow-test-reporter.ts @@ -1,9 +1,4 @@ import type { Reporter, TestCase, TestModule } from 'vitest/node'; -import { - ENFORCE_FACTOR, - INTEGRATION_BUDGET_MS, - UNIT_BUDGET_MS, -} from './vitest-slow-test-budgets.ts'; /** * Slow-test ratchet (see docs/agents/testing.md "Speed rules"). @@ -12,8 +7,18 @@ import { * wall clock was bounded by files whose tests slept through production * timeouts (a 10.8s test proving "times out" by waiting the full 10s budget). * This reporter fails the run when a unit test exceeds the enforced budget. - * The budgets themselves live in ./vitest-slow-test-budgets.ts. */ + +// Unit tests must not wait real time; integration scenarios drive a real +// daemon request path and get more room. +const UNIT_BUDGET_MS = 2_500; +const INTEGRATION_BUDGET_MS = 15_000; + +// Enforcement fires at 2x budget: host load legitimately stretches a +// borderline test by tens of percent, and a wall-clock gate that flakes under +// contention trains people to ignore it. Between budget and 2x budget the +// gate reports without failing. +const ENFORCE_FACTOR = 2; type Offender = { key: string; durationMs: number; budgetMs: number; enforce: boolean }; function budgetForPath(relativePath: string): number { diff --git a/test/output-economy/README.md b/test/output-economy/README.md index 293cf933b8..7400beee17 100644 --- a/test/output-economy/README.md +++ b/test/output-economy/README.md @@ -9,7 +9,7 @@ representative MCP tool result containing optimized text plus structured content ceiling it is measured against. - The actionability floors keep refs, generation pins, warnings, retry signals, and recovery guidance from being optimized away. -- SkillGym and the help-conformance benchmark remain the non-gating small-model outcome oracle: +- The help-conformance benchmark remains the non-gating small-model outcome oracle: byte reductions are not successful when the model needs an extra observation or chooses the wrong recovery command. - The routine-workflow oracle (`routine-workflow.ts`, `routine-workflow.test.ts`) is the gating diff --git a/test/skillgym/README.md b/test/skillgym/README.md deleted file mode 100644 index 6374b2afdc..0000000000 --- a/test/skillgym/README.md +++ /dev/null @@ -1,103 +0,0 @@ -# Skillgym For agent-device - -This folder benchmarks the one thing only an agentic runner can prove: given the `agent-device` skill, does a real agent route to it, consult local CLI help before answering, and read captured `agent-device` output correctly in that agentic setting? - -Everything else this suite used to cover moved to cheaper, stronger owners: - -- Command-planning knowledge checks (can a model choose the right next command from a help slice, a captured output, or an error?) live in the help conformance bench (`scripts/help-conformance-bench.mjs`). Its plans are validated by the production CLI parser, its case list is enumerated against the help-topic registry (`scripts/__tests__/help-conformance-topic-coverage.test.ts`), its recovery quizzes are enumerated against the error registry (`scripts/__tests__/help-conformance-error-recovery-coverage.test.ts`), and its quoted samples are pinned to the real renderers (`scripts/__tests__/help-conformance-sample-outputs.test.ts`). -- Live fixture-app behavior is owned by the deterministic iOS simulator e2e suite (`test/integration/ios-simulator-e2e/`) and its coverage manifest. - -## What stays here - -The suite uses two case tags: - -- `fixture-smoke`: one routing smoke (`open-and-snapshot`) — a sane fixture-app plan from the skill plus local help, without reading project source. -- `skill-guidance`: four output-interpretation cases that require a REAL observed local help probe before scoring (`requireLocalCliHelp` + `suites/local-cli-help-policy.ts` over observed command events): `settle-diff-is-observation`, `sample-output-settled-diff-next-target`, `sample-output-not-settled-needs-observe`, `sample-output-private-ax-recovery-continues`. Allowing help without verifying that the runner actually read it can misclassify a model-prior failure as a help-guidance failure. - -The captured output quoted in the skill-guidance cases is imported from `scripts/help-conformance-sample-outputs.mjs` — the same constants the bench embeds — so a renderer change fails the pinning test instead of silently grading agents against output the CLI no longer prints. - -## Included files - -- `../../examples/test-app/`: minimal Expo SDK 56 development-build fixture app -- `skillgym.config.ts`: starter config that runs Codex and Claude Haiku against this repo -- `suites/agent-device-smoke-suite.ts`: the routing + output-interpretation suite -- `suites/local-cli-help-policy.ts`: the observed-command policy that proves a real local help command ran - -For help-layout A/B testing and command-planning regressions, use the bench instead of adding cases here: it feeds only the top-level first screen or one help topic, runs runner x case pairs concurrently (`HELP_BENCH_CONCURRENCY`, default 4), grades draft help rewrites with zero rebuild via `--override-doc =`, measures variance with `--repeat ` plus an aggregate pass-rate and failure-taxonomy report, and filters with `--cases`/`--case` and `--runners`/`--runner` (both repeatable/CSV) the same way as this suite. - -`assertAgentDeviceEvidence` is intentionally soft when a runner does not expose skill-detection telemetry. When telemetry exists, the suite asserts that `agent-device` was loaded; when it is absent, the cases still judge command-planning output instead of failing on missing runner metadata. - -The `codex-mini` baseline is a benchmark signal, not a required all-green gate. Its failures should map to command-planning regressions called out by individual case IDs; do not treat the historical pass/fail count as a fixed threshold. - -SkillGym v0.8 command assertions are for observed command events. This suite primarily validates the command plan in the final answer, so it converts final-output command lines into a small planned-command report before calling `assert.commands.includes` or `assert.commands.notIncludes`. -The source-read guardrails use `assert.soft.*` plus deferred explain questions so one failing run can report multiple routing mistakes and can later be inspected with `skillgym explain`. -Suite types use the v0.8 root export name `Case`; older `TestCase` imports no longer typecheck. - -## Running the suite - -`skillgym` is installed as a repo dev dependency, so run the suite from the project root: - -```bash -cd /absolute/path/to/agent-device -pnpm install -pnpm test:skillgym -``` - -Prefer the package scripts so the environment guard and local CLI build run consistently: - -```bash -cd /absolute/path/to/agent-device -pnpm test:skillgym -pnpm test:skillgym:case open-and-snapshot -``` - -Useful v0.8 filters, reporters, and recovery options: - -```bash -pnpm test:skillgym -- --tag skill-guidance -pnpm test:skillgym -- --reporter json -pnpm test:skillgym -- --repeat 3 --repeat-failure 1 -``` - -Optional Vercel AI Gateway runner: - -```bash -AI_GATEWAY_API_KEY= \ -SKILLGYM_ENABLE_VERCEL_GATEWAY=1 \ -pnpm test:skillgym:case open-and-snapshot --runner gpt-nano-gateway -``` - -`gpt-nano-gateway` uses SkillGym's OpenCode adapter with a repo-injected `@ai-sdk/openai-compatible` provider pointed at `https://ai-gateway.vercel.sh/v1` and model `openai/gpt-5.4-nano`. It is disabled by default so normal runs do not require Gateway credentials, OpenCode auth, or Gateway spend. `VERCEL_OIDC_TOKEN` can be used instead of `AI_GATEWAY_API_KEY`; the config passes either token as the bearer credential for Gateway. - -If you need to run `skillgym` directly while developing the runner itself, build first so agents can call `node bin/agent-device.mjs help workflow`: - -```bash -pnpm build -pnpm exec skillgym run \ - ./test/skillgym/suites/agent-device-smoke-suite.ts \ - --config ./test/skillgym/skillgym.config.ts \ - --case open-and-snapshot -``` - -Use `--reporter github-actions` in CI when you want annotations in GitHub Actions logs. - -The config uses `schedule: parallel` so the planning suite can run case/runner pairs concurrently up to SkillGym v0.8's default available-machine parallelism cap. This is safe for the included suite because cases validate command plans and local CLI help, not live shared device state or workspace edits. Override with `--max-parallel ` for local experiments that need a different cap. -Use `--repeat ` when you want stability sampling rather than a single pass. Use `--repeat-failure ` for local benchmark recovery from transient runner failures; keep it off for strict regression checks unless you explicitly want retry artifacts. -When a run fails on an assertion that records explain questions, run `pnpm exec skillgym explain ` against the failed `repeat-*` artifact directory to resume the runner and collect its explanation. - -Prerequisites: - -- `codex` CLI installed and authenticated, because the starter config uses the Codex runner -- `claude` CLI installed and authenticated, because the same cases also run against Claude Haiku -- repo dependencies installed with `pnpm install` -- if you want the fixture app running locally, use `pnpm test-app:install` and then `pnpm test-app:ios` or `pnpm test-app:android` - -Sandbox note: - -The configured runners call external Codex and Claude model backends. In Codex sandboxes with `CODEX_SANDBOX_NETWORK_DISABLED=1`, `pnpm test:skillgym` and direct `skillgym run --config ./test/skillgym/skillgym.config.ts` fail fast before building or launching runners. Run the suite from a normal authenticated local shell instead. If you are in a sandbox that has explicitly approved network access and you still want to launch external runners, set `SKILLGYM_ALLOW_EXTERNAL_RUNNERS_IN_SANDBOX=1`. - -## Where to extend next - -- Add a case here ONLY when it measures agentic behavior: skill routing, local-help consultation, or multi-turn discovery a single non-agentic call cannot show. -- Add command-planning knowledge checks to `scripts/help-conformance-cases.mjs` instead; the topic-coverage gate tells you which help topics are unbenchmarked. -- Add local-only cases that drive a live simulator only when the deterministic e2e suite cannot own the behavior. diff --git a/test/skillgym/bin/agent-device b/test/skillgym/bin/agent-device deleted file mode 100755 index 07f567f4bc..0000000000 --- a/test/skillgym/bin/agent-device +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exec node "$(dirname "$0")/../../../bin/agent-device.mjs" "$@" diff --git a/test/skillgym/runner-environment.test.ts b/test/skillgym/runner-environment.test.ts deleted file mode 100644 index f3a26bd55b..0000000000 --- a/test/skillgym/runner-environment.test.ts +++ /dev/null @@ -1,47 +0,0 @@ -import assert from 'node:assert/strict'; -import test from 'node:test'; -import { - allowsExternalRunnersInSandbox, - isNetworkRestrictedCodexSandbox, - skillGymRunnerEnvironmentError, -} from './runner-environment.ts'; - -test('detects network-restricted Codex sandboxes', () => { - assert.equal(isNetworkRestrictedCodexSandbox({ CODEX_SANDBOX_NETWORK_DISABLED: '1' }), true); - assert.equal(isNetworkRestrictedCodexSandbox({ CODEX_SANDBOX_NETWORK_DISABLED: 'true' }), true); - assert.equal(isNetworkRestrictedCodexSandbox({ CODEX_SANDBOX_NETWORK_DISABLED: '0' }), false); - assert.equal(isNetworkRestrictedCodexSandbox({}), false); -}); - -test('allows explicit SkillGym sandbox override', () => { - assert.equal( - allowsExternalRunnersInSandbox({ SKILLGYM_ALLOW_EXTERNAL_RUNNERS_IN_SANDBOX: '1' }), - true, - ); - assert.equal( - allowsExternalRunnersInSandbox({ SKILLGYM_ALLOW_EXTERNAL_RUNNERS_IN_SANDBOX: 'true' }), - true, - ); - assert.equal(allowsExternalRunnersInSandbox({}), false); -}); - -test('explains why external SkillGym runners are blocked in restricted sandboxes', () => { - const message = skillGymRunnerEnvironmentError({ - CODEX_SANDBOX_NETWORK_DISABLED: '1', - }); - - assert.match(message ?? '', /external Codex and Claude runners/); - assert.match(message ?? '', /pnpm test:skillgym:case open-and-snapshot/); - assert.match(message ?? '', /SKILLGYM_ALLOW_EXTERNAL_RUNNERS_IN_SANDBOX=1/); -}); - -test('does not block normal local SkillGym runs', () => { - assert.equal(skillGymRunnerEnvironmentError({}), null); - assert.equal( - skillGymRunnerEnvironmentError({ - CODEX_SANDBOX_NETWORK_DISABLED: '1', - SKILLGYM_ALLOW_EXTERNAL_RUNNERS_IN_SANDBOX: '1', - }), - null, - ); -}); diff --git a/test/skillgym/runner-environment.ts b/test/skillgym/runner-environment.ts deleted file mode 100644 index 435f96860a..0000000000 --- a/test/skillgym/runner-environment.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { pathToFileURL } from 'node:url'; - -const SANDBOX_OVERRIDE_ENV = 'SKILLGYM_ALLOW_EXTERNAL_RUNNERS_IN_SANDBOX'; - -export function isNetworkRestrictedCodexSandbox(env: NodeJS.ProcessEnv = process.env): boolean { - return ( - env.CODEX_SANDBOX_NETWORK_DISABLED === '1' || env.CODEX_SANDBOX_NETWORK_DISABLED === 'true' - ); -} - -export function allowsExternalRunnersInSandbox(env: NodeJS.ProcessEnv = process.env): boolean { - return env[SANDBOX_OVERRIDE_ENV] === '1' || env[SANDBOX_OVERRIDE_ENV] === 'true'; -} - -export function skillGymRunnerEnvironmentError( - env: NodeJS.ProcessEnv = process.env, -): string | null { - if (!isNetworkRestrictedCodexSandbox(env) || allowsExternalRunnersInSandbox(env)) { - return null; - } - - return [ - 'SkillGym uses external Codex and Claude runners, but this Codex sandbox has network disabled.', - '', - 'Run this benchmark from a normal authenticated local shell:', - ' pnpm test:skillgym', - '', - 'For one case:', - ' pnpm test:skillgym:case open-and-snapshot', - '', - `If your sandbox has approved network access and you intentionally want to run external runners, set ${SANDBOX_OVERRIDE_ENV}=1.`, - ].join('\n'); -} - -export function enforceSkillGymRunnerEnvironment(): void { - const message = skillGymRunnerEnvironmentError(); - if (!message) return; - - console.error(message); - process.exit(2); -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - enforceSkillGymRunnerEnvironment(); -} diff --git a/test/skillgym/skillgym.config.ts b/test/skillgym/skillgym.config.ts deleted file mode 100644 index 754c46f300..0000000000 --- a/test/skillgym/skillgym.config.ts +++ /dev/null @@ -1,84 +0,0 @@ -import type { SkillgymConfig } from 'skillgym'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { enforceSkillGymRunnerEnvironment } from './runner-environment.ts'; - -enforceSkillGymRunnerEnvironment(); - -const localBinDir = fileURLToPath(new URL('./bin', import.meta.url)); -const runnerEnv = { - PATH: [localBinDir, process.env.PATH].filter(Boolean).join(path.delimiter), -}; - -const gatewayAuthToken = process.env.AI_GATEWAY_API_KEY ?? process.env.VERCEL_OIDC_TOKEN; -const gatewayOpenCodeConfig = { - provider: { - 'vercel-gateway': { - npm: '@ai-sdk/openai-compatible', - name: 'Vercel AI Gateway', - options: { - apiKey: gatewayAuthToken, - baseURL: 'https://ai-gateway.vercel.sh/v1', - }, - models: { - 'openai/gpt-5.4-nano': { - name: 'GPT 5.4 Nano', - }, - }, - }, - }, -}; - -const runners: SkillgymConfig['runners'] = { - 'codex-mini': { - agent: { - type: 'codex', - model: 'gpt-5.4-mini', - env: runnerEnv, - }, - }, - 'claude-haiku': { - agent: { - type: 'claude-code', - model: 'haiku', - env: runnerEnv, - }, - }, -}; - -if (process.env.SKILLGYM_ENABLE_VERCEL_GATEWAY === '1') { - if (gatewayAuthToken === undefined || gatewayAuthToken.length === 0) { - throw new Error( - 'SKILLGYM_ENABLE_VERCEL_GATEWAY=1 requires AI_GATEWAY_API_KEY or VERCEL_OIDC_TOKEN.', - ); - } - - runners['gpt-nano-gateway'] = { - agent: { - type: 'opencode', - command: 'opencode', - model: 'vercel-gateway/openai/gpt-5.4-nano', - env: { - ...runnerEnv, - AI_GATEWAY_API_KEY: gatewayAuthToken, - OPENCODE_CONFIG_CONTENT: JSON.stringify(gatewayOpenCodeConfig), - }, - }, - }; -} - -const config: SkillgymConfig = { - run: { - // Relative to this config file; points SkillGym at the repository root. - cwd: '../..', - outputDir: '.skillgym-results', - reporter: 'standard', - schedule: 'parallel', - }, - defaults: { - timeoutMs: 600_000, - }, - runners, -}; - -export default config; diff --git a/test/skillgym/suites/agent-device-smoke-suite.ts b/test/skillgym/suites/agent-device-smoke-suite.ts deleted file mode 100644 index 01493b578a..0000000000 --- a/test/skillgym/suites/agent-device-smoke-suite.ts +++ /dev/null @@ -1,465 +0,0 @@ -import { assert, commandMatcher, type Case, type CommandMatcher } from 'skillgym'; -import { - NOT_SETTLED_SAMPLE, - PRIVATE_AX_RECOVERY_SAMPLE, - SETTLE_DIFF_SAMPLE, - SETTLE_TAIL_SAMPLE, - sampleText, -} from '../../../scripts/help-conformance-sample-outputs.mjs'; -import { - extractCommandEvents, - isActualLocalCliHelpProbe, - isAllowedLocalCliHelpCommand, -} from './local-cli-help-policy.ts'; - -// This suite is deliberately small. It keeps only what an agentic runner can -// prove and a single non-agentic LLM call cannot: that a runner with the -// agent-device skill routes to it, consults LOCAL CLI help before answering -// (isActualLocalCliHelpProbe over observed command events), and reads -// captured agent-device output correctly in that agentic setting. Knowledge -// checks about command planning live in the cheaper, parser-validated help -// conformance bench (scripts/help-conformance-bench.mjs); live fixture -// behavior is owned by the iOS simulator e2e suite. The quoted samples are -// shared with the bench and pinned to the real renderers by -// scripts/__tests__/help-conformance-sample-outputs.test.ts. - -type SessionReport = Parameters[0]; -type AssertionContext = Parameters[1]; -type OutputMatcher = string | RegExp | PlannedCommandMatcher; - -interface PlannedCommandMatcher { - kind: 'planned-command'; - matchers: CommandMatcher[]; -} - -interface OutputAbsenceContext { - output: string; - finalOutput: string; - plannedReport: SessionReport; -} - -const WORKSPACE_ROOT = process.cwd().replaceAll('\\', '/'); -const APP_SOURCE = workspacePathPattern('examples/test-app', 'directory'); -const REPO_SOURCE = workspacePathPattern('src', 'directory'); -const COMMAND_DOCS = workspacePathPattern('website/docs/docs/commands.md', 'file'); -const SUITE_FILE = workspacePathPattern('test/skillgym/suites/agent-device-smoke-suite.ts', 'file'); - -const BASE_INSTRUCTIONS = ` -You are benchmarking agent-device command planning for a known fixture app. - -Do not read project source files or project docs. -Do not inspect examples/test-app, src/, README.md, or website/docs. -Do not browse the web. -Use only this prompt plus local CLI help as private reference. -Do not execute live app/device commands while planning; only local CLI help commands are allowed before final output. -For local CLI help in this repo, use node bin/agent-device.mjs help or --help; final commands still use agent-device. -If the app contract names an expected id, selector, or visible text, include that exact target in a final verification command instead of stopping at the action that reaches or reveals it. -`.trim(); - -const DEFAULT_FINAL_OUTPUT_INSTRUCTIONS = ` -Final output: only commands, one per line. Use agent-device for app/device automation; shell setup commands are allowed only when this prompt explicitly requires them. Any prose or Markdown fails. -Every final output line must start with agent-device. -Do not combine final commands with shell operators such as &&, ||, pipes, or semicolons. -`.trim(); - -function workspacePathPattern(relativePath: string, kind: 'directory' | 'file') { - const normalizedPath = relativePath.replaceAll('\\', '/').replace(/^\.\//, ''); - const escapedRoot = escapeRegExp(WORKSPACE_ROOT); - const escapedRelativePath = escapeRegExp(normalizedPath); - const boundary = kind === 'directory' ? '(?:/|$)' : '$'; - return new RegExp(`^(?:${escapedRelativePath}|${escapedRoot}/${escapedRelativePath})${boundary}`); -} - -function escapeRegExp(value: string) { - return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); -} - -function buildPrompt(options: { contract: string[]; task: string; requireLocalCliHelp?: boolean }) { - const contractLines = options.contract.map((line) => `- ${line}`).join('\n'); - const helpRequirement = options.requireLocalCliHelp - ? '\nBefore planning, run node bin/agent-device.mjs --help and use that output as the command contract.' - : ''; - return `${BASE_INSTRUCTIONS}${helpRequirement}\n${DEFAULT_FINAL_OUTPUT_INSTRUCTIONS}\n\nApp contract:\n${contractLines}\n\nTask:\n${options.task}`; -} - -function assertAgentDeviceEvidence(report: SessionReport) { - const detectedSkills = report.detectedSkills ?? []; - const hasDetectedSkills = detectedSkills.length > 0; - const hasBundledDeviceSkill = detectedSkills.some((skill) => - ['agent-device', 'dogfood'].includes(skill.skill), - ); - - // Some SkillGym runners do not expose skill telemetry. Keep this as a conditional routing - // assertion instead of failing otherwise valid command-planning runs on missing metadata. - if (hasDetectedSkills) { - assert.soft.ok( - hasBundledDeviceSkill, - `Expected detectedSkills to include an agent-device bundled skill. Observed detectedSkills: ${detectedSkills - .map((skill) => `${skill.skill} (${skill.confidence})`) - .join(', ')}`, - ); - } -} - -function assertNoProjectSourceReads(report: SessionReport) { - assert.soft.fileReads.notIncludes(report, APP_SOURCE, { - explain: { question: 'Why did you read the fixture app source instead of using CLI help?' }, - }); - assert.soft.fileReads.notIncludes(report, REPO_SOURCE, { - explain: { question: 'Why did you read repo source files instead of using CLI help?' }, - }); - assert.soft.fileReads.notIncludes(report, COMMAND_DOCS, { - explain: { question: 'Why did you read website command docs instead of local CLI help?' }, - }); -} - -function plannedCommand(command: string): PlannedCommandMatcher { - return plannedCommandAlternatives([command]); -} - -function plannedCommandAlternatives(commands: string[]): PlannedCommandMatcher { - return { - kind: 'planned-command', - matchers: commands.flatMap((command) => { - const [executable, ...args] = commandParts(command); - assert.ok(executable, 'planned command must not be empty'); - - return [ - commandMatcher('agent-device').args(executable, ...args), - commandMatcher(executable).args(...args), - ]; - }), - }; -} - -function assertOutputs(finalOutput: string, matchers: OutputMatcher[]) { - const output = normalizedFinalOutput(finalOutput); - const plannedReport = plannedCommandReport(output); - for (const matcher of matchers) { - if (isPlannedCommandMatcher(matcher)) { - assertPlannedCommandIncludes(plannedReport, matcher); - continue; - } - - assert.output.includes(normalizedOutputReport(output), matcher); - } -} - -function assertNoOutputs(finalOutput: string, matchers: OutputMatcher[]) { - const output = normalizedFinalOutput(finalOutput); - const plannedReport = plannedCommandReport(output); - const context = { output, finalOutput, plannedReport }; - - for (const matcher of matchers) { - assertOutputAbsent(context, matcher); - } -} - -function assertOutputAbsent(context: OutputAbsenceContext, matcher: OutputMatcher) { - if (isPlannedCommandMatcher(matcher)) { - assertPlannedCommandNotIncludes(context.plannedReport, matcher); - return; - } - - if (typeof matcher === 'string') { - assertStringOutputAbsent(context, matcher); - return; - } - - assert.doesNotMatch(context.output, matcher); -} - -function assertStringOutputAbsent(context: OutputAbsenceContext, matcher: string) { - assert.ok( - !context.output.includes(matcher), - `Expected final output not to include ${JSON.stringify(matcher)}. Observed final output: ${context.finalOutput}`, - ); -} - -function isPlannedCommandMatcher(matcher: OutputMatcher): matcher is PlannedCommandMatcher { - return ( - typeof matcher === 'object' && - !(matcher instanceof RegExp) && - matcher.kind === 'planned-command' - ); -} - -function assertPlannedCommandIncludes(report: SessionReport, matcher: PlannedCommandMatcher) { - if (matcher.matchers.length === 1) { - assert.commands.includes(report, matcher.matchers[0]!); - return; - } - - const failures: Error[] = []; - for (const command of matcher.matchers) { - try { - assert.commands.includes(report, command); - return; - } catch (error) { - failures.push(error as Error); - } - } - - assert.fail(failures.map((error) => error.message).join('\n')); -} - -function assertPlannedCommandNotIncludes(report: SessionReport, matcher: PlannedCommandMatcher) { - for (const command of matcher.matchers) { - assert.commands.notIncludes(report, command); - } -} - -function normalizedFinalOutput(output: string): string { - return output - .replace(/```[a-z]*\n?/gi, '') - .replace(/```/g, '') - .replace(/`([^`\n]+)`/g, '$1') - .trim(); -} - -function plannedCommandReport(output: string): SessionReport { - return { - events: output - .split('\n') - .map((line) => line.trim()) - .filter(Boolean) - .map((command) => ({ type: 'command' as const, command })), - } as SessionReport; -} - -function normalizedOutputReport(output: string): SessionReport { - return { finalOutput: output } as SessionReport; -} - -function commandParts(command: string): string[] { - return command.split(' ').filter(Boolean); -} - -function assertExpectedOutput( - report: SessionReport, - ctx: AssertionContext, - matchers: OutputMatcher[] = [], -) { - if (matchers.length === 0) { - assert.output.notEmpty(report); - return; - } - - assertOutputs(ctx.finalOutput(), matchers); -} - -function assertOnlyLocalCliHelpCommands(report: SessionReport) { - const commandEvents = extractCommandEvents(report); - const forbiddenCommands = commandEvents.filter( - (command) => !isAllowedLocalCliHelpCommand(command), - ); - - assert.deepEqual( - forbiddenCommands, - [], - `Expected planning runs to execute only local CLI help commands. Observed runtime commands: ${forbiddenCommands - .map((command) => JSON.stringify(command)) - .join(', ')}`, - ); -} - -function assertLocalCliHelpUsed(report: SessionReport) { - const commandEvents = extractCommandEvents(report); - assert.ok( - commandEvents.some(isActualLocalCliHelpProbe), - 'Expected the planning run to inspect local CLI help before answering.', - ); -} - -const RAW_COORDINATE_TARGET = - /(?:^|\n)(?:agent-device\s+)?(?:click|fill|press)\s+-?\d+(?:\.\d+)?\s+-?\d+(?:\.\d+)?/i; -const SHELL_OUTPUT_PROJECTION = /(?:2>\s*\/dev\/null|\|\s*(?:jq|grep|head|tail)\b)/i; -const IOS_TEST_APP_DEV_BUILD_OPEN = new RegExp( - String.raw`(?:^|\n)(?:agent-device\s+)?open\s+` + - String.raw`(?:(?:"Agent Device Tester")|(?:'Agent Device Tester')|com\.callstack\.agentdevicelab)\b`, - 'i', -); - -function makeCase(options: { - id: string; - contract: string[]; - task: string; - tags?: string[]; - outputs?: OutputMatcher[]; - forbiddenOutputs?: OutputMatcher[]; - allowOnlyLocalCliHelpCommands?: boolean; - requireLocalCliHelp?: boolean; -}): Case { - return { - id: options.id, - tags: options.tags, - prompt: buildPrompt({ - contract: options.contract, - task: options.task, - requireLocalCliHelp: options.requireLocalCliHelp, - }), - assert(report, ctx) { - assertAgentDeviceEvidence(report); - assertNoProjectSourceReads(report); - assert.soft.fileReads.notIncludes(report, SUITE_FILE, { - explain: { question: 'Why did you inspect the benchmark suite while answering?' }, - }); - assertExpectedOutput(report, ctx, options.outputs); - assertNoOutputs(ctx.finalOutput(), options.forbiddenOutputs ?? []); - if (options.allowOnlyLocalCliHelpCommands) assertOnlyLocalCliHelpCommands(report); - if (options.requireLocalCliHelp) assertLocalCliHelpUsed(report); - }, - }; -} - -function withTags(tags: string[], cases: Case[]): Case[] { - return cases.map((testCase) => ({ - ...testCase, - tags: [...new Set([...(testCase.tags ?? []), ...tags])], - })); -} - -function quiz(sample: { command: string; output: string }, question: string): string { - return `Read this previous agent-device output, then plan the next command: - -${sampleText(sample)} - -${question}`; -} - -// Routing smoke: proves the runner reaches a sane plan for the fixture app -// from the skill plus local help, without reading project source. -const FIXTURE_SMOKE_CASES: Case[] = [ - makeCase({ - id: 'open-and-snapshot', - contract: [ - 'App name: Agent Device Tester', - 'Platform: iOS', - 'Launch context: installed Expo development build', - 'Bundle identifier: com.callstack.agentdevicelab', - ], - task: 'Plan the commands to open Agent Device Tester as an installed Expo development build on iOS, take a snapshot -i to verify the app UI loaded, then close.', - outputs: [IOS_TEST_APP_DEV_BUILD_OPEN, /snapshot -i/i, plannedCommand('close')], - forbiddenOutputs: [ - /open\s+["']Expo Go["']/i, - /host\.exp\.Exponent/i, - /exp:\/\/127\.0\.0\.1:8081/i, - ], - // The suite's claim is "a sane plan from the skill plus local help" — so - // the run must actually consult local help, and must not execute anything - // else while planning. Without these, the case can pass on model prior - // alone and the claim is unproven. - allowOnlyLocalCliHelpCommands: true, - requireLocalCliHelp: true, - }), -]; - -// Output-interpretation cases: each run must consult real local CLI help -// (requireLocalCliHelp + the local-cli-help-policy events check) and then read -// captured agent-device output correctly. The bench carries the same cases as -// single-call knowledge checks; here they measure the agentic path. -const SKILL_GUIDANCE_CASES: Case[] = [ - makeCase({ - id: 'settle-diff-is-observation', - contract: [ - 'App name: Agent Device Tester', - 'Previous command output is from agent-device, not a task description', - 'The task was to confirm the feed-search UI and close the session', - ], - task: quiz( - SETTLE_TAIL_SAMPLE, - 'Use the output already shown to determine whether the feed-search UI is present, then close the session. What command should run next?', - ), - outputs: [plannedCommand('close')], - forbiddenOutputs: [ - plannedCommand('snapshot'), - plannedCommand('wait'), - plannedCommand('find'), - plannedCommand('get'), - plannedCommand('is'), - plannedCommandAlternatives(['press', 'click']), - ], - allowOnlyLocalCliHelpCommands: true, - requireLocalCliHelp: true, - }), - makeCase({ - id: 'sample-output-settled-diff-next-target', - contract: [ - 'App name: Agent Device Tester', - 'Previous command output is from agent-device, not a task description', - 'Need to continue from the settled diff without taking another snapshot', - ], - task: quiz( - SETTLE_DIFF_SAMPLE, - 'The task is to open the matching account result. What command should run next?', - ), - outputs: [ - plannedCommandAlternatives(['press', 'click', 'tap']), - /@e64(?:~s\d+)?\b|(?:label|text)=(?:["']@callstack\.com["']|@callstack\.com)/i, - /--settle\b/i, - ], - forbiddenOutputs: [ - plannedCommand('snapshot'), - plannedCommand('wait stable'), - plannedCommand('fill'), - RAW_COORDINATE_TARGET, - ], - allowOnlyLocalCliHelpCommands: true, - requireLocalCliHelp: true, - }), - makeCase({ - id: 'sample-output-not-settled-needs-observe', - contract: [ - 'App name: Agent Device Tester', - 'Previous command output is from agent-device, not a task description', - 'The next target is unknown because no settled tree was printed', - 'Old refs may be stale after the mutation', - ], - task: quiz( - NOT_SETTLED_SAMPLE, - 'The next target is not known yet. What command should run next?', - ), - outputs: [/(?:^|\n)(?:agent-device\s+)?(?:wait\b|snapshot\b[^\n]*-i\b)/i], - forbiddenOutputs: [ - /(?:^|\n)(?:agent-device\s+)?(?:press|click|fill|longpress)\s+@e\d+/i, - RAW_COORDINATE_TARGET, - SHELL_OUTPUT_PROJECTION, - ], - allowOnlyLocalCliHelpCommands: true, - requireLocalCliHelp: true, - }), - makeCase({ - id: 'sample-output-private-ax-recovery-continues', - contract: [ - 'Platform: iOS', - 'Previous command output is from agent-device, not a task description', - 'The fallback snapshot still exposed an actionable Search button', - 'Need to open Search and observe the resulting UI', - ], - task: quiz( - PRIVATE_AX_RECOVERY_SAMPLE, - 'Treat the recovery warning as a warning, not a fatal error. Use the exposed Search button. What command should run next?', - ), - outputs: [ - plannedCommandAlternatives(['press', 'click', 'tap']), - /@e5\b|label=(?:["']Search["']|Search)/i, - /(?:--settle\b|(?:^|\n)(?:agent-device\s+)?snapshot\b[^\n]*-i\b)/i, - ], - forbiddenOutputs: [ - /--debug|--verbose/i, - plannedCommand('screenshot'), - plannedCommand('close'), - plannedCommand('help'), - RAW_COORDINATE_TARGET, - ], - allowOnlyLocalCliHelpCommands: true, - requireLocalCliHelp: true, - }), -]; - -const suite: Case[] = [ - ...withTags(['fixture-smoke'], FIXTURE_SMOKE_CASES), - ...withTags(['skill-guidance'], SKILL_GUIDANCE_CASES), -]; - -export default suite; diff --git a/test/skillgym/suites/local-cli-help-policy.test.ts b/test/skillgym/suites/local-cli-help-policy.test.ts deleted file mode 100644 index 5dcd16a355..0000000000 --- a/test/skillgym/suites/local-cli-help-policy.test.ts +++ /dev/null @@ -1,46 +0,0 @@ -import assert from 'node:assert/strict'; -import { test } from 'vitest'; -import { - extractCommandEvents, - isActualLocalCliHelpProbe, - isAllowedLocalCliHelpCommand, -} from './local-cli-help-policy.ts'; - -test('allowed wrapper segments do not satisfy the local-help requirement by themselves', () => { - for (const command of ['cat', `printf 'help'`]) { - assert.equal(isAllowedLocalCliHelpCommand(command), true); - assert.equal(isActualLocalCliHelpProbe(command), false); - } -}); - -test('a real local agent-device help segment satisfies the requirement through allowed wrappers', () => { - for (const command of [ - 'node bin/agent-device.mjs --help', - 'agent-device help workflow', - 'node bin/agent-device.mjs help workflow | cat', - `/bin/zsh -lc 'node bin/agent-device.mjs snapshot --help 2>&1 | cat'`, - ]) { - assert.equal(isAllowedLocalCliHelpCommand(command), true); - assert.equal(isActualLocalCliHelpProbe(command), true); - } -}); - -test('a help probe combined with a forbidden runtime segment is neither allowed nor sufficient', () => { - const command = 'node bin/agent-device.mjs --help && agent-device open Settings'; - assert.equal(isAllowedLocalCliHelpCommand(command), false); - assert.equal(isActualLocalCliHelpProbe(command), false); -}); - -test('command events are extracted from direct and tool-call event shapes', () => { - assert.deepEqual( - extractCommandEvents({ - events: [ - { type: 'command', command: 'cat' }, - { type: 'toolCall', args: { command: 'node bin/agent-device.mjs --help' } }, - { type: 'toolCall', args: { cmd: 'agent-device help workflow' } }, - { type: 'message', command: 'ignored' }, - ], - }), - ['cat', 'node bin/agent-device.mjs --help', 'agent-device help workflow'], - ); -}); diff --git a/test/skillgym/suites/local-cli-help-policy.ts b/test/skillgym/suites/local-cli-help-policy.ts deleted file mode 100644 index 09a757ce67..0000000000 --- a/test/skillgym/suites/local-cli-help-policy.ts +++ /dev/null @@ -1,83 +0,0 @@ -interface CommandEventRecord { - type?: unknown; - command?: unknown; - args?: { - command?: unknown; - cmd?: unknown; - }; -} - -interface EventReport { - events?: unknown[]; -} - -export function extractCommandEvents(report: EventReport): string[] { - return (report.events ?? []).flatMap(commandFromEvent); -} - -export function isAllowedLocalCliHelpCommand(command: string) { - const segments = localCliHelpSegments(command); - return segments.length > 0 && segments.every(isAllowedLocalCliHelpSegment); -} - -export function isActualLocalCliHelpProbe(command: string) { - const segments = localCliHelpSegments(command); - return ( - segments.length > 0 && - segments.every(isAllowedLocalCliHelpSegment) && - segments.some(isAgentDeviceHelpSegment) - ); -} - -function commandFromEvent(event: unknown): string[] { - if (!isCommandEventRecord(event)) return []; - const command = directCommandEvent(event) ?? toolCallCommandEvent(event); - return command === undefined ? [] : [command]; -} - -function isCommandEventRecord(event: unknown): event is CommandEventRecord { - return typeof event === 'object' && event !== null; -} - -function directCommandEvent(record: CommandEventRecord): string | undefined { - if (record.type === 'command' && typeof record.command === 'string') { - return record.command; - } - return undefined; -} - -function toolCallCommandEvent(record: CommandEventRecord): string | undefined { - if (record.type !== 'toolCall') return undefined; - return firstString([record.args?.command, record.args?.cmd]); -} - -function firstString(values: unknown[]): string | undefined { - return values.find((value): value is string => typeof value === 'string'); -} - -function localCliHelpSegments(command: string): string[] { - const strippedCommand = command - .trim() - .replace(/^\/bin\/zsh\s+-lc\s+'(.+)'$/, '$1') - .trim(); - - return strippedCommand - .split(/\s*(?:&&|\|\||[;|])\s*/) - .map((segment) => segment.trim()) - .filter(Boolean); -} - -function isAllowedLocalCliHelpSegment(command: string) { - return ( - isAgentDeviceHelpSegment(command) || - /^printf\s+["'][^"']*["']$/.test(command) || - command === 'cat' - ); -} - -function isAgentDeviceHelpSegment(command: string) { - const helpToken = '[^\\s;&|]+'; - return new RegExp( - `^(?:node\\s+bin\\/agent-device\\.mjs|agent-device)\\s+(?:(?:help(?:\\s+${helpToken})*)|(?:${helpToken}\\s+)?--help)(?:\\s+2>&1)?$`, - ).test(command); -} diff --git a/vitest.config.ts b/vitest.config.ts index 7bdc3c39c4..d8e17153d4 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -22,9 +22,6 @@ export const SUBPROCESS_STUB_TESTS = [ // Replays the fuzz regression corpus (#1414) through that same worker watchdog, so a promoted // hang case fails against its per-case budget instead of wedging the unit job. 'scripts/fuzz/corpus-replay.test.ts', - // Spawns the `pnpm repo-health` CLI as a subprocess to exercise its arg handling and the JSON - // it writes over the real tree (#1423). - 'scripts/repo-health/run.test.ts', ]; export default defineConfig({ @@ -50,13 +47,8 @@ export default defineConfig({ 'scripts/__tests__/help-conformance-error-recovery-coverage.test.ts', 'scripts/__tests__/help-conformance-sample-outputs.test.ts', 'scripts/__tests__/help-conformance-topic-coverage.test.ts', - // Repo-health snapshot model (#1423): pure aggregation over the analyzers, no - // subprocess or device work, so it runs in the fast lane. The CLI's subprocess test - // lives in the subprocess-stub project. - 'scripts/repo-health/model.test.ts', // Parses CI configuration only, so this action guard needs no device or subprocess lane. 'test/ci/upload-agent-device-artifacts.test.ts', - 'test/skillgym/suites/local-cli-help-policy.test.ts', // The frozen replay-compat corpus (#1417): parse-only, no device or // subprocess work, so it belongs in the fast lane next to the // grammar it guards.