diff --git a/package.json b/package.json index 5c01c08040..3d67419dae 100644 --- a/package.json +++ b/package.json @@ -128,6 +128,7 @@ "check:layering": "node --experimental-strip-types --test scripts/layering/model.test.ts scripts/layering/zone-policy.test.ts && node --experimental-strip-types scripts/layering/check.ts", "depgraph": "node --experimental-strip-types scripts/depgraph/build.ts", "depgraph:test": "node --experimental-strip-types --test scripts/depgraph/model.test.ts scripts/depgraph/affected.test.ts", + "repo-health": "node --experimental-strip-types scripts/repo-health/run.ts", "check:production-exports": "fallow dead-code --config fallow-production-exports.json --production --unused-exports --fail-on-issues", "check:bundle-owner-files": "node --experimental-strip-types scripts/check-bundle-owner-files.ts", "check:command-docs": "vitest run --project unit-core src/__tests__/command-doc-coverage.test.ts", diff --git a/scripts/layering/check.ts b/scripts/layering/check.ts index 07a5ac386b..098cfaf907 100644 --- a/scripts/layering/check.ts +++ b/scripts/layering/check.ts @@ -261,7 +261,9 @@ function checkTypeInversions(edges: readonly ResolvedImportEdge[]): Violation[] // moves here bring it to 102. Measured on the rebased tree — an earlier 87 was taken against an // older main and was stale rather than violated, which is exactly the kind of drift a ratchet // measured at merge time prevents. -const TYPE_CYCLE_BASELINE = 102; +// Exported so the repo-health snapshot (scripts/repo-health) can record the R9 ratchet value as +// observatory data without re-deriving it. The gate remains the authority; the snapshot follows. +export const TYPE_CYCLE_BASELINE = 102; function checkTypeCycleGrowth(actual: number): Violation[] { if (actual <= TYPE_CYCLE_BASELINE) return []; diff --git a/scripts/lib/run-as-main.ts b/scripts/lib/run-as-main.ts new file mode 100644 index 0000000000..a5b4dd5e9f --- /dev/null +++ b/scripts/lib/run-as-main.ts @@ -0,0 +1,29 @@ +import { pathToFileURL } from 'node:url'; + +/** + * Run `main` as the process entrypoint, but only when `moduleUrl` (pass `import.meta.url`) is the + * script node was actually invoked with — so importing the module for its exports never runs it. + * Exits with `main`'s status and turns a throw into a `label`-prefixed non-zero exit. Keeps the + * `import.meta.url === pathToFileURL(process.argv[1])` guard from being copy-pasted per script. + */ +function isEntrypoint(moduleUrl: string): boolean { + return moduleUrl === pathToFileURL(process.argv[1] ?? '').href; +} + +function messageOf(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +export function runAsMain( + moduleUrl: string, + label: string, + main: (argv: string[]) => number, +): void { + if (!isEntrypoint(moduleUrl)) return; + try { + process.exit(main(process.argv.slice(2))); + } catch (error: unknown) { + process.stderr.write(`${label}: ${messageOf(error)}\n`); + process.exit(1); + } +} diff --git a/scripts/repo-health/README.md b/scripts/repo-health/README.md new file mode 100644 index 0000000000..1f41a1bbfb --- /dev/null +++ b/scripts/repo-health/README.md @@ -0,0 +1,136 @@ +# Repo-health snapshot + +```sh +pnpm repo-health # human-readable summary +pnpm repo-health --json # the raw JSON snapshot on stdout +pnpm repo-health --out p # also write the JSON snapshot to a file +``` + +One JSON snapshot aggregating the signals this repo already computes across 24+ CI checks, so an +agent can query the repo's state in one read instead of scraping job logs. It is **data, not a +dashboard** (#1409's lesson): there is no rendering, and it draws no conclusions for you. + +It reuses each analyzer rather than reimplementing any metric, runs from a clean checkout in well +under two minutes, and needs no network — everything comes from the working tree, `git`, and the +artifacts other gates already write. + +## What it does NOT do + +- **No history / trends / PR comments.** That is the follow-up quality-delta issue (#1424); this + snapshot is the input it persists. +- **No gating on component metrics.** Instability / abstractness / main-sequence distance are + observatory data to locate concrete high-fan-in modules worth pinning harder (CONTEXT.md + "Principles and their gates"). They must **never** become CI thresholds. +- **The one thing that can fail the command** is the depgraph-vs-layering R6 consistency + assertion — see below. + +## The R6 consistency assertion (the only wired-into-CI part) + +The dependency graph the snapshot builds must reproduce the layering gate's +`TYPE_INVERSION_BASELINE` (`scripts/layering/check.ts`), pair for pair (#1410). `pnpm repo-health` +exits non-zero and prints the difference when they diverge. The **identical** assertion is already +wired into CI by the **Layering Guard** job, which runs `scripts/depgraph/model.test.ts` — so no +new CI job or cost is added here. The gate stays the authority: a mismatch means the tree or the +baseline changed, never the snapshot. + +## Schema (`schemaVersion: 1`) + +Field names are the stable **trend/delta contract** consumed by #1424. Any change bumps +`schemaVersion`; #1424 refuses to diff across schema versions without a migration note. + +```jsonc +{ + "schemaVersion": 1, + "provenance": { + "schemaVersion": 1, + "commit": "", + "ref": "", + "node": "v22.x", + "generatedAt": "", + // Content hash per analyzer/config, standing in for a version: a source change is a version bump. + "tool": { "depgraph": "…", "layering": "…", "fallow": "…", "coverage": "…", "size": "…", + "slowTest": "…", "bench": "…", "skillgym": "…" }, + "inputs": { + "sourceFiles": 932, // production files fed to the graph build + "lockfile": { "path": "pnpm-lock.yaml", "sha256": "…" }, + // coverage and size are read artifacts this command does NOT produce. Their freshness relative + // to `commit` is proven ONLY from a producing commit the artifact stamps in its own bytes — + // never mtime (a copy/restore/CI-cache download resets it) nor an enumerated input list (never + // provably complete). status: "fresh" = stamped commit == HEAD; "stale" = stamped commit != + // HEAD; "unknown" = no stamp. Today neither producer stamps a commit, so both report "unknown" + // (honest — cannot be proven current); if one starts emitting `commit`, it is verified. The + // bytes are still hashed so #1424 keys history on the exact metrics. A consumer must treat + // "unknown"/"stale" alike as not-current, never as `commit`. + "coverageSummary": { "path": "coverage/coverage-summary.json", "sha256": "…", + "producerCommit": null, "status": "unknown" } | null, + "sizeReport": { "path": ".tmp/size-report.json", "sha256": "…", + "producerCommit": null, "status": "unknown" } | null + } + }, + "metrics": { + "depgraph": { // from scripts/depgraph (reuses the layering edge model) + "files": 932, "edges": 4791, + "valueCycles": 0, "typeCycles": 7, "dynamicCycles": 1, + "redundantEdges": 1366, // value edges also reachable at distance >= 2 (NOT removable) + "backEdges": 0, // R5 ranked-spine value back-edges (gate keeps this at 0) + "typeInversionEdges": 7 // R6 type-only spine inversions, summed + }, + "layering": { // ratchet values from scripts/layering/check.ts + "typeInversionBaseline": { "commands -> client": 3, "…": 0 }, + "typeInversionTotal": 7, // R6 ratchet (may only shrink) + "typeCycleBaseline": 102 // R9 largest-type-cycle ceiling (growth-only) + }, + "coverage": { // from coverage/coverage-summary.json (json-summary reporter) + "available": true, + "lines": { "total": …, "covered": …, "pct": … }, + "statements": …, "functions": …, "branches": … + }, // { "available": false } when the artifact is absent + "size": { // from .tmp/size-report.json (scripts/size-report.mjs) + "available": true, + "jsRawBytes": …, "jsGzipBytes": …, "npmTarballBytes": …, "npmUnpackedBytes": … + }, // { "available": false } when the artifact is absent + "fallow": { // from fallow-baselines/*.json + .fallowrc.json + "deadCodeFindings": 0, "healthFindings": 161, + "suppressions": { "ignoredExports": …, "ignoredDependencies": …, "total": 19 } + }, + "slowTest": { // ratchet state from scripts/vitest-slow-test-reporter.ts + "unitBudgetMs": 2500, "integrationBudgetMs": 15000, "enforceFactor": 2 + }, + "bench": { "cases": 19, "topics": 10 }, // scripts/help-conformance-cases.mjs registry + "skillgym": { "cases": 5 }, // test/skillgym/suites/agent-device-smoke-suite.ts + "components": { // OBSERVATORY ONLY — never a CI threshold + "byZone": [ + { "zone": "kernel", "rank": 0, "classification": "ranked", + "files": …, "loc": …, + "afferent": 764, "efferent": 0, // cross-zone couplings (Ca / Ce) + "instability": 0.0, // I = Ce / (Ca + Ce) + "abstractness": 0.36, // A ≈ type-only share of afferent edges (approx.) + "distance": 0.64 } // |A + I - 1| + ] + }, + "mainSequence": { // OBSERVATORY ONLY + "concreteHighFanIn": [ // concrete (A < 0.5), fan-in >= 10, most-depended-on first + { "file": "utils/exec.ts", "zone": "utils", "fanIn": 81, "fanOut": 3, + "instability": 0.04, "abstractness": 0.17, "distance": 0.79, "concrete": true } + ] + } + }, + "consistency": { "r6": { "ok": true, "expected": { … }, "actual": { … } } } +} +``` + +### Coverage and size availability + +`coverage` and `size` are read from the artifacts the coverage and size gates already produce +(`coverage/coverage-summary.json` via the vitest `json-summary` reporter; `.tmp/size-report.json` +via `pnpm size:markdown`). Recomputing either would blow the two-minute / no-network budget, so a +clean checkout that has run neither gets `{ "available": false }` with the producing command +named in the summary. Run `pnpm test:coverage` and/or `pnpm size:markdown` first (or point #1424's +history job at a run that already did) to populate them. + +### Abstractness is a first approximation + +Per the issue, `abstractness` approximates a component's type-only export share by the share of +its **incoming edges that are type-only**. A module most of whose dependents import it for values +(e.g. `src/utils/exec.ts`, `src/daemon/ref-frame.ts`) reads as concrete; when its fan-in is also +high, it is a foundation worth pinning harder. This is a locator, not a verdict. diff --git a/scripts/repo-health/components.ts b/scripts/repo-health/components.ts new file mode 100644 index 0000000000..e90e084efd --- /dev/null +++ b/scripts/repo-health/components.ts @@ -0,0 +1,132 @@ +// Component metrics for the repo-health snapshot (#1423): instability, abstractness, and +// main-sequence distance derived from the dependency graph's fan-in/fan-out. +// +// These are OBSERVATORY DATA ONLY. They locate concrete, high-fan-in modules worth pinning harder; +// they must NEVER become CI thresholds (CONTEXT.md "Principles and their gates"). Kept in their own +// module so that constraint — and the approximations these metrics make — stays visible instead of +// blending into the analyzer adapters that feed the gate-adjacent numbers. + +import { zoneRank } from '../layering/model.ts'; +import type { GraphData } from '../depgraph/model.ts'; + +export type ZoneComponent = { + zone: string; + rank: number | null; + classification: string; + files: number; + loc: number; + /** Afferent couplings: cross-zone edges INTO the zone (how much depends on it). */ + afferent: number; + /** Efferent couplings: cross-zone edges OUT of the zone (how much it depends on). */ + efferent: number; + /** Instability I = efferent / (afferent + efferent); 0 when the zone has no cross-zone edges. */ + instability: number; + /** + * Abstractness A, approximated by the type-only share of afferent edges (first approximation + * per the issue). 0 when nothing depends on the zone. + */ + abstractness: number; + /** Distance from the main sequence, |A + I - 1|. Lower is "more balanced". */ + distance: number; +}; + +function round(value: number): number { + return Math.round(value * 1000) / 1000; +} + +function zoneMapOf(graph: GraphData): Map { + return new Map(graph.nodes.map((node) => [node.id, node.zone])); +} + +/** + * Per-zone instability / abstractness / main-sequence distance from the graph's cross-zone edges. + * Observatory data — never a CI threshold. + */ +export function zoneComponents(graph: GraphData): ZoneComponent[] { + const zoneOf = zoneMapOf(graph); + const afferent = new Map(); + const efferent = new Map(); + const typeAfferent = new Map(); + const bump = (map: Map, key: string): void => + map.set(key, (map.get(key) ?? 0) + 1); + + for (const edge of graph.edges) { + const from = zoneOf.get(edge.from); + const to = zoneOf.get(edge.to); + if (from === undefined || to === undefined || from === to) continue; + bump(efferent, from); + bump(afferent, to); + if (edge.kind === 'type') bump(typeAfferent, to); + } + + return graph.zones + .map((zone) => { + const ca = afferent.get(zone.id) ?? 0; + const ce = efferent.get(zone.id) ?? 0; + const instability = ca + ce === 0 ? 0 : ce / (ca + ce); + const abstractness = ca === 0 ? 0 : (typeAfferent.get(zone.id) ?? 0) / ca; + return { + zone: zone.id, + rank: zoneRank(zone.id), + classification: zone.classification, + files: zone.files, + loc: zone.loc, + afferent: ca, + efferent: ce, + instability: round(instability), + abstractness: round(abstractness), + distance: round(Math.abs(abstractness + instability - 1)), + }; + }) + .sort((left, right) => right.afferent - left.afferent); +} + +export type MainSequenceModule = { + file: string; + zone: string; + fanIn: number; + fanOut: number; + instability: number; + abstractness: number; + distance: number; + concrete: boolean; +}; + +/** + * The main-sequence report: concrete, high-fan-in modules worth pinning harder. Per-file + * abstractness is approximated by the module's type-only dependent share; a module most of whose + * dependents import it for VALUES (e.g. src/utils/exec.ts, src/daemon/ref-frame.ts) is concrete + * and, when its fan-in is high, a foundation the tree leans on. Observatory data only. + */ +export function mainSequenceModules( + graph: GraphData, + options: { minFanIn?: number; limit?: number } = {}, +): MainSequenceModule[] { + const minFanIn = options.minFanIn ?? 10; + const limit = options.limit ?? 40; + const typeIn = new Map(); + for (const edge of graph.edges) { + if (edge.kind === 'type') typeIn.set(edge.to, (typeIn.get(edge.to) ?? 0) + 1); + } + + return graph.nodes + .filter((node) => node.fanIn >= minFanIn) + .map((node) => { + const abstractness = node.fanIn === 0 ? 0 : (typeIn.get(node.id) ?? 0) / node.fanIn; + const total = node.fanIn + node.fanOut; + const instability = total === 0 ? 0 : node.fanOut / total; + return { + file: node.id.replace(/^src\//, ''), + zone: node.zone, + fanIn: node.fanIn, + fanOut: node.fanOut, + instability: round(instability), + abstractness: round(abstractness), + distance: round(Math.abs(abstractness + instability - 1)), + concrete: abstractness < 0.5, + }; + }) + .filter((module) => module.concrete) + .sort((left, right) => right.fanIn - left.fanIn) + .slice(0, limit); +} diff --git a/scripts/repo-health/model.test.ts b/scripts/repo-health/model.test.ts new file mode 100644 index 0000000000..dea1a43ab1 --- /dev/null +++ b/scripts/repo-health/model.test.ts @@ -0,0 +1,215 @@ +import { readFileSync } from 'node:fs'; +import { describe, expect, test } from 'vitest'; +import { listSourceFiles, TYPE_INVERSION_BASELINE } from '../layering/check.ts'; +import { resolveImportEdges } from '../layering/model.ts'; +import { buildGraph, type GraphData } from '../depgraph/model.ts'; +import { mainSequenceModules, zoneComponents } from './components.ts'; +import { + collectArtifactProvenance, + benchMetrics, + buildSnapshot, + coverageMetrics, + depgraphMetrics, + fallowMetrics, + layeringRatchets, + r6Consistency, + ratchetTotal, + sizeMetrics, + SNAPSHOT_SCHEMA_VERSION, + type Provenance, +} from './model.ts'; + +/** Build the report's graph over the real production tree, the way the command does. */ +function realGraph(): GraphData { + const files = listSourceFiles(); + const sources = new Map(files.map((file) => [file, readFileSync(file, 'utf8')])); + return buildGraph(sources, resolveImportEdges(sources)); +} + +/** + * A tiny hand-made graph: two zones depend on kernel for VALUES and one for TYPES, so kernel is + * concrete (abstractness 1/3) with fan-in 3 — a miniature of the real exec.ts/ref-frame.ts shape. + */ +function fixtureGraph(): GraphData { + const sources = new Map([ + ['src/kernel/errors.ts', 'export const fail = 1;\nexport type Err = string;\n'], + ['src/core/run.ts', "import { fail } from '../kernel/errors.ts';\nexport const run = 1;\n"], + ['src/core/wrap.ts', "import { fail } from '../kernel/errors.ts';\nexport const wrap = 1;\n"], + [ + 'src/commands/tap.ts', + [ + "import { run } from '../core/run.ts';", + "import type { Err } from '../kernel/errors.ts';", + ].join('\n'), + ], + ]); + return buildGraph(sources, resolveImportEdges(sources)); +} + +test('depgraphMetrics reuses the graph rather than recomputing any number', () => { + const metrics = depgraphMetrics(fixtureGraph()); + expect(metrics.files).toBe(4); + expect(metrics.edges).toBe(4); + expect(metrics.valueCycles).toBe(0); + // commands -> kernel is a type-only import; the ranked spine outranks kernel so it is an R6 + // inversion, not a value/back edge. + expect(metrics.backEdges).toBe(0); +}); + +test('ratchetTotal sums a per-pair ratchet map', () => { + expect(ratchetTotal({ a: 3, b: 1, c: 2 })).toBe(6); +}); + +test('layeringRatchets reports the gate ratchets without re-deriving them', () => { + const ratchets = layeringRatchets(); + expect(ratchets.typeInversionBaseline).toBe(TYPE_INVERSION_BASELINE); + expect(ratchets.typeInversionTotal).toBe(ratchetTotal(TYPE_INVERSION_BASELINE)); + expect(ratchets.typeCycleBaseline).toBeGreaterThan(0); +}); + +describe('component metrics are derived from fan-in/fan-out', () => { + test('zoneComponents computes instability, abstractness and main-sequence distance', () => { + const byZone = zoneComponents(fixtureGraph()); + const kernel = byZone.find((zone) => zone.zone === 'kernel')!; + // Nothing in the fixture leaves kernel, and three files depend on it (one type-only). + expect(kernel.efferent).toBe(0); + expect(kernel.afferent).toBe(3); + expect(kernel.instability).toBe(0); + expect(kernel.abstractness).toBe(0.333); + }); + + test('mainSequenceModules surfaces concrete high-fan-in modules only', () => { + const modules = mainSequenceModules(fixtureGraph(), { minFanIn: 1 }); + const kernel = modules.find((module) => module.file === 'kernel/errors.ts')!; + expect(kernel.fanIn).toBe(3); + // Most dependents import it for values, so abstractness is 1/3 and it is concrete. + expect(kernel.abstractness).toBe(0.333); + expect(kernel.concrete).toBe(true); + }); +}); + +test('r6Consistency passes when the graph reproduces the baseline and fails on drift', () => { + const passing = r6Consistency(TYPE_INVERSION_BASELINE); + expect(passing.ok).toBe(true); + + const drifted = r6Consistency({ ...TYPE_INVERSION_BASELINE, 'commands -> client': 99 }); + expect(drifted.ok).toBe(false); + expect(drifted.expected).toEqual(passing.expected); +}); + +test('fallowMetrics counts findings from the baselines and suppressions from the config', () => { + const metrics = fallowMetrics( + { unused_exports: [{}, {}], circular_dependencies: [], unused_files: [{}] }, + { + finding_counts: { + 'src/a.ts': { complexity_critical: { count: 1 }, crap_moderate: { count: 2 } }, + 'src/b.ts': { crap_moderate: { count: 1 } }, + }, + }, + { + ignoreExports: [{ exports: ['x', 'y'] }, { exports: ['z'] }], + ignoreDependencies: ['@theme'], + }, + ); + expect(metrics.deadCodeFindings).toBe(3); + expect(metrics.healthFindings).toBe(4); + expect(metrics.suppressions).toEqual({ ignoredExports: 3, ignoredDependencies: 1, total: 4 }); +}); + +test('coverage and size degrade to unavailable when the artifact is absent', () => { + expect(coverageMetrics(null)).toEqual({ available: false }); + expect(sizeMetrics(null)).toEqual({ available: false }); + expect( + coverageMetrics({ total: { lines: { total: 10, covered: 8, pct: 80 } } as never }).available, + ).toBe(true); +}); + +// Drives the real collector (parse → read stamped commit → hash → status), so removing the +// commit-verification breaks these — unlike a fabricated status. Freshness is proven ONLY from a +// commit the artifact stamps, never mtime or a producer-input list. +test('collectArtifactProvenance proves freshness only from a commit the artifact stamps', () => { + const path = 'coverage/coverage-summary.json'; + + // A real coverage-summary shape stamps no commit — we cannot prove it matches HEAD, so `unknown`. + const coverageShaped = JSON.stringify({ total: { lines: { total: 10, covered: 8, pct: 80 } } }); + const unknown = collectArtifactProvenance(path, coverageShaped, 'abc123'); + expect(unknown?.status).toBe('unknown'); + expect(unknown?.producerCommit).toBeNull(); + expect(unknown?.sha256).toMatch(/^[0-9a-f]{12}$/); // bytes still hashed for #1424 + + // An artifact that stamps the current commit is proven current. + expect( + collectArtifactProvenance(path, JSON.stringify({ commit: 'abc123', total: {} }), 'abc123') + ?.status, + ).toBe('fresh'); + + // A stamped commit that differs is stale — metrics predate HEAD. + const stale = collectArtifactProvenance(path, JSON.stringify({ commit: 'old999' }), 'abc123'); + expect(stale?.status).toBe('stale'); + expect(stale?.producerCommit).toBe('old999'); + + // Absent artifact degrades cleanly to null (coverage/size become { available: false }). + expect(collectArtifactProvenance(path, null, 'abc123')).toBeNull(); +}); + +test('benchMetrics counts cases and distinct topics from the case registry', () => { + expect(benchMetrics([{ docs: ['a', 'b'] }, { docs: ['b'] }, { docs: ['c'] }])).toEqual({ + cases: 3, + topics: 3, + }); +}); + +const PROVENANCE: Provenance = { + schemaVersion: SNAPSHOT_SCHEMA_VERSION, + commit: 'abc', + ref: 'main', + node: 'v22', + generatedAt: '2026-07-28T00:00:00.000Z', + tool: {}, + inputs: { sourceFiles: 3, lockfile: null, coverageSummary: null, sizeReport: null }, +}; + +test('buildSnapshot assembles every metric family and pins the schema version', () => { + const snapshot = buildSnapshot({ + provenance: PROVENANCE, + graph: fixtureGraph(), + fallow: { deadCode: {}, health: {}, config: {} }, + coverage: null, + size: null, + slowTest: { unitBudgetMs: 2500, integrationBudgetMs: 15000, enforceFactor: 2 }, + benchCases: [{ docs: ['a'] }], + skillgymCaseCount: 5, + }); + expect(snapshot.schemaVersion).toBe(SNAPSHOT_SCHEMA_VERSION); + expect(Object.keys(snapshot.metrics).sort()).toEqual([ + 'bench', + 'components', + 'coverage', + 'depgraph', + 'fallow', + 'layering', + 'mainSequence', + 'size', + 'skillgym', + 'slowTest', + ]); + expect(snapshot.metrics.skillgym.cases).toBe(5); +}); + +// The acceptance spot-check (issue #1423): the main-sequence report must locate the concrete, +// high-fan-in foundations. src/utils/exec.ts (the process-exec seam) and src/daemon/ref-frame.ts +// both have many value dependents, so both must appear as concrete. Guards the metric against a +// sign flip or an abstractness inversion that would make the report point at the wrong modules. +test('the real-tree main-sequence report surfaces exec.ts and ref-frame.ts as concrete', () => { + const graph = realGraph(); + const concrete = new Map(mainSequenceModules(graph).map((module) => [module.file, module])); + for (const file of ['utils/exec.ts', 'daemon/ref-frame.ts']) { + const module = concrete.get(file); + expect(module, `${file} should surface as a concrete high-fan-in module`).toBeDefined(); + expect(module!.concrete).toBe(true); + expect(module!.fanIn).toBeGreaterThanOrEqual(10); + } + + // And the graph the report builds must reproduce the gate's R6 baseline over the real tree. + expect(r6Consistency(graph.typeInversions).ok).toBe(true); +}); diff --git a/scripts/repo-health/model.ts b/scripts/repo-health/model.ts new file mode 100644 index 0000000000..9246d2ecb3 --- /dev/null +++ b/scripts/repo-health/model.ts @@ -0,0 +1,376 @@ +// Repo-health snapshot model — pure aggregation over the analyzers this repo already runs. +// +// The snapshot is DATA for agents to query (issue #1423, Track C of #1412), not a dashboard and +// not a new gate. Every metric here is REUSED from an existing analyzer rather than recomputed: +// the dependency graph comes from scripts/depgraph (which itself reuses the layering gate's edge +// model), the R6/R9 ratchet values come from scripts/layering/check.ts, fallow counts come from +// the checked-in baselines, coverage/size come from the artifacts their own gates already write, +// the slow-test ratchet comes from the vitest reporter, and the bench/skillgym counts come from +// their case registries. +// +// Field names are the trend/delta contract consumed by the follow-up quality-delta comment +// (#1424): they must stay stable, and any change bumps SNAPSHOT_SCHEMA_VERSION so a consumer +// refuses to diff across schema versions without a migration note. +// +// The component metrics (instability / abstractness / main-sequence distance) are OBSERVATORY +// DATA ONLY. They locate concrete, high-fan-in modules worth pinning harder; they must NEVER +// become CI thresholds (CONTEXT.md "Principles and their gates"). The single thing wired into CI +// is the depgraph-vs-layering R6 consistency assertion, which lives in the Layering Guard job. + +import { createHash } from 'node:crypto'; +import { TYPE_CYCLE_BASELINE, TYPE_INVERSION_BASELINE } from '../layering/check.ts'; +import type { EdgeKind, GraphData } from '../depgraph/model.ts'; +import { + mainSequenceModules, + zoneComponents, + type MainSequenceModule, + type ZoneComponent, +} from './components.ts'; + +export const SNAPSHOT_SCHEMA_VERSION = 1; + +/** Total across a per-pair ratchet map, e.g. TYPE_INVERSION_BASELINE. */ +export function ratchetTotal(byPair: Readonly>): number { + return Object.values(byPair).reduce((sum, count) => sum + count, 0); +} + +export type DepgraphMetrics = { + files: number; + edges: number; + valueCycles: number; + typeCycles: number; + dynamicCycles: number; + /** Value edges whose target is ALSO reachable at distance >= 2. Reachability, not removability. */ + redundantEdges: number; + /** R5 ranked-spine back-edges over value edges (the gate keeps this at zero). */ + backEdges: number; + /** R6 type-only spine-inversion edges, summed across zone pairs. */ + typeInversionEdges: number; +}; + +export function depgraphMetrics(graph: GraphData): DepgraphMetrics { + const cyclesByKind = (kind: EdgeKind): number => + graph.cycles.filter((cycle) => cycle.kind === kind).length; + return { + files: graph.nodes.length, + edges: graph.edges.length, + valueCycles: cyclesByKind('value'), + typeCycles: cyclesByKind('type'), + dynamicCycles: cyclesByKind('dynamic'), + redundantEdges: graph.edges.filter((edge) => edge.transitivelyReachable).length, + backEdges: graph.edges.filter((edge) => edge.backEdge !== null).length, + typeInversionEdges: ratchetTotal(graph.typeInversions), + }; +} + +export type LayeringRatchets = { + /** R6 baseline, per zone pair, and its total. The counts may only shrink (gate authority). */ + typeInversionBaseline: Readonly>; + typeInversionTotal: number; + /** R9 largest-type-cycle ceiling (growth-only ratchet). */ + typeCycleBaseline: number; +}; + +export function layeringRatchets(): LayeringRatchets { + return { + typeInversionBaseline: TYPE_INVERSION_BASELINE, + typeInversionTotal: ratchetTotal(TYPE_INVERSION_BASELINE), + typeCycleBaseline: TYPE_CYCLE_BASELINE, + }; +} + +/** + * The depgraph-vs-layering R6 cross-check (#1410): the graph the report builds over the real tree + * must reproduce the gate's TYPE_INVERSION_BASELINE, pair for pair. `pnpm repo-health` fails when + * this is false, and the identical assertion is wired into the Layering Guard CI job via + * scripts/depgraph/model.test.ts. The gate stays the authority; a mismatch means the tree or the + * baseline changed, never this check. + */ +export type R6Consistency = { + ok: boolean; + expected: Readonly>; + actual: Readonly>; +}; + +function sortedEntries(byPair: Readonly>): [string, number][] { + return Object.entries(byPair).sort(([left], [right]) => left.localeCompare(right)); +} + +export function r6Consistency( + graphTypeInversions: Readonly>, +): R6Consistency { + const expected = Object.fromEntries(sortedEntries(TYPE_INVERSION_BASELINE)); + const actual = Object.fromEntries(sortedEntries(graphTypeInversions)); + return { + ok: JSON.stringify(actual) === JSON.stringify(expected), + expected, + actual, + }; +} + +// ---- Fallow ----------------------------------------------------------------------------------- + +/** The fallow dead-code baseline: arrays of findings keyed by category. */ +export type FallowDeadCodeBaseline = Record; +/** The fallow health baseline: per-file maps of finding kind -> { count }. */ +export type FallowHealthBaseline = { + finding_counts?: Record>; +}; +export type FallowConfig = { + ignoreExports?: { exports: string[] }[]; + ignoreDependencies?: string[]; +}; + +export type FallowMetrics = { + /** Total dead-code findings recorded in the dead-code baseline. */ + deadCodeFindings: number; + /** Total complexity/health findings recorded in the health baseline. */ + healthFindings: number; + suppressions: { + /** Distinct export entries suppressed via `ignoreExports` in .fallowrc.json. */ + ignoredExports: number; + ignoredDependencies: number; + total: number; + }; +}; + +export function fallowMetrics( + deadCode: FallowDeadCodeBaseline, + health: FallowHealthBaseline, + config: FallowConfig, +): FallowMetrics { + const deadCodeFindings = Object.values(deadCode).reduce( + (sum, list) => sum + (Array.isArray(list) ? list.length : 0), + 0, + ); + let healthFindings = 0; + for (const perKind of Object.values(health.finding_counts ?? {})) { + for (const finding of Object.values(perKind)) healthFindings += finding.count; + } + const ignoredExports = (config.ignoreExports ?? []).reduce( + (sum, entry) => sum + (entry.exports?.length ?? 0), + 0, + ); + const ignoredDependencies = (config.ignoreDependencies ?? []).length; + return { + deadCodeFindings, + healthFindings, + suppressions: { + ignoredExports, + ignoredDependencies, + total: ignoredExports + ignoredDependencies, + }, + }; +} + +// ---- Coverage & size (read from the artifacts their own gates already produce) ---------------- + +type CoverageTotalEntry = { total: number; covered: number; pct: number }; +export type CoverageSummary = { + total?: Record<'lines' | 'statements' | 'functions' | 'branches', CoverageTotalEntry>; +}; +export type CoverageMetrics = { + available: boolean; + lines?: CoverageTotalEntry; + statements?: CoverageTotalEntry; + functions?: CoverageTotalEntry; + branches?: CoverageTotalEntry; +}; + +export function coverageMetrics(summary: CoverageSummary | null): CoverageMetrics { + if (!summary?.total) return { available: false }; + const { lines, statements, functions, branches } = summary.total; + return { available: true, lines, statements, functions, branches }; +} + +export type SizeReport = { + js?: { rawBytes: number; gzipBytes: number }; + npmPack?: { tarballBytes: number; unpackedBytes: number }; +}; +export type SizeMetrics = { + available: boolean; + jsRawBytes?: number; + jsGzipBytes?: number; + npmTarballBytes?: number; + npmUnpackedBytes?: number; +}; + +export function sizeMetrics(report: SizeReport | null): SizeMetrics { + if (!report?.js || !report.npmPack) return { available: false }; + return { + available: true, + jsRawBytes: report.js.rawBytes, + jsGzipBytes: report.js.gzipBytes, + npmTarballBytes: report.npmPack.tarballBytes, + npmUnpackedBytes: report.npmPack.unpackedBytes, + }; +} + +// ---- Slow-test / bench / skillgym (registry-derived counts) ----------------------------------- + +export type SlowTestRatchet = { + unitBudgetMs: number; + integrationBudgetMs: number; + enforceFactor: number; +}; + +export type BenchMetrics = { + /** Help-conformance bench cases (scripts/help-conformance-cases.mjs). */ + cases: number; + /** Distinct help topics the cases exercise. */ + topics: number; +}; + +export function benchMetrics(cases: readonly { docs: readonly string[] }[]): BenchMetrics { + return { + cases: cases.length, + topics: new Set(cases.flatMap((testCase) => testCase.docs)).size, + }; +} + +// ---- Provenance ------------------------------------------------------------------------------- + +/** + * Freshness of a READ artifact relative to the snapshot's `commit`. `fresh` requires proof; anything + * unproven is `unknown`, never silently trusted. + */ +export type ArtifactStatus = 'fresh' | 'stale' | 'unknown'; + +/** + * Provenance for an analyzer artifact this snapshot READ but did not itself produce (coverage, + * size). Its freshness relative to `commit` is proven ONLY from a producing commit the artifact + * stamps in its own bytes. We deliberately reject the two signals that cannot prove it: mtime (a + * copy, restore, or CI cache download resets it) and an enumerated producer-input list (never + * provably complete — the last review showed it silently omitted inputs). So a stamped commit equal + * to `commit` is `fresh`; a different stamped commit is `stale`; NO stamp is `unknown`. Today + * neither producer stamps a commit, so both report `unknown` — honest, since we cannot prove they + * match; if a producer starts emitting one, it is verified automatically. `sha256` still records the + * bytes so #1424 keys history on the exact metrics read. A consumer must treat `stale` and `unknown` + * alike as not-current, never as `commit`. + */ +export type ArtifactProvenance = { + path: string; + sha256: string; + /** The producing commit read from the artifact's own bytes, or null when it stamps none. */ + producerCommit: string | null; + status: ArtifactStatus; +}; + +/** The producing commit an artifact stamps in its own JSON, if any (`producerCommit` or `commit`). */ +function readStampedCommit(raw: string): string | null { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return null; + } + if (parsed === null || typeof parsed !== 'object') return null; + const record = parsed as Record; + for (const key of ['producerCommit', 'commit']) { + const value = record[key]; + if (typeof value === 'string' && value.length > 0) return value; + } + return null; +} + +/** + * The whole read-artifact collector: hash the bytes, read any producing commit the artifact stamps, + * and derive freshness against `currentCommit`. Returns null when the artifact is absent. This is + * the function the runner wires to disk, so a test that drives it with real artifact bytes exercises + * the actual freshness path end-to-end (not a fabricated status). + */ +export function collectArtifactProvenance( + path: string, + raw: string | null, + currentCommit: string, +): ArtifactProvenance | null { + if (raw === null) return null; + const producerCommit = readStampedCommit(raw); + const status: ArtifactStatus = + producerCommit === null ? 'unknown' : producerCommit === currentCommit ? 'fresh' : 'stale'; + return { + path, + sha256: createHash('sha256').update(raw).digest('hex').slice(0, 12), + producerCommit, + status, + }; +} + +/** + * What produced the snapshot and from which inputs. Mandatory in v1 (issue #1423 amendment) so + * #1424 can persist history and refuse to diff across schema versions. `tool` holds a content + * hash per analyzer (its own source/config, standing in for a version); `inputs` records the + * lockfile and the read-only artifacts each family of metrics was computed from, each hashed and + * given a proven freshness `status` so a false commit-indexed history entry cannot slip through. + */ +export type Provenance = { + schemaVersion: number; + commit: string; + ref: string | null; + node: string; + generatedAt: string; + tool: Record; + inputs: { + /** Production source files fed to the graph build. */ + sourceFiles: number; + lockfile: { path: string; sha256: string } | null; + /** Read artifacts, hashed with a proven freshness status; null when absent (degrade cleanly). */ + coverageSummary: ArtifactProvenance | null; + sizeReport: ArtifactProvenance | null; + }; +}; + +// ---- Full snapshot ---------------------------------------------------------------------------- + +export type RepoHealthSnapshot = { + schemaVersion: number; + provenance: Provenance; + /** + * Component metrics are observatory data — they locate concrete high-fan-in modules and must + * NEVER become CI thresholds. Only `consistency.r6` gates anything. + */ + metrics: { + depgraph: DepgraphMetrics; + layering: LayeringRatchets; + coverage: CoverageMetrics; + size: SizeMetrics; + fallow: FallowMetrics; + slowTest: SlowTestRatchet; + bench: BenchMetrics; + skillgym: { cases: number }; + components: { byZone: ZoneComponent[] }; + mainSequence: { concreteHighFanIn: MainSequenceModule[] }; + }; + consistency: { r6: R6Consistency }; +}; + +export type SnapshotInputs = { + provenance: Provenance; + graph: GraphData; + fallow: { deadCode: FallowDeadCodeBaseline; health: FallowHealthBaseline; config: FallowConfig }; + coverage: CoverageSummary | null; + size: SizeReport | null; + slowTest: SlowTestRatchet; + benchCases: readonly { docs: readonly string[] }[]; + skillgymCaseCount: number; +}; + +export function buildSnapshot(inputs: SnapshotInputs): RepoHealthSnapshot { + return { + schemaVersion: SNAPSHOT_SCHEMA_VERSION, + provenance: inputs.provenance, + metrics: { + depgraph: depgraphMetrics(inputs.graph), + layering: layeringRatchets(), + coverage: coverageMetrics(inputs.coverage), + size: sizeMetrics(inputs.size), + fallow: fallowMetrics(inputs.fallow.deadCode, inputs.fallow.health, inputs.fallow.config), + slowTest: inputs.slowTest, + bench: benchMetrics(inputs.benchCases), + skillgym: { cases: inputs.skillgymCaseCount }, + components: { byZone: zoneComponents(inputs.graph) }, + mainSequence: { concreteHighFanIn: mainSequenceModules(inputs.graph) }, + }, + consistency: { r6: r6Consistency(inputs.graph.typeInversions) }, + }; +} diff --git a/scripts/repo-health/run.test.ts b/scripts/repo-health/run.test.ts new file mode 100644 index 0000000000..bcaa4c3c60 --- /dev/null +++ b/scripts/repo-health/run.test.ts @@ -0,0 +1,70 @@ +// End-to-end coverage of the `pnpm repo-health` CLI. It spawns the command as a subprocess (so it +// exercises argument handling and the file it writes over the real tree), which is why this lives +// in the `subprocess-stub` vitest project rather than `unit-core` — see #1412 coordination rule 4. + +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { expect, test } from 'vitest'; +import type { RepoHealthSnapshot } from './model.ts'; + +function runRepoHealth(args: readonly string[]): { + status: number | null; + stdout: string; + stderr: string; +} { + const result = spawnSync( + process.execPath, + ['--experimental-strip-types', 'scripts/repo-health/run.ts', ...args], + { encoding: 'utf8', maxBuffer: 1024 * 1024 * 32 }, + ); + return { status: result.status, stdout: result.stdout ?? '', stderr: result.stderr ?? '' }; +} + +test('the default run prints a summary and exits zero over the real tree', () => { + const { status, stdout } = runRepoHealth([]); + expect(status, stdout).toBe(0); + expect(stdout).toContain('Repo health snapshot'); + expect(stdout).toContain('graph reproduces baseline: yes'); + expect(stdout).toContain('observatory only'); +}); + +test('--json emits a schema-versioned snapshot with every metric family', () => { + const { status, stdout } = runRepoHealth(['--json']); + expect(status, stdout).toBe(0); + + const snapshot = JSON.parse(stdout) as RepoHealthSnapshot; + expect(snapshot.schemaVersion).toBe(1); + + // Provenance is mandatory in v1 (issue amendment): schemaVersion, commit, per-analyzer tool + // hashes, and input provenance must all be present so #1424 can persist history. + expect(snapshot.provenance.commit).toMatch(/^[0-9a-f]{7,40}$/); + expect(Object.keys(snapshot.provenance.tool)).toContain('depgraph'); + expect(snapshot.provenance.inputs.sourceFiles).toBeGreaterThan(0); + + // Field names are the #1424 delta contract — pin the ones the comment consumes. + expect(snapshot.metrics.depgraph.redundantEdges).toBeGreaterThan(0); + expect(snapshot.metrics.layering.typeInversionTotal).toBeGreaterThanOrEqual(0); + expect(snapshot.metrics.slowTest.unitBudgetMs).toBeGreaterThan(0); + expect(snapshot.metrics.fallow.suppressions.total).toBeGreaterThanOrEqual(0); + expect(snapshot.metrics.bench.cases).toBeGreaterThan(0); + expect(snapshot.metrics.skillgym.cases).toBeGreaterThan(0); + + // The consistency assertion must pass on a clean tree. + expect(snapshot.consistency.r6.ok).toBe(true); + + // The main-sequence spot-check the acceptance names. + const concrete = snapshot.metrics.mainSequence.concreteHighFanIn.map((module) => module.file); + expect(concrete).toContain('utils/exec.ts'); + expect(concrete).toContain('daemon/ref-frame.ts'); +}); + +test('--out writes the same JSON snapshot to a file', () => { + const out = join(mkdtempSync(join(tmpdir(), 'repo-health-')), 'snapshot.json'); + const { status } = runRepoHealth(['--out', out]); + expect(status).toBe(0); + const written = JSON.parse(readFileSync(out, 'utf8')) as RepoHealthSnapshot; + expect(written.schemaVersion).toBe(1); + expect(written.consistency.r6.ok).toBe(true); +}); diff --git a/scripts/repo-health/run.ts b/scripts/repo-health/run.ts new file mode 100644 index 0000000000..0a2ff9528d --- /dev/null +++ b/scripts/repo-health/run.ts @@ -0,0 +1,264 @@ +// Repo-health snapshot command (issue #1423, Track C of #1412): +// +// pnpm repo-health # human-readable summary +// pnpm repo-health --json # the raw JSON snapshot on stdout +// pnpm repo-health --out p # also write the JSON snapshot to a file +// +// One JSON snapshot aggregating the signals this repo already computes, by REUSING each analyzer +// rather than reimplementing any metric (see model.ts for which analyzer owns each number). It is +// DATA for agents to query, not a dashboard: no rendering, no history (that is #1424), and the +// component metrics are observatory-only and must never gate. +// +// It runs from a clean checkout in well under two minutes with no network: everything is read +// from the working tree, git, and the artifacts other gates already write. Coverage and package +// size are read from coverage/coverage-summary.json and .tmp/size-report.json when present and +// degrade to `available: false` (with the producing command) when they are not, so a fresh +// checkout still produces a complete, honest snapshot. Those two artifacts carry no producing-commit +// stamp today, so provenance hashes them and reports freshness as `unknown` unless the artifact +// itself stamps a commit equal to HEAD (see collectArtifactProvenance in model.ts) — we never infer +// freshness from mtime or a producer-input list, neither of which can prove it. A consumer like +// #1424 must treat `unknown`/`stale` as not-current, never as HEAD. +// +// The one thing that can FAIL the command is the depgraph-vs-layering R6 consistency assertion: +// the graph built here must reproduce scripts/layering/check.ts's TYPE_INVERSION_BASELINE. The +// identical assertion is wired into CI by the Layering Guard job (scripts/depgraph/model.test.ts). + +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { runCmdSync } from '../../src/utils/exec.ts'; +import { listSourceFiles } from '../layering/check.ts'; +import { resolveImportEdges } from '../layering/model.ts'; +import { buildGraph, type GraphData } from '../depgraph/model.ts'; +import { SLOW_TEST_RATCHET } from '../vitest-slow-test-budgets.ts'; +import { parseScriptArgs } from '../lib/cli-args.ts'; +import { runAsMain } from '../lib/run-as-main.ts'; +import { CASES } from '../help-conformance-cases.mjs'; +import skillgymSuite from '../../test/skillgym/suites/agent-device-smoke-suite.ts'; +import { + buildSnapshot, + collectArtifactProvenance, + SNAPSHOT_SCHEMA_VERSION, + type ArtifactProvenance, + type FallowConfig, + type FallowDeadCodeBaseline, + type FallowHealthBaseline, + type Provenance, + type RepoHealthSnapshot, +} from './model.ts'; + +const USAGE = 'Usage: pnpm repo-health [--json] [--out ]\n'; + +const COVERAGE_SUMMARY_PATH = 'coverage/coverage-summary.json'; +const SIZE_REPORT_PATH = '.tmp/size-report.json'; +const LOCKFILE_PATH = 'pnpm-lock.yaml'; + +const repoRoot = runCmdSync('git', ['rev-parse', '--show-toplevel']).stdout.trim(); + +function absolute(relativePath: string): string { + return path.join(repoRoot, relativePath); +} + +function readIfExists(relativePath: string): string | null { + const abs = absolute(relativePath); + return fs.existsSync(abs) ? fs.readFileSync(abs, 'utf8') : null; +} + +function readJsonIfExists(relativePath: string): T | null { + const raw = readIfExists(relativePath); + return raw === null ? null : (JSON.parse(raw) as T); +} + +/** A read artifact's raw bytes (for content-hash provenance) and its parsed form (for metrics). */ +function readArtifact(relativePath: string): { raw: string | null; parsed: T | null } { + const raw = readIfExists(relativePath); + return { raw, parsed: raw === null ? null : (JSON.parse(raw) as T) }; +} + +function shortSha(raw: string): string { + return createHash('sha256').update(raw).digest('hex').slice(0, 12); +} + +/** Short content hash standing in for an analyzer's version — a source change is a version bump. */ +function contentHash(relativePath: string): string { + const raw = readIfExists(relativePath); + return raw === null ? 'absent' : shortSha(raw); +} + +function lockfileProvenance(): { path: string; sha256: string } | null { + const raw = readIfExists(LOCKFILE_PATH); + return raw === null ? null : { path: LOCKFILE_PATH, sha256: shortSha(raw) }; +} + +function gitOutput(args: string[]): string | null { + const result = runCmdSync('git', args, { cwd: repoRoot, allowFailure: true }); + return result.exitCode === 0 ? result.stdout.trim() : null; +} + +function buildGraphFromTree(files: readonly string[]): GraphData { + const sources = new Map(files.map((file) => [file, fs.readFileSync(absolute(file), 'utf8')])); + return buildGraph(sources, resolveImportEdges(sources)); +} + +function currentCommit(): string { + return gitOutput(['rev-parse', 'HEAD']) ?? 'unknown'; +} + +function currentRef(): string | null { + return process.env.GITHUB_REF_NAME ?? gitOutput(['rev-parse', '--abbrev-ref', 'HEAD']); +} + +// A content hash per analyzer, standing in for a version: a change to the analyzer's own source +// is a version bump the snapshot records without a real semver to point at. +function toolHashes(): Record { + return { + depgraph: contentHash('scripts/depgraph/model.ts'), + layering: contentHash('scripts/layering/check.ts'), + fallow: contentHash('.fallowrc.json'), + coverage: contentHash('vitest.config.ts'), + size: contentHash('scripts/size-report.mjs'), + slowTest: contentHash('scripts/vitest-slow-test-budgets.ts'), + bench: contentHash('scripts/help-conformance-cases.mjs'), + skillgym: contentHash('test/skillgym/suites/agent-device-smoke-suite.ts'), + }; +} + +function collectProvenance( + graph: GraphData, + commit: string, + coverage: ArtifactProvenance | null, + size: ArtifactProvenance | null, +): Provenance { + return { + schemaVersion: SNAPSHOT_SCHEMA_VERSION, + commit, + ref: currentRef(), + node: process.version, + generatedAt: new Date().toISOString(), + tool: toolHashes(), + inputs: { + sourceFiles: graph.nodes.length, + lockfile: lockfileProvenance(), + coverageSummary: coverage, + sizeReport: size, + }, + }; +} + +function collectSnapshot(): RepoHealthSnapshot { + const graph = buildGraphFromTree(listSourceFiles()); + const commit = currentCommit(); + const coverage = readArtifact(COVERAGE_SUMMARY_PATH); + const size = readArtifact(SIZE_REPORT_PATH); + return buildSnapshot({ + provenance: collectProvenance( + graph, + commit, + collectArtifactProvenance(COVERAGE_SUMMARY_PATH, coverage.raw, commit), + collectArtifactProvenance(SIZE_REPORT_PATH, size.raw, commit), + ), + graph, + fallow: { + deadCode: readJsonIfExists('fallow-baselines/dead-code.json') ?? {}, + health: readJsonIfExists('fallow-baselines/health.json') ?? {}, + config: readJsonIfExists('.fallowrc.json') ?? {}, + }, + coverage: coverage.parsed, + size: size.parsed, + slowTest: SLOW_TEST_RATCHET, + benchCases: CASES, + skillgymCaseCount: skillgymSuite.length, + }); +} + +function pct(entry: { covered: number; total: number; pct: number } | undefined): string { + return entry + ? `${entry.pct.toFixed(2)}% (${entry.covered}/${entry.total})` + : 'n/a (run pnpm test:coverage)'; +} + +function bytes(value: number | undefined): string { + if (value === undefined) return 'n/a (run pnpm size:markdown)'; + if (value < 1000) return `${value} B`; + if (value < 1_000_000) return `${(value / 1000).toFixed(1)} kB`; + return `${(value / 1_000_000).toFixed(2)} MB`; +} + +function staleMark(artifact: ArtifactProvenance | null): string { + if (artifact === null || artifact.status === 'fresh') return ''; + return artifact.status === 'stale' + ? ' [STALE — artifact stamps a different commit]' + : ' [UNKNOWN — artifact stamps no commit; may not reflect HEAD]'; +} + +function renderSummary(snapshot: RepoHealthSnapshot): string { + const { metrics, provenance, consistency } = snapshot; + const lines: string[] = [ + `Repo health snapshot (schemaVersion ${snapshot.schemaVersion}) @ ${provenance.commit.slice(0, 12)}`, + ` depgraph: ${metrics.depgraph.files} files, ${metrics.depgraph.edges} edges, ` + + `${metrics.depgraph.redundantEdges} redundant; cycles value=${metrics.depgraph.valueCycles} ` + + `type=${metrics.depgraph.typeCycles} dynamic=${metrics.depgraph.dynamicCycles}`, + ` layering: R6 type-inversions=${metrics.layering.typeInversionTotal} ` + + `(graph reproduces baseline: ${consistency.r6.ok ? 'yes' : 'NO'}); R9 type-cycle baseline=${metrics.layering.typeCycleBaseline}`, + ` coverage: lines ${pct(metrics.coverage.lines)}; statements ${pct(metrics.coverage.statements)}` + + staleMark(provenance.inputs.coverageSummary), + ` size: js raw ${bytes(metrics.size.jsRawBytes)}, gzip ${bytes(metrics.size.jsGzipBytes)}; ` + + `npm tarball ${bytes(metrics.size.npmTarballBytes)}` + + staleMark(provenance.inputs.sizeReport), + ` fallow: dead-code findings=${metrics.fallow.deadCodeFindings}, health findings=${metrics.fallow.healthFindings}, ` + + `suppressions=${metrics.fallow.suppressions.total}`, + ` slow-test: unit ${metrics.slowTest.unitBudgetMs}ms / integration ${metrics.slowTest.integrationBudgetMs}ms (enforce ${metrics.slowTest.enforceFactor}x)`, + ` bench: ${metrics.bench.cases} cases across ${metrics.bench.topics} help topics; skillgym ${metrics.skillgym.cases} cases`, + '', + ' Component metrics (observatory only — never a CI threshold):', + ...metrics.components.byZone + .slice(0, 8) + .map( + (zone) => + ` ${zone.zone.padEnd(16)} Ca=${String(zone.afferent).padStart(4)} Ce=${String(zone.efferent).padStart(4)} ` + + `I=${zone.instability.toFixed(2)} A=${zone.abstractness.toFixed(2)} D=${zone.distance.toFixed(2)}`, + ), + ' Concrete high-fan-in modules worth pinning harder:', + ...metrics.mainSequence.concreteHighFanIn + .slice(0, 8) + .map( + (module) => + ` ${module.file.padEnd(40)} fanIn=${String(module.fanIn).padStart(3)} A=${module.abstractness.toFixed(2)}`, + ), + ]; + return `${lines.join('\n')}\n`; +} + +function run(argv: readonly string[]): number { + const values = parseScriptArgs(argv, USAGE, { + json: { type: 'boolean', default: false }, + out: { type: 'string' }, + }); + + const snapshot = collectSnapshot(); + const serialized = `${JSON.stringify(snapshot, null, 2)}\n`; + + if (typeof values.out === 'string') { + const outPath = path.resolve(values.out); + fs.mkdirSync(path.dirname(outPath), { recursive: true }); + fs.writeFileSync(outPath, serialized); + } + + process.stdout.write(values.json ? serialized : renderSummary(snapshot)); + + const { r6 } = snapshot.consistency; + if (!r6.ok) { + process.stderr.write( + 'repo-health: the dependency graph disagrees with the layering gate about R6 type-only ' + + 'spine inversions.\n' + + ` gate baseline (TYPE_INVERSION_BASELINE): ${JSON.stringify(r6.expected)}\n` + + ` graph over the current tree: ${JSON.stringify(r6.actual)}\n` + + 'The gate is the authority — fix the inverting edge, or update TYPE_INVERSION_BASELINE in ' + + 'scripts/layering/check.ts. See scripts/depgraph/README.md.\n', + ); + return 1; + } + return 0; +} + +runAsMain(import.meta.url, 'repo-health', run); diff --git a/scripts/vitest-slow-test-budgets.ts b/scripts/vitest-slow-test-budgets.ts new file mode 100644 index 0000000000..e1d42efdb4 --- /dev/null +++ b/scripts/vitest-slow-test-budgets.ts @@ -0,0 +1,21 @@ +// The slow-test ratchet's budgets, as a single source of truth (see docs/agents/testing.md +// "Speed rules"). Kept in a data-only module so consumers that only need the numbers — the +// repo-health snapshot (scripts/repo-health), which records them as observatory data — can import +// them without pulling the vitest reporter (a config-loaded module) into their dependency graph. + +// Unit tests must not wait real time; integration scenarios drive a real daemon request path and +// get more room. +export const UNIT_BUDGET_MS = 2_500; +export const INTEGRATION_BUDGET_MS = 15_000; + +// Enforcement fires at 2x budget: host load legitimately stretches a borderline test by tens of +// percent, and a wall-clock gate that flakes under contention trains people to ignore it. Between +// budget and 2x budget the gate reports without failing. +export const ENFORCE_FACTOR = 2; + +/** The ratchet as one record, for callers that report it rather than enforce it. */ +export const SLOW_TEST_RATCHET = { + unitBudgetMs: UNIT_BUDGET_MS, + integrationBudgetMs: INTEGRATION_BUDGET_MS, + enforceFactor: ENFORCE_FACTOR, +} as const; diff --git a/scripts/vitest-slow-test-reporter.ts b/scripts/vitest-slow-test-reporter.ts index c55b0ad621..6963d812b1 100644 --- a/scripts/vitest-slow-test-reporter.ts +++ b/scripts/vitest-slow-test-reporter.ts @@ -1,4 +1,9 @@ import type { Reporter, TestCase, TestModule } from 'vitest/node'; +import { + ENFORCE_FACTOR, + INTEGRATION_BUDGET_MS, + UNIT_BUDGET_MS, +} from './vitest-slow-test-budgets.ts'; /** * Slow-test ratchet (see docs/agents/testing.md "Speed rules"). @@ -7,19 +12,8 @@ import type { Reporter, TestCase, TestModule } from 'vitest/node'; * wall clock was bounded by files whose tests slept through production * timeouts (a 10.8s test proving "times out" by waiting the full 10s budget). * This reporter fails the run when a unit test exceeds the enforced budget. - * - * Budgets are per suite family: integration scenarios drive a real daemon - * request path and get more room; unit tests get 2.5s, which is already - * generous for injected-time tests. + * The budgets themselves live in ./vitest-slow-test-budgets.ts. */ -const UNIT_BUDGET_MS = 2_500; -const INTEGRATION_BUDGET_MS = 15_000; -// Enforcement fires at 2x budget: host load legitimately stretches a -// borderline test by tens of percent, and a wall-clock gate that flakes under -// contention trains people to ignore it. Between budget and 2x budget the -// gate reports without failing. -const ENFORCE_FACTOR = 2; - type Offender = { key: string; durationMs: number; budgetMs: number; enforce: boolean }; function budgetForPath(relativePath: string): number { diff --git a/vitest.config.ts b/vitest.config.ts index 7b6a7d118c..762349033e 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -22,6 +22,9 @@ export const SUBPROCESS_STUB_TESTS = [ // Replays the fuzz regression corpus (#1414) through that same worker watchdog, so a promoted // hang case fails against its per-case budget instead of wedging the unit job. 'scripts/fuzz/corpus-replay.test.ts', + // Spawns the `pnpm repo-health` CLI as a subprocess to exercise its arg handling and the JSON + // it writes over the real tree (#1423). + 'scripts/repo-health/run.test.ts', ]; export default defineConfig({ @@ -47,6 +50,10 @@ export default defineConfig({ 'scripts/__tests__/help-conformance-error-recovery-coverage.test.ts', 'scripts/__tests__/help-conformance-sample-outputs.test.ts', 'scripts/__tests__/help-conformance-topic-coverage.test.ts', + // Repo-health snapshot model (#1423): pure aggregation over the analyzers, no + // subprocess or device work, so it runs in the fast lane. The CLI's subprocess test + // lives in the subprocess-stub project. + 'scripts/repo-health/model.test.ts', // Parses CI configuration only, so this action guard needs no device or subprocess lane. 'test/ci/upload-agent-device-artifacts.test.ts', 'test/skillgym/suites/local-cli-help-policy.test.ts',