diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b6b626f --- /dev/null +++ b/.gitattributes @@ -0,0 +1,17 @@ +# Normalize line endings. The pipeline runs on Linux; shell scripts MUST stay LF +# or they break on the runner. Keep repo text LF-normalized regardless of the OS +# a contributor commits from (silences the CRLF warnings on Windows checkouts). +* text=auto eol=lf + +# Explicitly LF for executable/interpreted files that are line-ending sensitive. +*.sh text eol=lf +*.py text eol=lf +*.js text eol=lf +*.yml text eol=lf +*.yaml text eol=lf +*.md text eol=lf + +# Treat these as binary (no EOL munging, no diffs). +*.pdf binary +*.png binary +*.jpg binary diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..696340b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,54 @@ +name: CI + +# Repo self-validation, distinct from the CodeWeave pipeline (codeweave.yml). +# Runs on hosted runners and does NOT touch the target repo, secrets, or Copilot. +# It exercises the deterministic tooling that decides every verdict, and lints the +# pipeline's own workflow YAML and shell scripts. + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + tools-tests: + name: Verdict-engine tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install dev dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + - name: Run tooling tests + run: python -m pytest tests -q + + lint-workflows: + name: Lint workflows & shell + runs-on: ubuntu-latest + # The pipeline's workflow shell is authored (and partly Copilot-generated) and + # already runs in production. We gate on genuine defects (shellcheck severity + # >= warning: unused vars, quoting bugs that change behavior, etc.), not on + # info/style advisories (SC2086 word-splitting hints, SC2129 redirect grouping, + # ...), which would be high-churn/low-value to rewrite blindly here. Raise the + # bar in a dedicated shell-hardening pass, not a docs PR. + env: + SHELLCHECK_OPTS: --severity=warning + steps: + - uses: actions/checkout@v4 + - name: actionlint (GitHub Actions workflow linter, incl. embedded shellcheck) + uses: raven-actions/actionlint@v2 + with: + fail-on-error: true + - name: shellcheck (standalone .sh scripts) + uses: ludeeus/action-shellcheck@master + with: + scandir: "." + severity: warning diff --git a/.github/workflows/phase-2-adr.yml b/.github/workflows/phase-2-adr.yml index 4d5c73f..2292764 100644 --- a/.github/workflows/phase-2-adr.yml +++ b/.github/workflows/phase-2-adr.yml @@ -249,7 +249,6 @@ jobs: - name: Generate navigation indexes id: build-indexes run: | - BOOK_INDEX_BUILT=0 ADR_INDEX_BUILT=0 node .github/scripts/generate-indexes.js --generate-adr-index 2>&1 | tee "proof/adr-index.md" || true @@ -263,7 +262,6 @@ jobs: git add book/BOOK-INDEX.md if ! git diff --cached --quiet; then git commit -m "Phase 2: update BOOK-INDEX.md" - BOOK_INDEX_BUILT=1 fi fi diff --git a/.github/workflows/phase-5-6-build-baseline.yml b/.github/workflows/phase-5-6-build-baseline.yml index 4511998..65e4da8 100644 --- a/.github/workflows/phase-5-6-build-baseline.yml +++ b/.github/workflows/phase-5-6-build-baseline.yml @@ -145,7 +145,6 @@ jobs: # Copilot AUTHORS integration-test/build-source.sh (read/write/edit only — # no shell needed); the workflow EXECUTES it directly so build output # streams into the job log and no tool permissions can block the build. - VENV_PY="$PWD/integration-test/${HARNESS_VENV_PYTHON:-.venv/bin/python}" BUILD_OK=false for ATTEMPT in $(seq 1 "${PHASE5_MAX_ITERATIONS}"); do if [ -f integration-test/build-error.md ]; then MODE=repair; else MODE=author; fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..9025e07 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,78 @@ +# Contributing to CodeWeave + +Thanks for your interest in CodeWeave. It is an experimental, research-grade +automation pipeline, so the contributions that help most are the ones that make it +more portable, more auditable, or more statistically rigorous. Please read this +guide before opening an issue or a pull request. + +## Ways to contribute + +The most valuable areas are portability, measurement science, and reporting real +behavior. On portability, CodeWeave is designed to run against any codebase by +reading the target's toolchain from a manifest, and driving it end to end against a +new kind of target (through the `integration-test/harness-manifest.json` seam) is +the highest-leverage work available. On measurement science, the verdict logic +lives in `integration-test/_tools/ab_compare.py`, and changes there should arrive +with a written rationale and, ideally, a worked example. Reporting pipeline +behavior is also genuinely useful: a phase that stalls, a gate that misfires, or a +verdict that looks wrong all make good issues, as do clarifications to `docs/`, the +`README.md`, or the executive summary. + +## Before you start + +CodeWeave runs as GitHub Actions workflows on a self-hosted runner rather than as a +local command-line tool. There is no local install-and-test loop; changes are +exercised by dispatching the workflows, as described in the +[Quickstart](README.md#quickstart). Smoke-test structural changes cheaply with a +dry run before a real run, because a dry run skips the Copilot invocations and the +prerequisite checks: + +```bash +gh workflow run codeweave.yml -f dry_run=true +``` + +The deterministic tooling under `integration-test/_tools/` is covered by tests you +can run locally. Install the development dependencies and run the suite: + +```bash +pip install -r requirements-dev.txt +python -m pytest +``` + +## Reporting an issue + +A good report names the phase involved and the workflow that ran it, states what +you expected against what actually happened, and includes the relevant `proof/` +artifacts. Those artifacts, the generate and validate logs, the session +transcripts, the gate diagnostics, and the measurement records, are the primary +evidence for any pipeline behavior, so please attach or paste the relevant files +rather than describing them from memory. Include the relevant parts of your +`.github/codeweave.config` and your `constraints/` files as well, with anything +sensitive redacted. Never include the values of the `COPILOT_TOKEN` or `PUSH_TOKEN` +secrets. + +## Pull requests + +Use a descriptive branch prefix that matches the existing history, such as `docs/`, +`pipeline/`, `config/`, or `work/`. Follow the repository's commit style: a subject +line of the form `scope: imperative summary`, then a body that explains why. Keep +one logical change per commit, and keep pull requests focused, so that a change to +prompt files under `work/` stays separate from a change to workflow logic under +`.github/workflows/`. + +One invariant matters above the rest. CodeWeave's whole premise is that generators +propose and an independent, deterministic pipeline disposes, so any change must +preserve that separation. An agent must never be able to certify its own phase as +complete, and the pipeline must remain the single owner of commits and verdicts. +Changes to the verdict logic in `ab_compare.py` or to the correctness gate should +explain how they affect the trustworthiness of a verdict, in terms of significance, +the noise floor, drift control, or the family-wise correction. + +## Code of conduct + +Be respectful and constructive. `[ADD CODE_OF_CONDUCT.md IF ADOPTING ONE]` + +## License + +By contributing, you agree that your contributions are licensed under the project's +GNU General Public License v3.0 or later. See [`LICENSE`](LICENSE). diff --git a/README.md b/README.md index 5d62c1d..0608c47 100644 --- a/README.md +++ b/README.md @@ -1,205 +1,248 @@ # CodeWeave -CodeWeave is an automation system that drives the GitHub Copilot CLI across an **8-phase pipeline** to analyse an external codebase and then improve it under measurement. It clones a target repository and runs Copilot through two stages: +**An autonomous pipeline that hunts energy and CO₂ hotspots in a codebase and ships the efficiency fixes it can prove real.** It works against a codebase of any language or stack, reading the target's toolchain from a manifest rather than assuming one. Every candidate change is measured for energy and carbon, cleared by a correctness gate, and accepted only when a statistical test says the improvement is genuine. The result is greener code that you do not have to take an agent's word for. -- **Documentation (Phases 1–4)** — generate an architecture book (+ PDF), Architecture Decision Records (+ PDF), a performance-measurement harness specification, and the runnable integration-test harness. Each phase is a generate→validate loop with file-based early exit: only the validator writes the completion marker, and the generator never self-certifies. -- **Measurement & optimization (Phases 5–8)** — build the target from source (ccache-backed), establish a statistical performance baseline, run autonomous optimization cycles each gated by a correctness suite (import/smoke/unit/op-suite/differential-fuzz) and an A/B verdict, then synthesise the results into a ranked report with per-optimization PR drafts. +[![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](LICENSE) +![Pipeline: GitHub Actions](https://img.shields.io/badge/pipeline-GitHub%20Actions-2088FF?logo=githubactions&logoColor=white) +![Status: Experimental](https://img.shields.io/badge/status-experimental-orange) +![Objective: energy + CO₂](https://img.shields.io/badge/objective-energy%20%2B%20CO%E2%82%82-2ea44f) -Every iteration's changes are committed and all run artifacts are saved to `proof/`. The pipeline runs as a set of **GitHub Actions workflows** (below). **Full per-phase documentation lives in [`docs/`](docs/index.md)** (the phase table below links into it). +> _CodeWeave exists to make software cost less energy. It drives the GitHub Copilot CLI across an eight-phase pipeline: it clones a target repository, documents it from the ground up, builds an energy- and latency-aware measurement harness, profiles the target for the paths that burn the most, and then runs autonomous optimization cycles. It measures energy and carbon (via CodeCarbon) on every change, and it accepts or rejects each change by experiment rather than by the judgement of the agent that wrote it._ -## Workflow: CodeWeave + -**Orchestrator:** `.github/workflows/codeweave.yml` — a thin dispatcher that wires the per-phase **reusable workflows** (`.github/workflows/phase-1-book.yml` … `phase-5-6-build-baseline.yml`) via `needs`/`if` (resume + skip logic), with `finalize` inline. Shared bootstrap (Node + Copilot CLI + `codeweave.config` + git identity) lives in the `.github/actions/codeweave-setup` composite action. Phases 7–8 are the separate auto-chaining workflows `phase-7-optimize.yml` / `phase-8-report.yml`; once Phase 6 produces a baseline, `codeweave.yml`'s `trigger-phase-7` job dispatches the first optimization cycle, so a single dispatch runs Phases 1–8 end-to-end. +### What it does -### Overview +CodeWeave begins by finding where the energy actually goes. It profiles the target to rank the hotspots that dominate its energy and carbon cost, and it measures energy and carbon on every candidate change through CodeCarbon, normalized per iteration so the figure reflects the code rather than the wall clock. Energy and CO₂ are the objective that the entire pipeline is pointed at. -Phases 1–6 run inside `codeweave.yml`; the Phase 7 optimization cycles (`phase-7-optimize.yml`) and the Phase 8 report (`phase-8-report.yml`) run as two dedicated, auto-chaining workflows. After Phase 6 produces a baseline, `codeweave.yml` automatically dispatches the first Phase 7 cycle, which self-chains through the remaining cycles and into Phase 8 — so one dispatch of `codeweave.yml` carries the run through all eight phases. The eight phases are summarised below — each row links to its full `docs/` page (gate, inputs, process, outputs). +It then fixes those hotspots without breaking the program. Each proposed change must pass a six-stage correctness gate, covering a build and import check, an integration smoke test, a targeted unit test, a run of the target's own broad test suite, an output diff, and a blocking differential fuzz against a golden output captured on the base build. No measurement run is spent on a change until it has been shown to be correct. -| # | Phase | What it does | -|---|-------|--------------| -| 1 | [Book generation](docs/phase-1-book-generation.md) | Iterative generate→validate passes produce an architecture book + PDF. | -| 2 | [ADR generation](docs/phase-2-adr-generation.md) | Produce Architecture Decision Records (pushed to the work branch) + PDF. | -| 3 | [Harness design](docs/phase-3-harness-design.md) | Design the three-document performance-measurement harness specification. | -| 4 | [Integration test generation](docs/phase-4-test-generation.md) | Generate the runnable harness (tests, `setup.sh`, `run.sh`, `_tools/`) behind a smoke-test gate. | -| 5 | [Source build](docs/phase-5-source-build.md) | Copilot authors `build-source.sh`; the pipeline builds the target from source (ccache-backed). | -| 6 | [Baseline execution](docs/phase-6-baseline-execution.md) | Deterministic baseline measurement (per-iteration latency + energy) → `baseline.json` + flamegraph. | -| 7 | [Optimization cycles](docs/phase-7-optimization-cycles.md) | Per-optimization generate→build→correctness-gate→A/B verdict; auto-chains one optimization per dispatch. | -| 8 | [Aggregate report](docs/phase-8-aggregate-report.md) | Synthesise all verdicts into a ranked report + per-optimization PR drafts. | +Finally, it ships only the improvements it can prove. A change is accepted when the improvement is statistically real, which requires Welch's t-test to reject the null hypothesis at `p < 0.05` and the effect size to clear a measured noise floor. A Holm-Bonferroni correction is then applied across the whole campaign so that running many experiments does not manufacture a false winner. Every log, session transcript, gate diagnostic, and measurement record is written to `proof/`, so nothing is self-certified and every decision can be audited after the fact. -> The optimization-cycle and Phase 8 workflows assume a self-hosted runner with the `src/` checkout, built `integration-test/.venv`, and warm ccache persisted from Phase 5/6 (`clean: false` checkout). Dispatch with `dry_run=true` first to smoke-test structure and the auto-chain. +> **The honest mechanism.** Energy and carbon are the goal, and they are measured and reported on every change. CodeCarbon's resolution, however, is too coarse to arbitrate a single optimization on its own. The accept-or-reject verdict therefore runs on per-iteration latency, which is the tightly resolvable proxy for the same hot path, with energy and carbon reported next to each verdict. Put simply, energy is the target and latency is the lever precise enough to act on. -### Configuration +### In action -Settings are stored in `.github/codeweave.config` and loaded at runtime: +You point CodeWeave at a repository and dispatch a single workflow: -```env -# External repository to process -EXTERNAL_REPO_NAME=example -EXTERNAL_REPO_URL=https://github.com/example/repo.git +```bash +# .github/codeweave.config: the whole run is configured here; nothing is hardcoded +EXTERNAL_REPO_NAME=your-project +EXTERNAL_REPO_URL=https://github.com/your-org/your-repo EXTERNAL_REPO_BRANCH=main -EXTERNAL_REPO_WORK_BRANCH=copilot-work - -# Iteration settings -PHASE1_MAX_ITERATIONS=10 -PHASE2_MAX_ITERATIONS=5 -PHASE3_MAX_ITERATIONS=5 -PHASE4_MAX_ITERATIONS=3 -PHASE6_BASELINE_RUNS=5 - -# Model schedule per phase: model:iterations,model:iterations,...,model_for_remaining -# The last entry without a count covers all remaining iterations. -PHASE1_MODEL_SCHEDULE=claude-sonnet-4.6:3,claude-haiku-4.5 -PHASE2_MODEL_SCHEDULE=claude-haiku-4.5 -PHASE3_MODEL_SCHEDULE=claude-opus-4.6:1,claude-sonnet-4.6 -PHASE4_MODEL_SCHEDULE=claude-opus-4.6:1,claude-sonnet-4.6 -PHASE6_MODEL=claude-sonnet-4.6 - -# Git commit author identity -GIT_USER_NAME=github-actions[bot] -GIT_USER_EMAIL=41898282+github-actions[bot]@users.noreply.github.com +PHASE7_MAX_OPTIMIZATIONS=5 # how many hotspots to attempt + +# one dispatch carries the run through all eight phases (Phase 7 and 8 auto-chain) +gh workflow run codeweave.yml -f dry_run=true # smoke-test the structure first +gh workflow run codeweave.yml # then the real run ``` -| Setting | Description | -|---|---| -| `EXTERNAL_REPO_NAME` | Name of the external repository (used for organizing output) | -| `EXTERNAL_REPO_URL` | HTTPS URL of the external Git repository | -| `EXTERNAL_REPO_BRANCH` | Branch to clone from the external repository | -| `EXTERNAL_REPO_WORK_BRANCH` | Branch name to create and work on (keeps original branch clean) | -| `PHASE1_MAX_ITERATIONS` | Maximum number of Phase 1 (book generation) iterations (1-99) | -| `PHASE2_MAX_ITERATIONS` | Maximum number of Phase 2 (ADR generation) iterations (1-99) | -| `PHASE3_MAX_ITERATIONS` | Maximum number of Phase 3 (performance measurement) iterations (1-99) | -| `PHASE4_MAX_ITERATIONS` | Maximum number of Phase 4 (integration test code generation) iterations (1-99) | -| `PHASE6_BASELINE_RUNS` | Number of measurement runs to collect in Phase 6 (default 5) | -| `PHASE1_MODEL_SCHEDULE` | Model schedule for Phase 1. Format: `model:count,...,model_for_remaining` (e.g. `claude-sonnet-4.6:3,claude-haiku-4.5`) | -| `PHASE2_MODEL_SCHEDULE` | Model schedule for Phase 2. A single model name is valid (e.g. `claude-haiku-4.5`) | -| `PHASE3_MODEL_SCHEDULE` | Model schedule for Phase 3. Format same as above (e.g. `claude-opus-4.6:1,claude-sonnet-4.6`) | -| `PHASE4_MODEL_SCHEDULE` | Model schedule for Phase 4. Format same as above (e.g. `claude-opus-4.6:1,claude-sonnet-4.6`) | -| `PHASE6_MODEL` | Model for Phase 6's single execution pass (no schedule needed, e.g. `claude-sonnet-4.6`) | -| `PHASE5_MODEL` / `PHASE5_MAX_ITERATIONS` | Phase 5 source-build model and repair-iteration cap (default 3) | -| `PHASE7_MAX_OPTIMIZATIONS` / `PHASE7_MAX_ITERATIONS` | Number of optimization points (default 5) and per-optimization repair-loop cap (default 3) | -| `PHASE7_MODEL_SCHEDULE` / `PHASE7_SELECTION_MODEL` | Model schedule for the optimization passes; model for the hotspot-selection pass | -| `PHASE7_FUZZ_REQUIRED` | 7f differential fuzz gate, blocking by default (`1`); set `0` only for an op that cannot be fuzzed | -| `PHASE7_BASELINE_RUNS` / `PHASE7_MICROBENCH_MIN_SECONDS` | A/B measurement runs per side (default 5) and microbenchmark min run-time per side (default 10) | -| `PHASE8_MODEL` | Model for the Phase 8 aggregate-report pass | -| `GIT_USER_NAME` | Git commit author name | -| `GIT_USER_EMAIL` | Git commit author email | - -### Trigger - -Manually triggered via `workflow_dispatch`. Optional inputs: `start_from_phase` (choices `1`–`6`) resumes from a specific phase (defaults to `1`; later phases still check that prerequisite artifacts exist); `dry_run` (boolean) skips Copilot invocations and overrides prerequisite checks to smoke-test workflow structure. - -### Permissions - -- `contents: write` — required to commit and push changes back to the branch. +Each optimization cycle ends in one of five recorded terminal states. Only the first is worth turning into a pull request: + +| State | Measured? | Meaning | +|-------|-----------|---------| +| **KEEP** | yes | A genuine improvement on the point's primary signal, so Phase 8 drafts a pull request | +| **INVESTIGATE** | yes | Measured but ambiguous, so the report recommends a manual re-measurement rather than a pull request | +| **REVERT** | yes | A regression, or no detectable effect, so the change is not submitted (the conservative default) | +| **FAILED** | no | The change was incorrect and failed the gate, so its branch is never pushed and it is never counted as a regression | +| **INCOMPLETE** | no | Built and gated, but the statistics could not be trusted, so it is excluded and flagged for a re-run | + +**Start here:** [Quickstart](#quickstart), the [full per-phase docs](docs/index.md), the [executive summary](executive-summary.md), and [how it works](#how-it-works). + +--- + +## Why CodeWeave exists + +Software has an energy bill, and most of it hides in a small number of hot paths. Shrinking that bill by hand is slow work: you have to find the paths that matter, propose a change, confirm the change is still correct, and then prove that it actually saved energy rather than merely appearing to. The last step is the hardest, because energy and timing measurements are noisy and a plausible-looking win is often just thermal drift or a warm cache. CodeWeave was built to carry that whole loop autonomously, from finding the hotspot to proving the fix, on any codebase you point it at. + +Coding agents make the first part easy and the last part dangerous. Ask one to speed up a hot path and you get a confident diff and a confident claim, for example that it is roughly ten percent faster. Verifying that claim is the real work. Is the change correct on the edge cases? Is the improvement real, or is it noise? Would it survive a second measurement? Multiply that by dozens of candidates and the cost of verification dwarfs the cost of generation, which is why agent-proposed optimizations so rarely reach production. The bottleneck was never generating ideas; it was trusting them. + +CodeWeave is built on a single conviction: the agent that writes a change must never be the thing that certifies it. Generators propose, and a deterministic, statistically disciplined pipeline disposes. The agent edits source and authors specifications, while the pipeline builds, fuzzes, measures, and rules on the result. The verdict tool `ab_compare.py` is the single source of truth for every number, and the signal that a phase is complete is always a file written by an independent check rather than a claim made by the generator. + +The long-term aim is greener software produced without a human babysitting every experiment. Energy and carbon are measured directly and treated as the ultimate objective, while per-iteration latency serves as the lever precise enough to resolve at the scale of one change. What emerges is an autonomous loop that can walk into an unfamiliar codebase, understand it, improve it under measurement, and leave behind an audit trail a reviewer can actually check. + +A second goal follows from the way that loop is built. To optimize a system safely, CodeWeave has to understand it first, and the understanding it produces does not evaporate once the run ends. The architecture book, the per-area Architecture Decision Records committed next to the code they describe, and the grounded measurement harness are durable artifacts that outlive the optimization campaign. Software that had drifted into being effectively unmaintainable, opaque to the people who own it and risky to touch, comes back documented, mapped, and safe to change again. In this sense CodeWeave fights technical debt as it works: the same context it builds to find energy wins is also the context a team needs to maintain the system for years afterward. + +--- + +## What CodeWeave takes seriously + +Three commitments shape the design more than any single feature, and each is deliberate rather than incidental. + +**The statistical proof is embedded on purpose, not bolted on afterward.** A result cannot be reported unless it has survived the statistics, because the statistics are wired into the acceptance path itself. Every comparison runs Welch's t-test and must also clear a Minimum Detectable Effect, a noise floor computed from the baseline's own run-to-run variance, so that a difference which is significant but trivially small cannot pass. Because the workload is a fixed-time loop that pins the wall clock, every verdict uses per-iteration metrics rather than elapsed time. The base build is re-measured in the same cycle as each variant, back to back, so that slow machine drift cancels instead of masquerading as a result. And across a long campaign of many experiments, a Holm-Bonferroni correction demotes any winner that does not survive family-wise control. The point of all of this is that trust is designed in: the pipeline is built so a finding you did not statistically earn simply cannot reach the report. + +**The quality of the context it builds is what makes a change worth measuring.** Before it edits a single line, CodeWeave reads the system into an architecture book and a set of Architecture Decision Records, capturing subsystems, ownership boundaries, runtime behavior, and the paths that are sensitive to performance. Every later claim, including the choice of what to optimize and how, is grounded in those documents rather than in a keyhole view of one file. A generate-then-validate loop drives this understanding forward: a generation pass writes, an independent validation pass checks the result against an explicit checklist and records precise gaps, and the next generation pass closes those gaps before extending coverage, so the understanding deepens monotonically instead of churning. Good optimizations begin with a genuine model of the system, and building that model well is treated as part of the engineering, not a preamble to it. + +**The integration tests are reverse-engineered from the real system, not assumed.** The measurement harness and the correctness gate are derived from how the target actually behaves, on the actual toolchain the target uses, discovered and pinned rather than guessed. The differential fuzz compares a changed build against a golden output captured from the base build of the same source tree, so that "correct" means "indistinguishable from the real system's own behavior on inputs that matter." Finding the right tests, the ones that truly exercise the hot path and would catch a regression there, is treated as a first-class problem: the harness specification is authored, validated, and only then turned into runnable code behind a smoke-test gate, so the tests that guard every optimization are ones that reflect the system as it really runs. + +--- + +## Who it is for and what you would use it for + +CodeWeave is aimed at the people responsible for large, compute-heavy systems where reducing the energy and time a program spends is a real and recurring job. It is designed to be independent of language and stack, reading the target's build and test toolchain from a manifest, so the same pipeline can be pointed at different kinds of codebase. + +The most direct use is to find and prove efficiency wins in a hot codebase. You run the full pipeline and receive a ranked set of pull-request-ready branches, each carrying a statistical verdict and a per-iteration energy and carbon figure, together with honest REVERT and INVESTIGATE records for the ideas that did not pan out. + +A second use is to recover a codebase that has become hard to maintain. The first four phases on their own produce an architecture book with a PDF, per-area Architecture Decision Records committed alongside the code they describe, and a runnable measurement harness, all grounded in the actual source rather than in generic assumptions about it. For software that had drifted into being effectively unmaintainable, this is a direct way to pay down technical debt: the system comes back documented, mapped, and safe to change, whether or not you go on to run the optimization phases. + +A third use is to vet an agent's optimization before you trust it. The correctness gate combined with drift-controlled A/B measurement is exactly the review you would otherwise perform by hand for every candidate, except that it runs automatically and records its reasoning. + +A fourth use is to stand up repeatable energy and performance measurement as reusable infrastructure. The measurement phases give you a fixed-time hot-loop harness, a statistical baseline with a computed noise floor, and a hotspot profile, all of which are useful independently of the optimization stage. + +--- + +## Quickstart + +CodeWeave runs as GitHub Actions workflows rather than as a local command-line tool. The later phases build and measure a native target, so they require a machine that persists state between phases. + +### Prerequisites + +You will need a self-hosted GitHub Actions runner (labelled `self-hosted, Linux, X64`) that keeps the built `src/` tree, the editable build environment, and a warm build cache between phases. + +You will need access to the GitHub Copilot CLI, which the workflow installs automatically through `npm i -g @github/copilot`. + +You will need two fine-grained personal access tokens stored as repository secrets. `COPILOT_TOKEN` authenticates the Copilot CLI and needs only the Copilot user requests Read account permission, with no repository permissions. `PUSH_TOKEN` needs Contents Read and write on the target repository alone, because Phase 2 pushes Architecture Decision Records and Phase 7 pushes optimization branches. + +You will need the target's own build and test toolchain available on the runner. CodeWeave does not assume a particular language or package manager; you pin the toolchain your target needs in [`constraints/project.md`](constraints/project.md) and the optional [`constraints/harness.md`](constraints/harness.md), and the pipeline reads it from the generated manifest rather than hardcoding it. ### Steps -1. **Checkout** — Checks out this repository with full history and credential persistence. -2. **Setup Node.js 22** — Required to install the Copilot CLI. -3. **Install Copilot CLI** — Installs `@github/copilot` globally via npm. -4. **Load configuration** — Sources `.github/codeweave.config`, extracts the first repo's configuration, and exports settings as environment variables. -5. **Configure git identity** — Sets the git commit author from `codeweave.config` variables. -6. **Clone external repository** — Clones the specified branch from the external repo (shallow, single branch) into `src`, then creates and checks out the work branch. The `src/` directory is excluded from git tracking via `.git/info/exclude`. -7. **Run the CodeWeave pipeline (Phases 1–6)** — The `codeweave.yml` orchestrator calls a per-phase reusable workflow for each phase: Phases 1–4 are generate+validate loops with file-based early exit, Phase 5 builds the target from source, and Phase 6 collects the deterministic baseline. Each phase's gate, inputs, and outputs are documented in [`docs/`](docs/index.md) (see the phase table above). It finally writes `proof/final-status.md` summarising all phases. -8. **Push** — Pushes all outer-repo commits back to the triggering branch. - -> **Phases 7–8 are separate workflows, dispatched automatically.** When Phase 6 produces a baseline, `codeweave.yml`'s `trigger-phase-7` job dispatches the first optimization cycle (Phase 7, `phase-7-optimize.yml`); each cycle self-chains to the next, and the last chains into the aggregate report (Phase 8, `phase-8-report.yml`) — one optimization per dispatch. No manual step is needed; to start them by hand instead, run `gh workflow run phase-7-optimize.yml -f optimization_index=1`. See the Overview above for what each does. - -### Proof Artifacts - -Each iteration produces output files organized in `proof/`: - -| File | Contents | -|---|---| -| `1-book-generation-N.md` | Generate pass output log for Phase 1 iteration N | -| `1-book-generation-session-N.md` | Generate pass session transcript for Phase 1 iteration N | -| `1-book-validation-N.md` | Validate pass output log for Phase 1 iteration N | -| `1-book-validation-session-N.md` | Validate pass session transcript for Phase 1 iteration N | -| `1-book-validation-report-N.md` | Copy of `book/BOOK-VALIDATION.md` after Phase 1 iteration N | -| `2-adrs-generation-N.md` | Generate pass output log for Phase 2 iteration N | -| `2-adrs-generation-session-N.md` | Generate pass session transcript for Phase 2 iteration N | -| `2-adrs-validation-N.md` | Validate pass output log for Phase 2 iteration N | -| `2-adrs-validation-session-N.md` | Validate pass session transcript for Phase 2 iteration N | -| `2-adrs-validation-report-N.md` | Copy of `src/ADR-VALIDATION.md` after Phase 2 iteration N | -| `3-harness-generation-N.md` | Generate pass output log for Phase 3 iteration N | -| `3-harness-generation-session-N.md` | Generate pass session transcript for Phase 3 iteration N | -| `3-harness-validation-N.md` | Validate pass output log for Phase 3 iteration N | -| `3-harness-validation-session-N.md` | Validate pass session transcript for Phase 3 iteration N | -| `3-harness-validation-report-N.md` | Copy of `integration-test/HARNESS-VALIDATION.md` after Phase 3 iteration N | -| `4-tests-generation-N.md` | Generate pass output log for Phase 4 iteration N | -| `4-tests-generation-session-N.md` | Generate pass session transcript for Phase 4 iteration N | -| `4-tests-validation-N.md` | Validate pass output log for Phase 4 iteration N | -| `4-tests-validation-session-N.md` | Validate pass session transcript for Phase 4 iteration N | -| `4-tests-validation-report-N.md` | Copy of `integration-test/TESTS-VALIDATION.md` after Phase 4 iteration N | -| `5-build-source-N.md` / `5-build-source-session-N.md` | Phase 5 build authoring pass log / transcript (attempt N) | -| `5-build-output-N.log` | Phase 5 build execution output (attempt N) | -| `6-baseline-run-N.log` | Phase 6 baseline measurement run N output | -| `6-trace.log` | Phase 6 tracing-pass output | -| `6-repair-N.md` / `6-repair-session-N.md` | Phase 6 repair pass N log / transcript (if any) | -| `final-status.md` | Summary of all phases: PDF generation, ADR coverage, strategy status, test code status, and baseline execution status | - -Git commit history provides the natural diff trail between iterations. - -### Required Secrets - -| Secret | Purpose | -|---|---| -| `COPILOT_TOKEN` | Fine-grained PAT used to authenticate the Copilot CLI (`GH_TOKEN` in the step environment). Needs the **Copilot user requests: Read** user permission (Account permissions — no repository permissions required). | -| `PUSH_TOKEN` | Fine-grained PAT used to push to the target repository — Phase 2 pushes ADR changes to the work branch, and Phase 7 pushes each gate-passing optimization branch. Needs the **Contents: Read and write** repository permission on the target repository only. | - -## Work Definition and Constraints - -The workflow improves code based on tasks and constraints you define: - -- **`work/1-generate-book.md`** — Phase 1 generation prompt. Copilot writes/expands chapters each iteration without self-certifying completion. -- **`work/1-validate-book.md`** — Phase 1 validation prompt. Run after each generation pass; enforces all quality gates and exclusively writes `book/manuscript-complete.md` on PASS. -- **`work/2-generate-adrs.md`** — Phase 2 generation prompt. Directs Copilot to generate per-folder `ADR.md` files in `./src` using the book markdown sources as reference. -- **`work/2-validate-adrs.md`** — Phase 2 validation prompt. Run after each ADR generation pass; validates coverage against the scope map and exclusively writes `src/adrs-complete.md` on PASS. -- **`work/3-generate-harness.md`** — Phase 3 generation prompt. Directs Copilot to produce a three-tier performance measurement document set: `integration-test/AGENTS.md` (permanent harness operating rules), `integration-test/SOURCE-UNDER-INVESTIGATION.md` (target-specific profile derived from the book, ADRs, and constraints), and `integration-test/WORK.md` (execution agent checklist). -- **`work/3-validate-harness.md`** — Phase 3 validation prompt. Run after each harness design pass; validates all three harness documents and exclusively writes `integration-test/harness-complete.md` on PASS. -- **`work/4-generate-tests.md`** — Phase 4 generation prompt. Directs Copilot to read `integration-test/AGENTS.md`, `SOURCE-UNDER-INVESTIGATION.md`, and `WORK.md` for context, check `./src/` for existing build documentation, and generate all test files, `setup.sh`, `run.sh`, and `_tools/` helpers. Instructs the generator to derive all install commands and environment variables from `SOURCE-UNDER-INVESTIGATION.md` rather than from general knowledge. -- **`work/4-validate-tests.md`** — Phase 4 validation prompt. Run after each code generation pass; validates all generated files against `SOURCE-UNDER-INVESTIGATION.md`, reads `integration-test/smoke-test-report.md` as part of Check 1, and exclusively writes `integration-test/tests-complete.md` on PASS. -- **`work/6-repair-tests.md`** — Phase 6 repair agent prompt. Invoked only when a probe run of `run.sh` fails; fixes the specific runtime error in `tests/` or `_tools/`. Baseline collection itself (build via `setup.sh`, `PHASE6_BASELINE_RUNS` runs of `run.sh`, `_tools/ab_compare.py --mode baseline`) is deterministic pipeline logic, not a prompt. -- **`work/5-build-source.md`** — Phase 5 prompt. Copilot *authors* `integration-test/build-source.sh`; the pipeline *executes* it to build the target from source (the agent does not run the build itself). -- **`work/7-select-hotspots.md`** — Phase 7 hotspot-selection prompt (index 1 only). Reads `profiler-summary.md` + the ADR index and writes `integration-test/optimization-plan.md` (ranked target ops + per-point measurement path). -- **`work/7-generate-optimization.md`** — Phase 7 optimization prompt. Implements one source change in `./src` and authors the gate-7f differential-fuzz spec; `work/fuzz-examples/opt{1,2}_fuzz.py` are worked templates it references. -- **`work/8-aggregate-report.md`** — Phase 8 prompt. Synthesises every Phase 7 verdict into a ranked report + per-optimization PR drafts — an authoring task: it does not build, run, or measure (the verdicts are already computed by `ab_compare.py`). -- **`constraints/project.md`** — Repository-specific constraints and requirements that must be respected (e.g., framework versions, architecture decisions, tech stack limitations). The default includes a sample constraint; **when forking, replace it with your actual constraints**. -- **`constraints/harness.md`** — *(optional)* Target execution constraints for Phase 3: hardware requirements, scope limitations, time budgets, and isolation rules. Phase 3 incorporates this content into `integration-test/SOURCE-UNDER-INVESTIGATION.md`. -- **`constraints/harness-context.md`** — *(optional)* Domain context for Phase 3 scenario and observability design. Provides target-specific knowledge about what to instrument, what representative scenarios look like, and known performance-sensitive paths. - - -## Key Files to Know - -| File | Purpose | -|---|---| -| `.github/workflows/codeweave.yml` | Orchestrator (glue): dispatch + `needs`/`if` calling the per-phase reusable workflows; `finalize` inline | -| `.github/workflows/phase-*.yml` | Per-phase reusable workflows (1–6) + `phase-7-optimize.yml` / `phase-8-report.yml` | -| `.github/actions/codeweave-setup/action.yml` | Composite action: shared bootstrap (Node + Copilot CLI + `codeweave.config` + git identity) | -| `.github/codeweave.config` | Runtime configuration (external repo, branch, iterations, git identity) | -| `.github/scripts/generate-indexes.js` | Generates `book/BOOK-INDEX.md` (Phase 1) and `src/ADR-INDEX.md` (Phase 2) | -| `work/1-generate-book.md` | Phase 1 generation prompt (no self-certification) | -| `work/1-validate-book.md` | Phase 1 validation prompt (exclusively owns `manuscript-complete.md`) | -| `work/2-generate-adrs.md` | Phase 2 generation prompt (no self-certification) | -| `work/2-validate-adrs.md` | Phase 2 validation prompt (exclusively owns `adrs-complete.md`) | -| `work/3-generate-harness.md` | Phase 3 generation prompt (no self-certification) | -| `work/3-validate-harness.md` | Phase 3 validation prompt (exclusively owns `harness-complete.md`) | -| `work/4-generate-tests.md` | Phase 4 generation prompt (no self-certification) | -| `work/4-validate-tests.md` | Phase 4 validation prompt (exclusively owns `tests-complete.md`) | -| `work/6-repair-tests.md` | Phase 6 repair agent prompt — fixes runtime errors in `tests/`/`_tools/` (baseline collection is deterministic pipeline logic) | -| `work/5-build-source.md` | Phase 5 prompt — authors `build-source.sh` (pipeline executes it) | -| `work/7-select-hotspots.md` / `work/7-generate-optimization.md` | Phase 7 hotspot selection (→ `optimization-plan.md`) and per-optimization implementation + 7f fuzz spec | -| `work/8-aggregate-report.md` | Phase 8 prompt — ranked aggregate report + PR drafts | -| `integration-test/harness-manifest.json` | Machine-readable toolchain manifest (venv layout, smoke-check commands, profiler enable-env, hotspot-report path, incremental-build recipe, and op-suite/import-op gate commands) the deterministic pipeline reads instead of hardcoding Python/pytest/venv. Emitted by Phase 4 from `SOURCE-UNDER-INVESTIGATION.md §08`; a target whose toolchain matches the built-in defaults emits a no-op manifest. | -| `integration-test/_tools/ab_compare.py` | A/B + baseline statistics (v2.2: Welch's t-test + MDE floor, directional, measurement-path-aware verdict; `--mode family` Holm-Bonferroni) | -| `integration-test/_tools/op_microbench.py` / `diff_fuzz.py` | Per-op microbenchmark (framework-native benchmark timer) and the 7f differential-fuzz gate driver | -| `constraints/project.md` | Repository-specific constraints and requirements (customize for your fork) | -| `constraints/harness.md` | *(optional)* Target execution constraints (hardware, scope, time budgets) consumed by Phase 3 | -| `constraints/harness-context.md` | *(optional)* Domain context for Phase 3 scenario and observability design | -| `proof/` | Output artifacts directory (auto-created): run logs, session transcripts, final status | -| `.git/info/exclude` | Excludes `src/` from git tracking (auto-configured by workflow) | +Fork this repository. Configure the run in [`.github/codeweave.config`](.github/codeweave.config), which holds the target repository URL and branch, the per-phase iteration caps, and the per-phase model schedules; none of this is hardcoded in the workflow logic. Replace the sample constraint in [`constraints/project.md`](constraints/project.md) with your target's real constraints, such as toolchain versions, build environment variables, and scope limits, and note that `constraints/harness.md` and `constraints/harness-context.md` are optional target-specific inputs for the harness-design phase. Add the two secrets. Then smoke-test the structure before spending model time: + +```bash +gh workflow run codeweave.yml -f dry_run=true +``` + +The dry run skips the Copilot invocations and the prerequisite checks, so you can confirm that the eight phases are wired correctly and that the Phase 7 auto-chain fires. When that looks right, start the real run: + +```bash +gh workflow run codeweave.yml +``` + +### What you should see + +A single dispatch cascades through the whole system. The first six phases document and baseline the target, after which `codeweave.yml` automatically dispatches Phase 7. Phase 7 optimizes one hotspot, gates it, measures a paired A/B experiment, records a verdict with its energy and carbon figures, and then dispatches the next cycle itself. The final cycle chains into Phase 8, which writes a ranked report and drafts a pull request for each result worth submitting. You dispatched once, and the pipeline ran an entire measurement campaign and handed back reviewable branches. + +You can resume from any phase with `-f start_from_phase=N` for N from 1 to 6. The gates, inputs, and outputs of every phase are documented in [`docs/index.md`](docs/index.md). + +--- + +## Core capabilities + +### Understand an unfamiliar codebase (Phases 1 to 4) + +The documentation phases produce an architecture book with a PDF rendered through Pandoc and Typst, a set of per-area Architecture Decision Records committed into the target's own source tree, and a runnable measurement harness, all grounded in the real source. This matters because every later measurement decision traces back to a documented claim, so the harness reflects the actual system rather than the model's priors. + +The mechanism that keeps this honest is a generate-then-validate loop. A generate pass edits the artifacts, and a separate validate pass checks them against an explicit checklist and is the only step permitted to write the phase's completion marker. The validator writes a list of required actions, the next generator clears that list before doing anything else, and durable state files record the high-water mark, so the artifacts deepen across iterations instead of churning. The main cost is model budget: each phase runs several passes, and the iteration caps are set per phase in the configuration. + +### Measure on a trustworthy substrate (Phases 5 and 6) + +Phase 5 builds the target from source with a warm build cache, and Phase 6 establishes a statistical baseline together with a hotspot profile. Building from source means that later A/B comparisons compare two builds of the same source tree, and the baseline computes a noise floor, expressed as a coefficient of variation and a Minimum Detectable Effect, that every later verdict is held to. + +A deliberate design choice sits underneath all of this. The workload is a fixed-time hot loop, so wall-clock time carries no signal: a faster build simply completes more iterations in the same budget. Every verdict therefore uses per-iteration metrics, such as median iteration latency, iteration count, and joules per iteration, and never raw wall clock. + +### Optimize behind a correctness-first gate (Phase 7) + +Phase 7 handles one optimization per dispatch. It generates the change, rebuilds incrementally with a verification step so that a change which compiles nothing cannot slip through, runs the six-stage correctness gate, and only then measures a paired A/B experiment. The change must be provably correct, including a blocking differential fuzz against a golden captured on the base build, before any measurement run is spent on it. The base side is re-measured in every cycle, back to back with the variant, so that slow machine drift over a long run cancels out instead of polluting the comparison. + +There are two measurement paths, chosen per optimization from the baseline. A point whose end-to-end effect falls below the noise floor by construction is judged on a per-operation microbenchmark using the target's native benchmark timer, while other points are judged on end-to-end latency. The verdict logic is aware of which path applies and tests for regressions first. Energy and carbon are recorded on every change, but they inform rather than gate the verdict, because their resolution is too coarse to arbitrate a single optimization; a latency win that appears to cost energy is kept and flagged rather than silently discarded. + +### Report with campaign-wide rigor (Phase 8) + +Phase 8 reads every per-cycle verdict, applies the family-wise correction, ranks the results, summarizes any environment drift, and drafts a pull request for each result worth submitting. Running many cycles at a per-cycle significance of 0.05 inflates the chance of at least one false winner, so the Holm-Bonferroni correction across all cycles demotes any KEEP that does not survive the campaign-wide test. Phase 8 drafts the pull requests; opening them remains a deliberate human step. An INVESTIGATE result is a classification rather than an action: the system surfaces the ambiguous candidate and records why it is ambiguous, but it never re-measures or investigates on its own. + +--- + +## How it works + +A single `workflow_dispatch` runs eight gated phases. The first six run inside `codeweave.yml`, while Phases 7 and 8 are dedicated auto-chaining workflows: Phase 7 handles one optimization per dispatch and triggers the next, and the last chains into Phase 8. + +```mermaid +flowchart TD + start(["workflow_dispatch"]) --> clone["Clone target into src/, create work branch"] + clone --> P1 + subgraph gen["Phases 1 to 4: generate then validate loops"] + P1["1. Book"] --> P2["2. ADRs"] --> P3["3. Harness design"] --> P4["4. Integration tests"] + end + P4 --> P5["5. Build from source"] + subgraph meas["Phases 5 to 8: measure and optimize"] + P5 --> P6["6. Statistical baseline to baseline.json"] + P6 --> P7["7. Optimize one point, gate, paired A/B, verdict"] + P7 --> more{"more points?"} + more -- "yes, auto-chain N+1" --> P7 + more -- no --> P8["8. Aggregate report and PR drafts"] + end + P8 --> done(["ranked report and PR-worthy branches"]) +``` + +The generate-then-validate loop is the structural backbone of the first four phases. + +```mermaid +flowchart TD + enter(["enter phase"]) --> del["Delete completion marker"] + del --> genp["Generate pass: Copilot edits artifacts"] + genp --> smoke{"Phase 4 only: smoke test?"} + smoke -- fail --> report["Write findings, skip validator"] --> iter + smoke -- "pass or N/A" --> val["Validate pass: writes marker on PASS"] + val --> marker{"marker present?"} + marker -- yes --> exitok(["early exit"]) + marker -- no --> iter{"iterations left?"} + iter -- yes --> del + iter -- no --> stopmax(["stop, max iterations"]) +``` + +Three operating rules make the loop dependable. The pipeline owns every commit: Copilot runs non-interactively and is denied git in every phase except the Phase 7 optimization agent, which works on its own branch and reviews its own diff. Completion markers are deleted before each generate pass, so a stale marker can never short-circuit the next cycle. And the target's toolchain lives in a manifest rather than in the pipeline code: Phase 4 emits `integration-test/harness-manifest.json`, describing the build environment layout, smoke checks, profiler enable-environment, hotspot-report path, and gate commands, so the deterministic pipeline reads the toolchain instead of assuming a particular language or test runner. That manifest is the seam that lets the same engine run against different codebases. + +The full detail, including every gate, input, and output per phase, lives in [`docs/`](docs/index.md) and in the [executive summary](executive-summary.md). + +--- + +## How it is different + +CodeWeave is neither a coding assistant nor a benchmark runner. It is the pipeline between them that turns an agent's efficiency claims into something you can trust. The table below compares it fairly with the two obvious alternatives, and the point is one of discipline rather than a criticism of coding agents; CodeWeave uses one, the GitHub Copilot CLI, as its generator. What differs is what happens to a change after it is written. + +| | Ask an agent directly | Hand-roll a benchmark and review | CodeWeave | +|---|---|---|---| +| Energy and carbon measured | No | Rarely | On every change, per iteration | +| Correctness check before measuring | You do it, per change | You do it | Automated six-stage gate, including a differential fuzz | +| Improvement versus noise | The agent's word | Manual statistics, if any | Welch's t-test and a measured noise floor | +| Machine drift over a long run | Ignored | Manual re-runs | Contemporaneous A/B, base re-measured each cycle | +| False positives across many changes | Unaddressed | Rarely corrected | Holm-Bonferroni across the campaign | +| Output | A diff and a claim | A number you produced | Ranked, gated branches with a full `proof/` trail | +| Self-certification | The agent says it is done | Not applicable | Only an independent validator writes "done" | + +--- + +## Project status + +CodeWeave is experimental. It is a working, end-to-end pipeline that has been built and iterated against a large real-world codebase, but it has not yet been exercised across a range of targets and it has not been published with reproducible headline results. + +What works today is the full eight-phase run: the generate-then-validate documentation loops, the source build, the statistical baseline, the correctness-gated optimization cycles with paired A/B verdicts, and the aggregate report with pull-request drafts. + +What is now tested is the verdict engine itself. The code that decides every KEEP, INVESTIGATE, and REVERT, `integration-test/_tools/ab_compare.py`, is covered by a unit and integration suite exercising the decision table, the noise-floor gate, the Holm-Bonferroni correction, and all three command-line modes. The continuous integration workflow at [`.github/workflows/ci.yml`](.github/workflows/ci.yml) runs that suite on every push and pull request and lints the pipeline's own workflow YAML and shell scripts. + +What remains experimental is portability across a wide range of targets. The manifest seam is designed to make the engine language- and stack-agnostic, but only a limited set of toolchains has been driven end to end so far. `[VERIFY portability on additional targets]` + +Some behavior is manual by design. Phase 8 drafts pull requests but does not open them, and INVESTIGATE results are surfaced rather than acted on. + +Several limitations are worth stating plainly. The pipeline requires a self-hosted, persistent runner, because the later phases reuse the build and the build environment in place and there is no ephemeral-runner path. It requires GitHub Copilot CLI access and two fine-grained tokens. Energy and carbon are measured and reported but do not gate individual verdicts, because their per-change resolution is too coarse. The model and CI cost scales with the iteration caps and the number of optimizations. And there are no published benchmark results yet. `[ADD BENCHMARK: headline energy and latency results from a real run]` + +> **Maturity note.** Treat CodeWeave as a research-grade automation harness. Review every drafted pull request and read the `proof/` trail before shipping anything it produces. + +--- + +## Documentation, community, and trust + +The per-phase reference lives in [`docs/index.md`](docs/index.md), and the design rationale is in [`executive-summary.md`](executive-summary.md). Worked examples of the prompt and constraint files live under [`work/`](work) and [`constraints/`](constraints), and a sample run's artifacts appear in `proof/`, which is created automatically. `[ADD LINK TO A PUBLISHED EXAMPLE RUN]` + +The near-term direction is set out in [`ROADMAP.md`](ROADMAP.md), whose current focus is publishing a real run and verifying portability across additional targets through the manifest seam. Contributions are welcome; please read [`CONTRIBUTING.md`](CONTRIBUTING.md) first, and include the relevant `proof/` artifacts when you report pipeline behavior. For support, open a [GitHub issue](../../issues). `[ADD DISCUSSIONS LINK IF ENABLED]` Because the pipeline handles two access tokens and pushes branches to a target repository, please scope the tokens minimally as described under [Prerequisites](#quickstart) and report any vulnerability through [`SECURITY.md`](SECURITY.md). ## License -Copyright (C) 2026 Hightech ICT B.V. -This project is licensed under the GNU General Public License v3.0 or later. See the LICENSE file for details. +Copyright © 2026 Hightech ICT B.V. + +Licensed under the GNU General Public License v3.0 or later. See [`LICENSE`](LICENSE). diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..e7aa20f --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,51 @@ +# CodeWeave Roadmap + +CodeWeave is experimental. It runs end to end against a large real-world codebase, +but it has not yet been exercised across a range of targets and it has no published +headline results. This roadmap describes the direction rather than a dated +commitment, and priorities may change. + +If you have an opinion on ordering, or want to pick something up, open an issue or +read [`CONTRIBUTING.md`](CONTRIBUTING.md). + +## Now: proving the core + +The immediate focus is turning "it works against one codebase" into "it demonstrably +works, and you can see the evidence." That means publishing a real run, with a +linked `proof/` trail and a Phase 8 report that includes at least one KEEP verdict +alongside its statistics and its energy and carbon figures, which fills the benchmark +and example-run gaps flagged in the README. It also means producing a demo asset: a +terminal recording, or an annotated report, showing a cycle move from correctness +gate to paired A/B measurement to verdict. + +## Next: portability across many targets + +CodeWeave is designed to be independent of language and stack. The seam that makes +this possible already exists, because `integration-test/harness-manifest.json` lets +the deterministic pipeline read a target's toolchain instead of assuming a +particular build system, test runner, or benchmark timer. The work here is to drive +additional and materially different targets end to end through that seam, and to +document what a new target requires. A related goal is to reduce the runner +assumptions, exploring paths that do not require a single persistent self-hosted +runner to carry state across the measurement phases. + +## Later: sharper science and less manual glue + +Several improvements would deepen the system once the core is proven. An INVESTIGATE +result is today a classification rather than an action, since the pipeline surfaces +an ambiguous candidate and a reason but never re-measures; a guarded automatic +re-measurement path is a natural extension. Phase 8 currently drafts pull requests +but leaves opening them as a human step, so an opt-in flag to open the drafts +directly is a candidate. Energy is measured but does not gate, because its +per-change resolution is too coarse to arbitrate a single optimization, and finding +a measurement approach precise enough to make energy a first-class gating signal +would align the mechanism fully with the project's stated goal. Finally, additional +verdict models and configurable multiple-comparison corrections beyond +Holm-Bonferroni would give users more control over the statistical treatment. + +## Non-goals for now + +CodeWeave is not trying to become a general-purpose local coding assistant. It is a +measurement and verification pipeline that uses a coding agent, rather than a +replacement for one. It also does not auto-merge changes: a human reviews every +drafted pull request and the `proof/` trail before anything ships. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..8066cf1 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,70 @@ +# Security Policy + +CodeWeave is an automation pipeline that authenticates as a GitHub Copilot user, +holds two fine-grained Personal Access Tokens, and pushes branches to a target +repository. Its security posture matters. This document explains how to report a +vulnerability and how the pipeline handles sensitive credentials. + +## Reporting a vulnerability + +**Please do not open a public issue for security vulnerabilities.** + +Preferred channel: use GitHub's **private vulnerability reporting** on this +repository (the *Security* tab → *Report a vulnerability*). This keeps the report +confidential until a fix is available. + +Alternatively, email `[ADD SECURITY CONTACT EMAIL]`. + +When reporting, please include: + +- A description of the issue and its impact. +- Steps to reproduce, or a proof of concept. +- The affected component (a workflow, a `work/` prompt, `ab_compare.py`, etc.). +- Any suggested remediation. + +**Please redact secrets** from anything you attach, and never include the values of +`COPILOT_TOKEN`, `PUSH_TOKEN`, or any other credential. + +### What to expect + +- **Acknowledgement:** `[CONFIRM RESPONSE-TIME SLA]` (suggested: within 5 business days). +- We will confirm the issue, assess severity, and keep you updated on remediation. +- Please give us a reasonable window to release a fix before public disclosure. + +## Supported versions + +CodeWeave is **experimental** and pre-release. Security fixes are applied to the +default branch (`main`) only. `[CONFIRM SUPPORTED-VERSION POLICY IF RELEASES BEGIN]` + +## Credential handling (how the pipeline treats secrets) + +CodeWeave requires two fine-grained PATs, both stored as GitHub Actions repository +secrets and scoped as narrowly as possible: + +| Secret | Scope it needs | Used by | +|--------|----------------|---------| +| `COPILOT_TOKEN` | **Copilot user requests: Read** (account permission only, no repository permissions) | Every Copilot phase, as `GH_TOKEN` | +| `PUSH_TOKEN` | **Contents: Read and write** on the *target* repository only | Phase 2 (push ADRs) and Phase 7 (push optimization branches) | + +Guidance: + +- **Scope tokens minimally.** Grant only the permissions listed above. `PUSH_TOKEN` + should be limited to the single target repository, not an org-wide token. +- **Rotate tokens** on the cadence your organization requires, and immediately if a + runner or log is suspected of exposure. +- **Self-hosted runner trust.** Phases 5–8 run on a persistent self-hosted runner + that retains the built `src/` tree, the `.venv`, and a warm ccache between phases. + Treat that runner as a sensitive host: restrict who can dispatch workflows and who + can access the runner, since a compromised runner has access to both PATs at + runtime. +- **`proof/` artifacts** contain logs and session transcripts. Review them before + publishing; do not attach them to public issues without checking for anything + sensitive. + +## Scope + +This policy covers the CodeWeave pipeline in this repository (workflows, composite +action, prompts, and tooling under `integration-test/_tools/`). Vulnerabilities in +the *target* repositories CodeWeave analyzes, or in third-party dependencies such as +the GitHub Copilot CLI or CodeCarbon, should be reported to their respective +maintainers. diff --git a/integration-test/_tools/ab_compare.py b/integration-test/_tools/ab_compare.py index 2a28832..863b28d 100644 --- a/integration-test/_tools/ab_compare.py +++ b/integration-test/_tools/ab_compare.py @@ -76,7 +76,7 @@ def load_v2_records(path: Path, label: str): wall_clock_ms. Exits with a clear error if any record is stale-format: mixing schemas silently is exactly the contamination v2 exists to prevent. """ - with open(path, "r") as f: + with open(path, "r", encoding="utf-8") as f: records = json.load(f) if not records: @@ -145,10 +145,10 @@ def baseline_mode(runs_path: Path, output_dir: Path): } output_dir.mkdir(parents=True, exist_ok=True) - with open(output_dir / "baseline.json", "w") as f: + with open(output_dir / "baseline.json", "w", encoding="utf-8") as f: json.dump(baseline_result, f, indent=2) - with open(output_dir / "baseline-summary.md", "w") as f: + with open(output_dir / "baseline-summary.md", "w", encoding="utf-8") as f: f.write("# Baseline Statistics (v2 — per-iteration metrics)\n\n") f.write("| Metric | Value |\n") f.write("|--------|-------|\n") @@ -163,7 +163,7 @@ def baseline_mode(runs_path: Path, output_dir: Path): f.write(f"| Wall clock, mean (ms) — informational | {wall_stats['mean']:.0f} |\n") f.write(f"| Mean CO₂ (g) | {mean_co2:.6f} |\n") - with open(output_dir / "baseline-complete.md", "w") as f: + with open(output_dir / "baseline-complete.md", "w", encoding="utf-8") as f: f.write("# Baseline Complete\n\n") f.write(f"Baseline measurement completed with {iter_stats['n']} runs " f"(v2 per-iteration metrics).\n\n") @@ -186,7 +186,7 @@ def baseline_mode(runs_path: Path, output_dir: Path): def load_micro(path: Path): """Load an op_microbench.py output file: median_ns + raw_times_ns.""" - with open(path, "r") as f: + with open(path, "r", encoding="utf-8") as f: data = json.load(f) if "median_ns" not in data: print(f"Error: {path} is not a valid microbench file (no median_ns)", @@ -312,7 +312,7 @@ def drift_check(phase6_baseline_path: Path, current_a_iter_stats): Drift does not change the verdict (the paired design protects it) but is flagged for the Phase 8 report. """ - with open(phase6_baseline_path, "r") as f: + with open(phase6_baseline_path, "r", encoding="utf-8") as f: ref = json.load(f) if "median_iter_ms_mean" not in ref or "cv_iter" not in ref: @@ -434,10 +434,10 @@ def stat_block(iter_s, iterations_s, jpi_s, wall_s, energy_s): } output_prefix.parent.mkdir(parents=True, exist_ok=True) - with open(output_prefix.with_suffix(".json"), "w") as f: + with open(output_prefix.with_suffix(".json"), "w", encoding="utf-8") as f: json.dump(comparison_result, f, indent=2) - with open(output_prefix.with_suffix(".md"), "w") as f: + with open(output_prefix.with_suffix(".md"), "w", encoding="utf-8") as f: f.write("# A/B Comparison Results (v2)\n\n") f.write(f"## Decision: **{decision}** (signal: `{decision_signal}`)\n\n") f.write(f"Measurement path: `{measurement_path}`\n\n") @@ -533,7 +533,7 @@ def family_mode(reports_dir: Path, output_dir: Path, alpha=0.05): rows = [] for fp in sorted(reports_dir.glob("ab-comparison-opt*.json")): try: - with open(fp) as fh: + with open(fp, encoding="utf-8") as fh: d = json.load(fh) except (OSError, json.JSONDecodeError): continue @@ -573,10 +573,10 @@ def family_mode(reports_dir: Path, output_dir: Path, alpha=0.05): "family_size": m, "n_demoted": len(demoted), "cycles": rows} output_dir.mkdir(parents=True, exist_ok=True) - with open(output_dir / "family-correction.json", "w") as f: + with open(output_dir / "family-correction.json", "w", encoding="utf-8") as f: json.dump(result, f, indent=2) - with open(output_dir / "family-correction.md", "w") as f: + with open(output_dir / "family-correction.md", "w", encoding="utf-8") as f: f.write("# Family-Wise Correction (Holm-Bonferroni, FWER = " f"{alpha})\n\n") f.write(f"Family of {m} measured comparison(s); each cycle's primary-path " diff --git a/integration-test/_tools/diff_fuzz.py b/integration-test/_tools/diff_fuzz.py index ff663ad..33c76f8 100644 --- a/integration-test/_tools/diff_fuzz.py +++ b/integration-test/_tools/diff_fuzz.py @@ -358,7 +358,7 @@ def compare(mod, golden_path: Path, report_path: Path | None, spec_hash: str): } if report_path: report_path.parent.mkdir(parents=True, exist_ok=True) - with open(report_path, "w") as f: + with open(report_path, "w", encoding="utf-8") as f: json.dump(report, f, indent=2) n_fail = sum(1 for r in results if not r["passed"]) diff --git a/integration-test/_tools/op_microbench.py b/integration-test/_tools/op_microbench.py index 9d526f0..5827bf0 100644 --- a/integration-test/_tools/op_microbench.py +++ b/integration-test/_tools/op_microbench.py @@ -111,7 +111,7 @@ def main(): } args.output.parent.mkdir(parents=True, exist_ok=True) - with open(args.output, "w") as f: + with open(args.output, "w", encoding="utf-8") as f: json.dump(result, f, indent=2) print(f"Microbench {spec['op']}: median={measurement.median * 1e6:.1f}us " diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..1b592f6 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,7 @@ +[pytest] +# Repo-tooling tests only (the deterministic verdict/measurement engine). +# The target harness under integration-test/tests/ is generated per-target and +# runs inside a pipeline job against the built target, not in repo CI. +testpaths = tests +python_files = test_*.py +addopts = -q diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..32ea15c --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,7 @@ +# Development / CI dependencies for the CodeWeave repo tooling tests. +# These cover only the deterministic pipeline tools under integration-test/_tools/ +# (the statistics/verdict engine), NOT the target harness dependencies, which are +# installed per-target from SOURCE-UNDER-INVESTIGATION.md during a pipeline run. +pytest>=8,<10 +numpy>=1.26 +scipy>=1.11 diff --git a/tests/tools/test_ab_compare.py b/tests/tools/test_ab_compare.py new file mode 100644 index 0000000..1f5e9ac --- /dev/null +++ b/tests/tools/test_ab_compare.py @@ -0,0 +1,336 @@ +"""Unit + integration tests for the verdict engine (integration-test/_tools/ab_compare.py). + +`ab_compare.py` is the single source of truth for every KEEP / INVESTIGATE / REVERT +verdict CodeWeave produces. These tests lock in the behaviour that matters for +correctness: + +- the directional decision table (`decide`) — including the noise-floor gate that + stops a statistically-significant-but-trivial change from becoming a KEEP, +- the Holm-Bonferroni family-wise correction (`holm_bonferroni`), +- the microbench significance rule (`micro_comparison`), +- basic descriptive stats (`compute_stats`), +- and the three CLI modes end to end (baseline / compare / family) against + synthetic run records. + +The module is a CLI script; we load it by path so the tests run regardless of cwd. +""" + +import importlib.util +import json +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +MODULE_PATH = REPO_ROOT / "integration-test" / "_tools" / "ab_compare.py" + + +def _load_module(): + spec = importlib.util.spec_from_file_location("ab_compare", MODULE_PATH) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +ab = _load_module() + + +# --------------------------------------------------------------------------- # +# compute_stats +# --------------------------------------------------------------------------- # + +def test_compute_stats_empty(): + assert ab.compute_stats([]) == {"mean": 0, "std": 0, "cv": 0, "n": 0} + + +def test_compute_stats_single_value_has_no_spread(): + s = ab.compute_stats([42.0]) + assert s["n"] == 1 + assert s["mean"] == 42.0 + assert s["std"] == 0 + assert s["cv"] == 0 + + +def test_compute_stats_known_values(): + # Sample std of [2,4,4,4,5,5,7,9] is 2.13809 (n-1 denominator), mean 5. + s = ab.compute_stats([2, 4, 4, 4, 5, 5, 7, 9]) + assert s["n"] == 8 + assert s["mean"] == pytest.approx(5.0) + assert s["std"] == pytest.approx(2.138090, abs=1e-5) + assert s["cv"] == pytest.approx(2.138090 / 5.0, abs=1e-5) + + +def test_compute_stats_zero_mean_cv_is_zero_not_error(): + s = ab.compute_stats([-1, 0, 1]) + assert s["mean"] == 0 + assert s["cv"] == 0 # guarded division, must not raise + + +# --------------------------------------------------------------------------- # +# decide — the directional decision table +# --------------------------------------------------------------------------- # + +def _micro(delta, significant): + return {"delta_micro_pct": delta, "micro_significant": significant} + + +class TestDecideEndToEnd: + PATH = "end-to-end" + + def test_significant_faster_is_keep(self): + d, sig = ab.decide(5.0, 0.001, None, self.PATH, mde_pct=2.0) + assert (d, sig) == ("KEEP", "e2e") + + def test_significant_slower_is_revert_regression(self): + d, sig = ab.decide(-5.0, 0.001, None, self.PATH, mde_pct=2.0) + assert (d, sig) == ("REVERT", "e2e-regression") + + def test_significant_but_below_noise_floor_is_only_a_trend(self): + # p rejects, but |Δ| (1%) < MDE (2%): the floor gate must block the KEEP. + d, sig = ab.decide(1.0, 0.001, None, self.PATH, mde_pct=2.0) + assert (d, sig) == ("INVESTIGATE", "trend") + + def test_not_significant_positive_is_trend(self): + d, sig = ab.decide(5.0, 0.20, None, self.PATH, mde_pct=2.0) + assert (d, sig) == ("INVESTIGATE", "trend") + + def test_flat_no_signal_is_revert_no_effect(self): + d, sig = ab.decide(0.0, 0.9, None, self.PATH, mde_pct=2.0) + assert (d, sig) == ("REVERT", "no-effect") + + def test_micro_corroboration_can_keep_when_e2e_flat(self): + # e2e not significant, but a significant positive microbench → KEEP. + d, sig = ab.decide(0.5, 0.4, _micro(8.0, True), self.PATH, mde_pct=2.0) + assert (d, sig) == ("KEEP", "microbench") + + def test_micro_regression_takes_priority_over_path(self): + d, sig = ab.decide(0.0, 0.9, _micro(-9.0, True), self.PATH, mde_pct=2.0) + assert (d, sig) == ("REVERT", "micro-regression") + + +class TestDecideMicrobenchPath: + PATH = "microbench" + + def test_absent_microbench_reverts_not_falls_back_to_e2e(self): + # e2e looks like a big significant win, but on the microbench path a + # missing primary signal must REVERT rather than act on the sub-floor e2e. + d, sig = ab.decide(9.0, 0.001, None, self.PATH, mde_pct=2.0) + assert (d, sig) == ("REVERT", "no-microbench") + + def test_unknown_significance_reverts(self): + d, sig = ab.decide(9.0, 0.001, _micro(9.0, None), self.PATH, mde_pct=2.0) + assert (d, sig) == ("REVERT", "no-microbench") + + def test_significant_micro_win_is_keep(self): + d, sig = ab.decide(0.0, 0.9, _micro(6.0, True), self.PATH, mde_pct=2.0) + assert (d, sig) == ("KEEP", "microbench") + + def test_e2e_win_unsupported_by_micro_is_investigate(self): + # e2e significant + positive, micro present but not significant → the e2e + # "win" is implausible for a sub-floor op, so INVESTIGATE (drift suspect). + d, sig = ab.decide(9.0, 0.001, _micro(0.3, False), self.PATH, mde_pct=2.0) + assert (d, sig) == ("INVESTIGATE", "e2e-unsupported-by-micro") + + def test_micro_regression_reverts(self): + d, sig = ab.decide(0.0, 0.9, _micro(-6.0, True), self.PATH, mde_pct=2.0) + assert (d, sig) == ("REVERT", "micro-regression") + + +# --------------------------------------------------------------------------- # +# holm_bonferroni +# --------------------------------------------------------------------------- # + +def test_holm_step_down_known_example(): + # m=3, alpha=0.05. Sorted thresholds: 0.05/3, 0.05/2, 0.05/1. + pvals = [0.001, 0.04, 0.03] + survives, thresholds = ab.holm_bonferroni(pvals, 0.05) + assert survives == [True, False, False] + assert thresholds[0] == pytest.approx(0.05 / 3) + assert thresholds[2] == pytest.approx(0.05 / 2) + assert thresholds[1] == pytest.approx(0.05 / 1) + + +def test_holm_all_survive_when_all_tiny(): + survives, _ = ab.holm_bonferroni([0.001, 0.002, 0.003], 0.05) + assert survives == [True, True, True] + + +def test_holm_is_monotone_once_one_fails_all_larger_fail(): + # 0.02 passes 0.05/3=0.0167? No. So nothing survives despite 0.001 being tiny? + # 0.001 is rank 1 (thr 0.0167) -> survives. 0.02 rank2 (thr 0.025) -> 0.02<=0.025 survives. + # 0.049 rank3 (thr 0.05) -> survives. Use a clearer failing case: + survives, _ = ab.holm_bonferroni([0.001, 0.03, 0.9], 0.05) + # rank1 0.001<=0.0167 T; rank2 0.03<=0.025 F -> stop; rank3 F + assert survives == [True, False, False] + + +def test_holm_preserves_input_order(): + # Smallest p is last in input; survival must map back to index 2. + survives, _ = ab.holm_bonferroni([0.9, 0.9, 0.0001], 0.05) + assert survives == [False, False, True] + + +# --------------------------------------------------------------------------- # +# micro_comparison +# --------------------------------------------------------------------------- # + +def test_micro_comparison_invalid_baseline_median(): + out = ab.micro_comparison({"median_ns": 0}, {"median_ns": 100}) + assert out["micro_significant"] is False + assert "invalid" in out["micro_note"] + + +def test_micro_comparison_significant_speedup_with_raw_samples(): + # Base ~1000ns, variant ~900ns, tight samples → significant + clears 2% floor. + base = {"median_ns": 1000.0, "raw_times_ns": [1000, 1001, 999, 1000, 1002, 998]} + var = {"median_ns": 900.0, "raw_times_ns": [900, 901, 899, 900, 902, 898]} + out = ab.micro_comparison(base, var) + assert out["delta_micro_pct"] == pytest.approx(10.0) + assert out["micro_significant"] is True + assert out["micro_p_value"] < 0.05 + + +def test_micro_comparison_without_raw_samples_is_unknown_not_true(): + base = {"median_ns": 1000.0} + var = {"median_ns": 800.0} + out = ab.micro_comparison(base, var) + # A bare median delta must NOT read as significant; significance is UNKNOWN. + assert out["micro_significant"] is None + assert out["micro_p_value"] is None + + +def test_micro_comparison_tiny_delta_not_significant_even_if_p_small(): + # Very tight, but only a 0.5% median delta — below the 2% practical floor. + # (Small real spread so the t-test is well-defined, not degenerate.) + base = {"median_ns": 1000.0, "raw_times_ns": [999, 1000, 1001, 1000, 999, 1001, 1000, 1000]} + var = {"median_ns": 995.0, "raw_times_ns": [994, 995, 996, 995, 994, 996, 995, 995]} + out = ab.micro_comparison(base, var) + assert abs(out["delta_micro_pct"]) < ab.MICRO_PRACTICAL_FLOOR_PCT + assert out["micro_significant"] is False + + +# --------------------------------------------------------------------------- # +# CLI modes end to end (synthetic records) +# --------------------------------------------------------------------------- # + +def _run_record(run_id, median_iter_ms, iterations, wall_ms=30000, energy=100.0): + return { + "run_id": run_id, + "median_iter_ms": median_iter_ms, + "iterations": iterations, + "wall_clock_ms": wall_ms, + "energy_joules": energy, + "co2_grams": 0.01, + } + + +def _write_json(path, obj): + path.write_text(json.dumps(obj), encoding="utf-8") + + +def test_baseline_mode_writes_expected_stats(tmp_path): + runs = [_run_record(i, 10.0 + (i % 2) * 0.1, 3000) for i in range(5)] + runs_path = tmp_path / "run-records.json" + _write_json(runs_path, runs) + out_dir = tmp_path / "reports" + + result = ab.baseline_mode(runs_path, out_dir) + + assert result["schema"] == "v2" + assert result["n"] == 5 + # mde_pct is exactly MDE_FACTOR * cv * 100 by construction. + assert result["mde_pct"] == pytest.approx(ab.MDE_FACTOR * result["cv_iter"] * 100) + assert result["energy_valid"] is True + assert (out_dir / "baseline.json").exists() + assert (out_dir / "baseline-summary.md").exists() + assert (out_dir / "baseline-complete.md").exists() + + +def test_baseline_mode_flags_zero_energy_as_invalid(tmp_path): + runs = [_run_record(i, 10.0, 3000, energy=0.0) for i in range(5)] + runs_path = tmp_path / "run-records.json" + _write_json(runs_path, runs) + result = ab.baseline_mode(runs_path, tmp_path / "reports") + assert result["energy_valid"] is False + + +def test_load_v2_records_rejects_stale_format(tmp_path): + stale = [{"run_id": 1, "wall_clock_ms": 30000}] # no iterations / median_iter_ms + p = tmp_path / "stale.json" + _write_json(p, stale) + with pytest.raises(SystemExit): + ab.load_v2_records(p, "stale") + + +def test_compare_mode_end_to_end_keep(tmp_path): + # Variant clearly faster per iteration (9 vs 10 ms), tight variance → KEEP. + base = [_run_record(i, 10.0 + (i % 2) * 0.05, 3000) for i in range(5)] + var = [_run_record(i, 9.0 + (i % 2) * 0.05, 3333) for i in range(5)] + base_p = tmp_path / "base.json" + var_p = tmp_path / "var.json" + _write_json(base_p, base) + _write_json(var_p, var) + prefix = tmp_path / "ab-comparison-opt1" + + result = ab.compare_mode(base_p, var_p, prefix, measurement_path="end-to-end") + + assert result["decision"] == "KEEP" + assert result["decision_signal"] == "e2e" + assert result["delta_e2e_pct"] > 0 + assert (tmp_path / "ab-comparison-opt1.json").exists() + assert (tmp_path / "ab-comparison-opt1.md").exists() + + +def test_compare_mode_requires_two_runs_per_group(tmp_path): + base = [_run_record(0, 10.0, 3000)] + var = [_run_record(0, 9.0, 3333)] + base_p, var_p = tmp_path / "b.json", tmp_path / "v.json" + _write_json(base_p, base) + _write_json(var_p, var) + with pytest.raises(SystemExit): + ab.compare_mode(base_p, var_p, tmp_path / "out", measurement_path="end-to-end") + + +def test_family_mode_demotes_non_surviving_keep(tmp_path): + reports = tmp_path / "reports" + reports.mkdir() + # Family of 3 KEEPs at alpha=0.05. Holm step-down thresholds (ascending): + # 0.05/3=0.0167, 0.05/2=0.025, 0.05/1=0.05. + # opt1 p=0.001 -> rank1, 0.001<=0.0167 survives (stays KEEP) + # opt2 p=0.03 -> rank2, 0.03 >0.025 fails (demoted) + # opt3 p=0.04 -> rank3, step-down stops after opt2 fails (demoted) + _write_json(reports / "ab-comparison-opt1.json", { + "measurement_path": "end-to-end", "decision": "KEEP", + "decision_signal": "e2e", "p_value_e2e": 0.001, + }) + _write_json(reports / "ab-comparison-opt2.json", { + "measurement_path": "end-to-end", "decision": "KEEP", + "decision_signal": "e2e", "p_value_e2e": 0.03, + }) + _write_json(reports / "ab-comparison-opt3.json", { + "measurement_path": "end-to-end", "decision": "KEEP", + "decision_signal": "e2e", "p_value_e2e": 0.04, + }) + + result = ab.family_mode(reports, reports, alpha=0.05) + + assert result["family_size"] == 3 + by_opt = {r["opt"]: r for r in result["cycles"]} + assert by_opt["1"]["corrected_decision"] == "KEEP" + assert by_opt["2"]["corrected_decision"] == "INVESTIGATE" + assert by_opt["2"]["corrected_signal"] == "keep-not-family-significant" + assert by_opt["3"]["corrected_decision"] == "INVESTIGATE" + assert result["n_demoted"] == 2 + assert (reports / "family-correction.json").exists() + assert (reports / "family-correction.md").exists() + + +def test_family_mode_never_promotes_a_revert(tmp_path): + reports = tmp_path / "reports" + reports.mkdir() + _write_json(reports / "ab-comparison-opt1.json", { + "measurement_path": "end-to-end", "decision": "REVERT", + "decision_signal": "no-effect", "p_value_e2e": 0.0001, + }) + result = ab.family_mode(reports, reports, alpha=0.05) + assert result["cycles"][0]["corrected_decision"] == "REVERT"