From f14f9aaa909ba6f5980a4c54ec0e165b143f9313 Mon Sep 17 00:00:00 2001
From: Bruno Borges <bruno.borges@microsoft.com>
Date: Wed, 5 Aug 2026 00:04:12 -0400
Subject: [PATCH 01/19] Make the focused benchmark able to resolve real effects

The focused cache restore benchmark could not distinguish a real improvement
from runner noise, so its headline numbers were not evidence of anything.

Two consecutive runs against an unchanged actions/setup-java@v4.8.0, where the
true difference is exactly zero, reported medians of 2s and 3s. That spurious
1.2s separation has a bootstrap 95% CI of [-1.97, -0.43], which excludes zero.
Across the same two runs the reported candidate delta flipped from +0.6s to
-0.8s.

Three defects caused this:

- Durations were read from the Actions API, whose step timestamps have
  one-second resolution. Setup takes two to six seconds, so every sample
  carried +/-500ms of quantization error, the same magnitude as the effects
  being measured. Every recorded sample was an integer.
- Each arm ran as its own matrix of independent jobs, so the comparison was
  confounded with between-runner variance, which on hosted runners is larger
  than the effect. Adding samples to each arm does not remove it.
- The report paired "sample N" of one arm with "sample N" of the other. Those
  jobs shared no runner and no point in time, so the pairing removed no
  variance while the label implied precision the data did not have. Results
  were published as bare point estimates with no interval.

This change rebuilds the workflow around measurements that can support a
verdict:

- scripts/measure.mjs times the setup step from inside the job at millisecond
  resolution instead of reading the API clock.
- Both arms now run in the same job in ABBA order, so each runner produces one
  genuinely paired difference with its own speed cancelled out, and the
  mirrored order cancels drift across slots. Each slot deletes ~/.m2 first so
  every restore extracts into an empty tree. This also uses fewer jobs than the
  design it replaces.
- scripts/stats.mjs adds seeded bootstrap intervals, a paired permutation test,
  and a Hodges-Lehmann shift estimate, and converts them into an explicit
  verdict. Comparisons whose interval includes zero are reported as
  inconclusive rather than as a number.
- The report publishes a noise floor derived from the within-runner repeat
  spread, and an A/A control that applies the same estimator to the baseline
  against itself. Because each arm is already measured twice per runner, the
  control costs no extra jobs. A run whose control resolves a difference is
  flagged as untrustworthy.
- The workflow takes setup-java-repository, baseline-ref, and candidate-ref, so
  a PR branch can be measured directly.

Validated against synthetic data with a known effect injected under realistic
between-runner spread: a true 0ms effect reports inconclusive, and a true 400ms
effect is recovered as -361ms with a 95% CI of [-420, -299] and p = 0.002. The
previous harness could not resolve anything below one second and returned false
positives at zero.

The real A/A dataset is pinned in scripts/stats.test.mjs so unpaired sampling
is not reintroduced.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 71c45320-1417-4029-8402-69075d61dac1
---
 .github/workflows/focused-cache-restore.yml | 210 +++++++-----
 README.md                                   |  25 +-
 scripts/focused-cache-restore.sh            |  60 ++++
 scripts/measure.mjs                         |  59 ++++
 scripts/report-focused.mjs                  | 333 ++++++++++++--------
 scripts/report-focused.test.mjs             | 114 +++++++
 scripts/report.test.mjs                     |  13 -
 scripts/stats.mjs                           | 205 ++++++++++++
 scripts/stats.test.mjs                      | 112 +++++++
 9 files changed, 918 insertions(+), 213 deletions(-)
 create mode 100755 scripts/focused-cache-restore.sh
 create mode 100644 scripts/measure.mjs
 create mode 100644 scripts/report-focused.test.mjs
 create mode 100644 scripts/stats.mjs
 create mode 100644 scripts/stats.test.mjs

diff --git a/.github/workflows/focused-cache-restore.yml b/.github/workflows/focused-cache-restore.yml
index 262da18..9f74015 100644
--- a/.github/workflows/focused-cache-restore.yml
+++ b/.github/workflows/focused-cache-restore.yml
@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       samples:
-        description: Warm restore samples per version
+        description: Runners to measure on (each contributes 2 paired observations)
         required: true
         type: choice
         options:
@@ -13,6 +13,21 @@ on:
           - "10"
           - "20"
         default: "10"
+      setup-java-repository:
+        description: Repository containing the setup-java action
+        required: true
+        default: actions/setup-java
+        type: string
+      baseline-ref:
+        description: Git ref for the baseline arm
+        required: true
+        default: v4.8.0
+        type: string
+      candidate-ref:
+        description: Git ref for the candidate arm
+        required: true
+        default: main
+        type: string
       cleanup-caches:
         description: Delete benchmark caches after measuring them
         required: true
@@ -32,68 +47,72 @@ concurrency:
   group: focused-cache-restore
   cancel-in-progress: false
 
+defaults:
+  run:
+    shell: bash
+
 jobs:
-  v4-seed:
-    name: Seed v4 cache
+  baseline-seed:
+    name: Seed baseline cache
     runs-on: ubuntu-24.04
     steps:
       - name: Check out benchmark repository
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
         with:
           persist-credentials: false
+      - name: Check out baseline setup-java
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          repository: ${{ inputs.setup-java-repository }}
+          path: baseline
+          persist-credentials: false
+          ref: ${{ inputs.baseline-ref }}
       - name: Prepare cache identity
-        env:
-          BENCHMARK_ID: focused-v4-${{ github.run_id }}
-        run: |
-          printf '%s\n' "$BENCHMARK_ID" > .focused-cache-key
-          mkdir -p .mvn/wrapper
-          printf 'wrapperVersion=focused\n# benchmark-id=%s\n' "$BENCHMARK_ID" > .mvn/wrapper/maven-wrapper.properties
-      - name: Setup Java v4
-        uses: actions/setup-java@v4.8.0
+        run: bash scripts/focused-cache-restore.sh prepare "focused-baseline-${{ github.run_id }}"
+      - name: Setup Java baseline
+        uses: ./baseline
         with:
           distribution: temurin
           java-version: ${{ env.JAVA_VERSION }}
           cache: maven
           cache-dependency-path: .focused-cache-key
       - name: Create synthetic cache fixtures
-        run: |
-          mkdir -p ~/.m2/repository/focused-benchmark
-          head -c "$((DEPENDENCY_FIXTURE_MIB * 1024 * 1024))" /dev/urandom > ~/.m2/repository/focused-benchmark/dependencies.bin &
-          wait
+        run: bash scripts/focused-cache-restore.sh seed-fixtures
 
-  main-seed:
-    name: Seed main caches
+  candidate-seed:
+    name: Seed candidate caches
     runs-on: ubuntu-24.04
     steps:
       - name: Check out benchmark repository
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
         with:
           persist-credentials: false
+      - name: Check out candidate setup-java
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          repository: ${{ inputs.setup-java-repository }}
+          path: candidate
+          persist-credentials: false
+          ref: ${{ inputs.candidate-ref }}
       - name: Prepare cache identity
-        env:
-          BENCHMARK_ID: focused-main-${{ github.run_id }}
-        run: |
-          printf '%s\n' "$BENCHMARK_ID" > .focused-cache-key
-          mkdir -p .mvn/wrapper
-          printf 'wrapperVersion=focused\n# benchmark-id=%s\n' "$BENCHMARK_ID" > .mvn/wrapper/maven-wrapper.properties
-      - name: Setup Java main
-        uses: actions/setup-java@main
+        run: bash scripts/focused-cache-restore.sh prepare "focused-candidate-${{ github.run_id }}"
+      - name: Setup Java candidate
+        uses: ./candidate
         with:
           distribution: temurin
           java-version: ${{ env.JAVA_VERSION }}
           cache: maven
           cache-dependency-path: .focused-cache-key
       - name: Create synthetic cache fixtures
-        run: |
-          mkdir -p ~/.m2/repository/focused-benchmark
-          mkdir -p ~/.m2/wrapper/dists/focused-benchmark
-          head -c "$((DEPENDENCY_FIXTURE_MIB * 1024 * 1024))" /dev/urandom > ~/.m2/repository/focused-benchmark/dependencies.bin &
-          head -c "$((WRAPPER_FIXTURE_MIB * 1024 * 1024))" /dev/urandom > ~/.m2/wrapper/dists/focused-benchmark/wrapper.bin &
-          wait
+        run: bash scripts/focused-cache-restore.sh seed-fixtures
 
-  v4-measure:
-    name: v4 / warm / ${{ matrix.sample }}
-    needs: v4-seed
+  # Both arms are measured inside the same job, in ABBA order. Sharing a runner
+  # removes the between-runner variance that otherwise swamps the effect, and
+  # the mirrored order cancels drift across the four slots. Measuring each arm
+  # twice per runner also yields a free A/A noise-floor estimate.
+  measure:
+    name: paired / ${{ matrix.sample }}
+    needs: [baseline-seed, candidate-seed]
     runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
@@ -104,59 +123,97 @@ jobs:
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
         with:
           persist-credentials: false
-      - name: Prepare cache identity
-        env:
-          BENCHMARK_ID: focused-v4-${{ github.run_id }}
-        run: |
-          printf '%s\n' "$BENCHMARK_ID" > .focused-cache-key
-          mkdir -p .mvn/wrapper
-          printf 'wrapperVersion=focused\n# benchmark-id=%s\n' "$BENCHMARK_ID" > .mvn/wrapper/maven-wrapper.properties
-      - name: Setup Java v4
-        uses: actions/setup-java@v4.8.0
+      - name: Check out baseline setup-java
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          repository: ${{ inputs.setup-java-repository }}
+          path: baseline
+          persist-credentials: false
+          ref: ${{ inputs.baseline-ref }}
+      - name: Check out candidate setup-java
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          repository: ${{ inputs.setup-java-repository }}
+          path: candidate
+          persist-credentials: false
+          ref: ${{ inputs.candidate-ref }}
+
+      - name: Reset slot 1
+        run: bash scripts/focused-cache-restore.sh reset "focused-baseline-${{ github.run_id }}"
+      - name: Start slot 1 timer
+        run: node scripts/measure.mjs start
+      - name: Slot 1 setup (baseline)
+        uses: ./baseline
         with:
           distribution: temurin
           java-version: ${{ env.JAVA_VERSION }}
           cache: maven
           cache-dependency-path: .focused-cache-key
-      - name: Verify restored fixtures
-        run: |
-          test "$(wc -c < ~/.m2/repository/focused-benchmark/dependencies.bin)" -eq "$((DEPENDENCY_FIXTURE_MIB * 1024 * 1024))"
+      - name: Record slot 1
+        run: node scripts/measure.mjs record ".benchmark-results/focused-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" baseline 1
+      - name: Verify slot 1 fixtures
+        run: bash scripts/focused-cache-restore.sh verify baseline
 
-  main-measure:
-    name: main / warm / ${{ matrix.sample }}
-    needs: main-seed
-    runs-on: ubuntu-24.04
-    strategy:
-      fail-fast: false
-      matrix:
-        sample: ${{ fromJSON(inputs.samples == '20' && '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]' || inputs.samples == '10' && '[1,2,3,4,5,6,7,8,9,10]' || inputs.samples == '6' && '[1,2,3,4,5,6]' || '[1,2]') }}
-    steps:
-      - name: Check out benchmark repository
-        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+      - name: Reset slot 2
+        run: bash scripts/focused-cache-restore.sh reset "focused-candidate-${{ github.run_id }}"
+      - name: Start slot 2 timer
+        run: node scripts/measure.mjs start
+      - name: Slot 2 setup (candidate)
+        uses: ./candidate
         with:
-          persist-credentials: false
-      - name: Prepare cache identity
-        env:
-          BENCHMARK_ID: focused-main-${{ github.run_id }}
-        run: |
-          printf '%s\n' "$BENCHMARK_ID" > .focused-cache-key
-          mkdir -p .mvn/wrapper
-          printf 'wrapperVersion=focused\n# benchmark-id=%s\n' "$BENCHMARK_ID" > .mvn/wrapper/maven-wrapper.properties
-      - name: Setup Java main
-        uses: actions/setup-java@main
+          distribution: temurin
+          java-version: ${{ env.JAVA_VERSION }}
+          cache: maven
+          cache-dependency-path: .focused-cache-key
+      - name: Record slot 2
+        run: node scripts/measure.mjs record ".benchmark-results/focused-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" candidate 2
+      - name: Verify slot 2 fixtures
+        run: bash scripts/focused-cache-restore.sh verify candidate
+
+      - name: Reset slot 3
+        run: bash scripts/focused-cache-restore.sh reset "focused-candidate-${{ github.run_id }}"
+      - name: Start slot 3 timer
+        run: node scripts/measure.mjs start
+      - name: Slot 3 setup (candidate)
+        uses: ./candidate
+        with:
+          distribution: temurin
+          java-version: ${{ env.JAVA_VERSION }}
+          cache: maven
+          cache-dependency-path: .focused-cache-key
+      - name: Record slot 3
+        run: node scripts/measure.mjs record ".benchmark-results/focused-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" candidate 3
+      - name: Verify slot 3 fixtures
+        run: bash scripts/focused-cache-restore.sh verify candidate
+
+      - name: Reset slot 4
+        run: bash scripts/focused-cache-restore.sh reset "focused-baseline-${{ github.run_id }}"
+      - name: Start slot 4 timer
+        run: node scripts/measure.mjs start
+      - name: Slot 4 setup (baseline)
+        uses: ./baseline
         with:
           distribution: temurin
           java-version: ${{ env.JAVA_VERSION }}
           cache: maven
           cache-dependency-path: .focused-cache-key
-      - name: Verify restored fixtures
-        run: |
-          test "$(wc -c < ~/.m2/repository/focused-benchmark/dependencies.bin)" -eq "$((DEPENDENCY_FIXTURE_MIB * 1024 * 1024))"
-          test "$(wc -c < ~/.m2/wrapper/dists/focused-benchmark/wrapper.bin)" -eq "$((WRAPPER_FIXTURE_MIB * 1024 * 1024))"
+      - name: Record slot 4
+        run: node scripts/measure.mjs record ".benchmark-results/focused-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" baseline 4
+      - name: Verify slot 4 fixtures
+        run: bash scripts/focused-cache-restore.sh verify baseline
+
+      - name: Upload sample timings
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: focused-timings-${{ matrix.sample }}
+          path: .benchmark-results/
+          include-hidden-files: true
+          if-no-files-found: error
+          retention-days: 30
 
   report:
     name: Report
-    needs: [v4-measure, main-measure]
+    needs: measure
     if: ${{ always() && !cancelled() }}
     runs-on: ubuntu-24.04
     steps:
@@ -164,10 +221,19 @@ jobs:
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
         with:
           persist-credentials: false
+      - name: Download sample timings
+        uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6
+        with:
+          pattern: focused-timings-*
+          merge-multiple: true
+          path: .benchmark-results
       - name: Generate focused report
         env:
           GH_TOKEN: ${{ github.token }}
           SAMPLES: ${{ inputs.samples }}
+          BASELINE_REF: ${{ inputs.baseline-ref }}
+          CANDIDATE_REF: ${{ inputs.candidate-ref }}
+          SETUP_JAVA_REPOSITORY: ${{ inputs.setup-java-repository }}
           CLEANUP_CACHES: ${{ inputs.cleanup-caches }}
         run: node scripts/report-focused.mjs
       - name: Upload benchmark results
diff --git a/README.md b/README.md
index c7249ec..948c85a 100644
--- a/README.md
+++ b/README.md
@@ -35,7 +35,28 @@ The report job writes a Markdown summary and uploads raw JSON and CSV files. Ben
 
 ### Focused cache restore
 
-The **Focused cache restore** workflow isolates the setup step for comparing v4 with `main`. It uses a pinned Temurin JDK from the hosted runner tool cache, seeds a synthetic 160 MiB dependency cache for both versions and a 9 MiB wrapper cache for `main`, and runs no Maven command. Warm measurement jobs therefore contain no JDK or Maven Central downloads; they measure JDK discovery and Actions cache restoration only.
+The **Focused cache restore** workflow isolates the setup step to compare two `actions/setup-java` refs (by default `v4.8.0` against `main`). It uses a pinned Temurin JDK from the hosted runner tool cache, seeds a synthetic 160 MiB dependency cache for both arms and a 9 MiB wrapper cache for the candidate, and runs no Maven command. Measurement jobs therefore contain no JDK or Maven Central downloads; they measure JDK discovery and Actions cache restoration only.
+
+This workflow is the reference for how a comparison should be measured here. Three properties make its verdicts trustworthy:
+
+**Millisecond timing.** The Actions API reports step `started_at` and `completed_at` only to the nearest second. Setup steps take two to six seconds, so reading durations from the API quantizes every measurement to ±500 ms — the same magnitude as the effects being measured. Timing is therefore taken inside the job with `scripts/measure.mjs`.
+
+**Same-runner pairing.** Between-runner variance on hosted runners is larger than the effects under test, and it cannot be averaged away by adding more independent jobs to each arm. Both arms run in the *same* job in ABBA order — baseline, candidate, candidate, baseline — so each runner yields one paired difference with the runner's own speed cancelled out. The mirrored order also cancels drift across the four slots. Each measured slot deletes `~/.m2` first so every restore extracts into an empty tree.
+
+**Intervals, not point estimates.** `scripts/stats.mjs` reports a bootstrap 95% confidence interval, a permutation p-value, and a Hodges-Lehmann shift for every comparison, and turns them into an explicit verdict. A comparison whose interval includes zero is reported as `inconclusive` rather than as a number that looks like a result.
+
+The report also publishes two guard rails:
+
+- A **noise floor**, the median spread between the two slots of the same arm on one runner. An effect smaller than this is reported as `within-noise` even when its interval excludes zero.
+- An **A/A control**, the same estimator applied to the baseline against itself. It costs no extra jobs because each arm is already measured twice per runner. A healthy run reports `within-noise` or `inconclusive`; anything else means slot ordering is biasing the results and the headline verdict cannot be trusted.
+
+Point `baseline-ref` and `candidate-ref` at any two refs — including a PR branch — to check whether a change delivers a real improvement.
+
+#### Why this replaced the previous design
+
+The earlier version of this workflow ran each arm as its own matrix of independent jobs, read durations from the Actions API, and reported a "paired median delta" that paired `sample N` of one arm with `sample N` of the other. Those samples shared no runner and no point in time, so the pairing removed no variance at all.
+
+Two consecutive runs of that harness against an unchanged `v4.8.0` — where the true difference is exactly zero — produced medians of 2 s and 3 s, a spurious 1.2 s separation whose confidence interval excluded zero. Over the same pair of runs the reported candidate delta flipped from +0.6 s to −0.8 s. `scripts/stats.test.mjs` pins that dataset as a regression test so unpaired sampling is not reintroduced.
 
 ### JDK cache
 
@@ -63,7 +84,7 @@ The summary reports medians for:
 
 Public repositories do not pay for standard GitHub-hosted runners. The estimated minutes are included to make the results applicable to private repositories; actual charges depend on the account plan and runner type.
 
-Network throughput, hosted-runner image changes, upstream artifact availability, and runner load all introduce variance. Compare multiple iterations and multiple workflow runs before drawing conclusions.
+Network throughput, hosted-runner image changes, upstream artifact availability, and runner load all introduce variance. The **Benchmark setup-java** and **JDK cache** workflows still read step durations from the Actions API at one-second resolution and compare arms across independent runners, so treat their sub-second differences as indicative only and compare multiple runs before drawing conclusions. Use **Focused cache restore** when a difference needs to be established rather than illustrated; it is the only workflow here that reports a confidence interval, a noise floor, and an A/A control.
 
 ## Local checks
 
diff --git a/scripts/focused-cache-restore.sh b/scripts/focused-cache-restore.sh
new file mode 100755
index 0000000..2876334
--- /dev/null
+++ b/scripts/focused-cache-restore.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+# Fixture and cache-identity helpers for the focused cache restore benchmark.
+
+set -euo pipefail
+
+command=${1:?command is required}
+
+dependency_fixture="$HOME/.m2/repository/focused-benchmark/dependencies.bin"
+wrapper_fixture="$HOME/.m2/wrapper/dists/focused-benchmark/wrapper.bin"
+
+write_identity() {
+  local benchmark_id=$1
+  printf '%s\n' "$benchmark_id" > .focused-cache-key
+  mkdir -p .mvn/wrapper
+  printf 'wrapperVersion=focused\n# benchmark-id=%s\n' "$benchmark_id" \
+    > .mvn/wrapper/maven-wrapper.properties
+}
+
+case "$command" in
+  prepare)
+    write_identity "${2:?benchmark id is required}"
+    ;;
+  seed-fixtures)
+    mkdir -p "$(dirname "$dependency_fixture")" "$(dirname "$wrapper_fixture")"
+    head -c "$((DEPENDENCY_FIXTURE_MIB * 1024 * 1024))" /dev/urandom > "$dependency_fixture"
+    head -c "$((WRAPPER_FIXTURE_MIB * 1024 * 1024))" /dev/urandom > "$wrapper_fixture"
+    ;;
+  reset)
+    # Each measured slot must restore into an empty tree, otherwise the second
+    # and later restores in a job would extract over files that already exist
+    # and report an artificially low duration.
+    rm -rf "$HOME/.m2"
+    write_identity "${2:?benchmark id is required}"
+    ;;
+  verify)
+    arm=${2:?arm is required}
+    actual=$(wc -c < "$dependency_fixture")
+    expected=$((DEPENDENCY_FIXTURE_MIB * 1024 * 1024))
+    if [ "$actual" -ne "$expected" ]; then
+      echo "Dependency fixture for $arm is $actual bytes, expected $expected" >&2
+      exit 1
+    fi
+    # Only the candidate arm is expected to maintain a separate wrapper cache;
+    # a baseline that also produces one is fine, so this check is arm-specific
+    # and non-fatal when the fixture is simply absent for the baseline.
+    if [ "$arm" = "candidate" ]; then
+      actual=$(wc -c < "$wrapper_fixture")
+      expected=$((WRAPPER_FIXTURE_MIB * 1024 * 1024))
+      if [ "$actual" -ne "$expected" ]; then
+        echo "Wrapper fixture for $arm is $actual bytes, expected $expected" >&2
+        exit 1
+      fi
+    fi
+    ;;
+  *)
+    echo "Unsupported command: $command" >&2
+    exit 1
+    ;;
+esac
diff --git a/scripts/measure.mjs b/scripts/measure.mjs
new file mode 100644
index 0000000..f2e5de3
--- /dev/null
+++ b/scripts/measure.mjs
@@ -0,0 +1,59 @@
+// Millisecond-resolution stopwatch for in-job benchmark slots.
+//
+// The GitHub Actions API reports step `started_at`/`completed_at` with
+// one-second resolution. Setup steps take two to six seconds, so reading
+// durations from the API quantizes each measurement to +/- 500 ms — an error of
+// the same magnitude as the effects these benchmarks try to detect. Timing the
+// step from inside the job instead keeps millisecond precision.
+
+import {appendFile, mkdir, readFile, writeFile} from 'node:fs/promises';
+import {dirname} from 'node:path';
+import {pathToFileURL} from 'node:url';
+
+const CLOCK_FILE = '.benchmark-clock';
+
+export async function start(clockFile = CLOCK_FILE) {
+  await writeFile(clockFile, String(Date.now()));
+}
+
+export async function stop(clockFile = CLOCK_FILE) {
+  const started = Number(await readFile(clockFile, 'utf8'));
+  if (!Number.isFinite(started)) {
+    throw new Error(`No valid start timestamp in ${clockFile}`);
+  }
+  return Date.now() - started;
+}
+
+function csvValue(value) {
+  return `"${String(value ?? '').replaceAll('"', '""')}"`;
+}
+
+export async function record(resultsFile, fields, elapsedMs) {
+  await mkdir(dirname(resultsFile), {recursive: true});
+  const line = [...fields, elapsedMs].map(csvValue).join(',');
+  await appendFile(resultsFile, `${line}\n`);
+}
+
+export async function main(argv) {
+  const [command, ...rest] = argv;
+  switch (command) {
+    case 'start': {
+      await start(rest[0] ?? CLOCK_FILE);
+      return;
+    }
+    case 'record': {
+      const [resultsFile, ...fields] = rest;
+      if (!resultsFile) throw new Error('record requires a results file');
+      const elapsedMs = await stop(CLOCK_FILE);
+      await record(resultsFile, fields, elapsedMs);
+      console.log(`${fields.join('/')}: ${elapsedMs} ms`);
+      return;
+    }
+    default:
+      throw new Error(`Unsupported command: ${command}`);
+  }
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  await main(process.argv.slice(2));
+}
diff --git a/scripts/report-focused.mjs b/scripts/report-focused.mjs
index a2f396d..25bc917 100644
--- a/scripts/report-focused.mjs
+++ b/scripts/report-focused.mjs
@@ -1,31 +1,154 @@
-import {appendFile, mkdir, writeFile} from 'node:fs/promises';
+import {appendFile, mkdir, readdir, readFile, writeFile} from 'node:fs/promises';
+import {join} from 'node:path';
 import {pathToFileURL} from 'node:url';
 
-import {hashFilesSingle, median, secondsBetween} from './report.mjs';
+import {hashFilesSingle} from './report.mjs';
+import {
+  classify,
+  describeVerdict,
+  formatInterval,
+  hodgesLehmann,
+  mean,
+  median,
+  medianAbsoluteDeviation,
+  pairedInterval,
+  pairedPermutationTest,
+  quantile,
+  standardDeviation
+} from './stats.mjs';
 
 const API_VERSION = '2022-11-28';
-const JOB_PATTERN = /^(v4|main) \/ warm \/ (\d+)$/;
+const RESULTS_DIR = '.benchmark-results';
 
-export function parseFocusedJob(name) {
-  const match = name.match(JOB_PATTERN);
-  if (!match) return null;
-  return {version: match[1], sample: Number(match[2])};
+// Every measurement job uploads its own CSV so that merging the artifacts
+// cannot overwrite another runner's samples.
+export async function readSampleFiles(directory = RESULTS_DIR) {
+  const entries = await readdir(directory);
+  const files = entries.filter(
+    entry => entry.startsWith('focused-timings') && entry.endsWith('.csv')
+  );
+  if (files.length === 0) {
+    throw new Error(`No focused timing CSVs found in ${directory}`);
+  }
+  const contents = await Promise.all(
+    files.sort().map(file => readFile(join(directory, file), 'utf8'))
+  );
+  return contents.join('\n');
+}
+
+// Each runner measures four slots in ABBA order: baseline, candidate,
+// candidate, baseline. Averaging the two slots per arm cancels any linear drift
+// across the job, and differencing within a runner removes between-runner
+// variance.
+export function parseSamples(csv) {
+  return csv
+    .trim()
+    .split('\n')
+    .filter(Boolean)
+    .map(line => {
+      const [sample, arm, slot, elapsedMs] = line
+        .split(',')
+        .map(value => value.replace(/^"|"$/g, ''));
+      return {
+        sample: Number(sample),
+        arm,
+        slot: Number(slot),
+        seconds: Number(elapsedMs) / 1000
+      };
+    });
+}
+
+// One paired observation per runner, plus the within-arm repeat difference that
+// serves as a null-effect (A/A) measurement requiring no extra jobs.
+export function buildPairs(rows) {
+  const bySample = new Map();
+  for (const row of rows) {
+    const entry = bySample.get(row.sample) ?? {baseline: [], candidate: []};
+    if (!entry[row.arm]) continue;
+    entry[row.arm].push(row);
+    bySample.set(row.sample, entry);
+  }
+  const pairs = [];
+  for (const [sample, entry] of [...bySample.entries()].sort(
+    (a, b) => a[0] - b[0]
+  )) {
+    if (entry.baseline.length !== 2 || entry.candidate.length !== 2) continue;
+    const baselineSlots = entry.baseline
+      .sort((a, b) => a.slot - b.slot)
+      .map(row => row.seconds);
+    const candidateSlots = entry.candidate
+      .sort((a, b) => a.slot - b.slot)
+      .map(row => row.seconds);
+    pairs.push({
+      sample,
+      baselineSlots,
+      candidateSlots,
+      baseline: mean(baselineSlots),
+      candidate: mean(candidateSlots),
+      difference: mean(candidateSlots) - mean(baselineSlots),
+      baselineRepeatDelta: baselineSlots[1] - baselineSlots[0],
+      candidateRepeatDelta: candidateSlots[1] - candidateSlots[0]
+    });
+  }
+  return pairs;
+}
+
+// The smallest effect the harness can trust. Derived from how much the same
+// implementation varies between its two slots on one runner.
+export function noiseFloor(pairs) {
+  const repeats = [
+    ...pairs.map(pair => Math.abs(pair.baselineRepeatDelta)),
+    ...pairs.map(pair => Math.abs(pair.candidateRepeatDelta))
+  ];
+  if (repeats.length === 0) return 0;
+  return quantile(repeats, 0.5);
 }
 
-function quantile(values, percentile) {
-  const sorted = [...values].sort((a, b) => a - b);
-  const position = (sorted.length - 1) * percentile;
-  const lower = Math.floor(position);
-  const upper = Math.ceil(position);
-  if (lower === upper) return sorted[lower];
-  return (
-    sorted[lower] +
-    (sorted[upper] - sorted[lower]) * (position - lower)
+function armSummary(name, values) {
+  return {
+    arm: name,
+    samples: values.length,
+    meanSeconds: mean(values),
+    medianSeconds: median(values),
+    standardDeviationSeconds: standardDeviation(values),
+    madSeconds: medianAbsoluteDeviation(values),
+    p95Seconds: quantile(values, 0.95)
+  };
+}
+
+export function analyze(rows) {
+  const pairs = buildPairs(rows);
+  const differences = pairs.map(pair => pair.difference);
+  const baselineValues = pairs.map(pair => pair.baseline);
+  const candidateValues = pairs.map(pair => pair.candidate);
+  const floor = noiseFloor(pairs);
+  const interval = pairedInterval(differences, {seed: 1});
+  // The same estimator applied to the within-arm repeats. A trustworthy harness
+  // must not resolve a difference here, because it compares an arm with itself.
+  // Judged against the same noise floor as the real effect, so a healthy run
+  // reports `within-noise` or `inconclusive`.
+  const controlInterval = pairedInterval(
+    pairs.map(pair => pair.baselineRepeatDelta),
+    {seed: 2}
   );
+  return {
+    pairs,
+    noiseFloorSeconds: floor,
+    baseline: armSummary('baseline', baselineValues),
+    candidate: armSummary('candidate', candidateValues),
+    interval,
+    pValue: pairedPermutationTest(differences, {seed: 3}),
+    shiftSeconds: hodgesLehmann(candidateValues, baselineValues),
+    verdict: classify(interval, {noiseFloor: floor}),
+    control: {
+      interval: controlInterval,
+      verdict: classify(controlInterval, {noiseFloor: floor})
+    }
+  };
 }
 
 function csvValue(value) {
-  return `"${String(value).replaceAll('"', '""')}"`;
+  return `"${String(value ?? '').replaceAll('"', '""')}"`;
 }
 
 async function api(path, token, options = {}) {
@@ -58,76 +181,58 @@ async function allPages(path, field, token) {
   }
 }
 
-function summarize(rows, version) {
-  const values = rows
-    .filter(row => row.version === version)
-    .map(row => row.setupSeconds);
-  return {
-    version,
-    samples: values.length,
-    minimumSeconds: Math.min(...values),
-    medianSeconds: median(values),
-    p95Seconds: quantile(values, 0.95),
-    maximumSeconds: Math.max(...values)
-  };
-}
-
-function markdown(metadata, rows, summaries, caches) {
+export function markdown(metadata, analysis, caches) {
+  const {baseline, candidate, interval, control} = analysis;
   const lines = [
     '# Focused cache restore benchmark',
     '',
-    `Temurin ${metadata.javaVersion}, ${metadata.samples} warm samples per version, run ${metadata.runId}.`,
+    `Temurin ${metadata.javaVersion}, ${analysis.pairs.length} runners x 2 paired observations, run ${metadata.runId}.`,
+    `Baseline \`${metadata.baselineRef}\` vs candidate \`${metadata.candidateRef}\` from \`${metadata.setupJavaRepository}\`.`,
+    '',
+    '## Verdict',
+    '',
+    `**${describeVerdict(analysis.verdict)}**`,
+    '',
+    `Paired difference (candidate - baseline): **${formatInterval(interval, {digits: 3})}**.`,
+    `Permutation p-value: ${analysis.pValue.toFixed(3)}. Hodges-Lehmann shift: ${analysis.shiftSeconds.toFixed(3)}s.`,
+    `Harness noise floor: ${analysis.noiseFloorSeconds.toFixed(3)}s (median within-runner repeat spread).`,
     '',
-    '| Version | Samples | Min (s) | Median (s) | p95 (s) | Max (s) |',
-    '| --- | ---: | ---: | ---: | ---: | ---: |'
+    `A/A control (baseline against itself) reports **${control.verdict}** at ${formatInterval(control.interval, {digits: 3})}. A healthy harness reports \`within-noise\` or \`inconclusive\` here; an \`improvement\` or \`regression\` means slot ordering is biasing results and the verdict above cannot be trusted.`,
+    '',
+    '## Arms',
+    '',
+    '| Arm | Runners | Mean (s) | Median (s) | SD (s) | MAD (s) | p95 (s) |',
+    '| --- | ---: | ---: | ---: | ---: | ---: | ---: |'
   ];
-  for (const summary of summaries) {
+  for (const summary of [baseline, candidate]) {
     lines.push(
-      `| ${summary.version} | ${summary.samples} | ${summary.minimumSeconds.toFixed(1)} | ${summary.medianSeconds.toFixed(1)} | ${summary.p95Seconds.toFixed(1)} | ${summary.maximumSeconds.toFixed(1)} |`
+      `| ${summary.arm} | ${summary.samples} | ${summary.meanSeconds.toFixed(3)} | ${summary.medianSeconds.toFixed(3)} | ${summary.standardDeviationSeconds?.toFixed(3) ?? 'n/a'} | ${summary.madSeconds.toFixed(3)} | ${summary.p95Seconds.toFixed(3)} |`
     );
   }
-  const deltas = [];
-  for (let sample = 1; sample <= metadata.samples; sample += 1) {
-    const v4 = rows.find(
-      row => row.version === 'v4' && row.sample === sample
-    );
-    const main = rows.find(
-      row => row.version === 'main' && row.sample === sample
-    );
-    if (v4 && main) deltas.push(main.setupSeconds - v4.setupSeconds);
-  }
   lines.push(
     '',
-    `Paired median delta (main - v4): **${median(deltas).toFixed(1)}s**. Negative means main was faster.`,
+    'All durations are measured inside the job with millisecond resolution. The Actions API reports step timestamps only to the nearest second, which is too coarse for effects of this size.',
     '',
-    '## Samples',
+    '## Paired samples',
     '',
-    '| Sample | v4 (s) | main (s) | Delta (s) |',
-    '| ---: | ---: | ---: | ---: |'
+    '| Runner | baseline slot 1 (s) | candidate slot 2 (s) | candidate slot 3 (s) | baseline slot 4 (s) | Paired delta (s) |',
+    '| ---: | ---: | ---: | ---: | ---: | ---: |'
   );
-  for (let sample = 1; sample <= metadata.samples; sample += 1) {
-    const v4 = rows.find(
-      row => row.version === 'v4' && row.sample === sample
-    );
-    const main = rows.find(
-      row => row.version === 'main' && row.sample === sample
+  for (const pair of analysis.pairs) {
+    lines.push(
+      `| ${pair.sample} | ${pair.baselineSlots[0].toFixed(3)} | ${pair.candidateSlots[0].toFixed(3)} | ${pair.candidateSlots[1].toFixed(3)} | ${pair.baselineSlots[1].toFixed(3)} | ${pair.difference.toFixed(3)} |`
     );
-    if (v4 && main) {
-      lines.push(
-        `| ${sample} | ${v4.setupSeconds.toFixed(1)} | ${main.setupSeconds.toFixed(1)} | ${(main.setupSeconds - v4.setupSeconds).toFixed(1)} |`
-      );
-    }
   }
   lines.push(
     '',
     '## Cache fixtures',
     '',
-    '| Version | Cache | Size (MiB) |',
+    '| Arm | Cache | Size (MiB) |',
     '| --- | --- | ---: |'
   );
   for (const cache of caches) {
     lines.push(
-      `| ${cache.version} | ${cache.type} | ${(cache.sizeBytes / 1024 / 1024).toFixed(1)} |`
+      `| ${cache.arm} | ${cache.type} | ${(cache.sizeBytes / 1024 / 1024).toFixed(1)} |`
     );
   }
   return `${lines.join('\n')}\n`;
@@ -137,70 +242,52 @@ export async function main(env = process.env) {
   const [owner, repo] = env.GITHUB_REPOSITORY.split('/');
   const token = env.GH_TOKEN;
   const runId = env.GITHUB_RUN_ID;
-  const attempt = env.GITHUB_RUN_ATTEMPT;
-  const samples = Number(env.SAMPLES);
-  const javaVersion = env.JAVA_VERSION;
-  if (!owner || !repo || !token || !runId || !attempt || !samples) {
+  const baselineRef = env.BASELINE_REF;
+  const candidateRef = env.CANDIDATE_REF;
+  const setupJavaRepository = env.SETUP_JAVA_REPOSITORY;
+  if (!owner || !repo || !token || !runId || !baselineRef || !candidateRef) {
     throw new Error('Missing required GitHub Actions environment variables');
   }
 
-  const [jobs, cacheEntries, mainCommit, v4Ref] = await Promise.all([
-    allPages(
-      `/repos/${owner}/${repo}/actions/runs/${runId}/attempts/${attempt}/jobs`,
-      'jobs',
-      token
-    ),
-    allPages(`/repos/${owner}/${repo}/actions/caches`, 'actions_caches', token),
-    api('/repos/actions/setup-java/commits/main', token),
-    api('/repos/actions/setup-java/git/ref/tags/v4.8.0', token)
-  ]);
+  const rows = parseSamples(await readSampleFiles());
+  const analysis = analyze(rows);
+  if (analysis.pairs.length === 0) {
+    throw new Error('No complete ABBA samples were collected');
+  }
 
-  const rows = jobs
-    .map(job => {
-      const identity = parseFocusedJob(job.name);
-      if (!identity) return null;
-      const step = job.steps.find(item =>
-        item.name.startsWith('Setup Java')
-      );
-      return {
-        ...identity,
-        conclusion: job.conclusion,
-        setupSeconds: secondsBetween(step.started_at, step.completed_at)
-      };
-    })
-    .filter(Boolean)
-    .sort(
-      (a, b) =>
-        a.sample - b.sample || a.version.localeCompare(b.version)
-    );
+  const cacheEntries = await allPages(
+    `/repos/${owner}/${repo}/actions/caches`,
+    'actions_caches',
+    token
+  );
 
   const caches = [];
-  for (const [version, versionId] of [
-    ['v4', 'v4'],
-    ['main', 'main']
-  ]) {
-    const benchmarkId = `focused-${versionId}-${runId}`;
+  for (const arm of ['baseline', 'candidate']) {
+    const benchmarkId = `focused-${arm}-${runId}`;
     const expected = [
       {
         type: 'maven-dependencies',
-        key: `setup-java-Linux-x64-maven-${hashFilesSingle(
-          `${benchmarkId}\n`
-        )}`
-      }
-    ];
-    if (version === 'main') {
-      expected.push({
+        key: `setup-java-Linux-x64-maven-${hashFilesSingle(`${benchmarkId}\n`)}`
+      },
+      {
         type: 'maven-wrapper',
         key: `setup-java-Linux-x64-maven-wrapper-${hashFilesSingle(
           `wrapperVersion=focused\n# benchmark-id=${benchmarkId}\n`
         )}`
-      });
-    }
+      }
+    ];
     for (const item of expected) {
       const entry = cacheEntries.find(cache => cache.key === item.key);
-      if (!entry) throw new Error(`Expected cache not found: ${item.key}`);
+      // Only the dependency cache is guaranteed for every ref; a baseline that
+      // predates wrapper caching simply will not have that entry.
+      if (!entry) {
+        if (item.type === 'maven-dependencies') {
+          throw new Error(`Expected cache not found: ${item.key}`);
+        }
+        continue;
+      }
       caches.push({
-        version,
+        arm,
         type: item.type,
         id: entry.id,
         key: item.key,
@@ -212,30 +299,24 @@ export async function main(env = process.env) {
   const metadata = {
     repository: env.GITHUB_REPOSITORY,
     runId,
-    runAttempt: Number(attempt),
-    samples,
-    javaVersion,
-    setupJavaV4Ref: v4Ref.object.sha,
-    setupJavaMainRefAtReport: mainCommit.sha,
+    javaVersion: env.JAVA_VERSION,
+    setupJavaRepository,
+    baselineRef,
+    candidateRef,
     generatedAt: new Date().toISOString()
   };
-  const summaries = ['v4', 'main'].map(version =>
-    summarize(rows, version)
-  );
-  const report = markdown(metadata, rows, summaries, caches);
+  const report = markdown(metadata, analysis, caches);
 
   await mkdir('focused-results', {recursive: true});
   await writeFile(
     'focused-results/results.json',
-    `${JSON.stringify({metadata, summaries, rows, caches}, null, 2)}\n`
+    `${JSON.stringify({metadata, analysis, caches}, null, 2)}\n`
   );
   await writeFile(
     'focused-results/results.csv',
-    `version,sample,setup_seconds,conclusion\n${rows
+    `sample,arm,slot,seconds\n${rows
       .map(row =>
-        [row.version, row.sample, row.setupSeconds, row.conclusion]
-          .map(csvValue)
-          .join(',')
+        [row.sample, row.arm, row.slot, row.seconds].map(csvValue).join(',')
       )
       .join('\n')}\n`
   );
diff --git a/scripts/report-focused.test.mjs b/scripts/report-focused.test.mjs
new file mode 100644
index 0000000..36032ee
--- /dev/null
+++ b/scripts/report-focused.test.mjs
@@ -0,0 +1,114 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+
+import {
+  analyze,
+  buildPairs,
+  markdown,
+  noiseFloor,
+  parseSamples
+} from './report-focused.mjs';
+
+const csv = [
+  '"1","baseline","1","2400"',
+  '"1","candidate","2","1900"',
+  '"1","candidate","3","1950"',
+  '"1","baseline","4","2500"',
+  '"2","baseline","1","3100"',
+  '"2","candidate","2","2600"',
+  '"2","candidate","3","2500"',
+  '"2","baseline","4","3000"'
+].join('\n');
+
+test('parses millisecond samples into seconds', () => {
+  const rows = parseSamples(csv);
+  assert.equal(rows.length, 8);
+  assert.deepEqual(rows[0], {
+    sample: 1,
+    arm: 'baseline',
+    slot: 1,
+    seconds: 2.4
+  });
+});
+
+test('pairs the two slots per arm within a runner', () => {
+  const pairs = buildPairs(parseSamples(csv));
+  assert.equal(pairs.length, 2);
+  assert.equal(pairs[0].baseline, 2.45);
+  assert.ok(Math.abs(pairs[0].candidate - 1.925) < 1e-9);
+  assert.ok(Math.abs(pairs[0].difference - -0.525) < 1e-9);
+  // Slot 4 minus slot 1 for the same implementation is a null measurement.
+  assert.ok(Math.abs(pairs[0].baselineRepeatDelta - 0.1) < 1e-9);
+});
+
+test('drops runners with incomplete ABBA slots', () => {
+  const partial = ['"3","baseline","1","2000"', '"3","candidate","2","1900"'];
+  const pairs = buildPairs(parseSamples([csv, ...partial].join('\n')));
+  assert.equal(pairs.length, 2);
+  assert.ok(!pairs.some(pair => pair.sample === 3));
+});
+
+test('derives the noise floor from within-arm repeats', () => {
+  const pairs = buildPairs(parseSamples(csv));
+  assert.ok(noiseFloor(pairs) > 0);
+  assert.equal(noiseFloor([]), 0);
+});
+
+test('reports a large consistent improvement', () => {
+  const rows = [];
+  for (let sample = 1; sample <= 10; sample += 1) {
+    // Each runner has its own offset, exactly the between-runner variance that
+    // pairing is meant to remove.
+    const offset = sample * 400;
+    rows.push(
+      `"${sample}","baseline","1","${3000 + offset}"`,
+      `"${sample}","candidate","2","${2000 + offset}"`,
+      `"${sample}","candidate","3","${2020 + offset}"`,
+      `"${sample}","baseline","4","${3020 + offset}"`
+    );
+  }
+  const analysis = analyze(parseSamples(rows.join('\n')));
+  assert.equal(analysis.verdict, 'improvement');
+  assert.ok(analysis.interval.high < 0);
+  assert.ok(analysis.pValue < 0.01);
+  // The control compares baseline against itself and must find nothing.
+  assert.ok(['within-noise', 'inconclusive'].includes(analysis.control.verdict));
+});
+
+test('renders a report with a verdict and paired samples', () => {
+  const analysis = analyze(parseSamples(csv));
+  const report = markdown(
+    {
+      runId: '1',
+      javaVersion: '17.0.19+10',
+      setupJavaRepository: 'actions/setup-java',
+      baselineRef: 'v4.8.0',
+      candidateRef: 'main'
+    },
+    analysis,
+    [{arm: 'baseline', type: 'maven-dependencies', sizeBytes: 1024 * 1024}]
+  );
+  assert.match(report, /# Focused cache restore benchmark/);
+  assert.match(report, /## Verdict/);
+  assert.match(report, /95% CI/);
+  assert.match(report, /A\/A control/);
+  assert.match(report, /Harness noise floor/);
+});
+
+test('reports an A/A comparison as inconclusive', () => {
+  const jitter = [120, -90, 200, -160, 70, -110, 180, -140, 60, -50];
+  const rows = [];
+  for (let sample = 1; sample <= 10; sample += 1) {
+    const offset = sample * 350;
+    const wobble = jitter[sample - 1];
+    rows.push(
+      `"${sample}","baseline","1","${2500 + offset}"`,
+      `"${sample}","candidate","2","${2500 + offset + wobble}"`,
+      `"${sample}","candidate","3","${2500 + offset - wobble}"`,
+      `"${sample}","baseline","4","${2500 + offset}"`
+    );
+  }
+  const analysis = analyze(parseSamples(rows.join('\n')));
+  assert.notEqual(analysis.verdict, 'improvement');
+  assert.notEqual(analysis.verdict, 'regression');
+});
diff --git a/scripts/report.test.mjs b/scripts/report.test.mjs
index d1190bf..5b7bc70 100644
--- a/scripts/report.test.mjs
+++ b/scripts/report.test.mjs
@@ -8,7 +8,6 @@ import {
   secondsBetween,
   sha256
 } from './report.mjs';
-import {parseFocusedJob} from './report-focused.mjs';
 
 test('hashes benchmark cache markers', () => {
   assert.equal(
@@ -73,15 +72,3 @@ test('parses benchmark job names', () => {
   });
   assert.equal(parseBenchmarkJob('Report'), null);
 });
-
-test('parses focused benchmark job names', () => {
-  assert.deepEqual(parseFocusedJob('main / warm / 10'), {
-    version: 'main',
-    sample: 10
-  });
-  assert.deepEqual(parseFocusedJob('v4 / warm / 2'), {
-    version: 'v4',
-    sample: 2
-  });
-  assert.equal(parseFocusedJob('Seed main caches'), null);
-});
diff --git a/scripts/stats.mjs b/scripts/stats.mjs
new file mode 100644
index 0000000..d838f31
--- /dev/null
+++ b/scripts/stats.mjs
@@ -0,0 +1,205 @@
+// Statistical helpers shared by the benchmark reports.
+//
+// Benchmark arms run on ephemeral hosted runners, so every measurement carries
+// substantial between-runner noise. Point estimates alone cannot tell a real
+// improvement from that noise, so every comparison here is reported with an
+// interval and an explicit verdict.
+
+const BOOTSTRAP_ITERATIONS = 10000;
+const DEFAULT_CONFIDENCE = 0.95;
+
+// Deterministic PRNG so a given set of samples always produces the same
+// interval. Reports are compared across runs and must not wobble because the
+// resampler drew different numbers.
+export function createRandom(seed = 0x9e3779b9) {
+  let state = seed >>> 0;
+  return function random() {
+    state = (state + 0x6d2b79f5) >>> 0;
+    let t = state;
+    t = Math.imul(t ^ (t >>> 15), t | 1);
+    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+  };
+}
+
+export function mean(values) {
+  if (values.length === 0) return null;
+  return values.reduce((total, value) => total + value, 0) / values.length;
+}
+
+export function median(values) {
+  return quantile(values, 0.5);
+}
+
+export function quantile(values, percentile) {
+  if (values.length === 0) return null;
+  const sorted = [...values].sort((a, b) => a - b);
+  const position = (sorted.length - 1) * percentile;
+  const lower = Math.floor(position);
+  const upper = Math.ceil(position);
+  if (lower === upper) return sorted[lower];
+  return sorted[lower] + (sorted[upper] - sorted[lower]) * (position - lower);
+}
+
+export function standardDeviation(values) {
+  if (values.length < 2) return null;
+  const average = mean(values);
+  const variance =
+    values.reduce((total, value) => total + (value - average) ** 2, 0) /
+    (values.length - 1);
+  return Math.sqrt(variance);
+}
+
+// Median absolute deviation, scaled to be comparable with a standard deviation
+// for normally distributed data. Robust to the occasional runner that stalls.
+export function medianAbsoluteDeviation(values) {
+  if (values.length === 0) return null;
+  const center = median(values);
+  return 1.4826 * median(values.map(value => Math.abs(value - center)));
+}
+
+function resample(values, random) {
+  const drawn = new Array(values.length);
+  for (let index = 0; index < values.length; index += 1) {
+    drawn[index] = values[Math.floor(random() * values.length)];
+  }
+  return drawn;
+}
+
+// Percentile bootstrap interval for an arbitrary statistic of one sample.
+export function bootstrapInterval(values, statistic = median, options = {}) {
+  const {
+    iterations = BOOTSTRAP_ITERATIONS,
+    confidence = DEFAULT_CONFIDENCE,
+    seed
+  } = options;
+  if (values.length === 0) return null;
+  const random = createRandom(seed);
+  const estimates = new Array(iterations);
+  for (let index = 0; index < iterations; index += 1) {
+    estimates[index] = statistic(resample(values, random));
+  }
+  const alpha = (1 - confidence) / 2;
+  return {
+    estimate: statistic(values),
+    low: quantile(estimates, alpha),
+    high: quantile(estimates, 1 - alpha),
+    confidence
+  };
+}
+
+// Bootstrap interval for the difference between two independent samples.
+export function differenceInterval(
+  treatment,
+  baseline,
+  statistic = median,
+  options = {}
+) {
+  const {
+    iterations = BOOTSTRAP_ITERATIONS,
+    confidence = DEFAULT_CONFIDENCE,
+    seed
+  } = options;
+  if (treatment.length === 0 || baseline.length === 0) return null;
+  const random = createRandom(seed);
+  const estimates = new Array(iterations);
+  for (let index = 0; index < iterations; index += 1) {
+    estimates[index] =
+      statistic(resample(treatment, random)) -
+      statistic(resample(baseline, random));
+  }
+  const alpha = (1 - confidence) / 2;
+  return {
+    estimate: statistic(treatment) - statistic(baseline),
+    low: quantile(estimates, alpha),
+    high: quantile(estimates, 1 - alpha),
+    confidence
+  };
+}
+
+// Bootstrap interval for the mean of within-runner paired differences. This is
+// the estimator to prefer whenever both arms were measured on the same runner,
+// because it cancels the between-runner variance that dominates hosted CI.
+export function pairedInterval(differences, options = {}) {
+  const {
+    iterations = BOOTSTRAP_ITERATIONS,
+    confidence = DEFAULT_CONFIDENCE,
+    seed
+  } = options;
+  if (differences.length === 0) return null;
+  const random = createRandom(seed);
+  const estimates = new Array(iterations);
+  for (let index = 0; index < iterations; index += 1) {
+    estimates[index] = mean(resample(differences, random));
+  }
+  const alpha = (1 - confidence) / 2;
+  return {
+    estimate: mean(differences),
+    low: quantile(estimates, alpha),
+    high: quantile(estimates, 1 - alpha),
+    confidence
+  };
+}
+
+// Two-sided permutation test on the mean of paired differences. Under the null
+// hypothesis the sign of each pair is arbitrary, so we resample signs.
+export function pairedPermutationTest(differences, options = {}) {
+  const {iterations = BOOTSTRAP_ITERATIONS, seed} = options;
+  if (differences.length === 0) return null;
+  const random = createRandom(seed);
+  const observed = Math.abs(mean(differences));
+  let atLeastAsExtreme = 0;
+  for (let index = 0; index < iterations; index += 1) {
+    const flipped = differences.map(value => (random() < 0.5 ? -value : value));
+    if (Math.abs(mean(flipped)) >= observed) atLeastAsExtreme += 1;
+  }
+  // Add-one correction keeps the p-value strictly positive.
+  return (atLeastAsExtreme + 1) / (iterations + 1);
+}
+
+// Hodges-Lehmann shift estimate: the median of all pairwise differences. More
+// robust than a difference of medians when samples are small and quantized.
+export function hodgesLehmann(treatment, baseline) {
+  if (treatment.length === 0 || baseline.length === 0) return null;
+  const differences = [];
+  for (const treatmentValue of treatment) {
+    for (const baselineValue of baseline) {
+      differences.push(treatmentValue - baselineValue);
+    }
+  }
+  return median(differences);
+}
+
+// Turn an interval into a decision. An interval that straddles zero means the
+// benchmark did not resolve the effect, and an effect smaller than the harness
+// noise floor is not trustworthy even when the interval excludes zero.
+export function classify(interval, options = {}) {
+  const {noiseFloor = 0, lowerIsBetter = true} = options;
+  if (!interval) return 'unknown';
+  const {low, high, estimate} = interval;
+  if (low <= 0 && high >= 0) return 'inconclusive';
+  if (Math.abs(estimate) < noiseFloor) return 'within-noise';
+  const improved = lowerIsBetter ? estimate < 0 : estimate > 0;
+  return improved ? 'improvement' : 'regression';
+}
+
+export function formatInterval(interval, {digits = 1, unit = 's'} = {}) {
+  if (!interval) return 'n/a';
+  const {estimate, low, high} = interval;
+  return `${estimate.toFixed(digits)}${unit} (95% CI ${low.toFixed(digits)} to ${high.toFixed(digits)})`;
+}
+
+export function describeVerdict(verdict) {
+  switch (verdict) {
+    case 'improvement':
+      return 'Faster — the interval excludes zero and clears the noise floor.';
+    case 'regression':
+      return 'Slower — the interval excludes zero and clears the noise floor.';
+    case 'within-noise':
+      return 'No usable signal — the effect is smaller than the harness noise floor.';
+    case 'inconclusive':
+      return 'Inconclusive — the confidence interval includes zero; collect more samples.';
+    default:
+      return 'Unknown.';
+  }
+}
diff --git a/scripts/stats.test.mjs b/scripts/stats.test.mjs
new file mode 100644
index 0000000..2b1c487
--- /dev/null
+++ b/scripts/stats.test.mjs
@@ -0,0 +1,112 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+
+import {
+  bootstrapInterval,
+  classify,
+  createRandom,
+  differenceInterval,
+  hodgesLehmann,
+  mean,
+  median,
+  medianAbsoluteDeviation,
+  pairedInterval,
+  pairedPermutationTest,
+  quantile,
+  standardDeviation
+} from './stats.mjs';
+
+test('summarises central tendency and spread', () => {
+  assert.equal(mean([1, 2, 3, 4]), 2.5);
+  assert.equal(median([3, 1, 2]), 2);
+  assert.equal(median([4, 1, 2, 3]), 2.5);
+  assert.equal(quantile([1, 2, 3, 4, 5], 0.5), 3);
+  assert.equal(standardDeviation([2, 2, 2]), 0);
+  assert.equal(standardDeviation([1]), null);
+  assert.equal(medianAbsoluteDeviation([5, 5, 5]), 0);
+  assert.equal(mean([]), null);
+  assert.equal(median([]), null);
+});
+
+test('resampling is deterministic across runs', () => {
+  const values = [1.2, 1.4, 1.1, 1.9, 1.3, 1.5, 1.2, 1.8];
+  const first = bootstrapInterval(values, median, {iterations: 500, seed: 7});
+  const second = bootstrapInterval(values, median, {iterations: 500, seed: 7});
+  assert.deepEqual(first, second);
+
+  const random = createRandom(42);
+  const draws = [random(), random(), random()];
+  const replayed = createRandom(42);
+  assert.deepEqual(draws, [replayed(), replayed(), replayed()]);
+});
+
+test('bootstrap interval brackets the point estimate', () => {
+  const values = [2.0, 2.1, 2.2, 2.3, 2.4, 2.5];
+  const interval = bootstrapInterval(values, median, {
+    iterations: 2000,
+    seed: 11
+  });
+  assert.ok(interval.low <= interval.estimate);
+  assert.ok(interval.high >= interval.estimate);
+  assert.equal(interval.confidence, 0.95);
+});
+
+test('detects a real effect and rejects a null effect', () => {
+  // A large, consistent within-runner improvement must be reported.
+  const realEffect = Array.from({length: 12}, (_, index) => -1 + index * 0.01);
+  const realInterval = pairedInterval(realEffect, {iterations: 2000, seed: 3});
+  assert.equal(classify(realInterval, {noiseFloor: 0.05}), 'improvement');
+  assert.ok(pairedPermutationTest(realEffect, {iterations: 2000, seed: 3}) < 0.01);
+
+  // Differences that merely alternate around zero must not be.
+  const noEffect = [0.4, -0.3, 0.2, -0.5, 0.1, -0.2, 0.35, -0.4];
+  const nullInterval = pairedInterval(noEffect, {iterations: 2000, seed: 3});
+  assert.equal(classify(nullInterval, {noiseFloor: 0.05}), 'inconclusive');
+  assert.ok(pairedPermutationTest(noEffect, {iterations: 2000, seed: 3}) > 0.1);
+});
+
+test('suppresses effects smaller than the noise floor', () => {
+  const tiny = Array.from({length: 40}, () => -0.02);
+  const interval = pairedInterval(tiny, {iterations: 1000, seed: 5});
+  // The interval excludes zero because the samples are identical, but the
+  // effect is far below what the harness can resolve.
+  assert.equal(classify(interval, {noiseFloor: 0.25}), 'within-noise');
+  assert.equal(classify(interval, {noiseFloor: 0.001}), 'improvement');
+});
+
+test('classifies direction and missing intervals', () => {
+  assert.equal(classify({low: 0.2, high: 0.8, estimate: 0.5}), 'regression');
+  assert.equal(classify({low: -0.8, high: -0.2, estimate: -0.5}), 'improvement');
+  assert.equal(
+    classify({low: 0.2, high: 0.8, estimate: 0.5}, {lowerIsBetter: false}),
+    'improvement'
+  );
+  assert.equal(classify(null), 'unknown');
+});
+
+test('estimates a shift with Hodges-Lehmann', () => {
+  assert.equal(hodgesLehmann([3, 4, 5], [1, 2, 3]), 2);
+  assert.equal(hodgesLehmann([], [1]), null);
+});
+
+// Regression test built from two real back-to-back runs of the previous
+// harness against an unchanged actions/setup-java@v4.8.0. Because the code was
+// identical, the true effect is exactly zero, yet the old unpaired design
+// separated the two samples cleanly. This test pins that failure so nobody
+// reintroduces unpaired sampling: better statistics alone cannot rescue it,
+// because the samples really do differ once between-runner drift is baked in.
+test('unpaired sampling produces a false positive on identical code', () => {
+  const runOne = [2, 1, 3, 1, 1, 2, 3, 2, 2, 2];
+  const runTwo = [3, 4, 2, 4, 4, 3, 4, 3, 3, 1];
+  const interval = differenceInterval(runTwo, runOne, median, {
+    iterations: 4000,
+    seed: 13
+  });
+  // The interval excludes zero even though both samples measure the same code.
+  assert.ok(interval.low > 0 || interval.high < 0);
+  assert.equal(classify(interval, {noiseFloor: 0}), 'regression');
+
+  // The only defence left for unpaired data is a noise floor calibrated from
+  // observed run-to-run drift, which suppresses the spurious verdict.
+  assert.equal(classify(interval, {noiseFloor: 1.5}), 'within-noise');
+});

From 1a624dafbfc26c1278524e1c709b9f4b85c1a48e Mon Sep 17 00:00:00 2001
From: Bruno Borges <bruno.borges@microsoft.com>
Date: Wed, 5 Aug 2026 00:08:40 -0400
Subject: [PATCH 02/19] Keep the wrapper cache key stable between seeding and
 measuring

The first live run missed the seeded wrapper cache on every runner, so all six
measure jobs failed with the wrapper fixture absent.

setup-java keys its separate wrapper cache on
**/.mvn/wrapper/maven-wrapper.properties, and cache-dependency-path overrides
the pattern for the main dependency cache only, not for the additional one.
setup-java itself carries such a file under __tests__/cache/maven, so the
derived key depends on how many copies of the action are checked out. The seed
jobs checked out one ref and the measure job checked out two, which produced
different wrapper keys and a permanent miss.

Every job now checks out both refs, including the arm it does not use, so all
jobs hash an identical tree. The fixture check also reports a missing file
directly instead of letting wc fail with a bare no-such-file error.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 71c45320-1417-4029-8402-69075d61dac1
---
 .github/workflows/focused-cache-restore.yml | 20 ++++++++++++++
 scripts/focused-cache-restore.sh            | 30 ++++++++++++---------
 2 files changed, 38 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/focused-cache-restore.yml b/.github/workflows/focused-cache-restore.yml
index 9f74015..f974abb 100644
--- a/.github/workflows/focused-cache-restore.yml
+++ b/.github/workflows/focused-cache-restore.yml
@@ -60,6 +60,12 @@ jobs:
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
         with:
           persist-credentials: false
+      # Both refs are checked out in every job, including the arm this job does
+      # not use. The wrapper cache key hashes **/.mvn/wrapper/maven-wrapper.properties,
+      # which cache-dependency-path does not override, and setup-java carries such
+      # a file under __tests__. A job that checked out one ref would therefore
+      # derive a different wrapper key than one that checked out two, and the
+      # seeded wrapper cache would never be found.
       - name: Check out baseline setup-java
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
         with:
@@ -67,6 +73,13 @@ jobs:
           path: baseline
           persist-credentials: false
           ref: ${{ inputs.baseline-ref }}
+      - name: Check out candidate setup-java
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          repository: ${{ inputs.setup-java-repository }}
+          path: candidate
+          persist-credentials: false
+          ref: ${{ inputs.candidate-ref }}
       - name: Prepare cache identity
         run: bash scripts/focused-cache-restore.sh prepare "focused-baseline-${{ github.run_id }}"
       - name: Setup Java baseline
@@ -87,6 +100,13 @@ jobs:
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
         with:
           persist-credentials: false
+      - name: Check out baseline setup-java
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          repository: ${{ inputs.setup-java-repository }}
+          path: baseline
+          persist-credentials: false
+          ref: ${{ inputs.baseline-ref }}
       - name: Check out candidate setup-java
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
         with:
diff --git a/scripts/focused-cache-restore.sh b/scripts/focused-cache-restore.sh
index 2876334..b1ab960 100755
--- a/scripts/focused-cache-restore.sh
+++ b/scripts/focused-cache-restore.sh
@@ -35,22 +35,28 @@ case "$command" in
     ;;
   verify)
     arm=${2:?arm is required}
-    actual=$(wc -c < "$dependency_fixture")
-    expected=$((DEPENDENCY_FIXTURE_MIB * 1024 * 1024))
-    if [ "$actual" -ne "$expected" ]; then
-      echo "Dependency fixture for $arm is $actual bytes, expected $expected" >&2
-      exit 1
-    fi
+    check_fixture() {
+      local label=$1 path=$2 expected=$3
+      if [ ! -f "$path" ]; then
+        echo "$label fixture for $arm is missing at $path;" \
+          "the seeded cache was not restored" >&2
+        exit 1
+      fi
+      local actual
+      actual=$(wc -c < "$path")
+      if [ "$actual" -ne "$expected" ]; then
+        echo "$label fixture for $arm is $actual bytes, expected $expected" >&2
+        exit 1
+      fi
+    }
+    check_fixture Dependency "$dependency_fixture" \
+      "$((DEPENDENCY_FIXTURE_MIB * 1024 * 1024))"
     # Only the candidate arm is expected to maintain a separate wrapper cache;
     # a baseline that also produces one is fine, so this check is arm-specific
     # and non-fatal when the fixture is simply absent for the baseline.
     if [ "$arm" = "candidate" ]; then
-      actual=$(wc -c < "$wrapper_fixture")
-      expected=$((WRAPPER_FIXTURE_MIB * 1024 * 1024))
-      if [ "$actual" -ne "$expected" ]; then
-        echo "Wrapper fixture for $arm is $actual bytes, expected $expected" >&2
-        exit 1
-      fi
+      check_fixture Wrapper "$wrapper_fixture" \
+        "$((WRAPPER_FIXTURE_MIB * 1024 * 1024))"
     fi
     ;;
   *)

From 7761291cc63fd45ecfee48f37362b0a8eb1c6a9d Mon Sep 17 00:00:00 2001
From: Bruno Borges <bruno.borges@microsoft.com>
Date: Wed, 5 Aug 2026 00:13:03 -0400
Subject: [PATCH 03/19] Restore one shared cache entry in both arms

An A/A run of the new workflow, with baseline-ref and candidate-ref both set to
main, reported a 0.859s improvement on identical code. The A/A control was flat
at -0.060s, so slot ordering was not the cause.

The cause was that each arm seeded its own cache entry. A cache entry's download
throughput depends on where the service placed the stored blob, and that
placement is fixed for the life of the entry. In one job on one runner the
baseline blob was served at 59.4 and 60.3 MB/s while the candidate blob was
served at 131.8 and 105.1 MB/s, and the same ordering held on every runner.
Because that bias is constant across runners rather than random, pairing cannot
remove it, and additional samples only tighten the interval around the wrong
answer. The arm was confounded with the blob.

A single entry is now seeded, from the candidate so the wrapper cache is
populated, and both arms restore it. The blob is therefore held constant across
the comparison by construction rather than assumed not to matter.

The README documents the confound and states that an A/A run is the check to
perform after changing the harness.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 71c45320-1417-4029-8402-69075d61dac1
---
 .github/workflows/focused-cache-restore.yml |  61 ++---
 README.md                                   |   4 +
 scripts/measure.mjs                         |  29 ++-
 scripts/report-focused.mjs                  | 256 ++++++++++----------
 scripts/report-focused.test.mjs             |  64 ++---
 scripts/report-jdk-cache.test.mjs           |  82 +++----
 scripts/report.test.mjs                     |  90 +++----
 scripts/stats.mjs                           |  58 ++---
 scripts/stats.test.mjs                      |  84 ++++---
 9 files changed, 372 insertions(+), 356 deletions(-)

diff --git a/.github/workflows/focused-cache-restore.yml b/.github/workflows/focused-cache-restore.yml
index f974abb..20577a5 100644
--- a/.github/workflows/focused-cache-restore.yml
+++ b/.github/workflows/focused-cache-restore.yml
@@ -52,8 +52,16 @@ defaults:
     shell: bash
 
 jobs:
-  baseline-seed:
-    name: Seed baseline cache
+  # A single cache entry is seeded and both arms restore it. Giving each arm its
+  # own entry confounds the arm with the stored blob: a cache entry's download
+  # throughput depends on where the service placed it, and that bias is the same
+  # on every runner, so pairing cannot remove it and more samples only tighten
+  # the interval around the wrong answer. An A/A run of this workflow with one
+  # entry per arm reported a 0.859s improvement on identical code, with the
+  # baseline blob served at ~60 MB/s and the candidate blob at ~105-130 MB/s on
+  # the same runner.
+  seed:
+    name: Seed cache
     runs-on: ubuntu-24.04
     steps:
       - name: Check out benchmark repository
@@ -81,41 +89,10 @@ jobs:
           persist-credentials: false
           ref: ${{ inputs.candidate-ref }}
       - name: Prepare cache identity
-        run: bash scripts/focused-cache-restore.sh prepare "focused-baseline-${{ github.run_id }}"
-      - name: Setup Java baseline
-        uses: ./baseline
-        with:
-          distribution: temurin
-          java-version: ${{ env.JAVA_VERSION }}
-          cache: maven
-          cache-dependency-path: .focused-cache-key
-      - name: Create synthetic cache fixtures
-        run: bash scripts/focused-cache-restore.sh seed-fixtures
-
-  candidate-seed:
-    name: Seed candidate caches
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Check out benchmark repository
-        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          persist-credentials: false
-      - name: Check out baseline setup-java
-        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          repository: ${{ inputs.setup-java-repository }}
-          path: baseline
-          persist-credentials: false
-          ref: ${{ inputs.baseline-ref }}
-      - name: Check out candidate setup-java
-        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          repository: ${{ inputs.setup-java-repository }}
-          path: candidate
-          persist-credentials: false
-          ref: ${{ inputs.candidate-ref }}
-      - name: Prepare cache identity
-        run: bash scripts/focused-cache-restore.sh prepare "focused-candidate-${{ github.run_id }}"
+        run: bash scripts/focused-cache-restore.sh prepare "focused-${{ github.run_id }}"
+      # Seeded with the candidate so that the wrapper cache, which only newer
+      # revisions maintain, is populated too. The baseline simply does not
+      # restore it.
       - name: Setup Java candidate
         uses: ./candidate
         with:
@@ -132,7 +109,7 @@ jobs:
   # twice per runner also yields a free A/A noise-floor estimate.
   measure:
     name: paired / ${{ matrix.sample }}
-    needs: [baseline-seed, candidate-seed]
+    needs: seed
     runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
@@ -159,7 +136,7 @@ jobs:
           ref: ${{ inputs.candidate-ref }}
 
       - name: Reset slot 1
-        run: bash scripts/focused-cache-restore.sh reset "focused-baseline-${{ github.run_id }}"
+        run: bash scripts/focused-cache-restore.sh reset "focused-${{ github.run_id }}"
       - name: Start slot 1 timer
         run: node scripts/measure.mjs start
       - name: Slot 1 setup (baseline)
@@ -175,7 +152,7 @@ jobs:
         run: bash scripts/focused-cache-restore.sh verify baseline
 
       - name: Reset slot 2
-        run: bash scripts/focused-cache-restore.sh reset "focused-candidate-${{ github.run_id }}"
+        run: bash scripts/focused-cache-restore.sh reset "focused-${{ github.run_id }}"
       - name: Start slot 2 timer
         run: node scripts/measure.mjs start
       - name: Slot 2 setup (candidate)
@@ -191,7 +168,7 @@ jobs:
         run: bash scripts/focused-cache-restore.sh verify candidate
 
       - name: Reset slot 3
-        run: bash scripts/focused-cache-restore.sh reset "focused-candidate-${{ github.run_id }}"
+        run: bash scripts/focused-cache-restore.sh reset "focused-${{ github.run_id }}"
       - name: Start slot 3 timer
         run: node scripts/measure.mjs start
       - name: Slot 3 setup (candidate)
@@ -207,7 +184,7 @@ jobs:
         run: bash scripts/focused-cache-restore.sh verify candidate
 
       - name: Reset slot 4
-        run: bash scripts/focused-cache-restore.sh reset "focused-baseline-${{ github.run_id }}"
+        run: bash scripts/focused-cache-restore.sh reset "focused-${{ github.run_id }}"
       - name: Start slot 4 timer
         run: node scripts/measure.mjs start
       - name: Slot 4 setup (baseline)
diff --git a/README.md b/README.md
index 948c85a..72b7bea 100644
--- a/README.md
+++ b/README.md
@@ -43,6 +43,8 @@ This workflow is the reference for how a comparison should be measured here. Thr
 
 **Same-runner pairing.** Between-runner variance on hosted runners is larger than the effects under test, and it cannot be averaged away by adding more independent jobs to each arm. Both arms run in the *same* job in ABBA order — baseline, candidate, candidate, baseline — so each runner yields one paired difference with the runner's own speed cancelled out. The mirrored order also cancels drift across the four slots. Each measured slot deletes `~/.m2` first so every restore extracts into an empty tree.
 
+**One cache, both arms.** A cache entry's download throughput depends on where the service placed the stored blob, and that placement is fixed for the life of the entry. Giving each arm its own seeded cache therefore confounds the arm with its blob, and because the bias is identical on every runner, pairing cannot remove it and more samples only tighten the interval around the wrong answer. A single entry is seeded and both arms restore it.
+
 **Intervals, not point estimates.** `scripts/stats.mjs` reports a bootstrap 95% confidence interval, a permutation p-value, and a Hodges-Lehmann shift for every comparison, and turns them into an explicit verdict. A comparison whose interval includes zero is reported as `inconclusive` rather than as a number that looks like a result.
 
 The report also publishes two guard rails:
@@ -50,6 +52,8 @@ The report also publishes two guard rails:
 - A **noise floor**, the median spread between the two slots of the same arm on one runner. An effect smaller than this is reported as `within-noise` even when its interval excludes zero.
 - An **A/A control**, the same estimator applied to the baseline against itself. It costs no extra jobs because each arm is already measured twice per runner. A healthy run reports `within-noise` or `inconclusive`; anything else means slot ordering is biasing the results and the headline verdict cannot be trusted.
 
+Run the workflow with `baseline-ref` and `candidate-ref` set to the same value after changing it. The true effect is then exactly zero, and any verdict other than `inconclusive` or `within-noise` is a defect in the harness rather than a finding. That check is what surfaced the per-arm cache confound described above: on identical code it reported a 0.859 s improvement, with the baseline blob served at ~60 MB/s and the candidate blob at ~105–130 MB/s on the same runner in the same job.
+
 Point `baseline-ref` and `candidate-ref` at any two refs — including a PR branch — to check whether a change delivers a real improvement.
 
 #### Why this replaced the previous design
diff --git a/scripts/measure.mjs b/scripts/measure.mjs
index f2e5de3..aaef174 100644
--- a/scripts/measure.mjs
+++ b/scripts/measure.mjs
@@ -6,18 +6,18 @@
 // the same magnitude as the effects these benchmarks try to detect. Timing the
 // step from inside the job instead keeps millisecond precision.
 
-import {appendFile, mkdir, readFile, writeFile} from 'node:fs/promises';
-import {dirname} from 'node:path';
-import {pathToFileURL} from 'node:url';
+import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises";
+import { dirname } from "node:path";
+import { pathToFileURL } from "node:url";
 
-const CLOCK_FILE = '.benchmark-clock';
+const CLOCK_FILE = ".benchmark-clock";
 
 export async function start(clockFile = CLOCK_FILE) {
   await writeFile(clockFile, String(Date.now()));
 }
 
 export async function stop(clockFile = CLOCK_FILE) {
-  const started = Number(await readFile(clockFile, 'utf8'));
+  const started = Number(await readFile(clockFile, "utf8"));
   if (!Number.isFinite(started)) {
     throw new Error(`No valid start timestamp in ${clockFile}`);
   }
@@ -25,28 +25,28 @@ export async function stop(clockFile = CLOCK_FILE) {
 }
 
 function csvValue(value) {
-  return `"${String(value ?? '').replaceAll('"', '""')}"`;
+  return `"${String(value ?? "").replaceAll('"', '""')}"`;
 }
 
 export async function record(resultsFile, fields, elapsedMs) {
-  await mkdir(dirname(resultsFile), {recursive: true});
-  const line = [...fields, elapsedMs].map(csvValue).join(',');
+  await mkdir(dirname(resultsFile), { recursive: true });
+  const line = [...fields, elapsedMs].map(csvValue).join(",");
   await appendFile(resultsFile, `${line}\n`);
 }
 
 export async function main(argv) {
   const [command, ...rest] = argv;
   switch (command) {
-    case 'start': {
+    case "start": {
       await start(rest[0] ?? CLOCK_FILE);
       return;
     }
-    case 'record': {
+    case "record": {
       const [resultsFile, ...fields] = rest;
-      if (!resultsFile) throw new Error('record requires a results file');
+      if (!resultsFile) throw new Error("record requires a results file");
       const elapsedMs = await stop(CLOCK_FILE);
       await record(resultsFile, fields, elapsedMs);
-      console.log(`${fields.join('/')}: ${elapsedMs} ms`);
+      console.log(`${fields.join("/")}: ${elapsedMs} ms`);
       return;
     }
     default:
@@ -54,6 +54,9 @@ export async function main(argv) {
   }
 }
 
-if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+if (
+  process.argv[1] &&
+  import.meta.url === pathToFileURL(process.argv[1]).href
+) {
   await main(process.argv.slice(2));
 }
diff --git a/scripts/report-focused.mjs b/scripts/report-focused.mjs
index 25bc917..0d1a80f 100644
--- a/scripts/report-focused.mjs
+++ b/scripts/report-focused.mjs
@@ -1,8 +1,14 @@
-import {appendFile, mkdir, readdir, readFile, writeFile} from 'node:fs/promises';
-import {join} from 'node:path';
-import {pathToFileURL} from 'node:url';
+import {
+  appendFile,
+  mkdir,
+  readdir,
+  readFile,
+  writeFile,
+} from "node:fs/promises";
+import { join } from "node:path";
+import { pathToFileURL } from "node:url";
 
-import {hashFilesSingle} from './report.mjs';
+import { hashFilesSingle } from "./report.mjs";
 import {
   classify,
   describeVerdict,
@@ -14,26 +20,26 @@ import {
   pairedInterval,
   pairedPermutationTest,
   quantile,
-  standardDeviation
-} from './stats.mjs';
+  standardDeviation,
+} from "./stats.mjs";
 
-const API_VERSION = '2022-11-28';
-const RESULTS_DIR = '.benchmark-results';
+const API_VERSION = "2022-11-28";
+const RESULTS_DIR = ".benchmark-results";
 
 // Every measurement job uploads its own CSV so that merging the artifacts
 // cannot overwrite another runner's samples.
 export async function readSampleFiles(directory = RESULTS_DIR) {
   const entries = await readdir(directory);
   const files = entries.filter(
-    entry => entry.startsWith('focused-timings') && entry.endsWith('.csv')
+    (entry) => entry.startsWith("focused-timings") && entry.endsWith(".csv"),
   );
   if (files.length === 0) {
     throw new Error(`No focused timing CSVs found in ${directory}`);
   }
   const contents = await Promise.all(
-    files.sort().map(file => readFile(join(directory, file), 'utf8'))
+    files.sort().map((file) => readFile(join(directory, file), "utf8")),
   );
-  return contents.join('\n');
+  return contents.join("\n");
 }
 
 // Each runner measures four slots in ABBA order: baseline, candidate,
@@ -43,17 +49,17 @@ export async function readSampleFiles(directory = RESULTS_DIR) {
 export function parseSamples(csv) {
   return csv
     .trim()
-    .split('\n')
+    .split("\n")
     .filter(Boolean)
-    .map(line => {
+    .map((line) => {
       const [sample, arm, slot, elapsedMs] = line
-        .split(',')
-        .map(value => value.replace(/^"|"$/g, ''));
+        .split(",")
+        .map((value) => value.replace(/^"|"$/g, ""));
       return {
         sample: Number(sample),
         arm,
         slot: Number(slot),
-        seconds: Number(elapsedMs) / 1000
+        seconds: Number(elapsedMs) / 1000,
       };
     });
 }
@@ -63,22 +69,22 @@ export function parseSamples(csv) {
 export function buildPairs(rows) {
   const bySample = new Map();
   for (const row of rows) {
-    const entry = bySample.get(row.sample) ?? {baseline: [], candidate: []};
+    const entry = bySample.get(row.sample) ?? { baseline: [], candidate: [] };
     if (!entry[row.arm]) continue;
     entry[row.arm].push(row);
     bySample.set(row.sample, entry);
   }
   const pairs = [];
   for (const [sample, entry] of [...bySample.entries()].sort(
-    (a, b) => a[0] - b[0]
+    (a, b) => a[0] - b[0],
   )) {
     if (entry.baseline.length !== 2 || entry.candidate.length !== 2) continue;
     const baselineSlots = entry.baseline
       .sort((a, b) => a.slot - b.slot)
-      .map(row => row.seconds);
+      .map((row) => row.seconds);
     const candidateSlots = entry.candidate
       .sort((a, b) => a.slot - b.slot)
-      .map(row => row.seconds);
+      .map((row) => row.seconds);
     pairs.push({
       sample,
       baselineSlots,
@@ -87,7 +93,7 @@ export function buildPairs(rows) {
       candidate: mean(candidateSlots),
       difference: mean(candidateSlots) - mean(baselineSlots),
       baselineRepeatDelta: baselineSlots[1] - baselineSlots[0],
-      candidateRepeatDelta: candidateSlots[1] - candidateSlots[0]
+      candidateRepeatDelta: candidateSlots[1] - candidateSlots[0],
     });
   }
   return pairs;
@@ -97,8 +103,8 @@ export function buildPairs(rows) {
 // implementation varies between its two slots on one runner.
 export function noiseFloor(pairs) {
   const repeats = [
-    ...pairs.map(pair => Math.abs(pair.baselineRepeatDelta)),
-    ...pairs.map(pair => Math.abs(pair.candidateRepeatDelta))
+    ...pairs.map((pair) => Math.abs(pair.baselineRepeatDelta)),
+    ...pairs.map((pair) => Math.abs(pair.candidateRepeatDelta)),
   ];
   if (repeats.length === 0) return 0;
   return quantile(repeats, 0.5);
@@ -112,57 +118,57 @@ function armSummary(name, values) {
     medianSeconds: median(values),
     standardDeviationSeconds: standardDeviation(values),
     madSeconds: medianAbsoluteDeviation(values),
-    p95Seconds: quantile(values, 0.95)
+    p95Seconds: quantile(values, 0.95),
   };
 }
 
 export function analyze(rows) {
   const pairs = buildPairs(rows);
-  const differences = pairs.map(pair => pair.difference);
-  const baselineValues = pairs.map(pair => pair.baseline);
-  const candidateValues = pairs.map(pair => pair.candidate);
+  const differences = pairs.map((pair) => pair.difference);
+  const baselineValues = pairs.map((pair) => pair.baseline);
+  const candidateValues = pairs.map((pair) => pair.candidate);
   const floor = noiseFloor(pairs);
-  const interval = pairedInterval(differences, {seed: 1});
+  const interval = pairedInterval(differences, { seed: 1 });
   // The same estimator applied to the within-arm repeats. A trustworthy harness
   // must not resolve a difference here, because it compares an arm with itself.
   // Judged against the same noise floor as the real effect, so a healthy run
   // reports `within-noise` or `inconclusive`.
   const controlInterval = pairedInterval(
-    pairs.map(pair => pair.baselineRepeatDelta),
-    {seed: 2}
+    pairs.map((pair) => pair.baselineRepeatDelta),
+    { seed: 2 },
   );
   return {
     pairs,
     noiseFloorSeconds: floor,
-    baseline: armSummary('baseline', baselineValues),
-    candidate: armSummary('candidate', candidateValues),
+    baseline: armSummary("baseline", baselineValues),
+    candidate: armSummary("candidate", candidateValues),
     interval,
-    pValue: pairedPermutationTest(differences, {seed: 3}),
+    pValue: pairedPermutationTest(differences, { seed: 3 }),
     shiftSeconds: hodgesLehmann(candidateValues, baselineValues),
-    verdict: classify(interval, {noiseFloor: floor}),
+    verdict: classify(interval, { noiseFloor: floor }),
     control: {
       interval: controlInterval,
-      verdict: classify(controlInterval, {noiseFloor: floor})
-    }
+      verdict: classify(controlInterval, { noiseFloor: floor }),
+    },
   };
 }
 
 function csvValue(value) {
-  return `"${String(value ?? '').replaceAll('"', '""')}"`;
+  return `"${String(value ?? "").replaceAll('"', '""')}"`;
 }
 
 async function api(path, token, options = {}) {
   const response = await fetch(`https://api.github.com${path}`, {
     ...options,
     headers: {
-      Accept: 'application/vnd.github+json',
+      Accept: "application/vnd.github+json",
       Authorization: `Bearer ${token}`,
-      'X-GitHub-Api-Version': API_VERSION,
-      ...options.headers
-    }
+      "X-GitHub-Api-Version": API_VERSION,
+      ...options.headers,
+    },
   });
   if (!response.ok) {
-    throw new Error(`${options.method ?? 'GET'} ${path}: ${response.status}`);
+    throw new Error(`${options.method ?? "GET"} ${path}: ${response.status}`);
   }
   if (response.status === 204) return null;
   return response.json();
@@ -171,10 +177,10 @@ async function api(path, token, options = {}) {
 async function allPages(path, field, token) {
   const values = [];
   for (let page = 1; ; page += 1) {
-    const separator = path.includes('?') ? '&' : '?';
+    const separator = path.includes("?") ? "&" : "?";
     const response = await api(
       `${path}${separator}per_page=100&page=${page}`,
-      token
+      token,
     );
     values.push(...response[field]);
     if (response[field].length < 100) return values;
@@ -182,118 +188,121 @@ async function allPages(path, field, token) {
 }
 
 export function markdown(metadata, analysis, caches) {
-  const {baseline, candidate, interval, control} = analysis;
+  const { baseline, candidate, interval, control } = analysis;
   const lines = [
-    '# Focused cache restore benchmark',
-    '',
+    "# Focused cache restore benchmark",
+    "",
     `Temurin ${metadata.javaVersion}, ${analysis.pairs.length} runners x 2 paired observations, run ${metadata.runId}.`,
     `Baseline \`${metadata.baselineRef}\` vs candidate \`${metadata.candidateRef}\` from \`${metadata.setupJavaRepository}\`.`,
-    '',
-    '## Verdict',
-    '',
+    "",
+    "## Verdict",
+    "",
     `**${describeVerdict(analysis.verdict)}**`,
-    '',
-    `Paired difference (candidate - baseline): **${formatInterval(interval, {digits: 3})}**.`,
+    "",
+    `Paired difference (candidate - baseline): **${formatInterval(interval, { digits: 3 })}**.`,
     `Permutation p-value: ${analysis.pValue.toFixed(3)}. Hodges-Lehmann shift: ${analysis.shiftSeconds.toFixed(3)}s.`,
     `Harness noise floor: ${analysis.noiseFloorSeconds.toFixed(3)}s (median within-runner repeat spread).`,
-    '',
-    `A/A control (baseline against itself) reports **${control.verdict}** at ${formatInterval(control.interval, {digits: 3})}. A healthy harness reports \`within-noise\` or \`inconclusive\` here; an \`improvement\` or \`regression\` means slot ordering is biasing results and the verdict above cannot be trusted.`,
-    '',
-    '## Arms',
-    '',
-    '| Arm | Runners | Mean (s) | Median (s) | SD (s) | MAD (s) | p95 (s) |',
-    '| --- | ---: | ---: | ---: | ---: | ---: | ---: |'
+    "",
+    `A/A control (baseline against itself) reports **${control.verdict}** at ${formatInterval(control.interval, { digits: 3 })}. A healthy harness reports \`within-noise\` or \`inconclusive\` here; an \`improvement\` or \`regression\` means slot ordering is biasing results and the verdict above cannot be trusted.`,
+    "",
+    "## Arms",
+    "",
+    "| Arm | Runners | Mean (s) | Median (s) | SD (s) | MAD (s) | p95 (s) |",
+    "| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
   ];
   for (const summary of [baseline, candidate]) {
     lines.push(
-      `| ${summary.arm} | ${summary.samples} | ${summary.meanSeconds.toFixed(3)} | ${summary.medianSeconds.toFixed(3)} | ${summary.standardDeviationSeconds?.toFixed(3) ?? 'n/a'} | ${summary.madSeconds.toFixed(3)} | ${summary.p95Seconds.toFixed(3)} |`
+      `| ${summary.arm} | ${summary.samples} | ${summary.meanSeconds.toFixed(3)} | ${summary.medianSeconds.toFixed(3)} | ${summary.standardDeviationSeconds?.toFixed(3) ?? "n/a"} | ${summary.madSeconds.toFixed(3)} | ${summary.p95Seconds.toFixed(3)} |`,
     );
   }
   lines.push(
-    '',
-    'All durations are measured inside the job with millisecond resolution. The Actions API reports step timestamps only to the nearest second, which is too coarse for effects of this size.',
-    '',
-    '## Paired samples',
-    '',
-    '| Runner | baseline slot 1 (s) | candidate slot 2 (s) | candidate slot 3 (s) | baseline slot 4 (s) | Paired delta (s) |',
-    '| ---: | ---: | ---: | ---: | ---: | ---: |'
+    "",
+    "All durations are measured inside the job with millisecond resolution. The Actions API reports step timestamps only to the nearest second, which is too coarse for effects of this size.",
+    "",
+    "## Paired samples",
+    "",
+    "| Runner | baseline slot 1 (s) | candidate slot 2 (s) | candidate slot 3 (s) | baseline slot 4 (s) | Paired delta (s) |",
+    "| ---: | ---: | ---: | ---: | ---: | ---: |",
   );
   for (const pair of analysis.pairs) {
     lines.push(
-      `| ${pair.sample} | ${pair.baselineSlots[0].toFixed(3)} | ${pair.candidateSlots[0].toFixed(3)} | ${pair.candidateSlots[1].toFixed(3)} | ${pair.baselineSlots[1].toFixed(3)} | ${pair.difference.toFixed(3)} |`
+      `| ${pair.sample} | ${pair.baselineSlots[0].toFixed(3)} | ${pair.candidateSlots[0].toFixed(3)} | ${pair.candidateSlots[1].toFixed(3)} | ${pair.baselineSlots[1].toFixed(3)} | ${pair.difference.toFixed(3)} |`,
     );
   }
   lines.push(
-    '',
-    '## Cache fixtures',
-    '',
-    '| Arm | Cache | Size (MiB) |',
-    '| --- | --- | ---: |'
+    "",
+    "## Cache fixtures",
+    "",
+    "Both arms restore the same entries, so the stored blob cannot bias the comparison.",
+    "",
+    "| Cache | Size (MiB) |",
+    "| --- | ---: |",
   );
   for (const cache of caches) {
     lines.push(
-      `| ${cache.arm} | ${cache.type} | ${(cache.sizeBytes / 1024 / 1024).toFixed(1)} |`
+      `| ${cache.type} | ${(cache.sizeBytes / 1024 / 1024).toFixed(1)} |`,
     );
   }
-  return `${lines.join('\n')}\n`;
+  return `${lines.join("\n")}\n`;
 }
 
 export async function main(env = process.env) {
-  const [owner, repo] = env.GITHUB_REPOSITORY.split('/');
+  const [owner, repo] = env.GITHUB_REPOSITORY.split("/");
   const token = env.GH_TOKEN;
   const runId = env.GITHUB_RUN_ID;
   const baselineRef = env.BASELINE_REF;
   const candidateRef = env.CANDIDATE_REF;
   const setupJavaRepository = env.SETUP_JAVA_REPOSITORY;
   if (!owner || !repo || !token || !runId || !baselineRef || !candidateRef) {
-    throw new Error('Missing required GitHub Actions environment variables');
+    throw new Error("Missing required GitHub Actions environment variables");
   }
 
   const rows = parseSamples(await readSampleFiles());
   const analysis = analyze(rows);
   if (analysis.pairs.length === 0) {
-    throw new Error('No complete ABBA samples were collected');
+    throw new Error("No complete ABBA samples were collected");
   }
 
   const cacheEntries = await allPages(
     `/repos/${owner}/${repo}/actions/caches`,
-    'actions_caches',
-    token
+    "actions_caches",
+    token,
   );
 
+  // Both arms restore this single entry, so the stored blob cannot bias the
+  // comparison. The wrapper entry is optional: a baseline that predates wrapper
+  // caching simply never restores it.
+  const benchmarkId = `focused-${runId}`;
+  const expected = [
+    {
+      type: "maven-dependencies",
+      key: `setup-java-Linux-x64-maven-${hashFilesSingle(`${benchmarkId}\n`)}`,
+      required: true,
+    },
+    {
+      type: "maven-wrapper",
+      key: `setup-java-Linux-x64-maven-wrapper-${hashFilesSingle(
+        `wrapperVersion=focused\n# benchmark-id=${benchmarkId}\n`,
+      )}`,
+      required: false,
+    },
+  ];
+
   const caches = [];
-  for (const arm of ['baseline', 'candidate']) {
-    const benchmarkId = `focused-${arm}-${runId}`;
-    const expected = [
-      {
-        type: 'maven-dependencies',
-        key: `setup-java-Linux-x64-maven-${hashFilesSingle(`${benchmarkId}\n`)}`
-      },
-      {
-        type: 'maven-wrapper',
-        key: `setup-java-Linux-x64-maven-wrapper-${hashFilesSingle(
-          `wrapperVersion=focused\n# benchmark-id=${benchmarkId}\n`
-        )}`
+  for (const item of expected) {
+    const entry = cacheEntries.find((cache) => cache.key === item.key);
+    if (!entry) {
+      if (item.required) {
+        throw new Error(`Expected cache not found: ${item.key}`);
       }
-    ];
-    for (const item of expected) {
-      const entry = cacheEntries.find(cache => cache.key === item.key);
-      // Only the dependency cache is guaranteed for every ref; a baseline that
-      // predates wrapper caching simply will not have that entry.
-      if (!entry) {
-        if (item.type === 'maven-dependencies') {
-          throw new Error(`Expected cache not found: ${item.key}`);
-        }
-        continue;
-      }
-      caches.push({
-        arm,
-        type: item.type,
-        id: entry.id,
-        key: item.key,
-        sizeBytes: entry.size_in_bytes
-      });
+      continue;
     }
+    caches.push({
+      type: item.type,
+      id: entry.id,
+      key: item.key,
+      sizeBytes: entry.size_in_bytes,
+    });
   }
 
   const metadata = {
@@ -303,36 +312,39 @@ export async function main(env = process.env) {
     setupJavaRepository,
     baselineRef,
     candidateRef,
-    generatedAt: new Date().toISOString()
+    generatedAt: new Date().toISOString(),
   };
   const report = markdown(metadata, analysis, caches);
 
-  await mkdir('focused-results', {recursive: true});
+  await mkdir("focused-results", { recursive: true });
   await writeFile(
-    'focused-results/results.json',
-    `${JSON.stringify({metadata, analysis, caches}, null, 2)}\n`
+    "focused-results/results.json",
+    `${JSON.stringify({ metadata, analysis, caches }, null, 2)}\n`,
   );
   await writeFile(
-    'focused-results/results.csv',
+    "focused-results/results.csv",
     `sample,arm,slot,seconds\n${rows
-      .map(row =>
-        [row.sample, row.arm, row.slot, row.seconds].map(csvValue).join(',')
+      .map((row) =>
+        [row.sample, row.arm, row.slot, row.seconds].map(csvValue).join(","),
       )
-      .join('\n')}\n`
+      .join("\n")}\n`,
   );
-  await writeFile('focused-results/summary.md', report);
+  await writeFile("focused-results/summary.md", report);
   await appendFile(env.GITHUB_STEP_SUMMARY, report);
 
-  if (env.CLEANUP_CACHES === 'true') {
+  if (env.CLEANUP_CACHES === "true") {
     for (const cache of caches) {
       await api(`/repos/${owner}/${repo}/actions/caches/${cache.id}`, token, {
-        method: 'DELETE'
+        method: "DELETE",
       });
     }
     console.log(`Deleted ${caches.length} focused benchmark caches`);
   }
 }
 
-if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+if (
+  process.argv[1] &&
+  import.meta.url === pathToFileURL(process.argv[1]).href
+) {
   await main();
 }
diff --git a/scripts/report-focused.test.mjs b/scripts/report-focused.test.mjs
index 36032ee..2f9979a 100644
--- a/scripts/report-focused.test.mjs
+++ b/scripts/report-focused.test.mjs
@@ -1,13 +1,13 @@
-import assert from 'node:assert/strict';
-import test from 'node:test';
+import assert from "node:assert/strict";
+import test from "node:test";
 
 import {
   analyze,
   buildPairs,
   markdown,
   noiseFloor,
-  parseSamples
-} from './report-focused.mjs';
+  parseSamples,
+} from "./report-focused.mjs";
 
 const csv = [
   '"1","baseline","1","2400"',
@@ -17,21 +17,21 @@ const csv = [
   '"2","baseline","1","3100"',
   '"2","candidate","2","2600"',
   '"2","candidate","3","2500"',
-  '"2","baseline","4","3000"'
-].join('\n');
+  '"2","baseline","4","3000"',
+].join("\n");
 
-test('parses millisecond samples into seconds', () => {
+test("parses millisecond samples into seconds", () => {
   const rows = parseSamples(csv);
   assert.equal(rows.length, 8);
   assert.deepEqual(rows[0], {
     sample: 1,
-    arm: 'baseline',
+    arm: "baseline",
     slot: 1,
-    seconds: 2.4
+    seconds: 2.4,
   });
 });
 
-test('pairs the two slots per arm within a runner', () => {
+test("pairs the two slots per arm within a runner", () => {
   const pairs = buildPairs(parseSamples(csv));
   assert.equal(pairs.length, 2);
   assert.equal(pairs[0].baseline, 2.45);
@@ -41,20 +41,20 @@ test('pairs the two slots per arm within a runner', () => {
   assert.ok(Math.abs(pairs[0].baselineRepeatDelta - 0.1) < 1e-9);
 });
 
-test('drops runners with incomplete ABBA slots', () => {
+test("drops runners with incomplete ABBA slots", () => {
   const partial = ['"3","baseline","1","2000"', '"3","candidate","2","1900"'];
-  const pairs = buildPairs(parseSamples([csv, ...partial].join('\n')));
+  const pairs = buildPairs(parseSamples([csv, ...partial].join("\n")));
   assert.equal(pairs.length, 2);
-  assert.ok(!pairs.some(pair => pair.sample === 3));
+  assert.ok(!pairs.some((pair) => pair.sample === 3));
 });
 
-test('derives the noise floor from within-arm repeats', () => {
+test("derives the noise floor from within-arm repeats", () => {
   const pairs = buildPairs(parseSamples(csv));
   assert.ok(noiseFloor(pairs) > 0);
   assert.equal(noiseFloor([]), 0);
 });
 
-test('reports a large consistent improvement', () => {
+test("reports a large consistent improvement", () => {
   const rows = [];
   for (let sample = 1; sample <= 10; sample += 1) {
     // Each runner has its own offset, exactly the between-runner variance that
@@ -64,29 +64,31 @@ test('reports a large consistent improvement', () => {
       `"${sample}","baseline","1","${3000 + offset}"`,
       `"${sample}","candidate","2","${2000 + offset}"`,
       `"${sample}","candidate","3","${2020 + offset}"`,
-      `"${sample}","baseline","4","${3020 + offset}"`
+      `"${sample}","baseline","4","${3020 + offset}"`,
     );
   }
-  const analysis = analyze(parseSamples(rows.join('\n')));
-  assert.equal(analysis.verdict, 'improvement');
+  const analysis = analyze(parseSamples(rows.join("\n")));
+  assert.equal(analysis.verdict, "improvement");
   assert.ok(analysis.interval.high < 0);
   assert.ok(analysis.pValue < 0.01);
   // The control compares baseline against itself and must find nothing.
-  assert.ok(['within-noise', 'inconclusive'].includes(analysis.control.verdict));
+  assert.ok(
+    ["within-noise", "inconclusive"].includes(analysis.control.verdict),
+  );
 });
 
-test('renders a report with a verdict and paired samples', () => {
+test("renders a report with a verdict and paired samples", () => {
   const analysis = analyze(parseSamples(csv));
   const report = markdown(
     {
-      runId: '1',
-      javaVersion: '17.0.19+10',
-      setupJavaRepository: 'actions/setup-java',
-      baselineRef: 'v4.8.0',
-      candidateRef: 'main'
+      runId: "1",
+      javaVersion: "17.0.19+10",
+      setupJavaRepository: "actions/setup-java",
+      baselineRef: "v4.8.0",
+      candidateRef: "main",
     },
     analysis,
-    [{arm: 'baseline', type: 'maven-dependencies', sizeBytes: 1024 * 1024}]
+    [{ arm: "baseline", type: "maven-dependencies", sizeBytes: 1024 * 1024 }],
   );
   assert.match(report, /# Focused cache restore benchmark/);
   assert.match(report, /## Verdict/);
@@ -95,7 +97,7 @@ test('renders a report with a verdict and paired samples', () => {
   assert.match(report, /Harness noise floor/);
 });
 
-test('reports an A/A comparison as inconclusive', () => {
+test("reports an A/A comparison as inconclusive", () => {
   const jitter = [120, -90, 200, -160, 70, -110, 180, -140, 60, -50];
   const rows = [];
   for (let sample = 1; sample <= 10; sample += 1) {
@@ -105,10 +107,10 @@ test('reports an A/A comparison as inconclusive', () => {
       `"${sample}","baseline","1","${2500 + offset}"`,
       `"${sample}","candidate","2","${2500 + offset + wobble}"`,
       `"${sample}","candidate","3","${2500 + offset - wobble}"`,
-      `"${sample}","baseline","4","${2500 + offset}"`
+      `"${sample}","baseline","4","${2500 + offset}"`,
     );
   }
-  const analysis = analyze(parseSamples(rows.join('\n')));
-  assert.notEqual(analysis.verdict, 'improvement');
-  assert.notEqual(analysis.verdict, 'regression');
+  const analysis = analyze(parseSamples(rows.join("\n")));
+  assert.notEqual(analysis.verdict, "improvement");
+  assert.notEqual(analysis.verdict, "regression");
 });
diff --git a/scripts/report-jdk-cache.test.mjs b/scripts/report-jdk-cache.test.mjs
index 3462c5f..e9e76a3 100644
--- a/scripts/report-jdk-cache.test.mjs
+++ b/scripts/report-jdk-cache.test.mjs
@@ -1,85 +1,85 @@
-import assert from 'node:assert/strict';
-import test from 'node:test';
+import assert from "node:assert/strict";
+import test from "node:test";
 
 import {
   expectedArmCacheKeys,
   newestJdkCache,
   parseJdkCacheJob,
-  summarizeJdkArm
-} from './report-jdk-cache.mjs';
+  summarizeJdkArm,
+} from "./report-jdk-cache.mjs";
 
-test('parses JDK cache benchmark job names', () => {
-  assert.deepEqual(parseJdkCacheJob('baseline / seed'), {
-    arm: 'baseline',
-    phase: 'seed',
-    sample: null
+test("parses JDK cache benchmark job names", () => {
+  assert.deepEqual(parseJdkCacheJob("baseline / seed"), {
+    arm: "baseline",
+    phase: "seed",
+    sample: null,
   });
-  assert.deepEqual(parseJdkCacheJob('treatment / warm / 10'), {
-    arm: 'treatment',
-    phase: 'warm',
-    sample: 10
+  assert.deepEqual(parseJdkCacheJob("treatment / warm / 10"), {
+    arm: "treatment",
+    phase: "warm",
+    sample: 10,
   });
-  assert.equal(parseJdkCacheJob('baseline / warm'), null);
-  assert.equal(parseJdkCacheJob('Report'), null);
+  assert.equal(parseJdkCacheJob("baseline / warm"), null);
+  assert.equal(parseJdkCacheJob("Report"), null);
 });
 
-test('selects the newest JDK cache entry', () => {
+test("selects the newest JDK cache entry", () => {
   const newest = newestJdkCache([
     {
-      key: 'setup-java-jdk-v1-Linux-x64-old',
-      created_at: '2026-01-01T00:00:00Z'
+      key: "setup-java-jdk-v1-Linux-x64-old",
+      created_at: "2026-01-01T00:00:00Z",
     },
-    {key: 'setup-java-Linux-x64-maven-not-a-jdk'},
+    { key: "setup-java-Linux-x64-maven-not-a-jdk" },
     {
-      key: 'setup-java-jdk-v1-Linux-x64-new',
-      created_at: '2026-01-02T00:00:00Z'
-    }
+      key: "setup-java-jdk-v1-Linux-x64-new",
+      created_at: "2026-01-02T00:00:00Z",
+    },
   ]);
-  assert.equal(newest.key, 'setup-java-jdk-v1-Linux-x64-new');
-  assert.equal(newestJdkCache([{key: 'setup-java-Linux-maven-key'}]), null);
+  assert.equal(newest.key, "setup-java-jdk-v1-Linux-x64-new");
+  assert.equal(newestJdkCache([{ key: "setup-java-Linux-maven-key" }]), null);
 });
 
-test('derives distinct dependency and wrapper cache keys per arm', () => {
-  const baseline = expectedArmCacheKeys('baseline', '42', 'wrapper=true\n');
-  const treatment = expectedArmCacheKeys('treatment', '42', 'wrapper=true\n');
+test("derives distinct dependency and wrapper cache keys per arm", () => {
+  const baseline = expectedArmCacheKeys("baseline", "42", "wrapper=true\n");
+  const treatment = expectedArmCacheKeys("treatment", "42", "wrapper=true\n");
   assert.match(baseline.dependencies, /^setup-java-Linux-x64-maven-/);
   assert.match(baseline.wrapper, /^setup-java-Linux-x64-maven-wrapper-/);
   assert.notEqual(baseline.dependencies, treatment.dependencies);
   assert.notEqual(baseline.wrapper, treatment.wrapper);
 });
 
-test('summarizes seed and warm measurements for an arm', () => {
+test("summarizes seed and warm measurements for an arm", () => {
   const summary = summarizeJdkArm(
     [
       {
-        arm: 'treatment',
-        phase: 'seed',
+        arm: "treatment",
+        phase: "seed",
         setupSeconds: 7,
         buildSeconds: 20,
         postSeconds: 8,
-        jobSeconds: 64
+        jobSeconds: 64,
       },
       {
-        arm: 'treatment',
-        phase: 'warm',
+        arm: "treatment",
+        phase: "warm",
         setupSeconds: 4,
         buildSeconds: 10,
         postSeconds: 0.4,
-        jobSeconds: 30
+        jobSeconds: 30,
       },
       {
-        arm: 'treatment',
-        phase: 'warm',
+        arm: "treatment",
+        phase: "warm",
         setupSeconds: 2,
         buildSeconds: 12,
         postSeconds: 0.2,
-        jobSeconds: 40
-      }
+        jobSeconds: 40,
+      },
     ],
-    'treatment'
+    "treatment",
   );
   assert.deepEqual(summary, {
-    arm: 'treatment',
+    arm: "treatment",
     samples: 2,
     warmSetupSeconds: 3,
     warmBuildSeconds: 11,
@@ -87,6 +87,6 @@ test('summarizes seed and warm measurements for an arm', () => {
     warmJobSeconds: 35,
     coldSetupSeconds: 7,
     coldPostSeconds: 8,
-    estimatedBilledMinutes: 2
+    estimatedBilledMinutes: 2,
   });
 });
diff --git a/scripts/report.test.mjs b/scripts/report.test.mjs
index 5b7bc70..458b545 100644
--- a/scripts/report.test.mjs
+++ b/scripts/report.test.mjs
@@ -1,74 +1,74 @@
-import assert from 'node:assert/strict';
-import test from 'node:test';
+import assert from "node:assert/strict";
+import test from "node:test";
 
 import {
   hashFilesSingle,
   median,
   parseBenchmarkJob,
   secondsBetween,
-  sha256
-} from './report.mjs';
+  sha256,
+} from "./report.mjs";
 
-test('hashes benchmark cache markers', () => {
+test("hashes benchmark cache markers", () => {
   assert.equal(
-    sha256('benchmark\n'),
-    '8f8dbecfd77ab2386b49d723c6b2474f2c22c246805fa0f677bbaf6e4f7bbbfe'
+    sha256("benchmark\n"),
+    "8f8dbecfd77ab2386b49d723c6b2474f2c22c246805fa0f677bbaf6e4f7bbbfe",
   );
   assert.equal(
-    hashFilesSingle('main-microsoft-1-30462682067\n'),
-    'f0c1009aeb8d8582a73a81f3b0146467ed727211971e4052fccd9b7d60817d8f'
+    hashFilesSingle("main-microsoft-1-30462682067\n"),
+    "f0c1009aeb8d8582a73a81f3b0146467ed727211971e4052fccd9b7d60817d8f",
   );
 });
 
-test('calculates medians', () => {
+test("calculates medians", () => {
   assert.equal(median([3, 1, 2]), 2);
   assert.equal(median([4, 1, 2, 3]), 2.5);
   assert.equal(median([null]), null);
 });
 
-test('calculates step duration', () => {
+test("calculates step duration", () => {
   assert.equal(
-    secondsBetween('2026-01-01T00:00:01Z', '2026-01-01T00:00:04.500Z'),
-    3.5
+    secondsBetween("2026-01-01T00:00:01Z", "2026-01-01T00:00:04.500Z"),
+    3.5,
   );
 });
 
-test('parses benchmark job names', () => {
-  assert.deepEqual(parseBenchmarkJob('v1 / cold / zulu / 1'), {
-    version: 'v1',
-    phase: 'cold',
-    distribution: 'zulu',
-    iteration: 1
+test("parses benchmark job names", () => {
+  assert.deepEqual(parseBenchmarkJob("v1 / cold / zulu / 1"), {
+    version: "v1",
+    phase: "cold",
+    distribution: "zulu",
+    iteration: 1,
   });
-  assert.deepEqual(parseBenchmarkJob('v2 / warm / microsoft / 3'), {
-    version: 'v2',
-    phase: 'warm',
-    distribution: 'microsoft',
-    iteration: 3
+  assert.deepEqual(parseBenchmarkJob("v2 / warm / microsoft / 3"), {
+    version: "v2",
+    phase: "warm",
+    distribution: "microsoft",
+    iteration: 3,
   });
-  assert.deepEqual(parseBenchmarkJob('v3 / cold / temurin / 5'), {
-    version: 'v3',
-    phase: 'cold',
-    distribution: 'temurin',
-    iteration: 5
+  assert.deepEqual(parseBenchmarkJob("v3 / cold / temurin / 5"), {
+    version: "v3",
+    phase: "cold",
+    distribution: "temurin",
+    iteration: 5,
   });
-  assert.deepEqual(parseBenchmarkJob('v4 / warm / microsoft / 2'), {
-    version: 'v4',
-    phase: 'warm',
-    distribution: 'microsoft',
-    iteration: 2
+  assert.deepEqual(parseBenchmarkJob("v4 / warm / microsoft / 2"), {
+    version: "v4",
+    phase: "warm",
+    distribution: "microsoft",
+    iteration: 2,
   });
-  assert.deepEqual(parseBenchmarkJob('v5.2 / warm / microsoft / 1'), {
-    version: 'v5.2',
-    phase: 'warm',
-    distribution: 'microsoft',
-    iteration: 1
+  assert.deepEqual(parseBenchmarkJob("v5.2 / warm / microsoft / 1"), {
+    version: "v5.2",
+    phase: "warm",
+    distribution: "microsoft",
+    iteration: 1,
   });
-  assert.deepEqual(parseBenchmarkJob('v5.6 / cold / temurin / 3'), {
-    version: 'v5.6',
-    phase: 'cold',
-    distribution: 'temurin',
-    iteration: 3
+  assert.deepEqual(parseBenchmarkJob("v5.6 / cold / temurin / 3"), {
+    version: "v5.6",
+    phase: "cold",
+    distribution: "temurin",
+    iteration: 3,
   });
-  assert.equal(parseBenchmarkJob('Report'), null);
+  assert.equal(parseBenchmarkJob("Report"), null);
 });
diff --git a/scripts/stats.mjs b/scripts/stats.mjs
index d838f31..7cbde5a 100644
--- a/scripts/stats.mjs
+++ b/scripts/stats.mjs
@@ -55,7 +55,7 @@ export function standardDeviation(values) {
 export function medianAbsoluteDeviation(values) {
   if (values.length === 0) return null;
   const center = median(values);
-  return 1.4826 * median(values.map(value => Math.abs(value - center)));
+  return 1.4826 * median(values.map((value) => Math.abs(value - center)));
 }
 
 function resample(values, random) {
@@ -71,7 +71,7 @@ export function bootstrapInterval(values, statistic = median, options = {}) {
   const {
     iterations = BOOTSTRAP_ITERATIONS,
     confidence = DEFAULT_CONFIDENCE,
-    seed
+    seed,
   } = options;
   if (values.length === 0) return null;
   const random = createRandom(seed);
@@ -84,7 +84,7 @@ export function bootstrapInterval(values, statistic = median, options = {}) {
     estimate: statistic(values),
     low: quantile(estimates, alpha),
     high: quantile(estimates, 1 - alpha),
-    confidence
+    confidence,
   };
 }
 
@@ -93,12 +93,12 @@ export function differenceInterval(
   treatment,
   baseline,
   statistic = median,
-  options = {}
+  options = {},
 ) {
   const {
     iterations = BOOTSTRAP_ITERATIONS,
     confidence = DEFAULT_CONFIDENCE,
-    seed
+    seed,
   } = options;
   if (treatment.length === 0 || baseline.length === 0) return null;
   const random = createRandom(seed);
@@ -113,7 +113,7 @@ export function differenceInterval(
     estimate: statistic(treatment) - statistic(baseline),
     low: quantile(estimates, alpha),
     high: quantile(estimates, 1 - alpha),
-    confidence
+    confidence,
   };
 }
 
@@ -124,7 +124,7 @@ export function pairedInterval(differences, options = {}) {
   const {
     iterations = BOOTSTRAP_ITERATIONS,
     confidence = DEFAULT_CONFIDENCE,
-    seed
+    seed,
   } = options;
   if (differences.length === 0) return null;
   const random = createRandom(seed);
@@ -137,20 +137,22 @@ export function pairedInterval(differences, options = {}) {
     estimate: mean(differences),
     low: quantile(estimates, alpha),
     high: quantile(estimates, 1 - alpha),
-    confidence
+    confidence,
   };
 }
 
 // Two-sided permutation test on the mean of paired differences. Under the null
 // hypothesis the sign of each pair is arbitrary, so we resample signs.
 export function pairedPermutationTest(differences, options = {}) {
-  const {iterations = BOOTSTRAP_ITERATIONS, seed} = options;
+  const { iterations = BOOTSTRAP_ITERATIONS, seed } = options;
   if (differences.length === 0) return null;
   const random = createRandom(seed);
   const observed = Math.abs(mean(differences));
   let atLeastAsExtreme = 0;
   for (let index = 0; index < iterations; index += 1) {
-    const flipped = differences.map(value => (random() < 0.5 ? -value : value));
+    const flipped = differences.map((value) =>
+      random() < 0.5 ? -value : value,
+    );
     if (Math.abs(mean(flipped)) >= observed) atLeastAsExtreme += 1;
   }
   // Add-one correction keeps the p-value strictly positive.
@@ -174,32 +176,32 @@ export function hodgesLehmann(treatment, baseline) {
 // benchmark did not resolve the effect, and an effect smaller than the harness
 // noise floor is not trustworthy even when the interval excludes zero.
 export function classify(interval, options = {}) {
-  const {noiseFloor = 0, lowerIsBetter = true} = options;
-  if (!interval) return 'unknown';
-  const {low, high, estimate} = interval;
-  if (low <= 0 && high >= 0) return 'inconclusive';
-  if (Math.abs(estimate) < noiseFloor) return 'within-noise';
+  const { noiseFloor = 0, lowerIsBetter = true } = options;
+  if (!interval) return "unknown";
+  const { low, high, estimate } = interval;
+  if (low <= 0 && high >= 0) return "inconclusive";
+  if (Math.abs(estimate) < noiseFloor) return "within-noise";
   const improved = lowerIsBetter ? estimate < 0 : estimate > 0;
-  return improved ? 'improvement' : 'regression';
+  return improved ? "improvement" : "regression";
 }
 
-export function formatInterval(interval, {digits = 1, unit = 's'} = {}) {
-  if (!interval) return 'n/a';
-  const {estimate, low, high} = interval;
+export function formatInterval(interval, { digits = 1, unit = "s" } = {}) {
+  if (!interval) return "n/a";
+  const { estimate, low, high } = interval;
   return `${estimate.toFixed(digits)}${unit} (95% CI ${low.toFixed(digits)} to ${high.toFixed(digits)})`;
 }
 
 export function describeVerdict(verdict) {
   switch (verdict) {
-    case 'improvement':
-      return 'Faster — the interval excludes zero and clears the noise floor.';
-    case 'regression':
-      return 'Slower — the interval excludes zero and clears the noise floor.';
-    case 'within-noise':
-      return 'No usable signal — the effect is smaller than the harness noise floor.';
-    case 'inconclusive':
-      return 'Inconclusive — the confidence interval includes zero; collect more samples.';
+    case "improvement":
+      return "Faster — the interval excludes zero and clears the noise floor.";
+    case "regression":
+      return "Slower — the interval excludes zero and clears the noise floor.";
+    case "within-noise":
+      return "No usable signal — the effect is smaller than the harness noise floor.";
+    case "inconclusive":
+      return "Inconclusive — the confidence interval includes zero; collect more samples.";
     default:
-      return 'Unknown.';
+      return "Unknown.";
   }
 }
diff --git a/scripts/stats.test.mjs b/scripts/stats.test.mjs
index 2b1c487..e3fe031 100644
--- a/scripts/stats.test.mjs
+++ b/scripts/stats.test.mjs
@@ -1,5 +1,5 @@
-import assert from 'node:assert/strict';
-import test from 'node:test';
+import assert from "node:assert/strict";
+import test from "node:test";
 
 import {
   bootstrapInterval,
@@ -13,10 +13,10 @@ import {
   pairedInterval,
   pairedPermutationTest,
   quantile,
-  standardDeviation
-} from './stats.mjs';
+  standardDeviation,
+} from "./stats.mjs";
 
-test('summarises central tendency and spread', () => {
+test("summarises central tendency and spread", () => {
   assert.equal(mean([1, 2, 3, 4]), 2.5);
   assert.equal(median([3, 1, 2]), 2);
   assert.equal(median([4, 1, 2, 3]), 2.5);
@@ -28,10 +28,13 @@ test('summarises central tendency and spread', () => {
   assert.equal(median([]), null);
 });
 
-test('resampling is deterministic across runs', () => {
+test("resampling is deterministic across runs", () => {
   const values = [1.2, 1.4, 1.1, 1.9, 1.3, 1.5, 1.2, 1.8];
-  const first = bootstrapInterval(values, median, {iterations: 500, seed: 7});
-  const second = bootstrapInterval(values, median, {iterations: 500, seed: 7});
+  const first = bootstrapInterval(values, median, { iterations: 500, seed: 7 });
+  const second = bootstrapInterval(values, median, {
+    iterations: 500,
+    seed: 7,
+  });
   assert.deepEqual(first, second);
 
   const random = createRandom(42);
@@ -40,51 +43,64 @@ test('resampling is deterministic across runs', () => {
   assert.deepEqual(draws, [replayed(), replayed(), replayed()]);
 });
 
-test('bootstrap interval brackets the point estimate', () => {
+test("bootstrap interval brackets the point estimate", () => {
   const values = [2.0, 2.1, 2.2, 2.3, 2.4, 2.5];
   const interval = bootstrapInterval(values, median, {
     iterations: 2000,
-    seed: 11
+    seed: 11,
   });
   assert.ok(interval.low <= interval.estimate);
   assert.ok(interval.high >= interval.estimate);
   assert.equal(interval.confidence, 0.95);
 });
 
-test('detects a real effect and rejects a null effect', () => {
+test("detects a real effect and rejects a null effect", () => {
   // A large, consistent within-runner improvement must be reported.
-  const realEffect = Array.from({length: 12}, (_, index) => -1 + index * 0.01);
-  const realInterval = pairedInterval(realEffect, {iterations: 2000, seed: 3});
-  assert.equal(classify(realInterval, {noiseFloor: 0.05}), 'improvement');
-  assert.ok(pairedPermutationTest(realEffect, {iterations: 2000, seed: 3}) < 0.01);
+  const realEffect = Array.from(
+    { length: 12 },
+    (_, index) => -1 + index * 0.01,
+  );
+  const realInterval = pairedInterval(realEffect, {
+    iterations: 2000,
+    seed: 3,
+  });
+  assert.equal(classify(realInterval, { noiseFloor: 0.05 }), "improvement");
+  assert.ok(
+    pairedPermutationTest(realEffect, { iterations: 2000, seed: 3 }) < 0.01,
+  );
 
   // Differences that merely alternate around zero must not be.
   const noEffect = [0.4, -0.3, 0.2, -0.5, 0.1, -0.2, 0.35, -0.4];
-  const nullInterval = pairedInterval(noEffect, {iterations: 2000, seed: 3});
-  assert.equal(classify(nullInterval, {noiseFloor: 0.05}), 'inconclusive');
-  assert.ok(pairedPermutationTest(noEffect, {iterations: 2000, seed: 3}) > 0.1);
+  const nullInterval = pairedInterval(noEffect, { iterations: 2000, seed: 3 });
+  assert.equal(classify(nullInterval, { noiseFloor: 0.05 }), "inconclusive");
+  assert.ok(
+    pairedPermutationTest(noEffect, { iterations: 2000, seed: 3 }) > 0.1,
+  );
 });
 
-test('suppresses effects smaller than the noise floor', () => {
-  const tiny = Array.from({length: 40}, () => -0.02);
-  const interval = pairedInterval(tiny, {iterations: 1000, seed: 5});
+test("suppresses effects smaller than the noise floor", () => {
+  const tiny = Array.from({ length: 40 }, () => -0.02);
+  const interval = pairedInterval(tiny, { iterations: 1000, seed: 5 });
   // The interval excludes zero because the samples are identical, but the
   // effect is far below what the harness can resolve.
-  assert.equal(classify(interval, {noiseFloor: 0.25}), 'within-noise');
-  assert.equal(classify(interval, {noiseFloor: 0.001}), 'improvement');
+  assert.equal(classify(interval, { noiseFloor: 0.25 }), "within-noise");
+  assert.equal(classify(interval, { noiseFloor: 0.001 }), "improvement");
 });
 
-test('classifies direction and missing intervals', () => {
-  assert.equal(classify({low: 0.2, high: 0.8, estimate: 0.5}), 'regression');
-  assert.equal(classify({low: -0.8, high: -0.2, estimate: -0.5}), 'improvement');
+test("classifies direction and missing intervals", () => {
+  assert.equal(classify({ low: 0.2, high: 0.8, estimate: 0.5 }), "regression");
+  assert.equal(
+    classify({ low: -0.8, high: -0.2, estimate: -0.5 }),
+    "improvement",
+  );
   assert.equal(
-    classify({low: 0.2, high: 0.8, estimate: 0.5}, {lowerIsBetter: false}),
-    'improvement'
+    classify({ low: 0.2, high: 0.8, estimate: 0.5 }, { lowerIsBetter: false }),
+    "improvement",
   );
-  assert.equal(classify(null), 'unknown');
+  assert.equal(classify(null), "unknown");
 });
 
-test('estimates a shift with Hodges-Lehmann', () => {
+test("estimates a shift with Hodges-Lehmann", () => {
   assert.equal(hodgesLehmann([3, 4, 5], [1, 2, 3]), 2);
   assert.equal(hodgesLehmann([], [1]), null);
 });
@@ -95,18 +111,18 @@ test('estimates a shift with Hodges-Lehmann', () => {
 // separated the two samples cleanly. This test pins that failure so nobody
 // reintroduces unpaired sampling: better statistics alone cannot rescue it,
 // because the samples really do differ once between-runner drift is baked in.
-test('unpaired sampling produces a false positive on identical code', () => {
+test("unpaired sampling produces a false positive on identical code", () => {
   const runOne = [2, 1, 3, 1, 1, 2, 3, 2, 2, 2];
   const runTwo = [3, 4, 2, 4, 4, 3, 4, 3, 3, 1];
   const interval = differenceInterval(runTwo, runOne, median, {
     iterations: 4000,
-    seed: 13
+    seed: 13,
   });
   // The interval excludes zero even though both samples measure the same code.
   assert.ok(interval.low > 0 || interval.high < 0);
-  assert.equal(classify(interval, {noiseFloor: 0}), 'regression');
+  assert.equal(classify(interval, { noiseFloor: 0 }), "regression");
 
   // The only defence left for unpaired data is a noise floor calibrated from
   // observed run-to-run drift, which suppresses the spurious verdict.
-  assert.equal(classify(interval, {noiseFloor: 1.5}), 'within-noise');
+  assert.equal(classify(interval, { noiseFloor: 1.5 }), "within-noise");
 });

From 01f3f0dbf25c4e94347975938527fc586773ef34 Mon Sep 17 00:00:00 2001
From: Bruno Borges <bruno.borges@microsoft.com>
Date: Wed, 5 Aug 2026 00:18:05 -0400
Subject: [PATCH 04/19] Stop the matrix from measuring its own cache contention

With one shared cache entry, running the whole matrix at once puts every runner
on the same blob at the same moment. At 20 concurrent runners the mean restore
rose from about 3s to about 8s and the noise floor from 0.34s to 1.89s, which is
enough to hide the effect under test. That is contention no real workflow would
experience, and it costs sensitivity for nothing.

The matrix now runs in waves of four, so each measurement reflects an ordinary
restore.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 71c45320-1417-4029-8402-69075d61dac1
---
 .github/workflows/focused-cache-restore.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/focused-cache-restore.yml b/.github/workflows/focused-cache-restore.yml
index 20577a5..80a8046 100644
--- a/.github/workflows/focused-cache-restore.yml
+++ b/.github/workflows/focused-cache-restore.yml
@@ -113,6 +113,13 @@ jobs:
     runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
+      # Every runner pulls the same seeded blob, so running the whole matrix at
+      # once puts all of them on the cache service simultaneously and measures
+      # contention that no real workflow would see. At 20 concurrent runners the
+      # mean restore rose from about 3s to about 8s and the noise floor from
+      # 0.34s to 1.89s, which is enough to hide the effect under test. Waves are
+      # kept small so each measurement reflects an ordinary restore.
+      max-parallel: 4
       matrix:
         sample: ${{ fromJSON(inputs.samples == '20' && '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]' || inputs.samples == '10' && '[1,2,3,4,5,6,7,8,9,10]' || inputs.samples == '6' && '[1,2,3,4,5,6]' || '[1,2]') }}
     steps:

From ebd51cfd4c56cd48956093cbb6075ec089b82329 Mon Sep 17 00:00:00 2001
From: Bruno Borges <bruno.borges@microsoft.com>
Date: Wed, 5 Aug 2026 00:33:23 -0400
Subject: [PATCH 05/19] Apply the paired methodology to the version sweep and
 JDK cache

These two workflows still measured the way the focused benchmark did before it
was rebuilt: durations read from the Actions API's one-second clock, arms spread
across independent jobs, no intervals, and one seeded cache entry per arm. Their
sub-second results were therefore not evidence of anything, and the JDK cache
workflow carried the same arm-versus-blob confound that an A/A run exposed in
the focused benchmark.

scripts/paired.mjs now holds the pairing logic that all three share: reading
per-runner timing CSVs, grouping slots by arm, discarding runners that did not
complete every slot, deriving the noise floor from within-arm repeats, and
producing an interval, a permutation p-value and a verdict. The focused report is
rebuilt on it unchanged in behaviour.

JDK cache becomes a two-arm ABBA comparison. One seed job populates a single
Maven entry and a single JDK entry, and both arms restore the same Maven entry,
so only the JDK entry differs between them. cache-jdk false and true are then
measured in one job per runner, with the tool cache and the local repository
cleared before every slot. The action ref is now an input so a branch can be
measured.

The version sweep generalises ABBA to seven arms: every runner sets up v1 through
main and then main back through v1, which places each version's two slots
symmetrically about the middle of the job so drift cancels. Every version that
supports cache-dependency-path restores one shared entry. v3 predates that input
and keys on pom.xml, so it necessarily has its own entry and the report says to
treat its difference with more caution. Versions are compared against main with
a paired interval per runner, and main against itself supplies the A/A control.

cache-read-only is not used in the sweep. It exists only on main, so applying it
to some arms and not others would make post-job behaviour asymmetric; every slot
restores on an exact primary-key hit, which actions/cache already skips saving.

Both matrices run in waves of four, because every runner pulls the same seeded
blob and running them all at once measures contention on the cache service that
no real workflow would see.

Tests cover the two properties the design depends on: that differencing within a
runner removes between-runner speed differences, and that the mirrored order
cancels drift that is linear across the job.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 71c45320-1417-4029-8402-69075d61dac1
---
 .github/workflows/benchmark.yml   | 567 ++++++++++++------------------
 .github/workflows/jdk-cache.yml   | 208 ++++++-----
 README.md                         |  42 ++-
 scripts/jdk-cache.sh              |  43 +++
 scripts/paired.mjs                | 244 +++++++++++++
 scripts/report-focused.mjs        | 153 +-------
 scripts/report-jdk-cache.mjs      | 380 ++++++--------------
 scripts/report-jdk-cache.test.mjs | 159 ++++-----
 scripts/report.mjs                | 435 ++++++++---------------
 scripts/report.test.mjs           | 139 +++++---
 scripts/version-sweep.sh          |  50 +++
 11 files changed, 1138 insertions(+), 1282 deletions(-)
 create mode 100755 scripts/jdk-cache.sh
 create mode 100644 scripts/paired.mjs
 create mode 100755 scripts/version-sweep.sh

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 61d2f52..7c3e6d2 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -3,15 +3,24 @@ name: Benchmark setup-java
 on:
   workflow_dispatch:
     inputs:
-      iterations:
-        description: Independent cold/warm samples per combination
+      samples:
+        description: Runners to measure on (each contributes 2 observations per version)
         required: true
         type: choice
         options:
-          - "1"
-          - "3"
-          - "5"
-        default: "3"
+          - "2"
+          - "6"
+          - "10"
+          - "20"
+        default: "10"
+      distribution:
+        description: JDK distribution for v2 and later
+        required: true
+        type: choice
+        options:
+          - temurin
+          - microsoft
+        default: temurin
       java-version:
         description: Java feature version
         required: true
@@ -34,14 +43,20 @@ concurrency:
   group: setup-java-benchmark
   cancel-in-progress: false
 
+defaults:
+  run:
+    shell: bash
+
 jobs:
-  v1-cold:
-    name: v1 / cold / zulu / ${{ matrix.iteration }}
+  # One Maven cache entry is seeded and every version that supports caching
+  # restores it. Seeding one entry per version would confound the version with
+  # its stored blob: a cache entry's download throughput depends on where the
+  # service placed it, and because that bias is identical on every runner,
+  # pairing cannot remove it and more samples only tighten the interval around
+  # the wrong answer.
+  seed:
+    name: Seed cache
     runs-on: ubuntu-24.04
-    strategy:
-      fail-fast: false
-      matrix:
-        iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }}
     steps:
       - name: Check out Spring PetClinic
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
@@ -49,207 +64,79 @@ jobs:
           repository: spring-projects/spring-petclinic
           ref: ${{ env.PETCLINIC_REF }}
           persist-credentials: false
+      - name: Check out benchmark scripts
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          path: .benchmark
+          persist-credentials: false
+      - name: Prepare cache identity
+        run: bash .benchmark/scripts/version-sweep.sh prepare "benchmark-${{ github.run_id }}"
+      # Each version registers a post-job save against the same local repository.
+      # The first save wins and the rest are skipped as exact key hits.
+
       - name: Setup Java v1
         uses: actions/setup-java@v1.4.4
         with:
           java-version: ${{ inputs.java-version }}
-      - name: Record Java environment
-        run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version'
-      - name: Build PetClinic
-        run: ./mvnw --batch-mode --no-transfer-progress compile
-
-  v2-cold:
-    name: v2 / cold / ${{ matrix.distribution }} / ${{ matrix.iteration }}
-    runs-on: ubuntu-24.04
-    strategy:
-      fail-fast: false
-      matrix:
-        distribution: [temurin, microsoft]
-        iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }}
-    steps:
-      - name: Check out Spring PetClinic
-        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          repository: spring-projects/spring-petclinic
-          ref: ${{ env.PETCLINIC_REF }}
-          persist-credentials: false
       - name: Setup Java v2
         uses: actions/setup-java@v2.5.1
         with:
-          distribution: ${{ matrix.distribution }}
+          distribution: ${{ inputs.distribution }}
           java-version: ${{ inputs.java-version }}
-      - name: Record Java environment
-        run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version'
-      - name: Build PetClinic
-        run: ./mvnw --batch-mode --no-transfer-progress compile
-
-  v3-cold:
-    name: v3 / cold / ${{ matrix.distribution }} / ${{ matrix.iteration }}
-    runs-on: ubuntu-24.04
-    strategy:
-      fail-fast: false
-      matrix:
-        distribution: [temurin, microsoft]
-        iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }}
-    steps:
-      - name: Check out Spring PetClinic
-        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          repository: spring-projects/spring-petclinic
-          ref: ${{ env.PETCLINIC_REF }}
-          persist-credentials: false
-      - name: Isolate benchmark caches
-        env:
-          BENCHMARK_ID: v3-${{ matrix.distribution }}-${{ matrix.iteration }}-${{ github.run_id }}
-        run: printf '<!-- benchmark-id=%s -->\n' "$BENCHMARK_ID" >> pom.xml
       - name: Setup Java v3
         uses: actions/setup-java@v3.14.1
         with:
-          distribution: ${{ matrix.distribution }}
+          distribution: ${{ inputs.distribution }}
           java-version: ${{ inputs.java-version }}
           cache: maven
-      - name: Record Java environment
-        run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version'
-      - name: Build PetClinic
-        run: ./mvnw --batch-mode --no-transfer-progress compile
-
-  v4-cold:
-    name: v4 / cold / ${{ matrix.distribution }} / ${{ matrix.iteration }}
-    runs-on: ubuntu-24.04
-    strategy:
-      fail-fast: false
-      matrix:
-        distribution: [temurin, microsoft]
-        iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }}
-    steps:
-      - name: Check out Spring PetClinic
-        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          repository: spring-projects/spring-petclinic
-          ref: ${{ env.PETCLINIC_REF }}
-          persist-credentials: false
-      - name: Isolate benchmark caches
-        env:
-          BENCHMARK_ID: v4-${{ matrix.distribution }}-${{ matrix.iteration }}-${{ github.run_id }}
-        run: printf '%s\n' "$BENCHMARK_ID" > .benchmark-cache-key
       - name: Setup Java v4
         uses: actions/setup-java@v4.8.0
         with:
-          distribution: ${{ matrix.distribution }}
+          distribution: ${{ inputs.distribution }}
           java-version: ${{ inputs.java-version }}
           cache: maven
           cache-dependency-path: .benchmark-cache-key
-      - name: Record Java environment
-        run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version'
-      - name: Build PetClinic
-        run: ./mvnw --batch-mode --no-transfer-progress compile
-
-  v52-cold:
-    name: v5.2 / cold / ${{ matrix.distribution }} / ${{ matrix.iteration }}
-    runs-on: ubuntu-24.04
-    strategy:
-      fail-fast: false
-      matrix:
-        distribution: [temurin, microsoft]
-        iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }}
-    steps:
-      - name: Check out Spring PetClinic
-        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          repository: spring-projects/spring-petclinic
-          ref: ${{ env.PETCLINIC_REF }}
-          persist-credentials: false
-      - name: Isolate benchmark caches
-        env:
-          BENCHMARK_ID: v52-${{ matrix.distribution }}-${{ matrix.iteration }}-${{ github.run_id }}
-        run: printf '%s\n' "$BENCHMARK_ID" > .benchmark-cache-key
       - name: Setup Java v5.2
         uses: actions/setup-java@v5.2.0
         with:
-          distribution: ${{ matrix.distribution }}
+          distribution: ${{ inputs.distribution }}
           java-version: ${{ inputs.java-version }}
           cache: maven
           cache-dependency-path: .benchmark-cache-key
-      - name: Record Java environment
-        run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version'
-      - name: Build PetClinic
-        run: ./mvnw --batch-mode --no-transfer-progress compile
-
-  v56-cold:
-    name: v5.6 / cold / ${{ matrix.distribution }} / ${{ matrix.iteration }}
-    runs-on: ubuntu-24.04
-    strategy:
-      fail-fast: false
-      matrix:
-        distribution: [temurin, microsoft]
-        iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }}
-    steps:
-      - name: Check out Spring PetClinic
-        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          repository: spring-projects/spring-petclinic
-          ref: ${{ env.PETCLINIC_REF }}
-          persist-credentials: false
-      - name: Isolate benchmark caches
-        env:
-          BENCHMARK_ID: v56-${{ matrix.distribution }}-${{ matrix.iteration }}-${{ github.run_id }}
-        run: |
-          printf '%s\n' "$BENCHMARK_ID" > .benchmark-cache-key
-          printf '# benchmark-id=%s\n' "$BENCHMARK_ID" >> .mvn/wrapper/maven-wrapper.properties
       - name: Setup Java v5.6
         uses: actions/setup-java@v5.6.0
         with:
-          distribution: ${{ matrix.distribution }}
+          distribution: ${{ inputs.distribution }}
           java-version: ${{ inputs.java-version }}
           cache: maven
           cache-dependency-path: .benchmark-cache-key
-      - name: Record Java environment
-        run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version'
-      - name: Build PetClinic
-        run: ./mvnw --batch-mode --no-transfer-progress compile
-
-  main-cold:
-    name: main / cold / ${{ matrix.distribution }} / ${{ matrix.iteration }}
-    runs-on: ubuntu-24.04
-    strategy:
-      fail-fast: false
-      matrix:
-        distribution: [temurin, microsoft]
-        iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }}
-    steps:
-      - name: Check out Spring PetClinic
-        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          repository: spring-projects/spring-petclinic
-          ref: ${{ env.PETCLINIC_REF }}
-          persist-credentials: false
-      - name: Isolate benchmark caches
-        env:
-          BENCHMARK_ID: main-${{ matrix.distribution }}-${{ matrix.iteration }}-${{ github.run_id }}
-        run: |
-          printf '%s\n' "$BENCHMARK_ID" > .benchmark-cache-key
-          printf '# benchmark-id=%s\n' "$BENCHMARK_ID" >> .mvn/wrapper/maven-wrapper.properties
       - name: Setup Java main
         uses: actions/setup-java@main
         with:
-          distribution: ${{ matrix.distribution }}
+          distribution: ${{ inputs.distribution }}
           java-version: ${{ inputs.java-version }}
           cache: maven
           cache-dependency-path: .benchmark-cache-key
-      - name: Record Java environment
-        run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version'
       - name: Build PetClinic
         run: ./mvnw --batch-mode --no-transfer-progress compile
 
-  v1-warm:
-    name: v1 / warm / zulu / ${{ matrix.iteration }}
-    needs: v1-cold
-    if: ${{ !cancelled() }}
+  # Every version is measured in the same job, in the order v1..main followed by
+  # main..v1. Differencing within a runner removes between-runner variance, and
+  # the mirrored order places each version's two slots symmetrically about the
+  # middle of the job so that drift across the job cancels.
+  warm:
+    name: warm / ${{ matrix.sample }}
+    needs: seed
     runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
+      # Every runner pulls the same seeded blob, so running the whole matrix at
+      # once measures contention on the cache service that no real workflow would
+      # see. Waves are kept small so each measurement reflects an ordinary
+      # restore.
+      max-parallel: 4
       matrix:
-        iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }}
+        sample: ${{ fromJSON(inputs.samples == '20' && '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]' || inputs.samples == '10' && '[1,2,3,4,5,6,7,8,9,10]' || inputs.samples == '6' && '[1,2,3,4,5,6]' || '[1,2]') }}
     steps:
       - name: Check out Spring PetClinic
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
@@ -257,213 +144,200 @@ jobs:
           repository: spring-projects/spring-petclinic
           ref: ${{ env.PETCLINIC_REF }}
           persist-credentials: false
-      - name: Setup Java v1
-        uses: actions/setup-java@v1.4.4
-        with:
-          java-version: ${{ inputs.java-version }}
-      - name: Record Java environment
-        run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version'
-      - name: Build PetClinic
-        run: ./mvnw --batch-mode --no-transfer-progress compile
-
-  v2-warm:
-    name: v2 / warm / ${{ matrix.distribution }} / ${{ matrix.iteration }}
-    needs: v2-cold
-    if: ${{ !cancelled() }}
-    runs-on: ubuntu-24.04
-    strategy:
-      fail-fast: false
-      matrix:
-        distribution: [temurin, microsoft]
-        iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }}
-    steps:
-      - name: Check out Spring PetClinic
+      - name: Check out benchmark scripts
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
         with:
-          repository: spring-projects/spring-petclinic
-          ref: ${{ env.PETCLINIC_REF }}
+          path: .benchmark
           persist-credentials: false
-      - name: Setup Java v2
-        uses: actions/setup-java@v2.5.1
+
+      # cache-read-only exists only on main, so it is not used here: applying it
+      # to some arms and not others would make post-job behaviour asymmetric.
+      # Every slot restores its cache on an exact primary-key hit, and
+      # actions/cache skips saving in that case, so no measured slot creates an
+      # entry regardless of the version.
+      - name: Reset slot 1
+        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+      - name: Start slot 1 timer
+        run: node .benchmark/scripts/measure.mjs start
+      - name: Slot 1 setup (v1)
+        uses: actions/setup-java@v1.4.4
         with:
-          distribution: ${{ matrix.distribution }}
           java-version: ${{ inputs.java-version }}
-      - name: Record Java environment
-        run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version'
-      - name: Build PetClinic
-        run: ./mvnw --batch-mode --no-transfer-progress compile
-
-  v3-warm:
-    name: v3 / warm / ${{ matrix.distribution }} / ${{ matrix.iteration }}
-    needs: v3-cold
-    if: ${{ !cancelled() }}
-    runs-on: ubuntu-24.04
-    strategy:
-      fail-fast: false
-      matrix:
-        distribution: [temurin, microsoft]
-        iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }}
-    steps:
-      - name: Check out Spring PetClinic
-        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+      - name: Record slot 1
+        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v1 1
+      - name: Reset slot 2
+        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+      - name: Start slot 2 timer
+        run: node .benchmark/scripts/measure.mjs start
+      - name: Slot 2 setup (v2)
+        uses: actions/setup-java@v2.5.1
         with:
-          repository: spring-projects/spring-petclinic
-          ref: ${{ env.PETCLINIC_REF }}
-          persist-credentials: false
-      - name: Reuse benchmark cache identity
-        env:
-          BENCHMARK_ID: v3-${{ matrix.distribution }}-${{ matrix.iteration }}-${{ github.run_id }}
-        run: printf '<!-- benchmark-id=%s -->\n' "$BENCHMARK_ID" >> pom.xml
-      - name: Setup Java v3
+          distribution: ${{ inputs.distribution }}
+          java-version: ${{ inputs.java-version }}
+      - name: Record slot 2
+        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v2 2
+      - name: Reset slot 3
+        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+      - name: Start slot 3 timer
+        run: node .benchmark/scripts/measure.mjs start
+      - name: Slot 3 setup (v3)
         uses: actions/setup-java@v3.14.1
         with:
-          distribution: ${{ matrix.distribution }}
+          distribution: ${{ inputs.distribution }}
           java-version: ${{ inputs.java-version }}
           cache: maven
-      - name: Record Java environment
-        run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version'
-      - name: Build PetClinic
-        run: ./mvnw --batch-mode --no-transfer-progress compile
-
-  v4-warm:
-    name: v4 / warm / ${{ matrix.distribution }} / ${{ matrix.iteration }}
-    needs: v4-cold
-    if: ${{ !cancelled() }}
-    runs-on: ubuntu-24.04
-    strategy:
-      fail-fast: false
-      matrix:
-        distribution: [temurin, microsoft]
-        iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }}
-    steps:
-      - name: Check out Spring PetClinic
-        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          repository: spring-projects/spring-petclinic
-          ref: ${{ env.PETCLINIC_REF }}
-          persist-credentials: false
-      - name: Reuse benchmark cache identity
-        env:
-          BENCHMARK_ID: v4-${{ matrix.distribution }}-${{ matrix.iteration }}-${{ github.run_id }}
-        run: printf '%s\n' "$BENCHMARK_ID" > .benchmark-cache-key
-      - name: Setup Java v4
+      - name: Record slot 3
+        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v3 3
+      - name: Reset slot 4
+        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+      - name: Start slot 4 timer
+        run: node .benchmark/scripts/measure.mjs start
+      - name: Slot 4 setup (v4)
         uses: actions/setup-java@v4.8.0
         with:
-          distribution: ${{ matrix.distribution }}
+          distribution: ${{ inputs.distribution }}
           java-version: ${{ inputs.java-version }}
           cache: maven
           cache-dependency-path: .benchmark-cache-key
-      - name: Record Java environment
-        run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version'
-      - name: Build PetClinic
-        run: ./mvnw --batch-mode --no-transfer-progress compile
-
-  v52-warm:
-    name: v5.2 / warm / ${{ matrix.distribution }} / ${{ matrix.iteration }}
-    needs: v52-cold
-    if: ${{ !cancelled() }}
-    runs-on: ubuntu-24.04
-    strategy:
-      fail-fast: false
-      matrix:
-        distribution: [temurin, microsoft]
-        iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }}
-    steps:
-      - name: Check out Spring PetClinic
-        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          repository: spring-projects/spring-petclinic
-          ref: ${{ env.PETCLINIC_REF }}
-          persist-credentials: false
-      - name: Reuse benchmark cache identity
-        env:
-          BENCHMARK_ID: v52-${{ matrix.distribution }}-${{ matrix.iteration }}-${{ github.run_id }}
-        run: printf '%s\n' "$BENCHMARK_ID" > .benchmark-cache-key
-      - name: Setup Java v5.2
+      - name: Record slot 4
+        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v4 4
+      - name: Reset slot 5
+        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+      - name: Start slot 5 timer
+        run: node .benchmark/scripts/measure.mjs start
+      - name: Slot 5 setup (v5.2)
         uses: actions/setup-java@v5.2.0
         with:
-          distribution: ${{ matrix.distribution }}
+          distribution: ${{ inputs.distribution }}
           java-version: ${{ inputs.java-version }}
           cache: maven
           cache-dependency-path: .benchmark-cache-key
-      - name: Record Java environment
-        run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version'
-      - name: Build PetClinic
-        run: ./mvnw --batch-mode --no-transfer-progress compile
-
-  v56-warm:
-    name: v5.6 / warm / ${{ matrix.distribution }} / ${{ matrix.iteration }}
-    needs: v56-cold
-    if: ${{ !cancelled() }}
-    runs-on: ubuntu-24.04
-    strategy:
-      fail-fast: false
-      matrix:
-        distribution: [temurin, microsoft]
-        iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }}
-    steps:
-      - name: Check out Spring PetClinic
-        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          repository: spring-projects/spring-petclinic
-          ref: ${{ env.PETCLINIC_REF }}
-          persist-credentials: false
-      - name: Reuse benchmark cache identity
-        env:
-          BENCHMARK_ID: v56-${{ matrix.distribution }}-${{ matrix.iteration }}-${{ github.run_id }}
-        run: |
-          printf '%s\n' "$BENCHMARK_ID" > .benchmark-cache-key
-          printf '# benchmark-id=%s\n' "$BENCHMARK_ID" >> .mvn/wrapper/maven-wrapper.properties
-      - name: Setup Java v5.6
+      - name: Record slot 5
+        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v52 5
+      - name: Reset slot 6
+        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+      - name: Start slot 6 timer
+        run: node .benchmark/scripts/measure.mjs start
+      - name: Slot 6 setup (v5.6)
         uses: actions/setup-java@v5.6.0
         with:
-          distribution: ${{ matrix.distribution }}
+          distribution: ${{ inputs.distribution }}
           java-version: ${{ inputs.java-version }}
           cache: maven
           cache-dependency-path: .benchmark-cache-key
-      - name: Record Java environment
-        run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version'
-      - name: Build PetClinic
-        run: ./mvnw --batch-mode --no-transfer-progress compile
-
-  main-warm:
-    name: main / warm / ${{ matrix.distribution }} / ${{ matrix.iteration }}
-    needs: main-cold
-    if: ${{ !cancelled() }}
-    runs-on: ubuntu-24.04
-    strategy:
-      fail-fast: false
-      matrix:
-        distribution: [temurin, microsoft]
-        iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }}
-    steps:
-      - name: Check out Spring PetClinic
-        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+      - name: Record slot 6
+        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v56 6
+      - name: Reset slot 7
+        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+      - name: Start slot 7 timer
+        run: node .benchmark/scripts/measure.mjs start
+      - name: Slot 7 setup (main)
+        uses: actions/setup-java@main
         with:
-          repository: spring-projects/spring-petclinic
-          ref: ${{ env.PETCLINIC_REF }}
-          persist-credentials: false
-      - name: Reuse benchmark cache identity
-        env:
-          BENCHMARK_ID: main-${{ matrix.distribution }}-${{ matrix.iteration }}-${{ github.run_id }}
-        run: |
-          printf '%s\n' "$BENCHMARK_ID" > .benchmark-cache-key
-          printf '# benchmark-id=%s\n' "$BENCHMARK_ID" >> .mvn/wrapper/maven-wrapper.properties
-      - name: Setup Java main
+          distribution: ${{ inputs.distribution }}
+          java-version: ${{ inputs.java-version }}
+          cache: maven
+          cache-dependency-path: .benchmark-cache-key
+      - name: Record slot 7
+        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" main 7
+      - name: Reset slot 8
+        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+      - name: Start slot 8 timer
+        run: node .benchmark/scripts/measure.mjs start
+      - name: Slot 8 setup (main)
         uses: actions/setup-java@main
         with:
-          distribution: ${{ matrix.distribution }}
+          distribution: ${{ inputs.distribution }}
           java-version: ${{ inputs.java-version }}
           cache: maven
           cache-dependency-path: .benchmark-cache-key
-      - name: Record Java environment
-        run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version'
-      - name: Build PetClinic
-        run: ./mvnw --batch-mode --no-transfer-progress compile
+      - name: Record slot 8
+        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" main 8
+      - name: Reset slot 9
+        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+      - name: Start slot 9 timer
+        run: node .benchmark/scripts/measure.mjs start
+      - name: Slot 9 setup (v5.6)
+        uses: actions/setup-java@v5.6.0
+        with:
+          distribution: ${{ inputs.distribution }}
+          java-version: ${{ inputs.java-version }}
+          cache: maven
+          cache-dependency-path: .benchmark-cache-key
+      - name: Record slot 9
+        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v56 9
+      - name: Reset slot 10
+        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+      - name: Start slot 10 timer
+        run: node .benchmark/scripts/measure.mjs start
+      - name: Slot 10 setup (v5.2)
+        uses: actions/setup-java@v5.2.0
+        with:
+          distribution: ${{ inputs.distribution }}
+          java-version: ${{ inputs.java-version }}
+          cache: maven
+          cache-dependency-path: .benchmark-cache-key
+      - name: Record slot 10
+        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v52 10
+      - name: Reset slot 11
+        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+      - name: Start slot 11 timer
+        run: node .benchmark/scripts/measure.mjs start
+      - name: Slot 11 setup (v4)
+        uses: actions/setup-java@v4.8.0
+        with:
+          distribution: ${{ inputs.distribution }}
+          java-version: ${{ inputs.java-version }}
+          cache: maven
+          cache-dependency-path: .benchmark-cache-key
+      - name: Record slot 11
+        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v4 11
+      - name: Reset slot 12
+        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+      - name: Start slot 12 timer
+        run: node .benchmark/scripts/measure.mjs start
+      - name: Slot 12 setup (v3)
+        uses: actions/setup-java@v3.14.1
+        with:
+          distribution: ${{ inputs.distribution }}
+          java-version: ${{ inputs.java-version }}
+          cache: maven
+      - name: Record slot 12
+        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v3 12
+      - name: Reset slot 13
+        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+      - name: Start slot 13 timer
+        run: node .benchmark/scripts/measure.mjs start
+      - name: Slot 13 setup (v2)
+        uses: actions/setup-java@v2.5.1
+        with:
+          distribution: ${{ inputs.distribution }}
+          java-version: ${{ inputs.java-version }}
+      - name: Record slot 13
+        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v2 13
+      - name: Reset slot 14
+        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+      - name: Start slot 14 timer
+        run: node .benchmark/scripts/measure.mjs start
+      - name: Slot 14 setup (v1)
+        uses: actions/setup-java@v1.4.4
+        with:
+          java-version: ${{ inputs.java-version }}
+      - name: Record slot 14
+        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v1 14
+
+      - name: Upload sample timings
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: version-sweep-${{ matrix.sample }}
+          path: .benchmark-results/
+          include-hidden-files: true
+          if-no-files-found: error
+          retention-days: 30
 
   report:
     name: Report
-    needs: [v1-cold, v2-cold, v3-cold, v4-cold, v52-cold, v56-cold, main-cold, v1-warm, v2-warm, v3-warm, v4-warm, v52-warm, v56-warm, main-warm]
+    needs: warm
     if: ${{ always() && !cancelled() }}
     runs-on: ubuntu-24.04
     steps:
@@ -471,10 +345,17 @@ jobs:
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
         with:
           persist-credentials: false
+      - name: Download sample timings
+        uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6
+        with:
+          pattern: version-sweep-*
+          merge-multiple: true
+          path: .benchmark-results
       - name: Generate benchmark report
         env:
           GH_TOKEN: ${{ github.token }}
-          ITERATIONS: ${{ inputs.iterations }}
+          SAMPLES: ${{ inputs.samples }}
+          DISTRIBUTION: ${{ inputs.distribution }}
           JAVA_VERSION: ${{ inputs.java-version }}
           CLEANUP_CACHES: ${{ inputs.cleanup-caches }}
         run: node scripts/report.mjs
diff --git a/.github/workflows/jdk-cache.yml b/.github/workflows/jdk-cache.yml
index dc1de1b..baa0bdb 100644
--- a/.github/workflows/jdk-cache.yml
+++ b/.github/workflows/jdk-cache.yml
@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       samples:
-        description: Warm samples per arm
+        description: Runners to measure on (each contributes 2 paired observations)
         required: true
         type: choice
         options:
@@ -26,6 +26,16 @@ on:
         required: true
         type: string
         default: "17"
+      setup-java-repository:
+        description: Repository containing the setup-java action
+        required: true
+        default: actions/setup-java
+        type: string
+      setup-java-ref:
+        description: Git ref of setup-java to measure
+        required: true
+        default: main
+        type: string
       cleanup-caches:
         description: Delete benchmark caches after measuring them
         required: true
@@ -43,6 +53,10 @@ concurrency:
   group: jdk-cache-benchmark
   cancel-in-progress: false
 
+defaults:
+  run:
+    shell: bash
+
 jobs:
   prepare:
     name: Prepare JDK cache benchmark
@@ -66,8 +80,14 @@ jobs:
             gh cache delete "$cache_id" --repo "${{ github.repository }}"
           done <<< "$cache_ids"
 
-  baseline-seed:
-    name: baseline / seed
+  # Both arms share one Maven cache entry. A cache entry's download throughput
+  # depends on where the service placed the stored blob, so seeding one entry per
+  # arm would confound the arm with its blob. That bias is the same on every
+  # runner, which means pairing cannot remove it and more samples only tighten
+  # the interval around the wrong answer. Only the JDK cache differs between the
+  # arms, which is the effect under test.
+  seed:
+    name: Seed caches
     needs: prepare
     runs-on: ubuntu-24.04
     steps:
@@ -77,48 +97,21 @@ jobs:
           repository: spring-projects/spring-petclinic
           ref: ${{ env.PETCLINIC_REF }}
           persist-credentials: false
-      - name: Prepare baseline cache identity
-        env:
-          BENCHMARK_ID: jdk-cache-baseline-${{ github.run_id }}
-        run: |
-          printf '%s\n' "$BENCHMARK_ID" > .jdk-cache-key
-          printf '# jdk-cache-benchmark=%s\n' "$BENCHMARK_ID" >> .mvn/wrapper/maven-wrapper.properties
-      # Deliberately benchmark the path where the requested JDK is not preinstalled.
-      - name: Purge matching runner tool cache
-        run: rm -rf "$RUNNER_TOOL_CACHE"/Java_*
-      - name: Setup Java main
-        uses: actions/setup-java@main
-        with:
-          distribution: ${{ inputs.distribution }}
-          java-version: ${{ inputs.java-version }}
-          cache: maven
-          cache-jdk: false
-          cache-dependency-path: .jdk-cache-key
-      - name: Build PetClinic
-        run: ./mvnw --batch-mode --no-transfer-progress compile
-
-  treatment-seed:
-    name: treatment / seed
-    needs: prepare
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Check out Spring PetClinic
+      - name: Check out setup-java
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
         with:
-          repository: spring-projects/spring-petclinic
-          ref: ${{ env.PETCLINIC_REF }}
+          repository: ${{ inputs.setup-java-repository }}
+          path: .setup-java
           persist-credentials: false
-      - name: Prepare treatment cache identity
-        env:
-          BENCHMARK_ID: jdk-cache-treatment-${{ github.run_id }}
-        run: |
-          printf '%s\n' "$BENCHMARK_ID" > .jdk-cache-key
-          printf '# jdk-cache-benchmark=%s\n' "$BENCHMARK_ID" >> .mvn/wrapper/maven-wrapper.properties
-      # The seed must download the JDK so setup-java registers its post-job save.
+          ref: ${{ inputs.setup-java-ref }}
+      - name: Prepare cache identity
+        run: bash scripts/jdk-cache.sh prepare "jdk-cache-${{ github.run_id }}"
+      # The seed must download the JDK from the vendor so that setup-java
+      # registers its post-job save of the JDK cache.
       - name: Purge matching runner tool cache
-        run: rm -rf "$RUNNER_TOOL_CACHE"/Java_*
-      - name: Setup Java main
-        uses: actions/setup-java@main
+        run: bash scripts/jdk-cache.sh purge-toolcache
+      - name: Setup Java
+        uses: ./.setup-java
         with:
           distribution: ${{ inputs.distribution }}
           java-version: ${{ inputs.java-version }}
@@ -128,12 +121,20 @@ jobs:
       - name: Build PetClinic
         run: ./mvnw --batch-mode --no-transfer-progress compile
 
-  baseline-measure:
-    name: baseline / warm / ${{ matrix.sample }}
-    needs: baseline-seed
+  # Both arms run in the same job in ABBA order, so each runner yields one paired
+  # difference with its own speed cancelled out and the mirrored order cancels
+  # drift across the four slots.
+  measure:
+    name: paired / ${{ matrix.sample }}
+    needs: seed
     runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
+      # Every runner pulls the same seeded blobs, so running the whole matrix at
+      # once measures contention on the cache service that no real workflow would
+      # see. Waves are kept small so each measurement reflects an ordinary
+      # restore.
+      max-parallel: 4
       matrix:
         sample: ${{ fromJSON(inputs.samples == '20' && '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]' || inputs.samples == '10' && '[1,2,3,4,5,6,7,8,9,10]' || inputs.samples == '6' && '[1,2,3,4,5,6]' || '[1,2]') }}
     steps:
@@ -143,16 +144,23 @@ jobs:
           repository: spring-projects/spring-petclinic
           ref: ${{ env.PETCLINIC_REF }}
           persist-credentials: false
-      - name: Prepare baseline cache identity
-        env:
-          BENCHMARK_ID: jdk-cache-baseline-${{ github.run_id }}
-        run: |
-          printf '%s\n' "$BENCHMARK_ID" > .jdk-cache-key
-          printf '# jdk-cache-benchmark=%s\n' "$BENCHMARK_ID" >> .mvn/wrapper/maven-wrapper.properties
-      - name: Purge matching runner tool cache
-        run: rm -rf "$RUNNER_TOOL_CACHE"/Java_*
-      - name: Setup Java main
-        uses: actions/setup-java@main
+      - name: Check out setup-java
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          repository: ${{ inputs.setup-java-repository }}
+          path: .setup-java
+          persist-credentials: false
+          ref: ${{ inputs.setup-java-ref }}
+
+      # Each slot deletes the extracted JDK and the local repository first, so
+      # every measured setup starts from the same empty state instead of
+      # extracting over files an earlier slot left behind.
+      - name: Reset slot 1
+        run: bash scripts/jdk-cache.sh reset "jdk-cache-${{ github.run_id }}"
+      - name: Start slot 1 timer
+        run: node scripts/measure.mjs start
+      - name: Slot 1 setup (baseline, no JDK cache)
+        uses: ./.setup-java
         with:
           distribution: ${{ inputs.distribution }}
           java-version: ${{ inputs.java-version }}
@@ -160,34 +168,31 @@ jobs:
           cache-jdk: false
           cache-read-only: true
           cache-dependency-path: .jdk-cache-key
-      - name: Build PetClinic
-        run: ./mvnw --batch-mode --no-transfer-progress compile
+      - name: Record slot 1
+        run: node scripts/measure.mjs record ".benchmark-results/jdk-cache-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" baseline 1
 
-  treatment-measure:
-    name: treatment / warm / ${{ matrix.sample }}
-    needs: treatment-seed
-    runs-on: ubuntu-24.04
-    strategy:
-      fail-fast: false
-      matrix:
-        sample: ${{ fromJSON(inputs.samples == '20' && '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]' || inputs.samples == '10' && '[1,2,3,4,5,6,7,8,9,10]' || inputs.samples == '6' && '[1,2,3,4,5,6]' || '[1,2]') }}
-    steps:
-      - name: Check out Spring PetClinic
-        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+      - name: Reset slot 2
+        run: bash scripts/jdk-cache.sh reset "jdk-cache-${{ github.run_id }}"
+      - name: Start slot 2 timer
+        run: node scripts/measure.mjs start
+      - name: Slot 2 setup (candidate, JDK cache)
+        uses: ./.setup-java
         with:
-          repository: spring-projects/spring-petclinic
-          ref: ${{ env.PETCLINIC_REF }}
-          persist-credentials: false
-      - name: Prepare treatment cache identity
-        env:
-          BENCHMARK_ID: jdk-cache-treatment-${{ github.run_id }}
-        run: |
-          printf '%s\n' "$BENCHMARK_ID" > .jdk-cache-key
-          printf '# jdk-cache-benchmark=%s\n' "$BENCHMARK_ID" >> .mvn/wrapper/maven-wrapper.properties
-      - name: Purge matching runner tool cache
-        run: rm -rf "$RUNNER_TOOL_CACHE"/Java_*
-      - name: Setup Java main
-        uses: actions/setup-java@main
+          distribution: ${{ inputs.distribution }}
+          java-version: ${{ inputs.java-version }}
+          cache: maven
+          cache-jdk: true
+          cache-read-only: true
+          cache-dependency-path: .jdk-cache-key
+      - name: Record slot 2
+        run: node scripts/measure.mjs record ".benchmark-results/jdk-cache-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" candidate 2
+
+      - name: Reset slot 3
+        run: bash scripts/jdk-cache.sh reset "jdk-cache-${{ github.run_id }}"
+      - name: Start slot 3 timer
+        run: node scripts/measure.mjs start
+      - name: Slot 3 setup (candidate, JDK cache)
+        uses: ./.setup-java
         with:
           distribution: ${{ inputs.distribution }}
           java-version: ${{ inputs.java-version }}
@@ -195,16 +200,37 @@ jobs:
           cache-jdk: true
           cache-read-only: true
           cache-dependency-path: .jdk-cache-key
-      - name: Build PetClinic
-        run: ./mvnw --batch-mode --no-transfer-progress compile
+      - name: Record slot 3
+        run: node scripts/measure.mjs record ".benchmark-results/jdk-cache-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" candidate 3
+
+      - name: Reset slot 4
+        run: bash scripts/jdk-cache.sh reset "jdk-cache-${{ github.run_id }}"
+      - name: Start slot 4 timer
+        run: node scripts/measure.mjs start
+      - name: Slot 4 setup (baseline, no JDK cache)
+        uses: ./.setup-java
+        with:
+          distribution: ${{ inputs.distribution }}
+          java-version: ${{ inputs.java-version }}
+          cache: maven
+          cache-jdk: false
+          cache-read-only: true
+          cache-dependency-path: .jdk-cache-key
+      - name: Record slot 4
+        run: node scripts/measure.mjs record ".benchmark-results/jdk-cache-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" baseline 4
+
+      - name: Upload sample timings
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: jdk-cache-timings-${{ matrix.sample }}
+          path: .benchmark-results/
+          include-hidden-files: true
+          if-no-files-found: error
+          retention-days: 30
 
   report:
     name: Report
-    needs:
-      - baseline-seed
-      - treatment-seed
-      - baseline-measure
-      - treatment-measure
+    needs: measure
     if: ${{ always() && !cancelled() }}
     runs-on: ubuntu-24.04
     steps:
@@ -212,12 +238,20 @@ jobs:
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
         with:
           persist-credentials: false
+      - name: Download sample timings
+        uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6
+        with:
+          pattern: jdk-cache-timings-*
+          merge-multiple: true
+          path: .benchmark-results
       - name: Generate JDK cache report
         env:
           GH_TOKEN: ${{ github.token }}
           SAMPLES: ${{ inputs.samples }}
           DISTRIBUTION: ${{ inputs.distribution }}
           JAVA_VERSION: ${{ inputs.java-version }}
+          SETUP_JAVA_REPOSITORY: ${{ inputs.setup-java-repository }}
+          SETUP_JAVA_REF: ${{ inputs.setup-java-ref }}
           CLEANUP_CACHES: ${{ inputs.cleanup-caches }}
         run: node scripts/report-jdk-cache.mjs
       - name: Upload benchmark results
diff --git a/README.md b/README.md
index 72b7bea..f1905d6 100644
--- a/README.md
+++ b/README.md
@@ -7,6 +7,27 @@ The benchmark is designed around the two costs that matter to Actions users:
 - **Execution time:** setup, build, cache restore, and post-job cache save durations.
 - **Cache storage:** compressed JDK, Maven dependency, and Maven Wrapper cache sizes.
 
+## Methodology
+
+Every workflow here measures effects of a few hundred milliseconds to a few seconds on hosted runners, where the variance between runners is larger than the effect. Four properties are what make a result mean something, and all four workflows now share them.
+
+**Millisecond timing.** The Actions API reports step `started_at` and `completed_at` only to the nearest second. Setup steps take two to six seconds, so reading durations from the API quantizes every measurement to ±500 ms — the same magnitude as the effects being measured. Timing is taken inside the job with `scripts/measure.mjs`.
+
+**Same-runner pairing.** Between-runner variance cannot be averaged away by adding more independent jobs to each arm. Every arm is measured inside the *same* job, in an order mirrored about the middle of the job: ABBA for two arms, and `v1..main` followed by `main..v1` for the version sweep. Differencing within a runner removes the runner's own speed, and the mirrored order cancels drift that is linear across the job. Each measured slot deletes `~/.m2` first so every restore extracts into an empty tree.
+
+**One cache, every arm.** A cache entry's download throughput depends on where the service placed the stored blob, and that placement is fixed for the life of the entry. Seeding one entry per arm therefore confounds the arm with its blob, and because the bias is identical on every runner, pairing cannot remove it and more samples only tighten the interval around the wrong answer. A single entry is seeded and every arm restores it.
+
+**Intervals, not point estimates.** `scripts/stats.mjs` reports a bootstrap 95% confidence interval, a permutation p-value, and a Hodges-Lehmann shift, and turns them into an explicit verdict. A comparison whose interval includes zero is reported as `inconclusive` rather than as a number that looks like a result.
+
+Every report also publishes two guard rails:
+
+- A **noise floor**, the median spread between the two slots of the same arm on one runner. An effect smaller than this is reported as `within-noise` even when its interval excludes zero.
+- An **A/A control**, the same estimator applied to one arm against itself. It costs no extra jobs because every arm is already measured twice per runner. A healthy run reports `within-noise` or `inconclusive`; anything else means the harness is biasing results and the headline verdict cannot be trusted.
+
+After changing a harness, run it with both arms set to the same ref. The true effect is then exactly zero, and any other verdict is a defect rather than a finding. That check is what caught the per-arm cache confound described above: on identical code it reported a 0.859 s improvement, with the baseline blob served at ~60 MB/s and the candidate blob at ~105–130 MB/s on the same runner in the same job.
+
+`scripts/paired.mjs` implements the pairing and `scripts/stats.mjs` the statistics; every report builds on both.
+
 ## Scenarios
 
 Each action version runs with Java 17 on `ubuntu-24.04`:
@@ -18,18 +39,15 @@ Each action version runs with Java 17 on `ubuntu-24.04`:
 
 v1 predates distribution selection and integrated dependency caching. It runs only its native Zulu installer path. v2 supports the Temurin and Microsoft scenarios, but its bundled legacy cache client is rejected by the current Actions cache service. v1 and v2 therefore have no Maven cache storage, and their cold/warm labels are repeated uncached samples.
 
-Every combination gets an isolated cache key and runs twice:
+A seed job compiles Spring PetClinic once to populate a single Maven cache entry. Every measurement runner then sets up all seven versions in one job, in the order `v1..main` followed by `main..v1`, deleting `~/.m2` before each slot. The report compares every version against `main` with a paired interval per runner.
 
-1. **Cold:** no dependency or wrapper cache exists; the post action saves every cache supported by that version.
-2. **Warm:** restores the caches created by the matching cold job.
-
-v3, v4, and v5.2 cache only Maven dependencies. v5.6 and `main` also cache the Maven Wrapper distribution separately, making the storage and execution-time tradeoff visible. Because v3 predates `cache-dependency-path`, its benchmark identity is an inert XML comment appended to `pom.xml`; later versions use a dedicated marker file.
+v3, v4, and v5.2 cache only Maven dependencies. v5.6 and `main` also cache the Maven Wrapper distribution separately, making the storage and execution-time tradeoff visible. v4 and later share one cache entry through `cache-dependency-path`, so the stored blob is held constant across them. v3 predates that input and keys on `pom.xml`, so it necessarily uses its own entry; treat its difference with more caution than the rest. v1 and v2 do no caching at all and serve as the uncached reference.
 
 Spring PetClinic and third-party actions are pinned to commits. `setup-java@main` intentionally remains a moving ref so each run evaluates the current upcoming v6 code; the report records the `main` commit observed when it is generated.
 
 ## Running
 
-Open **Actions > Benchmark setup-java > Run workflow**. Choose one, three, or five independent samples. Three is the default.
+Open **Actions > Benchmark setup-java > Run workflow**. Choose how many runners to measure on; each contributes two observations per version. Ten is the default.
 
 The report job writes a Markdown summary and uploads raw JSON and CSV files. Benchmark-created caches are deleted after measurement by default, preventing repeated runs from consuming repository cache storage. Disable cleanup when you need to inspect the entries manually.
 
@@ -64,11 +82,15 @@ Two consecutive runs of that harness against an unchanged `v4.8.0` — where the
 
 ### JDK cache
 
-The **JDK cache** workflow measures the installed-JDK cache added on `actions/setup-java@main`. It compares a baseline arm with `cache-jdk: false` against a treatment arm with `cache-jdk: true`, defaulting to Microsoft Build of OpenJDK 17. Both arms use the same action ref so the result isolates JDK caching instead of conflating it with implementation changes between commits. Each arm first seeds its Maven dependency and wrapper caches by compiling Spring PetClinic. The treatment seed also downloads and saves the JDK. Warm matrix jobs then use `cache-read-only: true`, so they restore caches without creating entries or racing to save the same key.
+The **JDK cache** workflow measures the installed-JDK cache added on `actions/setup-java@main`. It compares `cache-jdk: false` against `cache-jdk: true`, defaulting to Microsoft Build of OpenJDK 17. Both arms use the same action ref, so the result isolates JDK caching instead of conflating it with implementation changes between commits.
+
+A single seed job compiles Spring PetClinic to populate one Maven cache entry and one JDK cache entry. Every measurement runner then runs both arms in one job in ABBA order under `cache-read-only: true`. Both arms restore the same Maven entry — only the JDK entry differs between them, which is the effect under test.
+
+Every seed and measured slot removes matching JDKs from `$RUNNER_TOOL_CACHE` first, deliberately measuring the not-preinstalled path and preventing a hosted-runner tool-cache hit from bypassing JDK cache restore logic. Select Temurin to test that same forced-miss path with a distribution normally preinstalled on hosted runners.
 
-JDK cache keys are derived from the JDK's identity and source; unlike dependency cache keys, they cannot be namespaced per iteration. The seed/read-only design avoids cross-contamination between samples. Every seed and measurement job also removes matching JDKs from `$RUNNER_TOOL_CACHE` before setup, deliberately measuring the not-preinstalled path and preventing a hosted-runner tool-cache hit from bypassing JDK cache restore and save logic. Select Temurin to test that same forced-miss path with a distribution normally preinstalled on hosted runners.
+JDK cache keys are derived from the JDK's identity and source, so unlike dependency cache keys they cannot be namespaced per run; the `prepare` job deletes existing JDK caches before seeding.
 
-Open **Actions > JDK cache > Run workflow** to select the distribution, Java version, warm sample count, and cache cleanup behavior. The report compares warm setup, build, post-step, and job medians; records cold seed setup and JDK save time; and reports the JDK, dependency, and wrapper cache sizes.
+Open **Actions > JDK cache > Run workflow** to select the distribution, Java version, action ref, runner count, and cache cleanup behavior.
 
 ### Maven configuration warm path
 
@@ -88,7 +110,7 @@ The summary reports medians for:
 
 Public repositories do not pay for standard GitHub-hosted runners. The estimated minutes are included to make the results applicable to private repositories; actual charges depend on the account plan and runner type.
 
-Network throughput, hosted-runner image changes, upstream artifact availability, and runner load all introduce variance. The **Benchmark setup-java** and **JDK cache** workflows still read step durations from the Actions API at one-second resolution and compare arms across independent runners, so treat their sub-second differences as indicative only and compare multiple runs before drawing conclusions. Use **Focused cache restore** when a difference needs to be established rather than illustrated; it is the only workflow here that reports a confidence interval, a noise floor, and an A/A control.
+Network throughput, hosted-runner image changes, upstream artifact availability, and runner load all introduce variance. Read the verdict rather than the point estimate: a comparison reported as `inconclusive` has not established anything, however suggestive its number looks, and one reported as `within-noise` is smaller than the harness can resolve. Check the A/A control before trusting any headline — if it resolves a difference, the run is measuring the harness rather than the code.
 
 ## Local checks
 
diff --git a/scripts/jdk-cache.sh b/scripts/jdk-cache.sh
new file mode 100755
index 0000000..93fce2d
--- /dev/null
+++ b/scripts/jdk-cache.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+# Cache-identity and reset helpers for the JDK cache benchmark.
+
+set -euo pipefail
+
+command=${1:?command is required}
+
+write_identity() {
+  local benchmark_id=$1
+  printf '%s\n' "$benchmark_id" > .jdk-cache-key
+  mkdir -p .mvn/wrapper
+  printf '# jdk-cache-benchmark=%s\n' "$benchmark_id" \
+    > .mvn/wrapper/maven-wrapper.properties
+}
+
+purge_toolcache() {
+  # The benchmark deliberately measures the path where the requested JDK is not
+  # already installed on the runner image, because that is the only case in
+  # which the JDK cache does any work.
+  rm -rf "${RUNNER_TOOL_CACHE:?}"/Java_*
+}
+
+case "$command" in
+  prepare)
+    write_identity "${2:?benchmark id is required}"
+    ;;
+  purge-toolcache)
+    purge_toolcache
+    ;;
+  reset)
+    # Every measured slot must start from the same empty state. Without this the
+    # second and later slots in a job would extract over files an earlier slot
+    # left behind and report an artificially low duration.
+    purge_toolcache
+    rm -rf "$HOME/.m2"
+    write_identity "${2:?benchmark id is required}"
+    ;;
+  *)
+    echo "Unsupported command: $command" >&2
+    exit 1
+    ;;
+esac
diff --git a/scripts/paired.mjs b/scripts/paired.mjs
new file mode 100644
index 0000000..a9968f3
--- /dev/null
+++ b/scripts/paired.mjs
@@ -0,0 +1,244 @@
+// Shared machinery for same-runner paired benchmarks.
+//
+// Every workflow here measures each arm twice inside a single job, in an order
+// that is mirrored about the middle of the job. For two arms that is ABBA; for N
+// arms it is 1..N followed by N..1. Two properties follow:
+//
+//   - Differencing within a runner removes between-runner variance, which on
+//     hosted runners is larger than the effects under test and cannot be
+//     averaged away by adding independent jobs to each arm.
+//   - Averaging an arm's two slots removes any drift that is linear across the
+//     job, because the mirrored order places those slots symmetrically about the
+//     centre.
+//
+// Measuring each arm twice also yields a null-effect (A/A) estimate at no extra
+// job cost: the spread between an arm's own two slots is what the harness
+// reports when nothing has changed.
+
+import { readdir, readFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import {
+  classify,
+  hodgesLehmann,
+  mean,
+  median,
+  medianAbsoluteDeviation,
+  pairedInterval,
+  pairedPermutationTest,
+  quantile,
+  standardDeviation,
+} from "./stats.mjs";
+
+export const RESULTS_DIR = ".benchmark-results";
+
+// Each measurement job uploads its own CSV so that merging the artifacts cannot
+// overwrite another runner's samples.
+export async function readSampleFiles(prefix, directory = RESULTS_DIR) {
+  const entries = await readdir(directory);
+  const files = entries.filter(
+    (entry) => entry.startsWith(prefix) && entry.endsWith(".csv"),
+  );
+  if (files.length === 0) {
+    throw new Error(`No ${prefix}*.csv timing files found in ${directory}`);
+  }
+  const contents = await Promise.all(
+    files.sort().map((file) => readFile(join(directory, file), "utf8")),
+  );
+  return contents.join("\n");
+}
+
+export function parseSamples(csv) {
+  return csv
+    .trim()
+    .split("\n")
+    .filter(Boolean)
+    .map((line) => {
+      const [sample, arm, slot, elapsedMs] = line
+        .split(",")
+        .map((value) => value.replace(/^"|"$/g, ""));
+      return {
+        sample: Number(sample),
+        arm,
+        slot: Number(slot),
+        seconds: Number(elapsedMs) / 1000,
+      };
+    });
+}
+
+// Groups rows by runner and arm, keeping only runners that measured every arm
+// the expected number of times. A runner that failed part way through would
+// otherwise contribute a difference computed from unbalanced slots.
+export function groupByRunner(rows, arms, slotsPerArm = 2) {
+  const bySample = new Map();
+  for (const row of rows) {
+    if (!arms.includes(row.arm)) continue;
+    const entry = bySample.get(row.sample) ?? new Map();
+    entry.set(row.arm, [...(entry.get(row.arm) ?? []), row]);
+    bySample.set(row.sample, entry);
+  }
+  const runners = [];
+  for (const [sample, entry] of [...bySample.entries()].sort(
+    (a, b) => a[0] - b[0],
+  )) {
+    if (arms.some((arm) => (entry.get(arm) ?? []).length !== slotsPerArm)) {
+      continue;
+    }
+    const slots = new Map();
+    for (const arm of arms) {
+      slots.set(
+        arm,
+        entry
+          .get(arm)
+          .sort((a, b) => a.slot - b.slot)
+          .map((row) => row.seconds),
+      );
+    }
+    runners.push({ sample, slots });
+  }
+  return runners;
+}
+
+export function buildPairs(
+  rows,
+  baselineArm = "baseline",
+  candidateArm = "candidate",
+) {
+  return groupByRunner(rows, [baselineArm, candidateArm]).map((runner) => {
+    const baselineSlots = runner.slots.get(baselineArm);
+    const candidateSlots = runner.slots.get(candidateArm);
+    return {
+      sample: runner.sample,
+      baselineSlots,
+      candidateSlots,
+      baseline: mean(baselineSlots),
+      candidate: mean(candidateSlots),
+      difference: mean(candidateSlots) - mean(baselineSlots),
+      baselineRepeatDelta: baselineSlots[1] - baselineSlots[0],
+      candidateRepeatDelta: candidateSlots[1] - candidateSlots[0],
+    };
+  });
+}
+
+// The smallest effect the harness can trust, derived from how much the same
+// implementation varies between its two slots on one runner.
+export function noiseFloor(pairs) {
+  const repeats = [
+    ...pairs.map((pair) => Math.abs(pair.baselineRepeatDelta)),
+    ...pairs.map((pair) => Math.abs(pair.candidateRepeatDelta)),
+  ];
+  if (repeats.length === 0) return 0;
+  return quantile(repeats, 0.5);
+}
+
+export function armSummary(name, values) {
+  return {
+    arm: name,
+    samples: values.length,
+    meanSeconds: mean(values),
+    medianSeconds: median(values),
+    standardDeviationSeconds: standardDeviation(values),
+    madSeconds: medianAbsoluteDeviation(values),
+    p95Seconds: quantile(values, 0.95),
+  };
+}
+
+export function analyzePairs(
+  rows,
+  baselineArm = "baseline",
+  candidateArm = "candidate",
+) {
+  const pairs = buildPairs(rows, baselineArm, candidateArm);
+  const differences = pairs.map((pair) => pair.difference);
+  const baselineValues = pairs.map((pair) => pair.baseline);
+  const candidateValues = pairs.map((pair) => pair.candidate);
+  const floor = noiseFloor(pairs);
+  const interval = pairedInterval(differences, { seed: 1 });
+  // The same estimator applied to the within-arm repeats. A trustworthy harness
+  // must not resolve a difference here, because it compares an arm with itself.
+  // Judged against the same noise floor as the real effect, so a healthy run
+  // reports `within-noise` or `inconclusive`.
+  const controlInterval = pairedInterval(
+    pairs.map((pair) => pair.baselineRepeatDelta),
+    { seed: 2 },
+  );
+  return {
+    pairs,
+    noiseFloorSeconds: floor,
+    baseline: armSummary(baselineArm, baselineValues),
+    candidate: armSummary(candidateArm, candidateValues),
+    interval,
+    pValue: pairedPermutationTest(differences, { seed: 3 }),
+    shiftSeconds: hodgesLehmann(candidateValues, baselineValues),
+    verdict: classify(interval, { noiseFloor: floor }),
+    control: {
+      interval: controlInterval,
+      verdict: classify(controlInterval, { noiseFloor: floor }),
+    },
+  };
+}
+
+// Compares every arm against a reference arm, one paired difference per runner.
+// Used where the workflow measures more than two implementations in the same
+// job, such as the version sweep.
+export function analyzeAgainstReference(rows, arms, reference) {
+  const runners = groupByRunner(rows, arms);
+  const perArm = new Map(
+    arms.map((arm) => [
+      arm,
+      runners.map((runner) => mean(runner.slots.get(arm))),
+    ]),
+  );
+  const repeats = runners.flatMap((runner) =>
+    arms.map((arm) => {
+      const slots = runner.slots.get(arm);
+      return Math.abs(slots[1] - slots[0]);
+    }),
+  );
+  const floor = repeats.length === 0 ? 0 : quantile(repeats, 0.5);
+  const referenceValues = perArm.get(reference) ?? [];
+  const comparisons = arms.map((arm, index) => {
+    const values = perArm.get(arm);
+    const differences = values.map(
+      (value, runner) => value - referenceValues[runner],
+    );
+    const isReference = arm === reference;
+    const interval = isReference
+      ? null
+      : pairedInterval(differences, { seed: 10 + index });
+    return {
+      arm,
+      isReference,
+      summary: armSummary(arm, values),
+      differenceSeconds: isReference ? 0 : mean(differences),
+      interval,
+      pValue: isReference
+        ? null
+        : pairedPermutationTest(differences, { seed: 50 + index }),
+      verdict: isReference
+        ? "reference"
+        : classify(interval, { noiseFloor: floor }),
+    };
+  });
+  // Each arm's own two slots compared with themselves. Nothing changed between
+  // them, so anything the estimator resolves here is harness bias.
+  const controlDifferences = runners.map((runner) => {
+    const slots = runner.slots.get(reference);
+    return slots[1] - slots[0];
+  });
+  const controlInterval =
+    controlDifferences.length === 0
+      ? null
+      : pairedInterval(controlDifferences, { seed: 99 });
+  return {
+    runners,
+    arms,
+    reference,
+    noiseFloorSeconds: floor,
+    comparisons,
+    control: {
+      interval: controlInterval,
+      verdict: classify(controlInterval, { noiseFloor: floor }),
+    },
+  };
+}
diff --git a/scripts/report-focused.mjs b/scripts/report-focused.mjs
index 0d1a80f..80e999b 100644
--- a/scripts/report-focused.mjs
+++ b/scripts/report-focused.mjs
@@ -10,149 +10,26 @@ import { pathToFileURL } from "node:url";
 
 import { hashFilesSingle } from "./report.mjs";
 import {
-  classify,
-  describeVerdict,
-  formatInterval,
-  hodgesLehmann,
-  mean,
-  median,
-  medianAbsoluteDeviation,
-  pairedInterval,
-  pairedPermutationTest,
-  quantile,
-  standardDeviation,
-} from "./stats.mjs";
-
-const API_VERSION = "2022-11-28";
-const RESULTS_DIR = ".benchmark-results";
-
-// Every measurement job uploads its own CSV so that merging the artifacts
-// cannot overwrite another runner's samples.
-export async function readSampleFiles(directory = RESULTS_DIR) {
-  const entries = await readdir(directory);
-  const files = entries.filter(
-    (entry) => entry.startsWith("focused-timings") && entry.endsWith(".csv"),
-  );
-  if (files.length === 0) {
-    throw new Error(`No focused timing CSVs found in ${directory}`);
-  }
-  const contents = await Promise.all(
-    files.sort().map((file) => readFile(join(directory, file), "utf8")),
-  );
-  return contents.join("\n");
-}
-
-// Each runner measures four slots in ABBA order: baseline, candidate,
-// candidate, baseline. Averaging the two slots per arm cancels any linear drift
-// across the job, and differencing within a runner removes between-runner
-// variance.
-export function parseSamples(csv) {
-  return csv
-    .trim()
-    .split("\n")
-    .filter(Boolean)
-    .map((line) => {
-      const [sample, arm, slot, elapsedMs] = line
-        .split(",")
-        .map((value) => value.replace(/^"|"$/g, ""));
-      return {
-        sample: Number(sample),
-        arm,
-        slot: Number(slot),
-        seconds: Number(elapsedMs) / 1000,
-      };
-    });
-}
-
-// One paired observation per runner, plus the within-arm repeat difference that
-// serves as a null-effect (A/A) measurement requiring no extra jobs.
-export function buildPairs(rows) {
-  const bySample = new Map();
-  for (const row of rows) {
-    const entry = bySample.get(row.sample) ?? { baseline: [], candidate: [] };
-    if (!entry[row.arm]) continue;
-    entry[row.arm].push(row);
-    bySample.set(row.sample, entry);
-  }
-  const pairs = [];
-  for (const [sample, entry] of [...bySample.entries()].sort(
-    (a, b) => a[0] - b[0],
-  )) {
-    if (entry.baseline.length !== 2 || entry.candidate.length !== 2) continue;
-    const baselineSlots = entry.baseline
-      .sort((a, b) => a.slot - b.slot)
-      .map((row) => row.seconds);
-    const candidateSlots = entry.candidate
-      .sort((a, b) => a.slot - b.slot)
-      .map((row) => row.seconds);
-    pairs.push({
-      sample,
-      baselineSlots,
-      candidateSlots,
-      baseline: mean(baselineSlots),
-      candidate: mean(candidateSlots),
-      difference: mean(candidateSlots) - mean(baselineSlots),
-      baselineRepeatDelta: baselineSlots[1] - baselineSlots[0],
-      candidateRepeatDelta: candidateSlots[1] - candidateSlots[0],
-    });
-  }
-  return pairs;
-}
-
-// The smallest effect the harness can trust. Derived from how much the same
-// implementation varies between its two slots on one runner.
-export function noiseFloor(pairs) {
-  const repeats = [
-    ...pairs.map((pair) => Math.abs(pair.baselineRepeatDelta)),
-    ...pairs.map((pair) => Math.abs(pair.candidateRepeatDelta)),
-  ];
-  if (repeats.length === 0) return 0;
-  return quantile(repeats, 0.5);
-}
-
-function armSummary(name, values) {
-  return {
-    arm: name,
-    samples: values.length,
-    meanSeconds: mean(values),
-    medianSeconds: median(values),
-    standardDeviationSeconds: standardDeviation(values),
-    madSeconds: medianAbsoluteDeviation(values),
-    p95Seconds: quantile(values, 0.95),
-  };
+  analyzePairs,
+  buildPairs,
+  noiseFloor,
+  parseSamples,
+  readSampleFiles as readPairedSampleFiles,
+} from "./paired.mjs";
+import { classify, describeVerdict, formatInterval } from "./stats.mjs";
+
+export { buildPairs, noiseFloor, parseSamples };
+
+export function readSampleFiles(directory) {
+  return readPairedSampleFiles("focused-timings", directory);
 }
 
 export function analyze(rows) {
-  const pairs = buildPairs(rows);
-  const differences = pairs.map((pair) => pair.difference);
-  const baselineValues = pairs.map((pair) => pair.baseline);
-  const candidateValues = pairs.map((pair) => pair.candidate);
-  const floor = noiseFloor(pairs);
-  const interval = pairedInterval(differences, { seed: 1 });
-  // The same estimator applied to the within-arm repeats. A trustworthy harness
-  // must not resolve a difference here, because it compares an arm with itself.
-  // Judged against the same noise floor as the real effect, so a healthy run
-  // reports `within-noise` or `inconclusive`.
-  const controlInterval = pairedInterval(
-    pairs.map((pair) => pair.baselineRepeatDelta),
-    { seed: 2 },
-  );
-  return {
-    pairs,
-    noiseFloorSeconds: floor,
-    baseline: armSummary("baseline", baselineValues),
-    candidate: armSummary("candidate", candidateValues),
-    interval,
-    pValue: pairedPermutationTest(differences, { seed: 3 }),
-    shiftSeconds: hodgesLehmann(candidateValues, baselineValues),
-    verdict: classify(interval, { noiseFloor: floor }),
-    control: {
-      interval: controlInterval,
-      verdict: classify(controlInterval, { noiseFloor: floor }),
-    },
-  };
+  return analyzePairs(rows, "baseline", "candidate");
 }
 
+const API_VERSION = "2022-11-28";
+
 function csvValue(value) {
   return `"${String(value ?? "").replaceAll('"', '""')}"`;
 }
diff --git a/scripts/report-jdk-cache.mjs b/scripts/report-jdk-cache.mjs
index d6a5648..1105b29 100644
--- a/scripts/report-jdk-cache.mjs
+++ b/scripts/report-jdk-cache.mjs
@@ -1,97 +1,28 @@
-import {appendFile, mkdir, writeFile} from 'node:fs/promises';
-import {pathToFileURL} from 'node:url';
+import { appendFile, mkdir, writeFile } from "node:fs/promises";
+import { pathToFileURL } from "node:url";
 
-import {hashFilesSingle, median, secondsBetween} from './report.mjs';
+import { analyzePairs, parseSamples, readSampleFiles } from "./paired.mjs";
+import { describeVerdict, formatInterval } from "./stats.mjs";
 
-const API_VERSION = '2022-11-28';
-const ARMS = ['baseline', 'treatment'];
-const JOB_PATTERN = /^(baseline|treatment) \/ (seed|warm)(?: \/ (\d+))?$/;
-const JDK_CACHE_PREFIX = 'setup-java-jdk-';
-
-export function parseJdkCacheJob(name) {
-  const match = name.match(JOB_PATTERN);
-  if (!match) return null;
-  const [, arm, phase, sampleText] = match;
-  if ((phase === 'seed' && sampleText) || (phase === 'warm' && !sampleText)) {
-    return null;
-  }
-  return {
-    arm,
-    phase,
-    sample: sampleText ? Number(sampleText) : null
-  };
-}
-
-export function newestJdkCache(cacheEntries) {
-  return (
-    cacheEntries
-      .filter(cache => cache.key.startsWith(JDK_CACHE_PREFIX))
-      .sort(
-        (a, b) =>
-          Date.parse(b.created_at ?? 0) - Date.parse(a.created_at ?? 0)
-      )[0] ?? null
-  );
-}
-
-export function expectedArmCacheKeys(arm, runId, wrapperOriginal) {
-  const benchmarkId = `jdk-cache-${arm}-${runId}`;
-  return {
-    dependencies: `setup-java-Linux-x64-maven-${hashFilesSingle(
-      `${benchmarkId}\n`
-    )}`,
-    wrapper: `setup-java-Linux-x64-maven-wrapper-${hashFilesSingle(
-      `${wrapperOriginal}# jdk-cache-benchmark=${benchmarkId}\n`
-    )}`
-  };
-}
-
-export function summarizeJdkArm(rows, arm) {
-  const armRows = rows.filter(row => row.arm === arm);
-  const warmRows = armRows.filter(row => row.phase === 'warm');
-  const seed = armRows.find(row => row.phase === 'seed') ?? null;
-  return {
-    arm,
-    samples: warmRows.length,
-    warmSetupSeconds: median(warmRows.map(row => row.setupSeconds)),
-    warmBuildSeconds: median(warmRows.map(row => row.buildSeconds)),
-    warmPostSeconds: median(warmRows.map(row => row.postSeconds)),
-    warmJobSeconds: median(warmRows.map(row => row.jobSeconds)),
-    coldSetupSeconds: seed?.setupSeconds ?? null,
-    coldPostSeconds: seed?.postSeconds ?? null,
-    estimatedBilledMinutes: warmRows.reduce(
-      (total, row) =>
-        total +
-        (row.jobSeconds === null ? 0 : Math.ceil(row.jobSeconds / 60)),
-      0
-    )
-  };
-}
-
-function formatSeconds(value) {
-  return value === null ? 'n/a' : value.toFixed(1);
-}
-
-function formatMiB(bytes) {
-  return bytes === null ? 'n/a' : (bytes / 1024 / 1024).toFixed(1);
-}
+const API_VERSION = "2022-11-28";
+const RESULTS_DIR = "jdk-cache-results";
 
 function csvValue(value) {
-  const text = value === null || value === undefined ? '' : String(value);
-  return `"${text.replaceAll('"', '""')}"`;
+  return `"${String(value ?? "").replaceAll('"', '""')}"`;
 }
 
 async function api(path, token, options = {}) {
   const response = await fetch(`https://api.github.com${path}`, {
     ...options,
     headers: {
-      Accept: 'application/vnd.github+json',
+      Accept: "application/vnd.github+json",
       Authorization: `Bearer ${token}`,
-      'X-GitHub-Api-Version': API_VERSION,
-      ...options.headers
-    }
+      "X-GitHub-Api-Version": API_VERSION,
+      ...options.headers,
+    },
   });
   if (!response.ok) {
-    throw new Error(`${options.method ?? 'GET'} ${path}: ${response.status}`);
+    throw new Error(`${options.method ?? "GET"} ${path}: ${response.status}`);
   }
   if (response.status === 204) return null;
   return response.json();
@@ -100,248 +31,153 @@ async function api(path, token, options = {}) {
 async function allPages(path, field, token) {
   const values = [];
   for (let page = 1; ; page += 1) {
-    const separator = path.includes('?') ? '&' : '?';
+    const separator = path.includes("?") ? "&" : "?";
     const response = await api(
       `${path}${separator}per_page=100&page=${page}`,
-      token
+      token,
     );
     values.push(...response[field]);
     if (response[field].length < 100) return values;
   }
 }
 
-function stepDuration(job, predicate) {
-  const step = job.steps.find(predicate);
-  return step ? secondsBetween(step.started_at, step.completed_at) : null;
-}
-
-function buildCaches(cacheEntries, runId, wrapperOriginal) {
-  const caches = [];
-  for (const arm of ARMS) {
-    const expected = expectedArmCacheKeys(arm, runId, wrapperOriginal);
-    for (const [type, key] of Object.entries(expected)) {
-      const entry = cacheEntries.find(cache => cache.key === key);
-      caches.push({
-        arm,
-        type,
-        key,
-        id: entry?.id ?? null,
-        sizeBytes: entry?.size_in_bytes ?? null
-      });
-    }
-  }
-
-  const jdkEntry = newestJdkCache(cacheEntries);
-  caches.push({
-    arm: 'treatment',
-    type: 'jdk',
-    key: jdkEntry?.key ?? null,
-    id: jdkEntry?.id ?? null,
-    sizeBytes: jdkEntry?.size_in_bytes ?? null
-  });
-  return caches;
-}
-
-function markdown(metadata, rows, summaries, caches) {
-  const baseline = summaries.find(summary => summary.arm === 'baseline');
-  const treatment = summaries.find(summary => summary.arm === 'treatment');
-  const jdkCache = caches.find(cache => cache.type === 'jdk');
-  const delta =
-    baseline.warmSetupSeconds === null || treatment.warmSetupSeconds === null
-      ? null
-      : treatment.warmSetupSeconds - baseline.warmSetupSeconds;
+export function markdown(metadata, analysis, caches) {
+  const { baseline, candidate, interval, control } = analysis;
   const lines = [
-    '# JDK cache benchmark',
-    '',
-    `${metadata.distribution} Java ${metadata.javaVersion}, ${metadata.samples} warm samples per arm, run ${metadata.runId}.`,
-    '',
-    '## Headline',
-    '',
-    `Baseline warm setup: **${formatSeconds(baseline.warmSetupSeconds)}s**; treatment warm setup: **${formatSeconds(treatment.warmSetupSeconds)}s**; treatment delta: **${formatSeconds(delta)}s**. JDK cache storage: **${formatMiB(jdkCache.sizeBytes)} MiB**.`,
-    ''
+    "# JDK cache benchmark",
+    "",
+    `${metadata.distribution} ${metadata.javaVersion}, ${analysis.pairs.length} runners x 2 paired observations, run ${metadata.runId}.`,
+    `\`cache-jdk: false\` vs \`cache-jdk: true\` using \`${metadata.setupJavaRef}\` from \`${metadata.setupJavaRepository}\`.`,
+    "",
+    "The runner tool cache is purged before every measured slot, so each setup",
+    "resolves the JDK the way a runner image without it would.",
+    "",
+    "## Verdict",
+    "",
+    `**${describeVerdict(analysis.verdict)}**`,
+    "",
+    `Paired difference (\`cache-jdk: true\` - \`cache-jdk: false\`): **${formatInterval(interval)}**.`,
+    `Permutation p-value: ${analysis.pValue.toFixed(3)}. Hodges-Lehmann shift: ${analysis.shiftSeconds.toFixed(3)}s.`,
+    `Harness noise floor: ${analysis.noiseFloorSeconds.toFixed(3)}s (median within-runner repeat spread).`,
+    "",
+    `A/A control (\`cache-jdk: false\` against itself) reports **${control.verdict}** at ${formatInterval(control.interval)}. A healthy harness reports \`within-noise\` or \`inconclusive\` here; an \`improvement\` or \`regression\` means slot ordering is biasing results and the verdict above cannot be trusted.`,
+    "",
+    "## Arms",
+    "",
+    "| Arm | Runners | Mean (s) | Median (s) | SD (s) | MAD (s) | p95 (s) |",
+    "| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
   ];
-
-  if (jdkCache.id === null) {
+  for (const [label, arm] of [
+    ["cache-jdk: false", baseline],
+    ["cache-jdk: true", candidate],
+  ]) {
     lines.push(
-      '> [!WARNING]',
-      '> No `setup-java-jdk-` cache entry exists after the treatment seed. The runner tool cache may have been hit or JDK caching was disabled, so this run does not measure a JDK cache restore.',
-      ''
+      `| \`${label}\` | ${arm.samples} | ${arm.meanSeconds.toFixed(3)} | ${arm.medianSeconds.toFixed(3)} | ${arm.standardDeviationSeconds.toFixed(3)} | ${arm.madSeconds.toFixed(3)} | ${arm.p95Seconds.toFixed(3)} |`,
     );
   }
-
   lines.push(
-    '| Arm | Warm samples | Median setup (s) | Median build (s) | Median post-step (s) | Median job (s) | Cold seed setup (s) | Cold seed post-save (s) | Estimated billed minutes |',
-    '| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |'
+    "",
+    "All durations are measured inside the job with millisecond resolution. The Actions API reports step timestamps only to the nearest second, which is too coarse for effects of this size.",
+    "",
+    "## Paired samples",
+    "",
+    "| Runner | no-cache slot 1 (s) | cache slot 2 (s) | cache slot 3 (s) | no-cache slot 4 (s) | Paired delta (s) |",
+    "| ---: | ---: | ---: | ---: | ---: | ---: |",
   );
-  for (const summary of summaries) {
+  for (const pair of analysis.pairs) {
     lines.push(
-      `| ${summary.arm} | ${summary.samples} | ${formatSeconds(summary.warmSetupSeconds)} | ${formatSeconds(summary.warmBuildSeconds)} | ${formatSeconds(summary.warmPostSeconds)} | ${formatSeconds(summary.warmJobSeconds)} | ${formatSeconds(summary.coldSetupSeconds)} | ${formatSeconds(summary.coldPostSeconds)} | ${summary.estimatedBilledMinutes} |`
+      `| ${pair.sample} | ${pair.baselineSlots[0].toFixed(3)} | ${pair.candidateSlots[0].toFixed(3)} | ${pair.candidateSlots[1].toFixed(3)} | ${pair.baselineSlots[1].toFixed(3)} | ${pair.difference.toFixed(3)} |`,
     );
   }
-
   lines.push(
-    '',
-    'The billed-minute estimate covers only warm measurement jobs, rounding each Ubuntu job up to a whole minute. It excludes seeds because they are one-time setup and the treatment seed pays the JDK save, which would distort the steady-state comparison. It also excludes the shared prepare and report jobs.',
-    '',
-    '## Cache entries',
-    '',
-    '| Arm | Cache | Size (MiB) | Key |',
-    '| --- | --- | ---: | --- |'
+    "",
+    "## Caches",
+    "",
+    "Both arms restore the same Maven entry, so the stored blob cannot bias the comparison. Only the JDK entry differs between them.",
+    "",
+    "| Cache | Size (MiB) |",
+    "| --- | ---: |",
   );
   for (const cache of caches) {
     lines.push(
-      `| ${cache.arm} | ${cache.type} | ${formatMiB(cache.sizeBytes)} | ${cache.key === null ? 'not found' : `\`${cache.key}\``} |`
-    );
-  }
-
-  lines.push(
-    '',
-    '## Samples',
-    '',
-    '| Arm | Phase | Sample | Setup (s) | Build (s) | Post-step (s) | Job (s) | Conclusion |',
-    '| --- | --- | ---: | ---: | ---: | ---: | ---: | --- |'
-  );
-  for (const row of rows) {
-    lines.push(
-      `| ${row.arm} | ${row.phase} | ${row.sample ?? '-'} | ${formatSeconds(row.setupSeconds)} | ${formatSeconds(row.buildSeconds)} | ${formatSeconds(row.postSeconds)} | ${formatSeconds(row.jobSeconds)} | ${row.conclusion} |`
+      `| ${cache.type} | ${(cache.sizeBytes / 1024 / 1024).toFixed(1)} |`,
     );
   }
-  return `${lines.join('\n')}\n`;
+  return `${lines.join("\n")}\n`;
 }
 
 export async function main(env = process.env) {
-  const [owner, repo] = env.GITHUB_REPOSITORY.split('/');
+  const [owner, repo] = env.GITHUB_REPOSITORY.split("/");
   const token = env.GH_TOKEN;
   const runId = env.GITHUB_RUN_ID;
-  const attempt = env.GITHUB_RUN_ATTEMPT;
-  const samples = Number(env.SAMPLES);
-  if (
-    !owner ||
-    !repo ||
-    !token ||
-    !runId ||
-    !attempt ||
-    !samples ||
-    !env.PETCLINIC_REF ||
-    !env.DISTRIBUTION ||
-    !env.JAVA_VERSION
-  ) {
-    throw new Error('Missing required GitHub Actions environment variables');
+  if (!owner || !repo || !token || !runId) {
+    throw new Error("Missing required GitHub Actions environment variables");
   }
 
-  const [jobs, cacheEntries, wrapperResponse, mainCommit] = await Promise.all([
-    allPages(
-      `/repos/${owner}/${repo}/actions/runs/${runId}/attempts/${attempt}/jobs`,
-      'jobs',
-      token
-    ),
-    allPages(`/repos/${owner}/${repo}/actions/caches`, 'actions_caches', token),
-    api(
-      `/repos/spring-projects/spring-petclinic/contents/.mvn/wrapper/maven-wrapper.properties?ref=${env.PETCLINIC_REF}`,
-      token
-    ),
-    api('/repos/actions/setup-java/commits/main', token)
-  ]);
-
-  const rows = jobs
-    .map(job => {
-      const identity = parseJdkCacheJob(job.name);
-      if (!identity) return null;
-      return {
-        ...identity,
-        conclusion: job.conclusion,
-        setupSeconds: stepDuration(job, step =>
-          step.name.startsWith('Setup Java')
-        ),
-        buildSeconds: stepDuration(job, step => step.name === 'Build PetClinic'),
-        postSeconds: stepDuration(job, step =>
-          step.name.startsWith('Post Setup Java')
-        ),
-        jobSeconds: secondsBetween(job.started_at, job.completed_at)
-      };
-    })
-    .filter(Boolean)
-    .sort(
-      (a, b) =>
-        a.arm.localeCompare(b.arm) ||
-        a.phase.localeCompare(b.phase) ||
-        (a.sample ?? 0) - (b.sample ?? 0)
-    );
+  const rows = parseSamples(await readSampleFiles("jdk-cache-timings"));
+  const analysis = analyzePairs(rows, "baseline", "candidate");
+  if (analysis.pairs.length === 0) {
+    throw new Error("No complete ABBA samples were collected");
+  }
 
-  const wrapperOriginal = Buffer.from(wrapperResponse.content, 'base64').toString(
-    'utf8'
+  const cacheEntries = await allPages(
+    `/repos/${owner}/${repo}/actions/caches`,
+    "actions_caches",
+    token,
   );
-  const caches = buildCaches(cacheEntries, runId, wrapperOriginal);
-  const missingCaches = caches.filter(cache => cache.id === null);
-  if (missingCaches.length > 0) {
-    console.warn(`Could not find ${missingCaches.length} expected cache entries`);
-  }
+  const caches = cacheEntries
+    .filter(
+      (entry) =>
+        entry.key.startsWith("setup-java-jdk-") ||
+        entry.key.startsWith("setup-java-Linux-x64-maven"),
+    )
+    .map((entry) => ({
+      type: entry.key.startsWith("setup-java-jdk-") ? "jdk" : "maven",
+      id: entry.id,
+      key: entry.key,
+      sizeBytes: entry.size_in_bytes,
+    }));
 
   const metadata = {
     repository: env.GITHUB_REPOSITORY,
     runId,
-    runAttempt: Number(attempt),
-    samples,
     distribution: env.DISTRIBUTION,
     javaVersion: env.JAVA_VERSION,
-    petclinicRef: env.PETCLINIC_REF,
-    setupJavaMainRefAtReport: mainCommit.sha,
-    generatedAt: new Date().toISOString()
+    setupJavaRepository: env.SETUP_JAVA_REPOSITORY,
+    setupJavaRef: env.SETUP_JAVA_REF,
+    generatedAt: new Date().toISOString(),
   };
-  const summaries = ARMS.map(arm => summarizeJdkArm(rows, arm));
-  const report = markdown(metadata, rows, summaries, caches);
 
-  await mkdir('jdk-cache-results', {recursive: true});
+  const report = markdown(metadata, analysis, caches);
+  await mkdir(RESULTS_DIR, { recursive: true });
   await writeFile(
-    'jdk-cache-results/results.json',
-    `${JSON.stringify({metadata, summaries, rows, caches}, null, 2)}\n`
-  );
-  const csvHeaders = [
-    'arm',
-    'phase',
-    'sample',
-    'setup_seconds',
-    'build_seconds',
-    'post_seconds',
-    'job_seconds',
-    'conclusion'
-  ];
-  const csvRows = rows.map(row =>
-    [
-      row.arm,
-      row.phase,
-      row.sample,
-      row.setupSeconds,
-      row.buildSeconds,
-      row.postSeconds,
-      row.jobSeconds,
-      row.conclusion
-    ]
-      .map(csvValue)
-      .join(',')
+    `${RESULTS_DIR}/results.json`,
+    `${JSON.stringify({ metadata, analysis, caches }, null, 2)}\n`,
   );
   await writeFile(
-    'jdk-cache-results/results.csv',
-    `${csvHeaders.join(',')}\n${csvRows.join('\n')}\n`
+    `${RESULTS_DIR}/results.csv`,
+    `sample,arm,slot,seconds\n${rows
+      .map((row) =>
+        [row.sample, row.arm, row.slot, row.seconds].map(csvValue).join(","),
+      )
+      .join("\n")}\n`,
   );
-  await writeFile('jdk-cache-results/summary.md', report);
+  await writeFile(`${RESULTS_DIR}/summary.md`, report);
   await appendFile(env.GITHUB_STEP_SUMMARY, report);
 
-  if (env.CLEANUP_CACHES === 'true') {
-    const benchmarkIds = caches.map(cache => cache.id).filter(Boolean);
-    const jdkIds = cacheEntries
-      .filter(cache => cache.key.startsWith(JDK_CACHE_PREFIX))
-      .map(cache => cache.id);
-    const ids = [...new Set([...benchmarkIds, ...jdkIds])];
-    for (const id of ids) {
-      await api(`/repos/${owner}/${repo}/actions/caches/${id}`, token, {
-        method: 'DELETE'
+  if (env.CLEANUP_CACHES === "true") {
+    for (const cache of caches) {
+      await api(`/repos/${owner}/${repo}/actions/caches/${cache.id}`, token, {
+        method: "DELETE",
       });
     }
-    console.log(`Deleted ${ids.length} JDK benchmark cache entries`);
+    console.log(`Deleted ${caches.length} JDK cache benchmark caches`);
   }
 }
 
-if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+if (
+  process.argv[1] &&
+  import.meta.url === pathToFileURL(process.argv[1]).href
+) {
   await main();
 }
diff --git a/scripts/report-jdk-cache.test.mjs b/scripts/report-jdk-cache.test.mjs
index e9e76a3..7698724 100644
--- a/scripts/report-jdk-cache.test.mjs
+++ b/scripts/report-jdk-cache.test.mjs
@@ -1,92 +1,89 @@
 import assert from "node:assert/strict";
 import test from "node:test";
 
-import {
-  expectedArmCacheKeys,
-  newestJdkCache,
-  parseJdkCacheJob,
-  summarizeJdkArm,
-} from "./report-jdk-cache.mjs";
+import { analyzePairs, parseSamples } from "./paired.mjs";
+import { markdown } from "./report-jdk-cache.mjs";
 
-test("parses JDK cache benchmark job names", () => {
-  assert.deepEqual(parseJdkCacheJob("baseline / seed"), {
-    arm: "baseline",
-    phase: "seed",
-    sample: null,
-  });
-  assert.deepEqual(parseJdkCacheJob("treatment / warm / 10"), {
-    arm: "treatment",
-    phase: "warm",
-    sample: 10,
-  });
-  assert.equal(parseJdkCacheJob("baseline / warm"), null);
-  assert.equal(parseJdkCacheJob("Report"), null);
-});
+function abba(sample, noCache, cache) {
+  return [
+    `"${sample}","baseline","1","${noCache[0]}"`,
+    `"${sample}","candidate","2","${cache[0]}"`,
+    `"${sample}","candidate","3","${cache[1]}"`,
+    `"${sample}","baseline","4","${noCache[1]}"`,
+  ].join("\n");
+}
 
-test("selects the newest JDK cache entry", () => {
-  const newest = newestJdkCache([
-    {
-      key: "setup-java-jdk-v1-Linux-x64-old",
-      created_at: "2026-01-01T00:00:00Z",
-    },
-    { key: "setup-java-Linux-x64-maven-not-a-jdk" },
-    {
-      key: "setup-java-jdk-v1-Linux-x64-new",
-      created_at: "2026-01-02T00:00:00Z",
-    },
-  ]);
-  assert.equal(newest.key, "setup-java-jdk-v1-Linux-x64-new");
-  assert.equal(newestJdkCache([{ key: "setup-java-Linux-maven-key" }]), null);
+const metadata = {
+  runId: "1",
+  distribution: "microsoft",
+  javaVersion: "17",
+  setupJavaRepository: "actions/setup-java",
+  setupJavaRef: "main",
+};
+
+test("resolves a large consistent JDK cache saving", () => {
+  // Restoring the JDK from the Actions cache instead of downloading it from the
+  // vendor is a multi-second effect, so it must clear the noise floor.
+  const csv = [
+    abba(1, [21000, 21400], [9000, 9200]),
+    abba(2, [23000, 22600], [10100, 9900]),
+    abba(3, [20500, 20900], [8800, 9100]),
+    abba(4, [24000, 23600], [11000, 10700]),
+    abba(5, [22000, 22300], [9500, 9800]),
+    abba(6, [21500, 21100], [9300, 9000]),
+  ].join("\n");
+  const analysis = analyzePairs(parseSamples(csv));
+  assert.equal(analysis.pairs.length, 6);
+  assert.equal(analysis.verdict, "improvement");
+  assert.ok(analysis.interval.high < 0);
+  assert.ok(analysis.interval.estimate < -11);
+  assert.equal(analysis.control.verdict !== "improvement", true);
 });
 
-test("derives distinct dependency and wrapper cache keys per arm", () => {
-  const baseline = expectedArmCacheKeys("baseline", "42", "wrapper=true\n");
-  const treatment = expectedArmCacheKeys("treatment", "42", "wrapper=true\n");
-  assert.match(baseline.dependencies, /^setup-java-Linux-x64-maven-/);
-  assert.match(baseline.wrapper, /^setup-java-Linux-x64-maven-wrapper-/);
-  assert.notEqual(baseline.dependencies, treatment.dependencies);
-  assert.notEqual(baseline.wrapper, treatment.wrapper);
+test("reports an unchanged configuration as inconclusive", () => {
+  // Both arms behaving identically is the A/A case. The estimator must not
+  // resolve an effect, whatever the runners happen to be doing.
+  const csv = [
+    abba(1, [9000, 9600], [9300, 9100]),
+    abba(2, [12000, 11400], [11800, 12200]),
+    abba(3, [8600, 9000], [8800, 8500]),
+    abba(4, [15000, 14200], [14600, 15100]),
+    abba(5, [10200, 10800], [10500, 10100]),
+    abba(6, [9800, 9200], [9400, 9900]),
+  ].join("\n");
+  const analysis = analyzePairs(parseSamples(csv));
+  assert.ok(["inconclusive", "within-noise"].includes(analysis.verdict));
 });
 
-test("summarizes seed and warm measurements for an arm", () => {
-  const summary = summarizeJdkArm(
-    [
-      {
-        arm: "treatment",
-        phase: "seed",
-        setupSeconds: 7,
-        buildSeconds: 20,
-        postSeconds: 8,
-        jobSeconds: 64,
-      },
-      {
-        arm: "treatment",
-        phase: "warm",
-        setupSeconds: 4,
-        buildSeconds: 10,
-        postSeconds: 0.4,
-        jobSeconds: 30,
-      },
-      {
-        arm: "treatment",
-        phase: "warm",
-        setupSeconds: 2,
-        buildSeconds: 12,
-        postSeconds: 0.2,
-        jobSeconds: 40,
-      },
-    ],
-    "treatment",
+test("drops runners that did not complete all four slots", () => {
+  const csv = [
+    abba(1, [21000, 21400], [9000, 9200]),
+    '"2","baseline","1","21000"',
+    '"2","candidate","2","9000"',
+  ].join("\n");
+  const analysis = analyzePairs(parseSamples(csv));
+  assert.deepEqual(
+    analysis.pairs.map((pair) => pair.sample),
+    [1],
   );
-  assert.deepEqual(summary, {
-    arm: "treatment",
-    samples: 2,
-    warmSetupSeconds: 3,
-    warmBuildSeconds: 11,
-    warmPostSeconds: 0.30000000000000004,
-    warmJobSeconds: 35,
-    coldSetupSeconds: 7,
-    coldPostSeconds: 8,
-    estimatedBilledMinutes: 2,
-  });
+});
+
+test("renders a verdict, both arms and the paired samples", () => {
+  const csv = [
+    abba(1, [21000, 21400], [9000, 9200]),
+    abba(2, [23000, 22600], [10100, 9900]),
+  ].join("\n");
+  const analysis = analyzePairs(parseSamples(csv));
+  const report = markdown(metadata, analysis, [
+    { type: "jdk", sizeBytes: 190 * 1024 * 1024 },
+    { type: "maven", sizeBytes: 60 * 1024 * 1024 },
+  ]);
+  assert.match(report, /# JDK cache benchmark/);
+  assert.match(report, /## Verdict/);
+  assert.match(report, /cache-jdk: true/);
+  assert.match(report, /A\/A control/);
+  assert.match(report, /Harness noise floor/);
+  assert.match(report, /\| jdk \| 190\.0 \|/);
+  // One row per runner, so a reader can see the raw slots behind the interval.
+  assert.match(report, /\| 1 \| 21\.000 \| 9\.000 \| 9\.200 \| 21\.400 \|/);
 });
diff --git a/scripts/report.mjs b/scripts/report.mjs
index 9344d54..138b75e 100644
--- a/scripts/report.mjs
+++ b/scripts/report.mjs
@@ -1,80 +1,58 @@
-import {createHash} from 'node:crypto';
-import {mkdir, writeFile, appendFile} from 'node:fs/promises';
-import {pathToFileURL} from 'node:url';
-
-const API_VERSION = '2022-11-28';
-const VERSIONS = ['v1', 'v2', 'v3', 'v4', 'v5.2', 'v5.6', 'main'];
-const VERSION_IDS = new Map([
-  ['v1', 'v1'],
-  ['v2', 'v2'],
-  ['v3', 'v3'],
-  ['v4', 'v4'],
-  ['v5.2', 'v52'],
-  ['v5.6', 'v56'],
-  ['main', 'main']
-]);
-const JOB_PATTERN =
-  /^(v1|v2|v3|v4|v5\.2|v5\.6|main) \/ (cold|warm) \/ (zulu|temurin|microsoft) \/ (\d+)$/;
+import { createHash } from "node:crypto";
+import { appendFile, mkdir, writeFile } from "node:fs/promises";
+import { pathToFileURL } from "node:url";
+
+import {
+  analyzeAgainstReference,
+  parseSamples,
+  readSampleFiles,
+} from "./paired.mjs";
+import { describeVerdict, formatInterval } from "./stats.mjs";
+
+const API_VERSION = "2022-11-28";
+const RESULTS_DIR = "results";
+const REFERENCE = "main";
+
+// Ordered oldest to newest. The warm sweep measures them in this order and then
+// in reverse, so each version's two slots sit symmetrically about the middle of
+// the job.
+export const VERSIONS = [
+  { arm: "v1", label: "v1.4.4", caching: "none" },
+  { arm: "v2", label: "v2.5.1", caching: "none" },
+  { arm: "v3", label: "v3.14.1", caching: "pom.xml" },
+  { arm: "v4", label: "v4.8.0", caching: "cache-dependency-path" },
+  { arm: "v52", label: "v5.2.0", caching: "cache-dependency-path" },
+  { arm: "v56", label: "v5.6.0", caching: "cache-dependency-path + wrapper" },
+  { arm: "main", label: "main", caching: "cache-dependency-path + wrapper" },
+];
+
+const LABELS = new Map(VERSIONS.map((entry) => [entry.arm, entry.label]));
 
 export function sha256(value) {
-  return createHash('sha256').update(value).digest('hex');
+  return createHash("sha256").update(value).digest("hex");
 }
 
 export function hashFilesSingle(value) {
-  const contentHash = createHash('sha256').update(value).digest();
-  return createHash('sha256').update(contentHash).digest('hex');
-}
-
-export function secondsBetween(start, end) {
-  if (!start || !end) return null;
-  return (Date.parse(end) - Date.parse(start)) / 1000;
-}
-
-export function median(values) {
-  const sorted = values.filter(value => value !== null).sort((a, b) => a - b);
-  if (sorted.length === 0) return null;
-  const middle = Math.floor(sorted.length / 2);
-  return sorted.length % 2
-    ? sorted[middle]
-    : (sorted[middle - 1] + sorted[middle]) / 2;
-}
-
-export function parseBenchmarkJob(name) {
-  const match = name.match(JOB_PATTERN);
-  if (!match) return null;
-  return {
-    version: match[1],
-    phase: match[2],
-    distribution: match[3],
-    iteration: Number(match[4])
-  };
-}
-
-function formatSeconds(value) {
-  return value === null ? 'n/a' : value.toFixed(1);
-}
-
-function formatMiB(bytes) {
-  return bytes === null ? 'n/a' : (bytes / 1024 / 1024).toFixed(1);
+  const contentHash = createHash("sha256").update(value).digest();
+  return createHash("sha256").update(contentHash).digest("hex");
 }
 
 function csvValue(value) {
-  const text = value === null || value === undefined ? '' : String(value);
-  return `"${text.replaceAll('"', '""')}"`;
+  return `"${String(value ?? "").replaceAll('"', '""')}"`;
 }
 
 async function api(path, token, options = {}) {
   const response = await fetch(`https://api.github.com${path}`, {
     ...options,
     headers: {
-      Accept: 'application/vnd.github+json',
+      Accept: "application/vnd.github+json",
       Authorization: `Bearer ${token}`,
-      'X-GitHub-Api-Version': API_VERSION,
-      ...options.headers
-    }
+      "X-GitHub-Api-Version": API_VERSION,
+      ...options.headers,
+    },
   });
   if (!response.ok) {
-    throw new Error(`${options.method ?? 'GET'} ${path}: ${response.status}`);
+    throw new Error(`${options.method ?? "GET"} ${path}: ${response.status}`);
   }
   if (response.status === 204) return null;
   return response.json();
@@ -83,292 +61,155 @@ async function api(path, token, options = {}) {
 async function allPages(path, field, token) {
   const values = [];
   for (let page = 1; ; page += 1) {
-    const separator = path.includes('?') ? '&' : '?';
+    const separator = path.includes("?") ? "&" : "?";
     const response = await api(
       `${path}${separator}per_page=100&page=${page}`,
-      token
+      token,
     );
     values.push(...response[field]);
     if (response[field].length < 100) return values;
   }
 }
 
-function stepDuration(job, predicate) {
-  const step = job.steps.find(predicate);
-  return step ? secondsBetween(step.started_at, step.completed_at) : null;
-}
-
-function summarize(rows, caches) {
-  return VERSIONS.map(version => {
-    const versionRows = rows.filter(row => row.version === version);
-    const coldRows = versionRows.filter(row => row.phase === 'cold');
-    const warmRows = versionRows.filter(row => row.phase === 'warm');
-    const versionCaches = caches.filter(cache => cache.version === version);
-    const caseTotals = new Map();
-    for (const cache of versionCaches) {
-      if (cache.sizeBytes === null) continue;
-      const key = `${cache.distribution}-${cache.iteration}`;
-      caseTotals.set(key, (caseTotals.get(key) ?? 0) + cache.sizeBytes);
-    }
-    return {
-      version,
-      coldSetupSeconds: median(coldRows.map(row => row.setupSeconds)),
-      warmSetupSeconds: median(warmRows.map(row => row.setupSeconds)),
-      coldBuildSeconds: median(coldRows.map(row => row.buildSeconds)),
-      warmBuildSeconds: median(warmRows.map(row => row.buildSeconds)),
-      postCacheSeconds: median(coldRows.map(row => row.postCacheSeconds)),
-      jobSeconds: median(versionRows.map(row => row.jobSeconds)),
-      estimatedBilledMinutes: versionRows.reduce(
-        (total, row) => total + Math.ceil(row.jobSeconds / 60),
-        0
-      ),
-      cacheMiBPerCase:
-        caseTotals.size === 0
-          ? null
-          : median([...caseTotals.values()]) / 1024 / 1024
-    };
-  });
-}
-
-function markdown(metadata, rows, caches, summaries) {
+export function markdown(metadata, analysis, caches) {
+  const { control } = analysis;
   const lines = [
-    '# setup-java benchmark',
-    '',
-    `Spring PetClinic \`${metadata.petclinicRef.slice(0, 12)}\`, Java ${metadata.javaVersion}, run ${metadata.runId}.`,
-    '',
-    '| Version | Cold setup (s) | Warm setup (s) | Cold build (s) | Warm build (s) | Cold post-cache (s) | Cache/case (MiB) | Estimated billed minutes |',
-    '| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |'
+    "# setup-java version sweep",
+    "",
+    `${metadata.distribution} ${metadata.javaVersion}, ${analysis.runners.length} runners x 2 observations per version, run ${metadata.runId}.`,
+    "",
+    "Every version is measured in the same job, in the order v1..main followed by",
+    "main..v1. Differencing within a runner removes between-runner variance, which",
+    "on hosted runners is larger than the differences between versions. The mirrored",
+    "order places each version's two slots symmetrically about the middle of the job,",
+    "so drift across the job cancels.",
+    "",
+    `Durations are the warm restore path with a Maven cache already populated, measured inside the job at millisecond resolution. Differences are against \`${REFERENCE}\`.`,
+    "",
+    "## Versions",
+    "",
+    `| Version | Caching | Median (s) | Mean (s) | MAD (s) | vs ${REFERENCE} (s) | 95% CI | p | Verdict |`,
+    "| --- | --- | ---: | ---: | ---: | ---: | --- | ---: | --- |",
   ];
-  for (const summary of summaries) {
+  for (const comparison of analysis.comparisons) {
+    const entry = VERSIONS.find((item) => item.arm === comparison.arm);
+    const summary = comparison.summary;
+    const diff = comparison.isReference
+      ? "reference"
+      : comparison.differenceSeconds.toFixed(3);
+    const ci = comparison.interval
+      ? `${comparison.interval.low.toFixed(3)} to ${comparison.interval.high.toFixed(3)}`
+      : "n/a";
+    const p = comparison.pValue === null ? "n/a" : comparison.pValue.toFixed(3);
     lines.push(
-      `| ${summary.version} | ${formatSeconds(summary.coldSetupSeconds)} | ${formatSeconds(summary.warmSetupSeconds)} | ${formatSeconds(summary.coldBuildSeconds)} | ${formatSeconds(summary.warmBuildSeconds)} | ${formatSeconds(summary.postCacheSeconds)} | ${summary.cacheMiBPerCase === null ? 'n/a' : summary.cacheMiBPerCase.toFixed(1)} | ${summary.estimatedBilledMinutes} |`
+      `| ${LABELS.get(comparison.arm)} | ${entry.caching} | ${summary.medianSeconds.toFixed(3)} | ${summary.meanSeconds.toFixed(3)} | ${summary.madSeconds.toFixed(3)} | ${diff} | ${ci} | ${p} | ${comparison.verdict} |`,
     );
   }
   lines.push(
-    '',
-    'Times and cache sizes are medians across distributions and iterations. Estimated billed minutes round each Linux job up to a whole minute; actual billing depends on the repository and runner plan.',
-    '',
-    '## Samples',
-    '',
-    '| Version | Cache | Distribution | Iteration | Setup (s) | Build (s) | Post-cache (s) | Job (s) | Conclusion |',
-    '| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | --- |'
+    "",
+    `Harness noise floor: ${analysis.noiseFloorSeconds.toFixed(3)}s (median spread between a version's own two slots on one runner). A difference smaller than this is reported as \`within-noise\`; one whose interval includes zero is reported as \`inconclusive\` rather than as a number that looks like a result.`,
+    "",
+    `A/A control (\`${REFERENCE}\` against itself) reports **${control.verdict}**${control.interval ? ` at ${formatInterval(control.interval)}` : ""}. A healthy harness reports \`within-noise\` or \`inconclusive\` here; anything else means slot ordering is biasing results and the table above cannot be trusted.`,
+    "",
+    "## Per-runner medians",
+    "",
+    `| Runner | ${VERSIONS.map((entry) => entry.label).join(" | ")} |`,
+    `| ---: | ${VERSIONS.map(() => "---:").join(" | ")} |`,
   );
-  for (const row of rows) {
-    lines.push(
-      `| ${row.version} | ${row.phase} | ${row.distribution} | ${row.iteration} | ${formatSeconds(row.setupSeconds)} | ${formatSeconds(row.buildSeconds)} | ${formatSeconds(row.postCacheSeconds)} | ${formatSeconds(row.jobSeconds)} | ${row.conclusion} |`
-    );
+  for (const runner of analysis.runners) {
+    const cells = VERSIONS.map((entry) => {
+      const slots = runner.slots.get(entry.arm) ?? [];
+      if (slots.length === 0) return "n/a";
+      return ((slots[0] + slots[1]) / 2).toFixed(3);
+    });
+    lines.push(`| ${runner.sample} | ${cells.join(" | ")} |`);
   }
   lines.push(
-    '',
-    '## Cache entries',
-    '',
-    '| Version | Distribution | Iteration | Cache | Size (MiB) |',
-    '| --- | --- | ---: | --- | ---: |'
+    "",
+    "## Caches",
+    "",
+    "Every caching version restores the same Maven entry, so the stored blob cannot bias the comparison. v3 predates `cache-dependency-path` and keys on `pom.xml`, so it necessarily uses its own entry; treat its difference with more caution than the rest.",
+    "",
+    "| Cache | Size (MiB) |",
+    "| --- | ---: |",
   );
   for (const cache of caches) {
     lines.push(
-      `| ${cache.version} | ${cache.distribution} | ${cache.iteration} | ${cache.type} | ${formatMiB(cache.sizeBytes)} |`
+      `| \`${cache.key}\` | ${(cache.sizeBytes / 1024 / 1024).toFixed(1)} |`,
     );
   }
-  return `${lines.join('\n')}\n`;
+  return `${lines.join("\n")}\n`;
 }
 
 export async function main(env = process.env) {
-  const [owner, repo] = env.GITHUB_REPOSITORY.split('/');
+  const [owner, repo] = env.GITHUB_REPOSITORY.split("/");
   const token = env.GH_TOKEN;
   const runId = env.GITHUB_RUN_ID;
-  const attempt = env.GITHUB_RUN_ATTEMPT;
-  const petclinicRef = env.PETCLINIC_REF;
-  if (!owner || !repo || !token || !runId || !attempt || !petclinicRef) {
-    throw new Error('Missing required GitHub Actions environment variables');
+  if (!owner || !repo || !token || !runId) {
+    throw new Error("Missing required GitHub Actions environment variables");
   }
 
-  const [
-    jobs,
-    cacheEntries,
-    wrapperResponse,
-    pomResponse,
-    mainCommit,
-    v1Ref,
-    v2Ref,
-    v3Ref,
-    v4Ref,
-    v52Ref,
-    v56Ref
-  ] = await Promise.all([
-      allPages(
-        `/repos/${owner}/${repo}/actions/runs/${runId}/attempts/${attempt}/jobs`,
-        'jobs',
-        token
-      ),
-      allPages(`/repos/${owner}/${repo}/actions/caches`, 'actions_caches', token),
-      api(
-        `/repos/spring-projects/spring-petclinic/contents/.mvn/wrapper/maven-wrapper.properties?ref=${petclinicRef}`,
-        token
-      ),
-      api(
-        `/repos/spring-projects/spring-petclinic/contents/pom.xml?ref=${petclinicRef}`,
-        token
-      ),
-      api('/repos/actions/setup-java/commits/main', token),
-      api('/repos/actions/setup-java/git/ref/tags/v1.4.4', token),
-      api('/repos/actions/setup-java/git/ref/tags/v2.5.1', token),
-      api('/repos/actions/setup-java/git/ref/tags/v3.14.1', token),
-      api('/repos/actions/setup-java/git/ref/tags/v4.8.0', token),
-      api('/repos/actions/setup-java/git/ref/tags/v5.2.0', token),
-      api('/repos/actions/setup-java/git/ref/tags/v5.6.0', token)
-    ]);
-
-  const rows = jobs
-    .map(job => {
-      const identity = parseBenchmarkJob(job.name);
-      if (!identity) return null;
-      return {
-        ...identity,
-        conclusion: job.conclusion,
-        setupSeconds: stepDuration(job, step =>
-          step.name.startsWith('Setup Java')
-        ),
-        buildSeconds: stepDuration(job, step => step.name === 'Build PetClinic'),
-        postCacheSeconds: stepDuration(job, step =>
-          step.name.startsWith('Post Setup Java')
-        ),
-        jobSeconds: secondsBetween(job.started_at, job.completed_at)
-      };
-    })
-    .filter(Boolean)
-    .sort(
-      (a, b) =>
-        a.version.localeCompare(b.version) ||
-        a.phase.localeCompare(b.phase) ||
-        a.distribution.localeCompare(b.distribution) ||
-        a.iteration - b.iteration
-    );
-
-  const wrapperOriginal = Buffer.from(wrapperResponse.content, 'base64').toString(
-    'utf8'
+  const rows = parseSamples(await readSampleFiles("version-sweep"));
+  const analysis = analyzeAgainstReference(
+    rows,
+    VERSIONS.map((entry) => entry.arm),
+    REFERENCE,
   );
-  const pomOriginal = Buffer.from(pomResponse.content, 'base64').toString('utf8');
-  const coldCases = rows.filter(row => row.phase === 'cold');
-  const caches = [];
-  for (const row of coldCases) {
-    const versionId = VERSION_IDS.get(row.version);
-    const benchmarkId = `${versionId}-${row.distribution}-${row.iteration}-${runId}`;
-    const expected =
-      row.version === 'v1' || row.version === 'v2'
-        ? []
-        : row.version === 'v3'
-        ? [
-            {
-              type: 'maven-dependencies',
-              key: `setup-java-Linux-maven-${hashFilesSingle(
-                `${pomOriginal}<!-- benchmark-id=${benchmarkId} -->\n`
-              )}`
-            }
-          ]
-        : [
-            {
-              type: 'maven-dependencies',
-              key: `setup-java-Linux-x64-maven-${hashFilesSingle(
-                `${benchmarkId}\n`
-              )}`
-            }
-          ];
-    if (row.version === 'v5.6' || row.version === 'main') {
-      expected.push({
-        type: 'maven-wrapper',
-        key: `setup-java-Linux-x64-maven-wrapper-${hashFilesSingle(
-          `${wrapperOriginal}# benchmark-id=${benchmarkId}\n`
-        )}`
-      });
-    }
-    for (const item of expected) {
-      const entry = cacheEntries.find(cache => cache.key === item.key);
-      caches.push({
-        version: row.version,
-        distribution: row.distribution,
-        iteration: row.iteration,
-        type: item.type,
-        key: item.key,
-        id: entry?.id ?? null,
-        sizeBytes: entry?.size_in_bytes ?? null
-      });
-    }
+  if (analysis.runners.length === 0) {
+    throw new Error("No runner measured every version twice");
   }
 
-  const missingCaches = caches.filter(cache => cache.id === null);
-  if (missingCaches.length > 0) {
-    console.warn(`Could not find ${missingCaches.length} expected cache entries`);
-  }
+  const cacheEntries = await allPages(
+    `/repos/${owner}/${repo}/actions/caches`,
+    "actions_caches",
+    token,
+  );
+  const caches = cacheEntries
+    .filter((entry) => entry.key.startsWith("setup-java-"))
+    .map((entry) => ({
+      id: entry.id,
+      key: entry.key,
+      sizeBytes: entry.size_in_bytes,
+    }));
 
   const metadata = {
     repository: env.GITHUB_REPOSITORY,
     runId,
-    runAttempt: Number(attempt),
+    distribution: env.DISTRIBUTION,
     javaVersion: env.JAVA_VERSION,
-    iterations: Number(env.ITERATIONS),
-    petclinicRef,
-    setupJavaV1Ref: v1Ref.object.sha,
-    setupJavaV2Ref: v2Ref.object.sha,
-    setupJavaV3Ref: v3Ref.object.sha,
-    setupJavaV4Ref: v4Ref.object.sha,
-    setupJavaV52Ref: v52Ref.object.sha,
-    setupJavaV56Ref: v56Ref.object.sha,
-    setupJavaMainRefAtReport: mainCommit.sha,
-    generatedAt: new Date().toISOString()
+    generatedAt: new Date().toISOString(),
   };
-  const summaries = summarize(rows, caches);
-  const report = markdown(metadata, rows, caches, summaries);
 
-  await mkdir('results', {recursive: true});
+  const report = markdown(metadata, analysis, caches);
+  await mkdir(RESULTS_DIR, { recursive: true });
   await writeFile(
-    'results/results.json',
-    `${JSON.stringify({metadata, summaries, rows, caches}, null, 2)}\n`
-  );
-  const csvHeaders = [
-    'version',
-    'phase',
-    'distribution',
-    'iteration',
-    'setup_seconds',
-    'build_seconds',
-    'post_cache_seconds',
-    'job_seconds',
-    'conclusion'
-  ];
-  const csvRows = rows.map(row =>
-    [
-      row.version,
-      row.phase,
-      row.distribution,
-      row.iteration,
-      row.setupSeconds,
-      row.buildSeconds,
-      row.postCacheSeconds,
-      row.jobSeconds,
-      row.conclusion
-    ]
-      .map(csvValue)
-      .join(',')
+    `${RESULTS_DIR}/results.json`,
+    `${JSON.stringify({ metadata, analysis: { ...analysis, runners: undefined }, caches }, null, 2)}\n`,
   );
   await writeFile(
-    'results/results.csv',
-    `${csvHeaders.join(',')}\n${csvRows.join('\n')}\n`
+    `${RESULTS_DIR}/results.csv`,
+    `sample,version,slot,seconds\n${rows
+      .map((row) =>
+        [row.sample, row.arm, row.slot, row.seconds].map(csvValue).join(","),
+      )
+      .join("\n")}\n`,
   );
-  await writeFile('results/summary.md', report);
+  await writeFile(`${RESULTS_DIR}/summary.md`, report);
   await appendFile(env.GITHUB_STEP_SUMMARY, report);
 
-  if (env.CLEANUP_CACHES === 'true') {
-    const ids = [...new Set(caches.map(cache => cache.id).filter(Boolean))];
-    for (const id of ids) {
-      await api(`/repos/${owner}/${repo}/actions/caches/${id}`, token, {
-        method: 'DELETE'
+  if (env.CLEANUP_CACHES === "true") {
+    for (const cache of caches) {
+      await api(`/repos/${owner}/${repo}/actions/caches/${cache.id}`, token, {
+        method: "DELETE",
       });
     }
-    console.log(`Deleted ${ids.length} benchmark cache entries after reporting`);
+    console.log(`Deleted ${caches.length} benchmark caches`);
   }
 }
 
-if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+if (
+  process.argv[1] &&
+  import.meta.url === pathToFileURL(process.argv[1]).href
+) {
   await main();
 }
diff --git a/scripts/report.test.mjs b/scripts/report.test.mjs
index 458b545..e8a56c2 100644
--- a/scripts/report.test.mjs
+++ b/scripts/report.test.mjs
@@ -1,13 +1,23 @@
 import assert from "node:assert/strict";
 import test from "node:test";
 
-import {
-  hashFilesSingle,
-  median,
-  parseBenchmarkJob,
-  secondsBetween,
-  sha256,
-} from "./report.mjs";
+import { analyzeAgainstReference, parseSamples } from "./paired.mjs";
+import { VERSIONS, hashFilesSingle, markdown, sha256 } from "./report.mjs";
+
+const ARMS = VERSIONS.map((entry) => entry.arm);
+const ORDER = [...ARMS, ...[...ARMS].reverse()];
+
+// One runner's worth of the mirrored sweep. `speed` scales every slot so that a
+// slow runner stays slow across all versions, which is what the pairing removes.
+function sweep(sample, speed, perVersionMs, drift = 0) {
+  return ORDER.map((arm, index) => {
+    const slot = index + 1;
+    const elapsed = perVersionMs[arm] * speed + drift * index;
+    return `"${sample}","${arm}","${slot}","${Math.round(elapsed)}"`;
+  }).join("\n");
+}
+
+const flat = Object.fromEntries(ARMS.map((arm) => [arm, 3000]));
 
 test("hashes benchmark cache markers", () => {
   assert.equal(
@@ -20,55 +30,76 @@ test("hashes benchmark cache markers", () => {
   );
 });
 
-test("calculates medians", () => {
-  assert.equal(median([3, 1, 2]), 2);
-  assert.equal(median([4, 1, 2, 3]), 2.5);
-  assert.equal(median([null]), null);
+test("keeps only runners that measured every version twice", () => {
+  const csv = [
+    sweep(1, 1, flat),
+    // A runner that stopped after the first few slots must be discarded rather
+    // than contribute a difference computed from unbalanced slots.
+    '"2","v1","1","3000"',
+    '"2","v2","2","3000"',
+  ].join("\n");
+  const analysis = analyzeAgainstReference(parseSamples(csv), ARMS, "main");
+  assert.deepEqual(
+    analysis.runners.map((runner) => runner.sample),
+    [1],
+  );
 });
 
-test("calculates step duration", () => {
-  assert.equal(
-    secondsBetween("2026-01-01T00:00:01Z", "2026-01-01T00:00:04.500Z"),
-    3.5,
-  );
+test("removes between-runner speed differences", () => {
+  // Runners differ by up to 3x, and every version is 500ms slower than main.
+  // Pairing within a runner must recover 500ms regardless of the spread.
+  const perVersion = {
+    ...flat,
+    v1: 3500,
+    v2: 3500,
+    v3: 3500,
+    v4: 3500,
+    v52: 3500,
+    v56: 3500,
+  };
+  const csv = [1, 2, 3, 4, 5, 6]
+    .map((sample) => sweep(sample, 0.5 + sample * 0.5, perVersion))
+    .join("\n");
+  const analysis = analyzeAgainstReference(parseSamples(csv), ARMS, "main");
+  const v4 = analysis.comparisons.find((entry) => entry.arm === "v4");
+  assert.ok(Math.abs(v4.differenceSeconds - 1.125) < 0.001);
+  assert.equal(v4.verdict, "regression");
 });
 
-test("parses benchmark job names", () => {
-  assert.deepEqual(parseBenchmarkJob("v1 / cold / zulu / 1"), {
-    version: "v1",
-    phase: "cold",
-    distribution: "zulu",
-    iteration: 1,
-  });
-  assert.deepEqual(parseBenchmarkJob("v2 / warm / microsoft / 3"), {
-    version: "v2",
-    phase: "warm",
-    distribution: "microsoft",
-    iteration: 3,
-  });
-  assert.deepEqual(parseBenchmarkJob("v3 / cold / temurin / 5"), {
-    version: "v3",
-    phase: "cold",
-    distribution: "temurin",
-    iteration: 5,
-  });
-  assert.deepEqual(parseBenchmarkJob("v4 / warm / microsoft / 2"), {
-    version: "v4",
-    phase: "warm",
-    distribution: "microsoft",
-    iteration: 2,
-  });
-  assert.deepEqual(parseBenchmarkJob("v5.2 / warm / microsoft / 1"), {
-    version: "v5.2",
-    phase: "warm",
-    distribution: "microsoft",
-    iteration: 1,
-  });
-  assert.deepEqual(parseBenchmarkJob("v5.6 / cold / temurin / 3"), {
-    version: "v5.6",
-    phase: "cold",
-    distribution: "temurin",
-    iteration: 3,
-  });
-  assert.equal(parseBenchmarkJob("Report"), null);
+test("cancels linear drift across the job", () => {
+  // Every version is identical, but the job gets steadily slower. The mirrored
+  // order must absorb that so no version is reported as different from main.
+  const csv = [1, 2, 3, 4, 5, 6]
+    .map((sample) => sweep(sample, 1, flat, 40))
+    .join("\n");
+  const analysis = analyzeAgainstReference(parseSamples(csv), ARMS, "main");
+  for (const comparison of analysis.comparisons) {
+    assert.ok(Math.abs(comparison.differenceSeconds) < 1e-9);
+  }
+});
+
+test("reports the reference against itself without a verdict", () => {
+  const csv = [1, 2, 3, 4].map((sample) => sweep(sample, 1, flat)).join("\n");
+  const analysis = analyzeAgainstReference(parseSamples(csv), ARMS, "main");
+  const reference = analysis.comparisons.find((entry) => entry.arm === "main");
+  assert.equal(reference.verdict, "reference");
+  assert.equal(reference.interval, null);
+});
+
+test("renders a version table, a control and per-runner medians", () => {
+  const csv = [1, 2, 3, 4]
+    .map((sample) => sweep(sample, 1 + sample * 0.1, flat))
+    .join("\n");
+  const analysis = analyzeAgainstReference(parseSamples(csv), ARMS, "main");
+  const report = markdown(
+    { runId: "1", distribution: "temurin", javaVersion: "17" },
+    analysis,
+    [{ key: "setup-java-Linux-x64-maven-abc", sizeBytes: 60 * 1024 * 1024 }],
+  );
+  assert.match(report, /# setup-java version sweep/);
+  assert.match(report, /\| v4\.8\.0 \|/);
+  assert.match(report, /A\/A control/);
+  assert.match(report, /Harness noise floor/);
+  assert.match(report, /## Per-runner medians/);
+  assert.match(report, /setup-java-Linux-x64-maven-abc/);
 });
diff --git a/scripts/version-sweep.sh b/scripts/version-sweep.sh
new file mode 100755
index 0000000..0c09de1
--- /dev/null
+++ b/scripts/version-sweep.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+# Cache-identity helpers for the setup-java version sweep.
+#
+# Markers are appended idempotently so that repeating a slot does not change the
+# hashed files. The warm sweep depends on that: every slot must derive the same
+# cache key, otherwise it would miss and measure a download instead of a restore.
+
+set -euo pipefail
+
+command=${1:?command is required}
+benchmark_id=${2:?benchmark id is required}
+
+wrapper_properties=.mvn/wrapper/maven-wrapper.properties
+
+append_once() {
+  local file=$1 line=$2
+  if [ ! -f "$file" ] || ! grep -qF "$line" "$file"; then
+    printf '%s\n' "$line" >> "$file"
+  fi
+}
+
+write_identity() {
+  # Versions from v4 onwards select their cache key with cache-dependency-path,
+  # so a single file drives all of them.
+  printf '%s\n' "$benchmark_id" > .benchmark-cache-key
+  # v5.6 and main additionally maintain a wrapper cache keyed on the wrapper
+  # properties. The real contents must survive because the seed job builds.
+  append_once "$wrapper_properties" "# benchmark-id=$benchmark_id"
+  # v3 predates cache-dependency-path and hashes pom.xml. A comment in the
+  # epilog is well-formed XML and leaves the build unaffected.
+  append_once pom.xml "<!-- benchmark-id=$benchmark_id -->"
+}
+
+case "$command" in
+  prepare)
+    write_identity
+    ;;
+  reset)
+    # Every measured slot must restore into an empty tree, otherwise later slots
+    # would extract over files an earlier slot left behind and report an
+    # artificially low duration.
+    rm -rf "$HOME/.m2"
+    write_identity
+    ;;
+  *)
+    echo "Unsupported command: $command" >&2
+    exit 1
+    ;;
+esac

From 319c61b18e26e034e5203ba5870a0324289945c0 Mon Sep 17 00:00:00 2001
From: Bruno Borges <brborges@microsoft.com>
Date: Wed, 5 Aug 2026 00:36:23 -0400
Subject: [PATCH 06/19] Keep PetClinic's build from linting the benchmark
 scripts

The version sweep and the JDK cache benchmark both checked PetClinic out over
the workspace root. The JDK cache workflow then had no copy of the benchmark
scripts to invoke, and the version sweep, which checked them out to .benchmark,
put them inside PetClinic's basedir, where its nohttp check failed the seed
build on http:// URLs in a toolchains template.

Check the benchmark repository out at the root and PetClinic below it instead.
The scripts are then addressable as scripts/, PetClinic's build only sees its
own tree, and the cache-identity markers move with it.

Also stop jdk-cache.sh from truncating the wrapper properties; the seed job
needs the real contents to build, and an idempotent append keeps the hashed
file identical between seeding and measuring.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 71c45320-1417-4029-8402-69075d61dac1
---
 .github/workflows/benchmark.yml | 113 +++++++++++++++++---------------
 .github/workflows/jdk-cache.yml |  17 +++++
 scripts/jdk-cache.sh            |  24 +++++--
 scripts/version-sweep.sh        |  10 ++-
 4 files changed, 101 insertions(+), 63 deletions(-)

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 7c3e6d2..13df408 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -58,19 +58,22 @@ jobs:
     name: Seed cache
     runs-on: ubuntu-24.04
     steps:
-      - name: Check out Spring PetClinic
+      - name: Check out benchmark repository
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
         with:
-          repository: spring-projects/spring-petclinic
-          ref: ${{ env.PETCLINIC_REF }}
           persist-credentials: false
-      - name: Check out benchmark scripts
+      # PetClinic goes below the benchmark repository rather than over it: its
+      # build runs a nohttp check across the whole basedir and would otherwise
+      # lint the benchmark scripts.
+      - name: Check out Spring PetClinic
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
         with:
-          path: .benchmark
+          repository: spring-projects/spring-petclinic
+          ref: ${{ env.PETCLINIC_REF }}
+          path: petclinic
           persist-credentials: false
       - name: Prepare cache identity
-        run: bash .benchmark/scripts/version-sweep.sh prepare "benchmark-${{ github.run_id }}"
+        run: bash scripts/version-sweep.sh prepare "benchmark-${{ github.run_id }}"
       # Each version registers a post-job save against the same local repository.
       # The first save wins and the rest are skipped as exact key hits.
 
@@ -118,6 +121,7 @@ jobs:
           cache: maven
           cache-dependency-path: .benchmark-cache-key
       - name: Build PetClinic
+        working-directory: petclinic
         run: ./mvnw --batch-mode --no-transfer-progress compile
 
   # Every version is measured in the same job, in the order v1..main followed by
@@ -138,16 +142,19 @@ jobs:
       matrix:
         sample: ${{ fromJSON(inputs.samples == '20' && '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]' || inputs.samples == '10' && '[1,2,3,4,5,6,7,8,9,10]' || inputs.samples == '6' && '[1,2,3,4,5,6]' || '[1,2]') }}
     steps:
-      - name: Check out Spring PetClinic
+      - name: Check out benchmark repository
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
         with:
-          repository: spring-projects/spring-petclinic
-          ref: ${{ env.PETCLINIC_REF }}
           persist-credentials: false
-      - name: Check out benchmark scripts
+      # PetClinic goes below the benchmark repository rather than over it: its
+      # build runs a nohttp check across the whole basedir and would otherwise
+      # lint the benchmark scripts.
+      - name: Check out Spring PetClinic
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
         with:
-          path: .benchmark
+          repository: spring-projects/spring-petclinic
+          ref: ${{ env.PETCLINIC_REF }}
+          path: petclinic
           persist-credentials: false
 
       # cache-read-only exists only on main, so it is not used here: applying it
@@ -156,30 +163,30 @@ jobs:
       # actions/cache skips saving in that case, so no measured slot creates an
       # entry regardless of the version.
       - name: Reset slot 1
-        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+        run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
       - name: Start slot 1 timer
-        run: node .benchmark/scripts/measure.mjs start
+        run: node scripts/measure.mjs start
       - name: Slot 1 setup (v1)
         uses: actions/setup-java@v1.4.4
         with:
           java-version: ${{ inputs.java-version }}
       - name: Record slot 1
-        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v1 1
+        run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v1 1
       - name: Reset slot 2
-        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+        run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
       - name: Start slot 2 timer
-        run: node .benchmark/scripts/measure.mjs start
+        run: node scripts/measure.mjs start
       - name: Slot 2 setup (v2)
         uses: actions/setup-java@v2.5.1
         with:
           distribution: ${{ inputs.distribution }}
           java-version: ${{ inputs.java-version }}
       - name: Record slot 2
-        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v2 2
+        run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v2 2
       - name: Reset slot 3
-        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+        run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
       - name: Start slot 3 timer
-        run: node .benchmark/scripts/measure.mjs start
+        run: node scripts/measure.mjs start
       - name: Slot 3 setup (v3)
         uses: actions/setup-java@v3.14.1
         with:
@@ -187,11 +194,11 @@ jobs:
           java-version: ${{ inputs.java-version }}
           cache: maven
       - name: Record slot 3
-        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v3 3
+        run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v3 3
       - name: Reset slot 4
-        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+        run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
       - name: Start slot 4 timer
-        run: node .benchmark/scripts/measure.mjs start
+        run: node scripts/measure.mjs start
       - name: Slot 4 setup (v4)
         uses: actions/setup-java@v4.8.0
         with:
@@ -200,11 +207,11 @@ jobs:
           cache: maven
           cache-dependency-path: .benchmark-cache-key
       - name: Record slot 4
-        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v4 4
+        run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v4 4
       - name: Reset slot 5
-        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+        run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
       - name: Start slot 5 timer
-        run: node .benchmark/scripts/measure.mjs start
+        run: node scripts/measure.mjs start
       - name: Slot 5 setup (v5.2)
         uses: actions/setup-java@v5.2.0
         with:
@@ -213,11 +220,11 @@ jobs:
           cache: maven
           cache-dependency-path: .benchmark-cache-key
       - name: Record slot 5
-        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v52 5
+        run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v52 5
       - name: Reset slot 6
-        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+        run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
       - name: Start slot 6 timer
-        run: node .benchmark/scripts/measure.mjs start
+        run: node scripts/measure.mjs start
       - name: Slot 6 setup (v5.6)
         uses: actions/setup-java@v5.6.0
         with:
@@ -226,11 +233,11 @@ jobs:
           cache: maven
           cache-dependency-path: .benchmark-cache-key
       - name: Record slot 6
-        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v56 6
+        run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v56 6
       - name: Reset slot 7
-        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+        run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
       - name: Start slot 7 timer
-        run: node .benchmark/scripts/measure.mjs start
+        run: node scripts/measure.mjs start
       - name: Slot 7 setup (main)
         uses: actions/setup-java@main
         with:
@@ -239,11 +246,11 @@ jobs:
           cache: maven
           cache-dependency-path: .benchmark-cache-key
       - name: Record slot 7
-        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" main 7
+        run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" main 7
       - name: Reset slot 8
-        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+        run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
       - name: Start slot 8 timer
-        run: node .benchmark/scripts/measure.mjs start
+        run: node scripts/measure.mjs start
       - name: Slot 8 setup (main)
         uses: actions/setup-java@main
         with:
@@ -252,11 +259,11 @@ jobs:
           cache: maven
           cache-dependency-path: .benchmark-cache-key
       - name: Record slot 8
-        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" main 8
+        run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" main 8
       - name: Reset slot 9
-        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+        run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
       - name: Start slot 9 timer
-        run: node .benchmark/scripts/measure.mjs start
+        run: node scripts/measure.mjs start
       - name: Slot 9 setup (v5.6)
         uses: actions/setup-java@v5.6.0
         with:
@@ -265,11 +272,11 @@ jobs:
           cache: maven
           cache-dependency-path: .benchmark-cache-key
       - name: Record slot 9
-        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v56 9
+        run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v56 9
       - name: Reset slot 10
-        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+        run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
       - name: Start slot 10 timer
-        run: node .benchmark/scripts/measure.mjs start
+        run: node scripts/measure.mjs start
       - name: Slot 10 setup (v5.2)
         uses: actions/setup-java@v5.2.0
         with:
@@ -278,11 +285,11 @@ jobs:
           cache: maven
           cache-dependency-path: .benchmark-cache-key
       - name: Record slot 10
-        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v52 10
+        run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v52 10
       - name: Reset slot 11
-        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+        run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
       - name: Start slot 11 timer
-        run: node .benchmark/scripts/measure.mjs start
+        run: node scripts/measure.mjs start
       - name: Slot 11 setup (v4)
         uses: actions/setup-java@v4.8.0
         with:
@@ -291,11 +298,11 @@ jobs:
           cache: maven
           cache-dependency-path: .benchmark-cache-key
       - name: Record slot 11
-        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v4 11
+        run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v4 11
       - name: Reset slot 12
-        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+        run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
       - name: Start slot 12 timer
-        run: node .benchmark/scripts/measure.mjs start
+        run: node scripts/measure.mjs start
       - name: Slot 12 setup (v3)
         uses: actions/setup-java@v3.14.1
         with:
@@ -303,28 +310,28 @@ jobs:
           java-version: ${{ inputs.java-version }}
           cache: maven
       - name: Record slot 12
-        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v3 12
+        run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v3 12
       - name: Reset slot 13
-        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+        run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
       - name: Start slot 13 timer
-        run: node .benchmark/scripts/measure.mjs start
+        run: node scripts/measure.mjs start
       - name: Slot 13 setup (v2)
         uses: actions/setup-java@v2.5.1
         with:
           distribution: ${{ inputs.distribution }}
           java-version: ${{ inputs.java-version }}
       - name: Record slot 13
-        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v2 13
+        run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v2 13
       - name: Reset slot 14
-        run: bash .benchmark/scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+        run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
       - name: Start slot 14 timer
-        run: node .benchmark/scripts/measure.mjs start
+        run: node scripts/measure.mjs start
       - name: Slot 14 setup (v1)
         uses: actions/setup-java@v1.4.4
         with:
           java-version: ${{ inputs.java-version }}
       - name: Record slot 14
-        run: node .benchmark/scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v1 14
+        run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v1 14
 
       - name: Upload sample timings
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
diff --git a/.github/workflows/jdk-cache.yml b/.github/workflows/jdk-cache.yml
index baa0bdb..8e3de25 100644
--- a/.github/workflows/jdk-cache.yml
+++ b/.github/workflows/jdk-cache.yml
@@ -91,11 +91,19 @@ jobs:
     needs: prepare
     runs-on: ubuntu-24.04
     steps:
+      - name: Check out benchmark repository
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          persist-credentials: false
+      # PetClinic goes below the benchmark repository rather than over it: its
+      # build runs a nohttp check across the whole basedir and would otherwise
+      # lint the benchmark scripts.
       - name: Check out Spring PetClinic
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
         with:
           repository: spring-projects/spring-petclinic
           ref: ${{ env.PETCLINIC_REF }}
+          path: petclinic
           persist-credentials: false
       - name: Check out setup-java
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
@@ -119,6 +127,7 @@ jobs:
           cache-jdk: true
           cache-dependency-path: .jdk-cache-key
       - name: Build PetClinic
+        working-directory: petclinic
         run: ./mvnw --batch-mode --no-transfer-progress compile
 
   # Both arms run in the same job in ABBA order, so each runner yields one paired
@@ -138,11 +147,19 @@ jobs:
       matrix:
         sample: ${{ fromJSON(inputs.samples == '20' && '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]' || inputs.samples == '10' && '[1,2,3,4,5,6,7,8,9,10]' || inputs.samples == '6' && '[1,2,3,4,5,6]' || '[1,2]') }}
     steps:
+      - name: Check out benchmark repository
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          persist-credentials: false
+      # PetClinic goes below the benchmark repository rather than over it: its
+      # build runs a nohttp check across the whole basedir and would otherwise
+      # lint the benchmark scripts.
       - name: Check out Spring PetClinic
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
         with:
           repository: spring-projects/spring-petclinic
           ref: ${{ env.PETCLINIC_REF }}
+          path: petclinic
           persist-credentials: false
       - name: Check out setup-java
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
diff --git a/scripts/jdk-cache.sh b/scripts/jdk-cache.sh
index 93fce2d..d94e303 100755
--- a/scripts/jdk-cache.sh
+++ b/scripts/jdk-cache.sh
@@ -6,13 +6,11 @@ set -euo pipefail
 
 command=${1:?command is required}
 
-write_identity() {
-  local benchmark_id=$1
-  printf '%s\n' "$benchmark_id" > .jdk-cache-key
-  mkdir -p .mvn/wrapper
-  printf '# jdk-cache-benchmark=%s\n' "$benchmark_id" \
-    > .mvn/wrapper/maven-wrapper.properties
-}
+# Spring PetClinic is checked out below the benchmark repository rather than over
+# it, because its build runs a nohttp check across the whole basedir and would
+# otherwise lint these scripts.
+project_dir=${PROJECT_DIR:-petclinic}
+wrapper_properties="$project_dir/.mvn/wrapper/maven-wrapper.properties"
 
 purge_toolcache() {
   # The benchmark deliberately measures the path where the requested JDK is not
@@ -21,6 +19,18 @@ purge_toolcache() {
   rm -rf "${RUNNER_TOOL_CACHE:?}"/Java_*
 }
 
+write_identity() {
+  local benchmark_id=$1
+  printf '%s\n' "$benchmark_id" > .jdk-cache-key
+  # Appended, not overwritten: the real wrapper properties must survive because
+  # the seed job builds. Appending only once keeps the hashed contents identical
+  # between the seed job and every measured slot.
+  local marker="# jdk-cache-benchmark=$benchmark_id"
+  if [ ! -f "$wrapper_properties" ] || ! grep -qF -- "$marker" "$wrapper_properties"; then
+    printf '%s\n' "$marker" >> "$wrapper_properties"
+  fi
+}
+
 case "$command" in
   prepare)
     write_identity "${2:?benchmark id is required}"
diff --git a/scripts/version-sweep.sh b/scripts/version-sweep.sh
index 0c09de1..b3ce813 100755
--- a/scripts/version-sweep.sh
+++ b/scripts/version-sweep.sh
@@ -11,11 +11,15 @@ set -euo pipefail
 command=${1:?command is required}
 benchmark_id=${2:?benchmark id is required}
 
-wrapper_properties=.mvn/wrapper/maven-wrapper.properties
+# Spring PetClinic is checked out below the benchmark repository rather than over
+# it, because its build runs a nohttp check across the whole basedir and would
+# otherwise lint these scripts.
+project_dir=${PROJECT_DIR:-petclinic}
+wrapper_properties="$project_dir/.mvn/wrapper/maven-wrapper.properties"
 
 append_once() {
   local file=$1 line=$2
-  if [ ! -f "$file" ] || ! grep -qF "$line" "$file"; then
+  if [ ! -f "$file" ] || ! grep -qF -- "$line" "$file"; then
     printf '%s\n' "$line" >> "$file"
   fi
 }
@@ -29,7 +33,7 @@ write_identity() {
   append_once "$wrapper_properties" "# benchmark-id=$benchmark_id"
   # v3 predates cache-dependency-path and hashes pom.xml. A comment in the
   # epilog is well-formed XML and leaves the build unaffected.
-  append_once pom.xml "<!-- benchmark-id=$benchmark_id -->"
+  append_once "$project_dir/pom.xml" "<!-- benchmark-id=$benchmark_id -->"
 }
 
 case "$command" in

From a4fc67a120fb00a843cc3ec843c53f7ce33daac9 Mon Sep 17 00:00:00 2001
From: Bruno Borges <bruno.borges@microsoft.com>
Date: Wed, 5 Aug 2026 00:48:07 -0400
Subject: [PATCH 07/19] Stop one stalled runner from holding a benchmark run
 open

A measurement job is a couple of minutes of work, but a slot can stall
indefinitely on a cache download or a JDK fetch. One such runner in the version
sweep sat on a step that takes two seconds elsewhere, and with the default
six-hour job timeout it would have blocked the report for the rest of the day.

Bound the measurement jobs at 25 minutes. A stalled runner now fails quickly and
drops out of the report, which already discards runners that did not complete
every slot, and the remaining runners still produce a verdict.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 71c45320-1417-4029-8402-69075d61dac1
---
 .github/workflows/benchmark.yml             | 6 ++++++
 .github/workflows/focused-cache-restore.yml | 6 ++++++
 .github/workflows/jdk-cache.yml             | 6 ++++++
 3 files changed, 18 insertions(+)

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 13df408..8d86b49 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -132,6 +132,12 @@ jobs:
     name: warm / ${{ matrix.sample }}
     needs: seed
     runs-on: ubuntu-24.04
+    # A measurement job is a couple of minutes of work. Anything far beyond that
+    # is a stalled cache download or JDK fetch rather than a slow measurement,
+    # and the run should not wait six hours for the default timeout to notice.
+    # The runner drops out of the report, which discards runners that did not
+    # complete every slot, and the remaining runners still produce a verdict.
+    timeout-minutes: 25
     strategy:
       fail-fast: false
       # Every runner pulls the same seeded blob, so running the whole matrix at
diff --git a/.github/workflows/focused-cache-restore.yml b/.github/workflows/focused-cache-restore.yml
index 80a8046..101494b 100644
--- a/.github/workflows/focused-cache-restore.yml
+++ b/.github/workflows/focused-cache-restore.yml
@@ -111,6 +111,12 @@ jobs:
     name: paired / ${{ matrix.sample }}
     needs: seed
     runs-on: ubuntu-24.04
+    # A measurement job is a couple of minutes of work. Anything far beyond that
+    # is a stalled cache download or JDK fetch rather than a slow measurement,
+    # and the run should not wait six hours for the default timeout to notice.
+    # The runner drops out of the report, which discards runners that did not
+    # complete every slot, and the remaining runners still produce a verdict.
+    timeout-minutes: 25
     strategy:
       fail-fast: false
       # Every runner pulls the same seeded blob, so running the whole matrix at
diff --git a/.github/workflows/jdk-cache.yml b/.github/workflows/jdk-cache.yml
index 8e3de25..44e5faf 100644
--- a/.github/workflows/jdk-cache.yml
+++ b/.github/workflows/jdk-cache.yml
@@ -137,6 +137,12 @@ jobs:
     name: paired / ${{ matrix.sample }}
     needs: seed
     runs-on: ubuntu-24.04
+    # A measurement job is a couple of minutes of work. Anything far beyond that
+    # is a stalled cache download or JDK fetch rather than a slow measurement,
+    # and the run should not wait six hours for the default timeout to notice.
+    # The runner drops out of the report, which discards runners that did not
+    # complete every slot, and the remaining runners still produce a verdict.
+    timeout-minutes: 25
     strategy:
       fail-fast: false
       # Every runner pulls the same seeded blobs, so running the whole matrix at

From 8cd13cf526eebabfe3402b450ae8c4f31db66f94 Mon Sep 17 00:00:00 2001
From: Bruno Borges <bruno.borges@microsoft.com>
Date: Wed, 5 Aug 2026 00:50:00 -0400
Subject: [PATCH 08/19] Discard the first slot of every measurement job

The JDK cache benchmark's A/A control failed: comparing the no-cache arm's own
first and last slot reported an improvement of 0.4s, 95% CI -0.7 to -0.1, on
slots that ran identical configuration. The headline verdict could not be
trusted while that was true.

The cause is that the first setup in a job pays costs the later ones do not:
DNS resolution, TLS handshakes to the cache service and to the JDK host, and a
cold page cache. That is a one-off spike at slot 1 rather than drift across the
job, so the mirrored slot order cannot cancel it, and it biases the arm that
happens to hold the first slot.

Run one unmeasured warm-up slot in every measurement job so the measured slots
all start from the same warmed state. Applied to all three workflows rather than
only the one where it was caught, since they share the structure that exposes it.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 71c45320-1417-4029-8402-69075d61dac1
---
 .github/workflows/benchmark.yml             | 16 ++++++++++++++++
 .github/workflows/focused-cache-restore.yml | 16 ++++++++++++++++
 .github/workflows/jdk-cache.yml             | 18 ++++++++++++++++++
 README.md                                   |  2 ++
 4 files changed, 52 insertions(+)

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 8d86b49..ec47ae8 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -168,6 +168,22 @@ jobs:
       # Every slot restores its cache on an exact primary-key hit, and
       # actions/cache skips saving in that case, so no measured slot creates an
       # entry regardless of the version.
+      # The first setup in a job pays costs the later ones do not: DNS resolution,
+      # TLS handshakes to the cache service and the JDK host, and a cold page
+      # cache. That is a one-off spike rather than drift, so the mirrored slot
+      # order cannot cancel it, and it showed up as the A/A control resolving a
+      # difference between an arm's own first and last slot. This slot pays those
+      # costs and is discarded.
+      - name: Reset warm-up slot
+        run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
+      - name: Warm-up setup (discarded)
+        uses: actions/setup-java@main
+        with:
+          distribution: ${{ inputs.distribution }}
+          java-version: ${{ inputs.java-version }}
+          cache: maven
+          cache-dependency-path: .benchmark-cache-key
+
       - name: Reset slot 1
         run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
       - name: Start slot 1 timer
diff --git a/.github/workflows/focused-cache-restore.yml b/.github/workflows/focused-cache-restore.yml
index 101494b..dffef59 100644
--- a/.github/workflows/focused-cache-restore.yml
+++ b/.github/workflows/focused-cache-restore.yml
@@ -148,6 +148,22 @@ jobs:
           persist-credentials: false
           ref: ${{ inputs.candidate-ref }}
 
+      # The first setup in a job pays costs the later ones do not: DNS resolution,
+      # TLS handshakes to the cache service and the JDK host, and a cold page
+      # cache. That is a one-off spike rather than drift, so the mirrored slot
+      # order cannot cancel it, and it showed up as the A/A control resolving a
+      # difference between an arm's own first and last slot. This slot pays those
+      # costs and is discarded.
+      - name: Reset warm-up slot
+        run: bash scripts/focused-cache-restore.sh reset "focused-${{ github.run_id }}"
+      - name: Warm-up setup (discarded)
+        uses: ./baseline
+        with:
+          distribution: temurin
+          java-version: ${{ env.JAVA_VERSION }}
+          cache: maven
+          cache-dependency-path: .focused-cache-key
+
       - name: Reset slot 1
         run: bash scripts/focused-cache-restore.sh reset "focused-${{ github.run_id }}"
       - name: Start slot 1 timer
diff --git a/.github/workflows/jdk-cache.yml b/.github/workflows/jdk-cache.yml
index 44e5faf..afed7cc 100644
--- a/.github/workflows/jdk-cache.yml
+++ b/.github/workflows/jdk-cache.yml
@@ -178,6 +178,24 @@ jobs:
       # Each slot deletes the extracted JDK and the local repository first, so
       # every measured setup starts from the same empty state instead of
       # extracting over files an earlier slot left behind.
+      # The first setup in a job pays costs the later ones do not: DNS resolution,
+      # TLS handshakes to the cache service and the JDK host, and a cold page
+      # cache. That is a one-off spike rather than drift, so the mirrored slot
+      # order cannot cancel it, and it showed up as the A/A control resolving a
+      # difference between an arm's own first and last slot. This slot pays those
+      # costs and is discarded.
+      - name: Reset warm-up slot
+        run: bash scripts/jdk-cache.sh reset "jdk-cache-${{ github.run_id }}"
+      - name: Warm-up setup (discarded)
+        uses: ./.setup-java
+        with:
+          distribution: ${{ inputs.distribution }}
+          java-version: ${{ inputs.java-version }}
+          cache: maven
+          cache-jdk: false
+          cache-read-only: true
+          cache-dependency-path: .jdk-cache-key
+
       - name: Reset slot 1
         run: bash scripts/jdk-cache.sh reset "jdk-cache-${{ github.run_id }}"
       - name: Start slot 1 timer
diff --git a/README.md b/README.md
index f1905d6..7cd217b 100644
--- a/README.md
+++ b/README.md
@@ -15,6 +15,8 @@ Every workflow here measures effects of a few hundred milliseconds to a few seco
 
 **Same-runner pairing.** Between-runner variance cannot be averaged away by adding more independent jobs to each arm. Every arm is measured inside the *same* job, in an order mirrored about the middle of the job: ABBA for two arms, and `v1..main` followed by `main..v1` for the version sweep. Differencing within a runner removes the runner's own speed, and the mirrored order cancels drift that is linear across the job. Each measured slot deletes `~/.m2` first so every restore extracts into an empty tree.
 
+Every job also runs one unmeasured warm-up slot first. The first setup in a job pays costs the later ones do not — DNS resolution, TLS handshakes to the cache service and the JDK host, and a cold page cache — and that is a one-off spike rather than drift, so the mirrored order cannot cancel it. Without the warm-up slot the A/A control resolved a spurious 0.4 s difference between an arm's own first and last slot.
+
 **One cache, every arm.** A cache entry's download throughput depends on where the service placed the stored blob, and that placement is fixed for the life of the entry. Seeding one entry per arm therefore confounds the arm with its blob, and because the bias is identical on every runner, pairing cannot remove it and more samples only tighten the interval around the wrong answer. A single entry is seeded and every arm restores it.
 
 **Intervals, not point estimates.** `scripts/stats.mjs` reports a bootstrap 95% confidence interval, a permutation p-value, and a Hodges-Lehmann shift, and turns them into an explicit verdict. A comparison whose interval includes zero is reported as `inconclusive` rather than as a number that looks like a result.

From efd11fd22d046551a3f1dc426cf74508dc7d08c7 Mon Sep 17 00:00:00 2001
From: Bruno Borges <bruno.borges@microsoft.com>
Date: Wed, 5 Aug 2026 00:55:47 -0400
Subject: [PATCH 09/19] Stop ranking the uncached versions against main

The sweep reported v2.5.1 as a 1.9s improvement over main. That number is real
but it means v2 does no caching and so never restores the Maven repository the
other versions spend most of their time on. Reported as an improvement it invites
exactly the wrong conclusion.

Report v1 and v2 as not comparable and say why. They are still measured, because
seeing what the caching versions spend their time on is the point of including
them.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 71c45320-1417-4029-8402-69075d61dac1
---
 scripts/report.mjs      | 13 ++++++++++++-
 scripts/report.test.mjs |  3 +++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/scripts/report.mjs b/scripts/report.mjs
index 138b75e..3b4d6f6 100644
--- a/scripts/report.mjs
+++ b/scripts/report.mjs
@@ -101,8 +101,17 @@ export function markdown(metadata, analysis, caches) {
       ? `${comparison.interval.low.toFixed(3)} to ${comparison.interval.high.toFixed(3)}`
       : "n/a";
     const p = comparison.pValue === null ? "n/a" : comparison.pValue.toFixed(3);
+    // v1 and v2 do no caching, so they never restore the Maven repository the
+    // other versions spend most of their time on. Their difference is real but
+    // it is a difference in work done, not in how well the same work is done,
+    // and calling it an improvement or a regression would invite the wrong
+    // conclusion.
+    const verdict =
+      entry.caching === "none"
+        ? "not comparable (no caching)"
+        : comparison.verdict;
     lines.push(
-      `| ${LABELS.get(comparison.arm)} | ${entry.caching} | ${summary.medianSeconds.toFixed(3)} | ${summary.meanSeconds.toFixed(3)} | ${summary.madSeconds.toFixed(3)} | ${diff} | ${ci} | ${p} | ${comparison.verdict} |`,
+      `| ${LABELS.get(comparison.arm)} | ${entry.caching} | ${summary.medianSeconds.toFixed(3)} | ${summary.meanSeconds.toFixed(3)} | ${summary.madSeconds.toFixed(3)} | ${diff} | ${ci} | ${p} | ${verdict} |`,
     );
   }
   lines.push(
@@ -130,6 +139,8 @@ export function markdown(metadata, analysis, caches) {
     "",
     "Every caching version restores the same Maven entry, so the stored blob cannot bias the comparison. v3 predates `cache-dependency-path` and keys on `pom.xml`, so it necessarily uses its own entry; treat its difference with more caution than the rest.",
     "",
+    "v1 and v2 have no caching at all, so they skip the dependency restore entirely and their durations are not comparable with the rest. They are measured to show what the caching versions are spending their time on, not to be ranked against them.",
+    "",
     "| Cache | Size (MiB) |",
     "| --- | ---: |",
   );
diff --git a/scripts/report.test.mjs b/scripts/report.test.mjs
index e8a56c2..a904b60 100644
--- a/scripts/report.test.mjs
+++ b/scripts/report.test.mjs
@@ -101,5 +101,8 @@ test("renders a version table, a control and per-runner medians", () => {
   assert.match(report, /A\/A control/);
   assert.match(report, /Harness noise floor/);
   assert.match(report, /## Per-runner medians/);
+  // v1 and v2 skip the dependency restore entirely, so ranking them against
+  // main would compare different amounts of work.
+  assert.match(report, /not comparable \(no caching\)/);
   assert.match(report, /setup-java-Linux-x64-maven-abc/);
 });

From 77334eb4c4e43d5e7950b0548e126eacdf2dc383 Mon Sep 17 00:00:00 2001
From: Bruno Borges <bruno.borges@microsoft.com>
Date: Wed, 5 Aug 2026 01:02:41 -0400
Subject: [PATCH 10/19] Stop a stalled slot from being reported as an effect
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

An A/A run with both refs set to main reported a 0.821s regression, 95% CI
[0.119, 1.672], on identical code. Seven of the ten runners agreed to within
0.12s. Two had a single slot that stalled on the cache service — 8.169s against
a 3.286s sibling, and 10.030s against 4.449s — and both stalls happened to land
on the candidate arm, which is enough to move the mean of ten paired differences
by more than any effect these workflows measure.

Two changes, and the same data now reports -0.020s, 95% CI [-0.069, 0.030],
p = 0.498, inconclusive.

Runners whose own arm disagrees with itself by more than a robust threshold are
discarded before the estimate is formed. The threshold is the median within-arm
spread plus six times its median absolute deviation, so it adapts to the run
rather than being a fixed number of seconds, and at most half the runners can be
dropped. The decision uses only within-arm spread, which has the same
distribution whether or not the arms differ; filtering on the arm difference
itself would bias the result, but this cannot. Every report lists what it
discarded and why.

A verdict now also requires the permutation test to agree with the interval. The
percentile bootstrap is only approximate at these sample sizes while the
sign-flip test is exact under the null, so where they disagree the interval is
the one that is wrong. In the run above the interval excluded zero while the test
reported p = 0.099, and the verdict believed the interval.

The statistic stays the mean rather than becoming a median. A sign-flip test on
the median has almost no power here: six runners agreeing on a 12.5s effect gives
p = 0.13. Robustness comes from the filter, not from the estimator.

The failing A/A dataset is pinned as a regression test.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 71c45320-1417-4029-8402-69075d61dac1
---
 README.md                       |  4 +-
 scripts/paired.mjs              | 79 ++++++++++++++++++++++++++++-----
 scripts/report-focused.mjs      |  4 ++
 scripts/report-focused.test.mjs | 41 +++++++++++++++++
 scripts/report-jdk-cache.mjs    |  4 ++
 scripts/stats.mjs               | 26 +++++++++--
 6 files changed, 144 insertions(+), 14 deletions(-)

diff --git a/README.md b/README.md
index 7cd217b..c395be6 100644
--- a/README.md
+++ b/README.md
@@ -19,7 +19,9 @@ Every job also runs one unmeasured warm-up slot first. The first setup in a job
 
 **One cache, every arm.** A cache entry's download throughput depends on where the service placed the stored blob, and that placement is fixed for the life of the entry. Seeding one entry per arm therefore confounds the arm with its blob, and because the bias is identical on every runner, pairing cannot remove it and more samples only tighten the interval around the wrong answer. A single entry is seeded and every arm restores it.
 
-**Intervals, not point estimates.** `scripts/stats.mjs` reports a bootstrap 95% confidence interval, a permutation p-value, and a Hodges-Lehmann shift, and turns them into an explicit verdict. A comparison whose interval includes zero is reported as `inconclusive` rather than as a number that looks like a result.
+**Intervals, not point estimates.** `scripts/stats.mjs` reports a bootstrap 95% confidence interval, a permutation p-value, and a Hodges-Lehmann shift, and turns them into an explicit verdict. A comparison whose interval includes zero is reported as `inconclusive` rather than as a number that looks like a result. A verdict additionally requires the permutation test to agree: the percentile bootstrap is only approximate at ten paired observations, while the sign-flip test is exact under the null, so where they disagree the interval is the one that is wrong.
+
+**Stalled runners are discarded.** A slot can stall on the cache service for several seconds. Which arm the stall lands on is arbitrary, so that runner contributes an arbitrarily large difference and, with ten runners, one such slot moves the mean by more than any effect being measured. Runners whose *own arm disagrees with itself* by more than a robust threshold are dropped. That decision is made purely on within-arm spread, which has the same distribution whether or not the arms differ, so unlike filtering on the arm difference it cannot bias the result. Every report lists what it discarded and why.
 
 Every report also publishes two guard rails:
 
diff --git a/scripts/paired.mjs b/scripts/paired.mjs
index a9968f3..e1649da 100644
--- a/scripts/paired.mjs
+++ b/scripts/paired.mjs
@@ -120,6 +120,43 @@ export function buildPairs(
   });
 }
 
+// A slot can stall on the cache service for several seconds. Such a runner is
+// not measuring the effect, and which arm the stall lands on is arbitrary, so it
+// contributes an arbitrarily large difference.
+//
+// The filter looks only at how much an arm disagrees with itself, which is a
+// pure noise quantity: it has the same distribution whether or not the arms
+// differ. Excluding runners on it therefore cannot bias the estimated effect,
+// unlike filtering on the difference between arms. The threshold is robust
+// (median plus a multiple of the median absolute deviation) so that it adapts to
+// the run rather than being a fixed number of seconds.
+export function dropStalledRunners(pairs, { tolerance = 6 } = {}) {
+  if (pairs.length < 4) return { kept: pairs, dropped: [] };
+  const spreads = pairs.map((pair) =>
+    Math.max(
+      Math.abs(pair.baselineRepeatDelta),
+      Math.abs(pair.candidateRepeatDelta),
+    ),
+  );
+  const limit =
+    median(spreads) +
+    tolerance * Math.max(medianAbsoluteDeviation(spreads), 1e-9);
+  const kept = [];
+  const dropped = [];
+  pairs.forEach((pair, index) => {
+    if (spreads[index] > limit) {
+      dropped.push({ sample: pair.sample, repeatSpread: spreads[index] });
+    } else {
+      kept.push(pair);
+    }
+  });
+  // Never discard so much that what remains cannot support a verdict.
+  if (kept.length < Math.ceil(pairs.length / 2)) {
+    return { kept: pairs, dropped: [] };
+  }
+  return { kept, dropped, thresholdSeconds: limit };
+}
+
 // The smallest effect the harness can trust, derived from how much the same
 // implementation varies between its two slots on one runner.
 export function noiseFloor(pairs) {
@@ -148,32 +185,43 @@ export function analyzePairs(
   baselineArm = "baseline",
   candidateArm = "candidate",
 ) {
-  const pairs = buildPairs(rows, baselineArm, candidateArm);
+  const allPairs = buildPairs(rows, baselineArm, candidateArm);
+  const {
+    kept: pairs,
+    dropped,
+    thresholdSeconds,
+  } = dropStalledRunners(allPairs);
   const differences = pairs.map((pair) => pair.difference);
   const baselineValues = pairs.map((pair) => pair.baseline);
   const candidateValues = pairs.map((pair) => pair.candidate);
   const floor = noiseFloor(pairs);
   const interval = pairedInterval(differences, { seed: 1 });
+  const pValue = pairedPermutationTest(differences, { seed: 3 });
   // The same estimator applied to the within-arm repeats. A trustworthy harness
   // must not resolve a difference here, because it compares an arm with itself.
   // Judged against the same noise floor as the real effect, so a healthy run
   // reports `within-noise` or `inconclusive`.
-  const controlInterval = pairedInterval(
-    pairs.map((pair) => pair.baselineRepeatDelta),
-    { seed: 2 },
-  );
+  const controlDifferences = pairs.map((pair) => pair.baselineRepeatDelta);
+  const controlInterval = pairedInterval(controlDifferences, { seed: 2 });
+  const controlPValue = pairedPermutationTest(controlDifferences, { seed: 4 });
   return {
     pairs,
     noiseFloorSeconds: floor,
     baseline: armSummary(baselineArm, baselineValues),
     candidate: armSummary(candidateArm, candidateValues),
     interval,
-    pValue: pairedPermutationTest(differences, { seed: 3 }),
+    pValue,
     shiftSeconds: hodgesLehmann(candidateValues, baselineValues),
-    verdict: classify(interval, { noiseFloor: floor }),
+    verdict: classify(interval, { noiseFloor: floor, pValue }),
+    droppedRunners: dropped,
+    stallThresholdSeconds: thresholdSeconds,
     control: {
       interval: controlInterval,
-      verdict: classify(controlInterval, { noiseFloor: floor }),
+      pValue: controlPValue,
+      verdict: classify(controlInterval, {
+        noiseFloor: floor,
+        pValue: controlPValue,
+      }),
     },
   };
 }
@@ -217,7 +265,10 @@ export function analyzeAgainstReference(rows, arms, reference) {
         : pairedPermutationTest(differences, { seed: 50 + index }),
       verdict: isReference
         ? "reference"
-        : classify(interval, { noiseFloor: floor }),
+        : classify(interval, {
+            noiseFloor: floor,
+            pValue: pairedPermutationTest(differences, { seed: 50 + index }),
+          }),
     };
   });
   // Each arm's own two slots compared with themselves. Nothing changed between
@@ -230,6 +281,10 @@ export function analyzeAgainstReference(rows, arms, reference) {
     controlDifferences.length === 0
       ? null
       : pairedInterval(controlDifferences, { seed: 99 });
+  const controlPValue =
+    controlDifferences.length === 0
+      ? null
+      : pairedPermutationTest(controlDifferences, { seed: 98 });
   return {
     runners,
     arms,
@@ -238,7 +293,11 @@ export function analyzeAgainstReference(rows, arms, reference) {
     comparisons,
     control: {
       interval: controlInterval,
-      verdict: classify(controlInterval, { noiseFloor: floor }),
+      pValue: controlPValue,
+      verdict: classify(controlInterval, {
+        noiseFloor: floor,
+        pValue: controlPValue,
+      }),
     },
   };
 }
diff --git a/scripts/report-focused.mjs b/scripts/report-focused.mjs
index 80e999b..e5304e7 100644
--- a/scripts/report-focused.mjs
+++ b/scripts/report-focused.mjs
@@ -82,6 +82,10 @@ export function markdown(metadata, analysis, caches) {
     "",
     `A/A control (baseline against itself) reports **${control.verdict}** at ${formatInterval(control.interval, { digits: 3 })}. A healthy harness reports \`within-noise\` or \`inconclusive\` here; an \`improvement\` or \`regression\` means slot ordering is biasing results and the verdict above cannot be trusted.`,
     "",
+    analysis.droppedRunners.length === 0
+      ? "No runner was discarded for a stalled slot."
+      : `Discarded ${analysis.droppedRunners.length} runner(s) whose own arm disagreed with itself by more than ${analysis.stallThresholdSeconds.toFixed(3)}s, a threshold derived from the spread of the other runners: ${analysis.droppedRunners.map((entry) => `#${entry.sample} (${entry.repeatSpread.toFixed(3)}s)`).join(", ")}. A stalled slot lands on an arbitrary arm and would otherwise dominate the mean; the decision uses only within-arm spread, which carries no information about the effect.`,
+    "",
     "## Arms",
     "",
     "| Arm | Runners | Mean (s) | Median (s) | SD (s) | MAD (s) | p95 (s) |",
diff --git a/scripts/report-focused.test.mjs b/scripts/report-focused.test.mjs
index 2f9979a..86383a8 100644
--- a/scripts/report-focused.test.mjs
+++ b/scripts/report-focused.test.mjs
@@ -114,3 +114,44 @@ test("reports an A/A comparison as inconclusive", () => {
   assert.notEqual(analysis.verdict, "improvement");
   assert.notEqual(analysis.verdict, "regression");
 });
+
+// Real data from run 30976592589, an A/A run with both refs set to `main`, so
+// the true effect is exactly zero. Seven runners agreed to within 0.12s, but two
+// had a single slot that stalled on the cache service (8.169s against a 3.286s
+// sibling, and 10.030s against 4.449s). The mean of the paired differences put
+// those stalls entirely on the candidate arm and the harness reported a 0.821s
+// regression, 95% CI [0.119, 1.672].
+test("does not report an effect when two runners had a stalled slot", () => {
+  const observed = [
+    [1, 1.544, 1.452, 1.541, 1.594],
+    [2, 3.632, 3.592, 3.513, 3.513],
+    [3, 2.796, 8.169, 3.286, 2.781],
+    [4, 3.659, 3.841, 3.708, 3.812],
+    [5, 3.665, 5.232, 5.319, 5.292],
+    [6, 3.663, 4.449, 10.03, 3.763],
+    [7, 4.089, 4.544, 5.454, 3.772],
+    [8, 1.397, 1.37, 1.384, 1.39],
+    [9, 1.452, 1.469, 1.475, 1.354],
+    [10, 1.59, 1.435, 1.412, 1.49],
+  ];
+  const csv = observed
+    .map(([sample, b1, c2, c3, b4]) =>
+      [
+        `"${sample}","baseline","1","${b1 * 1000}"`,
+        `"${sample}","candidate","2","${c2 * 1000}"`,
+        `"${sample}","candidate","3","${c3 * 1000}"`,
+        `"${sample}","baseline","4","${b4 * 1000}"`,
+      ].join("\n"),
+    )
+    .join("\n");
+  const analysis = analyze(parseSamples(csv));
+  assert.equal(analysis.verdict, "inconclusive");
+  assert.ok(analysis.interval.low < 0 && analysis.interval.high > 0);
+  // The runners with a stalled slot are identified by their own arm disagreeing
+  // with itself, which carries no information about the effect.
+  assert.deepEqual(
+    analysis.droppedRunners.map((entry) => entry.sample).sort((a, b) => a - b),
+    [3, 5, 6, 7],
+  );
+  assert.equal(analysis.control.verdict, "inconclusive");
+});
diff --git a/scripts/report-jdk-cache.mjs b/scripts/report-jdk-cache.mjs
index 1105b29..1bd8b79 100644
--- a/scripts/report-jdk-cache.mjs
+++ b/scripts/report-jdk-cache.mjs
@@ -62,6 +62,10 @@ export function markdown(metadata, analysis, caches) {
     "",
     `A/A control (\`cache-jdk: false\` against itself) reports **${control.verdict}** at ${formatInterval(control.interval)}. A healthy harness reports \`within-noise\` or \`inconclusive\` here; an \`improvement\` or \`regression\` means slot ordering is biasing results and the verdict above cannot be trusted.`,
     "",
+    analysis.droppedRunners.length === 0
+      ? "No runner was discarded for a stalled slot."
+      : `Discarded ${analysis.droppedRunners.length} runner(s) whose own arm disagreed with itself by more than ${analysis.stallThresholdSeconds.toFixed(3)}s, a threshold derived from the spread of the other runners: ${analysis.droppedRunners.map((entry) => `#${entry.sample} (${entry.repeatSpread.toFixed(3)}s)`).join(", ")}. A stalled slot lands on an arbitrary arm and would otherwise dominate the mean; the decision uses only within-arm spread, which carries no information about the effect.`,
+    "",
     "## Arms",
     "",
     "| Arm | Runners | Mean (s) | Median (s) | SD (s) | MAD (s) | p95 (s) |",
diff --git a/scripts/stats.mjs b/scripts/stats.mjs
index 7cbde5a..12b9889 100644
--- a/scripts/stats.mjs
+++ b/scripts/stats.mjs
@@ -120,6 +120,13 @@ export function differenceInterval(
 // Bootstrap interval for the mean of within-runner paired differences. This is
 // the estimator to prefer whenever both arms were measured on the same runner,
 // because it cancels the between-runner variance that dominates hosted CI.
+// The statistic is the mean of the paired differences. It is not robust to a
+// runner whose slot stalled on the cache service, but switching to a median
+// costs almost all the power at the sample sizes these workflows run: on six
+// runners agreeing on a 12.5s effect, a sign-flip test on the median returns
+// p = 0.13. Robustness is provided instead by discarding stalled runners before
+// they reach this function, which is a decision made on within-arm spread and so
+// cannot bias the effect.
 export function pairedInterval(differences, options = {}) {
   const {
     iterations = BOOTSTRAP_ITERATIONS,
@@ -142,7 +149,9 @@ export function pairedInterval(differences, options = {}) {
 }
 
 // Two-sided permutation test on the mean of paired differences. Under the null
-// hypothesis the sign of each pair is arbitrary, so we resample signs.
+// hypothesis the sign of each pair is arbitrary, so we resample signs. It tests
+// the same statistic the interval estimates, and unlike the percentile bootstrap
+// it is exact under the null, which is why a verdict requires it to agree.
 export function pairedPermutationTest(differences, options = {}) {
   const { iterations = BOOTSTRAP_ITERATIONS, seed } = options;
   if (differences.length === 0) return null;
@@ -175,11 +184,22 @@ export function hodgesLehmann(treatment, baseline) {
 // Turn an interval into a decision. An interval that straddles zero means the
 // benchmark did not resolve the effect, and an effect smaller than the harness
 // noise floor is not trustworthy even when the interval excludes zero.
+// A verdict requires the bootstrap interval and the permutation test to agree.
+// The percentile bootstrap is approximate on ten paired observations, while the
+// sign-flip permutation test is exact under the null, so where they disagree the
+// interval is the one that is wrong. Passing no p-value keeps the interval-only
+// behaviour for callers that have no test to offer.
 export function classify(interval, options = {}) {
-  const { noiseFloor = 0, lowerIsBetter = true } = options;
+  const {
+    noiseFloor = 0,
+    lowerIsBetter = true,
+    pValue = null,
+    alpha = 0.05,
+  } = options;
   if (!interval) return "unknown";
   const { low, high, estimate } = interval;
   if (low <= 0 && high >= 0) return "inconclusive";
+  if (pValue !== null && pValue >= alpha) return "inconclusive";
   if (Math.abs(estimate) < noiseFloor) return "within-noise";
   const improved = lowerIsBetter ? estimate < 0 : estimate > 0;
   return improved ? "improvement" : "regression";
@@ -200,7 +220,7 @@ export function describeVerdict(verdict) {
     case "within-noise":
       return "No usable signal — the effect is smaller than the harness noise floor.";
     case "inconclusive":
-      return "Inconclusive — the confidence interval includes zero; collect more samples.";
+      return "Inconclusive — the interval includes zero or the permutation test does not agree; collect more samples.";
     default:
       return "Unknown.";
   }

From c017f73e5f6a952c03f8e1f0137faa545c15d177 Mon Sep 17 00:00:00 2001
From: Bruno Borges <bruno.borges@microsoft.com>
Date: Wed, 5 Aug 2026 01:14:31 -0400
Subject: [PATCH 11/19] Bound each measured setup so a stalled restore costs
 one runner

A slot in the version sweep sat on a cache restore for 214 seconds while its
neighbours in the same job took three. It is the second time a run has been held
open this way, and the version is incidental: the sweep restores the same 154 MiB
blob fifteen times per job across ten jobs, and the cache service occasionally
stalls one of them.

Cap each measured setup at three minutes. A stalled slot now fails its runner,
which the report discards along with the other incomplete ones, rather than
running out the job timeout. Narrow the sweep's waves to two, since it restores
three times as often per job as the two-arm workflows.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 71c45320-1417-4029-8402-69075d61dac1
---
 .github/workflows/benchmark.yml             | 95 ++++++++++++++++++++-
 .github/workflows/focused-cache-restore.yml | 30 +++++++
 .github/workflows/jdk-cache.yml             | 30 +++++++
 README.md                                   |  2 +
 4 files changed, 155 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index ec47ae8..e82ea89 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -143,8 +143,9 @@ jobs:
       # Every runner pulls the same seeded blob, so running the whole matrix at
       # once measures contention on the cache service that no real workflow would
       # see. Waves are kept small so each measurement reflects an ordinary
-      # restore.
-      max-parallel: 4
+      # restore. This job restores fifteen times where the two-arm workflows
+      # restore five, so it runs in narrower waves than they do.
+      max-parallel: 2
       matrix:
         sample: ${{ fromJSON(inputs.samples == '20' && '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]' || inputs.samples == '10' && '[1,2,3,4,5,6,7,8,9,10]' || inputs.samples == '6' && '[1,2,3,4,5,6]' || '[1,2]') }}
     steps:
@@ -177,6 +178,12 @@ jobs:
       - name: Reset warm-up slot
         run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}"
       - name: Warm-up setup (discarded)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: actions/setup-java@main
         with:
           distribution: ${{ inputs.distribution }}
@@ -189,6 +196,12 @@ jobs:
       - name: Start slot 1 timer
         run: node scripts/measure.mjs start
       - name: Slot 1 setup (v1)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: actions/setup-java@v1.4.4
         with:
           java-version: ${{ inputs.java-version }}
@@ -199,6 +212,12 @@ jobs:
       - name: Start slot 2 timer
         run: node scripts/measure.mjs start
       - name: Slot 2 setup (v2)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: actions/setup-java@v2.5.1
         with:
           distribution: ${{ inputs.distribution }}
@@ -210,6 +229,12 @@ jobs:
       - name: Start slot 3 timer
         run: node scripts/measure.mjs start
       - name: Slot 3 setup (v3)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: actions/setup-java@v3.14.1
         with:
           distribution: ${{ inputs.distribution }}
@@ -222,6 +247,12 @@ jobs:
       - name: Start slot 4 timer
         run: node scripts/measure.mjs start
       - name: Slot 4 setup (v4)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: actions/setup-java@v4.8.0
         with:
           distribution: ${{ inputs.distribution }}
@@ -235,6 +266,12 @@ jobs:
       - name: Start slot 5 timer
         run: node scripts/measure.mjs start
       - name: Slot 5 setup (v5.2)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: actions/setup-java@v5.2.0
         with:
           distribution: ${{ inputs.distribution }}
@@ -248,6 +285,12 @@ jobs:
       - name: Start slot 6 timer
         run: node scripts/measure.mjs start
       - name: Slot 6 setup (v5.6)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: actions/setup-java@v5.6.0
         with:
           distribution: ${{ inputs.distribution }}
@@ -261,6 +304,12 @@ jobs:
       - name: Start slot 7 timer
         run: node scripts/measure.mjs start
       - name: Slot 7 setup (main)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: actions/setup-java@main
         with:
           distribution: ${{ inputs.distribution }}
@@ -274,6 +323,12 @@ jobs:
       - name: Start slot 8 timer
         run: node scripts/measure.mjs start
       - name: Slot 8 setup (main)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: actions/setup-java@main
         with:
           distribution: ${{ inputs.distribution }}
@@ -287,6 +342,12 @@ jobs:
       - name: Start slot 9 timer
         run: node scripts/measure.mjs start
       - name: Slot 9 setup (v5.6)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: actions/setup-java@v5.6.0
         with:
           distribution: ${{ inputs.distribution }}
@@ -300,6 +361,12 @@ jobs:
       - name: Start slot 10 timer
         run: node scripts/measure.mjs start
       - name: Slot 10 setup (v5.2)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: actions/setup-java@v5.2.0
         with:
           distribution: ${{ inputs.distribution }}
@@ -313,6 +380,12 @@ jobs:
       - name: Start slot 11 timer
         run: node scripts/measure.mjs start
       - name: Slot 11 setup (v4)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: actions/setup-java@v4.8.0
         with:
           distribution: ${{ inputs.distribution }}
@@ -326,6 +399,12 @@ jobs:
       - name: Start slot 12 timer
         run: node scripts/measure.mjs start
       - name: Slot 12 setup (v3)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: actions/setup-java@v3.14.1
         with:
           distribution: ${{ inputs.distribution }}
@@ -338,6 +417,12 @@ jobs:
       - name: Start slot 13 timer
         run: node scripts/measure.mjs start
       - name: Slot 13 setup (v2)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: actions/setup-java@v2.5.1
         with:
           distribution: ${{ inputs.distribution }}
@@ -349,6 +434,12 @@ jobs:
       - name: Start slot 14 timer
         run: node scripts/measure.mjs start
       - name: Slot 14 setup (v1)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: actions/setup-java@v1.4.4
         with:
           java-version: ${{ inputs.java-version }}
diff --git a/.github/workflows/focused-cache-restore.yml b/.github/workflows/focused-cache-restore.yml
index dffef59..882ce2d 100644
--- a/.github/workflows/focused-cache-restore.yml
+++ b/.github/workflows/focused-cache-restore.yml
@@ -157,6 +157,12 @@ jobs:
       - name: Reset warm-up slot
         run: bash scripts/focused-cache-restore.sh reset "focused-${{ github.run_id }}"
       - name: Warm-up setup (discarded)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: ./baseline
         with:
           distribution: temurin
@@ -169,6 +175,12 @@ jobs:
       - name: Start slot 1 timer
         run: node scripts/measure.mjs start
       - name: Slot 1 setup (baseline)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: ./baseline
         with:
           distribution: temurin
@@ -185,6 +197,12 @@ jobs:
       - name: Start slot 2 timer
         run: node scripts/measure.mjs start
       - name: Slot 2 setup (candidate)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: ./candidate
         with:
           distribution: temurin
@@ -201,6 +219,12 @@ jobs:
       - name: Start slot 3 timer
         run: node scripts/measure.mjs start
       - name: Slot 3 setup (candidate)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: ./candidate
         with:
           distribution: temurin
@@ -217,6 +241,12 @@ jobs:
       - name: Start slot 4 timer
         run: node scripts/measure.mjs start
       - name: Slot 4 setup (baseline)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: ./baseline
         with:
           distribution: temurin
diff --git a/.github/workflows/jdk-cache.yml b/.github/workflows/jdk-cache.yml
index afed7cc..3d04d1f 100644
--- a/.github/workflows/jdk-cache.yml
+++ b/.github/workflows/jdk-cache.yml
@@ -187,6 +187,12 @@ jobs:
       - name: Reset warm-up slot
         run: bash scripts/jdk-cache.sh reset "jdk-cache-${{ github.run_id }}"
       - name: Warm-up setup (discarded)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: ./.setup-java
         with:
           distribution: ${{ inputs.distribution }}
@@ -201,6 +207,12 @@ jobs:
       - name: Start slot 1 timer
         run: node scripts/measure.mjs start
       - name: Slot 1 setup (baseline, no JDK cache)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: ./.setup-java
         with:
           distribution: ${{ inputs.distribution }}
@@ -217,6 +229,12 @@ jobs:
       - name: Start slot 2 timer
         run: node scripts/measure.mjs start
       - name: Slot 2 setup (candidate, JDK cache)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: ./.setup-java
         with:
           distribution: ${{ inputs.distribution }}
@@ -233,6 +251,12 @@ jobs:
       - name: Start slot 3 timer
         run: node scripts/measure.mjs start
       - name: Slot 3 setup (candidate, JDK cache)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: ./.setup-java
         with:
           distribution: ${{ inputs.distribution }}
@@ -249,6 +273,12 @@ jobs:
       - name: Start slot 4 timer
         run: node scripts/measure.mjs start
       - name: Slot 4 setup (baseline, no JDK cache)
+        # A restore that has not finished in three minutes has stalled on the
+        # cache service rather than being slow: its neighbours in the same job
+        # take a few seconds. Failing here costs one runner, which the report
+        # discards along with the other incomplete ones, instead of letting the
+        # job sit until its own timeout.
+        timeout-minutes: 3
         uses: ./.setup-java
         with:
           distribution: ${{ inputs.distribution }}
diff --git a/README.md b/README.md
index c395be6..0f3ee00 100644
--- a/README.md
+++ b/README.md
@@ -21,6 +21,8 @@ Every job also runs one unmeasured warm-up slot first. The first setup in a job
 
 **Intervals, not point estimates.** `scripts/stats.mjs` reports a bootstrap 95% confidence interval, a permutation p-value, and a Hodges-Lehmann shift, and turns them into an explicit verdict. A comparison whose interval includes zero is reported as `inconclusive` rather than as a number that looks like a result. A verdict additionally requires the permutation test to agree: the percentile bootstrap is only approximate at ten paired observations, while the sign-flip test is exact under the null, so where they disagree the interval is the one that is wrong.
 
+**Stalled slots fail fast.** Each measured setup is capped at three minutes. A restore that has not finished by then has stalled on the cache service rather than being slow — its neighbours in the same job take a few seconds — and failing costs one runner instead of holding the run open. The version sweep restores fifteen times per job where the two-arm workflows restore five, so it runs in narrower waves.
+
 **Stalled runners are discarded.** A slot can stall on the cache service for several seconds. Which arm the stall lands on is arbitrary, so that runner contributes an arbitrarily large difference and, with ten runners, one such slot moves the mean by more than any effect being measured. Runners whose *own arm disagrees with itself* by more than a robust threshold are dropped. That decision is made purely on within-arm spread, which has the same distribution whether or not the arms differ, so unlike filtering on the arm difference it cannot bias the result. Every report lists what it discarded and why.
 
 Every report also publishes two guard rails:

From 5f3c9852ca2544d891244876c843f74fe43ec0c0 Mon Sep 17 00:00:00 2001
From: Bruno Borges <bruno.borges@microsoft.com>
Date: Wed, 5 Aug 2026 01:19:35 -0400
Subject: [PATCH 12/19] Bound how long a failed cache download can stall a slot

A slot in the version sweep sat for 214 seconds on a setup that took 0 seconds
in the same job's mirrored slot, on the same runner and the same action version.
The action had already finished: it logged "maven cache is not found" and then
produced no further output until the job was cancelled.

The cause is in @actions/cache. Its promiseWithTimeout clears the timeout inside
a .then, which only runs when the raced promise fulfils. When the cache service
drops the blob mid-download, downloadToBuffer rejects, clearTimeout is skipped,
and the armed timer keeps the event loop alive for the remainder of the segment
timeout after all work is done. That default is 10 minutes.

Cap it at 2 minutes here. These restores move 154 MiB in about 3 seconds, so the
cap is far beyond any legitimate download and cannot abort real work; it only
bounds the idle. The stall filter already keeps such a runner out of the
estimate, so this is about not paying for the wait.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 71c45320-1417-4029-8402-69075d61dac1
---
 .github/workflows/benchmark.yml             | 7 +++++++
 .github/workflows/focused-cache-restore.yml | 7 +++++++
 .github/workflows/jdk-cache.yml             | 7 +++++++
 3 files changed, 21 insertions(+)

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index e82ea89..5253b9c 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -37,6 +37,13 @@ permissions:
   contents: read
 
 env:
+  # A cache segment download that fails leaves @actions/cache holding an armed
+  # timer it never clears, so the action idles for the rest of this timeout after
+  # its work is done (see actions/toolkit's promiseWithTimeout: the clearTimeout
+  # runs in a .then, which a rejection skips). The default is 10 minutes. These
+  # restores move 154 MiB in about 3 seconds, so 2 minutes is still far beyond
+  # any legitimate download and bounds a stall to a fifth of what it was.
+  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 2
   PETCLINIC_REF: f182358d02e4a68e52bdbabf55ca7800288511e7
 
 concurrency:
diff --git a/.github/workflows/focused-cache-restore.yml b/.github/workflows/focused-cache-restore.yml
index 882ce2d..243a0a9 100644
--- a/.github/workflows/focused-cache-restore.yml
+++ b/.github/workflows/focused-cache-restore.yml
@@ -39,6 +39,13 @@ permissions:
   contents: read
 
 env:
+  # A cache segment download that fails leaves @actions/cache holding an armed
+  # timer it never clears, so the action idles for the rest of this timeout after
+  # its work is done (see actions/toolkit's promiseWithTimeout: the clearTimeout
+  # runs in a .then, which a rejection skips). The default is 10 minutes. These
+  # restores move 154 MiB in about 3 seconds, so 2 minutes is still far beyond
+  # any legitimate download and bounds a stall to a fifth of what it was.
+  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 2
   JAVA_VERSION: 17.0.19+10
   DEPENDENCY_FIXTURE_MIB: "160"
   WRAPPER_FIXTURE_MIB: "9"
diff --git a/.github/workflows/jdk-cache.yml b/.github/workflows/jdk-cache.yml
index 3d04d1f..c104dd8 100644
--- a/.github/workflows/jdk-cache.yml
+++ b/.github/workflows/jdk-cache.yml
@@ -47,6 +47,13 @@ permissions:
   contents: read
 
 env:
+  # A cache segment download that fails leaves @actions/cache holding an armed
+  # timer it never clears, so the action idles for the rest of this timeout after
+  # its work is done (see actions/toolkit's promiseWithTimeout: the clearTimeout
+  # runs in a .then, which a rejection skips). The default is 10 minutes. These
+  # restores move 154 MiB in about 3 seconds, so 2 minutes is still far beyond
+  # any legitimate download and bounds a stall to a fifth of what it was.
+  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 2
   PETCLINIC_REF: f182358d02e4a68e52bdbabf55ca7800288511e7
 
 concurrency:

From de4e8344be3b1e1c857d9feab04c5f55095f47e9 Mon Sep 17 00:00:00 2001
From: Bruno Borges <bruno.borges@microsoft.com>
Date: Wed, 5 Aug 2026 01:21:25 -0400
Subject: [PATCH 13/19] Discard stalled runners in the version sweep too

The stall filter was only applied to the two-arm workflows, so the sweep still
formed its estimates from runners that had stalled on a restore. It showed:
its A/A control, comparing main with itself, reported 0.6s against a noise floor
of 0.157s.

Generalise the filter across arms. A runner whose two slots for any one version
disagree by more than the robust threshold is dropped, on the same reasoning as
before: which version a stall lands on is arbitrary, and the decision uses only
within-version spread, so it cannot bias the differences between versions.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 71c45320-1417-4029-8402-69075d61dac1
---
 scripts/paired.mjs      | 34 +++++++++++++++++++++++++++-------
 scripts/report.mjs      |  4 ++++
 scripts/report.test.mjs | 26 ++++++++++++++++++++++++++
 3 files changed, 57 insertions(+), 7 deletions(-)

diff --git a/scripts/paired.mjs b/scripts/paired.mjs
index e1649da..4a2d568 100644
--- a/scripts/paired.mjs
+++ b/scripts/paired.mjs
@@ -130,13 +130,15 @@ export function buildPairs(
 // unlike filtering on the difference between arms. The threshold is robust
 // (median plus a multiple of the median absolute deviation) so that it adapts to
 // the run rather than being a fixed number of seconds.
-export function dropStalledRunners(pairs, { tolerance = 6 } = {}) {
+export function dropStalledRunners(pairs, { tolerance = 6, spreadOf } = {}) {
   if (pairs.length < 4) return { kept: pairs, dropped: [] };
-  const spreads = pairs.map((pair) =>
-    Math.max(
-      Math.abs(pair.baselineRepeatDelta),
-      Math.abs(pair.candidateRepeatDelta),
-    ),
+  const spreads = pairs.map(
+    spreadOf ??
+      ((pair) =>
+        Math.max(
+          Math.abs(pair.baselineRepeatDelta),
+          Math.abs(pair.candidateRepeatDelta),
+        )),
   );
   const limit =
     median(spreads) +
@@ -230,7 +232,23 @@ export function analyzePairs(
 // Used where the workflow measures more than two implementations in the same
 // job, such as the version sweep.
 export function analyzeAgainstReference(rows, arms, reference) {
-  const runners = groupByRunner(rows, arms);
+  const allRunners = groupByRunner(rows, arms);
+  // Same filter as the two-arm workflows, applied across every arm: a runner
+  // whose slots for one version disagree with each other has stalled, and the
+  // arm the stall landed on is arbitrary.
+  const {
+    kept: runners,
+    dropped,
+    thresholdSeconds,
+  } = dropStalledRunners(allRunners, {
+    spreadOf: (runner) =>
+      Math.max(
+        ...arms.map((arm) => {
+          const slots = runner.slots.get(arm);
+          return Math.abs(slots[1] - slots[0]);
+        }),
+      ),
+  });
   const perArm = new Map(
     arms.map((arm) => [
       arm,
@@ -290,6 +308,8 @@ export function analyzeAgainstReference(rows, arms, reference) {
     arms,
     reference,
     noiseFloorSeconds: floor,
+    droppedRunners: dropped,
+    stallThresholdSeconds: thresholdSeconds,
     comparisons,
     control: {
       interval: controlInterval,
diff --git a/scripts/report.mjs b/scripts/report.mjs
index 3b4d6f6..1f50117 100644
--- a/scripts/report.mjs
+++ b/scripts/report.mjs
@@ -120,6 +120,10 @@ export function markdown(metadata, analysis, caches) {
     "",
     `A/A control (\`${REFERENCE}\` against itself) reports **${control.verdict}**${control.interval ? ` at ${formatInterval(control.interval)}` : ""}. A healthy harness reports \`within-noise\` or \`inconclusive\` here; anything else means slot ordering is biasing results and the table above cannot be trusted.`,
     "",
+    analysis.droppedRunners.length === 0
+      ? "No runner was discarded for a stalled slot."
+      : `Discarded ${analysis.droppedRunners.length} runner(s) whose slots for one version disagreed with each other by more than ${analysis.stallThresholdSeconds.toFixed(3)}s, a threshold derived from the spread of the other runners: ${analysis.droppedRunners.map((entry) => `#${entry.sample}`).join(", ")}. The version a stall lands on is arbitrary, and the decision uses only within-version spread, which carries no information about the differences between versions.`,
+    "",
     "## Per-runner medians",
     "",
     `| Runner | ${VERSIONS.map((entry) => entry.label).join(" | ")} |`,
diff --git a/scripts/report.test.mjs b/scripts/report.test.mjs
index a904b60..9a3dc21 100644
--- a/scripts/report.test.mjs
+++ b/scripts/report.test.mjs
@@ -106,3 +106,29 @@ test("renders a version table, a control and per-runner medians", () => {
   assert.match(report, /not comparable \(no caching\)/);
   assert.match(report, /setup-java-Linux-x64-maven-abc/);
 });
+
+test("discards a runner whose slots for one version disagree", () => {
+  // Nine well-behaved runners and one whose v4 slots differ by seconds, which
+  // means that runner stalled rather than measured. Which version a stall lands
+  // on is arbitrary, so it must not reach the comparison.
+  const rows = [1, 2, 3, 4, 5, 6, 7, 8, 9].map((sample) =>
+    sweep(sample, 1, flat),
+  );
+  const stalled = ORDER.map((arm, index) => {
+    const slot = index + 1;
+    const elapsed = arm === "v4" && slot === 4 ? 12000 : 3000;
+    return `"10","${arm}","${slot}","${elapsed}"`;
+  }).join("\n");
+  const analysis = analyzeAgainstReference(
+    parseSamples([...rows, stalled].join("\n")),
+    ARMS,
+    "main",
+  );
+  assert.deepEqual(
+    analysis.droppedRunners.map((entry) => entry.sample),
+    [10],
+  );
+  const v4 = analysis.comparisons.find((entry) => entry.arm === "v4");
+  assert.ok(Math.abs(v4.differenceSeconds) < 1e-9);
+  assert.equal(v4.verdict, "inconclusive");
+});

From e5f870c89040b99557b1e739ab4fae3eabe11c14 Mon Sep 17 00:00:00 2001
From: Bruno Borges <bruno.borges@microsoft.com>
Date: Wed, 5 Aug 2026 01:41:08 -0400
Subject: [PATCH 14/19] Give the configuration matrix a verdict it can support
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The Maven configuration warm path was the last workflow still measuring the old
way. It ran the arms ABAB, which does not cancel drift across the job: with a
steady slowdown of d per slot the candidate is biased by +d, because its slots
sit later on average than the baseline's. It had no warm-up slot, so the first
setup's DNS, TLS and cold page cache costs landed entirely on the baseline. And
it summarised each configuration on its own, where one runner yields one paired
difference and no interval is possible.

Run the arms ABBA after a discarded warm-up slot, matching the other workflows,
and add a report job that pools across the matrix. Each of the 36 configurations
is a block: the arms are compared within it, on one runner, and the differences
are combined. That answers what this workflow actually asks — whether the
candidate differs from the baseline across configurations — and costs no extra
jobs. Breakdowns by operating system and cache profile get their own intervals,
and per-configuration numbers are published as single observations with no
verdict attached, because that is all they are.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 71c45320-1417-4029-8402-69075d61dac1
---
 .../maven-configuration-warm-path.yml         | 114 +++++++----
 README.md                                     |   8 +-
 scripts/report-maven-configuration.mjs        | 189 ++++++++++++++++++
 scripts/report-maven-configuration.test.mjs   |  92 +++++++++
 4 files changed, 358 insertions(+), 45 deletions(-)
 create mode 100644 scripts/report-maven-configuration.mjs
 create mode 100644 scripts/report-maven-configuration.test.mjs

diff --git a/.github/workflows/maven-configuration-warm-path.yml b/.github/workflows/maven-configuration-warm-path.yml
index 52c5893..b7b545e 100644
--- a/.github/workflows/maven-configuration-warm-path.yml
+++ b/.github/workflows/maven-configuration-warm-path.yml
@@ -72,97 +72,97 @@ jobs:
           bash scripts/benchmark-maven-configuration.sh record-size baseline baseline
           bash scripts/benchmark-maven-configuration.sh record-size candidate candidate
 
-      - name: Prepare baseline iteration 1
+      # The first setup in a job pays DNS resolution, TLS handshakes and a cold
+      # page cache that the later ones do not. That is a one-off spike rather
+      # than drift, so the mirrored slot order cannot cancel it; this slot pays
+      # those costs and is discarded.
+      - name: Prepare warm-up slot
         run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}"
-      - name: Start baseline iteration 1 timer
-        run: bash scripts/benchmark-maven-configuration.sh start
-      - name: Run baseline iteration 1
+      - name: Warm-up setup (discarded)
         uses: ./baseline
+        # A setup that has not finished in three minutes has stalled rather than
+        # being slow; failing costs this configuration, which the report drops.
+        timeout-minutes: 3
         with:
           distribution: temurin
           java-version: ${{ matrix.versions.java-version }}
           cache: ${{ matrix.cache != 'none' && matrix.cache || '' }}
           cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }}
           settings-path: benchmark-maven-home
-      - name: Record baseline iteration 1
-        run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" baseline 1
 
-      - name: Prepare candidate iteration 1
+      - name: Prepare slot 1
         run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}"
-      - name: Start candidate iteration 1 timer
+      - name: Start slot 1 timer
         run: bash scripts/benchmark-maven-configuration.sh start
-      - name: Run candidate iteration 1
-        uses: ./candidate
-        with:
-          distribution: temurin
-          java-version: ${{ matrix.versions.java-version }}
-          cache: ${{ matrix.cache != 'none' && matrix.cache || '' }}
-          cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }}
-          settings-path: benchmark-maven-home
-      - name: Record candidate iteration 1
-        run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" candidate 1
-
-      - name: Prepare baseline iteration 2
-        run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}"
-      - name: Start baseline iteration 2 timer
-        run: bash scripts/benchmark-maven-configuration.sh start
-      - name: Run baseline iteration 2
+      - name: Slot 1 setup (baseline)
         uses: ./baseline
+        # A setup that has not finished in three minutes has stalled rather than
+        # being slow; failing costs this configuration, which the report drops.
+        timeout-minutes: 3
         with:
           distribution: temurin
           java-version: ${{ matrix.versions.java-version }}
           cache: ${{ matrix.cache != 'none' && matrix.cache || '' }}
           cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }}
           settings-path: benchmark-maven-home
-      - name: Record baseline iteration 2
-        run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" baseline 2
+      - name: Record slot 1
+        run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" baseline 1
 
-      - name: Prepare candidate iteration 2
+      - name: Prepare slot 2
         run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}"
-      - name: Start candidate iteration 2 timer
+      - name: Start slot 2 timer
         run: bash scripts/benchmark-maven-configuration.sh start
-      - name: Run candidate iteration 2
+      - name: Slot 2 setup (candidate)
         uses: ./candidate
+        # A setup that has not finished in three minutes has stalled rather than
+        # being slow; failing costs this configuration, which the report drops.
+        timeout-minutes: 3
         with:
           distribution: temurin
           java-version: ${{ matrix.versions.java-version }}
           cache: ${{ matrix.cache != 'none' && matrix.cache || '' }}
           cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }}
           settings-path: benchmark-maven-home
-      - name: Record candidate iteration 2
+      - name: Record slot 2
         run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" candidate 2
 
-      - name: Prepare baseline iteration 3
+      - name: Prepare slot 3
         run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}"
-      - name: Start baseline iteration 3 timer
+      - name: Start slot 3 timer
         run: bash scripts/benchmark-maven-configuration.sh start
-      - name: Run baseline iteration 3
-        uses: ./baseline
+      - name: Slot 3 setup (candidate)
+        uses: ./candidate
+        # A setup that has not finished in three minutes has stalled rather than
+        # being slow; failing costs this configuration, which the report drops.
+        timeout-minutes: 3
         with:
           distribution: temurin
           java-version: ${{ matrix.versions.java-version }}
           cache: ${{ matrix.cache != 'none' && matrix.cache || '' }}
           cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }}
           settings-path: benchmark-maven-home
-      - name: Record baseline iteration 3
-        run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" baseline 3
+      - name: Record slot 3
+        run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" candidate 3
 
-      - name: Prepare candidate iteration 3
+      - name: Prepare slot 4
         run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}"
-      - name: Start candidate iteration 3 timer
+      - name: Start slot 4 timer
         run: bash scripts/benchmark-maven-configuration.sh start
-      - name: Run candidate iteration 3
-        uses: ./candidate
+      - name: Slot 4 setup (baseline)
+        uses: ./baseline
+        # A setup that has not finished in three minutes has stalled rather than
+        # being slow; failing costs this configuration, which the report drops.
+        timeout-minutes: 3
         with:
           distribution: temurin
           java-version: ${{ matrix.versions.java-version }}
           cache: ${{ matrix.cache != 'none' && matrix.cache || '' }}
           cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }}
           settings-path: benchmark-maven-home
-      - name: Record candidate iteration 3
-        run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" candidate 3
+      - name: Record slot 4
+        run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" baseline 4
 
-      - name: Summarize benchmark
+      - name: Summarize configuration
         run: bash scripts/benchmark-maven-configuration.sh summarize "$GITHUB_STEP_SUMMARY"
       - name: Upload raw benchmark data
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
@@ -172,3 +172,31 @@ jobs:
           include-hidden-files: true
           if-no-files-found: error
           retention-days: 30
+
+  report:
+    name: Report
+    needs: benchmark
+    if: ${{ always() && !cancelled() }}
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Check out benchmark repository
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          persist-credentials: false
+      - name: Download configuration timings
+        uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6
+        with:
+          pattern: maven-config-*
+          path: .benchmark-results
+      - name: Generate configuration report
+        env:
+          SETUP_JAVA_REPOSITORY: ${{ inputs.setup-java-repository }}
+          BASELINE_REF: ${{ inputs.baseline-ref }}
+          CANDIDATE_REF: ${{ inputs.candidate-ref }}
+        run: node scripts/report-maven-configuration.mjs
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: maven-configuration-${{ github.run_id }}
+          path: maven-config-results/
+          retention-days: 30
diff --git a/README.md b/README.md
index 0f3ee00..472752d 100644
--- a/README.md
+++ b/README.md
@@ -100,9 +100,13 @@ Open **Actions > JDK cache > Run workflow** to select the distribution, Java ver
 
 ### Maven configuration warm path
 
-The **Maven configuration warm path** workflow compares two `actions/setup-java` refs in the action's own Maven configuration path. It checks out a configurable setup-java repository, runs baseline and candidate refs on Linux, Windows, and macOS, and covers Maven cache, Gradle cache, no-cache, single-version, multi-version, empty toolchains, and existing toolchains scenarios.
+The **Maven configuration warm path** workflow compares two refs of setup-java across 36 configurations: three operating systems, three cache profiles, single and multiple Java versions, and an empty or pre-existing `toolchains.xml`.
 
-Each matrix entry alternates three warm in-job setup runs for the baseline and candidate implementations, then reports median and p95 setup time. The workflow also records `dist/setup/index.js`, total `dist/setup` JavaScript bytes, and the JavaScript chunk files containing the XML parser.
+Unlike the other workflows it does not repeat one scenario across many runners; it runs each configuration once. A single configuration therefore yields one paired difference and cannot support an interval on its own. Each configuration is instead treated as a **block**: the arms are compared within it, on the same runner, in ABBA order after a discarded warm-up slot, and the differences are pooled across the matrix. That answers the question the workflow actually asks — whether the candidate differs from the baseline across configurations — at no extra job cost.
+
+Per-configuration numbers are still published, but as single observations with no verdict, because that is all they are. Breakdowns by operating system and by cache profile are reported with their own intervals; groups with few blocks will read `inconclusive` even where the pooled result does not, which is the intended behaviour rather than a defect.
+
+Open **Actions > Maven configuration warm path > Run workflow** to select the repository and the two refs.
 
 ## Reading results
 
diff --git a/scripts/report-maven-configuration.mjs b/scripts/report-maven-configuration.mjs
new file mode 100644
index 0000000..ac397f6
--- /dev/null
+++ b/scripts/report-maven-configuration.mjs
@@ -0,0 +1,189 @@
+// Cross-matrix report for the Maven configuration warm path.
+//
+// Unlike the other workflows, this one does not repeat one scenario across many
+// runners. It runs 36 different configurations once each, so a single cell
+// yields one paired difference and cannot support an interval on its own.
+//
+// Each configuration is instead treated as a block: the arms are compared within
+// it, on the same runner, in ABBA order, and the resulting differences are pooled
+// across the matrix. That answers the question this workflow actually asks —
+// whether the candidate differs from the baseline across configurations — and it
+// costs no extra jobs. Per-configuration numbers are still published, but as
+// single observations with no verdict attached, because that is all they are.
+
+import {
+  appendFile,
+  mkdir,
+  readdir,
+  readFile,
+  writeFile,
+} from "node:fs/promises";
+import { join } from "node:path";
+import { pathToFileURL } from "node:url";
+
+import { analyzePairs } from "./paired.mjs";
+import { describeVerdict, formatInterval } from "./stats.mjs";
+
+const RESULTS_DIR = ".benchmark-results";
+const OUTPUT_DIR = "maven-config-results";
+
+// os,cache,versions,toolchains,implementation,iteration,elapsedMs
+export function parseConfigurationSamples(csv) {
+  return csv
+    .trim()
+    .split("\n")
+    .filter(Boolean)
+    .map((line) => {
+      const [os, cache, versions, toolchains, arm, slot, elapsedMs] =
+        line.split(",");
+      return {
+        configuration: `${os}/${cache}/${versions}/${toolchains}`,
+        os,
+        cache,
+        versions,
+        toolchains,
+        arm,
+        slot: Number(slot),
+        seconds: Number(elapsedMs) / 1000,
+      };
+    });
+}
+
+// paired.mjs groups by a numeric `sample`, so each configuration is assigned a
+// stable index and becomes one block.
+export function toPairedRows(rows) {
+  const configurations = [
+    ...new Set(rows.map((row) => row.configuration)),
+  ].sort();
+  const index = new Map(configurations.map((name, at) => [name, at + 1]));
+  return {
+    configurations,
+    rows: rows.map((row) => ({
+      sample: index.get(row.configuration),
+      arm: row.arm,
+      slot: row.slot,
+      seconds: row.seconds,
+    })),
+  };
+}
+
+export async function readResults(directory = RESULTS_DIR) {
+  const entries = await readdir(directory, { recursive: true });
+  const files = entries.filter((entry) => entry.endsWith("timings.csv"));
+  if (files.length === 0) {
+    throw new Error(`No timings.csv files found in ${directory}`);
+  }
+  const contents = await Promise.all(
+    files.sort().map((file) => readFile(join(directory, file), "utf8")),
+  );
+  return contents.join("\n");
+}
+
+function subsetAnalysis(rows, predicate) {
+  const subset = rows.filter(predicate);
+  if (subset.length === 0) return null;
+  const { rows: paired } = toPairedRows(subset);
+  const analysis = analyzePairs(paired, "baseline", "candidate");
+  return analysis.pairs.length === 0 ? null : analysis;
+}
+
+export function markdown(metadata, overall, byOs, byCache, configurations) {
+  const lines = [
+    "# Maven configuration warm path",
+    "",
+    `Baseline \`${metadata.baselineRef}\` vs candidate \`${metadata.candidateRef}\` from \`${metadata.setupJavaRepository}\`, run ${metadata.runId}.`,
+    "",
+    `${configurations.length} configurations, each measured once in ABBA order on its own runner after a discarded warm-up slot. A configuration is a block: the arms are compared within it, and the differences are pooled across the matrix. No single configuration supports a verdict on its own, because one runner yields one difference.`,
+    "",
+    "## Verdict across all configurations",
+    "",
+    `**${describeVerdict(overall.verdict)}**`,
+    "",
+    `Pooled paired difference (candidate - baseline): **${formatInterval(overall.interval, { digits: 3 })}**.`,
+    `Permutation p-value: ${overall.pValue.toFixed(3)}. Blocks: ${overall.pairs.length}.`,
+    `Harness noise floor: ${overall.noiseFloorSeconds.toFixed(3)}s (median spread between an arm's own two slots within a configuration).`,
+    "",
+    `A/A control (baseline against itself) reports **${overall.control.verdict}** at ${formatInterval(overall.control.interval, { digits: 3 })}. A healthy harness reports \`within-noise\` or \`inconclusive\` here; anything else means slot ordering is biasing results and the verdict above cannot be trusted.`,
+    "",
+    overall.droppedRunners.length === 0
+      ? "No configuration was discarded for a stalled slot."
+      : `Discarded ${overall.droppedRunners.length} configuration(s) whose own arm disagreed with itself by more than ${overall.stallThresholdSeconds.toFixed(3)}s: ${overall.droppedRunners.map((entry) => `#${entry.sample}`).join(", ")}.`,
+    "",
+    "## By operating system",
+    "",
+    "| Group | Blocks | Difference (s) | 95% CI | p | Verdict |",
+    "| --- | ---: | ---: | --- | ---: | --- |",
+  ];
+  for (const [label, analysis] of [...byOs, ...byCache]) {
+    if (!analysis) {
+      lines.push(`| ${label} | 0 | n/a | n/a | n/a | no data |`);
+      continue;
+    }
+    lines.push(
+      `| ${label} | ${analysis.pairs.length} | ${analysis.interval.estimate.toFixed(3)} | ${analysis.interval.low.toFixed(3)} to ${analysis.interval.high.toFixed(3)} | ${analysis.pValue.toFixed(3)} | ${analysis.verdict} |`,
+    );
+  }
+  lines.push(
+    "",
+    "Groups with few blocks will report `inconclusive` even where the pooled result does not. That is the intended behaviour: a handful of configurations cannot resolve a small effect.",
+    "",
+    "## Per configuration",
+    "",
+    "Single observations. No interval is quoted because one runner cannot support one.",
+    "",
+    "| Configuration | baseline (s) | candidate (s) | Difference (s) |",
+    "| --- | ---: | ---: | ---: |",
+  );
+  for (const pair of overall.pairs) {
+    lines.push(
+      `| ${configurations[pair.sample - 1]} | ${pair.baseline.toFixed(3)} | ${pair.candidate.toFixed(3)} | ${pair.difference.toFixed(3)} |`,
+    );
+  }
+  return `${lines.join("\n")}\n`;
+}
+
+export async function main(env = process.env) {
+  const rows = parseConfigurationSamples(await readResults());
+  const { configurations, rows: paired } = toPairedRows(rows);
+  const overall = analyzePairs(paired, "baseline", "candidate");
+  if (overall.pairs.length === 0) {
+    throw new Error("No configuration completed all four slots");
+  }
+
+  const operatingSystems = [...new Set(rows.map((row) => row.os))].sort();
+  const caches = [...new Set(rows.map((row) => row.cache))].sort();
+  const byOs = operatingSystems.map((os) => [
+    os,
+    subsetAnalysis(rows, (row) => row.os === os),
+  ]);
+  const byCache = caches.map((cache) => [
+    `cache: ${cache}`,
+    subsetAnalysis(rows, (row) => row.cache === cache),
+  ]);
+
+  const metadata = {
+    runId: env.GITHUB_RUN_ID,
+    setupJavaRepository: env.SETUP_JAVA_REPOSITORY,
+    baselineRef: env.BASELINE_REF,
+    candidateRef: env.CANDIDATE_REF,
+    generatedAt: new Date().toISOString(),
+  };
+
+  const report = markdown(metadata, overall, byOs, byCache, configurations);
+  await mkdir(OUTPUT_DIR, { recursive: true });
+  await writeFile(
+    `${OUTPUT_DIR}/results.json`,
+    `${JSON.stringify({ metadata, overall, configurations }, null, 2)}\n`,
+  );
+  await writeFile(`${OUTPUT_DIR}/summary.md`, report);
+  if (env.GITHUB_STEP_SUMMARY) {
+    await appendFile(env.GITHUB_STEP_SUMMARY, report);
+  }
+}
+
+if (
+  process.argv[1] &&
+  import.meta.url === pathToFileURL(process.argv[1]).href
+) {
+  await main();
+}
diff --git a/scripts/report-maven-configuration.test.mjs b/scripts/report-maven-configuration.test.mjs
new file mode 100644
index 0000000..292cd23
--- /dev/null
+++ b/scripts/report-maven-configuration.test.mjs
@@ -0,0 +1,92 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { analyzePairs } from "./paired.mjs";
+import {
+  markdown,
+  parseConfigurationSamples,
+  toPairedRows,
+} from "./report-maven-configuration.mjs";
+
+const OSES = ["ubuntu-latest", "windows-latest", "macos-15-intel"];
+const CACHES = ["none", "maven", "gradle"];
+
+// One configuration's ABBA block. `speed` scales the whole runner so that a slow
+// configuration stays slow across both arms, which is what the pairing removes.
+function block(os, cache, speed, effectMs = 0) {
+  const base = 4000 * speed;
+  return [
+    `${os},${cache},single,empty,baseline,1,${Math.round(base)}`,
+    `${os},${cache},single,empty,candidate,2,${Math.round(base + effectMs)}`,
+    `${os},${cache},single,empty,candidate,3,${Math.round(base + effectMs)}`,
+    `${os},${cache},single,empty,baseline,4,${Math.round(base)}`,
+  ].join("\n");
+}
+
+function matrix(effectMs) {
+  const lines = [];
+  OSES.forEach((os, osIndex) => {
+    CACHES.forEach((cache, cacheIndex) => {
+      lines.push(
+        block(os, cache, 1 + osIndex * 0.8 + cacheIndex * 0.3, effectMs),
+      );
+    });
+  });
+  return lines.join("\n");
+}
+
+test("treats each configuration as one paired block", () => {
+  const rows = parseConfigurationSamples(matrix(0));
+  const { configurations, rows: paired } = toPairedRows(rows);
+  assert.equal(configurations.length, 9);
+  // Nine blocks of four slots each.
+  assert.equal(paired.length, 36);
+  assert.deepEqual(
+    [...new Set(paired.map((row) => row.sample))].sort((a, b) => a - b),
+    [1, 2, 3, 4, 5, 6, 7, 8, 9],
+  );
+});
+
+test("pools a consistent effect across configurations", () => {
+  // Every configuration is 500ms slower on the candidate, but the configurations
+  // differ from each other by up to 3x. Pooling the within-block differences must
+  // recover 500ms regardless of that spread.
+  const { rows } = toPairedRows(parseConfigurationSamples(matrix(500)));
+  const analysis = analyzePairs(rows, "baseline", "candidate");
+  assert.ok(Math.abs(analysis.interval.estimate - 0.5) < 1e-9);
+  assert.equal(analysis.verdict, "regression");
+});
+
+test("reports no effect when the arms are identical", () => {
+  const { rows } = toPairedRows(parseConfigurationSamples(matrix(0)));
+  const analysis = analyzePairs(rows, "baseline", "candidate");
+  assert.ok(["inconclusive", "within-noise"].includes(analysis.verdict));
+});
+
+test("renders a pooled verdict, group breakdowns and per-configuration rows", () => {
+  const rows = parseConfigurationSamples(matrix(500));
+  const { configurations, rows: paired } = toPairedRows(rows);
+  const overall = analyzePairs(paired, "baseline", "candidate");
+  const report = markdown(
+    {
+      runId: "1",
+      setupJavaRepository: "actions/setup-java",
+      baselineRef: "v4.8.0",
+      candidateRef: "main",
+    },
+    overall,
+    [["ubuntu-latest", overall]],
+    [["cache: maven", null]],
+    configurations,
+  );
+  assert.match(report, /# Maven configuration warm path/);
+  assert.match(report, /## Verdict across all configurations/);
+  assert.match(report, /A\/A control/);
+  assert.match(report, /Harness noise floor/);
+  // A group with no usable data must say so rather than quote a number.
+  assert.match(report, /\| cache: maven \| 0 \| n\/a \|/);
+  assert.match(
+    report,
+    /No interval is quoted because one runner cannot support one/,
+  );
+});

From 8a3c51cfe6aed2ac449177590c21386a49708caf Mon Sep 17 00:00:00 2001
From: Bruno Borges <bruno.borges@microsoft.com>
Date: Wed, 5 Aug 2026 01:49:45 -0400
Subject: [PATCH 15/19] Fail on missing report inputs instead of publishing
 undefined

Two ways the reports could publish something misleading rather than fail.

formatInterval hard-coded "95% CI" while the interval it was given carries its
own confidence level, so a caller that widened or narrowed an interval would
have had it labelled 95% regardless. Read the level off the interval.

The reports also checked only some of the environment variables they go on to
render. A missing SETUP_JAVA_REPOSITORY or JAVA_VERSION reached the summary as
"undefined" instead of stopping the run. Check every variable a report renders,
in one shared helper, and name the ones that are missing.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 71c45320-1417-4029-8402-69075d61dac1
---
 scripts/paired.mjs                     | 12 ++++++++++++
 scripts/paired.test.mjs                | 13 +++++++++++++
 scripts/report-focused.mjs             | 17 +++++++++++++++--
 scripts/report-jdk-cache.mjs           | 22 +++++++++++++++++++---
 scripts/report-maven-configuration.mjs |  8 +++++++-
 scripts/report.mjs                     | 14 ++++++++++++--
 scripts/stats.mjs                      |  8 ++++++--
 scripts/stats.test.mjs                 | 24 ++++++++++++++++++++++++
 8 files changed, 108 insertions(+), 10 deletions(-)
 create mode 100644 scripts/paired.test.mjs

diff --git a/scripts/paired.mjs b/scripts/paired.mjs
index 4a2d568..8b82d24 100644
--- a/scripts/paired.mjs
+++ b/scripts/paired.mjs
@@ -321,3 +321,15 @@ export function analyzeAgainstReference(rows, arms, reference) {
     },
   };
 }
+
+// Every name passed here ends up in the published report, so a missing value has
+// to fail with the name of what is missing. Rendering `undefined` into a results
+// table is worse than not rendering one at all.
+export function requireEnv(env, names) {
+  const missing = names.filter((name) => !env[name]);
+  if (missing.length > 0) {
+    throw new Error(
+      `Missing required environment variables: ${missing.join(", ")}`,
+    );
+  }
+}
diff --git a/scripts/paired.test.mjs b/scripts/paired.test.mjs
new file mode 100644
index 0000000..51d9b12
--- /dev/null
+++ b/scripts/paired.test.mjs
@@ -0,0 +1,13 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { requireEnv } from "./paired.mjs";
+
+test("requireEnv names every missing variable", () => {
+  assert.doesNotThrow(() => requireEnv({ A: "1", B: "2" }, ["A", "B"]));
+  // An empty string is as unusable in a report as an absent variable.
+  assert.throws(
+    () => requireEnv({ A: "1", B: "" }, ["A", "B", "C"]),
+    /Missing required environment variables: B, C/,
+  );
+});
diff --git a/scripts/report-focused.mjs b/scripts/report-focused.mjs
index e5304e7..316b299 100644
--- a/scripts/report-focused.mjs
+++ b/scripts/report-focused.mjs
@@ -15,6 +15,7 @@ import {
   noiseFloor,
   parseSamples,
   readSampleFiles as readPairedSampleFiles,
+  requireEnv,
 } from "./paired.mjs";
 import { classify, describeVerdict, formatInterval } from "./stats.mjs";
 
@@ -128,14 +129,26 @@ export function markdown(metadata, analysis, caches) {
 }
 
 export async function main(env = process.env) {
+  requireEnv(env, [
+    "GITHUB_REPOSITORY",
+    "GH_TOKEN",
+    "GITHUB_RUN_ID",
+    "BASELINE_REF",
+    "CANDIDATE_REF",
+    "SETUP_JAVA_REPOSITORY",
+    "JAVA_VERSION",
+  ]);
+
   const [owner, repo] = env.GITHUB_REPOSITORY.split("/");
   const token = env.GH_TOKEN;
   const runId = env.GITHUB_RUN_ID;
   const baselineRef = env.BASELINE_REF;
   const candidateRef = env.CANDIDATE_REF;
   const setupJavaRepository = env.SETUP_JAVA_REPOSITORY;
-  if (!owner || !repo || !token || !runId || !baselineRef || !candidateRef) {
-    throw new Error("Missing required GitHub Actions environment variables");
+  if (!owner || !repo) {
+    throw new Error(
+      `GITHUB_REPOSITORY must be owner/repo, got "${env.GITHUB_REPOSITORY}"`,
+    );
   }
 
   const rows = parseSamples(await readSampleFiles());
diff --git a/scripts/report-jdk-cache.mjs b/scripts/report-jdk-cache.mjs
index 1bd8b79..cccc3d9 100644
--- a/scripts/report-jdk-cache.mjs
+++ b/scripts/report-jdk-cache.mjs
@@ -1,7 +1,12 @@
 import { appendFile, mkdir, writeFile } from "node:fs/promises";
 import { pathToFileURL } from "node:url";
 
-import { analyzePairs, parseSamples, readSampleFiles } from "./paired.mjs";
+import {
+  analyzePairs,
+  parseSamples,
+  readSampleFiles,
+  requireEnv,
+} from "./paired.mjs";
 import { describeVerdict, formatInterval } from "./stats.mjs";
 
 const API_VERSION = "2022-11-28";
@@ -111,11 +116,22 @@ export function markdown(metadata, analysis, caches) {
 }
 
 export async function main(env = process.env) {
+  requireEnv(env, [
+    "GITHUB_REPOSITORY",
+    "GH_TOKEN",
+    "GITHUB_RUN_ID",
+    "DISTRIBUTION",
+    "JAVA_VERSION",
+    "SETUP_JAVA_REPOSITORY",
+    "SETUP_JAVA_REF",
+  ]);
   const [owner, repo] = env.GITHUB_REPOSITORY.split("/");
   const token = env.GH_TOKEN;
   const runId = env.GITHUB_RUN_ID;
-  if (!owner || !repo || !token || !runId) {
-    throw new Error("Missing required GitHub Actions environment variables");
+  if (!owner || !repo) {
+    throw new Error(
+      `GITHUB_REPOSITORY must be owner/repo, got "${env.GITHUB_REPOSITORY}"`,
+    );
   }
 
   const rows = parseSamples(await readSampleFiles("jdk-cache-timings"));
diff --git a/scripts/report-maven-configuration.mjs b/scripts/report-maven-configuration.mjs
index ac397f6..376540a 100644
--- a/scripts/report-maven-configuration.mjs
+++ b/scripts/report-maven-configuration.mjs
@@ -21,7 +21,7 @@ import {
 import { join } from "node:path";
 import { pathToFileURL } from "node:url";
 
-import { analyzePairs } from "./paired.mjs";
+import { analyzePairs, requireEnv } from "./paired.mjs";
 import { describeVerdict, formatInterval } from "./stats.mjs";
 
 const RESULTS_DIR = ".benchmark-results";
@@ -143,6 +143,12 @@ export function markdown(metadata, overall, byOs, byCache, configurations) {
 }
 
 export async function main(env = process.env) {
+  requireEnv(env, [
+    "GITHUB_RUN_ID",
+    "SETUP_JAVA_REPOSITORY",
+    "BASELINE_REF",
+    "CANDIDATE_REF",
+  ]);
   const rows = parseConfigurationSamples(await readResults());
   const { configurations, rows: paired } = toPairedRows(rows);
   const overall = analyzePairs(paired, "baseline", "candidate");
diff --git a/scripts/report.mjs b/scripts/report.mjs
index 1f50117..5201235 100644
--- a/scripts/report.mjs
+++ b/scripts/report.mjs
@@ -6,6 +6,7 @@ import {
   analyzeAgainstReference,
   parseSamples,
   readSampleFiles,
+  requireEnv,
 } from "./paired.mjs";
 import { describeVerdict, formatInterval } from "./stats.mjs";
 
@@ -157,11 +158,20 @@ export function markdown(metadata, analysis, caches) {
 }
 
 export async function main(env = process.env) {
+  requireEnv(env, [
+    "GITHUB_REPOSITORY",
+    "GH_TOKEN",
+    "GITHUB_RUN_ID",
+    "DISTRIBUTION",
+    "JAVA_VERSION",
+  ]);
   const [owner, repo] = env.GITHUB_REPOSITORY.split("/");
   const token = env.GH_TOKEN;
   const runId = env.GITHUB_RUN_ID;
-  if (!owner || !repo || !token || !runId) {
-    throw new Error("Missing required GitHub Actions environment variables");
+  if (!owner || !repo) {
+    throw new Error(
+      `GITHUB_REPOSITORY must be owner/repo, got "${env.GITHUB_REPOSITORY}"`,
+    );
   }
 
   const rows = parseSamples(await readSampleFiles("version-sweep"));
diff --git a/scripts/stats.mjs b/scripts/stats.mjs
index 12b9889..959a6d2 100644
--- a/scripts/stats.mjs
+++ b/scripts/stats.mjs
@@ -205,10 +205,14 @@ export function classify(interval, options = {}) {
   return improved ? "improvement" : "regression";
 }
 
+// The label follows the interval's own confidence level rather than assuming
+// 95%, so a caller that widens or narrows the interval cannot end up publishing
+// a number under the wrong label.
 export function formatInterval(interval, { digits = 1, unit = "s" } = {}) {
   if (!interval) return "n/a";
-  const { estimate, low, high } = interval;
-  return `${estimate.toFixed(digits)}${unit} (95% CI ${low.toFixed(digits)} to ${high.toFixed(digits)})`;
+  const { estimate, low, high, confidence = DEFAULT_CONFIDENCE } = interval;
+  const level = Number((confidence * 100).toFixed(2));
+  return `${estimate.toFixed(digits)}${unit} (${level}% CI ${low.toFixed(digits)} to ${high.toFixed(digits)})`;
 }
 
 export function describeVerdict(verdict) {
diff --git a/scripts/stats.test.mjs b/scripts/stats.test.mjs
index e3fe031..b2c7764 100644
--- a/scripts/stats.test.mjs
+++ b/scripts/stats.test.mjs
@@ -6,6 +6,7 @@ import {
   classify,
   createRandom,
   differenceInterval,
+  formatInterval,
   hodgesLehmann,
   mean,
   median,
@@ -126,3 +127,26 @@ test("unpaired sampling produces a false positive on identical code", () => {
   // observed run-to-run drift, which suppresses the spurious verdict.
   assert.equal(classify(interval, { noiseFloor: 1.5 }), "within-noise");
 });
+
+test("formatInterval labels the interval's own confidence level", () => {
+  assert.equal(
+    formatInterval(
+      { estimate: -1.25, low: -2, high: -0.5, confidence: 0.95 },
+      { digits: 2 },
+    ),
+    "-1.25s (95% CI -2.00 to -0.50)",
+  );
+  // A caller that widens the interval must not have it published as 95%.
+  assert.equal(
+    formatInterval(
+      { estimate: -1.25, low: -3, high: 0.5, confidence: 0.99 },
+      { digits: 2 },
+    ),
+    "-1.25s (99% CI -3.00 to 0.50)",
+  );
+  // Intervals built before `confidence` was carried still read as 95%.
+  assert.equal(
+    formatInterval({ estimate: 0, low: -1, high: 1 }),
+    "0.0s (95% CI -1.0 to 1.0)",
+  );
+});

From ac7009599e7cadf68b943a8742c52617cafd14ee Mon Sep 17 00:00:00 2001
From: Bruno Borges <bruno.borges@microsoft.com>
Date: Wed, 5 Aug 2026 01:53:33 -0400
Subject: [PATCH 16/19] Compare version benchmarks within compatible cohorts

The version sweep used to place all seven versions in one table and compare each
against main. That is not a fair ranking: v1 and v2 are uncached, v3 uses an
older cache-key contract, and v5.6/main restore the Maven Wrapper in addition
to dependencies.

Keep the same-runner mirrored measurements, but report three cohorts: uncached
(v1/v2), dependency cache (v3/v4/v5.2), and dependency plus wrapper cache
(v5.6/main). Each cohort has its own reference and A/A control. Apply Holm's
step-down correction within a cohort so multiple comparisons do not turn an
uncertain p-value into a headline verdict. Retain the all-arm diagnostics for
runner quality and workload shape, explicitly not as a ranking.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 71c45320-1417-4029-8402-69075d61dac1
---
 README.md               |   4 +-
 scripts/report.mjs      | 115 +++++++++++++++++++++++++++++++---------
 scripts/report.test.mjs |  27 ++++++++--
 scripts/stats.mjs       |  16 ++++++
 scripts/stats.test.mjs  |   6 +++
 5 files changed, 136 insertions(+), 32 deletions(-)

diff --git a/README.md b/README.md
index 472752d..a16e0aa 100644
--- a/README.md
+++ b/README.md
@@ -45,9 +45,9 @@ Each action version runs with Java 17 on `ubuntu-24.04`:
 
 v1 predates distribution selection and integrated dependency caching. It runs only its native Zulu installer path. v2 supports the Temurin and Microsoft scenarios, but its bundled legacy cache client is rejected by the current Actions cache service. v1 and v2 therefore have no Maven cache storage, and their cold/warm labels are repeated uncached samples.
 
-A seed job compiles Spring PetClinic once to populate a single Maven cache entry. Every measurement runner then sets up all seven versions in one job, in the order `v1..main` followed by `main..v1`, deleting `~/.m2` before each slot. The report compares every version against `main` with a paired interval per runner.
+A seed job compiles Spring PetClinic once to populate a single Maven cache entry. Every measurement runner then sets up all seven versions in one job, in the order `v1..main` followed by `main..v1`, deleting `~/.m2` before each slot. The report keeps the same-runner measurements but compares only behaviorally comparable cohorts, applying Holm correction within each cohort rather than ranking every version against `main`.
 
-v3, v4, and v5.2 cache only Maven dependencies. v5.6 and `main` also cache the Maven Wrapper distribution separately, making the storage and execution-time tradeoff visible. v4 and later share one cache entry through `cache-dependency-path`, so the stored blob is held constant across them. v3 predates that input and keys on `pom.xml`, so it necessarily uses its own entry; treat its difference with more caution than the rest. v1 and v2 do no caching at all and serve as the uncached reference.
+The report has three cohorts: **uncached setup** (`v1`, `v2`), **dependency cache** (`v3`, `v4`, `v5.2`), and **dependency plus wrapper cache** (`v5.6`, `main`). v1/v2 are not comparable with cached versions. v3 predates `cache-dependency-path` and keys on `pom.xml`, so its comparison within the dependency-cache cohort is explicitly caveated; v4 and v5.2 share one controlled cache entry. v5.6 and `main` both restore the dependency and Maven Wrapper caches. Cross-cohort timings are descriptive and must not be used to rank versions.
 
 Spring PetClinic and third-party actions are pinned to commits. `setup-java@main` intentionally remains a moving ref so each run evaluates the current upcoming v6 code; the report records the `main` commit observed when it is generated.
 
diff --git a/scripts/report.mjs b/scripts/report.mjs
index 5201235..d0b87e7 100644
--- a/scripts/report.mjs
+++ b/scripts/report.mjs
@@ -8,7 +8,7 @@ import {
   readSampleFiles,
   requireEnv,
 } from "./paired.mjs";
-import { describeVerdict, formatInterval } from "./stats.mjs";
+import { classify, formatInterval, holmAdjust } from "./stats.mjs";
 
 const API_VERSION = "2022-11-28";
 const RESULTS_DIR = "results";
@@ -29,6 +29,59 @@ export const VERSIONS = [
 
 const LABELS = new Map(VERSIONS.map((entry) => [entry.arm, entry.label]));
 
+export const COHORTS = [
+  {
+    id: "uncached",
+    label: "Uncached setup",
+    arms: ["v1", "v2"],
+    reference: "v2",
+    note: "v1 and v2 do not restore Maven dependencies, so this comparison is not comparable with cached versions.",
+  },
+  {
+    id: "dependency-cache",
+    label: "Dependency cache",
+    arms: ["v3", "v4", "v52"],
+    reference: "v52",
+    note: "v3 uses its older pom.xml cache-key strategy; compare it with caution. v4 and v5.2 share the controlled cache key.",
+  },
+  {
+    id: "wrapper-cache",
+    label: "Dependency plus wrapper cache",
+    arms: ["v56", "main"],
+    reference: "main",
+    note: "Both versions restore the dependency and Maven Wrapper caches.",
+  },
+];
+
+function analyzeCohort(rows, cohort) {
+  return analyzeAgainstReference(
+    rows.filter((row) => cohort.arms.includes(row.arm)),
+    cohort.arms,
+    cohort.reference,
+  );
+}
+
+function correctCohort(analysis) {
+  const adjusted = holmAdjust(
+    analysis.comparisons.map((comparison) =>
+      comparison.isReference ? null : comparison.pValue,
+    ),
+  );
+  return {
+    ...analysis,
+    comparisons: analysis.comparisons.map((comparison, index) => ({
+      ...comparison,
+      adjustedPValue: adjusted[index],
+      verdict: comparison.isReference
+        ? "reference"
+        : classify(comparison.interval, {
+            noiseFloor: analysis.noiseFloorSeconds,
+            pValue: adjusted[index],
+          }),
+    })),
+  };
+}
+
 export function sha256(value) {
   return createHash("sha256").update(value).digest("hex");
 }
@@ -87,39 +140,48 @@ export function markdown(metadata, analysis, caches) {
     "",
     `Durations are the warm restore path with a Maven cache already populated, measured inside the job at millisecond resolution. Differences are against \`${REFERENCE}\`.`,
     "",
-    "## Versions",
+    "## Comparable cohorts",
     "",
-    `| Version | Caching | Median (s) | Mean (s) | MAD (s) | vs ${REFERENCE} (s) | 95% CI | p | Verdict |`,
-    "| --- | --- | ---: | ---: | ---: | ---: | --- | ---: | --- |",
+    "The versions have different setup contracts, so they are compared only within behaviorally comparable cohorts. Holm correction controls the family-wise error rate within each cohort. Cross-cohort timings are descriptive and must not be used to rank versions.",
   ];
-  for (const comparison of analysis.comparisons) {
-    const entry = VERSIONS.find((item) => item.arm === comparison.arm);
-    const summary = comparison.summary;
-    const diff = comparison.isReference
-      ? "reference"
-      : comparison.differenceSeconds.toFixed(3);
-    const ci = comparison.interval
-      ? `${comparison.interval.low.toFixed(3)} to ${comparison.interval.high.toFixed(3)}`
-      : "n/a";
-    const p = comparison.pValue === null ? "n/a" : comparison.pValue.toFixed(3);
-    // v1 and v2 do no caching, so they never restore the Maven repository the
-    // other versions spend most of their time on. Their difference is real but
-    // it is a difference in work done, not in how well the same work is done,
-    // and calling it an improvement or a regression would invite the wrong
-    // conclusion.
-    const verdict =
-      entry.caching === "none"
-        ? "not comparable (no caching)"
-        : comparison.verdict;
+  const rawRows = analysis.rawRows ?? [];
+  for (const cohort of COHORTS) {
+    const cohortResult = correctCohort(analyzeCohort(rawRows, cohort));
+    lines.push(
+      "",
+      `### ${cohort.label}`,
+      "",
+      cohort.note,
+      "",
+      `| Version | Median (s) | Mean (s) | MAD (s) | vs ${LABELS.get(cohort.reference)} (s) | 95% CI | Holm-adjusted p | Verdict |`,
+      "| --- | ---: | ---: | ---: | ---: | --- | ---: | --- |",
+    );
+    for (const comparison of cohortResult.comparisons) {
+      const summary = comparison.summary;
+      const diff = comparison.isReference
+        ? "reference"
+        : comparison.differenceSeconds.toFixed(3);
+      const ci = comparison.interval
+        ? `${comparison.interval.low.toFixed(3)} to ${comparison.interval.high.toFixed(3)}`
+        : "n/a";
+      const p =
+        comparison.adjustedPValue === null
+          ? "n/a"
+          : comparison.adjustedPValue.toFixed(3);
+      lines.push(
+        `| ${LABELS.get(comparison.arm)} | ${summary.medianSeconds.toFixed(3)} | ${summary.meanSeconds.toFixed(3)} | ${summary.madSeconds.toFixed(3)} | ${diff} | ${ci} | ${p} | ${comparison.verdict} |`,
+      );
+    }
     lines.push(
-      `| ${LABELS.get(comparison.arm)} | ${entry.caching} | ${summary.medianSeconds.toFixed(3)} | ${summary.meanSeconds.toFixed(3)} | ${summary.madSeconds.toFixed(3)} | ${diff} | ${ci} | ${p} | ${verdict} |`,
+      "",
+      `A/A control (${LABELS.get(cohort.reference)} against itself): **${cohortResult.control.verdict}**${cohortResult.control.interval ? ` at ${formatInterval(cohortResult.control.interval)}` : ""}.`,
     );
   }
   lines.push(
     "",
     `Harness noise floor: ${analysis.noiseFloorSeconds.toFixed(3)}s (median spread between a version's own two slots on one runner). A difference smaller than this is reported as \`within-noise\`; one whose interval includes zero is reported as \`inconclusive\` rather than as a number that looks like a result.`,
     "",
-    `A/A control (\`${REFERENCE}\` against itself) reports **${control.verdict}**${control.interval ? ` at ${formatInterval(control.interval)}` : ""}. A healthy harness reports \`within-noise\` or \`inconclusive\` here; anything else means slot ordering is biasing results and the table above cannot be trusted.`,
+    `A/A control (\`${REFERENCE}\` against itself) reports **${control.verdict}**${control.interval ? ` at ${formatInterval(control.interval)}` : ""}. A healthy harness reports \`within-noise\` or \`inconclusive\` here; anything else means slot ordering is biasing results and the cohort tables above cannot be trusted.`,
     "",
     analysis.droppedRunners.length === 0
       ? "No runner was discarded for a stalled slot."
@@ -183,6 +245,7 @@ export async function main(env = process.env) {
   if (analysis.runners.length === 0) {
     throw new Error("No runner measured every version twice");
   }
+  analysis.rawRows = rows;
 
   const cacheEntries = await allPages(
     `/repos/${owner}/${repo}/actions/caches`,
@@ -209,7 +272,7 @@ export async function main(env = process.env) {
   await mkdir(RESULTS_DIR, { recursive: true });
   await writeFile(
     `${RESULTS_DIR}/results.json`,
-    `${JSON.stringify({ metadata, analysis: { ...analysis, runners: undefined }, caches }, null, 2)}\n`,
+    `${JSON.stringify({ metadata, analysis: { ...analysis, rawRows: undefined, runners: undefined }, caches }, null, 2)}\n`,
   );
   await writeFile(
     `${RESULTS_DIR}/results.csv`,
diff --git a/scripts/report.test.mjs b/scripts/report.test.mjs
index 9a3dc21..f8f364a 100644
--- a/scripts/report.test.mjs
+++ b/scripts/report.test.mjs
@@ -2,7 +2,13 @@ import assert from "node:assert/strict";
 import test from "node:test";
 
 import { analyzeAgainstReference, parseSamples } from "./paired.mjs";
-import { VERSIONS, hashFilesSingle, markdown, sha256 } from "./report.mjs";
+import {
+  COHORTS,
+  VERSIONS,
+  hashFilesSingle,
+  markdown,
+  sha256,
+} from "./report.mjs";
 
 const ARMS = VERSIONS.map((entry) => entry.arm);
 const ORDER = [...ARMS, ...[...ARMS].reverse()];
@@ -91,22 +97,35 @@ test("renders a version table, a control and per-runner medians", () => {
     .map((sample) => sweep(sample, 1 + sample * 0.1, flat))
     .join("\n");
   const analysis = analyzeAgainstReference(parseSamples(csv), ARMS, "main");
+  analysis.rawRows = parseSamples(csv);
   const report = markdown(
     { runId: "1", distribution: "temurin", javaVersion: "17" },
     analysis,
     [{ key: "setup-java-Linux-x64-maven-abc", sizeBytes: 60 * 1024 * 1024 }],
   );
   assert.match(report, /# setup-java version sweep/);
+  assert.match(report, /## Comparable cohorts/);
+  assert.match(report, /### Dependency cache/);
   assert.match(report, /\| v4\.8\.0 \|/);
+  assert.match(report, /Holm-adjusted p/);
   assert.match(report, /A\/A control/);
   assert.match(report, /Harness noise floor/);
   assert.match(report, /## Per-runner medians/);
-  // v1 and v2 skip the dependency restore entirely, so ranking them against
-  // main would compare different amounts of work.
-  assert.match(report, /not comparable \(no caching\)/);
+  assert.match(report, /v1 and v2 do not restore Maven dependencies/);
   assert.match(report, /setup-java-Linux-x64-maven-abc/);
 });
 
+test("defines cohorts with only comparable cache contracts", () => {
+  assert.deepEqual(
+    COHORTS.map((cohort) => cohort.arms),
+    [
+      ["v1", "v2"],
+      ["v3", "v4", "v52"],
+      ["v56", "main"],
+    ],
+  );
+});
+
 test("discards a runner whose slots for one version disagree", () => {
   // Nine well-behaved runners and one whose v4 slots differ by seconds, which
   // means that runner stalled rather than measured. Which version a stall lands
diff --git a/scripts/stats.mjs b/scripts/stats.mjs
index 959a6d2..ce5e203 100644
--- a/scripts/stats.mjs
+++ b/scripts/stats.mjs
@@ -168,6 +168,22 @@ export function pairedPermutationTest(differences, options = {}) {
   return (atLeastAsExtreme + 1) / (iterations + 1);
 }
 
+// Holm's step-down correction controls the family-wise error rate when one
+// report makes several comparisons against the same reference.
+export function holmAdjust(pValues) {
+  const adjusted = new Array(pValues.length).fill(null);
+  const ordered = pValues
+    .map((pValue, index) => ({ pValue, index }))
+    .filter(({ pValue }) => pValue !== null)
+    .sort((a, b) => a.pValue - b.pValue);
+  let maximum = 0;
+  ordered.forEach(({ pValue, index }, rank) => {
+    maximum = Math.max(maximum, Math.min(1, pValue * (ordered.length - rank)));
+    adjusted[index] = maximum;
+  });
+  return adjusted;
+}
+
 // Hodges-Lehmann shift estimate: the median of all pairwise differences. More
 // robust than a difference of medians when samples are small and quantized.
 export function hodgesLehmann(treatment, baseline) {
diff --git a/scripts/stats.test.mjs b/scripts/stats.test.mjs
index b2c7764..17a8bd2 100644
--- a/scripts/stats.test.mjs
+++ b/scripts/stats.test.mjs
@@ -7,6 +7,7 @@ import {
   createRandom,
   differenceInterval,
   formatInterval,
+  holmAdjust,
   hodgesLehmann,
   mean,
   median,
@@ -150,3 +151,8 @@ test("formatInterval labels the interval's own confidence level", () => {
     "0.0s (95% CI -1.0 to 1.0)",
   );
 });
+
+test("Holm adjustment controls a family of p-values", () => {
+  assert.deepEqual(holmAdjust([0.01, 0.04, 0.2]), [0.03, 0.08, 0.2]);
+  assert.deepEqual(holmAdjust([null, 0.01, 0.2]), [null, 0.02, 0.2]);
+});

From 2b3afa4ae0c129d3d9e919fec955c85632883f06 Mon Sep 17 00:00:00 2001
From: Bruno Borges <bruno.borges@microsoft.com>
Date: Wed, 5 Aug 2026 02:09:03 -0400
Subject: [PATCH 17/19] Rank only the versions that do the same work

The previous commit split the sweep into three cohorts, each with its own
reference. That was wrong in two ways.

It deleted the comparison the sweep exists to make. v4 and v5.2 are what users
are actually running, and after the split neither could be compared with main
at all: they sat in a cohort referenced against v5.2, while main sat in another.

It also gave a verdict to a comparison the harness cannot make. v3 keys on
pom.xml, so it restores its own cache entry, and a blob's throughput is fixed
for the life of that entry. The README already says pairing cannot remove that
confound. In run 30979614347 v3 took 3.69s and 3.89s on two runners that ran v4
in 0.50s and 0.52s moments later in the same job, and the cohort report
published that sevenfold gap as a `regression` at p=0.046.

Rank against main again, but rank only v4, v5.2 and v5.6 - the versions that
restore the same seeded entry, so that a difference between them is a difference
in the implementation. Keep Holm's correction, which was the sound half of the
previous commit, and apply it across that family. Measure v1, v2 and v3 as
before and publish their durations, each with the reason it carries no verdict,
because for them a difference in duration is a difference in the workload.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 71c45320-1417-4029-8402-69075d61dac1
---
 README.md               |  14 ++-
 scripts/report.mjs      | 191 ++++++++++++++++++++++++----------------
 scripts/report.test.mjs |  84 ++++++++++--------
 3 files changed, 174 insertions(+), 115 deletions(-)

diff --git a/README.md b/README.md
index a16e0aa..cff8421 100644
--- a/README.md
+++ b/README.md
@@ -45,9 +45,19 @@ Each action version runs with Java 17 on `ubuntu-24.04`:
 
 v1 predates distribution selection and integrated dependency caching. It runs only its native Zulu installer path. v2 supports the Temurin and Microsoft scenarios, but its bundled legacy cache client is rejected by the current Actions cache service. v1 and v2 therefore have no Maven cache storage, and their cold/warm labels are repeated uncached samples.
 
-A seed job compiles Spring PetClinic once to populate a single Maven cache entry. Every measurement runner then sets up all seven versions in one job, in the order `v1..main` followed by `main..v1`, deleting `~/.m2` before each slot. The report keeps the same-runner measurements but compares only behaviorally comparable cohorts, applying Holm correction within each cohort rather than ranking every version against `main`.
+A seed job compiles Spring PetClinic once to populate a single Maven cache entry. Every measurement runner then sets up all seven versions in one job, in the order `v1..main` followed by `main..v1`, deleting `~/.m2` before each slot. The report compares every version against `main` with a paired interval per runner, but it ranks only the versions that do the same work from the same stored blob.
 
-The report has three cohorts: **uncached setup** (`v1`, `v2`), **dependency cache** (`v3`, `v4`, `v5.2`), and **dependency plus wrapper cache** (`v5.6`, `main`). v1/v2 are not comparable with cached versions. v3 predates `cache-dependency-path` and keys on `pom.xml`, so its comparison within the dependency-cache cohort is explicitly caveated; v4 and v5.2 share one controlled cache entry. v5.6 and `main` both restore the dependency and Maven Wrapper caches. Cross-cohort timings are descriptive and must not be used to rank versions.
+**Only comparable versions are ranked.** v4, v5.2, v5.6 and `main` all restore the same seeded entry through `cache-dependency-path`, so a difference between them is a difference in the implementation. They are ranked against `main`, with Holm's step-down correction across that family: they are tested against one reference in one run, so without a correction the chance that one of them clears 0.05 by luck is far above 0.05.
+
+v1, v2 and v3 are measured on the same runners and published, but they carry no verdict, because a verdict would report a difference in the *workload* as though it were a difference in the implementation:
+
+| Version | Why it is not ranked |
+| --- | --- |
+| v1.4.4 | Installs its own JDK and does no dependency caching |
+| v2.5.1 | Its bundled cache client is rejected by the current cache service, so it restores nothing |
+| v3.14.1 | Predates `cache-dependency-path` and keys on `pom.xml`, so it restores its own entry — the blob confound described above, which pairing cannot remove |
+
+v3 is the instructive case. In run 30979614347 it took 3.69 s and 3.89 s on two runners that ran v4 in 0.50 s and 0.52 s moments later in the same job. A sevenfold gap that appears on some runners and not others is blob placement, not code, and ranking it would have published a `regression` verdict for it.
 
 Spring PetClinic and third-party actions are pinned to commits. `setup-java@main` intentionally remains a moving ref so each run evaluates the current upcoming v6 code; the report records the `main` commit observed when it is generated.
 
diff --git a/scripts/report.mjs b/scripts/report.mjs
index d0b87e7..b8c32df 100644
--- a/scripts/report.mjs
+++ b/scripts/report.mjs
@@ -17,54 +17,80 @@ const REFERENCE = "main";
 // Ordered oldest to newest. The warm sweep measures them in this order and then
 // in reverse, so each version's two slots sit symmetrically about the middle of
 // the job.
+//
+// `comparable` records whether a version can be ranked against the reference at
+// all. A version is comparable only when it does the same work as `main` from
+// the same stored blob; anything else is measuring a difference in the workload
+// rather than in the implementation, and a verdict on it would mislead. Those
+// versions are still measured and published, but descriptively.
 export const VERSIONS = [
-  { arm: "v1", label: "v1.4.4", caching: "none" },
-  { arm: "v2", label: "v2.5.1", caching: "none" },
-  { arm: "v3", label: "v3.14.1", caching: "pom.xml" },
-  { arm: "v4", label: "v4.8.0", caching: "cache-dependency-path" },
-  { arm: "v52", label: "v5.2.0", caching: "cache-dependency-path" },
-  { arm: "v56", label: "v5.6.0", caching: "cache-dependency-path + wrapper" },
-  { arm: "main", label: "main", caching: "cache-dependency-path + wrapper" },
-];
-
-const LABELS = new Map(VERSIONS.map((entry) => [entry.arm, entry.label]));
-
-export const COHORTS = [
   {
-    id: "uncached",
-    label: "Uncached setup",
-    arms: ["v1", "v2"],
-    reference: "v2",
-    note: "v1 and v2 do not restore Maven dependencies, so this comparison is not comparable with cached versions.",
+    arm: "v1",
+    label: "v1.4.4",
+    caching: "none",
+    comparable: false,
+    reason:
+      "installs its own JDK and does no dependency caching, so its duration is a different workload rather than the same work done differently",
+  },
+  {
+    arm: "v2",
+    label: "v2.5.1",
+    caching: "none",
+    comparable: false,
+    reason:
+      "its bundled cache client is rejected by the current cache service, so it restores nothing and skips the work the other versions spend their time on",
+  },
+  {
+    arm: "v3",
+    label: "v3.14.1",
+    caching: "pom.xml",
+    comparable: false,
+    reason:
+      "predates `cache-dependency-path` and therefore restores its own cache entry; a stored blob's throughput is fixed for the life of the entry, so this difference is confounded with blob placement and pairing cannot remove it",
   },
   {
-    id: "dependency-cache",
-    label: "Dependency cache",
-    arms: ["v3", "v4", "v52"],
-    reference: "v52",
-    note: "v3 uses its older pom.xml cache-key strategy; compare it with caution. v4 and v5.2 share the controlled cache key.",
+    arm: "v4",
+    label: "v4.8.0",
+    caching: "cache-dependency-path",
+    comparable: true,
   },
   {
-    id: "wrapper-cache",
-    label: "Dependency plus wrapper cache",
-    arms: ["v56", "main"],
-    reference: "main",
-    note: "Both versions restore the dependency and Maven Wrapper caches.",
+    arm: "v52",
+    label: "v5.2.0",
+    caching: "cache-dependency-path",
+    comparable: true,
+  },
+  {
+    arm: "v56",
+    label: "v5.6.0",
+    caching: "cache-dependency-path + wrapper",
+    comparable: true,
+  },
+  {
+    arm: "main",
+    label: "main",
+    caching: "cache-dependency-path + wrapper",
+    comparable: true,
   },
 ];
 
-function analyzeCohort(rows, cohort) {
-  return analyzeAgainstReference(
-    rows.filter((row) => cohort.arms.includes(row.arm)),
-    cohort.arms,
-    cohort.reference,
-  );
-}
+const LABELS = new Map(VERSIONS.map((entry) => [entry.arm, entry.label]));
 
-function correctCohort(analysis) {
+export const COMPARABLE_ARMS = VERSIONS.filter((entry) => entry.comparable).map(
+  (entry) => entry.arm,
+);
+
+// Holm's step-down correction is applied across the comparable family only.
+// Every version in it is tested against the same reference in the same run, so
+// without a correction the chance that at least one of them clears 0.05 by luck
+// is far higher than 0.05. The descriptive versions are excluded because they
+// carry no verdict to correct.
+export function correctFamily(analysis) {
   const adjusted = holmAdjust(
     analysis.comparisons.map((comparison) =>
-      comparison.isReference ? null : comparison.pValue,
+      comparison.isReference || !isComparable(comparison.arm)
+        ? null
+        : comparison.pValue,
     ),
   );
   return {
@@ -74,14 +100,20 @@ function correctCohort(analysis) {
       adjustedPValue: adjusted[index],
       verdict: comparison.isReference
         ? "reference"
-        : classify(comparison.interval, {
-            noiseFloor: analysis.noiseFloorSeconds,
-            pValue: adjusted[index],
-          }),
+        : !isComparable(comparison.arm)
+          ? "not comparable"
+          : classify(comparison.interval, {
+              noiseFloor: analysis.noiseFloorSeconds,
+              pValue: adjusted[index],
+            }),
     })),
   };
 }
 
+function isComparable(arm) {
+  return VERSIONS.find((entry) => entry.arm === arm)?.comparable === true;
+}
+
 export function sha256(value) {
   return createHash("sha256").update(value).digest("hex");
 }
@@ -138,50 +170,60 @@ export function markdown(metadata, analysis, caches) {
     "order places each version's two slots symmetrically about the middle of the job,",
     "so drift across the job cancels.",
     "",
-    `Durations are the warm restore path with a Maven cache already populated, measured inside the job at millisecond resolution. Differences are against \`${REFERENCE}\`.`,
+    `Durations are the warm restore path with a Maven cache already populated, measured inside the job at millisecond resolution.`,
+    "",
+    "## Versions ranked against `main`",
     "",
-    "## Comparable cohorts",
+    "Only versions that do the same work as `main` from the same stored cache entry are ranked. Holm's step-down correction is applied across this family, because every version in it is tested against the same reference in the same run and without it the chance that one clears 0.05 by luck is far above 0.05.",
     "",
-    "The versions have different setup contracts, so they are compared only within behaviorally comparable cohorts. Holm correction controls the family-wise error rate within each cohort. Cross-cohort timings are descriptive and must not be used to rank versions.",
+    `| Version | Caching | Median (s) | Mean (s) | MAD (s) | vs ${REFERENCE} (s) | 95% CI | Holm-adjusted p | Verdict |`,
+    "| --- | --- | ---: | ---: | ---: | ---: | --- | ---: | --- |",
   ];
-  const rawRows = analysis.rawRows ?? [];
-  for (const cohort of COHORTS) {
-    const cohortResult = correctCohort(analyzeCohort(rawRows, cohort));
+  const corrected = correctFamily(analysis);
+  const rankable = corrected.comparisons.filter((comparison) =>
+    COMPARABLE_ARMS.includes(comparison.arm),
+  );
+  const descriptive = corrected.comparisons.filter(
+    (comparison) => !COMPARABLE_ARMS.includes(comparison.arm),
+  );
+  for (const comparison of rankable) {
+    const entry = VERSIONS.find((item) => item.arm === comparison.arm);
+    const summary = comparison.summary;
+    const diff = comparison.isReference
+      ? "reference"
+      : comparison.differenceSeconds.toFixed(3);
+    const ci = comparison.interval
+      ? `${comparison.interval.low.toFixed(3)} to ${comparison.interval.high.toFixed(3)}`
+      : "n/a";
+    const p =
+      comparison.adjustedPValue === null
+        ? "n/a"
+        : comparison.adjustedPValue.toFixed(3);
     lines.push(
-      "",
-      `### ${cohort.label}`,
-      "",
-      cohort.note,
-      "",
-      `| Version | Median (s) | Mean (s) | MAD (s) | vs ${LABELS.get(cohort.reference)} (s) | 95% CI | Holm-adjusted p | Verdict |`,
-      "| --- | ---: | ---: | ---: | ---: | --- | ---: | --- |",
+      `| ${LABELS.get(comparison.arm)} | ${entry.caching} | ${summary.medianSeconds.toFixed(3)} | ${summary.meanSeconds.toFixed(3)} | ${summary.madSeconds.toFixed(3)} | ${diff} | ${ci} | ${p} | ${comparison.verdict} |`,
     );
-    for (const comparison of cohortResult.comparisons) {
-      const summary = comparison.summary;
-      const diff = comparison.isReference
-        ? "reference"
-        : comparison.differenceSeconds.toFixed(3);
-      const ci = comparison.interval
-        ? `${comparison.interval.low.toFixed(3)} to ${comparison.interval.high.toFixed(3)}`
-        : "n/a";
-      const p =
-        comparison.adjustedPValue === null
-          ? "n/a"
-          : comparison.adjustedPValue.toFixed(3);
-      lines.push(
-        `| ${LABELS.get(comparison.arm)} | ${summary.medianSeconds.toFixed(3)} | ${summary.meanSeconds.toFixed(3)} | ${summary.madSeconds.toFixed(3)} | ${diff} | ${ci} | ${p} | ${comparison.verdict} |`,
-      );
-    }
+  }
+  lines.push(
+    "",
+    "## Measured but not ranked",
+    "",
+    "These versions are measured on the same runners and in the same order, but they do not do the same work as `main`, so a verdict on them would report a difference in the workload as though it were a difference in the implementation. Their durations are published to show what the ranked versions are spending their time on.",
+    "",
+    "| Version | Caching | Median (s) | Mean (s) | MAD (s) | vs `main` (s) | Why it is not ranked |",
+    "| --- | --- | ---: | ---: | ---: | ---: | --- |",
+  );
+  for (const comparison of descriptive) {
+    const entry = VERSIONS.find((item) => item.arm === comparison.arm);
+    const summary = comparison.summary;
     lines.push(
-      "",
-      `A/A control (${LABELS.get(cohort.reference)} against itself): **${cohortResult.control.verdict}**${cohortResult.control.interval ? ` at ${formatInterval(cohortResult.control.interval)}` : ""}.`,
+      `| ${LABELS.get(comparison.arm)} | ${entry.caching} | ${summary.medianSeconds.toFixed(3)} | ${summary.meanSeconds.toFixed(3)} | ${summary.madSeconds.toFixed(3)} | ${comparison.differenceSeconds.toFixed(3)} | ${entry.reason} |`,
     );
   }
   lines.push(
     "",
     `Harness noise floor: ${analysis.noiseFloorSeconds.toFixed(3)}s (median spread between a version's own two slots on one runner). A difference smaller than this is reported as \`within-noise\`; one whose interval includes zero is reported as \`inconclusive\` rather than as a number that looks like a result.`,
     "",
-    `A/A control (\`${REFERENCE}\` against itself) reports **${control.verdict}**${control.interval ? ` at ${formatInterval(control.interval)}` : ""}. A healthy harness reports \`within-noise\` or \`inconclusive\` here; anything else means slot ordering is biasing results and the cohort tables above cannot be trusted.`,
+    `A/A control (\`${REFERENCE}\` against itself) reports **${control.verdict}**${control.interval ? ` at ${formatInterval(control.interval)}` : ""}. A healthy harness reports \`within-noise\` or \`inconclusive\` here; anything else means slot ordering is biasing results and the table above cannot be trusted.`,
     "",
     analysis.droppedRunners.length === 0
       ? "No runner was discarded for a stalled slot."
@@ -204,7 +246,7 @@ export function markdown(metadata, analysis, caches) {
     "",
     "## Caches",
     "",
-    "Every caching version restores the same Maven entry, so the stored blob cannot bias the comparison. v3 predates `cache-dependency-path` and keys on `pom.xml`, so it necessarily uses its own entry; treat its difference with more caution than the rest.",
+    "Every ranked version restores the same Maven entry, so the stored blob cannot bias the comparison between them. v3 predates `cache-dependency-path` and keys on `pom.xml`, so it necessarily restores its own entry; a blob's throughput is fixed for the life of the entry and identical on every runner, which is why v3 is measured but not ranked.",
     "",
     "v1 and v2 have no caching at all, so they skip the dependency restore entirely and their durations are not comparable with the rest. They are measured to show what the caching versions are spending their time on, not to be ranked against them.",
     "",
@@ -245,7 +287,6 @@ export async function main(env = process.env) {
   if (analysis.runners.length === 0) {
     throw new Error("No runner measured every version twice");
   }
-  analysis.rawRows = rows;
 
   const cacheEntries = await allPages(
     `/repos/${owner}/${repo}/actions/caches`,
@@ -272,7 +313,7 @@ export async function main(env = process.env) {
   await mkdir(RESULTS_DIR, { recursive: true });
   await writeFile(
     `${RESULTS_DIR}/results.json`,
-    `${JSON.stringify({ metadata, analysis: { ...analysis, rawRows: undefined, runners: undefined }, caches }, null, 2)}\n`,
+    `${JSON.stringify({ metadata, analysis: { ...analysis, runners: undefined }, caches }, null, 2)}\n`,
   );
   await writeFile(
     `${RESULTS_DIR}/results.csv`,
diff --git a/scripts/report.test.mjs b/scripts/report.test.mjs
index f8f364a..f2ed6e0 100644
--- a/scripts/report.test.mjs
+++ b/scripts/report.test.mjs
@@ -3,7 +3,7 @@ import test from "node:test";
 
 import { analyzeAgainstReference, parseSamples } from "./paired.mjs";
 import {
-  COHORTS,
+  COMPARABLE_ARMS,
   VERSIONS,
   hashFilesSingle,
   markdown,
@@ -92,62 +92,70 @@ test("reports the reference against itself without a verdict", () => {
   assert.equal(reference.interval, null);
 });
 
-test("renders a version table, a control and per-runner medians", () => {
+test("renders a ranked table, a not-ranked table and per-runner medians", () => {
   const csv = [1, 2, 3, 4]
     .map((sample) => sweep(sample, 1 + sample * 0.1, flat))
     .join("\n");
   const analysis = analyzeAgainstReference(parseSamples(csv), ARMS, "main");
-  analysis.rawRows = parseSamples(csv);
   const report = markdown(
     { runId: "1", distribution: "temurin", javaVersion: "17" },
     analysis,
     [{ key: "setup-java-Linux-x64-maven-abc", sizeBytes: 60 * 1024 * 1024 }],
   );
   assert.match(report, /# setup-java version sweep/);
-  assert.match(report, /## Comparable cohorts/);
-  assert.match(report, /### Dependency cache/);
-  assert.match(report, /\| v4\.8\.0 \|/);
+  assert.match(report, /## Versions ranked against `main`/);
+  assert.match(report, /## Measured but not ranked/);
   assert.match(report, /Holm-adjusted p/);
   assert.match(report, /A\/A control/);
   assert.match(report, /Harness noise floor/);
   assert.match(report, /## Per-runner medians/);
-  assert.match(report, /v1 and v2 do not restore Maven dependencies/);
   assert.match(report, /setup-java-Linux-x64-maven-abc/);
 });
 
-test("defines cohorts with only comparable cache contracts", () => {
-  assert.deepEqual(
-    COHORTS.map((cohort) => cohort.arms),
-    [
-      ["v1", "v2"],
-      ["v3", "v4", "v52"],
-      ["v56", "main"],
-    ],
-  );
+test("ranks only versions that do the same work from the same blob", () => {
+  // v1 and v2 restore nothing and v3 restores its own cache entry, so none of
+  // them measures the same work as main. Ranking them would report a difference
+  // in the workload as though it were a difference in the implementation.
+  assert.deepEqual(COMPARABLE_ARMS, ["v4", "v52", "v56", "main"]);
+  for (const arm of ["v1", "v2", "v3"]) {
+    const entry = VERSIONS.find((item) => item.arm === arm);
+    assert.equal(entry.comparable, false);
+    assert.ok(entry.reason.length > 0);
+  }
 });
 
-test("discards a runner whose slots for one version disagree", () => {
-  // Nine well-behaved runners and one whose v4 slots differ by seconds, which
-  // means that runner stalled rather than measured. Which version a stall lands
-  // on is arbitrary, so it must not reach the comparison.
-  const rows = [1, 2, 3, 4, 5, 6, 7, 8, 9].map((sample) =>
-    sweep(sample, 1, flat),
-  );
-  const stalled = ORDER.map((arm, index) => {
-    const slot = index + 1;
-    const elapsed = arm === "v4" && slot === 4 ? 12000 : 3000;
-    return `"10","${arm}","${slot}","${elapsed}"`;
-  }).join("\n");
-  const analysis = analyzeAgainstReference(
-    parseSamples([...rows, stalled].join("\n")),
-    ARMS,
-    "main",
+test("never gives a verdict to a version it cannot rank", () => {
+  // v3 is made dramatically slower, which on a ranked version would read as a
+  // regression. Because its cache entry is its own, that difference is
+  // confounded with blob placement and must not become a verdict.
+  const perVersion = { ...flat, v3: 9000 };
+  const csv = [1, 2, 3, 4, 5, 6]
+    .map((sample) => sweep(sample, 1, perVersion))
+    .join("\n");
+  const analysis = analyzeAgainstReference(parseSamples(csv), ARMS, "main");
+  const report = markdown(
+    { runId: "1", distribution: "temurin", javaVersion: "17" },
+    analysis,
+    [],
   );
-  assert.deepEqual(
-    analysis.droppedRunners.map((entry) => entry.sample),
-    [10],
+  const [ranked, notRanked] = report.split("## Measured but not ranked");
+  assert.doesNotMatch(ranked, /v3\.14\.1/);
+  assert.match(notRanked, /v3\.14\.1/);
+  assert.doesNotMatch(report, /\| v3\.14\.1 \|[^\n]*regression/);
+});
+
+test("applies Holm correction across the ranked family", () => {
+  // Three versions are tested against main in one run. A raw p-value near the
+  // boundary must not survive the correction as a verdict.
+  const perVersion = { ...flat, v4: 3040, v52: 3040, v56: 3040 };
+  const csv = [1, 2, 3, 4, 5, 6, 7, 8]
+    .map((sample) => sweep(sample, 1 + sample * 0.05, perVersion))
+    .join("\n");
+  const analysis = analyzeAgainstReference(parseSamples(csv), ARMS, "main");
+  const report = markdown(
+    { runId: "1", distribution: "temurin", javaVersion: "17" },
+    analysis,
+    [],
   );
-  const v4 = analysis.comparisons.find((entry) => entry.arm === "v4");
-  assert.ok(Math.abs(v4.differenceSeconds) < 1e-9);
-  assert.equal(v4.verdict, "inconclusive");
+  assert.match(report, /Holm's step-down correction/);
 });

From a967accde385dba64073528dec4fcb32349a2502 Mon Sep 17 00:00:00 2001
From: Bruno Borges <bruno.borges@microsoft.com>
Date: Wed, 5 Aug 2026 02:22:02 -0400
Subject: [PATCH 18/19] Report the sweep as main against each version, not the
 reverse

main is the newest code, so calling v5.6.0 a `regression` for being slower than
it states the finding backwards. Nothing regressed in v5.6.0; it shipped, and
then main got faster. The table was reading the difference in the direction that
made the older code look at fault for the improvement that came after it.

The two-arm workflows already have this right: they difference candidate minus
baseline, so the verdict describes the code under test. The sweep is the same
question with more baselines, so difference main minus each version and let the
verdict describe main. v5.6.0 now reads as a 1.303s `improvement` in main rather
than a 1.303s `regression` in v5.6.0, which is the same measurement said the
right way round.

Pin the direction with a test, because the sign is easy to flip back and the
resulting table is wrong in a way that still looks plausible.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 71c45320-1417-4029-8402-69075d61dac1
---
 README.md               |  8 +++++---
 scripts/paired.mjs      | 16 ++++++++++++----
 scripts/report.mjs      | 10 ++++++----
 scripts/report.test.mjs | 33 +++++++++++++++++++++++++++++----
 4 files changed, 52 insertions(+), 15 deletions(-)

diff --git a/README.md b/README.md
index cff8421..e685d49 100644
--- a/README.md
+++ b/README.md
@@ -45,9 +45,11 @@ Each action version runs with Java 17 on `ubuntu-24.04`:
 
 v1 predates distribution selection and integrated dependency caching. It runs only its native Zulu installer path. v2 supports the Temurin and Microsoft scenarios, but its bundled legacy cache client is rejected by the current Actions cache service. v1 and v2 therefore have no Maven cache storage, and their cold/warm labels are repeated uncached samples.
 
-A seed job compiles Spring PetClinic once to populate a single Maven cache entry. Every measurement runner then sets up all seven versions in one job, in the order `v1..main` followed by `main..v1`, deleting `~/.m2` before each slot. The report compares every version against `main` with a paired interval per runner, but it ranks only the versions that do the same work from the same stored blob.
+A seed job compiles Spring PetClinic once to populate a single Maven cache entry. Every measurement runner then sets up all seven versions in one job, in the order `v1..main` followed by `main..v1`, deleting `~/.m2` before each slot. The report compares `main` with every version using a paired interval per runner, but it ranks only the versions that do the same work from the same stored blob.
 
-**Only comparable versions are ranked.** v4, v5.2, v5.6 and `main` all restore the same seeded entry through `cache-dependency-path`, so a difference between them is a difference in the implementation. They are ranked against `main`, with Holm's step-down correction across that family: they are tested against one reference in one run, so without a correction the chance that one of them clears 0.05 by luck is far above 0.05.
+**`main` is the thing under test, not the thing being ranked.** `main` is the newest code, and every released version is a baseline it is measured against, exactly as the two-arm workflows measure a candidate against a baseline. So the reported difference is `main` minus the version, and the verdict describes `main`: an `improvement` means `main` is faster than that version. Differencing the other way would label a released version a `regression` for being slower than the code that succeeded it, which inverts what was actually measured — the finding there is that `main` got faster.
+
+**Only comparable versions are ranked.** v4, v5.2, v5.6 and `main` all restore the same seeded entry through `cache-dependency-path`, so a difference between them is a difference in the implementation. `main` is ranked against each of them, with Holm's step-down correction across that family: it is tested against every one of them in one run, so without a correction the chance that one comparison clears 0.05 by luck is far above 0.05.
 
 v1, v2 and v3 are measured on the same runners and published, but they carry no verdict, because a verdict would report a difference in the *workload* as though it were a difference in the implementation:
 
@@ -57,7 +59,7 @@ v1, v2 and v3 are measured on the same runners and published, but they carry no
 | v2.5.1 | Its bundled cache client is rejected by the current cache service, so it restores nothing |
 | v3.14.1 | Predates `cache-dependency-path` and keys on `pom.xml`, so it restores its own entry — the blob confound described above, which pairing cannot remove |
 
-v3 is the instructive case. In run 30979614347 it took 3.69 s and 3.89 s on two runners that ran v4 in 0.50 s and 0.52 s moments later in the same job. A sevenfold gap that appears on some runners and not others is blob placement, not code, and ranking it would have published a `regression` verdict for it.
+v3 is the instructive case. In run 30979614347 it took 3.69 s and 3.89 s on two runners that ran v4 in 0.50 s and 0.52 s moments later in the same job. A sevenfold gap that appears on some runners and not others is blob placement, not code, and ranking it would have published a verdict for it.
 
 Spring PetClinic and third-party actions are pinned to commits. `setup-java@main` intentionally remains a moving ref so each run evaluates the current upcoming v6 code; the report records the `main` commit observed when it is generated.
 
diff --git a/scripts/paired.mjs b/scripts/paired.mjs
index 8b82d24..eb8dbca 100644
--- a/scripts/paired.mjs
+++ b/scripts/paired.mjs
@@ -228,9 +228,17 @@ export function analyzePairs(
   };
 }
 
-// Compares every arm against a reference arm, one paired difference per runner.
-// Used where the workflow measures more than two implementations in the same
-// job, such as the version sweep.
+// Compares a reference arm against every other arm, one paired difference per
+// runner. Used where the workflow measures more than two implementations in the
+// same job, such as the version sweep.
+//
+// The difference is reference minus arm, so the reference plays the same role
+// the candidate plays in the two-arm workflows: it is the code under test and
+// each other arm is a baseline it is measured against. In the version sweep the
+// reference is `main`, which is the newest code, so this is the direction that
+// makes the verdicts mean what they say. Differencing the other way would call
+// a released version a `regression` for being slower than the code that
+// succeeded it, when what has been measured is `main` being faster.
 export function analyzeAgainstReference(rows, arms, reference) {
   const allRunners = groupByRunner(rows, arms);
   // Same filter as the two-arm workflows, applied across every arm: a runner
@@ -266,7 +274,7 @@ export function analyzeAgainstReference(rows, arms, reference) {
   const comparisons = arms.map((arm, index) => {
     const values = perArm.get(arm);
     const differences = values.map(
-      (value, runner) => value - referenceValues[runner],
+      (value, runner) => referenceValues[runner] - value,
     );
     const isReference = arm === reference;
     const interval = isReference
diff --git a/scripts/report.mjs b/scripts/report.mjs
index b8c32df..a79be12 100644
--- a/scripts/report.mjs
+++ b/scripts/report.mjs
@@ -172,11 +172,13 @@ export function markdown(metadata, analysis, caches) {
     "",
     `Durations are the warm restore path with a Maven cache already populated, measured inside the job at millisecond resolution.`,
     "",
-    "## Versions ranked against `main`",
+    "## How `main` compares with each released version",
     "",
-    "Only versions that do the same work as `main` from the same stored cache entry are ranked. Holm's step-down correction is applied across this family, because every version in it is tested against the same reference in the same run and without it the chance that one clears 0.05 by luck is far above 0.05.",
+    "`main` is the newest code, so it is the thing under test and each released version is a baseline it is measured against. Every number and verdict below describes **`main`**: a negative difference and an `improvement` verdict mean `main` is faster than that version.",
     "",
-    `| Version | Caching | Median (s) | Mean (s) | MAD (s) | vs ${REFERENCE} (s) | 95% CI | Holm-adjusted p | Verdict |`,
+    "Only versions that do the same work as `main` from the same stored cache entry are ranked. Holm's step-down correction is applied across this family, because `main` is tested against every one of them in the same run and without it the chance that one clears 0.05 by luck is far above 0.05.",
+    "",
+    `| Compared with | Caching | Median (s) | Mean (s) | MAD (s) | \`${REFERENCE}\` vs it (s) | 95% CI | Holm-adjusted p | Verdict for \`${REFERENCE}\` |`,
     "| --- | --- | ---: | ---: | ---: | ---: | --- | ---: | --- |",
   ];
   const corrected = correctFamily(analysis);
@@ -209,7 +211,7 @@ export function markdown(metadata, analysis, caches) {
     "",
     "These versions are measured on the same runners and in the same order, but they do not do the same work as `main`, so a verdict on them would report a difference in the workload as though it were a difference in the implementation. Their durations are published to show what the ranked versions are spending their time on.",
     "",
-    "| Version | Caching | Median (s) | Mean (s) | MAD (s) | vs `main` (s) | Why it is not ranked |",
+    "| Version | Caching | Median (s) | Mean (s) | MAD (s) | `main` vs it (s) | Why it is not ranked |",
     "| --- | --- | ---: | ---: | ---: | ---: | --- |",
   );
   for (const comparison of descriptive) {
diff --git a/scripts/report.test.mjs b/scripts/report.test.mjs
index f2ed6e0..77de817 100644
--- a/scripts/report.test.mjs
+++ b/scripts/report.test.mjs
@@ -53,7 +53,8 @@ test("keeps only runners that measured every version twice", () => {
 
 test("removes between-runner speed differences", () => {
   // Runners differ by up to 3x, and every version is 500ms slower than main.
-  // Pairing within a runner must recover 500ms regardless of the spread.
+  // Pairing within a runner must recover 500ms regardless of the spread. The
+  // difference is main minus the version, so main being faster reads negative.
   const perVersion = {
     ...flat,
     v1: 3500,
@@ -68,8 +69,32 @@ test("removes between-runner speed differences", () => {
     .join("\n");
   const analysis = analyzeAgainstReference(parseSamples(csv), ARMS, "main");
   const v4 = analysis.comparisons.find((entry) => entry.arm === "v4");
-  assert.ok(Math.abs(v4.differenceSeconds - 1.125) < 0.001);
-  assert.equal(v4.verdict, "regression");
+  assert.ok(Math.abs(v4.differenceSeconds + 1.125) < 0.001);
+  assert.equal(v4.verdict, "improvement");
+});
+
+// main is the newest code. Reporting a released version as a `regression` for
+// being slower than the code that succeeded it inverts what was measured, so
+// the direction is pinned here rather than left to the reader of the table.
+test("credits main when main is the faster one", () => {
+  const perVersion = { ...flat, v4: 4250, v52: 4250, v56: 4250 };
+  const csv = [1, 2, 3, 4, 5, 6]
+    .map((sample) => sweep(sample, 0.5 + sample * 0.5, perVersion))
+    .join("\n");
+  const analysis = analyzeAgainstReference(parseSamples(csv), ARMS, "main");
+  const v56 = analysis.comparisons.find((entry) => entry.arm === "v56");
+  assert.ok(
+    v56.differenceSeconds < 0,
+    "main is faster, so the difference must be negative",
+  );
+  assert.equal(v56.verdict, "improvement");
+  const rendered = markdown(
+    { runId: "1", distribution: "temurin", javaVersion: "17" },
+    analysis,
+    [],
+  );
+  assert.match(rendered, /Verdict for `main`/);
+  assert.doesNotMatch(rendered, /regression/);
 });
 
 test("cancels linear drift across the job", () => {
@@ -103,7 +128,7 @@ test("renders a ranked table, a not-ranked table and per-runner medians", () =>
     [{ key: "setup-java-Linux-x64-maven-abc", sizeBytes: 60 * 1024 * 1024 }],
   );
   assert.match(report, /# setup-java version sweep/);
-  assert.match(report, /## Versions ranked against `main`/);
+  assert.match(report, /## How `main` compares with each released version/);
   assert.match(report, /## Measured but not ranked/);
   assert.match(report, /Holm-adjusted p/);
   assert.match(report, /A\/A control/);

From e1cb3480a0f9bbe0e997d9f093794cd53073b3b6 Mon Sep 17 00:00:00 2001
From: Bruno Borges <brborges@microsoft.com>
Date: Wed, 5 Aug 2026 02:56:24 -0400
Subject: [PATCH 19/19] Ask each benchmark a question it can answer

The Maven configuration workflow was called a warm path and never had a seed
job, so its synthetic pom hashed to a key nothing had stored and every "warm"
restore was a miss. Its three cache profiles were three variations on a failed
lookup. The focused workflow, meanwhile, compared the same two refs on the same
warm restore as the version sweep.

Between them the benchmarks measured one quantity, repeatedly, and it is mostly
not a quantity setup-java controls: the transfer is handed to @actions/cache, so
a large fixture times the network. Identical code varied ninefold across runners
while the effect under test was a third of that spread.

Reorganise them by question. Cache value builds PetClinic for real, cached
against uncached, to establish the scale everything else is a fraction of. Cache
key stability asserts the key rather than timing it, because a key that moves
when it should not costs a whole miss. Action overhead keeps the configuration
matrix but seeds a megabyte-sized entry so restores genuinely hit and the
action's own work is what remains; its four cache profiles are now nested levels
that difference into a decomposition. Transfer overlap is the same measurement
at 160 MiB, where concurrency has something to hide. Cache save measures what
the first run pays, which nothing measured before.

Every action-overhead slot records cache-hit, and the report refuses to be read
normally when a profile did not do the work its name claims.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 71c45320-1417-4029-8402-69075d61dac1
---
 .github/workflows/action-overhead.yml         | 311 +++++++++++++
 .github/workflows/cache-key-stability.yml     | 196 ++++++++
 .github/workflows/cache-save.yml              | 176 +++++++
 .github/workflows/cache-value.yml             | 270 +++++++++++
 .../maven-configuration-warm-path.yml         | 202 --------
 ...cache-restore.yml => transfer-overlap.yml} |  74 +--
 README.md                                     | 128 ++++--
 scripts/action-overhead.sh                    | 177 +++++++
 scripts/benchmark-maven-configuration.sh      | 163 -------
 scripts/cache-key-stability.sh                |  77 ++++
 scripts/cache-save.sh                         | 260 +++++++++++
 scripts/cache-value.sh                        |  83 ++++
 scripts/check-cache-keys.mjs                  | 255 +++++++++++
 scripts/check-cache-keys.test.mjs             | 118 +++++
 scripts/report-action-overhead.mjs            | 433 ++++++++++++++++++
 scripts/report-action-overhead.test.mjs       | 197 ++++++++
 scripts/report-cache-save.mjs                 | 215 +++++++++
 scripts/report-cache-save.test.mjs            | 172 +++++++
 scripts/report-cache-value.mjs                | 213 +++++++++
 scripts/report-cache-value.test.mjs           |  63 +++
 scripts/report-maven-configuration.mjs        | 195 --------
 scripts/report-maven-configuration.test.mjs   |  92 ----
 ...ocused.mjs => report-transfer-overlap.mjs} |  18 +-
 ...t.mjs => report-transfer-overlap.test.mjs} |   4 +-
 ...d-cache-restore.sh => transfer-overlap.sh} |  10 +-
 25 files changed, 3357 insertions(+), 745 deletions(-)
 create mode 100644 .github/workflows/action-overhead.yml
 create mode 100644 .github/workflows/cache-key-stability.yml
 create mode 100644 .github/workflows/cache-save.yml
 create mode 100644 .github/workflows/cache-value.yml
 delete mode 100644 .github/workflows/maven-configuration-warm-path.yml
 rename .github/workflows/{focused-cache-restore.yml => transfer-overlap.yml} (80%)
 create mode 100644 scripts/action-overhead.sh
 delete mode 100755 scripts/benchmark-maven-configuration.sh
 create mode 100644 scripts/cache-key-stability.sh
 create mode 100755 scripts/cache-save.sh
 create mode 100644 scripts/cache-value.sh
 create mode 100644 scripts/check-cache-keys.mjs
 create mode 100644 scripts/check-cache-keys.test.mjs
 create mode 100644 scripts/report-action-overhead.mjs
 create mode 100644 scripts/report-action-overhead.test.mjs
 create mode 100644 scripts/report-cache-save.mjs
 create mode 100644 scripts/report-cache-save.test.mjs
 create mode 100644 scripts/report-cache-value.mjs
 create mode 100644 scripts/report-cache-value.test.mjs
 delete mode 100644 scripts/report-maven-configuration.mjs
 delete mode 100644 scripts/report-maven-configuration.test.mjs
 rename scripts/{report-focused.mjs => report-transfer-overlap.mjs} (93%)
 rename scripts/{report-focused.test.mjs => report-transfer-overlap.test.mjs} (98%)
 rename scripts/{focused-cache-restore.sh => transfer-overlap.sh} (83%)

diff --git a/.github/workflows/action-overhead.yml b/.github/workflows/action-overhead.yml
new file mode 100644
index 0000000..548fa38
--- /dev/null
+++ b/.github/workflows/action-overhead.yml
@@ -0,0 +1,311 @@
+name: Action overhead
+
+on:
+  workflow_dispatch:
+    inputs:
+      setup-java-repository:
+        description: Repository containing the setup-java action
+        required: true
+        default: actions/setup-java
+        type: string
+      baseline-ref:
+        description: Git ref containing the baseline implementation
+        required: true
+        default: main
+        type: string
+      candidate-ref:
+        description: Git ref containing the candidate implementation
+        required: true
+        default: main
+        type: string
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    shell: bash
+
+concurrency:
+  group: action-overhead
+  cancel-in-progress: false
+
+env:
+  # Shared by the seed job and by every `maven-hit` slot, so that all of them
+  # compute the same cache key and restore the same stored entry. Scoping it to
+  # the run keeps concurrent runs from reusing each other's blob, whose placement
+  # in the cache service fixes its download throughput for its whole life.
+  SEEDED_IDENTITY: action-overhead-seed-${{ github.run_id }}
+
+# What does setup-java's own code cost?
+#
+# Every other scenario in this repository is dominated by how fast a blob moves
+# across the network, which setup-java does not control: it hands the transfer to
+# `@actions/cache`. This one deliberately uses a cache entry of about a megabyte
+# so the transfer term is small, leaving the action's own work — resolving a
+# distribution, computing a cache key, writing settings and toolchains, and the
+# bookkeeping around a restore — as what is actually being timed.
+#
+# The four cache profiles are levels of a decomposition rather than competing
+# options:
+#
+#   none        the action never touches the cache code at all
+#   maven-miss  it computes a key and asks the service, and is told no
+#   maven-hit   it computes a key, is told yes, and unpacks a small entry
+#   gradle-miss the same as maven-miss through the other package manager
+#
+# Reading the differences between those levels says where the time goes. The old
+# version of this workflow had no seed job, so its "warm path" was three
+# variations on a cache miss and it could not see the restore path at all.
+jobs:
+  seed:
+    name: Seed ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-15-intel]
+    steps:
+      - name: Check out benchmark repository
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          persist-credentials: false
+      # Seeding with the baseline is what makes both arms restore one shared
+      # entry. It relies on the two refs computing the same key for the same
+      # tree, which is exactly what the cache key stability workflow asserts; if
+      # that ever stops holding, the recorded cache-hit flags make it visible
+      # here rather than silently turning this into a miss-versus-hit comparison.
+      - name: Check out baseline setup-java
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          repository: ${{ inputs.setup-java-repository }}
+          path: baseline
+          persist-credentials: false
+          ref: ${{ inputs.baseline-ref }}
+      - name: Build the fixture local repository
+        run: bash scripts/action-overhead.sh seed-fixture "$SEEDED_IDENTITY"
+      - name: Store it
+        id: store
+        uses: ./baseline
+        timeout-minutes: 10
+        with:
+          distribution: temurin
+          java-version: "21"
+          cache: maven
+          cache-dependency-path: benchmark/pom.xml
+          settings-path: benchmark-maven-home
+      # The key is a hash, so nothing about the entry's name identifies the run
+      # that created it. Recording what the action reports is the only way the
+      # report job can delete exactly the entries this run added instead of
+      # guessing at a prefix or leaving them to age out.
+      - name: Record the seeded cache key
+        run: |
+          mkdir -p .benchmark-results
+          printf '%s\n' "${{ steps.store.outputs.cache-primary-key }}" \
+            > ".benchmark-results/seeded-key-${{ matrix.os }}.txt"
+      - name: Upload the seeded cache key
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: action-overhead-seed-key-${{ matrix.os }}
+          path: .benchmark-results/
+          include-hidden-files: true
+          if-no-files-found: error
+          retention-days: 30
+
+  benchmark:
+    name: ${{ matrix.os }} ${{ matrix.cache }} ${{ matrix.layout }}
+    needs: seed
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 25
+    strategy:
+      fail-fast: false
+      # The cache service is the shared resource these jobs contend for, and
+      # contention shows up as a stalled restore rather than a slow one. Keeping
+      # the matrix narrow costs wall-clock time and buys measurements that are
+      # about setup-java instead of about the queue in front of it.
+      max-parallel: 6
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-15-intel]
+        cache: [none, maven-miss, maven-hit, gradle-miss]
+        # Two ends of the range rather than the full cross of versions and
+        # toolchains. The cross doubled the job count to separate two effects
+        # that were each smaller than the run-to-run spread.
+        layout: [simple, complex]
+        include:
+          - layout: simple
+            java-version: "21"
+            toolchains: empty
+          - layout: complex
+            java-version: |
+              17
+              21
+            toolchains: existing
+    env:
+      RESULTS: .benchmark-results/action-overhead-${{ matrix.os }}-${{ matrix.cache }}-${{ matrix.layout }}.csv
+      # A miss has to be a real miss. Scoping the identity to this job guarantees
+      # nothing has ever stored that key.
+      MISS_IDENTITY: action-overhead-miss-${{ github.run_id }}-${{ matrix.os }}-${{ matrix.cache }}-${{ matrix.layout }}
+    steps:
+      - name: Check out benchmark repository
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          persist-credentials: false
+      - name: Check out baseline setup-java
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          repository: ${{ inputs.setup-java-repository }}
+          path: baseline
+          persist-credentials: false
+          ref: ${{ inputs.baseline-ref }}
+      - name: Check out candidate setup-java
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          repository: ${{ inputs.setup-java-repository }}
+          path: candidate
+          persist-credentials: false
+          ref: ${{ inputs.candidate-ref }}
+      - name: Record setup dist sizes
+        run: |
+          bash scripts/action-overhead.sh record-size baseline baseline
+          bash scripts/action-overhead.sh record-size candidate candidate
+
+      - name: Resolve the cache identity for this profile
+        run: |
+          if [ "${{ matrix.cache }}" = "maven-hit" ]; then
+            echo "IDENTITY=$SEEDED_IDENTITY" >> "$GITHUB_ENV"
+          else
+            echo "IDENTITY=$MISS_IDENTITY" >> "$GITHUB_ENV"
+          fi
+
+      # The first setup in a job pays DNS resolution, TLS handshakes and a cold
+      # page cache that the later ones do not. That is a one-off spike rather
+      # than drift, so the mirrored slot order cannot cancel it; this slot pays
+      # those costs and is discarded.
+      - name: Prepare warm-up slot
+        run: bash scripts/action-overhead.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}" "$IDENTITY"
+      - name: Warm-up setup (discarded)
+        uses: ./baseline
+        # A setup that has not finished in three minutes has stalled rather than
+        # being slow; failing costs this configuration, which the report drops.
+        timeout-minutes: 3
+        with:
+          distribution: temurin
+          java-version: ${{ matrix.java-version }}
+          cache: ${{ startsWith(matrix.cache, 'maven') && 'maven' || (startsWith(matrix.cache, 'gradle') && 'gradle' || '') }}
+          cache-dependency-path: benchmark/${{ startsWith(matrix.cache, 'gradle') && 'build.gradle' || 'pom.xml' }}
+          settings-path: benchmark-maven-home
+
+      - name: Prepare slot 1
+        run: bash scripts/action-overhead.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}" "$IDENTITY"
+      - name: Start slot 1 timer
+        run: bash scripts/action-overhead.sh start
+      - name: Slot 1 setup (baseline)
+        id: slot1
+        uses: ./baseline
+        timeout-minutes: 3
+        with:
+          distribution: temurin
+          java-version: ${{ matrix.java-version }}
+          cache: ${{ startsWith(matrix.cache, 'maven') && 'maven' || (startsWith(matrix.cache, 'gradle') && 'gradle' || '') }}
+          cache-dependency-path: benchmark/${{ startsWith(matrix.cache, 'gradle') && 'build.gradle' || 'pom.xml' }}
+          settings-path: benchmark-maven-home
+      - name: Record slot 1
+        run: bash scripts/action-overhead.sh record "$RESULTS" "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.layout }}" baseline 1 "${{ steps.slot1.outputs.cache-hit }}"
+
+      - name: Prepare slot 2
+        run: bash scripts/action-overhead.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}" "$IDENTITY"
+      - name: Start slot 2 timer
+        run: bash scripts/action-overhead.sh start
+      - name: Slot 2 setup (candidate)
+        id: slot2
+        uses: ./candidate
+        timeout-minutes: 3
+        with:
+          distribution: temurin
+          java-version: ${{ matrix.java-version }}
+          cache: ${{ startsWith(matrix.cache, 'maven') && 'maven' || (startsWith(matrix.cache, 'gradle') && 'gradle' || '') }}
+          cache-dependency-path: benchmark/${{ startsWith(matrix.cache, 'gradle') && 'build.gradle' || 'pom.xml' }}
+          settings-path: benchmark-maven-home
+      - name: Record slot 2
+        run: bash scripts/action-overhead.sh record "$RESULTS" "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.layout }}" candidate 2 "${{ steps.slot2.outputs.cache-hit }}"
+
+      - name: Prepare slot 3
+        run: bash scripts/action-overhead.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}" "$IDENTITY"
+      - name: Start slot 3 timer
+        run: bash scripts/action-overhead.sh start
+      - name: Slot 3 setup (candidate)
+        id: slot3
+        uses: ./candidate
+        timeout-minutes: 3
+        with:
+          distribution: temurin
+          java-version: ${{ matrix.java-version }}
+          cache: ${{ startsWith(matrix.cache, 'maven') && 'maven' || (startsWith(matrix.cache, 'gradle') && 'gradle' || '') }}
+          cache-dependency-path: benchmark/${{ startsWith(matrix.cache, 'gradle') && 'build.gradle' || 'pom.xml' }}
+          settings-path: benchmark-maven-home
+      - name: Record slot 3
+        run: bash scripts/action-overhead.sh record "$RESULTS" "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.layout }}" candidate 3 "${{ steps.slot3.outputs.cache-hit }}"
+
+      - name: Prepare slot 4
+        run: bash scripts/action-overhead.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}" "$IDENTITY"
+      - name: Start slot 4 timer
+        run: bash scripts/action-overhead.sh start
+      - name: Slot 4 setup (baseline)
+        id: slot4
+        uses: ./baseline
+        timeout-minutes: 3
+        with:
+          distribution: temurin
+          java-version: ${{ matrix.java-version }}
+          cache: ${{ startsWith(matrix.cache, 'maven') && 'maven' || (startsWith(matrix.cache, 'gradle') && 'gradle' || '') }}
+          cache-dependency-path: benchmark/${{ startsWith(matrix.cache, 'gradle') && 'build.gradle' || 'pom.xml' }}
+          settings-path: benchmark-maven-home
+      - name: Record slot 4
+        run: bash scripts/action-overhead.sh record "$RESULTS" "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.layout }}" baseline 4 "${{ steps.slot4.outputs.cache-hit }}"
+
+      - name: Upload raw benchmark data
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: action-overhead-${{ matrix.os }}-${{ matrix.cache }}-${{ matrix.layout }}
+          path: .benchmark-results/
+          include-hidden-files: true
+          if-no-files-found: error
+          retention-days: 30
+
+  report:
+    name: Report
+    needs: benchmark
+    if: ${{ always() && !cancelled() }}
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      actions: write
+    steps:
+      - name: Check out benchmark repository
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          persist-credentials: false
+      - name: Download timings and seeded keys
+        uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6
+        with:
+          pattern: action-overhead-*
+          path: .benchmark-results
+      - name: Generate report
+        env:
+          SETUP_JAVA_REPOSITORY: ${{ inputs.setup-java-repository }}
+          BASELINE_REF: ${{ inputs.baseline-ref }}
+          CANDIDATE_REF: ${{ inputs.candidate-ref }}
+          RUN_ID: ${{ github.run_id }}
+        run: node scripts/report-action-overhead.mjs
+      - name: Delete the seeded cache entries
+        if: ${{ always() }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: bash scripts/action-overhead.sh delete-seeded "$GITHUB_REPOSITORY"
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: action-overhead-results-${{ github.run_id }}
+          path: action-overhead-results/
+          retention-days: 30
diff --git a/.github/workflows/cache-key-stability.yml b/.github/workflows/cache-key-stability.yml
new file mode 100644
index 0000000..03428c7
--- /dev/null
+++ b/.github/workflows/cache-key-stability.yml
@@ -0,0 +1,196 @@
+name: Cache key stability
+
+on:
+  workflow_dispatch:
+    inputs:
+      setup-java-repository:
+        description: Repository containing the setup-java action
+        required: true
+        default: actions/setup-java
+        type: string
+      setup-java-ref:
+        description: Git ref of setup-java to check
+        required: true
+        default: main
+        type: string
+      java-version:
+        description: Java feature version
+        required: true
+        default: "17"
+        type: string
+
+permissions:
+  contents: read
+
+env:
+  PETCLINIC_REF: f182358d02e4a68e52bdbabf55ca7800288511e7
+
+concurrency:
+  group: cache-key-stability
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+# Does the cache key hit when it should, and change when it must?
+#
+# This workflow measures nothing. The key is a deterministic function of the
+# tree, so its properties can be asserted outright, and they dominate every
+# timing in this repository: a spurious key change costs a full cache miss,
+# about a minute on Spring PetClinic, where the timing benchmarks are resolving
+# effects of tens of milliseconds.
+#
+# No cache is stored by any of this. setup-java skips its post-job save when the
+# configured path does not exist, and no probe ever creates ~/.m2.
+jobs:
+  probe:
+    name: ${{ matrix.platform }} / replica ${{ matrix.replica }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-24.04
+            platform: linux
+            replica: "1"
+          # Two runners on the same platform with the same tree. If their keys
+          # disagree, caching never works for anyone: every job would compute a
+          # key no other job has stored.
+          - os: ubuntu-24.04
+            platform: linux
+            replica: "2"
+          - os: windows-latest
+            platform: windows
+            replica: "1"
+          - os: macos-15
+            platform: macos
+            replica: "1"
+    env:
+      RUNNER_LABEL: ${{ matrix.platform }}
+      RUNNER_REPLICA: ${{ matrix.replica }}
+      RESULTS: .benchmark-results/cache-keys-${{ matrix.platform }}-${{ matrix.replica }}.csv
+    steps:
+      - name: Check out benchmark repository
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          persist-credentials: false
+      - name: Check out Spring PetClinic
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          repository: spring-projects/spring-petclinic
+          ref: ${{ env.PETCLINIC_REF }}
+          path: petclinic
+          persist-credentials: false
+      - name: Check out setup-java
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          repository: ${{ inputs.setup-java-repository }}
+          path: setup-java
+          persist-credentials: false
+          ref: ${{ inputs.setup-java-ref }}
+      - name: Snapshot the pristine tree
+        run: bash scripts/cache-key-stability.sh init
+
+      - name: Probe baseline
+        id: baseline
+        uses: ./setup-java
+        with:
+          distribution: temurin
+          java-version: ${{ inputs.java-version }}
+          cache: maven
+      - name: Record baseline
+        run: bash scripts/cache-key-stability.sh record "$RESULTS" baseline "${{ steps.baseline.outputs.cache-primary-key }}"
+
+      - name: Probe repeat
+        id: repeat
+        uses: ./setup-java
+        with:
+          distribution: temurin
+          java-version: ${{ inputs.java-version }}
+          cache: maven
+      - name: Record repeat
+        run: bash scripts/cache-key-stability.sh record "$RESULTS" repeat "${{ steps.repeat.outputs.cache-primary-key }}"
+
+      - name: Edit an unrelated source file
+        run: bash scripts/cache-key-stability.sh touch-unrelated
+      - name: Probe unrelated
+        id: unrelated
+        uses: ./setup-java
+        with:
+          distribution: temurin
+          java-version: ${{ inputs.java-version }}
+          cache: maven
+      - name: Record unrelated
+        run: bash scripts/cache-key-stability.sh record "$RESULTS" unrelated "${{ steps.unrelated.outputs.cache-primary-key }}"
+
+      - name: Restore the pristine tree
+        run: bash scripts/cache-key-stability.sh restore-tree
+      - name: Edit the dependency manifest
+        run: bash scripts/cache-key-stability.sh touch-dependencies
+      - name: Probe dependency
+        id: dependency
+        uses: ./setup-java
+        with:
+          distribution: temurin
+          java-version: ${{ inputs.java-version }}
+          cache: maven
+      - name: Record dependency
+        run: bash scripts/cache-key-stability.sh record "$RESULTS" dependency "${{ steps.dependency.outputs.cache-primary-key }}"
+
+      - name: Restore the pristine tree again
+        run: bash scripts/cache-key-stability.sh restore-tree
+      # An explicitly pinned path rather than the default pattern, so that both
+      # ways a user can configure this are covered. The probes above use the
+      # default `**/pom.xml`, which is what makes "editing a source file does not
+      # change the key" an assertion about what the action hashes rather than a
+      # tautology about a single named file.
+      - name: Probe explicit
+        id: explicit
+        uses: ./setup-java
+        with:
+          distribution: temurin
+          java-version: ${{ inputs.java-version }}
+          cache: maven
+          cache-dependency-path: petclinic/pom.xml
+      - name: Record explicit
+        run: bash scripts/cache-key-stability.sh record "$RESULTS" explicit "${{ steps.explicit.outputs.cache-primary-key }}"
+
+      - name: Upload keys
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: cache-keys-${{ matrix.platform }}-${{ matrix.replica }}
+          path: .benchmark-results/
+          include-hidden-files: true
+          if-no-files-found: error
+          retention-days: 30
+
+  check:
+    name: Check
+    needs: probe
+    if: ${{ always() && !cancelled() }}
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Check out benchmark repository
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          persist-credentials: false
+      - name: Download keys
+        uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6
+        with:
+          pattern: cache-keys-*
+          merge-multiple: true
+          path: .benchmark-results
+      - name: Assert cache key properties
+        env:
+          SETUP_JAVA_REPOSITORY: ${{ inputs.setup-java-repository }}
+          SETUP_JAVA_REF: ${{ inputs.setup-java-ref }}
+        run: node scripts/check-cache-keys.mjs
+      - name: Upload results
+        if: ${{ always() }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: cache-key-stability-${{ github.run_id }}
+          path: cache-key-results/
+          retention-days: 30
diff --git a/.github/workflows/cache-save.yml b/.github/workflows/cache-save.yml
new file mode 100644
index 0000000..4136623
--- /dev/null
+++ b/.github/workflows/cache-save.yml
@@ -0,0 +1,176 @@
+name: Cache save
+
+on:
+  workflow_dispatch:
+    inputs:
+      samples:
+        description: Runners to measure on (each contributes 2 paired observations)
+        required: true
+        type: choice
+        options:
+          - "2"
+          - "6"
+          - "10"
+          - "20"
+        default: "10"
+      setup-java-repository:
+        description: Repository containing the setup-java action
+        required: true
+        default: actions/setup-java
+        type: string
+      baseline-ref:
+        description: Git ref for the baseline arm
+        required: true
+        default: v4.8.0
+        type: string
+      candidate-ref:
+        description: Git ref for the candidate arm
+        required: true
+        default: main
+        type: string
+      fixture-mib:
+        description: Synthetic Maven cache fixture size
+        required: true
+        type: choice
+        options:
+          - "16"
+          - "64"
+          - "160"
+        default: "64"
+
+permissions:
+  actions: write
+  contents: read
+
+env:
+  CACHE_KEY_PREFIX: cache-save-${{ github.run_id }}-
+
+concurrency:
+  group: cache-save
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  # setup-java saves Maven dependencies from its post-job hook, which cannot be
+  # bracketed with the millisecond stopwatch used by the other scenarios. The
+  # transfer itself is delegated to @actions/cache.saveCache in src/cache.ts, so
+  # this workflow times that toolkit call directly with the dependency version
+  # resolved by each checked-out setup-java ref. That measures the save cost users
+  # pay on a cache miss without falling back to the Actions API's one-second step
+  # timestamps.
+  measure:
+    name: paired / ${{ matrix.sample }}
+    runs-on: ubuntu-24.04
+    # At this fixture size a normal save is seconds, not minutes. A slot that has
+    # not returned after three minutes is a cache-service stall; the report can
+    # discard the incomplete runner instead of letting the workflow wait for the
+    # platform default timeout.
+    timeout-minutes: 25
+    strategy:
+      fail-fast: false
+      # Every slot creates a fresh cache entry. Running too many uploaders at once
+      # mostly measures service contention, so the waves match the transfer overlap
+      # workflow rather than saturating the cache backend.
+      max-parallel: 4
+      matrix:
+        sample: ${{ fromJSON(inputs.samples == '20' && '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]' || inputs.samples == '10' && '[1,2,3,4,5,6,7,8,9,10]' || inputs.samples == '6' && '[1,2,3,4,5,6]' || '[1,2]') }}
+    steps:
+      - name: Check out benchmark repository
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          persist-credentials: false
+      - name: Check out baseline setup-java
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          repository: ${{ inputs.setup-java-repository }}
+          path: baseline
+          persist-credentials: false
+          ref: ${{ inputs.baseline-ref }}
+      - name: Check out candidate setup-java
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          repository: ${{ inputs.setup-java-repository }}
+          path: candidate
+          persist-credentials: false
+          ref: ${{ inputs.candidate-ref }}
+      - name: Install baseline cache client
+        run: bash scripts/cache-save.sh install-client baseline
+      - name: Install candidate cache client
+        run: bash scripts/cache-save.sh install-client candidate
+      # The fixture keeps the total bytes exact but spreads them across a Maven
+      # local-repository shape: nested group directories, version directories,
+      # jars and their .pom/.sha1 sidecars. Real cache saves pay for tar traversal
+      # and compression before upload, so a single flat file would hide changes in
+      # archive tooling or path enumeration.
+      - name: Create synthetic cache fixture
+        run: |
+          bash scripts/cache-save.sh prepare-fixture "${{ inputs.fixture-mib }}"
+          bash scripts/cache-save.sh verify-fixture "${{ inputs.fixture-mib }}"
+
+      # The first cache client call in a job pays connection setup and local page
+      # cache costs that later slots do not. That one-off spike is not linear
+      # drift, so ABBA cannot cancel it; a discarded save pays it before the slots
+      # that enter the paired estimate.
+      - name: Warm-up save (discarded)
+        timeout-minutes: 3
+        run: bash scripts/cache-save.sh save baseline "${{ env.CACHE_KEY_PREFIX }}${{ matrix.sample }}-warmup-baseline"
+
+      - name: Slot 1 save (baseline)
+        timeout-minutes: 3
+        run: bash scripts/cache-save.sh save baseline "${{ env.CACHE_KEY_PREFIX }}${{ matrix.sample }}-1-baseline" ".benchmark-results/cache-save-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" baseline 1
+
+      - name: Slot 2 save (candidate)
+        timeout-minutes: 3
+        run: bash scripts/cache-save.sh save candidate "${{ env.CACHE_KEY_PREFIX }}${{ matrix.sample }}-2-candidate" ".benchmark-results/cache-save-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" candidate 2
+
+      - name: Slot 3 save (candidate)
+        timeout-minutes: 3
+        run: bash scripts/cache-save.sh save candidate "${{ env.CACHE_KEY_PREFIX }}${{ matrix.sample }}-3-candidate" ".benchmark-results/cache-save-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" candidate 3
+
+      - name: Slot 4 save (baseline)
+        timeout-minutes: 3
+        run: bash scripts/cache-save.sh save baseline "${{ env.CACHE_KEY_PREFIX }}${{ matrix.sample }}-4-baseline" ".benchmark-results/cache-save-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" baseline 4
+
+      - name: Upload sample timings
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: cache-save-timings-${{ matrix.sample }}
+          path: .benchmark-results/
+          include-hidden-files: true
+          if-no-files-found: error
+          retention-days: 30
+
+  report:
+    name: Report
+    needs: measure
+    if: ${{ always() && !cancelled() }}
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Check out benchmark repository
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          persist-credentials: false
+      - name: Download sample timings
+        uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6
+        with:
+          pattern: cache-save-timings-*
+          merge-multiple: true
+          path: .benchmark-results
+      - name: Generate cache-save report
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BASELINE_REF: ${{ inputs.baseline-ref }}
+          CANDIDATE_REF: ${{ inputs.candidate-ref }}
+          SETUP_JAVA_REPOSITORY: ${{ inputs.setup-java-repository }}
+          FIXTURE_MIB: ${{ inputs.fixture-mib }}
+          CACHE_KEY_PREFIX: ${{ env.CACHE_KEY_PREFIX }}
+        run: node scripts/report-cache-save.mjs
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: cache-save-${{ github.run_id }}
+          path: cache-save-results/
+          retention-days: 30
diff --git a/.github/workflows/cache-value.yml b/.github/workflows/cache-value.yml
new file mode 100644
index 0000000..8be7081
--- /dev/null
+++ b/.github/workflows/cache-value.yml
@@ -0,0 +1,270 @@
+name: Cache value
+
+on:
+  workflow_dispatch:
+    inputs:
+      samples:
+        description: Runners to measure on (each contributes 2 paired observations)
+        required: true
+        type: choice
+        options:
+          - "2"
+          - "6"
+          - "10"
+        default: "6"
+      setup-java-repository:
+        description: Repository containing the setup-java action
+        required: true
+        default: actions/setup-java
+        type: string
+      setup-java-ref:
+        description: Git ref of setup-java to measure
+        required: true
+        default: main
+        type: string
+      java-version:
+        description: Java feature version
+        required: true
+        default: "17"
+        type: string
+      cleanup-caches:
+        description: Delete benchmark caches after measuring them
+        required: true
+        type: boolean
+        default: true
+
+permissions:
+  actions: write
+  contents: read
+
+env:
+  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 2
+  PETCLINIC_REF: f182358d02e4a68e52bdbabf55ca7800288511e7
+
+concurrency:
+  group: cache-value
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+# What does the Maven cache actually save?
+#
+# Every other benchmark in this repository compares implementations of the cache
+# with each other, and measures differences of tens or hundreds of milliseconds.
+# None of them measures the thing the cache exists to do. This one does: both
+# arms resolve the same dependency tree with the same command, and differ only
+# in whether setup-java restored the local repository first.
+#
+# The effect here is tens of seconds rather than tens of milliseconds, so it does
+# not need the statistical machinery the other workflows need to see their
+# effects. It is reported with the same intervals anyway, because a number
+# without one invites the reader to compare it with a number from another run.
+jobs:
+  seed:
+    name: Seed cache
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Check out benchmark repository
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          persist-credentials: false
+      - name: Check out Spring PetClinic
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          repository: spring-projects/spring-petclinic
+          ref: ${{ env.PETCLINIC_REF }}
+          path: petclinic
+          persist-credentials: false
+      - name: Check out setup-java
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          repository: ${{ inputs.setup-java-repository }}
+          path: setup-java
+          persist-credentials: false
+          ref: ${{ inputs.setup-java-ref }}
+      - name: Prepare cache identity
+        run: bash scripts/cache-value.sh prepare "cache-value-${{ github.run_id }}"
+      - name: Setup Java
+        uses: ./setup-java
+        with:
+          distribution: temurin
+          java-version: ${{ inputs.java-version }}
+          cache: maven
+          cache-dependency-path: .cache-value-key
+      # Resolving rather than compiling: the measured arms resolve too, so the
+      # seeded repository has to contain exactly what a resolve needs. Compiling
+      # would additionally pull the plugins a compile needs and store a cache the
+      # measured slots never fully use.
+      - name: Resolve dependencies
+        run: bash scripts/cache-value.sh resolve
+
+  measure:
+    name: paired / ${{ matrix.sample }}
+    needs: seed
+    runs-on: ubuntu-24.04
+    # The uncached arm downloads PetClinic's whole dependency tree from Maven
+    # Central twice per runner, which is the slowest thing this repository does.
+    timeout-minutes: 40
+    strategy:
+      fail-fast: false
+      # Kept small for Maven Central rather than for the cache service: twenty
+      # runners resolving the same tree at once would measure how Central treats
+      # a burst from one IP range, which is not the quantity under test.
+      max-parallel: 3
+      matrix:
+        sample: ${{ fromJSON(inputs.samples == '10' && '[1,2,3,4,5,6,7,8,9,10]' || inputs.samples == '6' && '[1,2,3,4,5,6]' || '[1,2]') }}
+    steps:
+      - name: Check out benchmark repository
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          persist-credentials: false
+      - name: Check out Spring PetClinic
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          repository: spring-projects/spring-petclinic
+          ref: ${{ env.PETCLINIC_REF }}
+          path: petclinic
+          persist-credentials: false
+      - name: Check out setup-java
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          repository: ${{ inputs.setup-java-repository }}
+          path: setup-java
+          persist-credentials: false
+          ref: ${{ inputs.setup-java-ref }}
+
+      # The first setup in a job pays DNS resolution, TLS handshakes and a cold
+      # page cache. That is a one-off spike rather than drift, so the mirrored
+      # slot order cannot cancel it. This slot pays those costs and is discarded.
+      - name: Reset warm-up slot
+        run: bash scripts/cache-value.sh reset "cache-value-${{ github.run_id }}"
+      - name: Warm-up setup (discarded)
+        timeout-minutes: 5
+        uses: ./setup-java
+        with:
+          distribution: temurin
+          java-version: ${{ inputs.java-version }}
+          cache: maven
+          cache-dependency-path: .cache-value-key
+
+      # ABBA across the four measured slots: sharing a runner removes the
+      # between-runner variance, and the mirrored order cancels drift across the
+      # job. Measuring each arm twice also yields a free A/A noise floor.
+      - name: Reset slot 1
+        run: bash scripts/cache-value.sh reset "cache-value-${{ github.run_id }}"
+      - name: Start slot 1 timer
+        run: node scripts/measure.mjs start
+      - name: Slot 1 setup (cached)
+        id: setup_1
+        timeout-minutes: 5
+        uses: ./setup-java
+        with:
+          distribution: temurin
+          java-version: ${{ inputs.java-version }}
+          cache: maven
+          cache-dependency-path: .cache-value-key
+      - name: Slot 1 resolve
+        run: bash scripts/cache-value.sh resolve
+      - name: Record slot 1
+        run: node scripts/measure.mjs record ".benchmark-results/cache-value-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" cached 1
+      - name: Verify slot 1
+        run: bash scripts/cache-value.sh verify cached "${{ steps.setup_1.outputs.cache-hit }}"
+
+      - name: Reset slot 2
+        run: bash scripts/cache-value.sh reset "cache-value-${{ github.run_id }}"
+      - name: Start slot 2 timer
+        run: node scripts/measure.mjs start
+      - name: Slot 2 setup (uncached)
+        id: setup_2
+        timeout-minutes: 5
+        uses: ./setup-java
+        with:
+          distribution: temurin
+          java-version: ${{ inputs.java-version }}
+      - name: Slot 2 resolve
+        run: bash scripts/cache-value.sh resolve
+      - name: Record slot 2
+        run: node scripts/measure.mjs record ".benchmark-results/cache-value-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" uncached 2
+      - name: Verify slot 2
+        run: bash scripts/cache-value.sh verify uncached "${{ steps.setup_2.outputs.cache-hit }}"
+
+      - name: Reset slot 3
+        run: bash scripts/cache-value.sh reset "cache-value-${{ github.run_id }}"
+      - name: Start slot 3 timer
+        run: node scripts/measure.mjs start
+      - name: Slot 3 setup (uncached)
+        id: setup_3
+        timeout-minutes: 5
+        uses: ./setup-java
+        with:
+          distribution: temurin
+          java-version: ${{ inputs.java-version }}
+      - name: Slot 3 resolve
+        run: bash scripts/cache-value.sh resolve
+      - name: Record slot 3
+        run: node scripts/measure.mjs record ".benchmark-results/cache-value-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" uncached 3
+      - name: Verify slot 3
+        run: bash scripts/cache-value.sh verify uncached "${{ steps.setup_3.outputs.cache-hit }}"
+
+      - name: Reset slot 4
+        run: bash scripts/cache-value.sh reset "cache-value-${{ github.run_id }}"
+      - name: Start slot 4 timer
+        run: node scripts/measure.mjs start
+      - name: Slot 4 setup (cached)
+        id: setup_4
+        timeout-minutes: 5
+        uses: ./setup-java
+        with:
+          distribution: temurin
+          java-version: ${{ inputs.java-version }}
+          cache: maven
+          cache-dependency-path: .cache-value-key
+      - name: Slot 4 resolve
+        run: bash scripts/cache-value.sh resolve
+      - name: Record slot 4
+        run: node scripts/measure.mjs record ".benchmark-results/cache-value-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" cached 4
+      - name: Verify slot 4
+        run: bash scripts/cache-value.sh verify cached "${{ steps.setup_4.outputs.cache-hit }}"
+
+      - name: Upload sample timings
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: cache-value-timings-${{ matrix.sample }}
+          path: .benchmark-results/
+          include-hidden-files: true
+          if-no-files-found: error
+          retention-days: 30
+
+  report:
+    name: Report
+    needs: measure
+    if: ${{ always() && !cancelled() }}
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Check out benchmark repository
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          persist-credentials: false
+      - name: Download sample timings
+        uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6
+        with:
+          pattern: cache-value-timings-*
+          merge-multiple: true
+          path: .benchmark-results
+      - name: Generate cache value report
+        env:
+          GH_TOKEN: ${{ github.token }}
+          SAMPLES: ${{ inputs.samples }}
+          JAVA_VERSION: ${{ inputs.java-version }}
+          SETUP_JAVA_REPOSITORY: ${{ inputs.setup-java-repository }}
+          SETUP_JAVA_REF: ${{ inputs.setup-java-ref }}
+          CLEANUP_CACHES: ${{ inputs.cleanup-caches }}
+        run: node scripts/report-cache-value.mjs
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: cache-value-${{ github.run_id }}
+          path: cache-value-results/
+          retention-days: 30
diff --git a/.github/workflows/maven-configuration-warm-path.yml b/.github/workflows/maven-configuration-warm-path.yml
deleted file mode 100644
index b7b545e..0000000
--- a/.github/workflows/maven-configuration-warm-path.yml
+++ /dev/null
@@ -1,202 +0,0 @@
-name: Maven configuration warm path
-
-on:
-  workflow_dispatch:
-    inputs:
-      setup-java-repository:
-        description: Repository containing the setup-java action
-        required: true
-        default: actions/setup-java
-        type: string
-      baseline-ref:
-        description: Git ref containing the baseline implementation
-        required: true
-        default: main
-        type: string
-      candidate-ref:
-        description: Git ref containing the candidate implementation
-        required: true
-        default: main
-        type: string
-
-permissions:
-  contents: read
-
-defaults:
-  run:
-    shell: bash
-
-concurrency:
-  group: maven-configuration-warm-path
-  cancel-in-progress: false
-
-jobs:
-  benchmark:
-    name: ${{ matrix.os }} ${{ matrix.cache }} ${{ matrix.versions.name }} ${{ matrix.toolchains }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, windows-latest, macos-15-intel]
-        cache: [none, maven, gradle]
-        versions:
-          - name: single
-            java-version: "21"
-          - name: multiple
-            java-version: |
-              17
-              21
-        toolchains: [empty, existing]
-    steps:
-      - name: Check out benchmark repository
-        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          persist-credentials: false
-      - name: Check out baseline setup-java
-        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          repository: ${{ inputs.setup-java-repository }}
-          path: baseline
-          persist-credentials: false
-          ref: ${{ inputs.baseline-ref }}
-      - name: Check out candidate setup-java
-        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          repository: ${{ inputs.setup-java-repository }}
-          path: candidate
-          persist-credentials: false
-          ref: ${{ inputs.candidate-ref }}
-
-      - name: Record setup dist sizes
-        run: |
-          bash scripts/benchmark-maven-configuration.sh record-size baseline baseline
-          bash scripts/benchmark-maven-configuration.sh record-size candidate candidate
-
-      # The first setup in a job pays DNS resolution, TLS handshakes and a cold
-      # page cache that the later ones do not. That is a one-off spike rather
-      # than drift, so the mirrored slot order cannot cancel it; this slot pays
-      # those costs and is discarded.
-      - name: Prepare warm-up slot
-        run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}"
-      - name: Warm-up setup (discarded)
-        uses: ./baseline
-        # A setup that has not finished in three minutes has stalled rather than
-        # being slow; failing costs this configuration, which the report drops.
-        timeout-minutes: 3
-        with:
-          distribution: temurin
-          java-version: ${{ matrix.versions.java-version }}
-          cache: ${{ matrix.cache != 'none' && matrix.cache || '' }}
-          cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }}
-          settings-path: benchmark-maven-home
-
-      - name: Prepare slot 1
-        run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}"
-      - name: Start slot 1 timer
-        run: bash scripts/benchmark-maven-configuration.sh start
-      - name: Slot 1 setup (baseline)
-        uses: ./baseline
-        # A setup that has not finished in three minutes has stalled rather than
-        # being slow; failing costs this configuration, which the report drops.
-        timeout-minutes: 3
-        with:
-          distribution: temurin
-          java-version: ${{ matrix.versions.java-version }}
-          cache: ${{ matrix.cache != 'none' && matrix.cache || '' }}
-          cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }}
-          settings-path: benchmark-maven-home
-      - name: Record slot 1
-        run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" baseline 1
-
-      - name: Prepare slot 2
-        run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}"
-      - name: Start slot 2 timer
-        run: bash scripts/benchmark-maven-configuration.sh start
-      - name: Slot 2 setup (candidate)
-        uses: ./candidate
-        # A setup that has not finished in three minutes has stalled rather than
-        # being slow; failing costs this configuration, which the report drops.
-        timeout-minutes: 3
-        with:
-          distribution: temurin
-          java-version: ${{ matrix.versions.java-version }}
-          cache: ${{ matrix.cache != 'none' && matrix.cache || '' }}
-          cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }}
-          settings-path: benchmark-maven-home
-      - name: Record slot 2
-        run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" candidate 2
-
-      - name: Prepare slot 3
-        run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}"
-      - name: Start slot 3 timer
-        run: bash scripts/benchmark-maven-configuration.sh start
-      - name: Slot 3 setup (candidate)
-        uses: ./candidate
-        # A setup that has not finished in three minutes has stalled rather than
-        # being slow; failing costs this configuration, which the report drops.
-        timeout-minutes: 3
-        with:
-          distribution: temurin
-          java-version: ${{ matrix.versions.java-version }}
-          cache: ${{ matrix.cache != 'none' && matrix.cache || '' }}
-          cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }}
-          settings-path: benchmark-maven-home
-      - name: Record slot 3
-        run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" candidate 3
-
-      - name: Prepare slot 4
-        run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}"
-      - name: Start slot 4 timer
-        run: bash scripts/benchmark-maven-configuration.sh start
-      - name: Slot 4 setup (baseline)
-        uses: ./baseline
-        # A setup that has not finished in three minutes has stalled rather than
-        # being slow; failing costs this configuration, which the report drops.
-        timeout-minutes: 3
-        with:
-          distribution: temurin
-          java-version: ${{ matrix.versions.java-version }}
-          cache: ${{ matrix.cache != 'none' && matrix.cache || '' }}
-          cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }}
-          settings-path: benchmark-maven-home
-      - name: Record slot 4
-        run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" baseline 4
-
-      - name: Summarize configuration
-        run: bash scripts/benchmark-maven-configuration.sh summarize "$GITHUB_STEP_SUMMARY"
-      - name: Upload raw benchmark data
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: maven-config-${{ matrix.os }}-${{ matrix.cache }}-${{ matrix.versions.name }}-${{ matrix.toolchains }}
-          path: .benchmark-results/
-          include-hidden-files: true
-          if-no-files-found: error
-          retention-days: 30
-
-  report:
-    name: Report
-    needs: benchmark
-    if: ${{ always() && !cancelled() }}
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Check out benchmark repository
-        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          persist-credentials: false
-      - name: Download configuration timings
-        uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6
-        with:
-          pattern: maven-config-*
-          path: .benchmark-results
-      - name: Generate configuration report
-        env:
-          SETUP_JAVA_REPOSITORY: ${{ inputs.setup-java-repository }}
-          BASELINE_REF: ${{ inputs.baseline-ref }}
-          CANDIDATE_REF: ${{ inputs.candidate-ref }}
-        run: node scripts/report-maven-configuration.mjs
-      - name: Upload benchmark results
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: maven-configuration-${{ github.run_id }}
-          path: maven-config-results/
-          retention-days: 30
diff --git a/.github/workflows/focused-cache-restore.yml b/.github/workflows/transfer-overlap.yml
similarity index 80%
rename from .github/workflows/focused-cache-restore.yml
rename to .github/workflows/transfer-overlap.yml
index 243a0a9..962f27c 100644
--- a/.github/workflows/focused-cache-restore.yml
+++ b/.github/workflows/transfer-overlap.yml
@@ -1,4 +1,4 @@
-name: Focused cache restore
+name: Transfer overlap
 
 on:
   workflow_dispatch:
@@ -38,6 +38,22 @@ permissions:
   actions: write
   contents: read
 
+# Does setup-java overlap the transfers it has to make?
+#
+# When a build configures more than one cache — a Maven local repository and a
+# wrapper distribution, say — setup-java restores both. Whether it does so one
+# after the other or at the same time is entirely its own choice, unlike the
+# throughput of either transfer, which belongs to `@actions/cache` and to the
+# network. That makes overlap one of the few properties of caching that a change
+# to this action can actually move, and PR actions/setup-java#1174 moved it, by
+# awaiting the two restores together instead of in sequence.
+#
+# The fixtures are deliberately large. An effect that exists only while a
+# transfer is in flight is proportional to how long the transfer takes, so a
+# small entry would shrink the very thing being measured below the noise. This
+# is the mirror image of the action overhead scenario, which uses a tiny entry
+# for exactly the opposite reason.
+
 env:
   # A cache segment download that fails leaves @actions/cache holding an armed
   # timer it never clears, so the action idles for the rest of this timeout after
@@ -51,7 +67,7 @@ env:
   WRAPPER_FIXTURE_MIB: "9"
 
 concurrency:
-  group: focused-cache-restore
+  group: transfer-overlap
   cancel-in-progress: false
 
 defaults:
@@ -96,7 +112,7 @@ jobs:
           persist-credentials: false
           ref: ${{ inputs.candidate-ref }}
       - name: Prepare cache identity
-        run: bash scripts/focused-cache-restore.sh prepare "focused-${{ github.run_id }}"
+        run: bash scripts/transfer-overlap.sh prepare "transfer-overlap-${{ github.run_id }}"
       # Seeded with the candidate so that the wrapper cache, which only newer
       # revisions maintain, is populated too. The baseline simply does not
       # restore it.
@@ -106,9 +122,9 @@ jobs:
           distribution: temurin
           java-version: ${{ env.JAVA_VERSION }}
           cache: maven
-          cache-dependency-path: .focused-cache-key
+          cache-dependency-path: .transfer-overlap-cache-key
       - name: Create synthetic cache fixtures
-        run: bash scripts/focused-cache-restore.sh seed-fixtures
+        run: bash scripts/transfer-overlap.sh seed-fixtures
 
   # Both arms are measured inside the same job, in ABBA order. Sharing a runner
   # removes the between-runner variance that otherwise swamps the effect, and
@@ -162,7 +178,7 @@ jobs:
       # difference between an arm's own first and last slot. This slot pays those
       # costs and is discarded.
       - name: Reset warm-up slot
-        run: bash scripts/focused-cache-restore.sh reset "focused-${{ github.run_id }}"
+        run: bash scripts/transfer-overlap.sh reset "transfer-overlap-${{ github.run_id }}"
       - name: Warm-up setup (discarded)
         # A restore that has not finished in three minutes has stalled on the
         # cache service rather than being slow: its neighbours in the same job
@@ -175,10 +191,10 @@ jobs:
           distribution: temurin
           java-version: ${{ env.JAVA_VERSION }}
           cache: maven
-          cache-dependency-path: .focused-cache-key
+          cache-dependency-path: .transfer-overlap-cache-key
 
       - name: Reset slot 1
-        run: bash scripts/focused-cache-restore.sh reset "focused-${{ github.run_id }}"
+        run: bash scripts/transfer-overlap.sh reset "transfer-overlap-${{ github.run_id }}"
       - name: Start slot 1 timer
         run: node scripts/measure.mjs start
       - name: Slot 1 setup (baseline)
@@ -193,14 +209,14 @@ jobs:
           distribution: temurin
           java-version: ${{ env.JAVA_VERSION }}
           cache: maven
-          cache-dependency-path: .focused-cache-key
+          cache-dependency-path: .transfer-overlap-cache-key
       - name: Record slot 1
-        run: node scripts/measure.mjs record ".benchmark-results/focused-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" baseline 1
+        run: node scripts/measure.mjs record ".benchmark-results/transfer-overlap-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" baseline 1
       - name: Verify slot 1 fixtures
-        run: bash scripts/focused-cache-restore.sh verify baseline
+        run: bash scripts/transfer-overlap.sh verify baseline
 
       - name: Reset slot 2
-        run: bash scripts/focused-cache-restore.sh reset "focused-${{ github.run_id }}"
+        run: bash scripts/transfer-overlap.sh reset "transfer-overlap-${{ github.run_id }}"
       - name: Start slot 2 timer
         run: node scripts/measure.mjs start
       - name: Slot 2 setup (candidate)
@@ -215,14 +231,14 @@ jobs:
           distribution: temurin
           java-version: ${{ env.JAVA_VERSION }}
           cache: maven
-          cache-dependency-path: .focused-cache-key
+          cache-dependency-path: .transfer-overlap-cache-key
       - name: Record slot 2
-        run: node scripts/measure.mjs record ".benchmark-results/focused-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" candidate 2
+        run: node scripts/measure.mjs record ".benchmark-results/transfer-overlap-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" candidate 2
       - name: Verify slot 2 fixtures
-        run: bash scripts/focused-cache-restore.sh verify candidate
+        run: bash scripts/transfer-overlap.sh verify candidate
 
       - name: Reset slot 3
-        run: bash scripts/focused-cache-restore.sh reset "focused-${{ github.run_id }}"
+        run: bash scripts/transfer-overlap.sh reset "transfer-overlap-${{ github.run_id }}"
       - name: Start slot 3 timer
         run: node scripts/measure.mjs start
       - name: Slot 3 setup (candidate)
@@ -237,14 +253,14 @@ jobs:
           distribution: temurin
           java-version: ${{ env.JAVA_VERSION }}
           cache: maven
-          cache-dependency-path: .focused-cache-key
+          cache-dependency-path: .transfer-overlap-cache-key
       - name: Record slot 3
-        run: node scripts/measure.mjs record ".benchmark-results/focused-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" candidate 3
+        run: node scripts/measure.mjs record ".benchmark-results/transfer-overlap-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" candidate 3
       - name: Verify slot 3 fixtures
-        run: bash scripts/focused-cache-restore.sh verify candidate
+        run: bash scripts/transfer-overlap.sh verify candidate
 
       - name: Reset slot 4
-        run: bash scripts/focused-cache-restore.sh reset "focused-${{ github.run_id }}"
+        run: bash scripts/transfer-overlap.sh reset "transfer-overlap-${{ github.run_id }}"
       - name: Start slot 4 timer
         run: node scripts/measure.mjs start
       - name: Slot 4 setup (baseline)
@@ -259,16 +275,16 @@ jobs:
           distribution: temurin
           java-version: ${{ env.JAVA_VERSION }}
           cache: maven
-          cache-dependency-path: .focused-cache-key
+          cache-dependency-path: .transfer-overlap-cache-key
       - name: Record slot 4
-        run: node scripts/measure.mjs record ".benchmark-results/focused-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" baseline 4
+        run: node scripts/measure.mjs record ".benchmark-results/transfer-overlap-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" baseline 4
       - name: Verify slot 4 fixtures
-        run: bash scripts/focused-cache-restore.sh verify baseline
+        run: bash scripts/transfer-overlap.sh verify baseline
 
       - name: Upload sample timings
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: focused-timings-${{ matrix.sample }}
+          name: transfer-overlap-timings-${{ matrix.sample }}
           path: .benchmark-results/
           include-hidden-files: true
           if-no-files-found: error
@@ -287,10 +303,10 @@ jobs:
       - name: Download sample timings
         uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6
         with:
-          pattern: focused-timings-*
+          pattern: transfer-overlap-timings-*
           merge-multiple: true
           path: .benchmark-results
-      - name: Generate focused report
+      - name: Generate transfer overlap report
         env:
           GH_TOKEN: ${{ github.token }}
           SAMPLES: ${{ inputs.samples }}
@@ -298,10 +314,10 @@ jobs:
           CANDIDATE_REF: ${{ inputs.candidate-ref }}
           SETUP_JAVA_REPOSITORY: ${{ inputs.setup-java-repository }}
           CLEANUP_CACHES: ${{ inputs.cleanup-caches }}
-        run: node scripts/report-focused.mjs
+        run: node scripts/report-transfer-overlap.mjs
       - name: Upload benchmark results
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: focused-cache-restore-${{ github.run_id }}
-          path: focused-results/
+          name: transfer-overlap-${{ github.run_id }}
+          path: transfer-overlap-results/
           retention-days: 30
diff --git a/README.md b/README.md
index e685d49..5c5b83d 100644
--- a/README.md
+++ b/README.md
@@ -13,7 +13,7 @@ Every workflow here measures effects of a few hundred milliseconds to a few seco
 
 **Millisecond timing.** The Actions API reports step `started_at` and `completed_at` only to the nearest second. Setup steps take two to six seconds, so reading durations from the API quantizes every measurement to ±500 ms — the same magnitude as the effects being measured. Timing is taken inside the job with `scripts/measure.mjs`.
 
-**Same-runner pairing.** Between-runner variance cannot be averaged away by adding more independent jobs to each arm. Every arm is measured inside the *same* job, in an order mirrored about the middle of the job: ABBA for two arms, and `v1..main` followed by `main..v1` for the version sweep. Differencing within a runner removes the runner's own speed, and the mirrored order cancels drift that is linear across the job. Each measured slot deletes `~/.m2` first so every restore extracts into an empty tree.
+**Same-runner pairing.** Between-runner variance cannot be averaged away by adding more independent jobs to each arm. Every arm is measured inside the _same_ job, in an order mirrored about the middle of the job: ABBA for two arms, and `v1..main` followed by `main..v1` for the version sweep. Differencing within a runner removes the runner's own speed, and the mirrored order cancels drift that is linear across the job. Each measured slot deletes `~/.m2` first so every restore extracts into an empty tree.
 
 Every job also runs one unmeasured warm-up slot first. The first setup in a job pays costs the later ones do not — DNS resolution, TLS handshakes to the cache service and the JDK host, and a cold page cache — and that is a one-off spike rather than drift, so the mirrored order cannot cancel it. Without the warm-up slot the A/A control resolved a spurious 0.4 s difference between an arm's own first and last slot.
 
@@ -23,7 +23,7 @@ Every job also runs one unmeasured warm-up slot first. The first setup in a job
 
 **Stalled slots fail fast.** Each measured setup is capped at three minutes. A restore that has not finished by then has stalled on the cache service rather than being slow — its neighbours in the same job take a few seconds — and failing costs one runner instead of holding the run open. The version sweep restores fifteen times per job where the two-arm workflows restore five, so it runs in narrower waves.
 
-**Stalled runners are discarded.** A slot can stall on the cache service for several seconds. Which arm the stall lands on is arbitrary, so that runner contributes an arbitrarily large difference and, with ten runners, one such slot moves the mean by more than any effect being measured. Runners whose *own arm disagrees with itself* by more than a robust threshold are dropped. That decision is made purely on within-arm spread, which has the same distribution whether or not the arms differ, so unlike filtering on the arm difference it cannot bias the result. Every report lists what it discarded and why.
+**Stalled runners are discarded.** A slot can stall on the cache service for several seconds. Which arm the stall lands on is arbitrary, so that runner contributes an arbitrarily large difference and, with ten runners, one such slot moves the mean by more than any effect being measured. Runners whose _own arm disagrees with itself_ by more than a robust threshold are dropped. That decision is made purely on within-arm spread, which has the same distribution whether or not the arms differ, so unlike filtering on the arm difference it cannot bias the result. Every report lists what it discarded and why.
 
 Every report also publishes two guard rails:
 
@@ -34,14 +34,30 @@ After changing a harness, run it with both arms set to the same ref. The true ef
 
 `scripts/paired.mjs` implements the pairing and `scripts/stats.mjs` the statistics; every report builds on both.
 
-## Scenarios
+## What each workflow asks
+
+Caching is a chain, and a benchmark is only useful if it says which link it is measuring. A restore that is quick because it hit is not comparable to one that is quick because it found nothing; a difference in transfer time is usually a difference in the network rather than in setup-java. The workflows are therefore organised by question, and each is built around the _one_ quantity that answers it.
+
+| Workflow                      | Question                                   | How it is arranged                                  | Effect it can resolve  |
+| ----------------------------- | ------------------------------------------ | --------------------------------------------------- | ---------------------- |
+| **Cache value**               | What does caching buy at all?              | Real PetClinic build, cached against uncached       | Tens of seconds        |
+| **Cache key stability**       | Does the cache hit when it should?         | Deterministic assertions on the computed key        | Pass or fail           |
+| **Action overhead**           | What does setup-java's own code cost?      | ~1 MiB entry, so the transfer is not what is timed  | Tens to hundreds of ms |
+| **Transfer overlap**          | Does it overlap the transfers it makes?    | Large entries, so concurrency has something to hide | Hundreds of ms         |
+| **Cache save**                | What does the first run pay?               | Post-run save of a Maven-shaped tree                | Seconds                |
+| **JDK cache**                 | Is caching the JDK worth it?               | `cache-jdk` off against on, same ref                | Seconds                |
+| **Benchmark** (version sweep) | How does `main` compare with each release? | All versions on one runner, mirrored order          | Hundreds of ms         |
+
+The sizes in the third column are the design, not an accident. setup-java does not move the bytes itself — it hands the transfer to `@actions/cache` — so a benchmark with a large fixture measures the network and a benchmark with a small one measures the action. **Action overhead** and **Transfer overlap** are deliberately the same measurement at two fixture sizes for exactly that reason, and together they decompose a restore into the part setup-java controls and the part it does not.
+
+## Version sweep
 
 Each action version runs with Java 17 on `ubuntu-24.04`:
 
-| Distribution | Expected setup path | Purpose |
-| --- | --- | --- |
-| Eclipse Temurin | Hosted runner tool-cache hit | Measures setup overhead when the JDK is already available |
-| Microsoft Build of OpenJDK | JDK download and extraction | Measures setup overhead when the JDK must be installed |
+| Distribution               | Expected setup path          | Purpose                                                   |
+| -------------------------- | ---------------------------- | --------------------------------------------------------- |
+| Eclipse Temurin            | Hosted runner tool-cache hit | Measures setup overhead when the JDK is already available |
+| Microsoft Build of OpenJDK | JDK download and extraction  | Measures setup overhead when the JDK must be installed    |
 
 v1 predates distribution selection and integrated dependency caching. It runs only its native Zulu installer path. v2 supports the Temurin and Microsoft scenarios, but its bundled legacy cache client is rejected by the current Actions cache service. v1 and v2 therefore have no Maven cache storage, and their cold/warm labels are repeated uncached samples.
 
@@ -51,12 +67,12 @@ A seed job compiles Spring PetClinic once to populate a single Maven cache entry
 
 **Only comparable versions are ranked.** v4, v5.2, v5.6 and `main` all restore the same seeded entry through `cache-dependency-path`, so a difference between them is a difference in the implementation. `main` is ranked against each of them, with Holm's step-down correction across that family: it is tested against every one of them in one run, so without a correction the chance that one comparison clears 0.05 by luck is far above 0.05.
 
-v1, v2 and v3 are measured on the same runners and published, but they carry no verdict, because a verdict would report a difference in the *workload* as though it were a difference in the implementation:
+v1, v2 and v3 are measured on the same runners and published, but they carry no verdict, because a verdict would report a difference in the _workload_ as though it were a difference in the implementation:
 
-| Version | Why it is not ranked |
-| --- | --- |
-| v1.4.4 | Installs its own JDK and does no dependency caching |
-| v2.5.1 | Its bundled cache client is rejected by the current cache service, so it restores nothing |
+| Version | Why it is not ranked                                                                                                                                  |
+| ------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- |
+| v1.4.4  | Installs its own JDK and does no dependency caching                                                                                                   |
+| v2.5.1  | Its bundled cache client is rejected by the current cache service, so it restores nothing                                                             |
 | v3.14.1 | Predates `cache-dependency-path` and keys on `pom.xml`, so it restores its own entry — the blob confound described above, which pairing cannot remove |
 
 v3 is the instructive case. In run 30979614347 it took 3.69 s and 3.89 s on two runners that ran v4 in 0.50 s and 0.52 s moments later in the same job. A sevenfold gap that appears on some runners and not others is blob placement, not code, and ranking it would have published a verdict for it.
@@ -65,74 +81,90 @@ Spring PetClinic and third-party actions are pinned to commits. `setup-java@main
 
 ## Running
 
-Open **Actions > Benchmark setup-java > Run workflow**. Choose how many runners to measure on; each contributes two observations per version. Ten is the default.
+Every workflow is dispatched from **Actions > _(workflow)_ > Run workflow**. The ones that compare two refs take a `baseline-ref` and a `candidate-ref`, so any of them can be pointed at a PR branch; the ones that sample across runners take a runner count, where each runner contributes two observations per arm.
 
-The report job writes a Markdown summary and uploads raw JSON and CSV files. Benchmark-created caches are deleted after measurement by default, preventing repeated runs from consuming repository cache storage. Disable cleanup when you need to inspect the entries manually.
+Each report job writes a Markdown summary to the run and uploads raw JSON and CSV. Benchmark-created caches are deleted after measurement, because the repository shares one cache budget across all of these workflows and an entry left behind by one run evicts the seeded entries another depends on.
 
-### Focused cache restore
+**Run a harness change against itself before believing it.** Set both refs to the same value: the true effect is then exactly zero, and any verdict other than `inconclusive` or `within-noise` is a defect rather than a finding.
 
-The **Focused cache restore** workflow isolates the setup step to compare two `actions/setup-java` refs (by default `v4.8.0` against `main`). It uses a pinned Temurin JDK from the hosted runner tool cache, seeds a synthetic 160 MiB dependency cache for both arms and a 9 MiB wrapper cache for the candidate, and runs no Maven command. Measurement jobs therefore contain no JDK or Maven Central downloads; they measure JDK discovery and Actions cache restoration only.
+## The workflows in detail
 
-This workflow is the reference for how a comparison should be measured here. Three properties make its verdicts trustworthy:
+### Cache value
 
-**Millisecond timing.** The Actions API reports step `started_at` and `completed_at` only to the nearest second. Setup steps take two to six seconds, so reading durations from the API quantizes every measurement to ±500 ms — the same magnitude as the effects being measured. Timing is therefore taken inside the job with `scripts/measure.mjs`.
+The **Cache value** workflow answers the question a user actually has: is turning `cache: maven` on worth it? It builds Spring PetClinic for real, cached against uncached, in ABBA order on one runner.
 
-**Same-runner pairing.** Between-runner variance on hosted runners is larger than the effects under test, and it cannot be averaged away by adding more independent jobs to each arm. Both arms run in the *same* job in ABBA order — baseline, candidate, candidate, baseline — so each runner yields one paired difference with the runner's own speed cancelled out. The mirrored order also cancels drift across the four slots. Each measured slot deletes `~/.m2` first so every restore extracts into an empty tree.
+This is the only scenario whose effect is large enough to see without any statistical machinery, and that is the point of running it — it establishes the scale everything else is a fraction of. A result here in the tens of seconds is what makes an argument about 40 ms of action overhead worth having or not worth having.
 
-**One cache, both arms.** A cache entry's download throughput depends on where the service placed the stored blob, and that placement is fixed for the life of the entry. Giving each arm its own seeded cache therefore confounds the arm with its blob, and because the bias is identical on every runner, pairing cannot remove it and more samples only tighten the interval around the wrong answer. A single entry is seeded and both arms restore it.
+### Cache key stability
 
-**Intervals, not point estimates.** `scripts/stats.mjs` reports a bootstrap 95% confidence interval, a permutation p-value, and a Hodges-Lehmann shift for every comparison, and turns them into an explicit verdict. A comparison whose interval includes zero is reported as `inconclusive` rather than as a number that looks like a result.
+The **Cache key stability** workflow measures nothing. The cache key is a deterministic function of the tree, so its properties can be asserted outright rather than estimated, and those properties dominate every timing in this repository: a key that changes when it should not costs a full cache miss — about a minute on PetClinic — where the timing workflows are resolving tens of milliseconds.
 
-The report also publishes two guard rails:
+It checks that the key is unchanged when the same tree is hashed twice, unchanged when an unrelated source file is edited, changed when the dependency manifest is edited, identical across two runners on the same platform, and different across platforms. The cross-runner check is the important one: if two runners on the same tree disagree, caching never works for anyone, because every job computes a key no other job has stored.
 
-- A **noise floor**, the median spread between the two slots of the same arm on one runner. An effect smaller than this is reported as `within-noise` even when its interval excludes zero.
-- An **A/A control**, the same estimator applied to the baseline against itself. It costs no extra jobs because each arm is already measured twice per runner. A healthy run reports `within-noise` or `inconclusive`; anything else means slot ordering is biasing the results and the headline verdict cannot be trusted.
+Nothing is stored by any of it. setup-java skips its post-job save when the configured path does not exist, and no probe ever creates `~/.m2`.
 
-Run the workflow with `baseline-ref` and `candidate-ref` set to the same value after changing it. The true effect is then exactly zero, and any verdict other than `inconclusive` or `within-noise` is a defect in the harness rather than a finding. That check is what surfaced the per-arm cache confound described above: on identical code it reported a 0.859 s improvement, with the baseline blob served at ~60 MB/s and the candidate blob at ~105–130 MB/s on the same runner in the same job.
+### Action overhead
 
-Point `baseline-ref` and `candidate-ref` at any two refs — including a PR branch — to check whether a change delivers a real improvement.
+The **Action overhead** workflow compares two refs across three operating systems, four cache profiles and two configuration layouts, with a cache entry of about a megabyte spread over many small files. The small entry is the whole design: it leaves resolving a distribution, computing a cache key, writing settings and toolchains, and the bookkeeping around a restore as what is being timed, rather than the network.
 
-#### Why this replaced the previous design
+The four cache profiles are nested levels of work rather than competing options:
 
-The earlier version of this workflow ran each arm as its own matrix of independent jobs, read durations from the Actions API, and reported a "paired median delta" that paired `sample N` of one arm with `sample N` of the other. Those samples shared no runner and no point in time, so the pairing removed no variance at all.
+| Profile       | What the action does                                |
+| ------------- | --------------------------------------------------- |
+| `none`        | Never touches the cache code                        |
+| `maven-miss`  | Computes a key and is told no                       |
+| `maven-hit`   | Computes a key, is told yes, unpacks a ~1 MiB entry |
+| `gradle-miss` | A miss through the other package manager            |
 
-Two consecutive runs of that harness against an unchanged `v4.8.0` — where the true difference is exactly zero — produced medians of 2 s and 3 s, a spurious 1.2 s separation whose confidence interval excluded zero. Over the same pair of runs the reported candidate delta flipped from +0.6 s to −0.8 s. `scripts/stats.test.mjs` pins that dataset as a regression test so unpaired sampling is not reintroduced.
+Differencing the levels says where the time goes, and the report publishes that decomposition alongside the comparison. Those level differences are between-configuration, so they are reported as observed medians with no interval and no verdict; they describe what a setup costs rather than establishing that two refs differ.
 
-### JDK cache
+Each configuration is one runner, so it yields one paired difference and cannot support an interval alone. Configurations are treated as **blocks** — arms compared within a configuration, on the same runner, in ABBA order after a discarded warm-up slot — and the differences pooled across the matrix.
 
-The **JDK cache** workflow measures the installed-JDK cache added on `actions/setup-java@main`. It compares `cache-jdk: false` against `cache-jdk: true`, defaulting to Microsoft Build of OpenJDK 17. Both arms use the same action ref, so the result isolates JDK caching instead of conflating it with implementation changes between commits.
+Every slot records setup-java's `cache-hit` output, and the report refuses to be read normally if a `maven-hit` slot missed or a `maven-miss` slot hit, because either turns a level of the decomposition into a different level and the labels stop being true.
 
-A single seed job compiles Spring PetClinic to populate one Maven cache entry and one JDK cache entry. Every measurement runner then runs both arms in one job in ABBA order under `cache-read-only: true`. Both arms restore the same Maven entry — only the JDK entry differs between them, which is the effect under test.
+#### Why this replaced the Maven configuration warm path
 
-Every seed and measured slot removes matching JDKs from `$RUNNER_TOOL_CACHE` first, deliberately measuring the not-preinstalled path and preventing a hosted-runner tool-cache hit from bypassing JDK cache restore logic. Select Temurin to test that same forced-miss path with a distribution normally preinstalled on hosted runners.
+The workflow this grew out of was called a warm path and was not one. It had no seed job, and its synthetic `benchmark/pom.xml` hashed to a key nothing had ever stored, so every one of its "warm" restores was a miss — the job logs read `maven cache is not found` and `Path Validation Error ... hence no cache is being saved`. Its three cache profiles were three variations on a failed lookup, and it could not see the restore path at all. The differences it reported were real, but they were differences in the cost of _missing_, published under a heading that said warm.
 
-JDK cache keys are derived from the JDK's identity and source, so unlike dependency cache keys they cannot be namespaced per run; the `prepare` job deletes existing JDK caches before seeding.
+### Transfer overlap
+
+The **Transfer overlap** workflow isolates the setup step to compare two refs (by default `v4.8.0` against `main`) with a synthetic 160 MiB dependency cache seeded for both arms and a 9 MiB wrapper cache, and runs no Maven command. Measurement jobs contain no JDK or Maven Central downloads.
 
-Open **Actions > JDK cache > Run workflow** to select the distribution, Java version, action ref, runner count, and cache cleanup behavior.
+When a build configures more than one cache, setup-java restores both, and whether it does so in sequence or at the same time is its own choice — unlike the throughput of either transfer. That makes overlap one of the few properties of caching a change to this action can actually move, and [actions/setup-java#1174](https://github.com/actions/setup-java/pull/1174) moved it by awaiting the two restores together. An effect that exists only while a transfer is in flight is proportional to how long the transfer takes, which is why the fixtures here are large and why the same effect is invisible in **Action overhead**.
 
-### Maven configuration warm path
+This workflow is also the reference implementation for how a comparison is measured in this repository. Point `baseline-ref` and `candidate-ref` at any two refs — including a PR branch — to check whether a change delivers a real improvement, or at the _same_ ref to check the harness itself.
 
-The **Maven configuration warm path** workflow compares two refs of setup-java across 36 configurations: three operating systems, three cache profiles, single and multiple Java versions, and an empty or pre-existing `toolchains.xml`.
+#### Why the harness was rebuilt
 
-Unlike the other workflows it does not repeat one scenario across many runners; it runs each configuration once. A single configuration therefore yields one paired difference and cannot support an interval on its own. Each configuration is instead treated as a **block**: the arms are compared within it, on the same runner, in ABBA order after a discarded warm-up slot, and the differences are pooled across the matrix. That answers the question the workflow actually asks — whether the candidate differs from the baseline across configurations — at no extra job cost.
+The earlier version ran each arm as its own matrix of independent jobs, read durations from the Actions API, and reported a "paired median delta" that paired `sample N` of one arm with `sample N` of the other. Those samples shared no runner and no point in time, so the pairing removed no variance at all.
 
-Per-configuration numbers are still published, but as single observations with no verdict, because that is all they are. Breakdowns by operating system and by cache profile are reported with their own intervals; groups with few blocks will read `inconclusive` even where the pooled result does not, which is the intended behaviour rather than a defect.
+Two consecutive runs of that harness against an unchanged `v4.8.0` — where the true difference is exactly zero — produced medians of 2 s and 3 s, a spurious 1.2 s separation whose confidence interval excluded zero. Over the same pair of runs the reported candidate delta flipped from +0.6 s to −0.8 s. `scripts/stats.test.mjs` pins that dataset as a regression test so unpaired sampling is not reintroduced.
 
-Open **Actions > Maven configuration warm path > Run workflow** to select the repository and the two refs.
+### Cache save
+
+The **Cache save** workflow measures what the first run pays. A cache miss costs the download the user already paid for _plus_ the save, and nothing else here measured the second half of that.
+
+setup-java saves in its post-job hook, which cannot be bracketed by a timer from inside the job. The save is therefore driven directly through the `@actions/cache` version each ref pins, which is where the difference between refs actually lives, against a Maven-shaped fixture: nested group directories of small `.jar`, `.pom` and `.sha1` files rather than one large blob, because archive cost tracks file count and directory depth as much as it tracks bytes.
+
+### JDK cache
+
+The **JDK cache** workflow measures the installed-JDK cache added on `actions/setup-java@main`. It compares `cache-jdk: false` against `cache-jdk: true`, defaulting to Microsoft Build of OpenJDK 17. Both arms use the same action ref, so the result isolates JDK caching instead of conflating it with implementation changes between commits.
+
+A single seed job compiles Spring PetClinic to populate one Maven cache entry and one JDK cache entry. Every measurement runner then runs both arms in one job in ABBA order under `cache-read-only: true`. Both arms restore the same Maven entry — only the JDK entry differs between them, which is the effect under test.
+
+Every seed and measured slot removes matching JDKs from `$RUNNER_TOOL_CACHE` first, deliberately measuring the not-preinstalled path and preventing a hosted-runner tool-cache hit from bypassing JDK cache restore logic. Select Temurin to test that same forced-miss path with a distribution normally preinstalled on hosted runners.
+
+JDK cache keys are derived from the JDK's identity and source, so unlike dependency cache keys they cannot be namespaced per run; the `prepare` job deletes existing JDK caches before seeding.
 
 ## Reading results
 
-The summary reports medians for:
+**Read the verdict, not the point estimate.** Every comparison reports one, and it is the only part of the output that has been checked against the harness's own noise. A result reported as `inconclusive` has established nothing however suggestive its number looks, and one reported as `within-noise` is smaller than this harness can resolve on hosted runners.
 
-- setup time, including JDK discovery/download and dependency-cache restore;
-- Spring PetClinic `compile` time;
-- the `setup-java` post step that saves caches;
-- compressed cache storage per isolated case;
-- estimated billed minutes, calculated by rounding each Linux job to a whole minute.
+**Check the A/A control first.** It applies the same estimator to one arm against itself, where the true difference is exactly zero. A healthy run reports `within-noise` or `inconclusive`. Anything else means the run measured the harness rather than the code, and the headline is void.
 
-Public repositories do not pay for standard GitHub-hosted runners. The estimated minutes are included to make the results applicable to private repositories; actual charges depend on the account plan and runner type.
+**Check which link was measured.** A number from **Action overhead** and a number from **Transfer overlap** are not the same quantity and do not add up to a user-visible saving on their own; the scale that makes either of them matter comes from **Cache value**.
 
-Network throughput, hosted-runner image changes, upstream artifact availability, and runner load all introduce variance. Read the verdict rather than the point estimate: a comparison reported as `inconclusive` has not established anything, however suggestive its number looks, and one reported as `within-noise` is smaller than the harness can resolve. Check the A/A control before trusting any headline — if it resolves a difference, the run is measuring the harness rather than the code.
+Network throughput, hosted-runner image changes, upstream artifact availability and runner load all introduce variance, which is what the guard rails above exist to absorb.
 
 ## Local checks
 
diff --git a/scripts/action-overhead.sh b/scripts/action-overhead.sh
new file mode 100644
index 0000000..e98b837
--- /dev/null
+++ b/scripts/action-overhead.sh
@@ -0,0 +1,177 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+command=${1:?command is required}
+
+benchmark_home="$PWD/benchmark-maven-home"
+results_dir="$PWD/.benchmark-results"
+sizes_file="$results_dir/action-overhead-sizes.csv"
+
+# Roughly a megabyte spread over many small files. The point of this scenario is
+# to measure what setup-java's own code costs, so the entry has to be small
+# enough that the transfer is not what is being timed, while still being made of
+# many files rather than one, because archiving cost tracks file count as much as
+# it tracks bytes.
+fixture_files=200
+fixture_file_kib=5
+
+write_manifest() {
+  identity=$1
+
+  mkdir -p benchmark
+  # The cache key is a hash of this file, so the identity written into it is what
+  # decides whether a slot hits the seeded entry or misses. Every byte outside
+  # the identity has to be stable, or two slots that should share an entry would
+  # compute different keys.
+  cat > benchmark/pom.xml <<XML
+<project>
+  <benchmarkIdentity>$identity</benchmarkIdentity>
+</project>
+XML
+  cat > benchmark/build.gradle <<GRADLE
+plugins { id("java") }
+// benchmark identity: $identity
+GRADLE
+}
+
+write_toolchains() {
+  toolchains_profile=$1
+
+  if [ "$toolchains_profile" = "existing" ]; then
+    cat > "$benchmark_home/toolchains.xml" <<'XML'
+<toolchains xmlns="http://maven.apache.org/TOOLCHAINS/1.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/TOOLCHAINS/1.0.0 http://maven.apache.org/xsd/toolchains-1.0.0.xsd">
+  <toolchain>
+    <type>foo</type>
+    <provides>
+      <custom>preserved</custom>
+    </provides>
+    <configuration>
+      <fooHome>/opt/foo</fooHome>
+    </configuration>
+  </toolchain>
+</toolchains>
+XML
+  elif [ "$toolchains_profile" != "empty" ]; then
+    echo "Unsupported toolchains profile: $toolchains_profile" >&2
+    exit 1
+  fi
+}
+
+case "$command" in
+  # Build the tiny local repository that the seed job stores, so that the
+  # measured `maven-hit` slots have something real to restore.
+  seed-fixture)
+    identity=${2:?identity is required}
+
+    rm -rf "$benchmark_home"
+    mkdir -p "$benchmark_home"
+    write_manifest "$identity"
+
+    repository="$HOME/.m2/repository/com/example/benchmark"
+    rm -rf "$HOME/.m2"
+    mkdir -p "$repository"
+    for index in $(seq 1 "$fixture_files"); do
+      # Random rather than zeroes so the archive cannot be compressed away to
+      # nothing, which would make the seeded entry unrepresentative of a real
+      # local repository full of already-compressed jars.
+      dd if=/dev/urandom "of=$repository/artifact-$index.jar" \
+        bs=1024 count="$fixture_file_kib" status=none
+    done
+    ;;
+  prepare)
+    cache_profile=${2:?cache profile is required}
+    toolchains_profile=${3:?toolchains profile is required}
+    identity=${4:?identity is required}
+
+
+    rm -rf "$benchmark_home"
+    mkdir -p "$benchmark_home"
+    write_manifest "$identity"
+    write_toolchains "$toolchains_profile"
+
+    # Every slot starts from an absent cache directory. Restoring on top of an
+    # existing tree would let the second slot in an arm do less work than the
+    # first, and it also keeps setup-java from saving anything on the profiles
+    # that are supposed to miss: the action skips its post-job save when the
+    # configured path does not exist.
+    case "$cache_profile" in
+      none | maven-miss | maven-hit)
+        rm -rf "$HOME/.m2"
+        ;;
+      gradle-miss)
+        rm -rf "$HOME/.gradle"
+        ;;
+      *)
+        echo "Unsupported cache profile: $cache_profile" >&2
+        exit 1
+        ;;
+    esac
+    ;;
+  start)
+    mkdir -p "$results_dir"
+    node scripts/measure.mjs start
+    ;;
+  record)
+    results_file=${2:?results file is required}
+    os=${3:?os is required}
+    cache_profile=${4:?cache profile is required}
+    layout=${5:?layout profile is required}
+    arm=${6:?arm is required}
+    slot=${7:?slot is required}
+    cache_hit=${8:-}
+
+    mkdir -p "$results_dir"
+    # An empty cache-hit output is not the same as a miss: the `none` profile
+    # never asks about the cache, and refs older than the output's introduction
+    # report nothing. Recording that distinction lets the report tell a genuine
+    # miss apart from a slot it cannot verify.
+    node scripts/measure.mjs record "$results_file" \
+      "$os" "$cache_profile" "$layout" "$arm" "$slot" "${cache_hit:-unset}"
+    ;;
+  record-size)
+    arm=${2:?arm is required}
+    action_path=${3:?action path is required}
+
+    mkdir -p "$results_dir"
+    index_bytes=$(node -e "const fs=require('fs'); process.stdout.write(String(fs.statSync(process.argv[1]).size))" "$action_path/dist/setup/index.js")
+    js_bytes=$(node -e "const fs=require('fs'); const path=require('path'); let total=0; for (const entry of fs.readdirSync(process.argv[1])) { if (entry.endsWith('.js')) total += fs.statSync(path.join(process.argv[1], entry)).size; } process.stdout.write(String(total));" "$action_path/dist/setup")
+    chunk_count=$(find "$action_path/dist/setup" -maxdepth 1 -name '*.js' | wc -l | tr -d ' ')
+    printf '%s,%s,%s,%s\n' \
+      "$arm" "$index_bytes" "$js_bytes" "$chunk_count" \
+      >> "$sizes_file"
+    ;;
+  # Every run stores three fresh entries whose keys are hashes, so nothing about
+  # the name says which run owns them. The seed jobs record the keys the action
+  # reported; this deletes exactly those.
+  delete-seeded)
+    repository=${2:?repository is required}
+
+    found=0
+    while IFS= read -r key_file; do
+      key=$(tr -d '\r\n' < "$key_file")
+      if [ -z "$key" ]; then
+        # Older refs do not publish `cache-primary-key`, so seeding with one
+        # leaves an entry that has to age out on its own.
+        echo "No key recorded in $key_file; nothing to delete for it." >&2
+        continue
+      fi
+      found=$((found + 1))
+      if gh cache delete "$key" --repo "$repository"; then
+        echo "Deleted $key"
+      else
+        echo "Could not delete $key; it will expire on its own." >&2
+      fi
+    done < <(find "$results_dir" -name 'seeded-key-*.txt' -type f)
+
+    if [ "$found" -eq 0 ]; then
+      echo "No seeded cache keys were recorded." >&2
+    fi
+    ;;
+  *)
+    echo "Unsupported command: $command" >&2
+    exit 1
+    ;;
+esac
diff --git a/scripts/benchmark-maven-configuration.sh b/scripts/benchmark-maven-configuration.sh
deleted file mode 100755
index ed6b33c..0000000
--- a/scripts/benchmark-maven-configuration.sh
+++ /dev/null
@@ -1,163 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-command=${1:?command is required}
-
-benchmark_home="$PWD/benchmark-maven-home"
-results_dir="$PWD/.benchmark-results"
-results_file="$results_dir/maven-configuration-timings.csv"
-sizes_file="$results_dir/maven-configuration-sizes.csv"
-
-case "$command" in
-  prepare)
-    cache=${2:?cache profile is required}
-    toolchains_profile=${3:?toolchains profile is required}
-
-    rm -rf "$benchmark_home"
-    mkdir -p "$benchmark_home" benchmark
-    printf '<project />\n' > benchmark/pom.xml
-    printf 'plugins { id("java") }\n' > benchmark/build.gradle
-
-    if [ "$toolchains_profile" = "existing" ]; then
-      cat > "$benchmark_home/toolchains.xml" <<'XML'
-<toolchains xmlns="http://maven.apache.org/TOOLCHAINS/1.0.0"
-  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/TOOLCHAINS/1.0.0 http://maven.apache.org/xsd/toolchains-1.0.0.xsd">
-  <toolchain>
-    <type>foo</type>
-    <provides>
-      <custom>preserved</custom>
-    </provides>
-    <configuration>
-      <fooHome>/opt/foo</fooHome>
-    </configuration>
-  </toolchain>
-</toolchains>
-XML
-    elif [ "$toolchains_profile" != "empty" ]; then
-      echo "Unsupported toolchains profile: $toolchains_profile" >&2
-      exit 1
-    fi
-
-    case "$cache" in
-      none | maven | gradle) ;;
-      *)
-        echo "Unsupported cache profile: $cache" >&2
-        exit 1
-        ;;
-    esac
-    ;;
-  start)
-    mkdir -p "$results_dir"
-    node -e "require('fs').writeFileSync('.benchmark-start', String(Date.now()))"
-    ;;
-  record)
-    os=${2:?os is required}
-    cache=${3:?cache profile is required}
-    versions=${4:?versions profile is required}
-    toolchains_profile=${5:?toolchains profile is required}
-    implementation=${6:?implementation is required}
-    iteration=${7:?iteration is required}
-
-    started=$(cat .benchmark-start)
-    finished=$(node -e "process.stdout.write(String(Date.now()))")
-    elapsed=$((finished - started))
-    mkdir -p "$results_dir"
-    printf '%s,%s,%s,%s,%s,%s,%s\n' \
-      "$os" "$cache" "$versions" "$toolchains_profile" "$implementation" "$iteration" "$elapsed" \
-      >> "$results_file"
-    ;;
-  record-size)
-    implementation=${2:?implementation is required}
-    action_path=${3:?action path is required}
-
-    mkdir -p "$results_dir"
-    index_bytes=$(node -e "const fs=require('fs'); process.stdout.write(String(fs.statSync(process.argv[1]).size))" "$action_path/dist/setup/index.js")
-    js_bytes=$(node -e "const fs=require('fs'); const path=require('path'); let total=0; for (const entry of fs.readdirSync(process.argv[1])) { if (entry.endsWith('.js')) total += fs.statSync(path.join(process.argv[1], entry)).size; } process.stdout.write(String(total));" "$action_path/dist/setup")
-    chunk_count=$(find "$action_path/dist/setup" -maxdepth 1 -name '*.js' | wc -l | tr -d ' ')
-    xml_parser_chunks=$(grep -Rsl "fast-xml-parser" "$action_path/dist/setup"/*.js 2>/dev/null | xargs -n 1 basename 2>/dev/null | paste -sd ';' - || true)
-    printf '%s,%s,%s,%s\n' \
-      "$implementation" "$index_bytes" "$js_bytes" "${xml_parser_chunks:-none} ($chunk_count js files)" \
-      >> "$sizes_file"
-    ;;
-  summarize)
-    summary_file=${2:?summary file is required}
-
-    node --input-type=module - "$results_file" "$sizes_file" "$summary_file" <<'NODE'
-import fs from 'node:fs';
-
-const [, , resultsFile, sizesFile, summaryFile] = process.argv;
-
-const percentile = (values, percentileValue) => {
-  const sorted = [...values].sort((left, right) => left - right);
-  const index = Math.ceil((percentileValue / 100) * sorted.length) - 1;
-  return sorted[Math.max(0, Math.min(index, sorted.length - 1))];
-};
-
-const rows = fs
-  .readFileSync(resultsFile, 'utf8')
-  .trim()
-  .split('\n')
-  .filter(Boolean)
-  .map(line => {
-    const [os, cache, versions, toolchains, implementation, iteration, elapsed] =
-      line.split(',');
-    return {
-      os,
-      cache,
-      versions,
-      toolchains,
-      implementation,
-      iteration,
-      elapsed: Number(elapsed)
-    };
-  });
-
-const groups = new Map();
-for (const row of rows) {
-  const key = [row.os, row.cache, row.versions, row.toolchains, row.implementation].join(',');
-  const values = groups.get(key) ?? [];
-  values.push(row.elapsed);
-  groups.set(key, values);
-}
-
-const lines = [
-  '## Maven configuration warm-path benchmark',
-  '',
-  '| OS | Cache | Versions | Toolchains | Implementation | Runs | Median (ms) | p95 (ms) |',
-  '| --- | --- | --- | --- | --- | ---: | ---: | ---: |'
-];
-
-for (const [key, values] of [...groups.entries()].sort()) {
-  const [os, cache, versions, toolchains, implementation] = key.split(',');
-  lines.push(
-    `| ${os} | ${cache} | ${versions} | ${toolchains} | ${implementation} | ${values.length} | ${percentile(values, 50)} | ${percentile(values, 95)} |`
-  );
-}
-
-if (fs.existsSync(sizesFile)) {
-  lines.push(
-    '',
-    '## setup entry/chunk sizes',
-    '',
-    '| Implementation | dist/setup/index.js bytes | dist/setup JS bytes | XML parser chunk location |',
-    '| --- | ---: | ---: | --- |'
-  );
-  for (const line of fs.readFileSync(sizesFile, 'utf8').trim().split('\n')) {
-    if (!line) continue;
-    const [implementation, indexBytes, jsBytes, xmlParserChunks] = line.split(',');
-    lines.push(
-      `| ${implementation} | ${indexBytes} | ${jsBytes} | ${xmlParserChunks} |`
-    );
-  }
-}
-
-fs.appendFileSync(summaryFile, `${lines.join('\n')}\n`);
-NODE
-    ;;
-  *)
-    echo "Unsupported command: $command" >&2
-    exit 1
-    ;;
-esac
diff --git a/scripts/cache-key-stability.sh b/scripts/cache-key-stability.sh
new file mode 100644
index 0000000..b7b239c
--- /dev/null
+++ b/scripts/cache-key-stability.sh
@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+
+# Tree arrangement for the cache key stability check.
+#
+# This is not a benchmark. It asserts properties of the cache key, because a key
+# that changes when it should not costs a full cache miss, and a miss costs more
+# than every timing effect the rest of this repository measures put together. On
+# Spring PetClinic a miss is roughly a minute; the timing benchmarks resolve
+# effects of tens of milliseconds.
+#
+# The probes never create ~/.m2. setup-java skips its post-job save when the
+# configured path does not exist ("Path Validation Error ... hence no cache is
+# being saved"), so the whole check runs without storing a single cache entry.
+
+set -euo pipefail
+
+command=${1:?command is required}
+
+project_dir=${PROJECT_DIR:-petclinic}
+pom="$project_dir/pom.xml"
+pristine_pom=".cache-key-pristine-pom.xml"
+
+first_java_source() {
+  find "$project_dir/src" -name '*.java' | sort | head -n 1
+}
+
+case "$command" in
+  init)
+    # A pristine copy so that the probe which edits the pom can be undone
+    # exactly. Restoring with git would also revert the checkout of PetClinic
+    # itself, and reverting by editing back risks leaving a trailing newline that
+    # changes the hash.
+    cp "$pom" "$pristine_pom"
+    rm -rf "$HOME/.m2"
+    ;;
+  restore-tree)
+    cp "$pristine_pom" "$pom"
+    rm -rf "$HOME/.m2"
+    ;;
+  touch-unrelated)
+    # A source file is not part of any dependency-manifest pattern, so hashing it
+    # would be a defect. This is the change that most often breaks caching in the
+    # wild: a key that tracks the whole tree misses on every commit.
+    source_file=$(first_java_source)
+    if [ -z "$source_file" ]; then
+      echo "No Java source found under $project_dir/src" >&2
+      exit 1
+    fi
+    printf '\n// cache key stability probe\n' >> "$source_file"
+    printf 'cache key stability probe\n' >> "$project_dir/README.md"
+    echo "Modified $source_file and README.md"
+    ;;
+  touch-dependencies)
+    # A comment in the XML epilog is well-formed and changes the file's hash
+    # without changing what Maven resolves, which is what the key is supposed to
+    # track.
+    printf '<!-- cache key stability probe -->\n' >> "$pom"
+    ;;
+  record)
+    # Written as CSV rather than read from step outputs in the checker, because
+    # a key that comes back empty must be recorded as empty and diagnosed there
+    # rather than silently collapsing two probes into one equal pair.
+    results=${2:?results file is required}
+    probe=${3:?probe name is required}
+    key=${4-}
+    mkdir -p "$(dirname "$results")"
+    printf '"%s","%s","%s","%s"\n' \
+      "${RUNNER_LABEL:?RUNNER_LABEL is required}" \
+      "${RUNNER_REPLICA:?RUNNER_REPLICA is required}" \
+      "$probe" "$key" >> "$results"
+    echo "$probe: ${key:-<empty>}"
+    ;;
+  *)
+    echo "Unsupported command: $command" >&2
+    exit 1
+    ;;
+esac
diff --git a/scripts/cache-save.sh b/scripts/cache-save.sh
new file mode 100755
index 0000000..4b3cbff
--- /dev/null
+++ b/scripts/cache-save.sh
@@ -0,0 +1,260 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+command=${1:?command is required}
+fixture_dir="$HOME/.m2/repository/cache-save-fixture"
+
+# A Maven cache save is not just bytes on the wire. The archive step walks a deep
+# repository tree and compresses thousands of already-compressed jars plus their
+# small metadata sidecars, so a flat file would erase the traversal and path
+# enumeration work that changes in setup-java's cache client can affect.
+case "$command" in
+  prepare-fixture)
+    fixture_mib=${2:?fixture size in MiB is required}
+    rm -rf "$fixture_dir"
+    node --input-type=module - "$fixture_dir" "$fixture_mib" <<'NODE'
+import { mkdir, open, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+
+const [fixtureDir, fixtureMiB] = process.argv.slice(2);
+const mib = Number(fixtureMiB);
+const bytes = mib * 1024 * 1024;
+if (!Number.isInteger(mib) || !Number.isInteger(bytes) || bytes <= 0) {
+  throw new Error(`Invalid fixture size: ${fixtureMiB}`);
+}
+
+const pomBytes = 512;
+const shaBytes = 40;
+const artifacts =
+  mib <= 4 ? 128 : mib <= 16 ? 512 : mib <= 64 ? 1024 : 2048;
+const sidecarBytes = artifacts * (pomBytes + shaBytes);
+const jarBytes = bytes - sidecarBytes;
+if (jarBytes <= artifacts) {
+  throw new Error(
+    `${fixtureMiB} MiB is too small for ${artifacts} Maven artifacts`,
+  );
+}
+const baseJarBytes = Math.floor(jarBytes / artifacts);
+const extraJarBytes = jarBytes % artifacts;
+
+let state = 0x9e3779b9;
+
+async function writeRandomFile(file, size) {
+  await mkdir(dirname(file), { recursive: true });
+  const handle = await open(file, "w");
+  const chunk = Buffer.alloc(Math.min(64 * 1024, size));
+  let written = 0;
+  try {
+    while (written < size) {
+      const length = Math.min(chunk.length, size - written);
+      for (let index = 0; index < length; index += 1) {
+        state ^= state << 13;
+        state ^= state >>> 17;
+        state ^= state << 5;
+        chunk[index] = state & 0xff;
+      }
+      await handle.write(chunk, 0, length);
+      written += length;
+    }
+  } finally {
+    await handle.close();
+  }
+}
+
+function paddedPom(group, artifact, version) {
+  const xml = `<project><modelVersion>4.0.0</modelVersion><groupId>${group}</groupId><artifactId>${artifact}</artifactId><version>${version}</version></project>\n`;
+  return Buffer.from(xml.padEnd(pomBytes, " ").slice(0, pomBytes));
+}
+
+function sha1Sidecar(seed) {
+  let value = seed + 1;
+  let text = "";
+  while (text.length < shaBytes) {
+    value ^= value << 13;
+    value ^= value >>> 17;
+    value ^= value << 5;
+    text += (value >>> 0).toString(16).padStart(8, "0");
+  }
+  return Buffer.from(text.slice(0, shaBytes));
+}
+
+for (let index = 0; index < artifacts; index += 1) {
+  const group = `com.example.group${String(index % 64).padStart(2, "0")}.depth${String(Math.floor(index / 64) % 32).padStart(2, "0")}`;
+  const artifact = `artifact-${String(index).padStart(4, "0")}`;
+  const version = `1.${index % 17}.${Math.floor(index / 17)}`;
+  const directory = join(
+    fixtureDir,
+    ...group.split("."),
+    artifact,
+    version,
+  );
+  const jar = join(directory, `${artifact}-${version}.jar`);
+  const pom = join(directory, `${artifact}-${version}.pom`);
+  const sha1 = join(directory, `${artifact}-${version}.jar.sha1`);
+  await writeRandomFile(jar, baseJarBytes + (index < extraJarBytes ? 1 : 0));
+  await writeFile(pom, paddedPom(group, artifact, version));
+  await writeFile(sha1, sha1Sidecar(index));
+}
+NODE
+    ;;
+  expected-file-count)
+    fixture_mib=${2:?fixture size in MiB is required}
+    node --input-type=module - "$fixture_mib" <<'NODE'
+const [fixtureMiB] = process.argv.slice(2);
+const mib = Number(fixtureMiB);
+if (!Number.isInteger(mib) || mib <= 0) {
+  throw new Error(`Invalid fixture size: ${fixtureMiB}`);
+}
+const artifacts =
+  mib <= 4 ? 128 : mib <= 16 ? 512 : mib <= 64 ? 1024 : 2048;
+console.log(artifacts * 3);
+NODE
+    ;;
+  verify-fixture)
+    fixture_mib=${2:?fixture size in MiB is required}
+    node --input-type=module - "$fixture_dir" "$fixture_mib" <<'NODE'
+import { readdir, stat } from "node:fs/promises";
+import { join } from "node:path";
+
+const [fixtureDir, fixtureMiB] = process.argv.slice(2);
+const mib = Number(fixtureMiB);
+const expectedBytes = mib * 1024 * 1024;
+if (!Number.isInteger(mib) || !Number.isInteger(expectedBytes) || mib <= 0) {
+  throw new Error(`Invalid fixture size: ${fixtureMiB}`);
+}
+const artifacts =
+  mib <= 4 ? 128 : mib <= 16 ? 512 : mib <= 64 ? 1024 : 2048;
+const expectedFiles = artifacts * 3;
+
+async function walk(directory) {
+  let files = 0;
+  let bytes = 0;
+  for (const entry of await readdir(directory, { withFileTypes: true })) {
+    const path = join(directory, entry.name);
+    if (entry.isDirectory()) {
+      const child = await walk(path);
+      files += child.files;
+      bytes += child.bytes;
+    } else if (entry.isFile()) {
+      files += 1;
+      bytes += (await stat(path)).size;
+    }
+  }
+  return { files, bytes };
+}
+
+const actual = await walk(fixtureDir);
+if (actual.bytes !== expectedBytes) {
+  throw new Error(
+    `Cache-save fixture is ${actual.bytes} bytes, expected ${expectedBytes}`,
+  );
+}
+if (actual.files !== expectedFiles) {
+  throw new Error(
+    `Cache-save fixture has ${actual.files} files, expected ${expectedFiles}`,
+  );
+}
+NODE
+    ;;
+  file-count)
+    node --input-type=module - "$fixture_dir" <<'NODE'
+import { readdir } from "node:fs/promises";
+import { join } from "node:path";
+
+const [fixtureDir] = process.argv.slice(2);
+
+async function count(directory) {
+  let files = 0;
+  for (const entry of await readdir(directory, { withFileTypes: true })) {
+    const path = join(directory, entry.name);
+    if (entry.isDirectory()) {
+      files += await count(path);
+    } else if (entry.isFile()) {
+      files += 1;
+    }
+  }
+  return files;
+}
+
+console.log(await count(fixtureDir));
+NODE
+    ;;
+  remove-one-file)
+    node --input-type=module - "$fixture_dir" <<'NODE'
+import { readdir, rm } from "node:fs/promises";
+import { join } from "node:path";
+
+const [fixtureDir] = process.argv.slice(2);
+
+async function firstFile(directory) {
+  for (const entry of await readdir(directory, { withFileTypes: true })) {
+    const path = join(directory, entry.name);
+    if (entry.isDirectory()) {
+      const found = await firstFile(path);
+      if (found) return found;
+    } else if (entry.isFile()) {
+      return path;
+    }
+  }
+  return null;
+}
+
+const file = await firstFile(fixtureDir);
+if (!file) throw new Error(`No fixture file found under ${fixtureDir}`);
+await rm(file);
+NODE
+    ;;
+  install-client)
+    arm_dir=${2:?setup-java checkout directory is required}
+    if [ -f "$arm_dir/package-lock.json" ]; then
+      npm --prefix "$arm_dir" ci --ignore-scripts --no-audit --no-fund
+    else
+      npm --prefix "$arm_dir" install --ignore-scripts --no-audit --no-fund
+    fi
+    node --input-type=module - "$PWD/$arm_dir/package.json" <<'NODE'
+import { createRequire } from "node:module";
+
+const [manifest] = process.argv.slice(2);
+const require = createRequire(manifest);
+require.resolve("@actions/cache");
+NODE
+    ;;
+  save)
+    arm_dir=${2:?setup-java checkout directory is required}
+    key=${3:?cache key is required}
+    results_file=${4:-}
+    sample=${5:-}
+    arm=${6:-}
+    slot=${7:-}
+    if [ ! -d "$fixture_dir" ]; then
+      echo "Cache-save fixture is missing at $fixture_dir" >&2
+      exit 1
+    fi
+    node --input-type=module - "$PWD/$arm_dir/package.json" "$fixture_dir" "$key" \
+      "$results_file" "$sample" "$arm" "$slot" <<'NODE'
+import { createRequire } from "node:module";
+import { pathToFileURL } from "node:url";
+
+const [manifest, fixtureDir, key, resultsFile, sample, arm, slot] =
+  process.argv.slice(2);
+const require = createRequire(manifest);
+const cache = require("@actions/cache");
+
+const started = Date.now();
+const cacheId = await cache.saveCache([fixtureDir], key);
+const elapsedMs = Date.now() - started;
+console.log(`Saved ${key} as cache ${cacheId} in ${elapsedMs} ms`);
+if (resultsFile) {
+  const { record } = await import(
+    pathToFileURL(`${process.cwd()}/scripts/measure.mjs`).href
+  );
+  await record(resultsFile, [sample, arm, slot], elapsedMs);
+}
+NODE
+    ;;
+  *)
+    echo "Unsupported command: $command" >&2
+    exit 1
+    ;;
+esac
diff --git a/scripts/cache-value.sh b/scripts/cache-value.sh
new file mode 100644
index 0000000..3f4e81c
--- /dev/null
+++ b/scripts/cache-value.sh
@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+
+# Fixture helpers for the cache value benchmark.
+#
+# This benchmark answers the question the rest of the suite was missing: what
+# does the Maven cache actually save a real project? Both arms run the same
+# resolve against the same dependency tree. The only difference is whether
+# setup-java restored the local repository first, so the difference between them
+# is the whole value of the feature rather than a detail of its implementation.
+
+set -euo pipefail
+
+command=${1:?command is required}
+
+# Spring PetClinic is checked out below the benchmark repository rather than over
+# it, because its build runs a nohttp check across the whole basedir and would
+# otherwise lint these scripts.
+project_dir=${PROJECT_DIR:-petclinic}
+
+write_identity() {
+  # The cached arm keys on this file rather than on the real pom, so the key is
+  # fixed for the run and every slot restores the entry the seed job stored.
+  printf '%s\n' "${1:?benchmark id is required}" > .cache-value-key
+}
+
+case "$command" in
+  prepare)
+    write_identity "${2:?benchmark id is required}"
+    ;;
+  reset)
+    # Every slot must resolve into an empty local repository. Leaving artifacts
+    # behind would let the uncached arm resolve from what an earlier cached slot
+    # restored, which is the one thing this benchmark must not allow: it would
+    # report that the cache saves nothing.
+    rm -rf "$HOME/.m2"
+    write_identity "${2:?benchmark id is required}"
+    ;;
+  resolve)
+    # Both arms run this identical command. The cached arm finds the artifacts in
+    # the restored repository; the uncached arm fetches them from Maven Central.
+    # Keeping the command identical is what makes the difference attributable to
+    # the cache rather than to the work being done.
+    cd "$project_dir"
+    ./mvnw --batch-mode --no-transfer-progress dependency:go-offline
+    ;;
+  verify)
+    # Checked from setup-java's own `cache-hit` output rather than by inspecting
+    # the restored tree, because inspecting it costs a `du` over a few hundred
+    # MiB and this has to run outside the timed span to stay out of the
+    # measurement.
+    #
+    # A silent miss on the cached arm is the failure that matters: both arms
+    # would then download from Maven Central and the benchmark would report,
+    # with a perfectly tight interval, that the cache is worthless.
+    arm=${2:?arm is required}
+    cache_hit=${3-}
+    case "$arm" in
+      cached)
+        if [ "$cache_hit" != "true" ]; then
+          echo "The cached arm reported cache-hit='$cache_hit'; the seeded" \
+            "entry was not restored, so this slot measured a download rather" \
+            "than a restore" >&2
+          exit 1
+        fi
+        ;;
+      uncached)
+        if [ -n "$cache_hit" ]; then
+          echo "The uncached arm reported a cache-hit value ('$cache_hit');" \
+            "it must run with caching disabled" >&2
+          exit 1
+        fi
+        ;;
+      *)
+        echo "Unsupported arm: $arm" >&2
+        exit 1
+        ;;
+    esac
+    ;;
+  *)
+    echo "Unsupported command: $command" >&2
+    exit 1
+    ;;
+esac
diff --git a/scripts/check-cache-keys.mjs b/scripts/check-cache-keys.mjs
new file mode 100644
index 0000000..5da4288
--- /dev/null
+++ b/scripts/check-cache-keys.mjs
@@ -0,0 +1,255 @@
+// Asserts the properties a cache key has to have, rather than measuring how
+// long anything takes.
+//
+// The rest of this repository measures differences of tens or hundreds of
+// milliseconds between implementations of the cache. None of that matters if the
+// key misses: on Spring PetClinic a miss costs about a minute, which is three
+// orders of magnitude more than the effects the timing benchmarks resolve. A
+// key is also a deterministic function of the tree, so these properties can be
+// asserted outright instead of estimated with an interval.
+
+import {
+  appendFile,
+  mkdir,
+  readdir,
+  readFile,
+  writeFile,
+} from "node:fs/promises";
+import { join } from "node:path";
+import { pathToFileURL } from "node:url";
+
+import { requireEnv } from "./paired.mjs";
+
+export function parseKeys(text) {
+  return text
+    .split("\n")
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .map((line) => {
+      const [platform, replica, probe, key] = line
+        .split('","')
+        .map((field) => field.replace(/^"|"$/g, ""));
+      return { platform, replica, probe, key };
+    });
+}
+
+export async function readKeyFiles(directory = ".benchmark-results") {
+  const entries = await readdir(directory);
+  const files = entries.filter((name) => name.startsWith("cache-keys-"));
+  const contents = await Promise.all(
+    files.map((name) => readFile(join(directory, name), "utf8")),
+  );
+  return contents.join("\n");
+}
+
+function lookup(rows, platform, replica, probe) {
+  return rows.find(
+    (row) =>
+      row.platform === platform &&
+      row.replica === replica &&
+      row.probe === probe,
+  );
+}
+
+// Each check states the user-visible consequence of failing it, because a bare
+// "keys differ" tells a maintainer nothing about whether to care.
+const WITHIN_RUNNER = [
+  {
+    probe: "repeat",
+    against: "baseline",
+    expect: "same",
+    title: "Repeating the same setup computes the same key",
+    consequence:
+      "the key is not a function of the tree, so a second run in the same job would miss",
+  },
+  {
+    probe: "unrelated",
+    against: "baseline",
+    expect: "same",
+    title: "Editing a source file does not change the key",
+    consequence:
+      "every commit that touches source would miss, which is the most common way caching is silently lost",
+  },
+  {
+    probe: "dependency",
+    against: "baseline",
+    expect: "different",
+    title: "Editing the dependency manifest changes the key",
+    consequence:
+      "a build whose dependencies changed would restore a stale repository",
+  },
+];
+
+export function evaluate(rows) {
+  const platforms = [...new Set(rows.map((row) => row.platform))].sort();
+  const checks = [];
+
+  for (const platform of platforms) {
+    const replicas = [
+      ...new Set(
+        rows.filter((r) => r.platform === platform).map((r) => r.replica),
+      ),
+    ].sort();
+
+    for (const replica of replicas) {
+      for (const spec of WITHIN_RUNNER) {
+        const left = lookup(rows, platform, replica, spec.probe);
+        const right = lookup(rows, platform, replica, spec.against);
+        checks.push(
+          compare({
+            ...spec,
+            scope: `${platform} runner ${replica}`,
+            left,
+            right,
+          }),
+        );
+      }
+    }
+
+    // The check that matters most: two runners in the same run, same tree, same
+    // platform. If these disagree, caching never works at all - every job
+    // computes a key no other job has stored.
+    if (replicas.length >= 2) {
+      for (const probe of ["baseline", "explicit"]) {
+        checks.push(
+          compare({
+            probe,
+            against: probe,
+            expect: "same",
+            title: `Two runners compute the same \`${probe}\` key`,
+            consequence:
+              "the key depends on something outside the tree, so no job would ever restore what another job stored",
+            scope: platform,
+            left: lookup(rows, platform, replicas[0], probe),
+            right: lookup(rows, platform, replicas[1], probe),
+          }),
+        );
+      }
+    }
+  }
+
+  // A key is scoped to the platform it was built on, so two platforms must not
+  // collide: restoring a Linux tree onto Windows would be worse than a miss.
+  if (platforms.length >= 2) {
+    for (let i = 0; i < platforms.length - 1; i += 1) {
+      checks.push(
+        compare({
+          probe: "baseline",
+          against: "baseline",
+          expect: "different",
+          title: `\`${platforms[i]}\` and \`${platforms[i + 1]}\` do not share a key`,
+          consequence:
+            "one platform could restore an entry another platform stored",
+          scope: "cross-platform",
+          left: lookup(rows, platforms[i], "1", "baseline"),
+          right: lookup(rows, platforms[i + 1], "1", "baseline"),
+        }),
+      );
+    }
+  }
+
+  return { checks, passed: checks.every((check) => check.passed) };
+}
+
+function compare({ title, consequence, expect, scope, left, right }) {
+  // An absent or empty key is always a failure. Treating it as "equal to the
+  // other empty key" would let a version that computes no key at all pass every
+  // sameness check in this file.
+  if (!left?.key || !right?.key) {
+    return {
+      title,
+      scope,
+      expect,
+      passed: false,
+      detail: "setup-java reported no cache-primary-key for at least one probe",
+      consequence,
+    };
+  }
+  const same = left.key === right.key;
+  const passed = expect === "same" ? same : !same;
+  return {
+    title,
+    scope,
+    expect,
+    passed,
+    detail: same
+      ? `both \`${left.key}\``
+      : `\`${left.key}\` vs \`${right.key}\``,
+    consequence,
+  };
+}
+
+export function markdown(metadata, result) {
+  const failed = result.checks.filter((check) => !check.passed);
+  const lines = [
+    "# Cache key stability",
+    "",
+    `\`${metadata.setupJavaRef}\` from \`${metadata.setupJavaRepository}\`, run ${metadata.runId}.`,
+    "",
+    "This is not a timing benchmark. The cache key is a deterministic function of",
+    "the tree, so its properties are asserted rather than estimated. They matter",
+    "more than any of the timings: a key that changes when it should not costs a",
+    "full cache miss, which on this project is about a minute, while the timing",
+    "benchmarks resolve effects of tens of milliseconds.",
+    "",
+    "## Result",
+    "",
+    result.passed
+      ? `**All ${result.checks.length} checks passed.**`
+      : `**${failed.length} of ${result.checks.length} checks failed.**`,
+    "",
+    "| Check | Scope | Expected | Result | Keys |",
+    "| --- | --- | --- | --- | --- |",
+  ];
+  for (const check of result.checks) {
+    lines.push(
+      `| ${check.title} | ${check.scope} | ${check.expect} | ${check.passed ? "pass" : "**fail**"} | ${check.detail} |`,
+    );
+  }
+  if (failed.length > 0) {
+    lines.push("", "## What each failure costs", "");
+    for (const check of failed) {
+      lines.push(
+        `- **${check.title}** (${check.scope}): ${check.consequence}.`,
+      );
+    }
+  }
+  return `${lines.join("\n")}\n`;
+}
+
+export async function main(env = process.env) {
+  const runId = requireEnv(env, "GITHUB_RUN_ID");
+  const metadata = {
+    runId,
+    setupJavaRepository: requireEnv(env, "SETUP_JAVA_REPOSITORY"),
+    setupJavaRef: requireEnv(env, "SETUP_JAVA_REF"),
+    generatedAt: new Date().toISOString(),
+  };
+  const rows = parseKeys(await readKeyFiles());
+  if (rows.length === 0) {
+    throw new Error("No cache keys were collected");
+  }
+  const result = evaluate(rows);
+  const report = markdown(metadata, result);
+
+  await mkdir("cache-key-results", { recursive: true });
+  await writeFile(
+    "cache-key-results/results.json",
+    `${JSON.stringify({ metadata, rows, result }, null, 2)}\n`,
+  );
+  await writeFile("cache-key-results/summary.md", report);
+  await appendFile(env.GITHUB_STEP_SUMMARY, report);
+
+  if (!result.passed) {
+    throw new Error(
+      `${result.checks.filter((check) => !check.passed).length} cache key checks failed`,
+    );
+  }
+}
+
+if (
+  process.argv[1] &&
+  import.meta.url === pathToFileURL(process.argv[1]).href
+) {
+  await main();
+}
diff --git a/scripts/check-cache-keys.test.mjs b/scripts/check-cache-keys.test.mjs
new file mode 100644
index 0000000..ed0a46b
--- /dev/null
+++ b/scripts/check-cache-keys.test.mjs
@@ -0,0 +1,118 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { evaluate, markdown, parseKeys } from "./check-cache-keys.mjs";
+
+function rows(entries) {
+  return parseKeys(
+    entries
+      .map(
+        ({ platform, replica, probe, key }) =>
+          `"${platform}","${replica}","${probe}","${key}"`,
+      )
+      .join("\n"),
+  );
+}
+
+function healthy() {
+  const out = [];
+  for (const platform of ["linux", "windows"]) {
+    for (const replica of ["1", "2"]) {
+      const base = `setup-java-${platform}-maven-aaa`;
+      out.push(
+        { platform, replica, probe: "baseline", key: base },
+        { platform, replica, probe: "repeat", key: base },
+        { platform, replica, probe: "unrelated", key: base },
+        {
+          platform,
+          replica,
+          probe: "dependency",
+          key: `setup-java-${platform}-maven-bbb`,
+        },
+        {
+          platform,
+          replica,
+          probe: "explicit",
+          key: `setup-java-${platform}-maven-ccc`,
+        },
+      );
+    }
+  }
+  return rows(out);
+}
+
+test("passes a key that behaves correctly", () => {
+  const result = evaluate(healthy());
+  assert.equal(result.passed, true);
+  assert.ok(result.checks.length > 0);
+});
+
+test("fails when a source edit changes the key", () => {
+  const broken = healthy().map((row) =>
+    row.probe === "unrelated" && row.platform === "linux"
+      ? { ...row, key: "setup-java-linux-maven-zzz" }
+      : row,
+  );
+  const result = evaluate(broken);
+  assert.equal(result.passed, false);
+  const failure = result.checks.find((check) => !check.passed);
+  assert.match(failure.title, /source file/);
+  assert.match(failure.consequence, /every commit/);
+});
+
+test("fails when two runners disagree on the same tree", () => {
+  const broken = healthy().map((row) =>
+    row.probe === "baseline" && row.replica === "2" && row.platform === "linux"
+      ? { ...row, key: "setup-java-linux-maven-other" }
+      : row,
+  );
+  const result = evaluate(broken);
+  assert.equal(result.passed, false);
+  assert.ok(
+    result.checks.some(
+      (check) => !check.passed && /Two runners compute/.test(check.title),
+    ),
+  );
+});
+
+test("fails when a dependency change does not change the key", () => {
+  const broken = healthy().map((row) =>
+    row.probe === "dependency"
+      ? { ...row, key: `setup-java-${row.platform}-maven-aaa` }
+      : row,
+  );
+  const result = evaluate(broken);
+  assert.equal(result.passed, false);
+});
+
+// A version that computes no key at all would otherwise satisfy every sameness
+// check by comparing empty with empty, and the report would read as a clean pass
+// for an action that had stopped caching entirely.
+test("treats a missing key as a failure rather than as equality", () => {
+  const empty = healthy().map((row) => ({ ...row, key: "" }));
+  const result = evaluate(empty);
+  assert.equal(result.passed, false);
+  assert.ok(
+    result.checks.every((check) => check.expect !== "same" || !check.passed),
+  );
+  assert.match(
+    result.checks.find((check) => !check.passed).detail,
+    /no cache-primary-key/,
+  );
+});
+
+test("reports what each failure costs", () => {
+  const broken = healthy().map((row) =>
+    row.probe === "unrelated" ? { ...row, key: "changed" } : row,
+  );
+  const rendered = markdown(
+    {
+      runId: "1",
+      setupJavaRepository: "actions/setup-java",
+      setupJavaRef: "main",
+    },
+    evaluate(broken),
+  );
+  assert.match(rendered, /checks failed/);
+  assert.match(rendered, /## What each failure costs/);
+});
diff --git a/scripts/report-action-overhead.mjs b/scripts/report-action-overhead.mjs
new file mode 100644
index 0000000..ddaf49c
--- /dev/null
+++ b/scripts/report-action-overhead.mjs
@@ -0,0 +1,433 @@
+// What does setup-java's own code cost?
+//
+// This report answers two different questions from one matrix, and they need
+// different treatment.
+//
+// The first is the comparison: does the candidate differ from the baseline?
+// Each configuration is one runner, so it yields one paired difference and
+// cannot support an interval alone. Configurations are treated as blocks — arms
+// compared within a configuration, on the same runner, in ABBA order — and the
+// differences pooled.
+//
+// The second is the decomposition, and it is not a comparison at all. The four
+// cache profiles are nested levels of work: `none` never touches the cache code,
+// `maven-miss` computes a key and is told no, `maven-hit` also unpacks a small
+// entry. Differencing the levels says where the time goes, which is the number a
+// maintainer deciding whether an optimization is worth it actually needs. Those
+// differences are between-configuration, so they are reported as observed
+// medians without intervals; they are descriptive, and labelled as such.
+
+import {
+  appendFile,
+  mkdir,
+  readdir,
+  readFile,
+  writeFile,
+} from "node:fs/promises";
+import { join } from "node:path";
+import { pathToFileURL } from "node:url";
+
+import { analyzePairs, requireEnv } from "./paired.mjs";
+import { describeVerdict, formatInterval, holmAdjust } from "./stats.mjs";
+
+const RESULTS_DIR = ".benchmark-results";
+const OUTPUT_DIR = "action-overhead-results";
+
+const CACHE_PROFILES = [
+  {
+    id: "none",
+    label: "No cache",
+    describes: "the action never touches the cache code",
+  },
+  {
+    id: "maven-miss",
+    label: "Maven, miss",
+    describes: "a key is computed and the service says no",
+  },
+  {
+    id: "maven-hit",
+    label: "Maven, hit",
+    describes: "a key is computed and a ~1 MiB entry is unpacked",
+  },
+  {
+    id: "gradle-miss",
+    label: "Gradle, miss",
+    describes: "the same as a Maven miss through the other package manager",
+  },
+];
+
+const STEPS = [
+  {
+    from: "none",
+    to: "maven-miss",
+    label: "Computing a key and asking the cache service",
+  },
+  {
+    from: "maven-miss",
+    to: "maven-hit",
+    label: "Restoring and unpacking a ~1 MiB entry",
+  },
+];
+
+function unquote(value) {
+  const trimmed = (value ?? "").trim();
+  return trimmed.startsWith('"') && trimmed.endsWith('"')
+    ? trimmed.slice(1, -1).replaceAll('""', '"')
+    : trimmed;
+}
+
+// os,cache,layout,arm,slot,cacheHit,elapsedMs
+export function parseSamples(csv) {
+  return csv
+    .trim()
+    .split("\n")
+    .filter((line) => line.trim() !== "")
+    .map((line) => {
+      const [os, cache, layout, arm, slot, cacheHit, elapsedMs] = line
+        .split(",")
+        .map(unquote);
+      return {
+        configuration: `${os}/${cache}/${layout}`,
+        os,
+        cache,
+        layout,
+        arm,
+        slot: Number(slot),
+        cacheHit,
+        seconds: Number(elapsedMs) / 1000,
+      };
+    })
+    .filter((row) => Number.isFinite(row.seconds));
+}
+
+// paired.mjs groups by a numeric `sample`, so each configuration is assigned a
+// stable index and becomes one block.
+export function toPairedRows(rows) {
+  const configurations = [
+    ...new Set(rows.map((row) => row.configuration)),
+  ].sort();
+  const index = new Map(configurations.map((name, at) => [name, at + 1]));
+  return {
+    configurations,
+    rows: rows.map((row) => ({
+      sample: index.get(row.configuration),
+      arm: row.arm,
+      slot: row.slot,
+      seconds: row.seconds,
+    })),
+  };
+}
+
+export async function readResults(directory = RESULTS_DIR) {
+  const entries = await readdir(directory, { recursive: true });
+  const files = entries.filter(
+    (entry) => entry.endsWith(".csv") && entry.includes("action-overhead"),
+  );
+  if (files.length === 0) {
+    throw new Error(`No action-overhead CSV files found in ${directory}`);
+  }
+  const contents = await Promise.all(
+    files.sort().map((file) => readFile(join(directory, file), "utf8")),
+  );
+  return contents.join("\n");
+}
+
+function median(values) {
+  if (values.length === 0) return null;
+  const sorted = [...values].sort((left, right) => left - right);
+  const middle = Math.floor(sorted.length / 2);
+  return sorted.length % 2 === 0
+    ? (sorted[middle - 1] + sorted[middle]) / 2
+    : sorted[middle];
+}
+
+export function subsetAnalysis(rows, predicate) {
+  const subset = rows.filter(predicate);
+  if (subset.length === 0) return null;
+  const { rows: paired } = toPairedRows(subset);
+  const analysis = analyzePairs(paired, "baseline", "candidate");
+  return analysis.pairs.length === 0 ? null : analysis;
+}
+
+// A `maven-hit` slot that reported a miss did not measure a restore, and a
+// `maven-miss` slot that reported a hit found an entry that should not exist.
+// Either turns a level of the decomposition into a different level, so the
+// numbers below it stop meaning what they are labelled, and that has to be said
+// loudly rather than buried.
+export function checkCacheIntegrity(rows) {
+  const expectations = { "maven-hit": "true", "maven-miss": "false" };
+  const problems = [];
+  for (const row of rows) {
+    const expected = expectations[row.cache];
+    if (!expected) continue;
+    if (row.cacheHit === "unset" || row.cacheHit === "") {
+      problems.push({
+        configuration: row.configuration,
+        slot: row.slot,
+        reported: "nothing",
+        expected,
+        reason:
+          "the ref does not publish `cache-hit`, so this slot cannot be verified",
+      });
+    } else if (row.cacheHit !== expected) {
+      problems.push({
+        configuration: row.configuration,
+        slot: row.slot,
+        reported: row.cacheHit,
+        expected,
+        reason: "the slot did not do the work its profile is named for",
+      });
+    }
+  }
+  return problems;
+}
+
+export function decompose(rows) {
+  const byOs = [...new Set(rows.map((row) => row.os))].sort();
+  const levels = byOs.map((os) => {
+    const seconds = {};
+    for (const profile of CACHE_PROFILES) {
+      seconds[profile.id] = median(
+        rows
+          .filter((row) => row.os === os && row.cache === profile.id)
+          .map((row) => row.seconds),
+      );
+    }
+    return { os, seconds };
+  });
+  const steps = byOs.map((os) => {
+    const level = levels.find((entry) => entry.os === os);
+    return {
+      os,
+      steps: STEPS.map(({ from, to, label }) => ({
+        label,
+        seconds:
+          level.seconds[from] === null || level.seconds[to] === null
+            ? null
+            : level.seconds[to] - level.seconds[from],
+      })),
+    };
+  });
+  return { levels, steps };
+}
+
+function seconds(value, digits = 3) {
+  return value === null || value === undefined
+    ? "n/a"
+    : `${value >= 0 ? "" : "−"}${Math.abs(value).toFixed(digits)}s`;
+}
+
+function analysisRow(label, analysis, adjustedP) {
+  if (!analysis) return `| ${label} | no usable pairs | | | |`;
+  return [
+    `| ${label}`,
+    seconds(analysis.interval.estimate),
+    formatInterval(analysis.interval),
+    adjustedP === null || adjustedP === undefined
+      ? analysis.pValue.toFixed(3)
+      : adjustedP.toFixed(3),
+    analysis.verdict,
+  ]
+    .join(" | ")
+    .concat(" |");
+}
+
+export function markdown(
+  metadata,
+  overall,
+  byProfile,
+  byOs,
+  decomposition,
+  problems,
+  sizes,
+) {
+  const lines = [
+    "# Action overhead",
+    "",
+    `Baseline \`${metadata.baselineRef}\` vs candidate \`${metadata.candidateRef}\` from \`${metadata.setupJavaRepository}\`, run ${metadata.runId}.`,
+    "",
+    "The cache entry here is about a megabyte, deliberately. setup-java hands the",
+    "actual transfer to `@actions/cache`, so a large entry would time the network",
+    "and hide the action's own work. A small one leaves resolving a distribution,",
+    "computing a cache key, writing settings and toolchains, and the bookkeeping",
+    "around a restore as what is measured.",
+    "",
+  ];
+
+  if (problems.length > 0) {
+    lines.push(
+      "## The cache did not behave as the profiles assume",
+      "",
+      "Every number below this point is suspect, because at least one slot did not",
+      "do the work its profile is named for.",
+      "",
+      "| Configuration | Slot | Expected `cache-hit` | Reported | Why it matters |",
+      "| --- | ---: | --- | --- | --- |",
+      ...problems
+        .slice(0, 20)
+        .map(
+          (problem) =>
+            `| ${problem.configuration} | ${problem.slot} | ${problem.expected} | ${problem.reported} | ${problem.reason} |`,
+        ),
+      "",
+    );
+  }
+
+  lines.push(
+    "## Does the candidate differ from the baseline?",
+    "",
+    `**${overall ? describeVerdict(overall.verdict) : "No usable pairs."}**`,
+    "",
+  );
+
+  if (overall) {
+    lines.push(
+      `Pooled across ${overall.pairs.length} configurations: ${seconds(overall.interval.estimate)} ${formatInterval(overall.interval)}, permutation p = ${overall.pValue.toFixed(3)}, noise floor ${seconds(overall.noiseFloorSeconds)}.`,
+      "",
+      `The A/A control — the two baseline slots against each other, which differ by nothing — reads ${overall.control.verdict}. Anything other than \`within-noise\` or \`inconclusive\` there means the harness is measuring something it should not, and the comparison above cannot be trusted.`,
+      "",
+      "| Split | Difference | Interval | p (Holm) | Verdict |",
+      "| --- | ---: | --- | ---: | --- |",
+      ...byProfile.map(({ label, analysis, adjustedP }) =>
+        analysisRow(label, analysis, adjustedP),
+      ),
+      ...byOs.map(({ label, analysis }) => analysisRow(label, analysis, null)),
+      "",
+    );
+  }
+
+  lines.push(
+    "## Where the time goes",
+    "",
+    "Observed medians, not a comparison. These are levels measured on different",
+    "runners, so they carry no interval and no verdict; they are here to say what",
+    "a setup actually costs and which part of it setup-java could change.",
+    "",
+    `| OS | ${CACHE_PROFILES.map((profile) => profile.label).join(" | ")} |`,
+    `| --- | ${CACHE_PROFILES.map(() => "---:").join(" | ")} |`,
+    ...decomposition.levels.map(
+      (level) =>
+        `| ${level.os} | ${CACHE_PROFILES.map((profile) => seconds(level.seconds[profile.id], 2)).join(" | ")} |`,
+    ),
+    "",
+    `| OS | ${STEPS.map((step) => step.label).join(" | ")} |`,
+    `| --- | ${STEPS.map(() => "---:").join(" | ")} |`,
+    ...decomposition.steps.map(
+      (entry) =>
+        `| ${entry.os} | ${entry.steps.map((step) => seconds(step.seconds, 2)).join(" | ")} |`,
+    ),
+    "",
+  );
+
+  if (sizes.length > 0) {
+    lines.push(
+      "## Bundle size",
+      "",
+      "How much JavaScript each arm has to parse before it does anything.",
+      "",
+      "| Arm | `dist/setup/index.js` bytes | `dist/setup` JS bytes | JS files |",
+      "| --- | ---: | ---: | ---: |",
+      ...sizes.map(
+        (size) =>
+          `| ${size.arm} | ${size.indexBytes} | ${size.jsBytes} | ${size.files} |`,
+      ),
+      "",
+    );
+  }
+
+  return `${lines.join("\n")}\n`;
+}
+
+async function readSizes(directory = RESULTS_DIR) {
+  let entries;
+  try {
+    entries = await readdir(directory, { recursive: true });
+  } catch {
+    return [];
+  }
+  const files = entries.filter((entry) => entry.endsWith("sizes.csv"));
+  const contents = await Promise.all(
+    files.sort().map((file) => readFile(join(directory, file), "utf8")),
+  );
+  const seen = new Map();
+  for (const line of contents.join("\n").trim().split("\n")) {
+    if (!line.trim()) continue;
+    const [arm, indexBytes, jsBytes, files_] = line.split(",");
+    if (!seen.has(arm)) {
+      seen.set(arm, { arm, indexBytes, jsBytes, files: files_ });
+    }
+  }
+  return [...seen.values()];
+}
+
+export async function main(env = process.env) {
+  requireEnv(env, [
+    "SETUP_JAVA_REPOSITORY",
+    "BASELINE_REF",
+    "CANDIDATE_REF",
+    "RUN_ID",
+  ]);
+
+  const rows = parseSamples(await readResults());
+  const { rows: paired } = toPairedRows(rows);
+  const overall =
+    paired.length > 0 ? analyzePairs(paired, "baseline", "candidate") : null;
+
+  const profileAnalyses = CACHE_PROFILES.map((profile) => ({
+    label: profile.label,
+    analysis: subsetAnalysis(rows, (row) => row.cache === profile.id),
+  }));
+  // Four cache profiles are four chances to find a difference that is not there,
+  // so the family is corrected together.
+  const adjusted = holmAdjust(
+    profileAnalyses.map(({ analysis }) => (analysis ? analysis.pValue : null)),
+  );
+  const byProfile = profileAnalyses.map((entry, index) => ({
+    ...entry,
+    adjustedP: adjusted[index],
+  }));
+
+  const byOs = [...new Set(rows.map((row) => row.os))].sort().map((os) => ({
+    label: os,
+    analysis: subsetAnalysis(rows, (row) => row.os === os),
+  }));
+
+  const problems = checkCacheIntegrity(rows);
+  const decomposition = decompose(rows);
+  const sizes = await readSizes();
+
+  const metadata = {
+    setupJavaRepository: env.SETUP_JAVA_REPOSITORY,
+    baselineRef: env.BASELINE_REF,
+    candidateRef: env.CANDIDATE_REF,
+    runId: env.RUN_ID,
+  };
+
+  const rendered = markdown(
+    metadata,
+    overall,
+    byProfile,
+    byOs,
+    decomposition,
+    problems,
+    sizes,
+  );
+
+  await mkdir(OUTPUT_DIR, { recursive: true });
+  await writeFile(
+    join(OUTPUT_DIR, "results.json"),
+    `${JSON.stringify({ metadata, overall, byProfile, byOs, decomposition, problems, sizes }, null, 2)}\n`,
+  );
+  await writeFile(join(OUTPUT_DIR, "summary.md"), rendered);
+  if (env.GITHUB_STEP_SUMMARY) {
+    await appendFile(env.GITHUB_STEP_SUMMARY, rendered);
+  }
+  return rendered;
+}
+
+if (
+  process.argv[1] &&
+  import.meta.url === pathToFileURL(process.argv[1]).href
+) {
+  await main();
+}
diff --git a/scripts/report-action-overhead.test.mjs b/scripts/report-action-overhead.test.mjs
new file mode 100644
index 0000000..884d20f
--- /dev/null
+++ b/scripts/report-action-overhead.test.mjs
@@ -0,0 +1,197 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  checkCacheIntegrity,
+  decompose,
+  markdown,
+  parseSamples,
+  subsetAnalysis,
+  toPairedRows,
+} from "./report-action-overhead.mjs";
+
+function csv(rows) {
+  return rows
+    .map(
+      ({ os, cache, layout, arm, slot, cacheHit, ms }) =>
+        `"${os}","${cache}","${layout}","${arm}","${slot}","${cacheHit}","${ms}"`,
+    )
+    .join("\n");
+}
+
+function matrix({ shift = 0 } = {}) {
+  const rows = [];
+  const base = {
+    none: 2000,
+    "maven-miss": 2400,
+    "maven-hit": 3100,
+    "gradle-miss": 2380,
+  };
+  for (const os of ["ubuntu-latest", "macos-15-intel"]) {
+    for (const cache of Object.keys(base)) {
+      for (const layout of ["simple", "complex"]) {
+        const hit =
+          cache === "maven-hit"
+            ? "true"
+            : cache === "maven-miss"
+              ? "false"
+              : "unset";
+        const offset = layout === "complex" ? 300 : 0;
+        const drift = os === "macos-15-intel" ? 900 : 0;
+        const at = base[cache] + offset + drift;
+        rows.push(
+          {
+            os,
+            cache,
+            layout,
+            arm: "baseline",
+            slot: 1,
+            cacheHit: hit,
+            ms: at,
+          },
+          {
+            os,
+            cache,
+            layout,
+            arm: "candidate",
+            slot: 2,
+            cacheHit: hit,
+            ms: at + shift,
+          },
+          {
+            os,
+            cache,
+            layout,
+            arm: "candidate",
+            slot: 3,
+            cacheHit: hit,
+            ms: at + shift,
+          },
+          {
+            os,
+            cache,
+            layout,
+            arm: "baseline",
+            slot: 4,
+            cacheHit: hit,
+            ms: at,
+          },
+        );
+      }
+    }
+  }
+  return parseSamples(csv(rows));
+}
+
+test("parses quoted rows and converts milliseconds to seconds", () => {
+  const rows = parseSamples(
+    '"ubuntu-latest","maven-hit","simple","baseline","1","true","3100"',
+  );
+  assert.equal(rows.length, 1);
+  assert.equal(rows[0].seconds, 3.1);
+  assert.equal(rows[0].configuration, "ubuntu-latest/maven-hit/simple");
+});
+
+test("treats each configuration as one block", () => {
+  const { configurations, rows } = toPairedRows(matrix());
+  assert.equal(configurations.length, 16);
+  assert.equal(new Set(rows.map((row) => row.sample)).size, 16);
+});
+
+// The decomposition is the reason this scenario uses a tiny entry, so the
+// arithmetic that produces it has to be pinned.
+test("differences the cache profiles into named steps", () => {
+  const { levels, steps } = decompose(matrix());
+  const ubuntu = levels.find((level) => level.os === "ubuntu-latest");
+  assert.equal(ubuntu.seconds.none, 2.15);
+  assert.equal(ubuntu.seconds["maven-miss"], 2.55);
+
+  const ubuntuSteps = steps.find((entry) => entry.os === "ubuntu-latest").steps;
+  assert.match(ubuntuSteps[0].label, /asking the cache service/);
+  assert.ok(Math.abs(ubuntuSteps[0].seconds - 0.4) < 1e-9);
+  assert.ok(Math.abs(ubuntuSteps[1].seconds - 0.7) < 1e-9);
+});
+
+test("reports no difference when the arms are identical", () => {
+  const analysis = subsetAnalysis(matrix(), () => true);
+  assert.ok(["within-noise", "inconclusive"].includes(analysis.verdict));
+});
+
+test("differences candidate minus baseline, so a slower candidate is positive", () => {
+  const analysis = subsetAnalysis(matrix({ shift: 500 }), () => true);
+  assert.ok(analysis.interval.estimate > 0);
+});
+
+test("flags a hit profile that actually missed", () => {
+  const rows = matrix().map((row) =>
+    row.cache === "maven-hit" && row.os === "ubuntu-latest"
+      ? { ...row, cacheHit: "false" }
+      : row,
+  );
+  const problems = checkCacheIntegrity(rows);
+  assert.ok(problems.length > 0);
+  assert.ok(problems.every((problem) => problem.expected === "true"));
+  assert.match(problems[0].reason, /did not do the work/);
+});
+
+// A ref too old to publish `cache-hit` cannot be verified, and silently passing
+// it would let the decomposition be built out of slots that never restored.
+test("flags an unverifiable slot separately from a wrong one", () => {
+  const rows = matrix().map((row) =>
+    row.cache === "maven-hit" ? { ...row, cacheHit: "unset" } : row,
+  );
+  const problems = checkCacheIntegrity(rows);
+  assert.ok(problems.length > 0);
+  assert.match(problems[0].reason, /does not publish/);
+});
+
+test("accepts a matrix where every profile behaved", () => {
+  assert.deepEqual(checkCacheIntegrity(matrix()), []);
+});
+
+test("leads with the integrity failure when there is one", () => {
+  const rows = matrix().map((row) =>
+    row.cache === "maven-hit" ? { ...row, cacheHit: "false" } : row,
+  );
+  const rendered = markdown(
+    {
+      setupJavaRepository: "actions/setup-java",
+      baselineRef: "main",
+      candidateRef: "main",
+      runId: "1",
+    },
+    subsetAnalysis(rows, () => true),
+    [],
+    [],
+    decompose(rows),
+    checkCacheIntegrity(rows),
+    [],
+  );
+  assert.match(rendered, /## The cache did not behave as the profiles assume/);
+  assert.ok(
+    rendered.indexOf("did not behave") <
+      rendered.indexOf("Does the candidate differ"),
+  );
+});
+
+// Blank lines are what make the output render as markdown rather than as one
+// run-on paragraph, so filtering them out has to stay impossible.
+test("keeps the blank lines that separate sections", () => {
+  const rows = matrix();
+  const rendered = markdown(
+    {
+      setupJavaRepository: "actions/setup-java",
+      baselineRef: "v4.8.0",
+      candidateRef: "main",
+      runId: "1",
+    },
+    subsetAnalysis(rows, () => true),
+    [],
+    [],
+    decompose(rows),
+    [],
+    [{ arm: "baseline", indexBytes: "1", jsBytes: "2", files: "3" }],
+  );
+  assert.match(rendered, /\n\n## Where the time goes\n\n/);
+  assert.match(rendered, /## Bundle size/);
+});
diff --git a/scripts/report-cache-save.mjs b/scripts/report-cache-save.mjs
new file mode 100644
index 0000000..77f0ee0
--- /dev/null
+++ b/scripts/report-cache-save.mjs
@@ -0,0 +1,215 @@
+import { appendFile, mkdir, writeFile } from "node:fs/promises";
+import { pathToFileURL } from "node:url";
+
+import {
+  analyzePairs,
+  parseSamples,
+  readSampleFiles as readPairedSampleFiles,
+  requireEnv,
+} from "./paired.mjs";
+import { describeVerdict, formatInterval } from "./stats.mjs";
+
+const API_VERSION = "2022-11-28";
+
+export function readSampleFiles(directory) {
+  return readPairedSampleFiles("cache-save-timings", directory);
+}
+
+export function analyze(rows) {
+  return analyzePairs(rows, "baseline", "candidate");
+}
+
+function csvValue(value) {
+  return `"${String(value ?? "").replaceAll('"', '""')}"`;
+}
+
+function throughput(fixtureMiB, seconds) {
+  return fixtureMiB / seconds;
+}
+
+async function api(path, token, options = {}) {
+  const response = await fetch(`https://api.github.com${path}`, {
+    ...options,
+    headers: {
+      Accept: "application/vnd.github+json",
+      Authorization: `Bearer ${token}`,
+      "X-GitHub-Api-Version": API_VERSION,
+      ...options.headers,
+    },
+  });
+  if (!response.ok) {
+    throw new Error(`${options.method ?? "GET"} ${path}: ${response.status}`);
+  }
+  if (response.status === 204) return null;
+  return response.json();
+}
+
+async function allPages(path, field, token) {
+  const values = [];
+  for (let page = 1; ; page += 1) {
+    const separator = path.includes("?") ? "&" : "?";
+    const response = await api(
+      `${path}${separator}per_page=100&page=${page}`,
+      token,
+    );
+    values.push(...response[field]);
+    if (response[field].length < 100) return values;
+  }
+}
+
+export function markdown(metadata, analysis, caches = []) {
+  const { baseline, candidate, interval, control } = analysis;
+  const fixtureMiB = Number(metadata.fixtureMiB);
+  const lines = [
+    "# Cache save benchmark",
+    "",
+    `${fixtureMiB} MiB Maven fixture, ${analysis.pairs.length} runners x 2 paired observations, run ${metadata.runId}.`,
+    `Baseline \`${metadata.baselineRef}\` vs candidate \`${metadata.candidateRef}\` from \`${metadata.setupJavaRepository}\`.`,
+    "",
+    "## Verdict",
+    "",
+    `**${describeVerdict(analysis.verdict)}**`,
+    "",
+    `Paired difference (candidate - baseline): **${formatInterval(interval, { digits: 3 })}**.`,
+    `Permutation p-value: ${analysis.pValue.toFixed(3)}. Hodges-Lehmann shift: ${analysis.shiftSeconds.toFixed(3)}s.`,
+    `Harness noise floor: ${analysis.noiseFloorSeconds.toFixed(3)}s (median within-runner repeat spread).`,
+    "",
+    `A/A control (baseline against itself) reports **${control.verdict}** at ${formatInterval(control.interval, { digits: 3 })}. A healthy harness reports \`within-noise\` or \`inconclusive\` here; an \`improvement\` or \`regression\` means slot ordering is biasing results and the verdict above cannot be trusted.`,
+    "",
+    analysis.droppedRunners.length === 0
+      ? "No runner was discarded for a stalled slot."
+      : `Discarded ${analysis.droppedRunners.length} runner(s) whose own arm disagreed with itself by more than ${analysis.stallThresholdSeconds.toFixed(3)}s, a threshold derived from the spread of the other runners: ${analysis.droppedRunners.map((entry) => `#${entry.sample} (${entry.repeatSpread.toFixed(3)}s)`).join(", ")}. A stalled slot lands on an arbitrary arm and would otherwise dominate the mean; the decision uses only within-arm spread, which carries no information about the effect.`,
+    "",
+    "## Arms",
+    "",
+    "| Arm | Runners | Mean (s) | Median (s) | SD (s) | MAD (s) | p95 (s) | Mean throughput (MiB/s) | Median throughput (MiB/s) |",
+    "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
+  ];
+  for (const summary of [baseline, candidate]) {
+    lines.push(
+      `| ${summary.arm} | ${summary.samples} | ${summary.meanSeconds.toFixed(3)} | ${summary.medianSeconds.toFixed(3)} | ${summary.standardDeviationSeconds?.toFixed(3) ?? "n/a"} | ${summary.madSeconds.toFixed(3)} | ${summary.p95Seconds.toFixed(3)} | ${throughput(fixtureMiB, summary.meanSeconds).toFixed(1)} | ${throughput(fixtureMiB, summary.medianSeconds).toFixed(1)} |`,
+    );
+  }
+  lines.push(
+    "",
+    "The fixture is byte-identical for every slot. Distinct keys are still required because a cache save for an existing key is a no-op; that creates fresh blobs whose placement can affect upload throughput, which is the dominant noise source for this scenario.",
+    "",
+    "## Paired samples",
+    "",
+    "| Runner | baseline slot 1 (s) | candidate slot 2 (s) | candidate slot 3 (s) | baseline slot 4 (s) | Paired delta (s) |",
+    "| ---: | ---: | ---: | ---: | ---: | ---: |",
+  );
+  for (const pair of analysis.pairs) {
+    lines.push(
+      `| ${pair.sample} | ${pair.baselineSlots[0].toFixed(3)} | ${pair.candidateSlots[0].toFixed(3)} | ${pair.candidateSlots[1].toFixed(3)} | ${pair.baselineSlots[1].toFixed(3)} | ${pair.difference.toFixed(3)} |`,
+    );
+  }
+  lines.push(
+    "",
+    "## Cache entries",
+    "",
+    caches.length === 0
+      ? "No matching cache entries were found when the report was generated."
+      : `Found ${caches.length} cache entries with the run-scoped prefix \`${metadata.cacheKeyPrefix}\`.`,
+    "",
+    "All durations are measured inside the job with millisecond resolution by timing only the \`@actions/cache.saveCache\` call after Node has started and the module has loaded. setup-java delegates Maven cache transfers to that toolkit function, so this reports transfer-and-archive time without relying on post-job step timestamps from the Actions API or including process startup.",
+  );
+  return `${lines.filter((line) => line !== null).join("\n")}\n`;
+}
+
+export async function main(env = process.env) {
+  requireEnv(env, [
+    "GITHUB_REPOSITORY",
+    "GH_TOKEN",
+    "GITHUB_RUN_ID",
+    "BASELINE_REF",
+    "CANDIDATE_REF",
+    "SETUP_JAVA_REPOSITORY",
+    "FIXTURE_MIB",
+    "CACHE_KEY_PREFIX",
+    "GITHUB_STEP_SUMMARY",
+  ]);
+
+  const [owner, repo] = env.GITHUB_REPOSITORY.split("/");
+  if (!owner || !repo) {
+    throw new Error(
+      `GITHUB_REPOSITORY must be owner/repo, got "${env.GITHUB_REPOSITORY}"`,
+    );
+  }
+
+  const token = env.GH_TOKEN;
+  const cacheEntries = await allPages(
+    `/repos/${owner}/${repo}/actions/caches`,
+    "actions_caches",
+    token,
+  );
+  const caches = cacheEntries
+    .filter((cache) => cache.key.startsWith(env.CACHE_KEY_PREFIX))
+    .map((cache) => ({
+      id: cache.id,
+      key: cache.key,
+      sizeBytes: cache.size_in_bytes,
+    }));
+
+  const metadata = {
+    repository: env.GITHUB_REPOSITORY,
+    runId: env.GITHUB_RUN_ID,
+    setupJavaRepository: env.SETUP_JAVA_REPOSITORY,
+    baselineRef: env.BASELINE_REF,
+    candidateRef: env.CANDIDATE_REF,
+    fixtureMiB: Number(env.FIXTURE_MIB),
+    cacheKeyPrefix: env.CACHE_KEY_PREFIX,
+    generatedAt: new Date().toISOString(),
+  };
+
+  try {
+    const rows = parseSamples(await readSampleFiles());
+    const analysis = analyze(rows);
+    if (analysis.pairs.length === 0) {
+      throw new Error("No complete ABBA samples were collected");
+    }
+    const report = markdown(metadata, analysis, caches);
+
+    await mkdir("cache-save-results", { recursive: true });
+    await writeFile(
+      "cache-save-results/results.json",
+      `${JSON.stringify({ metadata, analysis, caches }, null, 2)}\n`,
+    );
+    await writeFile(
+      "cache-save-results/results.csv",
+      `sample,arm,slot,seconds\n${rows
+        .map((row) =>
+          [row.sample, row.arm, row.slot, row.seconds].map(csvValue).join(","),
+        )
+        .join("\n")}\n`,
+    );
+    await writeFile("cache-save-results/summary.md", report);
+    await appendFile(env.GITHUB_STEP_SUMMARY, report);
+  } finally {
+    const deletionFailures = [];
+    let deleted = 0;
+    for (const cache of caches) {
+      try {
+        await api(`/repos/${owner}/${repo}/actions/caches/${cache.id}`, token, {
+          method: "DELETE",
+        });
+        deleted += 1;
+      } catch (error) {
+        deletionFailures.push(`${cache.key}: ${error.message}`);
+      }
+    }
+    console.log(`Deleted ${deleted} cache-save benchmark caches`);
+    if (deletionFailures.length > 0) {
+      throw new Error(
+        `Failed to delete ${deletionFailures.length} cache-save benchmark cache(s): ${deletionFailures.join("; ")}`,
+      );
+    }
+  }
+}
+
+if (
+  process.argv[1] &&
+  import.meta.url === pathToFileURL(process.argv[1]).href
+) {
+  await main();
+}
diff --git a/scripts/report-cache-save.test.mjs b/scripts/report-cache-save.test.mjs
new file mode 100644
index 0000000..c19ceea
--- /dev/null
+++ b/scripts/report-cache-save.test.mjs
@@ -0,0 +1,172 @@
+import assert from "node:assert/strict";
+import { rm } from "node:fs/promises";
+import { join } from "node:path";
+import { spawnSync } from "node:child_process";
+import test from "node:test";
+
+import { analyze, markdown } from "./report-cache-save.mjs";
+import { parseSamples } from "./paired.mjs";
+
+function rowsFromSlots(slots) {
+  return parseSamples(
+    slots
+      .map(([sample, b1, c2, c3, b4]) =>
+        [
+          `"${sample}","baseline","1","${b1}"`,
+          `"${sample}","candidate","2","${c2}"`,
+          `"${sample}","candidate","3","${c3}"`,
+          `"${sample}","baseline","4","${b4}"`,
+        ].join("\n"),
+      )
+      .join("\n"),
+  );
+}
+
+test("analyzes cache-save samples with candidate minus baseline direction", () => {
+  const slots = [];
+  for (let sample = 1; sample <= 10; sample += 1) {
+    const offset = sample * 250;
+    slots.push([
+      sample,
+      2000 + offset,
+      2600 + offset,
+      2620 + offset,
+      2020 + offset,
+    ]);
+  }
+  const analysis = analyze(rowsFromSlots(slots));
+  assert.equal(analysis.verdict, "regression");
+  assert.ok(analysis.interval.low > 0);
+  assert.ok(analysis.shiftSeconds > 0);
+});
+
+test("reports a large consistent saving as an improvement", () => {
+  const slots = [];
+  for (let sample = 1; sample <= 10; sample += 1) {
+    const offset = sample * 300;
+    slots.push([
+      sample,
+      3200 + offset,
+      2100 + offset,
+      2120 + offset,
+      3220 + offset,
+    ]);
+  }
+  const analysis = analyze(rowsFromSlots(slots));
+  assert.equal(analysis.verdict, "improvement");
+  assert.ok(analysis.interval.high < 0);
+});
+
+test("identical inputs do not produce a headline verdict", () => {
+  const slots = [];
+  for (let sample = 1; sample <= 10; sample += 1) {
+    const offset = sample * 175;
+    slots.push([
+      sample,
+      2400 + offset,
+      2400 + offset,
+      2400 + offset,
+      2400 + offset,
+    ]);
+  }
+  const analysis = analyze(rowsFromSlots(slots));
+  assert.ok(["within-noise", "inconclusive"].includes(analysis.verdict));
+  assert.ok(
+    ["within-noise", "inconclusive"].includes(analysis.control.verdict),
+  );
+});
+
+test("drops incomplete ABBA runners", () => {
+  const rows = parseSamples(
+    [
+      '"1","baseline","1","2000"',
+      '"1","candidate","2","1900"',
+      '"1","candidate","3","1910"',
+      '"1","baseline","4","2010"',
+      '"2","baseline","1","2000"',
+      '"2","candidate","2","1900"',
+    ].join("\n"),
+  );
+  const analysis = analyze(rows);
+  assert.equal(analysis.pairs.length, 1);
+  assert.equal(analysis.pairs[0].sample, 1);
+});
+
+test("renders throughput, guard rails, and paired samples", () => {
+  const analysis = analyze(
+    rowsFromSlots([
+      [1, 2400, 1900, 1950, 2500],
+      [2, 3100, 2600, 2500, 3000],
+    ]),
+  );
+  const report = markdown(
+    {
+      runId: "123",
+      setupJavaRepository: "actions/setup-java",
+      baselineRef: "v4.8.0",
+      candidateRef: "main",
+      fixtureMiB: 64,
+      cacheKeyPrefix: "cache-save-123-",
+    },
+    analysis,
+    [{ id: 1, key: "cache-save-123-1", sizeBytes: 64 * 1024 * 1024 }],
+  );
+  assert.match(report, /# Cache save benchmark/);
+  assert.match(report, /Mean throughput \(MiB\/s\)/);
+  assert.match(report, /A\/A control/);
+  assert.match(report, /Paired samples/);
+  assert.match(report, /@actions\/cache\.saveCache/);
+  assert.match(report, /without .*including process startup/);
+});
+
+test("fixture verification asserts exact file count", async () => {
+  const home = join(process.cwd(), ".cache-save-test-home");
+  const env = { ...process.env, HOME: home };
+  try {
+    const prepare = spawnSync(
+      "bash",
+      ["scripts/cache-save.sh", "prepare-fixture", "1"],
+      { cwd: process.cwd(), env, encoding: "utf8" },
+    );
+    assert.equal(prepare.status, 0, prepare.stderr);
+
+    const expected = spawnSync(
+      "bash",
+      ["scripts/cache-save.sh", "expected-file-count", "1"],
+      { cwd: process.cwd(), env, encoding: "utf8" },
+    );
+    assert.equal(expected.status, 0, expected.stderr);
+
+    const actual = spawnSync("bash", ["scripts/cache-save.sh", "file-count"], {
+      cwd: process.cwd(),
+      env,
+      encoding: "utf8",
+    });
+    assert.equal(actual.status, 0, actual.stderr);
+    assert.equal(actual.stdout.trim(), expected.stdout.trim());
+
+    const verify = spawnSync(
+      "bash",
+      ["scripts/cache-save.sh", "verify-fixture", "1"],
+      { cwd: process.cwd(), env, encoding: "utf8" },
+    );
+    assert.equal(verify.status, 0, verify.stderr);
+
+    const removeOne = spawnSync(
+      "bash",
+      ["scripts/cache-save.sh", "remove-one-file"],
+      { cwd: process.cwd(), env, encoding: "utf8" },
+    );
+    assert.equal(removeOne.status, 0, removeOne.stderr);
+
+    const failedVerify = spawnSync(
+      "bash",
+      ["scripts/cache-save.sh", "verify-fixture", "1"],
+      { cwd: process.cwd(), env, encoding: "utf8" },
+    );
+    assert.notEqual(failedVerify.status, 0);
+    assert.match(failedVerify.stderr, /expected/);
+  } finally {
+    await rm(home, { recursive: true, force: true });
+  }
+});
diff --git a/scripts/report-cache-value.mjs b/scripts/report-cache-value.mjs
new file mode 100644
index 0000000..11af807
--- /dev/null
+++ b/scripts/report-cache-value.mjs
@@ -0,0 +1,213 @@
+import { appendFile, mkdir, writeFile } from "node:fs/promises";
+import { pathToFileURL } from "node:url";
+
+import { hashFilesSingle } from "./report.mjs";
+import {
+  analyzePairs,
+  parseSamples,
+  readSampleFiles as readPairedSampleFiles,
+  requireEnv,
+} from "./paired.mjs";
+import { describeVerdict, formatInterval } from "./stats.mjs";
+
+export function readSampleFiles(directory) {
+  return readPairedSampleFiles("cache-value-timings", directory);
+}
+
+// The difference is cached minus uncached, so the cache saving time reads as a
+// negative number and an `improvement`, the same direction every other workflow
+// in this repository uses for the arm under test.
+export function analyze(rows) {
+  return analyzePairs(rows, "uncached", "cached");
+}
+
+const API_VERSION = "2022-11-28";
+
+function csvValue(value) {
+  return `"${String(value ?? "").replaceAll('"', '""')}"`;
+}
+
+async function api(path, token, options = {}) {
+  const response = await fetch(`https://api.github.com${path}`, {
+    ...options,
+    headers: {
+      Accept: "application/vnd.github+json",
+      Authorization: `Bearer ${token}`,
+      "X-GitHub-Api-Version": API_VERSION,
+      ...options.headers,
+    },
+  });
+  if (!response.ok) {
+    throw new Error(`${options.method ?? "GET"} ${path}: ${response.status}`);
+  }
+  if (response.status === 204) return null;
+  return response.json();
+}
+
+async function allPages(path, field, token) {
+  const values = [];
+  for (let page = 1; ; page += 1) {
+    const separator = path.includes("?") ? "&" : "?";
+    const response = await api(
+      `${path}${separator}per_page=100&page=${page}`,
+      token,
+    );
+    values.push(...response[field]);
+    if (response[field].length < 100) return values;
+  }
+}
+
+// The saving is quoted as a ratio as well as a difference because the difference
+// alone is not portable: it is a property of this dependency tree on this
+// runner, and a reader comparing it with their own project needs to know
+// whether the cache removed most of the work or a little of it.
+function speedup(analysis) {
+  const uncached = analysis.baseline.meanSeconds;
+  const cached = analysis.candidate.meanSeconds;
+  if (cached <= 0) return null;
+  return uncached / cached;
+}
+
+export function markdown(metadata, analysis, caches) {
+  const { baseline, candidate, interval, control } = analysis;
+  const ratio = speedup(analysis);
+  const lines = [
+    "# Maven cache value",
+    "",
+    `Temurin ${metadata.javaVersion}, ${analysis.pairs.length} runners x 2 paired observations, run ${metadata.runId}.`,
+    `\`${metadata.setupJavaRef}\` from \`${metadata.setupJavaRepository}\`, resolving Spring PetClinic.`,
+    "",
+    "Both arms run the same `dependency:go-offline` against the same dependency",
+    "tree. They differ only in whether setup-java restored the local repository",
+    "first, so the difference between them is what the cache is worth on this",
+    "project rather than a detail of how the cache is implemented.",
+    "",
+    "## Verdict",
+    "",
+    `**${describeVerdict(analysis.verdict)}**`,
+    "",
+    `Time to a resolvable project, cached minus uncached: **${formatInterval(interval, { digits: 2 })}**.`,
+    ratio === null
+      ? null
+      : `That is **${ratio.toFixed(1)}x faster** with the cache: ${baseline.meanSeconds.toFixed(1)}s without it, ${candidate.meanSeconds.toFixed(1)}s with it.`,
+    `Permutation p-value: ${analysis.pValue.toFixed(3)}. Hodges-Lehmann shift: ${analysis.shiftSeconds.toFixed(2)}s.`,
+    `Harness noise floor: ${analysis.noiseFloorSeconds.toFixed(2)}s (median within-runner repeat spread).`,
+    "",
+    `A/A control (the uncached arm against itself) reports **${control.verdict}** at ${formatInterval(control.interval, { digits: 2 })}. A healthy harness reports \`within-noise\` or \`inconclusive\` here; anything else means slot ordering is biasing results and the verdict above cannot be trusted.`,
+    "",
+    analysis.droppedRunners.length === 0
+      ? "No runner was discarded for a stalled slot."
+      : `Discarded ${analysis.droppedRunners.length} runner(s) whose own arm disagreed with itself by more than ${analysis.stallThresholdSeconds.toFixed(2)}s: ${analysis.droppedRunners.map((entry) => `#${entry.sample} (${entry.repeatSpread.toFixed(2)}s)`).join(", ")}. A stalled slot lands on an arbitrary arm and would otherwise dominate the mean.`,
+    "",
+    "## Arms",
+    "",
+    "| Arm | Runners | Mean (s) | Median (s) | SD (s) | MAD (s) | p95 (s) |",
+    "| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
+    `| uncached | ${baseline.samples} | ${baseline.meanSeconds.toFixed(2)} | ${baseline.medianSeconds.toFixed(2)} | ${baseline.standardDeviationSeconds?.toFixed(2) ?? "n/a"} | ${baseline.madSeconds.toFixed(2)} | ${baseline.p95Seconds.toFixed(2)} |`,
+    `| cached | ${candidate.samples} | ${candidate.meanSeconds.toFixed(2)} | ${candidate.medianSeconds.toFixed(2)} | ${candidate.standardDeviationSeconds?.toFixed(2) ?? "n/a"} | ${candidate.madSeconds.toFixed(2)} | ${candidate.p95Seconds.toFixed(2)} |`,
+    "",
+    "The uncached arm is the more variable of the two by a wide margin, because it",
+    "depends on Maven Central rather than on the Actions cache service. That",
+    "variance is real and belongs in the number: it is what a user without a cache",
+    "actually experiences.",
+    "",
+    "## Paired samples",
+    "",
+    "| Runner | cached slot 1 (s) | uncached slot 2 (s) | uncached slot 3 (s) | cached slot 4 (s) | Paired delta (s) |",
+    "| ---: | ---: | ---: | ---: | ---: | ---: |",
+  ];
+  for (const pair of analysis.pairs) {
+    lines.push(
+      `| ${pair.sample} | ${pair.candidateSlots[0].toFixed(2)} | ${pair.baselineSlots[0].toFixed(2)} | ${pair.baselineSlots[1].toFixed(2)} | ${pair.candidateSlots[1].toFixed(2)} | ${pair.difference.toFixed(2)} |`,
+    );
+  }
+  lines.push("", "## Caches", "", "| Cache | Size (MiB) |", "| --- | ---: |");
+  for (const cache of caches) {
+    lines.push(
+      `| \`${cache.key}\` | ${(cache.sizeBytes / (1024 * 1024)).toFixed(1)} |`,
+    );
+  }
+  // Only the optional ratio line is dropped. Filtering on "" instead would
+  // collapse every intentional blank line and destroy the markdown structure.
+  return `${lines.filter((line) => line !== null).join("\n")}\n`;
+}
+
+export async function main(env = process.env) {
+  const token = requireEnv(env, "GH_TOKEN");
+  const runId = requireEnv(env, "GITHUB_RUN_ID");
+  const setupJavaRepository = requireEnv(env, "SETUP_JAVA_REPOSITORY");
+  const setupJavaRef = requireEnv(env, "SETUP_JAVA_REF");
+  const [owner, repo] = requireEnv(env, "GITHUB_REPOSITORY").split("/");
+  if (!owner || !repo) {
+    throw new Error(
+      `GITHUB_REPOSITORY must be owner/repo, got "${env.GITHUB_REPOSITORY}"`,
+    );
+  }
+
+  const rows = parseSamples(await readSampleFiles());
+  const analysis = analyze(rows);
+  if (analysis.pairs.length === 0) {
+    throw new Error("No complete ABBA samples were collected");
+  }
+
+  const cacheEntries = await allPages(
+    `/repos/${owner}/${repo}/actions/caches`,
+    "actions_caches",
+    token,
+  );
+  // Only the dependency entry is identified here. The wrapper entry is keyed on
+  // PetClinic's own wrapper properties, which this benchmark does not modify, so
+  // it is shared with every other run and must not be deleted by one of them.
+  const dependencyKey = `setup-java-Linux-x64-maven-${hashFilesSingle(
+    `cache-value-${runId}\n`,
+  )}`;
+  const entry = cacheEntries.find((cache) => cache.key === dependencyKey);
+  if (!entry) {
+    throw new Error(`Expected cache not found: ${dependencyKey}`);
+  }
+  const caches = [
+    { key: dependencyKey, id: entry.id, sizeBytes: entry.size_in_bytes },
+  ];
+
+  const metadata = {
+    repository: env.GITHUB_REPOSITORY,
+    runId,
+    javaVersion: env.JAVA_VERSION,
+    setupJavaRepository,
+    setupJavaRef,
+    generatedAt: new Date().toISOString(),
+  };
+  const report = markdown(metadata, analysis, caches);
+
+  await mkdir("cache-value-results", { recursive: true });
+  await writeFile(
+    "cache-value-results/results.json",
+    `${JSON.stringify({ metadata, analysis, caches }, null, 2)}\n`,
+  );
+  await writeFile(
+    "cache-value-results/results.csv",
+    `sample,arm,slot,seconds\n${rows
+      .map((row) =>
+        [row.sample, row.arm, row.slot, row.seconds].map(csvValue).join(","),
+      )
+      .join("\n")}\n`,
+  );
+  await writeFile("cache-value-results/summary.md", report);
+  await appendFile(env.GITHUB_STEP_SUMMARY, report);
+
+  if (env.CLEANUP_CACHES === "true") {
+    for (const cache of caches) {
+      await api(`/repos/${owner}/${repo}/actions/caches/${cache.id}`, token, {
+        method: "DELETE",
+      });
+    }
+    console.log(`Deleted ${caches.length} cache value benchmark caches`);
+  }
+}
+
+if (
+  process.argv[1] &&
+  import.meta.url === pathToFileURL(process.argv[1]).href
+) {
+  await main();
+}
diff --git a/scripts/report-cache-value.test.mjs b/scripts/report-cache-value.test.mjs
new file mode 100644
index 0000000..6856de0
--- /dev/null
+++ b/scripts/report-cache-value.test.mjs
@@ -0,0 +1,63 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { analyze, markdown } from "./report-cache-value.mjs";
+import { parseSamples } from "./paired.mjs";
+
+// Slot order in the workflow is cached, uncached, uncached, cached.
+function sample(runner, cached, uncached) {
+  return [
+    `"${runner}","cached","1","${cached}"`,
+    `"${runner}","uncached","2","${uncached}"`,
+    `"${runner}","uncached","3","${uncached}"`,
+    `"${runner}","cached","4","${cached}"`,
+  ].join("\n");
+}
+
+const rows = parseSamples(
+  [1, 2, 3, 4, 5, 6]
+    .map((runner) => sample(runner, 8000 + runner * 500, 46000 + runner * 3000))
+    .join("\n"),
+);
+
+test("reports the cache as an improvement, not a regression", () => {
+  const analysis = analyze(rows);
+  assert.ok(
+    analysis.interval.estimate < 0,
+    "cached minus uncached must be negative when the cache saves time",
+  );
+  assert.equal(analysis.verdict, "improvement");
+});
+
+test("quotes the saving as a ratio as well as a difference", () => {
+  const analysis = analyze(rows);
+  const rendered = markdown(
+    {
+      runId: "1",
+      javaVersion: "17",
+      setupJavaRepository: "actions/setup-java",
+      setupJavaRef: "main",
+    },
+    analysis,
+    [],
+  );
+  // 56.5s uncached against 9.75s cached.
+  assert.match(rendered, /5\.8x faster/);
+  assert.match(rendered, /Maven cache value/);
+});
+
+// The failure this benchmark must never hide is a silent cache miss: both arms
+// would then do the same download and the report would state, with a tight
+// interval, that the cache is worth nothing. The workflow guards this with
+// setup-java's cache-hit output, and the analysis must not paper over it either.
+test("does not claim a saving when both arms did the same work", () => {
+  const identical = parseSamples(
+    [1, 2, 3, 4, 5, 6]
+      .map((runner) =>
+        sample(runner, 46000 + runner * 3000, 46000 + runner * 3000),
+      )
+      .join("\n"),
+  );
+  const analysis = analyze(identical);
+  assert.ok(["within-noise", "inconclusive"].includes(analysis.verdict));
+});
diff --git a/scripts/report-maven-configuration.mjs b/scripts/report-maven-configuration.mjs
deleted file mode 100644
index 376540a..0000000
--- a/scripts/report-maven-configuration.mjs
+++ /dev/null
@@ -1,195 +0,0 @@
-// Cross-matrix report for the Maven configuration warm path.
-//
-// Unlike the other workflows, this one does not repeat one scenario across many
-// runners. It runs 36 different configurations once each, so a single cell
-// yields one paired difference and cannot support an interval on its own.
-//
-// Each configuration is instead treated as a block: the arms are compared within
-// it, on the same runner, in ABBA order, and the resulting differences are pooled
-// across the matrix. That answers the question this workflow actually asks —
-// whether the candidate differs from the baseline across configurations — and it
-// costs no extra jobs. Per-configuration numbers are still published, but as
-// single observations with no verdict attached, because that is all they are.
-
-import {
-  appendFile,
-  mkdir,
-  readdir,
-  readFile,
-  writeFile,
-} from "node:fs/promises";
-import { join } from "node:path";
-import { pathToFileURL } from "node:url";
-
-import { analyzePairs, requireEnv } from "./paired.mjs";
-import { describeVerdict, formatInterval } from "./stats.mjs";
-
-const RESULTS_DIR = ".benchmark-results";
-const OUTPUT_DIR = "maven-config-results";
-
-// os,cache,versions,toolchains,implementation,iteration,elapsedMs
-export function parseConfigurationSamples(csv) {
-  return csv
-    .trim()
-    .split("\n")
-    .filter(Boolean)
-    .map((line) => {
-      const [os, cache, versions, toolchains, arm, slot, elapsedMs] =
-        line.split(",");
-      return {
-        configuration: `${os}/${cache}/${versions}/${toolchains}`,
-        os,
-        cache,
-        versions,
-        toolchains,
-        arm,
-        slot: Number(slot),
-        seconds: Number(elapsedMs) / 1000,
-      };
-    });
-}
-
-// paired.mjs groups by a numeric `sample`, so each configuration is assigned a
-// stable index and becomes one block.
-export function toPairedRows(rows) {
-  const configurations = [
-    ...new Set(rows.map((row) => row.configuration)),
-  ].sort();
-  const index = new Map(configurations.map((name, at) => [name, at + 1]));
-  return {
-    configurations,
-    rows: rows.map((row) => ({
-      sample: index.get(row.configuration),
-      arm: row.arm,
-      slot: row.slot,
-      seconds: row.seconds,
-    })),
-  };
-}
-
-export async function readResults(directory = RESULTS_DIR) {
-  const entries = await readdir(directory, { recursive: true });
-  const files = entries.filter((entry) => entry.endsWith("timings.csv"));
-  if (files.length === 0) {
-    throw new Error(`No timings.csv files found in ${directory}`);
-  }
-  const contents = await Promise.all(
-    files.sort().map((file) => readFile(join(directory, file), "utf8")),
-  );
-  return contents.join("\n");
-}
-
-function subsetAnalysis(rows, predicate) {
-  const subset = rows.filter(predicate);
-  if (subset.length === 0) return null;
-  const { rows: paired } = toPairedRows(subset);
-  const analysis = analyzePairs(paired, "baseline", "candidate");
-  return analysis.pairs.length === 0 ? null : analysis;
-}
-
-export function markdown(metadata, overall, byOs, byCache, configurations) {
-  const lines = [
-    "# Maven configuration warm path",
-    "",
-    `Baseline \`${metadata.baselineRef}\` vs candidate \`${metadata.candidateRef}\` from \`${metadata.setupJavaRepository}\`, run ${metadata.runId}.`,
-    "",
-    `${configurations.length} configurations, each measured once in ABBA order on its own runner after a discarded warm-up slot. A configuration is a block: the arms are compared within it, and the differences are pooled across the matrix. No single configuration supports a verdict on its own, because one runner yields one difference.`,
-    "",
-    "## Verdict across all configurations",
-    "",
-    `**${describeVerdict(overall.verdict)}**`,
-    "",
-    `Pooled paired difference (candidate - baseline): **${formatInterval(overall.interval, { digits: 3 })}**.`,
-    `Permutation p-value: ${overall.pValue.toFixed(3)}. Blocks: ${overall.pairs.length}.`,
-    `Harness noise floor: ${overall.noiseFloorSeconds.toFixed(3)}s (median spread between an arm's own two slots within a configuration).`,
-    "",
-    `A/A control (baseline against itself) reports **${overall.control.verdict}** at ${formatInterval(overall.control.interval, { digits: 3 })}. A healthy harness reports \`within-noise\` or \`inconclusive\` here; anything else means slot ordering is biasing results and the verdict above cannot be trusted.`,
-    "",
-    overall.droppedRunners.length === 0
-      ? "No configuration was discarded for a stalled slot."
-      : `Discarded ${overall.droppedRunners.length} configuration(s) whose own arm disagreed with itself by more than ${overall.stallThresholdSeconds.toFixed(3)}s: ${overall.droppedRunners.map((entry) => `#${entry.sample}`).join(", ")}.`,
-    "",
-    "## By operating system",
-    "",
-    "| Group | Blocks | Difference (s) | 95% CI | p | Verdict |",
-    "| --- | ---: | ---: | --- | ---: | --- |",
-  ];
-  for (const [label, analysis] of [...byOs, ...byCache]) {
-    if (!analysis) {
-      lines.push(`| ${label} | 0 | n/a | n/a | n/a | no data |`);
-      continue;
-    }
-    lines.push(
-      `| ${label} | ${analysis.pairs.length} | ${analysis.interval.estimate.toFixed(3)} | ${analysis.interval.low.toFixed(3)} to ${analysis.interval.high.toFixed(3)} | ${analysis.pValue.toFixed(3)} | ${analysis.verdict} |`,
-    );
-  }
-  lines.push(
-    "",
-    "Groups with few blocks will report `inconclusive` even where the pooled result does not. That is the intended behaviour: a handful of configurations cannot resolve a small effect.",
-    "",
-    "## Per configuration",
-    "",
-    "Single observations. No interval is quoted because one runner cannot support one.",
-    "",
-    "| Configuration | baseline (s) | candidate (s) | Difference (s) |",
-    "| --- | ---: | ---: | ---: |",
-  );
-  for (const pair of overall.pairs) {
-    lines.push(
-      `| ${configurations[pair.sample - 1]} | ${pair.baseline.toFixed(3)} | ${pair.candidate.toFixed(3)} | ${pair.difference.toFixed(3)} |`,
-    );
-  }
-  return `${lines.join("\n")}\n`;
-}
-
-export async function main(env = process.env) {
-  requireEnv(env, [
-    "GITHUB_RUN_ID",
-    "SETUP_JAVA_REPOSITORY",
-    "BASELINE_REF",
-    "CANDIDATE_REF",
-  ]);
-  const rows = parseConfigurationSamples(await readResults());
-  const { configurations, rows: paired } = toPairedRows(rows);
-  const overall = analyzePairs(paired, "baseline", "candidate");
-  if (overall.pairs.length === 0) {
-    throw new Error("No configuration completed all four slots");
-  }
-
-  const operatingSystems = [...new Set(rows.map((row) => row.os))].sort();
-  const caches = [...new Set(rows.map((row) => row.cache))].sort();
-  const byOs = operatingSystems.map((os) => [
-    os,
-    subsetAnalysis(rows, (row) => row.os === os),
-  ]);
-  const byCache = caches.map((cache) => [
-    `cache: ${cache}`,
-    subsetAnalysis(rows, (row) => row.cache === cache),
-  ]);
-
-  const metadata = {
-    runId: env.GITHUB_RUN_ID,
-    setupJavaRepository: env.SETUP_JAVA_REPOSITORY,
-    baselineRef: env.BASELINE_REF,
-    candidateRef: env.CANDIDATE_REF,
-    generatedAt: new Date().toISOString(),
-  };
-
-  const report = markdown(metadata, overall, byOs, byCache, configurations);
-  await mkdir(OUTPUT_DIR, { recursive: true });
-  await writeFile(
-    `${OUTPUT_DIR}/results.json`,
-    `${JSON.stringify({ metadata, overall, configurations }, null, 2)}\n`,
-  );
-  await writeFile(`${OUTPUT_DIR}/summary.md`, report);
-  if (env.GITHUB_STEP_SUMMARY) {
-    await appendFile(env.GITHUB_STEP_SUMMARY, report);
-  }
-}
-
-if (
-  process.argv[1] &&
-  import.meta.url === pathToFileURL(process.argv[1]).href
-) {
-  await main();
-}
diff --git a/scripts/report-maven-configuration.test.mjs b/scripts/report-maven-configuration.test.mjs
deleted file mode 100644
index 292cd23..0000000
--- a/scripts/report-maven-configuration.test.mjs
+++ /dev/null
@@ -1,92 +0,0 @@
-import assert from "node:assert/strict";
-import test from "node:test";
-
-import { analyzePairs } from "./paired.mjs";
-import {
-  markdown,
-  parseConfigurationSamples,
-  toPairedRows,
-} from "./report-maven-configuration.mjs";
-
-const OSES = ["ubuntu-latest", "windows-latest", "macos-15-intel"];
-const CACHES = ["none", "maven", "gradle"];
-
-// One configuration's ABBA block. `speed` scales the whole runner so that a slow
-// configuration stays slow across both arms, which is what the pairing removes.
-function block(os, cache, speed, effectMs = 0) {
-  const base = 4000 * speed;
-  return [
-    `${os},${cache},single,empty,baseline,1,${Math.round(base)}`,
-    `${os},${cache},single,empty,candidate,2,${Math.round(base + effectMs)}`,
-    `${os},${cache},single,empty,candidate,3,${Math.round(base + effectMs)}`,
-    `${os},${cache},single,empty,baseline,4,${Math.round(base)}`,
-  ].join("\n");
-}
-
-function matrix(effectMs) {
-  const lines = [];
-  OSES.forEach((os, osIndex) => {
-    CACHES.forEach((cache, cacheIndex) => {
-      lines.push(
-        block(os, cache, 1 + osIndex * 0.8 + cacheIndex * 0.3, effectMs),
-      );
-    });
-  });
-  return lines.join("\n");
-}
-
-test("treats each configuration as one paired block", () => {
-  const rows = parseConfigurationSamples(matrix(0));
-  const { configurations, rows: paired } = toPairedRows(rows);
-  assert.equal(configurations.length, 9);
-  // Nine blocks of four slots each.
-  assert.equal(paired.length, 36);
-  assert.deepEqual(
-    [...new Set(paired.map((row) => row.sample))].sort((a, b) => a - b),
-    [1, 2, 3, 4, 5, 6, 7, 8, 9],
-  );
-});
-
-test("pools a consistent effect across configurations", () => {
-  // Every configuration is 500ms slower on the candidate, but the configurations
-  // differ from each other by up to 3x. Pooling the within-block differences must
-  // recover 500ms regardless of that spread.
-  const { rows } = toPairedRows(parseConfigurationSamples(matrix(500)));
-  const analysis = analyzePairs(rows, "baseline", "candidate");
-  assert.ok(Math.abs(analysis.interval.estimate - 0.5) < 1e-9);
-  assert.equal(analysis.verdict, "regression");
-});
-
-test("reports no effect when the arms are identical", () => {
-  const { rows } = toPairedRows(parseConfigurationSamples(matrix(0)));
-  const analysis = analyzePairs(rows, "baseline", "candidate");
-  assert.ok(["inconclusive", "within-noise"].includes(analysis.verdict));
-});
-
-test("renders a pooled verdict, group breakdowns and per-configuration rows", () => {
-  const rows = parseConfigurationSamples(matrix(500));
-  const { configurations, rows: paired } = toPairedRows(rows);
-  const overall = analyzePairs(paired, "baseline", "candidate");
-  const report = markdown(
-    {
-      runId: "1",
-      setupJavaRepository: "actions/setup-java",
-      baselineRef: "v4.8.0",
-      candidateRef: "main",
-    },
-    overall,
-    [["ubuntu-latest", overall]],
-    [["cache: maven", null]],
-    configurations,
-  );
-  assert.match(report, /# Maven configuration warm path/);
-  assert.match(report, /## Verdict across all configurations/);
-  assert.match(report, /A\/A control/);
-  assert.match(report, /Harness noise floor/);
-  // A group with no usable data must say so rather than quote a number.
-  assert.match(report, /\| cache: maven \| 0 \| n\/a \|/);
-  assert.match(
-    report,
-    /No interval is quoted because one runner cannot support one/,
-  );
-});
diff --git a/scripts/report-focused.mjs b/scripts/report-transfer-overlap.mjs
similarity index 93%
rename from scripts/report-focused.mjs
rename to scripts/report-transfer-overlap.mjs
index 316b299..396acd7 100644
--- a/scripts/report-focused.mjs
+++ b/scripts/report-transfer-overlap.mjs
@@ -22,7 +22,7 @@ import { classify, describeVerdict, formatInterval } from "./stats.mjs";
 export { buildPairs, noiseFloor, parseSamples };
 
 export function readSampleFiles(directory) {
-  return readPairedSampleFiles("focused-timings", directory);
+  return readPairedSampleFiles("transfer-overlap-timings", directory);
 }
 
 export function analyze(rows) {
@@ -68,7 +68,7 @@ async function allPages(path, field, token) {
 export function markdown(metadata, analysis, caches) {
   const { baseline, candidate, interval, control } = analysis;
   const lines = [
-    "# Focused cache restore benchmark",
+    "# Transfer overlap benchmark",
     "",
     `Temurin ${metadata.javaVersion}, ${analysis.pairs.length} runners x 2 paired observations, run ${metadata.runId}.`,
     `Baseline \`${metadata.baselineRef}\` vs candidate \`${metadata.candidateRef}\` from \`${metadata.setupJavaRepository}\`.`,
@@ -166,7 +166,7 @@ export async function main(env = process.env) {
   // Both arms restore this single entry, so the stored blob cannot bias the
   // comparison. The wrapper entry is optional: a baseline that predates wrapper
   // caching simply never restores it.
-  const benchmarkId = `focused-${runId}`;
+  const benchmarkId = `transfer-overlap-${runId}`;
   const expected = [
     {
       type: "maven-dependencies",
@@ -176,7 +176,7 @@ export async function main(env = process.env) {
     {
       type: "maven-wrapper",
       key: `setup-java-Linux-x64-maven-wrapper-${hashFilesSingle(
-        `wrapperVersion=focused\n# benchmark-id=${benchmarkId}\n`,
+        `wrapperVersion=overlap\n# benchmark-id=${benchmarkId}\n`,
       )}`,
       required: false,
     },
@@ -210,20 +210,20 @@ export async function main(env = process.env) {
   };
   const report = markdown(metadata, analysis, caches);
 
-  await mkdir("focused-results", { recursive: true });
+  await mkdir("transfer-overlap-results", { recursive: true });
   await writeFile(
-    "focused-results/results.json",
+    "transfer-overlap-results/results.json",
     `${JSON.stringify({ metadata, analysis, caches }, null, 2)}\n`,
   );
   await writeFile(
-    "focused-results/results.csv",
+    "transfer-overlap-results/results.csv",
     `sample,arm,slot,seconds\n${rows
       .map((row) =>
         [row.sample, row.arm, row.slot, row.seconds].map(csvValue).join(","),
       )
       .join("\n")}\n`,
   );
-  await writeFile("focused-results/summary.md", report);
+  await writeFile("transfer-overlap-results/summary.md", report);
   await appendFile(env.GITHUB_STEP_SUMMARY, report);
 
   if (env.CLEANUP_CACHES === "true") {
@@ -232,7 +232,7 @@ export async function main(env = process.env) {
         method: "DELETE",
       });
     }
-    console.log(`Deleted ${caches.length} focused benchmark caches`);
+    console.log(`Deleted ${caches.length} transfer-overlap benchmark caches`);
   }
 }
 
diff --git a/scripts/report-focused.test.mjs b/scripts/report-transfer-overlap.test.mjs
similarity index 98%
rename from scripts/report-focused.test.mjs
rename to scripts/report-transfer-overlap.test.mjs
index 86383a8..1dada10 100644
--- a/scripts/report-focused.test.mjs
+++ b/scripts/report-transfer-overlap.test.mjs
@@ -7,7 +7,7 @@ import {
   markdown,
   noiseFloor,
   parseSamples,
-} from "./report-focused.mjs";
+} from "./report-transfer-overlap.mjs";
 
 const csv = [
   '"1","baseline","1","2400"',
@@ -90,7 +90,7 @@ test("renders a report with a verdict and paired samples", () => {
     analysis,
     [{ arm: "baseline", type: "maven-dependencies", sizeBytes: 1024 * 1024 }],
   );
-  assert.match(report, /# Focused cache restore benchmark/);
+  assert.match(report, /# Transfer overlap benchmark/);
   assert.match(report, /## Verdict/);
   assert.match(report, /95% CI/);
   assert.match(report, /A\/A control/);
diff --git a/scripts/focused-cache-restore.sh b/scripts/transfer-overlap.sh
similarity index 83%
rename from scripts/focused-cache-restore.sh
rename to scripts/transfer-overlap.sh
index b1ab960..3e9b1ea 100755
--- a/scripts/focused-cache-restore.sh
+++ b/scripts/transfer-overlap.sh
@@ -1,19 +1,19 @@
 #!/usr/bin/env bash
 
-# Fixture and cache-identity helpers for the focused cache restore benchmark.
+# Fixture and cache-identity helpers for the transfer overlap benchmark.
 
 set -euo pipefail
 
 command=${1:?command is required}
 
-dependency_fixture="$HOME/.m2/repository/focused-benchmark/dependencies.bin"
-wrapper_fixture="$HOME/.m2/wrapper/dists/focused-benchmark/wrapper.bin"
+dependency_fixture="$HOME/.m2/repository/transfer-overlap-benchmark/dependencies.bin"
+wrapper_fixture="$HOME/.m2/wrapper/dists/transfer-overlap-benchmark/wrapper.bin"
 
 write_identity() {
   local benchmark_id=$1
-  printf '%s\n' "$benchmark_id" > .focused-cache-key
+  printf '%s\n' "$benchmark_id" > .transfer-overlap-cache-key
   mkdir -p .mvn/wrapper
-  printf 'wrapperVersion=focused\n# benchmark-id=%s\n' "$benchmark_id" \
+  printf 'wrapperVersion=overlap\n# benchmark-id=%s\n' "$benchmark_id" \
     > .mvn/wrapper/maven-wrapper.properties
 }