diff --git a/.github/workflows/action-overhead.yml b/.github/workflows/action-overhead.yml new file mode 100644 index 0000000..548fa38 --- /dev/null +++ b/.github/workflows/action-overhead.yml @@ -0,0 +1,311 @@ +name: Action overhead + +on: + workflow_dispatch: + inputs: + setup-java-repository: + description: Repository containing the setup-java action + required: true + default: actions/setup-java + type: string + baseline-ref: + description: Git ref containing the baseline implementation + required: true + default: main + type: string + candidate-ref: + description: Git ref containing the candidate implementation + required: true + default: main + type: string + +permissions: + contents: read + +defaults: + run: + shell: bash + +concurrency: + group: action-overhead + cancel-in-progress: false + +env: + # Shared by the seed job and by every `maven-hit` slot, so that all of them + # compute the same cache key and restore the same stored entry. Scoping it to + # the run keeps concurrent runs from reusing each other's blob, whose placement + # in the cache service fixes its download throughput for its whole life. + SEEDED_IDENTITY: action-overhead-seed-${{ github.run_id }} + +# What does setup-java's own code cost? +# +# Every other scenario in this repository is dominated by how fast a blob moves +# across the network, which setup-java does not control: it hands the transfer to +# `@actions/cache`. This one deliberately uses a cache entry of about a megabyte +# so the transfer term is small, leaving the action's own work — resolving a +# distribution, computing a cache key, writing settings and toolchains, and the +# bookkeeping around a restore — as what is actually being timed. +# +# The four cache profiles are levels of a decomposition rather than competing +# options: +# +# none the action never touches the cache code at all +# maven-miss it computes a key and asks the service, and is told no +# maven-hit it computes a key, is told yes, and unpacks a small entry +# gradle-miss the same as maven-miss through the other package manager +# +# Reading the differences between those levels says where the time goes. The old +# version of this workflow had no seed job, so its "warm path" was three +# variations on a cache miss and it could not see the restore path at all. +jobs: + seed: + name: Seed ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-15-intel] + steps: + - name: Check out benchmark repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + # Seeding with the baseline is what makes both arms restore one shared + # entry. It relies on the two refs computing the same key for the same + # tree, which is exactly what the cache key stability workflow asserts; if + # that ever stops holding, the recorded cache-hit flags make it visible + # here rather than silently turning this into a miss-versus-hit comparison. + - name: Check out baseline setup-java + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: ${{ inputs.setup-java-repository }} + path: baseline + persist-credentials: false + ref: ${{ inputs.baseline-ref }} + - name: Build the fixture local repository + run: bash scripts/action-overhead.sh seed-fixture "$SEEDED_IDENTITY" + - name: Store it + id: store + uses: ./baseline + timeout-minutes: 10 + with: + distribution: temurin + java-version: "21" + cache: maven + cache-dependency-path: benchmark/pom.xml + settings-path: benchmark-maven-home + # The key is a hash, so nothing about the entry's name identifies the run + # that created it. Recording what the action reports is the only way the + # report job can delete exactly the entries this run added instead of + # guessing at a prefix or leaving them to age out. + - name: Record the seeded cache key + run: | + mkdir -p .benchmark-results + printf '%s\n' "${{ steps.store.outputs.cache-primary-key }}" \ + > ".benchmark-results/seeded-key-${{ matrix.os }}.txt" + - name: Upload the seeded cache key + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: action-overhead-seed-key-${{ matrix.os }} + path: .benchmark-results/ + include-hidden-files: true + if-no-files-found: error + retention-days: 30 + + benchmark: + name: ${{ matrix.os }} ${{ matrix.cache }} ${{ matrix.layout }} + needs: seed + runs-on: ${{ matrix.os }} + timeout-minutes: 25 + strategy: + fail-fast: false + # The cache service is the shared resource these jobs contend for, and + # contention shows up as a stalled restore rather than a slow one. Keeping + # the matrix narrow costs wall-clock time and buys measurements that are + # about setup-java instead of about the queue in front of it. + max-parallel: 6 + matrix: + os: [ubuntu-latest, windows-latest, macos-15-intel] + cache: [none, maven-miss, maven-hit, gradle-miss] + # Two ends of the range rather than the full cross of versions and + # toolchains. The cross doubled the job count to separate two effects + # that were each smaller than the run-to-run spread. + layout: [simple, complex] + include: + - layout: simple + java-version: "21" + toolchains: empty + - layout: complex + java-version: | + 17 + 21 + toolchains: existing + env: + RESULTS: .benchmark-results/action-overhead-${{ matrix.os }}-${{ matrix.cache }}-${{ matrix.layout }}.csv + # A miss has to be a real miss. Scoping the identity to this job guarantees + # nothing has ever stored that key. + MISS_IDENTITY: action-overhead-miss-${{ github.run_id }}-${{ matrix.os }}-${{ matrix.cache }}-${{ matrix.layout }} + steps: + - name: Check out benchmark repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - name: Check out baseline setup-java + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: ${{ inputs.setup-java-repository }} + path: baseline + persist-credentials: false + ref: ${{ inputs.baseline-ref }} + - name: Check out candidate setup-java + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: ${{ inputs.setup-java-repository }} + path: candidate + persist-credentials: false + ref: ${{ inputs.candidate-ref }} + - name: Record setup dist sizes + run: | + bash scripts/action-overhead.sh record-size baseline baseline + bash scripts/action-overhead.sh record-size candidate candidate + + - name: Resolve the cache identity for this profile + run: | + if [ "${{ matrix.cache }}" = "maven-hit" ]; then + echo "IDENTITY=$SEEDED_IDENTITY" >> "$GITHUB_ENV" + else + echo "IDENTITY=$MISS_IDENTITY" >> "$GITHUB_ENV" + fi + + # The first setup in a job pays DNS resolution, TLS handshakes and a cold + # page cache that the later ones do not. That is a one-off spike rather + # than drift, so the mirrored slot order cannot cancel it; this slot pays + # those costs and is discarded. + - name: Prepare warm-up slot + run: bash scripts/action-overhead.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}" "$IDENTITY" + - name: Warm-up setup (discarded) + uses: ./baseline + # A setup that has not finished in three minutes has stalled rather than + # being slow; failing costs this configuration, which the report drops. + timeout-minutes: 3 + with: + distribution: temurin + java-version: ${{ matrix.java-version }} + cache: ${{ startsWith(matrix.cache, 'maven') && 'maven' || (startsWith(matrix.cache, 'gradle') && 'gradle' || '') }} + cache-dependency-path: benchmark/${{ startsWith(matrix.cache, 'gradle') && 'build.gradle' || 'pom.xml' }} + settings-path: benchmark-maven-home + + - name: Prepare slot 1 + run: bash scripts/action-overhead.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}" "$IDENTITY" + - name: Start slot 1 timer + run: bash scripts/action-overhead.sh start + - name: Slot 1 setup (baseline) + id: slot1 + uses: ./baseline + timeout-minutes: 3 + with: + distribution: temurin + java-version: ${{ matrix.java-version }} + cache: ${{ startsWith(matrix.cache, 'maven') && 'maven' || (startsWith(matrix.cache, 'gradle') && 'gradle' || '') }} + cache-dependency-path: benchmark/${{ startsWith(matrix.cache, 'gradle') && 'build.gradle' || 'pom.xml' }} + settings-path: benchmark-maven-home + - name: Record slot 1 + run: bash scripts/action-overhead.sh record "$RESULTS" "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.layout }}" baseline 1 "${{ steps.slot1.outputs.cache-hit }}" + + - name: Prepare slot 2 + run: bash scripts/action-overhead.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}" "$IDENTITY" + - name: Start slot 2 timer + run: bash scripts/action-overhead.sh start + - name: Slot 2 setup (candidate) + id: slot2 + uses: ./candidate + timeout-minutes: 3 + with: + distribution: temurin + java-version: ${{ matrix.java-version }} + cache: ${{ startsWith(matrix.cache, 'maven') && 'maven' || (startsWith(matrix.cache, 'gradle') && 'gradle' || '') }} + cache-dependency-path: benchmark/${{ startsWith(matrix.cache, 'gradle') && 'build.gradle' || 'pom.xml' }} + settings-path: benchmark-maven-home + - name: Record slot 2 + run: bash scripts/action-overhead.sh record "$RESULTS" "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.layout }}" candidate 2 "${{ steps.slot2.outputs.cache-hit }}" + + - name: Prepare slot 3 + run: bash scripts/action-overhead.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}" "$IDENTITY" + - name: Start slot 3 timer + run: bash scripts/action-overhead.sh start + - name: Slot 3 setup (candidate) + id: slot3 + uses: ./candidate + timeout-minutes: 3 + with: + distribution: temurin + java-version: ${{ matrix.java-version }} + cache: ${{ startsWith(matrix.cache, 'maven') && 'maven' || (startsWith(matrix.cache, 'gradle') && 'gradle' || '') }} + cache-dependency-path: benchmark/${{ startsWith(matrix.cache, 'gradle') && 'build.gradle' || 'pom.xml' }} + settings-path: benchmark-maven-home + - name: Record slot 3 + run: bash scripts/action-overhead.sh record "$RESULTS" "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.layout }}" candidate 3 "${{ steps.slot3.outputs.cache-hit }}" + + - name: Prepare slot 4 + run: bash scripts/action-overhead.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}" "$IDENTITY" + - name: Start slot 4 timer + run: bash scripts/action-overhead.sh start + - name: Slot 4 setup (baseline) + id: slot4 + uses: ./baseline + timeout-minutes: 3 + with: + distribution: temurin + java-version: ${{ matrix.java-version }} + cache: ${{ startsWith(matrix.cache, 'maven') && 'maven' || (startsWith(matrix.cache, 'gradle') && 'gradle' || '') }} + cache-dependency-path: benchmark/${{ startsWith(matrix.cache, 'gradle') && 'build.gradle' || 'pom.xml' }} + settings-path: benchmark-maven-home + - name: Record slot 4 + run: bash scripts/action-overhead.sh record "$RESULTS" "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.layout }}" baseline 4 "${{ steps.slot4.outputs.cache-hit }}" + + - name: Upload raw benchmark data + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: action-overhead-${{ matrix.os }}-${{ matrix.cache }}-${{ matrix.layout }} + path: .benchmark-results/ + include-hidden-files: true + if-no-files-found: error + retention-days: 30 + + report: + name: Report + needs: benchmark + if: ${{ always() && !cancelled() }} + runs-on: ubuntu-24.04 + permissions: + contents: read + actions: write + steps: + - name: Check out benchmark repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - name: Download timings and seeded keys + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6 + with: + pattern: action-overhead-* + path: .benchmark-results + - name: Generate report + env: + SETUP_JAVA_REPOSITORY: ${{ inputs.setup-java-repository }} + BASELINE_REF: ${{ inputs.baseline-ref }} + CANDIDATE_REF: ${{ inputs.candidate-ref }} + RUN_ID: ${{ github.run_id }} + run: node scripts/report-action-overhead.mjs + - name: Delete the seeded cache entries + if: ${{ always() }} + env: + GH_TOKEN: ${{ github.token }} + run: bash scripts/action-overhead.sh delete-seeded "$GITHUB_REPOSITORY" + - name: Upload benchmark results + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: action-overhead-results-${{ github.run_id }} + path: action-overhead-results/ + retention-days: 30 diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 61d2f52..5253b9c 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -3,15 +3,24 @@ name: Benchmark setup-java on: workflow_dispatch: inputs: - iterations: - description: Independent cold/warm samples per combination + samples: + description: Runners to measure on (each contributes 2 observations per version) required: true type: choice options: - - "1" - - "3" - - "5" - default: "3" + - "2" + - "6" + - "10" + - "20" + default: "10" + distribution: + description: JDK distribution for v2 and later + required: true + type: choice + options: + - temurin + - microsoft + default: temurin java-version: description: Java feature version required: true @@ -28,442 +37,434 @@ permissions: contents: read env: + # A cache segment download that fails leaves @actions/cache holding an armed + # timer it never clears, so the action idles for the rest of this timeout after + # its work is done (see actions/toolkit's promiseWithTimeout: the clearTimeout + # runs in a .then, which a rejection skips). The default is 10 minutes. These + # restores move 154 MiB in about 3 seconds, so 2 minutes is still far beyond + # any legitimate download and bounds a stall to a fifth of what it was. + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 2 PETCLINIC_REF: f182358d02e4a68e52bdbabf55ca7800288511e7 concurrency: group: setup-java-benchmark cancel-in-progress: false +defaults: + run: + shell: bash + jobs: - v1-cold: - name: v1 / cold / zulu / ${{ matrix.iteration }} + # One Maven cache entry is seeded and every version that supports caching + # restores it. Seeding one entry per version would confound the version with + # its stored blob: a cache entry's download throughput depends on where the + # service placed it, and because that bias is identical on every runner, + # pairing cannot remove it and more samples only tighten the interval around + # the wrong answer. + seed: + name: Seed cache runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }} steps: + - name: Check out benchmark repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + # PetClinic goes below the benchmark repository rather than over it: its + # build runs a nohttp check across the whole basedir and would otherwise + # lint the benchmark scripts. - name: Check out Spring PetClinic uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: repository: spring-projects/spring-petclinic ref: ${{ env.PETCLINIC_REF }} + path: petclinic persist-credentials: false + - name: Prepare cache identity + run: bash scripts/version-sweep.sh prepare "benchmark-${{ github.run_id }}" + # Each version registers a post-job save against the same local repository. + # The first save wins and the rest are skipped as exact key hits. + - name: Setup Java v1 uses: actions/setup-java@v1.4.4 with: java-version: ${{ inputs.java-version }} - - name: Record Java environment - run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version' - - name: Build PetClinic - run: ./mvnw --batch-mode --no-transfer-progress compile - - v2-cold: - name: v2 / cold / ${{ matrix.distribution }} / ${{ matrix.iteration }} - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - distribution: [temurin, microsoft] - iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }} - steps: - - name: Check out Spring PetClinic - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - repository: spring-projects/spring-petclinic - ref: ${{ env.PETCLINIC_REF }} - persist-credentials: false - name: Setup Java v2 uses: actions/setup-java@v2.5.1 with: - distribution: ${{ matrix.distribution }} + distribution: ${{ inputs.distribution }} java-version: ${{ inputs.java-version }} - - name: Record Java environment - run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version' - - name: Build PetClinic - run: ./mvnw --batch-mode --no-transfer-progress compile - - v3-cold: - name: v3 / cold / ${{ matrix.distribution }} / ${{ matrix.iteration }} - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - distribution: [temurin, microsoft] - iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }} - steps: - - name: Check out Spring PetClinic - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - repository: spring-projects/spring-petclinic - ref: ${{ env.PETCLINIC_REF }} - persist-credentials: false - - name: Isolate benchmark caches - env: - BENCHMARK_ID: v3-${{ matrix.distribution }}-${{ matrix.iteration }}-${{ github.run_id }} - run: printf '\n' "$BENCHMARK_ID" >> pom.xml - name: Setup Java v3 uses: actions/setup-java@v3.14.1 with: - distribution: ${{ matrix.distribution }} + distribution: ${{ inputs.distribution }} java-version: ${{ inputs.java-version }} cache: maven - - name: Record Java environment - run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version' - - name: Build PetClinic - run: ./mvnw --batch-mode --no-transfer-progress compile - - v4-cold: - name: v4 / cold / ${{ matrix.distribution }} / ${{ matrix.iteration }} - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - distribution: [temurin, microsoft] - iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }} - steps: - - name: Check out Spring PetClinic - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - repository: spring-projects/spring-petclinic - ref: ${{ env.PETCLINIC_REF }} - persist-credentials: false - - name: Isolate benchmark caches - env: - BENCHMARK_ID: v4-${{ matrix.distribution }}-${{ matrix.iteration }}-${{ github.run_id }} - run: printf '%s\n' "$BENCHMARK_ID" > .benchmark-cache-key - name: Setup Java v4 uses: actions/setup-java@v4.8.0 with: - distribution: ${{ matrix.distribution }} + distribution: ${{ inputs.distribution }} java-version: ${{ inputs.java-version }} cache: maven cache-dependency-path: .benchmark-cache-key - - name: Record Java environment - run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version' - - name: Build PetClinic - run: ./mvnw --batch-mode --no-transfer-progress compile - - v52-cold: - name: v5.2 / cold / ${{ matrix.distribution }} / ${{ matrix.iteration }} - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - distribution: [temurin, microsoft] - iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }} - steps: - - name: Check out Spring PetClinic - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - repository: spring-projects/spring-petclinic - ref: ${{ env.PETCLINIC_REF }} - persist-credentials: false - - name: Isolate benchmark caches - env: - BENCHMARK_ID: v52-${{ matrix.distribution }}-${{ matrix.iteration }}-${{ github.run_id }} - run: printf '%s\n' "$BENCHMARK_ID" > .benchmark-cache-key - name: Setup Java v5.2 uses: actions/setup-java@v5.2.0 with: - distribution: ${{ matrix.distribution }} + distribution: ${{ inputs.distribution }} java-version: ${{ inputs.java-version }} cache: maven cache-dependency-path: .benchmark-cache-key - - name: Record Java environment - run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version' - - name: Build PetClinic - run: ./mvnw --batch-mode --no-transfer-progress compile - - v56-cold: - name: v5.6 / cold / ${{ matrix.distribution }} / ${{ matrix.iteration }} - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - distribution: [temurin, microsoft] - iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }} - steps: - - name: Check out Spring PetClinic - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - repository: spring-projects/spring-petclinic - ref: ${{ env.PETCLINIC_REF }} - persist-credentials: false - - name: Isolate benchmark caches - env: - BENCHMARK_ID: v56-${{ matrix.distribution }}-${{ matrix.iteration }}-${{ github.run_id }} - run: | - printf '%s\n' "$BENCHMARK_ID" > .benchmark-cache-key - printf '# benchmark-id=%s\n' "$BENCHMARK_ID" >> .mvn/wrapper/maven-wrapper.properties - name: Setup Java v5.6 uses: actions/setup-java@v5.6.0 with: - distribution: ${{ matrix.distribution }} + distribution: ${{ inputs.distribution }} java-version: ${{ inputs.java-version }} cache: maven cache-dependency-path: .benchmark-cache-key - - name: Record Java environment - run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version' - - name: Build PetClinic - run: ./mvnw --batch-mode --no-transfer-progress compile - - main-cold: - name: main / cold / ${{ matrix.distribution }} / ${{ matrix.iteration }} - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - distribution: [temurin, microsoft] - iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }} - steps: - - name: Check out Spring PetClinic - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - repository: spring-projects/spring-petclinic - ref: ${{ env.PETCLINIC_REF }} - persist-credentials: false - - name: Isolate benchmark caches - env: - BENCHMARK_ID: main-${{ matrix.distribution }}-${{ matrix.iteration }}-${{ github.run_id }} - run: | - printf '%s\n' "$BENCHMARK_ID" > .benchmark-cache-key - printf '# benchmark-id=%s\n' "$BENCHMARK_ID" >> .mvn/wrapper/maven-wrapper.properties - name: Setup Java main uses: actions/setup-java@main with: - distribution: ${{ matrix.distribution }} + distribution: ${{ inputs.distribution }} java-version: ${{ inputs.java-version }} cache: maven cache-dependency-path: .benchmark-cache-key - - name: Record Java environment - run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version' - name: Build PetClinic + working-directory: petclinic run: ./mvnw --batch-mode --no-transfer-progress compile - v1-warm: - name: v1 / warm / zulu / ${{ matrix.iteration }} - needs: v1-cold - if: ${{ !cancelled() }} + # Every version is measured in the same job, in the order v1..main followed by + # main..v1. Differencing within a runner removes between-runner variance, and + # the mirrored order places each version's two slots symmetrically about the + # middle of the job so that drift across the job cancels. + warm: + name: warm / ${{ matrix.sample }} + needs: seed runs-on: ubuntu-24.04 + # A measurement job is a couple of minutes of work. Anything far beyond that + # is a stalled cache download or JDK fetch rather than a slow measurement, + # and the run should not wait six hours for the default timeout to notice. + # The runner drops out of the report, which discards runners that did not + # complete every slot, and the remaining runners still produce a verdict. + timeout-minutes: 25 strategy: fail-fast: false + # Every runner pulls the same seeded blob, so running the whole matrix at + # once measures contention on the cache service that no real workflow would + # see. Waves are kept small so each measurement reflects an ordinary + # restore. This job restores fifteen times where the two-arm workflows + # restore five, so it runs in narrower waves than they do. + max-parallel: 2 matrix: - iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }} + sample: ${{ fromJSON(inputs.samples == '20' && '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]' || inputs.samples == '10' && '[1,2,3,4,5,6,7,8,9,10]' || inputs.samples == '6' && '[1,2,3,4,5,6]' || '[1,2]') }} steps: - - name: Check out Spring PetClinic + - name: Check out benchmark repository uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: - repository: spring-projects/spring-petclinic - ref: ${{ env.PETCLINIC_REF }} persist-credentials: false - - name: Setup Java v1 - uses: actions/setup-java@v1.4.4 - with: - java-version: ${{ inputs.java-version }} - - name: Record Java environment - run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version' - - name: Build PetClinic - run: ./mvnw --batch-mode --no-transfer-progress compile - - v2-warm: - name: v2 / warm / ${{ matrix.distribution }} / ${{ matrix.iteration }} - needs: v2-cold - if: ${{ !cancelled() }} - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - distribution: [temurin, microsoft] - iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }} - steps: + # PetClinic goes below the benchmark repository rather than over it: its + # build runs a nohttp check across the whole basedir and would otherwise + # lint the benchmark scripts. - name: Check out Spring PetClinic uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: repository: spring-projects/spring-petclinic ref: ${{ env.PETCLINIC_REF }} + path: petclinic persist-credentials: false - - name: Setup Java v2 - uses: actions/setup-java@v2.5.1 + + # cache-read-only exists only on main, so it is not used here: applying it + # to some arms and not others would make post-job behaviour asymmetric. + # Every slot restores its cache on an exact primary-key hit, and + # actions/cache skips saving in that case, so no measured slot creates an + # entry regardless of the version. + # The first setup in a job pays costs the later ones do not: DNS resolution, + # TLS handshakes to the cache service and the JDK host, and a cold page + # cache. That is a one-off spike rather than drift, so the mirrored slot + # order cannot cancel it, and it showed up as the A/A control resolving a + # difference between an arm's own first and last slot. This slot pays those + # costs and is discarded. + - name: Reset warm-up slot + run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}" + - name: Warm-up setup (discarded) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 + uses: actions/setup-java@main with: - distribution: ${{ matrix.distribution }} + distribution: ${{ inputs.distribution }} java-version: ${{ inputs.java-version }} - - name: Record Java environment - run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version' - - name: Build PetClinic - run: ./mvnw --batch-mode --no-transfer-progress compile + cache: maven + cache-dependency-path: .benchmark-cache-key - v3-warm: - name: v3 / warm / ${{ matrix.distribution }} / ${{ matrix.iteration }} - needs: v3-cold - if: ${{ !cancelled() }} - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - distribution: [temurin, microsoft] - iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }} - steps: - - name: Check out Spring PetClinic - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - name: Reset slot 1 + run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}" + - name: Start slot 1 timer + run: node scripts/measure.mjs start + - name: Slot 1 setup (v1) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 + uses: actions/setup-java@v1.4.4 with: - repository: spring-projects/spring-petclinic - ref: ${{ env.PETCLINIC_REF }} - persist-credentials: false - - name: Reuse benchmark cache identity - env: - BENCHMARK_ID: v3-${{ matrix.distribution }}-${{ matrix.iteration }}-${{ github.run_id }} - run: printf '\n' "$BENCHMARK_ID" >> pom.xml - - name: Setup Java v3 + java-version: ${{ inputs.java-version }} + - name: Record slot 1 + run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v1 1 + - name: Reset slot 2 + run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}" + - name: Start slot 2 timer + run: node scripts/measure.mjs start + - name: Slot 2 setup (v2) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 + uses: actions/setup-java@v2.5.1 + with: + distribution: ${{ inputs.distribution }} + java-version: ${{ inputs.java-version }} + - name: Record slot 2 + run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v2 2 + - name: Reset slot 3 + run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}" + - name: Start slot 3 timer + run: node scripts/measure.mjs start + - name: Slot 3 setup (v3) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 uses: actions/setup-java@v3.14.1 with: - distribution: ${{ matrix.distribution }} + distribution: ${{ inputs.distribution }} java-version: ${{ inputs.java-version }} cache: maven - - name: Record Java environment - run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version' - - name: Build PetClinic - run: ./mvnw --batch-mode --no-transfer-progress compile - - v4-warm: - name: v4 / warm / ${{ matrix.distribution }} / ${{ matrix.iteration }} - needs: v4-cold - if: ${{ !cancelled() }} - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - distribution: [temurin, microsoft] - iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }} - steps: - - name: Check out Spring PetClinic - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - repository: spring-projects/spring-petclinic - ref: ${{ env.PETCLINIC_REF }} - persist-credentials: false - - name: Reuse benchmark cache identity - env: - BENCHMARK_ID: v4-${{ matrix.distribution }}-${{ matrix.iteration }}-${{ github.run_id }} - run: printf '%s\n' "$BENCHMARK_ID" > .benchmark-cache-key - - name: Setup Java v4 + - name: Record slot 3 + run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v3 3 + - name: Reset slot 4 + run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}" + - name: Start slot 4 timer + run: node scripts/measure.mjs start + - name: Slot 4 setup (v4) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 uses: actions/setup-java@v4.8.0 with: - distribution: ${{ matrix.distribution }} + distribution: ${{ inputs.distribution }} java-version: ${{ inputs.java-version }} cache: maven cache-dependency-path: .benchmark-cache-key - - name: Record Java environment - run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version' - - name: Build PetClinic - run: ./mvnw --batch-mode --no-transfer-progress compile - - v52-warm: - name: v5.2 / warm / ${{ matrix.distribution }} / ${{ matrix.iteration }} - needs: v52-cold - if: ${{ !cancelled() }} - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - distribution: [temurin, microsoft] - iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }} - steps: - - name: Check out Spring PetClinic - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - repository: spring-projects/spring-petclinic - ref: ${{ env.PETCLINIC_REF }} - persist-credentials: false - - name: Reuse benchmark cache identity - env: - BENCHMARK_ID: v52-${{ matrix.distribution }}-${{ matrix.iteration }}-${{ github.run_id }} - run: printf '%s\n' "$BENCHMARK_ID" > .benchmark-cache-key - - name: Setup Java v5.2 + - name: Record slot 4 + run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v4 4 + - name: Reset slot 5 + run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}" + - name: Start slot 5 timer + run: node scripts/measure.mjs start + - name: Slot 5 setup (v5.2) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 uses: actions/setup-java@v5.2.0 with: - distribution: ${{ matrix.distribution }} + distribution: ${{ inputs.distribution }} java-version: ${{ inputs.java-version }} cache: maven cache-dependency-path: .benchmark-cache-key - - name: Record Java environment - run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version' - - name: Build PetClinic - run: ./mvnw --batch-mode --no-transfer-progress compile - - v56-warm: - name: v5.6 / warm / ${{ matrix.distribution }} / ${{ matrix.iteration }} - needs: v56-cold - if: ${{ !cancelled() }} - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - distribution: [temurin, microsoft] - iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }} - steps: - - name: Check out Spring PetClinic - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - repository: spring-projects/spring-petclinic - ref: ${{ env.PETCLINIC_REF }} - persist-credentials: false - - name: Reuse benchmark cache identity - env: - BENCHMARK_ID: v56-${{ matrix.distribution }}-${{ matrix.iteration }}-${{ github.run_id }} - run: | - printf '%s\n' "$BENCHMARK_ID" > .benchmark-cache-key - printf '# benchmark-id=%s\n' "$BENCHMARK_ID" >> .mvn/wrapper/maven-wrapper.properties - - name: Setup Java v5.6 + - name: Record slot 5 + run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v52 5 + - name: Reset slot 6 + run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}" + - name: Start slot 6 timer + run: node scripts/measure.mjs start + - name: Slot 6 setup (v5.6) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 uses: actions/setup-java@v5.6.0 with: - distribution: ${{ matrix.distribution }} + distribution: ${{ inputs.distribution }} java-version: ${{ inputs.java-version }} cache: maven cache-dependency-path: .benchmark-cache-key - - name: Record Java environment - run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version' - - name: Build PetClinic - run: ./mvnw --batch-mode --no-transfer-progress compile - - main-warm: - name: main / warm / ${{ matrix.distribution }} / ${{ matrix.iteration }} - needs: main-cold - if: ${{ !cancelled() }} - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - distribution: [temurin, microsoft] - iteration: ${{ fromJSON(inputs.iterations == '5' && '[1,2,3,4,5]' || inputs.iterations == '3' && '[1,2,3]' || '[1]') }} - steps: - - name: Check out Spring PetClinic - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - name: Record slot 6 + run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v56 6 + - name: Reset slot 7 + run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}" + - name: Start slot 7 timer + run: node scripts/measure.mjs start + - name: Slot 7 setup (main) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 + uses: actions/setup-java@main with: - repository: spring-projects/spring-petclinic - ref: ${{ env.PETCLINIC_REF }} - persist-credentials: false - - name: Reuse benchmark cache identity - env: - BENCHMARK_ID: main-${{ matrix.distribution }}-${{ matrix.iteration }}-${{ github.run_id }} - run: | - printf '%s\n' "$BENCHMARK_ID" > .benchmark-cache-key - printf '# benchmark-id=%s\n' "$BENCHMARK_ID" >> .mvn/wrapper/maven-wrapper.properties - - name: Setup Java main + distribution: ${{ inputs.distribution }} + java-version: ${{ inputs.java-version }} + cache: maven + cache-dependency-path: .benchmark-cache-key + - name: Record slot 7 + run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" main 7 + - name: Reset slot 8 + run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}" + - name: Start slot 8 timer + run: node scripts/measure.mjs start + - name: Slot 8 setup (main) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 uses: actions/setup-java@main with: - distribution: ${{ matrix.distribution }} + distribution: ${{ inputs.distribution }} java-version: ${{ inputs.java-version }} cache: maven cache-dependency-path: .benchmark-cache-key - - name: Record Java environment - run: java -XshowSettings:properties -version 2>&1 | grep -E 'java.home|java.vendor|java.version' - - name: Build PetClinic - run: ./mvnw --batch-mode --no-transfer-progress compile + - name: Record slot 8 + run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" main 8 + - name: Reset slot 9 + run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}" + - name: Start slot 9 timer + run: node scripts/measure.mjs start + - name: Slot 9 setup (v5.6) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 + uses: actions/setup-java@v5.6.0 + with: + distribution: ${{ inputs.distribution }} + java-version: ${{ inputs.java-version }} + cache: maven + cache-dependency-path: .benchmark-cache-key + - name: Record slot 9 + run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v56 9 + - name: Reset slot 10 + run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}" + - name: Start slot 10 timer + run: node scripts/measure.mjs start + - name: Slot 10 setup (v5.2) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 + uses: actions/setup-java@v5.2.0 + with: + distribution: ${{ inputs.distribution }} + java-version: ${{ inputs.java-version }} + cache: maven + cache-dependency-path: .benchmark-cache-key + - name: Record slot 10 + run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v52 10 + - name: Reset slot 11 + run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}" + - name: Start slot 11 timer + run: node scripts/measure.mjs start + - name: Slot 11 setup (v4) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 + uses: actions/setup-java@v4.8.0 + with: + distribution: ${{ inputs.distribution }} + java-version: ${{ inputs.java-version }} + cache: maven + cache-dependency-path: .benchmark-cache-key + - name: Record slot 11 + run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v4 11 + - name: Reset slot 12 + run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}" + - name: Start slot 12 timer + run: node scripts/measure.mjs start + - name: Slot 12 setup (v3) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 + uses: actions/setup-java@v3.14.1 + with: + distribution: ${{ inputs.distribution }} + java-version: ${{ inputs.java-version }} + cache: maven + - name: Record slot 12 + run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v3 12 + - name: Reset slot 13 + run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}" + - name: Start slot 13 timer + run: node scripts/measure.mjs start + - name: Slot 13 setup (v2) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 + uses: actions/setup-java@v2.5.1 + with: + distribution: ${{ inputs.distribution }} + java-version: ${{ inputs.java-version }} + - name: Record slot 13 + run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v2 13 + - name: Reset slot 14 + run: bash scripts/version-sweep.sh reset "benchmark-${{ github.run_id }}" + - name: Start slot 14 timer + run: node scripts/measure.mjs start + - name: Slot 14 setup (v1) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 + uses: actions/setup-java@v1.4.4 + with: + java-version: ${{ inputs.java-version }} + - name: Record slot 14 + run: node scripts/measure.mjs record ".benchmark-results/version-sweep-${{ matrix.sample }}.csv" "${{ matrix.sample }}" v1 14 + + - name: Upload sample timings + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: version-sweep-${{ matrix.sample }} + path: .benchmark-results/ + include-hidden-files: true + if-no-files-found: error + retention-days: 30 report: name: Report - needs: [v1-cold, v2-cold, v3-cold, v4-cold, v52-cold, v56-cold, main-cold, v1-warm, v2-warm, v3-warm, v4-warm, v52-warm, v56-warm, main-warm] + needs: warm if: ${{ always() && !cancelled() }} runs-on: ubuntu-24.04 steps: @@ -471,10 +472,17 @@ jobs: uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: persist-credentials: false + - name: Download sample timings + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6 + with: + pattern: version-sweep-* + merge-multiple: true + path: .benchmark-results - name: Generate benchmark report env: GH_TOKEN: ${{ github.token }} - ITERATIONS: ${{ inputs.iterations }} + SAMPLES: ${{ inputs.samples }} + DISTRIBUTION: ${{ inputs.distribution }} JAVA_VERSION: ${{ inputs.java-version }} CLEANUP_CACHES: ${{ inputs.cleanup-caches }} run: node scripts/report.mjs diff --git a/.github/workflows/cache-key-stability.yml b/.github/workflows/cache-key-stability.yml new file mode 100644 index 0000000..03428c7 --- /dev/null +++ b/.github/workflows/cache-key-stability.yml @@ -0,0 +1,196 @@ +name: Cache key stability + +on: + workflow_dispatch: + inputs: + setup-java-repository: + description: Repository containing the setup-java action + required: true + default: actions/setup-java + type: string + setup-java-ref: + description: Git ref of setup-java to check + required: true + default: main + type: string + java-version: + description: Java feature version + required: true + default: "17" + type: string + +permissions: + contents: read + +env: + PETCLINIC_REF: f182358d02e4a68e52bdbabf55ca7800288511e7 + +concurrency: + group: cache-key-stability + cancel-in-progress: false + +defaults: + run: + shell: bash + +# Does the cache key hit when it should, and change when it must? +# +# This workflow measures nothing. The key is a deterministic function of the +# tree, so its properties can be asserted outright, and they dominate every +# timing in this repository: a spurious key change costs a full cache miss, +# about a minute on Spring PetClinic, where the timing benchmarks are resolving +# effects of tens of milliseconds. +# +# No cache is stored by any of this. setup-java skips its post-job save when the +# configured path does not exist, and no probe ever creates ~/.m2. +jobs: + probe: + name: ${{ matrix.platform }} / replica ${{ matrix.replica }} + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-24.04 + platform: linux + replica: "1" + # Two runners on the same platform with the same tree. If their keys + # disagree, caching never works for anyone: every job would compute a + # key no other job has stored. + - os: ubuntu-24.04 + platform: linux + replica: "2" + - os: windows-latest + platform: windows + replica: "1" + - os: macos-15 + platform: macos + replica: "1" + env: + RUNNER_LABEL: ${{ matrix.platform }} + RUNNER_REPLICA: ${{ matrix.replica }} + RESULTS: .benchmark-results/cache-keys-${{ matrix.platform }}-${{ matrix.replica }}.csv + steps: + - name: Check out benchmark repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - name: Check out Spring PetClinic + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: spring-projects/spring-petclinic + ref: ${{ env.PETCLINIC_REF }} + path: petclinic + persist-credentials: false + - name: Check out setup-java + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: ${{ inputs.setup-java-repository }} + path: setup-java + persist-credentials: false + ref: ${{ inputs.setup-java-ref }} + - name: Snapshot the pristine tree + run: bash scripts/cache-key-stability.sh init + + - name: Probe baseline + id: baseline + uses: ./setup-java + with: + distribution: temurin + java-version: ${{ inputs.java-version }} + cache: maven + - name: Record baseline + run: bash scripts/cache-key-stability.sh record "$RESULTS" baseline "${{ steps.baseline.outputs.cache-primary-key }}" + + - name: Probe repeat + id: repeat + uses: ./setup-java + with: + distribution: temurin + java-version: ${{ inputs.java-version }} + cache: maven + - name: Record repeat + run: bash scripts/cache-key-stability.sh record "$RESULTS" repeat "${{ steps.repeat.outputs.cache-primary-key }}" + + - name: Edit an unrelated source file + run: bash scripts/cache-key-stability.sh touch-unrelated + - name: Probe unrelated + id: unrelated + uses: ./setup-java + with: + distribution: temurin + java-version: ${{ inputs.java-version }} + cache: maven + - name: Record unrelated + run: bash scripts/cache-key-stability.sh record "$RESULTS" unrelated "${{ steps.unrelated.outputs.cache-primary-key }}" + + - name: Restore the pristine tree + run: bash scripts/cache-key-stability.sh restore-tree + - name: Edit the dependency manifest + run: bash scripts/cache-key-stability.sh touch-dependencies + - name: Probe dependency + id: dependency + uses: ./setup-java + with: + distribution: temurin + java-version: ${{ inputs.java-version }} + cache: maven + - name: Record dependency + run: bash scripts/cache-key-stability.sh record "$RESULTS" dependency "${{ steps.dependency.outputs.cache-primary-key }}" + + - name: Restore the pristine tree again + run: bash scripts/cache-key-stability.sh restore-tree + # An explicitly pinned path rather than the default pattern, so that both + # ways a user can configure this are covered. The probes above use the + # default `**/pom.xml`, which is what makes "editing a source file does not + # change the key" an assertion about what the action hashes rather than a + # tautology about a single named file. + - name: Probe explicit + id: explicit + uses: ./setup-java + with: + distribution: temurin + java-version: ${{ inputs.java-version }} + cache: maven + cache-dependency-path: petclinic/pom.xml + - name: Record explicit + run: bash scripts/cache-key-stability.sh record "$RESULTS" explicit "${{ steps.explicit.outputs.cache-primary-key }}" + + - name: Upload keys + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: cache-keys-${{ matrix.platform }}-${{ matrix.replica }} + path: .benchmark-results/ + include-hidden-files: true + if-no-files-found: error + retention-days: 30 + + check: + name: Check + needs: probe + if: ${{ always() && !cancelled() }} + runs-on: ubuntu-24.04 + steps: + - name: Check out benchmark repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - name: Download keys + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6 + with: + pattern: cache-keys-* + merge-multiple: true + path: .benchmark-results + - name: Assert cache key properties + env: + SETUP_JAVA_REPOSITORY: ${{ inputs.setup-java-repository }} + SETUP_JAVA_REF: ${{ inputs.setup-java-ref }} + run: node scripts/check-cache-keys.mjs + - name: Upload results + if: ${{ always() }} + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: cache-key-stability-${{ github.run_id }} + path: cache-key-results/ + retention-days: 30 diff --git a/.github/workflows/cache-save.yml b/.github/workflows/cache-save.yml new file mode 100644 index 0000000..4136623 --- /dev/null +++ b/.github/workflows/cache-save.yml @@ -0,0 +1,176 @@ +name: Cache save + +on: + workflow_dispatch: + inputs: + samples: + description: Runners to measure on (each contributes 2 paired observations) + required: true + type: choice + options: + - "2" + - "6" + - "10" + - "20" + default: "10" + setup-java-repository: + description: Repository containing the setup-java action + required: true + default: actions/setup-java + type: string + baseline-ref: + description: Git ref for the baseline arm + required: true + default: v4.8.0 + type: string + candidate-ref: + description: Git ref for the candidate arm + required: true + default: main + type: string + fixture-mib: + description: Synthetic Maven cache fixture size + required: true + type: choice + options: + - "16" + - "64" + - "160" + default: "64" + +permissions: + actions: write + contents: read + +env: + CACHE_KEY_PREFIX: cache-save-${{ github.run_id }}- + +concurrency: + group: cache-save + cancel-in-progress: false + +defaults: + run: + shell: bash + +jobs: + # setup-java saves Maven dependencies from its post-job hook, which cannot be + # bracketed with the millisecond stopwatch used by the other scenarios. The + # transfer itself is delegated to @actions/cache.saveCache in src/cache.ts, so + # this workflow times that toolkit call directly with the dependency version + # resolved by each checked-out setup-java ref. That measures the save cost users + # pay on a cache miss without falling back to the Actions API's one-second step + # timestamps. + measure: + name: paired / ${{ matrix.sample }} + runs-on: ubuntu-24.04 + # At this fixture size a normal save is seconds, not minutes. A slot that has + # not returned after three minutes is a cache-service stall; the report can + # discard the incomplete runner instead of letting the workflow wait for the + # platform default timeout. + timeout-minutes: 25 + strategy: + fail-fast: false + # Every slot creates a fresh cache entry. Running too many uploaders at once + # mostly measures service contention, so the waves match the transfer overlap + # workflow rather than saturating the cache backend. + max-parallel: 4 + matrix: + sample: ${{ fromJSON(inputs.samples == '20' && '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]' || inputs.samples == '10' && '[1,2,3,4,5,6,7,8,9,10]' || inputs.samples == '6' && '[1,2,3,4,5,6]' || '[1,2]') }} + steps: + - name: Check out benchmark repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - name: Check out baseline setup-java + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: ${{ inputs.setup-java-repository }} + path: baseline + persist-credentials: false + ref: ${{ inputs.baseline-ref }} + - name: Check out candidate setup-java + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: ${{ inputs.setup-java-repository }} + path: candidate + persist-credentials: false + ref: ${{ inputs.candidate-ref }} + - name: Install baseline cache client + run: bash scripts/cache-save.sh install-client baseline + - name: Install candidate cache client + run: bash scripts/cache-save.sh install-client candidate + # The fixture keeps the total bytes exact but spreads them across a Maven + # local-repository shape: nested group directories, version directories, + # jars and their .pom/.sha1 sidecars. Real cache saves pay for tar traversal + # and compression before upload, so a single flat file would hide changes in + # archive tooling or path enumeration. + - name: Create synthetic cache fixture + run: | + bash scripts/cache-save.sh prepare-fixture "${{ inputs.fixture-mib }}" + bash scripts/cache-save.sh verify-fixture "${{ inputs.fixture-mib }}" + + # The first cache client call in a job pays connection setup and local page + # cache costs that later slots do not. That one-off spike is not linear + # drift, so ABBA cannot cancel it; a discarded save pays it before the slots + # that enter the paired estimate. + - name: Warm-up save (discarded) + timeout-minutes: 3 + run: bash scripts/cache-save.sh save baseline "${{ env.CACHE_KEY_PREFIX }}${{ matrix.sample }}-warmup-baseline" + + - name: Slot 1 save (baseline) + timeout-minutes: 3 + run: bash scripts/cache-save.sh save baseline "${{ env.CACHE_KEY_PREFIX }}${{ matrix.sample }}-1-baseline" ".benchmark-results/cache-save-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" baseline 1 + + - name: Slot 2 save (candidate) + timeout-minutes: 3 + run: bash scripts/cache-save.sh save candidate "${{ env.CACHE_KEY_PREFIX }}${{ matrix.sample }}-2-candidate" ".benchmark-results/cache-save-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" candidate 2 + + - name: Slot 3 save (candidate) + timeout-minutes: 3 + run: bash scripts/cache-save.sh save candidate "${{ env.CACHE_KEY_PREFIX }}${{ matrix.sample }}-3-candidate" ".benchmark-results/cache-save-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" candidate 3 + + - name: Slot 4 save (baseline) + timeout-minutes: 3 + run: bash scripts/cache-save.sh save baseline "${{ env.CACHE_KEY_PREFIX }}${{ matrix.sample }}-4-baseline" ".benchmark-results/cache-save-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" baseline 4 + + - name: Upload sample timings + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: cache-save-timings-${{ matrix.sample }} + path: .benchmark-results/ + include-hidden-files: true + if-no-files-found: error + retention-days: 30 + + report: + name: Report + needs: measure + if: ${{ always() && !cancelled() }} + runs-on: ubuntu-24.04 + steps: + - name: Check out benchmark repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - name: Download sample timings + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6 + with: + pattern: cache-save-timings-* + merge-multiple: true + path: .benchmark-results + - name: Generate cache-save report + env: + GH_TOKEN: ${{ github.token }} + BASELINE_REF: ${{ inputs.baseline-ref }} + CANDIDATE_REF: ${{ inputs.candidate-ref }} + SETUP_JAVA_REPOSITORY: ${{ inputs.setup-java-repository }} + FIXTURE_MIB: ${{ inputs.fixture-mib }} + CACHE_KEY_PREFIX: ${{ env.CACHE_KEY_PREFIX }} + run: node scripts/report-cache-save.mjs + - name: Upload benchmark results + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: cache-save-${{ github.run_id }} + path: cache-save-results/ + retention-days: 30 diff --git a/.github/workflows/cache-value.yml b/.github/workflows/cache-value.yml new file mode 100644 index 0000000..8be7081 --- /dev/null +++ b/.github/workflows/cache-value.yml @@ -0,0 +1,270 @@ +name: Cache value + +on: + workflow_dispatch: + inputs: + samples: + description: Runners to measure on (each contributes 2 paired observations) + required: true + type: choice + options: + - "2" + - "6" + - "10" + default: "6" + setup-java-repository: + description: Repository containing the setup-java action + required: true + default: actions/setup-java + type: string + setup-java-ref: + description: Git ref of setup-java to measure + required: true + default: main + type: string + java-version: + description: Java feature version + required: true + default: "17" + type: string + cleanup-caches: + description: Delete benchmark caches after measuring them + required: true + type: boolean + default: true + +permissions: + actions: write + contents: read + +env: + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 2 + PETCLINIC_REF: f182358d02e4a68e52bdbabf55ca7800288511e7 + +concurrency: + group: cache-value + cancel-in-progress: false + +defaults: + run: + shell: bash + +# What does the Maven cache actually save? +# +# Every other benchmark in this repository compares implementations of the cache +# with each other, and measures differences of tens or hundreds of milliseconds. +# None of them measures the thing the cache exists to do. This one does: both +# arms resolve the same dependency tree with the same command, and differ only +# in whether setup-java restored the local repository first. +# +# The effect here is tens of seconds rather than tens of milliseconds, so it does +# not need the statistical machinery the other workflows need to see their +# effects. It is reported with the same intervals anyway, because a number +# without one invites the reader to compare it with a number from another run. +jobs: + seed: + name: Seed cache + runs-on: ubuntu-24.04 + steps: + - name: Check out benchmark repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - name: Check out Spring PetClinic + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: spring-projects/spring-petclinic + ref: ${{ env.PETCLINIC_REF }} + path: petclinic + persist-credentials: false + - name: Check out setup-java + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: ${{ inputs.setup-java-repository }} + path: setup-java + persist-credentials: false + ref: ${{ inputs.setup-java-ref }} + - name: Prepare cache identity + run: bash scripts/cache-value.sh prepare "cache-value-${{ github.run_id }}" + - name: Setup Java + uses: ./setup-java + with: + distribution: temurin + java-version: ${{ inputs.java-version }} + cache: maven + cache-dependency-path: .cache-value-key + # Resolving rather than compiling: the measured arms resolve too, so the + # seeded repository has to contain exactly what a resolve needs. Compiling + # would additionally pull the plugins a compile needs and store a cache the + # measured slots never fully use. + - name: Resolve dependencies + run: bash scripts/cache-value.sh resolve + + measure: + name: paired / ${{ matrix.sample }} + needs: seed + runs-on: ubuntu-24.04 + # The uncached arm downloads PetClinic's whole dependency tree from Maven + # Central twice per runner, which is the slowest thing this repository does. + timeout-minutes: 40 + strategy: + fail-fast: false + # Kept small for Maven Central rather than for the cache service: twenty + # runners resolving the same tree at once would measure how Central treats + # a burst from one IP range, which is not the quantity under test. + max-parallel: 3 + matrix: + sample: ${{ fromJSON(inputs.samples == '10' && '[1,2,3,4,5,6,7,8,9,10]' || inputs.samples == '6' && '[1,2,3,4,5,6]' || '[1,2]') }} + steps: + - name: Check out benchmark repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - name: Check out Spring PetClinic + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: spring-projects/spring-petclinic + ref: ${{ env.PETCLINIC_REF }} + path: petclinic + persist-credentials: false + - name: Check out setup-java + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: ${{ inputs.setup-java-repository }} + path: setup-java + persist-credentials: false + ref: ${{ inputs.setup-java-ref }} + + # The first setup in a job pays DNS resolution, TLS handshakes and a cold + # page cache. That is a one-off spike rather than drift, so the mirrored + # slot order cannot cancel it. This slot pays those costs and is discarded. + - name: Reset warm-up slot + run: bash scripts/cache-value.sh reset "cache-value-${{ github.run_id }}" + - name: Warm-up setup (discarded) + timeout-minutes: 5 + uses: ./setup-java + with: + distribution: temurin + java-version: ${{ inputs.java-version }} + cache: maven + cache-dependency-path: .cache-value-key + + # ABBA across the four measured slots: sharing a runner removes the + # between-runner variance, and the mirrored order cancels drift across the + # job. Measuring each arm twice also yields a free A/A noise floor. + - name: Reset slot 1 + run: bash scripts/cache-value.sh reset "cache-value-${{ github.run_id }}" + - name: Start slot 1 timer + run: node scripts/measure.mjs start + - name: Slot 1 setup (cached) + id: setup_1 + timeout-minutes: 5 + uses: ./setup-java + with: + distribution: temurin + java-version: ${{ inputs.java-version }} + cache: maven + cache-dependency-path: .cache-value-key + - name: Slot 1 resolve + run: bash scripts/cache-value.sh resolve + - name: Record slot 1 + run: node scripts/measure.mjs record ".benchmark-results/cache-value-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" cached 1 + - name: Verify slot 1 + run: bash scripts/cache-value.sh verify cached "${{ steps.setup_1.outputs.cache-hit }}" + + - name: Reset slot 2 + run: bash scripts/cache-value.sh reset "cache-value-${{ github.run_id }}" + - name: Start slot 2 timer + run: node scripts/measure.mjs start + - name: Slot 2 setup (uncached) + id: setup_2 + timeout-minutes: 5 + uses: ./setup-java + with: + distribution: temurin + java-version: ${{ inputs.java-version }} + - name: Slot 2 resolve + run: bash scripts/cache-value.sh resolve + - name: Record slot 2 + run: node scripts/measure.mjs record ".benchmark-results/cache-value-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" uncached 2 + - name: Verify slot 2 + run: bash scripts/cache-value.sh verify uncached "${{ steps.setup_2.outputs.cache-hit }}" + + - name: Reset slot 3 + run: bash scripts/cache-value.sh reset "cache-value-${{ github.run_id }}" + - name: Start slot 3 timer + run: node scripts/measure.mjs start + - name: Slot 3 setup (uncached) + id: setup_3 + timeout-minutes: 5 + uses: ./setup-java + with: + distribution: temurin + java-version: ${{ inputs.java-version }} + - name: Slot 3 resolve + run: bash scripts/cache-value.sh resolve + - name: Record slot 3 + run: node scripts/measure.mjs record ".benchmark-results/cache-value-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" uncached 3 + - name: Verify slot 3 + run: bash scripts/cache-value.sh verify uncached "${{ steps.setup_3.outputs.cache-hit }}" + + - name: Reset slot 4 + run: bash scripts/cache-value.sh reset "cache-value-${{ github.run_id }}" + - name: Start slot 4 timer + run: node scripts/measure.mjs start + - name: Slot 4 setup (cached) + id: setup_4 + timeout-minutes: 5 + uses: ./setup-java + with: + distribution: temurin + java-version: ${{ inputs.java-version }} + cache: maven + cache-dependency-path: .cache-value-key + - name: Slot 4 resolve + run: bash scripts/cache-value.sh resolve + - name: Record slot 4 + run: node scripts/measure.mjs record ".benchmark-results/cache-value-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" cached 4 + - name: Verify slot 4 + run: bash scripts/cache-value.sh verify cached "${{ steps.setup_4.outputs.cache-hit }}" + + - name: Upload sample timings + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: cache-value-timings-${{ matrix.sample }} + path: .benchmark-results/ + include-hidden-files: true + if-no-files-found: error + retention-days: 30 + + report: + name: Report + needs: measure + if: ${{ always() && !cancelled() }} + runs-on: ubuntu-24.04 + steps: + - name: Check out benchmark repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - name: Download sample timings + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6 + with: + pattern: cache-value-timings-* + merge-multiple: true + path: .benchmark-results + - name: Generate cache value report + env: + GH_TOKEN: ${{ github.token }} + SAMPLES: ${{ inputs.samples }} + JAVA_VERSION: ${{ inputs.java-version }} + SETUP_JAVA_REPOSITORY: ${{ inputs.setup-java-repository }} + SETUP_JAVA_REF: ${{ inputs.setup-java-ref }} + CLEANUP_CACHES: ${{ inputs.cleanup-caches }} + run: node scripts/report-cache-value.mjs + - name: Upload benchmark results + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: cache-value-${{ github.run_id }} + path: cache-value-results/ + retention-days: 30 diff --git a/.github/workflows/focused-cache-restore.yml b/.github/workflows/focused-cache-restore.yml deleted file mode 100644 index 262da18..0000000 --- a/.github/workflows/focused-cache-restore.yml +++ /dev/null @@ -1,178 +0,0 @@ -name: Focused cache restore - -on: - workflow_dispatch: - inputs: - samples: - description: Warm restore samples per version - required: true - type: choice - options: - - "2" - - "6" - - "10" - - "20" - default: "10" - cleanup-caches: - description: Delete benchmark caches after measuring them - required: true - type: boolean - default: true - -permissions: - actions: write - contents: read - -env: - JAVA_VERSION: 17.0.19+10 - DEPENDENCY_FIXTURE_MIB: "160" - WRAPPER_FIXTURE_MIB: "9" - -concurrency: - group: focused-cache-restore - cancel-in-progress: false - -jobs: - v4-seed: - name: Seed v4 cache - runs-on: ubuntu-24.04 - steps: - - name: Check out benchmark repository - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - persist-credentials: false - - name: Prepare cache identity - env: - BENCHMARK_ID: focused-v4-${{ github.run_id }} - run: | - printf '%s\n' "$BENCHMARK_ID" > .focused-cache-key - mkdir -p .mvn/wrapper - printf 'wrapperVersion=focused\n# benchmark-id=%s\n' "$BENCHMARK_ID" > .mvn/wrapper/maven-wrapper.properties - - name: Setup Java v4 - uses: actions/setup-java@v4.8.0 - with: - distribution: temurin - java-version: ${{ env.JAVA_VERSION }} - cache: maven - cache-dependency-path: .focused-cache-key - - name: Create synthetic cache fixtures - run: | - mkdir -p ~/.m2/repository/focused-benchmark - head -c "$((DEPENDENCY_FIXTURE_MIB * 1024 * 1024))" /dev/urandom > ~/.m2/repository/focused-benchmark/dependencies.bin & - wait - - main-seed: - name: Seed main caches - runs-on: ubuntu-24.04 - steps: - - name: Check out benchmark repository - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - persist-credentials: false - - name: Prepare cache identity - env: - BENCHMARK_ID: focused-main-${{ github.run_id }} - run: | - printf '%s\n' "$BENCHMARK_ID" > .focused-cache-key - mkdir -p .mvn/wrapper - printf 'wrapperVersion=focused\n# benchmark-id=%s\n' "$BENCHMARK_ID" > .mvn/wrapper/maven-wrapper.properties - - name: Setup Java main - uses: actions/setup-java@main - with: - distribution: temurin - java-version: ${{ env.JAVA_VERSION }} - cache: maven - cache-dependency-path: .focused-cache-key - - name: Create synthetic cache fixtures - run: | - mkdir -p ~/.m2/repository/focused-benchmark - mkdir -p ~/.m2/wrapper/dists/focused-benchmark - head -c "$((DEPENDENCY_FIXTURE_MIB * 1024 * 1024))" /dev/urandom > ~/.m2/repository/focused-benchmark/dependencies.bin & - head -c "$((WRAPPER_FIXTURE_MIB * 1024 * 1024))" /dev/urandom > ~/.m2/wrapper/dists/focused-benchmark/wrapper.bin & - wait - - v4-measure: - name: v4 / warm / ${{ matrix.sample }} - needs: v4-seed - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - sample: ${{ fromJSON(inputs.samples == '20' && '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]' || inputs.samples == '10' && '[1,2,3,4,5,6,7,8,9,10]' || inputs.samples == '6' && '[1,2,3,4,5,6]' || '[1,2]') }} - steps: - - name: Check out benchmark repository - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - persist-credentials: false - - name: Prepare cache identity - env: - BENCHMARK_ID: focused-v4-${{ github.run_id }} - run: | - printf '%s\n' "$BENCHMARK_ID" > .focused-cache-key - mkdir -p .mvn/wrapper - printf 'wrapperVersion=focused\n# benchmark-id=%s\n' "$BENCHMARK_ID" > .mvn/wrapper/maven-wrapper.properties - - name: Setup Java v4 - uses: actions/setup-java@v4.8.0 - with: - distribution: temurin - java-version: ${{ env.JAVA_VERSION }} - cache: maven - cache-dependency-path: .focused-cache-key - - name: Verify restored fixtures - run: | - test "$(wc -c < ~/.m2/repository/focused-benchmark/dependencies.bin)" -eq "$((DEPENDENCY_FIXTURE_MIB * 1024 * 1024))" - - main-measure: - name: main / warm / ${{ matrix.sample }} - needs: main-seed - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - sample: ${{ fromJSON(inputs.samples == '20' && '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]' || inputs.samples == '10' && '[1,2,3,4,5,6,7,8,9,10]' || inputs.samples == '6' && '[1,2,3,4,5,6]' || '[1,2]') }} - steps: - - name: Check out benchmark repository - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - persist-credentials: false - - name: Prepare cache identity - env: - BENCHMARK_ID: focused-main-${{ github.run_id }} - run: | - printf '%s\n' "$BENCHMARK_ID" > .focused-cache-key - mkdir -p .mvn/wrapper - printf 'wrapperVersion=focused\n# benchmark-id=%s\n' "$BENCHMARK_ID" > .mvn/wrapper/maven-wrapper.properties - - name: Setup Java main - uses: actions/setup-java@main - with: - distribution: temurin - java-version: ${{ env.JAVA_VERSION }} - cache: maven - cache-dependency-path: .focused-cache-key - - name: Verify restored fixtures - run: | - test "$(wc -c < ~/.m2/repository/focused-benchmark/dependencies.bin)" -eq "$((DEPENDENCY_FIXTURE_MIB * 1024 * 1024))" - test "$(wc -c < ~/.m2/wrapper/dists/focused-benchmark/wrapper.bin)" -eq "$((WRAPPER_FIXTURE_MIB * 1024 * 1024))" - - report: - name: Report - needs: [v4-measure, main-measure] - if: ${{ always() && !cancelled() }} - runs-on: ubuntu-24.04 - steps: - - name: Check out benchmark repository - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - persist-credentials: false - - name: Generate focused report - env: - GH_TOKEN: ${{ github.token }} - SAMPLES: ${{ inputs.samples }} - CLEANUP_CACHES: ${{ inputs.cleanup-caches }} - run: node scripts/report-focused.mjs - - name: Upload benchmark results - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: focused-cache-restore-${{ github.run_id }} - path: focused-results/ - retention-days: 30 diff --git a/.github/workflows/jdk-cache.yml b/.github/workflows/jdk-cache.yml index dc1de1b..c104dd8 100644 --- a/.github/workflows/jdk-cache.yml +++ b/.github/workflows/jdk-cache.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: samples: - description: Warm samples per arm + description: Runners to measure on (each contributes 2 paired observations) required: true type: choice options: @@ -26,6 +26,16 @@ on: required: true type: string default: "17" + setup-java-repository: + description: Repository containing the setup-java action + required: true + default: actions/setup-java + type: string + setup-java-ref: + description: Git ref of setup-java to measure + required: true + default: main + type: string cleanup-caches: description: Delete benchmark caches after measuring them required: true @@ -37,12 +47,23 @@ permissions: contents: read env: + # A cache segment download that fails leaves @actions/cache holding an armed + # timer it never clears, so the action idles for the rest of this timeout after + # its work is done (see actions/toolkit's promiseWithTimeout: the clearTimeout + # runs in a .then, which a rejection skips). The default is 10 minutes. These + # restores move 154 MiB in about 3 seconds, so 2 minutes is still far beyond + # any legitimate download and bounds a stall to a fifth of what it was. + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 2 PETCLINIC_REF: f182358d02e4a68e52bdbabf55ca7800288511e7 concurrency: group: jdk-cache-benchmark cancel-in-progress: false +defaults: + run: + shell: bash + jobs: prepare: name: Prepare JDK cache benchmark @@ -66,59 +87,46 @@ jobs: gh cache delete "$cache_id" --repo "${{ github.repository }}" done <<< "$cache_ids" - baseline-seed: - name: baseline / seed + # Both arms share one Maven cache entry. A cache entry's download throughput + # depends on where the service placed the stored blob, so seeding one entry per + # arm would confound the arm with its blob. That bias is the same on every + # runner, which means pairing cannot remove it and more samples only tighten + # the interval around the wrong answer. Only the JDK cache differs between the + # arms, which is the effect under test. + seed: + name: Seed caches needs: prepare runs-on: ubuntu-24.04 steps: - - name: Check out Spring PetClinic + - name: Check out benchmark repository uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: - repository: spring-projects/spring-petclinic - ref: ${{ env.PETCLINIC_REF }} persist-credentials: false - - name: Prepare baseline cache identity - env: - BENCHMARK_ID: jdk-cache-baseline-${{ github.run_id }} - run: | - printf '%s\n' "$BENCHMARK_ID" > .jdk-cache-key - printf '# jdk-cache-benchmark=%s\n' "$BENCHMARK_ID" >> .mvn/wrapper/maven-wrapper.properties - # Deliberately benchmark the path where the requested JDK is not preinstalled. - - name: Purge matching runner tool cache - run: rm -rf "$RUNNER_TOOL_CACHE"/Java_* - - name: Setup Java main - uses: actions/setup-java@main - with: - distribution: ${{ inputs.distribution }} - java-version: ${{ inputs.java-version }} - cache: maven - cache-jdk: false - cache-dependency-path: .jdk-cache-key - - name: Build PetClinic - run: ./mvnw --batch-mode --no-transfer-progress compile - - treatment-seed: - name: treatment / seed - needs: prepare - runs-on: ubuntu-24.04 - steps: + # PetClinic goes below the benchmark repository rather than over it: its + # build runs a nohttp check across the whole basedir and would otherwise + # lint the benchmark scripts. - name: Check out Spring PetClinic uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: repository: spring-projects/spring-petclinic ref: ${{ env.PETCLINIC_REF }} + path: petclinic persist-credentials: false - - name: Prepare treatment cache identity - env: - BENCHMARK_ID: jdk-cache-treatment-${{ github.run_id }} - run: | - printf '%s\n' "$BENCHMARK_ID" > .jdk-cache-key - printf '# jdk-cache-benchmark=%s\n' "$BENCHMARK_ID" >> .mvn/wrapper/maven-wrapper.properties - # The seed must download the JDK so setup-java registers its post-job save. + - name: Check out setup-java + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: ${{ inputs.setup-java-repository }} + path: .setup-java + persist-credentials: false + ref: ${{ inputs.setup-java-ref }} + - name: Prepare cache identity + run: bash scripts/jdk-cache.sh prepare "jdk-cache-${{ github.run_id }}" + # The seed must download the JDK from the vendor so that setup-java + # registers its post-job save of the JDK cache. - name: Purge matching runner tool cache - run: rm -rf "$RUNNER_TOOL_CACHE"/Java_* - - name: Setup Java main - uses: actions/setup-java@main + run: bash scripts/jdk-cache.sh purge-toolcache + - name: Setup Java + uses: ./.setup-java with: distribution: ${{ inputs.distribution }} java-version: ${{ inputs.java-version }} @@ -126,33 +134,73 @@ jobs: cache-jdk: true cache-dependency-path: .jdk-cache-key - name: Build PetClinic + working-directory: petclinic run: ./mvnw --batch-mode --no-transfer-progress compile - baseline-measure: - name: baseline / warm / ${{ matrix.sample }} - needs: baseline-seed + # Both arms run in the same job in ABBA order, so each runner yields one paired + # difference with its own speed cancelled out and the mirrored order cancels + # drift across the four slots. + measure: + name: paired / ${{ matrix.sample }} + needs: seed runs-on: ubuntu-24.04 + # A measurement job is a couple of minutes of work. Anything far beyond that + # is a stalled cache download or JDK fetch rather than a slow measurement, + # and the run should not wait six hours for the default timeout to notice. + # The runner drops out of the report, which discards runners that did not + # complete every slot, and the remaining runners still produce a verdict. + timeout-minutes: 25 strategy: fail-fast: false + # Every runner pulls the same seeded blobs, so running the whole matrix at + # once measures contention on the cache service that no real workflow would + # see. Waves are kept small so each measurement reflects an ordinary + # restore. + max-parallel: 4 matrix: sample: ${{ fromJSON(inputs.samples == '20' && '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]' || inputs.samples == '10' && '[1,2,3,4,5,6,7,8,9,10]' || inputs.samples == '6' && '[1,2,3,4,5,6]' || '[1,2]') }} steps: + - name: Check out benchmark repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + # PetClinic goes below the benchmark repository rather than over it: its + # build runs a nohttp check across the whole basedir and would otherwise + # lint the benchmark scripts. - name: Check out Spring PetClinic uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: repository: spring-projects/spring-petclinic ref: ${{ env.PETCLINIC_REF }} + path: petclinic persist-credentials: false - - name: Prepare baseline cache identity - env: - BENCHMARK_ID: jdk-cache-baseline-${{ github.run_id }} - run: | - printf '%s\n' "$BENCHMARK_ID" > .jdk-cache-key - printf '# jdk-cache-benchmark=%s\n' "$BENCHMARK_ID" >> .mvn/wrapper/maven-wrapper.properties - - name: Purge matching runner tool cache - run: rm -rf "$RUNNER_TOOL_CACHE"/Java_* - - name: Setup Java main - uses: actions/setup-java@main + - name: Check out setup-java + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: ${{ inputs.setup-java-repository }} + path: .setup-java + persist-credentials: false + ref: ${{ inputs.setup-java-ref }} + + # Each slot deletes the extracted JDK and the local repository first, so + # every measured setup starts from the same empty state instead of + # extracting over files an earlier slot left behind. + # The first setup in a job pays costs the later ones do not: DNS resolution, + # TLS handshakes to the cache service and the JDK host, and a cold page + # cache. That is a one-off spike rather than drift, so the mirrored slot + # order cannot cancel it, and it showed up as the A/A control resolving a + # difference between an arm's own first and last slot. This slot pays those + # costs and is discarded. + - name: Reset warm-up slot + run: bash scripts/jdk-cache.sh reset "jdk-cache-${{ github.run_id }}" + - name: Warm-up setup (discarded) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 + uses: ./.setup-java with: distribution: ${{ inputs.distribution }} java-version: ${{ inputs.java-version }} @@ -160,34 +208,41 @@ jobs: cache-jdk: false cache-read-only: true cache-dependency-path: .jdk-cache-key - - name: Build PetClinic - run: ./mvnw --batch-mode --no-transfer-progress compile - treatment-measure: - name: treatment / warm / ${{ matrix.sample }} - needs: treatment-seed - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - sample: ${{ fromJSON(inputs.samples == '20' && '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]' || inputs.samples == '10' && '[1,2,3,4,5,6,7,8,9,10]' || inputs.samples == '6' && '[1,2,3,4,5,6]' || '[1,2]') }} - steps: - - name: Check out Spring PetClinic - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - name: Reset slot 1 + run: bash scripts/jdk-cache.sh reset "jdk-cache-${{ github.run_id }}" + - name: Start slot 1 timer + run: node scripts/measure.mjs start + - name: Slot 1 setup (baseline, no JDK cache) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 + uses: ./.setup-java with: - repository: spring-projects/spring-petclinic - ref: ${{ env.PETCLINIC_REF }} - persist-credentials: false - - name: Prepare treatment cache identity - env: - BENCHMARK_ID: jdk-cache-treatment-${{ github.run_id }} - run: | - printf '%s\n' "$BENCHMARK_ID" > .jdk-cache-key - printf '# jdk-cache-benchmark=%s\n' "$BENCHMARK_ID" >> .mvn/wrapper/maven-wrapper.properties - - name: Purge matching runner tool cache - run: rm -rf "$RUNNER_TOOL_CACHE"/Java_* - - name: Setup Java main - uses: actions/setup-java@main + distribution: ${{ inputs.distribution }} + java-version: ${{ inputs.java-version }} + cache: maven + cache-jdk: false + cache-read-only: true + cache-dependency-path: .jdk-cache-key + - name: Record slot 1 + run: node scripts/measure.mjs record ".benchmark-results/jdk-cache-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" baseline 1 + + - name: Reset slot 2 + run: bash scripts/jdk-cache.sh reset "jdk-cache-${{ github.run_id }}" + - name: Start slot 2 timer + run: node scripts/measure.mjs start + - name: Slot 2 setup (candidate, JDK cache) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 + uses: ./.setup-java with: distribution: ${{ inputs.distribution }} java-version: ${{ inputs.java-version }} @@ -195,16 +250,65 @@ jobs: cache-jdk: true cache-read-only: true cache-dependency-path: .jdk-cache-key - - name: Build PetClinic - run: ./mvnw --batch-mode --no-transfer-progress compile + - name: Record slot 2 + run: node scripts/measure.mjs record ".benchmark-results/jdk-cache-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" candidate 2 + + - name: Reset slot 3 + run: bash scripts/jdk-cache.sh reset "jdk-cache-${{ github.run_id }}" + - name: Start slot 3 timer + run: node scripts/measure.mjs start + - name: Slot 3 setup (candidate, JDK cache) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 + uses: ./.setup-java + with: + distribution: ${{ inputs.distribution }} + java-version: ${{ inputs.java-version }} + cache: maven + cache-jdk: true + cache-read-only: true + cache-dependency-path: .jdk-cache-key + - name: Record slot 3 + run: node scripts/measure.mjs record ".benchmark-results/jdk-cache-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" candidate 3 + + - name: Reset slot 4 + run: bash scripts/jdk-cache.sh reset "jdk-cache-${{ github.run_id }}" + - name: Start slot 4 timer + run: node scripts/measure.mjs start + - name: Slot 4 setup (baseline, no JDK cache) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 + uses: ./.setup-java + with: + distribution: ${{ inputs.distribution }} + java-version: ${{ inputs.java-version }} + cache: maven + cache-jdk: false + cache-read-only: true + cache-dependency-path: .jdk-cache-key + - name: Record slot 4 + run: node scripts/measure.mjs record ".benchmark-results/jdk-cache-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" baseline 4 + + - name: Upload sample timings + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: jdk-cache-timings-${{ matrix.sample }} + path: .benchmark-results/ + include-hidden-files: true + if-no-files-found: error + retention-days: 30 report: name: Report - needs: - - baseline-seed - - treatment-seed - - baseline-measure - - treatment-measure + needs: measure if: ${{ always() && !cancelled() }} runs-on: ubuntu-24.04 steps: @@ -212,12 +316,20 @@ jobs: uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: persist-credentials: false + - name: Download sample timings + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6 + with: + pattern: jdk-cache-timings-* + merge-multiple: true + path: .benchmark-results - name: Generate JDK cache report env: GH_TOKEN: ${{ github.token }} SAMPLES: ${{ inputs.samples }} DISTRIBUTION: ${{ inputs.distribution }} JAVA_VERSION: ${{ inputs.java-version }} + SETUP_JAVA_REPOSITORY: ${{ inputs.setup-java-repository }} + SETUP_JAVA_REF: ${{ inputs.setup-java-ref }} CLEANUP_CACHES: ${{ inputs.cleanup-caches }} run: node scripts/report-jdk-cache.mjs - name: Upload benchmark results diff --git a/.github/workflows/maven-configuration-warm-path.yml b/.github/workflows/maven-configuration-warm-path.yml deleted file mode 100644 index 52c5893..0000000 --- a/.github/workflows/maven-configuration-warm-path.yml +++ /dev/null @@ -1,174 +0,0 @@ -name: Maven configuration warm path - -on: - workflow_dispatch: - inputs: - setup-java-repository: - description: Repository containing the setup-java action - required: true - default: actions/setup-java - type: string - baseline-ref: - description: Git ref containing the baseline implementation - required: true - default: main - type: string - candidate-ref: - description: Git ref containing the candidate implementation - required: true - default: main - type: string - -permissions: - contents: read - -defaults: - run: - shell: bash - -concurrency: - group: maven-configuration-warm-path - cancel-in-progress: false - -jobs: - benchmark: - name: ${{ matrix.os }} ${{ matrix.cache }} ${{ matrix.versions.name }} ${{ matrix.toolchains }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest, macos-15-intel] - cache: [none, maven, gradle] - versions: - - name: single - java-version: "21" - - name: multiple - java-version: | - 17 - 21 - toolchains: [empty, existing] - steps: - - name: Check out benchmark repository - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - persist-credentials: false - - name: Check out baseline setup-java - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - repository: ${{ inputs.setup-java-repository }} - path: baseline - persist-credentials: false - ref: ${{ inputs.baseline-ref }} - - name: Check out candidate setup-java - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - repository: ${{ inputs.setup-java-repository }} - path: candidate - persist-credentials: false - ref: ${{ inputs.candidate-ref }} - - - name: Record setup dist sizes - run: | - bash scripts/benchmark-maven-configuration.sh record-size baseline baseline - bash scripts/benchmark-maven-configuration.sh record-size candidate candidate - - - name: Prepare baseline iteration 1 - run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}" - - name: Start baseline iteration 1 timer - run: bash scripts/benchmark-maven-configuration.sh start - - name: Run baseline iteration 1 - uses: ./baseline - with: - distribution: temurin - java-version: ${{ matrix.versions.java-version }} - cache: ${{ matrix.cache != 'none' && matrix.cache || '' }} - cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }} - settings-path: benchmark-maven-home - - name: Record baseline iteration 1 - run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" baseline 1 - - - name: Prepare candidate iteration 1 - run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}" - - name: Start candidate iteration 1 timer - run: bash scripts/benchmark-maven-configuration.sh start - - name: Run candidate iteration 1 - uses: ./candidate - with: - distribution: temurin - java-version: ${{ matrix.versions.java-version }} - cache: ${{ matrix.cache != 'none' && matrix.cache || '' }} - cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }} - settings-path: benchmark-maven-home - - name: Record candidate iteration 1 - run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" candidate 1 - - - name: Prepare baseline iteration 2 - run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}" - - name: Start baseline iteration 2 timer - run: bash scripts/benchmark-maven-configuration.sh start - - name: Run baseline iteration 2 - uses: ./baseline - with: - distribution: temurin - java-version: ${{ matrix.versions.java-version }} - cache: ${{ matrix.cache != 'none' && matrix.cache || '' }} - cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }} - settings-path: benchmark-maven-home - - name: Record baseline iteration 2 - run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" baseline 2 - - - name: Prepare candidate iteration 2 - run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}" - - name: Start candidate iteration 2 timer - run: bash scripts/benchmark-maven-configuration.sh start - - name: Run candidate iteration 2 - uses: ./candidate - with: - distribution: temurin - java-version: ${{ matrix.versions.java-version }} - cache: ${{ matrix.cache != 'none' && matrix.cache || '' }} - cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }} - settings-path: benchmark-maven-home - - name: Record candidate iteration 2 - run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" candidate 2 - - - name: Prepare baseline iteration 3 - run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}" - - name: Start baseline iteration 3 timer - run: bash scripts/benchmark-maven-configuration.sh start - - name: Run baseline iteration 3 - uses: ./baseline - with: - distribution: temurin - java-version: ${{ matrix.versions.java-version }} - cache: ${{ matrix.cache != 'none' && matrix.cache || '' }} - cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }} - settings-path: benchmark-maven-home - - name: Record baseline iteration 3 - run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" baseline 3 - - - name: Prepare candidate iteration 3 - run: bash scripts/benchmark-maven-configuration.sh prepare "${{ matrix.cache }}" "${{ matrix.toolchains }}" - - name: Start candidate iteration 3 timer - run: bash scripts/benchmark-maven-configuration.sh start - - name: Run candidate iteration 3 - uses: ./candidate - with: - distribution: temurin - java-version: ${{ matrix.versions.java-version }} - cache: ${{ matrix.cache != 'none' && matrix.cache || '' }} - cache-dependency-path: benchmark/${{ matrix.cache == 'gradle' && 'build.gradle' || 'pom.xml' }} - settings-path: benchmark-maven-home - - name: Record candidate iteration 3 - run: bash scripts/benchmark-maven-configuration.sh record "${{ matrix.os }}" "${{ matrix.cache }}" "${{ matrix.versions.name }}" "${{ matrix.toolchains }}" candidate 3 - - - name: Summarize benchmark - run: bash scripts/benchmark-maven-configuration.sh summarize "$GITHUB_STEP_SUMMARY" - - name: Upload raw benchmark data - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: maven-config-${{ matrix.os }}-${{ matrix.cache }}-${{ matrix.versions.name }}-${{ matrix.toolchains }} - path: .benchmark-results/ - include-hidden-files: true - if-no-files-found: error - retention-days: 30 diff --git a/.github/workflows/transfer-overlap.yml b/.github/workflows/transfer-overlap.yml new file mode 100644 index 0000000..962f27c --- /dev/null +++ b/.github/workflows/transfer-overlap.yml @@ -0,0 +1,323 @@ +name: Transfer overlap + +on: + workflow_dispatch: + inputs: + samples: + description: Runners to measure on (each contributes 2 paired observations) + required: true + type: choice + options: + - "2" + - "6" + - "10" + - "20" + default: "10" + setup-java-repository: + description: Repository containing the setup-java action + required: true + default: actions/setup-java + type: string + baseline-ref: + description: Git ref for the baseline arm + required: true + default: v4.8.0 + type: string + candidate-ref: + description: Git ref for the candidate arm + required: true + default: main + type: string + cleanup-caches: + description: Delete benchmark caches after measuring them + required: true + type: boolean + default: true + +permissions: + actions: write + contents: read + +# Does setup-java overlap the transfers it has to make? +# +# When a build configures more than one cache — a Maven local repository and a +# wrapper distribution, say — setup-java restores both. Whether it does so one +# after the other or at the same time is entirely its own choice, unlike the +# throughput of either transfer, which belongs to `@actions/cache` and to the +# network. That makes overlap one of the few properties of caching that a change +# to this action can actually move, and PR actions/setup-java#1174 moved it, by +# awaiting the two restores together instead of in sequence. +# +# The fixtures are deliberately large. An effect that exists only while a +# transfer is in flight is proportional to how long the transfer takes, so a +# small entry would shrink the very thing being measured below the noise. This +# is the mirror image of the action overhead scenario, which uses a tiny entry +# for exactly the opposite reason. + +env: + # A cache segment download that fails leaves @actions/cache holding an armed + # timer it never clears, so the action idles for the rest of this timeout after + # its work is done (see actions/toolkit's promiseWithTimeout: the clearTimeout + # runs in a .then, which a rejection skips). The default is 10 minutes. These + # restores move 154 MiB in about 3 seconds, so 2 minutes is still far beyond + # any legitimate download and bounds a stall to a fifth of what it was. + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 2 + JAVA_VERSION: 17.0.19+10 + DEPENDENCY_FIXTURE_MIB: "160" + WRAPPER_FIXTURE_MIB: "9" + +concurrency: + group: transfer-overlap + cancel-in-progress: false + +defaults: + run: + shell: bash + +jobs: + # A single cache entry is seeded and both arms restore it. Giving each arm its + # own entry confounds the arm with the stored blob: a cache entry's download + # throughput depends on where the service placed it, and that bias is the same + # on every runner, so pairing cannot remove it and more samples only tighten + # the interval around the wrong answer. An A/A run of this workflow with one + # entry per arm reported a 0.859s improvement on identical code, with the + # baseline blob served at ~60 MB/s and the candidate blob at ~105-130 MB/s on + # the same runner. + seed: + name: Seed cache + runs-on: ubuntu-24.04 + steps: + - name: Check out benchmark repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + # Both refs are checked out in every job, including the arm this job does + # not use. The wrapper cache key hashes **/.mvn/wrapper/maven-wrapper.properties, + # which cache-dependency-path does not override, and setup-java carries such + # a file under __tests__. A job that checked out one ref would therefore + # derive a different wrapper key than one that checked out two, and the + # seeded wrapper cache would never be found. + - name: Check out baseline setup-java + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: ${{ inputs.setup-java-repository }} + path: baseline + persist-credentials: false + ref: ${{ inputs.baseline-ref }} + - name: Check out candidate setup-java + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: ${{ inputs.setup-java-repository }} + path: candidate + persist-credentials: false + ref: ${{ inputs.candidate-ref }} + - name: Prepare cache identity + run: bash scripts/transfer-overlap.sh prepare "transfer-overlap-${{ github.run_id }}" + # Seeded with the candidate so that the wrapper cache, which only newer + # revisions maintain, is populated too. The baseline simply does not + # restore it. + - name: Setup Java candidate + uses: ./candidate + with: + distribution: temurin + java-version: ${{ env.JAVA_VERSION }} + cache: maven + cache-dependency-path: .transfer-overlap-cache-key + - name: Create synthetic cache fixtures + run: bash scripts/transfer-overlap.sh seed-fixtures + + # Both arms are measured inside the same job, in ABBA order. Sharing a runner + # removes the between-runner variance that otherwise swamps the effect, and + # the mirrored order cancels drift across the four slots. Measuring each arm + # twice per runner also yields a free A/A noise-floor estimate. + measure: + name: paired / ${{ matrix.sample }} + needs: seed + runs-on: ubuntu-24.04 + # A measurement job is a couple of minutes of work. Anything far beyond that + # is a stalled cache download or JDK fetch rather than a slow measurement, + # and the run should not wait six hours for the default timeout to notice. + # The runner drops out of the report, which discards runners that did not + # complete every slot, and the remaining runners still produce a verdict. + timeout-minutes: 25 + strategy: + fail-fast: false + # Every runner pulls the same seeded blob, so running the whole matrix at + # once puts all of them on the cache service simultaneously and measures + # contention that no real workflow would see. At 20 concurrent runners the + # mean restore rose from about 3s to about 8s and the noise floor from + # 0.34s to 1.89s, which is enough to hide the effect under test. Waves are + # kept small so each measurement reflects an ordinary restore. + max-parallel: 4 + matrix: + sample: ${{ fromJSON(inputs.samples == '20' && '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]' || inputs.samples == '10' && '[1,2,3,4,5,6,7,8,9,10]' || inputs.samples == '6' && '[1,2,3,4,5,6]' || '[1,2]') }} + steps: + - name: Check out benchmark repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - name: Check out baseline setup-java + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: ${{ inputs.setup-java-repository }} + path: baseline + persist-credentials: false + ref: ${{ inputs.baseline-ref }} + - name: Check out candidate setup-java + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: ${{ inputs.setup-java-repository }} + path: candidate + persist-credentials: false + ref: ${{ inputs.candidate-ref }} + + # The first setup in a job pays costs the later ones do not: DNS resolution, + # TLS handshakes to the cache service and the JDK host, and a cold page + # cache. That is a one-off spike rather than drift, so the mirrored slot + # order cannot cancel it, and it showed up as the A/A control resolving a + # difference between an arm's own first and last slot. This slot pays those + # costs and is discarded. + - name: Reset warm-up slot + run: bash scripts/transfer-overlap.sh reset "transfer-overlap-${{ github.run_id }}" + - name: Warm-up setup (discarded) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 + uses: ./baseline + with: + distribution: temurin + java-version: ${{ env.JAVA_VERSION }} + cache: maven + cache-dependency-path: .transfer-overlap-cache-key + + - name: Reset slot 1 + run: bash scripts/transfer-overlap.sh reset "transfer-overlap-${{ github.run_id }}" + - name: Start slot 1 timer + run: node scripts/measure.mjs start + - name: Slot 1 setup (baseline) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 + uses: ./baseline + with: + distribution: temurin + java-version: ${{ env.JAVA_VERSION }} + cache: maven + cache-dependency-path: .transfer-overlap-cache-key + - name: Record slot 1 + run: node scripts/measure.mjs record ".benchmark-results/transfer-overlap-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" baseline 1 + - name: Verify slot 1 fixtures + run: bash scripts/transfer-overlap.sh verify baseline + + - name: Reset slot 2 + run: bash scripts/transfer-overlap.sh reset "transfer-overlap-${{ github.run_id }}" + - name: Start slot 2 timer + run: node scripts/measure.mjs start + - name: Slot 2 setup (candidate) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 + uses: ./candidate + with: + distribution: temurin + java-version: ${{ env.JAVA_VERSION }} + cache: maven + cache-dependency-path: .transfer-overlap-cache-key + - name: Record slot 2 + run: node scripts/measure.mjs record ".benchmark-results/transfer-overlap-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" candidate 2 + - name: Verify slot 2 fixtures + run: bash scripts/transfer-overlap.sh verify candidate + + - name: Reset slot 3 + run: bash scripts/transfer-overlap.sh reset "transfer-overlap-${{ github.run_id }}" + - name: Start slot 3 timer + run: node scripts/measure.mjs start + - name: Slot 3 setup (candidate) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 + uses: ./candidate + with: + distribution: temurin + java-version: ${{ env.JAVA_VERSION }} + cache: maven + cache-dependency-path: .transfer-overlap-cache-key + - name: Record slot 3 + run: node scripts/measure.mjs record ".benchmark-results/transfer-overlap-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" candidate 3 + - name: Verify slot 3 fixtures + run: bash scripts/transfer-overlap.sh verify candidate + + - name: Reset slot 4 + run: bash scripts/transfer-overlap.sh reset "transfer-overlap-${{ github.run_id }}" + - name: Start slot 4 timer + run: node scripts/measure.mjs start + - name: Slot 4 setup (baseline) + # A restore that has not finished in three minutes has stalled on the + # cache service rather than being slow: its neighbours in the same job + # take a few seconds. Failing here costs one runner, which the report + # discards along with the other incomplete ones, instead of letting the + # job sit until its own timeout. + timeout-minutes: 3 + uses: ./baseline + with: + distribution: temurin + java-version: ${{ env.JAVA_VERSION }} + cache: maven + cache-dependency-path: .transfer-overlap-cache-key + - name: Record slot 4 + run: node scripts/measure.mjs record ".benchmark-results/transfer-overlap-timings-${{ matrix.sample }}.csv" "${{ matrix.sample }}" baseline 4 + - name: Verify slot 4 fixtures + run: bash scripts/transfer-overlap.sh verify baseline + + - name: Upload sample timings + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: transfer-overlap-timings-${{ matrix.sample }} + path: .benchmark-results/ + include-hidden-files: true + if-no-files-found: error + retention-days: 30 + + report: + name: Report + needs: measure + if: ${{ always() && !cancelled() }} + runs-on: ubuntu-24.04 + steps: + - name: Check out benchmark repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - name: Download sample timings + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6 + with: + pattern: transfer-overlap-timings-* + merge-multiple: true + path: .benchmark-results + - name: Generate transfer overlap report + env: + GH_TOKEN: ${{ github.token }} + SAMPLES: ${{ inputs.samples }} + BASELINE_REF: ${{ inputs.baseline-ref }} + CANDIDATE_REF: ${{ inputs.candidate-ref }} + SETUP_JAVA_REPOSITORY: ${{ inputs.setup-java-repository }} + CLEANUP_CACHES: ${{ inputs.cleanup-caches }} + run: node scripts/report-transfer-overlap.mjs + - name: Upload benchmark results + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: transfer-overlap-${{ github.run_id }} + path: transfer-overlap-results/ + retention-days: 30 diff --git a/README.md b/README.md index c7249ec..5c5b83d 100644 --- a/README.md +++ b/README.md @@ -7,63 +7,164 @@ The benchmark is designed around the two costs that matter to Actions users: - **Execution time:** setup, build, cache restore, and post-job cache save durations. - **Cache storage:** compressed JDK, Maven dependency, and Maven Wrapper cache sizes. -## Scenarios +## Methodology + +Every workflow here measures effects of a few hundred milliseconds to a few seconds on hosted runners, where the variance between runners is larger than the effect. Four properties are what make a result mean something, and all four workflows now share them. + +**Millisecond timing.** The Actions API reports step `started_at` and `completed_at` only to the nearest second. Setup steps take two to six seconds, so reading durations from the API quantizes every measurement to ±500 ms — the same magnitude as the effects being measured. Timing is taken inside the job with `scripts/measure.mjs`. + +**Same-runner pairing.** Between-runner variance cannot be averaged away by adding more independent jobs to each arm. Every arm is measured inside the _same_ job, in an order mirrored about the middle of the job: ABBA for two arms, and `v1..main` followed by `main..v1` for the version sweep. Differencing within a runner removes the runner's own speed, and the mirrored order cancels drift that is linear across the job. Each measured slot deletes `~/.m2` first so every restore extracts into an empty tree. + +Every job also runs one unmeasured warm-up slot first. The first setup in a job pays costs the later ones do not — DNS resolution, TLS handshakes to the cache service and the JDK host, and a cold page cache — and that is a one-off spike rather than drift, so the mirrored order cannot cancel it. Without the warm-up slot the A/A control resolved a spurious 0.4 s difference between an arm's own first and last slot. + +**One cache, every arm.** A cache entry's download throughput depends on where the service placed the stored blob, and that placement is fixed for the life of the entry. Seeding one entry per arm therefore confounds the arm with its blob, and because the bias is identical on every runner, pairing cannot remove it and more samples only tighten the interval around the wrong answer. A single entry is seeded and every arm restores it. + +**Intervals, not point estimates.** `scripts/stats.mjs` reports a bootstrap 95% confidence interval, a permutation p-value, and a Hodges-Lehmann shift, and turns them into an explicit verdict. A comparison whose interval includes zero is reported as `inconclusive` rather than as a number that looks like a result. A verdict additionally requires the permutation test to agree: the percentile bootstrap is only approximate at ten paired observations, while the sign-flip test is exact under the null, so where they disagree the interval is the one that is wrong. + +**Stalled slots fail fast.** Each measured setup is capped at three minutes. A restore that has not finished by then has stalled on the cache service rather than being slow — its neighbours in the same job take a few seconds — and failing costs one runner instead of holding the run open. The version sweep restores fifteen times per job where the two-arm workflows restore five, so it runs in narrower waves. + +**Stalled runners are discarded.** A slot can stall on the cache service for several seconds. Which arm the stall lands on is arbitrary, so that runner contributes an arbitrarily large difference and, with ten runners, one such slot moves the mean by more than any effect being measured. Runners whose _own arm disagrees with itself_ by more than a robust threshold are dropped. That decision is made purely on within-arm spread, which has the same distribution whether or not the arms differ, so unlike filtering on the arm difference it cannot bias the result. Every report lists what it discarded and why. + +Every report also publishes two guard rails: + +- A **noise floor**, the median spread between the two slots of the same arm on one runner. An effect smaller than this is reported as `within-noise` even when its interval excludes zero. +- An **A/A control**, the same estimator applied to one arm against itself. It costs no extra jobs because every arm is already measured twice per runner. A healthy run reports `within-noise` or `inconclusive`; anything else means the harness is biasing results and the headline verdict cannot be trusted. + +After changing a harness, run it with both arms set to the same ref. The true effect is then exactly zero, and any other verdict is a defect rather than a finding. That check is what caught the per-arm cache confound described above: on identical code it reported a 0.859 s improvement, with the baseline blob served at ~60 MB/s and the candidate blob at ~105–130 MB/s on the same runner in the same job. + +`scripts/paired.mjs` implements the pairing and `scripts/stats.mjs` the statistics; every report builds on both. + +## What each workflow asks + +Caching is a chain, and a benchmark is only useful if it says which link it is measuring. A restore that is quick because it hit is not comparable to one that is quick because it found nothing; a difference in transfer time is usually a difference in the network rather than in setup-java. The workflows are therefore organised by question, and each is built around the _one_ quantity that answers it. + +| Workflow | Question | How it is arranged | Effect it can resolve | +| ----------------------------- | ------------------------------------------ | --------------------------------------------------- | ---------------------- | +| **Cache value** | What does caching buy at all? | Real PetClinic build, cached against uncached | Tens of seconds | +| **Cache key stability** | Does the cache hit when it should? | Deterministic assertions on the computed key | Pass or fail | +| **Action overhead** | What does setup-java's own code cost? | ~1 MiB entry, so the transfer is not what is timed | Tens to hundreds of ms | +| **Transfer overlap** | Does it overlap the transfers it makes? | Large entries, so concurrency has something to hide | Hundreds of ms | +| **Cache save** | What does the first run pay? | Post-run save of a Maven-shaped tree | Seconds | +| **JDK cache** | Is caching the JDK worth it? | `cache-jdk` off against on, same ref | Seconds | +| **Benchmark** (version sweep) | How does `main` compare with each release? | All versions on one runner, mirrored order | Hundreds of ms | + +The sizes in the third column are the design, not an accident. setup-java does not move the bytes itself — it hands the transfer to `@actions/cache` — so a benchmark with a large fixture measures the network and a benchmark with a small one measures the action. **Action overhead** and **Transfer overlap** are deliberately the same measurement at two fixture sizes for exactly that reason, and together they decompose a restore into the part setup-java controls and the part it does not. + +## Version sweep Each action version runs with Java 17 on `ubuntu-24.04`: -| Distribution | Expected setup path | Purpose | -| --- | --- | --- | -| Eclipse Temurin | Hosted runner tool-cache hit | Measures setup overhead when the JDK is already available | -| Microsoft Build of OpenJDK | JDK download and extraction | Measures setup overhead when the JDK must be installed | +| Distribution | Expected setup path | Purpose | +| -------------------------- | ---------------------------- | --------------------------------------------------------- | +| Eclipse Temurin | Hosted runner tool-cache hit | Measures setup overhead when the JDK is already available | +| Microsoft Build of OpenJDK | JDK download and extraction | Measures setup overhead when the JDK must be installed | v1 predates distribution selection and integrated dependency caching. It runs only its native Zulu installer path. v2 supports the Temurin and Microsoft scenarios, but its bundled legacy cache client is rejected by the current Actions cache service. v1 and v2 therefore have no Maven cache storage, and their cold/warm labels are repeated uncached samples. -Every combination gets an isolated cache key and runs twice: +A seed job compiles Spring PetClinic once to populate a single Maven cache entry. Every measurement runner then sets up all seven versions in one job, in the order `v1..main` followed by `main..v1`, deleting `~/.m2` before each slot. The report compares `main` with every version using a paired interval per runner, but it ranks only the versions that do the same work from the same stored blob. + +**`main` is the thing under test, not the thing being ranked.** `main` is the newest code, and every released version is a baseline it is measured against, exactly as the two-arm workflows measure a candidate against a baseline. So the reported difference is `main` minus the version, and the verdict describes `main`: an `improvement` means `main` is faster than that version. Differencing the other way would label a released version a `regression` for being slower than the code that succeeded it, which inverts what was actually measured — the finding there is that `main` got faster. -1. **Cold:** no dependency or wrapper cache exists; the post action saves every cache supported by that version. -2. **Warm:** restores the caches created by the matching cold job. +**Only comparable versions are ranked.** v4, v5.2, v5.6 and `main` all restore the same seeded entry through `cache-dependency-path`, so a difference between them is a difference in the implementation. `main` is ranked against each of them, with Holm's step-down correction across that family: it is tested against every one of them in one run, so without a correction the chance that one comparison clears 0.05 by luck is far above 0.05. -v3, v4, and v5.2 cache only Maven dependencies. v5.6 and `main` also cache the Maven Wrapper distribution separately, making the storage and execution-time tradeoff visible. Because v3 predates `cache-dependency-path`, its benchmark identity is an inert XML comment appended to `pom.xml`; later versions use a dedicated marker file. +v1, v2 and v3 are measured on the same runners and published, but they carry no verdict, because a verdict would report a difference in the _workload_ as though it were a difference in the implementation: + +| Version | Why it is not ranked | +| ------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | +| v1.4.4 | Installs its own JDK and does no dependency caching | +| v2.5.1 | Its bundled cache client is rejected by the current cache service, so it restores nothing | +| v3.14.1 | Predates `cache-dependency-path` and keys on `pom.xml`, so it restores its own entry — the blob confound described above, which pairing cannot remove | + +v3 is the instructive case. In run 30979614347 it took 3.69 s and 3.89 s on two runners that ran v4 in 0.50 s and 0.52 s moments later in the same job. A sevenfold gap that appears on some runners and not others is blob placement, not code, and ranking it would have published a verdict for it. Spring PetClinic and third-party actions are pinned to commits. `setup-java@main` intentionally remains a moving ref so each run evaluates the current upcoming v6 code; the report records the `main` commit observed when it is generated. ## Running -Open **Actions > Benchmark setup-java > Run workflow**. Choose one, three, or five independent samples. Three is the default. +Every workflow is dispatched from **Actions > _(workflow)_ > Run workflow**. The ones that compare two refs take a `baseline-ref` and a `candidate-ref`, so any of them can be pointed at a PR branch; the ones that sample across runners take a runner count, where each runner contributes two observations per arm. -The report job writes a Markdown summary and uploads raw JSON and CSV files. Benchmark-created caches are deleted after measurement by default, preventing repeated runs from consuming repository cache storage. Disable cleanup when you need to inspect the entries manually. +Each report job writes a Markdown summary to the run and uploads raw JSON and CSV. Benchmark-created caches are deleted after measurement, because the repository shares one cache budget across all of these workflows and an entry left behind by one run evicts the seeded entries another depends on. -### Focused cache restore +**Run a harness change against itself before believing it.** Set both refs to the same value: the true effect is then exactly zero, and any verdict other than `inconclusive` or `within-noise` is a defect rather than a finding. -The **Focused cache restore** workflow isolates the setup step for comparing v4 with `main`. It uses a pinned Temurin JDK from the hosted runner tool cache, seeds a synthetic 160 MiB dependency cache for both versions and a 9 MiB wrapper cache for `main`, and runs no Maven command. Warm measurement jobs therefore contain no JDK or Maven Central downloads; they measure JDK discovery and Actions cache restoration only. +## The workflows in detail -### JDK cache +### Cache value + +The **Cache value** workflow answers the question a user actually has: is turning `cache: maven` on worth it? It builds Spring PetClinic for real, cached against uncached, in ABBA order on one runner. + +This is the only scenario whose effect is large enough to see without any statistical machinery, and that is the point of running it — it establishes the scale everything else is a fraction of. A result here in the tens of seconds is what makes an argument about 40 ms of action overhead worth having or not worth having. -The **JDK cache** workflow measures the installed-JDK cache added on `actions/setup-java@main`. It compares a baseline arm with `cache-jdk: false` against a treatment arm with `cache-jdk: true`, defaulting to Microsoft Build of OpenJDK 17. Both arms use the same action ref so the result isolates JDK caching instead of conflating it with implementation changes between commits. Each arm first seeds its Maven dependency and wrapper caches by compiling Spring PetClinic. The treatment seed also downloads and saves the JDK. Warm matrix jobs then use `cache-read-only: true`, so they restore caches without creating entries or racing to save the same key. +### Cache key stability -JDK cache keys are derived from the JDK's identity and source; unlike dependency cache keys, they cannot be namespaced per iteration. The seed/read-only design avoids cross-contamination between samples. Every seed and measurement job also removes matching JDKs from `$RUNNER_TOOL_CACHE` before setup, deliberately measuring the not-preinstalled path and preventing a hosted-runner tool-cache hit from bypassing JDK cache restore and save logic. Select Temurin to test that same forced-miss path with a distribution normally preinstalled on hosted runners. +The **Cache key stability** workflow measures nothing. The cache key is a deterministic function of the tree, so its properties can be asserted outright rather than estimated, and those properties dominate every timing in this repository: a key that changes when it should not costs a full cache miss — about a minute on PetClinic — where the timing workflows are resolving tens of milliseconds. + +It checks that the key is unchanged when the same tree is hashed twice, unchanged when an unrelated source file is edited, changed when the dependency manifest is edited, identical across two runners on the same platform, and different across platforms. The cross-runner check is the important one: if two runners on the same tree disagree, caching never works for anyone, because every job computes a key no other job has stored. + +Nothing is stored by any of it. setup-java skips its post-job save when the configured path does not exist, and no probe ever creates `~/.m2`. + +### Action overhead + +The **Action overhead** workflow compares two refs across three operating systems, four cache profiles and two configuration layouts, with a cache entry of about a megabyte spread over many small files. The small entry is the whole design: it leaves resolving a distribution, computing a cache key, writing settings and toolchains, and the bookkeeping around a restore as what is being timed, rather than the network. + +The four cache profiles are nested levels of work rather than competing options: + +| Profile | What the action does | +| ------------- | --------------------------------------------------- | +| `none` | Never touches the cache code | +| `maven-miss` | Computes a key and is told no | +| `maven-hit` | Computes a key, is told yes, unpacks a ~1 MiB entry | +| `gradle-miss` | A miss through the other package manager | + +Differencing the levels says where the time goes, and the report publishes that decomposition alongside the comparison. Those level differences are between-configuration, so they are reported as observed medians with no interval and no verdict; they describe what a setup costs rather than establishing that two refs differ. + +Each configuration is one runner, so it yields one paired difference and cannot support an interval alone. Configurations are treated as **blocks** — arms compared within a configuration, on the same runner, in ABBA order after a discarded warm-up slot — and the differences pooled across the matrix. + +Every slot records setup-java's `cache-hit` output, and the report refuses to be read normally if a `maven-hit` slot missed or a `maven-miss` slot hit, because either turns a level of the decomposition into a different level and the labels stop being true. + +#### Why this replaced the Maven configuration warm path + +The workflow this grew out of was called a warm path and was not one. It had no seed job, and its synthetic `benchmark/pom.xml` hashed to a key nothing had ever stored, so every one of its "warm" restores was a miss — the job logs read `maven cache is not found` and `Path Validation Error ... hence no cache is being saved`. Its three cache profiles were three variations on a failed lookup, and it could not see the restore path at all. The differences it reported were real, but they were differences in the cost of _missing_, published under a heading that said warm. + +### Transfer overlap + +The **Transfer overlap** workflow isolates the setup step to compare two refs (by default `v4.8.0` against `main`) with a synthetic 160 MiB dependency cache seeded for both arms and a 9 MiB wrapper cache, and runs no Maven command. Measurement jobs contain no JDK or Maven Central downloads. + +When a build configures more than one cache, setup-java restores both, and whether it does so in sequence or at the same time is its own choice — unlike the throughput of either transfer. That makes overlap one of the few properties of caching a change to this action can actually move, and [actions/setup-java#1174](https://github.com/actions/setup-java/pull/1174) moved it by awaiting the two restores together. An effect that exists only while a transfer is in flight is proportional to how long the transfer takes, which is why the fixtures here are large and why the same effect is invisible in **Action overhead**. + +This workflow is also the reference implementation for how a comparison is measured in this repository. Point `baseline-ref` and `candidate-ref` at any two refs — including a PR branch — to check whether a change delivers a real improvement, or at the _same_ ref to check the harness itself. + +#### Why the harness was rebuilt + +The earlier version ran each arm as its own matrix of independent jobs, read durations from the Actions API, and reported a "paired median delta" that paired `sample N` of one arm with `sample N` of the other. Those samples shared no runner and no point in time, so the pairing removed no variance at all. + +Two consecutive runs of that harness against an unchanged `v4.8.0` — where the true difference is exactly zero — produced medians of 2 s and 3 s, a spurious 1.2 s separation whose confidence interval excluded zero. Over the same pair of runs the reported candidate delta flipped from +0.6 s to −0.8 s. `scripts/stats.test.mjs` pins that dataset as a regression test so unpaired sampling is not reintroduced. + +### Cache save + +The **Cache save** workflow measures what the first run pays. A cache miss costs the download the user already paid for _plus_ the save, and nothing else here measured the second half of that. + +setup-java saves in its post-job hook, which cannot be bracketed by a timer from inside the job. The save is therefore driven directly through the `@actions/cache` version each ref pins, which is where the difference between refs actually lives, against a Maven-shaped fixture: nested group directories of small `.jar`, `.pom` and `.sha1` files rather than one large blob, because archive cost tracks file count and directory depth as much as it tracks bytes. + +### JDK cache -Open **Actions > JDK cache > Run workflow** to select the distribution, Java version, warm sample count, and cache cleanup behavior. The report compares warm setup, build, post-step, and job medians; records cold seed setup and JDK save time; and reports the JDK, dependency, and wrapper cache sizes. +The **JDK cache** workflow measures the installed-JDK cache added on `actions/setup-java@main`. It compares `cache-jdk: false` against `cache-jdk: true`, defaulting to Microsoft Build of OpenJDK 17. Both arms use the same action ref, so the result isolates JDK caching instead of conflating it with implementation changes between commits. -### Maven configuration warm path +A single seed job compiles Spring PetClinic to populate one Maven cache entry and one JDK cache entry. Every measurement runner then runs both arms in one job in ABBA order under `cache-read-only: true`. Both arms restore the same Maven entry — only the JDK entry differs between them, which is the effect under test. -The **Maven configuration warm path** workflow compares two `actions/setup-java` refs in the action's own Maven configuration path. It checks out a configurable setup-java repository, runs baseline and candidate refs on Linux, Windows, and macOS, and covers Maven cache, Gradle cache, no-cache, single-version, multi-version, empty toolchains, and existing toolchains scenarios. +Every seed and measured slot removes matching JDKs from `$RUNNER_TOOL_CACHE` first, deliberately measuring the not-preinstalled path and preventing a hosted-runner tool-cache hit from bypassing JDK cache restore logic. Select Temurin to test that same forced-miss path with a distribution normally preinstalled on hosted runners. -Each matrix entry alternates three warm in-job setup runs for the baseline and candidate implementations, then reports median and p95 setup time. The workflow also records `dist/setup/index.js`, total `dist/setup` JavaScript bytes, and the JavaScript chunk files containing the XML parser. +JDK cache keys are derived from the JDK's identity and source, so unlike dependency cache keys they cannot be namespaced per run; the `prepare` job deletes existing JDK caches before seeding. ## Reading results -The summary reports medians for: +**Read the verdict, not the point estimate.** Every comparison reports one, and it is the only part of the output that has been checked against the harness's own noise. A result reported as `inconclusive` has established nothing however suggestive its number looks, and one reported as `within-noise` is smaller than this harness can resolve on hosted runners. -- setup time, including JDK discovery/download and dependency-cache restore; -- Spring PetClinic `compile` time; -- the `setup-java` post step that saves caches; -- compressed cache storage per isolated case; -- estimated billed minutes, calculated by rounding each Linux job to a whole minute. +**Check the A/A control first.** It applies the same estimator to one arm against itself, where the true difference is exactly zero. A healthy run reports `within-noise` or `inconclusive`. Anything else means the run measured the harness rather than the code, and the headline is void. -Public repositories do not pay for standard GitHub-hosted runners. The estimated minutes are included to make the results applicable to private repositories; actual charges depend on the account plan and runner type. +**Check which link was measured.** A number from **Action overhead** and a number from **Transfer overlap** are not the same quantity and do not add up to a user-visible saving on their own; the scale that makes either of them matter comes from **Cache value**. -Network throughput, hosted-runner image changes, upstream artifact availability, and runner load all introduce variance. Compare multiple iterations and multiple workflow runs before drawing conclusions. +Network throughput, hosted-runner image changes, upstream artifact availability and runner load all introduce variance, which is what the guard rails above exist to absorb. ## Local checks diff --git a/scripts/action-overhead.sh b/scripts/action-overhead.sh new file mode 100644 index 0000000..e98b837 --- /dev/null +++ b/scripts/action-overhead.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash + +set -euo pipefail + +command=${1:?command is required} + +benchmark_home="$PWD/benchmark-maven-home" +results_dir="$PWD/.benchmark-results" +sizes_file="$results_dir/action-overhead-sizes.csv" + +# Roughly a megabyte spread over many small files. The point of this scenario is +# to measure what setup-java's own code costs, so the entry has to be small +# enough that the transfer is not what is being timed, while still being made of +# many files rather than one, because archiving cost tracks file count as much as +# it tracks bytes. +fixture_files=200 +fixture_file_kib=5 + +write_manifest() { + identity=$1 + + mkdir -p benchmark + # The cache key is a hash of this file, so the identity written into it is what + # decides whether a slot hits the seeded entry or misses. Every byte outside + # the identity has to be stable, or two slots that should share an entry would + # compute different keys. + cat > benchmark/pom.xml < + $identity + +XML + cat > benchmark/build.gradle < "$benchmark_home/toolchains.xml" <<'XML' + + + foo + + preserved + + + /opt/foo + + + +XML + elif [ "$toolchains_profile" != "empty" ]; then + echo "Unsupported toolchains profile: $toolchains_profile" >&2 + exit 1 + fi +} + +case "$command" in + # Build the tiny local repository that the seed job stores, so that the + # measured `maven-hit` slots have something real to restore. + seed-fixture) + identity=${2:?identity is required} + + rm -rf "$benchmark_home" + mkdir -p "$benchmark_home" + write_manifest "$identity" + + repository="$HOME/.m2/repository/com/example/benchmark" + rm -rf "$HOME/.m2" + mkdir -p "$repository" + for index in $(seq 1 "$fixture_files"); do + # Random rather than zeroes so the archive cannot be compressed away to + # nothing, which would make the seeded entry unrepresentative of a real + # local repository full of already-compressed jars. + dd if=/dev/urandom "of=$repository/artifact-$index.jar" \ + bs=1024 count="$fixture_file_kib" status=none + done + ;; + prepare) + cache_profile=${2:?cache profile is required} + toolchains_profile=${3:?toolchains profile is required} + identity=${4:?identity is required} + + + rm -rf "$benchmark_home" + mkdir -p "$benchmark_home" + write_manifest "$identity" + write_toolchains "$toolchains_profile" + + # Every slot starts from an absent cache directory. Restoring on top of an + # existing tree would let the second slot in an arm do less work than the + # first, and it also keeps setup-java from saving anything on the profiles + # that are supposed to miss: the action skips its post-job save when the + # configured path does not exist. + case "$cache_profile" in + none | maven-miss | maven-hit) + rm -rf "$HOME/.m2" + ;; + gradle-miss) + rm -rf "$HOME/.gradle" + ;; + *) + echo "Unsupported cache profile: $cache_profile" >&2 + exit 1 + ;; + esac + ;; + start) + mkdir -p "$results_dir" + node scripts/measure.mjs start + ;; + record) + results_file=${2:?results file is required} + os=${3:?os is required} + cache_profile=${4:?cache profile is required} + layout=${5:?layout profile is required} + arm=${6:?arm is required} + slot=${7:?slot is required} + cache_hit=${8:-} + + mkdir -p "$results_dir" + # An empty cache-hit output is not the same as a miss: the `none` profile + # never asks about the cache, and refs older than the output's introduction + # report nothing. Recording that distinction lets the report tell a genuine + # miss apart from a slot it cannot verify. + node scripts/measure.mjs record "$results_file" \ + "$os" "$cache_profile" "$layout" "$arm" "$slot" "${cache_hit:-unset}" + ;; + record-size) + arm=${2:?arm is required} + action_path=${3:?action path is required} + + mkdir -p "$results_dir" + index_bytes=$(node -e "const fs=require('fs'); process.stdout.write(String(fs.statSync(process.argv[1]).size))" "$action_path/dist/setup/index.js") + js_bytes=$(node -e "const fs=require('fs'); const path=require('path'); let total=0; for (const entry of fs.readdirSync(process.argv[1])) { if (entry.endsWith('.js')) total += fs.statSync(path.join(process.argv[1], entry)).size; } process.stdout.write(String(total));" "$action_path/dist/setup") + chunk_count=$(find "$action_path/dist/setup" -maxdepth 1 -name '*.js' | wc -l | tr -d ' ') + printf '%s,%s,%s,%s\n' \ + "$arm" "$index_bytes" "$js_bytes" "$chunk_count" \ + >> "$sizes_file" + ;; + # Every run stores three fresh entries whose keys are hashes, so nothing about + # the name says which run owns them. The seed jobs record the keys the action + # reported; this deletes exactly those. + delete-seeded) + repository=${2:?repository is required} + + found=0 + while IFS= read -r key_file; do + key=$(tr -d '\r\n' < "$key_file") + if [ -z "$key" ]; then + # Older refs do not publish `cache-primary-key`, so seeding with one + # leaves an entry that has to age out on its own. + echo "No key recorded in $key_file; nothing to delete for it." >&2 + continue + fi + found=$((found + 1)) + if gh cache delete "$key" --repo "$repository"; then + echo "Deleted $key" + else + echo "Could not delete $key; it will expire on its own." >&2 + fi + done < <(find "$results_dir" -name 'seeded-key-*.txt' -type f) + + if [ "$found" -eq 0 ]; then + echo "No seeded cache keys were recorded." >&2 + fi + ;; + *) + echo "Unsupported command: $command" >&2 + exit 1 + ;; +esac diff --git a/scripts/benchmark-maven-configuration.sh b/scripts/benchmark-maven-configuration.sh deleted file mode 100755 index ed6b33c..0000000 --- a/scripts/benchmark-maven-configuration.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -command=${1:?command is required} - -benchmark_home="$PWD/benchmark-maven-home" -results_dir="$PWD/.benchmark-results" -results_file="$results_dir/maven-configuration-timings.csv" -sizes_file="$results_dir/maven-configuration-sizes.csv" - -case "$command" in - prepare) - cache=${2:?cache profile is required} - toolchains_profile=${3:?toolchains profile is required} - - rm -rf "$benchmark_home" - mkdir -p "$benchmark_home" benchmark - printf '\n' > benchmark/pom.xml - printf 'plugins { id("java") }\n' > benchmark/build.gradle - - if [ "$toolchains_profile" = "existing" ]; then - cat > "$benchmark_home/toolchains.xml" <<'XML' - - - foo - - preserved - - - /opt/foo - - - -XML - elif [ "$toolchains_profile" != "empty" ]; then - echo "Unsupported toolchains profile: $toolchains_profile" >&2 - exit 1 - fi - - case "$cache" in - none | maven | gradle) ;; - *) - echo "Unsupported cache profile: $cache" >&2 - exit 1 - ;; - esac - ;; - start) - mkdir -p "$results_dir" - node -e "require('fs').writeFileSync('.benchmark-start', String(Date.now()))" - ;; - record) - os=${2:?os is required} - cache=${3:?cache profile is required} - versions=${4:?versions profile is required} - toolchains_profile=${5:?toolchains profile is required} - implementation=${6:?implementation is required} - iteration=${7:?iteration is required} - - started=$(cat .benchmark-start) - finished=$(node -e "process.stdout.write(String(Date.now()))") - elapsed=$((finished - started)) - mkdir -p "$results_dir" - printf '%s,%s,%s,%s,%s,%s,%s\n' \ - "$os" "$cache" "$versions" "$toolchains_profile" "$implementation" "$iteration" "$elapsed" \ - >> "$results_file" - ;; - record-size) - implementation=${2:?implementation is required} - action_path=${3:?action path is required} - - mkdir -p "$results_dir" - index_bytes=$(node -e "const fs=require('fs'); process.stdout.write(String(fs.statSync(process.argv[1]).size))" "$action_path/dist/setup/index.js") - js_bytes=$(node -e "const fs=require('fs'); const path=require('path'); let total=0; for (const entry of fs.readdirSync(process.argv[1])) { if (entry.endsWith('.js')) total += fs.statSync(path.join(process.argv[1], entry)).size; } process.stdout.write(String(total));" "$action_path/dist/setup") - chunk_count=$(find "$action_path/dist/setup" -maxdepth 1 -name '*.js' | wc -l | tr -d ' ') - xml_parser_chunks=$(grep -Rsl "fast-xml-parser" "$action_path/dist/setup"/*.js 2>/dev/null | xargs -n 1 basename 2>/dev/null | paste -sd ';' - || true) - printf '%s,%s,%s,%s\n' \ - "$implementation" "$index_bytes" "$js_bytes" "${xml_parser_chunks:-none} ($chunk_count js files)" \ - >> "$sizes_file" - ;; - summarize) - summary_file=${2:?summary file is required} - - node --input-type=module - "$results_file" "$sizes_file" "$summary_file" <<'NODE' -import fs from 'node:fs'; - -const [, , resultsFile, sizesFile, summaryFile] = process.argv; - -const percentile = (values, percentileValue) => { - const sorted = [...values].sort((left, right) => left - right); - const index = Math.ceil((percentileValue / 100) * sorted.length) - 1; - return sorted[Math.max(0, Math.min(index, sorted.length - 1))]; -}; - -const rows = fs - .readFileSync(resultsFile, 'utf8') - .trim() - .split('\n') - .filter(Boolean) - .map(line => { - const [os, cache, versions, toolchains, implementation, iteration, elapsed] = - line.split(','); - return { - os, - cache, - versions, - toolchains, - implementation, - iteration, - elapsed: Number(elapsed) - }; - }); - -const groups = new Map(); -for (const row of rows) { - const key = [row.os, row.cache, row.versions, row.toolchains, row.implementation].join(','); - const values = groups.get(key) ?? []; - values.push(row.elapsed); - groups.set(key, values); -} - -const lines = [ - '## Maven configuration warm-path benchmark', - '', - '| OS | Cache | Versions | Toolchains | Implementation | Runs | Median (ms) | p95 (ms) |', - '| --- | --- | --- | --- | --- | ---: | ---: | ---: |' -]; - -for (const [key, values] of [...groups.entries()].sort()) { - const [os, cache, versions, toolchains, implementation] = key.split(','); - lines.push( - `| ${os} | ${cache} | ${versions} | ${toolchains} | ${implementation} | ${values.length} | ${percentile(values, 50)} | ${percentile(values, 95)} |` - ); -} - -if (fs.existsSync(sizesFile)) { - lines.push( - '', - '## setup entry/chunk sizes', - '', - '| Implementation | dist/setup/index.js bytes | dist/setup JS bytes | XML parser chunk location |', - '| --- | ---: | ---: | --- |' - ); - for (const line of fs.readFileSync(sizesFile, 'utf8').trim().split('\n')) { - if (!line) continue; - const [implementation, indexBytes, jsBytes, xmlParserChunks] = line.split(','); - lines.push( - `| ${implementation} | ${indexBytes} | ${jsBytes} | ${xmlParserChunks} |` - ); - } -} - -fs.appendFileSync(summaryFile, `${lines.join('\n')}\n`); -NODE - ;; - *) - echo "Unsupported command: $command" >&2 - exit 1 - ;; -esac diff --git a/scripts/cache-key-stability.sh b/scripts/cache-key-stability.sh new file mode 100644 index 0000000..b7b239c --- /dev/null +++ b/scripts/cache-key-stability.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +# Tree arrangement for the cache key stability check. +# +# This is not a benchmark. It asserts properties of the cache key, because a key +# that changes when it should not costs a full cache miss, and a miss costs more +# than every timing effect the rest of this repository measures put together. On +# Spring PetClinic a miss is roughly a minute; the timing benchmarks resolve +# effects of tens of milliseconds. +# +# The probes never create ~/.m2. setup-java skips its post-job save when the +# configured path does not exist ("Path Validation Error ... hence no cache is +# being saved"), so the whole check runs without storing a single cache entry. + +set -euo pipefail + +command=${1:?command is required} + +project_dir=${PROJECT_DIR:-petclinic} +pom="$project_dir/pom.xml" +pristine_pom=".cache-key-pristine-pom.xml" + +first_java_source() { + find "$project_dir/src" -name '*.java' | sort | head -n 1 +} + +case "$command" in + init) + # A pristine copy so that the probe which edits the pom can be undone + # exactly. Restoring with git would also revert the checkout of PetClinic + # itself, and reverting by editing back risks leaving a trailing newline that + # changes the hash. + cp "$pom" "$pristine_pom" + rm -rf "$HOME/.m2" + ;; + restore-tree) + cp "$pristine_pom" "$pom" + rm -rf "$HOME/.m2" + ;; + touch-unrelated) + # A source file is not part of any dependency-manifest pattern, so hashing it + # would be a defect. This is the change that most often breaks caching in the + # wild: a key that tracks the whole tree misses on every commit. + source_file=$(first_java_source) + if [ -z "$source_file" ]; then + echo "No Java source found under $project_dir/src" >&2 + exit 1 + fi + printf '\n// cache key stability probe\n' >> "$source_file" + printf 'cache key stability probe\n' >> "$project_dir/README.md" + echo "Modified $source_file and README.md" + ;; + touch-dependencies) + # A comment in the XML epilog is well-formed and changes the file's hash + # without changing what Maven resolves, which is what the key is supposed to + # track. + printf '\n' >> "$pom" + ;; + record) + # Written as CSV rather than read from step outputs in the checker, because + # a key that comes back empty must be recorded as empty and diagnosed there + # rather than silently collapsing two probes into one equal pair. + results=${2:?results file is required} + probe=${3:?probe name is required} + key=${4-} + mkdir -p "$(dirname "$results")" + printf '"%s","%s","%s","%s"\n' \ + "${RUNNER_LABEL:?RUNNER_LABEL is required}" \ + "${RUNNER_REPLICA:?RUNNER_REPLICA is required}" \ + "$probe" "$key" >> "$results" + echo "$probe: ${key:-}" + ;; + *) + echo "Unsupported command: $command" >&2 + exit 1 + ;; +esac diff --git a/scripts/cache-save.sh b/scripts/cache-save.sh new file mode 100755 index 0000000..4b3cbff --- /dev/null +++ b/scripts/cache-save.sh @@ -0,0 +1,260 @@ +#!/usr/bin/env bash + +set -euo pipefail + +command=${1:?command is required} +fixture_dir="$HOME/.m2/repository/cache-save-fixture" + +# A Maven cache save is not just bytes on the wire. The archive step walks a deep +# repository tree and compresses thousands of already-compressed jars plus their +# small metadata sidecars, so a flat file would erase the traversal and path +# enumeration work that changes in setup-java's cache client can affect. +case "$command" in + prepare-fixture) + fixture_mib=${2:?fixture size in MiB is required} + rm -rf "$fixture_dir" + node --input-type=module - "$fixture_dir" "$fixture_mib" <<'NODE' +import { mkdir, open, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; + +const [fixtureDir, fixtureMiB] = process.argv.slice(2); +const mib = Number(fixtureMiB); +const bytes = mib * 1024 * 1024; +if (!Number.isInteger(mib) || !Number.isInteger(bytes) || bytes <= 0) { + throw new Error(`Invalid fixture size: ${fixtureMiB}`); +} + +const pomBytes = 512; +const shaBytes = 40; +const artifacts = + mib <= 4 ? 128 : mib <= 16 ? 512 : mib <= 64 ? 1024 : 2048; +const sidecarBytes = artifacts * (pomBytes + shaBytes); +const jarBytes = bytes - sidecarBytes; +if (jarBytes <= artifacts) { + throw new Error( + `${fixtureMiB} MiB is too small for ${artifacts} Maven artifacts`, + ); +} +const baseJarBytes = Math.floor(jarBytes / artifacts); +const extraJarBytes = jarBytes % artifacts; + +let state = 0x9e3779b9; + +async function writeRandomFile(file, size) { + await mkdir(dirname(file), { recursive: true }); + const handle = await open(file, "w"); + const chunk = Buffer.alloc(Math.min(64 * 1024, size)); + let written = 0; + try { + while (written < size) { + const length = Math.min(chunk.length, size - written); + for (let index = 0; index < length; index += 1) { + state ^= state << 13; + state ^= state >>> 17; + state ^= state << 5; + chunk[index] = state & 0xff; + } + await handle.write(chunk, 0, length); + written += length; + } + } finally { + await handle.close(); + } +} + +function paddedPom(group, artifact, version) { + const xml = `4.0.0${group}${artifact}${version}\n`; + return Buffer.from(xml.padEnd(pomBytes, " ").slice(0, pomBytes)); +} + +function sha1Sidecar(seed) { + let value = seed + 1; + let text = ""; + while (text.length < shaBytes) { + value ^= value << 13; + value ^= value >>> 17; + value ^= value << 5; + text += (value >>> 0).toString(16).padStart(8, "0"); + } + return Buffer.from(text.slice(0, shaBytes)); +} + +for (let index = 0; index < artifacts; index += 1) { + const group = `com.example.group${String(index % 64).padStart(2, "0")}.depth${String(Math.floor(index / 64) % 32).padStart(2, "0")}`; + const artifact = `artifact-${String(index).padStart(4, "0")}`; + const version = `1.${index % 17}.${Math.floor(index / 17)}`; + const directory = join( + fixtureDir, + ...group.split("."), + artifact, + version, + ); + const jar = join(directory, `${artifact}-${version}.jar`); + const pom = join(directory, `${artifact}-${version}.pom`); + const sha1 = join(directory, `${artifact}-${version}.jar.sha1`); + await writeRandomFile(jar, baseJarBytes + (index < extraJarBytes ? 1 : 0)); + await writeFile(pom, paddedPom(group, artifact, version)); + await writeFile(sha1, sha1Sidecar(index)); +} +NODE + ;; + expected-file-count) + fixture_mib=${2:?fixture size in MiB is required} + node --input-type=module - "$fixture_mib" <<'NODE' +const [fixtureMiB] = process.argv.slice(2); +const mib = Number(fixtureMiB); +if (!Number.isInteger(mib) || mib <= 0) { + throw new Error(`Invalid fixture size: ${fixtureMiB}`); +} +const artifacts = + mib <= 4 ? 128 : mib <= 16 ? 512 : mib <= 64 ? 1024 : 2048; +console.log(artifacts * 3); +NODE + ;; + verify-fixture) + fixture_mib=${2:?fixture size in MiB is required} + node --input-type=module - "$fixture_dir" "$fixture_mib" <<'NODE' +import { readdir, stat } from "node:fs/promises"; +import { join } from "node:path"; + +const [fixtureDir, fixtureMiB] = process.argv.slice(2); +const mib = Number(fixtureMiB); +const expectedBytes = mib * 1024 * 1024; +if (!Number.isInteger(mib) || !Number.isInteger(expectedBytes) || mib <= 0) { + throw new Error(`Invalid fixture size: ${fixtureMiB}`); +} +const artifacts = + mib <= 4 ? 128 : mib <= 16 ? 512 : mib <= 64 ? 1024 : 2048; +const expectedFiles = artifacts * 3; + +async function walk(directory) { + let files = 0; + let bytes = 0; + for (const entry of await readdir(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + if (entry.isDirectory()) { + const child = await walk(path); + files += child.files; + bytes += child.bytes; + } else if (entry.isFile()) { + files += 1; + bytes += (await stat(path)).size; + } + } + return { files, bytes }; +} + +const actual = await walk(fixtureDir); +if (actual.bytes !== expectedBytes) { + throw new Error( + `Cache-save fixture is ${actual.bytes} bytes, expected ${expectedBytes}`, + ); +} +if (actual.files !== expectedFiles) { + throw new Error( + `Cache-save fixture has ${actual.files} files, expected ${expectedFiles}`, + ); +} +NODE + ;; + file-count) + node --input-type=module - "$fixture_dir" <<'NODE' +import { readdir } from "node:fs/promises"; +import { join } from "node:path"; + +const [fixtureDir] = process.argv.slice(2); + +async function count(directory) { + let files = 0; + for (const entry of await readdir(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + if (entry.isDirectory()) { + files += await count(path); + } else if (entry.isFile()) { + files += 1; + } + } + return files; +} + +console.log(await count(fixtureDir)); +NODE + ;; + remove-one-file) + node --input-type=module - "$fixture_dir" <<'NODE' +import { readdir, rm } from "node:fs/promises"; +import { join } from "node:path"; + +const [fixtureDir] = process.argv.slice(2); + +async function firstFile(directory) { + for (const entry of await readdir(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + if (entry.isDirectory()) { + const found = await firstFile(path); + if (found) return found; + } else if (entry.isFile()) { + return path; + } + } + return null; +} + +const file = await firstFile(fixtureDir); +if (!file) throw new Error(`No fixture file found under ${fixtureDir}`); +await rm(file); +NODE + ;; + install-client) + arm_dir=${2:?setup-java checkout directory is required} + if [ -f "$arm_dir/package-lock.json" ]; then + npm --prefix "$arm_dir" ci --ignore-scripts --no-audit --no-fund + else + npm --prefix "$arm_dir" install --ignore-scripts --no-audit --no-fund + fi + node --input-type=module - "$PWD/$arm_dir/package.json" <<'NODE' +import { createRequire } from "node:module"; + +const [manifest] = process.argv.slice(2); +const require = createRequire(manifest); +require.resolve("@actions/cache"); +NODE + ;; + save) + arm_dir=${2:?setup-java checkout directory is required} + key=${3:?cache key is required} + results_file=${4:-} + sample=${5:-} + arm=${6:-} + slot=${7:-} + if [ ! -d "$fixture_dir" ]; then + echo "Cache-save fixture is missing at $fixture_dir" >&2 + exit 1 + fi + node --input-type=module - "$PWD/$arm_dir/package.json" "$fixture_dir" "$key" \ + "$results_file" "$sample" "$arm" "$slot" <<'NODE' +import { createRequire } from "node:module"; +import { pathToFileURL } from "node:url"; + +const [manifest, fixtureDir, key, resultsFile, sample, arm, slot] = + process.argv.slice(2); +const require = createRequire(manifest); +const cache = require("@actions/cache"); + +const started = Date.now(); +const cacheId = await cache.saveCache([fixtureDir], key); +const elapsedMs = Date.now() - started; +console.log(`Saved ${key} as cache ${cacheId} in ${elapsedMs} ms`); +if (resultsFile) { + const { record } = await import( + pathToFileURL(`${process.cwd()}/scripts/measure.mjs`).href + ); + await record(resultsFile, [sample, arm, slot], elapsedMs); +} +NODE + ;; + *) + echo "Unsupported command: $command" >&2 + exit 1 + ;; +esac diff --git a/scripts/cache-value.sh b/scripts/cache-value.sh new file mode 100644 index 0000000..3f4e81c --- /dev/null +++ b/scripts/cache-value.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +# Fixture helpers for the cache value benchmark. +# +# This benchmark answers the question the rest of the suite was missing: what +# does the Maven cache actually save a real project? Both arms run the same +# resolve against the same dependency tree. The only difference is whether +# setup-java restored the local repository first, so the difference between them +# is the whole value of the feature rather than a detail of its implementation. + +set -euo pipefail + +command=${1:?command is required} + +# Spring PetClinic is checked out below the benchmark repository rather than over +# it, because its build runs a nohttp check across the whole basedir and would +# otherwise lint these scripts. +project_dir=${PROJECT_DIR:-petclinic} + +write_identity() { + # The cached arm keys on this file rather than on the real pom, so the key is + # fixed for the run and every slot restores the entry the seed job stored. + printf '%s\n' "${1:?benchmark id is required}" > .cache-value-key +} + +case "$command" in + prepare) + write_identity "${2:?benchmark id is required}" + ;; + reset) + # Every slot must resolve into an empty local repository. Leaving artifacts + # behind would let the uncached arm resolve from what an earlier cached slot + # restored, which is the one thing this benchmark must not allow: it would + # report that the cache saves nothing. + rm -rf "$HOME/.m2" + write_identity "${2:?benchmark id is required}" + ;; + resolve) + # Both arms run this identical command. The cached arm finds the artifacts in + # the restored repository; the uncached arm fetches them from Maven Central. + # Keeping the command identical is what makes the difference attributable to + # the cache rather than to the work being done. + cd "$project_dir" + ./mvnw --batch-mode --no-transfer-progress dependency:go-offline + ;; + verify) + # Checked from setup-java's own `cache-hit` output rather than by inspecting + # the restored tree, because inspecting it costs a `du` over a few hundred + # MiB and this has to run outside the timed span to stay out of the + # measurement. + # + # A silent miss on the cached arm is the failure that matters: both arms + # would then download from Maven Central and the benchmark would report, + # with a perfectly tight interval, that the cache is worthless. + arm=${2:?arm is required} + cache_hit=${3-} + case "$arm" in + cached) + if [ "$cache_hit" != "true" ]; then + echo "The cached arm reported cache-hit='$cache_hit'; the seeded" \ + "entry was not restored, so this slot measured a download rather" \ + "than a restore" >&2 + exit 1 + fi + ;; + uncached) + if [ -n "$cache_hit" ]; then + echo "The uncached arm reported a cache-hit value ('$cache_hit');" \ + "it must run with caching disabled" >&2 + exit 1 + fi + ;; + *) + echo "Unsupported arm: $arm" >&2 + exit 1 + ;; + esac + ;; + *) + echo "Unsupported command: $command" >&2 + exit 1 + ;; +esac diff --git a/scripts/check-cache-keys.mjs b/scripts/check-cache-keys.mjs new file mode 100644 index 0000000..5da4288 --- /dev/null +++ b/scripts/check-cache-keys.mjs @@ -0,0 +1,255 @@ +// Asserts the properties a cache key has to have, rather than measuring how +// long anything takes. +// +// The rest of this repository measures differences of tens or hundreds of +// milliseconds between implementations of the cache. None of that matters if the +// key misses: on Spring PetClinic a miss costs about a minute, which is three +// orders of magnitude more than the effects the timing benchmarks resolve. A +// key is also a deterministic function of the tree, so these properties can be +// asserted outright instead of estimated with an interval. + +import { + appendFile, + mkdir, + readdir, + readFile, + writeFile, +} from "node:fs/promises"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +import { requireEnv } from "./paired.mjs"; + +export function parseKeys(text) { + return text + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => { + const [platform, replica, probe, key] = line + .split('","') + .map((field) => field.replace(/^"|"$/g, "")); + return { platform, replica, probe, key }; + }); +} + +export async function readKeyFiles(directory = ".benchmark-results") { + const entries = await readdir(directory); + const files = entries.filter((name) => name.startsWith("cache-keys-")); + const contents = await Promise.all( + files.map((name) => readFile(join(directory, name), "utf8")), + ); + return contents.join("\n"); +} + +function lookup(rows, platform, replica, probe) { + return rows.find( + (row) => + row.platform === platform && + row.replica === replica && + row.probe === probe, + ); +} + +// Each check states the user-visible consequence of failing it, because a bare +// "keys differ" tells a maintainer nothing about whether to care. +const WITHIN_RUNNER = [ + { + probe: "repeat", + against: "baseline", + expect: "same", + title: "Repeating the same setup computes the same key", + consequence: + "the key is not a function of the tree, so a second run in the same job would miss", + }, + { + probe: "unrelated", + against: "baseline", + expect: "same", + title: "Editing a source file does not change the key", + consequence: + "every commit that touches source would miss, which is the most common way caching is silently lost", + }, + { + probe: "dependency", + against: "baseline", + expect: "different", + title: "Editing the dependency manifest changes the key", + consequence: + "a build whose dependencies changed would restore a stale repository", + }, +]; + +export function evaluate(rows) { + const platforms = [...new Set(rows.map((row) => row.platform))].sort(); + const checks = []; + + for (const platform of platforms) { + const replicas = [ + ...new Set( + rows.filter((r) => r.platform === platform).map((r) => r.replica), + ), + ].sort(); + + for (const replica of replicas) { + for (const spec of WITHIN_RUNNER) { + const left = lookup(rows, platform, replica, spec.probe); + const right = lookup(rows, platform, replica, spec.against); + checks.push( + compare({ + ...spec, + scope: `${platform} runner ${replica}`, + left, + right, + }), + ); + } + } + + // The check that matters most: two runners in the same run, same tree, same + // platform. If these disagree, caching never works at all - every job + // computes a key no other job has stored. + if (replicas.length >= 2) { + for (const probe of ["baseline", "explicit"]) { + checks.push( + compare({ + probe, + against: probe, + expect: "same", + title: `Two runners compute the same \`${probe}\` key`, + consequence: + "the key depends on something outside the tree, so no job would ever restore what another job stored", + scope: platform, + left: lookup(rows, platform, replicas[0], probe), + right: lookup(rows, platform, replicas[1], probe), + }), + ); + } + } + } + + // A key is scoped to the platform it was built on, so two platforms must not + // collide: restoring a Linux tree onto Windows would be worse than a miss. + if (platforms.length >= 2) { + for (let i = 0; i < platforms.length - 1; i += 1) { + checks.push( + compare({ + probe: "baseline", + against: "baseline", + expect: "different", + title: `\`${platforms[i]}\` and \`${platforms[i + 1]}\` do not share a key`, + consequence: + "one platform could restore an entry another platform stored", + scope: "cross-platform", + left: lookup(rows, platforms[i], "1", "baseline"), + right: lookup(rows, platforms[i + 1], "1", "baseline"), + }), + ); + } + } + + return { checks, passed: checks.every((check) => check.passed) }; +} + +function compare({ title, consequence, expect, scope, left, right }) { + // An absent or empty key is always a failure. Treating it as "equal to the + // other empty key" would let a version that computes no key at all pass every + // sameness check in this file. + if (!left?.key || !right?.key) { + return { + title, + scope, + expect, + passed: false, + detail: "setup-java reported no cache-primary-key for at least one probe", + consequence, + }; + } + const same = left.key === right.key; + const passed = expect === "same" ? same : !same; + return { + title, + scope, + expect, + passed, + detail: same + ? `both \`${left.key}\`` + : `\`${left.key}\` vs \`${right.key}\``, + consequence, + }; +} + +export function markdown(metadata, result) { + const failed = result.checks.filter((check) => !check.passed); + const lines = [ + "# Cache key stability", + "", + `\`${metadata.setupJavaRef}\` from \`${metadata.setupJavaRepository}\`, run ${metadata.runId}.`, + "", + "This is not a timing benchmark. The cache key is a deterministic function of", + "the tree, so its properties are asserted rather than estimated. They matter", + "more than any of the timings: a key that changes when it should not costs a", + "full cache miss, which on this project is about a minute, while the timing", + "benchmarks resolve effects of tens of milliseconds.", + "", + "## Result", + "", + result.passed + ? `**All ${result.checks.length} checks passed.**` + : `**${failed.length} of ${result.checks.length} checks failed.**`, + "", + "| Check | Scope | Expected | Result | Keys |", + "| --- | --- | --- | --- | --- |", + ]; + for (const check of result.checks) { + lines.push( + `| ${check.title} | ${check.scope} | ${check.expect} | ${check.passed ? "pass" : "**fail**"} | ${check.detail} |`, + ); + } + if (failed.length > 0) { + lines.push("", "## What each failure costs", ""); + for (const check of failed) { + lines.push( + `- **${check.title}** (${check.scope}): ${check.consequence}.`, + ); + } + } + return `${lines.join("\n")}\n`; +} + +export async function main(env = process.env) { + const runId = requireEnv(env, "GITHUB_RUN_ID"); + const metadata = { + runId, + setupJavaRepository: requireEnv(env, "SETUP_JAVA_REPOSITORY"), + setupJavaRef: requireEnv(env, "SETUP_JAVA_REF"), + generatedAt: new Date().toISOString(), + }; + const rows = parseKeys(await readKeyFiles()); + if (rows.length === 0) { + throw new Error("No cache keys were collected"); + } + const result = evaluate(rows); + const report = markdown(metadata, result); + + await mkdir("cache-key-results", { recursive: true }); + await writeFile( + "cache-key-results/results.json", + `${JSON.stringify({ metadata, rows, result }, null, 2)}\n`, + ); + await writeFile("cache-key-results/summary.md", report); + await appendFile(env.GITHUB_STEP_SUMMARY, report); + + if (!result.passed) { + throw new Error( + `${result.checks.filter((check) => !check.passed).length} cache key checks failed`, + ); + } +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + await main(); +} diff --git a/scripts/check-cache-keys.test.mjs b/scripts/check-cache-keys.test.mjs new file mode 100644 index 0000000..ed0a46b --- /dev/null +++ b/scripts/check-cache-keys.test.mjs @@ -0,0 +1,118 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { evaluate, markdown, parseKeys } from "./check-cache-keys.mjs"; + +function rows(entries) { + return parseKeys( + entries + .map( + ({ platform, replica, probe, key }) => + `"${platform}","${replica}","${probe}","${key}"`, + ) + .join("\n"), + ); +} + +function healthy() { + const out = []; + for (const platform of ["linux", "windows"]) { + for (const replica of ["1", "2"]) { + const base = `setup-java-${platform}-maven-aaa`; + out.push( + { platform, replica, probe: "baseline", key: base }, + { platform, replica, probe: "repeat", key: base }, + { platform, replica, probe: "unrelated", key: base }, + { + platform, + replica, + probe: "dependency", + key: `setup-java-${platform}-maven-bbb`, + }, + { + platform, + replica, + probe: "explicit", + key: `setup-java-${platform}-maven-ccc`, + }, + ); + } + } + return rows(out); +} + +test("passes a key that behaves correctly", () => { + const result = evaluate(healthy()); + assert.equal(result.passed, true); + assert.ok(result.checks.length > 0); +}); + +test("fails when a source edit changes the key", () => { + const broken = healthy().map((row) => + row.probe === "unrelated" && row.platform === "linux" + ? { ...row, key: "setup-java-linux-maven-zzz" } + : row, + ); + const result = evaluate(broken); + assert.equal(result.passed, false); + const failure = result.checks.find((check) => !check.passed); + assert.match(failure.title, /source file/); + assert.match(failure.consequence, /every commit/); +}); + +test("fails when two runners disagree on the same tree", () => { + const broken = healthy().map((row) => + row.probe === "baseline" && row.replica === "2" && row.platform === "linux" + ? { ...row, key: "setup-java-linux-maven-other" } + : row, + ); + const result = evaluate(broken); + assert.equal(result.passed, false); + assert.ok( + result.checks.some( + (check) => !check.passed && /Two runners compute/.test(check.title), + ), + ); +}); + +test("fails when a dependency change does not change the key", () => { + const broken = healthy().map((row) => + row.probe === "dependency" + ? { ...row, key: `setup-java-${row.platform}-maven-aaa` } + : row, + ); + const result = evaluate(broken); + assert.equal(result.passed, false); +}); + +// A version that computes no key at all would otherwise satisfy every sameness +// check by comparing empty with empty, and the report would read as a clean pass +// for an action that had stopped caching entirely. +test("treats a missing key as a failure rather than as equality", () => { + const empty = healthy().map((row) => ({ ...row, key: "" })); + const result = evaluate(empty); + assert.equal(result.passed, false); + assert.ok( + result.checks.every((check) => check.expect !== "same" || !check.passed), + ); + assert.match( + result.checks.find((check) => !check.passed).detail, + /no cache-primary-key/, + ); +}); + +test("reports what each failure costs", () => { + const broken = healthy().map((row) => + row.probe === "unrelated" ? { ...row, key: "changed" } : row, + ); + const rendered = markdown( + { + runId: "1", + setupJavaRepository: "actions/setup-java", + setupJavaRef: "main", + }, + evaluate(broken), + ); + assert.match(rendered, /checks failed/); + assert.match(rendered, /## What each failure costs/); +}); diff --git a/scripts/jdk-cache.sh b/scripts/jdk-cache.sh new file mode 100755 index 0000000..d94e303 --- /dev/null +++ b/scripts/jdk-cache.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +# Cache-identity and reset helpers for the JDK cache benchmark. + +set -euo pipefail + +command=${1:?command is required} + +# Spring PetClinic is checked out below the benchmark repository rather than over +# it, because its build runs a nohttp check across the whole basedir and would +# otherwise lint these scripts. +project_dir=${PROJECT_DIR:-petclinic} +wrapper_properties="$project_dir/.mvn/wrapper/maven-wrapper.properties" + +purge_toolcache() { + # The benchmark deliberately measures the path where the requested JDK is not + # already installed on the runner image, because that is the only case in + # which the JDK cache does any work. + rm -rf "${RUNNER_TOOL_CACHE:?}"/Java_* +} + +write_identity() { + local benchmark_id=$1 + printf '%s\n' "$benchmark_id" > .jdk-cache-key + # Appended, not overwritten: the real wrapper properties must survive because + # the seed job builds. Appending only once keeps the hashed contents identical + # between the seed job and every measured slot. + local marker="# jdk-cache-benchmark=$benchmark_id" + if [ ! -f "$wrapper_properties" ] || ! grep -qF -- "$marker" "$wrapper_properties"; then + printf '%s\n' "$marker" >> "$wrapper_properties" + fi +} + +case "$command" in + prepare) + write_identity "${2:?benchmark id is required}" + ;; + purge-toolcache) + purge_toolcache + ;; + reset) + # Every measured slot must start from the same empty state. Without this the + # second and later slots in a job would extract over files an earlier slot + # left behind and report an artificially low duration. + purge_toolcache + rm -rf "$HOME/.m2" + write_identity "${2:?benchmark id is required}" + ;; + *) + echo "Unsupported command: $command" >&2 + exit 1 + ;; +esac diff --git a/scripts/measure.mjs b/scripts/measure.mjs new file mode 100644 index 0000000..aaef174 --- /dev/null +++ b/scripts/measure.mjs @@ -0,0 +1,62 @@ +// Millisecond-resolution stopwatch for in-job benchmark slots. +// +// The GitHub Actions API reports step `started_at`/`completed_at` with +// one-second resolution. Setup steps take two to six seconds, so reading +// durations from the API quantizes each measurement to +/- 500 ms — an error of +// the same magnitude as the effects these benchmarks try to detect. Timing the +// step from inside the job instead keeps millisecond precision. + +import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname } from "node:path"; +import { pathToFileURL } from "node:url"; + +const CLOCK_FILE = ".benchmark-clock"; + +export async function start(clockFile = CLOCK_FILE) { + await writeFile(clockFile, String(Date.now())); +} + +export async function stop(clockFile = CLOCK_FILE) { + const started = Number(await readFile(clockFile, "utf8")); + if (!Number.isFinite(started)) { + throw new Error(`No valid start timestamp in ${clockFile}`); + } + return Date.now() - started; +} + +function csvValue(value) { + return `"${String(value ?? "").replaceAll('"', '""')}"`; +} + +export async function record(resultsFile, fields, elapsedMs) { + await mkdir(dirname(resultsFile), { recursive: true }); + const line = [...fields, elapsedMs].map(csvValue).join(","); + await appendFile(resultsFile, `${line}\n`); +} + +export async function main(argv) { + const [command, ...rest] = argv; + switch (command) { + case "start": { + await start(rest[0] ?? CLOCK_FILE); + return; + } + case "record": { + const [resultsFile, ...fields] = rest; + if (!resultsFile) throw new Error("record requires a results file"); + const elapsedMs = await stop(CLOCK_FILE); + await record(resultsFile, fields, elapsedMs); + console.log(`${fields.join("/")}: ${elapsedMs} ms`); + return; + } + default: + throw new Error(`Unsupported command: ${command}`); + } +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + await main(process.argv.slice(2)); +} diff --git a/scripts/paired.mjs b/scripts/paired.mjs new file mode 100644 index 0000000..eb8dbca --- /dev/null +++ b/scripts/paired.mjs @@ -0,0 +1,343 @@ +// Shared machinery for same-runner paired benchmarks. +// +// Every workflow here measures each arm twice inside a single job, in an order +// that is mirrored about the middle of the job. For two arms that is ABBA; for N +// arms it is 1..N followed by N..1. Two properties follow: +// +// - Differencing within a runner removes between-runner variance, which on +// hosted runners is larger than the effects under test and cannot be +// averaged away by adding independent jobs to each arm. +// - Averaging an arm's two slots removes any drift that is linear across the +// job, because the mirrored order places those slots symmetrically about the +// centre. +// +// Measuring each arm twice also yields a null-effect (A/A) estimate at no extra +// job cost: the spread between an arm's own two slots is what the harness +// reports when nothing has changed. + +import { readdir, readFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { + classify, + hodgesLehmann, + mean, + median, + medianAbsoluteDeviation, + pairedInterval, + pairedPermutationTest, + quantile, + standardDeviation, +} from "./stats.mjs"; + +export const RESULTS_DIR = ".benchmark-results"; + +// Each measurement job uploads its own CSV so that merging the artifacts cannot +// overwrite another runner's samples. +export async function readSampleFiles(prefix, directory = RESULTS_DIR) { + const entries = await readdir(directory); + const files = entries.filter( + (entry) => entry.startsWith(prefix) && entry.endsWith(".csv"), + ); + if (files.length === 0) { + throw new Error(`No ${prefix}*.csv timing files found in ${directory}`); + } + const contents = await Promise.all( + files.sort().map((file) => readFile(join(directory, file), "utf8")), + ); + return contents.join("\n"); +} + +export function parseSamples(csv) { + return csv + .trim() + .split("\n") + .filter(Boolean) + .map((line) => { + const [sample, arm, slot, elapsedMs] = line + .split(",") + .map((value) => value.replace(/^"|"$/g, "")); + return { + sample: Number(sample), + arm, + slot: Number(slot), + seconds: Number(elapsedMs) / 1000, + }; + }); +} + +// Groups rows by runner and arm, keeping only runners that measured every arm +// the expected number of times. A runner that failed part way through would +// otherwise contribute a difference computed from unbalanced slots. +export function groupByRunner(rows, arms, slotsPerArm = 2) { + const bySample = new Map(); + for (const row of rows) { + if (!arms.includes(row.arm)) continue; + const entry = bySample.get(row.sample) ?? new Map(); + entry.set(row.arm, [...(entry.get(row.arm) ?? []), row]); + bySample.set(row.sample, entry); + } + const runners = []; + for (const [sample, entry] of [...bySample.entries()].sort( + (a, b) => a[0] - b[0], + )) { + if (arms.some((arm) => (entry.get(arm) ?? []).length !== slotsPerArm)) { + continue; + } + const slots = new Map(); + for (const arm of arms) { + slots.set( + arm, + entry + .get(arm) + .sort((a, b) => a.slot - b.slot) + .map((row) => row.seconds), + ); + } + runners.push({ sample, slots }); + } + return runners; +} + +export function buildPairs( + rows, + baselineArm = "baseline", + candidateArm = "candidate", +) { + return groupByRunner(rows, [baselineArm, candidateArm]).map((runner) => { + const baselineSlots = runner.slots.get(baselineArm); + const candidateSlots = runner.slots.get(candidateArm); + return { + sample: runner.sample, + baselineSlots, + candidateSlots, + baseline: mean(baselineSlots), + candidate: mean(candidateSlots), + difference: mean(candidateSlots) - mean(baselineSlots), + baselineRepeatDelta: baselineSlots[1] - baselineSlots[0], + candidateRepeatDelta: candidateSlots[1] - candidateSlots[0], + }; + }); +} + +// A slot can stall on the cache service for several seconds. Such a runner is +// not measuring the effect, and which arm the stall lands on is arbitrary, so it +// contributes an arbitrarily large difference. +// +// The filter looks only at how much an arm disagrees with itself, which is a +// pure noise quantity: it has the same distribution whether or not the arms +// differ. Excluding runners on it therefore cannot bias the estimated effect, +// unlike filtering on the difference between arms. The threshold is robust +// (median plus a multiple of the median absolute deviation) so that it adapts to +// the run rather than being a fixed number of seconds. +export function dropStalledRunners(pairs, { tolerance = 6, spreadOf } = {}) { + if (pairs.length < 4) return { kept: pairs, dropped: [] }; + const spreads = pairs.map( + spreadOf ?? + ((pair) => + Math.max( + Math.abs(pair.baselineRepeatDelta), + Math.abs(pair.candidateRepeatDelta), + )), + ); + const limit = + median(spreads) + + tolerance * Math.max(medianAbsoluteDeviation(spreads), 1e-9); + const kept = []; + const dropped = []; + pairs.forEach((pair, index) => { + if (spreads[index] > limit) { + dropped.push({ sample: pair.sample, repeatSpread: spreads[index] }); + } else { + kept.push(pair); + } + }); + // Never discard so much that what remains cannot support a verdict. + if (kept.length < Math.ceil(pairs.length / 2)) { + return { kept: pairs, dropped: [] }; + } + return { kept, dropped, thresholdSeconds: limit }; +} + +// The smallest effect the harness can trust, derived from how much the same +// implementation varies between its two slots on one runner. +export function noiseFloor(pairs) { + const repeats = [ + ...pairs.map((pair) => Math.abs(pair.baselineRepeatDelta)), + ...pairs.map((pair) => Math.abs(pair.candidateRepeatDelta)), + ]; + if (repeats.length === 0) return 0; + return quantile(repeats, 0.5); +} + +export function armSummary(name, values) { + return { + arm: name, + samples: values.length, + meanSeconds: mean(values), + medianSeconds: median(values), + standardDeviationSeconds: standardDeviation(values), + madSeconds: medianAbsoluteDeviation(values), + p95Seconds: quantile(values, 0.95), + }; +} + +export function analyzePairs( + rows, + baselineArm = "baseline", + candidateArm = "candidate", +) { + const allPairs = buildPairs(rows, baselineArm, candidateArm); + const { + kept: pairs, + dropped, + thresholdSeconds, + } = dropStalledRunners(allPairs); + const differences = pairs.map((pair) => pair.difference); + const baselineValues = pairs.map((pair) => pair.baseline); + const candidateValues = pairs.map((pair) => pair.candidate); + const floor = noiseFloor(pairs); + const interval = pairedInterval(differences, { seed: 1 }); + const pValue = pairedPermutationTest(differences, { seed: 3 }); + // The same estimator applied to the within-arm repeats. A trustworthy harness + // must not resolve a difference here, because it compares an arm with itself. + // Judged against the same noise floor as the real effect, so a healthy run + // reports `within-noise` or `inconclusive`. + const controlDifferences = pairs.map((pair) => pair.baselineRepeatDelta); + const controlInterval = pairedInterval(controlDifferences, { seed: 2 }); + const controlPValue = pairedPermutationTest(controlDifferences, { seed: 4 }); + return { + pairs, + noiseFloorSeconds: floor, + baseline: armSummary(baselineArm, baselineValues), + candidate: armSummary(candidateArm, candidateValues), + interval, + pValue, + shiftSeconds: hodgesLehmann(candidateValues, baselineValues), + verdict: classify(interval, { noiseFloor: floor, pValue }), + droppedRunners: dropped, + stallThresholdSeconds: thresholdSeconds, + control: { + interval: controlInterval, + pValue: controlPValue, + verdict: classify(controlInterval, { + noiseFloor: floor, + pValue: controlPValue, + }), + }, + }; +} + +// Compares a reference arm against every other arm, one paired difference per +// runner. Used where the workflow measures more than two implementations in the +// same job, such as the version sweep. +// +// The difference is reference minus arm, so the reference plays the same role +// the candidate plays in the two-arm workflows: it is the code under test and +// each other arm is a baseline it is measured against. In the version sweep the +// reference is `main`, which is the newest code, so this is the direction that +// makes the verdicts mean what they say. Differencing the other way would call +// a released version a `regression` for being slower than the code that +// succeeded it, when what has been measured is `main` being faster. +export function analyzeAgainstReference(rows, arms, reference) { + const allRunners = groupByRunner(rows, arms); + // Same filter as the two-arm workflows, applied across every arm: a runner + // whose slots for one version disagree with each other has stalled, and the + // arm the stall landed on is arbitrary. + const { + kept: runners, + dropped, + thresholdSeconds, + } = dropStalledRunners(allRunners, { + spreadOf: (runner) => + Math.max( + ...arms.map((arm) => { + const slots = runner.slots.get(arm); + return Math.abs(slots[1] - slots[0]); + }), + ), + }); + const perArm = new Map( + arms.map((arm) => [ + arm, + runners.map((runner) => mean(runner.slots.get(arm))), + ]), + ); + const repeats = runners.flatMap((runner) => + arms.map((arm) => { + const slots = runner.slots.get(arm); + return Math.abs(slots[1] - slots[0]); + }), + ); + const floor = repeats.length === 0 ? 0 : quantile(repeats, 0.5); + const referenceValues = perArm.get(reference) ?? []; + const comparisons = arms.map((arm, index) => { + const values = perArm.get(arm); + const differences = values.map( + (value, runner) => referenceValues[runner] - value, + ); + const isReference = arm === reference; + const interval = isReference + ? null + : pairedInterval(differences, { seed: 10 + index }); + return { + arm, + isReference, + summary: armSummary(arm, values), + differenceSeconds: isReference ? 0 : mean(differences), + interval, + pValue: isReference + ? null + : pairedPermutationTest(differences, { seed: 50 + index }), + verdict: isReference + ? "reference" + : classify(interval, { + noiseFloor: floor, + pValue: pairedPermutationTest(differences, { seed: 50 + index }), + }), + }; + }); + // Each arm's own two slots compared with themselves. Nothing changed between + // them, so anything the estimator resolves here is harness bias. + const controlDifferences = runners.map((runner) => { + const slots = runner.slots.get(reference); + return slots[1] - slots[0]; + }); + const controlInterval = + controlDifferences.length === 0 + ? null + : pairedInterval(controlDifferences, { seed: 99 }); + const controlPValue = + controlDifferences.length === 0 + ? null + : pairedPermutationTest(controlDifferences, { seed: 98 }); + return { + runners, + arms, + reference, + noiseFloorSeconds: floor, + droppedRunners: dropped, + stallThresholdSeconds: thresholdSeconds, + comparisons, + control: { + interval: controlInterval, + pValue: controlPValue, + verdict: classify(controlInterval, { + noiseFloor: floor, + pValue: controlPValue, + }), + }, + }; +} + +// Every name passed here ends up in the published report, so a missing value has +// to fail with the name of what is missing. Rendering `undefined` into a results +// table is worse than not rendering one at all. +export function requireEnv(env, names) { + const missing = names.filter((name) => !env[name]); + if (missing.length > 0) { + throw new Error( + `Missing required environment variables: ${missing.join(", ")}`, + ); + } +} diff --git a/scripts/paired.test.mjs b/scripts/paired.test.mjs new file mode 100644 index 0000000..51d9b12 --- /dev/null +++ b/scripts/paired.test.mjs @@ -0,0 +1,13 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { requireEnv } from "./paired.mjs"; + +test("requireEnv names every missing variable", () => { + assert.doesNotThrow(() => requireEnv({ A: "1", B: "2" }, ["A", "B"])); + // An empty string is as unusable in a report as an absent variable. + assert.throws( + () => requireEnv({ A: "1", B: "" }, ["A", "B", "C"]), + /Missing required environment variables: B, C/, + ); +}); diff --git a/scripts/report-action-overhead.mjs b/scripts/report-action-overhead.mjs new file mode 100644 index 0000000..ddaf49c --- /dev/null +++ b/scripts/report-action-overhead.mjs @@ -0,0 +1,433 @@ +// What does setup-java's own code cost? +// +// This report answers two different questions from one matrix, and they need +// different treatment. +// +// The first is the comparison: does the candidate differ from the baseline? +// Each configuration is one runner, so it yields one paired difference and +// cannot support an interval alone. Configurations are treated as blocks — arms +// compared within a configuration, on the same runner, in ABBA order — and the +// differences pooled. +// +// The second is the decomposition, and it is not a comparison at all. The four +// cache profiles are nested levels of work: `none` never touches the cache code, +// `maven-miss` computes a key and is told no, `maven-hit` also unpacks a small +// entry. Differencing the levels says where the time goes, which is the number a +// maintainer deciding whether an optimization is worth it actually needs. Those +// differences are between-configuration, so they are reported as observed +// medians without intervals; they are descriptive, and labelled as such. + +import { + appendFile, + mkdir, + readdir, + readFile, + writeFile, +} from "node:fs/promises"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +import { analyzePairs, requireEnv } from "./paired.mjs"; +import { describeVerdict, formatInterval, holmAdjust } from "./stats.mjs"; + +const RESULTS_DIR = ".benchmark-results"; +const OUTPUT_DIR = "action-overhead-results"; + +const CACHE_PROFILES = [ + { + id: "none", + label: "No cache", + describes: "the action never touches the cache code", + }, + { + id: "maven-miss", + label: "Maven, miss", + describes: "a key is computed and the service says no", + }, + { + id: "maven-hit", + label: "Maven, hit", + describes: "a key is computed and a ~1 MiB entry is unpacked", + }, + { + id: "gradle-miss", + label: "Gradle, miss", + describes: "the same as a Maven miss through the other package manager", + }, +]; + +const STEPS = [ + { + from: "none", + to: "maven-miss", + label: "Computing a key and asking the cache service", + }, + { + from: "maven-miss", + to: "maven-hit", + label: "Restoring and unpacking a ~1 MiB entry", + }, +]; + +function unquote(value) { + const trimmed = (value ?? "").trim(); + return trimmed.startsWith('"') && trimmed.endsWith('"') + ? trimmed.slice(1, -1).replaceAll('""', '"') + : trimmed; +} + +// os,cache,layout,arm,slot,cacheHit,elapsedMs +export function parseSamples(csv) { + return csv + .trim() + .split("\n") + .filter((line) => line.trim() !== "") + .map((line) => { + const [os, cache, layout, arm, slot, cacheHit, elapsedMs] = line + .split(",") + .map(unquote); + return { + configuration: `${os}/${cache}/${layout}`, + os, + cache, + layout, + arm, + slot: Number(slot), + cacheHit, + seconds: Number(elapsedMs) / 1000, + }; + }) + .filter((row) => Number.isFinite(row.seconds)); +} + +// paired.mjs groups by a numeric `sample`, so each configuration is assigned a +// stable index and becomes one block. +export function toPairedRows(rows) { + const configurations = [ + ...new Set(rows.map((row) => row.configuration)), + ].sort(); + const index = new Map(configurations.map((name, at) => [name, at + 1])); + return { + configurations, + rows: rows.map((row) => ({ + sample: index.get(row.configuration), + arm: row.arm, + slot: row.slot, + seconds: row.seconds, + })), + }; +} + +export async function readResults(directory = RESULTS_DIR) { + const entries = await readdir(directory, { recursive: true }); + const files = entries.filter( + (entry) => entry.endsWith(".csv") && entry.includes("action-overhead"), + ); + if (files.length === 0) { + throw new Error(`No action-overhead CSV files found in ${directory}`); + } + const contents = await Promise.all( + files.sort().map((file) => readFile(join(directory, file), "utf8")), + ); + return contents.join("\n"); +} + +function median(values) { + if (values.length === 0) return null; + const sorted = [...values].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 + ? (sorted[middle - 1] + sorted[middle]) / 2 + : sorted[middle]; +} + +export function subsetAnalysis(rows, predicate) { + const subset = rows.filter(predicate); + if (subset.length === 0) return null; + const { rows: paired } = toPairedRows(subset); + const analysis = analyzePairs(paired, "baseline", "candidate"); + return analysis.pairs.length === 0 ? null : analysis; +} + +// A `maven-hit` slot that reported a miss did not measure a restore, and a +// `maven-miss` slot that reported a hit found an entry that should not exist. +// Either turns a level of the decomposition into a different level, so the +// numbers below it stop meaning what they are labelled, and that has to be said +// loudly rather than buried. +export function checkCacheIntegrity(rows) { + const expectations = { "maven-hit": "true", "maven-miss": "false" }; + const problems = []; + for (const row of rows) { + const expected = expectations[row.cache]; + if (!expected) continue; + if (row.cacheHit === "unset" || row.cacheHit === "") { + problems.push({ + configuration: row.configuration, + slot: row.slot, + reported: "nothing", + expected, + reason: + "the ref does not publish `cache-hit`, so this slot cannot be verified", + }); + } else if (row.cacheHit !== expected) { + problems.push({ + configuration: row.configuration, + slot: row.slot, + reported: row.cacheHit, + expected, + reason: "the slot did not do the work its profile is named for", + }); + } + } + return problems; +} + +export function decompose(rows) { + const byOs = [...new Set(rows.map((row) => row.os))].sort(); + const levels = byOs.map((os) => { + const seconds = {}; + for (const profile of CACHE_PROFILES) { + seconds[profile.id] = median( + rows + .filter((row) => row.os === os && row.cache === profile.id) + .map((row) => row.seconds), + ); + } + return { os, seconds }; + }); + const steps = byOs.map((os) => { + const level = levels.find((entry) => entry.os === os); + return { + os, + steps: STEPS.map(({ from, to, label }) => ({ + label, + seconds: + level.seconds[from] === null || level.seconds[to] === null + ? null + : level.seconds[to] - level.seconds[from], + })), + }; + }); + return { levels, steps }; +} + +function seconds(value, digits = 3) { + return value === null || value === undefined + ? "n/a" + : `${value >= 0 ? "" : "−"}${Math.abs(value).toFixed(digits)}s`; +} + +function analysisRow(label, analysis, adjustedP) { + if (!analysis) return `| ${label} | no usable pairs | | | |`; + return [ + `| ${label}`, + seconds(analysis.interval.estimate), + formatInterval(analysis.interval), + adjustedP === null || adjustedP === undefined + ? analysis.pValue.toFixed(3) + : adjustedP.toFixed(3), + analysis.verdict, + ] + .join(" | ") + .concat(" |"); +} + +export function markdown( + metadata, + overall, + byProfile, + byOs, + decomposition, + problems, + sizes, +) { + const lines = [ + "# Action overhead", + "", + `Baseline \`${metadata.baselineRef}\` vs candidate \`${metadata.candidateRef}\` from \`${metadata.setupJavaRepository}\`, run ${metadata.runId}.`, + "", + "The cache entry here is about a megabyte, deliberately. setup-java hands the", + "actual transfer to `@actions/cache`, so a large entry would time the network", + "and hide the action's own work. A small one leaves resolving a distribution,", + "computing a cache key, writing settings and toolchains, and the bookkeeping", + "around a restore as what is measured.", + "", + ]; + + if (problems.length > 0) { + lines.push( + "## The cache did not behave as the profiles assume", + "", + "Every number below this point is suspect, because at least one slot did not", + "do the work its profile is named for.", + "", + "| Configuration | Slot | Expected `cache-hit` | Reported | Why it matters |", + "| --- | ---: | --- | --- | --- |", + ...problems + .slice(0, 20) + .map( + (problem) => + `| ${problem.configuration} | ${problem.slot} | ${problem.expected} | ${problem.reported} | ${problem.reason} |`, + ), + "", + ); + } + + lines.push( + "## Does the candidate differ from the baseline?", + "", + `**${overall ? describeVerdict(overall.verdict) : "No usable pairs."}**`, + "", + ); + + if (overall) { + lines.push( + `Pooled across ${overall.pairs.length} configurations: ${seconds(overall.interval.estimate)} ${formatInterval(overall.interval)}, permutation p = ${overall.pValue.toFixed(3)}, noise floor ${seconds(overall.noiseFloorSeconds)}.`, + "", + `The A/A control — the two baseline slots against each other, which differ by nothing — reads ${overall.control.verdict}. Anything other than \`within-noise\` or \`inconclusive\` there means the harness is measuring something it should not, and the comparison above cannot be trusted.`, + "", + "| Split | Difference | Interval | p (Holm) | Verdict |", + "| --- | ---: | --- | ---: | --- |", + ...byProfile.map(({ label, analysis, adjustedP }) => + analysisRow(label, analysis, adjustedP), + ), + ...byOs.map(({ label, analysis }) => analysisRow(label, analysis, null)), + "", + ); + } + + lines.push( + "## Where the time goes", + "", + "Observed medians, not a comparison. These are levels measured on different", + "runners, so they carry no interval and no verdict; they are here to say what", + "a setup actually costs and which part of it setup-java could change.", + "", + `| OS | ${CACHE_PROFILES.map((profile) => profile.label).join(" | ")} |`, + `| --- | ${CACHE_PROFILES.map(() => "---:").join(" | ")} |`, + ...decomposition.levels.map( + (level) => + `| ${level.os} | ${CACHE_PROFILES.map((profile) => seconds(level.seconds[profile.id], 2)).join(" | ")} |`, + ), + "", + `| OS | ${STEPS.map((step) => step.label).join(" | ")} |`, + `| --- | ${STEPS.map(() => "---:").join(" | ")} |`, + ...decomposition.steps.map( + (entry) => + `| ${entry.os} | ${entry.steps.map((step) => seconds(step.seconds, 2)).join(" | ")} |`, + ), + "", + ); + + if (sizes.length > 0) { + lines.push( + "## Bundle size", + "", + "How much JavaScript each arm has to parse before it does anything.", + "", + "| Arm | `dist/setup/index.js` bytes | `dist/setup` JS bytes | JS files |", + "| --- | ---: | ---: | ---: |", + ...sizes.map( + (size) => + `| ${size.arm} | ${size.indexBytes} | ${size.jsBytes} | ${size.files} |`, + ), + "", + ); + } + + return `${lines.join("\n")}\n`; +} + +async function readSizes(directory = RESULTS_DIR) { + let entries; + try { + entries = await readdir(directory, { recursive: true }); + } catch { + return []; + } + const files = entries.filter((entry) => entry.endsWith("sizes.csv")); + const contents = await Promise.all( + files.sort().map((file) => readFile(join(directory, file), "utf8")), + ); + const seen = new Map(); + for (const line of contents.join("\n").trim().split("\n")) { + if (!line.trim()) continue; + const [arm, indexBytes, jsBytes, files_] = line.split(","); + if (!seen.has(arm)) { + seen.set(arm, { arm, indexBytes, jsBytes, files: files_ }); + } + } + return [...seen.values()]; +} + +export async function main(env = process.env) { + requireEnv(env, [ + "SETUP_JAVA_REPOSITORY", + "BASELINE_REF", + "CANDIDATE_REF", + "RUN_ID", + ]); + + const rows = parseSamples(await readResults()); + const { rows: paired } = toPairedRows(rows); + const overall = + paired.length > 0 ? analyzePairs(paired, "baseline", "candidate") : null; + + const profileAnalyses = CACHE_PROFILES.map((profile) => ({ + label: profile.label, + analysis: subsetAnalysis(rows, (row) => row.cache === profile.id), + })); + // Four cache profiles are four chances to find a difference that is not there, + // so the family is corrected together. + const adjusted = holmAdjust( + profileAnalyses.map(({ analysis }) => (analysis ? analysis.pValue : null)), + ); + const byProfile = profileAnalyses.map((entry, index) => ({ + ...entry, + adjustedP: adjusted[index], + })); + + const byOs = [...new Set(rows.map((row) => row.os))].sort().map((os) => ({ + label: os, + analysis: subsetAnalysis(rows, (row) => row.os === os), + })); + + const problems = checkCacheIntegrity(rows); + const decomposition = decompose(rows); + const sizes = await readSizes(); + + const metadata = { + setupJavaRepository: env.SETUP_JAVA_REPOSITORY, + baselineRef: env.BASELINE_REF, + candidateRef: env.CANDIDATE_REF, + runId: env.RUN_ID, + }; + + const rendered = markdown( + metadata, + overall, + byProfile, + byOs, + decomposition, + problems, + sizes, + ); + + await mkdir(OUTPUT_DIR, { recursive: true }); + await writeFile( + join(OUTPUT_DIR, "results.json"), + `${JSON.stringify({ metadata, overall, byProfile, byOs, decomposition, problems, sizes }, null, 2)}\n`, + ); + await writeFile(join(OUTPUT_DIR, "summary.md"), rendered); + if (env.GITHUB_STEP_SUMMARY) { + await appendFile(env.GITHUB_STEP_SUMMARY, rendered); + } + return rendered; +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + await main(); +} diff --git a/scripts/report-action-overhead.test.mjs b/scripts/report-action-overhead.test.mjs new file mode 100644 index 0000000..884d20f --- /dev/null +++ b/scripts/report-action-overhead.test.mjs @@ -0,0 +1,197 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + checkCacheIntegrity, + decompose, + markdown, + parseSamples, + subsetAnalysis, + toPairedRows, +} from "./report-action-overhead.mjs"; + +function csv(rows) { + return rows + .map( + ({ os, cache, layout, arm, slot, cacheHit, ms }) => + `"${os}","${cache}","${layout}","${arm}","${slot}","${cacheHit}","${ms}"`, + ) + .join("\n"); +} + +function matrix({ shift = 0 } = {}) { + const rows = []; + const base = { + none: 2000, + "maven-miss": 2400, + "maven-hit": 3100, + "gradle-miss": 2380, + }; + for (const os of ["ubuntu-latest", "macos-15-intel"]) { + for (const cache of Object.keys(base)) { + for (const layout of ["simple", "complex"]) { + const hit = + cache === "maven-hit" + ? "true" + : cache === "maven-miss" + ? "false" + : "unset"; + const offset = layout === "complex" ? 300 : 0; + const drift = os === "macos-15-intel" ? 900 : 0; + const at = base[cache] + offset + drift; + rows.push( + { + os, + cache, + layout, + arm: "baseline", + slot: 1, + cacheHit: hit, + ms: at, + }, + { + os, + cache, + layout, + arm: "candidate", + slot: 2, + cacheHit: hit, + ms: at + shift, + }, + { + os, + cache, + layout, + arm: "candidate", + slot: 3, + cacheHit: hit, + ms: at + shift, + }, + { + os, + cache, + layout, + arm: "baseline", + slot: 4, + cacheHit: hit, + ms: at, + }, + ); + } + } + } + return parseSamples(csv(rows)); +} + +test("parses quoted rows and converts milliseconds to seconds", () => { + const rows = parseSamples( + '"ubuntu-latest","maven-hit","simple","baseline","1","true","3100"', + ); + assert.equal(rows.length, 1); + assert.equal(rows[0].seconds, 3.1); + assert.equal(rows[0].configuration, "ubuntu-latest/maven-hit/simple"); +}); + +test("treats each configuration as one block", () => { + const { configurations, rows } = toPairedRows(matrix()); + assert.equal(configurations.length, 16); + assert.equal(new Set(rows.map((row) => row.sample)).size, 16); +}); + +// The decomposition is the reason this scenario uses a tiny entry, so the +// arithmetic that produces it has to be pinned. +test("differences the cache profiles into named steps", () => { + const { levels, steps } = decompose(matrix()); + const ubuntu = levels.find((level) => level.os === "ubuntu-latest"); + assert.equal(ubuntu.seconds.none, 2.15); + assert.equal(ubuntu.seconds["maven-miss"], 2.55); + + const ubuntuSteps = steps.find((entry) => entry.os === "ubuntu-latest").steps; + assert.match(ubuntuSteps[0].label, /asking the cache service/); + assert.ok(Math.abs(ubuntuSteps[0].seconds - 0.4) < 1e-9); + assert.ok(Math.abs(ubuntuSteps[1].seconds - 0.7) < 1e-9); +}); + +test("reports no difference when the arms are identical", () => { + const analysis = subsetAnalysis(matrix(), () => true); + assert.ok(["within-noise", "inconclusive"].includes(analysis.verdict)); +}); + +test("differences candidate minus baseline, so a slower candidate is positive", () => { + const analysis = subsetAnalysis(matrix({ shift: 500 }), () => true); + assert.ok(analysis.interval.estimate > 0); +}); + +test("flags a hit profile that actually missed", () => { + const rows = matrix().map((row) => + row.cache === "maven-hit" && row.os === "ubuntu-latest" + ? { ...row, cacheHit: "false" } + : row, + ); + const problems = checkCacheIntegrity(rows); + assert.ok(problems.length > 0); + assert.ok(problems.every((problem) => problem.expected === "true")); + assert.match(problems[0].reason, /did not do the work/); +}); + +// A ref too old to publish `cache-hit` cannot be verified, and silently passing +// it would let the decomposition be built out of slots that never restored. +test("flags an unverifiable slot separately from a wrong one", () => { + const rows = matrix().map((row) => + row.cache === "maven-hit" ? { ...row, cacheHit: "unset" } : row, + ); + const problems = checkCacheIntegrity(rows); + assert.ok(problems.length > 0); + assert.match(problems[0].reason, /does not publish/); +}); + +test("accepts a matrix where every profile behaved", () => { + assert.deepEqual(checkCacheIntegrity(matrix()), []); +}); + +test("leads with the integrity failure when there is one", () => { + const rows = matrix().map((row) => + row.cache === "maven-hit" ? { ...row, cacheHit: "false" } : row, + ); + const rendered = markdown( + { + setupJavaRepository: "actions/setup-java", + baselineRef: "main", + candidateRef: "main", + runId: "1", + }, + subsetAnalysis(rows, () => true), + [], + [], + decompose(rows), + checkCacheIntegrity(rows), + [], + ); + assert.match(rendered, /## The cache did not behave as the profiles assume/); + assert.ok( + rendered.indexOf("did not behave") < + rendered.indexOf("Does the candidate differ"), + ); +}); + +// Blank lines are what make the output render as markdown rather than as one +// run-on paragraph, so filtering them out has to stay impossible. +test("keeps the blank lines that separate sections", () => { + const rows = matrix(); + const rendered = markdown( + { + setupJavaRepository: "actions/setup-java", + baselineRef: "v4.8.0", + candidateRef: "main", + runId: "1", + }, + subsetAnalysis(rows, () => true), + [], + [], + decompose(rows), + [], + [{ arm: "baseline", indexBytes: "1", jsBytes: "2", files: "3" }], + ); + assert.match(rendered, /\n\n## Where the time goes\n\n/); + assert.match(rendered, /## Bundle size/); +}); diff --git a/scripts/report-cache-save.mjs b/scripts/report-cache-save.mjs new file mode 100644 index 0000000..77f0ee0 --- /dev/null +++ b/scripts/report-cache-save.mjs @@ -0,0 +1,215 @@ +import { appendFile, mkdir, writeFile } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; + +import { + analyzePairs, + parseSamples, + readSampleFiles as readPairedSampleFiles, + requireEnv, +} from "./paired.mjs"; +import { describeVerdict, formatInterval } from "./stats.mjs"; + +const API_VERSION = "2022-11-28"; + +export function readSampleFiles(directory) { + return readPairedSampleFiles("cache-save-timings", directory); +} + +export function analyze(rows) { + return analyzePairs(rows, "baseline", "candidate"); +} + +function csvValue(value) { + return `"${String(value ?? "").replaceAll('"', '""')}"`; +} + +function throughput(fixtureMiB, seconds) { + return fixtureMiB / seconds; +} + +async function api(path, token, options = {}) { + const response = await fetch(`https://api.github.com${path}`, { + ...options, + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "X-GitHub-Api-Version": API_VERSION, + ...options.headers, + }, + }); + if (!response.ok) { + throw new Error(`${options.method ?? "GET"} ${path}: ${response.status}`); + } + if (response.status === 204) return null; + return response.json(); +} + +async function allPages(path, field, token) { + const values = []; + for (let page = 1; ; page += 1) { + const separator = path.includes("?") ? "&" : "?"; + const response = await api( + `${path}${separator}per_page=100&page=${page}`, + token, + ); + values.push(...response[field]); + if (response[field].length < 100) return values; + } +} + +export function markdown(metadata, analysis, caches = []) { + const { baseline, candidate, interval, control } = analysis; + const fixtureMiB = Number(metadata.fixtureMiB); + const lines = [ + "# Cache save benchmark", + "", + `${fixtureMiB} MiB Maven fixture, ${analysis.pairs.length} runners x 2 paired observations, run ${metadata.runId}.`, + `Baseline \`${metadata.baselineRef}\` vs candidate \`${metadata.candidateRef}\` from \`${metadata.setupJavaRepository}\`.`, + "", + "## Verdict", + "", + `**${describeVerdict(analysis.verdict)}**`, + "", + `Paired difference (candidate - baseline): **${formatInterval(interval, { digits: 3 })}**.`, + `Permutation p-value: ${analysis.pValue.toFixed(3)}. Hodges-Lehmann shift: ${analysis.shiftSeconds.toFixed(3)}s.`, + `Harness noise floor: ${analysis.noiseFloorSeconds.toFixed(3)}s (median within-runner repeat spread).`, + "", + `A/A control (baseline against itself) reports **${control.verdict}** at ${formatInterval(control.interval, { digits: 3 })}. A healthy harness reports \`within-noise\` or \`inconclusive\` here; an \`improvement\` or \`regression\` means slot ordering is biasing results and the verdict above cannot be trusted.`, + "", + analysis.droppedRunners.length === 0 + ? "No runner was discarded for a stalled slot." + : `Discarded ${analysis.droppedRunners.length} runner(s) whose own arm disagreed with itself by more than ${analysis.stallThresholdSeconds.toFixed(3)}s, a threshold derived from the spread of the other runners: ${analysis.droppedRunners.map((entry) => `#${entry.sample} (${entry.repeatSpread.toFixed(3)}s)`).join(", ")}. A stalled slot lands on an arbitrary arm and would otherwise dominate the mean; the decision uses only within-arm spread, which carries no information about the effect.`, + "", + "## Arms", + "", + "| Arm | Runners | Mean (s) | Median (s) | SD (s) | MAD (s) | p95 (s) | Mean throughput (MiB/s) | Median throughput (MiB/s) |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", + ]; + for (const summary of [baseline, candidate]) { + lines.push( + `| ${summary.arm} | ${summary.samples} | ${summary.meanSeconds.toFixed(3)} | ${summary.medianSeconds.toFixed(3)} | ${summary.standardDeviationSeconds?.toFixed(3) ?? "n/a"} | ${summary.madSeconds.toFixed(3)} | ${summary.p95Seconds.toFixed(3)} | ${throughput(fixtureMiB, summary.meanSeconds).toFixed(1)} | ${throughput(fixtureMiB, summary.medianSeconds).toFixed(1)} |`, + ); + } + lines.push( + "", + "The fixture is byte-identical for every slot. Distinct keys are still required because a cache save for an existing key is a no-op; that creates fresh blobs whose placement can affect upload throughput, which is the dominant noise source for this scenario.", + "", + "## Paired samples", + "", + "| Runner | baseline slot 1 (s) | candidate slot 2 (s) | candidate slot 3 (s) | baseline slot 4 (s) | Paired delta (s) |", + "| ---: | ---: | ---: | ---: | ---: | ---: |", + ); + for (const pair of analysis.pairs) { + lines.push( + `| ${pair.sample} | ${pair.baselineSlots[0].toFixed(3)} | ${pair.candidateSlots[0].toFixed(3)} | ${pair.candidateSlots[1].toFixed(3)} | ${pair.baselineSlots[1].toFixed(3)} | ${pair.difference.toFixed(3)} |`, + ); + } + lines.push( + "", + "## Cache entries", + "", + caches.length === 0 + ? "No matching cache entries were found when the report was generated." + : `Found ${caches.length} cache entries with the run-scoped prefix \`${metadata.cacheKeyPrefix}\`.`, + "", + "All durations are measured inside the job with millisecond resolution by timing only the \`@actions/cache.saveCache\` call after Node has started and the module has loaded. setup-java delegates Maven cache transfers to that toolkit function, so this reports transfer-and-archive time without relying on post-job step timestamps from the Actions API or including process startup.", + ); + return `${lines.filter((line) => line !== null).join("\n")}\n`; +} + +export async function main(env = process.env) { + requireEnv(env, [ + "GITHUB_REPOSITORY", + "GH_TOKEN", + "GITHUB_RUN_ID", + "BASELINE_REF", + "CANDIDATE_REF", + "SETUP_JAVA_REPOSITORY", + "FIXTURE_MIB", + "CACHE_KEY_PREFIX", + "GITHUB_STEP_SUMMARY", + ]); + + const [owner, repo] = env.GITHUB_REPOSITORY.split("/"); + if (!owner || !repo) { + throw new Error( + `GITHUB_REPOSITORY must be owner/repo, got "${env.GITHUB_REPOSITORY}"`, + ); + } + + const token = env.GH_TOKEN; + const cacheEntries = await allPages( + `/repos/${owner}/${repo}/actions/caches`, + "actions_caches", + token, + ); + const caches = cacheEntries + .filter((cache) => cache.key.startsWith(env.CACHE_KEY_PREFIX)) + .map((cache) => ({ + id: cache.id, + key: cache.key, + sizeBytes: cache.size_in_bytes, + })); + + const metadata = { + repository: env.GITHUB_REPOSITORY, + runId: env.GITHUB_RUN_ID, + setupJavaRepository: env.SETUP_JAVA_REPOSITORY, + baselineRef: env.BASELINE_REF, + candidateRef: env.CANDIDATE_REF, + fixtureMiB: Number(env.FIXTURE_MIB), + cacheKeyPrefix: env.CACHE_KEY_PREFIX, + generatedAt: new Date().toISOString(), + }; + + try { + const rows = parseSamples(await readSampleFiles()); + const analysis = analyze(rows); + if (analysis.pairs.length === 0) { + throw new Error("No complete ABBA samples were collected"); + } + const report = markdown(metadata, analysis, caches); + + await mkdir("cache-save-results", { recursive: true }); + await writeFile( + "cache-save-results/results.json", + `${JSON.stringify({ metadata, analysis, caches }, null, 2)}\n`, + ); + await writeFile( + "cache-save-results/results.csv", + `sample,arm,slot,seconds\n${rows + .map((row) => + [row.sample, row.arm, row.slot, row.seconds].map(csvValue).join(","), + ) + .join("\n")}\n`, + ); + await writeFile("cache-save-results/summary.md", report); + await appendFile(env.GITHUB_STEP_SUMMARY, report); + } finally { + const deletionFailures = []; + let deleted = 0; + for (const cache of caches) { + try { + await api(`/repos/${owner}/${repo}/actions/caches/${cache.id}`, token, { + method: "DELETE", + }); + deleted += 1; + } catch (error) { + deletionFailures.push(`${cache.key}: ${error.message}`); + } + } + console.log(`Deleted ${deleted} cache-save benchmark caches`); + if (deletionFailures.length > 0) { + throw new Error( + `Failed to delete ${deletionFailures.length} cache-save benchmark cache(s): ${deletionFailures.join("; ")}`, + ); + } + } +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + await main(); +} diff --git a/scripts/report-cache-save.test.mjs b/scripts/report-cache-save.test.mjs new file mode 100644 index 0000000..c19ceea --- /dev/null +++ b/scripts/report-cache-save.test.mjs @@ -0,0 +1,172 @@ +import assert from "node:assert/strict"; +import { rm } from "node:fs/promises"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; +import test from "node:test"; + +import { analyze, markdown } from "./report-cache-save.mjs"; +import { parseSamples } from "./paired.mjs"; + +function rowsFromSlots(slots) { + return parseSamples( + slots + .map(([sample, b1, c2, c3, b4]) => + [ + `"${sample}","baseline","1","${b1}"`, + `"${sample}","candidate","2","${c2}"`, + `"${sample}","candidate","3","${c3}"`, + `"${sample}","baseline","4","${b4}"`, + ].join("\n"), + ) + .join("\n"), + ); +} + +test("analyzes cache-save samples with candidate minus baseline direction", () => { + const slots = []; + for (let sample = 1; sample <= 10; sample += 1) { + const offset = sample * 250; + slots.push([ + sample, + 2000 + offset, + 2600 + offset, + 2620 + offset, + 2020 + offset, + ]); + } + const analysis = analyze(rowsFromSlots(slots)); + assert.equal(analysis.verdict, "regression"); + assert.ok(analysis.interval.low > 0); + assert.ok(analysis.shiftSeconds > 0); +}); + +test("reports a large consistent saving as an improvement", () => { + const slots = []; + for (let sample = 1; sample <= 10; sample += 1) { + const offset = sample * 300; + slots.push([ + sample, + 3200 + offset, + 2100 + offset, + 2120 + offset, + 3220 + offset, + ]); + } + const analysis = analyze(rowsFromSlots(slots)); + assert.equal(analysis.verdict, "improvement"); + assert.ok(analysis.interval.high < 0); +}); + +test("identical inputs do not produce a headline verdict", () => { + const slots = []; + for (let sample = 1; sample <= 10; sample += 1) { + const offset = sample * 175; + slots.push([ + sample, + 2400 + offset, + 2400 + offset, + 2400 + offset, + 2400 + offset, + ]); + } + const analysis = analyze(rowsFromSlots(slots)); + assert.ok(["within-noise", "inconclusive"].includes(analysis.verdict)); + assert.ok( + ["within-noise", "inconclusive"].includes(analysis.control.verdict), + ); +}); + +test("drops incomplete ABBA runners", () => { + const rows = parseSamples( + [ + '"1","baseline","1","2000"', + '"1","candidate","2","1900"', + '"1","candidate","3","1910"', + '"1","baseline","4","2010"', + '"2","baseline","1","2000"', + '"2","candidate","2","1900"', + ].join("\n"), + ); + const analysis = analyze(rows); + assert.equal(analysis.pairs.length, 1); + assert.equal(analysis.pairs[0].sample, 1); +}); + +test("renders throughput, guard rails, and paired samples", () => { + const analysis = analyze( + rowsFromSlots([ + [1, 2400, 1900, 1950, 2500], + [2, 3100, 2600, 2500, 3000], + ]), + ); + const report = markdown( + { + runId: "123", + setupJavaRepository: "actions/setup-java", + baselineRef: "v4.8.0", + candidateRef: "main", + fixtureMiB: 64, + cacheKeyPrefix: "cache-save-123-", + }, + analysis, + [{ id: 1, key: "cache-save-123-1", sizeBytes: 64 * 1024 * 1024 }], + ); + assert.match(report, /# Cache save benchmark/); + assert.match(report, /Mean throughput \(MiB\/s\)/); + assert.match(report, /A\/A control/); + assert.match(report, /Paired samples/); + assert.match(report, /@actions\/cache\.saveCache/); + assert.match(report, /without .*including process startup/); +}); + +test("fixture verification asserts exact file count", async () => { + const home = join(process.cwd(), ".cache-save-test-home"); + const env = { ...process.env, HOME: home }; + try { + const prepare = spawnSync( + "bash", + ["scripts/cache-save.sh", "prepare-fixture", "1"], + { cwd: process.cwd(), env, encoding: "utf8" }, + ); + assert.equal(prepare.status, 0, prepare.stderr); + + const expected = spawnSync( + "bash", + ["scripts/cache-save.sh", "expected-file-count", "1"], + { cwd: process.cwd(), env, encoding: "utf8" }, + ); + assert.equal(expected.status, 0, expected.stderr); + + const actual = spawnSync("bash", ["scripts/cache-save.sh", "file-count"], { + cwd: process.cwd(), + env, + encoding: "utf8", + }); + assert.equal(actual.status, 0, actual.stderr); + assert.equal(actual.stdout.trim(), expected.stdout.trim()); + + const verify = spawnSync( + "bash", + ["scripts/cache-save.sh", "verify-fixture", "1"], + { cwd: process.cwd(), env, encoding: "utf8" }, + ); + assert.equal(verify.status, 0, verify.stderr); + + const removeOne = spawnSync( + "bash", + ["scripts/cache-save.sh", "remove-one-file"], + { cwd: process.cwd(), env, encoding: "utf8" }, + ); + assert.equal(removeOne.status, 0, removeOne.stderr); + + const failedVerify = spawnSync( + "bash", + ["scripts/cache-save.sh", "verify-fixture", "1"], + { cwd: process.cwd(), env, encoding: "utf8" }, + ); + assert.notEqual(failedVerify.status, 0); + assert.match(failedVerify.stderr, /expected/); + } finally { + await rm(home, { recursive: true, force: true }); + } +}); diff --git a/scripts/report-cache-value.mjs b/scripts/report-cache-value.mjs new file mode 100644 index 0000000..11af807 --- /dev/null +++ b/scripts/report-cache-value.mjs @@ -0,0 +1,213 @@ +import { appendFile, mkdir, writeFile } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; + +import { hashFilesSingle } from "./report.mjs"; +import { + analyzePairs, + parseSamples, + readSampleFiles as readPairedSampleFiles, + requireEnv, +} from "./paired.mjs"; +import { describeVerdict, formatInterval } from "./stats.mjs"; + +export function readSampleFiles(directory) { + return readPairedSampleFiles("cache-value-timings", directory); +} + +// The difference is cached minus uncached, so the cache saving time reads as a +// negative number and an `improvement`, the same direction every other workflow +// in this repository uses for the arm under test. +export function analyze(rows) { + return analyzePairs(rows, "uncached", "cached"); +} + +const API_VERSION = "2022-11-28"; + +function csvValue(value) { + return `"${String(value ?? "").replaceAll('"', '""')}"`; +} + +async function api(path, token, options = {}) { + const response = await fetch(`https://api.github.com${path}`, { + ...options, + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "X-GitHub-Api-Version": API_VERSION, + ...options.headers, + }, + }); + if (!response.ok) { + throw new Error(`${options.method ?? "GET"} ${path}: ${response.status}`); + } + if (response.status === 204) return null; + return response.json(); +} + +async function allPages(path, field, token) { + const values = []; + for (let page = 1; ; page += 1) { + const separator = path.includes("?") ? "&" : "?"; + const response = await api( + `${path}${separator}per_page=100&page=${page}`, + token, + ); + values.push(...response[field]); + if (response[field].length < 100) return values; + } +} + +// The saving is quoted as a ratio as well as a difference because the difference +// alone is not portable: it is a property of this dependency tree on this +// runner, and a reader comparing it with their own project needs to know +// whether the cache removed most of the work or a little of it. +function speedup(analysis) { + const uncached = analysis.baseline.meanSeconds; + const cached = analysis.candidate.meanSeconds; + if (cached <= 0) return null; + return uncached / cached; +} + +export function markdown(metadata, analysis, caches) { + const { baseline, candidate, interval, control } = analysis; + const ratio = speedup(analysis); + const lines = [ + "# Maven cache value", + "", + `Temurin ${metadata.javaVersion}, ${analysis.pairs.length} runners x 2 paired observations, run ${metadata.runId}.`, + `\`${metadata.setupJavaRef}\` from \`${metadata.setupJavaRepository}\`, resolving Spring PetClinic.`, + "", + "Both arms run the same `dependency:go-offline` against the same dependency", + "tree. They differ only in whether setup-java restored the local repository", + "first, so the difference between them is what the cache is worth on this", + "project rather than a detail of how the cache is implemented.", + "", + "## Verdict", + "", + `**${describeVerdict(analysis.verdict)}**`, + "", + `Time to a resolvable project, cached minus uncached: **${formatInterval(interval, { digits: 2 })}**.`, + ratio === null + ? null + : `That is **${ratio.toFixed(1)}x faster** with the cache: ${baseline.meanSeconds.toFixed(1)}s without it, ${candidate.meanSeconds.toFixed(1)}s with it.`, + `Permutation p-value: ${analysis.pValue.toFixed(3)}. Hodges-Lehmann shift: ${analysis.shiftSeconds.toFixed(2)}s.`, + `Harness noise floor: ${analysis.noiseFloorSeconds.toFixed(2)}s (median within-runner repeat spread).`, + "", + `A/A control (the uncached arm against itself) reports **${control.verdict}** at ${formatInterval(control.interval, { digits: 2 })}. A healthy harness reports \`within-noise\` or \`inconclusive\` here; anything else means slot ordering is biasing results and the verdict above cannot be trusted.`, + "", + analysis.droppedRunners.length === 0 + ? "No runner was discarded for a stalled slot." + : `Discarded ${analysis.droppedRunners.length} runner(s) whose own arm disagreed with itself by more than ${analysis.stallThresholdSeconds.toFixed(2)}s: ${analysis.droppedRunners.map((entry) => `#${entry.sample} (${entry.repeatSpread.toFixed(2)}s)`).join(", ")}. A stalled slot lands on an arbitrary arm and would otherwise dominate the mean.`, + "", + "## Arms", + "", + "| Arm | Runners | Mean (s) | Median (s) | SD (s) | MAD (s) | p95 (s) |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: |", + `| uncached | ${baseline.samples} | ${baseline.meanSeconds.toFixed(2)} | ${baseline.medianSeconds.toFixed(2)} | ${baseline.standardDeviationSeconds?.toFixed(2) ?? "n/a"} | ${baseline.madSeconds.toFixed(2)} | ${baseline.p95Seconds.toFixed(2)} |`, + `| cached | ${candidate.samples} | ${candidate.meanSeconds.toFixed(2)} | ${candidate.medianSeconds.toFixed(2)} | ${candidate.standardDeviationSeconds?.toFixed(2) ?? "n/a"} | ${candidate.madSeconds.toFixed(2)} | ${candidate.p95Seconds.toFixed(2)} |`, + "", + "The uncached arm is the more variable of the two by a wide margin, because it", + "depends on Maven Central rather than on the Actions cache service. That", + "variance is real and belongs in the number: it is what a user without a cache", + "actually experiences.", + "", + "## Paired samples", + "", + "| Runner | cached slot 1 (s) | uncached slot 2 (s) | uncached slot 3 (s) | cached slot 4 (s) | Paired delta (s) |", + "| ---: | ---: | ---: | ---: | ---: | ---: |", + ]; + for (const pair of analysis.pairs) { + lines.push( + `| ${pair.sample} | ${pair.candidateSlots[0].toFixed(2)} | ${pair.baselineSlots[0].toFixed(2)} | ${pair.baselineSlots[1].toFixed(2)} | ${pair.candidateSlots[1].toFixed(2)} | ${pair.difference.toFixed(2)} |`, + ); + } + lines.push("", "## Caches", "", "| Cache | Size (MiB) |", "| --- | ---: |"); + for (const cache of caches) { + lines.push( + `| \`${cache.key}\` | ${(cache.sizeBytes / (1024 * 1024)).toFixed(1)} |`, + ); + } + // Only the optional ratio line is dropped. Filtering on "" instead would + // collapse every intentional blank line and destroy the markdown structure. + return `${lines.filter((line) => line !== null).join("\n")}\n`; +} + +export async function main(env = process.env) { + const token = requireEnv(env, "GH_TOKEN"); + const runId = requireEnv(env, "GITHUB_RUN_ID"); + const setupJavaRepository = requireEnv(env, "SETUP_JAVA_REPOSITORY"); + const setupJavaRef = requireEnv(env, "SETUP_JAVA_REF"); + const [owner, repo] = requireEnv(env, "GITHUB_REPOSITORY").split("/"); + if (!owner || !repo) { + throw new Error( + `GITHUB_REPOSITORY must be owner/repo, got "${env.GITHUB_REPOSITORY}"`, + ); + } + + const rows = parseSamples(await readSampleFiles()); + const analysis = analyze(rows); + if (analysis.pairs.length === 0) { + throw new Error("No complete ABBA samples were collected"); + } + + const cacheEntries = await allPages( + `/repos/${owner}/${repo}/actions/caches`, + "actions_caches", + token, + ); + // Only the dependency entry is identified here. The wrapper entry is keyed on + // PetClinic's own wrapper properties, which this benchmark does not modify, so + // it is shared with every other run and must not be deleted by one of them. + const dependencyKey = `setup-java-Linux-x64-maven-${hashFilesSingle( + `cache-value-${runId}\n`, + )}`; + const entry = cacheEntries.find((cache) => cache.key === dependencyKey); + if (!entry) { + throw new Error(`Expected cache not found: ${dependencyKey}`); + } + const caches = [ + { key: dependencyKey, id: entry.id, sizeBytes: entry.size_in_bytes }, + ]; + + const metadata = { + repository: env.GITHUB_REPOSITORY, + runId, + javaVersion: env.JAVA_VERSION, + setupJavaRepository, + setupJavaRef, + generatedAt: new Date().toISOString(), + }; + const report = markdown(metadata, analysis, caches); + + await mkdir("cache-value-results", { recursive: true }); + await writeFile( + "cache-value-results/results.json", + `${JSON.stringify({ metadata, analysis, caches }, null, 2)}\n`, + ); + await writeFile( + "cache-value-results/results.csv", + `sample,arm,slot,seconds\n${rows + .map((row) => + [row.sample, row.arm, row.slot, row.seconds].map(csvValue).join(","), + ) + .join("\n")}\n`, + ); + await writeFile("cache-value-results/summary.md", report); + await appendFile(env.GITHUB_STEP_SUMMARY, report); + + if (env.CLEANUP_CACHES === "true") { + for (const cache of caches) { + await api(`/repos/${owner}/${repo}/actions/caches/${cache.id}`, token, { + method: "DELETE", + }); + } + console.log(`Deleted ${caches.length} cache value benchmark caches`); + } +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + await main(); +} diff --git a/scripts/report-cache-value.test.mjs b/scripts/report-cache-value.test.mjs new file mode 100644 index 0000000..6856de0 --- /dev/null +++ b/scripts/report-cache-value.test.mjs @@ -0,0 +1,63 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { analyze, markdown } from "./report-cache-value.mjs"; +import { parseSamples } from "./paired.mjs"; + +// Slot order in the workflow is cached, uncached, uncached, cached. +function sample(runner, cached, uncached) { + return [ + `"${runner}","cached","1","${cached}"`, + `"${runner}","uncached","2","${uncached}"`, + `"${runner}","uncached","3","${uncached}"`, + `"${runner}","cached","4","${cached}"`, + ].join("\n"); +} + +const rows = parseSamples( + [1, 2, 3, 4, 5, 6] + .map((runner) => sample(runner, 8000 + runner * 500, 46000 + runner * 3000)) + .join("\n"), +); + +test("reports the cache as an improvement, not a regression", () => { + const analysis = analyze(rows); + assert.ok( + analysis.interval.estimate < 0, + "cached minus uncached must be negative when the cache saves time", + ); + assert.equal(analysis.verdict, "improvement"); +}); + +test("quotes the saving as a ratio as well as a difference", () => { + const analysis = analyze(rows); + const rendered = markdown( + { + runId: "1", + javaVersion: "17", + setupJavaRepository: "actions/setup-java", + setupJavaRef: "main", + }, + analysis, + [], + ); + // 56.5s uncached against 9.75s cached. + assert.match(rendered, /5\.8x faster/); + assert.match(rendered, /Maven cache value/); +}); + +// The failure this benchmark must never hide is a silent cache miss: both arms +// would then do the same download and the report would state, with a tight +// interval, that the cache is worth nothing. The workflow guards this with +// setup-java's cache-hit output, and the analysis must not paper over it either. +test("does not claim a saving when both arms did the same work", () => { + const identical = parseSamples( + [1, 2, 3, 4, 5, 6] + .map((runner) => + sample(runner, 46000 + runner * 3000, 46000 + runner * 3000), + ) + .join("\n"), + ); + const analysis = analyze(identical); + assert.ok(["within-noise", "inconclusive"].includes(analysis.verdict)); +}); diff --git a/scripts/report-focused.mjs b/scripts/report-focused.mjs deleted file mode 100644 index a2f396d..0000000 --- a/scripts/report-focused.mjs +++ /dev/null @@ -1,257 +0,0 @@ -import {appendFile, mkdir, writeFile} from 'node:fs/promises'; -import {pathToFileURL} from 'node:url'; - -import {hashFilesSingle, median, secondsBetween} from './report.mjs'; - -const API_VERSION = '2022-11-28'; -const JOB_PATTERN = /^(v4|main) \/ warm \/ (\d+)$/; - -export function parseFocusedJob(name) { - const match = name.match(JOB_PATTERN); - if (!match) return null; - return {version: match[1], sample: Number(match[2])}; -} - -function quantile(values, percentile) { - const sorted = [...values].sort((a, b) => a - b); - const position = (sorted.length - 1) * percentile; - const lower = Math.floor(position); - const upper = Math.ceil(position); - if (lower === upper) return sorted[lower]; - return ( - sorted[lower] + - (sorted[upper] - sorted[lower]) * (position - lower) - ); -} - -function csvValue(value) { - return `"${String(value).replaceAll('"', '""')}"`; -} - -async function api(path, token, options = {}) { - const response = await fetch(`https://api.github.com${path}`, { - ...options, - headers: { - Accept: 'application/vnd.github+json', - Authorization: `Bearer ${token}`, - 'X-GitHub-Api-Version': API_VERSION, - ...options.headers - } - }); - if (!response.ok) { - throw new Error(`${options.method ?? 'GET'} ${path}: ${response.status}`); - } - if (response.status === 204) return null; - return response.json(); -} - -async function allPages(path, field, token) { - const values = []; - for (let page = 1; ; page += 1) { - const separator = path.includes('?') ? '&' : '?'; - const response = await api( - `${path}${separator}per_page=100&page=${page}`, - token - ); - values.push(...response[field]); - if (response[field].length < 100) return values; - } -} - -function summarize(rows, version) { - const values = rows - .filter(row => row.version === version) - .map(row => row.setupSeconds); - return { - version, - samples: values.length, - minimumSeconds: Math.min(...values), - medianSeconds: median(values), - p95Seconds: quantile(values, 0.95), - maximumSeconds: Math.max(...values) - }; -} - -function markdown(metadata, rows, summaries, caches) { - const lines = [ - '# Focused cache restore benchmark', - '', - `Temurin ${metadata.javaVersion}, ${metadata.samples} warm samples per version, run ${metadata.runId}.`, - '', - '| Version | Samples | Min (s) | Median (s) | p95 (s) | Max (s) |', - '| --- | ---: | ---: | ---: | ---: | ---: |' - ]; - for (const summary of summaries) { - lines.push( - `| ${summary.version} | ${summary.samples} | ${summary.minimumSeconds.toFixed(1)} | ${summary.medianSeconds.toFixed(1)} | ${summary.p95Seconds.toFixed(1)} | ${summary.maximumSeconds.toFixed(1)} |` - ); - } - const deltas = []; - for (let sample = 1; sample <= metadata.samples; sample += 1) { - const v4 = rows.find( - row => row.version === 'v4' && row.sample === sample - ); - const main = rows.find( - row => row.version === 'main' && row.sample === sample - ); - if (v4 && main) deltas.push(main.setupSeconds - v4.setupSeconds); - } - lines.push( - '', - `Paired median delta (main - v4): **${median(deltas).toFixed(1)}s**. Negative means main was faster.`, - '', - '## Samples', - '', - '| Sample | v4 (s) | main (s) | Delta (s) |', - '| ---: | ---: | ---: | ---: |' - ); - for (let sample = 1; sample <= metadata.samples; sample += 1) { - const v4 = rows.find( - row => row.version === 'v4' && row.sample === sample - ); - const main = rows.find( - row => row.version === 'main' && row.sample === sample - ); - if (v4 && main) { - lines.push( - `| ${sample} | ${v4.setupSeconds.toFixed(1)} | ${main.setupSeconds.toFixed(1)} | ${(main.setupSeconds - v4.setupSeconds).toFixed(1)} |` - ); - } - } - lines.push( - '', - '## Cache fixtures', - '', - '| Version | Cache | Size (MiB) |', - '| --- | --- | ---: |' - ); - for (const cache of caches) { - lines.push( - `| ${cache.version} | ${cache.type} | ${(cache.sizeBytes / 1024 / 1024).toFixed(1)} |` - ); - } - return `${lines.join('\n')}\n`; -} - -export async function main(env = process.env) { - const [owner, repo] = env.GITHUB_REPOSITORY.split('/'); - const token = env.GH_TOKEN; - const runId = env.GITHUB_RUN_ID; - const attempt = env.GITHUB_RUN_ATTEMPT; - const samples = Number(env.SAMPLES); - const javaVersion = env.JAVA_VERSION; - if (!owner || !repo || !token || !runId || !attempt || !samples) { - throw new Error('Missing required GitHub Actions environment variables'); - } - - const [jobs, cacheEntries, mainCommit, v4Ref] = await Promise.all([ - allPages( - `/repos/${owner}/${repo}/actions/runs/${runId}/attempts/${attempt}/jobs`, - 'jobs', - token - ), - allPages(`/repos/${owner}/${repo}/actions/caches`, 'actions_caches', token), - api('/repos/actions/setup-java/commits/main', token), - api('/repos/actions/setup-java/git/ref/tags/v4.8.0', token) - ]); - - const rows = jobs - .map(job => { - const identity = parseFocusedJob(job.name); - if (!identity) return null; - const step = job.steps.find(item => - item.name.startsWith('Setup Java') - ); - return { - ...identity, - conclusion: job.conclusion, - setupSeconds: secondsBetween(step.started_at, step.completed_at) - }; - }) - .filter(Boolean) - .sort( - (a, b) => - a.sample - b.sample || a.version.localeCompare(b.version) - ); - - const caches = []; - for (const [version, versionId] of [ - ['v4', 'v4'], - ['main', 'main'] - ]) { - const benchmarkId = `focused-${versionId}-${runId}`; - const expected = [ - { - type: 'maven-dependencies', - key: `setup-java-Linux-x64-maven-${hashFilesSingle( - `${benchmarkId}\n` - )}` - } - ]; - if (version === 'main') { - expected.push({ - type: 'maven-wrapper', - key: `setup-java-Linux-x64-maven-wrapper-${hashFilesSingle( - `wrapperVersion=focused\n# benchmark-id=${benchmarkId}\n` - )}` - }); - } - for (const item of expected) { - const entry = cacheEntries.find(cache => cache.key === item.key); - if (!entry) throw new Error(`Expected cache not found: ${item.key}`); - caches.push({ - version, - type: item.type, - id: entry.id, - key: item.key, - sizeBytes: entry.size_in_bytes - }); - } - } - - const metadata = { - repository: env.GITHUB_REPOSITORY, - runId, - runAttempt: Number(attempt), - samples, - javaVersion, - setupJavaV4Ref: v4Ref.object.sha, - setupJavaMainRefAtReport: mainCommit.sha, - generatedAt: new Date().toISOString() - }; - const summaries = ['v4', 'main'].map(version => - summarize(rows, version) - ); - const report = markdown(metadata, rows, summaries, caches); - - await mkdir('focused-results', {recursive: true}); - await writeFile( - 'focused-results/results.json', - `${JSON.stringify({metadata, summaries, rows, caches}, null, 2)}\n` - ); - await writeFile( - 'focused-results/results.csv', - `version,sample,setup_seconds,conclusion\n${rows - .map(row => - [row.version, row.sample, row.setupSeconds, row.conclusion] - .map(csvValue) - .join(',') - ) - .join('\n')}\n` - ); - await writeFile('focused-results/summary.md', report); - await appendFile(env.GITHUB_STEP_SUMMARY, report); - - if (env.CLEANUP_CACHES === 'true') { - for (const cache of caches) { - await api(`/repos/${owner}/${repo}/actions/caches/${cache.id}`, token, { - method: 'DELETE' - }); - } - console.log(`Deleted ${caches.length} focused benchmark caches`); - } -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - await main(); -} diff --git a/scripts/report-jdk-cache.mjs b/scripts/report-jdk-cache.mjs index d6a5648..cccc3d9 100644 --- a/scripts/report-jdk-cache.mjs +++ b/scripts/report-jdk-cache.mjs @@ -1,97 +1,33 @@ -import {appendFile, mkdir, writeFile} from 'node:fs/promises'; -import {pathToFileURL} from 'node:url'; +import { appendFile, mkdir, writeFile } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; -import {hashFilesSingle, median, secondsBetween} from './report.mjs'; +import { + analyzePairs, + parseSamples, + readSampleFiles, + requireEnv, +} from "./paired.mjs"; +import { describeVerdict, formatInterval } from "./stats.mjs"; -const API_VERSION = '2022-11-28'; -const ARMS = ['baseline', 'treatment']; -const JOB_PATTERN = /^(baseline|treatment) \/ (seed|warm)(?: \/ (\d+))?$/; -const JDK_CACHE_PREFIX = 'setup-java-jdk-'; - -export function parseJdkCacheJob(name) { - const match = name.match(JOB_PATTERN); - if (!match) return null; - const [, arm, phase, sampleText] = match; - if ((phase === 'seed' && sampleText) || (phase === 'warm' && !sampleText)) { - return null; - } - return { - arm, - phase, - sample: sampleText ? Number(sampleText) : null - }; -} - -export function newestJdkCache(cacheEntries) { - return ( - cacheEntries - .filter(cache => cache.key.startsWith(JDK_CACHE_PREFIX)) - .sort( - (a, b) => - Date.parse(b.created_at ?? 0) - Date.parse(a.created_at ?? 0) - )[0] ?? null - ); -} - -export function expectedArmCacheKeys(arm, runId, wrapperOriginal) { - const benchmarkId = `jdk-cache-${arm}-${runId}`; - return { - dependencies: `setup-java-Linux-x64-maven-${hashFilesSingle( - `${benchmarkId}\n` - )}`, - wrapper: `setup-java-Linux-x64-maven-wrapper-${hashFilesSingle( - `${wrapperOriginal}# jdk-cache-benchmark=${benchmarkId}\n` - )}` - }; -} - -export function summarizeJdkArm(rows, arm) { - const armRows = rows.filter(row => row.arm === arm); - const warmRows = armRows.filter(row => row.phase === 'warm'); - const seed = armRows.find(row => row.phase === 'seed') ?? null; - return { - arm, - samples: warmRows.length, - warmSetupSeconds: median(warmRows.map(row => row.setupSeconds)), - warmBuildSeconds: median(warmRows.map(row => row.buildSeconds)), - warmPostSeconds: median(warmRows.map(row => row.postSeconds)), - warmJobSeconds: median(warmRows.map(row => row.jobSeconds)), - coldSetupSeconds: seed?.setupSeconds ?? null, - coldPostSeconds: seed?.postSeconds ?? null, - estimatedBilledMinutes: warmRows.reduce( - (total, row) => - total + - (row.jobSeconds === null ? 0 : Math.ceil(row.jobSeconds / 60)), - 0 - ) - }; -} - -function formatSeconds(value) { - return value === null ? 'n/a' : value.toFixed(1); -} - -function formatMiB(bytes) { - return bytes === null ? 'n/a' : (bytes / 1024 / 1024).toFixed(1); -} +const API_VERSION = "2022-11-28"; +const RESULTS_DIR = "jdk-cache-results"; function csvValue(value) { - const text = value === null || value === undefined ? '' : String(value); - return `"${text.replaceAll('"', '""')}"`; + return `"${String(value ?? "").replaceAll('"', '""')}"`; } async function api(path, token, options = {}) { const response = await fetch(`https://api.github.com${path}`, { ...options, headers: { - Accept: 'application/vnd.github+json', + Accept: "application/vnd.github+json", Authorization: `Bearer ${token}`, - 'X-GitHub-Api-Version': API_VERSION, - ...options.headers - } + "X-GitHub-Api-Version": API_VERSION, + ...options.headers, + }, }); if (!response.ok) { - throw new Error(`${options.method ?? 'GET'} ${path}: ${response.status}`); + throw new Error(`${options.method ?? "GET"} ${path}: ${response.status}`); } if (response.status === 204) return null; return response.json(); @@ -100,248 +36,168 @@ async function api(path, token, options = {}) { async function allPages(path, field, token) { const values = []; for (let page = 1; ; page += 1) { - const separator = path.includes('?') ? '&' : '?'; + const separator = path.includes("?") ? "&" : "?"; const response = await api( `${path}${separator}per_page=100&page=${page}`, - token + token, ); values.push(...response[field]); if (response[field].length < 100) return values; } } -function stepDuration(job, predicate) { - const step = job.steps.find(predicate); - return step ? secondsBetween(step.started_at, step.completed_at) : null; -} - -function buildCaches(cacheEntries, runId, wrapperOriginal) { - const caches = []; - for (const arm of ARMS) { - const expected = expectedArmCacheKeys(arm, runId, wrapperOriginal); - for (const [type, key] of Object.entries(expected)) { - const entry = cacheEntries.find(cache => cache.key === key); - caches.push({ - arm, - type, - key, - id: entry?.id ?? null, - sizeBytes: entry?.size_in_bytes ?? null - }); - } - } - - const jdkEntry = newestJdkCache(cacheEntries); - caches.push({ - arm: 'treatment', - type: 'jdk', - key: jdkEntry?.key ?? null, - id: jdkEntry?.id ?? null, - sizeBytes: jdkEntry?.size_in_bytes ?? null - }); - return caches; -} - -function markdown(metadata, rows, summaries, caches) { - const baseline = summaries.find(summary => summary.arm === 'baseline'); - const treatment = summaries.find(summary => summary.arm === 'treatment'); - const jdkCache = caches.find(cache => cache.type === 'jdk'); - const delta = - baseline.warmSetupSeconds === null || treatment.warmSetupSeconds === null - ? null - : treatment.warmSetupSeconds - baseline.warmSetupSeconds; +export function markdown(metadata, analysis, caches) { + const { baseline, candidate, interval, control } = analysis; const lines = [ - '# JDK cache benchmark', - '', - `${metadata.distribution} Java ${metadata.javaVersion}, ${metadata.samples} warm samples per arm, run ${metadata.runId}.`, - '', - '## Headline', - '', - `Baseline warm setup: **${formatSeconds(baseline.warmSetupSeconds)}s**; treatment warm setup: **${formatSeconds(treatment.warmSetupSeconds)}s**; treatment delta: **${formatSeconds(delta)}s**. JDK cache storage: **${formatMiB(jdkCache.sizeBytes)} MiB**.`, - '' + "# JDK cache benchmark", + "", + `${metadata.distribution} ${metadata.javaVersion}, ${analysis.pairs.length} runners x 2 paired observations, run ${metadata.runId}.`, + `\`cache-jdk: false\` vs \`cache-jdk: true\` using \`${metadata.setupJavaRef}\` from \`${metadata.setupJavaRepository}\`.`, + "", + "The runner tool cache is purged before every measured slot, so each setup", + "resolves the JDK the way a runner image without it would.", + "", + "## Verdict", + "", + `**${describeVerdict(analysis.verdict)}**`, + "", + `Paired difference (\`cache-jdk: true\` - \`cache-jdk: false\`): **${formatInterval(interval)}**.`, + `Permutation p-value: ${analysis.pValue.toFixed(3)}. Hodges-Lehmann shift: ${analysis.shiftSeconds.toFixed(3)}s.`, + `Harness noise floor: ${analysis.noiseFloorSeconds.toFixed(3)}s (median within-runner repeat spread).`, + "", + `A/A control (\`cache-jdk: false\` against itself) reports **${control.verdict}** at ${formatInterval(control.interval)}. A healthy harness reports \`within-noise\` or \`inconclusive\` here; an \`improvement\` or \`regression\` means slot ordering is biasing results and the verdict above cannot be trusted.`, + "", + analysis.droppedRunners.length === 0 + ? "No runner was discarded for a stalled slot." + : `Discarded ${analysis.droppedRunners.length} runner(s) whose own arm disagreed with itself by more than ${analysis.stallThresholdSeconds.toFixed(3)}s, a threshold derived from the spread of the other runners: ${analysis.droppedRunners.map((entry) => `#${entry.sample} (${entry.repeatSpread.toFixed(3)}s)`).join(", ")}. A stalled slot lands on an arbitrary arm and would otherwise dominate the mean; the decision uses only within-arm spread, which carries no information about the effect.`, + "", + "## Arms", + "", + "| Arm | Runners | Mean (s) | Median (s) | SD (s) | MAD (s) | p95 (s) |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: |", ]; - - if (jdkCache.id === null) { + for (const [label, arm] of [ + ["cache-jdk: false", baseline], + ["cache-jdk: true", candidate], + ]) { lines.push( - '> [!WARNING]', - '> No `setup-java-jdk-` cache entry exists after the treatment seed. The runner tool cache may have been hit or JDK caching was disabled, so this run does not measure a JDK cache restore.', - '' + `| \`${label}\` | ${arm.samples} | ${arm.meanSeconds.toFixed(3)} | ${arm.medianSeconds.toFixed(3)} | ${arm.standardDeviationSeconds.toFixed(3)} | ${arm.madSeconds.toFixed(3)} | ${arm.p95Seconds.toFixed(3)} |`, ); } - lines.push( - '| Arm | Warm samples | Median setup (s) | Median build (s) | Median post-step (s) | Median job (s) | Cold seed setup (s) | Cold seed post-save (s) | Estimated billed minutes |', - '| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |' + "", + "All durations are measured inside the job with millisecond resolution. The Actions API reports step timestamps only to the nearest second, which is too coarse for effects of this size.", + "", + "## Paired samples", + "", + "| Runner | no-cache slot 1 (s) | cache slot 2 (s) | cache slot 3 (s) | no-cache slot 4 (s) | Paired delta (s) |", + "| ---: | ---: | ---: | ---: | ---: | ---: |", ); - for (const summary of summaries) { + for (const pair of analysis.pairs) { lines.push( - `| ${summary.arm} | ${summary.samples} | ${formatSeconds(summary.warmSetupSeconds)} | ${formatSeconds(summary.warmBuildSeconds)} | ${formatSeconds(summary.warmPostSeconds)} | ${formatSeconds(summary.warmJobSeconds)} | ${formatSeconds(summary.coldSetupSeconds)} | ${formatSeconds(summary.coldPostSeconds)} | ${summary.estimatedBilledMinutes} |` + `| ${pair.sample} | ${pair.baselineSlots[0].toFixed(3)} | ${pair.candidateSlots[0].toFixed(3)} | ${pair.candidateSlots[1].toFixed(3)} | ${pair.baselineSlots[1].toFixed(3)} | ${pair.difference.toFixed(3)} |`, ); } - lines.push( - '', - 'The billed-minute estimate covers only warm measurement jobs, rounding each Ubuntu job up to a whole minute. It excludes seeds because they are one-time setup and the treatment seed pays the JDK save, which would distort the steady-state comparison. It also excludes the shared prepare and report jobs.', - '', - '## Cache entries', - '', - '| Arm | Cache | Size (MiB) | Key |', - '| --- | --- | ---: | --- |' + "", + "## Caches", + "", + "Both arms restore the same Maven entry, so the stored blob cannot bias the comparison. Only the JDK entry differs between them.", + "", + "| Cache | Size (MiB) |", + "| --- | ---: |", ); for (const cache of caches) { lines.push( - `| ${cache.arm} | ${cache.type} | ${formatMiB(cache.sizeBytes)} | ${cache.key === null ? 'not found' : `\`${cache.key}\``} |` + `| ${cache.type} | ${(cache.sizeBytes / 1024 / 1024).toFixed(1)} |`, ); } - - lines.push( - '', - '## Samples', - '', - '| Arm | Phase | Sample | Setup (s) | Build (s) | Post-step (s) | Job (s) | Conclusion |', - '| --- | --- | ---: | ---: | ---: | ---: | ---: | --- |' - ); - for (const row of rows) { - lines.push( - `| ${row.arm} | ${row.phase} | ${row.sample ?? '-'} | ${formatSeconds(row.setupSeconds)} | ${formatSeconds(row.buildSeconds)} | ${formatSeconds(row.postSeconds)} | ${formatSeconds(row.jobSeconds)} | ${row.conclusion} |` - ); - } - return `${lines.join('\n')}\n`; + return `${lines.join("\n")}\n`; } export async function main(env = process.env) { - const [owner, repo] = env.GITHUB_REPOSITORY.split('/'); + requireEnv(env, [ + "GITHUB_REPOSITORY", + "GH_TOKEN", + "GITHUB_RUN_ID", + "DISTRIBUTION", + "JAVA_VERSION", + "SETUP_JAVA_REPOSITORY", + "SETUP_JAVA_REF", + ]); + const [owner, repo] = env.GITHUB_REPOSITORY.split("/"); const token = env.GH_TOKEN; const runId = env.GITHUB_RUN_ID; - const attempt = env.GITHUB_RUN_ATTEMPT; - const samples = Number(env.SAMPLES); - if ( - !owner || - !repo || - !token || - !runId || - !attempt || - !samples || - !env.PETCLINIC_REF || - !env.DISTRIBUTION || - !env.JAVA_VERSION - ) { - throw new Error('Missing required GitHub Actions environment variables'); + if (!owner || !repo) { + throw new Error( + `GITHUB_REPOSITORY must be owner/repo, got "${env.GITHUB_REPOSITORY}"`, + ); } - const [jobs, cacheEntries, wrapperResponse, mainCommit] = await Promise.all([ - allPages( - `/repos/${owner}/${repo}/actions/runs/${runId}/attempts/${attempt}/jobs`, - 'jobs', - token - ), - allPages(`/repos/${owner}/${repo}/actions/caches`, 'actions_caches', token), - api( - `/repos/spring-projects/spring-petclinic/contents/.mvn/wrapper/maven-wrapper.properties?ref=${env.PETCLINIC_REF}`, - token - ), - api('/repos/actions/setup-java/commits/main', token) - ]); - - const rows = jobs - .map(job => { - const identity = parseJdkCacheJob(job.name); - if (!identity) return null; - return { - ...identity, - conclusion: job.conclusion, - setupSeconds: stepDuration(job, step => - step.name.startsWith('Setup Java') - ), - buildSeconds: stepDuration(job, step => step.name === 'Build PetClinic'), - postSeconds: stepDuration(job, step => - step.name.startsWith('Post Setup Java') - ), - jobSeconds: secondsBetween(job.started_at, job.completed_at) - }; - }) - .filter(Boolean) - .sort( - (a, b) => - a.arm.localeCompare(b.arm) || - a.phase.localeCompare(b.phase) || - (a.sample ?? 0) - (b.sample ?? 0) - ); + const rows = parseSamples(await readSampleFiles("jdk-cache-timings")); + const analysis = analyzePairs(rows, "baseline", "candidate"); + if (analysis.pairs.length === 0) { + throw new Error("No complete ABBA samples were collected"); + } - const wrapperOriginal = Buffer.from(wrapperResponse.content, 'base64').toString( - 'utf8' + const cacheEntries = await allPages( + `/repos/${owner}/${repo}/actions/caches`, + "actions_caches", + token, ); - const caches = buildCaches(cacheEntries, runId, wrapperOriginal); - const missingCaches = caches.filter(cache => cache.id === null); - if (missingCaches.length > 0) { - console.warn(`Could not find ${missingCaches.length} expected cache entries`); - } + const caches = cacheEntries + .filter( + (entry) => + entry.key.startsWith("setup-java-jdk-") || + entry.key.startsWith("setup-java-Linux-x64-maven"), + ) + .map((entry) => ({ + type: entry.key.startsWith("setup-java-jdk-") ? "jdk" : "maven", + id: entry.id, + key: entry.key, + sizeBytes: entry.size_in_bytes, + })); const metadata = { repository: env.GITHUB_REPOSITORY, runId, - runAttempt: Number(attempt), - samples, distribution: env.DISTRIBUTION, javaVersion: env.JAVA_VERSION, - petclinicRef: env.PETCLINIC_REF, - setupJavaMainRefAtReport: mainCommit.sha, - generatedAt: new Date().toISOString() + setupJavaRepository: env.SETUP_JAVA_REPOSITORY, + setupJavaRef: env.SETUP_JAVA_REF, + generatedAt: new Date().toISOString(), }; - const summaries = ARMS.map(arm => summarizeJdkArm(rows, arm)); - const report = markdown(metadata, rows, summaries, caches); - await mkdir('jdk-cache-results', {recursive: true}); + const report = markdown(metadata, analysis, caches); + await mkdir(RESULTS_DIR, { recursive: true }); await writeFile( - 'jdk-cache-results/results.json', - `${JSON.stringify({metadata, summaries, rows, caches}, null, 2)}\n` - ); - const csvHeaders = [ - 'arm', - 'phase', - 'sample', - 'setup_seconds', - 'build_seconds', - 'post_seconds', - 'job_seconds', - 'conclusion' - ]; - const csvRows = rows.map(row => - [ - row.arm, - row.phase, - row.sample, - row.setupSeconds, - row.buildSeconds, - row.postSeconds, - row.jobSeconds, - row.conclusion - ] - .map(csvValue) - .join(',') + `${RESULTS_DIR}/results.json`, + `${JSON.stringify({ metadata, analysis, caches }, null, 2)}\n`, ); await writeFile( - 'jdk-cache-results/results.csv', - `${csvHeaders.join(',')}\n${csvRows.join('\n')}\n` + `${RESULTS_DIR}/results.csv`, + `sample,arm,slot,seconds\n${rows + .map((row) => + [row.sample, row.arm, row.slot, row.seconds].map(csvValue).join(","), + ) + .join("\n")}\n`, ); - await writeFile('jdk-cache-results/summary.md', report); + await writeFile(`${RESULTS_DIR}/summary.md`, report); await appendFile(env.GITHUB_STEP_SUMMARY, report); - if (env.CLEANUP_CACHES === 'true') { - const benchmarkIds = caches.map(cache => cache.id).filter(Boolean); - const jdkIds = cacheEntries - .filter(cache => cache.key.startsWith(JDK_CACHE_PREFIX)) - .map(cache => cache.id); - const ids = [...new Set([...benchmarkIds, ...jdkIds])]; - for (const id of ids) { - await api(`/repos/${owner}/${repo}/actions/caches/${id}`, token, { - method: 'DELETE' + if (env.CLEANUP_CACHES === "true") { + for (const cache of caches) { + await api(`/repos/${owner}/${repo}/actions/caches/${cache.id}`, token, { + method: "DELETE", }); } - console.log(`Deleted ${ids.length} JDK benchmark cache entries`); + console.log(`Deleted ${caches.length} JDK cache benchmark caches`); } } -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { await main(); } diff --git a/scripts/report-jdk-cache.test.mjs b/scripts/report-jdk-cache.test.mjs index 3462c5f..7698724 100644 --- a/scripts/report-jdk-cache.test.mjs +++ b/scripts/report-jdk-cache.test.mjs @@ -1,92 +1,89 @@ -import assert from 'node:assert/strict'; -import test from 'node:test'; +import assert from "node:assert/strict"; +import test from "node:test"; -import { - expectedArmCacheKeys, - newestJdkCache, - parseJdkCacheJob, - summarizeJdkArm -} from './report-jdk-cache.mjs'; +import { analyzePairs, parseSamples } from "./paired.mjs"; +import { markdown } from "./report-jdk-cache.mjs"; -test('parses JDK cache benchmark job names', () => { - assert.deepEqual(parseJdkCacheJob('baseline / seed'), { - arm: 'baseline', - phase: 'seed', - sample: null - }); - assert.deepEqual(parseJdkCacheJob('treatment / warm / 10'), { - arm: 'treatment', - phase: 'warm', - sample: 10 - }); - assert.equal(parseJdkCacheJob('baseline / warm'), null); - assert.equal(parseJdkCacheJob('Report'), null); -}); +function abba(sample, noCache, cache) { + return [ + `"${sample}","baseline","1","${noCache[0]}"`, + `"${sample}","candidate","2","${cache[0]}"`, + `"${sample}","candidate","3","${cache[1]}"`, + `"${sample}","baseline","4","${noCache[1]}"`, + ].join("\n"); +} -test('selects the newest JDK cache entry', () => { - const newest = newestJdkCache([ - { - key: 'setup-java-jdk-v1-Linux-x64-old', - created_at: '2026-01-01T00:00:00Z' - }, - {key: 'setup-java-Linux-x64-maven-not-a-jdk'}, - { - key: 'setup-java-jdk-v1-Linux-x64-new', - created_at: '2026-01-02T00:00:00Z' - } - ]); - assert.equal(newest.key, 'setup-java-jdk-v1-Linux-x64-new'); - assert.equal(newestJdkCache([{key: 'setup-java-Linux-maven-key'}]), null); +const metadata = { + runId: "1", + distribution: "microsoft", + javaVersion: "17", + setupJavaRepository: "actions/setup-java", + setupJavaRef: "main", +}; + +test("resolves a large consistent JDK cache saving", () => { + // Restoring the JDK from the Actions cache instead of downloading it from the + // vendor is a multi-second effect, so it must clear the noise floor. + const csv = [ + abba(1, [21000, 21400], [9000, 9200]), + abba(2, [23000, 22600], [10100, 9900]), + abba(3, [20500, 20900], [8800, 9100]), + abba(4, [24000, 23600], [11000, 10700]), + abba(5, [22000, 22300], [9500, 9800]), + abba(6, [21500, 21100], [9300, 9000]), + ].join("\n"); + const analysis = analyzePairs(parseSamples(csv)); + assert.equal(analysis.pairs.length, 6); + assert.equal(analysis.verdict, "improvement"); + assert.ok(analysis.interval.high < 0); + assert.ok(analysis.interval.estimate < -11); + assert.equal(analysis.control.verdict !== "improvement", true); }); -test('derives distinct dependency and wrapper cache keys per arm', () => { - const baseline = expectedArmCacheKeys('baseline', '42', 'wrapper=true\n'); - const treatment = expectedArmCacheKeys('treatment', '42', 'wrapper=true\n'); - assert.match(baseline.dependencies, /^setup-java-Linux-x64-maven-/); - assert.match(baseline.wrapper, /^setup-java-Linux-x64-maven-wrapper-/); - assert.notEqual(baseline.dependencies, treatment.dependencies); - assert.notEqual(baseline.wrapper, treatment.wrapper); +test("reports an unchanged configuration as inconclusive", () => { + // Both arms behaving identically is the A/A case. The estimator must not + // resolve an effect, whatever the runners happen to be doing. + const csv = [ + abba(1, [9000, 9600], [9300, 9100]), + abba(2, [12000, 11400], [11800, 12200]), + abba(3, [8600, 9000], [8800, 8500]), + abba(4, [15000, 14200], [14600, 15100]), + abba(5, [10200, 10800], [10500, 10100]), + abba(6, [9800, 9200], [9400, 9900]), + ].join("\n"); + const analysis = analyzePairs(parseSamples(csv)); + assert.ok(["inconclusive", "within-noise"].includes(analysis.verdict)); }); -test('summarizes seed and warm measurements for an arm', () => { - const summary = summarizeJdkArm( - [ - { - arm: 'treatment', - phase: 'seed', - setupSeconds: 7, - buildSeconds: 20, - postSeconds: 8, - jobSeconds: 64 - }, - { - arm: 'treatment', - phase: 'warm', - setupSeconds: 4, - buildSeconds: 10, - postSeconds: 0.4, - jobSeconds: 30 - }, - { - arm: 'treatment', - phase: 'warm', - setupSeconds: 2, - buildSeconds: 12, - postSeconds: 0.2, - jobSeconds: 40 - } - ], - 'treatment' +test("drops runners that did not complete all four slots", () => { + const csv = [ + abba(1, [21000, 21400], [9000, 9200]), + '"2","baseline","1","21000"', + '"2","candidate","2","9000"', + ].join("\n"); + const analysis = analyzePairs(parseSamples(csv)); + assert.deepEqual( + analysis.pairs.map((pair) => pair.sample), + [1], ); - assert.deepEqual(summary, { - arm: 'treatment', - samples: 2, - warmSetupSeconds: 3, - warmBuildSeconds: 11, - warmPostSeconds: 0.30000000000000004, - warmJobSeconds: 35, - coldSetupSeconds: 7, - coldPostSeconds: 8, - estimatedBilledMinutes: 2 - }); +}); + +test("renders a verdict, both arms and the paired samples", () => { + const csv = [ + abba(1, [21000, 21400], [9000, 9200]), + abba(2, [23000, 22600], [10100, 9900]), + ].join("\n"); + const analysis = analyzePairs(parseSamples(csv)); + const report = markdown(metadata, analysis, [ + { type: "jdk", sizeBytes: 190 * 1024 * 1024 }, + { type: "maven", sizeBytes: 60 * 1024 * 1024 }, + ]); + assert.match(report, /# JDK cache benchmark/); + assert.match(report, /## Verdict/); + assert.match(report, /cache-jdk: true/); + assert.match(report, /A\/A control/); + assert.match(report, /Harness noise floor/); + assert.match(report, /\| jdk \| 190\.0 \|/); + // One row per runner, so a reader can see the raw slots behind the interval. + assert.match(report, /\| 1 \| 21\.000 \| 9\.000 \| 9\.200 \| 21\.400 \|/); }); diff --git a/scripts/report-transfer-overlap.mjs b/scripts/report-transfer-overlap.mjs new file mode 100644 index 0000000..396acd7 --- /dev/null +++ b/scripts/report-transfer-overlap.mjs @@ -0,0 +1,244 @@ +import { + appendFile, + mkdir, + readdir, + readFile, + writeFile, +} from "node:fs/promises"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +import { hashFilesSingle } from "./report.mjs"; +import { + analyzePairs, + buildPairs, + noiseFloor, + parseSamples, + readSampleFiles as readPairedSampleFiles, + requireEnv, +} from "./paired.mjs"; +import { classify, describeVerdict, formatInterval } from "./stats.mjs"; + +export { buildPairs, noiseFloor, parseSamples }; + +export function readSampleFiles(directory) { + return readPairedSampleFiles("transfer-overlap-timings", directory); +} + +export function analyze(rows) { + return analyzePairs(rows, "baseline", "candidate"); +} + +const API_VERSION = "2022-11-28"; + +function csvValue(value) { + return `"${String(value ?? "").replaceAll('"', '""')}"`; +} + +async function api(path, token, options = {}) { + const response = await fetch(`https://api.github.com${path}`, { + ...options, + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "X-GitHub-Api-Version": API_VERSION, + ...options.headers, + }, + }); + if (!response.ok) { + throw new Error(`${options.method ?? "GET"} ${path}: ${response.status}`); + } + if (response.status === 204) return null; + return response.json(); +} + +async function allPages(path, field, token) { + const values = []; + for (let page = 1; ; page += 1) { + const separator = path.includes("?") ? "&" : "?"; + const response = await api( + `${path}${separator}per_page=100&page=${page}`, + token, + ); + values.push(...response[field]); + if (response[field].length < 100) return values; + } +} + +export function markdown(metadata, analysis, caches) { + const { baseline, candidate, interval, control } = analysis; + const lines = [ + "# Transfer overlap benchmark", + "", + `Temurin ${metadata.javaVersion}, ${analysis.pairs.length} runners x 2 paired observations, run ${metadata.runId}.`, + `Baseline \`${metadata.baselineRef}\` vs candidate \`${metadata.candidateRef}\` from \`${metadata.setupJavaRepository}\`.`, + "", + "## Verdict", + "", + `**${describeVerdict(analysis.verdict)}**`, + "", + `Paired difference (candidate - baseline): **${formatInterval(interval, { digits: 3 })}**.`, + `Permutation p-value: ${analysis.pValue.toFixed(3)}. Hodges-Lehmann shift: ${analysis.shiftSeconds.toFixed(3)}s.`, + `Harness noise floor: ${analysis.noiseFloorSeconds.toFixed(3)}s (median within-runner repeat spread).`, + "", + `A/A control (baseline against itself) reports **${control.verdict}** at ${formatInterval(control.interval, { digits: 3 })}. A healthy harness reports \`within-noise\` or \`inconclusive\` here; an \`improvement\` or \`regression\` means slot ordering is biasing results and the verdict above cannot be trusted.`, + "", + analysis.droppedRunners.length === 0 + ? "No runner was discarded for a stalled slot." + : `Discarded ${analysis.droppedRunners.length} runner(s) whose own arm disagreed with itself by more than ${analysis.stallThresholdSeconds.toFixed(3)}s, a threshold derived from the spread of the other runners: ${analysis.droppedRunners.map((entry) => `#${entry.sample} (${entry.repeatSpread.toFixed(3)}s)`).join(", ")}. A stalled slot lands on an arbitrary arm and would otherwise dominate the mean; the decision uses only within-arm spread, which carries no information about the effect.`, + "", + "## Arms", + "", + "| Arm | Runners | Mean (s) | Median (s) | SD (s) | MAD (s) | p95 (s) |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: |", + ]; + for (const summary of [baseline, candidate]) { + lines.push( + `| ${summary.arm} | ${summary.samples} | ${summary.meanSeconds.toFixed(3)} | ${summary.medianSeconds.toFixed(3)} | ${summary.standardDeviationSeconds?.toFixed(3) ?? "n/a"} | ${summary.madSeconds.toFixed(3)} | ${summary.p95Seconds.toFixed(3)} |`, + ); + } + lines.push( + "", + "All durations are measured inside the job with millisecond resolution. The Actions API reports step timestamps only to the nearest second, which is too coarse for effects of this size.", + "", + "## Paired samples", + "", + "| Runner | baseline slot 1 (s) | candidate slot 2 (s) | candidate slot 3 (s) | baseline slot 4 (s) | Paired delta (s) |", + "| ---: | ---: | ---: | ---: | ---: | ---: |", + ); + for (const pair of analysis.pairs) { + lines.push( + `| ${pair.sample} | ${pair.baselineSlots[0].toFixed(3)} | ${pair.candidateSlots[0].toFixed(3)} | ${pair.candidateSlots[1].toFixed(3)} | ${pair.baselineSlots[1].toFixed(3)} | ${pair.difference.toFixed(3)} |`, + ); + } + lines.push( + "", + "## Cache fixtures", + "", + "Both arms restore the same entries, so the stored blob cannot bias the comparison.", + "", + "| Cache | Size (MiB) |", + "| --- | ---: |", + ); + for (const cache of caches) { + lines.push( + `| ${cache.type} | ${(cache.sizeBytes / 1024 / 1024).toFixed(1)} |`, + ); + } + return `${lines.join("\n")}\n`; +} + +export async function main(env = process.env) { + requireEnv(env, [ + "GITHUB_REPOSITORY", + "GH_TOKEN", + "GITHUB_RUN_ID", + "BASELINE_REF", + "CANDIDATE_REF", + "SETUP_JAVA_REPOSITORY", + "JAVA_VERSION", + ]); + + const [owner, repo] = env.GITHUB_REPOSITORY.split("/"); + const token = env.GH_TOKEN; + const runId = env.GITHUB_RUN_ID; + const baselineRef = env.BASELINE_REF; + const candidateRef = env.CANDIDATE_REF; + const setupJavaRepository = env.SETUP_JAVA_REPOSITORY; + if (!owner || !repo) { + throw new Error( + `GITHUB_REPOSITORY must be owner/repo, got "${env.GITHUB_REPOSITORY}"`, + ); + } + + const rows = parseSamples(await readSampleFiles()); + const analysis = analyze(rows); + if (analysis.pairs.length === 0) { + throw new Error("No complete ABBA samples were collected"); + } + + const cacheEntries = await allPages( + `/repos/${owner}/${repo}/actions/caches`, + "actions_caches", + token, + ); + + // Both arms restore this single entry, so the stored blob cannot bias the + // comparison. The wrapper entry is optional: a baseline that predates wrapper + // caching simply never restores it. + const benchmarkId = `transfer-overlap-${runId}`; + const expected = [ + { + type: "maven-dependencies", + key: `setup-java-Linux-x64-maven-${hashFilesSingle(`${benchmarkId}\n`)}`, + required: true, + }, + { + type: "maven-wrapper", + key: `setup-java-Linux-x64-maven-wrapper-${hashFilesSingle( + `wrapperVersion=overlap\n# benchmark-id=${benchmarkId}\n`, + )}`, + required: false, + }, + ]; + + const caches = []; + for (const item of expected) { + const entry = cacheEntries.find((cache) => cache.key === item.key); + if (!entry) { + if (item.required) { + throw new Error(`Expected cache not found: ${item.key}`); + } + continue; + } + caches.push({ + type: item.type, + id: entry.id, + key: item.key, + sizeBytes: entry.size_in_bytes, + }); + } + + const metadata = { + repository: env.GITHUB_REPOSITORY, + runId, + javaVersion: env.JAVA_VERSION, + setupJavaRepository, + baselineRef, + candidateRef, + generatedAt: new Date().toISOString(), + }; + const report = markdown(metadata, analysis, caches); + + await mkdir("transfer-overlap-results", { recursive: true }); + await writeFile( + "transfer-overlap-results/results.json", + `${JSON.stringify({ metadata, analysis, caches }, null, 2)}\n`, + ); + await writeFile( + "transfer-overlap-results/results.csv", + `sample,arm,slot,seconds\n${rows + .map((row) => + [row.sample, row.arm, row.slot, row.seconds].map(csvValue).join(","), + ) + .join("\n")}\n`, + ); + await writeFile("transfer-overlap-results/summary.md", report); + await appendFile(env.GITHUB_STEP_SUMMARY, report); + + if (env.CLEANUP_CACHES === "true") { + for (const cache of caches) { + await api(`/repos/${owner}/${repo}/actions/caches/${cache.id}`, token, { + method: "DELETE", + }); + } + console.log(`Deleted ${caches.length} transfer-overlap benchmark caches`); + } +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + await main(); +} diff --git a/scripts/report-transfer-overlap.test.mjs b/scripts/report-transfer-overlap.test.mjs new file mode 100644 index 0000000..1dada10 --- /dev/null +++ b/scripts/report-transfer-overlap.test.mjs @@ -0,0 +1,157 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + analyze, + buildPairs, + markdown, + noiseFloor, + parseSamples, +} from "./report-transfer-overlap.mjs"; + +const csv = [ + '"1","baseline","1","2400"', + '"1","candidate","2","1900"', + '"1","candidate","3","1950"', + '"1","baseline","4","2500"', + '"2","baseline","1","3100"', + '"2","candidate","2","2600"', + '"2","candidate","3","2500"', + '"2","baseline","4","3000"', +].join("\n"); + +test("parses millisecond samples into seconds", () => { + const rows = parseSamples(csv); + assert.equal(rows.length, 8); + assert.deepEqual(rows[0], { + sample: 1, + arm: "baseline", + slot: 1, + seconds: 2.4, + }); +}); + +test("pairs the two slots per arm within a runner", () => { + const pairs = buildPairs(parseSamples(csv)); + assert.equal(pairs.length, 2); + assert.equal(pairs[0].baseline, 2.45); + assert.ok(Math.abs(pairs[0].candidate - 1.925) < 1e-9); + assert.ok(Math.abs(pairs[0].difference - -0.525) < 1e-9); + // Slot 4 minus slot 1 for the same implementation is a null measurement. + assert.ok(Math.abs(pairs[0].baselineRepeatDelta - 0.1) < 1e-9); +}); + +test("drops runners with incomplete ABBA slots", () => { + const partial = ['"3","baseline","1","2000"', '"3","candidate","2","1900"']; + const pairs = buildPairs(parseSamples([csv, ...partial].join("\n"))); + assert.equal(pairs.length, 2); + assert.ok(!pairs.some((pair) => pair.sample === 3)); +}); + +test("derives the noise floor from within-arm repeats", () => { + const pairs = buildPairs(parseSamples(csv)); + assert.ok(noiseFloor(pairs) > 0); + assert.equal(noiseFloor([]), 0); +}); + +test("reports a large consistent improvement", () => { + const rows = []; + for (let sample = 1; sample <= 10; sample += 1) { + // Each runner has its own offset, exactly the between-runner variance that + // pairing is meant to remove. + const offset = sample * 400; + rows.push( + `"${sample}","baseline","1","${3000 + offset}"`, + `"${sample}","candidate","2","${2000 + offset}"`, + `"${sample}","candidate","3","${2020 + offset}"`, + `"${sample}","baseline","4","${3020 + offset}"`, + ); + } + const analysis = analyze(parseSamples(rows.join("\n"))); + assert.equal(analysis.verdict, "improvement"); + assert.ok(analysis.interval.high < 0); + assert.ok(analysis.pValue < 0.01); + // The control compares baseline against itself and must find nothing. + assert.ok( + ["within-noise", "inconclusive"].includes(analysis.control.verdict), + ); +}); + +test("renders a report with a verdict and paired samples", () => { + const analysis = analyze(parseSamples(csv)); + const report = markdown( + { + runId: "1", + javaVersion: "17.0.19+10", + setupJavaRepository: "actions/setup-java", + baselineRef: "v4.8.0", + candidateRef: "main", + }, + analysis, + [{ arm: "baseline", type: "maven-dependencies", sizeBytes: 1024 * 1024 }], + ); + assert.match(report, /# Transfer overlap benchmark/); + assert.match(report, /## Verdict/); + assert.match(report, /95% CI/); + assert.match(report, /A\/A control/); + assert.match(report, /Harness noise floor/); +}); + +test("reports an A/A comparison as inconclusive", () => { + const jitter = [120, -90, 200, -160, 70, -110, 180, -140, 60, -50]; + const rows = []; + for (let sample = 1; sample <= 10; sample += 1) { + const offset = sample * 350; + const wobble = jitter[sample - 1]; + rows.push( + `"${sample}","baseline","1","${2500 + offset}"`, + `"${sample}","candidate","2","${2500 + offset + wobble}"`, + `"${sample}","candidate","3","${2500 + offset - wobble}"`, + `"${sample}","baseline","4","${2500 + offset}"`, + ); + } + const analysis = analyze(parseSamples(rows.join("\n"))); + assert.notEqual(analysis.verdict, "improvement"); + assert.notEqual(analysis.verdict, "regression"); +}); + +// Real data from run 30976592589, an A/A run with both refs set to `main`, so +// the true effect is exactly zero. Seven runners agreed to within 0.12s, but two +// had a single slot that stalled on the cache service (8.169s against a 3.286s +// sibling, and 10.030s against 4.449s). The mean of the paired differences put +// those stalls entirely on the candidate arm and the harness reported a 0.821s +// regression, 95% CI [0.119, 1.672]. +test("does not report an effect when two runners had a stalled slot", () => { + const observed = [ + [1, 1.544, 1.452, 1.541, 1.594], + [2, 3.632, 3.592, 3.513, 3.513], + [3, 2.796, 8.169, 3.286, 2.781], + [4, 3.659, 3.841, 3.708, 3.812], + [5, 3.665, 5.232, 5.319, 5.292], + [6, 3.663, 4.449, 10.03, 3.763], + [7, 4.089, 4.544, 5.454, 3.772], + [8, 1.397, 1.37, 1.384, 1.39], + [9, 1.452, 1.469, 1.475, 1.354], + [10, 1.59, 1.435, 1.412, 1.49], + ]; + const csv = observed + .map(([sample, b1, c2, c3, b4]) => + [ + `"${sample}","baseline","1","${b1 * 1000}"`, + `"${sample}","candidate","2","${c2 * 1000}"`, + `"${sample}","candidate","3","${c3 * 1000}"`, + `"${sample}","baseline","4","${b4 * 1000}"`, + ].join("\n"), + ) + .join("\n"); + const analysis = analyze(parseSamples(csv)); + assert.equal(analysis.verdict, "inconclusive"); + assert.ok(analysis.interval.low < 0 && analysis.interval.high > 0); + // The runners with a stalled slot are identified by their own arm disagreeing + // with itself, which carries no information about the effect. + assert.deepEqual( + analysis.droppedRunners.map((entry) => entry.sample).sort((a, b) => a - b), + [3, 5, 6, 7], + ); + assert.equal(analysis.control.verdict, "inconclusive"); +}); diff --git a/scripts/report.mjs b/scripts/report.mjs index 9344d54..a79be12 100644 --- a/scripts/report.mjs +++ b/scripts/report.mjs @@ -1,80 +1,144 @@ -import {createHash} from 'node:crypto'; -import {mkdir, writeFile, appendFile} from 'node:fs/promises'; -import {pathToFileURL} from 'node:url'; +import { createHash } from "node:crypto"; +import { appendFile, mkdir, writeFile } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; -const API_VERSION = '2022-11-28'; -const VERSIONS = ['v1', 'v2', 'v3', 'v4', 'v5.2', 'v5.6', 'main']; -const VERSION_IDS = new Map([ - ['v1', 'v1'], - ['v2', 'v2'], - ['v3', 'v3'], - ['v4', 'v4'], - ['v5.2', 'v52'], - ['v5.6', 'v56'], - ['main', 'main'] -]); -const JOB_PATTERN = - /^(v1|v2|v3|v4|v5\.2|v5\.6|main) \/ (cold|warm) \/ (zulu|temurin|microsoft) \/ (\d+)$/; +import { + analyzeAgainstReference, + parseSamples, + readSampleFiles, + requireEnv, +} from "./paired.mjs"; +import { classify, formatInterval, holmAdjust } from "./stats.mjs"; -export function sha256(value) { - return createHash('sha256').update(value).digest('hex'); -} +const API_VERSION = "2022-11-28"; +const RESULTS_DIR = "results"; +const REFERENCE = "main"; -export function hashFilesSingle(value) { - const contentHash = createHash('sha256').update(value).digest(); - return createHash('sha256').update(contentHash).digest('hex'); -} +// Ordered oldest to newest. The warm sweep measures them in this order and then +// in reverse, so each version's two slots sit symmetrically about the middle of +// the job. +// +// `comparable` records whether a version can be ranked against the reference at +// all. A version is comparable only when it does the same work as `main` from +// the same stored blob; anything else is measuring a difference in the workload +// rather than in the implementation, and a verdict on it would mislead. Those +// versions are still measured and published, but descriptively. +export const VERSIONS = [ + { + arm: "v1", + label: "v1.4.4", + caching: "none", + comparable: false, + reason: + "installs its own JDK and does no dependency caching, so its duration is a different workload rather than the same work done differently", + }, + { + arm: "v2", + label: "v2.5.1", + caching: "none", + comparable: false, + reason: + "its bundled cache client is rejected by the current cache service, so it restores nothing and skips the work the other versions spend their time on", + }, + { + arm: "v3", + label: "v3.14.1", + caching: "pom.xml", + comparable: false, + reason: + "predates `cache-dependency-path` and therefore restores its own cache entry; a stored blob's throughput is fixed for the life of the entry, so this difference is confounded with blob placement and pairing cannot remove it", + }, + { + arm: "v4", + label: "v4.8.0", + caching: "cache-dependency-path", + comparable: true, + }, + { + arm: "v52", + label: "v5.2.0", + caching: "cache-dependency-path", + comparable: true, + }, + { + arm: "v56", + label: "v5.6.0", + caching: "cache-dependency-path + wrapper", + comparable: true, + }, + { + arm: "main", + label: "main", + caching: "cache-dependency-path + wrapper", + comparable: true, + }, +]; -export function secondsBetween(start, end) { - if (!start || !end) return null; - return (Date.parse(end) - Date.parse(start)) / 1000; -} +const LABELS = new Map(VERSIONS.map((entry) => [entry.arm, entry.label])); -export function median(values) { - const sorted = values.filter(value => value !== null).sort((a, b) => a - b); - if (sorted.length === 0) return null; - const middle = Math.floor(sorted.length / 2); - return sorted.length % 2 - ? sorted[middle] - : (sorted[middle - 1] + sorted[middle]) / 2; -} +export const COMPARABLE_ARMS = VERSIONS.filter((entry) => entry.comparable).map( + (entry) => entry.arm, +); -export function parseBenchmarkJob(name) { - const match = name.match(JOB_PATTERN); - if (!match) return null; +// Holm's step-down correction is applied across the comparable family only. +// Every version in it is tested against the same reference in the same run, so +// without a correction the chance that at least one of them clears 0.05 by luck +// is far higher than 0.05. The descriptive versions are excluded because they +// carry no verdict to correct. +export function correctFamily(analysis) { + const adjusted = holmAdjust( + analysis.comparisons.map((comparison) => + comparison.isReference || !isComparable(comparison.arm) + ? null + : comparison.pValue, + ), + ); return { - version: match[1], - phase: match[2], - distribution: match[3], - iteration: Number(match[4]) + ...analysis, + comparisons: analysis.comparisons.map((comparison, index) => ({ + ...comparison, + adjustedPValue: adjusted[index], + verdict: comparison.isReference + ? "reference" + : !isComparable(comparison.arm) + ? "not comparable" + : classify(comparison.interval, { + noiseFloor: analysis.noiseFloorSeconds, + pValue: adjusted[index], + }), + })), }; } -function formatSeconds(value) { - return value === null ? 'n/a' : value.toFixed(1); +function isComparable(arm) { + return VERSIONS.find((entry) => entry.arm === arm)?.comparable === true; +} + +export function sha256(value) { + return createHash("sha256").update(value).digest("hex"); } -function formatMiB(bytes) { - return bytes === null ? 'n/a' : (bytes / 1024 / 1024).toFixed(1); +export function hashFilesSingle(value) { + const contentHash = createHash("sha256").update(value).digest(); + return createHash("sha256").update(contentHash).digest("hex"); } function csvValue(value) { - const text = value === null || value === undefined ? '' : String(value); - return `"${text.replaceAll('"', '""')}"`; + return `"${String(value ?? "").replaceAll('"', '""')}"`; } async function api(path, token, options = {}) { const response = await fetch(`https://api.github.com${path}`, { ...options, headers: { - Accept: 'application/vnd.github+json', + Accept: "application/vnd.github+json", Authorization: `Bearer ${token}`, - 'X-GitHub-Api-Version': API_VERSION, - ...options.headers - } + "X-GitHub-Api-Version": API_VERSION, + ...options.headers, + }, }); if (!response.ok) { - throw new Error(`${options.method ?? 'GET'} ${path}: ${response.status}`); + throw new Error(`${options.method ?? "GET"} ${path}: ${response.status}`); } if (response.status === 204) return null; return response.json(); @@ -83,292 +147,200 @@ async function api(path, token, options = {}) { async function allPages(path, field, token) { const values = []; for (let page = 1; ; page += 1) { - const separator = path.includes('?') ? '&' : '?'; + const separator = path.includes("?") ? "&" : "?"; const response = await api( `${path}${separator}per_page=100&page=${page}`, - token + token, ); values.push(...response[field]); if (response[field].length < 100) return values; } } -function stepDuration(job, predicate) { - const step = job.steps.find(predicate); - return step ? secondsBetween(step.started_at, step.completed_at) : null; -} - -function summarize(rows, caches) { - return VERSIONS.map(version => { - const versionRows = rows.filter(row => row.version === version); - const coldRows = versionRows.filter(row => row.phase === 'cold'); - const warmRows = versionRows.filter(row => row.phase === 'warm'); - const versionCaches = caches.filter(cache => cache.version === version); - const caseTotals = new Map(); - for (const cache of versionCaches) { - if (cache.sizeBytes === null) continue; - const key = `${cache.distribution}-${cache.iteration}`; - caseTotals.set(key, (caseTotals.get(key) ?? 0) + cache.sizeBytes); - } - return { - version, - coldSetupSeconds: median(coldRows.map(row => row.setupSeconds)), - warmSetupSeconds: median(warmRows.map(row => row.setupSeconds)), - coldBuildSeconds: median(coldRows.map(row => row.buildSeconds)), - warmBuildSeconds: median(warmRows.map(row => row.buildSeconds)), - postCacheSeconds: median(coldRows.map(row => row.postCacheSeconds)), - jobSeconds: median(versionRows.map(row => row.jobSeconds)), - estimatedBilledMinutes: versionRows.reduce( - (total, row) => total + Math.ceil(row.jobSeconds / 60), - 0 - ), - cacheMiBPerCase: - caseTotals.size === 0 - ? null - : median([...caseTotals.values()]) / 1024 / 1024 - }; - }); -} - -function markdown(metadata, rows, caches, summaries) { +export function markdown(metadata, analysis, caches) { + const { control } = analysis; const lines = [ - '# setup-java benchmark', - '', - `Spring PetClinic \`${metadata.petclinicRef.slice(0, 12)}\`, Java ${metadata.javaVersion}, run ${metadata.runId}.`, - '', - '| Version | Cold setup (s) | Warm setup (s) | Cold build (s) | Warm build (s) | Cold post-cache (s) | Cache/case (MiB) | Estimated billed minutes |', - '| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |' + "# setup-java version sweep", + "", + `${metadata.distribution} ${metadata.javaVersion}, ${analysis.runners.length} runners x 2 observations per version, run ${metadata.runId}.`, + "", + "Every version is measured in the same job, in the order v1..main followed by", + "main..v1. Differencing within a runner removes between-runner variance, which", + "on hosted runners is larger than the differences between versions. The mirrored", + "order places each version's two slots symmetrically about the middle of the job,", + "so drift across the job cancels.", + "", + `Durations are the warm restore path with a Maven cache already populated, measured inside the job at millisecond resolution.`, + "", + "## How `main` compares with each released version", + "", + "`main` is the newest code, so it is the thing under test and each released version is a baseline it is measured against. Every number and verdict below describes **`main`**: a negative difference and an `improvement` verdict mean `main` is faster than that version.", + "", + "Only versions that do the same work as `main` from the same stored cache entry are ranked. Holm's step-down correction is applied across this family, because `main` is tested against every one of them in the same run and without it the chance that one clears 0.05 by luck is far above 0.05.", + "", + `| Compared with | Caching | Median (s) | Mean (s) | MAD (s) | \`${REFERENCE}\` vs it (s) | 95% CI | Holm-adjusted p | Verdict for \`${REFERENCE}\` |`, + "| --- | --- | ---: | ---: | ---: | ---: | --- | ---: | --- |", ]; - for (const summary of summaries) { + const corrected = correctFamily(analysis); + const rankable = corrected.comparisons.filter((comparison) => + COMPARABLE_ARMS.includes(comparison.arm), + ); + const descriptive = corrected.comparisons.filter( + (comparison) => !COMPARABLE_ARMS.includes(comparison.arm), + ); + for (const comparison of rankable) { + const entry = VERSIONS.find((item) => item.arm === comparison.arm); + const summary = comparison.summary; + const diff = comparison.isReference + ? "reference" + : comparison.differenceSeconds.toFixed(3); + const ci = comparison.interval + ? `${comparison.interval.low.toFixed(3)} to ${comparison.interval.high.toFixed(3)}` + : "n/a"; + const p = + comparison.adjustedPValue === null + ? "n/a" + : comparison.adjustedPValue.toFixed(3); lines.push( - `| ${summary.version} | ${formatSeconds(summary.coldSetupSeconds)} | ${formatSeconds(summary.warmSetupSeconds)} | ${formatSeconds(summary.coldBuildSeconds)} | ${formatSeconds(summary.warmBuildSeconds)} | ${formatSeconds(summary.postCacheSeconds)} | ${summary.cacheMiBPerCase === null ? 'n/a' : summary.cacheMiBPerCase.toFixed(1)} | ${summary.estimatedBilledMinutes} |` + `| ${LABELS.get(comparison.arm)} | ${entry.caching} | ${summary.medianSeconds.toFixed(3)} | ${summary.meanSeconds.toFixed(3)} | ${summary.madSeconds.toFixed(3)} | ${diff} | ${ci} | ${p} | ${comparison.verdict} |`, ); } lines.push( - '', - 'Times and cache sizes are medians across distributions and iterations. Estimated billed minutes round each Linux job up to a whole minute; actual billing depends on the repository and runner plan.', - '', - '## Samples', - '', - '| Version | Cache | Distribution | Iteration | Setup (s) | Build (s) | Post-cache (s) | Job (s) | Conclusion |', - '| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | --- |' + "", + "## Measured but not ranked", + "", + "These versions are measured on the same runners and in the same order, but they do not do the same work as `main`, so a verdict on them would report a difference in the workload as though it were a difference in the implementation. Their durations are published to show what the ranked versions are spending their time on.", + "", + "| Version | Caching | Median (s) | Mean (s) | MAD (s) | `main` vs it (s) | Why it is not ranked |", + "| --- | --- | ---: | ---: | ---: | ---: | --- |", ); - for (const row of rows) { + for (const comparison of descriptive) { + const entry = VERSIONS.find((item) => item.arm === comparison.arm); + const summary = comparison.summary; lines.push( - `| ${row.version} | ${row.phase} | ${row.distribution} | ${row.iteration} | ${formatSeconds(row.setupSeconds)} | ${formatSeconds(row.buildSeconds)} | ${formatSeconds(row.postCacheSeconds)} | ${formatSeconds(row.jobSeconds)} | ${row.conclusion} |` + `| ${LABELS.get(comparison.arm)} | ${entry.caching} | ${summary.medianSeconds.toFixed(3)} | ${summary.meanSeconds.toFixed(3)} | ${summary.madSeconds.toFixed(3)} | ${comparison.differenceSeconds.toFixed(3)} | ${entry.reason} |`, ); } lines.push( - '', - '## Cache entries', - '', - '| Version | Distribution | Iteration | Cache | Size (MiB) |', - '| --- | --- | ---: | --- | ---: |' + "", + `Harness noise floor: ${analysis.noiseFloorSeconds.toFixed(3)}s (median spread between a version's own two slots on one runner). A difference smaller than this is reported as \`within-noise\`; one whose interval includes zero is reported as \`inconclusive\` rather than as a number that looks like a result.`, + "", + `A/A control (\`${REFERENCE}\` against itself) reports **${control.verdict}**${control.interval ? ` at ${formatInterval(control.interval)}` : ""}. A healthy harness reports \`within-noise\` or \`inconclusive\` here; anything else means slot ordering is biasing results and the table above cannot be trusted.`, + "", + analysis.droppedRunners.length === 0 + ? "No runner was discarded for a stalled slot." + : `Discarded ${analysis.droppedRunners.length} runner(s) whose slots for one version disagreed with each other by more than ${analysis.stallThresholdSeconds.toFixed(3)}s, a threshold derived from the spread of the other runners: ${analysis.droppedRunners.map((entry) => `#${entry.sample}`).join(", ")}. The version a stall lands on is arbitrary, and the decision uses only within-version spread, which carries no information about the differences between versions.`, + "", + "## Per-runner medians", + "", + `| Runner | ${VERSIONS.map((entry) => entry.label).join(" | ")} |`, + `| ---: | ${VERSIONS.map(() => "---:").join(" | ")} |`, + ); + for (const runner of analysis.runners) { + const cells = VERSIONS.map((entry) => { + const slots = runner.slots.get(entry.arm) ?? []; + if (slots.length === 0) return "n/a"; + return ((slots[0] + slots[1]) / 2).toFixed(3); + }); + lines.push(`| ${runner.sample} | ${cells.join(" | ")} |`); + } + lines.push( + "", + "## Caches", + "", + "Every ranked version restores the same Maven entry, so the stored blob cannot bias the comparison between them. v3 predates `cache-dependency-path` and keys on `pom.xml`, so it necessarily restores its own entry; a blob's throughput is fixed for the life of the entry and identical on every runner, which is why v3 is measured but not ranked.", + "", + "v1 and v2 have no caching at all, so they skip the dependency restore entirely and their durations are not comparable with the rest. They are measured to show what the caching versions are spending their time on, not to be ranked against them.", + "", + "| Cache | Size (MiB) |", + "| --- | ---: |", ); for (const cache of caches) { lines.push( - `| ${cache.version} | ${cache.distribution} | ${cache.iteration} | ${cache.type} | ${formatMiB(cache.sizeBytes)} |` + `| \`${cache.key}\` | ${(cache.sizeBytes / 1024 / 1024).toFixed(1)} |`, ); } - return `${lines.join('\n')}\n`; + return `${lines.join("\n")}\n`; } export async function main(env = process.env) { - const [owner, repo] = env.GITHUB_REPOSITORY.split('/'); + requireEnv(env, [ + "GITHUB_REPOSITORY", + "GH_TOKEN", + "GITHUB_RUN_ID", + "DISTRIBUTION", + "JAVA_VERSION", + ]); + const [owner, repo] = env.GITHUB_REPOSITORY.split("/"); const token = env.GH_TOKEN; const runId = env.GITHUB_RUN_ID; - const attempt = env.GITHUB_RUN_ATTEMPT; - const petclinicRef = env.PETCLINIC_REF; - if (!owner || !repo || !token || !runId || !attempt || !petclinicRef) { - throw new Error('Missing required GitHub Actions environment variables'); - } - - const [ - jobs, - cacheEntries, - wrapperResponse, - pomResponse, - mainCommit, - v1Ref, - v2Ref, - v3Ref, - v4Ref, - v52Ref, - v56Ref - ] = await Promise.all([ - allPages( - `/repos/${owner}/${repo}/actions/runs/${runId}/attempts/${attempt}/jobs`, - 'jobs', - token - ), - allPages(`/repos/${owner}/${repo}/actions/caches`, 'actions_caches', token), - api( - `/repos/spring-projects/spring-petclinic/contents/.mvn/wrapper/maven-wrapper.properties?ref=${petclinicRef}`, - token - ), - api( - `/repos/spring-projects/spring-petclinic/contents/pom.xml?ref=${petclinicRef}`, - token - ), - api('/repos/actions/setup-java/commits/main', token), - api('/repos/actions/setup-java/git/ref/tags/v1.4.4', token), - api('/repos/actions/setup-java/git/ref/tags/v2.5.1', token), - api('/repos/actions/setup-java/git/ref/tags/v3.14.1', token), - api('/repos/actions/setup-java/git/ref/tags/v4.8.0', token), - api('/repos/actions/setup-java/git/ref/tags/v5.2.0', token), - api('/repos/actions/setup-java/git/ref/tags/v5.6.0', token) - ]); - - const rows = jobs - .map(job => { - const identity = parseBenchmarkJob(job.name); - if (!identity) return null; - return { - ...identity, - conclusion: job.conclusion, - setupSeconds: stepDuration(job, step => - step.name.startsWith('Setup Java') - ), - buildSeconds: stepDuration(job, step => step.name === 'Build PetClinic'), - postCacheSeconds: stepDuration(job, step => - step.name.startsWith('Post Setup Java') - ), - jobSeconds: secondsBetween(job.started_at, job.completed_at) - }; - }) - .filter(Boolean) - .sort( - (a, b) => - a.version.localeCompare(b.version) || - a.phase.localeCompare(b.phase) || - a.distribution.localeCompare(b.distribution) || - a.iteration - b.iteration + if (!owner || !repo) { + throw new Error( + `GITHUB_REPOSITORY must be owner/repo, got "${env.GITHUB_REPOSITORY}"`, ); + } - const wrapperOriginal = Buffer.from(wrapperResponse.content, 'base64').toString( - 'utf8' + const rows = parseSamples(await readSampleFiles("version-sweep")); + const analysis = analyzeAgainstReference( + rows, + VERSIONS.map((entry) => entry.arm), + REFERENCE, ); - const pomOriginal = Buffer.from(pomResponse.content, 'base64').toString('utf8'); - const coldCases = rows.filter(row => row.phase === 'cold'); - const caches = []; - for (const row of coldCases) { - const versionId = VERSION_IDS.get(row.version); - const benchmarkId = `${versionId}-${row.distribution}-${row.iteration}-${runId}`; - const expected = - row.version === 'v1' || row.version === 'v2' - ? [] - : row.version === 'v3' - ? [ - { - type: 'maven-dependencies', - key: `setup-java-Linux-maven-${hashFilesSingle( - `${pomOriginal}\n` - )}` - } - ] - : [ - { - type: 'maven-dependencies', - key: `setup-java-Linux-x64-maven-${hashFilesSingle( - `${benchmarkId}\n` - )}` - } - ]; - if (row.version === 'v5.6' || row.version === 'main') { - expected.push({ - type: 'maven-wrapper', - key: `setup-java-Linux-x64-maven-wrapper-${hashFilesSingle( - `${wrapperOriginal}# benchmark-id=${benchmarkId}\n` - )}` - }); - } - for (const item of expected) { - const entry = cacheEntries.find(cache => cache.key === item.key); - caches.push({ - version: row.version, - distribution: row.distribution, - iteration: row.iteration, - type: item.type, - key: item.key, - id: entry?.id ?? null, - sizeBytes: entry?.size_in_bytes ?? null - }); - } + if (analysis.runners.length === 0) { + throw new Error("No runner measured every version twice"); } - const missingCaches = caches.filter(cache => cache.id === null); - if (missingCaches.length > 0) { - console.warn(`Could not find ${missingCaches.length} expected cache entries`); - } + const cacheEntries = await allPages( + `/repos/${owner}/${repo}/actions/caches`, + "actions_caches", + token, + ); + const caches = cacheEntries + .filter((entry) => entry.key.startsWith("setup-java-")) + .map((entry) => ({ + id: entry.id, + key: entry.key, + sizeBytes: entry.size_in_bytes, + })); const metadata = { repository: env.GITHUB_REPOSITORY, runId, - runAttempt: Number(attempt), + distribution: env.DISTRIBUTION, javaVersion: env.JAVA_VERSION, - iterations: Number(env.ITERATIONS), - petclinicRef, - setupJavaV1Ref: v1Ref.object.sha, - setupJavaV2Ref: v2Ref.object.sha, - setupJavaV3Ref: v3Ref.object.sha, - setupJavaV4Ref: v4Ref.object.sha, - setupJavaV52Ref: v52Ref.object.sha, - setupJavaV56Ref: v56Ref.object.sha, - setupJavaMainRefAtReport: mainCommit.sha, - generatedAt: new Date().toISOString() + generatedAt: new Date().toISOString(), }; - const summaries = summarize(rows, caches); - const report = markdown(metadata, rows, caches, summaries); - await mkdir('results', {recursive: true}); + const report = markdown(metadata, analysis, caches); + await mkdir(RESULTS_DIR, { recursive: true }); await writeFile( - 'results/results.json', - `${JSON.stringify({metadata, summaries, rows, caches}, null, 2)}\n` - ); - const csvHeaders = [ - 'version', - 'phase', - 'distribution', - 'iteration', - 'setup_seconds', - 'build_seconds', - 'post_cache_seconds', - 'job_seconds', - 'conclusion' - ]; - const csvRows = rows.map(row => - [ - row.version, - row.phase, - row.distribution, - row.iteration, - row.setupSeconds, - row.buildSeconds, - row.postCacheSeconds, - row.jobSeconds, - row.conclusion - ] - .map(csvValue) - .join(',') + `${RESULTS_DIR}/results.json`, + `${JSON.stringify({ metadata, analysis: { ...analysis, runners: undefined }, caches }, null, 2)}\n`, ); await writeFile( - 'results/results.csv', - `${csvHeaders.join(',')}\n${csvRows.join('\n')}\n` + `${RESULTS_DIR}/results.csv`, + `sample,version,slot,seconds\n${rows + .map((row) => + [row.sample, row.arm, row.slot, row.seconds].map(csvValue).join(","), + ) + .join("\n")}\n`, ); - await writeFile('results/summary.md', report); + await writeFile(`${RESULTS_DIR}/summary.md`, report); await appendFile(env.GITHUB_STEP_SUMMARY, report); - if (env.CLEANUP_CACHES === 'true') { - const ids = [...new Set(caches.map(cache => cache.id).filter(Boolean))]; - for (const id of ids) { - await api(`/repos/${owner}/${repo}/actions/caches/${id}`, token, { - method: 'DELETE' + if (env.CLEANUP_CACHES === "true") { + for (const cache of caches) { + await api(`/repos/${owner}/${repo}/actions/caches/${cache.id}`, token, { + method: "DELETE", }); } - console.log(`Deleted ${ids.length} benchmark cache entries after reporting`); + console.log(`Deleted ${caches.length} benchmark caches`); } } -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { await main(); } diff --git a/scripts/report.test.mjs b/scripts/report.test.mjs index d1190bf..77de817 100644 --- a/scripts/report.test.mjs +++ b/scripts/report.test.mjs @@ -1,87 +1,186 @@ -import assert from 'node:assert/strict'; -import test from 'node:test'; +import assert from "node:assert/strict"; +import test from "node:test"; +import { analyzeAgainstReference, parseSamples } from "./paired.mjs"; import { + COMPARABLE_ARMS, + VERSIONS, hashFilesSingle, - median, - parseBenchmarkJob, - secondsBetween, - sha256 -} from './report.mjs'; -import {parseFocusedJob} from './report-focused.mjs'; + markdown, + sha256, +} from "./report.mjs"; -test('hashes benchmark cache markers', () => { +const ARMS = VERSIONS.map((entry) => entry.arm); +const ORDER = [...ARMS, ...[...ARMS].reverse()]; + +// One runner's worth of the mirrored sweep. `speed` scales every slot so that a +// slow runner stays slow across all versions, which is what the pairing removes. +function sweep(sample, speed, perVersionMs, drift = 0) { + return ORDER.map((arm, index) => { + const slot = index + 1; + const elapsed = perVersionMs[arm] * speed + drift * index; + return `"${sample}","${arm}","${slot}","${Math.round(elapsed)}"`; + }).join("\n"); +} + +const flat = Object.fromEntries(ARMS.map((arm) => [arm, 3000])); + +test("hashes benchmark cache markers", () => { assert.equal( - sha256('benchmark\n'), - '8f8dbecfd77ab2386b49d723c6b2474f2c22c246805fa0f677bbaf6e4f7bbbfe' + sha256("benchmark\n"), + "8f8dbecfd77ab2386b49d723c6b2474f2c22c246805fa0f677bbaf6e4f7bbbfe", ); assert.equal( - hashFilesSingle('main-microsoft-1-30462682067\n'), - 'f0c1009aeb8d8582a73a81f3b0146467ed727211971e4052fccd9b7d60817d8f' + hashFilesSingle("main-microsoft-1-30462682067\n"), + "f0c1009aeb8d8582a73a81f3b0146467ed727211971e4052fccd9b7d60817d8f", ); }); -test('calculates medians', () => { - assert.equal(median([3, 1, 2]), 2); - assert.equal(median([4, 1, 2, 3]), 2.5); - assert.equal(median([null]), null); +test("keeps only runners that measured every version twice", () => { + const csv = [ + sweep(1, 1, flat), + // A runner that stopped after the first few slots must be discarded rather + // than contribute a difference computed from unbalanced slots. + '"2","v1","1","3000"', + '"2","v2","2","3000"', + ].join("\n"); + const analysis = analyzeAgainstReference(parseSamples(csv), ARMS, "main"); + assert.deepEqual( + analysis.runners.map((runner) => runner.sample), + [1], + ); }); -test('calculates step duration', () => { - assert.equal( - secondsBetween('2026-01-01T00:00:01Z', '2026-01-01T00:00:04.500Z'), - 3.5 +test("removes between-runner speed differences", () => { + // Runners differ by up to 3x, and every version is 500ms slower than main. + // Pairing within a runner must recover 500ms regardless of the spread. The + // difference is main minus the version, so main being faster reads negative. + const perVersion = { + ...flat, + v1: 3500, + v2: 3500, + v3: 3500, + v4: 3500, + v52: 3500, + v56: 3500, + }; + const csv = [1, 2, 3, 4, 5, 6] + .map((sample) => sweep(sample, 0.5 + sample * 0.5, perVersion)) + .join("\n"); + const analysis = analyzeAgainstReference(parseSamples(csv), ARMS, "main"); + const v4 = analysis.comparisons.find((entry) => entry.arm === "v4"); + assert.ok(Math.abs(v4.differenceSeconds + 1.125) < 0.001); + assert.equal(v4.verdict, "improvement"); +}); + +// main is the newest code. Reporting a released version as a `regression` for +// being slower than the code that succeeded it inverts what was measured, so +// the direction is pinned here rather than left to the reader of the table. +test("credits main when main is the faster one", () => { + const perVersion = { ...flat, v4: 4250, v52: 4250, v56: 4250 }; + const csv = [1, 2, 3, 4, 5, 6] + .map((sample) => sweep(sample, 0.5 + sample * 0.5, perVersion)) + .join("\n"); + const analysis = analyzeAgainstReference(parseSamples(csv), ARMS, "main"); + const v56 = analysis.comparisons.find((entry) => entry.arm === "v56"); + assert.ok( + v56.differenceSeconds < 0, + "main is faster, so the difference must be negative", + ); + assert.equal(v56.verdict, "improvement"); + const rendered = markdown( + { runId: "1", distribution: "temurin", javaVersion: "17" }, + analysis, + [], ); + assert.match(rendered, /Verdict for `main`/); + assert.doesNotMatch(rendered, /regression/); +}); + +test("cancels linear drift across the job", () => { + // Every version is identical, but the job gets steadily slower. The mirrored + // order must absorb that so no version is reported as different from main. + const csv = [1, 2, 3, 4, 5, 6] + .map((sample) => sweep(sample, 1, flat, 40)) + .join("\n"); + const analysis = analyzeAgainstReference(parseSamples(csv), ARMS, "main"); + for (const comparison of analysis.comparisons) { + assert.ok(Math.abs(comparison.differenceSeconds) < 1e-9); + } }); -test('parses benchmark job names', () => { - assert.deepEqual(parseBenchmarkJob('v1 / cold / zulu / 1'), { - version: 'v1', - phase: 'cold', - distribution: 'zulu', - iteration: 1 - }); - assert.deepEqual(parseBenchmarkJob('v2 / warm / microsoft / 3'), { - version: 'v2', - phase: 'warm', - distribution: 'microsoft', - iteration: 3 - }); - assert.deepEqual(parseBenchmarkJob('v3 / cold / temurin / 5'), { - version: 'v3', - phase: 'cold', - distribution: 'temurin', - iteration: 5 - }); - assert.deepEqual(parseBenchmarkJob('v4 / warm / microsoft / 2'), { - version: 'v4', - phase: 'warm', - distribution: 'microsoft', - iteration: 2 - }); - assert.deepEqual(parseBenchmarkJob('v5.2 / warm / microsoft / 1'), { - version: 'v5.2', - phase: 'warm', - distribution: 'microsoft', - iteration: 1 - }); - assert.deepEqual(parseBenchmarkJob('v5.6 / cold / temurin / 3'), { - version: 'v5.6', - phase: 'cold', - distribution: 'temurin', - iteration: 3 - }); - assert.equal(parseBenchmarkJob('Report'), null); +test("reports the reference against itself without a verdict", () => { + const csv = [1, 2, 3, 4].map((sample) => sweep(sample, 1, flat)).join("\n"); + const analysis = analyzeAgainstReference(parseSamples(csv), ARMS, "main"); + const reference = analysis.comparisons.find((entry) => entry.arm === "main"); + assert.equal(reference.verdict, "reference"); + assert.equal(reference.interval, null); }); -test('parses focused benchmark job names', () => { - assert.deepEqual(parseFocusedJob('main / warm / 10'), { - version: 'main', - sample: 10 - }); - assert.deepEqual(parseFocusedJob('v4 / warm / 2'), { - version: 'v4', - sample: 2 - }); - assert.equal(parseFocusedJob('Seed main caches'), null); +test("renders a ranked table, a not-ranked table and per-runner medians", () => { + const csv = [1, 2, 3, 4] + .map((sample) => sweep(sample, 1 + sample * 0.1, flat)) + .join("\n"); + const analysis = analyzeAgainstReference(parseSamples(csv), ARMS, "main"); + const report = markdown( + { runId: "1", distribution: "temurin", javaVersion: "17" }, + analysis, + [{ key: "setup-java-Linux-x64-maven-abc", sizeBytes: 60 * 1024 * 1024 }], + ); + assert.match(report, /# setup-java version sweep/); + assert.match(report, /## How `main` compares with each released version/); + assert.match(report, /## Measured but not ranked/); + assert.match(report, /Holm-adjusted p/); + assert.match(report, /A\/A control/); + assert.match(report, /Harness noise floor/); + assert.match(report, /## Per-runner medians/); + assert.match(report, /setup-java-Linux-x64-maven-abc/); +}); + +test("ranks only versions that do the same work from the same blob", () => { + // v1 and v2 restore nothing and v3 restores its own cache entry, so none of + // them measures the same work as main. Ranking them would report a difference + // in the workload as though it were a difference in the implementation. + assert.deepEqual(COMPARABLE_ARMS, ["v4", "v52", "v56", "main"]); + for (const arm of ["v1", "v2", "v3"]) { + const entry = VERSIONS.find((item) => item.arm === arm); + assert.equal(entry.comparable, false); + assert.ok(entry.reason.length > 0); + } +}); + +test("never gives a verdict to a version it cannot rank", () => { + // v3 is made dramatically slower, which on a ranked version would read as a + // regression. Because its cache entry is its own, that difference is + // confounded with blob placement and must not become a verdict. + const perVersion = { ...flat, v3: 9000 }; + const csv = [1, 2, 3, 4, 5, 6] + .map((sample) => sweep(sample, 1, perVersion)) + .join("\n"); + const analysis = analyzeAgainstReference(parseSamples(csv), ARMS, "main"); + const report = markdown( + { runId: "1", distribution: "temurin", javaVersion: "17" }, + analysis, + [], + ); + const [ranked, notRanked] = report.split("## Measured but not ranked"); + assert.doesNotMatch(ranked, /v3\.14\.1/); + assert.match(notRanked, /v3\.14\.1/); + assert.doesNotMatch(report, /\| v3\.14\.1 \|[^\n]*regression/); +}); + +test("applies Holm correction across the ranked family", () => { + // Three versions are tested against main in one run. A raw p-value near the + // boundary must not survive the correction as a verdict. + const perVersion = { ...flat, v4: 3040, v52: 3040, v56: 3040 }; + const csv = [1, 2, 3, 4, 5, 6, 7, 8] + .map((sample) => sweep(sample, 1 + sample * 0.05, perVersion)) + .join("\n"); + const analysis = analyzeAgainstReference(parseSamples(csv), ARMS, "main"); + const report = markdown( + { runId: "1", distribution: "temurin", javaVersion: "17" }, + analysis, + [], + ); + assert.match(report, /Holm's step-down correction/); }); diff --git a/scripts/stats.mjs b/scripts/stats.mjs new file mode 100644 index 0000000..ce5e203 --- /dev/null +++ b/scripts/stats.mjs @@ -0,0 +1,247 @@ +// Statistical helpers shared by the benchmark reports. +// +// Benchmark arms run on ephemeral hosted runners, so every measurement carries +// substantial between-runner noise. Point estimates alone cannot tell a real +// improvement from that noise, so every comparison here is reported with an +// interval and an explicit verdict. + +const BOOTSTRAP_ITERATIONS = 10000; +const DEFAULT_CONFIDENCE = 0.95; + +// Deterministic PRNG so a given set of samples always produces the same +// interval. Reports are compared across runs and must not wobble because the +// resampler drew different numbers. +export function createRandom(seed = 0x9e3779b9) { + let state = seed >>> 0; + return function random() { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +export function mean(values) { + if (values.length === 0) return null; + return values.reduce((total, value) => total + value, 0) / values.length; +} + +export function median(values) { + return quantile(values, 0.5); +} + +export function quantile(values, percentile) { + if (values.length === 0) return null; + const sorted = [...values].sort((a, b) => a - b); + const position = (sorted.length - 1) * percentile; + const lower = Math.floor(position); + const upper = Math.ceil(position); + if (lower === upper) return sorted[lower]; + return sorted[lower] + (sorted[upper] - sorted[lower]) * (position - lower); +} + +export function standardDeviation(values) { + if (values.length < 2) return null; + const average = mean(values); + const variance = + values.reduce((total, value) => total + (value - average) ** 2, 0) / + (values.length - 1); + return Math.sqrt(variance); +} + +// Median absolute deviation, scaled to be comparable with a standard deviation +// for normally distributed data. Robust to the occasional runner that stalls. +export function medianAbsoluteDeviation(values) { + if (values.length === 0) return null; + const center = median(values); + return 1.4826 * median(values.map((value) => Math.abs(value - center))); +} + +function resample(values, random) { + const drawn = new Array(values.length); + for (let index = 0; index < values.length; index += 1) { + drawn[index] = values[Math.floor(random() * values.length)]; + } + return drawn; +} + +// Percentile bootstrap interval for an arbitrary statistic of one sample. +export function bootstrapInterval(values, statistic = median, options = {}) { + const { + iterations = BOOTSTRAP_ITERATIONS, + confidence = DEFAULT_CONFIDENCE, + seed, + } = options; + if (values.length === 0) return null; + const random = createRandom(seed); + const estimates = new Array(iterations); + for (let index = 0; index < iterations; index += 1) { + estimates[index] = statistic(resample(values, random)); + } + const alpha = (1 - confidence) / 2; + return { + estimate: statistic(values), + low: quantile(estimates, alpha), + high: quantile(estimates, 1 - alpha), + confidence, + }; +} + +// Bootstrap interval for the difference between two independent samples. +export function differenceInterval( + treatment, + baseline, + statistic = median, + options = {}, +) { + const { + iterations = BOOTSTRAP_ITERATIONS, + confidence = DEFAULT_CONFIDENCE, + seed, + } = options; + if (treatment.length === 0 || baseline.length === 0) return null; + const random = createRandom(seed); + const estimates = new Array(iterations); + for (let index = 0; index < iterations; index += 1) { + estimates[index] = + statistic(resample(treatment, random)) - + statistic(resample(baseline, random)); + } + const alpha = (1 - confidence) / 2; + return { + estimate: statistic(treatment) - statistic(baseline), + low: quantile(estimates, alpha), + high: quantile(estimates, 1 - alpha), + confidence, + }; +} + +// Bootstrap interval for the mean of within-runner paired differences. This is +// the estimator to prefer whenever both arms were measured on the same runner, +// because it cancels the between-runner variance that dominates hosted CI. +// The statistic is the mean of the paired differences. It is not robust to a +// runner whose slot stalled on the cache service, but switching to a median +// costs almost all the power at the sample sizes these workflows run: on six +// runners agreeing on a 12.5s effect, a sign-flip test on the median returns +// p = 0.13. Robustness is provided instead by discarding stalled runners before +// they reach this function, which is a decision made on within-arm spread and so +// cannot bias the effect. +export function pairedInterval(differences, options = {}) { + const { + iterations = BOOTSTRAP_ITERATIONS, + confidence = DEFAULT_CONFIDENCE, + seed, + } = options; + if (differences.length === 0) return null; + const random = createRandom(seed); + const estimates = new Array(iterations); + for (let index = 0; index < iterations; index += 1) { + estimates[index] = mean(resample(differences, random)); + } + const alpha = (1 - confidence) / 2; + return { + estimate: mean(differences), + low: quantile(estimates, alpha), + high: quantile(estimates, 1 - alpha), + confidence, + }; +} + +// Two-sided permutation test on the mean of paired differences. Under the null +// hypothesis the sign of each pair is arbitrary, so we resample signs. It tests +// the same statistic the interval estimates, and unlike the percentile bootstrap +// it is exact under the null, which is why a verdict requires it to agree. +export function pairedPermutationTest(differences, options = {}) { + const { iterations = BOOTSTRAP_ITERATIONS, seed } = options; + if (differences.length === 0) return null; + const random = createRandom(seed); + const observed = Math.abs(mean(differences)); + let atLeastAsExtreme = 0; + for (let index = 0; index < iterations; index += 1) { + const flipped = differences.map((value) => + random() < 0.5 ? -value : value, + ); + if (Math.abs(mean(flipped)) >= observed) atLeastAsExtreme += 1; + } + // Add-one correction keeps the p-value strictly positive. + return (atLeastAsExtreme + 1) / (iterations + 1); +} + +// Holm's step-down correction controls the family-wise error rate when one +// report makes several comparisons against the same reference. +export function holmAdjust(pValues) { + const adjusted = new Array(pValues.length).fill(null); + const ordered = pValues + .map((pValue, index) => ({ pValue, index })) + .filter(({ pValue }) => pValue !== null) + .sort((a, b) => a.pValue - b.pValue); + let maximum = 0; + ordered.forEach(({ pValue, index }, rank) => { + maximum = Math.max(maximum, Math.min(1, pValue * (ordered.length - rank))); + adjusted[index] = maximum; + }); + return adjusted; +} + +// Hodges-Lehmann shift estimate: the median of all pairwise differences. More +// robust than a difference of medians when samples are small and quantized. +export function hodgesLehmann(treatment, baseline) { + if (treatment.length === 0 || baseline.length === 0) return null; + const differences = []; + for (const treatmentValue of treatment) { + for (const baselineValue of baseline) { + differences.push(treatmentValue - baselineValue); + } + } + return median(differences); +} + +// Turn an interval into a decision. An interval that straddles zero means the +// benchmark did not resolve the effect, and an effect smaller than the harness +// noise floor is not trustworthy even when the interval excludes zero. +// A verdict requires the bootstrap interval and the permutation test to agree. +// The percentile bootstrap is approximate on ten paired observations, while the +// sign-flip permutation test is exact under the null, so where they disagree the +// interval is the one that is wrong. Passing no p-value keeps the interval-only +// behaviour for callers that have no test to offer. +export function classify(interval, options = {}) { + const { + noiseFloor = 0, + lowerIsBetter = true, + pValue = null, + alpha = 0.05, + } = options; + if (!interval) return "unknown"; + const { low, high, estimate } = interval; + if (low <= 0 && high >= 0) return "inconclusive"; + if (pValue !== null && pValue >= alpha) return "inconclusive"; + if (Math.abs(estimate) < noiseFloor) return "within-noise"; + const improved = lowerIsBetter ? estimate < 0 : estimate > 0; + return improved ? "improvement" : "regression"; +} + +// The label follows the interval's own confidence level rather than assuming +// 95%, so a caller that widens or narrows the interval cannot end up publishing +// a number under the wrong label. +export function formatInterval(interval, { digits = 1, unit = "s" } = {}) { + if (!interval) return "n/a"; + const { estimate, low, high, confidence = DEFAULT_CONFIDENCE } = interval; + const level = Number((confidence * 100).toFixed(2)); + return `${estimate.toFixed(digits)}${unit} (${level}% CI ${low.toFixed(digits)} to ${high.toFixed(digits)})`; +} + +export function describeVerdict(verdict) { + switch (verdict) { + case "improvement": + return "Faster — the interval excludes zero and clears the noise floor."; + case "regression": + return "Slower — the interval excludes zero and clears the noise floor."; + case "within-noise": + return "No usable signal — the effect is smaller than the harness noise floor."; + case "inconclusive": + return "Inconclusive — the interval includes zero or the permutation test does not agree; collect more samples."; + default: + return "Unknown."; + } +} diff --git a/scripts/stats.test.mjs b/scripts/stats.test.mjs new file mode 100644 index 0000000..17a8bd2 --- /dev/null +++ b/scripts/stats.test.mjs @@ -0,0 +1,158 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + bootstrapInterval, + classify, + createRandom, + differenceInterval, + formatInterval, + holmAdjust, + hodgesLehmann, + mean, + median, + medianAbsoluteDeviation, + pairedInterval, + pairedPermutationTest, + quantile, + standardDeviation, +} from "./stats.mjs"; + +test("summarises central tendency and spread", () => { + assert.equal(mean([1, 2, 3, 4]), 2.5); + assert.equal(median([3, 1, 2]), 2); + assert.equal(median([4, 1, 2, 3]), 2.5); + assert.equal(quantile([1, 2, 3, 4, 5], 0.5), 3); + assert.equal(standardDeviation([2, 2, 2]), 0); + assert.equal(standardDeviation([1]), null); + assert.equal(medianAbsoluteDeviation([5, 5, 5]), 0); + assert.equal(mean([]), null); + assert.equal(median([]), null); +}); + +test("resampling is deterministic across runs", () => { + const values = [1.2, 1.4, 1.1, 1.9, 1.3, 1.5, 1.2, 1.8]; + const first = bootstrapInterval(values, median, { iterations: 500, seed: 7 }); + const second = bootstrapInterval(values, median, { + iterations: 500, + seed: 7, + }); + assert.deepEqual(first, second); + + const random = createRandom(42); + const draws = [random(), random(), random()]; + const replayed = createRandom(42); + assert.deepEqual(draws, [replayed(), replayed(), replayed()]); +}); + +test("bootstrap interval brackets the point estimate", () => { + const values = [2.0, 2.1, 2.2, 2.3, 2.4, 2.5]; + const interval = bootstrapInterval(values, median, { + iterations: 2000, + seed: 11, + }); + assert.ok(interval.low <= interval.estimate); + assert.ok(interval.high >= interval.estimate); + assert.equal(interval.confidence, 0.95); +}); + +test("detects a real effect and rejects a null effect", () => { + // A large, consistent within-runner improvement must be reported. + const realEffect = Array.from( + { length: 12 }, + (_, index) => -1 + index * 0.01, + ); + const realInterval = pairedInterval(realEffect, { + iterations: 2000, + seed: 3, + }); + assert.equal(classify(realInterval, { noiseFloor: 0.05 }), "improvement"); + assert.ok( + pairedPermutationTest(realEffect, { iterations: 2000, seed: 3 }) < 0.01, + ); + + // Differences that merely alternate around zero must not be. + const noEffect = [0.4, -0.3, 0.2, -0.5, 0.1, -0.2, 0.35, -0.4]; + const nullInterval = pairedInterval(noEffect, { iterations: 2000, seed: 3 }); + assert.equal(classify(nullInterval, { noiseFloor: 0.05 }), "inconclusive"); + assert.ok( + pairedPermutationTest(noEffect, { iterations: 2000, seed: 3 }) > 0.1, + ); +}); + +test("suppresses effects smaller than the noise floor", () => { + const tiny = Array.from({ length: 40 }, () => -0.02); + const interval = pairedInterval(tiny, { iterations: 1000, seed: 5 }); + // The interval excludes zero because the samples are identical, but the + // effect is far below what the harness can resolve. + assert.equal(classify(interval, { noiseFloor: 0.25 }), "within-noise"); + assert.equal(classify(interval, { noiseFloor: 0.001 }), "improvement"); +}); + +test("classifies direction and missing intervals", () => { + assert.equal(classify({ low: 0.2, high: 0.8, estimate: 0.5 }), "regression"); + assert.equal( + classify({ low: -0.8, high: -0.2, estimate: -0.5 }), + "improvement", + ); + assert.equal( + classify({ low: 0.2, high: 0.8, estimate: 0.5 }, { lowerIsBetter: false }), + "improvement", + ); + assert.equal(classify(null), "unknown"); +}); + +test("estimates a shift with Hodges-Lehmann", () => { + assert.equal(hodgesLehmann([3, 4, 5], [1, 2, 3]), 2); + assert.equal(hodgesLehmann([], [1]), null); +}); + +// Regression test built from two real back-to-back runs of the previous +// harness against an unchanged actions/setup-java@v4.8.0. Because the code was +// identical, the true effect is exactly zero, yet the old unpaired design +// separated the two samples cleanly. This test pins that failure so nobody +// reintroduces unpaired sampling: better statistics alone cannot rescue it, +// because the samples really do differ once between-runner drift is baked in. +test("unpaired sampling produces a false positive on identical code", () => { + const runOne = [2, 1, 3, 1, 1, 2, 3, 2, 2, 2]; + const runTwo = [3, 4, 2, 4, 4, 3, 4, 3, 3, 1]; + const interval = differenceInterval(runTwo, runOne, median, { + iterations: 4000, + seed: 13, + }); + // The interval excludes zero even though both samples measure the same code. + assert.ok(interval.low > 0 || interval.high < 0); + assert.equal(classify(interval, { noiseFloor: 0 }), "regression"); + + // The only defence left for unpaired data is a noise floor calibrated from + // observed run-to-run drift, which suppresses the spurious verdict. + assert.equal(classify(interval, { noiseFloor: 1.5 }), "within-noise"); +}); + +test("formatInterval labels the interval's own confidence level", () => { + assert.equal( + formatInterval( + { estimate: -1.25, low: -2, high: -0.5, confidence: 0.95 }, + { digits: 2 }, + ), + "-1.25s (95% CI -2.00 to -0.50)", + ); + // A caller that widens the interval must not have it published as 95%. + assert.equal( + formatInterval( + { estimate: -1.25, low: -3, high: 0.5, confidence: 0.99 }, + { digits: 2 }, + ), + "-1.25s (99% CI -3.00 to 0.50)", + ); + // Intervals built before `confidence` was carried still read as 95%. + assert.equal( + formatInterval({ estimate: 0, low: -1, high: 1 }), + "0.0s (95% CI -1.0 to 1.0)", + ); +}); + +test("Holm adjustment controls a family of p-values", () => { + assert.deepEqual(holmAdjust([0.01, 0.04, 0.2]), [0.03, 0.08, 0.2]); + assert.deepEqual(holmAdjust([null, 0.01, 0.2]), [null, 0.02, 0.2]); +}); diff --git a/scripts/transfer-overlap.sh b/scripts/transfer-overlap.sh new file mode 100755 index 0000000..3e9b1ea --- /dev/null +++ b/scripts/transfer-overlap.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +# Fixture and cache-identity helpers for the transfer overlap benchmark. + +set -euo pipefail + +command=${1:?command is required} + +dependency_fixture="$HOME/.m2/repository/transfer-overlap-benchmark/dependencies.bin" +wrapper_fixture="$HOME/.m2/wrapper/dists/transfer-overlap-benchmark/wrapper.bin" + +write_identity() { + local benchmark_id=$1 + printf '%s\n' "$benchmark_id" > .transfer-overlap-cache-key + mkdir -p .mvn/wrapper + printf 'wrapperVersion=overlap\n# benchmark-id=%s\n' "$benchmark_id" \ + > .mvn/wrapper/maven-wrapper.properties +} + +case "$command" in + prepare) + write_identity "${2:?benchmark id is required}" + ;; + seed-fixtures) + mkdir -p "$(dirname "$dependency_fixture")" "$(dirname "$wrapper_fixture")" + head -c "$((DEPENDENCY_FIXTURE_MIB * 1024 * 1024))" /dev/urandom > "$dependency_fixture" + head -c "$((WRAPPER_FIXTURE_MIB * 1024 * 1024))" /dev/urandom > "$wrapper_fixture" + ;; + reset) + # Each measured slot must restore into an empty tree, otherwise the second + # and later restores in a job would extract over files that already exist + # and report an artificially low duration. + rm -rf "$HOME/.m2" + write_identity "${2:?benchmark id is required}" + ;; + verify) + arm=${2:?arm is required} + check_fixture() { + local label=$1 path=$2 expected=$3 + if [ ! -f "$path" ]; then + echo "$label fixture for $arm is missing at $path;" \ + "the seeded cache was not restored" >&2 + exit 1 + fi + local actual + actual=$(wc -c < "$path") + if [ "$actual" -ne "$expected" ]; then + echo "$label fixture for $arm is $actual bytes, expected $expected" >&2 + exit 1 + fi + } + check_fixture Dependency "$dependency_fixture" \ + "$((DEPENDENCY_FIXTURE_MIB * 1024 * 1024))" + # Only the candidate arm is expected to maintain a separate wrapper cache; + # a baseline that also produces one is fine, so this check is arm-specific + # and non-fatal when the fixture is simply absent for the baseline. + if [ "$arm" = "candidate" ]; then + check_fixture Wrapper "$wrapper_fixture" \ + "$((WRAPPER_FIXTURE_MIB * 1024 * 1024))" + fi + ;; + *) + echo "Unsupported command: $command" >&2 + exit 1 + ;; +esac diff --git a/scripts/version-sweep.sh b/scripts/version-sweep.sh new file mode 100755 index 0000000..b3ce813 --- /dev/null +++ b/scripts/version-sweep.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# Cache-identity helpers for the setup-java version sweep. +# +# Markers are appended idempotently so that repeating a slot does not change the +# hashed files. The warm sweep depends on that: every slot must derive the same +# cache key, otherwise it would miss and measure a download instead of a restore. + +set -euo pipefail + +command=${1:?command is required} +benchmark_id=${2:?benchmark id is required} + +# Spring PetClinic is checked out below the benchmark repository rather than over +# it, because its build runs a nohttp check across the whole basedir and would +# otherwise lint these scripts. +project_dir=${PROJECT_DIR:-petclinic} +wrapper_properties="$project_dir/.mvn/wrapper/maven-wrapper.properties" + +append_once() { + local file=$1 line=$2 + if [ ! -f "$file" ] || ! grep -qF -- "$line" "$file"; then + printf '%s\n' "$line" >> "$file" + fi +} + +write_identity() { + # Versions from v4 onwards select their cache key with cache-dependency-path, + # so a single file drives all of them. + printf '%s\n' "$benchmark_id" > .benchmark-cache-key + # v5.6 and main additionally maintain a wrapper cache keyed on the wrapper + # properties. The real contents must survive because the seed job builds. + append_once "$wrapper_properties" "# benchmark-id=$benchmark_id" + # v3 predates cache-dependency-path and hashes pom.xml. A comment in the + # epilog is well-formed XML and leaves the build unaffected. + append_once "$project_dir/pom.xml" "" +} + +case "$command" in + prepare) + write_identity + ;; + reset) + # Every measured slot must restore into an empty tree, otherwise later slots + # would extract over files an earlier slot left behind and report an + # artificially low duration. + rm -rf "$HOME/.m2" + write_identity + ;; + *) + echo "Unsupported command: $command" >&2 + exit 1 + ;; +esac