From e613969191bfd17899f8a871e56554d787bb8e8e Mon Sep 17 00:00:00 2001 From: Stephan Linz Date: Fri, 11 Oct 2024 16:45:37 +0200 Subject: [PATCH] chore(ci): support fine graded multi-stages Use our own reusable GitHub workflows for Docker from: - tiacsys/github-workflows/.github/workflows/docker-stages.yml@main - tiacsys/github-workflows/.github/workflows/docker-retention.yml@main Signed-off-by: Stephan Linz --- .github/platforms.json | 10 ++ .github/stages.json | 7 + .github/workflows/clean-buildcache.yml | 14 ++ .github/workflows/docker-publish.yml | 238 ++----------------------- .github/workflows/pr-close-policy.yml | 32 +--- .github/workflows/retention-policy.yml | 35 +--- 6 files changed, 61 insertions(+), 275 deletions(-) create mode 100644 .github/platforms.json create mode 100644 .github/stages.json create mode 100644 .github/workflows/clean-buildcache.yml diff --git a/.github/platforms.json b/.github/platforms.json new file mode 100644 index 0000000..860299a --- /dev/null +++ b/.github/platforms.json @@ -0,0 +1,10 @@ +{ + "include": [ + { "platform": "linux/amd64" }, + { "platform": "linux/arm/v7" }, + { "platform": "linux/arm64/v8" }, + { "platform": "linux/riscv64" }, + { "platform": "linux/ppc64le" }, + { "platform": "linux/s390x" } + ] +} diff --git a/.github/stages.json b/.github/stages.json new file mode 100644 index 0000000..7c7a9a5 --- /dev/null +++ b/.github/stages.json @@ -0,0 +1,7 @@ +{ + "include": [ + { "stage": "base" }, + { "stage": "asdf" }, + { "stage": "push", "ghr-free-disk": false } + ] +} diff --git a/.github/workflows/clean-buildcache.yml b/.github/workflows/clean-buildcache.yml new file mode 100644 index 0000000..6243992 --- /dev/null +++ b/.github/workflows/clean-buildcache.yml @@ -0,0 +1,14 @@ +name: Clean Buildcache + +on: + workflow_dispatch: # Manually on button click + +jobs: + setup: + name: Setup + + # Delete the "buildcache-*" tagged Docker images on demand. + uses: tiacsys/github-workflows/.github/workflows/docker-mr-proper.yml@main + with: + delete-tags: buildcache-* + secrets: inherit # pass all secrets diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index b0bec57..dc70316 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -1,13 +1,5 @@ name: Docker -# Based on GitHub actions starter workflow for Docker publishing: -# https://github.com/actions/starter-workflows/blob/main/ci/docker-publish.yml - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - on: schedule: # every day at 00:15 UTC @@ -23,220 +15,18 @@ on: branches: - 'main' -env: - # Use docker.io for Docker Hub if empty - REGISTRY: ghcr.io - # github.repository as / - IMAGE_NAME: ${{ github.repository }} - # repository base URL - REPO_README: ${{ github.server_url }}/${{ github.repository }}/blob/main/README.rst - jobs: - build-linux: - name: Linux - - - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - platform: - # Distribute build across multiple runners, use a matrix strategy. - # https://docs.docker.com/build/ci/github-actions/multi-platform - - linux/amd64 - - linux/arm/v7 - - linux/arm64/v8 - - linux/riscv64 - - linux/ppc64le - - linux/s390x - - steps: - # Free Disk Space on Ubuntu runners, clear up to 35 GB disk space - # https://github.com/jlumbroso/free-disk-space - - name: Free disk space - uses: jlumbroso/free-disk-space@v1.3.1 - with: - # Tool cache: Saved 8.3GiB - tool-cache: true - # Android library: Saved 7.5GiB - android: true - # .NET runtime: Saved 1.6GiB - dotnet: true - # Haskell runtime: Saved 5.4GiB - haskell: true - # Large misc. packages: Saved 4.8GiB - large-packages: true - # Docker images: Saved 3.2GiB - docker-images: true - # Swap storage: Saved 4.0GiB - swap-storage: true - - - name: Prepare GitHub runner environment - run: | - platform=${{ matrix.platform }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Get Recent Commit Authors - id: recent-commits-authors - run: | - jsonlist=$(git log --format="%an <%ae>" | \ - sort -u | tr '\n' '\000' | \ - jq -R -s -c 'split("\u0000")') - echo "jsonlist=$jsonlist" >> $GITHUB_OUTPUT - - # Install QEMU static binaries for multi-arch image build - # https://github.com/docker/setup-qemu-action - # https://docs.docker.com/build/ci/github-actions/multi-platform - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - # Set up BuildKit Docker container builder to be able to build - # multi-platform images and export cache - # https://github.com/docker/setup-buildx-action - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - # Login against a Docker registry except on PR - # https://github.com/docker/login-action - - name: Login to registry ${{ env.REGISTRY }} - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - # Extract metadata (tags, labels) for Docker - # https://github.com/docker/metadata-action - - name: Extract Docker metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - labels: | - org.opencontainers.image.vendor=TiaC Systems Network - org.opencontainers.image.authors=${{ steps.recent-commits-authors.outputs.jsonlist }} - org.opencontainers.image.documentation=${{ env.REPO_README }} - - # Build and push Docker image with Buildx (don't push on PR) - # https://github.com/docker/build-push-action - # https://docs.docker.com/build/ci/github-actions/cache - - name: Build and push Docker image - id: build-and-push - uses: docker/build-push-action@v6 - with: - context: . - platforms: ${{ matrix.platform }} - labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ env.PLATFORM_PAIR }} - cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ env.PLATFORM_PAIR }},mode=max - - - name: Export digest - run: | - mkdir -p /tmp/digests - digest="${{ steps.build-and-push.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-${{ env.PLATFORM_PAIR }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - - merge: - name: Merge all platforms - needs: - - build-linux - - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - # This is used to complete the identity challenge - # with sigstore/fulcio when running outside of PRs. - id-token: write - - steps: - # Free Disk Space on Ubuntu runners, clear up to 35 GB disk space - # https://github.com/jlumbroso/free-disk-space - - name: Free disk space - uses: jlumbroso/free-disk-space@v1.3.1 - with: - # Tool cache: Saved 8.3GiB - tool-cache: true - # Android library: Saved 7.5GiB - android: true - # .NET runtime: Saved 1.6GiB - dotnet: true - # Haskell runtime: Saved 5.4GiB - haskell: true - # Large misc. packages: Saved 4.8GiB - large-packages: true - # Docker images: Saved 3.2GiB - docker-images: true - # Swap storage: Saved 4.0GiB - swap-storage: true - - # Checkout again (also in this merge job), to get - # the list of all recent commit authors again. - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Get Recent Commit Authors - id: recent-commits-authors - run: | - jsonlist=$(git log --format="%an <%ae>" | \ - sort -u | tr '\n' '\000' | \ - jq -R -s -c 'split("\u0000")') - echo "jsonlist=$jsonlist" >> $GITHUB_OUTPUT - - - name: Download digests - uses: actions/download-artifact@v4 - with: - path: /tmp/digests - pattern: digests-* - merge-multiple: true - - # Set up BuildKit Docker container builder to be able to build - # multi-platform images and export cache - # https://github.com/docker/setup-buildx-action - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - # Login against a Docker registry except on PR - # https://github.com/docker/login-action - - name: Login to registry ${{ env.REGISTRY }} - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - # Extract metadata (tags, labels) for Docker - # https://github.com/docker/metadata-action - - name: Extract Docker metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - labels: | - org.opencontainers.image.vendor=TiaC Systems Network - org.opencontainers.image.authors=${{ steps.recent-commits-authors.outputs.jsonlist }} - org.opencontainers.image.documentation=${{ env.REPO_README }} - - - name: Create manifest list and push - working-directory: /tmp/digests - run: | - docker buildx imagetools create \ - $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *) - - - name: Inspect image - run: | - docker buildx imagetools inspect \ - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }} + setup: + name: Setup + + # Build all multi-platform Docker images, all stages. The specified + # stages (in ./.github/stages.json) are built one after the other + # for the specified platforms (in ./.github/platforms.json) and, + # if successful, are finally merged via a Docker manifest and + # published to the Docker registry on GitHub. + uses: tiacsys/github-workflows/.github/workflows/docker-stages.yml@main + with: + ghr-free-disk: false + platforms-json: ./.github/platforms.json + stages-json: ./.github/stages.json + secrets: inherit # pass all secrets diff --git a/.github/workflows/pr-close-policy.yml b/.github/workflows/pr-close-policy.yml index 573dd70..40490bd 100644 --- a/.github/workflows/pr-close-policy.yml +++ b/.github/workflows/pr-close-policy.yml @@ -4,28 +4,12 @@ on: pull_request: types: [closed] -env: - # Use docker.io for Docker Hub if empty - REGISTRY: ghcr.io - # github.repository as / - IMAGE_NAME: ${{ github.repository }} - jobs: - clean: - name: Clean up the registry - - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - steps: - # Delete the "pr-#" tagged Docker images when a pull request is closed. - # https://github.com/dataaxiom/ghcr-cleanup-action - - name: Delete pr-tagged Docker images on registry ${{ env.REGISTRY }} - uses: dataaxiom/ghcr-cleanup-action@v1 - with: - delete-tags: pr-${{github.event.pull_request.number}} - packages: ${{ github.event.repository.name }} - owner: ${{ github.repository_owner }} - token: ${{ secrets.GITHUB_TOKEN }} + setup: + name: Setup + + # Delete the "pr-#" tagged Docker images when a pull request is closed. + uses: tiacsys/github-workflows/.github/workflows/docker-mr-proper.yml@main + with: + delete-tags: pr-${{github.event.pull_request.number}} + secrets: inherit # pass all secrets diff --git a/.github/workflows/retention-policy.yml b/.github/workflows/retention-policy.yml index 867ed6a..a5ed232 100644 --- a/.github/workflows/retention-policy.yml +++ b/.github/workflows/retention-policy.yml @@ -6,32 +6,13 @@ on: # every day at 00:05 UTC - cron: '5 0 * * *' -env: - # Use docker.io for Docker Hub if empty - REGISTRY: ghcr.io - # github.repository as / - IMAGE_NAME: ${{ github.repository }} - jobs: - clean: - name: Clean up the registry - - runs-on: ubuntu-latest - permissions: - contents: read - packages: write + setup: + name: Setup - steps: - # Delete all untagged Docker images older than 2 weeks. - # https://github.com/dataaxiom/ghcr-cleanup-action - - name: Delete untagged Docker images on registry ${{ env.REGISTRY }} - uses: dataaxiom/ghcr-cleanup-action@v1 - with: - older-than: 2 weeks - delete-untagged: true - delete-partial-images: true - exclude-tags: "^\\d+\\.\\d+\\.\\d+$|^latest$|^main$|^nightly$" - use-regex: true - packages: ${{ github.event.repository.name }} - owner: ${{ github.repository_owner }} - token: ${{ secrets.GITHUB_TOKEN }} + # Delete all untagged Docker images older than 2 weeks. + uses: tiacsys/github-workflows/.github/workflows/docker-retention.yml@main + with: + older-than: 2 weeks + exclude-tags: "^\\d+\\.\\d+\\.\\d+$|^latest$|^main$|^nightly$" + secrets: inherit # pass all secrets