chore(ci): support fine graded multi-stages

Use our own reusable GitHub workflows for Docker from: - tiacsys/github-workflows/.github/workflows/docker-stages.yml@main - tiacsys/github-workflows/.github/workflows/docker-retention.yml@main Use our own reusable GitHub workflows for Retry on Failure from: - tiacsys/github-workflows/.github/workflows/retry-workflow.yml@main Signed-off-by: Stephan Linz <[email protected]>
tiacsys · Oct 18, 2024 · c93314d · c93314d
1 parent dc09c41
commit c93314d
Show file tree

Hide file tree

Showing 7 changed files with 76 additions and 275 deletions.
diff --git a/.github/platforms.json b/.github/platforms.json
@@ -0,0 +1,10 @@
+{
+	"include": [
+		{ "platform": "linux/amd64" },
+		{ "platform": "linux/arm/v7" },
+		{ "platform": "linux/arm64/v8" },
+		{ "platform": "linux/riscv64" },
+		{ "platform": "linux/ppc64le" },
+		{ "platform": "linux/s390x" }
+	]
+}
diff --git a/.github/stages.json b/.github/stages.json
@@ -0,0 +1,7 @@
+{
+	"include": [
+		{ "stage": "base" },
+		{ "stage": "asdf" },
+		{ "stage": "push", "ghr-free-disk": false }
+	]
+}
diff --git a/.github/workflows/clean-buildcache.yml b/.github/workflows/clean-buildcache.yml
@@ -0,0 +1,14 @@
+name: Clean Buildcache
+
+on:
+  workflow_dispatch: # Manually on button click
+
+jobs:
+  setup:
+    name: Setup
+
+    # Delete the "buildcache-*" tagged Docker images on demand.
+    uses: tiacsys/github-workflows/.github/workflows/docker-mr-proper.yml@main
+    with:
+      delete-tags: buildcache-*
+    secrets: inherit # pass all secrets
diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml
@@ -1,13 +1,5 @@
 name: Docker
 
-# Based on GitHub actions starter workflow for Docker publishing:
-# https://github.com/actions/starter-workflows/blob/main/ci/docker-publish.yml
-
-# This workflow uses actions that are not certified by GitHub.
-# They are provided by a third-party and are governed by
-# separate terms of service, privacy policy, and support
-# documentation.
-
 on:
   schedule:
     # every day at 00:15 UTC
@@ -23,220 +15,18 @@ on:
     branches:
       - 'main'
 
-env:
-  # Use docker.io for Docker Hub if empty
-  REGISTRY: ghcr.io
-  # github.repository as <account>/<repo>
-  IMAGE_NAME: ${{ github.repository }}
-  # repository base URL
-  REPO_README: ${{ github.server_url }}/${{ github.repository }}/blob/main/README.rst
-
 jobs:
-  build-linux:
-    name: Linux
-
-
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        platform:
-          # Distribute build across multiple runners, use a matrix strategy.
-          # https://docs.docker.com/build/ci/github-actions/multi-platform
-          - linux/amd64
-          - linux/arm/v7
-          - linux/arm64/v8
-          - linux/riscv64
-          - linux/ppc64le
-          - linux/s390x
-
-    steps:
-      # Free Disk Space on Ubuntu runners, clear up to 35 GB disk space
-      # https://github.com/jlumbroso/free-disk-space
-      - name: Free disk space
-        uses: jlumbroso/[email protected]
-        with:
-          # Tool cache: Saved 8.3GiB
-          tool-cache: true
-          # Android library: Saved 7.5GiB
-          android: true
-          # .NET runtime: Saved 1.6GiB
-          dotnet: true
-          # Haskell runtime: Saved 5.4GiB
-          haskell: true
-          # Large misc. packages: Saved 4.8GiB
-          large-packages: true
-          # Docker images: Saved 3.2GiB
-          docker-images: true
-          # Swap storage: Saved 4.0GiB
-          swap-storage: true
-
-      - name: Prepare GitHub runner environment
-        run: |
-          platform=${{ matrix.platform }}
-          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
-
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Get Recent Commit Authors
-        id: recent-commits-authors
-        run: |
-          jsonlist=$(git log --format="%an <%ae>" | \
-                     sort -u | tr '\n' '\000' | \
-                     jq -R -s -c 'split("\u0000")')
-          echo "jsonlist=$jsonlist" >> $GITHUB_OUTPUT
-
-      # Install QEMU static binaries for multi-arch image build
-      # https://github.com/docker/setup-qemu-action
-      # https://docs.docker.com/build/ci/github-actions/multi-platform
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      # Set up BuildKit Docker container builder to be able to build
-      # multi-platform images and export cache
-      # https://github.com/docker/setup-buildx-action
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      # Login against a Docker registry except on PR
-      # https://github.com/docker/login-action
-      - name: Login to registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      # Extract metadata (tags, labels) for Docker
-      # https://github.com/docker/metadata-action
-      - name: Extract Docker metadata
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-          labels: |
-            org.opencontainers.image.vendor=TiaC Systems Network
-            org.opencontainers.image.authors=${{ steps.recent-commits-authors.outputs.jsonlist }}
-            org.opencontainers.image.documentation=${{ env.REPO_README }}
-
-      # Build and push Docker image with Buildx (don't push on PR)
-      # https://github.com/docker/build-push-action
-      # https://docs.docker.com/build/ci/github-actions/cache
-      - name: Build and push Docker image
-        id: build-and-push
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          platforms: ${{ matrix.platform }}
-          labels: ${{ steps.meta.outputs.labels }}
-          outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true
-          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ env.PLATFORM_PAIR }}
-          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ env.PLATFORM_PAIR }},mode=max
-
-      - name: Export digest
-        run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build-and-push.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-
-      - name: Upload digest
-        uses: actions/upload-artifact@v4
-        with:
-          name: digests-${{ env.PLATFORM_PAIR }}
-          path: /tmp/digests/*
-          if-no-files-found: error
-          retention-days: 1
-
-  merge:
-    name: Merge all platforms
-    needs:
-      - build-linux
-
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-      # This is used to complete the identity challenge
-      # with sigstore/fulcio when running outside of PRs.
-      id-token: write
-
-    steps:
-      # Free Disk Space on Ubuntu runners, clear up to 35 GB disk space
-      # https://github.com/jlumbroso/free-disk-space
-      - name: Free disk space
-        uses: jlumbroso/[email protected]
-        with:
-          # Tool cache: Saved 8.3GiB
-          tool-cache: true
-          # Android library: Saved 7.5GiB
-          android: true
-          # .NET runtime: Saved 1.6GiB
-          dotnet: true
-          # Haskell runtime: Saved 5.4GiB
-          haskell: true
-          # Large misc. packages: Saved 4.8GiB
-          large-packages: true
-          # Docker images: Saved 3.2GiB
-          docker-images: true
-          # Swap storage: Saved 4.0GiB
-          swap-storage: true
-
-      # Checkout again (also in this merge job), to get
-      # the list of all recent commit authors again.
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Get Recent Commit Authors
-        id: recent-commits-authors
-        run: |
-          jsonlist=$(git log --format="%an <%ae>" | \
-                     sort -u | tr '\n' '\000' | \
-                     jq -R -s -c 'split("\u0000")')
-          echo "jsonlist=$jsonlist" >> $GITHUB_OUTPUT
-
-      - name: Download digests
-        uses: actions/download-artifact@v4
-        with:
-          path: /tmp/digests
-          pattern: digests-*
-          merge-multiple: true
-
-      # Set up BuildKit Docker container builder to be able to build
-      # multi-platform images and export cache
-      # https://github.com/docker/setup-buildx-action
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      # Login against a Docker registry except on PR
-      # https://github.com/docker/login-action
-      - name: Login to registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      # Extract metadata (tags, labels) for Docker
-      # https://github.com/docker/metadata-action
-      - name: Extract Docker metadata
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-          labels: |
-            org.opencontainers.image.vendor=TiaC Systems Network
-            org.opencontainers.image.authors=${{ steps.recent-commits-authors.outputs.jsonlist }}
-            org.opencontainers.image.documentation=${{ env.REPO_README }}
-
-      - name: Create manifest list and push
-        working-directory: /tmp/digests
-        run: |
-          docker buildx imagetools create \
-            $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-            $(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
-
-      - name: Inspect image
-        run: |
-          docker buildx imagetools inspect \
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
+  setup:
+    name: Setup
+
+    # Build all multi-platform Docker images, all stages. The specified
+    # stages (in ./.github/stages.json) are built one after the other
+    # for the specified platforms (in ./.github/platforms.json) and,
+    # if successful, are finally merged via a Docker manifest and
+    # published to the Docker registry on GitHub.
+    uses: tiacsys/github-workflows/.github/workflows/docker-stages.yml@main
+    with:
+      ghr-free-disk: false
+      platforms-json: ./.github/platforms.json
+      stages-json: ./.github/stages.json
+    secrets: inherit # pass all secrets
diff --git a/.github/workflows/pr-close-policy.yml b/.github/workflows/pr-close-policy.yml
@@ -4,28 +4,12 @@ on:
   pull_request:
     types: [closed]
 
-env:
-  # Use docker.io for Docker Hub if empty
-  REGISTRY: ghcr.io
-  # github.repository as <account>/<repo>
-  IMAGE_NAME: ${{ github.repository }}
-
 jobs:
-  clean:
-    name: Clean up the registry
-
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-
-    steps:
-      # Delete the "pr-#" tagged Docker images when a pull request is closed.
-      # https://github.com/dataaxiom/ghcr-cleanup-action
-      - name: Delete pr-tagged Docker images on registry ${{ env.REGISTRY }}
-        uses: dataaxiom/ghcr-cleanup-action@v1
-        with:
-          delete-tags: pr-${{github.event.pull_request.number}}
-          packages: ${{ github.event.repository.name }}
-          owner: ${{ github.repository_owner }}
-          token: ${{ secrets.GITHUB_TOKEN }}
+  setup:
+    name: Setup
+
+    # Delete the "pr-#" tagged Docker images when a pull request is closed.
+    uses: tiacsys/github-workflows/.github/workflows/docker-mr-proper.yml@main
+    with:
+      delete-tags: pr-${{github.event.pull_request.number}}
+    secrets: inherit # pass all secrets
diff --git a/.github/workflows/retention-policy.yml b/.github/workflows/retention-policy.yml
@@ -6,32 +6,13 @@ on:
     # every day at 00:05 UTC
     - cron: '5 0 * * *'
 
-env:
-  # Use docker.io for Docker Hub if empty
-  REGISTRY: ghcr.io
-  # github.repository as <account>/<repo>
-  IMAGE_NAME: ${{ github.repository }}
-
 jobs:
-  clean:
-    name: Clean up the registry
-
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
+  setup:
+    name: Setup
 
-    steps:
-      # Delete all untagged Docker images older than 2 weeks.
-      # https://github.com/dataaxiom/ghcr-cleanup-action
-      - name: Delete untagged Docker images on registry ${{ env.REGISTRY }}
-        uses: dataaxiom/ghcr-cleanup-action@v1
-        with:
-          older-than: 2 weeks
-          delete-untagged: true
-          delete-partial-images: true
-          exclude-tags: "^\\d+\\.\\d+\\.\\d+$|^latest$|^main$|^nightly$"
-          use-regex: true
-          packages: ${{ github.event.repository.name }}
-          owner: ${{ github.repository_owner }}
-          token: ${{ secrets.GITHUB_TOKEN }}
+    # Delete all untagged Docker images older than 2 weeks.
+    uses: tiacsys/github-workflows/.github/workflows/docker-retention.yml@main
+    with:
+      older-than: 2 weeks
+      exclude-tags: "^\\d+\\.\\d+\\.\\d+$|^latest$|^main$|^nightly$"
+    secrets: inherit # pass all secrets
diff --git a/.github/workflows/retry-workflow.yml b/.github/workflows/retry-workflow.yml
@@ -0,0 +1,15 @@
+name: Retry Workflow
+
+on:
+  workflow_dispatch:
+    inputs:
+      runid:
+        required: true
+
+jobs:
+  fail:
+    name: Fail
+
+    uses: tiacsys/github-workflows/.github/workflows/retry-workflow.yml@main
+    with:
+      runid: ${{ inputs.runid }}