ci: correctly download and merge manifests #197
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build docker images | |
| concurrency: | |
| group: main-${{ github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| workflow_dispatch: | |
| push: | |
| tags: | |
| - "*" | |
| jobs: | |
| prepare: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| otp: ${{ steps.otp.outputs.version }} | |
| elixir: ${{ steps.elixir.outputs.version }} | |
| steps: | |
| - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 | |
| - name: get otp_version | |
| id: otp | |
| run: | | |
| otp_version=$(grep -E "^\+\sOTP-.*" ./RELEASE.md | sed 's/\+\sOTP-//g' | jq -R -s -c 'split("\n")[:-1]') | |
| echo "version=$otp_version" >> $GITHUB_OUTPUT | |
| - name: get elixir_version | |
| id: elixir | |
| run: | | |
| elixir_version=$(grep -E "^\+\sElixir-.*" ./RELEASE.md | sed 's/\+\sElixir-//g' | jq -R -s -c 'split("\n")[:-1]') | |
| echo "version=$elixir_version" >> $GITHUB_OUTPUT | |
| base: | |
| runs-on: ${{ github.repository_owner == 'emqx' && matrix.platform[2] || '["ubuntu-latest"]' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| base_image_vsn: | |
| - "5.0" | |
| platform: | |
| - [ubuntu24.04, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [ubuntu24.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [ubuntu22.04, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [ubuntu22.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [ubuntu20.04, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [ubuntu20.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [ubuntu18.04, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [ubuntu18.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [debian12, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [debian12, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [debian11, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [debian11, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [debian10, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [debian10, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [el9, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [el9, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [el8, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [el8, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [el7, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [el7, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [amzn2, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [amzn2, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [amzn2023, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [amzn2023, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [alpine3.15.1, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [alpine3.15.1, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| env: | |
| REGISTRY_IMAGE: ghcr.io/${{ github.repository }}/base-${{ matrix.base_image_vsn }} | |
| steps: | |
| - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 | |
| - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.AWS_DEFAULT_REGION }} | |
| - name: define base tag | |
| id: base_tag | |
| run: | | |
| PLATFORM=${{ matrix.platform[1] }} | |
| ARCH=${PLATFORM#linux/} | |
| echo "tag=${{ matrix.platform[0] }}-${ARCH}" | tee -a $GITHUB_OUTPUT | |
| - name: Get cache | |
| run: aws s3 sync s3://docker-buildx-cache/emqx-builder/${{ steps.base_tag.outputs.tag }} /tmp/.docker-buildx-cache | |
| - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 | |
| - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 | |
| - uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ github.token }} | |
| - uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 | |
| id: base_meta | |
| with: | |
| images: ${{ env.REGISTRY_IMAGE }} | |
| tags: type=raw,value=${{ steps.base_tag.outputs.tag }} | |
| - name: Build base image | |
| id: build | |
| uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 | |
| with: | |
| pull: true | |
| push: true | |
| cache-from: type=local,src=/tmp/.docker-buildx-cache,mode=max | |
| cache-to: type=local,dest=/tmp/.docker-buildx-cache-new,mode=max | |
| platforms: ${{ matrix.platform[1] }} | |
| tags: ${{ steps.base_meta.outputs.tags }} | |
| labels: ${{ steps.base_meta.outputs.labels }} | |
| file: ${{ matrix.platform[0] }}/Dockerfile | |
| context: . | |
| - name: Update cache | |
| run: aws s3 sync --delete /tmp/.docker-buildx-cache-new s3://docker-buildx-cache/emqx-builder/${{ steps.base_tag.outputs.tag }} | |
| build: | |
| runs-on: ${{ github.repository_owner == 'emqx' && matrix.platform[2] || '["ubuntu-latest"]' }} | |
| needs: | |
| - prepare | |
| - base | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| base_image_vsn: | |
| - "5.0" | |
| otp: ${{ fromJSON(needs.prepare.outputs.otp) }} | |
| elixir: ${{ fromJSON(needs.prepare.outputs.elixir) }} | |
| platform: | |
| - [ubuntu24.04, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [ubuntu24.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [ubuntu22.04, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [ubuntu22.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [ubuntu20.04, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [ubuntu20.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [ubuntu18.04, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [ubuntu18.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [ubuntu16.04, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [ubuntu16.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [debian12, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [debian12, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [debian11, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [debian11, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [debian10, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [debian10, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [debian9, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [debian9, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [el9, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [el9, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [el8, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [el8, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [el7, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [el7, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [amzn2, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [amzn2, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [amzn2023, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [amzn2023, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| - [alpine3.15.1, linux/amd64, [self-hosted, linux, x64, ephemeral]] | |
| - [alpine3.15.1, linux/arm64, [self-hosted, linux, arm64, ephemeral]] | |
| steps: | |
| - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 | |
| - name: Get ref | |
| id: ref | |
| run: echo "ref=${GITHUB_REF##*/}" >> $GITHUB_OUTPUT | |
| - name: Set registry image | |
| id: registry | |
| run: | | |
| echo "image=ghcr.io/${{ github.repository }}/${{ steps.ref.outputs.ref }}" | tee -a $GITHUB_OUTPUT | |
| - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 | |
| - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 | |
| - uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ github.token }} | |
| - uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 | |
| id: meta | |
| with: | |
| images: ${{ steps.registry.outputs.image }} | |
| - name: define base tag | |
| id: base_tag | |
| run: | | |
| PLATFORM=${{ matrix.platform[1] }} | |
| ARCH="${PLATFORM#linux/}" | |
| TAG="${{ matrix.platform[0] }}-${ARCH}" | |
| echo "tag=${TAG}" | tee -a $GITHUB_OUTPUT | |
| echo "image=ghcr.io/${{ github.repository }}/base-${{ matrix.base_image_vsn }}:${TAG}" | tee -a $GITHUB_OUTPUT | |
| - uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 | |
| id: build | |
| with: | |
| pull: true | |
| no-cache: true | |
| platforms: ${{ matrix.platform[1] }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| build-args: | | |
| BUILD_FROM=${{ steps.base_tag.outputs.image }} | |
| OTP_VERSION=${{ matrix.otp }} | |
| ELIXIR_VERSION=${{ matrix.elixir }} | |
| file: ./Dockerfile | |
| context: . | |
| outputs: type=image,name=${{ steps.registry.outputs.image }},push-by-digest=true,name-canonical=true,push=true | |
| - name: Export digests | |
| run: | | |
| mkdir -p /tmp/digests | |
| digest="${{ steps.build.outputs.digest }}" | |
| touch "/tmp/digests/${digest#sha256:}" | |
| - name: Upload digest | |
| uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 | |
| with: | |
| name: "digests-${{ matrix.platform[0] }}-${{ matrix.otp }}-${{ matrix.elixir }}-${{ matrix.platform[1] == 'linux/amd64' && 'amd64' || 'arm64' }}" | |
| path: /tmp/digests/* | |
| if-no-files-found: error | |
| retention-days: 1 | |
| merge: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - prepare | |
| - build | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| base_image_vsn: | |
| - "5.0" | |
| otp: ${{ fromJSON(needs.prepare.outputs.otp) }} | |
| elixir: ${{ fromJSON(needs.prepare.outputs.elixir) }} | |
| platform: | |
| - ubuntu24.04 | |
| - ubuntu22.04 | |
| - ubuntu20.04 | |
| - ubuntu18.04 | |
| - debian12 | |
| - debian11 | |
| - debian10 | |
| - el9 | |
| - el8 | |
| - el7 | |
| - amzn2 | |
| - amzn2023 | |
| - alpine3.15.1 | |
| steps: | |
| - name: Get ref | |
| id: ref | |
| run: echo "ref=${GITHUB_REF##*/}" >> $GITHUB_OUTPUT | |
| - name: Set registry image | |
| id: registry | |
| run: | | |
| echo "image=ghcr.io/${{ github.repository }}/${{ steps.ref.outputs.ref }}" | tee -a $GITHUB_OUTPUT | |
| - name: Download digests | |
| uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 | |
| with: | |
| pattern: "digests-${{ matrix.platform }}-${{ matrix.otp }}-${{ matrix.elixir }}-*" | |
| path: /tmp/digests | |
| merge-multiple: true | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 | |
| with: | |
| images: ${{ steps.registry.outputs.image }} | |
| tags: | | |
| type=raw,value=${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.platform }} | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ github.token }} | |
| - name: Create manifest list and push | |
| working-directory: /tmp/digests | |
| run: | | |
| docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ | |
| $(printf '${{ steps.registry.outputs.image }}@sha256:%s ' *) | |
| - name: Inspect image | |
| run: | | |
| docker buildx imagetools inspect ${{ steps.registry.outputs.image }}:${{ steps.meta.outputs.version }} | |
| release: | |
| runs-on: ubuntu-latest | |
| needs: merge | |
| if: startsWith(github.ref, 'refs/tags/') | |
| steps: | |
| - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 | |
| - name: Create Release | |
| id: create_release | |
| uses: actions/create-release@0cb9c9b65d5d1901c1f53e5e66eaf4afd303e70e # v1.1.4 | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| with: | |
| tag_name: ${{ github.ref }} | |
| release_name: EMQX Builder ${{ github.ref }} Released | |
| body_path: RELEASE.md | |
| draft: false | |
| prerelease: false |