From c51f3c037d666f5c2f0d51f816a0ee3d4a425c3f Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 9 Oct 2024 08:33:52 +0200 Subject: [PATCH 01/12] chore: add OTP 24 to support 4.x --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index 7b2817b..9ea72d1 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -2,6 +2,7 @@ List per major version used by EMQX, quic, rocksdb builds OTP version from emqx/otp.git: ++ OTP-24.3.4.2-4 + OTP-25.3.2-2 + OTP-26.2.5.2-1 From f28f57e819a9ec135cce3ad5ecd9ccc9614c61d3 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 9 Oct 2024 09:16:12 +0200 Subject: [PATCH 02/12] chore: add 27.1-1 --- RELEASE.md | 1 + get-otp.sh | 3 +++ 2 files changed, 4 insertions(+) diff --git a/RELEASE.md b/RELEASE.md index 9ea72d1..09fb2ae 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -5,6 +5,7 @@ OTP version from emqx/otp.git: + OTP-24.3.4.2-4 + OTP-25.3.2-2 + OTP-26.2.5.2-1 ++ OTP-27.1-1 Elixir version from elixir-lang/elixir.git: NOTE: Only one version is allowed. diff --git a/get-otp.sh b/get-otp.sh index c3c1407..e1845cf 100755 --- a/get-otp.sh +++ b/get-otp.sh @@ -64,6 +64,9 @@ erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "relea ######################### case "${OTP_VERSION}" in + 27*) + REBAR3_VERSION="${REBAR3_VERSION:-3.20.0-emqx-3}" + ;; 26*) REBAR3_VERSION="${REBAR3_VERSION:-3.20.0-emqx-1}" ;; From d1edeb168857e4ee29f4a86873d41f65f10d30f7 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 9 Oct 2024 14:35:39 +0200 Subject: [PATCH 03/12] feat: bump elixir to 1.17.3 --- RELEASE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index 09fb2ae..7918f95 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -10,4 +10,4 @@ OTP version from emqx/otp.git: Elixir version from elixir-lang/elixir.git: NOTE: Only one version is allowed. -+ Elixir-1.15.7 ++ Elixir-1.17.3 From 826d5202cbcbe016ed42e84dd402c28d2002f5c2 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 9 Oct 2024 15:14:38 +0200 Subject: [PATCH 04/12] ci: bind elixir version to otp version --- .github/workflows/test.yaml | 26 +++++++++++--------------- RELEASE.md | 15 +++++---------- 2 files changed, 16 insertions(+), 25 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7530c86..98817ba 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -19,30 +19,27 @@ jobs: wget https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VSN}/actionlint_${ACTIONLINT_VSN}_linux_amd64.tar.gz tar zxf actionlint_${ACTIONLINT_VSN}_linux_amd64.tar.gz actionlint # TODO: enable shellcheck when all the current issues are fixed - ./actionlint -color \ - -shellcheck= + ./actionlint -color -shellcheck= prepare: runs-on: ubuntu-latest needs: - sanity-checks outputs: - otp: ${{ steps.otp.outputs.version }} - elixir: ${{ steps.elixir.outputs.version }} + versions: ${{ steps.versions.outputs.versions }} steps: - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 - - name: get otp_version - id: otp + - name: get release versions + id: versions run: | - otp_version=$(grep -E "^\+\sOTP-.*" ./RELEASE.md | sed 's/\+\sOTP-//g' | jq -R -s -c 'split("\n")[:-1]') - echo "version=$otp_version" >> $GITHUB_OUTPUT - - name: get elixir_version - id: elixir - run: | - elixir_version=$(grep -E "^\+\sElixir-.*" ./RELEASE.md | sed 's/\+\sElixir-//g' | jq -R -s -c 'split("\n")[:-1]') - echo "version=$elixir_version" >> $GITHUB_OUTPUT + #!/bin/bash + versions="$(grep -E "^\+\sOTP-.*" ./RELEASE.md | \ + sed -E 's/\+\sOTP-([0-9.-]*),Elixir-([0-9.]*).*/{"otp":"\1","elixir":"\2"}/g' | \ + jq -sc .)" + echo "versions=$versions" | tee -a $GITHUB_OUTPUT build: + name: ${{ matrix.platform[0] }} ${{ matrix.platform[1] }} OTP-${{ matrix.otp }} Elixir-${{ matrix.elixir }} runs-on: ${{ github.repository_owner == 'emqx' && matrix.platform[2] || '["ubuntu-latest"]' }} needs: - prepare @@ -50,10 +47,9 @@ jobs: strategy: fail-fast: false matrix: + include: ${{ fromJSON(needs.prepare.outputs.versions) }} base_image_vsn: - "5.0" - otp: ${{ fromJSON(needs.prepare.outputs.otp) }} - elixir: ${{ fromJSON(needs.prepare.outputs.elixir) }} platform: - [ubuntu24.04, linux/amd64, [self-hosted, linux, x64, ephemeral]] - [ubuntu24.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]] diff --git a/RELEASE.md b/RELEASE.md index 7918f95..ae2fc1e 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,13 +1,8 @@ List per major version used by EMQX, quic, rocksdb builds -OTP version from emqx/otp.git: +OTP version from emqx/otp.git, Elixir version from elixir-lang/elixir.git. -+ OTP-24.3.4.2-4 -+ OTP-25.3.2-2 -+ OTP-26.2.5.2-1 -+ OTP-27.1-1 - -Elixir version from elixir-lang/elixir.git: -NOTE: Only one version is allowed. - -+ Elixir-1.17.3 ++ OTP-24.3.4.2-4,Elixir-1.15.7 ++ OTP-25.3.2-2,Elixir-1.15.7 ++ OTP-26.2.5.2-1,Elixir-1.15.7 ++ OTP-27.1-1,Elixir-1.17.3 From b6b8164bd1e78cb596c985265afdb69a662bdd54 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 9 Oct 2024 15:55:50 +0200 Subject: [PATCH 05/12] ci: fix matrix --- .github/workflows/test.yaml | 89 ++++++++++++++++++++++--------------- 1 file changed, 52 insertions(+), 37 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 98817ba..3f46968 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -26,7 +26,7 @@ jobs: needs: - sanity-checks outputs: - versions: ${{ steps.versions.outputs.versions }} + matrix: ${{ steps.versions.outputs.matrix }} steps: - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 - name: get release versions @@ -36,47 +36,62 @@ jobs: versions="$(grep -E "^\+\sOTP-.*" ./RELEASE.md | \ sed -E 's/\+\sOTP-([0-9.-]*),Elixir-([0-9.]*).*/{"otp":"\1","elixir":"\2"}/g' | \ jq -sc .)" - echo "versions=$versions" | tee -a $GITHUB_OUTPUT + + platforms='[ + {"os": "ubuntu24.04", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "ubuntu24.04", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}, + {"os": "ubuntu22.04", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "ubuntu22.04", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}, + {"os": "ubuntu20.04", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "ubuntu20.04", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}, + {"os": "ubuntu18.04", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "ubuntu18.04", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}, + {"os": "debian12", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "debian12", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}, + {"os": "debian11", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "debian11", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}, + {"os": "debian10", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "debian10", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}, + {"os": "el9", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "el9", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}, + {"os": "el8", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "el8", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}, + {"os": "el7", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "el7", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}, + {"os": "amzn2", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "amzn2", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}, + {"os": "amzn2023", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "amzn2023", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}, + {"os": "alpine3.15.1", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "alpine3.15.1", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]} + ]' + + matrix="$(jq -cn \ + --argjson versions "$versions" \ + --argjson platforms "$platforms" \ + '[ + $versions[] as $version | + $platforms[] as $platform | + { + base_image_vsn: "5.0", + otp: $version.otp, + elixir: $version.elixir, + platform: $platform + } + ]')" + + echo "matrix=$matrix" | tee -a $GITHUB_OUTPUT build: - name: ${{ matrix.platform[0] }} ${{ matrix.platform[1] }} OTP-${{ matrix.otp }} Elixir-${{ matrix.elixir }} - runs-on: ${{ github.repository_owner == 'emqx' && matrix.platform[2] || '["ubuntu-latest"]' }} + name: ${{ matrix.platform.os }} ${{ matrix.platform.arch }} OTP-${{ matrix.otp }} Elixir-${{ matrix.elixir }} + runs-on: ${{ github.repository_owner == 'emqx' && matrix.platform.runner || 'ubuntu-latest' }} needs: - prepare strategy: fail-fast: false matrix: - include: ${{ fromJSON(needs.prepare.outputs.versions) }} - base_image_vsn: - - "5.0" - platform: - - [ubuntu24.04, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [ubuntu24.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [ubuntu22.04, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [ubuntu22.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [ubuntu20.04, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [ubuntu20.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [ubuntu18.04, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [ubuntu18.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [debian12, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [debian12, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [debian11, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [debian11, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [debian10, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [debian10, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [el9, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [el9, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [el8, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [el8, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [el7, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [el7, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [amzn2, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [amzn2, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [amzn2023, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [amzn2023, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [alpine3.15.1, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [alpine3.15.1, linux/arm64, [self-hosted, linux, arm64, ephemeral]] + include: ${{ fromJSON(needs.prepare.outputs.matrix) }} steps: - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 @@ -93,9 +108,9 @@ jobs: - name: define base tag id: base_tag run: | - PLATFORM=${{ matrix.platform[1] }} + PLATFORM=${{ matrix.platform.arch }} ARCH="${PLATFORM#linux/}" - TAG="${{ matrix.platform[0] }}-${ARCH}" + TAG="${{ matrix.platform.os }}-${ARCH}" echo "tag=${TAG}" | tee -a $GITHUB_OUTPUT echo "image=ghcr.io/${{ github.repository }}/base-${{ matrix.base_image_vsn }}:${TAG}" | tee -a $GITHUB_OUTPUT - name: Get cache @@ -106,7 +121,7 @@ jobs: driver-opts: network=host - uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 with: - platforms: ${{ matrix.platform[1] }} + platforms: ${{ matrix.platform.arch }} cache-from: type=local,src=/tmp/.docker-buildx-cache,mode=max build-args: | BUILD_FROM=${{ steps.base_tag.outputs.image }} From a7ac41aadf83582c9bef92fcfd806e4cfbc7d8b1 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 11 Oct 2024 10:15:10 +0200 Subject: [PATCH 06/12] fix: use devtoolset v10 for amzn2 --- amzn2/Dockerfile | 76 +++++++++++-------- amzn2/RPM-GPG-KEY-CentOS-7 | 30 ++++++++ amzn2/RPM-GPG-KEY-CentOS-SIG-SCLo | 20 +++++ .../linux/amd64/CentOS-SCLo-rh.repo | 6 ++ .../linux/arm64/CentOS-SCLo-rh.repo | 6 ++ 5 files changed, 105 insertions(+), 33 deletions(-) create mode 100644 amzn2/RPM-GPG-KEY-CentOS-7 create mode 100644 amzn2/RPM-GPG-KEY-CentOS-SIG-SCLo create mode 100644 amzn2/yum.repos.d/linux/amd64/CentOS-SCLo-rh.repo create mode 100644 amzn2/yum.repos.d/linux/arm64/CentOS-SCLo-rh.repo diff --git a/amzn2/Dockerfile b/amzn2/Dockerfile index 64ec0d3..4476ea0 100644 --- a/amzn2/Dockerfile +++ b/amzn2/Dockerfile @@ -1,41 +1,53 @@ ARG BUILD_FROM=public.ecr.aws/amazonlinux/amazonlinux:2 FROM ${BUILD_FROM} -RUN yum update -y - -RUN yum groupinstall -y "Development Tools" && yum install -y \ - curl \ - git \ - jq \ - ncurses-devel \ - openssl11 \ - openssl11-devel \ - python3 \ - python3-pip \ - systemd \ - unixODBC \ - unixODBC-devel \ - vim \ - wget \ - which \ - krb5-workstation \ - cyrus-sasl-devel \ - cyrus-sasl \ - cyrus-sasl-gssapi - -# The following is not necessary for Kerberos authentication but makes testing -# https://github.com/kafka4beam/sasl_auth easier -RUN yum install -y \ - krb5-server \ - expect +ARG TARGETPLATFORM +COPY amzn2/RPM-GPG-KEY* /etc/pki/rpm-gpg/ +COPY amzn2/yum.repos.d/${TARGETPLATFORM}/* /etc/yum.repos.d/ -ADD get-cmake.sh get-automake.sh / +RUN yum clean all && \ + yum makecache && \ + yum update -y && \ + yum install -y wget && \ + if [ $(arch) == "x86_64" ]; then \ + wget https://vault.centos.org/7.9.2009/os/x86_64/Packages/libgfortran5-8.3.1-2.1.1.el7.x86_64.rpm; \ + else \ + wget https://vault.centos.org/altarch/7/os/aarch64/Packages/libgfortran5-8.3.1-2.1.1.el7.aarch64.rpm; \ + fi && \ + yum install -y libgfortran5-8.3.1-2.1.1.el7.*.rpm + +RUN yum update -y && \ + yum groupinstall -y "Development Tools" && \ + yum install -y curl \ + cyrus-sasl \ + cyrus-sasl-devel \ + cyrus-sasl-gssapi \ + devtoolset-10-gcc \ + devtoolset-10-gcc-c++ \ + expect \ + git \ + jq \ + krb5-server \ + krb5-workstation \ + ncurses-devel \ + openssl11 \ + openssl11-devel \ + python3 \ + python3-pip \ + systemd \ + unixODBC \ + unixODBC-devel \ + vim \ + wget \ + which WORKDIR / -RUN yum remove -y automake && /get-automake.sh +ADD get-cmake.sh get-automake.sh / + +RUN yum remove -y automake && /get-automake.sh && rm /get-automake.sh -RUN /get-cmake.sh build +RUN /get-cmake.sh build && rm /get-cmake.sh RUN alternatives --install /usr/bin/python python /usr/bin/python2 1 && \ alternatives --install /usr/bin/python python /usr/bin/python3 2 && \ @@ -47,9 +59,7 @@ RUN alternatives --install /usr/bin/python python /usr/bin/python2 1 && \ # cleanup RUN yum clean packages && \ - rm -rf /tmp/* && \ - rm -rf /var/tmp/* && \ - rm /get-cmake.sh /get-automake.sh + rm -rf /tmp/* /var/tmp/* /var/cache/yum # Elixir complains if runs without UTF-8 ENV LC_ALL=C.UTF-8 diff --git a/amzn2/RPM-GPG-KEY-CentOS-7 b/amzn2/RPM-GPG-KEY-CentOS-7 new file mode 100644 index 0000000..47f6d4d --- /dev/null +++ b/amzn2/RPM-GPG-KEY-CentOS-7 @@ -0,0 +1,30 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.5 (GNU/Linux) + +mQINBFOn/0sBEADLDyZ+DQHkcTHDQSE0a0B2iYAEXwpPvs67cJ4tmhe/iMOyVMh9 +Yw/vBIF8scm6T/vPN5fopsKiW9UsAhGKg0epC6y5ed+NAUHTEa6pSOdo7CyFDwtn +4HF61Esyb4gzPT6QiSr0zvdTtgYBRZjAEPFVu3Dio0oZ5UQZ7fzdZfeixMQ8VMTQ +4y4x5vik9B+cqmGiq9AW71ixlDYVWasgR093fXiD9NLT4DTtK+KLGYNjJ8eMRqfZ +Ws7g7C+9aEGHfsGZ/SxLOumx/GfiTloal0dnq8TC7XQ/JuNdB9qjoXzRF+faDUsj +WuvNSQEqUXW1dzJjBvroEvgTdfCJfRpIgOrc256qvDMp1SxchMFltPlo5mbSMKu1 +x1p4UkAzx543meMlRXOgx2/hnBm6H6L0FsSyDS6P224yF+30eeODD4Ju4BCyQ0jO +IpUxmUnApo/m0eRelI6TRl7jK6aGqSYUNhFBuFxSPKgKYBpFhVzRM63Jsvib82rY +438q3sIOUdxZY6pvMOWRkdUVoz7WBExTdx5NtGX4kdW5QtcQHM+2kht6sBnJsvcB +JYcYIwAUeA5vdRfwLKuZn6SgAUKdgeOtuf+cPR3/E68LZr784SlokiHLtQkfk98j +NXm6fJjXwJvwiM2IiFyg8aUwEEDX5U+QOCA0wYrgUQ/h8iathvBJKSc9jQARAQAB +tEJDZW50T1MtNyBLZXkgKENlbnRPUyA3IE9mZmljaWFsIFNpZ25pbmcgS2V5KSA8 +c2VjdXJpdHlAY2VudG9zLm9yZz6JAjUEEwECAB8FAlOn/0sCGwMGCwkIBwMCBBUC +CAMDFgIBAh4BAheAAAoJECTGqKf0qA61TN0P/2730Th8cM+d1pEON7n0F1YiyxqG +QzwpC2Fhr2UIsXpi/lWTXIG6AlRvrajjFhw9HktYjlF4oMG032SnI0XPdmrN29lL +F+ee1ANdyvtkw4mMu2yQweVxU7Ku4oATPBvWRv+6pCQPTOMe5xPG0ZPjPGNiJ0xw +4Ns+f5Q6Gqm927oHXpylUQEmuHKsCp3dK/kZaxJOXsmq6syY1gbrLj2Anq0iWWP4 +Tq8WMktUrTcc+zQ2pFR7ovEihK0Rvhmk6/N4+4JwAGijfhejxwNX8T6PCuYs5Jiv +hQvsI9FdIIlTP4XhFZ4N9ndnEwA4AH7tNBsmB3HEbLqUSmu2Rr8hGiT2Plc4Y9AO +aliW1kOMsZFYrX39krfRk2n2NXvieQJ/lw318gSGR67uckkz2ZekbCEpj/0mnHWD +3R6V7m95R6UYqjcw++Q5CtZ2tzmxomZTf42IGIKBbSVmIS75WY+cBULUx3PcZYHD +ZqAbB0Dl4MbdEH61kOI8EbN/TLl1i077r+9LXR1mOnlC3GLD03+XfY8eEBQf7137 +YSMiW5r/5xwQk7xEcKlbZdmUJp3ZDTQBXT06vavvp3jlkqqH9QOE8ViZZ6aKQLqv +pL+4bs52jzuGwTMT7gOR5MzD+vT0fVS7Xm8MjOxvZgbHsAgzyFGlI1ggUQmU7lu3 +uPNL0eRx4S1G4Jn5 +=OGYX +-----END PGP PUBLIC KEY BLOCK----- diff --git a/amzn2/RPM-GPG-KEY-CentOS-SIG-SCLo b/amzn2/RPM-GPG-KEY-CentOS-SIG-SCLo new file mode 100644 index 0000000..85fc96f --- /dev/null +++ b/amzn2/RPM-GPG-KEY-CentOS-SIG-SCLo @@ -0,0 +1,20 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v2.0.22 (GNU/Linux) + +mQENBFYM/AoBCADR9Q5cb+H5ndx+QkzNBQ88wcD+g112yvnHNlSiBMOnNEGHuKPJ +tujZ+eWXP3K6ucJckT91WxfQ2fxPr9jQ0xpZytcHcZdTfn3vKL9+OwR0npp+qmcz +rK8/EzVz/SWSgBQ5xT/HUvaeoVAbzBHSng0r2njnBAqABKAoTxgyRGKSCWduKD32 +7PF2ZpqeDFFhd99Ykt6ar8SlV8ToqH6F7An0ILeejINVbHUxd6+wsbpcOwQ4mGAa +/CPXeqqLGj62ASBv36xQr34hlN/9zQMViaKkacl8zkuvwhuHf4b4VlGVCe6VILpQ +8ytKMV/lcg7YpMfRq4KVWBjCwkvk6zg6KxaHABEBAAG0aENlbnRPUyBTb2Z0d2Fy +ZUNvbGxlY3Rpb25zIFNJRyAoaHR0cHM6Ly93aWtpLmNlbnRvcy5vcmcvU3BlY2lh +bEludGVyZXN0R3JvdXAvU0NMbykgPHNlY3VyaXR5QGNlbnRvcy5vcmc+iQE5BBMB +CgAjAhsDBwsJCAcDAgEGFQgCCQoLBBYCAwECHgECF4AFAmIgbg0ACgkQTrhOcfLu +nVUzkAgAgJSl++PWo69OGI8eUn53whhfJ6ZyKCOyI9+Flghtfwq1mdcP4dIoAubw +nlQcvEsE4Erob2I1S+n/dXYh9y2+Si3HdXRreKzC1rCSG5ebYRrEqeMoJhBoDFSd +xydtxi03l+Dh2iWtLtXpAM85qHvgIrPfdUnLRVbDBT52Ph5igZl8ux2ooJ94CPm4 +yTG/0aPIebNQsLP8LEpa2wad2WJfIeFgjEHranWIzhzapGgwyf2Iu9Sz3L8ljiz2 +2HixHpI2es3IqgLyRsTBo2/EdbvMJWlFwiESe1UQIW6ObKNnVabIHUAxZgOlj59o +JhPOzTBG+Ma2fLs+ZFBV88cpKdctFQ== +=+Gr2 +-----END PGP PUBLIC KEY BLOCK----- diff --git a/amzn2/yum.repos.d/linux/amd64/CentOS-SCLo-rh.repo b/amzn2/yum.repos.d/linux/amd64/CentOS-SCLo-rh.repo new file mode 100644 index 0000000..592e17c --- /dev/null +++ b/amzn2/yum.repos.d/linux/amd64/CentOS-SCLo-rh.repo @@ -0,0 +1,6 @@ +[centos-sclo-rh] +name=CentOS-7 - SCLo rh +baseurl=https://vault.centos.org/centos/7/sclo/$basearch/rh/ +gpgcheck=1 +enabled=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-SCLo diff --git a/amzn2/yum.repos.d/linux/arm64/CentOS-SCLo-rh.repo b/amzn2/yum.repos.d/linux/arm64/CentOS-SCLo-rh.repo new file mode 100644 index 0000000..7bbb087 --- /dev/null +++ b/amzn2/yum.repos.d/linux/arm64/CentOS-SCLo-rh.repo @@ -0,0 +1,6 @@ +[centos-sclo-rh] +name=CentOS-7 - SCLo rh +baseurl=https://vault.centos.org/altarch/7/sclo/$basearch/rh/ +gpgcheck=1 +enabled=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-SCLo From 951e43608ff0cf71a9ee146c911bb94823f49982 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 11 Oct 2024 10:16:23 +0200 Subject: [PATCH 07/12] fix: use devtoolset v10 for el7 --- el7/Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/el7/Dockerfile b/el7/Dockerfile index 08a80c9..308a16b 100644 --- a/el7/Dockerfile +++ b/el7/Dockerfile @@ -17,8 +17,8 @@ RUN yum groupinstall -y "Development Tools" && \ cyrus-sasl \ cyrus-sasl-devel \ cyrus-sasl-gssapi \ - devtoolset-8-gcc \ - devtoolset-8-gcc-c++ \ + devtoolset-10-gcc \ + devtoolset-10-gcc-c++ \ expect \ flex-devel \ jq \ @@ -88,9 +88,9 @@ RUN yum clean packages && \ rm -rf /var/tmp/* && \ rm /install-openssl11.sh /get-git.sh /get-automake.sh /get-cmake.sh -ENV BASH_ENV=/opt/rh/devtoolset-8/enable \ - ENV=/opt/rh/devtoolset-8/enable \ - PROMPT_COMMAND=". /opt/rh/devtoolset-8/enable" +ENV BASH_ENV=/opt/rh/devtoolset-10/enable \ + ENV=/opt/rh/devtoolset-10/enable \ + PROMPT_COMMAND=". /opt/rh/devtoolset-10/enable" # Elixir complains if runs without UTF-8 # CentOS 7, exceptionally, has an `en_US.UTF-8` override as does not From b22bbe86eda81b3dd56751f889de5dbc50f30141 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 11 Oct 2024 10:16:51 +0200 Subject: [PATCH 08/12] fix: use devtoolchain 11 on ubuntu18.04 --- ubuntu18.04/Dockerfile | 141 +++++++++++++++++++++-------------------- 1 file changed, 71 insertions(+), 70 deletions(-) diff --git a/ubuntu18.04/Dockerfile b/ubuntu18.04/Dockerfile index b56fec9..ea3f876 100644 --- a/ubuntu18.04/Dockerfile +++ b/ubuntu18.04/Dockerfile @@ -4,90 +4,91 @@ FROM ${BUILD_FROM} # Avoid getting asked questions during installation (tzdata) ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y \ - apt-transport-https \ - autoconf \ - automake \ - autotools-dev \ - bison \ - build-essential \ - ca-certificates \ - cmake \ - curl \ - debhelper \ - flex \ - g++ \ - gcc \ - git \ - jq \ - libffi-dev \ - libkrb5-3 \ - libkrb5-dev \ - libncurses5-dev \ - libsasl2-2 \ - libsasl2-dev \ - libsasl2-modules-gssapi-mit \ - libssl-dev \ - libtool \ - make \ - software-properties-common \ - unixodbc \ - unixodbc-dev \ - unzip \ - vim \ - wget \ - zip \ - zlib1g-dev - -# The following is not necessary for Kerberos authentication but makes testing -# https://github.com/kafka4beam/sasl_auth easier -RUN apt-get install -y \ - krb5-kdc \ - krb5-admin-server \ - expect - -ADD get-git.sh get-cmake.sh / +RUN apt-get update && \ + apt-get install -y software-properties-common && \ + add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ + apt-get update && \ + apt-get install -y \ + apt-transport-https \ + autoconf \ + automake \ + autotools-dev \ + bison \ + build-essential \ + ca-certificates \ + cmake \ + curl \ + debhelper \ + expect \ + flex \ + g++-11 \ + gcc-11 \ + git \ + jq \ + krb5-admin-server \ + krb5-kdc \ + libbz2-dev \ + libcurl4-openssl-dev \ + libffi-dev \ + libgdbm-dev \ + libkrb5-3 \ + libkrb5-dev \ + libncurses5-dev \ + libnss3-dev \ + libreadline-dev \ + libsasl2-2 \ + libsasl2-dev \ + libsasl2-modules-gssapi-mit \ + libsqlite3-dev \ + libssl-dev \ + libtool \ + make \ + manpages-dev \ + unixodbc \ + unixodbc-dev \ + unzip \ + vim \ + wget \ + zip \ + zlib1g-dev && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 90 --slave /usr/bin/g++ g++ /usr/bin/g++-11 --slave /usr/bin/gcov gcov /usr/bin/gcov-11 && \ + apt-get clean && rm -rf /var/lib/apt/lists/* WORKDIR / -# We need git >= 2.18 otherwise actions/checkout@v3 in github -# workflows will use REST API to download source code archive without -# .git directory instead of normal git clone. And if release is -# configured to take it's version from git tag, rebar3 will not be -# able to resolve the version correctly and will fallback to 0.0.0, -# like it happened with emqtt-bench. -# The error from actions/checkout@v3 is as follows: -# > The repository will be downloaded using the GitHub REST API -# > To create a local Git repository instead, add Git 2.18 or higher to the PATH -RUN apt-get install -y libcurl4-openssl-dev \ - && /get-git.sh \ - && git --version - -RUN wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz \ - && tar xvf Python-3.7.4.tgz \ - && cd Python-3.7.4 \ +ARG PYTHON_VERSION=3.9.2 +RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz \ + && tar xvf Python-${PYTHON_VERSION}.tgz \ + && cd Python-${PYTHON_VERSION} \ && echo "_socket socketmodule.c" >> Modules/Setup.dist \ && echo "_ssl _ssl.c -DUSE_SSL -I/usr/local/ssl/include -I/usr/local/ssl/include/openssl -L/usr/local/ssl/lib -lssl -lcrypto" >> Modules/Setup.dist \ - && ./configure --prefix=/usr/local/python3.7.4 \ + && ./configure --prefix=/usr/local/python${PYTHON_VERSION} \ && make \ && make install \ && rm -rf /usr/bin/python3 /usr/bin/python \ - && ln -s /usr/local/python3.7.4/bin/python3.7 /usr/bin/python3 \ - && ln -s /usr/local/python3.7.4/bin/python3.7 /usr/bin/python + && ln -s /usr/local/python${PYTHON_VERSION}/bin/python${PYTHON_VERSION%.*} /usr/bin/python3 \ + && ln -s /usr/local/python${PYTHON_VERSION}/bin/python${PYTHON_VERSION%.*} /usr/bin/python -RUN sed -i 's/python3/python2.7/1' /usr/bin/lsb_release \ - && curl -k -L -o /tmp/get-pip.py https://bootstrap.pypa.io/get-pip.py \ +RUN curl -k -L -o /tmp/get-pip.py https://bootstrap.pypa.io/get-pip.py \ && python /tmp/get-pip.py \ && python3 /tmp/get-pip.py -ENV PATH=/usr/local/python3.7.4/bin:$PATH +ENV PATH=/usr/local/python${PYTHON_VERSION}/bin:$PATH + +ADD get-git.sh get-cmake.sh / -RUN /get-cmake.sh build +# We need git >= 2.18 otherwise actions/checkout@v3 in github +# workflows will use REST API to download source code archive without +# .git directory instead of normal git clone. And if release is +# configured to take it's version from git tag, rebar3 will not be +# able to resolve the version correctly and will fallback to 0.0.0, +# like it happened with emqtt-bench. +# The error from actions/checkout@v3 is as follows: +# > The repository will be downloaded using the GitHub REST API +# > To create a local Git repository instead, add Git 2.18 or higher to the PATH +RUN /get-git.sh && git --version && rm /get-git.sh -# cleanup -RUN apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ - && rm /get-git.sh /get-cmake.sh +RUN /get-cmake.sh build && rm /get-cmake.sh # Elixir complains if runs without UTF-8 ENV LC_ALL=C.UTF-8 From 8308a6ec63fdfd6be04650bc852277c79aa6f45a Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 11 Oct 2024 10:17:38 +0200 Subject: [PATCH 09/12] feat: upgrade included apps and libraries OTP 27.1-1 Elixir 1.17.3 FoundationDB 7.3.43 emqtt-bench 0.4.25 Lux 3.0 --- Dockerfile | 21 ++++++++++++--------- Makefile | 4 ++-- get-emqtt-bench.sh | 2 +- get-fdb.sh | 2 ++ get-lux.sh | 2 +- 5 files changed, 18 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index 15890ad..9d44579 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,20 +4,23 @@ FROM ${BUILD_FROM} ENV EMQX_BUILDER_IMAGE=${BUILD_FROM} ENV ERL_AFLAGS="-kernel shell_history enabled" -ARG OTP_VERSION=26.2.3-1 -ARG ELIXIR_VERSION=1.15.7 -ARG FDB_VERSION=7.3.27 -ARG EMQTT_BENCH_REF=0.4.17 -ARG LUX_REF=lux-2.9.1 +ARG OTP_VERSION=27.1-1 +ARG ELIXIR_VERSION=1.17.3 +ARG FDB_VERSION=7.3.43 +ARG EMQTT_BENCH_VERSION=0.4.25 +ARG LUX_VERSION=lux-3.0 COPY get-otp.sh get-zsh.sh get-elixir.sh get-fdb.sh get-emqtt-bench.sh get-lux.sh / -RUN /get-zsh.sh && \ +RUN if [ -f /opt/rh/devtoolset-10/enable ]; then source /opt/rh/devtoolset-10/enable; fi && \ + which gcc && gcc --version && \ + which g++ && g++ --version && \ + /get-zsh.sh && \ /get-otp.sh ${OTP_VERSION} && \ /get-elixir.sh ${ELIXIR_VERSION} && \ - env FDB_VERSION=${FDB_VERSION} /get-fdb.sh && \ - env EMQTT_BENCH_REF=${EMQTT_BENCH_REF} /get-emqtt-bench.sh && \ - env LUX_REF=${LUX_REF} /get-lux.sh && \ + /get-fdb.sh ${FDB_VERSION} && \ + /get-emqtt-bench.sh ${EMQTT_BENCH_VERSION} && \ + /get-lux.sh ${LUX_VERSION} && \ rm /get-otp.sh /get-zsh.sh /get-elixir.sh /get-fdb.sh /get-emqtt-bench.sh /get-lux.sh WORKDIR / diff --git a/Makefile b/Makefile index c806e85..e518ee7 100644 --- a/Makefile +++ b/Makefile @@ -6,8 +6,8 @@ all: $(IMAGES) .PHONY: $(IMAGES) define gen-build-image-target $1: - @docker build -t ghcr.io/emqx/emqx-builder:$1-base $1 - @docker build --build-arg BUILD_FROM=ghcr.io/emqx/emqx-builder:$1-base -t ghcr.io/emqx/emqx-builder:$1 . + @docker build . -t ghcr.io/emqx/emqx-builder:$1-base -f $1/Dockerfile --progress plain + @docker build --build-arg BUILD_FROM=ghcr.io/emqx/emqx-builder:$1-base -t ghcr.io/emqx/emqx-builder:$1 --progress plain . endef $(foreach img,$(IMAGES),$(eval $(call gen-build-image-target,$(img)))) diff --git a/get-emqtt-bench.sh b/get-emqtt-bench.sh index 5eb1c98..c2a8103 100755 --- a/get-emqtt-bench.sh +++ b/get-emqtt-bench.sh @@ -2,7 +2,7 @@ set -xeuo pipefail -VSN="${1:-0.4.17}" +VSN="${1:-0.4.25}" . /etc/os-release if [[ "${ID_LIKE:-}" =~ rhel|fedora ]]; then diff --git a/get-fdb.sh b/get-fdb.sh index 869ee17..15d2933 100755 --- a/get-fdb.sh +++ b/get-fdb.sh @@ -8,6 +8,8 @@ if [ "${ARCH}" != "x86_64" ]; then exit 0 fi +FDB_VERSION=${1:-7.3.43} + BASE_URL="https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}" . /etc/os-release if [[ "${ID_LIKE:-}" =~ rhel|fedora ]]; then diff --git a/get-lux.sh b/get-lux.sh index dce9f0f..ddadda4 100755 --- a/get-lux.sh +++ b/get-lux.sh @@ -2,7 +2,7 @@ set -xeuo pipefail -LUX_REF="${LUX_REF:-lux-2.9.1}" +LUX_REF="${1:-lux-3.0}" mkdir -p /tools git clone --depth=1 --branch=${LUX_REF} https://github.com/hawk/lux /tools/lux From 0bd4691aa2f4d714c68f748fad402615f8f3c300 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 11 Oct 2024 10:35:33 +0200 Subject: [PATCH 10/12] ci: update base workflow to allow user input --- .github/workflows/base.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/base.yaml b/.github/workflows/base.yaml index 94e096b..d2f3d1c 100644 --- a/.github/workflows/base.yaml +++ b/.github/workflows/base.yaml @@ -6,6 +6,14 @@ concurrency: on: workflow_dispatch: + inputs: + base_image_vsn: + required: true + type: string + default: "5.0" + ref: + required: false + type: string ## Build base images to be used by other github workflows jobs: @@ -16,7 +24,7 @@ jobs: fail-fast: false matrix: base_image_vsn: - - "5.0" + - ${{ github.event.inputs.base_image_vsn }} platform: - [ubuntu24.04, linux/amd64, [self-hosted, linux, x64, ephemeral]] - [ubuntu24.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]] @@ -50,6 +58,8 @@ jobs: steps: - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + with: + ref: ${{ github.event.inputs.ref }} - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} From 9b9fb61007abf061f3e8354048ee7a84b0b9c122 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 11 Oct 2024 14:38:38 +0200 Subject: [PATCH 11/12] ci: override included apps version for OTP 24 --- .github/workflows/test.yaml | 15 +++++++++++++++ Dockerfile | 1 + 2 files changed, 16 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 3f46968..f0fcff9 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -120,6 +120,21 @@ jobs: with: driver-opts: network=host - uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 + if: ${{ startsWith(matrix.otp, '24') }} + with: + platforms: ${{ matrix.platform.arch }} + cache-from: type=local,src=/tmp/.docker-buildx-cache,mode=max + build-args: | + BUILD_FROM=${{ steps.base_tag.outputs.image }} + OTP_VERSION=${{ matrix.otp }} + ELIXIR_VERSION=${{ matrix.elixir }} + BUILD_WITHOUT_QUIC=1 + EMQTT_BENCH_VERSION=0.4.5 + LUX_VERSION=lux-2.6 + file: ./Dockerfile + context: . + - uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 + if: ${{ ! startsWith(matrix.otp, '24') }} with: platforms: ${{ matrix.platform.arch }} cache-from: type=local,src=/tmp/.docker-buildx-cache,mode=max diff --git a/Dockerfile b/Dockerfile index 9d44579..268191e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,7 @@ FROM ${BUILD_FROM} ENV EMQX_BUILDER_IMAGE=${BUILD_FROM} ENV ERL_AFLAGS="-kernel shell_history enabled" +ARG BUILD_WITHOUT_QUIC=false ARG OTP_VERSION=27.1-1 ARG ELIXIR_VERSION=1.17.3 ARG FDB_VERSION=7.3.43 From 46db3989bfabd1cb9a4633c8fdef060a363b31a2 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 11 Oct 2024 15:36:15 +0200 Subject: [PATCH 12/12] ci: update main.yaml --- .github/workflows/main.yaml | 241 +++++++++++++++++++++--------------- 1 file changed, 138 insertions(+), 103 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index c436133..39ee4fa 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -14,55 +14,103 @@ jobs: prepare: runs-on: ubuntu-latest outputs: - otp: ${{ steps.otp.outputs.version }} - elixir: ${{ steps.elixir.outputs.version }} + matrix: ${{ steps.versions.outputs.matrix }} + final_matrix: ${{ steps.versions.outputs.final_matrix }} steps: - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 - - name: get otp_version - id: otp + - name: get release versions + id: versions run: | - otp_version=$(grep -E "^\+\sOTP-.*" ./RELEASE.md | sed 's/\+\sOTP-//g' | jq -R -s -c 'split("\n")[:-1]') - echo "version=$otp_version" >> $GITHUB_OUTPUT - - name: get elixir_version - id: elixir - run: | - elixir_version=$(grep -E "^\+\sElixir-.*" ./RELEASE.md | sed 's/\+\sElixir-//g' | jq -R -s -c 'split("\n")[:-1]') - echo "version=$elixir_version" >> $GITHUB_OUTPUT + #!/bin/bash + versions="$(grep -E "^\+\sOTP-.*" ./RELEASE.md | \ + sed -E 's/\+\sOTP-([0-9.-]*),Elixir-([0-9.]*).*/{"otp":"\1","elixir":"\2"}/g' | \ + jq -sc .)" + + platforms='[ + {"os": "ubuntu24.04", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "ubuntu24.04", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}, + {"os": "ubuntu22.04", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "ubuntu22.04", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}, + {"os": "ubuntu20.04", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "ubuntu20.04", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}, + {"os": "ubuntu18.04", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "ubuntu18.04", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}, + {"os": "debian12", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "debian12", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}, + {"os": "debian11", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "debian11", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}, + {"os": "debian10", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "debian10", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}, + {"os": "el9", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "el9", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}, + {"os": "el8", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "el8", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}, + {"os": "el7", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "el7", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}, + {"os": "amzn2", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "amzn2", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}, + {"os": "amzn2023", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "amzn2023", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}, + {"os": "alpine3.15.1", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "alpine3.15.1", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]} + ]' + + final_platforms='[ + {"os": "ubuntu24.04", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "ubuntu22.04", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "ubuntu20.04", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "ubuntu18.04", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "debian12", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "debian11", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "debian10", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "el9", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "el8", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "el7", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "amzn2", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "amzn2023", "runner": ["self-hosted", "linux", "x64", "ephemeral"]}, + {"os": "alpine3.15.1", "runner": ["self-hosted", "linux", "x64", "ephemeral"]} + ]' + + matrix="$(jq -cn \ + --argjson versions "$versions" \ + --argjson platforms "$platforms" \ + '[ + $versions[] as $version | + $platforms[] as $platform | + { + base_image_vsn: "5.0", + otp: $version.otp, + elixir: $version.elixir, + platform: $platform + } + ]')" + + final_matrix="$(jq -cn \ + --argjson versions "$versions" \ + --argjson platforms "$final_platforms" \ + '[ + $versions[] as $version | + $platforms[] as $platform | + { + base_image_vsn: "5.0", + otp: $version.otp, + elixir: $version.elixir, + platform: $platform + } + ]')" + + echo "matrix=$matrix" | tee -a $GITHUB_OUTPUT + echo "final_matrix=$final_matrix" | tee -a $GITHUB_OUTPUT base: - runs-on: ${{ github.repository_owner == 'emqx' && matrix.platform[2] || '["ubuntu-latest"]' }} + runs-on: ${{ github.repository_owner == 'emqx' && matrix.platform.runner || '["ubuntu-latest"]' }} + needs: + - prepare + strategy: fail-fast: false matrix: - base_image_vsn: - - "5.0" - platform: - - [ubuntu24.04, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [ubuntu24.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [ubuntu22.04, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [ubuntu22.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [ubuntu20.04, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [ubuntu20.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [ubuntu18.04, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [ubuntu18.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [debian12, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [debian12, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [debian11, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [debian11, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [debian10, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [debian10, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [el9, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [el9, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [el8, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [el8, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [el7, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [el7, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [amzn2, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [amzn2, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [amzn2023, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [amzn2023, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [alpine3.15.1, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [alpine3.15.1, linux/arm64, [self-hosted, linux, arm64, ephemeral]] + include: ${{ fromJSON(needs.prepare.outputs.matrix) }} env: REGISTRY_IMAGE: ghcr.io/${{ github.repository }}/base-${{ matrix.base_image_vsn }} @@ -77,9 +125,9 @@ jobs: - name: define base tag id: base_tag run: | - PLATFORM=${{ matrix.platform[1] }} + PLATFORM=${{ matrix.platform.arch }} ARCH=${PLATFORM#linux/} - echo "tag=${{ matrix.platform[0] }}-${ARCH}" | tee -a $GITHUB_OUTPUT + echo "tag=${{ matrix.platform.os }}-${ARCH}" | tee -a $GITHUB_OUTPUT - name: Get cache run: aws s3 sync s3://docker-buildx-cache/emqx-builder/${{ steps.base_tag.outputs.tag }} /tmp/.docker-buildx-cache - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 @@ -102,16 +150,17 @@ jobs: push: true cache-from: type=local,src=/tmp/.docker-buildx-cache,mode=max cache-to: type=local,dest=/tmp/.docker-buildx-cache-new,mode=max - platforms: ${{ matrix.platform[1] }} + platforms: ${{ matrix.platform.arch }} tags: ${{ steps.base_meta.outputs.tags }} labels: ${{ steps.base_meta.outputs.labels }} - file: ${{ matrix.platform[0] }}/Dockerfile + file: ${{ matrix.platform.os }}/Dockerfile context: . - name: Update cache run: aws s3 sync --delete /tmp/.docker-buildx-cache-new s3://docker-buildx-cache/emqx-builder/${{ steps.base_tag.outputs.tag }} build: - runs-on: ${{ github.repository_owner == 'emqx' && matrix.platform[2] || '["ubuntu-latest"]' }} + name: ${{ matrix.platform.os }} ${{ matrix.platform.arch }} OTP-${{ matrix.otp }} Elixir-${{ matrix.elixir }} + runs-on: ${{ github.repository_owner == 'emqx' && matrix.platform.runner || 'ubuntu-latest' }} needs: - prepare - base @@ -119,37 +168,7 @@ jobs: strategy: fail-fast: false matrix: - base_image_vsn: - - "5.0" - otp: ${{ fromJSON(needs.prepare.outputs.otp) }} - elixir: ${{ fromJSON(needs.prepare.outputs.elixir) }} - platform: - - [ubuntu24.04, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [ubuntu24.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [ubuntu22.04, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [ubuntu22.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [ubuntu20.04, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [ubuntu20.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [ubuntu18.04, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [ubuntu18.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [debian12, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [debian12, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [debian11, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [debian11, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [debian10, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [debian10, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [el9, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [el9, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [el8, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [el8, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [el7, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [el7, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [amzn2, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [amzn2, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [amzn2023, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [amzn2023, linux/arm64, [self-hosted, linux, arm64, ephemeral]] - - [alpine3.15.1, linux/amd64, [self-hosted, linux, x64, ephemeral]] - - [alpine3.15.1, linux/arm64, [self-hosted, linux, arm64, ephemeral]] + include: ${{ fromJSON(needs.prepare.outputs.matrix) }} steps: - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 @@ -174,34 +193,67 @@ jobs: - name: define base tag id: base_tag run: | - PLATFORM=${{ matrix.platform[1] }} + PLATFORM=${{ matrix.platform.arch }} ARCH="${PLATFORM#linux/}" - TAG="${{ matrix.platform[0] }}-${ARCH}" + TAG="${{ matrix.platform.os }}-${ARCH}" echo "tag=${TAG}" | tee -a $GITHUB_OUTPUT echo "image=ghcr.io/${{ github.repository }}/base-${{ matrix.base_image_vsn }}:${TAG}" | tee -a $GITHUB_OUTPUT - uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 - id: build + id: build_24 + if: ${{ startsWith(matrix.otp, '24') }} with: pull: true no-cache: true - platforms: ${{ matrix.platform[1] }} + platforms: ${{ matrix.platform.arch }} labels: ${{ steps.meta.outputs.labels }} build-args: | BUILD_FROM=${{ steps.base_tag.outputs.image }} OTP_VERSION=${{ matrix.otp }} ELIXIR_VERSION=${{ matrix.elixir }} + BUILD_WITHOUT_QUIC=1 + EMQTT_BENCH_VERSION=0.4.5 + LUX_VERSION=lux-2.6 file: ./Dockerfile context: . outputs: type=image,name=${{ steps.registry.outputs.image }},push-by-digest=true,name-canonical=true,push=true + - uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 + id: build_not_24 + if: ${{ ! startsWith(matrix.otp, '24') }} + with: + pull: true + no-cache: true + platforms: ${{ matrix.platform.arch }} + build-args: | + BUILD_FROM=${{ steps.base_tag.outputs.image }} + OTP_VERSION=${{ matrix.otp }} + ELIXIR_VERSION=${{ matrix.elixir }} + file: ./Dockerfile + context: . + outputs: type=image,name=${{ steps.registry.outputs.image }},push-by-digest=true,name-canonical=true,push=true + - name: Export digests + if: ${{ startsWith(matrix.otp, '24') }} + run: | + mkdir -p /tmp/digests + digest="${{ steps.build_24.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + - name: Upload digest + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: "digests-${{ matrix.platform.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-${{ matrix.platform.arch == 'linux/amd64' && 'amd64' || 'arm64' }}" + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + - name: Export digests + if: ${{ ! startsWith(matrix.otp, '24') }} run: | mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" + digest="${{ steps.build_not_24.outputs.digest }}" touch "/tmp/digests/${digest#sha256:}" - name: Upload digest uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 with: - name: "digests-${{ matrix.platform[0] }}-${{ matrix.otp }}-${{ matrix.elixir }}-${{ matrix.platform[1] == 'linux/amd64' && 'amd64' || 'arm64' }}" + name: "digests-${{ matrix.platform.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-${{ matrix.platform.arch == 'linux/amd64' && 'amd64' || 'arm64' }}" path: /tmp/digests/* if-no-files-found: error retention-days: 1 @@ -214,24 +266,7 @@ jobs: strategy: fail-fast: false matrix: - base_image_vsn: - - "5.0" - otp: ${{ fromJSON(needs.prepare.outputs.otp) }} - elixir: ${{ fromJSON(needs.prepare.outputs.elixir) }} - platform: - - ubuntu24.04 - - ubuntu22.04 - - ubuntu20.04 - - ubuntu18.04 - - debian12 - - debian11 - - debian10 - - el9 - - el8 - - el7 - - amzn2 - - amzn2023 - - alpine3.15.1 + include: ${{ fromJSON(needs.prepare.outputs.final_matrix) }} steps: - name: Get ref @@ -244,7 +279,7 @@ jobs: - name: Download digests uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 with: - pattern: "digests-${{ matrix.platform }}-${{ matrix.otp }}-${{ matrix.elixir }}-*" + pattern: "digests-${{ matrix.platform.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-*" path: /tmp/digests merge-multiple: true - name: Set up Docker Buildx @@ -255,7 +290,7 @@ jobs: with: images: ${{ steps.registry.outputs.image }} tags: | - type=raw,value=${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.platform }} + type=raw,value=${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.platform.os }} - name: Login to Docker Hub uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 with: