Merge pull request #121 from emqx/20241009-add-otp24-to-support-4.x

feat: add OTP 24, OTP 27, bump elixir to 1.17.3

.github/workflows/base.yaml

@@ -6,6 +6,14 @@ concurrency:
 
 on:
   workflow_dispatch:
+    inputs:
+      base_image_vsn:
+        required: true
+        type: string
+        default: "5.0"
+      ref:
+        required: false
+        type: string
 
 ## Build base images to be used by other github workflows
 jobs:
@@ -16,7 +24,7 @@ jobs:
       fail-fast: false
       matrix:
         base_image_vsn:
-          - "5.0"
+          - ${{ github.event.inputs.base_image_vsn }}
         platform:
           - [ubuntu24.04, linux/amd64, [self-hosted, linux, x64, ephemeral]]
           - [ubuntu24.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]]
@@ -50,6 +58,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        with:
+          ref: ${{ github.event.inputs.ref }}
       - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}

.github/workflows/test.yaml

@@ -19,68 +19,79 @@ jobs:
           wget https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VSN}/actionlint_${ACTIONLINT_VSN}_linux_amd64.tar.gz
           tar zxf actionlint_${ACTIONLINT_VSN}_linux_amd64.tar.gz actionlint
           # TODO: enable shellcheck when all the current issues are fixed
-          ./actionlint -color \
-            -shellcheck=
+          ./actionlint -color -shellcheck=
 
   prepare:
     runs-on: ubuntu-latest
     needs:
       - sanity-checks
     outputs:
-      otp: ${{ steps.otp.outputs.version }}
-      elixir: ${{ steps.elixir.outputs.version }}
+      matrix: ${{ steps.versions.outputs.matrix }}
     steps:
     - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
-    - name: get otp_version
-      id: otp
+    - name: get release versions
+      id: versions
       run: |
-        otp_version=$(grep -E "^\+\sOTP-.*" ./RELEASE.md | sed 's/\+\sOTP-//g' | jq -R -s -c 'split("\n")[:-1]')
-        echo "version=$otp_version" >> $GITHUB_OUTPUT
-    - name: get elixir_version
-      id: elixir
-      run: |
-        elixir_version=$(grep -E "^\+\sElixir-.*" ./RELEASE.md | sed 's/\+\sElixir-//g' | jq -R -s -c 'split("\n")[:-1]')
-        echo "version=$elixir_version" >> $GITHUB_OUTPUT
+        #!/bin/bash
+        versions="$(grep -E "^\+\sOTP-.*" ./RELEASE.md | \
+          sed -E 's/\+\sOTP-([0-9.-]*),Elixir-([0-9.]*).*/{"otp":"\1","elixir":"\2"}/g' | \
+          jq -sc .)"
+
+        platforms='[
+          {"os": "ubuntu24.04", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]},
+          {"os": "ubuntu24.04", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]},
+          {"os": "ubuntu22.04", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]},
+          {"os": "ubuntu22.04", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]},
+          {"os": "ubuntu20.04", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]},
+          {"os": "ubuntu20.04", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]},
+          {"os": "ubuntu18.04", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]},
+          {"os": "ubuntu18.04", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]},
+          {"os": "debian12", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]},
+          {"os": "debian12", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]},
+          {"os": "debian11", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]},
+          {"os": "debian11", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]},
+          {"os": "debian10", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]},
+          {"os": "debian10", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]},
+          {"os": "el9", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]},
+          {"os": "el9", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]},
+          {"os": "el8", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]},
+          {"os": "el8", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]},
+          {"os": "el7", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]},
+          {"os": "el7", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]},
+          {"os": "amzn2", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]},
+          {"os": "amzn2", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]},
+          {"os": "amzn2023", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]},
+          {"os": "amzn2023", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]},
+          {"os": "alpine3.15.1", "arch": "linux/amd64", "runner": ["self-hosted", "linux", "x64", "ephemeral"]},
+          {"os": "alpine3.15.1", "arch": "linux/arm64", "runner": ["self-hosted", "linux", "arm64", "ephemeral"]}
+        ]'
+
+        matrix="$(jq -cn \
+          --argjson versions "$versions" \
+          --argjson platforms "$platforms" \
+          '[
+            $versions[] as $version |
+            $platforms[] as $platform |
+            {
+              base_image_vsn: "5.0",
+              otp: $version.otp,
+              elixir: $version.elixir,
+              platform: $platform
+            }
+          ]')"
+
+        echo "matrix=$matrix" | tee -a $GITHUB_OUTPUT
 
   build:
-    runs-on: ${{ github.repository_owner == 'emqx' && matrix.platform[2] || '["ubuntu-latest"]' }}
+    name: ${{ matrix.platform.os }} ${{ matrix.platform.arch }} OTP-${{ matrix.otp }} Elixir-${{ matrix.elixir }}
+    runs-on: ${{ github.repository_owner == 'emqx' && matrix.platform.runner || 'ubuntu-latest' }}
     needs:
       - prepare
 
     strategy:
       fail-fast: false
       matrix:
-        base_image_vsn:
-          - "5.0"
-        otp: ${{ fromJSON(needs.prepare.outputs.otp) }}
-        elixir: ${{ fromJSON(needs.prepare.outputs.elixir) }}
-        platform:
-          - [ubuntu24.04, linux/amd64, [self-hosted, linux, x64, ephemeral]]
-          - [ubuntu24.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]]
-          - [ubuntu22.04, linux/amd64, [self-hosted, linux, x64, ephemeral]]
-          - [ubuntu22.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]]
-          - [ubuntu20.04, linux/amd64, [self-hosted, linux, x64, ephemeral]]
-          - [ubuntu20.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]]
-          - [ubuntu18.04, linux/amd64, [self-hosted, linux, x64, ephemeral]]
-          - [ubuntu18.04, linux/arm64, [self-hosted, linux, arm64, ephemeral]]
-          - [debian12, linux/amd64, [self-hosted, linux, x64, ephemeral]]
-          - [debian12, linux/arm64, [self-hosted, linux, arm64, ephemeral]]
-          - [debian11, linux/amd64, [self-hosted, linux, x64, ephemeral]]
-          - [debian11, linux/arm64, [self-hosted, linux, arm64, ephemeral]]
-          - [debian10, linux/amd64, [self-hosted, linux, x64, ephemeral]]
-          - [debian10, linux/arm64, [self-hosted, linux, arm64, ephemeral]]
-          - [el9, linux/amd64, [self-hosted, linux, x64, ephemeral]]
-          - [el9, linux/arm64, [self-hosted, linux, arm64, ephemeral]]
-          - [el8, linux/amd64, [self-hosted, linux, x64, ephemeral]]
-          - [el8, linux/arm64, [self-hosted, linux, arm64, ephemeral]]
-          - [el7, linux/amd64, [self-hosted, linux, x64, ephemeral]]
-          - [el7, linux/arm64, [self-hosted, linux, arm64, ephemeral]]
-          - [amzn2, linux/amd64, [self-hosted, linux, x64, ephemeral]]
-          - [amzn2, linux/arm64, [self-hosted, linux, arm64, ephemeral]]
-          - [amzn2023, linux/amd64, [self-hosted, linux, x64, ephemeral]]
-          - [amzn2023, linux/arm64, [self-hosted, linux, arm64, ephemeral]]
-          - [alpine3.15.1, linux/amd64, [self-hosted, linux, x64, ephemeral]]
-          - [alpine3.15.1, linux/arm64, [self-hosted, linux, arm64, ephemeral]]
+        include: ${{ fromJSON(needs.prepare.outputs.matrix) }}
 
     steps:
     - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
@@ -97,9 +108,9 @@ jobs:
     - name: define base tag
       id: base_tag
       run: |
-        PLATFORM=${{ matrix.platform[1] }}
+        PLATFORM=${{ matrix.platform.arch }}
         ARCH="${PLATFORM#linux/}"
-        TAG="${{ matrix.platform[0] }}-${ARCH}"
+        TAG="${{ matrix.platform.os }}-${ARCH}"
         echo "tag=${TAG}" | tee -a $GITHUB_OUTPUT
         echo "image=ghcr.io/${{ github.repository }}/base-${{ matrix.base_image_vsn }}:${TAG}" | tee -a $GITHUB_OUTPUT
     - name: Get cache
@@ -109,8 +120,23 @@ jobs:
       with:
         driver-opts: network=host
     - uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
+      if: ${{ startsWith(matrix.otp, '24') }}
+      with:
+        platforms: ${{ matrix.platform.arch }}
+        cache-from: type=local,src=/tmp/.docker-buildx-cache,mode=max
+        build-args: |
+          BUILD_FROM=${{ steps.base_tag.outputs.image }}
+          OTP_VERSION=${{ matrix.otp }}
+          ELIXIR_VERSION=${{ matrix.elixir }}
+          BUILD_WITHOUT_QUIC=1
+          EMQTT_BENCH_VERSION=0.4.5
+          LUX_VERSION=lux-2.6
+        file: ./Dockerfile
+        context: .
+    - uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
+      if: ${{ ! startsWith(matrix.otp, '24') }}
       with:
-        platforms: ${{ matrix.platform[1] }}
+        platforms: ${{ matrix.platform.arch }}
         cache-from: type=local,src=/tmp/.docker-buildx-cache,mode=max
         build-args: |
           BUILD_FROM=${{ steps.base_tag.outputs.image }}

Dockerfile

@@ -4,20 +4,24 @@ FROM ${BUILD_FROM}
 ENV EMQX_BUILDER_IMAGE=${BUILD_FROM}
 ENV ERL_AFLAGS="-kernel shell_history enabled"
 
-ARG OTP_VERSION=26.2.3-1
-ARG ELIXIR_VERSION=1.15.7
-ARG FDB_VERSION=7.3.27
-ARG EMQTT_BENCH_REF=0.4.17
-ARG LUX_REF=lux-2.9.1
+ARG BUILD_WITHOUT_QUIC=false
+ARG OTP_VERSION=27.1-1
+ARG ELIXIR_VERSION=1.17.3
+ARG FDB_VERSION=7.3.43
+ARG EMQTT_BENCH_VERSION=0.4.25
+ARG LUX_VERSION=lux-3.0
 
 COPY get-otp.sh get-zsh.sh get-elixir.sh get-fdb.sh get-emqtt-bench.sh get-lux.sh /
 
-RUN /get-zsh.sh && \
+RUN if [ -f /opt/rh/devtoolset-10/enable ]; then source /opt/rh/devtoolset-10/enable; fi && \
+    which gcc && gcc --version && \
+    which g++ && g++ --version && \
+    /get-zsh.sh && \
     /get-otp.sh ${OTP_VERSION} && \
     /get-elixir.sh ${ELIXIR_VERSION} && \    
-    env FDB_VERSION=${FDB_VERSION} /get-fdb.sh && \
-    env EMQTT_BENCH_REF=${EMQTT_BENCH_REF} /get-emqtt-bench.sh && \
-    env LUX_REF=${LUX_REF} /get-lux.sh && \
+    /get-fdb.sh ${FDB_VERSION} && \
+    /get-emqtt-bench.sh ${EMQTT_BENCH_VERSION} && \
+    /get-lux.sh ${LUX_VERSION} && \
     rm /get-otp.sh /get-zsh.sh /get-elixir.sh /get-fdb.sh /get-emqtt-bench.sh /get-lux.sh
 
 WORKDIR /

Makefile

@@ -6,8 +6,8 @@ all: $(IMAGES)
 .PHONY: $(IMAGES)
 define gen-build-image-target
 $1:
-	@docker build -t ghcr.io/emqx/emqx-builder:$1-base $1
-	@docker build --build-arg BUILD_FROM=ghcr.io/emqx/emqx-builder:$1-base -t ghcr.io/emqx/emqx-builder:$1 .
+	@docker build . -t ghcr.io/emqx/emqx-builder:$1-base -f $1/Dockerfile --progress plain
+	@docker build --build-arg BUILD_FROM=ghcr.io/emqx/emqx-builder:$1-base -t ghcr.io/emqx/emqx-builder:$1 --progress plain .
 endef
 $(foreach img,$(IMAGES),$(eval $(call gen-build-image-target,$(img))))
 

RELEASE.md

@@ -1,11 +1,8 @@
 List per major version used by EMQX, quic, rocksdb builds
 
-OTP version from emqx/otp.git:
+OTP version from emqx/otp.git, Elixir version from elixir-lang/elixir.git.
 
-+ OTP-25.3.2-2
-+ OTP-26.2.5.2-1
-
-Elixir version from elixir-lang/elixir.git:
-NOTE: Only one version is allowed.
-
-+ Elixir-1.15.7
++ OTP-24.3.4.2-4,Elixir-1.15.7
++ OTP-25.3.2-2,Elixir-1.15.7
++ OTP-26.2.5.2-1,Elixir-1.15.7
++ OTP-27.1-1,Elixir-1.17.3

amzn2/Dockerfile

@@ -1,41 +1,53 @@
 ARG BUILD_FROM=public.ecr.aws/amazonlinux/amazonlinux:2
 FROM ${BUILD_FROM}
 
-RUN yum update -y
-
-RUN yum groupinstall -y "Development Tools" && yum install -y \
-    curl \
-    git \
-    jq \
-    ncurses-devel \
-    openssl11 \
-    openssl11-devel \
-    python3 \
-    python3-pip \
-    systemd \
-    unixODBC \
-    unixODBC-devel \
-    vim \
-    wget \
-    which \
-    krb5-workstation \
-    cyrus-sasl-devel \
-    cyrus-sasl \
-    cyrus-sasl-gssapi
-
-# The following is not necessary for Kerberos authentication but makes testing
-# https://github.com/kafka4beam/sasl_auth easier
-RUN yum install -y \
-    krb5-server \
-    expect
+ARG TARGETPLATFORM
+COPY amzn2/RPM-GPG-KEY* /etc/pki/rpm-gpg/
+COPY amzn2/yum.repos.d/${TARGETPLATFORM}/* /etc/yum.repos.d/
 
-ADD get-cmake.sh get-automake.sh  /
+RUN yum clean all && \
+    yum makecache && \
+    yum update -y && \
+    yum install -y wget && \
+    if [ $(arch) == "x86_64" ]; then \
+        wget https://vault.centos.org/7.9.2009/os/x86_64/Packages/libgfortran5-8.3.1-2.1.1.el7.x86_64.rpm; \
+    else \
+        wget https://vault.centos.org/altarch/7/os/aarch64/Packages/libgfortran5-8.3.1-2.1.1.el7.aarch64.rpm; \
+    fi && \
+    yum install -y libgfortran5-8.3.1-2.1.1.el7.*.rpm
+
+RUN yum update -y && \
+    yum groupinstall -y "Development Tools" && \
+    yum install -y curl \
+                   cyrus-sasl \
+                   cyrus-sasl-devel \
+                   cyrus-sasl-gssapi \
+                   devtoolset-10-gcc \
+                   devtoolset-10-gcc-c++ \
+                   expect \
+                   git \
+                   jq \
+                   krb5-server \
+                   krb5-workstation \
+                   ncurses-devel \
+                   openssl11 \
+                   openssl11-devel \
+                   python3 \
+                   python3-pip \
+                   systemd \
+                   unixODBC \
+                   unixODBC-devel \
+                   vim \
+                   wget \
+                   which
 
 WORKDIR /
 
-RUN yum remove -y automake && /get-automake.sh
+ADD get-cmake.sh get-automake.sh  /
+
+RUN yum remove -y automake && /get-automake.sh && rm /get-automake.sh
 
-RUN /get-cmake.sh build
+RUN /get-cmake.sh build && rm /get-cmake.sh
 
 RUN alternatives --install /usr/bin/python python /usr/bin/python2 1 && \
     alternatives --install /usr/bin/python python /usr/bin/python3 2 && \
@@ -47,9 +59,7 @@ RUN alternatives --install /usr/bin/python python /usr/bin/python2 1 && \
 
 # cleanup
 RUN yum clean packages && \
-    rm -rf /tmp/* && \
-    rm -rf /var/tmp/* && \
-    rm /get-cmake.sh /get-automake.sh
+    rm -rf /tmp/* /var/tmp/* /var/cache/yum
 
 # Elixir complains if runs without UTF-8
 ENV LC_ALL=C.UTF-8

amzn2/RPM-GPG-KEY-CentOS-7

@@ -0,0 +1,30 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1.4.5 (GNU/Linux)
+
+mQINBFOn/0sBEADLDyZ+DQHkcTHDQSE0a0B2iYAEXwpPvs67cJ4tmhe/iMOyVMh9
+Yw/vBIF8scm6T/vPN5fopsKiW9UsAhGKg0epC6y5ed+NAUHTEa6pSOdo7CyFDwtn
+4HF61Esyb4gzPT6QiSr0zvdTtgYBRZjAEPFVu3Dio0oZ5UQZ7fzdZfeixMQ8VMTQ
+4y4x5vik9B+cqmGiq9AW71ixlDYVWasgR093fXiD9NLT4DTtK+KLGYNjJ8eMRqfZ
+Ws7g7C+9aEGHfsGZ/SxLOumx/GfiTloal0dnq8TC7XQ/JuNdB9qjoXzRF+faDUsj
+WuvNSQEqUXW1dzJjBvroEvgTdfCJfRpIgOrc256qvDMp1SxchMFltPlo5mbSMKu1
+x1p4UkAzx543meMlRXOgx2/hnBm6H6L0FsSyDS6P224yF+30eeODD4Ju4BCyQ0jO
+IpUxmUnApo/m0eRelI6TRl7jK6aGqSYUNhFBuFxSPKgKYBpFhVzRM63Jsvib82rY
+438q3sIOUdxZY6pvMOWRkdUVoz7WBExTdx5NtGX4kdW5QtcQHM+2kht6sBnJsvcB
+JYcYIwAUeA5vdRfwLKuZn6SgAUKdgeOtuf+cPR3/E68LZr784SlokiHLtQkfk98j
+NXm6fJjXwJvwiM2IiFyg8aUwEEDX5U+QOCA0wYrgUQ/h8iathvBJKSc9jQARAQAB
+tEJDZW50T1MtNyBLZXkgKENlbnRPUyA3IE9mZmljaWFsIFNpZ25pbmcgS2V5KSA8
+c2VjdXJpdHlAY2VudG9zLm9yZz6JAjUEEwECAB8FAlOn/0sCGwMGCwkIBwMCBBUC
+CAMDFgIBAh4BAheAAAoJECTGqKf0qA61TN0P/2730Th8cM+d1pEON7n0F1YiyxqG
+QzwpC2Fhr2UIsXpi/lWTXIG6AlRvrajjFhw9HktYjlF4oMG032SnI0XPdmrN29lL
+F+ee1ANdyvtkw4mMu2yQweVxU7Ku4oATPBvWRv+6pCQPTOMe5xPG0ZPjPGNiJ0xw
+4Ns+f5Q6Gqm927oHXpylUQEmuHKsCp3dK/kZaxJOXsmq6syY1gbrLj2Anq0iWWP4
+Tq8WMktUrTcc+zQ2pFR7ovEihK0Rvhmk6/N4+4JwAGijfhejxwNX8T6PCuYs5Jiv
+hQvsI9FdIIlTP4XhFZ4N9ndnEwA4AH7tNBsmB3HEbLqUSmu2Rr8hGiT2Plc4Y9AO
+aliW1kOMsZFYrX39krfRk2n2NXvieQJ/lw318gSGR67uckkz2ZekbCEpj/0mnHWD
+3R6V7m95R6UYqjcw++Q5CtZ2tzmxomZTf42IGIKBbSVmIS75WY+cBULUx3PcZYHD
+ZqAbB0Dl4MbdEH61kOI8EbN/TLl1i077r+9LXR1mOnlC3GLD03+XfY8eEBQf7137
+YSMiW5r/5xwQk7xEcKlbZdmUJp3ZDTQBXT06vavvp3jlkqqH9QOE8ViZZ6aKQLqv
+pL+4bs52jzuGwTMT7gOR5MzD+vT0fVS7Xm8MjOxvZgbHsAgzyFGlI1ggUQmU7lu3
+uPNL0eRx4S1G4Jn5
+=OGYX
+-----END PGP PUBLIC KEY BLOCK-----