build-local.yaml

name: Build c4c local

# ========== TRIGGER ==========

on:
  push:
    branches:
      - 'public-main'   # default branch name
      - '*-build*'      # Temporary build, no testing
      - '*build-*'      # Temporary build, no testing
      - 'prerelease-*'  # Prerelease build
      - 'release-*'     # Release build
      - '*.check'
    paths:
      - 'Makefile'
      - 'src/**'
      - '.github/workflows/build-local.yaml'
  pull_request:
    types:
      - opened
      - synchronize
      - reopened
    paths:
      - 'Makefile'
      - 'src/**'
      - '.github/workflows/build-local.yaml'
    secrets:
      GHUB_TOKEN:
        required: true
      AZ_SAS_TOK:
        required: true
  workflow_dispatch:
    secrets:
      GHUB_TOKEN:
        required: true
      AZ_SAS_TOK:
        required: true
    inputs:
      whitelist:
        type: string
        required: false
        default: .*
        description: >-
          Whitelist of operating systems to build against, supplied in
          the form of a regular expression. (e.g. ".*anolis.*")
      force_download:
        type: string
        required: false
        default: false
        description: >-
          Whether or not to force the use of azcopy to download the
          local kernel-devel packages.
  workflow_call:
    secrets:
      GHUB_TOKEN:
        required: true
      AZ_SAS_TOK:
        required: true
    inputs:
      whitelist:
        type: string
        required: false
        default: .*
        description: >-
          Whitelist of operating systems to build against, supplied in
          the form of a regular expression. (e.g. ".*anolis.*")
      force_download:
        type: string
        required: false
        default: false
        description: >-
          Whether or not to force the use of azcopy to download the
          local kernel-devel packages.

env:
  DOWNLOAD: ".download"
  UNPACKED: ".unpacked"

# Cancel outdated workflows.
# --------------------------
# Group construction:
# - Cases: Workflow triggered by manual run or on default branch ("main").
#   Resolves to: ${{ github.workflow }}-${{ github.run_id }}
# - All other cases, resolves to:
#   ${{ github.workflow }}-${{ github.head_ref || github.ref_name }}
concurrency:
  cancel-in-progress: true
  group: >-
    ${{ github.workflow }}-${{
    (!((github.head_ref || github.ref_name) == github.event.repository.default_branch)
    && !((github.head_ref || github.ref_name) == 'main')
    && !(github.event_name == 'workflow_dispatch'))
    && (github.head_ref || github.ref_name)
    || (github.run_id)
    }}

jobs:
  prepare:
    name: Prepare build matrix
    runs-on:
      - self-hosted
      - ci-linter-vm
    outputs:
      releases: ${{ steps.matrix.outputs.releases }}
    env:
      BASE_DIR: binaries/misc/kernel-devel-c4c
      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
    steps:
      - name: Clean old artifacts
        run: |
          sudo find -delete

      - name: Check out repository
        uses: Corigine/ci-libs/github_actions/utilities/checkout_corigine@main
        with:
          token: ${{ secrets.GHUB_TOKEN }}

      - name: Determine list of accepted distros using repo kernel-devel metadata
        id: compat
        shell: bash
        run: |
          accepted="$(find .github/data/kernel-devel-c4c -type f -name buildinfo.json \
                      -exec sh -c "echo {} | awk -F'/' '{print \$(NF-2)\"/\"\$(NF-1)}'" \; | xargs)"

          echo "ACCEPTED_OS=${accepted}" | tee -a $GITHUB_OUTPUT

      - name: Get local kernel-devel metadata
        uses: Corigine/ci-libs/github_actions/azure/azcopy_download@main
        with:
          connection-string: ${{ secrets.AZ_SAS_TOK }}
          src: ${{ env.BASE_DIR }}/*
          dst: ${{ env.DOWNLOAD }}
          pattern: "*buildinfo.json"

      - name: Create build matrix
        id: matrix
        shell: python3 {0}
        working-directory: ${{ env.DOWNLOAD }}
        run: |
          import os
          import re
          import json

          if "${{ github.event_name }}" == "workflow_dispatch":
            whitelist = "${{ inputs.whitelist }}"
          else:
            whitelist = ".*"

          accepted_os = "${{ steps.compat.outputs.ACCEPTED_OS }}".split()

          releases = []
          architectures = [d for d in sorted(os.listdir()) if
                           (os.path.isdir(d) and re.search(whitelist, d))]
          for arch in architectures:
            dir = f"{os.getcwd()}/{arch}/"
            distros = [d for d in sorted(os.listdir(dir)) if
                       (os.path.isdir(f"{dir}/{d}") and re.search(whitelist, d))]
            for distro in distros:
              if f"{arch}/{distro}" in accepted_os:
                with open(f"{dir}/{distro}/buildinfo.json") as buildinfo:
                  release = json.load(buildinfo)
                  release["arch"] = arch
                  release["directory"] = f"${{ env.BASE_DIR }}/{arch}/{distro}"

                  releases.append(release)
              else:
                print(f"::notice::{arch}/{distro} is not supported for this version of the o-o-t driver")

          with open(os.environ['GITHUB_OUTPUT'], 'a') as github_output:
            github_output.write(f"releases={releases}")

  build-c4c-misc:
    name: '${{ matrix.release.name }} - ${{ matrix.release.arch }}'
    needs: prepare
    runs-on:
      - self-hosted
      - ${{ matrix.release.arch == 'aarch64' && 'builder-arm' || 'builder' }}
    timeout-minutes: 90
    env:
      KVER: "*"
      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
    strategy:
      fail-fast: false
      matrix:
        release: ${{ fromJSON(needs.prepare.outputs.releases) }}
    container:
      image: ${{ matrix.release.container }}
    steps:
      - name: Check for pre-requisite packages
        id: provision_check
        shell: bash
        run: |
          if [[ "${{ matrix.release.arch }}" == "x86_64" && \
                "${{ matrix.release.name }}" == "anolisos-7.9" ]]; then
            GIT="git236"
          else
            GIT="git"
          fi

          declare -A dependencies=( ["yum"]="ca-certificates cpio \
                                             elfutils-libelf-devel findutils \
                                             gcc ${GIT} gzip hostname make \
                                             sudo tar wget which" \
                                    ["dnf"]="ca-certificates cpio \
                                             elfutils-libelf-devel findutils \
                                             gcc git gzip hostname make sed \
                                             sudo tar wget" \
                                    ["zypper"]="gawk bc bison ca-certificates \
                                                cpio findutils flex gcc git \
                                                gzip hostname libelf-devel \
                                                libelf1 libopenssl-devel make \
                                                python3 sed sudo tar wget \
                                                which" )

          if [[ "${{ matrix.release.name }}" == "anolisos-7.9" ]]; then
            package_manager="yum"
            installed_packages="$(yum list installed | sed 1d | awk -F'\.' '{print $1}')"
          elif [[ "${{ matrix.release.name }}" == "opensuse"* ]]; then
            package_manager="zypper"
            installed_packages="$(zypper search -i | sed -n 's/^i[ \+] | \([^ ]*\).*$/\1/p')"
          else
            package_manager="dnf"
            installed_packages="$(dnf list --installed | sed 1d | awk -F'\.' '{print $1}')"
          fi

          if [[ "${{ matrix.release.name }}" == "openeuler"* || \
                "${{ matrix.release.name }}" == "kylinos"* ]]; then
            new_dependencies="$( echo "${dependencies[$package_manager]}" | \
                                 sed 's/elfutils-libelf-devel/elfutils-devel/' )"
            dependencies[$package_manager]="$new_dependencies"
          fi

          prerequisites="${dependencies[$package_manager]}"
          provision="false"
          for package in $prerequisites; do
            if ! $(echo "$installed_packages" | grep -w -q "$package"); then
              missing_deps=$missing_deps$package" "
              provision="true"
            fi
          done

          echo "missing_deps=$missing_deps" | tee -a $GITHUB_OUTPUT
          echo "provision=$provision" | tee -a $GITHUB_OUTPUT

      - name: Update OpenEuler mirror
        if: >-
          steps.provision_check.outputs.provision == 'true' &&
          startsWith(matrix.release.name, 'openeuler')
        run: |
          version=$(echo "${ matrix.release.upload_dir }" | cut -d "-" -f2)
          if [[ ! -f /etc/yum.repos.d/openEuler.repo ]]; then
            echo "[base]" >> /etc/yum.repos.d/openEuler.repo
            echo "name=base" >> /etc/yum.repos.d/openEuler.repo
            echo "baseurl=http://repo.openeuler.org/openEuler-${version}-LTS/OS/x86_64/" \
                 >> /etc/yum.repos.d/openEuler.repo
            echo "enabled=1" >> /etc/yum.repos.d/openEuler.repo
            echo "gpgcheck=1" >> /etc/yum.repos.d/openEuler.repo
            echo "gpgkey=http://repo.openeuler.org/openEuler-${version}-LTS/OS/x86_64/RPM-GPG-KEY-openEuler"\
                 >> /etc/yum.repos.d/openEuler.repo
          fi

          sed -i 's|repo.openeuler.org|mirrors.dotsrc.org/openeuler|g' /etc/yum.repos.d/openEuler.repo

      - name: Install pre-requisite packages | YUM
        if: >-
          steps.provision_check.outputs.provision == 'true' &&
          matrix.release.name == 'anolisos-7.9'
        run: |
          if [[ "${{ matrix.release.arch }}" == "x86_64" ]]; then
            if ! yum list installed | grep -q ius-release-el7; then
              yum install -y https://repo.ius.io/ius-release-el7.rpm
            fi
            if ! yum list installed | grep -q ius-release-el7; then
              yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
            fi
          fi

          yum install -y ${{ steps.provision_check.outputs.missing_deps }}

      - name: Install pre-requisite packages | DNF
        if: >-
          steps.provision_check.outputs.provision == 'true' &&
          matrix.release.name != 'anolisos-7.9' &&
          !contains(matrix.release.name, 'opensuse')
        run: |
          dnf clean all
          dnf distro-sync -y
          if [[ "${{ matrix.release.name }}" != "opencloudos-9.0" && \
                "${{ matrix.release.name }}" != "kylinos-v10"* && \
                "${{ matrix.release.name }}" != "openeuler"* ]]; then
            dnf install -y epel-release
          fi
          dnf install -y ${{ steps.provision_check.outputs.missing_deps }}
          dnf reinstall -y crypto-policies

      - name: Install pre-requisite packages | Zypper
        if: >-
          steps.provision_check.outputs.provision == 'true' &&
          contains(matrix.release.name, 'opensuse')
        run: |
          PACKAGES="${{ steps.provision_check.outputs.missing_deps }}"

          if ! zypper repos >/dev/null 2>&1; then
            if [[ "${{ matrix.release.name }}" == "opensuse-15"* ]]; then
              SUSE_VER="${{ matrix.release.name }}"
              REPO_URL="http://download.opensuse.org/distribution/leap/${SUSE_VER#*-}/repo/oss"
            else
              REPO_URL="http://download.opensuse.org/distribution/openSUSE-stable/repo/oss"
            fi

            zypper addrepo -f -p 50 "${REPO_URL}" repo-oss
            if [[ "${{ matrix.release.name }}" == "opensuse-15.1" ]]; then
              # Add repo at lower priority to install newer git version
              zypper addrepo -f "http://download.opensuse.org/distribution/leap/15.2/repo/oss" \
                repo-oss-git
              PACKAGES=$(echo ${PACKAGES} | sed 's/git/git-2.25.0-lp152.1.6/')
            fi
          fi
          zypper --gpg-auto-import-keys refresh
          zypper -n install ${PACKAGES}

      - name: Clean old artifacts
        run: |
          sudo find -delete

      - name: Check out repository | Corigine
        if: >-
          matrix.release.arch == 'x86_64' || matrix.release.name != 'anolisos-7.9'
        uses: Corigine/ci-libs/github_actions/utilities/checkout_corigine@main
        with:
          token: ${{ secrets.GHUB_TOKEN }}

      - name: Check out repository
        if: >-
          matrix.release.arch == 'aarch64' && matrix.release.name == 'anolisos-7.9'
        # This cannot use the 'checkout_corigine' action since it requires a
        # newer version of git, which are not installable on this combination
        # of OS and architecture.
        uses: actions/checkout@v3

      - name: Install azcopy binary
        if: >-
          steps.provision_check.outputs.provision == 'true'
        run: |
          if [[ '${{ matrix.release.arch }}' == 'aarch64' ]]; then
            binary_link=https://azcopyvnext.azureedge.net/releases/release-10.24.0-20240326/azcopy_linux_arm64_10.24.0.tar.gz
          else
            binary_link=https://azcopyvnext.azureedge.net/releases/release-10.24.0-20240326/azcopy_linux_amd64_10.24.0.tar.gz
          fi

          wget $binary_link -O /tmp/downloadazcopy-v10-linux.tgz
          tar -xvf /tmp/downloadazcopy-v10-linux.tgz -C /tmp/ --strip-components=1
          sudo cp /tmp/azcopy /usr/bin/

      - name: Get local kernel-devel packages
        uses: Corigine/ci-libs/github_actions/azure/azcopy_download@main
        with:
          connection-string: ${{ secrets.AZ_SAS_TOK }}
          src: ${{ matrix.release.directory }}
          dst: ${{ env.DOWNLOAD }}
          pattern: "*.rpm"
          force_download: ${{ inputs.force_download == 'true' }}

      - name: Unpack Kernel Headers
        uses: ./.github/actions/unpack-rpm
        with:
            input-path: ${{ env.DOWNLOAD }}
            output-path: ${{ env.UNPACKED }}

      - name: Set build environment variables | openSUSE
        if: contains(matrix.release.name, 'opensuse')
        working-directory: ${{ env.UNPACKED }}/usr/src
        shell: bash
        run: |
          mkdir -p kernels
          VERSIONS=$(ls ./ | grep "linux-.*[0-9]$")

          for SOURCE in $VERSIONS; do
            find $(pwd)/$SOURCE-obj/x86_64/default/ -maxdepth 1 -mindepth 1 -not -name Makefile \
              -exec cp -r {} $(pwd)/$SOURCE/ \;

            mv $(pwd)/$SOURCE/ $(pwd)/kernels/$SOURCE
          done

      - name: Build
        uses: ./.github/actions/build
        with:
            search-path: "${{ env.UNPACKED }}/usr/src/kernels"
            redirect-stderr: ${{ contains(matrix.release.name, 'opensuse') && 'false' || 'true' }}