From fb02af226831d1000f8b546d23b91bc9769f1dc8 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Mon, 6 Jan 2025 20:08:38 +0000 Subject: [PATCH] ensure zstd is installed in all stages that could access GHA cache --- .github/actions/fetch_ctk/action.yml | 34 +++------------ .github/actions/install_unix_deps/action.yml | 45 ++++++++++++++++++++ .github/workflows/build-and-test.yml | 43 ++++++------------- .github/workflows/build-docs.yml | 8 ++++ 4 files changed, 71 insertions(+), 59 deletions(-) create mode 100644 .github/actions/install_unix_deps/action.yml diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml index 7b8674ab..5850b4c7 100644 --- a/.github/actions/fetch_ctk/action.yml +++ b/.github/actions/fetch_ctk/action.yml @@ -20,35 +20,11 @@ runs: echo "CTK_CACHE_FILENAME=mini-ctk-${{ inputs.cuda-version }}-${{ inputs.host-platform }}.tar.gz" >> $GITHUB_ENV - name: Install dependencies - shell: bash --noprofile --norc -xeuo pipefail {0} - run: | - dependencies=(zstd curl xz-utils) - dependent_exes=(zstd curl xz) - - not_found=0 - for dep in ${dependent_exes[@]}; do - if ! (command -v curl 2>&1 >/dev/null); then - not_found=1 - break - fi - done - if [[ $not_found == 0 ]]; then - echo "All dependencies are found. Do nothing." - exit 0 - fi - if ! (command -v sudo 2>&1 >/dev/null); then - if [[ $EUID == 0 ]]; then - alias SUDO="" - else - echo "The following oprations require root access." - exit 1 - fi - else - alias SUDO="sudo" - fi - shopt -s expand_aliases - SUDO apt update - SUDO apt install -y ${dependencies[@]} + uses: ./.github/actions/install_unix_deps + continue-on-error: false + with: + dependencies: "zstd curl xz-utils" + dependent_exes: "zstd curl xz" - name: Download CTK cache id: ctk-get-cache diff --git a/.github/actions/install_unix_deps/action.yml b/.github/actions/install_unix_deps/action.yml new file mode 100644 index 00000000..f1ee73ab --- /dev/null +++ b/.github/actions/install_unix_deps/action.yml @@ -0,0 +1,45 @@ +name: Install dependencies on Ubuntu + +description: Install needed dependencies, regardless if using GitHub- or self- hosted runners, container, sudo or not. + +inputs: + dependencies: + required: true + type: string + dependent_exes: + required: true + type: string + +runs: + using: composite + steps: + - name: Install dependencies + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + dependencies=(${{ inputs.dependencies }}) + dependent_exes=(${{ inputs.dependent_exes }}) + + not_found=0 + for dep in ${dependent_exes[@]}; do + if ! (command -v $dep 2>&1 >/dev/null); then + not_found=1 + break + fi + done + if [[ $not_found == 0 ]]; then + echo "All dependencies are found. Do nothing." + exit 0 + fi + if ! (command -v sudo 2>&1 >/dev/null); then + if [[ $EUID == 0 ]]; then + alias SUDO="" + else + echo "The following oprations require root access." + exit 1 + fi + else + alias SUDO="sudo" + fi + shopt -s expand_aliases + SUDO apt update + SUDO apt install -y ${dependencies[@]} diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 67dad274..d25929d3 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -100,35 +100,12 @@ jobs: - name: Install dependencies if: ${{ env.USE_CACHE == '1' }} - run: | + uses: ./.github/actions/install_unix_deps + continue-on-error: false + with: # For GHA Cache - dependencies=(zstd) - dependent_exes=(zstd) - - not_found=0 - for dep in ${dependent_exes[@]}; do - if ! (command -v curl 2>&1 >/dev/null); then - not_found=1 - break - fi - done - if [[ $not_found == 0 ]]; then - echo "All dependencies are found. Do nothing." - exit 0 - fi - if ! (command -v sudo 2>&1 >/dev/null); then - if [[ $EUID == 0 ]]; then - alias SUDO="" - else - echo "The following oprations require root access." - exit 1 - fi - else - alias SUDO="sudo" - fi - shopt -s expand_aliases - SUDO apt update - SUDO apt install -y ${dependencies[@]} + dependencies: "zstd" + dependent_exes: "zstd" - name: Dump environment run: | @@ -354,6 +331,14 @@ jobs: echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/dist")" >> $GITHUB_ENV echo "SKIP_CUDA_BINDINGS_TEST=${SKIP_CUDA_BINDINGS_TEST}" >> $GITHUB_ENV + - name: Install dependencies + uses: ./.github/actions/install_unix_deps + continue-on-error: false + with: + # zstd for GHA Cache, gcc for Cython tests + dependencies: "zstd build-essential" + dependent_exes: "zstd gcc" + # We'll try GHA Artifacts first, and then fall back to GHA Cache - name: Download cuda.bindings build artifacts id: cuda-bindings-download @@ -453,8 +438,6 @@ jobs: pip install -r requirements.txt pytest -rxXs tests/ if [[ "${{ matrix.host-platform }}" == linux* ]]; then - # cython tests require gcc - apt install -y build-essential bash tests/cython/build_tests.sh elif [[ "${{ matrix.host-platform }}" == win* ]]; then # TODO: enable this once win-64 runners are up diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index c0dd7869..aedc716b 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -31,6 +31,14 @@ jobs: # TODO: cache conda env to speed up the workflow once conda-incubator/setup-miniconda#267 # is resolved + - name: Install dependencies + uses: ./.github/actions/install_unix_deps + continue-on-error: false + with: + # zstd for GHA Cache + dependencies: "zstd" + dependent_exes: "zstd" + - name: Set up miniforge uses: conda-incubator/setup-miniconda@v3 with: