build-and-test.yml

name: "CI: Build and test"

concurrency:
  group: ${{ github.workflow }}-${{
           github.ref_name == '11.8.x' && format('ci-main-build-test-{0}', github.run_id) ||
           format('ci-pr-build-test-on-{0}-against-branch-{1}', github.event_name, github.ref_name)
         }}
  cancel-in-progress: true

on:
  push:
    branches:
      - "pull-request/[0-9]+"
      - "11.8.x"

jobs:
  build:
    strategy:
      fail-fast: false
      matrix:
        host-platform:
          - linux-64
          - linux-aarch64
          - win-64
        python-version:
          - "3.13"
          - "3.12"
          - "3.11"
          - "3.10"
          - "3.9"
        cuda-version:
          # Note: this is for build-time only.
          - "11.8.0"
    name: Build (${{ matrix.host-platform }}, Python ${{ matrix.python-version }}, CUDA ${{ matrix.cuda-version }})
    if: ${{ github.repository_owner == 'nvidia' }}
    permissions:
      id-token: write # This is required for configure-aws-credentials
      contents: read  # This is required for actions/checkout
    runs-on: ${{ (matrix.host-platform == 'linux-64' && 'linux-amd64-cpu8') ||
                 (matrix.host-platform == 'linux-aarch64' && 'linux-arm64-cpu8') ||
                 (matrix.host-platform == 'win-64' && 'windows-2019') }}
               #  (matrix.host-platform == 'win-64' && 'windows-amd64-cpu8') }}
    outputs:
      BUILD_CTK_VER: ${{ steps.pass_env.outputs.CUDA_VERSION }}
    steps:
      - name: Checkout ${{ github.event.repository.name }}
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.head_ref || github.ref_name }}

      # WAR: setup-python is not relocatable...
      # see https://github.com/actions/setup-python/issues/871
      - name: Set up Python ${{ matrix.python-version }}
        if: ${{ startsWith(matrix.host-platform, 'linux') }}
        id: setup-python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
  
      - name: Set up MSVC
        if: ${{ startsWith(matrix.host-platform, 'win') }}
        uses: ilammy/msvc-dev-cmd@v1
  
      - name: Set environment variables
        shell: bash --noprofile --norc -xeuo pipefail {0}
        run: |
          PYTHON_VERSION_FORMATTED=$(echo '${{ matrix.python-version }}' | tr -d '.')
          if [[ "${{ matrix.host-platform }}" == linux* ]]; then
            CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-manylinux*"
            REPO_DIR=$(pwd)
          elif [[ "${{ matrix.host-platform }}" == win* ]]; then
            CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-win_amd64"
            PWD=$(pwd)
            REPO_DIR=$(cygpath -w $PWD)
          fi
  
          echo "PARALLEL_LEVEL=$(nproc)" >> $GITHUB_ENV
          echo "CUDA_BINDINGS_ARTIFACT_NAME=cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${{ matrix.cuda-version }}-${{ matrix.host-platform }}-${{ github.sha }}" >> $GITHUB_ENV
          echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/dist")" >> $GITHUB_ENV
          echo "CIBW_BUILD=${CIBW_BUILD}" >> $GITHUB_ENV
  
      - name: Dump environment
        shell: bash --noprofile --norc -xeuo pipefail {0}
        run: |
          env

      - name: Set up mini CTK
        uses: ./.github/actions/fetch_ctk
        continue-on-error: false
        with:
          host-platform: ${{ matrix.host-platform }}
          cuda-version: ${{ matrix.cuda-version }}

      - name: Build cuda.bindings wheel
        uses: pypa/cibuildwheel@v2.22.0
        env:
          CIBW_BUILD: ${{ env.CIBW_BUILD }}
          CIBW_ARCHS_LINUX: "native"
          CIBW_BUILD_VERBOSITY: 1
          # CIBW mounts the host filesystem under /host
          CIBW_ENVIRONMENT_LINUX: >
            CUDA_PATH=/host/${{ env.CUDA_PATH }}
            PARALLEL_LEVEL=${{ env.PARALLEL_LEVEL }}
          CIBW_ENVIRONMENT_WINDOWS: >
            CUDA_HOME="$(cygpath -w ${{ env.CUDA_PATH }})"
          #  PARALLEL_LEVEL=${{ env.PARALLEL_LEVEL }}
        with:
          package-dir: ./cuda_bindings/
          output-dir: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}

      - name: List the cuda.bindings artifacts directory
        shell: bash --noprofile --norc -xeuo pipefail {0}
        run: |
          if [[ "${{ matrix.host-platform }}" == win* ]]; then
            export CHOWN=chown
          else
            export CHOWN="sudo chown"
          fi
          $CHOWN -R $(whoami) ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
          ls -lahR ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}

      # TODO: enable this after NVIDIA/cuda-python#297 is resolved
      # - name: Check cuda.bindings wheel
      #   shell: bash --noprofile --norc -xeuo pipefail {0}
      #   run: |
      #     twine check ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl

      - name: Upload cuda.bindings build artifacts
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}
          path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl
          if-no-files-found: error

      # upload-artifact's "overwrite: true" option has a race condition among parallel
      # jobs, so we let job 0 do the work
      - name: Build and check cuda-python wheel
        if: ${{ strategy.job-index == 0 }}
        run: |
          pushd cuda_python
          pip wheel -v --no-deps .
          twine check *.whl
          popd

      - name: List the cuda-python artifacts directory
        if: ${{ strategy.job-index == 0 }}
        run: |
          if [[ "${{ matrix.host-platform }}" == win* ]]; then
            export CHOWN=chown
          else
            export CHOWN="sudo chown"
          fi
          $CHOWN -R $(whoami) cuda_python/*.whl
          ls -lahR cuda_python

      - name: Upload cuda-python build artifacts
        if: ${{ strategy.job-index == 0 }}
        uses: actions/upload-artifact@v4
        with:
          name: cuda-python-wheel
          path: cuda_python/*.whl
          if-no-files-found: error

      - name: Pass environment variables to the next runner
        id: pass_env
        run: |
          echo "CUDA_VERSION=${{ matrix.cuda-version }}" >> $GITHUB_OUTPUT

  test:
    strategy:
      fail-fast: false
      # TODO: add driver version here
      matrix:
        host-platform:
          - linux-64
          - linux-aarch64
          # TODO: enable testing once win-64 GPU runners are up
          # - win-64
        python-version:
          - "3.13"
          - "3.12"
          - "3.11"
          - "3.10"
          - "3.9"
        cuda-version:
          # Note: this is for test-time only.
          - "11.8.0"
        local-ctk:
          - 1  # use mini CTK
          - 0  # use CTK wheels
        runner:
          - default
        include:
          - host-platform: linux-64
            python-version: "3.12"
            cuda-version: "11.8.0"
            runner: H100
    name: Test (${{ matrix.host-platform }}, Python ${{ matrix.python-version }}, CUDA ${{ matrix.cuda-version }}, Runner ${{ matrix.runner }}, ${{ (matrix.local-ctk == '1' && 'local CTK') || 'CTK wheels' }})
    # The build stage could fail but we want the CI to keep moving.
    if: ${{ github.repository_owner == 'nvidia' && always() }}
    permissions:
      id-token: write # This is required for configure-aws-credentials
      contents: read  # This is required for actions/checkout
    runs-on: ${{ (matrix.runner == 'default' && matrix.host-platform == 'linux-64' && 'linux-amd64-gpu-v100-latest-1') ||
                 (matrix.runner == 'default' && matrix.host-platform == 'linux-aarch64' && 'linux-arm64-gpu-a100-latest-1') ||
                 (matrix.runner == 'H100' && 'linux-amd64-gpu-h100-latest-1-testing') }}
    # Our self-hosted runners require a container
    # TODO: use a different (nvidia?) container
    container:
      options: -u root --security-opt seccomp=unconfined --shm-size 16g
      image: ubuntu:22.04
      env:
        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
    needs:
      - build
    steps:
      - name: Ensure GPU is working
        shell: bash --noprofile --norc -xeuo pipefail {0}
        run: nvidia-smi

      - name: Checkout ${{ github.event.repository.name }}
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.head_ref || github.ref_name }}

      - name: Set environment variables
        shell: bash --noprofile --norc -xeuo pipefail {0}
        run: |
          PYTHON_VERSION_FORMATTED=$(echo '${{ matrix.python-version }}' | tr -d '.')
          if [[ "${{ matrix.host-platform }}" == linux* ]]; then
            REPO_DIR=$(pwd)
          elif [[ "${{ matrix.host-platform }}" == win* ]]; then
            PWD=$(pwd)
            REPO_DIR=$(cygpath -w $PWD)
          fi

          BUILD_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${{ needs.build.outputs.BUILD_CTK_VER }})"
          TEST_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${{ matrix.cuda-version }})"
          if [[ $BUILD_CUDA_MAJOR != $TEST_CUDA_MAJOR ]]; then
            SKIP_CUDA_BINDINGS_TEST=1
          else
            SKIP_CUDA_BINDINGS_TEST=0
          fi

          # make outputs from the previous job as env vars
          echo "CUDA_BINDINGS_ARTIFACT_NAME=cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${{ needs.build.outputs.BUILD_CTK_VER }}-${{ matrix.host-platform }}-${{ github.sha }}" >> $GITHUB_ENV
          echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/dist")" >> $GITHUB_ENV
          echo "SKIP_CUDA_BINDINGS_TEST=${SKIP_CUDA_BINDINGS_TEST}" >> $GITHUB_ENV

      - name: Download cuda-python build artifacts
        uses: actions/download-artifact@v4
        with:
          name: cuda-python-wheel
          path: .

      - name: Display structure of downloaded cuda-python artifacts
        run: |
          pwd
          ls -lahR .

      - name: Download cuda.bindings build artifacts
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}
          path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}

      - name: Display structure of downloaded cuda.bindings artifacts
        shell: bash --noprofile --norc -xeuo pipefail {0}
        run: |
          pwd
          ls -lahR $CUDA_BINDINGS_ARTIFACTS_DIR

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
        env:
          # we use self-hosted runners on which setup-python behaves weirdly...
          AGENT_TOOLSDIRECTORY: "/opt/hostedtoolcache"

      - name: Set up mini CTK
        if: ${{ matrix.local-ctk == '1' }}
        uses: ./.github/actions/fetch_ctk
        continue-on-error: false
        with:
          host-platform: ${{ matrix.host-platform }}
          cuda-version: ${{ matrix.cuda-version }}

      - name: Run cuda.bindings tests
        if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0' }}
        shell: bash --noprofile --norc -xeuo pipefail {0}
        run: |
          pushd "${CUDA_BINDINGS_ARTIFACTS_DIR}"
          if [[ "${{ matrix.local-ctk }}" == 1 ]]; then
            ls $CUDA_PATH
            pip install *.whl
          else
            pip install $(ls *.whl)[all]
          fi
          popd

          pushd ./cuda_bindings
          pip install -r requirements.txt
          pytest -rxXs tests/
          # It is a bit convoluted to run the Cython tests against CTK wheels,
          # so let's just skip them.
          if [[ "${{ matrix.local-ctk }}" == 1 ]]; then
            if [[ "${{ matrix.host-platform }}" == linux* ]]; then
              # cython tests require gcc
              apt install -y build-essential
              bash tests_cython/build_tests.sh
            elif [[ "${{ matrix.host-platform }}" == win* ]]; then
              # TODO: enable this once win-64 runners are up 
              exit 1
            fi  
            pytest -rxXs tests_cython
          fi
          popd

      - name: Ensure cuda-python installable
        run: |
          pip install cuda_python*.whl

  checks:
    name: Check job status
    permissions:
      checks: read
    needs:
      - build
      - test
    secrets: inherit
    uses:
      ./.github/workflows/status-check.yml