build-and-test-device.yaml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: test-c-device

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
    paths:
      - 'CMakeLists.txt'
      - '.github/workflows/build-and-test-device.yaml'
      - 'src/nanoarrow/**'
      - 'python/setup.py'

permissions:
  contents: read

jobs:
  test-c-device:

    runs-on: ${{ matrix.config.runner }}

    name: ${{ matrix.config.label }}

    strategy:
      fail-fast: false
      matrix:
        config:
          - {runner: ubuntu-latest, label: default-build}
          - {runner: ubuntu-latest, label: namespaced-build, cmake_args: "-DNANOARROW_NAMESPACE=SomeUserNamespace"}
          - {runner: ubuntu-latest, label: bundled-build, cmake_args: "-DNANOARROW_DEVICE_BUNDLE=ON"}
          - {runner: macOS-latest, label: with-metal, cmake_args: "-DNANOARROW_DEVICE_WITH_METAL=ON"}
          # The self-hosted cuda runner is currently not picking up any jobs
          # https://github.com/apache/arrow-nanoarrow/issues/646
          # - {runner: ["self-hosted", "cuda"], label: with-cuda, cmake_args: "-DNANOARROW_DEVICE_WITH_CUDA=ON"}


    steps:
      - uses: actions/checkout@v4

      - name: Install memcheck dependencies
        if: matrix.config.label == 'default-build'
        run: |
          sudo apt-get update && sudo apt-get install -y valgrind

      # The self-hosted runner for CUDA needs a number of packages installed that
      # are already installed on the GitHub runner
      - name: Install build dependencies
        if: matrix.config.label == 'with-cuda'
        run: |
          sudo apt-get update && \
            sudo apt-get install -y cmake build-essential cuda-toolkit tzdata
          # Debug install location of cuda toolkit
          ldconfig -p | grep libcuda

      - name: Cache Arrow C++ Build
        id: cache-arrow-build
        uses: actions/cache@v4
        with:
          path: arrow
          # Bump the number at the end of this line to force a new Arrow C++ build
          # The self-hosted runner needs its own Arrow build since it is running
          # a much older linux distribution. When the Arrow C++ requirement for tests is
          # dropped, we don't have to use this at all as part of the CI configuration.
          key: arrow-device-${{ runner.os }}-${{ runner.arch }}-${{ matrix.config.label }}-2

      - name: Build Arrow C++
        if: steps.cache-arrow-build.outputs.cache-hit != 'true'
        shell: bash
        run: |
          ci/scripts/build-arrow-cpp-minimal.sh 15.0.2 arrow

      - name: Build
        run: |
          ARROW_PATH="$(pwd)/arrow"

          # Debug arrow includes
          find "${ARROW_PATH}"

          mkdir build
          cd build
          export VERBOSE=1
          cmake .. -DCMAKE_BUILD_TYPE=Debug -DNANOARROW_DEVICE=ON \
            -DNANOARROW_BUILD_TESTS=ON -DCMAKE_PREFIX_PATH="${ARROW_PATH}" \
            ${{ matrix.config.cmake_args }}

          cmake --build .

      - name: Check for non-namespaced symbols in namespaced build
        if: matrix.config.label == 'namespaced-build'
        run: |
          # Dump all symbols
          nm --extern-only build/libnanoarrow_device.a

          # Check for non-namespaced ones
          ARROW_SYMBOLS=`nm --extern-only build/libnanoarrow_device.a | grep "T Arrow" || true`
          if [ -z "$ARROW_SYMBOLS" ]; then
            exit 0
          fi

          echo "Found the following non-namespaced extern symbols:"
          echo $ARROW_SYMBOLS
          exit 1

      - name: Run tests
        run: |
          cd build
          ctest -T test --output-on-failure .

      - name: Run tests with valgrind
        if: matrix.config.label == 'default-build' || matrix.config.label == 'default-noatomics'
        run: |
          cd build
          ctest -T memcheck .

      - name: Upload memcheck results
        if: failure() && matrix.config.label == 'default-build'
        uses: actions/upload-artifact@main
        with:
          name: nanoarrow-device-memcheck
          path: build/Testing/Temporary/MemoryChecker.*.log

      - name: Set up Python
        if: matrix.config.label == 'with-cuda'
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
          cache: 'pip'

      - name: Build Python bindings with CUDA
        if: matrix.config.label == 'with-cuda'
        env:
          NANOARROW_PYTHON_CUDA_HOME: "/usr/local/cuda"
        run: |
          cd python
          python3 -m pip install ".[test]" -vv
          python3 -m pip install cupy-cuda12x

      - name: Test Python bindings with CUDA
        if: matrix.config.label == 'with-cuda'
        run: |
          cd python
          python3 -m pytest -vv