(WIP) Update ingest to use coordinate system metadata #24
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This CI confirms the Python package continues to install properly in the many | |
| # different required contexts | |
| name: TileDB-SOMA Python CI (Packaging) | |
| on: | |
| push: | |
| paths: | |
| - '.github/workflows/python-ci-packaging.yml' | |
| - 'apis/python/MANIFEST.in' | |
| - 'apis/python/pyproject.toml' | |
| - 'apis/python/setup.py' | |
| - 'apis/python/src/tiledbsoma/__init__.py' | |
| - 'libtiledbsoma/cmake/inputs/Config.cmake.in' | |
| - 'libtiledbsoma/cmake/inputs/tiledbsoma.pc.in' | |
| - 'libtiledbsoma/cmake/Modules/FindTileDB_EP.cmake' | |
| - 'libtiledbsoma/cmake/Modules/TileDBCommon.cmake' | |
| - 'libtiledbsoma/cmake/Superbuild.cmake' | |
| - 'libtiledbsoma/CMakeLists.txt' | |
| pull_request: | |
| paths: | |
| - '.github/workflows/python-ci-packaging.yml' | |
| - 'apis/python/MANIFEST.in' | |
| - 'apis/python/pyproject.toml' | |
| - 'apis/python/setup.py' | |
| - 'apis/python/src/tiledbsoma/__init__.py' | |
| - 'libtiledbsoma/cmake/inputs/Config.cmake.in' | |
| - 'libtiledbsoma/cmake/inputs/tiledbsoma.pc.in' | |
| - 'libtiledbsoma/cmake/Modules/FindTileDB_EP.cmake' | |
| - 'libtiledbsoma/cmake/Modules/TileDBCommon.cmake' | |
| - 'libtiledbsoma/cmake/Superbuild.cmake' | |
| - 'libtiledbsoma/CMakeLists.txt' | |
| workflow_dispatch: | |
| jobs: | |
| # Confirm shared object copying when building the Python package | |
| # https://github.com/single-cell-data/TileDB-SOMA/pull/1937 | |
| docker: | |
| runs-on: ubuntu-latest | |
| name: "SO copying (docker) TILEDB_EXISTS: ${{ matrix.TILEDB_EXISTS }} TILEDBSOMA_EXISTS: ${{ matrix.TILEDBSOMA_EXISTS }}" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| TILEDB_EXISTS: ["no", "yes"] | |
| TILEDBSOMA_EXISTS: ["no", "yes"] | |
| exclude: | |
| - TILEDB_EXISTS: "no" | |
| TILEDBSOMA_EXISTS: "yes" | |
| container: | |
| image: ubuntu:22.04 | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Docker image info | |
| run: | | |
| uname -a | |
| cat /etc/lsb-release | |
| - name: Setup | |
| run: | | |
| apt-get update | |
| apt-get install --yes cmake git python-is-python3 python3 python3-pip python3-venv unzip wget | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # for setuptools-scm | |
| - name: Configure Git | |
| run: | | |
| # This is a permissions quirk due to running Git as root inside of a Docker container | |
| git config --global --add safe.directory $(pwd) | |
| git branch | |
| - name: Install pre-built libtiledb | |
| if: ${{ matrix.TILEDB_EXISTS == 'yes' }} | |
| run: | | |
| mkdir -p external | |
| # Please do not edit manually -- let scripts/update-tiledb-version.py update this | |
| wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.23.0/tiledb-linux-x86_64-2.23.0-152093b.tar.gz | |
| tar -C external -xzf tiledb-linux-x86_64-*.tar.gz | |
| ls external/lib/ | |
| echo "LD_LIBRARY_PATH=$(pwd)/external/lib" >> $GITHUB_ENV | |
| echo "PKG_CONFIG_PATH=$(pwd)/external/lib/pkgconfig" >> $GITHUB_ENV | |
| echo "TILEDB_PATH=$(pwd)/external" >> $GITHUB_ENV | |
| - name: Build and install libtiledbsoma | |
| if: ${{ matrix.TILEDBSOMA_EXISTS == 'yes' }} | |
| run: | | |
| cmake -S libtiledbsoma -B build-libtiledbsoma \ | |
| -D CMAKE_BUILD_TYPE=Release \ | |
| -D CMAKE_PREFIX_PATH=$(pwd)/external/ \ | |
| -D CMAKE_INSTALL_PREFIX:PATH=$(pwd)/external/ \ | |
| -D OVERRIDE_INSTALL_PREFIX=OFF \ | |
| -D DOWNLOAD_TILEDB_PREBUILT=OFF \ | |
| -D FORCE_BUILD_TILEDB=OFF | |
| cmake --build build-libtiledbsoma -j $(nproc) | |
| cmake --build build-libtiledbsoma --target install-libtiledbsoma | |
| ls external/lib/ | |
| echo "TILEDBSOMA_PATH=$(pwd)/external" >> $GITHUB_ENV | |
| - name: Setup Python | |
| run: | | |
| python --version | |
| python -m venv ./venv-soma | |
| ./venv-soma/bin/python -m pip install --prefer-binary pybind11-global typeguard sparse wheel | |
| ./venv-soma/bin/python -m pip list | |
| - name: Build wheel | |
| run: | | |
| echo env vars: $LD_LIBRARY_PATH $PKG_CONFIG_PATH $TILEDB_PATH $TILEDBSOMA_PATH | |
| cd apis/python | |
| ../../venv-soma/bin/python setup.py bdist_wheel | |
| - name: Inspect wheel | |
| run: unzip -l apis/python/dist/tiledbsoma-*.whl | grep '\.so' | |
| - name: Confirm libtiledb.so is copied | |
| if: ${{ matrix.TILEDB_EXISTS == 'no' }} | |
| run: unzip -l apis/python/dist/tiledbsoma-*.whl | grep -q libtiledb.so | |
| - name: Confirm libtiledb.so is **not** copied when using external shared object | |
| if: ${{ matrix.TILEDB_EXISTS == 'yes' }} | |
| run: | | |
| if unzip -l apis/python/dist/tiledbsoma-*.whl | grep -q libtiledb.so | |
| then | |
| echo "libtiledb.so was copied into the wheel when it was built against an external shared object" | |
| exit 1 | |
| fi | |
| - name: Confirm libtiledbsoma.so is copied | |
| if: ${{ matrix.TILEDBSOMA_EXISTS == 'no' }} | |
| run: unzip -l apis/python/dist/tiledbsoma-*.whl | grep -q libtiledbsoma.so | |
| - name: Confirm libtiledbsoma.so is **not** copied when using external shared object | |
| if: ${{ matrix.TILEDBSOMA_EXISTS == 'yes' }} | |
| run: | | |
| if unzip -l apis/python/dist/tiledbsoma-*.whl | grep -q libtiledbsoma.so | |
| then | |
| echo "libtiledbsoma.so was copied into the wheel when it was built against an external shared object" | |
| exit 1 | |
| fi | |
| - name: Install wheel | |
| run: ./venv-soma/bin/python -m pip install --prefer-binary apis/python/dist/tiledbsoma-*.whl | |
| - name: Check linking and RPATH | |
| run: | | |
| ldd ./venv-soma/lib/python*/site-packages/tiledbsoma/pytiledbsoma.*.so | |
| readelf -d ./venv-soma/lib/python*/site-packages/tiledbsoma/pytiledbsoma.*.so | grep R*PATH | |
| - name: Runtime test | |
| run: ./venv-soma/bin/python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())" | |
| - name: Confirm linking to installed shared objects | |
| run: | | |
| rm -fr build/ build-libtiledbsoma/ dist/ apis/python/build apis/python/src/tiledbsoma/*tile*.so* | |
| # should only show shared objects installed in virtual env or in ./external/ | |
| find . -name '*tile*.so*' | |
| ./venv-soma/bin/python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())" | |
| # Confirm shared object copying when building the Python package | |
| # https://github.com/single-cell-data/TileDB-SOMA/pull/1937 | |
| # Same as job above, but running on macOS instead of in a Docker container | |
| macos: | |
| runs-on: macos-13 | |
| name: "SO copying (macos) TILEDB_EXISTS: ${{ matrix.TILEDB_EXISTS }} TILEDBSOMA_EXISTS: ${{ matrix.TILEDBSOMA_EXISTS }}" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| TILEDB_EXISTS: ["no", "yes"] | |
| TILEDBSOMA_EXISTS: ["no", "yes"] | |
| exclude: | |
| - TILEDB_EXISTS: "no" | |
| TILEDBSOMA_EXISTS: "yes" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # for setuptools-scm | |
| - name: Check if System Integrity Protection (SIP) is enabled | |
| run: csrutil status | |
| - name: Install pre-built libtiledb | |
| if: ${{ matrix.TILEDB_EXISTS == 'yes' }} | |
| run: | | |
| mkdir -p external | |
| # Please do not edit manually -- let scripts/update-tiledb-version.py update this | |
| wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.23.0/tiledb-macos-x86_64-2.23.0-152093b.tar.gz | |
| tar -C external -xzf tiledb-macos-x86_64-*.tar.gz | |
| ls external/lib/ | |
| echo "DYLD_LIBRARY_PATH=$(pwd)/external/lib" >> $GITHUB_ENV | |
| echo "PKG_CONFIG_PATH=$(pwd)/external/lib/pkgconfig" >> $GITHUB_ENV | |
| echo "TILEDB_PATH=$(pwd)/external" >> $GITHUB_ENV | |
| - name: Build and install libtiledbsoma | |
| if: ${{ matrix.TILEDBSOMA_EXISTS == 'yes' }} | |
| run: | | |
| cmake -S libtiledbsoma -B build-libtiledbsoma \ | |
| -D CMAKE_BUILD_TYPE=Release \ | |
| -D CMAKE_PREFIX_PATH=$(pwd)/external/ \ | |
| -D CMAKE_INSTALL_PREFIX:PATH=$(pwd)/external/ \ | |
| -D OVERRIDE_INSTALL_PREFIX=OFF \ | |
| -D DOWNLOAD_TILEDB_PREBUILT=OFF \ | |
| -D FORCE_BUILD_TILEDB=OFF | |
| cmake --build build-libtiledbsoma -j $(nproc) | |
| cmake --build build-libtiledbsoma --target install-libtiledbsoma | |
| ls external/lib/ | |
| echo "TILEDBSOMA_PATH=$(pwd)/external" >> $GITHUB_ENV | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Setup Python virtual env | |
| run: | | |
| python --version | |
| python -m venv ./venv-soma | |
| ./venv-soma/bin/python -m pip install --prefer-binary pybind11-global typeguard sparse wheel setuptools | |
| ./venv-soma/bin/python -m pip list | |
| - name: Build wheel | |
| run: | | |
| echo env vars: $DYLD_LIBRARY_PATH $PKG_CONFIG_PATH $TILEDB_PATH $TILEDBSOMA_PATH | |
| cd apis/python | |
| ../../venv-soma/bin/python setup.py bdist_wheel | |
| - name: Inspect wheel | |
| run: unzip -l apis/python/dist/tiledbsoma-*.whl | grep -e '\.dylib' -e '\.so' | |
| - name: Confirm libtiledb.dylib is copied | |
| if: ${{ matrix.TILEDB_EXISTS == 'no' }} | |
| run: unzip -l apis/python/dist/tiledbsoma-*.whl | grep -q libtiledb.dylib | |
| - name: Confirm libtiledb.dylib is **not** copied when using external shared object | |
| if: ${{ matrix.TILEDB_EXISTS == 'yes' }} | |
| run: | | |
| if unzip -l apis/python/dist/tiledbsoma-*.whl | grep -q libtiledb.dylib | |
| then | |
| echo "libtiledb.dylib was copied into the wheel when it was built against an external shared object" | |
| exit 1 | |
| fi | |
| - name: Confirm libtiledbsoma.dylib is copied | |
| if: ${{ matrix.TILEDBSOMA_EXISTS == 'no' }} | |
| run: unzip -l apis/python/dist/tiledbsoma-*.whl | grep -q libtiledbsoma.dylib | |
| - name: Confirm libtiledbsoma.dylib is **not** copied when using external shared object | |
| if: ${{ matrix.TILEDBSOMA_EXISTS == 'yes' }} | |
| run: | | |
| if unzip -l apis/python/dist/tiledbsoma-*.whl | grep -q libtiledbsoma.dylib | |
| then | |
| echo "libtiledbsoma.dylib was copied into the wheel when it was built against an external shared object" | |
| exit 1 | |
| fi | |
| - name: Install wheel | |
| run: ./venv-soma/bin/python -m pip install --prefer-binary apis/python/dist/tiledbsoma-*.whl | |
| - name: Check linking and RPATH | |
| run: otool -L ./venv-soma/lib/python*/site-packages/tiledbsoma/pytiledbsoma.*.so | |
| - name: Runtime test | |
| run: ./venv-soma/bin/python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())" | |
| - name: Confirm linking to installed shared objects | |
| run: | | |
| rm -fr build/ build-libtiledbsoma/ dist/ apis/python/build apis/python/src/tiledbsoma/*tile*.dylib* | |
| # should only show shared objects installed in virtual env or in ./external/ | |
| find . -name '*tile*.so*' | |
| find . -name '*tile*.dylib*' | |
| ./venv-soma/bin/python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())" | |
| # Tests that the --libtiledbsoma flag to setup.py continues working | |
| setuptools: | |
| runs-on: ${{ matrix.os }} | |
| name: "setuptools (${{ matrix.os }})" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: ["ubuntu-22.04", "macos-12"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # for setuptools-scm | |
| - name: Install pre-built libtiledb | |
| run: | | |
| mkdir -p external | |
| if [ `uname -s` == "Darwin" ]; | |
| then | |
| # Please do not edit manually -- let scripts/update-tiledb-version.py update this | |
| wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.23.0/tiledb-macos-x86_64-2.23.0-152093b.tar.gz | |
| else | |
| # Please do not edit manually -- let scripts/update-tiledb-version.py update this | |
| wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.23.0/tiledb-linux-x86_64-2.23.0-152093b.tar.gz | |
| fi | |
| tar -C external -xzf tiledb-*.tar.gz | |
| ls external/lib/ | |
| - name: Build and install libtiledbsoma | |
| run: | | |
| cmake -S libtiledbsoma -B build-libtiledbsoma \ | |
| -D CMAKE_BUILD_TYPE=Release \ | |
| -D CMAKE_PREFIX_PATH=$(pwd)/external/ \ | |
| -D CMAKE_INSTALL_PREFIX:PATH=$(pwd)/external/ \ | |
| -D OVERRIDE_INSTALL_PREFIX=OFF \ | |
| -D DOWNLOAD_TILEDB_PREBUILT=OFF \ | |
| -D FORCE_BUILD_TILEDB=OFF | |
| cmake --build build-libtiledbsoma -j 2 | |
| cmake --build build-libtiledbsoma --target install-libtiledbsoma | |
| ls external/lib/ | |
| # Delete all cmake executables from the runner. This will ensure that | |
| # tiledbsoma-py has to use the cli flags to find the external | |
| # libtiledbsoma.so and not build it from source by shelling out to cmake | |
| - name: Delete cmake | |
| run: | | |
| echo before | |
| which -a cmake | |
| which -a cmake | xargs sudo rm -f | |
| echo after | |
| which -a cmake || echo cmake removed | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Setup Python virtual env | |
| run: | | |
| python --version | |
| python -m venv ./venv-soma | |
| ./venv-soma/bin/python -m pip install --prefer-binary pybind11-global typeguard sparse wheel setuptools | |
| ./venv-soma/bin/python -m pip list | |
| - name: Install TileDB-SOMA-Py with setuptools and --libtiledbsoma | |
| run: | | |
| cd apis/python | |
| ../../venv-soma/bin/python setup.py install \ | |
| --single-version-externally-managed \ | |
| --record record.txt \ | |
| --tiledb=$GITHUB_WORKSPACE/external/ \ | |
| --libtiledbsoma=$GITHUB_WORKSPACE/external/ | |
| - name: Check linking and RPATH (Linux) | |
| if: runner.os == 'Linux' | |
| run: | | |
| ldd ./venv-soma/lib/python*/site-packages/tiledbsoma/pytiledbsoma.*.so | |
| readelf -d ./venv-soma/lib/python*/site-packages/tiledbsoma/pytiledbsoma.*.so | grep R*PATH | |
| - name: Check linking and RPATH (macOS) | |
| if: runner.os == 'macOS' | |
| run: | | |
| otool -L ./venv-soma/lib/python*/site-packages/tiledbsoma/pytiledbsoma.*.so | |
| otool -l ./venv-soma/lib/python*/site-packages/tiledbsoma/pytiledbsoma.*.so | |
| - name: Install runtime dependencies | |
| run: | | |
| grep -v '^\[' apis/python/src/tiledbsoma.egg-info/requires.txt >/tmp/filtered-requirements.txt | |
| ./venv-soma/bin/pip install --prefer-binary -r /tmp/filtered-requirements.txt | |
| - name: Runtime test | |
| run: ./venv-soma/bin/python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())" | |
| # Build a wheel from a source tarball (confirms all required files are | |
| # distributed in the sdist). This is in preparation for the official building of | |
| # the PyPI wheels in python-packaging.yml. | |
| # https://github.com/single-cell-data/TileDB-SOMA/pull/2506 | |
| # | |
| # | |
| # Differences to python-packaging.yml: | |
| # | |
| # * Uses transparent python commands instead of the opaque action | |
| # `pypa/cibuildwheel`, and therefore easier to debug | |
| # * Only builds two wheels total (python 3.11 on Ubuntu and macOS), so it is | |
| # much faster. It's not creating the wheels for distribution, but instead just | |
| # providing a quick check that all the required files are distributed in the | |
| # source tarball | |
| # * Runs whenever an installation-related file is modified instead of only | |
| # after a release | |
| # | |
| # In summary, the goal is to identify any potential problems with building | |
| # the PyPI wheels when a PR is submitted, and not at release time. | |
| sdist: | |
| runs-on: ${{ matrix.os }} | |
| name: "Wheel from sdist (${{ matrix.os }})" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: ["ubuntu-22.04", "macos-12"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # for setuptools-scm | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Install dependencies | |
| run: python -m pip install --prefer-binary pybind11 wheel | |
| - name: Build source tarball (sdist) | |
| run: | | |
| cd apis/python | |
| python setup.py sdist | |
| - name: Extract source tarball | |
| run: | | |
| tar --list -f apis/python/dist/tiledbsoma-*.tar.gz | |
| tar -xzf apis/python/dist/tiledbsoma-*.tar.gz | |
| - name: Build wheel | |
| run: | | |
| cd tiledbsoma-*/ | |
| python setup.py bdist_wheel | |
| - name: Install wheel | |
| run: | | |
| python -m pip install --prefer-binary tiledbsoma-*/dist/tiledbsoma-*.whl | |
| python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())" |