Bulk-sync main to release-1.15 (#3251)

* Bulk-sync `main` to `release-1.15`

* add in some binary files

.github/workflows/libtiledb-ci.yml

@@ -19,7 +19,7 @@ jobs:
     - name: Checkout TileDB-SOMA
       uses: actions/checkout@v4
     - name: Build libTileDB-SOMA
-      run: TILEDBSOMA_COVERAGE="--coverage" ./scripts/bld --no-tiledb-deprecated=true
+      run: TILEDBSOMA_COVERAGE="--coverage" ./scripts/bld --no-tiledb-deprecated=true --werror=true
     - name: Run libTileDB-SOMA unittests
       run: ctest --test-dir build/libtiledbsoma -C Release --verbose --rerun-failed --output-on-failure
     - name: Upload coverage to Codecov

.github/workflows/python-ci-full.yml

@@ -21,16 +21,16 @@ jobs:
       fail-fast: false
       matrix:
         # TODO: decide on Windows CI coverage
-        os: [ubuntu-22.04, macos-12]
-        # os: [ubuntu-22.04, macos-12, windows-2019]
+        os: [ubuntu-latest, macos-latest]
+        # os: [ubuntu-latest, macos-latest, windows-2019]
         # TODO: add 3.12
         # https://github.com/single-cell-data/TileDB-SOMA/issues/1849
-        python-version: ['3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.11', '3.12']
         include:
-          - runs-on: ubuntu-22.04
+          - runs-on: ubuntu-latest
             cc: gcc-11
             cxx: g++-11
-          - runs-on: macos-12
+          - runs-on: macos-latest
             cc: clang
             cxx: clang++
     uses: ./.github/workflows/python-ci-single.yml
@@ -39,6 +39,6 @@ jobs:
       python_version: ${{ matrix.python-version }}
       cc: ${{ matrix.cc }}
       cxx: ${{ matrix.cxx }}
-      report_codecov: ${{ matrix.os == 'ubuntu-22.04' && matrix.python-version == '3.11' }}
-      run_lint: ${{ matrix.os == 'ubuntu-22.04' && matrix.python-version == '3.11' }}
+      report_codecov: ${{ matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11' }}
+      run_lint: ${{ matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11' }}
     secrets: inherit

.github/workflows/python-ci-minimal.yml

@@ -28,13 +28,13 @@ jobs:
       fail-fast: true
 
       matrix:
-        os: [ubuntu-22.04, macos-12]
-        python-version: ['3.9', '3.11']
+        os: [ubuntu-latest, macos-latest]
+        python-version: ['3.9', '3.12']
         include:
-          - os: ubuntu-22.04
+          - os: ubuntu-latest
             cc: gcc-11
             cxx: g++-11
-          - os: macos-12
+          - os: macos-latest
             cc: clang
             cxx: clang++
     uses: ./.github/workflows/python-ci-single.yml
@@ -43,6 +43,6 @@ jobs:
       python_version: ${{ matrix.python-version }}
       cc: ${{ matrix.cc }}
       cxx: ${{ matrix.cxx }}
-      report_codecov: ${{ matrix.python-version == '3.11' }}
-      run_lint: ${{ matrix.python-version == '3.11' }}
+      report_codecov: ${{ matrix.python-version == '3.12' }}
+      run_lint: ${{ matrix.python-version == '3.12' }}
     secrets: inherit

.github/workflows/python-ci-packaging.yml

@@ -50,7 +50,7 @@ jobs:
           - TILEDB_EXISTS: "no"
             TILEDBSOMA_EXISTS: "yes"
     container:
-      image: ubuntu:22.04
+      image: ubuntu:latest
     defaults:
       run:
         shell: bash
@@ -92,16 +92,17 @@ jobs:
             -D OVERRIDE_INSTALL_PREFIX=OFF \
             -D DOWNLOAD_TILEDB_PREBUILT=OFF \
             -D TILEDB_REMOVE_DEPRECATIONS=ON \
+            -D TILEDBSOMA_ENABLE_WERROR=ON \
             -D FORCE_BUILD_TILEDB=OFF
-          cmake --build build-libtiledbsoma -j $(nproc)
+          cmake --build build-libtiledbsoma -j $(nproc) --verbose
           cmake --build build-libtiledbsoma --target install-libtiledbsoma
           ls external/lib/
           echo "TILEDBSOMA_PATH=$(pwd)/external" >> $GITHUB_ENV
       - name: Setup Python
         run: |
           python --version
           python -m venv ./venv-soma
-          ./venv-soma/bin/pip install --prefer-binary pybind11-global typeguard sparse 'setuptools>=70.1' wheel
+          ./venv-soma/bin/pip install --prefer-binary pybind11-global typeguard sparse wheel 'setuptools>=70.1'
           ./venv-soma/bin/pip list
       - name: Build wheel
         run: |
@@ -172,8 +173,13 @@ jobs:
         run: |
           mkdir -p external
           # Please do not edit manually -- let scripts/update-tiledb-version.py update this
-          wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.26.2/tiledb-macos-x86_64-2.26.2-30fc114.tar.gz
-          tar -C external -xzf tiledb-macos-x86_64-*.tar.gz
+          if [ `uname -m` == "arm64" ]; then
+            wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.26.2/tiledb-macos-arm64-2.26.2-30fc114.tar.gz
+            tar -C external -xzf tiledb-macos-arm64-*.tar.gz
+          else
+            wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.26.2/tiledb-macos-x86_64-2.26.2-30fc114.tar.gz
+            tar -C external -xzf tiledb-macos-x86_64-*.tar.gz
+          fi
           ls external/lib/
           echo "DYLD_LIBRARY_PATH=$(pwd)/external/lib" >> $GITHUB_ENV
           echo "PKG_CONFIG_PATH=$(pwd)/external/lib/pkgconfig" >> $GITHUB_ENV
@@ -253,18 +259,22 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-22.04", "macos-12"]
+        os: ["ubuntu-latest", "macos-latest"]
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0  # for setuptools-scm
       - name: Install pre-built libtiledb
         run: |
           mkdir -p external
-          if [ `uname -s` == "Darwin" ];
-          then
-            # Please do not edit manually -- let scripts/update-tiledb-version.py update this
-            wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.26.2/tiledb-macos-x86_64-2.26.2-30fc114.tar.gz
+          if [ `uname -s` == "Darwin" ]; then
+            if [ `uname -m` == "arm64" ]; then
+                # Please do not edit manually -- let scripts/update-tiledb-version.py update this
+                wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.26.2/tiledb-macos-arm64-2.26.2-30fc114.tar.gz
+            else
+                # Please do not edit manually -- let scripts/update-tiledb-version.py update this
+                wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.26.2/tiledb-macos-x86_64-2.26.2-30fc114.tar.gz
+            fi
           else
             # Please do not edit manually -- let scripts/update-tiledb-version.py update this
             wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.26.2/tiledb-linux-x86_64-2.26.2-30fc114.tar.gz
@@ -323,7 +333,9 @@ jobs:
           otool -L ./venv-soma/lib/python*/site-packages/tiledbsoma/pytiledbsoma.*.so
           otool -l ./venv-soma/lib/python*/site-packages/tiledbsoma/pytiledbsoma.*.so
       - name: Install runtime dependencies
-        run: ./venv-soma/bin/pip install --prefer-binary `grep -v '^\[' apis/python/src/tiledbsoma.egg-info/requires.txt`
+        run: |
+          grep -v '^\[' apis/python/src/tiledbsoma.egg-info/requires.txt > runtime-reqs.txt
+          ./venv-soma/bin/pip install --prefer-binary -r runtime-reqs.txt
       - name: Runtime test
         run: ./venv-soma/bin/python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())"
 
@@ -352,7 +364,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-22.04", "macos-12"]
+        os: ["ubuntu-latest", "macos-latest"]
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/python-ci-single.yml

@@ -70,18 +70,17 @@ jobs:
       run: echo "inputs.os:" ${{ inputs.os }}
 
     - name: Linux CPU info
-      if: ${{ inputs.os == 'ubuntu-22.04' }}
+      if: ${{ inputs.os == 'ubuntu-latest' }}
       run: cat /proc/cpuinfo
 
     - name: MacOS CPU info
-      if: ${{ inputs.os == 'macos-12' }}
+      if: ${{ inputs.os == 'macos-latest' }}
       run: sysctl -a | grep cpu
     - name: Select XCode version
       if: startsWith(inputs.os, 'macos')
       uses: maxim-lobanov/setup-xcode@v1
       with:
-        # Pending https://github.com/actions/runner-images/issues/6350
-        xcode-version: '13.4'
+        xcode-version: '15.4'
 
     - name: Checkout TileDB-SOMA
       uses: actions/checkout@v4
@@ -134,14 +133,14 @@ jobs:
       # Setting PYTHONPATH ensures the tests load the in-tree source code under apis/python/src
       # instead of the copy we `pip install`ed to site-packages above. That's needed for the code
       # coverage analysis to work.
-      run: PYTHONPATH=$(pwd)/apis/python/src python -m pytest --cov=apis/python/src --cov-report=xml apis/python/tests -v --durations=20 --maxfail=50
+      run: export SOMA_PY_NEW_SHAPE=false; PYTHONPATH=$(pwd)/apis/python/src python -m pytest --cov=apis/python/src --cov-report=xml apis/python/tests -v --durations=20 --maxfail=50
 
     - name: Run pytests for Python with new shape
       shell: bash
       # Setting PYTHONPATH ensures the tests load the in-tree source code under apis/python/src
       # instead of the copy we `pip install`ed to site-packages above. That's needed for the code
       # coverage analysis to work.
-      run: export SOMA_PY_NEW_SHAPE=true; PYTHONPATH=$(pwd)/apis/python/src python -m pytest --cov=apis/python/src --cov-report=xml apis/python/tests -v --durations=20 --maxfail=50
+      run: PYTHONPATH=$(pwd)/apis/python/src python -m pytest --cov=apis/python/src --cov-report=xml apis/python/tests -v --durations=20 --maxfail=50
 
     - name: Report coverage to Codecov
       if: inputs.report_codecov

.github/workflows/python-packaging.yml

@@ -21,7 +21,7 @@ on:
 jobs:
   sdist:
     name: Build source distribution
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout TileDB-SOMA
         uses: actions/checkout@v4
@@ -50,14 +50,14 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: [ '39', '310', '311' ]
+        python-version: [ '39', '310', '311', '312' ]
         cibw_build: [ manylinux_x86_64, macosx_x86_64, macosx_arm64 ]
         include:
           - cibw_build: manylinux_x86_64
-            os: ubuntu-20.04
+            os: ubuntu-latest
             wheel-name: manylinux2014
           - cibw_build: macosx_x86_64
-            os: macos-12
+            os: macos-latest
             cibw_archs_macos: x86_64
             wheel-name: macos-x86_64
           - cibw_build: macosx_arm64
@@ -119,18 +119,20 @@ jobs:
             dotted-version: '3.10'
           - undotted-version: '311'
             dotted-version: '3.11'
+          - undotted-version: '312'
+            dotted-version: '3.12'
         wheel-name:
           - manylinux2014
           - macos-x86_64
           - macos-arm64
         include:
           - wheel-name: manylinux2014
-            os: ubuntu-20.04
+            os: ubuntu-latest
             arch: x86_64
             cc: gcc-11
             cxx: g++-11
           - wheel-name: macos-x86_64
-            os: macos-12
+            os: macos-13
             arch: x86_64
             cc: clang
             cxx: clang++
@@ -164,7 +166,7 @@ jobs:
         run: python -c 'import tiledbsoma; print(tiledbsoma.pytiledbsoma.__file__); tiledbsoma.show_package_versions()'
         # TODO: more thorough local smoke test
       - name: Smoke test in docker
-        if: ${{ matrix.os == 'ubuntu-20.04' }}
+        if: ${{ matrix.os == 'ubuntu-latest' }}
         run: |
           docker run -v $(pwd):/mnt python:${{ matrix.python.dotted-version }} bash -ec "
             apt-get -qq update && apt-get install -y python3-pip python3-wheel
@@ -176,7 +178,7 @@ jobs:
   publish-to-test-pypi:
     name: Publish package to TestPyPI
     needs: smoke-test
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     if: github.event_name == 'workflow_dispatch'
     steps:
       - name: Download artifacts
@@ -200,7 +202,7 @@ jobs:
   publish-to-pypi:
     name: Publish package to PyPI
     needs: smoke-test
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     if: github.event_name == 'release'
     steps:
       - name: Download artifacts

.github/workflows/r-ci.yml

@@ -150,12 +150,12 @@ jobs:
 
       - name: Test without new shape
         if: ${{ matrix.covr == 'no' }}
-        run: cd apis/r/tests && Rscript testthat.R
+        run: export SOMA_R_NEW_SHAPE=false && cd apis/r/tests && Rscript testthat.R
 
       # https://github.com/single-cell-data/TileDB-SOMA/issues/2407
       - name: Test with new shape
         if: ${{ matrix.covr == 'no' }}
-        run: export SOMA_R_NEW_SHAPE=true && cd apis/r/tests && Rscript testthat.R
+        run: cd apis/r/tests && Rscript testthat.R
 
       - name: Coverage
         if: ${{ matrix.os == 'ubuntu-latest' && matrix.covr == 'yes' && github.event_name == 'workflow_dispatch' }}

.github/workflows/r-python-interop-testing.yml

@@ -80,7 +80,7 @@ jobs:
       #  run: cd apis/r && Rscript -e "options(bspm.version.check=TRUE); install.packages('tiledb', repos = c('https://eddelbuettel.r-universe.dev/bin/linux/jammy/4.3/', 'https://cloud.r-project.org'))"
 
       - name: Build and install libtiledbsoma
-        run: sudo scripts/bld --prefix=/usr/local --no-tiledb-deprecated=true && sudo ldconfig
+        run: sudo scripts/bld --prefix=/usr/local --no-tiledb-deprecated=true --werror=true && sudo ldconfig
 
       - name: Install R-tiledbsoma
         run: |
@@ -100,7 +100,7 @@ jobs:
           cache-dependency-path: ./apis/python/setup.py
 
       - name: Install tiledbsoma
-        run: pip -v install -e apis/python[dev] -C "--build-option=--no-tiledb-deprecated"
+        run: pip -v install -e apis/python[dev] -C "--build-option=--no-tiledb-deprecated "
 
       - name: Show Python package versions
         run: |
@@ -110,7 +110,14 @@ jobs:
       - name: Update Packages
         run: Rscript -e 'update.packages(ask=FALSE)'
 
-      - name: Interop Tests
+      - name: Interop tests with new-shape feature flag off
         run: python -m pytest apis/system/tests/
         env:
           TILEDB_SOMA_INIT_BUFFER_BYTES: 33554432 # accommodate tiny runners
+
+      - name: Interop tests with new-shape feature flag on
+        run: python -m pytest apis/system/tests/
+        env:
+          TILEDB_SOMA_INIT_BUFFER_BYTES: 33554432 # accommodate tiny runners
+          SOMA_PY_NEW_SHAPE: true
+          SOMA_R_NEW_SHAPE: true

.pre-commit-config.yaml

@@ -18,7 +18,7 @@ repos:
         # Pandas 2.x types (e.g. `pd.Series[Any]`). See `_types.py` or https://github.com/single-cell-data/TileDB-SOMA/issues/2839
         # for more info.
         - "pandas-stubs>=2"
-        - "somacore==1.0.20"
+        - "somacore==1.0.21"
         - types-setuptools
       args: ["--config-file=apis/python/pyproject.toml", "apis/python/src", "apis/python/devtools"]
       pass_filenames: false

Makefile

@@ -46,13 +46,7 @@ ctest_update:
 
 .PHONY: data
 data:
-	rm -rvf test/soco
-	./apis/python/devtools/ingestor \
-		--soco \
-		-o test/soco \
-		-n \
-		data/pbmc3k_processed.h5ad \
-		data/10x-pbmc-multiome-v1.0/subset_100_100.h5ad
+	cd test && rm -rf soco && tar zxf soco.tgz && cd ..
 
 # format
 # -------------------------------------------------------------------

apis/python/MANIFEST.in

@@ -3,4 +3,4 @@ include src/tiledbsoma/*.cc
 include src/tiledbsoma/*.h
 graft dist_links
 prune dist_links/libtiledbsoma/test/__pycache__
-include requirements_dev.txt
+include requirements_*.txt

apis/python/README.md

@@ -52,6 +52,7 @@ If this comes up empty for your system, you'll definitely need to build from sou
   ```
 * In either case:
   ```shell
+  make data
   python -m pytest tests
   ```
 

apis/python/requirements_dev.txt

@@ -3,5 +3,9 @@ ruff
 pytest
 pytest-cov
 sparse
-typeguard==4.2.1
+# Python 3.12 support requires https://github.com/agronholm/typeguard/pull/490, which landed between 4.3.0 and 4.4.0.
+# However, 4.4.0 also included https://github.com/agronholm/typeguard/pull/496, which causes errors in Python 3.9 (e.g.
+# https://github.com/single-cell-data/TileDB-SOMA/actions/runs/11545217103/job/32131849817), so we are pinning to the
+# last useful commit here. See also: https://github.com/single-cell-data/TileDB-SOMA/issues/3216.
+typeguard @ git+https://github.com/agronholm/typeguard@afad2c7
 types-setuptools

apis/python/requirements_spatial.txt

@@ -0,0 +1,3 @@
+tifffile
+pillow
+spatialdata