chore(ci): fix wheel builds

- Update versions to match upstream
- Add 3.8 support since upstream uses 3.8 as the base
- Adjust CI to test both 3.8 and 3.12

Fixes #1610.

.env

@@ -31,7 +31,7 @@ ARCH_CONDA_FORGE=linux_64_
 JDK=8
 MANYLINUX=2014
 MAVEN=3.5.4
-PYTHON=3.10
+PYTHON=3.8
 GO=1.19.5
 ARROW_MAJOR_VERSION=14
 DOTNET=7.0
@@ -40,7 +40,7 @@ DOTNET=7.0
 # ci/scripts/install_vcpkg.sh script. Keep in sync with apache/arrow .env.
 # When updating, also update the docs, which list the version of libpq/SQLite
 # that vcpkg (and hence our wheels) ship
-VCPKG="501db0f17ef6df184fcdbfbe0f87cde2313b6ab1"
+VCPKG="a42af01b72c28a8e1d7b48107b33e4f286a55ef6"
 
 # These are used to tell tests where to find services for integration testing.
 # They are valid if the services are started with the docker-compose config.

.github/workflows/packaging.yml

@@ -544,16 +544,46 @@ jobs:
             adbc/python/adbc_driver_sqlite/repaired_wheels/*.whl
             adbc/python/adbc_driver_snowflake/repaired_wheels/*.whl
 
-      - name: Test wheel
+      - name: Test wheel 3.8
+        env:
+          ARCH: ${{ matrix.arch }}
+          MANYLINUX: ${{ matrix.manylinux_version }}
+        run: |
+          pushd adbc
+          env PYTHON=3.8 docker compose run python-wheel-manylinux-test
+
+      - name: Test wheel 3.8
         env:
           ARCH: ${{ matrix.arch }}
           MANYLINUX: ${{ matrix.manylinux_version }}
         run: |
           pushd adbc
           env PYTHON=3.9 docker compose run python-wheel-manylinux-test
+
+      - name: Test wheel 3.8
+        env:
+          ARCH: ${{ matrix.arch }}
+          MANYLINUX: ${{ matrix.manylinux_version }}
+        run: |
+          pushd adbc
           env PYTHON=3.10 docker compose run python-wheel-manylinux-test
+
+      - name: Test wheel 3.8
+        env:
+          ARCH: ${{ matrix.arch }}
+          MANYLINUX: ${{ matrix.manylinux_version }}
+        run: |
+          pushd adbc
           env PYTHON=3.11 docker compose run python-wheel-manylinux-test
 
+      - name: Test wheel 3.8
+        env:
+          ARCH: ${{ matrix.arch }}
+          MANYLINUX: ${{ matrix.manylinux_version }}
+        run: |
+          pushd adbc
+          env PYTHON=3.12 docker compose run python-wheel-manylinux-test
+
   python-macos:
     name: "Python ${{ matrix.arch }} macOS"
     runs-on: macos-latest
@@ -618,9 +648,11 @@ jobs:
       - name: Install Python
         run: |
           pushd adbc
+          sudo ci/scripts/install_python.sh macos 3.8
           sudo ci/scripts/install_python.sh macos 3.9
           sudo ci/scripts/install_python.sh macos 3.10
           sudo ci/scripts/install_python.sh macos 3.11
+          sudo ci/scripts/install_python.sh macos 3.12
           popd
 
       - name: Build wheel
@@ -645,7 +677,18 @@ jobs:
             adbc/python/adbc_driver_sqlite/repaired_wheels/*.whl
             adbc/python/adbc_driver_snowflake/repaired_wheels/*.whl
 
-      - name: Test wheel
+      - name: Test wheel 3.8
+        if: matrix.arch == 'amd64'
+        run: |
+          pushd adbc
+
+          /Library/Frameworks/Python.framework/Versions/3.8/bin/python3.8 -m venv test-env-38
+          source test-env-38/bin/activate
+          export PYTHON_VERSION=3.8
+          ./ci/scripts/python_wheel_unix_test.sh $(pwd)
+          deactivate
+
+      - name: Test wheel 3.9
         if: matrix.arch == 'amd64'
         run: |
           pushd adbc
@@ -656,19 +699,38 @@ jobs:
           ./ci/scripts/python_wheel_unix_test.sh $(pwd)
           deactivate
 
+      - name: Test wheel 3.10
+        if: matrix.arch == 'amd64'
+        run: |
+          pushd adbc
+
           /Library/Frameworks/Python.framework/Versions/3.10/bin/python3.10 -m venv test-env-310
           source test-env-310/bin/activate
           export PYTHON_VERSION=3.10
           ./ci/scripts/python_wheel_unix_test.sh $(pwd)
           deactivate
 
+      - name: Test wheel 3.11
+        if: matrix.arch == 'amd64'
+        run: |
+          pushd adbc
+
           /Library/Frameworks/Python.framework/Versions/3.11/bin/python3.11 -m venv test-env-311
           source test-env-311/bin/activate
           export PYTHON_VERSION=3.11
           ./ci/scripts/python_wheel_unix_test.sh $(pwd)
           deactivate
 
-          popd
+      - name: Test wheel 3.12
+        if: matrix.arch == 'amd64'
+        run: |
+          pushd adbc
+
+          /Library/Frameworks/Python.framework/Versions/3.12/bin/python3.12 -m venv test-env-312
+          source test-env-312/bin/activate
+          export PYTHON_VERSION=3.12
+          ./ci/scripts/python_wheel_unix_test.sh $(pwd)
+          deactivate
 
   python-windows:
     name: "Python ${{ matrix.python_version }} Windows"

ci/conda_env_python.txt

@@ -16,6 +16,7 @@
 # under the License.
 
 Cython
+importlib-resources
 pandas
 pyarrow>=8.0.0
 pyright

ci/scripts/install_python.sh

@@ -19,19 +19,19 @@
 
 # From apache/arrow/ci/scripts/install_python.sh
 
-set -eu
+set -euxo pipefail
 
 declare -A platforms
 platforms=([windows]=Windows
            [macos]=MacOSX
            [linux]=Linux)
 
 declare -A versions
-versions=([3.7]=3.7.9
-          [3.8]=3.8.10
+versions=([3.8]=3.8.10
           [3.9]=3.9.13
-          [3.10]=3.10.8
-          [3.11]=3.11.0)
+          [3.10]=3.10.11
+          [3.11]=3.11.8
+          [3.12]=3.12.2)
 
 if [ "$#" -ne 2 ]; then
   echo "Usage: $0 <platform> <version>"
@@ -48,7 +48,7 @@ full_version=${versions[$2]}
 if [ $platform = "MacOSX" ]; then
     echo "Downloading Python installer..."
 
-    if [ "$(uname -m)" = "arm64" ] || [ "$version" = "3.10" ] || [ "$version" = "3.11" ]; then
+    if [ "$(uname -m)" = "arm64" ] || [ "$version" = "3.10" ] || [ "$version" = "3.11" ] || [ "$version" = "3.12" ]; then
         fname="python-${full_version}-macos11.pkg"
     else
         fname="python-${full_version}-macosx10.9.pkg"

ci/scripts/python_sdist_test.sh

@@ -47,7 +47,7 @@ echo "=== Installing sdists ==="
 for component in ${COMPONENTS}; do
     pip install --no-deps --force-reinstall ${source_dir}/python/${component}/dist/*.tar.gz
 done
-pip install pytest pyarrow pandas protobuf
+pip install importlib-resources pytest pyarrow pandas protobuf
 
 echo "=== (${PYTHON_VERSION}) Testing sdists ==="
 test_packages

ci/scripts/python_wheel_unix_test.sh

@@ -49,7 +49,7 @@ for component in ${COMPONENTS}; do
         echo "NOTE: assuming wheels are already installed"
     fi
 done
-pip install pytest pyarrow pandas protobuf
+pip install importlib-resources pytest pyarrow pandas protobuf
 
 
 echo "=== (${PYTHON_VERSION}) Testing wheels ==="

ci/scripts/python_wheel_windows_test.bat

@@ -27,7 +27,7 @@ FOR %%c IN (adbc_driver_manager adbc_driver_flightsql adbc_driver_postgresql adb
     )
 )
 
-pip install pytest pyarrow pandas protobuf
+pip install importlib-resources pytest pyarrow pandas protobuf
 
 echo "=== (%PYTHON_VERSION%) Testing wheels ==="
 

python/adbc_driver_flightsql/adbc_driver_flightsql/__init__.py

@@ -176,16 +176,17 @@ def connect(
     )
 
 
-@functools.cache
+@functools.lru_cache
 def _driver_path() -> str:
-    import importlib.resources
     import pathlib
     import sys
 
+    import importlib_resources
+
     driver = "adbc_driver_flightsql"
 
     # Wheels bundle the shared library
-    root = importlib.resources.files(driver)
+    root = importlib_resources.files(driver)
     # The filename is always the same regardless of platform
     entrypoint = root.joinpath(f"lib{driver}.so")
     if entrypoint.is_file():

python/adbc_driver_flightsql/pyproject.toml

@@ -21,10 +21,11 @@ description = "An ADBC driver for working with Apache Arrow Flight SQL."
 authors = [{name = "Apache Arrow Developers", email = "[email protected]"}]
 license = {text = "Apache-2.0"}
 readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.8"
 dynamic = ["version"]
 dependencies = [
     "adbc-driver-manager",
+    "importlib-resources >=1.3",
 ]
 
 [project.optional-dependencies]

python/adbc_driver_manager/adbc_driver_manager/_lib.pyx

@@ -1498,7 +1498,7 @@ cdef extern from "_blocking_impl.h" nogil:
     c_string CClearBlockingCallback"pyadbc_driver_manager::ClearBlockingCallback"()
 
 
-@functools.cache
+@functools.lru_cache
 def _init_blocking_call():
     error = bytes(CInitBlockingCallback()).decode("utf-8")
     if error:

python/adbc_driver_manager/pyproject.toml

@@ -21,7 +21,7 @@ description = "A generic entrypoint for ADBC drivers."
 authors = [{name = "Apache Arrow Developers", email = "[email protected]"}]
 license = {text = "Apache-2.0"}
 readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.8"
 dynamic = ["version"]
 dependencies = ["typing-extensions"]
 

python/adbc_driver_postgresql/adbc_driver_postgresql/__init__.py

@@ -40,16 +40,17 @@ def connect(uri: str) -> adbc_driver_manager.AdbcDatabase:
     return adbc_driver_manager.AdbcDatabase(driver=_driver_path(), uri=uri)
 
 
-@functools.cache
+@functools.lru_cache
 def _driver_path() -> str:
-    import importlib.resources
     import pathlib
     import sys
 
+    import importlib_resources
+
     driver = "adbc_driver_postgresql"
 
     # Wheels bundle the shared library
-    root = importlib.resources.files(driver)
+    root = importlib_resources.files(driver)
     # The filename is always the same regardless of platform
     entrypoint = root.joinpath(f"lib{driver}.so")
     if entrypoint.is_file():

python/adbc_driver_postgresql/pyproject.toml

@@ -21,10 +21,11 @@ description = "A libpq-based ADBC driver for working with PostgreSQL."
 authors = [{name = "Apache Arrow Developers", email = "[email protected]"}]
 license = {text = "Apache-2.0"}
 readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.8"
 dynamic = ["version"]
 dependencies = [
     "adbc-driver-manager",
+    "importlib-resources >=1.3",
 ]
 
 [project.optional-dependencies]

python/adbc_driver_postgresql/tests/test_lowlevel.py

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import collections.abc
+import typing
 
 import pyarrow
 import pytest
@@ -27,7 +27,7 @@
 @pytest.fixture
 def postgres(
     postgres_uri: str,
-) -> collections.abc.Generator[adbc_driver_manager.AdbcConnection, None, None]:
+) -> typing.Generator[adbc_driver_manager.AdbcConnection, None, None]:
     with adbc_driver_postgresql.connect(postgres_uri) as db:
         with adbc_driver_manager.AdbcConnection(db) as conn:
             yield conn

python/adbc_driver_snowflake/adbc_driver_snowflake/__init__.py

@@ -138,16 +138,17 @@ def connect(
     return adbc_driver_manager.AdbcDatabase(driver=_driver_path(), **kwargs)
 
 
-@functools.cache
+@functools.lru_cache
 def _driver_path() -> str:
-    import importlib.resources
     import pathlib
     import sys
 
+    import importlib_resources
+
     driver = "adbc_driver_snowflake"
 
     # Wheels bundle the shared library
-    root = importlib.resources.files(driver)
+    root = importlib_resources.files(driver)
     # The filename is always the same regardless of platform
     entrypoint = root.joinpath(f"lib{driver}.so")
     if entrypoint.is_file():

python/adbc_driver_snowflake/pyproject.toml

@@ -21,10 +21,11 @@ description = "An ADBC driver for working with Snowflake."
 authors = [{name = "Apache Arrow Developers", email = "[email protected]"}]
 license = {text = "Apache-2.0"}
 readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.8"
 dynamic = ["version"]
 dependencies = [
     "adbc-driver-manager",
+    "importlib-resources >=1.3",
 ]
 
 [project.optional-dependencies]

python/adbc_driver_sqlite/adbc_driver_sqlite/__init__.py

@@ -60,16 +60,17 @@ def connect(uri: typing.Optional[str] = None) -> adbc_driver_manager.AdbcDatabas
     return adbc_driver_manager.AdbcDatabase(driver=_driver_path(), uri=uri)
 
 
-@functools.cache
+@functools.lru_cache
 def _driver_path() -> str:
-    import importlib.resources
     import pathlib
     import sys
 
+    import importlib_resources
+
     driver = "adbc_driver_sqlite"
 
     # Wheels bundle the shared library
-    root = importlib.resources.files(driver)
+    root = importlib_resources.files(driver)
     # The filename is always the same regardless of platform
     entrypoint = root.joinpath(f"lib{driver}.so")
     if entrypoint.is_file():

python/adbc_driver_sqlite/pyproject.toml

@@ -21,10 +21,11 @@ description = "An ADBC driver for working with SQLite."
 authors = [{name = "Apache Arrow Developers", email = "[email protected]"}]
 license = {text = "Apache-2.0"}
 readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.8"
 dynamic = ["version"]
 dependencies = [
     "adbc-driver-manager",
+    "importlib-resources >=1.3",
 ]
 
 [project.optional-dependencies]