diff --git a/.github/workflows/python-core-wheels.yml b/.github/workflows/python-core-wheels.yml index 2cd2699b..281feeff 100644 --- a/.github/workflows/python-core-wheels.yml +++ b/.github/workflows/python-core-wheels.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - target: [x86_64, i686] + target: [x86_64] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 @@ -22,8 +22,11 @@ jobs: uses: PyO3/maturin-action@v1 with: target: ${{ matrix.target }} - manylinux: auto + manylinux: manylinux_2_28 args: --release --out dist -m python/core/Cargo.toml + before-script-linux: | + yum update -y + yum install openssl openssl-devel perl-IPC-Cmd -y - name: Install built wheel if: matrix.target == 'x86_64' @@ -39,79 +42,82 @@ jobs: name: wheels path: dist/*.whl - linux-cross: - runs-on: ubuntu-latest - strategy: - matrix: - target: [aarch64, armv7, s390x, ppc64le, ppc64] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: Build wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.target }} - manylinux: auto - args: --release --out dist -m python/core/Cargo.toml + # linux-cross: + # runs-on: ubuntu-latest + # strategy: + # matrix: + # target: [aarch64, armv7, s390x, ppc64le, ppc64] + # steps: + # - uses: actions/checkout@v4 + # - uses: actions/setup-python@v4 + # with: + # python-version: 3.8 - # This is currently failing with - # python: command not found - - # - uses: uraimo/run-on-arch-action@v2.5.1 - # if: matrix.target == 'aarch64' - # name: Install built wheel - # with: - # arch: ${{ matrix.target }} - # distro: ubuntu20.04 - # githubToken: ${{ github.token }} - # install: | - # apt-get update - # apt-get install -y --no-install-recommends python3 python3-pip - # pip3 install -U pip - # run: | - # pip install geoarrow-rust-core --no-index --find-links dist --force-reinstall - # python -c "import geoarrow.rust.core" + # - name: Build wheels + # uses: PyO3/maturin-action@v1 + # with: + # target: ${{ matrix.target }} + # manylinux: auto + # args: --release --out dist -m python/core/Cargo.toml - # Have to set path from root - # https://github.com/actions/upload-artifact/issues/232#issuecomment-964235360 - - name: Upload wheels - uses: actions/upload-artifact@v3 - with: - name: wheels - path: dist/*.whl + # # This is currently failing with + # # python: command not found + + # # - uses: uraimo/run-on-arch-action@v2.5.1 + # # if: matrix.target == 'aarch64' + # # name: Install built wheel + # # with: + # # arch: ${{ matrix.target }} + # # distro: ubuntu20.04 + # # githubToken: ${{ github.token }} + # # install: | + # # apt-get update + # # apt-get install -y --no-install-recommends python3 python3-pip + # # pip3 install -U pip + # # run: | + # # pip install geoarrow-rust-core --no-index --find-links dist --force-reinstall + # # python -c "import geoarrow.rust.core" + + # # Have to set path from root + # # https://github.com/actions/upload-artifact/issues/232#issuecomment-964235360 + # - name: Upload wheels + # uses: actions/upload-artifact@v3 + # with: + # name: wheels + # path: dist/*.whl macos: - runs-on: macos-latest + name: Build ${{ matrix.arch }} wheels on ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: macos-11 + arch: x86_64 + target: x86_64 + python-version: 3.8 + - os: macos-14 + arch: arm64 + target: aarch64 + # Earliest python version on Github for macos arm64 + # https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json + python-version: 3.11.3 + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: - python-version: 3.8 - architecture: x64 + python-version: ${{ matrix.python-version }} - uses: dtolnay/rust-toolchain@stable - - name: Build wheels - x86_64 + - name: Build wheels - ${{ matrix.target }} uses: PyO3/maturin-action@v1 with: - target: x86_64 + target: ${{ matrix.target }} args: --release --out dist --sdist -m python/core/Cargo.toml - - name: Install built wheel - x86_64 - run: | - pip install geoarrow-rust-core --no-index --find-links dist --force-reinstall - python -c "import geoarrow.rust.core" - - - name: Build wheels - universal2 - uses: PyO3/maturin-action@v1 - with: - target: universal2-apple-darwin - args: --release --out dist -m python/core/Cargo.toml - - - name: Install built wheel - universal2 + - name: Install built wheel - ${{ matrix.target }} run: | pip install geoarrow-rust-core --no-index --find-links dist --force-reinstall python -c "import geoarrow.rust.core" @@ -251,7 +257,7 @@ jobs: id-token: write runs-on: ubuntu-latest if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') - needs: [macos, windows, linux, linux-cross] + needs: [macos, windows, linux] steps: - uses: actions/download-artifact@v3 with: diff --git a/python/core/CHANGELOG.md b/python/core/CHANGELOG.md index 5841875e..ff18657f 100644 --- a/python/core/CHANGELOG.md +++ b/python/core/CHANGELOG.md @@ -4,37 +4,46 @@ ### New! :sparkles: -- New I/O support for reading and writing to geospatial formats: - - [Asynchronous FlatGeobuf reader](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.read_flatgeobuf_async). This also supports passing a spatial filter. - - [Initial support for reading from PostGIS](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.read_postgis). Note that not all Postgres data types are supported yet. Create an issue if your query fails. - - [Reading](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.read_geojson_lines) and [writing](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.write_geojson_lines) [newline-delimited GeoJSON](https://stevage.github.io/ndgeojson/). - - [Pyogrio integration](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.read_pyogrio) for reading from OGR/GDAL. - - [Asynchronous GeoParquet reader](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.read_parquet_async) for reading from remote files. By @weiji14 in https://github.com/geoarrow/geoarrow-rs/pull/493 - - Also new support for writing GeoParquet files. - - Most I/O readers and writers support Python file-like objects (in binary mode). - - Support for [reading](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.read_ipc) and [writing](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.write_ipc) Arrow IPC files. -- Better interoperability with the Python geospatial ecosystem. - - [Import from](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/interop/#geoarrow.rust.core.from_geopandas) and [export to](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/interop/#geoarrow.rust.core.to_geopandas) GeoPandas GeoDataFrames. Refer to the [GeoPandas interoperability documentation](https://geoarrow.org/geoarrow-rs/python/v0.2.0/ecosystem/geopandas/). - - [Import from](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/interop/#geoarrow.rust.core.from_shapely) and [export to](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/interop/#geoarrow.rust.core.to_shapely) Shapely arrays. Refer to the [Shapely interoperability documentation](https://geoarrow.org/geoarrow-rs/python/v0.2.0/ecosystem/shapely/). - - [Better integration with Lonboard](https://geoarrow.org/geoarrow-rs/python/v0.2.0/ecosystem/lonboard/) for fast visualization in Jupyter. - - All scalars, arrays, chunked arrays, and table objects implement [`__geo_interface__`](https://gist.github.com/sgillies/2217756) for interoperability with existing tools. - - Numpy interoperability for float array output from algorithms. Pass any `Float64Array` or `BooleanArray` to `numpy.asarray`. You can also pass a numpy array as vectorized input into a function like [`line_interpolate_point`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.line_interpolate_point). -- New algorithms! - - [Explode a Table](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.explode) where each multi-geometry expands into multiple rows of single geometries. - - [`total_bounds`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.total_bounds) - - Unified [`area` function](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.area) for planar and geodesic area. - - Unified [`simplify` function](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.simplify) for multiple simplification methods. Also new support for [topology-preserving simplification](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/enums/#geoarrow.rust.core.enums.SimplifyMethod.VW_Preserve). - - Unified [`length` function](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.length) for euclidean and geodesic length calculations. - - [`frechet_distance`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.frechet_distance) for LineString similarity. - - [`affine_transform`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.affine_transform), which integrates with the Python [`affine`](https://github.com/rasterio/affine) library. - - [`line_interpolate_point`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.line_interpolate_point) and [`line_locate_point`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.line_locate_point) for linear referencing. - - [`polylabel`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.polylabel) for polygon labeling. -- Improved display of Python objects: - - Scalars now display as SVG geometries inside Jupyter environments. - - Tables, arrays, and chunked arrays implement `__repr__` so that you can inspect data easily. -- Improved usability: - - [`PointArray.from_xy`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/geometry/array/#geoarrow.rust.core.PointArray.from_xy) to simplify creating a point array from numpy arrays of coordinates. - - Index into arrays and chunked arrays with square brackets. E.g. `point_array[0]` will access the first point in the array. Negative indexing is also supported, so `point_array[-1]` will get the last item in the array. +#### New I/O support for reading and writing to geospatial formats + +- [Asynchronous FlatGeobuf reader](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.read_flatgeobuf_async). This also supports passing a spatial filter. +- [Initial support for reading from PostGIS](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.read_postgis). Note that not all Postgres data types are supported yet. Create an issue if your query fails. +- [Reading](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.read_geojson_lines) and [writing](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.write_geojson_lines) [newline-delimited GeoJSON](https://stevage.github.io/ndgeojson/). +- [Pyogrio integration](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.read_pyogrio) for reading from OGR/GDAL. +- [Asynchronous GeoParquet reader](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.read_parquet_async) for reading from remote files. By @weiji14 in https://github.com/geoarrow/geoarrow-rs/pull/493 +- Also new support for writing GeoParquet files. +- Most I/O readers and writers support Python file-like objects (in binary mode). +- Support for [reading](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.read_ipc) and [writing](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/io/#geoarrow.rust.core.write_ipc) Arrow IPC files. + +#### Better interoperability with the Python geospatial ecosystem. + +- [Import from](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/interop/#geoarrow.rust.core.from_geopandas) and [export to](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/interop/#geoarrow.rust.core.to_geopandas) GeoPandas GeoDataFrames. Refer to the [GeoPandas interoperability documentation](https://geoarrow.org/geoarrow-rs/python/v0.2.0/ecosystem/geopandas/). +- [Import from](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/interop/#geoarrow.rust.core.from_shapely) and [export to](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/interop/#geoarrow.rust.core.to_shapely) Shapely arrays. Refer to the [Shapely interoperability documentation](https://geoarrow.org/geoarrow-rs/python/v0.2.0/ecosystem/shapely/). +- [Better integration with Lonboard](https://geoarrow.org/geoarrow-rs/python/v0.2.0/ecosystem/lonboard/) for fast visualization in Jupyter. +- All scalars, arrays, chunked arrays, and table objects implement [`__geo_interface__`](https://gist.github.com/sgillies/2217756) for interoperability with existing tools. +- Numpy interoperability for float array output from algorithms. Pass any `Float64Array` or `BooleanArray` to `numpy.asarray`. You can also pass a numpy array as vectorized input into a function like [`line_interpolate_point`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.line_interpolate_point). + +#### New algorithms! + +- [Explode a Table](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.explode) where each multi-geometry expands into multiple rows of single geometries. +- [`total_bounds`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.total_bounds) +- Unified [`area` function](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.area) for planar and geodesic area. +- Unified [`simplify` function](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.simplify) for multiple simplification methods. Also new support for [topology-preserving simplification](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/enums/#geoarrow.rust.core.enums.SimplifyMethod.VW_Preserve). +- Unified [`length` function](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.length) for euclidean and geodesic length calculations. +- [`frechet_distance`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.frechet_distance) for LineString similarity. +- [`affine_transform`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.affine_transform), which integrates with the Python [`affine`](https://github.com/rasterio/affine) library. +- [`line_interpolate_point`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.line_interpolate_point) and [`line_locate_point`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.line_locate_point) for linear referencing. +- [`polylabel`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/functions/#geoarrow.rust.core.polylabel) for polygon labeling. + +#### Improved display of Python objects: + +- Scalars now display as SVG geometries inside Jupyter environments. +- Tables, arrays, and chunked arrays implement `__repr__` so that you can inspect data easily. + +#### Improved usability: + +- [`PointArray.from_xy`](https://geoarrow.org/geoarrow-rs/python/v0.2.0/api/core/geometry/array/#geoarrow.rust.core.PointArray.from_xy) to simplify creating a point array from numpy arrays of coordinates. +- Index into arrays and chunked arrays with square brackets. E.g. `point_array[0]` will access the first point in the array. Negative indexing is also supported, so `point_array[-1]` will get the last item in the array. - New [top-level docs website](https://geoarrow.org/geoarrow-rs/). diff --git a/python/core/Cargo.lock b/python/core/Cargo.lock index e3c2a5ff..1731dbfc 100644 --- a/python/core/Cargo.lock +++ b/python/core/Cargo.lock @@ -1022,6 +1022,7 @@ dependencies = [ "geozero", "numpy", "object_store", + "openssl", "parquet", "pyo3", "pyo3-asyncio", diff --git a/python/core/Cargo.toml b/python/core/Cargo.toml index 1286b146..b2ac8cb2 100644 --- a/python/core/Cargo.toml +++ b/python/core/Cargo.toml @@ -55,5 +55,11 @@ tokio = { version = "1.9", features = ["rt"] } url = "2.5" # reqwest is pulled in by object store, but not used by python binding itself -# for binary wheel best practice, statically link openssl -reqwest = { version = "*", features = ["native-tls-vendored"] } +# We statically link the rust-native rustls. For a binary wheel we want a +# static-compiled openssl anyways, and having it be native rust seems to make +# things simpler. +# See native roots discussion here: +# https://github.com/rustls/rustls-native-certs/blob/0ebeee01ed61ebeacd5ebb98e7f5694b0ee70a08/README.md#should-i-use-this-or-webpki-roots +reqwest = { version = "*", features = ["rustls-tls-native-roots"] } +# Compile openssl from source +openssl = { version = "0.10", features = ["vendored"] } diff --git a/python/core/src/io/input/sync.rs b/python/core/src/io/input/sync.rs index ca741a12..abe680b3 100644 --- a/python/core/src/io/input/sync.rs +++ b/python/core/src/io/input/sync.rs @@ -11,6 +11,7 @@ use pyo3::types::{PyBytes, PyString, PyType}; use std::fs::File; use std::io::{self, BufRead, BufReader, BufWriter}; use std::io::{Read, Seek, SeekFrom, Write}; +#[cfg(not(target_os = "windows"))] use std::os::fd::{AsRawFd, RawFd}; use std::path::Path; @@ -203,6 +204,7 @@ impl Seek for PyFileLikeObject { } } +#[cfg(not(target_os = "windows"))] impl AsRawFd for PyFileLikeObject { fn as_raw_fd(&self) -> RawFd { Python::with_gil(|py| {