From c0317723f4ef0889e834cc24d40a48da3f1538db Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Fri, 25 Oct 2024 14:53:21 -0400 Subject: [PATCH] Bootstrap into subprojects directory --- .github/workflows/build-and-test-device.yaml | 3 +- .github/workflows/python-wheels.yaml | 4 +- .gitignore | 1 + ci/scripts/coverage.sh | 2 +- dev/release/rat_exclude_files.txt | 1 + dev/release/source_build.sh | 2 +- meson.build | 6 +- python/bootstrap.py | 17 +----- python/generate_dist.py | 59 ++++++++++++++++++++ python/meson.build | 10 +++- python/pyproject.toml | 7 ++- python/src/nanoarrow/_ipc_lib.pyx | 2 +- python/src/nanoarrow/meson.build | 40 ++++++++++++- python/subprojects/arrow-nanoarrow | 1 + python/vendor/meson.build | 53 ------------------ 15 files changed, 127 insertions(+), 81 deletions(-) create mode 100644 python/generate_dist.py create mode 120000 python/subprojects/arrow-nanoarrow delete mode 100644 python/vendor/meson.build diff --git a/.github/workflows/build-and-test-device.yaml b/.github/workflows/build-and-test-device.yaml index 5a7cb8a16..10ecaf70f 100644 --- a/.github/workflows/build-and-test-device.yaml +++ b/.github/workflows/build-and-test-device.yaml @@ -28,7 +28,8 @@ on: - 'CMakeLists.txt' - '.github/workflows/build-and-test-device.yaml' - 'src/nanoarrow/**' - - 'python/setup.py' + - 'python/meson.build' + - 'python/src/nanoarrow/meson.build' permissions: contents: read diff --git a/.github/workflows/python-wheels.yaml b/.github/workflows/python-wheels.yaml index e35ea9645..6f1a40492 100644 --- a/.github/workflows/python-wheels.yaml +++ b/.github/workflows/python-wheels.yaml @@ -24,9 +24,11 @@ on: - main paths: - '.github/workflows/python-wheels.yaml' - - 'python/setup.py' + - 'python/meson.build' + - 'python/src/nanoarrow/meson.build' - 'python/pyproject.toml' - 'python/bootstrap.py' + - 'python/generate_dist.py' - 'python/MANIFEST.in' push: branches: diff --git a/.gitignore b/.gitignore index 89fb9fe2f..48bd6537a 100644 --- a/.gitignore +++ b/.gitignore @@ -33,5 +33,6 @@ __pycache__ subprojects/* !subprojects/packagefiles !subprojects/*.wrap +python/subprojects/* compile_commands.json diff --git a/ci/scripts/coverage.sh b/ci/scripts/coverage.sh index 532fe8dd9..c09e47a47 100755 --- a/ci/scripts/coverage.sh +++ b/ci/scripts/coverage.sh @@ -128,7 +128,7 @@ function main() { TARGET_NANOARROW_PYTHON_DIR="${TARGET_NANOARROW_DIR}/python" pushd "${TARGET_NANOARROW_PYTHON_DIR}" - NANOARROW_PYTHON_COVERAGE=1 python -m pip install -e . + NANOARROW_PYTHON_COVERAGE=1 python -m pip install . # Run tests + coverage.py (generates .coverage with absolute file paths) python -m pytest --cov ./src/nanoarrow diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index cb0a83409..819866645 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -15,3 +15,4 @@ dist/flatcc.c src/nanoarrow/ipc/flatcc_generated.h thirdparty/* python/src/nanoarrow/dlpack_abi.h +python/subprojects/arrow-nanoarrow diff --git a/dev/release/source_build.sh b/dev/release/source_build.sh index e9fa56974..92d182a5e 100755 --- a/dev/release/source_build.sh +++ b/dev/release/source_build.sh @@ -43,7 +43,7 @@ main() { # Resolve all hard and symbolic links rm -rf "${base_name}.tmp/" mv "${base_name}/" "${base_name}.tmp/" - cp -R -L "${base_name}.tmp" "${base_name}" + cp -R -d "${base_name}.tmp" "${base_name}" rm -rf "${base_name}.tmp/" # Create new tarball diff --git a/meson.build b/meson.build index ba68167a1..144319c77 100644 --- a/meson.build +++ b/meson.build @@ -103,7 +103,7 @@ if get_option('ipc') ) nanoarrow_ipc_dep = declare_dependency(include_directories: [incdir], link_with: nanoarrow_ipc_lib, - dependencies: [nanoarrow_dep, flatcc_dep]) + dependencies: [nanoarrow_dep]) endif needs_device = get_option('device') or get_option('metal') or get_option('cuda') @@ -137,6 +137,10 @@ if needs_device install: true, cpp_args: device_defines, ) + + nanoarrow_device_dep = declare_dependency(include_directories: [incdir], + link_with: nanoarrow_device_lib, + dependencies: device_deps) endif needs_testing = get_option('testing') or get_option('tests') diff --git a/python/bootstrap.py b/python/bootstrap.py index 60c7a04f7..b9f6d5ef6 100644 --- a/python/bootstrap.py +++ b/python/bootstrap.py @@ -223,10 +223,6 @@ def _write_defs(self, output): output.write(b"\n") -# Runs cmake -DNANOARROW_BUNDLE=ON if cmake exists or copies nanoarrow.c/h -# from ../dist if it does not. Running cmake is safer because it will sync -# any changes from nanoarrow C library sources in the checkout but is not -# strictly necessary for things like installing from GitHub. def copy_or_generate_nanoarrow_c(target_dir: pathlib.Path): vendored_files = [ "nanoarrow.h", @@ -239,21 +235,12 @@ def copy_or_generate_nanoarrow_c(target_dir: pathlib.Path): dst = {name: target_dir / name for name in vendored_files} this_dir = pathlib.Path(__file__).parent.resolve() - source_dir = this_dir.parent - is_cmake_dir = (source_dir / "CMakeLists.txt").exists() - is_in_nanoarrow_repo = ( - is_cmake_dir and (source_dir / "src" / "nanoarrow" / "nanoarrow.h").exists() - ) - - if not is_in_nanoarrow_repo: - raise ValueError( - "Attempt to build source distribution outside the nanoarrow repo" - ) + arrow_proj_dir = this_dir / "subprojects" / "arrow-nanoarrow" subprocess.run( [ sys.executable, - source_dir / "ci" / "scripts" / "bundle.py", + arrow_proj_dir / "ci" / "scripts" / "bundle.py", "--symbol-namespace", "PythonPkg", "--header-namespace", diff --git a/python/generate_dist.py b/python/generate_dist.py new file mode 100644 index 000000000..abf4104bd --- /dev/null +++ b/python/generate_dist.py @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import pathlib +import shutil + + +def main(): + src_dir = pathlib.Path(os.environ["MESON_SOURCE_ROOT"]).parent.resolve() + dist_dir = pathlib.Path(os.environ["MESON_DIST_ROOT"]).resolve() + subproj_dir = dist_dir / "subprojects" / "arrow-nanoarrow" + + if subproj_dir.is_symlink(): + subproj_dir.unlink() + + subproj_dir.mkdir(parents=True) + shutil.copy(src_dir / "meson.build", subproj_dir / "meson.build") + shutil.copy(src_dir / "meson.options", subproj_dir / "meson.options") + + # Copy over any subproject dependency / wrap files + subproj_subproj_dir = subproj_dir / "subprojects" + subproj_subproj_dir.mkdir() + for f in (src_dir / "subprojects").glob("*.wrap"): + shutil.copy(f, subproj_subproj_dir / f.name) + shutil.copytree( + src_dir / "subprojects" / "packagefiles", subproj_subproj_dir / "packagefiles" + ) + + target_src_dir = subproj_dir / "src" + shutil.copytree(src_dir / "src", target_src_dir) + + # CMake isn't actually required for building, but the bundle.py script reads from + # its configuration + shutil.copy(src_dir / "CMakeLists.txt", subproj_dir / "CMakeLists.txt") + + subproj_ci_scripts_dir = subproj_dir / "ci" / "scripts" + subproj_ci_scripts_dir.mkdir(parents=True) + shutil.copy( + src_dir / "ci" / "scripts" / "bundle.py", subproj_ci_scripts_dir / "bundle.py" + ) + + +if __name__ == "__main__": + main() diff --git a/python/meson.build b/python/meson.build index d2336be51..c5f9792d8 100644 --- a/python/meson.build +++ b/python/meson.build @@ -25,10 +25,14 @@ project( 'warning_level=2', 'c_std=c99', 'default_library=static', + # We need to set these options at the project default_option level + # due to https://github.com/mesonbuild/meson/issues/6728 + 'arrow-nanoarrow:ipc=true', + 'arrow-nanoarrow:device=true', + 'arrow-nanoarrow:namespace=PythonPkg', ], ) -py = import('python').find_installation(pure: false) - -subdir('vendor') subdir('src/nanoarrow') + +meson.add_dist_script('python', meson.current_source_dir() / 'generate_dist.py') diff --git a/python/pyproject.toml b/python/pyproject.toml index 3cfeb6b95..b686b2c43 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -40,7 +40,12 @@ Changelog = "https://github.com/apache/arrow-nanoarrow/blob/main/CHANGELOG.md" requires = [ "meson>=1.3.0", "meson-python", - "cmake>=3.14", "Cython" ] build-backend = "mesonpy" + +[tool.meson-python.args] +install = ['--skip-subprojects'] + +[tool.pytest.ini_options] +norecursedirs = "subprojects/arrow-nanoarrow" diff --git a/python/src/nanoarrow/_ipc_lib.pyx b/python/src/nanoarrow/_ipc_lib.pyx index fcb6530ad..163644d9e 100644 --- a/python/src/nanoarrow/_ipc_lib.pyx +++ b/python/src/nanoarrow/_ipc_lib.pyx @@ -38,7 +38,7 @@ from nanoarrow._array cimport CArrayView from nanoarrow._utils cimport Error -cdef extern from "nanoarrow_ipc.h" nogil: +cdef extern from "nanoarrow/nanoarrow_ipc.h" nogil: struct ArrowIpcInputStream: ArrowErrorCode (*read)(ArrowIpcInputStream* stream, uint8_t* buf, int64_t buf_size_bytes, int64_t* size_read_out, diff --git a/python/src/nanoarrow/meson.build b/python/src/nanoarrow/meson.build index e56bbcd3d..e1b1449f6 100644 --- a/python/src/nanoarrow/meson.build +++ b/python/src/nanoarrow/meson.build @@ -15,6 +15,40 @@ # specific language governing permissions and limitations # under the License. +nanoarrow_proj = subproject('arrow-nanoarrow') +nanoarrow_dep = nanoarrow_proj.get_variable('nanoarrow_dep') +nanoarrow_ipc_dep = nanoarrow_proj.get_variable('nanoarrow_ipc_dep') +nanoarrow_device_dep = nanoarrow_proj.get_variable('nanoarrow_device_dep') + +py = import('python').find_installation(pure: false) + +vendored_files = custom_target( + 'generate-pyx', + output: [ + 'flatcc.c', + 'nanoarrow.c', + 'nanoarrow_c.pxd', + 'nanoarrow_device.c', + 'nanoarrow_device_c.pxd', + 'nanoarrow_device.h', + 'nanoarrow_device.hpp', + 'nanoarrow.h', + 'nanoarrow.hpp', + 'nanoarrow_ipc.c', + 'nanoarrow_ipc.h', + 'nanoarrow_ipc.hpp', + ], + command: [ + py, + meson.current_source_dir() + '/../../bootstrap.py', + '--output-dir', meson.current_build_dir() + ], +) + +nanoarrow_pyx_dep = declare_dependency( + sources: vendored_files[2], +) + cyfiles = [ '_array.pyx', '_array_stream.pyx', @@ -30,7 +64,7 @@ cython_args = [ '--include-dir', meson.current_source_dir(), '--include-dir', - meson.project_build_root() / 'vendor', + meson.current_build_dir(), ] if get_option('buildtype') == 'debug' cython_args += ['--gdb'] @@ -38,11 +72,11 @@ endif fs = import('fs') foreach cyf : cyfiles - cyfile_deps = [nanoarrow_pyx_dep] + cyfile_deps = [nanoarrow_pyx_dep, nanoarrow_dep] stem = fs.stem(cyf) if stem in ['_array', '_device'] - cyfile_deps += [nanoarrow_device_pyx_dep] + cyfile_deps += [nanoarrow_device_dep] elif stem == '_ipc_lib' cyfile_deps += [nanoarrow_ipc_dep] endif diff --git a/python/subprojects/arrow-nanoarrow b/python/subprojects/arrow-nanoarrow new file mode 120000 index 000000000..c25bddb6d --- /dev/null +++ b/python/subprojects/arrow-nanoarrow @@ -0,0 +1 @@ +../.. \ No newline at end of file diff --git a/python/vendor/meson.build b/python/vendor/meson.build deleted file mode 100644 index a3505e682..000000000 --- a/python/vendor/meson.build +++ /dev/null @@ -1,53 +0,0 @@ -vendored_files = custom_target( - 'generate-pyx', - output: [ - 'flatcc.c', - 'nanoarrow.c', - 'nanoarrow_c.pxd', - 'nanoarrow_device.c', - 'nanoarrow_device_c.pxd', - 'nanoarrow_device.h', - 'nanoarrow_device.hpp', - 'nanoarrow.h', - 'nanoarrow.hpp', - 'nanoarrow_ipc.c', - 'nanoarrow_ipc.h', - 'nanoarrow_ipc.hpp', - ], - command: [ - py, - meson.current_source_dir() + '/../bootstrap.py', - '--output-dir', meson.current_build_dir() - ], -) - -nanoarrow_lib = static_library( - 'nanoarrow', - sources: vendored_files[1], -) - -nanoarrow_pyx_dep = declare_dependency( - sources: vendored_files[2], - link_with: nanoarrow_lib -) - -nanoarrow_ipc_lib = static_library( - 'nanoarrow_ipc', - sources: [vendored_files[0], vendored_files[9]], - link_with: nanoarrow_lib, -) - -nanoarrow_ipc_dep = declare_dependency( - link_with: nanoarrow_ipc_lib -) - -nanoarrow_device_lib = static_library( - 'nanoarrow_device', - sources: vendored_files[3], - link_with: nanoarrow_lib, -) - -nanoarrow_device_pyx_dep = declare_dependency( - sources: vendored_files[4], - link_with: nanoarrow_device_lib, -)