Use bootstrap approach instead of subproject

.gitignore

@@ -33,8 +33,5 @@ __pycache__
 subprojects/*
 !subprojects/packagefiles
 !subprojects/*.wrap
-python/subprojects/*
-!python/subprojects/packagefiles
-!python/subprojects/*.wrap
 
 compile_commands.json

dev/release/rat_exclude_files.txt

@@ -15,4 +15,3 @@ dist/flatcc.c
 src/nanoarrow/ipc/flatcc_generated.h
 thirdparty/*
 python/src/nanoarrow/dlpack_abi.h
-python/subprojects/arrow-nanoarrow

dev/release/release_tools.py

@@ -84,11 +84,11 @@ def set_python_dev_version_command(args):
     _, last_dev_tag = find_last_dev_tag()
     dev_distance = len(find_commits_since(last_dev_tag))
 
-    version_file = src_path("python", "meson.build")
+    config_file = src_path("python", "meson.build")
     file_regex_replace(
         r'"([0-9]+\.[0-9]+\.[0-9]+)\.dev[0-9]+"',
         f'"\\1.dev{dev_distance}"',
-        version_file,
+        config_file,
     )
 
 

dev/release/source_build.sh

@@ -43,7 +43,7 @@ main() {
     # Resolve all hard and symbolic links
     rm -rf "${base_name}.tmp/"
     mv "${base_name}/" "${base_name}.tmp/"
-    cp -R -d "${base_name}.tmp" "${base_name}"
+    cp -R -L "${base_name}.tmp" "${base_name}"
     rm -rf "${base_name}.tmp/"
 
     # Create new tarball

meson.build

@@ -103,7 +103,7 @@ if get_option('ipc')
     )
     nanoarrow_ipc_dep = declare_dependency(include_directories: [incdir],
                                            link_with: nanoarrow_ipc_lib,
-                                           dependencies: [nanoarrow_dep])
+                                           dependencies: [nanoarrow_dep, flatcc_dep])
 endif
 
 needs_device = get_option('device') or get_option('metal') or get_option('cuda')
@@ -137,10 +137,6 @@ if needs_device
       install: true,
       cpp_args: device_defines,
   )
-
-  nanoarrow_device_dep = declare_dependency(include_directories: [incdir],
-                                            link_with: nanoarrow_device_lib,
-                                            dependencies: device_deps)
 endif
 
 needs_testing = get_option('testing') or get_option('tests')

python/.gitignore

@@ -16,7 +16,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-vendor/
 src/nanoarrow/_*.c
 
 # Byte-compiled / optimized / DLL files

python/MANIFEST.in

@@ -20,6 +20,5 @@ recursive-include src/nanoarrow *.pxd
 recursive-include src/nanoarrow *.h
 
 exclude bootstrap.py
-exclude generate_dist.py
 exclude src/nanoarrow/*.c
 recursive-exclude src/nanoarrow *.o *.so

python/bootstrap.py

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import argparse
 import pathlib
 import re
 import subprocess
@@ -58,9 +59,7 @@ def generate_pxd(self, file_in, file_out):
             output.write(header.encode("UTF-8"))
 
             output.write(
-                f'\ncdef extern from "nanoarrow/{file_in_name}" nogil:\n'.encode(
-                    "UTF-8"
-                )
+                f'\ncdef extern from "{file_in_name}" nogil:\n'.encode("UTF-8")
             )
 
             # A few things we add in manually
@@ -228,13 +227,7 @@ def _write_defs(self, output):
 # from ../dist if it does not. Running cmake is safer because it will sync
 # any changes from nanoarrow C library sources in the checkout but is not
 # strictly necessary for things like installing from GitHub.
-def copy_or_generate_nanoarrow_c():
-    this_dir = pathlib.Path(__file__).parent.resolve()
-    subproj_dir = this_dir / "subprojects"
-    source_dir = subproj_dir / "arrow-nanoarrow"
-
-    vendor_dir = this_dir / "vendor"
-
+def copy_or_generate_nanoarrow_c(target_dir: pathlib.Path):
     vendored_files = [
         "nanoarrow.h",
         "nanoarrow.c",
@@ -243,12 +236,20 @@ def copy_or_generate_nanoarrow_c():
         "nanoarrow_device.h",
         "nanoarrow_device.c",
     ]
-    dst = {name: vendor_dir / name for name in vendored_files}
+    dst = {name: target_dir / name for name in vendored_files}
+
+    this_dir = pathlib.Path(__file__).parent.resolve()
+    source_dir = this_dir.parent
+    is_cmake_dir = (source_dir / "CMakeLists.txt").exists()
+    is_in_nanoarrow_repo = (
+        is_cmake_dir and (source_dir / "src" / "nanoarrow" / "nanoarrow.h").exists()
+    )
 
-    for f in dst.values():
-        f.unlink(missing_ok=True)
+    if not is_in_nanoarrow_repo:
+        raise ValueError(
+            "Attempt to build source distribution outside the nanoarrow repo"
+        )
 
-    vendor_dir.mkdir(exist_ok=True)
     subprocess.run(
         [
             sys.executable,
@@ -258,9 +259,9 @@ def copy_or_generate_nanoarrow_c():
             "--header-namespace",
             "",
             "--source-output-dir",
-            vendor_dir,
+            target_dir,
             "--include-output-dir",
-            vendor_dir,
+            target_dir,
             "--with-device",
             "--with-ipc",
             "--with-flatcc",
@@ -272,18 +273,24 @@ def copy_or_generate_nanoarrow_c():
 
 
 # Runs the pxd generator with some information about the file name
-def generate_nanoarrow_pxds():
-    this_dir = pathlib.Path(__file__).parent.resolve()
-
+def generate_nanoarrow_pxds(target_dir: pathlib.Path):
     NanoarrowPxdGenerator().generate_pxd(
-        this_dir / "vendor" / "nanoarrow.h", this_dir / "vendor" / "nanoarrow_c.pxd"
+        target_dir / "nanoarrow.h", target_dir / "nanoarrow_c.pxd"
     )
     NanoarrowDevicePxdGenerator().generate_pxd(
-        this_dir / "vendor" / "nanoarrow_device.h",
-        this_dir / "vendor" / "nanoarrow_device_c.pxd",
+        target_dir / "nanoarrow_device.h",
+        target_dir / "nanoarrow_device_c.pxd",
     )
 
 
 if __name__ == "__main__":
-    copy_or_generate_nanoarrow_c()
-    generate_nanoarrow_pxds()
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--output-dir", help="Target directory where files should be written"
+    )
+
+    args = parser.parse_args()
+    target_dir = pathlib.Path(args.output_dir).resolve()
+
+    copy_or_generate_nanoarrow_c(target_dir)
+    generate_nanoarrow_pxds(target_dir)

python/meson.build

@@ -17,21 +17,18 @@
 
 project(
     'nanoarrow',
-    'cpp', 'cython',
+    'c', 'cython',
     version: '0.7.0.dev0',
     license: 'Apache-2.0',
     meson_version: '>=1.2.0',
     default_options: [
         'warning_level=2',
-        'cpp_std=c++17',
+        'c_std=c99',
         'default_library=static',
-        # We need to set these options at the project default_option level
-        # due to https://github.com/mesonbuild/meson/issues/6728
-        'arrow-nanoarrow:ipc=true',
-        'arrow-nanoarrow:device=true',
     ],
 )
 
-subdir('src/nanoarrow')
+py = import('python').find_installation(pure: false)
 
-meson.add_dist_script('python', meson.current_source_dir() / 'generate_dist.py')
+subdir('vendor')
+subdir('src/nanoarrow')

python/pyproject.toml

@@ -40,10 +40,7 @@ Changelog = "https://github.com/apache/arrow-nanoarrow/blob/main/CHANGELOG.md"
 requires = [
     "meson>=1.3.0",
     "meson-python",
+    "cmake>=3.14",
     "Cython"
 ]
 build-backend = "mesonpy"
-
-[tool.meson-python.args]
-install = ['--skip-subprojects']
-dist = ['--include-subprojects']

python/src/nanoarrow/_ipc_lib.pyx

@@ -38,7 +38,7 @@ from nanoarrow._array cimport CArrayView
 from nanoarrow._utils cimport Error
 
 
-cdef extern from "nanoarrow/nanoarrow_ipc.h" nogil:
+cdef extern from "nanoarrow_ipc.h" nogil:
     struct ArrowIpcInputStream:
         ArrowErrorCode (*read)(ArrowIpcInputStream* stream, uint8_t* buf,
                                int64_t buf_size_bytes, int64_t* size_read_out,

python/src/nanoarrow/meson.build

@@ -15,24 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# Try to resolve the symlink to a subproject first and fall back
-# to the wrap entry if that is unsuccessful. Ideally Meson would
-# take care of this for us, but there is a bug upstream
-# https://github.com/mesonbuild/meson/issues/13746#issuecomment-2392510954
-nanoarrow_proj = subproject('arrow-nanoarrow')
-nanoarrow_dep = nanoarrow_proj.get_variable('nanoarrow_dep')
-nanoarrow_ipc_dep = nanoarrow_proj.get_variable('nanoarrow_ipc_dep')
-nanoarrow_device_dep = nanoarrow_proj.get_variable('nanoarrow_device_dep')
-
-py = import('python').find_installation(pure: false)
-
-generated_pyx = custom_target(
-    'generate-pyx',
-    output: 'nanoarrow_c.pxd',
-    command: [py, meson.current_source_dir() + '/../../bootstrap.py'],
-)
-nanoarrow_c_dep = declare_dependency(sources: generated_pyx)
-
 cyfiles = [
     '_array.pyx',
     '_array_stream.pyx',
@@ -48,19 +30,19 @@ cython_args = [
     '--include-dir',
     meson.current_source_dir(),
     '--include-dir',
-    meson.project_source_root() / 'vendor',  # for generated nanoarrow_c file
+    meson.project_build_root() / 'vendor',
 ]
 if get_option('buildtype') == 'debug'
     cython_args += ['--gdb']
 endif
 
 fs = import('fs')
 foreach cyf : cyfiles
-  cyfile_deps = [nanoarrow_c_dep, nanoarrow_dep]
+  cyfile_deps = [nanoarrow_pyx_dep]
 
   stem = fs.stem(cyf)
   if stem in ['_array', '_device']
-    cyfile_deps += [nanoarrow_device_dep]
+    cyfile_deps += [nanoarrow_device_pyx_dep]
   elif stem == '_ipc_lib'
     cyfile_deps += [nanoarrow_ipc_dep]
   endif

python/vendor/meson.build

@@ -0,0 +1,53 @@
+vendored_files = custom_target(
+    'generate-pyx',
+    output: [
+        'flatcc.c',
+        'nanoarrow.c',
+        'nanoarrow_c.pxd',
+        'nanoarrow_device.c',
+        'nanoarrow_device_c.pxd',
+        'nanoarrow_device.h',
+        'nanoarrow_device.hpp',
+        'nanoarrow.h',
+        'nanoarrow.hpp',
+        'nanoarrow_ipc.c',
+        'nanoarrow_ipc.h',
+        'nanoarrow_ipc.hpp',
+    ],
+    command: [
+        py,
+        meson.current_source_dir() + '/../bootstrap.py',
+        '--output-dir', meson.current_build_dir()
+    ],
+)
+
+nanoarrow_lib = static_library(
+    'nanoarrow',
+    sources: vendored_files[1],
+)
+
+nanoarrow_pyx_dep = declare_dependency(
+    sources: vendored_files[2],
+    link_with: nanoarrow_lib
+)
+
+nanoarrow_ipc_lib = static_library(
+    'nanoarrow_ipc',
+    sources: [vendored_files[0], vendored_files[9]],
+    link_with: nanoarrow_lib,
+)
+
+nanoarrow_ipc_dep = declare_dependency(
+    link_with: nanoarrow_ipc_lib
+)
+
+nanoarrow_device_lib = static_library(
+    'nanoarrow_device',
+    sources: vendored_files[3],
+    link_with: nanoarrow_lib,
+)
+
+nanoarrow_device_pyx_dep = declare_dependency(
+    sources: vendored_files[4],
+    link_with: nanoarrow_device_lib,
+)