feat: Standalone Python installer using Meson

python/bootstrap.py

@@ -15,28 +15,17 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import os
+import argparse
 import pathlib
 import re
-import shutil
-import subprocess
-import tempfile
-import warnings
 
 
 # Generate the nanoarrow_c.pxd file used by the Cython extensions
 class NanoarrowPxdGenerator:
     def __init__(self):
         self._define_regexes()
 
-    def generate_nanoarrow_pxd(self, file_in, file_out):
-        file_in_name = pathlib.Path(file_in).name
-
-        # Read the nanoarrow.h header
-        content = None
-        with open(file_in, "r") as input:
-            content = input.read()
-
+    def generate_nanoarrow_pxd(self, content: str, build_dir: pathlib.Path) -> None:
         # Strip comments
         content = self.re_comment.sub("", content)
 
@@ -57,12 +46,11 @@ def generate_nanoarrow_pxd(self, file_in, file_out):
         header = self.re_newline_plus_indent.sub("\n", self._pxd_header())
 
         # Write nanoarrow_c.pxd
+        file_out = build_dir / "nanoarrow_c.pxd"
         with open(file_out, "wb") as output:
             output.write(header.encode("UTF-8"))
 
-            output.write(
-                f'\ncdef extern from "{file_in_name}" nogil:\n'.encode("UTF-8")
-            )
+            output.write('\ncdef extern from "nanoarrow.h" nogil:\n'.encode("UTF-8"))
 
             # A few things we add in manually
             output.write(b"\n")
@@ -175,105 +163,40 @@ def _pxd_header(self):
         """
 
 
-# Runs cmake -DNANOARROW_BUNDLE=ON if cmake exists or copies nanoarrow.c/h
-# from ../dist if it does not. Running cmake is safer because it will sync
-# any changes from nanoarrow C library sources in the checkout but is not
-# strictly necessary for things like installing from GitHub.
-def copy_or_generate_nanoarrow_c():
+def generate_nanoarrow_c() -> str:
     this_dir = pathlib.Path(__file__).parent.resolve()
-    source_dir = this_dir.parent
-    vendor_dir = this_dir / "vendor"
+    nanoarrow_dir = this_dir / "subprojects" / "nanoarrow" / "src" / "nanoarrow"
 
-    vendored_files = [
+    # This should match the NANOARROW_BUNDLE code in CMakeLists.txt
+    # With the only thing missing being the nanoarrow namespace. However, we
+    # assume the Python installation is sandboxed so should not be required (?)
+    header_data: list[str] = []
+
+    files = [
+        # TODO: - do we need the config file for Cython?
+        # 'nanoarrow_config.h',
+        "nanoarrow_types.h",
         "nanoarrow.h",
-        "nanoarrow.c",
-        "nanoarrow_ipc.h",
-        "nanoarrow_ipc.c",
-        "nanoarrow_device.h",
-        "nanoarrow_device.c",
+        "buffer_inline.h",
+        "array_inline.h",
     ]
-    dst = {name: vendor_dir / name for name in vendored_files}
-
-    for f in dst.values():
-        f.unlink(missing_ok=True)
-
-    is_cmake_dir = (source_dir / "CMakeLists.txt").exists()
-    is_in_nanoarrow_repo = (
-        is_cmake_dir and (source_dir / "src" / "nanoarrow" / "nanoarrow.h").exists()
-    )
-
-    if not is_in_nanoarrow_repo:
-        raise ValueError(
-            "Attempt to build source distribution outside the nanoarrow repo"
-        )
 
-    cmake_bin = os.getenv("CMAKE_BIN")
-    if not cmake_bin:
-        cmake_bin = "cmake"
-    has_cmake = os.system(f"{cmake_bin} --version") == 0
-    if not has_cmake:
-        raise ValueError("Attempt to build source distribution without CMake")
+    for file in files:
+        with open(nanoarrow_dir / file) as f:
+            header_data.append(f.read())
 
-    # The C library, IPC extension, and Device extension all currently have slightly
-    # different methods of bundling (hopefully this can be unified)
+    contents = "\n".join(header_data)
+    # Remove includes that aren't needed when the headers are concatenated
+    contents = re.sub(r"#include \".*", "", contents)
 
-    vendor_dir.mkdir(exist_ok=True)
-
-    # Copy device files
-    device_ext_src = (
-        source_dir / "extensions" / "nanoarrow_device" / "src" / "nanoarrow"
-    )
-
-    for device_file in ["nanoarrow_device.h", "nanoarrow_device.c"]:
-        shutil.copyfile(
-            device_ext_src / device_file,
-            dst[device_file],
-        )
-
-    ipc_source_dir = source_dir / "extensions/nanoarrow_ipc"
-
-    for cmake_project in [source_dir, ipc_source_dir]:
-        with tempfile.TemporaryDirectory() as build_dir:
-            try:
-                subprocess.run(
-                    [
-                        cmake_bin,
-                        "-B",
-                        build_dir,
-                        "-S",
-                        cmake_project,
-                        "-DNANOARROW_IPC_BUNDLE=ON",
-                        "-DNANOARROW_BUNDLE=ON",
-                        "-DNANOARROW_NAMESPACE=PythonPkg",
-                    ]
-                )
-                subprocess.run(
-                    [
-                        cmake_bin,
-                        "--install",
-                        build_dir,
-                        "--prefix",
-                        vendor_dir,
-                    ]
-                )
-            except Exception as e:
-                warnings.warn(f"cmake call failed: {e}")
-
-    if not dst["nanoarrow.h"].exists():
-        raise ValueError("Attempt to vendor nanoarrow.c/h failed")
-
-
-# Runs the pxd generator with some information about the file name
-def generate_nanoarrow_pxd():
-    this_dir = pathlib.Path(__file__).parent.resolve()
-    maybe_nanoarrow_h = this_dir / "vendor/nanoarrow.h"
-    maybe_nanoarrow_pxd = this_dir / "vendor/nanoarrow_c.pxd"
-
-    NanoarrowPxdGenerator().generate_nanoarrow_pxd(
-        maybe_nanoarrow_h, maybe_nanoarrow_pxd
-    )
+    return contents
 
 
 if __name__ == "__main__":
-    copy_or_generate_nanoarrow_c()
-    generate_nanoarrow_pxd()
+    parser = argparse.ArgumentParser()
+    parser.add_argument("build_dir", type=str)
+    args = parser.parse_args()
+    build_dir = pathlib.Path(args.build_dir).resolve()
+
+    contents = generate_nanoarrow_c()
+    NanoarrowPxdGenerator().generate_nanoarrow_pxd(contents, build_dir)

python/meson.build

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+project(
+    'nanoarrow-python',
+    'cython',
+    version: '0.15.0', # TODO: don't hard code this
+)
+
+subdir('src/nanoarrow')

python/pyproject.toml

@@ -38,7 +38,7 @@ Changelog = "https://github.com/apache/arrow-nanoarrow/blob/main/CHANGELOG.md"
 
 [build-system]
 requires = [
-    "setuptools >= 61.0.0",
+    "meson-python",
     "Cython"
 ]
-build-backend = "setuptools.build_meta"
+build-backend = "mesonpy"

python/src/nanoarrow/meson.build

@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+py = import('python').find_installation(pure: false)
+
+cython_args = [
+    '--include-dir',
+    meson.current_build_dir(),
+    '--include-dir',
+    meson.current_source_dir(), # TODO: should meson handle this natively?
+]
+if get_option('buildtype') == 'debug'
+    cython_args += ['--gdb']
+endif
+
+nanoarrow_dep = dependency('nanoarrow')
+
+generated_pyx = custom_target(
+    'generate-pyx',
+    input: meson.current_build_dir(),
+    output: 'nanoarrow_c.pxd',
+    command: [py, '../../bootstrap.py', '@INPUT@'],
+    install: true,
+    install_dir: '.',
+    #depends: nanoarrow_dep,
+    # ERROR: custom_target keyword argument 'depends' was of type
+    # array[InternalDependency] but should have been array[BuildTarget |
+    # CustomTarget] - seems fragile without this?
+
+)
+nanoarrow_c_dep = declare_dependency(sources: generated_pyx)
+
+# TODO: we need dependencies on nanoarrow_ipc and nanoarrow_device,
+# which should happen as part of https://github.com/apache/arrow-nanoarrow/pull/483
+
+py.extension_module(
+    '_lib',
+    sources: ['_lib.pyx'],
+    cython_args: cython_args,
+    dependencies: [nanoarrow_dep, nanoarrow_c_dep],
+    subdir: 'nanoarrow/',
+    install: true,
+)
+
+py.extension_module(
+    '_ipc_lib',
+    sources: ['_ipc_lib.pyx'],
+    cython_args: cython_args,
+    dependencies: [nanoarrow_dep, nanoarrow_c_dep],
+    subdir: 'nanoarrow',
+    install: true,
+)

python/subprojects/nanoarrow.wrap

@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[wrap-file]
+directory = arrow-nanoarrow-apache-arrow-nanoarrow-0.5.0
+source_url = https://github.com/apache/arrow-nanoarrow/archive/refs/tags/apache-arrow-nanoarrow-0.5.0.tar.gz
+source_filename = apache-arrow-nanoarrow-0.5.0.tar.gz
+source_hash = 0ceeaa1fb005dbc89c8c7d1b39f2dba07344e40aa9d885ee25fb55b4d57e331a
+source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/nanoarrow_0.5.0-1/apache-arrow-nanoarrow-0.5.0.tar.gz
+wrapdb_version = 0.5.0-1
+
+[provide]
+nanoarrow = nanoarrow_dep
+
+# For development in the arrow-nanoarrow source tree, you may want to provide
+# an alternate wrap specification. The following example will pull whatever
+# local project is committed to HEAD
+# [wrap-git]
+# url = ../..
+# revision = HEAD
+
+# [provide]
+# nanoarrow = nanoarrow_dep