diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3880e7397..a249e548f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,7 +28,7 @@ repos: - id: mypy name: mypy 3.11 on cibuildwheel/ args: ["--python-version=3.11"] - exclude: ^cibuildwheel/resources/_cross_venv.py$ # Requires Python 3.13 or later + exclude: ^cibuildwheel/resources/android/_cross_venv.py$ # Requires Python 3.13 or later additional_dependencies: &mypy-dependencies - bracex - build diff --git a/bin/generate_schema.py b/bin/generate_schema.py index b2f631185..f981e2cdf 100755 --- a/bin/generate_schema.py +++ b/bin/generate_schema.py @@ -15,6 +15,8 @@ parser.add_argument("--schemastore", action="store_true", help="Generate schema_store version") args = parser.parse_args() +# The defaults in the schema are used by external tools for validation and IDE support. They +# should match the values in defaults.toml, which are used by cibuildwheel itself. starter = """ $schema: http://json-schema.org/draft-07/schema# $id: https://github.com/pypa/cibuildwheel/blob/main/cibuildwheel/resources/cibuildwheel.schema.json @@ -367,14 +369,15 @@ def as_object(d: dict[str, Any]) -> dict[str, Any]: "ios": as_object(not_linux), } -oses["linux"]["properties"]["repair-wheel-command"] = { - **schema["properties"]["repair-wheel-command"], - "default": "auditwheel repair -w {dest_dir} {wheel}", -} -oses["macos"]["properties"]["repair-wheel-command"] = { - **schema["properties"]["repair-wheel-command"], - "default": "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}", -} +for os_name, command in [ + ("linux", "auditwheel repair -w {dest_dir} {wheel}"), + ("macos", "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}"), + ("android", "auditwheel repair --ldpaths {ldpaths} -w {dest_dir} {wheel}"), +]: + oses[os_name]["properties"]["repair-wheel-command"] = { + **schema["properties"]["repair-wheel-command"], + "default": command, + } del oses["linux"]["properties"]["dependency-versions"] diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 5f4839e58..7124f016c 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -1,15 +1,11 @@ -import csv -import hashlib import os import platform import re import shlex import shutil import subprocess -import sysconfig -from collections.abc import Iterable, Iterator, MutableMapping +import sys from dataclasses import dataclass -from os.path import relpath from pathlib import Path from pprint import pprint from runpy import run_path @@ -18,8 +14,6 @@ from build import ProjectBuilder from build.env import IsolatedEnv -from elftools.common.exceptions import ELFError -from elftools.elf.elffile import ELFFile from filelock import FileLock from .. import errors, platforms # pylint: disable=cyclic-import @@ -28,6 +22,7 @@ from ..logger import log from ..options import BuildOptions, Options from ..selector import BuildSelector +from ..typing import PathOrStr from ..util import resources from ..util.cmd import call, shell from ..util.file import CIBW_CACHE_PATH, copy_test_sources, download, move_file @@ -41,6 +36,15 @@ "x86_64": "x86_64-linux-android", } +CROSS_BUILD_FILES = { + "numpy": [ + "numpy/_core/include/numpy/numpyconfig.h", + "numpy/_core/include/numpy/_numpyconfig.h", + "numpy/_core/lib/libnpymath.a", + "numpy/random/lib/libnpyrandom.a", + ] +} + def parse_identifier(identifier: str) -> tuple[str, str]: match = re.fullmatch(r"cp(\d)(\d+)-android_(.+)", identifier) @@ -136,6 +140,7 @@ def build(options: Options, tmp_path: Path) -> None: state = BuildState( config, build_options, build_path, python_dir, build_env, android_env ) + setup_cross_build_files(state) compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) if compatible_wheel: @@ -149,7 +154,7 @@ def build(options: Options, tmp_path: Path) -> None: built_wheel = build_wheel(state) repaired_wheel = repair_wheel(state, built_wheel) - test_wheel(state, repaired_wheel, build_frontend=build_options.build_frontend.name) + test_wheel(state, repaired_wheel) output_wheel: Path | None = None if compatible_wheel is None: @@ -176,6 +181,12 @@ def setup_target_python(config: PythonConfiguration, build_path: Path) -> Path: python_dir = build_path / "python" python_dir.mkdir() shutil.unpack_archive(python_tgz, python_dir) + + # Work around https://github.com/python/cpython/issues/138800. This can be removed + # once we've updated to Python versions that include the fix. + pc_path = python_dir / f"prefix/lib/pkgconfig/python-{config.version}.pc" + pc_path.write_text(pc_path.read_text().replace("$(BLDLIBRARY)", f"-lpython{config.version}")) + return python_dir @@ -189,13 +200,7 @@ def setup_env( * android_env, which uses the environment while simulating running on Android. """ log.step("Setting up build environment...") - build_frontend = build_options.build_frontend.name - use_uv = build_frontend in {"build[uv]", "uv"} - uv_path = find_uv() - if use_uv and uv_path is None: - msg = "uv not found" - raise AssertionError(msg) - pip = ["pip"] if not use_uv else [str(uv_path), "pip"] + use_uv, pip = find_pip(build_options) # Create virtual environment python_exe = create_python_build_standalone_environment( @@ -224,14 +229,21 @@ def setup_env( raise errors.FatalError(msg) call(command, "--version", env=build_env) - # Construct an altered environment which simulates running on Android. - android_env = setup_android_env(config, python_dir, venv_dir, build_env) - # Install build tools - if build_frontend not in {"build", "build[uv]", "uv"}: - msg = "Android requires the build frontend to be 'build' or 'uv'" - raise errors.FatalError(msg) - call(*pip, "install", "build", *constraint_flags(dependency_constraint), env=build_env) + # TODO: use an official auditwheel version once + # https://github.com/pypa/auditwheel/pull/643 has been released, and add it to the + # constraints files. + tools = [ + "auditwheel @ git+https://github.com/mhsmith/auditwheel@android", + "patchelf", + "pkgconf", + ] + if build_options.build_frontend.name in {"build", "build[uv]"}: + tools.append("build") + call(*pip, "install", *tools, *constraint_flags(dependency_constraint), env=build_env) + + # Construct an altered environment which simulates running on Android. + android_env = setup_android_env(config, python_dir, build_env) # Build-time requirements must be queried within android_env, because # `get_requires_for_build` can run arbitrary code in setup.py scripts, which may be @@ -321,9 +333,9 @@ def localized_vars( if isinstance(final, str): final = final.replace(orig_prefix, str(prefix)) + # See platforms.md for the reason why we use this default API level. if key == "ANDROID_API_LEVEL": - if api_level := build_env.get(key): - final = int(api_level) + final = int(build_env.get(key, "24")) # Build systems vary in whether FLAGS variables are read from sysconfig, and if so, # whether they're replaced by environment variables or combined with them. Even @@ -346,11 +358,11 @@ def localized_vars( def setup_android_env( - config: PythonConfiguration, python_dir: Path, venv_dir: Path, build_env: dict[str, str] + config: PythonConfiguration, python_dir: Path, build_env: dict[str, str] ) -> dict[str, str]: - site_packages = next(venv_dir.glob("lib/python*/site-packages")) + site_packages = find_site_packages(build_env) for suffix in ["pth", "py"]: - shutil.copy(resources.PATH / f"_cross_venv.{suffix}", site_packages) + shutil.copy(resources.PATH / f"android/_cross_venv.{suffix}", site_packages) sysconfigdata_path = Path( shutil.copy( @@ -393,6 +405,15 @@ def setup_android_env( # Cargo target linker needs to be specified after CC is set setup_rust(config, python_dir, android_env) + # Create shims which install additional build tools on first use. + setup_fortran(android_env) + + # `android.py env` returns PKG_CONFIG="pkg-config --define-prefix", but some build + # systems can't handle arguments in that variable. Since we have a known version + # of pkgconf, it's safe to use PKG_CONFIG_RELOCATE_PATHS instead. + android_env["PKG_CONFIG"] = call("which", "pkgconf", env=build_env, capture_stdout=True).strip() + android_env["PKG_CONFIG_RELOCATE_PATHS"] = "1" + # Format the environment so it can be pasted into a shell when debugging. for key, value in sorted(android_env.items()): if os.environ.get(key) != value: @@ -401,11 +422,7 @@ def setup_android_env( return android_env -def setup_rust( - config: PythonConfiguration, - python_dir: Path, - env: MutableMapping[str, str], -) -> None: +def setup_rust(config: PythonConfiguration, python_dir: Path, env: dict[str, str]) -> None: cargo_target = android_triplet(config.identifier) # CARGO_BUILD_TARGET is the variable used by Cargo and setuptools_rust @@ -425,17 +442,83 @@ def setup_rust( venv_bin = Path(env["VIRTUAL_ENV"]) / "bin" for tool in ["cargo", "rustup"]: shim_path = venv_bin / tool - shutil.copy(resources.PATH / "_rust_shim.py", shim_path) + shutil.copy(resources.PATH / "android/rust_shim.py", shim_path) shim_path.chmod(0o755) +def setup_fortran(env: dict[str, str]) -> None: + # In case there's any autodetection based on the executable name, use the same name + # as the real executable (see fortran_shim.run_flang) + shim_in = resources.PATH / "android/fortran_shim.py" + shim_out = Path(env["VIRTUAL_ENV"]) / "bin/flang-new" + + # The hashbang line runs the shim in cibuildwheel's own virtual environment, so it + # has access to utility functions for downloading and caching files. + shim_out.write_text(f"#!{sys.executable}\n\n" + shim_in.read_text()) + shim_out.chmod(0o755) + env["FC"] = str(shim_out) + + +# Although the build environment must be installed for the build platform, some packages +# contain platform-specific files which should be replaced with their Android +# equivalents. We do this using a similar technique to Pyodide: +# * https://github.com/pyodide/pyodide-build/blob/v0.30.2/pyodide_build/recipe/builder.py#L638 +# * https://github.com/pyodide/pyodide-recipes/blob/20250606/packages/numpy/meta.yaml#L28 +def setup_cross_build_files(state: BuildState) -> None: + _, pip = find_pip(state.options) + cbf_dir = state.build_path / "cross_build_files" + cbf_dir.mkdir() + + for requirement in call(*pip, "freeze", env=state.build_env, capture_stdout=True).splitlines(): + name, _, _ = requirement.strip().partition("==") + cross_build_files = CROSS_BUILD_FILES.get(name.lower(), []) + if cross_build_files: + pip_install_android(state, cbf_dir, "--no-deps", requirement) + for cbf in cross_build_files: + if (cbf_dir / cbf).exists(): + shutil.copy( + cbf_dir / cbf, + find_site_packages(state.build_env) / cbf, + ) + else: + log.warning(f"{cbf_dir / cbf} does not exist") + + +def pip_install_android(state: BuildState, target: Path, *args: PathOrStr) -> None: + use_uv, pip = find_pip(state.options) + call( + *pip, + "install", + "--only-binary=:all:", + *(["--python-platform", android_triplet(state.config.identifier)] if use_uv else []), + "--target", + target, + *args, + env=state.android_env, + ) + + +def find_site_packages(env: dict[str, str]) -> Path: + return next(Path(env["VIRTUAL_ENV"]).glob("lib/python*/site-packages")) + + +def find_pip(build_options: BuildOptions) -> tuple[bool, list[str]]: + use_uv = build_options.build_frontend.name in {"build[uv]", "uv"} + uv_path = find_uv() + if use_uv and uv_path is None: + msg = "uv not found" + raise AssertionError(msg) + pip = ["pip"] if not use_uv else [str(uv_path), "pip"] + return use_uv, pip + + def before_build(state: BuildState) -> None: if state.options.before_build: log.step("Running before_build...") shell_prepared( state.options.before_build, build_options=state.options, - env=state.build_env, + env=state.android_env, ) @@ -480,8 +563,8 @@ def build_wheel(state: BuildState) -> Path: env=state.android_env, ) case x: - msg = f"Invalid build backend {x!r}" - raise AssertionError(msg) + msg = f"Android requires the build frontend to be 'build' or 'uv', not {x!r}" + raise errors.FatalError(msg) built_wheels = list(built_wheel_dir.glob("*.whl")) if len(built_wheels) != 1: @@ -500,9 +583,24 @@ def repair_wheel(state: BuildState, built_wheel: Path) -> Path: repaired_wheel_dir.mkdir() if state.options.repair_command: + toolchain = Path(state.android_env["CC"]).parent.parent + triplet = android_triplet(state.config.identifier) + ldpaths = ":".join( + # Pass ldpaths to help auditwheel find compiler libraries. If we implement + # PEP 725 in the future to provide non-Python libraries, we'll need to add + # their location here. + str(next(Path(toolchain).glob(path))) + for path in [ + # libc++_shared + f"sysroot/usr/lib/{triplet}", + # libomp + f"lib/clang/*/lib/linux/{triplet.split('-')[0]}", + ] + ) shell( prepare_command( state.options.repair_command, + ldpaths=ldpaths, wheel=built_wheel, dest_dir=repaired_wheel_dir, package=state.options.package_dir, @@ -511,7 +609,7 @@ def repair_wheel(state: BuildState, built_wheel: Path) -> Path: env=state.build_env, ) else: - repair_default(state.android_env, built_wheel, repaired_wheel_dir) + shutil.move(built_wheel, repaired_wheel_dir) repaired_wheels = list(repaired_wheel_dir.glob("*.whl")) if len(repaired_wheels) == 0: @@ -526,119 +624,12 @@ def repair_wheel(state: BuildState, built_wheel: Path) -> Path: return repaired_wheel -def repair_default( - android_env: dict[str, str], built_wheel: Path, repaired_wheel_dir: Path -) -> None: - """ - Adds libc++ to the wheel if anything links against it. In the future this should be - moved to auditwheel and generalized to support more libraries. - """ - if (match := re.search(r"^(.+?)-", built_wheel.name)) is None: - msg = f"Failed to parse wheel filename: {built_wheel.name}" - raise errors.FatalError(msg) - wheel_name = match[1] - - unpacked_dir = repaired_wheel_dir / "unpacked" - unpacked_dir.mkdir() - shutil.unpack_archive(built_wheel, unpacked_dir, format="zip") - - # Some build systems are inconsistent about name normalization, so don't assume the - # dist-info name is identical to the wheel name. - record_paths = list(unpacked_dir.glob("*.dist-info/RECORD")) - if len(record_paths) != 1: - msg = f"{built_wheel.name} contains {len(record_paths)} dist-info/RECORD files; expected 1" - raise errors.FatalError(msg) - - old_soname = "libc++_shared.so" - paths_to_patch = [] - for path, elffile in elf_file_filter( - unpacked_dir / filename - for filename, *_ in csv.reader(record_paths[0].read_text().splitlines()) - ): - if (dynamic := elffile.get_section_by_name(".dynamic")) and any( # type: ignore[no-untyped-call] - tag.entry.d_tag == "DT_NEEDED" and tag.needed == old_soname - for tag in dynamic.iter_tags() - ): - paths_to_patch.append(path) - - if not paths_to_patch: - shutil.copyfile(built_wheel, repaired_wheel_dir / built_wheel.name) - else: - # Android doesn't support DT_RPATH, but supports DT_RUNPATH since API level 24 - # (https://github.com/aosp-mirror/platform_bionic/blob/master/android-changes-for-ndk-developers.md). - if int(sysconfig_print('get_config_vars()["ANDROID_API_LEVEL"]', android_env)) < 24: - msg = f"Adding {old_soname} requires ANDROID_API_LEVEL to be at least 24" - raise errors.FatalError(msg) - - toolchain = Path(android_env["CC"]).parent.parent - src_path = toolchain / f"sysroot/usr/lib/{android_env['CIBW_HOST_TRIPLET']}/{old_soname}" - - # Use the same library location as auditwheel would. - libs_dir = unpacked_dir / (wheel_name + ".libs") - libs_dir.mkdir() - new_soname = soname_with_hash(src_path) - dst_path = libs_dir / new_soname - shutil.copyfile(src_path, dst_path) - call(which("patchelf"), "--set-soname", new_soname, dst_path) - - for path in paths_to_patch: - call(which("patchelf"), "--replace-needed", old_soname, new_soname, path) - call( - which("patchelf"), - "--set-rpath", - f"${{ORIGIN}}/{relpath(libs_dir, path.parent)}", - path, - ) - call(which("wheel"), "pack", unpacked_dir, "-d", repaired_wheel_dir) - - -# If cibuildwheel was called without activating its environment, its scripts directory -# will not be on the PATH. -def which(cmd: str) -> str: - scripts_dir = sysconfig.get_path("scripts") - result = shutil.which(cmd, path=scripts_dir + os.pathsep + os.environ["PATH"]) - if result is None: - msg = f"Couldn't find {cmd!r} in {scripts_dir} or on the PATH" - raise errors.FatalError(msg) - return result - - -def elf_file_filter(paths: Iterable[Path]) -> Iterator[tuple[Path, ELFFile]]: - """Filter through an iterator of filenames and load up only ELF files""" - for path in paths: - if not path.name.endswith(".py"): - try: - with open(path, "rb") as f: - candidate = ELFFile(f) # type: ignore[no-untyped-call] - yield path, candidate - except ELFError: - pass # Not an ELF file - - -def soname_with_hash(src_path: Path) -> str: - """Return the same library filename as auditwheel would""" - shorthash = hashlib.sha256(src_path.read_bytes()).hexdigest()[:8] - src_name = src_path.name - base, ext = src_name.split(".", 1) - if not base.endswith(f"-{shorthash}"): - return f"{base}-{shorthash}.{ext}" - else: - return src_name - - -def test_wheel(state: BuildState, wheel: Path, *, build_frontend: str) -> None: +def test_wheel(state: BuildState, wheel: Path) -> None: test_command = state.options.test_command if not (test_command and state.options.test_selector(state.config.identifier)): return log.step("Testing wheel...") - use_uv = build_frontend == "build[uv]" - uv_path = find_uv() - if use_uv and uv_path is None: - msg = "uv not found" - raise AssertionError(msg) - pip = ["pip"] if not use_uv else [str(uv_path), "pip"] - native_arch = arch_synonym(platform.machine(), platforms.native_platform(), "android") if state.config.arch != native_arch: log.warning( @@ -651,31 +642,17 @@ def test_wheel(state: BuildState, wheel: Path, *, build_frontend: str) -> None: shell_prepared( state.options.before_test, build_options=state.options, - env=state.build_env, + env=state.android_env, ) - platform_args = ( - ["--python-platform", android_triplet(state.config.identifier)] - if use_uv - else [ - "--platform", - sysconfig_print("get_platform()", state.android_env).replace("-", "_"), - ] - ) - # Install the wheel and test-requires. site_packages_dir = state.build_path / "site-packages" site_packages_dir.mkdir() - call( - *pip, - "install", - "--only-binary=:all:", - *platform_args, - "--target", + pip_install_android( + state, site_packages_dir, f"{wheel}{state.options.test_extras}", *state.options.test_requires, - env=state.android_env, ) # Copy test-sources. @@ -745,13 +722,3 @@ def test_wheel(state: BuildState, wheel: Path, *, build_frontend: str) -> None: *test_args, env=state.build_env, ) - - -def sysconfig_print(method_call: str, env: dict[str, str]) -> str: - return call( - "python", - "-c", - f'import sysconfig; print(sysconfig.{method_call}, end="")', - env=env, - capture_stdout=True, - ) diff --git a/cibuildwheel/resources/_cross_venv.pth b/cibuildwheel/resources/android/_cross_venv.pth similarity index 100% rename from cibuildwheel/resources/_cross_venv.pth rename to cibuildwheel/resources/android/_cross_venv.pth diff --git a/cibuildwheel/resources/_cross_venv.py b/cibuildwheel/resources/android/_cross_venv.py similarity index 95% rename from cibuildwheel/resources/_cross_venv.py rename to cibuildwheel/resources/android/_cross_venv.py index 40dfaca5f..0c50dcce2 100644 --- a/cibuildwheel/resources/_cross_venv.py +++ b/cibuildwheel/resources/android/_cross_venv.py @@ -14,6 +14,9 @@ def initialize() -> None: if not (host_triplet := os.environ.get("CIBW_HOST_TRIPLET")): return + # Pre-import any modules which would fail to import after the monkey-patching. + import ctypes # noqa: F401, PLC0415 - uses get_config_var("LDLIBRARY") + # os ###################################################################### def cross_os_uname() -> os.uname_result: return os.uname_result( diff --git a/cibuildwheel/resources/android/fortran_shim.py b/cibuildwheel/resources/android/fortran_shim.py new file mode 100644 index 000000000..62dba8bf0 --- /dev/null +++ b/cibuildwheel/resources/android/fortran_shim.py @@ -0,0 +1,114 @@ +# This file intentionally has no hashbang line in the source: cibuildwheel will add it +# above this comment when the file is deployed. + +import os +import re +import shutil +import sys +from itertools import chain +from pathlib import Path + +from filelock import FileLock + +from cibuildwheel.util.file import CIBW_CACHE_PATH, download + +# In the future we might pick a different Flang release depending on the NDK version, +# but so far all Python versions use the same NDK version, so there's no need. +RELEASE_URL = "https://github.com/termux/ndk-toolchain-clang-with-flang/releases/download" +RELEASE_VERSION = "r27c" +ARCHS = ["aarch64", "x86_64"] + +# The compiler is built for Linux x86_64, so we use Docker on macOS. +DOCKER_IMAGE = "debian:trixie" + + +def main() -> None: + cache_dir = CIBW_CACHE_PATH / f"flang-android-{RELEASE_VERSION}" + with FileLock(f"{cache_dir}.lock"): + if not cache_dir.exists(): + download_flang(cache_dir) + + run_flang(cache_dir) + + +def download_flang(cache_dir: Path) -> None: + tmp_dir = Path(f"{cache_dir}.tmp") + if tmp_dir.exists(): + shutil.rmtree(tmp_dir) + tmp_dir.mkdir(parents=True) + + for archive_name in [f"package-flang-{arch}.tar.bz2" for arch in ARCHS] + [ + "package-flang-host.tar.bz2", + "package-install.tar.bz2", + ]: + archive_path = tmp_dir / archive_name + download(f"{RELEASE_URL}/{RELEASE_VERSION}/{archive_name}", archive_path) + shutil.unpack_archive(archive_path, tmp_dir) + archive_path.unlink() + + # Merge the extracted trees together, along with the necessary parts of the NDK. Based on + # https://github.com/kivy/python-for-android/blob/develop/pythonforandroid/recipes/fortran/__init__.py) + flang_toolchain = tmp_dir / "toolchain" + (tmp_dir / "out/install/linux-x86/clang-dev").rename(flang_toolchain) + + ndk_toolchain = Path(os.environ["CC"]).parents[1] + if (clang_ver_flang := clang_ver(flang_toolchain)) != ( + clang_ver_ndk := clang_ver(ndk_toolchain) + ): + msg = f"Flang uses Clang {clang_ver_flang}, but NDK uses Clang {clang_ver_ndk}" + raise ValueError(msg) + + clang_lib_path = f"lib/clang/{clang_ver_ndk}/lib" + shutil.rmtree(flang_toolchain / clang_lib_path) + + for src, dst in [ + (f"{tmp_dir}/build-{arch}-install", f"sysroot/usr/lib/{arch}-linux-android") + for arch in ARCHS + ] + [ + (f"{tmp_dir}/build-host-install", ""), + (f"{ndk_toolchain}/{clang_lib_path}", clang_lib_path), + (f"{ndk_toolchain}/sysroot", "sysroot"), + ]: + shutil.copytree(src, flang_toolchain / dst, symlinks=True, dirs_exist_ok=True) + + flang_toolchain.rename(cache_dir) + shutil.rmtree(tmp_dir) + + +def clang_ver(toolchain: Path) -> str: + versions = [p.name for p in (toolchain / "lib/clang").iterdir()] + assert len(versions) == 1 + return versions[0] + + +def run_flang(cache_dir: Path) -> None: + match = re.fullmatch(r".+/(.+)-clang", os.environ["CC"]) + assert match is not None + target = match[1] + + # In a future Flang version the executable name will change to "flang" + # (https://blog.llvm.org/posts/2025-03-11-flang-new/). + args = [f"{cache_dir}/bin/flang-new", f"--target={target}", *sys.argv[1:]] + + if sys.platform == "linux": + pass + elif sys.platform == "darwin": + args = [ + *["docker", "run", "--rm", "--platform", "linux/amd64"], + *chain.from_iterable( + # Docker on macOS only allows certain directories to be mounted as volumes + # by default, but they include all the locations we're likely to need. + ["-v", f"{path}:{path}"] + for path in ["/private", "/Users", "/tmp"] + ), + *["-w", str(Path.cwd()), "--entrypoint", args[0], DOCKER_IMAGE, *args[1:]], + ] + else: + msg = f"unknown platform: {sys.platform}" + raise ValueError(msg) + + os.execvp(args[0], args) + + +if __name__ == "__main__": + main() diff --git a/cibuildwheel/resources/_rust_shim.py b/cibuildwheel/resources/android/rust_shim.py similarity index 100% rename from cibuildwheel/resources/_rust_shim.py rename to cibuildwheel/resources/android/rust_shim.py diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json index a82518f4b..ff8eed744 100644 --- a/cibuildwheel/resources/cibuildwheel.schema.json +++ b/cibuildwheel/resources/cibuildwheel.schema.json @@ -1166,7 +1166,20 @@ "$ref": "#/properties/pyodide-version" }, "repair-wheel-command": { - "$ref": "#/properties/repair-wheel-command" + "description": "Execute a shell command to repair each built wheel.", + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "title": "CIBW_REPAIR_WHEEL_COMMAND", + "default": "auditwheel repair --ldpaths {ldpaths} -w {dest_dir} {wheel}" }, "test-command": { "$ref": "#/properties/test-command" diff --git a/cibuildwheel/resources/constraints-python310.txt b/cibuildwheel/resources/constraints-python310.txt index 21031bc73..16ce38f59 100644 --- a/cibuildwheel/resources/constraints-python310.txt +++ b/cibuildwheel/resources/constraints-python310.txt @@ -20,8 +20,12 @@ packaging==26.0 # via # build # delocate +patchelf==0.17.2.4 + # via -r cibuildwheel/resources/constraints.in pip==26.0.1 # via -r cibuildwheel/resources/constraints.in +pkgconf==2.5.1.post1 + # via -r cibuildwheel/resources/constraints.in platformdirs==4.9.4 # via # python-discovery diff --git a/cibuildwheel/resources/constraints-python311.txt b/cibuildwheel/resources/constraints-python311.txt index 84fb2ec3e..acd252d94 100644 --- a/cibuildwheel/resources/constraints-python311.txt +++ b/cibuildwheel/resources/constraints-python311.txt @@ -18,8 +18,12 @@ packaging==26.0 # via # build # delocate +patchelf==0.17.2.4 + # via -r cibuildwheel/resources/constraints.in pip==26.0.1 # via -r cibuildwheel/resources/constraints.in +pkgconf==2.5.1.post1 + # via -r cibuildwheel/resources/constraints.in platformdirs==4.9.4 # via # python-discovery diff --git a/cibuildwheel/resources/constraints-python312.txt b/cibuildwheel/resources/constraints-python312.txt index 84fb2ec3e..acd252d94 100644 --- a/cibuildwheel/resources/constraints-python312.txt +++ b/cibuildwheel/resources/constraints-python312.txt @@ -18,8 +18,12 @@ packaging==26.0 # via # build # delocate +patchelf==0.17.2.4 + # via -r cibuildwheel/resources/constraints.in pip==26.0.1 # via -r cibuildwheel/resources/constraints.in +pkgconf==2.5.1.post1 + # via -r cibuildwheel/resources/constraints.in platformdirs==4.9.4 # via # python-discovery diff --git a/cibuildwheel/resources/constraints-python313.txt b/cibuildwheel/resources/constraints-python313.txt index 84fb2ec3e..acd252d94 100644 --- a/cibuildwheel/resources/constraints-python313.txt +++ b/cibuildwheel/resources/constraints-python313.txt @@ -18,8 +18,12 @@ packaging==26.0 # via # build # delocate +patchelf==0.17.2.4 + # via -r cibuildwheel/resources/constraints.in pip==26.0.1 # via -r cibuildwheel/resources/constraints.in +pkgconf==2.5.1.post1 + # via -r cibuildwheel/resources/constraints.in platformdirs==4.9.4 # via # python-discovery diff --git a/cibuildwheel/resources/constraints-python314.txt b/cibuildwheel/resources/constraints-python314.txt index 84fb2ec3e..acd252d94 100644 --- a/cibuildwheel/resources/constraints-python314.txt +++ b/cibuildwheel/resources/constraints-python314.txt @@ -18,8 +18,12 @@ packaging==26.0 # via # build # delocate +patchelf==0.17.2.4 + # via -r cibuildwheel/resources/constraints.in pip==26.0.1 # via -r cibuildwheel/resources/constraints.in +pkgconf==2.5.1.post1 + # via -r cibuildwheel/resources/constraints.in platformdirs==4.9.4 # via # python-discovery diff --git a/cibuildwheel/resources/constraints-python38.txt b/cibuildwheel/resources/constraints-python38.txt index 579b21e2c..435aa2d3a 100644 --- a/cibuildwheel/resources/constraints-python38.txt +++ b/cibuildwheel/resources/constraints-python38.txt @@ -13,15 +13,21 @@ filelock==3.16.1 # python-discovery # virtualenv importlib-metadata==8.5.0 - # via build + # via + # build + # pkgconf macholib==1.16.4 # via delocate packaging==26.0 # via # build # delocate +patchelf==0.17.2.4 + # via -r cibuildwheel/resources/constraints.in pip==25.0.1 # via -r cibuildwheel/resources/constraints.in +pkgconf==2.2.0.post0 + # via -r cibuildwheel/resources/constraints.in platformdirs==4.3.6 # via # python-discovery diff --git a/cibuildwheel/resources/constraints-python39.txt b/cibuildwheel/resources/constraints-python39.txt index 279adc673..8c8c41d8f 100644 --- a/cibuildwheel/resources/constraints-python39.txt +++ b/cibuildwheel/resources/constraints-python39.txt @@ -13,15 +13,21 @@ filelock==3.19.1 # python-discovery # virtualenv importlib-metadata==8.7.1 - # via build + # via + # build + # pkgconf macholib==1.16.4 # via delocate packaging==26.0 # via # build # delocate +patchelf==0.17.2.4 + # via -r cibuildwheel/resources/constraints.in pip==26.0.1 # via -r cibuildwheel/resources/constraints.in +pkgconf==2.4.3.post2 + # via -r cibuildwheel/resources/constraints.in platformdirs==4.4.0 # via # python-discovery diff --git a/cibuildwheel/resources/constraints.in b/cibuildwheel/resources/constraints.in index 50bfabb6e..562df62cf 100644 --- a/cibuildwheel/resources/constraints.in +++ b/cibuildwheel/resources/constraints.in @@ -1,4 +1,10 @@ pip build -delocate virtualenv + +# Android +patchelf +pkgconf + +# macOS +delocate diff --git a/cibuildwheel/resources/constraints.txt b/cibuildwheel/resources/constraints.txt index 84fb2ec3e..acd252d94 100644 --- a/cibuildwheel/resources/constraints.txt +++ b/cibuildwheel/resources/constraints.txt @@ -18,8 +18,12 @@ packaging==26.0 # via # build # delocate +patchelf==0.17.2.4 + # via -r cibuildwheel/resources/constraints.in pip==26.0.1 # via -r cibuildwheel/resources/constraints.in +pkgconf==2.5.1.post1 + # via -r cibuildwheel/resources/constraints.in platformdirs==4.9.4 # via # python-discovery diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml index 78895bf9a..041978153 100644 --- a/cibuildwheel/resources/defaults.toml +++ b/cibuildwheel/resources/defaults.toml @@ -1,3 +1,6 @@ +# These are the defaults used by cibuildwheel itself. They should match the values in +# generate_schema.py, which are used by external tools for validation and IDE support. + [tool.cibuildwheel] build = "*" skip = "" @@ -60,6 +63,7 @@ repair-wheel-command = "delocate-wheel --require-archs {delocate_archs} -w {dest [tool.cibuildwheel.windows] [tool.cibuildwheel.android] +repair-wheel-command = "auditwheel repair --ldpaths {ldpaths} -w {dest_dir} {wheel}" [tool.cibuildwheel.ios] diff --git a/docs/options.md b/docs/options.md index fd84d83c0..7f2d6a2b0 100644 --- a/docs/options.md +++ b/docs/options.md @@ -909,8 +909,7 @@ Default: - on Linux: `'auditwheel repair -w {dest_dir} {wheel}'` - on macOS: `'delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}'` -- on Android: There is no default command, but cibuildwheel will add `libc++` to the - wheel if anything links against it. Setting a command will replace this behavior. +- on Android: `'auditwheel repair --ldpaths {ldpaths} -w {dest_dir} {wheel}'` - on Pyodide: You can use `pyodide auditwheel repair --libdir /path/to/libraries --output-dir {dest_dir} {wheel}` command to repair the wheel. Unlike other platforms, this command is not set by default as you need to explicitly specify the library directory. You might not want to use the libraries in the system @@ -925,6 +924,7 @@ The following placeholders must be used inside the command and will be replaced - `{wheel}` for the absolute path to the built wheel - `{dest_dir}` for the absolute path of the directory where to create the repaired wheel - `{delocate_archs}` (macOS only) comma-separated list of architectures in the wheel. +- `{ldpaths}` (Android only) colon-separated list of directories to search for external libraries. cibuildwheel will set this to include any necessary locations in the NDK. To add your own locations, use the `LD_LIBRARY_PATH` environment variable. You can use the `{package}` or `{project}` placeholders in your `repair-wheel-command` to refer to the package being built or the project root, respectively. diff --git a/docs/platforms.md b/docs/platforms.md index 44d8afbf5..a31961e72 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -222,13 +222,9 @@ It also requires the following commands to be on the `PATH`: Android builds will honor the `ANDROID_API_LEVEL` environment variable to set the minimum supported [API level](https://developer.android.com/tools/releases/platforms) -for generated wheels. This will default to the minimum API level of the selected Python -version. - -If the [`repair-wheel-command`](options.md#repair-wheel-command) adds any libraries to -the wheel, then `ANDROID_API_LEVEL` must be at least 24. This is already the default -when building for Python 3.14 and later, but you may need to set it when building for -Python 3.13. +for generated wheels. This defaults to 24, which is supported by [99% of active +devices](https://dl.google.com/android/studio/metadata/distributions.json), and is the first +version to support RUNPATH, which auditwheel needs in order to graft external libraries. ### Build frontend support diff --git a/pyproject.toml b/pyproject.toml index f2b5169f9..fd2dd65bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,8 +44,6 @@ dependencies = [ "filelock", "humanize", "packaging>=20.9", - # patchelf is used for Android - "patchelf; (sys_platform == 'linux' or sys_platform == 'darwin') and (platform_machine == 'x86_64' or platform_machine == 'arm64' or platform_machine == 'aarch64')", "platformdirs", "pyelftools>=0.29", "wheel>=0.33.6", diff --git a/test/test_android.py b/test/test_android.py index eb16e2b13..fc57fb90d 100644 --- a/test/test_android.py +++ b/test/test_android.py @@ -125,7 +125,7 @@ def test_frontend_good(tmp_path, build_frontend_env): tmp_path, add_env={**cp313_env, **build_frontend_env, "CIBW_TEST_COMMAND": "python -m site"}, ) - assert wheels == [f"spam-0.1.0-cp313-cp313-android_21_{native_arch.android_abi}.whl"] + assert wheels == [f"spam-0.1.0-cp313-cp313-android_24_{native_arch.android_abi}.whl"] @pytest.mark.parametrize("frontend", ["pip"]) @@ -136,7 +136,10 @@ def test_frontend_bad(frontend, tmp_path, capfd): tmp_path, add_env={**cp313_env, "CIBW_BUILD_FRONTEND": frontend}, ) - assert "Android requires the build frontend to be 'build'" in capfd.readouterr().err + assert ( + f"Android requires the build frontend to be 'build' or 'uv', not '{frontend}'" + in capfd.readouterr().err + ) @needs_emulator @@ -162,7 +165,7 @@ def test_archs(tmp_path, capfd): ), }, ) - assert wheels == [f"spam-0.1.0-cp313-cp313-android_21_{arch.android_abi}.whl" for arch in archs] + assert wheels == [f"spam-0.1.0-cp313-cp313-android_24_{arch.android_abi}.whl" for arch in archs] stdout, stderr = capfd.readouterr() lines = (line for line in stdout.splitlines() if line.startswith("Hello from")) @@ -470,17 +473,16 @@ def test_libcxx(tmp_path, capfd): "PATH": non_venv_path, } - # Including external libraries requires API level 24. + # Including external libraries requires API level 24. This is enforced by auditwheel. + cp313_android_21_env = {**cp313_test_env, "ANDROID_API_LEVEL": "21"} with pytest.raises(CalledProcessError): - cibuildwheel_run(project_dir, add_env=cp313_test_env, output_dir=output_dir) - assert "libc++_shared.so requires ANDROID_API_LEVEL to be at least 24" in capfd.readouterr().err - - wheels = cibuildwheel_run( - project_dir, - add_env={**cp313_test_env, "ANDROID_API_LEVEL": "24"}, - output_dir=output_dir, + cibuildwheel_run(project_dir, add_env=cp313_android_21_env, output_dir=output_dir) + assert ( + "Grafting libraries with RUNPATH requires API level 24 or higher" in capfd.readouterr().err ) - assert len(wheels) == 1 + + wheels = cibuildwheel_run(project_dir, add_env=cp313_test_env, output_dir=output_dir) + assert wheels == [f"spam-0.1.0-cp313-cp313-android_24_{native_arch.android_abi}.whl"] names = ZipFile(output_dir / wheels[0]).namelist() libcxx_names = [ name for name in names if re.fullmatch(r"spam\.libs/libc\+\+_shared-[0-9a-f]{8}\.so", name) @@ -488,12 +490,12 @@ def test_libcxx(tmp_path, capfd): assert len(libcxx_names) == 1 assert "ham: 1, spam: 0" in capfd.readouterr().out - # A C package should not include libc++. + # A C package should not include libc++, and can therefore use an older API level. rmtree(project_dir) rmtree(output_dir) new_c_project().generate(project_dir) - wheels = cibuildwheel_run(project_dir, add_env=cp313_env, output_dir=output_dir) - assert len(wheels) == 1 + wheels = cibuildwheel_run(project_dir, add_env=cp313_android_21_env, output_dir=output_dir) + assert wheels == [f"spam-0.1.0-cp313-cp313-android_21_{native_arch.android_abi}.whl"] for name in ZipFile(output_dir / wheels[0]).namelist(): assert ".libs" not in name diff --git a/test/utils.py b/test/utils.py index bb599abd3..cf2102fd3 100644 --- a/test/utils.py +++ b/test/utils.py @@ -173,6 +173,7 @@ def expected_wheels( musllinux_versions: list[str] | None = None, macosx_deployment_target: str | None = None, iphoneos_deployment_target: str | None = None, + android_api_level: int | None = None, machine_arch: str | None = None, platform: str | None = None, python_abi_tags: list[str] | None = None, @@ -197,6 +198,9 @@ def expected_wheels( if iphoneos_deployment_target is None: iphoneos_deployment_target = os.environ.get("IPHONEOS_DEPLOYMENT_TARGET", "13.0") + if android_api_level is None: + android_api_level = int(os.environ.get("ANDROID_API_LEVEL", "24")) + architectures = [machine_arch] if not single_arch: if platform == "linux" and full_auto: @@ -222,6 +226,7 @@ def expected_wheels( musllinux_versions=musllinux_versions, macosx_deployment_target=macosx_deployment_target, iphoneos_deployment_target=iphoneos_deployment_target, + android_api_level=android_api_level, platform=platform, python_abi_tags=python_abi_tags, include_universal2=include_universal2, @@ -238,6 +243,7 @@ def _expected_wheels( musllinux_versions: list[str] | None, macosx_deployment_target: str, iphoneos_deployment_target: str, + android_api_level: int, platform: str, python_abi_tags: list[str] | None, include_universal2: bool, @@ -392,11 +398,7 @@ def _expected_wheels( platform_tags.append(f"macosx_{min_macosx.replace('.', '_')}_universal2") elif platform == "android": - api_level = { - "cp313-cp313": 21, - "cp314-cp314": 24, - }[python_abi_tag] - platform_tags = [f"android_{api_level}_{machine_arch}"] + platform_tags = [f"android_{android_api_level}_{machine_arch}"] elif platform == "ios": if machine_arch == "x86_64":