From a77fdc4fef3768f5490d2b77ba24785ddd2097c9 Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Thu, 7 May 2026 13:51:41 +1000 Subject: [PATCH 1/2] Add make_py_package_standalone.py PEX builder This is a claude oneshot except for the `zip_safe = True` (default) behaviour. The one Meta uses has never been open sourced. This is a version for OSS users. --- prelude/python/tools/BUCK | 6 + .../tools/make_py_package_standalone.py | 494 ++++++++++++++++++ prelude/toolchains/python.bzl | 4 + 3 files changed, 504 insertions(+) create mode 100644 prelude/python/tools/make_py_package_standalone.py diff --git a/prelude/python/tools/BUCK b/prelude/python/tools/BUCK index 1b5c3ec8c0abd..9e42fe6805d28 100644 --- a/prelude/python/tools/BUCK +++ b/prelude/python/tools/BUCK @@ -157,3 +157,9 @@ prelude.python_bootstrap_binary( main = "gather_libpython_symbols.py", visibility = ["PUBLIC"], ) + +prelude.python_bootstrap_binary( + name = "make_py_package_standalone", + main = "make_py_package_standalone.py", + visibility = ["PUBLIC"], +) diff --git a/prelude/python/tools/make_py_package_standalone.py b/prelude/python/tools/make_py_package_standalone.py new file mode 100644 index 0000000000000..94cfb9b2b3916 --- /dev/null +++ b/prelude/python/tools/make_py_package_standalone.py @@ -0,0 +1,494 @@ +#!/usr/bin/env python3 + +""" +Build a standalone PEX (Python EXecutable) file. + +A PEX is a zip file with a shebang prepended. Python can execute zip files +directly: when you run `python3 archive.pex`, Python finds `__main__.py` +inside the zip and runs it. With a shebang (`#!/usr/bin/env python3`), the +OS can run it directly too: `./archive.pex`. + +This tool replaces Meta's internal `make_par` standalone builder. It accepts +the same CLI interface that the Buck2 prelude's `make_py_package` action +generates (module manifests, bootstrap args, etc.) and produces a single +self-contained executable. + +The generated PEX bootstrap (__main__.py) works as follows: + 1. Extracts the zip to a cache directory (~/.cache/par//) + 2. Sets LD_LIBRARY_PATH for native libraries + 3. Sets PYTHONPATH to the extracted modules + 4. Re-execs Python so the dynamic linker picks up LD_LIBRARY_PATH + (glibc caches LD_LIBRARY_PATH at process start, so os.environ changes + after startup don't affect dlopen — re-exec is required) + 5. The re-exec'd Python loads the entry point via __par__.bootstrap.run_as_main + +Why extract instead of running from the zip? + - Native extensions (.so) cannot be dlopen'd from inside a zip file + - Even "zip-safe" pure Python code benefits from extraction when native + libraries are present, since LD_LIBRARY_PATH must be set before Python + starts loading extensions + +CLI interface (matches what make_py_package.bzl generates): + + Module args (via @argfile expansion): + --module-manifest=PATH JSON: [[dest, src, origin], ...] + --resource-manifest=PATH JSON: [[dest, src, origin], ...] + --native-library-src=PATH .so file to bundle + --native-library-dest=PATH relative dest for the .so + + Bootstrap args: + --python INTERP interpreter for shebang + --host-python INTERP host interpreter (unused, for compat) + --entry-point MODULE module to run (mutually exclusive with --main-function) + --main-function MOD.FUNC function entry point + --main-runner MOD.FUNC bootstrap runner (default: __par__.bootstrap.run_as_main) + --no-zip-safe force extraction mode (always on when native libs present) + --native-library-runtime-path=PATH extra LD_LIBRARY_PATH entries + --preload=PATH libraries to LD_PRELOAD + OUTPUT output .pex file path (positional) + + Passthrough: + --passthrough ARG forwarded to bootstrap (e.g. --runtime_env=K=V) +""" + +import argparse +import json +import os +import sys +import tempfile +import zipfile +from pathlib import Path + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Create a standalone PEX file", + fromfile_prefix_chars="@", + ) + + # -- Module args (from _pex_modules_args / _pex_modules_common_args) -- + parser.add_argument( + "--module-manifest", + action="append", + dest="module_manifests", + default=[], + ) + parser.add_argument( + "--resource-manifest", + action="append", + dest="resource_manifests", + default=[], + ) + parser.add_argument( + "--native-library-src", + type=Path, + dest="native_library_srcs", + action="append", + default=[], + ) + parser.add_argument( + "--native-library-dest", + type=Path, + dest="native_library_dests", + action="append", + default=[], + ) + + # -- Bootstrap args (from _pex_bootstrap_args) -- + parser.add_argument( + "--preload", + dest="preload_libraries", + action="append", + default=[], + ) + parser.add_argument("--python", required=True) + parser.add_argument("--host-python", required=True) + + entry_point = parser.add_mutually_exclusive_group(required=True) + entry_point.add_argument("--entry-point") + entry_point.add_argument("--main-function") + + parser.add_argument("--main-runner", required=True) + parser.add_argument("--no-zip-safe", action="store_true") + parser.add_argument( + "--native-library-runtime-path", + dest="native_library_runtime_paths", + action="append", + default=[], + ) + + # Positional output path + parser.add_argument("output", type=Path) + + # -- Compatibility / passthrough -- + parser.add_argument("--passthrough", action="append", default=[]) + parser.add_argument("--modules-dir", type=Path, default=None) + parser.add_argument("--dwp-src", dest="dwp_srcs", action="append", default=[]) + parser.add_argument("--dwp-dest", dest="dwp_dests", action="append", default=[]) + parser.add_argument("--debuginfo-src", dest="debuginfo_srcs", action="append", default=[]) + parser.add_argument( + "--bytecode-artifacts", + dest="bytecode_artifacts", + action="append", + default=[], + ) + parser.add_argument("--omnibus-debug-info", default=None) + + return parser.parse_args() + + +# --------------------------------------------------------------------------- +# File collection +# --------------------------------------------------------------------------- + + +MODULE_SUFFIXES = {".py", ".so", ".pyd", ".dll"} + + +def collect_files( + args: argparse.Namespace, +) -> dict[str, str]: + """Return {zip_dest_path: abs_src_path} for every file to bundle.""" + files: dict[str, str] = {} + + for manifest_path in args.module_manifests: + with open(manifest_path) as f: + for dest, src, _origin in json.load(f): + files[dest] = src + + for manifest_path in args.resource_manifests: + with open(manifest_path) as f: + for dest, src, _origin in json.load(f): + files[dest] = src + + for src, dest in zip(args.native_library_srcs, args.native_library_dests): + files[str(dest)] = str(src) + + return files + + +def compute_init_pys(files: dict[str, str]) -> set[str]: + """Compute __init__.py paths needed so Python recognises packages.""" + init_dirs: set[Path] = set() + for dest in files: + p = Path(dest) + if p.suffix in MODULE_SUFFIXES: + package = p.parent + while package != Path("") and package != Path("."): + init_dirs.add(package) + package = package.parent + + needed: set[str] = set() + for d in init_dirs: + init_path = str(d / "__init__.py") + if init_path not in files: + needed.add(init_path) + return needed + + +# --------------------------------------------------------------------------- +# Bootstrap generation +# --------------------------------------------------------------------------- + + +def parse_passthrough_env(passthroughs: list[str]) -> dict[str, str]: + env: dict[str, str] = {} + for arg in passthroughs: + if arg.startswith("--runtime_env="): + kv = arg[len("--runtime_env=") :] + k, _, v = kv.partition("=") + if k: + env[k] = v + return env + + +def generate_main_py(args: argparse.Namespace) -> str: + """Generate the __main__.py bootstrap that goes inside the zip.""" + main_module: str + main_function: str + if args.main_function: + mod, _, func = args.main_function.rpartition(".") + main_module = mod + main_function = func + else: + main_module = args.entry_point + main_function = "" + + main_runner_module, _, main_runner_function = args.main_runner.rpartition(".") + runtime_env = parse_passthrough_env(args.passthrough) + + if args.no_zip_safe: + preload_basenames = [os.path.basename(p) for p in args.preload_libraries] + template = _BOOTSTRAP_EXTRACT_TEMPLATE + template = template.replace("", repr(preload_basenames)) + template = template.replace("", repr(args.native_library_runtime_paths)) + else: + template = _BOOTSTRAP_ZIP_TEMPLATE + + template = template.replace("", repr(main_module)) + template = template.replace("", repr(main_function)) + template = template.replace("", main_runner_module) + template = template.replace("", main_runner_function) + template = template.replace("", repr(runtime_env)) + return template + + +_BOOTSTRAP_ZIP_TEMPLATE = """\ +import os +import sys + +MAIN_MODULE = +MAIN_FUNCTION = +RUNTIME_ENV = + +for k, v in RUNTIME_ENV.items(): + os.environ[k] = v + +from import as _run_as_main +_run_as_main(MAIN_MODULE, MAIN_FUNCTION) +""" + +_BOOTSTRAP_EXTRACT_TEMPLATE = """\ +import fcntl +import hashlib +import os +import platform +import signal +import subprocess +import sys +import time +import zipfile + +MAIN_MODULE = +MAIN_FUNCTION = +MAIN_RUNNER_MODULE = "" +MAIN_RUNNER_FUNCTION = "" +PRELOAD_BASENAMES = +NATIVE_RUNTIME_PATHS = +RUNTIME_ENV = + + +def _par_path(): + return os.path.abspath(sys.argv[0]) + + +def _cache_dir(par): + st = os.stat(par) + key = "{}:{}:{}".format(par, st.st_size, st.st_mtime_ns) + digest = hashlib.sha256(key.encode()).hexdigest()[:16] + base = os.environ.get( + "PAR_TEMP_DIR", + os.path.join(os.path.expanduser("~"), ".cache", "par"), + ) + return os.path.join(base, "{}--{}".format(os.path.basename(par), digest)) + + +def _extract(par, dest): + stamp = os.path.join(dest, ".par_extracted") + if os.path.exists(stamp): + return + + os.makedirs(dest, exist_ok=True) + lock_path = dest + ".lock" + with open(lock_path, "w") as lf: + # fcntl.lockf wraps POSIX fcntl() locks; prefer it over flock() which is not + # in POSIX. See https://apenwarr.ca/log/20101213 + fcntl.lockf(lf, fcntl.LOCK_EX) + try: + if os.path.exists(stamp): + return + with zipfile.ZipFile(par, "r") as zf: + zf.extractall(dest) + for root, _dirs, filenames in os.walk(dest): + for fn in filenames: + if fn.endswith(".so") or ".so." in fn: + p = os.path.join(root, fn) + os.chmod(p, os.stat(p).st_mode | 0o555) + with open(stamp, "w") as sf: + sf.write(par + "\\n") + finally: + fcntl.lockf(lf, fcntl.LOCK_UN) + + +def main(): + os.environ["PAR_LAUNCH_TIMESTAMP"] = str(time.time()) + + par = _par_path() + cache = _cache_dir(par) + _extract(par, cache) + + for k, v in RUNTIME_ENV.items(): + os.environ[k] = v + + if platform.system() == "Darwin": + native_env = "DYLD_LIBRARY_PATH" + else: + native_env = "LD_LIBRARY_PATH" + + lib_dirs = [cache] + NATIVE_RUNTIME_PATHS + old_lib_path = os.environ.get(native_env, "") + if old_lib_path: + os.environ["FB_SAVED_" + native_env] = old_lib_path + os.environ[native_env] = os.pathsep.join( + d for d in lib_dirs + [old_lib_path] if d + ) + + if PRELOAD_BASENAMES: + if platform.system() == "Darwin": + preload_env = "DYLD_INSERT_LIBRARIES" + else: + preload_env = "LD_PRELOAD" + preload_paths = [os.path.join(cache, b) for b in PRELOAD_BASENAMES] + old_preload = os.environ.get(preload_env, "") + if old_preload: + os.environ["FB_SAVED_" + preload_env] = old_preload + os.environ[preload_env] = os.pathsep.join( + p for p in preload_paths + [old_preload] if p + ) + + old_pythonpath = os.environ.get("PYTHONPATH", "") + if old_pythonpath: + os.environ["FB_SAVED_PYTHONPATH"] = old_pythonpath + os.environ["PYTHONPATH"] = cache + + os.environ["PAR_INVOKED_NAME_TAG"] = sys.argv[0] + + startup = ( + "# " + sys.argv[0] + "\\n" + "def __run():\\n" + " import sys\\n" + " assert sys.argv[0] == \\"-c\\"\\n" + " sys.argv[0] = " + repr(sys.argv[0]) + "\\n" + " if \\"\\" in sys.path:\\n" + " sys.path.remove(\\"\\")\\n" + " from import as run_as_main\\n" + " run_as_main(" + repr(MAIN_MODULE) + ", " + repr(MAIN_FUNCTION) + ")\\n" + "__run()\\n" + ) + + args = [sys.executable, "-c", startup] + sys.argv[1:] + + if platform.system() == "Windows": + p = subprocess.Popen(args) + def handler(signum, frame): + if signum == signal.SIGINT: + p.send_signal(signal.CTRL_C_EVENT) + else: + p.terminate() + signal.signal(signal.SIGINT, handler) + p.wait() + sys.exit(p.returncode) + else: + os.execv(sys.executable, args) + + +main() +""" + + +# --------------------------------------------------------------------------- +# PEX assembly +# --------------------------------------------------------------------------- + + +import time as _time + +_ZIP_EPOCH = (1980, 1, 1, 0, 0, 0) + + +def _zip_add_file(zf: zipfile.ZipFile, src: str, dest: str) -> bool: + """Add a file to the zip, clamping timestamps for Nix store files. + + Returns False if the file could not be read (e.g. broken symlink). + """ + try: + st = os.stat(src) + except FileNotFoundError: + print( + "warning: skipping {} (broken symlink or missing)".format(src), + file=sys.stderr, + ) + return False + mtime = _time.localtime(st.st_mtime) + date_time = ( + mtime.tm_year, + mtime.tm_mon, + mtime.tm_mday, + mtime.tm_hour, + mtime.tm_min, + mtime.tm_sec, + ) + if date_time < _ZIP_EPOCH: + date_time = _ZIP_EPOCH + info = zipfile.ZipInfo(dest, date_time) + info.compress_type = zipfile.ZIP_DEFLATED + info.external_attr = (st.st_mode & 0xFFFF) << 16 + with open(src, "rb") as f: + zf.writestr(info, f.read()) + return True + + +def build_pex(args: argparse.Namespace) -> None: + files = collect_files(args) + + if not args.no_zip_safe: + native_files = [ + d for d in files if d.endswith(".so") or ".so." in d or d.endswith(".pyd") or d.endswith(".dll") + ] + if native_files: + print( + "error: PEX contains native extensions but zip_safe is not False.\n" + "Native .so files cannot be loaded from inside a zip.\n" + "Set zip_safe = False on your python_binary target. The resulting\n" + "pex will extract itself to a cache directory and execute from there.\n" + "First 5 native files:\n" + "\n".join(" " + f for f in native_files[:5]), + file=sys.stderr, + ) + sys.exit(1) + + init_pys = compute_init_pys(files) + main_py = generate_main_py(args) + + python = args.python + if os.path.isabs(python): + shebang = "#!{}\n".format(python) + else: + shebang = "#!/usr/bin/env {}\n".format(python) + + output_dir = args.output.parent + output_dir.mkdir(parents=True, exist_ok=True) + + fd, tmp_path = tempfile.mkstemp( + dir=output_dir, + prefix=".{}.tmp.".format(args.output.name), + ) + try: + with os.fdopen(fd, "wb") as f: + f.write(shebang.encode("utf-8")) + + with zipfile.ZipFile(f, "w", zipfile.ZIP_DEFLATED) as zf: + zf.writestr("__main__.py", main_py) + + for init_path in sorted(init_pys): + zf.writestr(init_path, "") + + for dest, src in sorted(files.items()): + _zip_add_file(zf, src, dest) + + os.chmod(tmp_path, 0o755) + os.rename(tmp_path, args.output) + except BaseException: + try: + os.unlink(tmp_path) + except OSError: + pass + raise + + +def main() -> None: + args = parse_args() + build_pex(args) + + +if __name__ == "__main__": + main() diff --git a/prelude/toolchains/python.bzl b/prelude/toolchains/python.bzl index e83c65871ea10..d143a52d9396b 100644 --- a/prelude/toolchains/python.bzl +++ b/prelude/toolchains/python.bzl @@ -98,6 +98,8 @@ def _system_python_toolchain_impl(ctx): host_interpreter = RunInfo(args = [ctx.attrs.interpreter]), interpreter = RunInfo(args = [ctx.attrs.interpreter]), compile = RunInfo(args = ["echo", "COMPILEINFO"]), + make_py_package_standalone = ctx.attrs.make_py_package_standalone[RunInfo], + build_standalone_binaries_locally = ctx.attrs.build_standalone_binaries_locally, package_style = "inplace", pex_extension = ctx.attrs.pex_extension, native_link_strategy = "separate", @@ -111,6 +113,8 @@ system_python_toolchain = rule( "binary_linker_flags": attrs.default_only(attrs.list(attrs.arg(), default = [])), "interpreter": attrs.string(default = _INTERPRETER), "linker_flags": attrs.default_only(attrs.list(attrs.arg(), default = [])), + "build_standalone_binaries_locally": attrs.bool(default = True), + "make_py_package_standalone": attrs.exec_dep(default = "prelude//python/tools:make_py_package_standalone", providers = [RunInfo]), "pex_extension": attrs.string(default = ".pex"), }, is_toolchain_rule = True, From 663cc8db9b30bca2d70d48516187e859df910165 Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Mon, 11 May 2026 12:00:22 +1000 Subject: [PATCH 2/2] Split out the template strings into pex_bootstrap_zip.py.in (etc) --- prelude/python/tools/BUCK | 24 ++- .../tools/make_py_package_standalone.py | 159 +----------------- .../python/tools/pex_bootstrap_extract.py.in | 134 +++++++++++++++ prelude/python/tools/pex_bootstrap_zip.py.in | 12 ++ 4 files changed, 173 insertions(+), 156 deletions(-) create mode 100644 prelude/python/tools/pex_bootstrap_extract.py.in create mode 100644 prelude/python/tools/pex_bootstrap_zip.py.in diff --git a/prelude/python/tools/BUCK b/prelude/python/tools/BUCK index 9e42fe6805d28..44bb55f5dd519 100644 --- a/prelude/python/tools/BUCK +++ b/prelude/python/tools/BUCK @@ -159,7 +159,29 @@ prelude.python_bootstrap_binary( ) prelude.python_bootstrap_binary( - name = "make_py_package_standalone", + name = "make_py_package_standalone.py", main = "make_py_package_standalone.py", visibility = ["PUBLIC"], ) + +prelude.export_file( + name = "pex_bootstrap_zip.py.in", + src = "pex_bootstrap_zip.py.in", +) + +prelude.export_file( + name = "pex_bootstrap_extract.py.in", + src = "pex_bootstrap_extract.py.in", +) + +prelude.command_alias( + name = "make_py_package_standalone", + args = [ + "--zip-template", + "$(location :pex_bootstrap_zip.py.in)", + "--extract-template", + "$(location :pex_bootstrap_extract.py.in)", + ], + exe = ":make_py_package_standalone.py", + visibility = ["PUBLIC"], +) diff --git a/prelude/python/tools/make_py_package_standalone.py b/prelude/python/tools/make_py_package_standalone.py index 94cfb9b2b3916..170a9a1114f45 100644 --- a/prelude/python/tools/make_py_package_standalone.py +++ b/prelude/python/tools/make_py_package_standalone.py @@ -110,6 +110,8 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--main-runner", required=True) parser.add_argument("--no-zip-safe", action="store_true") + parser.add_argument("--zip-template", type=Path, required=True) + parser.add_argument("--extract-template", type=Path, required=True) parser.add_argument( "--native-library-runtime-path", dest="native_library_runtime_paths", @@ -219,11 +221,11 @@ def generate_main_py(args: argparse.Namespace) -> str: if args.no_zip_safe: preload_basenames = [os.path.basename(p) for p in args.preload_libraries] - template = _BOOTSTRAP_EXTRACT_TEMPLATE + template = args.extract_template.read_text() template = template.replace("", repr(preload_basenames)) template = template.replace("", repr(args.native_library_runtime_paths)) else: - template = _BOOTSTRAP_ZIP_TEMPLATE + template = args.zip_template.read_text() template = template.replace("", repr(main_module)) template = template.replace("", repr(main_function)) @@ -233,159 +235,6 @@ def generate_main_py(args: argparse.Namespace) -> str: return template -_BOOTSTRAP_ZIP_TEMPLATE = """\ -import os -import sys - -MAIN_MODULE = -MAIN_FUNCTION = -RUNTIME_ENV = - -for k, v in RUNTIME_ENV.items(): - os.environ[k] = v - -from import as _run_as_main -_run_as_main(MAIN_MODULE, MAIN_FUNCTION) -""" - -_BOOTSTRAP_EXTRACT_TEMPLATE = """\ -import fcntl -import hashlib -import os -import platform -import signal -import subprocess -import sys -import time -import zipfile - -MAIN_MODULE = -MAIN_FUNCTION = -MAIN_RUNNER_MODULE = "" -MAIN_RUNNER_FUNCTION = "" -PRELOAD_BASENAMES = -NATIVE_RUNTIME_PATHS = -RUNTIME_ENV = - - -def _par_path(): - return os.path.abspath(sys.argv[0]) - - -def _cache_dir(par): - st = os.stat(par) - key = "{}:{}:{}".format(par, st.st_size, st.st_mtime_ns) - digest = hashlib.sha256(key.encode()).hexdigest()[:16] - base = os.environ.get( - "PAR_TEMP_DIR", - os.path.join(os.path.expanduser("~"), ".cache", "par"), - ) - return os.path.join(base, "{}--{}".format(os.path.basename(par), digest)) - - -def _extract(par, dest): - stamp = os.path.join(dest, ".par_extracted") - if os.path.exists(stamp): - return - - os.makedirs(dest, exist_ok=True) - lock_path = dest + ".lock" - with open(lock_path, "w") as lf: - # fcntl.lockf wraps POSIX fcntl() locks; prefer it over flock() which is not - # in POSIX. See https://apenwarr.ca/log/20101213 - fcntl.lockf(lf, fcntl.LOCK_EX) - try: - if os.path.exists(stamp): - return - with zipfile.ZipFile(par, "r") as zf: - zf.extractall(dest) - for root, _dirs, filenames in os.walk(dest): - for fn in filenames: - if fn.endswith(".so") or ".so." in fn: - p = os.path.join(root, fn) - os.chmod(p, os.stat(p).st_mode | 0o555) - with open(stamp, "w") as sf: - sf.write(par + "\\n") - finally: - fcntl.lockf(lf, fcntl.LOCK_UN) - - -def main(): - os.environ["PAR_LAUNCH_TIMESTAMP"] = str(time.time()) - - par = _par_path() - cache = _cache_dir(par) - _extract(par, cache) - - for k, v in RUNTIME_ENV.items(): - os.environ[k] = v - - if platform.system() == "Darwin": - native_env = "DYLD_LIBRARY_PATH" - else: - native_env = "LD_LIBRARY_PATH" - - lib_dirs = [cache] + NATIVE_RUNTIME_PATHS - old_lib_path = os.environ.get(native_env, "") - if old_lib_path: - os.environ["FB_SAVED_" + native_env] = old_lib_path - os.environ[native_env] = os.pathsep.join( - d for d in lib_dirs + [old_lib_path] if d - ) - - if PRELOAD_BASENAMES: - if platform.system() == "Darwin": - preload_env = "DYLD_INSERT_LIBRARIES" - else: - preload_env = "LD_PRELOAD" - preload_paths = [os.path.join(cache, b) for b in PRELOAD_BASENAMES] - old_preload = os.environ.get(preload_env, "") - if old_preload: - os.environ["FB_SAVED_" + preload_env] = old_preload - os.environ[preload_env] = os.pathsep.join( - p for p in preload_paths + [old_preload] if p - ) - - old_pythonpath = os.environ.get("PYTHONPATH", "") - if old_pythonpath: - os.environ["FB_SAVED_PYTHONPATH"] = old_pythonpath - os.environ["PYTHONPATH"] = cache - - os.environ["PAR_INVOKED_NAME_TAG"] = sys.argv[0] - - startup = ( - "# " + sys.argv[0] + "\\n" - "def __run():\\n" - " import sys\\n" - " assert sys.argv[0] == \\"-c\\"\\n" - " sys.argv[0] = " + repr(sys.argv[0]) + "\\n" - " if \\"\\" in sys.path:\\n" - " sys.path.remove(\\"\\")\\n" - " from import as run_as_main\\n" - " run_as_main(" + repr(MAIN_MODULE) + ", " + repr(MAIN_FUNCTION) + ")\\n" - "__run()\\n" - ) - - args = [sys.executable, "-c", startup] + sys.argv[1:] - - if platform.system() == "Windows": - p = subprocess.Popen(args) - def handler(signum, frame): - if signum == signal.SIGINT: - p.send_signal(signal.CTRL_C_EVENT) - else: - p.terminate() - signal.signal(signal.SIGINT, handler) - p.wait() - sys.exit(p.returncode) - else: - os.execv(sys.executable, args) - - -main() -""" - - # --------------------------------------------------------------------------- # PEX assembly # --------------------------------------------------------------------------- diff --git a/prelude/python/tools/pex_bootstrap_extract.py.in b/prelude/python/tools/pex_bootstrap_extract.py.in new file mode 100644 index 0000000000000..7c08e950eb062 --- /dev/null +++ b/prelude/python/tools/pex_bootstrap_extract.py.in @@ -0,0 +1,134 @@ +import fcntl +import hashlib +import os +import platform +import signal +import subprocess +import sys +import time +import zipfile + +MAIN_MODULE = +MAIN_FUNCTION = +MAIN_RUNNER_MODULE = "" +MAIN_RUNNER_FUNCTION = "" +PRELOAD_BASENAMES = +NATIVE_RUNTIME_PATHS = +RUNTIME_ENV = + + +def _par_path(): + return os.path.abspath(sys.argv[0]) + + +def _cache_dir(par): + st = os.stat(par) + key = "{}:{}:{}".format(par, st.st_size, st.st_mtime_ns) + digest = hashlib.sha256(key.encode()).hexdigest()[:16] + base = os.environ.get( + "PAR_TEMP_DIR", + os.path.join(os.path.expanduser("~"), ".cache", "par"), + ) + return os.path.join(base, "{}--{}".format(os.path.basename(par), digest)) + + +def _extract(par, dest): + stamp = os.path.join(dest, ".par_extracted") + if os.path.exists(stamp): + return + + os.makedirs(dest, exist_ok=True) + lock_path = dest + ".lock" + with open(lock_path, "w") as lf: + # fcntl.lockf wraps POSIX fcntl() locks; prefer it over flock() which is not + # in POSIX. See https://apenwarr.ca/log/20101213 + fcntl.lockf(lf, fcntl.LOCK_EX) + try: + if os.path.exists(stamp): + return + with zipfile.ZipFile(par, "r") as zf: + zf.extractall(dest) + for root, _dirs, filenames in os.walk(dest): + for fn in filenames: + if fn.endswith(".so") or ".so." in fn: + p = os.path.join(root, fn) + os.chmod(p, os.stat(p).st_mode | 0o555) + with open(stamp, "w") as sf: + sf.write(par + "\n") + finally: + fcntl.lockf(lf, fcntl.LOCK_UN) + + +def main(): + os.environ["PAR_LAUNCH_TIMESTAMP"] = str(time.time()) + + par = _par_path() + cache = _cache_dir(par) + _extract(par, cache) + + for k, v in RUNTIME_ENV.items(): + os.environ[k] = v + + if platform.system() == "Darwin": + native_env = "DYLD_LIBRARY_PATH" + else: + native_env = "LD_LIBRARY_PATH" + + lib_dirs = [cache] + NATIVE_RUNTIME_PATHS + old_lib_path = os.environ.get(native_env, "") + if old_lib_path: + os.environ["FB_SAVED_" + native_env] = old_lib_path + os.environ[native_env] = os.pathsep.join( + d for d in lib_dirs + [old_lib_path] if d + ) + + if PRELOAD_BASENAMES: + if platform.system() == "Darwin": + preload_env = "DYLD_INSERT_LIBRARIES" + else: + preload_env = "LD_PRELOAD" + preload_paths = [os.path.join(cache, b) for b in PRELOAD_BASENAMES] + old_preload = os.environ.get(preload_env, "") + if old_preload: + os.environ["FB_SAVED_" + preload_env] = old_preload + os.environ[preload_env] = os.pathsep.join( + p for p in preload_paths + [old_preload] if p + ) + + old_pythonpath = os.environ.get("PYTHONPATH", "") + if old_pythonpath: + os.environ["FB_SAVED_PYTHONPATH"] = old_pythonpath + os.environ["PYTHONPATH"] = cache + + os.environ["PAR_INVOKED_NAME_TAG"] = sys.argv[0] + + startup = ( + "# " + sys.argv[0] + "\n" + "def __run():\n" + " import sys\n" + " assert sys.argv[0] == \"-c\"\n" + " sys.argv[0] = " + repr(sys.argv[0]) + "\n" + " if \"\" in sys.path:\n" + " sys.path.remove(\"\")\n" + " from import as run_as_main\n" + " run_as_main(" + repr(MAIN_MODULE) + ", " + repr(MAIN_FUNCTION) + ")\n" + "__run()\n" + ) + + args = [sys.executable, "-c", startup] + sys.argv[1:] + + if platform.system() == "Windows": + p = subprocess.Popen(args) + def handler(signum, frame): + if signum == signal.SIGINT: + p.send_signal(signal.CTRL_C_EVENT) + else: + p.terminate() + signal.signal(signal.SIGINT, handler) + p.wait() + sys.exit(p.returncode) + else: + os.execv(sys.executable, args) + + +main() diff --git a/prelude/python/tools/pex_bootstrap_zip.py.in b/prelude/python/tools/pex_bootstrap_zip.py.in new file mode 100644 index 0000000000000..c768dad46d383 --- /dev/null +++ b/prelude/python/tools/pex_bootstrap_zip.py.in @@ -0,0 +1,12 @@ +import os +import sys + +MAIN_MODULE = +MAIN_FUNCTION = +RUNTIME_ENV = + +for k, v in RUNTIME_ENV.items(): + os.environ[k] = v + +from import as _run_as_main +_run_as_main(MAIN_MODULE, MAIN_FUNCTION)