wip: make e2e tests runnable outside Meta

.buckconfig

@@ -8,14 +8,14 @@ root = gh_facebook_buck2
 [oss]
 internal_cell = fbcode
 stripped_root_dirs = buck2
+python_module_prefix = buck2
 
 [project]
 ignore = \
     app/buck2_explain, \
     app_dep_graph_rules, \
     examples, \
-    integrations/rust-project/tests, \
-    tests
+    integrations/rust-project/tests
 
 [rust]
 default_edition = 2024

BUCK

@@ -10,6 +10,7 @@ pagable_transition_alias(
     name = "buck2",
     actual = "//buck2/app/buck2:buck2-bin",
     labels = [ci.aarch64(ci.skip_test())],
+    visibility = ["PUBLIC"],
 )
 
 buck2_bundle(

defs.bzl

@@ -102,12 +102,13 @@ _pagable_transition_alias = rule(
     cfg = _pagable_transition,
 )
 
-def pagable_transition_alias(name: str, actual, labels):
+def pagable_transition_alias(name: str, actual, labels, visibility = None):
     platform = platform_utils.get_cxx_platform_for_base_path(native.package_name())
     default_target_platform = platform.target_platform
     _pagable_transition_alias(
         name = name,
         actual = translate_target(actual),
         labels = labels,
         default_target_platform = default_target_platform,
+        visibility = visibility,
     )

shed/completion_verify/BUCK

@@ -18,7 +18,7 @@ rust_binary(
         },
     }),
     deps = [
-        "fbsource//third-party/rust:buck-resources",
+        "fbcode//buck2/integrations/resources/rust:buck_resources",
         "fbsource//third-party/rust:clap",
         "fbsource//third-party/rust:ptyprocess",
         "fbsource//third-party/rust:tempfile",

shim/.buckconfig

@@ -10,7 +10,7 @@
 # error messages when the user doesn't run "reindeer" much more obvious,
 # because the default BUCK file just errors out. We then make reindeer generate
 # BUCK.reindeer to override that.
-name = BUCK.reindeer,BUCK
+name = BUCK.pydeer,BUCK.reindeer,BUCK
 
 [cells]
 gh_facebook_buck2_shims_meta = .

shim/buck2/app/modifier.bzl

@@ -0,0 +1,17 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is dual-licensed under either the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree or the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree. You may select, at your option, one of the
+# above-listed licenses.
+
+# OSS stubs for the Meta-internal `ovr_config//`-driven modifier set used by
+# `tests/buck_e2e.bzl`. The OSS build doesn't have those config targets, so
+# return empty lists.
+
+def buck2_modifiers():
+    return []
+
+def disable_buck2_modifiers():
+    return []

shim/buck2/tests/buck_e2e.bzl

@@ -0,0 +1,22 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is dual-licensed under either the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree or the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree. You may select, at your option, one of the
+# above-listed licenses.
+
+# Loads `@fbcode//buck2/tests:buck_e2e.bzl` resolve to the shim cell because
+# `fbcode` is aliased to the shim cell in OSS. Forward to the real file in the
+# buck2 root cell so the e2e tests can actually be parsed.
+
+load(
+    "@gh_facebook_buck2//tests:buck_e2e.bzl",
+    _buck2_core_tests = "buck2_core_tests",
+    _buck2_e2e_test = "buck2_e2e_test",
+    _buck_e2e_test = "buck_e2e_test",
+)
+
+buck2_core_tests = _buck2_core_tests
+buck2_e2e_test = _buck2_e2e_test
+buck_e2e_test = _buck_e2e_test

shim/build_defs/lib/oss.bzl

@@ -116,6 +116,18 @@ PREFIX_MAPPINGS = _parse_prefix_mappings(
     _filter_empty_strings(read_config("oss", "prefix_mappings", "").split(" ")),
 )
 
+# Prepended to default `base_module` in python rules, so that source files keep
+# the same dotted import path they have inside the internal monorepo (e.g.
+# `fbcode/buck2/tests/foo.py` is imported as `buck2.tests.foo`, but in OSS the
+# package_name is `tests/foo` and would otherwise be `tests.foo`).
+PYTHON_MODULE_PREFIX = read_config("oss", "python_module_prefix", "")
+
+def default_base_module():
+    pkg = native.package_name().replace("/", ".")
+    if PYTHON_MODULE_PREFIX:
+        return PYTHON_MODULE_PREFIX + "." + pkg if pkg else PYTHON_MODULE_PREFIX
+    return pkg
+
 # Hardcoded rewrite rules that apply to many projects and only produce targets
 # within the shim cell. They are applied after the rules from .buckconfig, and
 # will not be applied if any other rules match.
@@ -235,3 +247,4 @@ def _swap_root_dir_for_path(path: str, root_dir: str, new_root_dir) -> str:
         suffix = "/" + suffix
     replace_path = new_root_dir.removesuffix("/") + suffix
     return replace_path.removeprefix("/")
+

shim/build_defs/python_binary.bzl

@@ -6,8 +6,13 @@
 # of this source tree. You may select, at your option, one of the
 # above-listed licenses.
 
-def python_binary(srcs = [], **kwargs):
+load("@shim//build_defs/lib:oss.bzl", "default_base_module")
+
+def python_binary(srcs = [], base_module = None, **kwargs):
     _unused = srcs  # @unused
 
+    if base_module == None:
+        base_module = default_base_module()
+
     # @lint-ignore BUCKLINT: avoid "Direct usage of native rules is not allowed."
-    native.python_binary(**kwargs)
+    native.python_binary(base_module = base_module, **kwargs)

shim/build_defs/python_library.bzl

@@ -6,8 +6,14 @@
 # of this source tree. You may select, at your option, one of the
 # above-listed licenses.
 
-def python_library(srcs = [], visibility = ["PUBLIC"], **kwargs):
-    _unused = srcs  # @unused
+load("@shim//build_defs/lib:oss.bzl", "default_base_module", "translate_target")
+
+def python_library(srcs = [], visibility = ["PUBLIC"], deps = None, base_module = None, **kwargs):
+    if deps != None:
+        kwargs["deps"] = [translate_target(d) for d in deps]
+
+    if base_module == None:
+        base_module = default_base_module()
 
     # @lint-ignore BUCKLINT: avoid "Direct usage of native rules is not allowed."
-    native.python_library(visibility = visibility, **kwargs)
+    native.python_library(srcs = srcs, visibility = visibility, base_module = base_module, **kwargs)

shim/build_defs/python_pytest.bzl

@@ -0,0 +1,149 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is dual-licensed under either the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree or the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree. You may select, at your option, one of the
+# above-listed licenses.
+
+load("@prelude//utils:type_defs.bzl", "is_select")
+load("@shim//build_defs/lib:oss.bzl", "default_base_module", "translate_target")
+
+def _build_addopts(pytest_config, pytest_marks, pytest_expr, pytest_confcutdir):
+    parts = []
+    if pytest_config:
+        parts += ["-c", pytest_config]
+    if pytest_marks:
+        parts += ["-m", pytest_marks]
+    if pytest_expr:
+        parts += ["-k", pytest_expr]
+    if pytest_confcutdir:
+        parts += ["--confcutdir", pytest_confcutdir]
+    return parts
+
+def _src_to_module(src, base_module):
+    if not src.endswith(".py"):
+        fail("python_pytest sources must be .py files: {}".format(src))
+    relative = src[:-3].replace("/", ".")
+    return base_module + "." + relative if base_module else relative
+
+def _shell_single_quote(s):
+    return "'" + s.replace("'", "'\\''") + "'"
+
+def _translate_env_value_str(v):
+    # Rewrite any cell-qualified targets that appear inside `$(macro ...)`
+    # substrings (e.g. `$(location fbcode//buck2:buck2)`) so that env values
+    # produced by the Meta-internal `tests/buck_e2e.bzl` resolve correctly
+    # through the OSS shim cell mapping.
+    if "//" not in v:
+        return v
+    chunks = v.split("$(")
+    out = [chunks[0]]
+    for chunk in chunks[1:]:
+        if ")" not in chunk:
+            out.append("$(" + chunk)
+            continue
+        end = chunk.index(")")
+        macro_body = chunk[:end]
+        rest = chunk[end:]
+        tokens = macro_body.split(" ")
+        for i in range(len(tokens)):
+            if "//" in tokens[i]:
+                tokens[i] = translate_target(tokens[i])
+        out.append("$(" + " ".join(tokens) + rest)
+    return "".join(out)
+
+def _translate_env_value(v):
+    if is_select(v):
+        return select_map(v, _translate_env_value_str)
+    return _translate_env_value_str(v)
+
+def python_pytest(
+        name,
+        srcs = [],
+        env = None,
+        emails = None,
+        base_module = None,
+        deps = None,
+        resources = None,
+        pytest_config = None,
+        pytest_marks = None,
+        pytest_expr = None,
+        pytest_confcutdir = None,
+        skip_on_mode_mac = False,
+        skip_on_mode_win = False,
+        modifiers = None,
+        **kwargs):
+    # The shim has no native handling for these meta-internal attrs; tests
+    # silently run on all platforms in OSS.
+    _unused = (skip_on_mode_mac, skip_on_mode_win, modifiers)  # @unused
+
+    if deps != None:
+        kwargs["deps"] = [translate_target(d) for d in deps]
+
+    if resources != None:
+        # Meta's `python_pytest` takes resources as `{target: dest_path}`;
+        # the OSS `python_test` rule takes `{dest_path: target}`. Flip and
+        # rewrite the target paths to their shim equivalents.
+        kwargs["resources"] = {
+            dest: translate_target(target)
+            for target, dest in resources.items()
+        }
+
+    env = {k: _translate_env_value(v) for k, v in env.items()} if env else {}
+    extra = _build_addopts(pytest_config, pytest_marks, pytest_expr, pytest_confcutdir)
+
+    # The buck2 e2e tests use an async `buck` pytest fixture; pytest-asyncio's
+    # default "strict" mode requires explicit fixture marking, which the
+    # upstream tests don't have. Meta's internal harness sets this elsewhere.
+    extra = ["--asyncio-mode=auto"] + extra
+    if extra:
+        existing = env.get("PYTEST_ADDOPTS", "").strip()
+        joined = " ".join(extra)
+        env["PYTEST_ADDOPTS"] = (existing + " " + joined) if existing else joined
+
+    module_base = base_module if base_module != None else default_base_module()
+    test_modules = [_src_to_module(s, module_base) for s in srcs]
+
+    main_lines = [
+        "import sys",
+        "import pytest",
+        "",
+        "TEST_MODULES = [",
+    ] + ['    "{}",'.format(m) for m in test_modules] + [
+        "]",
+        "",
+        "if __name__ == \"__main__\":",
+        "    sys.exit(pytest.main([\"--pyargs\"] + TEST_MODULES + sys.argv[1:]))",
+    ]
+    quoted_lines = " ".join([_shell_single_quote(l) for l in main_lines])
+
+    main_genrule = "__{}__pytest_main".format(name)
+
+    # @lint-ignore BUCKLINT: avoid "Direct usage of native rules is not allowed."
+    native.genrule(
+        name = main_genrule,
+        out = "__pytest_main__.py",
+        cmd = "printf '%s\\n' " + quoted_lines + " > $OUT",
+    )
+
+    if emails != None:
+        kwargs["contacts"] = emails
+
+    # The buck2 e2e tests look up `__manifest__.fbmake["build_rule"]` (a Meta
+    # convention) to derive a stable isolation prefix per test. Provide a
+    # minimally compatible shape so tests that read this don't crash in OSS.
+    kwargs.setdefault(
+        "manifest_module_entries",
+        {"fbmake": {"build_rule": "{}:{}".format(native.package_name(), name)}},
+    )
+
+    # @lint-ignore BUCKLINT: avoid "Direct usage of native rules is not allowed."
+    native.python_test(
+        name = name,
+        srcs = list(srcs) + [":" + main_genrule],
+        env = env,
+        base_module = module_base,
+        main_module = (module_base + "." if module_base else "") + "__pytest_main__",
+        **kwargs
+    )

shim/build_defs/python_unittest.bzl

@@ -6,8 +6,13 @@
 # of this source tree. You may select, at your option, one of the
 # above-listed licenses.
 
-def python_unittest(srcs = [], py_version = None, **kwargs):
+load("@shim//build_defs/lib:oss.bzl", "default_base_module")
+
+def python_unittest(srcs = [], py_version = None, base_module = None, **kwargs):
     _unused = (srcs, py_version)  # @unused
 
+    if base_module == None:
+        base_module = default_base_module()
+
     # @lint-ignore BUCKLINT: avoid "Direct usage of native rules is not allowed."
-    native.python_test(**kwargs)
+    native.python_test(base_module = base_module, **kwargs)

shim/clifoundation/cli_target/buck_defs/cli.bzl

@@ -0,0 +1,45 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is dual-licensed under either the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree or the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree. You may select, at your option, one of the
+# above-listed licenses.
+
+# OSS stub for `@fbcode//clifoundation/cli_target/buck_defs:cli.bzl`. The Meta
+# CLI deployer infrastructure isn't ported to OSS — `cli.deployer(...)` is a
+# no-op so its argument-builder helpers can return any placeholder value.
+
+def _noop(**_kwargs):
+    pass
+
+def _placeholder(*_args, **_kwargs):
+    return None
+
+_TOKEN = struct()
+
+cli = struct(
+    deployer = _noop,
+    distribution = _placeholder,
+    bump_diffs = _placeholder,
+    destination = _placeholder,
+    repository = struct(FBSOURCE = _TOKEN),
+    metadata = _placeholder,
+    criticality = _placeholder,
+    level = struct(NOT_CRITICAL = _TOKEN),
+    packaging = _placeholder,
+    dotslash = _placeholder,
+    platform = struct(
+        linux = _placeholder,
+        macos = _placeholder,
+        windows = _placeholder,
+    ),
+    architecture = struct(AARCH64 = _TOKEN, X86_64 = _TOKEN),
+    release = _placeholder,
+    artifact_ci_config = _placeholder,
+    conveyor = _placeholder,
+    frequency = struct(DAILY = _TOKEN),
+    holiday_country = struct(UNITED_STATES = _TOKEN),
+    srconveyor = _placeholder,
+    rollout = struct(AT_ONCE = _TOKEN),
+)

shim/third-party/pypi/.gitignore

@@ -0,0 +1 @@
+**/BUCK.pydeer

shim/third-party/pypi/README.md

@@ -0,0 +1,4 @@
+# PyPI dependencies
+
+These have generated `BUCK.pydeer` files.
+If they're missing, you can generate them with `bootstrap/buck2 run shim//third-party/python/pydeer:buckify`.

shim/third-party/pypi/pyproject.toml

@@ -0,0 +1,14 @@
+[project]
+name = "buck2-third-party-deps"
+version = "0.1.0"
+requires-python = ">=3.11"
+dependencies = [
+    "pytest",
+    "pytest-asyncio",
+    "decorator",
+    "setuptools",
+    "typed-argument-parser",
+    "importlib-resources",
+    "dataclasses-json",
+    "packaging",
+]