Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions src/python/pants/backend/python/goals/pytest_runner_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,11 @@
from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
from pants.backend.python.util_rules.lockfile_metadata import PythonLockfileMetadataV3
from pants.backend.python.util_rules.pex import PexRequirementsInfo
from pants.backend.python.util_rules.pex_requirements import LoadedLockfile, Lockfile
from pants.backend.python.util_rules.pex_requirements import (
LoadedLockfile,
Lockfile,
LockfileFormat,
)
from pants.engine.fs import DigestContents, FileContent
from pants.engine.internals.native_engine import EMPTY_DIGEST
from pants.testutil.option_util import create_subsystem
Expand Down Expand Up @@ -98,7 +102,9 @@ def validate(reqs: list[str]) -> None:
only_binary=set(),
no_binary=set(),
)
loaded_lockfile = LoadedLockfile(EMPTY_DIGEST, "", metadata, 0, True, None, lockfile)
loaded_lockfile = LoadedLockfile(
EMPTY_DIGEST, "", metadata, 0, LockfileFormat.Pex, None, lockfile
)
run_rule_with_mocks(
validate_pytest_cov_included,
rule_args=[tool],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from pants.backend.python.util_rules.pex_requirements import (
LoadedLockfile,
Lockfile,
LockfileFormat,
PexRequirements,
Resolve,
)
Expand Down Expand Up @@ -52,7 +53,9 @@ def test_get_lockfile_metadata() -> None:
no_binary=set(),
)
lockfile = Lockfile("dummy_url", "dummy_description_of_origin", "dummy_resolve")
loaded_lockfile = LoadedLockfile(EMPTY_DIGEST, "", metadata, 0, True, None, lockfile)
loaded_lockfile = LoadedLockfile(
EMPTY_DIGEST, "", metadata, 0, LockfileFormat.Pex, None, lockfile
)
assert (
run_rule_with_mocks(
get_lockfile_metadata,
Expand Down
7 changes: 4 additions & 3 deletions src/python/pants/backend/python/util_rules/pex.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
LoadedLockfile,
LoadedLockfileRequest,
Lockfile,
LockfileFormat,
Resolve,
ResolvePexConfigRequest,
determine_resolve_pex_config,
Expand Down Expand Up @@ -571,7 +572,7 @@ async def _build_uv_venv(
uv_request.requirements.from_superset, **implicitly()
)
loaded_lockfile = await load_lockfile(LoadedLockfileRequest(lockfile), **implicitly())
if loaded_lockfile.is_pex_native:
if loaded_lockfile.lockfile_format == LockfileFormat.Pex:
try:
digest_contents = await get_digest_contents(loaded_lockfile.lockfile_digest)
lockfile_bytes = next(
Expand Down Expand Up @@ -793,7 +794,7 @@ async def _setup_pex_requirements(
if loaded_lockfile:
argv = (
["--lock", loaded_lockfile.lockfile_path, *pex_lock_resolver_args]
if loaded_lockfile.is_pex_native
if loaded_lockfile.lockfile_format == LockfileFormat.Pex
# We use pip to resolve a requirements.txt pseudo-lockfile, possibly with hashes.
else [
"--requirement",
Expand Down Expand Up @@ -843,7 +844,7 @@ async def _setup_pex_requirements(
loaded_lockfile = await load_lockfile(LoadedLockfileRequest(lockfile), **implicitly())

# NB: This is also validated in the constructor.
assert loaded_lockfile.is_pex_native
assert loaded_lockfile.lockfile_format == LockfileFormat.Pex
if not reqs_info.req_strings:
return _BuildPexRequirementsSetup([], [], concurrency_available)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
EntireLockfile,
LoadedLockfileRequest,
Lockfile,
LockfileFormat,
PexRequirements,
Resolve,
load_lockfile,
Expand Down Expand Up @@ -570,7 +571,7 @@ async def _determine_requirements_for_pex_from_targets(
loaded_lockfile = await load_lockfile(
LoadedLockfileRequest(chosen_resolve.lockfile), **implicitly()
)
pex_native_subsetting_supported = loaded_lockfile.is_pex_native
pex_native_subsetting_supported = loaded_lockfile.lockfile_format == LockfileFormat.Pex
if loaded_lockfile.as_constraints_strings:
requirements = dataclasses.replace(
requirements,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
)
from pants.backend.python.util_rules.pex_requirements import (
EntireLockfile,
LockfileFormat,
PexRequirements,
Resolve,
)
Expand Down Expand Up @@ -172,10 +173,12 @@ class RequirementMode(Enum):
global_requirement_constraints = ["constraint1", "constraint2"]

resolve__pex = Resolve("pex", False)
loaded_lockfile__pex = Mock(is_pex_native=True, as_constraints_strings=None)
loaded_lockfile__pex = Mock(lockfile_format=LockfileFormat.Pex, as_constraints_strings=None)
chosen_resolve__pex = Mock(lockfile=Mock())
chosen_resolve__pex.name = "pex" # name has special meaning in Mock(), so must set it here.
loaded_lockfile__not_pex = Mock(is_pex_native=False, as_constraints_strings=req_strings)
loaded_lockfile__not_pex = Mock(
lockfile_format=LockfileFormat.ConstraintsDeprecated, as_constraints_strings=req_strings
)
chosen_resolve__not_pex = Mock(lockfile=Mock())
chosen_resolve__not_pex.name = "not_pex" # ditto.

Expand Down
26 changes: 19 additions & 7 deletions src/python/pants/backend/python/util_rules/pex_requirements.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import logging
from collections.abc import Iterable, Iterator
from dataclasses import dataclass, field
from enum import StrEnum, auto
from typing import TYPE_CHECKING
from urllib.parse import urlparse

Expand Down Expand Up @@ -50,6 +51,13 @@
logger = logging.getLogger(__name__)


class LockfileFormat(StrEnum):
Pex = auto()
# The very old, deprecated constraints-based "lockfile" that should
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any reason not to start the deprecation process?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No reason. The current deprecation is for 3.0.0, which is silly. We can and should get rid of this.

# be removed entirely.
ConstraintsDeprecated = auto()


@dataclass(frozen=True)
class Resolve:
# A named resolve for a "user lockfile".
Expand Down Expand Up @@ -101,10 +109,10 @@ class LoadedLockfile:
# An estimate of the number of requirements in this lockfile, to be used as a heuristic for
# available parallelism.
requirement_estimate: int
# True if the loaded lockfile is in PEX's native format.
is_pex_native: bool
# If !is_pex_native, the lockfile parsed as constraints strings, for use when the lockfile
# needs to be subsetted (see #15031, ##12222).
# The format of the loaded lockfile.
lockfile_format: LockfileFormat
# If lockfile_format is ConstraintsDeprecated, the lockfile parsed as constraints strings,
# for use when the lockfile needs to be subsetted (see #15031, ##12222).
as_constraints_strings: FrozenOrderedSet[str] | None
# The original file or file content (which may not have identical content to the output
# `lockfile_digest`).
Expand Down Expand Up @@ -225,7 +233,11 @@ async def load_lockfile(

lockfile_contents = await get_digest_contents(lockfile_digest)
lock_bytes = lockfile_contents[0].content
is_pex_native = is_probably_pex_json_lockfile(lock_bytes)
lockfile_format = (
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does Python have a match expression or some other way to check easily when new enums have been added? 2-item enums end up being the bane of my existence as I always forget to look for places where I did if/else while an enum was 2-items, then updated to 3 and bugs introduced.

Maybe there is a mypy or ruff lint?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is match/case, but it is a statement, not an expression, so you can't use it on the RHS of an assignment. But that's ok on line 272 below, since it already has that structure (I assume you didn't mean to use such an expression here on line 236, since this is where we decide which enum value is in play, so the match doesn't figure into it)

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But also, I don't intend to support lockfile front matter for any new lockfile formats, so it's moot for line 272.

LockfileFormat.Pex
if is_probably_pex_json_lockfile(lock_bytes)
else LockfileFormat.ConstraintsDeprecated
)
constraints_strings = None

metadata_url = PythonLockfileMetadata.metadata_location_for_lockfile(lockfile.url)
Expand Down Expand Up @@ -256,7 +268,7 @@ async def load_lockfile(
pass

if not metadata:
if is_pex_native:
if lockfile_format == LockfileFormat.Pex:
header_delimiter = "//"
stripped_lock_bytes = strip_comments_from_pex_json_lockfile(lock_bytes)
lockfile_digest = await create_digest(
Expand Down Expand Up @@ -286,7 +298,7 @@ async def load_lockfile(
lockfile_path,
metadata,
requirement_estimate,
is_pex_native,
lockfile_format,
constraints_strings,
original_lockfile=lockfile,
)
Expand Down
5 changes: 4 additions & 1 deletion src/python/pants/backend/python/util_rules/pex_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
EntireLockfile,
LoadedLockfile,
Lockfile,
LockfileFormat,
PexRequirements,
Resolve,
ResolvePexConfig,
Expand Down Expand Up @@ -703,7 +704,9 @@ def create_loaded_lockfile(is_pex_lock: bool) -> LoadedLockfile:
lockfile_path,
metadata=None,
requirement_estimate=2,
is_pex_native=is_pex_lock,
lockfile_format=LockfileFormat.Pex
if is_pex_lock
else LockfileFormat.ConstraintsDeprecated,
as_constraints_strings=None,
original_lockfile=lockfile_obj,
)
Expand Down
8 changes: 6 additions & 2 deletions src/python/pants/core/goals/update_build_files_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,11 @@
from pants.backend.python.util_rules import pex
from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
from pants.backend.python.util_rules.lockfile_metadata import PythonLockfileMetadata
from pants.backend.python.util_rules.pex_requirements import LoadedLockfile, Lockfile
from pants.backend.python.util_rules.pex_requirements import (
LoadedLockfile,
Lockfile,
LockfileFormat,
)
from pants.core.goals.update_build_files import (
FormatWithBlackRequest,
FormatWithBuildifierRequest,
Expand Down Expand Up @@ -185,7 +189,7 @@ def assert_ics(
"black.lock",
metadata=metadata,
requirement_estimate=1,
is_pex_native=True,
lockfile_format=LockfileFormat.Pex,
as_constraints_strings=None,
original_lockfile=Lockfile(
"black.lock", url_description_of_origin="foo", resolve_name="black"
Expand Down
Loading