Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion crates/pixi_cli/src/workspace/platform.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ pub struct VirtualPackageArgs {
pub cuda_arch: Option<String>,

/// Declare a `__archspec` virtual package with the given microarchitecture
/// string, e.g. `x86-64-v3`. Valid on any subdir.
/// string, e.g. `x86_64_v3`. Valid on any subdir.
#[clap(long, value_name = "ARCH")]
pub archspec: Option<String>,

Expand Down Expand Up @@ -128,6 +128,8 @@ impl VirtualPackageArgs {
if value.is_empty() {
miette::bail!("--archspec requires a non-empty microarchitecture string");
}
pixi_manifest::platform::validate_archspec_name(&value)
.map_err(|message| miette::miette!("{message}"))?;
push_unique(
&mut specs,
&mut seen_names,
Expand Down Expand Up @@ -258,6 +260,10 @@ fn parse_raw_virtual_package(spec: &str) -> miette::Result<GenericVirtualPackage
.transpose()?
.unwrap_or_else(zero_version);
let build_string = parts.next().unwrap_or("").to_string();
if name.as_normalized() == "__archspec" && !build_string.is_empty() {
pixi_manifest::platform::validate_archspec_name(&build_string)
.map_err(|message| miette::miette!("{message}"))?;
}
Ok(GenericVirtualPackage {
name,
version,
Expand Down
34 changes: 34 additions & 0 deletions crates/pixi_core/src/environment/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -867,6 +867,40 @@ mod tests {
assert!(parsed.minimum_supported_platform.is_none());
}

/// The minimum platform can carry several `__archspec` entries -- one per
/// distinct microarchitecture the resolved packages require -- and they
/// must survive the marker-file round-trip.
#[test]
fn environment_file_roundtrips_multiple_archspec_entries() {
let archspec = |microarchitecture: &str| GenericVirtualPackage {
name: PackageName::from_str("__archspec").unwrap(),
version: Version::major(0),
build_string: microarchitecture.to_string(),
};
let file = EnvironmentFile {
manifest_path: PathBuf::from("/ws/pixi.toml"),
environment_name: "default".to_string(),
pixi_version: "0.1.0".to_string(),
environment_lock_file_hash: LockedEnvironmentHash::invalid(),
resolved_platform: None,
minimum_supported_platform: Some(PlatformData::new(
Platform::Linux64,
vec![archspec("skylake"), archspec("x86_64_v3")],
)),
};
let json = serde_json::to_string(&file).expect("marker serializes");
let parsed: EnvironmentFile = serde_json::from_str(&json).expect("marker parses");
let minimum = parsed
.minimum_supported_platform
.expect("minimum platform survives");
let microarchitectures: Vec<&str> = minimum
.virtual_packages()
.iter()
.map(|vp| vp.build_string.as_str())
.collect();
assert_eq!(microarchitectures, vec!["skylake", "x86_64_v3"]);
}

/// A linux-64 lock environment with no packages, so the quick-validate
/// hash varies only with the platform passed to `from_environment`.
fn empty_linux_lock() -> rattler_lock::LockFile {
Expand Down
97 changes: 83 additions & 14 deletions crates/pixi_core/src/lock_file/virtual_packages.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@ use pypi_modifiers::pypi_tags::{PyPITagError, get_tags_from_machine, is_python_r
use rattler_conda_types::ParseMatchSpecError;
use rattler_conda_types::ParseStrictness::Lenient;
use rattler_conda_types::{
GenericVirtualPackage, MatchSpec, Matches, PackageName, Platform, Version, VersionSpec,
GenericVirtualPackage, MatchSpec, Matches, PackageName, Platform, StringMatcher, Version,
VersionSpec,
};
use rattler_lock::{CondaPackageData, ConversionError, LockFile, PypiPackageData};
use rattler_virtual_packages::{
Expand All @@ -19,7 +20,10 @@ use thiserror::Error;
use uv_distribution_filename::WheelFilename;

/// Define accepted virtual packages as a constant set
/// These packages will be checked against the system virtual packages
/// These packages will be checked against the system virtual packages.
/// `__archspec` is intentionally excluded: the solver already matches it
/// through the microarchitecture DAG, so re-validating it here would reject
/// hosts the solver accepted (a `skylake` host running `x86_64_v3` packages).
const ACCEPTED_VIRTUAL_PACKAGES: &[&str] = &[
"__glibc", "__musl", "__eglibc", "__cuda", "__osx", "__win", "__linux",
];
Expand All @@ -41,7 +45,9 @@ impl VirtualPackageNotFoundError {
let help = required_package
.name
.as_exact()
.and_then(|name| conda_override_hint(name.as_normalized(), required_version))
// `__archspec` is skipped before it reaches here, so no accepted
// virtual package needs a build string for its override hint.
.and_then(|name| conda_override_hint(name.as_normalized(), required_version, None))
.map(|hint| {
format!(
" You can mock the virtual package by overriding the environment variable, e.g.: '`{hint}`'"
Expand Down Expand Up @@ -182,10 +188,25 @@ pub fn minimal_required_virtual_packages(depends: &[&str]) -> Vec<GenericVirtual
};

let mut aggregated: HashMap<PackageName, GenericVirtualPackage> = HashMap::new();
// `__archspec` aggregates by distinct microarchitecture instead of by
// version: the machine must satisfy every requirement, and DAG-incomparable
// pairs have no single representative. Bare or pattern-matched
// requirements only assert presence (an empty build string).
let mut archspec_microarchitectures: Vec<String> = Vec::new();
for spec in specs {
let Some(name) = spec.name.as_exact() else {
continue;
};
if name.as_normalized() == "__archspec" {
let microarchitecture = match &spec.build {
Some(StringMatcher::Exact(microarchitecture)) => microarchitecture.clone(),
Some(_) | None => String::new(),
};
if !archspec_microarchitectures.contains(&microarchitecture) {
archspec_microarchitectures.push(microarchitecture);
}
continue;
}
let version = spec
.version
.as_ref()
Expand All @@ -207,7 +228,18 @@ pub fn minimal_required_virtual_packages(depends: &[&str]) -> Vec<GenericVirtual
}

let mut vps: Vec<GenericVirtualPackage> = aggregated.into_values().collect();
vps.sort_by(|a, b| a.name.as_normalized().cmp(b.name.as_normalized()));
vps.extend(
archspec_microarchitectures
.into_iter()
.map(|microarchitecture| GenericVirtualPackage {
name: PackageName::new_unchecked("__archspec"),
version: Version::major(0),
build_string: microarchitecture,
}),
);
vps.sort_by(|a, b| {
(a.name.as_normalized(), &a.build_string).cmp(&(b.name.as_normalized(), &b.build_string))
});
vps
}

Expand Down Expand Up @@ -711,26 +743,34 @@ packages:
}
}

/// `__archspec` is not validated here: the solver already matched it
/// through the microarchitecture DAG, so lock-file validation skips it
/// even when the host microarchitecture would not match by name. The
/// fixture's package requires `__archspec 1 x86_64`.
#[test]
fn test_archspec_skip() {
fn test_archspec_skipped() {
let root_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
let lock_file_path = root_dir.join("../../tests/data/lock_files/archspec.lock");
let lock_file = LockFile::from_path(&lock_file_path).unwrap();
let platform = pixi_manifest::PixiPlatform::from_subdir(Platform::Linux64);

let overrides = VirtualPackageOverrides {
let overrides = |archspec: &str| VirtualPackageOverrides {
libc: Some(Override::String("2.17".to_string())),
archspec: Some(Override::String(archspec.to_string())),
..VirtualPackageOverrides::default()
};

// validate that the archspec is skipped
validate_system_meets_environment_requirements(
&lock_file,
&platform,
&EnvironmentName::default(),
Some(overrides),
)
.unwrap();
// A matching host passes, but so does an unknown or unrelated one:
// `__archspec` is never checked here.
for archspec in ["x86_64", "0", "m1"] {
validate_system_meets_environment_requirements(
&lock_file,
&platform,
&EnvironmentName::default(),
Some(overrides(archspec)),
)
.unwrap_or_else(|error| panic!("archspec {archspec} should be skipped: {error:?}"));
}
}

#[test]
Expand All @@ -756,4 +796,33 @@ packages:
)
.unwrap();
}

/// `__archspec` requirements aggregate into the minimal set by distinct
/// microarchitecture -- the machine must satisfy each one -- while other
/// virtual packages keep the highest version seen.
#[test]
fn test_minimal_required_virtual_packages_archspec() {
let depends = [
"__cuda >=11",
"__cuda >=12",
"__archspec 1 x86_64_v3",
"__archspec 1 skylake",
"__archspec 1 x86_64_v3",
];
let minimal = minimal_required_virtual_packages(&depends);

let archspec: Vec<&str> = minimal
.iter()
.filter(|vp| vp.name.as_normalized() == "__archspec")
.map(|vp| vp.build_string.as_str())
.collect();
assert_eq!(archspec, vec!["skylake", "x86_64_v3"]);

let cuda = minimal
.iter()
.find(|vp| vp.name.as_normalized() == "__cuda")
.expect("__cuda is required");
assert_eq!(cuda.version, Version::from_str("12").unwrap());
assert_eq!(minimal.len(), 3);
}
}
35 changes: 31 additions & 4 deletions crates/pixi_core/src/workspace/errors.rs
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,13 @@ impl Diagnostic for UnsupportedPlatformError {
let overrides: Vec<String> = self
.unsatisfied_requirements
.iter()
.filter_map(|req| conda_override_hint(req.name.as_normalized(), Some(&req.version)))
.filter_map(|req| {
conda_override_hint(
req.name.as_normalized(),
Some(&req.version),
Some(&req.build_string),
)
})
.collect();

let base = if overrides.is_empty() {
Expand Down Expand Up @@ -105,6 +111,14 @@ impl Diagnostic for UnsupportedPlatformError {
fn format_requirements(reqs: &[GenericVirtualPackage]) -> String {
reqs.iter()
.map(|r| {
// __archspec's requirement is the microarchitecture in its build
// string; its version is a meaningless constant.
if r.name.as_normalized() == "__archspec"
&& !r.build_string.is_empty()
&& r.build_string != "0"
{
return format!("{} {}", r.name.as_normalized(), r.build_string);
}
// Version 0 encodes a version-less requirement (a bare `__cuda`
// dependency): the package must be present at any version.
if r.version == Version::major(0) {
Expand All @@ -117,9 +131,14 @@ fn format_requirements(reqs: &[GenericVirtualPackage]) -> String {
}

/// `CONDA_OVERRIDE_*` hint for a missing virtual package: the required
/// version when known, a realistic example otherwise. `None` for virtual
/// packages without a known override (e.g. `__unix`).
pub(crate) fn conda_override_hint(name: &str, version: Option<&Version>) -> Option<String> {
/// version (or, for `__archspec`, the required microarchitecture) when
/// known, a realistic example otherwise. `None` for virtual packages
/// without a known override (e.g. `__unix`).
pub(crate) fn conda_override_hint(
name: &str,
version: Option<&Version>,
build_string: Option<&str>,
) -> Option<String> {
let env_var = match name {
"__glibc" => "CONDA_OVERRIDE_GLIBC",
"__cuda" => "CONDA_OVERRIDE_CUDA",
Expand All @@ -129,6 +148,14 @@ pub(crate) fn conda_override_hint(name: &str, version: Option<&Version>) -> Opti
"__archspec" => "CONDA_OVERRIDE_ARCHSPEC",
_ => return None,
};
// __archspec's payload is the microarchitecture in its build string,
// not its version; suggest overriding to exactly the required one.
if name == "__archspec" {
let example = build_string
.filter(|build| !build.is_empty() && *build != "0")
.unwrap_or("x86_64_v3");
return Some(format!("{env_var}={example}"));
}
// A version-0 requirement means "any version"; "=0" reads like nonsense,
// so suggest a realistic value instead.
let example = match version.filter(|v| **v != Version::major(0)) {
Expand Down
34 changes: 33 additions & 1 deletion crates/pixi_core/src/workspace/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ use crate::lock_file::LockedPackageKind;
use rattler_networking::{LazyClient, s3_middleware};
use rattler_repodata_gateway::Gateway;
use rattler_virtual_packages::{
Cuda, EnvOverride, LibC, Linux, Osx, Override, VirtualPackageOverrides, VirtualPackages,
Archspec, Cuda, EnvOverride, LibC, Linux, Osx, Override, VirtualPackageOverrides,
VirtualPackages,
};
pub use registry::{WorkspaceRegistry, WorkspaceRegistryError};
pub use solve_group::SolveGroup;
Expand Down Expand Up @@ -302,6 +303,37 @@ fn apply_environment_variable_overrides(packages: &mut Vec<GenericVirtualPackage
);

apply_glibc_override(packages);
apply_archspec_override(packages);
}

/// Apply `CONDA_OVERRIDE_ARCHSPEC` to `packages`: unset leaves the detected
/// `__archspec` untouched, an empty value removes it, and a microarchitecture
/// name (or `0` for "unknown") replaces or inserts it. Unknown names are
/// rejected by rattler's parser; the override is then ignored with a warning.
fn apply_archspec_override(packages: &mut Vec<GenericVirtualPackage>) {
let Ok(value) = std::env::var(Archspec::DEFAULT_ENV_NAME) else {
return;
};
match Archspec::parse_version_opt(&value) {
Ok(None) => packages.retain(|p| p.name.as_normalized() != "__archspec"),
Ok(Some(archspec)) => {
let overridden = GenericVirtualPackage::from(archspec);
if let Some(existing) = packages
.iter_mut()
.find(|p| p.name.as_normalized() == "__archspec")
{
*existing = overridden;
} else {
packages.push(overridden);
}
}
Err(error) => {
tracing::warn!(
"Ignoring invalid {}='{value}': {error}",
Archspec::DEFAULT_ENV_NAME,
);
}
}
}

/// Apply `CONDA_OVERRIDE_GLIBC` (rattler's only libc slot) to `packages`. The
Expand Down
25 changes: 7 additions & 18 deletions crates/pixi_core/src/workspace/virtual_packages.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ use pixi_manifest::{
};
use rattler_conda_types::{GenericVirtualPackage, Platform};
use rattler_lock::LockFile;
use rattler_virtual_packages::{Archspec, Cuda, CudaArch, LibC, Linux, Osx, VirtualPackage};
use rattler_virtual_packages::{Cuda, CudaArch, LibC, Linux, Osx, VirtualPackage};
use std::collections::HashSet;
use std::path::PathBuf;
use std::sync::{LazyLock, Mutex};
Expand Down Expand Up @@ -67,17 +67,9 @@ fn generic_to_virtual_package(gvp: &GenericVirtualPackage) -> Option<VirtualPack
"__cuda_arch" => Some(VirtualPackage::CudaArch(CudaArch {
version: gvp.version.clone(),
})),
"__archspec" => {
// Rattler maps an archspec string through a microarch database
// lookup; an empty/"0" build-string means "unknown microarch"
// and `from_name` returns the generic catch-all in that case.
if gvp.build_string.is_empty() || gvp.build_string == "0" {
return Some(VirtualPackage::Archspec(Archspec::Unknown));
}
Some(VirtualPackage::Archspec(Archspec::from_name(
gvp.build_string.as_str(),
)))
}
"__archspec" => Some(VirtualPackage::Archspec(
pixi_manifest::platform::archspec_from_build_string(&gvp.build_string),
)),
_ => None,
}
}
Expand Down Expand Up @@ -238,19 +230,16 @@ pub fn minimum_compatible_declared_platform<'p>(
}

/// The declared virtual packages of `platform` that the machine does not
/// provide (missing entirely, or present at a lower version).
/// provide (missing entirely, at a lower version, or with a different
/// build string), per [`pixi_manifest::platform::satisfied_by_system`].
fn unsatisfied_virtual_packages(
platform: &PixiPlatform,
system: &[GenericVirtualPackage],
) -> Vec<GenericVirtualPackage> {
platform
.declared_virtual_packages()
.iter()
.filter(|required| {
!system
.iter()
.any(|sys| sys.name == required.name && sys.version >= required.version)
})
.filter(|required| !pixi_manifest::platform::satisfied_by_system(required, system))
.cloned()
.collect()
}
Expand Down
Loading