diff --git a/Cargo.lock b/Cargo.lock index ae950ace3c..0e6d1e6785 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6901,6 +6901,7 @@ dependencies = [ "toml_edit 0.23.10+spec-1.0.0", "tracing", "url", + "xxhash-rust", "zstd", ] diff --git a/crates/pixi_cli/src/global/global_specs.rs b/crates/pixi_cli/src/global/global_specs.rs index 4c9f978943..17b39dc3b7 100644 --- a/crates/pixi_cli/src/global/global_specs.rs +++ b/crates/pixi_cli/src/global/global_specs.rs @@ -10,7 +10,8 @@ use pixi_consts::consts; use pixi_global::project::FromMatchSpecError; use pixi_spec::{PixiSpec, Subdirectory, SubdirectoryError}; use rattler_conda_types::{ - ChannelConfig, MatchSpec, ParseMatchSpecError, ParseMatchSpecOptions, RepodataRevision, + ChannelConfig, MatchSpec, PackageName, ParseMatchSpecError, ParseMatchSpecOptions, + RepodataRevision, }; use typed_path::Utf8NativePathBuf; @@ -37,6 +38,20 @@ pub struct GlobalSpecs { /// The path to the local package #[clap(long, conflicts_with = "git")] pub path: Option, + + /// The build backend to build the source with, when the source does not + /// provide its own package manifest (or to override the one it has). + /// Accepts a name with an optional version constraint, e.g. + /// `pixi-build-rust` or `"pixi-build-rust>=0.3,<0.4"`. + #[clap(long, value_name = "BUILD_BACKEND")] + pub build_backend: Option, + + /// Additional fields of the inline package definition, as + /// `DOTTED_KEY=TOML_VALUE` pairs that are recorded under the `package` + /// key of the dependency, e.g. `host-dependencies.hatchling="*"` or + /// `build.config.extra-args=["--all-features"]`. + #[clap(long = "package", value_name = "KEY=VALUE")] + pub package: Vec, } impl HasSpecs for GlobalSpecs { @@ -56,6 +71,25 @@ pub enum GlobalSpecsConversionError { help = "Use a full package specification like `python==3.12` instead of just `==3.12`" )] NameRequired, + #[error("`--build-backend` and `--package` require a source location")] + #[diagnostic(help = "Add `--git ` or `--path ` to specify where the source lives")] + InlineRequiresSource, + #[error("invalid `--build-backend` value '{input}': {reason}")] + #[diagnostic( + help = "Pass a package name with an optional version constraint, e.g. `pixi-build-rust` or `\"pixi-build-rust>=0.3,<0.4\"`" + )] + InvalidBuildBackend { input: String, reason: String }, + #[error("invalid `--package` value '{input}': {reason}")] + #[diagnostic( + help = "Pass a `DOTTED_KEY=TOML_VALUE` pair, e.g. `--package 'host-dependencies.hatchling=\"*\"'`" + )] + InvalidPackageFragment { input: String, reason: String }, + #[error("the key '{key}' of the inline package definition is set more than once")] + #[diagnostic(help = "Each `--package` key (and `--build-backend`) may only be set once")] + PackageKeyCollision { key: String }, + #[error(transparent)] + #[diagnostic(transparent)] + InlinePackageValue(#[from] pixi_global::project::InlinePackageValueError), #[error("couldn't construct a relative path from {} to {}", .0, .1)] RelativePath(String, String), #[error("could not absolutize path: {0}")] @@ -86,7 +120,112 @@ pub enum GlobalSpecsConversionError { InvalidSubdirectory(#[from] SubdirectoryError), } +/// Recursively merges `source` into `target`, failing when a key would be +/// set twice. `path` is the dotted prefix for error reporting. +fn merge_inline_tables( + target: &mut toml_edit::InlineTable, + source: toml_edit::InlineTable, + path: &str, +) -> Result<(), GlobalSpecsConversionError> { + for (key, value) in source.into_iter() { + let key_path = if path.is_empty() { + key.to_string() + } else { + format!("{path}.{key}") + }; + match target.get_mut(&key) { + None => { + target.insert(&key, value); + } + Some(toml_edit::Value::InlineTable(existing)) => { + if let toml_edit::Value::InlineTable(source) = value { + merge_inline_tables(existing, source, &key_path)?; + } else { + return Err(GlobalSpecsConversionError::PackageKeyCollision { key: key_path }); + } + } + Some(_) => { + return Err(GlobalSpecsConversionError::PackageKeyCollision { key: key_path }); + } + } + } + Ok(()) +} + +/// Parses a single `--package DOTTED_KEY=TOML_VALUE` fragment into an inline +/// table. The right-hand side must be a valid TOML value, so strings need +/// their quotes: `--package 'build.config.profile="release"'`. +fn parse_package_fragment( + input: &str, +) -> Result { + let document = input.parse::().map_err(|error| { + GlobalSpecsConversionError::InvalidPackageFragment { + input: input.to_string(), + reason: error.message().to_string(), + } + })?; + Ok(document.as_table().clone().into_inline_table()) +} + impl GlobalSpecs { + /// Builds the inline package definition from `--build-backend` and + /// `--package`, or `None` when neither is given. + fn inline_package_value( + &self, + ) -> Result, GlobalSpecsConversionError> { + if self.build_backend.is_none() && self.package.is_empty() { + return Ok(None); + } + + let mut package = toml_edit::InlineTable::new(); + + // `--build-backend NAME` is sugar for + // `--package 'build.backend.name="NAME"'` (plus the version + // constraint when one is given). + if let Some(input) = &self.build_backend { + let match_spec = + MatchSpec::from_str(input, ParseMatchSpecOptions::lenient()).map_err(|e| { + GlobalSpecsConversionError::InvalidBuildBackend { + input: input.clone(), + reason: e.to_string(), + } + })?; + let name = match_spec.name.as_exact().cloned().ok_or_else(|| { + GlobalSpecsConversionError::InvalidBuildBackend { + input: input.clone(), + reason: "a package name is required".to_string(), + } + })?; + let name_and_version_only = MatchSpec { + name: match_spec.name.clone(), + version: match_spec.version.clone(), + ..MatchSpec::default() + }; + if match_spec != name_and_version_only { + return Err(GlobalSpecsConversionError::InvalidBuildBackend { + input: input.clone(), + reason: "only a name and a version constraint are supported".to_string(), + }); + } + + let mut backend = toml_edit::InlineTable::new(); + backend.insert("name", name.as_source().into()); + if let Some(version) = &match_spec.version { + backend.insert("version", version.to_string().into()); + } + let mut build = toml_edit::InlineTable::new(); + build.insert("backend", toml_edit::Value::InlineTable(backend)); + package.insert("build", toml_edit::Value::InlineTable(build)); + } + + for fragment in &self.package { + let table = parse_package_fragment(fragment)?; + merge_inline_tables(&mut package, table, "")?; + } + + Ok(Some(pixi_global::project::InlinePackageValue::new(package))) + } + /// Convert GlobalSpecs to a vector of GlobalSpec instances pub async fn to_global_specs( &self, @@ -173,14 +312,34 @@ impl GlobalSpecs { } None }; + // The inline package definition assembled from `--build-backend` and + // `--package`, if any. It only makes sense next to a source location. + let inline = self.inline_package_value()?; + if let Some(pixi_spec) = git_or_path_spec { if self.specs.is_empty() { - // Infer the package name from the path/git spec - let inferred_name = project.infer_package_name_from_spec(&pixi_spec).await?; - return Ok(vec![pixi_global::project::GlobalSpec::new( - inferred_name, - pixi_spec, - )]); + // Infer the package name from the path/git spec. With an + // inline definition present, backend discovery uses it + // instead of reading a manifest from the checkout; the + // placeholder name only serves the inference call, the + // definition is re-anchored to the inferred name below. + let inline_manifest = inline + .as_ref() + .map(|value| { + value.to_inline_manifest( + &PackageName::new_unchecked("uninferred-package"), + manifest_root, + ) + }) + .transpose()?; + let inferred_name = project + .infer_package_name_from_spec(&pixi_spec, inline_manifest.as_ref()) + .await?; + let mut spec = pixi_global::project::GlobalSpec::new(inferred_name, pixi_spec); + if let Some(inline) = inline { + spec = spec.with_inline(inline); + } + return Ok(vec![spec]); } self.specs @@ -195,12 +354,20 @@ impl GlobalSpecs { .as_exact() .cloned() .ok_or(GlobalSpecsConversionError::NameRequired)?; - Ok(pixi_global::project::GlobalSpec::new( - name, - pixi_spec.clone(), - )) + // Validate the inline definition early, before anything + // is recorded in the manifest. + if let Some(inline) = &inline { + inline.to_inline_manifest(&name, manifest_root)?; + } + let mut spec = pixi_global::project::GlobalSpec::new(name, pixi_spec.clone()); + if let Some(inline) = &inline { + spec = spec.with_inline(inline.clone()); + } + Ok(spec) }) .collect() + } else if inline.is_some() { + Err(GlobalSpecsConversionError::InlineRequiresSource) } else { self.specs .iter() @@ -229,10 +396,7 @@ mod tests { async fn test_to_global_specs_named() { let specs = GlobalSpecs { specs: vec!["numpy==1.21.0".to_string(), "scipy>=1.7".to_string()], - git: None, - rev: None, - subdir: None, - path: None, + ..Default::default() }; let channel_config = ChannelConfig::default_with_root_dir(PathBuf::from(".")); @@ -255,9 +419,7 @@ mod tests { let specs = GlobalSpecs { specs: vec!["mypackage".to_string()], git: Some("https://github.com/user/repo.git".parse().unwrap()), - rev: None, - subdir: None, - path: None, + ..Default::default() }; let channel_config = ChannelConfig::default_with_root_dir(PathBuf::from(".")); @@ -343,10 +505,7 @@ mod tests { async fn test_to_global_specs_nameless() { let specs = GlobalSpecs { specs: vec![">=1.0".to_string()], - git: None, - rev: None, - subdir: None, - path: None, + ..Default::default() }; let channel_config = ChannelConfig::default_with_root_dir(PathBuf::from(".")); @@ -360,4 +519,262 @@ mod tests { .await; assert!(global_specs.is_err()); } + + /// `--build-backend NAME` is sugar for a `build.backend.name` entry in the + /// inline package definition. + #[test] + fn test_build_backend_sugar() { + let specs = GlobalSpecs { + build_backend: Some("pixi-build-rust".to_string()), + ..Default::default() + }; + let inline = specs.inline_package_value().unwrap().unwrap(); + assert_eq!( + inline.to_toml_value().to_string(), + r#"{ build = { backend = { name = "pixi-build-rust" } } }"# + ); + } + + /// A version constraint on `--build-backend` lands as `build.backend.version`. + #[test] + fn test_build_backend_with_version() { + let specs = GlobalSpecs { + build_backend: Some("pixi-build-rust>=0.3,<0.4".to_string()), + ..Default::default() + }; + let inline = specs.inline_package_value().unwrap().unwrap(); + assert_eq!( + inline.to_toml_value().to_string(), + r#"{ build = { backend = { name = "pixi-build-rust", version = ">=0.3,<0.4" } } }"# + ); + } + + /// `--package` fragments merge into the definition. + #[test] + fn test_package_fragments_merge() { + let specs = GlobalSpecs { + build_backend: Some("pixi-build-python".to_string()), + package: vec![ + "host-dependencies.hatchling=\"*\"".to_string(), + "build.config.profile=\"release\"".to_string(), + ], + ..Default::default() + }; + let inline = specs.inline_package_value().unwrap().unwrap(); + let rendered = inline.to_toml_value().to_string(); + assert!( + rendered.contains(r#"name = "pixi-build-python""#), + "{rendered}" + ); + assert!(rendered.contains(r#"hatchling = "*""#), "{rendered}"); + assert!(rendered.contains(r#"profile = "release""#), "{rendered}"); + } + + /// Setting the same key via `--build-backend` and `--package` is an error. + #[test] + fn test_package_collision_with_build_backend() { + let specs = GlobalSpecs { + build_backend: Some("pixi-build-rust".to_string()), + package: vec!["build.backend.name=\"other\"".to_string()], + ..Default::default() + }; + let err = specs.inline_package_value().unwrap_err(); + assert!( + matches!(err, GlobalSpecsConversionError::PackageKeyCollision { .. }), + "expected a collision error, got {err:?}" + ); + } + + /// Neither flag means no inline definition. + #[test] + fn test_no_inline_definition() { + let specs = GlobalSpecs::default(); + assert!(specs.inline_package_value().unwrap().is_none()); + } + + /// One inline definition given next to several named packages is attached + /// to each of them. + #[tokio::test] + async fn test_inline_applies_to_all_named_packages() { + let specs = GlobalSpecs { + specs: vec!["foo".to_string(), "bar".to_string()], + git: Some("https://github.com/user/repo.git".parse().unwrap()), + build_backend: Some("pixi-build-rust".to_string()), + ..Default::default() + }; + let channel_config = ChannelConfig::default_with_root_dir(PathBuf::from(".")); + let manifest_root = PathBuf::from("."); + let project = pixi_global::Project::discover_or_create().await.unwrap(); + let global_specs = specs + .to_global_specs(&channel_config, &manifest_root, &project) + .await + .unwrap(); + assert_eq!(global_specs.len(), 2); + assert!( + global_specs.iter().all(|spec| spec.inline.is_some()), + "every named package should carry the inline definition" + ); + } + + /// `--build-backend`/`--package` without a source location is an error. + #[tokio::test] + async fn test_inline_requires_source() { + let specs = GlobalSpecs { + specs: vec!["xsv".to_string()], + build_backend: Some("pixi-build-rust".to_string()), + ..Default::default() + }; + let channel_config = ChannelConfig::default_with_root_dir(PathBuf::from(".")); + let manifest_root = PathBuf::from("."); + let project = pixi_global::Project::discover_or_create().await.unwrap(); + let err = specs + .to_global_specs(&channel_config, &manifest_root, &project) + .await + .unwrap_err(); + assert!( + matches!(err, GlobalSpecsConversionError::InlineRequiresSource), + "expected an inline-requires-source error, got {err:?}" + ); + } + + /// `--build-backend` accepts only a name and a version constraint; any other + /// matchspec component (channel, build string, subdir, ...) is rejected. + #[test] + fn test_build_backend_rejects_non_version_matchspec() { + for input in [ + "conda-forge::pixi-build-rust", + "pixi-build-rust[build=foo]", + "pixi-build-rust[subdir=noarch]", + ] { + let specs = GlobalSpecs { + build_backend: Some(input.to_string()), + ..Default::default() + }; + let err = specs.inline_package_value().unwrap_err(); + assert!( + matches!(err, GlobalSpecsConversionError::InvalidBuildBackend { .. }), + "expected an invalid-build-backend error for '{input}', got {err:?}" + ); + } + } + + /// A `--package` fragment that isn't a `DOTTED_KEY=TOML_VALUE` pair is + /// rejected. In particular, the right-hand side must be a valid TOML + /// value: unquoted strings and malformed arrays error instead of being + /// silently recorded as strings. + #[test] + fn test_invalid_package_fragment() { + for fragment in [ + "no-equals-sign", + "=novalue", + "build.config.profile=release", + "build.config.extra-args=[1,2", + ] { + let specs = GlobalSpecs { + package: vec![fragment.to_string()], + ..Default::default() + }; + let err = specs.inline_package_value().unwrap_err(); + assert!( + matches!( + err, + GlobalSpecsConversionError::InvalidPackageFragment { .. } + ), + "expected an invalid-package-fragment error for '{fragment}', got {err:?}" + ); + } + } + + /// Two `--package` fragments setting the same leaf key collide. + #[test] + fn test_two_package_fragments_collide() { + let specs = GlobalSpecs { + package: vec![ + "host-dependencies.hatchling=\"1\"".to_string(), + "host-dependencies.hatchling=\"2\"".to_string(), + ], + ..Default::default() + }; + let err = specs.inline_package_value().unwrap_err(); + assert!( + matches!(err, GlobalSpecsConversionError::PackageKeyCollision { .. }), + "expected a collision error, got {err:?}" + ); + } + + /// `--build-backend NAME` records the same definition as + /// `--package 'build.backend.name="NAME"'`. + #[test] + fn test_build_backend_equivalent_to_package() { + let sugar = GlobalSpecs { + build_backend: Some("pixi-build-rust".to_string()), + ..Default::default() + }; + let explicit = GlobalSpecs { + package: vec!["build.backend.name=\"pixi-build-rust\"".to_string()], + ..Default::default() + }; + assert_eq!( + sugar + .inline_package_value() + .unwrap() + .unwrap() + .to_toml_value() + .to_string(), + explicit + .inline_package_value() + .unwrap() + .unwrap() + .to_toml_value() + .to_string(), + ); + } + + /// The inline definition may not set `name` (it comes from the dependency + /// key) nor `build.source` (it comes from the dependency spec). Both checks + /// live past the `build.backend` requirement, so a backend must be present + /// for the definition to parse far enough to reach them. + #[test] + fn test_inline_definition_rejects_reserved_keys() { + let name = PackageName::new_unchecked("xsv"); + let root = std::path::Path::new("."); + + let explicit_name = GlobalSpecs { + build_backend: Some("pixi-build-rust".to_string()), + package: vec!["name=\"other\"".to_string()], + ..Default::default() + }; + let err = explicit_name + .inline_package_value() + .unwrap() + .unwrap() + .to_inline_manifest(&name, root) + .unwrap_err(); + assert!( + matches!( + err, + pixi_global::project::InlinePackageValueError::ExplicitName + ), + "expected an explicit-name error, got {err:?}" + ); + + let explicit_source = GlobalSpecs { + build_backend: Some("pixi-build-rust".to_string()), + package: vec!["build.source.path=\"elsewhere\"".to_string()], + ..Default::default() + }; + let err = explicit_source + .inline_package_value() + .unwrap() + .unwrap() + .to_inline_manifest(&name, root) + .unwrap_err(); + assert!( + matches!( + err, + pixi_global::project::InlinePackageValueError::ExplicitBuildSource + ), + "expected an explicit-build-source error, got {err:?}" + ); + } } diff --git a/crates/pixi_core/src/environment/mod.rs b/crates/pixi_core/src/environment/mod.rs index 1e1ca8f815..1177107afb 100644 --- a/crates/pixi_core/src/environment/mod.rs +++ b/crates/pixi_core/src/environment/mod.rs @@ -20,7 +20,7 @@ use rattler_lock::{LockFile, LockedPackage}; use serde::{Deserialize, Serialize}; use std::fmt::{Display, Formatter}; use std::{ - collections::HashMap, + collections::{BTreeMap, HashMap}, hash::{Hash, Hasher}, io::ErrorKind, path::{Path, PathBuf}, @@ -386,6 +386,17 @@ pub struct EnvironmentFile { /// be weaker than [`Self::resolved_platform`]. `None` as above. #[serde(default)] pub minimum_supported_platform: Option, + /// Fingerprints of the manifest's source dependencies at install time, + /// keyed by package name: a hash of the source spec plus any inline + /// package definition. Only written by `pixi global`, which has no lock + /// file; `pixi global sync` compares them against the manifest to decide + /// whether a source dependency's *specification* changed (source content + /// changes are `pixi global update`'s job). Empty for workspace + /// environments and environments written by an older pixi — for + /// environments with source dependencies that means out-of-sync, which + /// degrades gracefully to a one-time rebuild. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub source_fingerprints: BTreeMap, } /// The path to the environment file in the `conda-meta` directory of the @@ -434,9 +445,7 @@ pub fn write_environment_file( /// Reading the environment file of the environment. /// Removing it if it's not valid. -pub(crate) fn read_environment_file( - environment_dir: &Path, -) -> miette::Result> { +pub fn read_environment_file(environment_dir: &Path) -> miette::Result> { let path = environment_file_path(environment_dir); let contents = match fs_err::read_to_string(&path) { diff --git a/crates/pixi_core/src/lock_file/update.rs b/crates/pixi_core/src/lock_file/update.rs index 57c3e293db..dce461d2bd 100644 --- a/crates/pixi_core/src/lock_file/update.rs +++ b/crates/pixi_core/src/lock_file/update.rs @@ -810,6 +810,7 @@ impl<'p> LockFileDerivedData<'p> { environment_lock_file_hash: hash, resolved_platform, minimum_supported_platform, + source_fingerprints: Default::default(), }, )?; diff --git a/crates/pixi_global/Cargo.toml b/crates/pixi_global/Cargo.toml index 74df9ed142..3e5be3a5f1 100644 --- a/crates/pixi_global/Cargo.toml +++ b/crates/pixi_global/Cargo.toml @@ -55,6 +55,7 @@ toml-span = { workspace = true } toml_edit = { workspace = true } tracing = { workspace = true } url = { workspace = true } +xxhash-rust = { workspace = true, features = ["xxh3"] } zstd = { workspace = true } [target.'cfg(unix)'.dependencies] diff --git a/crates/pixi_global/src/project/global_spec.rs b/crates/pixi_global/src/project/global_spec.rs index 079dee29b5..0a36828a3d 100644 --- a/crates/pixi_global/src/project/global_spec.rs +++ b/crates/pixi_global/src/project/global_spec.rs @@ -1,4 +1,10 @@ //! This module makes it a bit easier to pass around a package name and the pixi specification +use std::path::Path; + +use pixi_manifest::{ + InlinePackageManifest, KnownPreviewFeature, Preview, + toml::{TomlPackage, WorkspacePackageProperties}, +}; use pixi_spec::PixiSpec; use rattler_conda_types::{ MatchSpec, NamelessMatchSpec, PackageName, ParseMatchSpecOptions, RepodataRevision, @@ -10,6 +16,10 @@ use rattler_conda_types::{ pub struct GlobalSpec { pub name: PackageName, pub spec: PixiSpec, + /// An inline package definition (`package = { ... }`) accompanying a + /// source spec: it describes how the source is built when the source does + /// not provide its own package manifest (or to override the one it has). + pub inline: Option, } #[derive(Debug, thiserror::Error, miette::Diagnostic)] @@ -23,7 +33,17 @@ pub enum FromMatchSpecError { impl GlobalSpec { /// Creates a new `GlobalSpec` with a package name and a Pixi specification. pub fn new(name: PackageName, spec: PixiSpec) -> Self { - Self { name, spec } + Self { + name, + spec, + inline: None, + } + } + + /// Attaches an inline package definition to this spec. + pub fn with_inline(mut self, inline: InlinePackageValue) -> Self { + self.inline = Some(inline); + self } /// Returns the package name. @@ -63,3 +83,88 @@ impl GlobalSpec { Ok(GlobalSpec::new(name.clone(), pixi_spec)) } } + +#[derive(Debug, thiserror::Error, miette::Diagnostic)] +pub enum InlinePackageValueError { + #[error("failed to parse the inline package definition: {0}")] + Parse(String), + #[error("an inline package definition cannot set `name`; it is taken from the dependency key")] + ExplicitName, + #[error( + "an inline package definition cannot set `build.source`; the source is taken from the dependency spec" + )] + ExplicitBuildSource, + #[error("failed to convert the inline package definition: {0}")] + Convert(String), +} + +/// An inline package definition as the TOML value of the `package` key of a +/// dependency entry (e.g. `package.build.backend.name = "pixi-build-rust"`). +/// +/// The raw TOML representation is kept so the definition can be written +/// verbatim into the global manifest document; [`Self::to_inline_manifest`] +/// converts it into the resolved [`InlinePackageManifest`] when it is needed +/// before the manifest is saved and re-parsed (e.g. for package name +/// inference). +#[derive(Debug, Clone)] +pub struct InlinePackageValue(toml_edit::InlineTable); + +impl InlinePackageValue { + /// Wraps a `package` table. + pub fn new(table: toml_edit::InlineTable) -> Self { + Self(table) + } + + /// Returns the definition as a TOML value, for insertion into a dependency + /// entry under the `package` key. + pub fn to_toml_value(&self) -> toml_edit::Value { + toml_edit::Value::InlineTable(self.0.clone()) + } + + /// Parses and converts the definition into an [`InlinePackageManifest`] + /// for the dependency `name`, mirroring the checks of the manifest parser: + /// the definition may not set `name` (it comes from the dependency key) + /// nor `build.source` (it comes from the dependency spec). + pub fn to_inline_manifest( + &self, + name: &PackageName, + root_directory: &Path, + ) -> Result { + let source = format!("package = {}", self.0); + let mut value = + toml_span::parse(&source).map_err(|e| InlinePackageValueError::Parse(e.to_string()))?; + let mut th = toml_span::de_helpers::TableHelper::new(&mut value) + .map_err(|e| InlinePackageValueError::Parse(e.to_string()))?; + let (_, mut package_value) = th + .take("package") + .expect("the `package` key was just serialized"); + let package = ::deserialize(&mut package_value) + .map_err(|e| { + InlinePackageValueError::Parse( + e.errors + .iter() + .map(ToString::to_string) + .collect::>() + .join("; "), + ) + })?; + + if package.name.is_some() { + return Err(InlinePackageValueError::ExplicitName); + } + if package.build.source.is_some() { + return Err(InlinePackageValueError::ExplicitBuildSource); + } + + let preview = Preview::from_iter([KnownPreviewFeature::PixiBuild]); + InlinePackageManifest::from_toml_package( + name, + package, + WorkspacePackageProperties::default(), + &preview, + root_directory, + ) + .map(|with_warnings| with_warnings.value) + .map_err(|e| InlinePackageValueError::Convert(e.to_string())) + } +} diff --git a/crates/pixi_global/src/project/manifest.rs b/crates/pixi_global/src/project/manifest.rs index 46a5dd3e90..0c571fab1d 100644 --- a/crates/pixi_global/src/project/manifest.rs +++ b/crates/pixi_global/src/project/manifest.rs @@ -51,7 +51,8 @@ impl Manifest { /// Creates a new manifest from a string pub fn from_str(manifest_path: &Path, contents: impl Into) -> miette::Result { let contents = contents.into(); - let parsed = ParsedManifest::from_toml_str(&contents); + let root_directory = manifest_path.parent().unwrap_or(Path::new("")); + let parsed = ParsedManifest::from_toml_str(&contents, root_directory); let (manifest, document) = match parsed.and_then(|manifest| { contents @@ -143,28 +144,76 @@ impl Manifest { let name = named_spec.name(); let spec = named_spec.spec(); - // Update self.parsed - self.parsed - .envs - .get_mut(env_name) - .ok_or_else(|| { - miette::miette!("Environment {} doesn't exist.", env_name.fancy_display()) - })? - .dependencies - .specs - .insert(name.clone(), spec.clone()); + // The value to write into the document: the source spec, extended + // with the inline package definition under the `package` key when one + // is supplied. + let toml_value = match &named_spec.inline { + None => spec.to_toml_value(), + Some(inline) => { + let toml_edit::Value::InlineTable(mut table) = spec.to_toml_value() else { + miette::bail!( + "an inline package definition requires a source spec, but the spec of '{}' is `{}`", + name.as_normalized(), + spec.to_toml_value() + ); + }; + table.insert("package", inline.to_toml_value()); + toml_edit::Value::InlineTable(table) + } + }; - // Update self.document - self.document.insert_into_inline_table( - &["envs", env_name.as_str(), "dependencies"], - name.as_normalized(), - spec.to_toml_value(), - )?; + if !self.parsed.envs.contains_key(env_name) { + miette::bail!("Environment {} doesn't exist.", env_name.fancy_display()); + } + + if named_spec.inline.is_some() { + // Inline package definitions are converted with root-directory + // context by the manifest parser, so `self.parsed` has to come + // from a re-parse of the document. Apply the edit to a scratch + // copy first and only commit document and parsed manifest + // together once the re-parse succeeds, keeping both consistent + // when it doesn't. + let mut document = self.document.clone(); + document.insert_into_inline_table( + &["envs", env_name.as_str(), "dependencies"], + name.as_normalized(), + toml_value.clone(), + )?; + let contents = document.to_string(); + let root_directory = self.path.parent().unwrap_or(Path::new("")); + match ParsedManifest::from_toml_str(&contents, root_directory) { + Ok(parsed) => { + self.document = document; + self.parsed = parsed; + } + Err(e) => { + return e.to_fancy(consts::GLOBAL_MANIFEST_DEFAULT_NAME, &contents, &self.path); + } + } + } else { + // Update self.parsed + let env = self + .parsed + .envs + .get_mut(env_name) + .expect("environment existence was checked above"); + env.dependencies.specs.insert(name.clone(), spec.clone()); + // A previously recorded inline definition no longer applies when + // the dependency is overwritten without one. + env.inline_packages.swap_remove(name); + + // Update self.document + self.document.insert_into_inline_table( + &["envs", env_name.as_str(), "dependencies"], + name.as_normalized(), + toml_value.clone(), + )?; + } tracing::debug!( "Added dependency {}={} to toml document for environment {}", name.as_normalized(), - spec.to_toml_value().to_string(), + toml_value.to_string(), env_name.fancy_display() ); Ok(()) @@ -177,12 +226,10 @@ impl Manifest { name: &PackageName, ) -> miette::Result { // Update self.parsed - self.parsed - .envs - .get_mut(env_name) - .ok_or_else(|| { - miette::miette!("Environment {} doesn't exist.", env_name.fancy_display()) - })? + let parsed_env = self.parsed.envs.get_mut(env_name).ok_or_else(|| { + miette::miette!("Environment {} doesn't exist.", env_name.fancy_display()) + })?; + parsed_env .dependencies .specs .swap_remove(name) @@ -191,6 +238,7 @@ impl Manifest { console::style(name.as_normalized()).green(), env_name.fancy_display() ))?; + parsed_env.inline_packages.swap_remove(name); // Update self.document self.document @@ -1060,6 +1108,110 @@ mod tests { ); } + /// Re-adding a source dependency replaces its inline `package` definition + /// wholesale: attaching a definition records it, and re-adding without one + /// removes it rather than leaving a stale table behind. + #[test] + fn test_add_dependency_overwrites_inline_definition() { + let contents = r#" + [envs.xsv] + channels = ["conda-forge"] + [envs.xsv.dependencies] + xsv = { path = "some-source" } + "#; + let mut manifest = + Manifest::from_str(std::path::Path::new("pixi-global.toml"), contents).unwrap(); + + let env_name = EnvironmentName::from_str("xsv").unwrap(); + let name = rattler_conda_types::PackageName::from_str("xsv").unwrap(); + let source_spec = manifest + .parsed + .envs + .get(&env_name) + .unwrap() + .dependencies + .specs + .get(&name) + .unwrap() + .clone(); + + let dependency_toml = |manifest: &mut Manifest| { + manifest + .document + .get_or_insert_nested_table(&["envs", env_name.as_str(), "dependencies"]) + .unwrap() + .get(name.as_normalized()) + .unwrap() + .to_string() + }; + + // Attach an inline definition to the existing source spec. + let value: toml_edit::Value = r#"{ build = { backend = { name = "pixi-build-rust" } } }"# + .parse() + .unwrap(); + let inline = + crate::project::InlinePackageValue::new(value.as_inline_table().unwrap().clone()); + let with_inline = GlobalSpec::new(name.clone(), source_spec.clone()).with_inline(inline); + manifest.add_dependency(&env_name, &with_inline).unwrap(); + assert!( + dependency_toml(&mut manifest).contains("package"), + "inline definition was not recorded" + ); + + // Re-add the same dependency without an inline definition. + let without_inline = GlobalSpec::new(name.clone(), source_spec); + manifest.add_dependency(&env_name, &without_inline).unwrap(); + assert!( + !dependency_toml(&mut manifest).contains("package"), + "stale inline definition lingered after overwrite" + ); + assert!( + manifest + .parsed + .envs + .get(&env_name) + .unwrap() + .inline_packages + .is_empty(), + "parsed inline definition lingered after overwrite" + ); + } + + /// Removing a source dependency also drops its parsed inline definition. + #[test] + fn test_remove_dependency_drops_inline_definition() { + let contents = r#" + [envs.xsv] + channels = ["conda-forge"] + [envs.xsv.dependencies] + xsv = { path = "some-source", package.build.backend.name = "pixi-build-rust" } + "#; + let mut manifest = + Manifest::from_str(std::path::Path::new("pixi-global.toml"), contents).unwrap(); + + let env_name = EnvironmentName::from_str("xsv").unwrap(); + let name = rattler_conda_types::PackageName::from_str("xsv").unwrap(); + assert!( + manifest + .parsed + .envs + .get(&env_name) + .unwrap() + .inline_packages + .contains_key(&name), + "inline definition should be parsed from the manifest" + ); + + manifest.remove_dependency(&env_name, &name).unwrap(); + + let env = manifest.parsed.envs.get(&env_name).unwrap(); + assert!(env.dependencies.specs.is_empty()); + assert!( + env.inline_packages.is_empty(), + "parsed inline definition lingered after removal" + ); + } + #[test] fn test_set_platform() { let mut manifest = Manifest::default(); diff --git a/crates/pixi_global/src/project/mod.rs b/crates/pixi_global/src/project/mod.rs index 50ded265c2..91840ea2cd 100644 --- a/crates/pixi_global/src/project/mod.rs +++ b/crates/pixi_global/src/project/mod.rs @@ -22,8 +22,8 @@ pub use parsed_manifest::{ExposedName, ParsedEnvironment, ParsedManifest}; use pixi_build_frontend::BackendOverride; use pixi_command_dispatcher::{ BuildBackendMetadataSpec, BuildEnvironment, CommandDispatcher, ComputeResultExt, - EnvironmentRef, EnvironmentSpec, EphemeralEnv, InstallPixiEnvironmentSpec, Limits, - SourceCheckoutExt, + EnvironmentRef, EnvironmentSpec, EphemeralEnv, InlinePackage, InstallPixiEnvironmentSpec, + Limits, SourceCheckoutExt, keys::{SolvePixiEnvironmentKey, SolvePixiEnvironmentSpec}, }; use pixi_config::{Config, RunPostLinkScripts, default_channel_config, pixi_home}; @@ -33,7 +33,7 @@ use pixi_core::environment::{ }; use pixi_core::lock_file::virtual_packages::minimal_required_virtual_packages; use pixi_core::repodata::Repodata; -use pixi_manifest::PrioritizedChannel; +use pixi_manifest::{InlinePackageManifest, PrioritizedChannel, WorkspaceManifest}; use pixi_path::AbsPathBuf; use pixi_reporters::TopLevelProgress; use pixi_spec::{BinarySpec, PathBinarySpec}; @@ -45,8 +45,8 @@ use pixi_utils::{ rlimit::try_increase_rlimit_to_sensible, }; use rattler_conda_types::{ - ChannelConfig, GenericVirtualPackage, MatchSpec, PackageName, Platform, PrefixRecord, - menuinst::MenuMode, package::CondaArchiveIdentifier, + ChannelConfig, GenericVirtualPackage, MatchSpec, NamedChannelOrUrl, PackageName, Platform, + PrefixRecord, menuinst::MenuMode, package::CondaArchiveIdentifier, }; use rattler_networking::LazyClient; use rattler_repodata_gateway::Gateway; @@ -77,7 +77,9 @@ mod environment; mod global_spec; mod manifest; mod parsed_manifest; -pub use global_spec::{FromMatchSpecError, GlobalSpec}; +pub use global_spec::{ + FromMatchSpecError, GlobalSpec, InlinePackageValue, InlinePackageValueError, +}; use pixi_utils::reqwest::{LazyReqwestClient, build_lazy_reqwest_clients}; #[derive(Debug, thiserror::Error, miette::Diagnostic)] @@ -96,20 +98,18 @@ pub enum CommandDispatcherError { pub enum InferPackageNameError { #[error("only source specifications are supported")] UnsupportedSpecType, - #[error( - "git package name inference is not yet supported. Please specify the package name explicitly" - )] - GitNotSupported, #[error("failed to get command dispatcher")] CommandDispatcher(#[from] CommandDispatcherError), #[error("failed to get build backend metadata for package name inference")] BuildBackendMetadata(#[source] Box), #[error("no package outputs found in the specified path/repository")] NoPackageOutputs, - #[error( - "multiple package outputs found: {package_names}. Please specify the package name explicitly" - )] - MultiplePackageOutputs { package_names: String }, + #[error("multiple package outputs found: {}", .package_names.join(", "))] + #[diagnostic(help( + "Specify which package(s) to install by naming them explicitly, e.g. `pixi global install {} ...`", + .package_names.first().map(String::as_str).unwrap_or_default() + ))] + MultiplePackageOutputs { package_names: Vec }, } pub(crate) const MANIFESTS_DIR: &str = "manifests"; @@ -596,6 +596,25 @@ impl Project { self.install_environment_with_options(env_name, false).await } + /// Builds the [`InlinePackage`] to thread through the command dispatcher + /// for a CLI-supplied inline package definition, before the definition is + /// recorded in the manifest (e.g. for package name inference). The + /// synthesized workspace manifest carries the channels the backend's + /// dependencies fall back to. + fn inline_package_for_channels( + &self, + inline: &InlinePackageManifest, + channels: impl IntoIterator, + ) -> InlinePackage { + InlinePackage { + manifest: Arc::new(inline.manifest.clone()), + workspace: Arc::new(workspace_manifest_with_channels( + channels.into_iter().map(PrioritizedChannel::from), + )), + content_hash: inline.content_hash, + } + } + pub async fn install_environment_with_options( &self, env_name: &EnvironmentName, @@ -616,6 +635,28 @@ impl Project { .into_diagnostic()?; let platform = environment.platform.unwrap_or_else(Platform::current); + + // Source dependencies are built on this machine, so they can only + // target the platform we are running on. + if platform != Platform::current() + && let Some(source_package) = environment + .dependencies + .specs + .iter() + .find(|(_, spec)| spec.is_source()) + .map(|(name, _)| name) + { + return Err(miette::miette!( + help = format!( + "Remove `platform = \"{platform}\"` from the environment, or install the package from a channel instead of from source." + ), + "environment {} requests platform '{platform}', but '{}' is a source dependency that has to be built on the current machine ('{}'); cross-platform source builds are not supported", + env_name.fancy_display(), + source_package.as_normalized(), + Platform::current(), + )); + } + let solve_virtual_packages = Self::virtual_packages_for(&platform).into_diagnostic()?; // Convert dependency specs to binary specs for CommandDispatcher @@ -635,6 +676,12 @@ impl Project { .collect::>(); let build_environment = BuildEnvironment::simple(platform, solve_virtual_packages.clone()); + + // Inline package definitions from the manifest, threaded into the + // solve and install so backend discovery uses them instead of reading + // a manifest from the source checkout. + let inline_packages = inline_packages_for_environment(environment); + // Create solve spec (compute-engine keys path). let solve_spec = SolvePixiEnvironmentSpec { dependencies: pixi_specs, @@ -654,15 +701,43 @@ impl Project { channel_priority: Default::default(), }, )), - inline_packages: Default::default(), + inline_packages: Arc::new(inline_packages.clone()), }; // Solve via SolvePixiEnvironmentKey (new keys path). + // + // A source dependency without an inline package definition relies on + // the source providing its own manifest. When it doesn't, backend + // discovery fails; point the user at `--build-backend` in that case, + // but leave unrelated solve errors (unsatisfiable specs, network + // failures, ...) unwrapped. + let has_source_without_inline = + environment.dependencies.specs.iter().any(|(name, spec)| { + spec.is_source() && !environment.inline_packages.contains_key(name) + }); let records_arc = command_dispatcher .engine() .compute(&SolvePixiEnvironmentKey::new(solve_spec)) .await - .map_err_into_dispatcher(std::convert::identity)?; + .map_err_into_dispatcher(std::convert::identity) + .map_err(miette::Report::from) + .map_err(|err| { + // Matches `DiscoveryError::FailedToDiscover`, which is + // forwarded transparently through the dispatcher's error + // wrappers. + let is_discovery_failure = err.chain().any(|cause| { + cause + .to_string() + .contains("does not contain a supported manifest") + }); + if has_source_without_inline && is_discovery_failure { + err.wrap_err( + "If the source does not provide a pixi package manifest, specify how to build it with `--build-backend`, e.g. `--build-backend pixi-build-rust`.", + ) + } else { + err + } + })?; let pixi_records: Vec<_> = (*records_arc).clone(); // Move this to a separate function to avoid code duplication @@ -711,7 +786,7 @@ impl Project { force_reinstall: force_reinstall_packages, variant_configuration: None, variant_files: None, - inline_packages: Default::default(), + inline_packages: inline_packages.into_iter().collect(), }) .await?; @@ -721,6 +796,7 @@ impl Project { platform, solve_virtual_packages, &resolved_depends, + source_fingerprints_for_environment(environment), )?; let install_changes = get_install_changes(result.transaction); @@ -732,7 +808,8 @@ impl Project { /// write. The resolved platform is the subdir plus the virtual packages the /// solve ran against (machine detection honoring `CONDA_OVERRIDE_*`); the /// minimum-supported platform keeps only the virtual packages some installed - /// record actually depends on. + /// record actually depends on. `source_fingerprints` records the source + /// dependency specifications so `pixi global sync` can detect edits. fn write_environment_file( &self, env_name: &EnvironmentName, @@ -740,6 +817,7 @@ impl Project { platform: Platform, resolved_virtual_packages: Vec, resolved_depends: &[String], + source_fingerprints: BTreeMap, ) -> miette::Result<()> { let resolved_platform = PlatformData::new(platform, resolved_virtual_packages); let depends: Vec<&str> = resolved_depends.iter().map(String::as_str).collect(); @@ -757,6 +835,7 @@ impl Project { environment_lock_file_hash: LockedEnvironmentHash::invalid(), resolved_platform: Some(resolved_platform), minimum_supported_platform: Some(minimum_supported_platform), + source_fingerprints, }, )?; Ok(()) @@ -1045,6 +1124,38 @@ impl Project { let env_dir = EnvDir::from_path(self.env_root.clone().path().join(env_name.clone().as_str())); + // Source dependencies are matched by name against the prefix, so an + // edit to a source spec (git rev, path, inline `package.*` table) + // wouldn't otherwise register as out of sync. Compare the fingerprints + // recorded at install time against the ones the current manifest + // implies. A missing record (older pixi, or never installed) counts as + // out of sync, which triggers a one-time rebuild. + if !source_package_names.is_empty() { + let expected_fingerprints = source_fingerprints_for_environment(environment); + let recorded_fingerprints = match pixi_core::environment::read_environment_file( + env_dir.path(), + ) { + Ok(env_file) => env_file + .map(|env_file| env_file.source_fingerprints) + .unwrap_or_default(), + Err(e) => { + tracing::debug!( + "Failed to read environment file of environment {}, treating source dependency fingerprints as missing: {e}", + env_name.fancy_display() + ); + Default::default() + } + }; + + if expected_fingerprints != recorded_fingerprints { + tracing::debug!( + "Environment {} out of sync because source dependency fingerprints changed", + env_name.fancy_display() + ); + return Ok(false); + } + } + let prefix = self.environment_prefix(env_name).await?; let prefix_records = prefix.find_installed_packages()?; let specs_in_sync = environment_specs_in_sync( @@ -1545,10 +1656,13 @@ impl Project { }) } - /// Infer the package name from a SourceSpec by examining build outputs + /// Infer the package name from a SourceSpec by examining build outputs. + /// When `inline` is set, backend discovery uses the inline package + /// definition instead of reading a manifest from the checkout. async fn infer_package_name_from_source_spec( &self, source_spec: pixi_spec::SourceSpec, + inline: Option, ) -> Result { let command_dispatcher = self.command_dispatcher()?; let checkout = command_dispatcher @@ -1582,7 +1696,7 @@ impl Project { )), build_string_prefix: None, build_number: None, - inline: None, + inline, }; // Get the metadata using the command dispatcher @@ -1592,31 +1706,37 @@ impl Project { .await .map_err(|e| InferPackageNameError::BuildBackendMetadata(Box::new(e)))?; - // Get the available outputs and use exactly_one to handle the single output - // case - let packages = metadata.metadata.output_names(); + // Get the available output names. Several outputs may share one name + // (e.g. variant builds of the same package); those describe a single + // package, so dedupe before deciding whether the source is ambiguous. + let packages: IndexSet = + metadata.metadata.output_names().into_iter().collect(); match packages.len() { 0 => Err(InferPackageNameError::NoPackageOutputs), 1 => Ok(packages[0].clone()), - _ => { - let package_names: Vec<_> = packages.iter().map(|p| p.as_source()).collect(); - Err(InferPackageNameError::MultiplePackageOutputs { - package_names: package_names.join(", "), - }) - } + _ => Err(InferPackageNameError::MultiplePackageOutputs { + package_names: packages.iter().map(|p| p.as_source().to_string()).collect(), + }), } } /// Infer the package name from a PixiSpec (path or git) by examining build - /// outputs + /// outputs. When `inline` carries an inline package definition, backend + /// discovery uses it instead of reading a manifest from the source + /// checkout. pub async fn infer_package_name_from_spec( &self, pixi_spec: &pixi_spec::PixiSpec, + inline: Option<&InlinePackageManifest>, ) -> Result { match pixi_spec.clone().into_source_or_binary() { Either::Left(source_spec) => { - self.infer_package_name_from_source_spec(source_spec).await + let inline = inline.map(|inline| { + self.inline_package_for_channels(inline, self.config().default_channels()) + }); + self.infer_package_name_from_source_spec(source_spec, inline) + .await } Either::Right(binary_spec) => match binary_spec { BinarySpec::Path(PathBinarySpec { path }) => path @@ -1630,6 +1750,78 @@ impl Project { } } +/// Synthesizes the minimal [`WorkspaceManifest`] that inline package +/// definitions need: there is no real workspace behind a global environment, +/// and backend discovery only consults the workspace's channels (as the +/// fallback for the build backend's own dependencies when the definition +/// doesn't pin `build.backend.channels`). +fn workspace_manifest_with_channels( + channels: impl IntoIterator, +) -> WorkspaceManifest { + let mut workspace_manifest = WorkspaceManifest::default(); + workspace_manifest.workspace.channels = channels.into_iter().collect(); + workspace_manifest +} + +/// Computes a fingerprint for each source dependency in the environment: a +/// hash of the source spec together with any inline package definition's +/// content hash. `pixi global sync` compares these against the ones recorded +/// at install time to detect when a source dependency's *specification* +/// changed (e.g. an edited git `rev` or inline `package.*` table). Binary +/// dependencies are matched against the prefix directly and are not included. +fn source_fingerprints_for_environment(environment: &ParsedEnvironment) -> BTreeMap { + use std::hash::{Hash, Hasher}; + + use xxhash_rust::xxh3::Xxh3; + + environment + .dependencies + .specs + .iter() + .filter(|(_, spec)| spec.is_source()) + .map(|(name, spec)| { + let mut hasher = Xxh3::new(); + // The source spec's TOML form is a stable, complete rendering of + // the source location (git url + rev, path, ...). + spec.to_toml_value().to_string().hash(&mut hasher); + // Fold in the inline package definition, if any, via its content + // hash so edits to `package.*` invalidate the fingerprint. + if let Some(inline) = environment.inline_packages.get(name) { + inline.content_hash.as_u64().hash(&mut hasher); + } + (name.as_normalized().to_string(), hasher.finish()) + }) + .collect() +} + +/// Converts an environment's inline package definitions into the +/// [`InlinePackage`]s the command dispatcher threads through solve, metadata +/// and build keys. +fn inline_packages_for_environment( + environment: &ParsedEnvironment, +) -> BTreeMap { + if environment.inline_packages.is_empty() { + return BTreeMap::new(); + } + let workspace = Arc::new(workspace_manifest_with_channels( + environment.channels.iter().cloned(), + )); + environment + .inline_packages + .iter() + .map(|(name, inline)| { + ( + name.clone(), + InlinePackage { + manifest: Arc::new(inline.manifest.clone()), + workspace: workspace.clone(), + content_hash: inline.content_hash, + }, + ) + }) + .collect() +} + impl Repodata for Project { /// Returns the [`Gateway`] used by this project. fn repodata_gateway(&self) -> miette::Result<&Gateway> { diff --git a/crates/pixi_global/src/project/parsed_manifest.rs b/crates/pixi_global/src/project/parsed_manifest.rs index 40fe65768f..1cb874ca47 100644 --- a/crates/pixi_global/src/project/parsed_manifest.rs +++ b/crates/pixi_global/src/project/parsed_manifest.rs @@ -6,7 +6,11 @@ use indexmap::{IndexMap, IndexSet}; use itertools::{Either, Itertools}; use miette::{Context, Diagnostic, IntoDiagnostic, LabeledSpan, NamedSource, Report}; use pixi_consts::consts; -use pixi_manifest::{PrioritizedChannel, toml::TomlPlatform, utils::package_map::UniquePackageMap}; +use pixi_manifest::{ + InlinePackageManifest, KnownPreviewFeature, Preview, PrioritizedChannel, + toml::{TomlPlatform, WorkspacePackageProperties}, + utils::package_map::{DependencyTable, UniquePackageMap}, +}; use pixi_spec::PixiSpec; use pixi_toml::{TomlFromStr, TomlIndexMap, TomlIndexSet, TomlWith}; use rattler_conda_types::{NamedChannelOrUrl, PackageName, Platform}; @@ -118,7 +122,15 @@ pub struct ParsedManifest { pub envs: IndexMap, } -impl<'de> toml_span::Deserialize<'de> for ParsedManifest { +/// The toml-stage of [`ParsedManifest`]: deserialized as-is from the document, +/// before inline package definitions are converted (which requires the +/// manifest's root directory as context). +struct TomlParsedManifest { + version: ManifestVersion, + envs: IndexMap, +} + +impl<'de> toml_span::Deserialize<'de> for TomlParsedManifest { fn deserialize(value: &mut Value<'de>) -> Result { let mut th = TableHelper::new(value)?; @@ -127,7 +139,7 @@ impl<'de> toml_span::Deserialize<'de> for ParsedManifest { .map(ManifestVersion) .unwrap_or_default(); let envs = th - .optional::>("envs") + .optional::>("envs") .map(TomlIndexMap::into_inner) .unwrap_or_default(); @@ -142,7 +154,7 @@ impl<'de> toml_span::Deserialize<'de> for ParsedManifest { fn ensure_unique_exposed_names( value: &mut Value<'_>, - envs: &IndexMap, + envs: &IndexMap, ) -> Result<(), DeserError> { let mut exposed_names = IndexSet::new(); let mut duplicates = IndexMap::new(); @@ -176,7 +188,7 @@ fn ensure_unique_exposed_names( fn ensure_unique_shortcut_names( value: &mut Value<'_>, - envs: &IndexMap, + envs: &IndexMap, ) -> Result<(), DeserError> { let mut shortcut_names = IndexSet::new(); let mut duplicates = IndexMap::new(); @@ -207,7 +219,14 @@ fn ensure_unique_shortcut_names( impl ParsedManifest { /// Parses a toml string into a project manifest. - pub(crate) fn from_toml_str(source: &str) -> Result { + /// + /// `root_directory` is the directory containing the manifest file; it is + /// used to resolve and convert inline package definitions attached to + /// source dependencies. + pub(crate) fn from_toml_str( + source: &str, + root_directory: &Path, + ) -> Result { let mut toml_value = toml_span::parse(source)?; let version = toml_value @@ -215,8 +234,8 @@ impl ParsedManifest { .and_then(|table| table.get("version")) .and_then(|version| version.as_integer()); - match ParsedManifest::deserialize(&mut toml_value) { - Ok(manifest) => Ok(manifest), + match TomlParsedManifest::deserialize(&mut toml_value) { + Ok(manifest) => manifest.into_parsed_manifest(root_directory), Err(e) => { let error = e .errors @@ -241,6 +260,71 @@ impl ParsedManifest { } } +impl TomlParsedManifest { + /// Converts the toml-stage manifest into a [`ParsedManifest`], turning + /// each environment's inline package definitions into + /// [`InlinePackageManifest`]s. + fn into_parsed_manifest( + self, + root_directory: &Path, + ) -> Result { + // The global manifest has no preview mechanism; source dependencies + // (and with them inline package definitions) are simply supported, so + // conversion runs with the pixi-build feature enabled. + let preview = Preview::from_iter([KnownPreviewFeature::PixiBuild]); + + let mut envs = IndexMap::with_capacity(self.envs.len()); + for (env_name, env) in self.envs { + let DependencyTable { + specs: dependencies, + inline_packages: inline_toml, + } = env.dependencies; + + let mut inline_packages = IndexMap::with_capacity(inline_toml.len()); + for (name, package) in inline_toml { + let span = package.span.clone(); + // There is no workspace to inherit from, so `{ workspace = + // true }` fields in inline definitions fail to resolve, and + // package defaults stay empty. + let converted = InlinePackageManifest::from_toml_package( + &name, + package.value, + WorkspacePackageProperties::default(), + &preview, + root_directory, + ) + .map_err(|e| { + ManifestParsingError::TomlError(toml_span::Error { + kind: toml_span::ErrorKind::Custom(e.to_string().into()), + span: span + .map(|range| toml_span::Span::new(range.start, range.end)) + .unwrap_or_default(), + line_info: None, + }) + })?; + inline_packages.insert(name, converted.value); + } + + envs.insert( + env_name, + ParsedEnvironment { + channels: env.channels, + platform: env.platform, + dependencies, + inline_packages, + exposed: env.exposed, + shortcuts: env.shortcuts, + }, + ); + } + + Ok(ParsedManifest { + version: self.version, + envs, + }) + } +} + impl From for ParsedManifest where I: IntoIterator, @@ -297,12 +381,30 @@ pub struct ParsedEnvironment { /// Platform used by the environment. pub platform: Option, pub dependencies: UniquePackageMap, + /// Inline package definitions attached to source dependencies. Keyed by + /// dependency name; the matching source spec lives in + /// [`Self::dependencies`]. + #[serde(skip)] + pub inline_packages: IndexMap, #[serde(default, serialize_with = "serialize_expose_mappings")] pub exposed: IndexSet, pub shortcuts: Option>, } -impl<'de> toml_span::Deserialize<'de> for ParsedEnvironment { +/// The toml-stage of [`ParsedEnvironment`]: inline package definitions are +/// still raw [`TomlPackage`](pixi_manifest::toml::TomlPackage)s inside the +/// [`DependencyTable`]; they are converted by +/// [`TomlParsedManifest::into_parsed_manifest`], which has the root directory +/// as context. +struct TomlParsedEnvironment { + channels: IndexSet, + platform: Option, + dependencies: DependencyTable, + exposed: IndexSet, + shortcuts: Option>, +} + +impl<'de> toml_span::Deserialize<'de> for TomlParsedEnvironment { fn deserialize(value: &mut toml_span::Value<'de>) -> Result { let mut th = TableHelper::new(value)?; @@ -413,9 +515,12 @@ impl AsRef for ExposedName { #[cfg(test)] mod tests { + use std::str::FromStr; + use insta::assert_snapshot; + use rattler_conda_types::PackageName; - use super::ParsedManifest; + use super::{EnvironmentName, ParsedManifest}; #[test] fn test_invalid_key() { @@ -427,9 +532,11 @@ mod tests { assert_snapshot!( examples .into_iter() - .map(|example| ParsedManifest::from_toml_str(example) - .unwrap_err() - .to_string()) + .map( + |example| ParsedManifest::from_toml_str(example, std::path::Path::new("")) + .unwrap_err() + .to_string() + ) .collect::>() .join("\n") ) @@ -453,7 +560,7 @@ mod tests { "python" = "python" "python3" = "python" "#; - let manifest = ParsedManifest::from_toml_str(contents); + let manifest = ParsedManifest::from_toml_str(contents, std::path::Path::new("")); fn remove_ansi_escape_sequences(input: &str) -> String { use regex::Regex; @@ -482,7 +589,7 @@ mod tests { [envs.python.exposed] python = "python" "#; - let manifest = ParsedManifest::from_toml_str(contents); + let manifest = ParsedManifest::from_toml_str(contents, std::path::Path::new("")); assert!(manifest.is_err()); assert_snapshot!(manifest.unwrap_err()); @@ -501,7 +608,7 @@ mod tests { // Replace the pixi-bin-name with the actual executable name that can be variable at runtime in tests let contents = contents.replace("pixi-bin-name", pixi_utils::executable_name()); - let manifest = ParsedManifest::from_toml_str(contents.as_str()); + let manifest = ParsedManifest::from_toml_str(contents.as_str(), std::path::Path::new("")); assert!(manifest.is_err()); // Replace back the executable name with "pixi" to satisfy the snapshot @@ -541,6 +648,47 @@ mod tests { [envs.python3-10.exposed] "python3.10" = "python" "#; - let _manifest = ParsedManifest::from_toml_str(contents).unwrap(); + let _manifest = ParsedManifest::from_toml_str(contents, std::path::Path::new("")).unwrap(); + } + + /// An inline package definition attached to a git source dependency is + /// parsed into an `InlinePackageManifest`, and the surrounding spec + /// remains a source spec in the dependency map. + #[test] + fn test_inline_package_definition() { + let contents = r#" + [envs.xsv] + channels = ["conda-forge"] + [envs.xsv.dependencies] + xsv = { git = "https://github.com/BurntSushi/xsv.git", package.build.backend.name = "pixi-build-rust" } + "#; + let manifest = ParsedManifest::from_toml_str(contents, std::path::Path::new("")).unwrap(); + + let env_name = EnvironmentName::from_str("xsv").unwrap(); + let env = manifest.envs.get(&env_name).unwrap(); + let name = PackageName::from_str("xsv").unwrap(); + + // The source spec is retained as a dependency. + assert!(env.dependencies.specs.get(&name).unwrap().is_source()); + + // The inline definition is converted and keyed by dependency name. + let inline = env.inline_packages.get(&name).unwrap(); + assert_eq!( + inline.manifest.build.backend.name.as_normalized(), + "pixi-build-rust" + ); + } + + /// An inline definition without a source location is meaningless and + /// rejected, matching the workspace inline syntax. + #[test] + fn test_inline_package_requires_source() { + let contents = r#" + [envs.xsv] + channels = ["conda-forge"] + [envs.xsv.dependencies] + xsv = { version = "*", package.build.backend.name = "pixi-build-rust" } + "#; + assert!(ParsedManifest::from_toml_str(contents, std::path::Path::new("")).is_err()); } } diff --git a/crates/pixi_manifest/src/target.rs b/crates/pixi_manifest/src/target.rs index 114a21e374..d4ea17c646 100644 --- a/crates/pixi_manifest/src/target.rs +++ b/crates/pixi_manifest/src/target.rs @@ -90,6 +90,54 @@ pub struct InlinePackageManifest { pub content_hash: InlineContentHash, } +impl InlinePackageManifest { + /// Converts a parsed inline `package` table into an + /// [`InlinePackageManifest`], fingerprinting the assembled manifest so + /// editing the inline definition invalidates the content-addressed build + /// caches it feeds. The dependency name is folded in so two identical + /// inline tables declared under different names stay distinct. + /// + /// The manifest's build source is taken from the surrounding dependency + /// spec, so the converted manifest carries no `build.source` of its own. + /// Package defaults stay empty: an inline definition describes a + /// dependency, not the consuming project, so it must not pick up the + /// consumer's metadata implicitly. + pub fn from_toml_package( + dependency_name: &PackageName, + package: crate::toml::TomlPackage, + workspace_package_properties: crate::toml::WorkspacePackageProperties, + preview: &crate::Preview, + root_directory: &std::path::Path, + ) -> Result, crate::TomlError> { + use xxhash_rust::xxh3::Xxh3; + + let crate::WithWarnings { + value: manifest, + warnings, + } = package.into_manifest( + workspace_package_properties, + crate::toml::PackageDefaults::default(), + preview, + root_directory, + )?; + + let content_hash = { + let mut hasher = Xxh3::new(); + dependency_name.as_normalized().hash(&mut hasher); + manifest.hash(&mut hasher); + InlineContentHash(hasher.finish()) + }; + + Ok(crate::WithWarnings { + value: InlinePackageManifest { + manifest, + content_hash, + }, + warnings, + }) + } +} + /// A package target describes the dependencies for a specific platform. #[derive(Default, Debug, Clone)] pub struct PackageTarget { diff --git a/crates/pixi_manifest/src/toml/target.rs b/crates/pixi_manifest/src/toml/target.rs index 4ac51f3349..3e5b187ee5 100644 --- a/crates/pixi_manifest/src/toml/target.rs +++ b/crates/pixi_manifest/src/toml/target.rs @@ -1,24 +1,16 @@ -use std::{ - collections::HashMap, - hash::{Hash, Hasher}, - path::Path, -}; +use std::{collections::HashMap, path::Path}; use indexmap::IndexMap; use pixi_spec::{PixiSpec, TomlLocationSpec}; use pixi_spec_containers::DependencyMap; use pixi_toml::{TomlHashMap, TomlIndexMap}; use toml_span::{DeserError, Value, de_helpers::TableHelper}; -use xxhash_rust::xxh3::Xxh3; use crate::{ - Activation, InlineContentHash, InlinePackageManifest, KnownPreviewFeature, SpecType, - TargetSelector, Task, TaskName, TomlError, Warning, WithWarnings, WorkspaceTarget, + Activation, InlinePackageManifest, KnownPreviewFeature, SpecType, TargetSelector, Task, + TaskName, TomlError, Warning, WithWarnings, WorkspaceTarget, error::GenericError, - toml::{ - PackageDefaults, TomlPackage, WorkspacePackageProperties, preview::TomlPreview, - task::TomlTask, - }, + toml::{TomlPackage, WorkspacePackageProperties, preview::TomlPreview, task::TomlTask}, utils::{ PixiSpanned, package_map::{DependencyTable, UniquePackageMap}, @@ -124,34 +116,18 @@ impl TomlTarget { let mut inline_packages: IndexMap = IndexMap::new(); for (name, package) in inline_toml { let WithWarnings { - value: manifest, + value: inline_manifest, warnings: mut package_warnings, - } = package.value.into_manifest( + } = InlinePackageManifest::from_toml_package( + &name, + package.value, workspace_package_properties.clone(), - PackageDefaults::default(), &full_preview, root_directory, )?; warnings.append(&mut package_warnings); - // Fingerprint the assembled manifest so editing the inline - // definition invalidates the content-addressed build caches it - // feeds. The dependency name is folded in so two identical inline - // tables declared under different names stay distinct. - let content_hash = { - let mut hasher = Xxh3::new(); - name.as_normalized().hash(&mut hasher); - manifest.hash(&mut hasher); - InlineContentHash(hasher.finish()) - }; - - inline_packages.insert( - name, - InlinePackageManifest { - manifest, - content_hash, - }, - ); + inline_packages.insert(name, inline_manifest); } // Convert dev dependencies from TomlLocationSpec to SourceLocationSpec diff --git a/docs/global_tools/manifest.md b/docs/global_tools/manifest.md index 5a22e772fd..b6a61962b2 100644 --- a/docs/global_tools/manifest.md +++ b/docs/global_tools/manifest.md @@ -125,6 +125,70 @@ pixi global remove --environment my-env package-a package-b ``` +## Source dependencies + +Instead of a Conda package from a channel, you can install a tool built from source. +Point `pixi global install` at a git repository or a local path: + +```shell +pixi global install --git https://github.com/prefix-dev/rattler-build rattler-build +pixi global install --path ./my-tool +``` + +If the source contains a pixi package manifest, that's all you need. +If it doesn't, tell pixi how to build it with `--build-backend`: + +```shell +pixi global install --git https://github.com/BurntSushi/xsv.git --build-backend pixi-build-rust +``` + +This records an *inline package definition* under the `package` key of the dependency: + +```toml +[envs.xsv] +channels = ["conda-forge"] +[envs.xsv.dependencies] +xsv = { git = "https://github.com/BurntSushi/xsv.git", package = { build = { backend = { name = "pixi-build-rust" } } } } +``` + +When pixi can infer the package name from the source you can omit it, so the command above +installs the `xsv` environment without naming it explicitly. +If a source produces several differently-named packages, name the ones you want: + +```shell +pixi global install --path ./workspace foo bar +``` + +When `--build-backend` or `--package` is combined with several named packages, +the same inline definition is recorded for each of them. + +The `--build-backend` value accepts an optional version constraint, e.g. +`--build-backend "pixi-build-rust>=0.3,<0.4"`. +Any other field of the package definition can be set with `--package DOTTED_KEY=TOML_VALUE`, +which maps directly onto the keys under `package`. +The value must be valid TOML, so strings need their quotes: + +```shell +pixi global install --git https://github.com/some/tool \ + --build-backend pixi-build-python \ + --package 'host-dependencies.hatchling="*"' +``` + +Anything expressible in a [pixi package manifest](../build/getting_started.md) is allowed here; +the CLI flags are just a shortcut for editing the definition by hand with +[`pixi global edit`](../reference/cli/pixi/global/edit.md). + +!!! note + Source dependencies are built on your machine, so an environment that contains one + can only target your current platform. Setting a different `platform` for such an + environment is an error. + +`pixi global sync` rebuilds a source dependency when its *specification* changes +(for example an edited git revision or `package` table). +To pick up new commits of an unpinned git dependency or new content of a local path, +run [`pixi global update`](../reference/cli/pixi/global/update.md). + + ## Platform Each environment is solved for a single platform, defaulting to your current one. diff --git a/docs/reference/cli/pixi/global/add.md b/docs/reference/cli/pixi/global/add.md index 3e0b5eaf5c..77b0561c6e 100644 --- a/docs/reference/cli/pixi/global/add.md +++ b/docs/reference/cli/pixi/global/add.md @@ -21,6 +21,11 @@ pixi global add [OPTIONS] --environment [PACKAGE]... ## Options - `--path ` : The path to the local package +- `--build-backend ` +: The build backend to build the source with, when the source does not provide its own package manifest (or to override the one it has). Accepts a name with an optional version constraint, e.g. `pixi-build-rust` or `"pixi-build-rust>=0.3,<0.4"` +- `--package ` +: Additional fields of the inline package definition, as `DOTTED_KEY=TOML_VALUE` pairs that are recorded under the `package` key of the dependency, e.g. `host-dependencies.hatchling="*"` or `build.config.extra-args=["--all-features"]` +
May be provided more than once. - `--environment (-e) ` : Specifies the environment that the dependencies need to be added to
**required**: `true` diff --git a/docs/reference/cli/pixi/global/install.md b/docs/reference/cli/pixi/global/install.md index f6e13a4569..1e74a1f03f 100644 --- a/docs/reference/cli/pixi/global/install.md +++ b/docs/reference/cli/pixi/global/install.md @@ -21,6 +21,11 @@ pixi global install [OPTIONS] [PACKAGE]... ## Options - `--path ` : The path to the local package +- `--build-backend ` +: The build backend to build the source with, when the source does not provide its own package manifest (or to override the one it has). Accepts a name with an optional version constraint, e.g. `pixi-build-rust` or `"pixi-build-rust>=0.3,<0.4"` +- `--package ` +: Additional fields of the inline package definition, as `DOTTED_KEY=TOML_VALUE` pairs that are recorded under the `package` key of the dependency, e.g. `host-dependencies.hatchling="*"` or `build.config.extra-args=["--all-features"]` +
May be provided more than once. - `--channel (-c) ` : The channels to consider as a name or a url. Multiple channels can be specified by using this field multiple times
May be provided more than once. diff --git a/tests/data/pixi-build/inline-package/recipe.yaml b/tests/data/pixi-build/inline-package/recipe.yaml new file mode 100644 index 0000000000..3d41d5c6bb --- /dev/null +++ b/tests/data/pixi-build/inline-package/recipe.yaml @@ -0,0 +1,17 @@ +package: + name: simple-package + version: 0.1.0 + +build: + number: 0 + script: + - if: win + then: + - if not exist "%PREFIX%\bin" mkdir "%PREFIX%\bin" + - echo @echo off > %PREFIX%\bin\simple-package.bat + - echo echo hello from simple-package >> %PREFIX%\bin\simple-package.bat + else: + - mkdir -p $PREFIX/bin + - echo "#!/usr/bin/env bash" > $PREFIX/bin/simple-package + - echo "echo hello from simple-package" >> $PREFIX/bin/simple-package + - chmod +x $PREFIX/bin/simple-package diff --git a/tests/integration_python/pixi_build/test_global.py b/tests/integration_python/pixi_build/test_global.py index 257edfa798..e7fd2a29ef 100644 --- a/tests/integration_python/pixi_build/test_global.py +++ b/tests/integration_python/pixi_build/test_global.py @@ -5,6 +5,7 @@ import tomllib from .common import ( + CURRENT_PLATFORM, ExitCode, copy_manifest, copytree_with_local_backend, @@ -300,6 +301,130 @@ def test_install_multi_output_multiple( verify_cli_command([bizbar], env=env, stdout_contains="Hello from bizbar") +def test_install_inline_package( + pixi: Path, + tmp_path: Path, + build_data: Path, +) -> None: + """Install a source that has no pixi manifest, supplying the build backend + with `--build-backend` so an inline package definition is recorded.""" + pixi_home = tmp_path / "pixi_home" + env = {"PIXI_HOME": str(pixi_home)} + + # A directory with only a `recipe.yaml`, i.e. no pixi package manifest. + source_project = build_data.joinpath("inline-package") + + # The package name is inferred from the recipe by the backend. + verify_cli_command( + [ + pixi, + "global", + "install", + "--path", + source_project, + "--build-backend", + "pixi-build-rattler-build", + ], + env=env, + ) + + # The manifest records the inline package definition under the `package` + # key of the dependency. + manifest_path = pixi_home.joinpath("manifests", "pixi-global.toml") + manifest = tomllib.loads(manifest_path.read_text()) + dependency = manifest["envs"]["simple-package"]["dependencies"]["simple-package"] + assert dependency["package"]["build"]["backend"]["name"] == "pixi-build-rattler-build" + + # The tool was built and installed. + simple_package = pixi_home / "bin" / exec_extension("simple-package") + verify_cli_command([simple_package], env=env, stdout_contains="hello from simple-package") + + +def test_install_inline_package_collision( + pixi: Path, + tmp_path: Path, + build_data: Path, +) -> None: + """Setting the same inline key via `--build-backend` and `--package` fails.""" + pixi_home = tmp_path / "pixi_home" + env = {"PIXI_HOME": str(pixi_home)} + + source_project = build_data.joinpath("inline-package") + + verify_cli_command( + [ + pixi, + "global", + "install", + "--path", + source_project, + "--build-backend", + "pixi-build-rattler-build", + "--package", + 'build.backend.name="other"', + ], + ExitCode.FAILURE, + env=env, + stderr_contains="set more than once", + ) + + +def test_install_build_backend_requires_source( + pixi: Path, + tmp_path: Path, +) -> None: + """`--build-backend` without a source location is an error.""" + pixi_home = tmp_path / "pixi_home" + env = {"PIXI_HOME": str(pixi_home)} + + verify_cli_command( + [ + pixi, + "global", + "install", + "xsv", + "--build-backend", + "pixi-build-rust", + ], + ExitCode.FAILURE, + env=env, + stderr_contains="require a source location", + ) + + +def test_install_source_platform_guard( + pixi: Path, + tmp_path: Path, + build_data: Path, +) -> None: + """A source dependency cannot be installed for a non-current platform.""" + pixi_home = tmp_path / "pixi_home" + env = {"PIXI_HOME": str(pixi_home)} + + source_project = build_data.joinpath("simple-package") + + # Pick a platform that is never the current one. + other_platform = "linux-aarch64" if CURRENT_PLATFORM != "linux-aarch64" else "win-64" + + # Pass the name explicitly so the guard (which fires before any solve or + # build) is reached without needing the backend for name inference. + verify_cli_command( + [ + pixi, + "global", + "install", + "--path", + source_project, + "simple-package", + "--platform", + other_platform, + ], + ExitCode.FAILURE, + env=env, + stderr_contains="source dependency", + ) + + @pytest.mark.slow def test_install_recursive_source_run_dependencies( pixi: Path, diff --git a/tests/integration_python/pixi_build/test_global_inline.py b/tests/integration_python/pixi_build/test_global_inline.py new file mode 100644 index 0000000000..a68901563d --- /dev/null +++ b/tests/integration_python/pixi_build/test_global_inline.py @@ -0,0 +1,197 @@ +"""Integration tests for inline package definitions in `pixi global` +(`--build-backend` / `--package`), introduced in #6521. + +These cover the end-to-end paths that can't be unit-tested in Rust: building a +manifest-less source through the real backend, name inference, and the +fingerprint-driven rebuild. The pure spec-conversion and manifest-overwrite +logic is covered by Rust unit tests in `pixi_cli`/`pixi_global`. + +Run a single one with `pixi run test-specific-test `. +""" + +from pathlib import Path + +import pytest +import tomli_w +import tomllib + +from .common import ( + copy_manifest, + copytree_with_local_backend, + exec_extension, + git_test_repo, + verify_cli_command, +) + +# The hermetic fixture is a directory with only a `recipe.yaml` (no pixi +# package manifest); `pixi-build-rattler-build` builds it into a +# `simple-package` tool that prints "hello from simple-package". +RATTLER_BUILD = "pixi-build-rattler-build" + + +def _modifiable_recipe_source(build_data: Path, dest: Path) -> Path: + """Copy the recipe-only fixture so its recipe can be edited between builds. + `copy_manifest` bumps the mtime so a stale build cache isn't reused.""" + copytree_with_local_backend( + build_data.joinpath("inline-package"), + dest, + copy_function=copy_manifest, + ) + return dest + + +def _rewrite_recipe_message(source_project: Path, old: str, new: str) -> None: + recipe_path = source_project / "recipe.yaml" + recipe_path.write_text(recipe_path.read_text().replace(old, new)) + + +def test_install_inline_from_git_infers_name(pixi: Path, tmp_path: Path, build_data: Path) -> None: + """A git source with no pixi manifest builds via `--build-backend`, and the + package name is inferred from the recipe.""" + pixi_home = tmp_path / "pixi_home" + env = {"PIXI_HOME": str(pixi_home)} + + git_url = git_test_repo(build_data.joinpath("inline-package"), "inline-git", tmp_path) + + verify_cli_command( + [ + pixi, + "global", + "install", + "--git", + git_url, + "--build-backend", + RATTLER_BUILD, + ], + env=env, + ) + + simple_package = pixi_home / "bin" / exec_extension("simple-package") + verify_cli_command([simple_package], env=env, stdout_contains="hello from simple-package") + + +@pytest.mark.slow +def test_manifest_inline_edit_triggers_rebuild( + pixi: Path, tmp_path: Path, build_data: Path +) -> None: + """Hand-editing the inline `package` table in the manifest changes the source + fingerprint, so `sync` detects the env as out of sync and rebuilds. Guards the + new behavior that source fingerprints include the inline definition.""" + pixi_home = tmp_path / "pixi_home" + env = {"PIXI_HOME": str(pixi_home)} + source_project = _modifiable_recipe_source(build_data, tmp_path / "src") + + verify_cli_command( + [ + pixi, + "global", + "install", + "--path", + source_project, + "simple-package", + "--build-backend", + RATTLER_BUILD, + ], + env=env, + ) + simple_package = pixi_home / "bin" / exec_extension("simple-package") + + # Edit the source; `sync` alone would ignore this (the spec is unchanged) ... + _rewrite_recipe_message( + source_project, "hello from simple-package", "goodbye from simple-package" + ) + + # ... and edit the inline definition in the manifest. This changes the + # recorded fingerprint, which should mark the env out of sync. + manifest_path = pixi_home.joinpath("manifests", "pixi-global.toml") + manifest = tomllib.loads(manifest_path.read_text()) + backend = manifest["envs"]["simple-package"]["dependencies"]["simple-package"]["package"][ + "build" + ]["backend"] + assert "version" not in backend + backend["version"] = "*" + manifest_path.write_text(tomli_w.dumps(manifest)) + + verify_cli_command([pixi, "global", "sync"], env=env) + verify_cli_command([simple_package], env=env, stdout_contains="goodbye from simple-package") + + +def test_add_inline_to_existing_environment( + pixi: Path, tmp_path: Path, build_data: Path, dummy_channel_1: str +) -> None: + """`pixi global add --build-backend` records an inline definition for a + source added to an existing environment and builds it.""" + pixi_home = tmp_path / "pixi_home" + env = {"PIXI_HOME": str(pixi_home)} + + verify_cli_command( + [ + pixi, + "global", + "install", + "--channel", + dummy_channel_1, + "--environment", + "test-env", + "dummy-f", + ], + env=env, + ) + + verify_cli_command( + [ + pixi, + "global", + "add", + "--environment", + "test-env", + "--path", + build_data.joinpath("inline-package"), + "--build-backend", + RATTLER_BUILD, + "--expose", + "simple-package=simple-package", + ], + env=env, + ) + + manifest_path = pixi_home.joinpath("manifests", "pixi-global.toml") + manifest = tomllib.loads(manifest_path.read_text()) + dependency = manifest["envs"]["test-env"]["dependencies"]["simple-package"] + assert dependency["package"]["build"]["backend"]["name"] == RATTLER_BUILD + + simple_package = pixi_home / "bin" / exec_extension("simple-package") + verify_cli_command([simple_package], env=env, stdout_contains="hello from simple-package") + + +def test_sync_unchanged_inline_env_is_noop(pixi: Path, tmp_path: Path, build_data: Path) -> None: + """`pixi global sync` leaves an unchanged source environment alone. The + fingerprints recorded at install time must match the ones recomputed from + the manifest; if they ever diverge for an unchanged manifest, every sync + rebuilds every source environment.""" + pixi_home = tmp_path / "pixi_home" + env = {"PIXI_HOME": str(pixi_home)} + + verify_cli_command( + [ + pixi, + "global", + "install", + "--path", + build_data.joinpath("inline-package"), + "--build-backend", + RATTLER_BUILD, + ], + env=env, + ) + + # The environment file is rewritten whenever the environment is + # (re)installed, so a stable mtime means the sync was a no-op. + env_file = pixi_home.joinpath("envs", "simple-package", "conda-meta", "pixi") + mtime_before = env_file.stat().st_mtime_ns + + verify_cli_command([pixi, "global", "sync"], env=env) + + assert env_file.stat().st_mtime_ns == mtime_before, ( + "sync rebuilt an unchanged source environment" + )