diff --git a/Cargo.lock b/Cargo.lock index eeed7abf4e..857db57496 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6401,6 +6401,7 @@ dependencies = [ "futures", "hex", "human_bytes", + "ignore", "indexmap 2.14.0", "indicatif", "insta", @@ -6414,6 +6415,7 @@ dependencies = [ "pixi_api", "pixi_auth", "pixi_build_frontend", + "pixi_build_types", "pixi_command_dispatcher", "pixi_compute_reporters", "pixi_config", diff --git a/Cargo.toml b/Cargo.toml index d5d3333ac0..38914f3fc5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,6 +66,7 @@ http = "1.3.1" http-cache-reqwest = "1.0.0-alpha.6" human_bytes = "0.4.3" humantime = "2.1.0" +ignore = "0.4" indexmap = "2.10.0" indicatif = "0.18.0" insta = "1.42.1" diff --git a/crates/pixi/tests/integration_rust/source_package_tests.rs b/crates/pixi/tests/integration_rust/source_package_tests.rs index 49495594b8..44053d8048 100644 --- a/crates/pixi/tests/integration_rust/source_package_tests.rs +++ b/crates/pixi/tests/integration_rust/source_package_tests.rs @@ -1005,7 +1005,8 @@ async fn test_publish_fails_before_build_or_upload_when_one_variant_is_unsatisfi target_channel: Some(target_url.to_string()), target_dir: None, force: false, - skip_existing: true, + no_skip_existing: false, + dry_run: false, generate_attestation: false, variant: Vec::new(), variant_config: Vec::new(), @@ -2925,7 +2926,8 @@ async fn test_publish_without_target_builds_but_does_not_upload() { target_channel: None, target_dir: None, force: false, - skip_existing: true, + no_skip_existing: false, + dry_run: false, generate_attestation: false, variant: Vec::new(), variant_config: Vec::new(), @@ -2940,6 +2942,420 @@ async fn test_publish_without_target_builds_but_does_not_upload() { ); } +/// Serializes tests that must change the process working directory: +/// workspace-wide publishing (no `--path`) anchors workspace discovery at the +/// current directory. +static PUBLISH_CWD_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +/// Run `pixi publish` with the process working directory set to +/// `workspace_root`, restoring the previous working directory afterwards. +async fn publish_from_directory( + workspace_root: &std::path::Path, + args: publish::Args, +) -> miette::Result<()> { + let _guard = PUBLISH_CWD_LOCK.lock().await; + let original_cwd = std::env::current_dir().unwrap(); + std::env::set_current_dir(workspace_root).unwrap(); + let result = publish::execute(args).await; + std::env::set_current_dir(original_cwd).unwrap(); + result +} + +fn publish_args_for_test( + backend_override: Option, + path: Option, + target_dir: Option, +) -> publish::Args { + publish::Args { + backend_override, + config_cli: Default::default(), + config_source: isolated_config_source(), + target_platform: Platform::current(), + build_platform: Platform::current(), + build_string_prefix: None, + build_number: None, + build_dir: None, + clean: false, + path, + target_channel: None, + target_dir, + force: false, + no_skip_existing: false, + dry_run: false, + generate_attestation: false, + variant: Vec::new(), + variant_config: Vec::new(), + package_format: None, + } +} + +/// The names (without version/build suffix) of the `.conda` artifacts below +/// `dir`, sorted. +fn conda_artifact_names(dir: &std::path::Path) -> Vec { + let mut found = Vec::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(current) = stack.pop() { + for entry in fs::read_dir(¤t).unwrap().filter_map(Result::ok) { + let path = entry.path(); + if path.is_dir() { + stack.push(path); + } else if path.extension().is_some_and(|ext| ext == "conda") { + let file_name = path.file_name().unwrap().to_string_lossy(); + let name = file_name.split('-').next().unwrap_or_default(); + found.push(name.to_string()); + } + } + } + found.sort(); + found +} + +/// Renders the `publish` line of a `[package]` section. +fn publish_flag(publish: Option) -> String { + match publish { + Some(value) => format!("publish = {value}\n"), + None => String::new(), + } +} + +/// Writes a three-package workspace: the root manifest holds the workspace +/// and a `kit` package that run-depends on member `cpp`, which in turn +/// run-depends on member `core`. The `publish` arguments set the `publish` +/// flag of the respective package, selecting which of them a workspace-wide +/// publish operates on. +fn write_three_package_workspace( + root: &std::path::Path, + kit_publish: Option, + cpp_publish: Option, + core_publish: Option, +) { + let platform = Platform::current(); + fs::write( + root.join("pixi.toml"), + format!( + r#" +[workspace] +channels = [] +platforms = ["{platform}"] +preview = ["pixi-build"] + +[package] +name = "kit" +version = "1.0.0" +{kit_flag} +[package.build] +backend = {{ name = "in-memory", version = "0.1.0" }} + +[package.run-dependencies] +cpp = {{ path = "./cpp" }} +"#, + kit_flag = publish_flag(kit_publish), + ), + ) + .unwrap(); + + let member_manifest = |name: &str, publish: Option, extra: &str| { + format!( + r#" +[package] +name = "{name}" +version = "1.0.0" +{flag} +[package.build] +backend = {{ name = "in-memory", version = "0.1.0" }} +{extra} +"#, + flag = publish_flag(publish), + ) + }; + + let cpp_dir = root.join("cpp"); + fs::create_dir_all(&cpp_dir).unwrap(); + fs::write( + cpp_dir.join("pixi.toml"), + member_manifest( + "cpp", + cpp_publish, + r#" +[package.run-dependencies] +core = { path = "../core" } +"#, + ), + ) + .unwrap(); + + let core_dir = root.join("core"); + fs::create_dir_all(&core_dir).unwrap(); + fs::write( + core_dir.join("pixi.toml"), + member_manifest("core", core_publish, ""), + ) + .unwrap(); +} + +/// `pixi publish` without `--path` must build every package that opts in +/// with `publish = true` - including the root package - and upload all of +/// them in dependency order. +#[tokio::test] +async fn test_publish_workspace_publishes_all_opted_in_packages() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + write_three_package_workspace(pixi.workspace_path(), Some(true), Some(true), Some(true)); + + let publish_dir = tempfile::tempdir().unwrap(); + publish_from_directory( + pixi.workspace_path(), + publish_args_for_test( + Some(BackendOverride::from_memory( + PassthroughBackend::instantiator(), + )), + None, + Some(publish_dir.path().to_path_buf()), + ), + ) + .await + .expect("workspace-wide publish should succeed"); + + assert_eq!( + conda_artifact_names(publish_dir.path()), + vec!["core", "cpp", "kit"], + "all three opted-in workspace packages should be published" + ); +} + +/// A published package whose source dependency does not opt into publishing +/// must fail the publish: uploading it would leave the target with an +/// unsatisfiable run dependency. +#[tokio::test] +async fn test_publish_workspace_rejects_unpublished_source_dependency() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + write_three_package_workspace(pixi.workspace_path(), Some(true), Some(true), None); + + let err = publish_from_directory( + pixi.workspace_path(), + publish_args_for_test( + Some(BackendOverride::from_memory( + PassthroughBackend::instantiator(), + )), + None, + None, + ), + ) + .await + .expect_err("publishing with a source dependency that does not opt in should fail"); + + let rendered = format_diagnostic(err.as_ref()); + assert!( + rendered.contains("not part of the publish set") && rendered.contains("core"), + "{rendered}" + ); +} + +/// An explicit `publish = false` behaves exactly like an absent flag: the +/// package is not part of the publish set, so depending on it fails. +#[tokio::test] +async fn test_publish_workspace_rejects_publish_false_source_dependency() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + write_three_package_workspace(pixi.workspace_path(), Some(true), Some(true), Some(false)); + + let err = publish_from_directory( + pixi.workspace_path(), + publish_args_for_test( + Some(BackendOverride::from_memory( + PassthroughBackend::instantiator(), + )), + None, + None, + ), + ) + .await + .expect_err("publishing with a `publish = false` source dependency should fail"); + + let rendered = format_diagnostic(err.as_ref()); + assert!( + rendered.contains("not part of the publish set") && rendered.contains("core"), + "{rendered}" + ); +} + +/// `pixi publish --path ` keeps the single-package behavior for +/// self-contained packages: only the addressed package is built and +/// uploaded, whether or not it sets `publish = true`. +#[tokio::test] +async fn test_publish_with_path_publishes_a_single_package() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + write_three_package_workspace(pixi.workspace_path(), Some(true), Some(true), None); + + let publish_dir = tempfile::tempdir().unwrap(); + publish::execute(publish_args_for_test( + Some(BackendOverride::from_memory( + PassthroughBackend::instantiator(), + )), + Some(pixi.workspace_path().join("core")), + Some(publish_dir.path().to_path_buf()), + )) + .await + .expect("single-package publish should succeed"); + + assert_eq!( + conda_artifact_names(publish_dir.path()), + vec!["core"], + "only the package addressed with --path should be published" + ); +} + +/// `pixi publish --path` is a batch of one: a package with source +/// dependencies cannot be published alone, because nothing in the batch +/// satisfies them on the target. +#[tokio::test] +async fn test_publish_with_path_rejects_source_dependencies() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + write_three_package_workspace(pixi.workspace_path(), Some(true), Some(true), Some(true)); + + let err = publish::execute(publish_args_for_test( + Some(BackendOverride::from_memory( + PassthroughBackend::instantiator(), + )), + Some(pixi.workspace_path().join("cpp")), + None, + )) + .await + .expect_err("--path on a package with source dependencies should fail"); + + let rendered = format_diagnostic(err.as_ref()); + assert!( + rendered.contains("cannot be published on its own") && rendered.contains("core"), + "{rendered}" + ); +} + +/// `--dry-run` resolves and prints the publish set but must not build or +/// upload anything. +#[tokio::test] +async fn test_publish_dry_run_builds_and_uploads_nothing() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + write_three_package_workspace(pixi.workspace_path(), Some(true), Some(true), Some(true)); + + let publish_dir = tempfile::tempdir().unwrap(); + let mut args = publish_args_for_test( + Some(BackendOverride::from_memory( + PassthroughBackend::instantiator(), + )), + None, + Some(publish_dir.path().to_path_buf()), + ); + args.dry_run = true; + publish_from_directory(pixi.workspace_path(), args) + .await + .expect("a dry run of a valid publish set should succeed"); + + assert_eq!( + conda_artifact_names(publish_dir.path()), + Vec::::new(), + "a dry run must not upload any artifacts" + ); +} + +/// A package without `publish = true` is not published: a workspace whose +/// only package does not opt in must fail instead of falling back to the +/// closest package. +#[tokio::test] +async fn test_publish_without_opt_in_fails() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + let platform = Platform::current(); + fs::write( + pixi.manifest_path(), + format!( + r#" +[workspace] +channels = [] +platforms = ["{platform}"] +preview = ["pixi-build"] + +[package] +name = "solo" +version = "1.0.0" + +[package.build] +backend = {{ name = "in-memory", version = "0.1.0" }} +"# + ), + ) + .unwrap(); + + let err = publish_from_directory( + pixi.workspace_path(), + publish_args_for_test( + Some(BackendOverride::from_memory( + PassthroughBackend::instantiator(), + )), + None, + None, + ), + ) + .await + .expect_err("publishing a workspace whose packages do not opt in should fail"); + + let rendered = format_diagnostic(err.as_ref()); + assert!( + rendered.contains("no package in the workspace opts into publishing"), + "{rendered}" + ); +} + +/// A workspace without any package must fail with the same opt-in hint. +#[tokio::test] +async fn test_publish_workspace_without_packages_fails() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + fs::write( + pixi.manifest_path(), + format!( + r#" +[workspace] +channels = [] +platforms = ["{}"] +preview = ["pixi-build"] +"#, + Platform::current() + ), + ) + .unwrap(); + + let err = publish_from_directory( + pixi.workspace_path(), + publish_args_for_test( + Some(BackendOverride::from_memory( + PassthroughBackend::instantiator(), + )), + None, + None, + ), + ) + .await + .expect_err("publishing an empty workspace should fail"); + + let rendered = format_diagnostic(err.as_ref()); + assert!( + rendered.contains("no package in the workspace opts into publishing"), + "{rendered}" + ); +} + /// Regression test for #4761: `.pixi/.gitignore` must be created during /// `sanity_check_workspace`, even when the publish itself fails because the /// configured backend cannot be resolved. Without the gitignore, rattler-build @@ -2993,7 +3409,8 @@ backend.version = "0.1.0" target_channel: None, target_dir: None, force: false, - skip_existing: true, + no_skip_existing: false, + dry_run: false, generate_attestation: false, variant: Vec::new(), variant_config: Vec::new(), @@ -3117,7 +3534,8 @@ host-lib = "*" target_channel: None, target_dir: Some(target_dir.path().to_path_buf()), force: false, - skip_existing: true, + no_skip_existing: false, + dry_run: false, generate_attestation: false, variant: Vec::new(), variant_config: Vec::new(), diff --git a/crates/pixi_cli/Cargo.toml b/crates/pixi_cli/Cargo.toml index 9aa6267829..446b3ec04e 100644 --- a/crates/pixi_cli/Cargo.toml +++ b/crates/pixi_cli/Cargo.toml @@ -35,6 +35,7 @@ fs-err = { workspace = true, features = ["tokio"] } futures = { workspace = true } hex = { workspace = true } human_bytes = { workspace = true } +ignore = { workspace = true } indexmap = { workspace = true, features = ["serde"] } indicatif = { workspace = true } is_executable = { workspace = true } @@ -46,6 +47,7 @@ pep508_rs = { workspace = true } pixi_api = { workspace = true } pixi_auth = { workspace = true } pixi_build_frontend = { workspace = true } +pixi_build_types = { workspace = true } pixi_command_dispatcher = { workspace = true } pixi_compute_reporters = { workspace = true } pixi_config = { workspace = true } diff --git a/crates/pixi_cli/src/build.rs b/crates/pixi_cli/src/build.rs index cdb856601a..4221f0a627 100644 --- a/crates/pixi_cli/src/build.rs +++ b/crates/pixi_cli/src/build.rs @@ -60,6 +60,12 @@ pub struct Args { } pub async fn execute(args: Args) -> miette::Result<()> { + // `pixi build` predates workspace-wide publishing and always operated on + // a single package. Anchoring the publish invocation to the current + // directory when no `--path` is given keeps it that way instead of + // fanning out to every workspace member. + let path = args.path.unwrap_or_else(|| PathBuf::from(".")); + let mut cmd_parts = vec!["pixi publish".to_string()]; if args.target_platform != Platform::current() { @@ -74,9 +80,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { if args.clean { cmd_parts.push("--clean".to_string()); } - if let Some(ref path) = args.path { - cmd_parts.push(format!("--path {}", path.display())); - } + cmd_parts.push(format!("--path {}", path.display())); if args.output_dir != Path::new(".") { cmd_parts.push(format!("--target-dir {}", args.output_dir.display())); } @@ -102,11 +106,12 @@ pub async fn execute(args: Args) -> miette::Result<()> { build_number: None, build_dir: args.build_dir, clean: args.clean, - path: args.path, + path: Some(path), target_channel: None, target_dir: Some(args.output_dir), force: false, - skip_existing: true, + no_skip_existing: false, + dry_run: false, generate_attestation: false, variant: Vec::new(), variant_config: Vec::new(), diff --git a/crates/pixi_cli/src/publish/discovery.rs b/crates/pixi_cli/src/publish/discovery.rs new file mode 100644 index 0000000000..95474289a6 --- /dev/null +++ b/crates/pixi_cli/src/publish/discovery.rs @@ -0,0 +1,699 @@ +//! Discovery of the publish set for `pixi publish`. +//! +//! A workspace-wide `pixi publish` operates on every package in the workspace +//! that opts into publishing with `publish = true` in its `[package]` +//! section. Package manifests are discovered by walking the workspace +//! directory tree; the walk respects ignore files (such as `.gitignore`), +//! skips hidden directories, and skips subtrees that belong to a nested +//! workspace. +//! +//! Every publish is a closed batch: each source dependency (build, host, or +//! run) of every published package must itself be part of the batch. Source +//! dependencies that do not opt into publishing - whether they point at a +//! directory inside the workspace or at something external (git or url +//! sources, paths escaping the workspace root) - fail the publish. + +use std::{ + collections::{BTreeMap, BTreeSet}, + path::{Path, PathBuf}, +}; + +use miette::{Context, IntoDiagnostic}; +use pixi_build_types::{PackageSpec, SourcePackageSpec, procedures::conda_outputs::CondaOutput}; +use pixi_command_dispatcher::{ + BuildBackendMetadataSpec, CommandDispatcher, build::conversion::from_source_spec_v1, +}; +use pixi_consts::consts::{MOJOPROJECT_MANIFEST, PYPROJECT_MANIFEST, WORKSPACE_MANIFEST}; +use pixi_core::Workspace; +use pixi_record::{PinnedPathSpec, PinnedSourceSpec}; +use pixi_spec::{SourceAnchor, SourceLocationSpec}; +use typed_path::Utf8TypedPathBuf; + +/// The packages a workspace-wide `pixi publish` operates on, plus diagnostics +/// gathered while resolving them. +pub(crate) struct WorkspacePackageSet { + /// Workspace-relative package sources, ordered so that every package + /// appears after the packages it depends on (dependencies first). Upload + /// in this order never leaves the target channel with a package whose + /// run dependencies are missing. + pub packages: Vec, + + /// Members that had to be forced out of a dependency cycle. Uploads + /// involving these packages cannot be fully dependency-ordered. + pub cycle_members: Vec, +} + +/// Where a source dependency points, relative to the workspace. +enum SourceTarget { + /// A workspace-relative directory. + Member(String), + + /// A source that lives outside the workspace (git/url source or a path + /// escaping the workspace root). + External(String), +} + +/// Resolve the set of packages that opt into publishing with +/// `publish = true` and return them in dependency order (dependencies before +/// dependents). +/// +/// `make_metadata_spec` builds the per-package metadata request; the caller +/// owns the environment/channel/variant configuration that goes into it. The +/// metadata computed here is cached by the command dispatcher, so the +/// subsequent build phase does not pay for it twice. +pub(crate) async fn resolve_publish_set( + workspace: &Workspace, + command_dispatcher: &CommandDispatcher, + make_metadata_spec: impl Fn(PinnedSourceSpec) -> BuildBackendMetadataSpec, +) -> miette::Result { + let workspace_root = workspace.root(); + + let members = discover_opted_in_packages(workspace_root)?; + if members.is_empty() { + return Err(miette::diagnostic!( + help = "Set `publish = true` in the `[package]` section of every package that \ + `pixi publish` should publish. To publish a single package, pass \ + `--path `.", + "no package in the workspace opts into publishing", + ) + .into()); + } + + // Query each package's build backend metadata and validate the closure: + // every source dependency must itself opt into publishing. Output names + // must be unique across the set; the target cannot hold two different + // packages under the same name. + let mut member_dependencies: BTreeMap> = BTreeMap::new(); + let mut output_owners: BTreeMap = BTreeMap::new(); + let mut violations: BTreeSet = BTreeSet::new(); + for dir in &members { + let manifest_source: PinnedSourceSpec = PinnedPathSpec { + path: dir.clone().into(), + } + .into(); + let backend_metadata = command_dispatcher + .build_backend_metadata(make_metadata_spec(manifest_source.clone())) + .await?; + + // Relative path specs in the metadata are anchored to the package + // that declares them, and resolve to workspace-relative paths. + let anchor = SourceAnchor::from(SourceLocationSpec::from(manifest_source)); + + for output in &backend_metadata.metadata.outputs { + let output_name = output.metadata.name.as_normalized().to_string(); + if let Some(owner) = output_owners.get(&output_name) { + if owner != dir { + return Err(miette::diagnostic!( + help = "Packages in the publish set must have unique names. Remove \ + `publish = true` from one of the two packages.", + "packages '{owner}' and '{dir}' both produce an output named '{output_name}'", + ) + .into()); + } + } else { + output_owners.insert(output_name, dir.clone()); + } + + for (_name, source_spec) in output_source_dependencies(output) { + let resolved = anchor.resolve(from_source_spec_v1(source_spec.clone())); + match classify_source_location(workspace_root, resolved.location) { + SourceTarget::Member(dep_dir) => { + if members.contains(&dep_dir) { + if dep_dir != *dir { + member_dependencies + .entry(dir.clone()) + .or_default() + .insert(dep_dir); + } + } else { + violations.insert(format!( + "package '{dir}' depends on '{dep_dir}', which is not part of the publish set" + )); + } + } + SourceTarget::External(location) => { + violations.insert(format!( + "package '{dir}' depends on '{location}', which lives outside the workspace" + )); + } + } + } + } + } + + if !violations.is_empty() { + return Err(miette::diagnostic!( + help = "Every source dependency of a published package must itself be published \ + in the same batch. Set `publish = true` in the `[package]` section of \ + the missing packages, or replace the source dependencies with binary \ + dependencies.", + "the packages cannot be published as a self-contained set:\n {}", + violations.iter().cloned().collect::>().join("\n "), + ) + .into()); + } + + let ordering = dependency_order(&members, &member_dependencies); + let packages = ordering + .order + .into_iter() + .map(|dir| PinnedPathSpec { path: dir.into() }.into()) + .collect(); + + Ok(WorkspacePackageSet { + packages, + cycle_members: ordering.cycle_members, + }) +} + +/// All source dependencies of a backend output as `(name, spec)` pairs: +/// build, host, and run dependencies plus the dependencies of every extra +/// group. +pub(super) fn output_source_dependencies( + output: &CondaOutput, +) -> impl Iterator { + let dependency_sets = output + .build_dependencies + .iter() + .chain(output.host_dependencies.iter()) + .chain(std::iter::once(&output.run_dependencies)); + dependency_sets + .flat_map(|deps| deps.depends.iter()) + .chain(output.extra_dependencies.values().flatten()) + .filter_map(|named| match &named.spec { + PackageSpec::Source(source) => Some((named.name.as_str(), source)), + PackageSpec::Binary(_) | PackageSpec::PinCompatible(_) => None, + }) +} + +/// The publish-relevant contents of a manifest, read with a lightweight TOML +/// peek instead of a full manifest parse. +struct ManifestPeek { + /// The manifest declares a workspace section (`[workspace]`, or the + /// legacy `[project]` alias). + declares_workspace: bool, + + /// The value of `publish` in the package section, if the manifest + /// declares one. + publish: Option, +} + +/// A pixi manifest found while walking the workspace directory tree. +struct DiscoveredManifest { + /// Absolute directory containing the manifest. + dir: PathBuf, + peek: ManifestPeek, +} + +/// Read the publish-relevant keys of a manifest. For pyproject manifests the +/// pixi sections live under `[tool.pixi]`. +fn peek_manifest(manifest_path: &Path) -> miette::Result { + let source = fs_err::read_to_string(manifest_path).into_diagnostic()?; + let doc = source + .parse::() + .into_diagnostic() + .with_context(|| format!("failed to parse '{}'", manifest_path.display()))?; + + let is_pyproject = manifest_path + .file_name() + .is_some_and(|name| name == PYPROJECT_MANIFEST); + let root: Option<&toml_edit::Item> = if is_pyproject { + doc.get("tool").and_then(|tool| tool.get("pixi")) + } else { + Some(doc.as_item()) + }; + let Some(root) = root else { + return Ok(ManifestPeek { + declares_workspace: false, + publish: None, + }); + }; + + let declares_workspace = root.get("workspace").is_some() || root.get("project").is_some(); + let publish = match root + .get("package") + .and_then(|package| package.get("publish")) + { + None => None, + Some(item) => Some(item.as_bool().ok_or_else(|| { + miette::diagnostic!( + "the `publish` key of the package in '{}' must be a boolean", + manifest_path.display(), + ) + })?), + }; + + Ok(ManifestPeek { + declares_workspace, + publish, + }) +} + +/// Precedence of manifest files within one directory: a `pixi.toml` shadows a +/// `pyproject.toml`, which shadows a `mojoproject.toml`. +fn manifest_priority(file_name: &str) -> Option { + [WORKSPACE_MANIFEST, PYPROJECT_MANIFEST, MOJOPROJECT_MANIFEST] + .iter() + .position(|name| *name == file_name) +} + +/// Walk the workspace directory tree and collect the member directories of +/// every package that opts into publishing with `publish = true`. +/// +/// The walk respects ignore files (such as `.gitignore`) and skips hidden +/// directories. A manifest in a subdirectory that declares its own workspace +/// starts a nested, independent workspace: its entire subtree is skipped. +fn discover_opted_in_packages(workspace_root: &Path) -> miette::Result> { + // The highest-priority manifest of every visited directory. + let mut manifest_paths: BTreeMap = BTreeMap::new(); + for entry in ignore::WalkBuilder::new(workspace_root) + .require_git(false) + .build() + { + let entry = match entry { + Ok(entry) => entry, + Err(error) => { + tracing::debug!("skipping unreadable entry while discovering packages: {error}"); + continue; + } + }; + if !entry.file_type().is_some_and(|kind| kind.is_file()) { + continue; + } + let Some(priority) = entry.file_name().to_str().and_then(manifest_priority) else { + continue; + }; + let Some(dir) = entry.path().parent() else { + continue; + }; + match manifest_paths.entry(dir.to_path_buf()) { + std::collections::btree_map::Entry::Vacant(vacant) => { + vacant.insert((priority, entry.path().to_path_buf())); + } + std::collections::btree_map::Entry::Occupied(mut occupied) => { + if priority < occupied.get().0 { + occupied.insert((priority, entry.path().to_path_buf())); + } + } + } + } + + let mut discovered = Vec::with_capacity(manifest_paths.len()); + for (dir, (_priority, manifest_path)) in manifest_paths { + discovered.push(DiscoveredManifest { + dir, + peek: peek_manifest(&manifest_path)?, + }); + } + + // A workspace declaration below the root starts a nested workspace; its + // packages opt into publishing for that workspace, not for this one. + let nested_workspace_roots: Vec = discovered + .iter() + .filter(|manifest| manifest.peek.declares_workspace && manifest.dir != workspace_root) + .map(|manifest| manifest.dir.clone()) + .collect(); + + let mut members = BTreeSet::new(); + for manifest in discovered { + if manifest.peek.publish != Some(true) { + continue; + } + if nested_workspace_roots + .iter() + .any(|nested_root| manifest.dir.starts_with(nested_root)) + { + continue; + } + let Some(relative) = pathdiff::diff_paths(&manifest.dir, workspace_root) else { + continue; + }; + members.insert(member_dir_from_relative_path(&relative)); + } + + Ok(members) +} + +/// Canonical member-directory form of a workspace-root-relative filesystem +/// path. The workspace root itself is represented as `.`. +fn member_dir_from_relative_path(path: &Path) -> String { + normalize_member_dir(native_relative_path_to_member(path)) +} + +/// Convert a relative filesystem path into the forward-slash form used to +/// identify members, so the same directory referenced through a manifest path +/// spec and through a filesystem path (backslash-separated on Windows) +/// compares equal. +fn native_relative_path_to_member(path: &Path) -> Utf8TypedPathBuf { + Utf8TypedPathBuf::from( + path.to_string_lossy() + .replace(std::path::MAIN_SEPARATOR, "/"), + ) +} + +/// Determine whether a workspace-anchored source location refers to a +/// directory inside the workspace or to something external. +fn classify_source_location(workspace_root: &Path, location: SourceLocationSpec) -> SourceTarget { + let SourceLocationSpec::Path(path_spec) = &location else { + return SourceTarget::External(location.to_string()); + }; + + let path = path_spec.path.to_path(); + if path.is_absolute() || path.starts_with("~") { + // Absolute paths can still point into the workspace; try to re-anchor + // them. `~` cannot be resolved without the filesystem, treat it as + // external. + let std_path = Path::new(path_spec.path.as_str()); + let (Ok(canonical_path), Ok(canonical_root)) = ( + dunce::canonicalize(std_path), + dunce::canonicalize(workspace_root), + ) else { + return SourceTarget::External(location.to_string()); + }; + return match canonical_path.strip_prefix(&canonical_root) { + Ok(relative) => SourceTarget::Member(member_dir_for( + workspace_root, + native_relative_path_to_member(relative), + )), + Err(_) => SourceTarget::External(location.to_string()), + }; + } + + // Workspace-relative path. A normalized path that still starts with `..` + // escapes the workspace root. + let as_str = path_spec.path.as_str(); + if as_str == ".." || as_str.starts_with("../") || as_str.starts_with("..\\") { + return SourceTarget::External(location.to_string()); + } + + SourceTarget::Member(member_dir_for(workspace_root, path_spec.path.clone())) +} + +/// Reduce a workspace-relative source path to the directory that identifies +/// the member package: paths that point at a manifest file are replaced by +/// their parent directory so that `pkg/pixi.toml` and `pkg` name the same +/// member. +fn member_dir_for(workspace_root: &Path, path: Utf8TypedPathBuf) -> String { + let is_file = workspace_root.join(path.as_str()).is_file(); + let dir = if is_file { + path.to_path() + .parent() + .map(|parent| parent.to_path_buf()) + .unwrap_or(path) + } else { + path + }; + normalize_member_dir(dir) +} + +/// Canonical string form of a member directory. The workspace root itself is +/// represented as `.`. +fn normalize_member_dir(dir: Utf8TypedPathBuf) -> String { + let dir = dir.as_str().trim_end_matches(['/', '\\']); + if dir.is_empty() || dir == "." { + ".".to_string() + } else { + dir.to_string() + } +} + +/// The result of ordering members by their dependencies. +pub(super) struct DependencyOrdering { + /// Members with dependencies before their dependents. + pub order: Vec, + + /// Members that were forced out of a dependency cycle. When this is + /// non-empty the order is best-effort: a forced member is emitted before + /// its dependencies are complete. + pub cycle_members: Vec, +} + +/// Order members so that dependencies come before their dependents. Should a +/// dependency cycle exist, one member of the cycle is forced out instead of +/// failing; members that merely depend on the cycle still come after it. The +/// forced members are reported so the caller can warn the user. +/// +/// Also used by the publish flow to order the outputs of a single +/// multi-output package. +pub(super) fn dependency_order( + members: &BTreeSet, + dependencies: &BTreeMap>, +) -> DependencyOrdering { + let mut order = Vec::with_capacity(members.len()); + let mut cycle_members = Vec::new(); + let mut emitted: BTreeSet = BTreeSet::new(); + let mut remaining: BTreeSet = members.clone(); + + while !remaining.is_empty() { + let ready: Vec = remaining + .iter() + .filter(|member| { + dependencies.get(*member).is_none_or(|deps| { + deps.iter() + .all(|dep| emitted.contains(dep) || !members.contains(dep)) + }) + }) + .cloned() + .collect(); + + if ready.is_empty() { + let member = find_cycle_member(&remaining, dependencies); + remaining.remove(&member); + emitted.insert(member.clone()); + cycle_members.push(member.clone()); + order.push(member); + continue; + } + + for member in ready { + remaining.remove(&member); + emitted.insert(member.clone()); + order.push(member); + } + } + + DependencyOrdering { + order, + cycle_members, + } +} + +/// Pick a member that lies on a dependency cycle among `remaining`. +/// +/// Walks unmet dependencies (always the lexicographically first one, for +/// determinism) starting from the first remaining member; the first member +/// visited twice is on a cycle. Only called when no member is ready, which +/// guarantees every walk eventually revisits a member. +fn find_cycle_member( + remaining: &BTreeSet, + dependencies: &BTreeMap>, +) -> String { + let mut current = remaining + .first() + .expect("caller ensures remaining is not empty") + .clone(); + let mut visited: BTreeSet = BTreeSet::new(); + visited.insert(current.clone()); + loop { + let next = dependencies + .get(¤t) + .and_then(|deps| deps.iter().find(|dep| remaining.contains(*dep))) + .cloned(); + match next { + // No unmet dependency: the walk cannot continue, treat the + // current member as the one to force out. + None => return current, + Some(next) => { + if !visited.insert(next.clone()) { + return next; + } + current = next; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn set(items: &[&str]) -> BTreeSet { + items.iter().map(|s| s.to_string()).collect() + } + + fn write_manifest(root: &Path, dir: &str, contents: &str) { + let dir = root.join(dir); + fs_err::create_dir_all(&dir).unwrap(); + fs_err::write(dir.join("pixi.toml"), contents).unwrap(); + } + + #[test] + fn dependency_order_puts_dependencies_first() { + let members = set(&["kit", "core", "cpp"]); + let mut deps = BTreeMap::new(); + deps.insert("kit".to_string(), set(&["cpp"])); + deps.insert("cpp".to_string(), set(&["core"])); + + let ordering = dependency_order(&members, &deps); + assert_eq!(ordering.order, vec!["core", "cpp", "kit"]); + assert!(ordering.cycle_members.is_empty()); + } + + #[test] + fn dependency_order_ignores_non_member_dependencies() { + let members = set(&["a"]); + let mut deps = BTreeMap::new(); + deps.insert("a".to_string(), set(&["not-a-member"])); + + let ordering = dependency_order(&members, &deps); + assert_eq!(ordering.order, vec!["a"]); + assert!(ordering.cycle_members.is_empty()); + } + + #[test] + fn dependency_order_survives_cycles() { + let members = set(&["a", "b", "c"]); + let mut deps = BTreeMap::new(); + deps.insert("a".to_string(), set(&["b"])); + deps.insert("b".to_string(), set(&["a"])); + deps.insert("c".to_string(), set(&["a"])); + + let ordering = dependency_order(&members, &deps); + // One cycle member is forced out, then the rest resolves normally; + // `c` only depends on `a` and follows it. + assert_eq!(ordering.order, vec!["a", "b", "c"]); + assert_eq!(ordering.cycle_members, vec!["a"]); + } + + #[test] + fn dependency_order_emits_cycle_before_its_dependents() { + // `0-app` sorts before the cycle members but merely depends on the + // cycle; it must still be emitted after its dependency `a`. + let members = set(&["0-app", "a", "b"]); + let mut deps = BTreeMap::new(); + deps.insert("0-app".to_string(), set(&["a"])); + deps.insert("a".to_string(), set(&["b"])); + deps.insert("b".to_string(), set(&["a"])); + + let ordering = dependency_order(&members, &deps); + // `a` is forced out of the cycle first; `0-app` and `b` both become + // ready and follow in name order. + assert_eq!(ordering.order, vec!["a", "0-app", "b"]); + assert_eq!(ordering.cycle_members, vec!["a"]); + } + + #[test] + fn normalize_member_dir_maps_root_to_dot() { + assert_eq!(normalize_member_dir(Utf8TypedPathBuf::from("")), "."); + assert_eq!(normalize_member_dir(Utf8TypedPathBuf::from(".")), "."); + assert_eq!(normalize_member_dir(Utf8TypedPathBuf::from("pkg/")), "pkg"); + } + + #[test] + fn discovery_collects_only_packages_that_opt_in() { + let root = tempfile::tempdir().unwrap(); + let root_path = root.path(); + + write_manifest( + root_path, + ".", + "[workspace]\nchannels = []\n\n[package]\nname = \"root\"\npublish = true\n", + ); + write_manifest( + root_path, + "packages/foo", + "[package]\nname = \"foo\"\npublish = true\n", + ); + write_manifest( + root_path, + "packages/bar", + "[package]\nname = \"bar\"\npublish = false\n", + ); + write_manifest(root_path, "packages/baz", "[package]\nname = \"baz\"\n"); + + let members = discover_opted_in_packages(root_path).unwrap(); + assert_eq!(members, set(&[".", "packages/foo"])); + } + + #[test] + fn discovery_skips_nested_workspaces() { + let root = tempfile::tempdir().unwrap(); + let root_path = root.path(); + + write_manifest(root_path, ".", "[workspace]\nchannels = []\n"); + write_manifest( + root_path, + "packages/foo", + "[package]\nname = \"foo\"\npublish = true\n", + ); + write_manifest( + root_path, + "examples/demo", + "[workspace]\nchannels = []\n\n[package]\nname = \"demo\"\npublish = true\n", + ); + write_manifest( + root_path, + "examples/demo/member", + "[package]\nname = \"member\"\npublish = true\n", + ); + + let members = discover_opted_in_packages(root_path).unwrap(); + assert_eq!(members, set(&["packages/foo"])); + } + + #[test] + fn discovery_respects_ignore_files() { + let root = tempfile::tempdir().unwrap(); + let root_path = root.path(); + + write_manifest(root_path, ".", "[workspace]\nchannels = []\n"); + write_manifest( + root_path, + "packages/foo", + "[package]\nname = \"foo\"\npublish = true\n", + ); + write_manifest( + root_path, + "vendored/dep", + "[package]\nname = \"dep\"\npublish = true\n", + ); + fs_err::write(root_path.join(".gitignore"), "vendored/\n").unwrap(); + + let members = discover_opted_in_packages(root_path).unwrap(); + assert_eq!(members, set(&["packages/foo"])); + } + + #[test] + fn discovery_reads_pyproject_manifests() { + let root = tempfile::tempdir().unwrap(); + let root_path = root.path(); + + write_manifest(root_path, ".", "[workspace]\nchannels = []\n"); + let dir = root_path.join("packages/py"); + fs_err::create_dir_all(&dir).unwrap(); + fs_err::write( + dir.join("pyproject.toml"), + "[tool.pixi.package]\nname = \"py\"\npublish = true\n", + ) + .unwrap(); + + let members = discover_opted_in_packages(root_path).unwrap(); + assert_eq!(members, set(&["packages/py"])); + } + + #[test] + fn discovery_rejects_non_boolean_publish() { + let root = tempfile::tempdir().unwrap(); + let root_path = root.path(); + + write_manifest(root_path, ".", "[workspace]\nchannels = []\n"); + write_manifest( + root_path, + "packages/foo", + "[package]\nname = \"foo\"\npublish = \"yes\"\n", + ); + + let err = discover_opted_in_packages(root_path).unwrap_err(); + assert!(err.to_string().contains("must be a boolean")); + } +} diff --git a/crates/pixi_cli/src/publish.rs b/crates/pixi_cli/src/publish/mod.rs similarity index 82% rename from crates/pixi_cli/src/publish.rs rename to crates/pixi_cli/src/publish/mod.rs index 4e274e9fdc..81018f0e39 100644 --- a/crates/pixi_cli/src/publish.rs +++ b/crates/pixi_cli/src/publish/mod.rs @@ -37,11 +37,18 @@ use rattler_conda_types::{GenericVirtualPackage, Platform}; use rattler_networking::{AuthenticationStorage, s3_middleware}; use rattler_package_streaming::seek::read_package_file; -/// Build a conda package and publish it to a channel. +mod discovery; + +/// Build the conda packages of a workspace and publish them to a channel. /// -/// Builds the package from your workspace and either uploads it to a channel -/// (`--target-channel`) or copies the artifact into a local directory -/// (`--target-dir`). +/// Builds every package in the workspace that opts into publishing with +/// `publish = true` in its `[package]` section - in dependency order, so +/// packages that depend on other workspace packages are built after them - +/// and either uploads the artifacts to a channel (`--target-channel`) or +/// copies them into a local directory (`--target-dir`). Every source +/// dependency of a published package must itself opt into publishing; the +/// publish fails otherwise. Use `--path` to build and publish a single +/// self-contained package instead. /// /// Supported destinations for `--target-channel` (alias `--to`): /// - prefix.dev: `https://prefix.dev/` @@ -94,6 +101,10 @@ pub struct Args { /// The path to a directory containing a package manifest, or to a specific manifest file. /// + /// When given, only that package is built and published - whether or not + /// it sets `publish = true`. The package must be self-contained: + /// publishing it alone fails if it has any source dependencies. + /// /// Supported manifest files: `package.xml`, `recipe.yaml`, `pixi.toml`, `pyproject.toml`, or `mojoproject.toml`. #[arg(long)] pub path: Option, @@ -114,10 +125,20 @@ pub struct Args { #[arg(long)] pub force: bool, - /// Skip uploading packages that already exist at the target. - /// This is enabled by default. Use `--no-skip-existing` to disable. - #[arg(long, default_value_t = true, action = clap::ArgAction::Set)] - pub skip_existing: bool, + /// Do not skip packages that already exist at the target. + /// + /// Packages that already exist at the target are skipped by default. + /// With this flag they fail the publish instead, unless `--force` is + /// also given to overwrite them. + #[arg(long)] + pub no_skip_existing: bool, + + /// Resolve and print the publish set without building or uploading. + /// + /// Discovers the packages, validates that they form a self-contained + /// set, and prints them in the order they would be built and uploaded. + #[arg(long)] + pub dry_run: bool, /// Generate sigstore attestation (prefix.dev only) #[arg(long)] @@ -523,7 +544,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { let mut workspace = WorkspaceLocator::for_cli() .with_global_config_source(args.config_source.source()) .with_search_start(workspace_locator.clone()) - .with_closest_package(false) + .with_closest_package(true) .locate()? .with_cli_config(args.config_cli); if let Some(backend_override) = args.backend_override.clone() { @@ -532,10 +553,13 @@ pub async fn execute(args: Args) -> miette::Result<()> { sanity_check_workspace(&workspace).await?; + // `--force` requests an overwrite, so existing packages must not be + // skipped even though skipping is the default. + let skip_existing = !args.no_skip_existing && !args.force; let ctx = PublishContext::new( &workspace, args.force, - args.skip_existing, + skip_existing, args.generate_attestation, )?; @@ -633,70 +657,162 @@ pub async fn execute(args: Args) -> miette::Result<()> { host_virtual_packages, }; - let Ok(manifest_path) = workspace_locator.path() else { - miette::bail!("could not determine the current working directory to locate the workspace"); + let channel_config = workspace.channel_config(); + let channels = workspace + .default_environment() + .channel_urls(&channel_config) + .into_diagnostic()?; + + // When running `pixi publish`, the exclude_newer config is ignored; + // it only matters when using the package as a source dependency. + let make_env_ref = |manifest_source: &PinnedSourceSpec| { + EnvironmentRef::Ephemeral(EphemeralEnv::new( + manifest_source.to_string(), + EnvironmentSpec { + channels: channels.clone(), + build_environment: build_environment.clone(), + variants: pixi_utils::variants::VariantConfig { + variant_configuration: variant_configuration.clone(), + variant_files: variant_files.clone(), + }, + exclude_newer: None, + channel_priority: Default::default(), + }, + )) + }; + let make_metadata_spec = |manifest_source: PinnedSourceSpec| { + let env_ref = make_env_ref(&manifest_source); + BuildBackendMetadataSpec { + manifest_source, + preferred_build_source: None, + env_ref, + build_string_prefix: args.build_string_prefix.clone(), + build_number: args.build_number, + inline: None, + } }; - let package_manifest_path = match args.path { + // Determine which packages to publish: a single one when `--path` is + // given, otherwise every package in the workspace that opts in with + // `publish = true`, ordered so that dependencies are built and uploaded + // before their dependents. + let single_package_mode = args.path.is_some(); + let (package_sources, cycle_members) = match &args.path { Some(path) => { - validate_package_manifest(&path).await?; - path + validate_package_manifest(path).await?; + let package_manifest_path_canonical = dunce::canonicalize(path) + .into_diagnostic() + .with_context(|| { + format!("failed to canonicalize manifest path '{}'", path.display()) + })?; + let manifest_path_spec = + pathdiff::diff_paths(&package_manifest_path_canonical, workspace.root()) + .unwrap_or_else(|| package_manifest_path_canonical.clone()); + let manifest_source: PinnedSourceSpec = PinnedPathSpec { + path: manifest_path_spec.to_string_lossy().into_owned().into(), + } + .into(); + (vec![manifest_source], Vec::new()) + } + None => { + let resolved = discovery::resolve_publish_set( + &workspace, + &command_dispatcher, + &make_metadata_spec, + ) + .await?; + (resolved.packages, resolved.cycle_members) } - None => manifest_path.clone(), }; - let package_manifest_path_canonical = dunce::canonicalize(&package_manifest_path) - .into_diagnostic() - .with_context(|| { - format!( - "failed to canonicalize manifest path '{}'", - package_manifest_path.display() - ) - })?; + if !cycle_members.is_empty() { + pixi_progress::println!( + "{}dependency cycle among workspace packages involving {}; uploads cannot be fully \ + dependency-ordered, so the target may be inconsistent until every upload finishes", + console::style(console::Emoji("âš ī¸ ", "warning: ")).yellow(), + cycle_members.join(", "), + ); + } - let manifest_path_spec = - pathdiff::diff_paths(&package_manifest_path_canonical, workspace.root()) - .unwrap_or_else(|| package_manifest_path_canonical.to_path_buf()); + // Fetch the backend metadata of every package. For packages that were + // just resolved from the publish set this is a cache hit. Packages + // without any output for the target platform are skipped. + let mut package_plans = Vec::with_capacity(package_sources.len()); + for manifest_source in package_sources { + let backend_metadata = command_dispatcher + .build_backend_metadata(make_metadata_spec(manifest_source.clone())) + .await?; + if backend_metadata.metadata.outputs.is_empty() { + pixi_progress::println!( + "{}skipping '{}': no outputs for platform {}", + console::style(console::Emoji("â„šī¸ ", "")).blue(), + manifest_source, + args.target_platform, + ); + continue; + } + package_plans.push((manifest_source, backend_metadata)); + } - let channel_config = workspace.channel_config(); - let channels = workspace - .default_environment() - .channel_urls(&channel_config) - .into_diagnostic()?; + if package_plans.is_empty() { + miette::bail!( + "no package produces outputs for platform {}", + args.target_platform + ); + } - let manifest_source: PinnedSourceSpec = PinnedPathSpec { - path: manifest_path_spec.to_string_lossy().into_owned().into(), + // A single-package publish is a batch of one, so the closure rule demands + // that it has no source dependencies beyond its own outputs: nothing else + // in the batch could satisfy them on the target. Dependencies on sibling + // outputs (e.g. through `pin_subpackage`) are published together with the + // package and are therefore fine. + if single_package_mode { + for (manifest_source, backend_metadata) in &package_plans { + let output_names: BTreeSet = backend_metadata + .metadata + .outputs + .iter() + .map(|output| output.metadata.name.as_normalized().to_string()) + .collect(); + let source_dependencies: BTreeSet<&str> = backend_metadata + .metadata + .outputs + .iter() + .flat_map(discovery::output_source_dependencies) + .map(|(name, _)| name) + .filter(|name| !output_names.contains(&name.to_lowercase())) + .collect(); + if !source_dependencies.is_empty() { + return Err(miette::diagnostic!( + help = "A single-package publish must be self-contained. Set \ + `publish = true` in the `[package]` section of the package and \ + its source dependencies, then run `pixi publish` without `--path` \ + to publish them together.", + "package '{}' has source dependencies ({}) and cannot be published on its own", + manifest_source, + source_dependencies + .iter() + .copied() + .collect::>() + .join(", "), + ) + .into()); + } + } } - .into(); - // When running `pixi publish`, the exclude_newer config is ignored; - // it only matters when using the package as a source dependency. - let env_ref = EnvironmentRef::Ephemeral(EphemeralEnv::new( - manifest_source.to_string(), - EnvironmentSpec { - channels: channels.clone(), - build_environment: build_environment.clone(), - variants: pixi_utils::variants::VariantConfig { - variant_configuration: variant_configuration.clone(), - variant_files: variant_files.clone(), - }, - exclude_newer: None, - channel_priority: Default::default(), - }, - )); - let backend_metadata_spec = BuildBackendMetadataSpec { - manifest_source: manifest_source.clone(), - preferred_build_source: None, - env_ref: env_ref.clone(), - build_string_prefix: args.build_string_prefix.clone(), - build_number: args.build_number, - inline: None, - }; - let backend_metadata = command_dispatcher - .build_backend_metadata(backend_metadata_spec.clone()) - .await?; + if package_plans.len() > 1 { + pixi_progress::println!( + "\n{}Publishing {} workspace packages that set `publish = true`", + console::style(console::Emoji("🔍 ", "")).cyan(), + package_plans.len(), + ); + } - let packages = &backend_metadata.metadata.outputs; + let packages: Vec<_> = package_plans + .iter() + .flat_map(|(_, backend_metadata)| backend_metadata.metadata.outputs.iter()) + .collect(); // The CondaOutput metadata uses `pixi_build_types::VariantValue`, while the // rest of the publish flow (and our helpers) work with the @@ -724,8 +840,13 @@ pub async fn execute(args: Args) -> miette::Result<()> { // Print initial build summary pixi_progress::println!( - "\n{}Building {} package(s):", + "\n{}{} {} package(s):", console::style(console::Emoji("📋 ", "")).cyan(), + if args.dry_run { + "Would build" + } else { + "Building" + }, packages.len() ); for (pkg, variants) in packages.iter().zip(&pkg_variant_maps_owned) { @@ -740,35 +861,106 @@ pub async fn execute(args: Args) -> miette::Result<()> { } pixi_progress::println!(""); + // A dry run stops after the publish set has been resolved, validated, + // and printed in build order. + if args.dry_run { + pixi_progress::println!( + "{}Dry run: nothing was built or uploaded", + console::style(console::Emoji("â„šī¸ ", "")).blue(), + ); + return Ok(()); + } + // Pre-resolve a SourceRecord per unique package name via RSP; each - // returned variant becomes a separate SourceBuildKey invocation. - let unique_names: BTreeSet<_> = packages.iter().map(|p| p.metadata.name.clone()).collect(); - let source_location: SourceLocationSpec = manifest_source.clone().into(); + // returned variant becomes a separate SourceBuildKey invocation. The + // records keep the dependency order of `package_plans`, so dependencies + // are built and uploaded before the packages that depend on them. let mut resolved_records = Vec::new(); - for name in unique_names { - let rsp = ResolveSourcePackageSpec { - package: name, - source_location: source_location.clone(), - preferred_build_source: Arc::new(BTreeMap::new()), - env_ref: env_ref.clone(), - inline: None, - installed_source_hints: Default::default(), - }; - let records = command_dispatcher - .engine() - .compute(&ResolveSourcePackageKey::new(rsp)) - .await - .map_err_into_dispatcher(std::convert::identity) - .into_diagnostic()?; - resolved_records.extend(records.iter().cloned()); + for (manifest_source, backend_metadata) in &package_plans { + let unique_names: BTreeSet<_> = backend_metadata + .metadata + .outputs + .iter() + .map(|p| p.metadata.name.clone()) + .collect(); + + // Order the outputs of this package so that an output consumed by a + // sibling output (e.g. through `pin_subpackage`) is built and + // uploaded before its dependents; iterating `unique_names` directly + // would order them alphabetically. + let output_names: BTreeSet = unique_names + .iter() + .map(|name| name.as_normalized().to_string()) + .collect(); + let mut output_dependencies: BTreeMap> = BTreeMap::new(); + for output in &backend_metadata.metadata.outputs { + let name = output.metadata.name.as_normalized().to_string(); + let sibling_deps: BTreeSet = output + .build_dependencies + .iter() + .chain(output.host_dependencies.iter()) + .chain(std::iter::once(&output.run_dependencies)) + .flat_map(|deps| deps.depends.iter()) + .map(|named| named.name.as_str().to_lowercase()) + .filter(|dep| *dep != name && output_names.contains(dep)) + .collect(); + if !sibling_deps.is_empty() { + output_dependencies + .entry(name) + .or_default() + .extend(sibling_deps); + } + } + let mut by_normalized_name: BTreeMap = unique_names + .into_iter() + .map(|name| (name.as_normalized().to_string(), name)) + .collect(); + + let source_location: SourceLocationSpec = manifest_source.clone().into(); + let env_ref = make_env_ref(manifest_source); + let output_ordering = discovery::dependency_order(&output_names, &output_dependencies); + if !output_ordering.cycle_members.is_empty() { + pixi_progress::println!( + "{}dependency cycle among the outputs of '{}' involving {}; uploads cannot be \ + fully dependency-ordered", + console::style(console::Emoji("âš ī¸ ", "warning: ")).yellow(), + manifest_source, + output_ordering.cycle_members.join(", "), + ); + } + for name in output_ordering.order { + let name = by_normalized_name + .remove(&name) + .expect("dependency_order returns exactly the members it was given"); + let rsp = ResolveSourcePackageSpec { + package: name, + source_location: source_location.clone(), + preferred_build_source: Arc::new(BTreeMap::new()), + env_ref: env_ref.clone(), + inline: None, + installed_source_hints: Default::default(), + }; + let records = command_dispatcher + .engine() + .compute(&ResolveSourcePackageKey::new(rsp)) + .await + .map_err_into_dispatcher(std::convert::identity) + .into_diagnostic()?; + resolved_records.extend(records.iter().cloned()); + } } // `--clean` nukes the per-package artifact + workspace caches so - // the upcoming SourceBuildKey calls rebuild from scratch. + // the upcoming SourceBuildKey calls rebuild from scratch. Clear before + // any build starts so a build never invalidates an earlier one. if args.clean { - for record in &resolved_records { + let clean_names: BTreeSet<_> = resolved_records + .iter() + .map(|record| record.data.package_record.name.clone()) + .collect(); + for name in clean_names { command_dispatcher - .clear_source_build_cache(&record.data.package_record.name) + .clear_source_build_cache(&name) .into_diagnostic()?; } } @@ -777,6 +969,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { // built with, so the publish summary can attribute every artifact back to // the variant that produced it. let mut built_packages: Vec<(PathBuf, BTreeMap)> = Vec::new(); + let mut seen_artifacts: BTreeSet = BTreeSet::new(); for record in resolved_records { let record = Arc::unwrap_or_clone(record); @@ -809,7 +1002,9 @@ pub async fn execute(args: Args) -> miette::Result<()> { let package_path = dunce::canonicalize(&built.artifact) .expect("failed to canonicalize output file which must now exist"); - built_packages.push((package_path, variants)); + if seen_artifacts.insert(package_path.clone()) { + built_packages.push((package_path, variants)); + } } // Release the repodata gateway before indexing. It memory-maps the target @@ -1337,7 +1532,9 @@ async fn upload_to_s3( write_shards: true, repodata_revisions: vec![], package_revision_assignment: Default::default(), - force: false, + // `--force` may have replaced an existing artifact under its old + // filename; only a forced index re-extracts its metadata. + force: ctx.force, max_parallel: std::thread::available_parallelism() .map(|p| p.get()) .unwrap_or(1), @@ -1414,7 +1611,9 @@ async fn upload_to_local_filesystem_channel( write_shards: true, repodata_revisions: vec![], package_revision_assignment: Default::default(), - force: false, + // `--force` may have replaced an existing artifact under its old + // filename; only a forced index re-extracts its metadata. + force: ctx.force, max_parallel: std::thread::available_parallelism() .map(|p| p.get()) .unwrap_or(1), diff --git a/crates/pixi_command_dispatcher/src/cache/artifact.rs b/crates/pixi_command_dispatcher/src/cache/artifact.rs index ad75744e71..96e56996a8 100644 --- a/crates/pixi_command_dispatcher/src/cache/artifact.rs +++ b/crates/pixi_command_dispatcher/src/cache/artifact.rs @@ -41,6 +41,7 @@ use rattler_digest::Sha256Hash; use serde::{Deserialize, Serialize}; use serde_with::serde_as; use thiserror::Error; +use url::Url; use xxhash_rust::xxh3::Xxh3; /// Opaque content-addressed handle for an artifact cache entry. @@ -332,10 +333,16 @@ impl ArtifactCache { return Ok(None); } + // Point the record at the artifact being returned. Sidecars written + // by older versions carry the work-directory output path, which does + // not survive workspace-cache cleanups. + let mut record = sidecar.record; + record.url = Url::from_file_path(&artifact_path).expect("cache entry paths are absolute"); + Ok(Some(CachedArtifact { artifact: artifact_path, sha256: sidecar.artifact_sha256, - record: sidecar.record, + record, })) } @@ -358,7 +365,7 @@ impl ArtifactCache { artifact_source: &Path, input_glob_sets: Vec, input_files: impl IntoIterator, - record: RepoDataRecord, + mut record: RepoDataRecord, ) -> Result { let entry_dir = self.entry_dir(package, key); let lock_file = self.open_lock_file(package, key).await?; @@ -386,6 +393,12 @@ impl ArtifactCache { ArtifactCacheError::io("copying artifact into cache", dest.clone(), err) })?; + // The record was synthesized from the freshly-built artifact and + // points at its work-directory location. Persist and return the + // cached copy instead; the work directory is mutable and may be + // cleaned before the record is consumed again. + record.url = Url::from_file_path(&dest).expect("cache entry paths are absolute"); + let sha256 = { let path = dest.clone(); tokio::task::spawn_blocking(move || { @@ -668,9 +681,10 @@ mod tests { #[tokio::test] async fn sidecar_preserves_record_fields_across_lookup() { // Verify that every RepoDataRecord field we care about - // (identifier, url, package_record.version, .build, .subdir, - // .depends) round-trips through the sidecar. A future refactor - // that accidentally drops a field will fail this test. + // (identifier, package_record.version, .build, .subdir, .depends) + // round-trips through the sidecar, and that the url is rewritten to + // the cached artifact. A future refactor that accidentally drops a + // field will fail this test. let tmp = TempDir::new().unwrap(); let cache = ArtifactCache::new(tmp.path().join("artifacts")); let engine = ComputeEngine::new(); @@ -722,8 +736,13 @@ mod tests { record.package_record.depends ); - // RepoDataRecord top-level fields. - assert_eq!(hit.record.url, record.url); + // RepoDataRecord top-level fields. The url always points at the + // cached artifact, replacing whatever location the record carried + // when it was stored. + assert_eq!( + hit.record.url, + url::Url::from_file_path(&hit.artifact).unwrap() + ); assert_eq!(hit.record.identifier, record.identifier); } diff --git a/crates/pixi_command_dispatcher/src/keys/source_build.rs b/crates/pixi_command_dispatcher/src/keys/source_build.rs index 2be1efb814..57465c3504 100644 --- a/crates/pixi_command_dispatcher/src/keys/source_build.rs +++ b/crates/pixi_command_dispatcher/src/keys/source_build.rs @@ -140,10 +140,20 @@ async fn compute_inner( ctx: &mut ComputeCtx, spec: Arc, ) -> Result { - // sha256s are collected in a stable (build, host) order so the - // artifact cache key stays deterministic across buckets. - let (build_source_dep_sha256s, host_source_dep_sha256s) = + // Results are collected in a stable (build, host) order so the + // artifact cache key stays deterministic across buckets. The full + // build results feed the prefix installs below so the source deps + // are not built a second time through the installer. + let (build_source_dep_results, host_source_dep_results) = recurse_source_deps(ctx, &spec).await?; + let build_source_dep_sha256s: Vec = build_source_dep_results + .iter() + .map(|result| result.artifact_sha256) + .collect(); + let host_source_dep_sha256s: Vec = host_source_dep_results + .iter() + .map(|result| result.artifact_sha256) + .collect(); let manifest_source = spec.record.manifest_source.clone(); let manifest_checkout = ctx @@ -277,11 +287,13 @@ async fn compute_inner( // could dedup. let output = fetch_matching_output(&backend, &spec, &work_directory).await?; - // install_prefix recurses into source entries via SourceBuildKey, - // so build_records / host_records are all binaries on disk. Build - // and host packages come pre-resolved on the input record (v7+ - // lock file), so the two installs are independent and can run - // concurrently. + // Source entries were already built by `recurse_source_deps`; hand + // their binary records to `install_prefix` so the installer links + // them instead of triggering another build. Build and host packages + // come pre-resolved on the input record (v7+ lock file), so the two + // installs are independent and can run concurrently. + let build_source_dep_records = records_by_name(&build_source_dep_results); + let host_source_dep_records = records_by_name(&host_source_dep_results); let directories = Directories::new(&work_directory, spec.build_environment.host_platform); let ((build_records, _build_install_result), (host_records, _host_install_result)) = ctx .try_compute2( @@ -292,6 +304,7 @@ async fn compute_inner( InstallTarget::Build, directories.build_prefix.clone(), spec.record.build_packages.clone(), + &build_source_dep_records, ) .await }, @@ -302,6 +315,7 @@ async fn compute_inner( InstallTarget::Host, directories.host_prefix.clone(), spec.record.host_packages.clone(), + &host_source_dep_records, ) .await }, @@ -476,14 +490,30 @@ async fn compute_inner( }) } +/// Index source-dependency build results by package name for the prefix +/// installs. +fn records_by_name( + results: &[Arc], +) -> std::collections::HashMap> { + results + .iter() + .map(|result| { + ( + result.record.package_record.name.clone(), + Arc::new(result.record.clone()), + ) + }) + .collect() +} + /// Fan out over every source entry in `build_packages` and /// `host_packages`, recursively build each via [`SourceBuildKey`], and -/// return their sha256s split by bucket. The two buckets feed into the +/// return the build results split by bucket. The buckets feed into the /// cache key separately so a dep moving build ↔ host invalidates. async fn recurse_source_deps( ctx: &mut ComputeCtx, spec: &Arc, -) -> Result<(Vec, Vec), SourceBuildError> { +) -> Result<(Vec>, Vec>), SourceBuildError> { // build_packages run on the build platform. The nested build's // HOST platform is therefore the outer's BUILD platform. let build = build_source_deps( @@ -511,7 +541,7 @@ async fn build_source_deps( spec: Arc, packages: Vec, nested_build_environment: BuildEnvironment, -) -> Result, SourceBuildError> { +) -> Result>, SourceBuildError> { let sources: Vec> = packages .into_iter() .filter_map(|r| match r { @@ -527,7 +557,7 @@ async fn build_source_deps( let build_env = nested_build_environment; async move |sub_ctx: &mut ComputeCtx, src: Arc| - -> Result { + -> Result, SourceBuildError> { let nested_spec = SourceBuildSpec { record: src, channels: spec.channels.clone(), @@ -541,15 +571,18 @@ async fn build_source_deps( // dependency closure builds against consistent values. build_string_prefix: spec.build_string_prefix.clone(), build_number: spec.build_number, - // Nested source deps are unpacked into the parent's - // prefix immediately; use the cheapest compression. - package_format: Some(CondaPackageFormat::fast()), + // The format is inherited too, so a package that is both + // built in its own right and consumed as a source + // dependency (e.g. two members of one workspace during + // `pixi publish`) shares a single cache entry instead of + // being built once per format. + package_format: spec.package_format, // A nested source dependency carries its own on-disk manifest; // inline definitions apply only to the consumer's direct deps. inline: None, }; let result = sub_ctx.compute(&SourceBuildKey::new(nested_spec)).await?; - Ok(result.artifact_sha256) + Ok(result) } }; ctx.try_compute_join(sources, mapper).await @@ -638,6 +671,10 @@ async fn install_prefix( target: InstallTarget, prefix_path: PathBuf, packages: Vec, + resolved_source_deps: &std::collections::HashMap< + rattler_conda_types::PackageName, + Arc, + >, ) -> Result< ( Vec, @@ -662,9 +699,31 @@ async fn install_prefix( InstallTarget::Build => format!("{} (build)", spec.record.name().as_source()), InstallTarget::Host => format!("{} (host)", spec.record.name().as_source()), }; + // Substitute every source entry with the binary record its build + // produced. Handing the installer the pre-built binaries keeps it + // from launching another build of the same package with different + // build parameters (a different package format, notably). + let mut records = Vec::with_capacity(packages.len()); + let install_records = packages + .into_iter() + .map(|record| match record { + UnresolvedPixiRecord::Binary(binary) => { + records.push((*binary).clone()); + UnresolvedPixiRecord::Binary(binary) + } + UnresolvedPixiRecord::Source(source) => { + let built = resolved_source_deps + .get(source.name()) + .expect("source dependency should have been built by recurse_source_deps") + .clone(); + records.push((*built).clone()); + UnresolvedPixiRecord::Binary(built) + } + }) + .collect(); let install_spec = InstallPixiEnvironmentSpec { name: label, - records: packages.clone(), + records: install_records, prefix, installed: None, ignore_packages: None, @@ -687,25 +746,6 @@ async fn install_prefix( }) .map_err(unwrap_dispatcher_err)?; - // Collect the RepoDataRecords that were installed: binaries pass - // through, sources come from the resolved_source_records map the - // ctx install just populated. - let mut records = Vec::with_capacity(packages.len()); - for r in packages { - match r { - UnresolvedPixiRecord::Binary(rec) => records.push((*rec).clone()), - UnresolvedPixiRecord::Source(src) => { - let built = result - .resolved_source_records - .get(src.name()) - .cloned() - .expect( - "source package should have been built by ctx.install_pixi_environment", - ); - records.push((*built).clone()); - } - } - } Ok((records, Some(result))) } diff --git a/crates/pixi_glob/Cargo.toml b/crates/pixi_glob/Cargo.toml index 9bbb8e3eda..1dd1d4510f 100644 --- a/crates/pixi_glob/Cargo.toml +++ b/crates/pixi_glob/Cargo.toml @@ -12,7 +12,7 @@ version = "0.1.0" [dependencies] dashmap = { workspace = true } fs-err = { workspace = true } -ignore = "0.4" +ignore = { workspace = true } itertools = { workspace = true } memchr = { workspace = true } parking_lot = { workspace = true } diff --git a/crates/pixi_manifest/src/package.rs b/crates/pixi_manifest/src/package.rs index f2425847a7..3f71efed97 100644 --- a/crates/pixi_manifest/src/package.rs +++ b/crates/pixi_manifest/src/package.rs @@ -35,6 +35,10 @@ pub struct Package { /// URL of the project documentation pub documentation: Option, + + /// Whether a workspace-wide `pixi publish` publishes this package. + /// Packages that do not opt in are left out of the publish set. + pub publish: bool, } impl Package { @@ -51,6 +55,7 @@ impl Package { homepage: None, repository: None, documentation: None, + publish: false, } } } diff --git a/crates/pixi_manifest/src/toml/package.rs b/crates/pixi_manifest/src/toml/package.rs index e50244683b..ecf7b479a5 100644 --- a/crates/pixi_manifest/src/toml/package.rs +++ b/crates/pixi_manifest/src/toml/package.rs @@ -141,6 +141,7 @@ pub struct TomlPackage { pub documentation: Option>, // Fields that are package-specific and cannot be inherited + pub publish: Option, pub build: TomlPackageBuild, pub host_dependencies: Option>, pub build_dependencies: Option>, @@ -191,6 +192,7 @@ impl<'de> toml_span::Deserialize<'de> for TomlPackage { .map(TomlWith::into_inner) .unwrap_or_default(); let run_constraints = th.optional("run-constraints"); + let publish = th.optional("publish"); let build = th.required("build")?; let target = th .optional::>>("target") @@ -214,6 +216,7 @@ impl<'de> toml_span::Deserialize<'de> for TomlPackage { run_dependencies, extra_dependencies, run_constraints, + publish, build, target, span: value.span, @@ -612,6 +615,7 @@ impl TomlPackage { package_defaults.documentation, "documentation", )?, + publish: self.publish.unwrap_or(false), }, build: build_result.value, dependencies: default_package_target, diff --git a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__package__test__invalid_extra_key.snap b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__package__test__invalid_extra_key.snap index 63ed0126b1..b8ce7d889c 100644 --- a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__package__test__invalid_extra_key.snap +++ b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__package__test__invalid_extra_key.snap @@ -1,9 +1,10 @@ --- source: crates/pixi_manifest/src/toml/package.rs +assertion_line: 811 expression: "expect_parse_failure(r#\"\n foo = \"bar\"\n name = \"bla\"\n extra = \"key\"\n\n [build]\n backend = { name = \"bla\", version = \"1.0\" }\n \"#,)" --- × Unexpected keys, expected only 'name', 'version', 'description', 'authors', 'license', 'license-file', 'readme', 'homepage', 'repository', 'documentation', 'host-dependencies', 'build- - │ dependencies', 'run-dependencies', 'extra-dependencies', 'run-constraints', 'build', 'target' + │ dependencies', 'run-dependencies', 'extra-dependencies', 'run-constraints', 'publish', 'build', 'target' ╭─[pixi.toml:2:9] 1 │ 2 │ foo = "bar" diff --git a/docs/build/workspace.md b/docs/build/workspace.md index 508471afcd..981c97abf4 100644 --- a/docs/build/workspace.md +++ b/docs/build/workspace.md @@ -103,6 +103,31 @@ each member opt in per entry with `{ workspace = true }`. See [Workspace Dependencies](workspace_dependencies.md) for the syntax, override rules, and error semantics. +## Publishing the Workspace + +To publish a workspace's packages, opt each of them in with `publish = true` +in its `[package]` section: + +```toml title="packages/cpp_math/pixi.toml" +[package] +name = "cpp_math" +publish = true +``` + +`pixi publish` walks the workspace directory tree, finds every package that +opts in, and builds and uploads them in dependency order. +The discovery respects ignore files such as `.gitignore` and skips +subdirectories that contain their own workspace. +The set must be self-contained: every source dependency of a published +package has to opt in as well, and `pixi publish` fails otherwise. +This guarantees that the target channel never ends up with a package whose +dependencies were not uploaded. +Use `pixi publish --dry-run` to see which packages would be published, in +which order, without building or uploading anything. + +See [`pixi publish`](../reference/cli/pixi/publish.md) for the full +behavior, including single-package publishes with `--path`. + ## Conclusion In this tutorial, we created a Pixi workspace containing two packages. diff --git a/docs/reference/cli/pixi.md b/docs/reference/cli/pixi.md index e6b8c7bb9c..cb029113e0 100644 --- a/docs/reference/cli/pixi.md +++ b/docs/reference/cli/pixi.md @@ -28,7 +28,7 @@ pixi [OPTIONS] [COMMAND] | [`list`](pixi/list.md) | List the packages of the current workspace | | [`lock`](pixi/lock.md) | Solve environment and update the lock file without installing the environments | | [`reinstall`](pixi/reinstall.md) | Re-install an environment, both updating the lock file and re-installing the environment | -| [`publish`](pixi/publish.md) | Build a conda package and publish it to a channel. | +| [`publish`](pixi/publish.md) | Build the conda packages of a workspace and publish them to a channel. | | [`remove`](pixi/remove.md) | Removes dependencies from the workspace | | [`run`](pixi/run.md) | Runs task in the pixi environment | | [`search`](pixi/search.md) | Search a conda package | diff --git a/docs/reference/cli/pixi/publish.md b/docs/reference/cli/pixi/publish.md index b88de238ba..8e08130303 100644 --- a/docs/reference/cli/pixi/publish.md +++ b/docs/reference/cli/pixi/publish.md @@ -4,7 +4,7 @@ title: pixi publish # [pixi](../pixi.md) publish -Build a conda package and publish it to a channel. +Build the conda packages of a workspace and publish them to a channel. --8<-- "docs/reference/cli/pixi/publish_extender:description" @@ -37,10 +37,10 @@ pixi publish [OPTIONS] : The local filesystem path to copy the built package(s) into (no channel indexing) - `--force` : Force overwrite existing packages -- `--skip-existing ` -: Skip uploading packages that already exist at the target. This is enabled by default. Use `--no-skip-existing` to disable -
**default**: `true` -
**options**: `true`, `false` +- `--no-skip-existing` +: Do not skip packages that already exist at the target +- `--dry-run` +: Resolve and print the publish set without building or uploading - `--generate-attestation` : Generate sigstore attestation (prefix.dev only) - `--variant ` @@ -96,11 +96,16 @@ pixi publish [OPTIONS] : Use environment activation cache (experimental) ## Description -Build a conda package and publish it to a channel. +Build the conda packages of a workspace and publish them to a channel. -Builds the package from your workspace and either uploads it to a channel -(`--target-channel`) or copies the artifact into a local directory -(`--target-dir`). +Builds every package in the workspace that opts into publishing with +`publish = true` in its `[package]` section - in dependency order, so +packages that depend on other workspace packages are built after them - +and either uploads the artifacts to a channel (`--target-channel`) or +copies them into a local directory (`--target-dir`). Every source +dependency of a published package must itself opt into publishing; the +publish fails otherwise. Use `--path` to build and publish a single +self-contained package instead. Supported destinations for `--target-channel` (alias `--to`): - prefix.dev: `https://prefix.dev/` diff --git a/docs/reference/cli/pixi/publish_extender b/docs/reference/cli/pixi/publish_extender index 7fedbae79f..531db11dd2 100644 --- a/docs/reference/cli/pixi/publish_extender +++ b/docs/reference/cli/pixi/publish_extender @@ -1,7 +1,23 @@ --8<-- [start:description] -`pixi publish` **builds** a conda package from your workspace and **uploads** it to a channel or copies it into a directory. +`pixi publish` **builds** the conda packages of your workspace and **uploads** them to a channel or copies them into a directory. + +By default every workspace package that opts into publishing with `publish = true` in its `[package]` section is published: + +```toml +[package] +name = "my-package" +publish = true +``` + +Packages are discovered by walking the workspace directory tree. The walk respects ignore files such as `.gitignore` and skips subdirectories that contain their own workspace. Packages that do not set `publish = true` are left out of the publish set; if no package opts in, the publish fails. + +Packages are built and uploaded in dependency order, so a package is never uploaded before the workspace packages it depends on. The published set must be closed: every source dependency (build, host, or run) of a published package must itself opt into publishing. Depending on a package outside the publish set, on a git or url source, or on a path outside the workspace root fails the publish - intentionally, so a publish can never leave the target channel referencing packages that were not uploaded. Replace external source dependencies with binary dependencies to publish. + +Use `--dry-run` to resolve, validate, and print the publish set without building or uploading anything. + +Use `--path ` to build and publish a single package instead - whether or not it sets `publish = true`. A `--path` publish is a batch of one, so the package must be self-contained: it may not have any source dependencies. - With `--target-channel ` (alias `--to`): builds and uploads to the specified channel. - With `--target-dir `: builds and copies the package(s) into the given directory (no channel indexing). @@ -116,14 +132,16 @@ pixi publish --target-dir /path/to/output/dir pixi publish --target-dir ../my-packages ``` -### Publishing from a specific manifest +### Publishing a single package + +When `--path` is given, only the addressed package is built and published, whether or not it sets `publish = true`. The package must not have any source dependencies. ```shell +# Publish a single package of the workspace +pixi publish --target-channel https://prefix.dev/my-channel --path ./my-package/ + # Publish a package from a specific recipe pixi publish --target-channel https://prefix.dev/my-channel --path ./my-recipe/recipe.yaml - -# Publish from a different workspace -pixi publish --target-channel https://prefix.dev/my-channel --path ./my-project/ ``` ### Clean rebuild and publish diff --git a/docs/reference/pixi_manifest.md b/docs/reference/pixi_manifest.md index e7063292b4..a72849d86c 100644 --- a/docs/reference/pixi_manifest.md +++ b/docs/reference/pixi_manifest.md @@ -1446,6 +1446,7 @@ The package section is defined using the following fields: - `run-dependencies`: The run dependencies of the package. - `run-constraints`: Version constraints applied to the package's run environment. - `target`: The target table to configure target specific dependencies. (Similar to the [target](#the-target-table) table) +- `publish`: Whether a workspace-wide [`pixi publish`](cli/pixi/publish.md) publishes this package. Packages that do not opt in with `publish = true` are left out of the publish set. And to extend the basics, it can also contain the following fields: diff --git a/docs/switching_from/uv.md b/docs/switching_from/uv.md index eaaa041a10..e341f3d1aa 100644 --- a/docs/switching_from/uv.md +++ b/docs/switching_from/uv.md @@ -154,7 +154,7 @@ Both tools support multi-package workspaces. uv defines workspace members with a members = ["packages/*"] ``` -Pixi takes a different approach: you reference local packages as path dependencies directly in the workspace manifest. Any subdirectory with its own `pixi.toml` (containing a `[package]` section) can be pulled in this way: +Pixi's environments work similarly: you reference local packages as path dependencies directly in the workspace manifest. Any subdirectory with its own `pixi.toml` (containing a `[package]` section) can be pulled in this way: ```toml title="pixi.toml" [workspace] @@ -165,6 +165,14 @@ platforms = ["linux-64", "osx-arm64", "win-64"] my_lib = { path = "packages/my_lib" } ``` +For publishing, each package opts in individually: a workspace-wide `pixi publish` builds and uploads every package that sets `publish = true` in its `[package]` section: + +```toml title="packages/my_lib/pixi.toml" +[package] +name = "my_lib" +publish = true +``` + Both tools share a single lock file across the workspace. See [Building Multiple Packages](../build/workspace.md). ### Standalone scripts diff --git a/schema/model.py b/schema/model.py index 70b3ae1c7e..412f759b32 100644 --- a/schema/model.py +++ b/schema/model.py @@ -1143,6 +1143,11 @@ class Package(StrictBaseModel): description="The URL of the documentation of the project. Can be a URL or { workspace = true } to inherit from workspace", ) + publish: bool | None = Field( + None, + description="Whether a workspace-wide `pixi publish` publishes this package. Packages that do not opt in with `publish = true` are left out of the publish set.", + ) + build: Build = Field(..., description="The build configuration of the package") host_dependencies: ConditionalInheritableDependencies = HostDependenciesField diff --git a/schema/pyproject/partial-pixi.json b/schema/pyproject/partial-pixi.json index 7942821528..19546603dd 100644 --- a/schema/pyproject/partial-pixi.json +++ b/schema/pyproject/partial-pixi.json @@ -1932,6 +1932,11 @@ } ] }, + "publish": { + "title": "Publish", + "description": "Whether a workspace-wide `pixi publish` publishes this package. Packages that do not opt in with `publish = true` are left out of the publish set.", + "type": "boolean" + }, "readme": { "title": "Readme", "description": "The path to the readme file of the project. Can be a path or { workspace = true } to inherit from workspace", diff --git a/schema/pyproject/schema.json b/schema/pyproject/schema.json index 41f9153c60..ef68711964 100644 --- a/schema/pyproject/schema.json +++ b/schema/pyproject/schema.json @@ -1675,6 +1675,11 @@ } ] }, + "publish": { + "title": "Publish", + "description": "Whether a workspace-wide `pixi publish` publishes this package. Packages that do not opt in with `publish = true` are left out of the publish set.", + "type": "boolean" + }, "readme": { "title": "Readme", "description": "The path to the readme file of the project. Can be a path or { workspace = true } to inherit from workspace", diff --git a/schema/schema.json b/schema/schema.json index b814e023ca..4f5a1ae11d 100644 --- a/schema/schema.json +++ b/schema/schema.json @@ -1956,6 +1956,11 @@ } ] }, + "publish": { + "title": "Publish", + "description": "Whether a workspace-wide `pixi publish` publishes this package. Packages that do not opt in with `publish = true` are left out of the publish set.", + "type": "boolean" + }, "readme": { "title": "Readme", "description": "The path to the readme file of the project. Can be a path or { workspace = true } to inherit from workspace", diff --git a/tests/integration_python/pixi_build/test_publish.py b/tests/integration_python/pixi_build/test_publish.py new file mode 100644 index 0000000000..5d0f1a35c4 --- /dev/null +++ b/tests/integration_python/pixi_build/test_publish.py @@ -0,0 +1,703 @@ +"""Integration tests for multi-package `pixi publish`. + +`pixi publish` without `--path` builds every workspace package that opts in +with `publish = true` in its `[package]` section, in dependency order, and +uploads them. Every source dependency of a published package must itself opt +in; the publish fails otherwise. These tests exercise the package discovery, +the closure validation, ordering and upload behavior end to end. They build +real packages and download backends from prefix.dev, so they are all marked +`slow`. +""" + +from __future__ import annotations + +import shutil +import tomllib +from dataclasses import dataclass, field +from pathlib import Path + +import pytest +import tomli_w +import yaml + +from .common import CURRENT_PLATFORM, ExitCode, git_test_repo, verify_cli_command + + +BACKEND_CHANNELS = [ + "https://prefix.dev/pixi-build-backends", + "https://prefix.dev/conda-forge", +] + + +@dataclass +class Pkg: + """A member package, its publish opt-in, and the members it depends on.""" + + host: list[str] = field(default_factory=list) + run: list[str] = field(default_factory=list) + build: list[str] = field(default_factory=list) + publish: bool = True + + +def _module(name: str) -> str: + return name.replace("-", "_") + + +def _package_table(name: str, pkg: Pkg, backend: str, *, dep_prefix: str = "../") -> dict: + backend_name = f"pixi-build-{backend}" + table: dict = { + "name": name, + "version": "1.0.0", + "publish": pkg.publish, + "build": { + "backend": { + "name": backend_name, + "channels": BACKEND_CHANNELS, + "version": "*", + }, + }, + } + host: dict = {} + if backend == "python": + host["hatchling"] = "*" + host.update({dep: {"path": f"{dep_prefix}{dep}"} for dep in pkg.host}) + if host: + table["host-dependencies"] = host + if pkg.run: + table["run-dependencies"] = {dep: {"path": f"{dep_prefix}{dep}"} for dep in pkg.run} + if pkg.build: + table["build-dependencies"] = {dep: {"path": f"{dep_prefix}{dep}"} for dep in pkg.build} + return table + + +def _write_package_sources(pkg_dir: Path, name: str, backend: str) -> None: + """Write the backend-specific source files a package needs to build.""" + if backend == "python": + module = _module(name) + pyproject = { + "project": { + "name": name, + "version": "1.0.0", + "requires-python": ">= 3.11", + "dependencies": [], + "scripts": {name: f"{module}:main"}, + }, + "build-system": { + "build-backend": "hatchling.build", + "requires": ["hatchling"], + }, + } + pkg_dir.joinpath("pyproject.toml").write_text(tomli_w.dumps(pyproject)) + src = pkg_dir.joinpath("src", module) + src.mkdir(parents=True, exist_ok=True) + src.joinpath("__init__.py").write_text( + f'def main() -> None:\n print("hello from {name}")\n' + ) + elif backend == "rattler-build": + recipe = {"package": {"name": name, "version": "1.0.0"}} + pkg_dir.joinpath("recipe.yaml").write_text(yaml.dump(recipe)) + else: + raise ValueError(f"unsupported backend {backend!r}") + + +def write_workspace( + root: Path, + packages: dict[str, Pkg], + *, + backend: str = "python", + root_package: str | None = None, + workspace_deps: list[str] | None = None, + channels: list[str] | None = None, +) -> None: + """Write a multi-package pixi workspace under `root`. + + Each entry in `packages` becomes a member directory `root/` with a + `pixi.toml` and the backend-specific source files; its `Pkg.publish` flag + becomes the `publish` key of the `[package]` section. `workspace_deps` + selects which members are referenced from the workspace `[dependencies]` + table (defaults to all of them except the root package). When + `root_package` is given, the workspace manifest also carries a `[package]` + section so the root itself is a publishable package. + """ + if workspace_deps is None: + workspace_deps = [name for name in packages if name != root_package] + + workspace_manifest: dict = { + "workspace": { + "channels": channels or ["https://prefix.dev/conda-forge"], + "preview": ["pixi-build"], + "platforms": [CURRENT_PLATFORM], + }, + "dependencies": {dep: {"path": dep} for dep in workspace_deps}, + } + + if root_package is not None: + workspace_manifest["package"] = _package_table( + root_package, packages.get(root_package, Pkg()), backend, dep_prefix="" + ) + _write_package_sources(root, root_package, backend) + + root.joinpath("pixi.toml").write_text(tomli_w.dumps(workspace_manifest)) + + for name, pkg in packages.items(): + if name == root_package: + continue + pkg_dir = root.joinpath(name) + pkg_dir.mkdir(parents=True, exist_ok=True) + manifest = {"package": _package_table(name, pkg, backend)} + pkg_dir.joinpath("pixi.toml").write_text(tomli_w.dumps(manifest)) + _write_package_sources(pkg_dir, name, backend) + + +def publish_to_dir(pixi: Path, workspace: Path, *, args: list[str] | None = None): + """Run `pixi publish` (whole-workspace) from `workspace` into `workspace/dist`.""" + target_dir = workspace.joinpath("dist") + output = verify_cli_command( + [pixi, "publish", "--target-dir", str(target_dir), *(args or [])], + cwd=workspace, + ) + return output, target_dir + + +def write_standalone_package(pkg_dir: Path, name: str, *, backend: str = "python") -> None: + """Write a package that lives outside any workspace under test.""" + pkg_dir.mkdir(parents=True, exist_ok=True) + manifest = {"package": _package_table(name, Pkg(), backend)} + pkg_dir.joinpath("pixi.toml").write_text(tomli_w.dumps(manifest)) + _write_package_sources(pkg_dir, name, backend) + + +def built_package_names(target_dir: Path) -> list[str]: + """Names of the packages in `target_dir`, derived from the artifact files.""" + return sorted(p.name.split("-1.0.0-")[0] for p in target_dir.glob("*.conda")) + + +def assert_first_occurrence_order(stderr: str, *names: str) -> None: + """Assert `names` first appear in `stderr` in the given order. + + All publish output is on stderr and the "Building N package(s):" list is + the first place the package names show up, so first occurrences reflect + the build order. + """ + indexes = [stderr.index(name) for name in names] + assert indexes == sorted(indexes), stderr + + +def _nest_workspace(tmp_pixi_workspace: Path, name: str) -> Path: + """Create a sub-workspace directory that keeps the fixture's pixi config.""" + nested = tmp_pixi_workspace.joinpath(name) + nested.joinpath(".pixi").mkdir(parents=True) + config = tmp_pixi_workspace.joinpath(".pixi", "config.toml") + if config.is_file(): + shutil.copy(config, nested.joinpath(".pixi", "config.toml")) + return nested + + +@pytest.mark.slow +def test_all_opted_in_packages_are_published(pixi: Path, tmp_pixi_workspace: Path) -> None: + """Two independent opted-in packages are both built and published.""" + write_workspace(tmp_pixi_workspace, {"alpha": Pkg(), "bravo": Pkg()}) + output, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + assert built_package_names(target_dir) == ["alpha", "bravo"] + assert "Publishing 2 workspace packages" in output.stderr + + +@pytest.mark.slow +def test_root_package_is_published_with_members(pixi: Path, tmp_pixi_workspace: Path) -> None: + """A `[package]` in the workspace manifest is published like any member. + + The root package also host-depends on a member, which exercises source + anchoring of relative paths declared in the root manifest. + """ + write_workspace( + tmp_pixi_workspace, + { + "rootpkg": Pkg(host=["member-a"]), + "member-a": Pkg(), + }, + root_package="rootpkg", + ) + output, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + assert built_package_names(target_dir) == ["member-a", "rootpkg"] + assert_first_occurrence_order(output.stderr, "member-a", "rootpkg") + + +@pytest.mark.slow +def test_non_opted_in_package_is_not_published(pixi: Path, tmp_pixi_workspace: Path) -> None: + """A package without `publish = true` is not published. + + Being referenced from the workspace `[dependencies]` table does not make a + package part of the publish set; only the `publish` flag does. + """ + write_workspace( + tmp_pixi_workspace, + {"alpha": Pkg(), "orphan": Pkg(publish=False)}, + ) + _, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + assert built_package_names(target_dir) == ["alpha"] + + +@pytest.mark.slow +def test_unpublished_source_dependency_fails(pixi: Path, tmp_pixi_workspace: Path) -> None: + """A source dependency on a package that does not opt in fails the publish. + + Uploading `alpha` while skipping `bravo` would leave the target channel + with an unsatisfiable dependency, so the whole publish is refused. + """ + write_workspace( + tmp_pixi_workspace, + {"alpha": Pkg(host=["bravo"]), "bravo": Pkg(publish=False)}, + ) + verify_cli_command( + [pixi, "publish", "--target-dir", str(tmp_pixi_workspace.joinpath("dist"))], + ExitCode.FAILURE, + cwd=tmp_pixi_workspace, + stderr_contains="not part of the publish set", + ) + + +@pytest.mark.slow +def test_transitive_chain_publishes_in_dependency_order( + pixi: Path, tmp_pixi_workspace: Path +) -> None: + """alpha -> bravo -> charlie (host deps) publishes charlie, bravo, alpha.""" + write_workspace( + tmp_pixi_workspace, + { + "alpha": Pkg(host=["bravo"]), + "bravo": Pkg(host=["charlie"]), + "charlie": Pkg(), + }, + ) + output, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + assert built_package_names(target_dir) == ["alpha", "bravo", "charlie"] + assert_first_occurrence_order(output.stderr, "charlie", "bravo", "alpha") + + +@pytest.mark.slow +def test_diamond_members_published_once_each(pixi: Path, tmp_pixi_workspace: Path) -> None: + """A diamond (alpha -> bravo+charlie -> delta) publishes each member once.""" + write_workspace( + tmp_pixi_workspace, + { + "alpha": Pkg(host=["bravo", "charlie"]), + "bravo": Pkg(host=["delta"]), + "charlie": Pkg(host=["delta"]), + "delta": Pkg(), + }, + ) + output, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + assert built_package_names(target_dir) == ["alpha", "bravo", "charlie", "delta"] + assert_first_occurrence_order(output.stderr, "delta", "bravo", "alpha") + assert_first_occurrence_order(output.stderr, "delta", "charlie", "alpha") + + +@pytest.mark.slow +def test_run_dependency_orders_publish(pixi: Path, tmp_pixi_workspace: Path) -> None: + """Run dependencies between members also order the publish.""" + write_workspace( + tmp_pixi_workspace, + {"alpha": Pkg(run=["bravo"]), "bravo": Pkg()}, + workspace_deps=["alpha"], + ) + output, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + assert built_package_names(target_dir) == ["alpha", "bravo"] + assert_first_occurrence_order(output.stderr, "bravo", "alpha") + + +@pytest.mark.slow +def test_path_option_publishes_only_that_package(pixi: Path, tmp_pixi_workspace: Path) -> None: + """`--path` restricts the publish to a single package.""" + write_workspace(tmp_pixi_workspace, {"alpha": Pkg(), "bravo": Pkg()}) + _, target_dir = publish_to_dir(pixi, tmp_pixi_workspace, args=["--path", "alpha"]) + + assert built_package_names(target_dir) == ["alpha"] + + +@pytest.mark.slow +def test_workspace_without_opted_in_packages_fails(pixi: Path, tmp_pixi_workspace: Path) -> None: + """A workspace where no package sets `publish = true` errors out.""" + write_workspace(tmp_pixi_workspace, {}) + verify_cli_command( + [pixi, "publish", "--target-dir", str(tmp_pixi_workspace.joinpath("dist"))], + ExitCode.FAILURE, + cwd=tmp_pixi_workspace, + stderr_contains="no package in the workspace opts into publishing", + ) + + +@pytest.mark.slow +def test_gitignored_package_is_not_published(pixi: Path, tmp_pixi_workspace: Path) -> None: + """Package discovery respects ignore files. + + A package that opts in with `publish = true` but sits in a gitignored + directory is not discovered and therefore not published. + """ + write_workspace( + tmp_pixi_workspace, + {"alpha": Pkg(), "vendored": Pkg()}, + workspace_deps=["alpha"], + ) + tmp_pixi_workspace.joinpath(".gitignore").write_text("vendored/\n") + + _, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + assert built_package_names(target_dir) == ["alpha"] + + +@pytest.mark.slow +def test_external_path_dependency_fails(pixi: Path, tmp_pixi_workspace: Path) -> None: + """A source dependency on a path outside the workspace root fails. + + An external source cannot be part of the published batch, so the closure + rule refuses the publish. + """ + workspace = _nest_workspace(tmp_pixi_workspace, "ws") + write_workspace(workspace, {"alpha": Pkg()}) + write_standalone_package(tmp_pixi_workspace.joinpath("external", "extpkg"), "extpkg") + + manifest_path = workspace.joinpath("alpha", "pixi.toml") + manifest = tomllib.loads(manifest_path.read_text()) + manifest["package"].setdefault("host-dependencies", {})["extpkg"] = { + "path": "../../external/extpkg" + } + manifest_path.write_text(tomli_w.dumps(manifest)) + + verify_cli_command( + [pixi, "publish", "--target-dir", str(workspace.joinpath("dist"))], + ExitCode.FAILURE, + cwd=workspace, + stderr_contains="outside the workspace", + ) + + +@pytest.mark.slow +def test_external_git_dependency_fails(pixi: Path, tmp_pixi_workspace: Path) -> None: + """A git source dependency fails the publish.""" + workspace = _nest_workspace(tmp_pixi_workspace, "ws") + write_workspace(workspace, {"alpha": Pkg()}) + pkg_src = tmp_pixi_workspace.joinpath("gitsrc") + write_standalone_package(pkg_src, "gitpkg") + repo_url = git_test_repo(pkg_src, "gitpkg-repo", tmp_pixi_workspace.joinpath("repos")) + + manifest_path = workspace.joinpath("alpha", "pixi.toml") + manifest = tomllib.loads(manifest_path.read_text()) + manifest["package"].setdefault("host-dependencies", {})["gitpkg"] = {"git": repo_url} + manifest_path.write_text(tomli_w.dumps(manifest)) + + verify_cli_command( + [pixi, "publish", "--target-dir", str(workspace.joinpath("dist"))], + ExitCode.FAILURE, + cwd=workspace, + stderr_contains="outside the workspace", + ) + + +@pytest.mark.slow +def test_source_dependency_on_manifest_path_matches_listed_directory( + pixi: Path, tmp_pixi_workspace: Path +) -> None: + """A `pkg/pixi.toml` source dependency matches the discovered `pkg` directory. + + The closure check must identify a dependency declared through the manifest + file path with the member directory of the opted-in package. + """ + write_workspace(tmp_pixi_workspace, {"alpha": Pkg(host=["bravo"]), "bravo": Pkg()}) + manifest_path = tmp_pixi_workspace.joinpath("alpha", "pixi.toml") + manifest = tomllib.loads(manifest_path.read_text()) + manifest["package"]["host-dependencies"]["bravo"] = {"path": "../bravo/pixi.toml"} + manifest_path.write_text(tomli_w.dumps(manifest)) + + output, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + assert built_package_names(target_dir) == ["alpha", "bravo"] + assert "Publishing 2 workspace packages" in output.stderr + + +@pytest.mark.slow +def test_multi_output_member_publishes_all_outputs(pixi: Path, tmp_pixi_workspace: Path) -> None: + """Every output of a multi-output recipe member is published.""" + write_workspace(tmp_pixi_workspace, {"multi": Pkg()}, backend="rattler-build") + recipe = { + "recipe": {"name": "multi", "version": "1.0.0"}, + "outputs": [ + {"package": {"name": "multi"}}, + {"package": {"name": "multi-tools"}}, + ], + } + tmp_pixi_workspace.joinpath("multi", "recipe.yaml").write_text(yaml.dump(recipe)) + + _, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + assert built_package_names(target_dir) == ["multi", "multi-tools"] + + +@pytest.mark.slow +def test_second_publish_skips_existing(pixi: Path, tmp_pixi_workspace: Path) -> None: + """Publishing into the same target twice skips the existing artifacts.""" + write_workspace(tmp_pixi_workspace, {"alpha": Pkg()}) + publish_to_dir(pixi, tmp_pixi_workspace) + output, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + assert built_package_names(target_dir) == ["alpha"] + assert "Skipping" in output.stderr, output.stderr + assert "already exists" in output.stderr, output.stderr + + +@pytest.mark.slow +def test_target_channel_file_url_installs_end_to_end(pixi: Path, tmp_pixi_workspace: Path) -> None: + """Packages published to a `file://` channel install and run elsewhere.""" + workspace = _nest_workspace(tmp_pixi_workspace, "ws") + write_workspace(workspace, {"alpha": Pkg()}) + channel = tmp_pixi_workspace.joinpath("channel") + verify_cli_command( + [pixi, "publish", "--target-channel", channel.as_uri()], + cwd=workspace, + ) + assert channel.joinpath("noarch", "repodata.json").is_file() + + consumer = _nest_workspace(tmp_pixi_workspace, "consumer") + consumer_manifest = { + "workspace": { + "channels": [channel.as_uri(), "https://prefix.dev/conda-forge"], + "platforms": [CURRENT_PLATFORM], + }, + "dependencies": {"alpha": "*"}, + } + consumer.joinpath("pixi.toml").write_text(tomli_w.dumps(consumer_manifest)) + + verify_cli_command( + [pixi, "run", "alpha"], + cwd=consumer, + stdout_contains="hello from alpha", + ) + + +@pytest.mark.slow +def test_smoke_host_dep_orders_and_builds_once(pixi: Path, tmp_pixi_workspace: Path) -> None: + """Validation smoke test: `alpha` host-depends on `bravo`. + + `bravo` must be built before `alpha` and exactly once, even though it is + both a standalone member and a host dependency of `alpha`. + """ + write_workspace( + tmp_pixi_workspace, + { + "alpha": Pkg(host=["bravo"]), + "bravo": Pkg(), + }, + ) + output, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + built = sorted(p.name for p in target_dir.glob("*.conda")) + assert len(built) == 2, built + + # bravo is built before alpha (dependency order). All publish output is on + # stderr; the "Building N package(s):" list is the first place both names + # appear, so first-occurrence order reflects build order. + assert output.stderr.index("bravo") < output.stderr.index("alpha"), output.stderr + + +@pytest.mark.slow +def test_host_dep_member_is_built_only_once(pixi: Path, tmp_pixi_workspace: Path) -> None: + """A member that is also a host dependency of another member builds once. + + `bravo` is published as a member and consumed as a host dependency of + `alpha`. Its member build artifact must be reused when assembling + `alpha`'s host environment instead of triggering a second build. + """ + write_workspace( + tmp_pixi_workspace, + { + "alpha": Pkg(host=["bravo"]), + "bravo": Pkg(), + }, + ) + output, _ = publish_to_dir(pixi, tmp_pixi_workspace) + + # The backend logs one "Running build for recipe:" line per package it + # actually builds; cache hits produce none. Two packages, two builds. + assert output.stderr.count("Running build for recipe") == 2, output.stderr + + +@pytest.mark.slow +def test_build_dep_member_is_built_only_once(pixi: Path, tmp_pixi_workspace: Path) -> None: + """A member that is also a build dependency of another member builds once. + + Build dependencies target the build platform. Since publish targets the + current platform by default, the build-dependency build of `bravo` must + reuse the member build instead of running again. + """ + write_workspace( + tmp_pixi_workspace, + { + "alpha": Pkg(build=["bravo"]), + "bravo": Pkg(), + }, + ) + output, _ = publish_to_dir(pixi, tmp_pixi_workspace) + + assert output.stderr.count("Running build for recipe") == 2, output.stderr + + +@pytest.mark.slow +def test_force_overwrites_existing_artifact(pixi: Path, tmp_pixi_workspace: Path) -> None: + """`--force` replaces an artifact that already exists at the target.""" + write_workspace(tmp_pixi_workspace, {"alpha": Pkg()}) + _, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + artifact = next(target_dir.glob("*.conda")) + first_bytes = artifact.read_bytes() + + # Change the package contents without changing name/version/build string, + # so the replacement artifact keeps the same filename. Comparing bytes + # instead of mtime: `fs::copy` preserves timestamps on Windows and macOS. + module = tmp_pixi_workspace.joinpath("alpha", "src", "alpha", "__init__.py") + module.write_text('def main() -> None:\n print("changed contents")\n') + + output, _ = publish_to_dir(pixi, tmp_pixi_workspace, args=["--force"]) + + assert "already exists" not in output.stderr, output.stderr + assert artifact.read_bytes() != first_bytes + + +@pytest.mark.slow +def test_no_skip_existing_fails_on_existing_artifact(pixi: Path, tmp_pixi_workspace: Path) -> None: + """`--no-skip-existing` without `--force` refuses to overwrite.""" + write_workspace(tmp_pixi_workspace, {"alpha": Pkg()}) + _, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + verify_cli_command( + [pixi, "publish", "--target-dir", str(target_dir), "--no-skip-existing"], + ExitCode.FAILURE, + cwd=tmp_pixi_workspace, + stderr_contains="--force", + ) + + +@pytest.mark.slow +def test_dry_run_prints_set_without_building(pixi: Path, tmp_pixi_workspace: Path) -> None: + """`--dry-run` prints the resolved publish set and uploads nothing.""" + write_workspace(tmp_pixi_workspace, {"alpha": Pkg(host=["bravo"]), "bravo": Pkg()}) + target_dir = tmp_pixi_workspace.joinpath("dist") + output = verify_cli_command( + [pixi, "publish", "--target-dir", str(target_dir), "--dry-run"], + cwd=tmp_pixi_workspace, + ) + + assert "Would build 2 package(s)" in output.stderr + assert_first_occurrence_order(output.stderr, "bravo", "alpha") + assert not target_dir.exists() or built_package_names(target_dir) == [] + + +@pytest.mark.slow +def test_force_reindexes_local_channel(pixi: Path, tmp_pixi_workspace: Path) -> None: + """`--force` on a `file://` channel updates repodata for the new artifact. + + Overwriting the `.conda` file alone is not enough: the channel index must + pick up the replaced artifact's checksums, otherwise consumers fail with + checksum mismatches. + """ + workspace = _nest_workspace(tmp_pixi_workspace, "ws") + write_workspace(workspace, {"alpha": Pkg()}) + channel = tmp_pixi_workspace.joinpath("channel") + + verify_cli_command([pixi, "publish", "--target-channel", channel.as_uri()], cwd=workspace) + repodata_path = channel.joinpath("noarch", "repodata.json") + first_repodata = repodata_path.read_text() + + # Change the package contents without changing name/version/build string, + # so the replacement artifact keeps the same filename. + module = workspace.joinpath("alpha", "src", "alpha", "__init__.py") + module.write_text('def main() -> None:\n print("changed contents")\n') + + verify_cli_command( + [pixi, "publish", "--target-channel", channel.as_uri(), "--force"], + cwd=workspace, + ) + second_repodata = repodata_path.read_text() + + assert second_repodata != first_repodata, "repodata must reflect the replaced artifact" + + +@pytest.mark.slow +def test_multi_output_sibling_dependency_uploads_in_order( + pixi: Path, tmp_pixi_workspace: Path +) -> None: + """Outputs of one recipe upload dependencies-first, not alphabetically.""" + write_workspace(tmp_pixi_workspace, {"multi": Pkg()}, backend="rattler-build") + recipe = { + "recipe": {"name": "multi", "version": "1.0.0"}, + "outputs": [ + {"package": {"name": "zzz-lib"}}, + { + "package": {"name": "aaa-app"}, + "requirements": {"run": ['${{ pin_subpackage("zzz-lib", exact=True) }}']}, + }, + ], + } + tmp_pixi_workspace.joinpath("multi", "recipe.yaml").write_text(yaml.dump(recipe)) + + output, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + assert built_package_names(target_dir) == ["aaa-app", "zzz-lib"] + # The upload listing reflects the build/upload order; the library that + # `aaa-app` run-depends on must come first. + published = output.stderr[output.stderr.index("Successfully published") :] + assert published.index("zzz-lib") < published.index("aaa-app"), output.stderr + + +@pytest.mark.slow +def test_publish_from_member_directory_includes_root_package( + pixi: Path, tmp_pixi_workspace: Path +) -> None: + """The published set does not depend on the invocation directory. + + Every opted-in package - including the root `[package]` - is published + even when the command runs inside a member directory. + """ + write_workspace( + tmp_pixi_workspace, + {"rootpkg": Pkg(), "member-a": Pkg()}, + root_package="rootpkg", + workspace_deps=["member-a"], + ) + target_dir = tmp_pixi_workspace.joinpath("dist") + verify_cli_command( + [pixi, "publish", "--target-dir", str(target_dir)], + cwd=tmp_pixi_workspace.joinpath("member-a"), + ) + + assert built_package_names(target_dir) == ["member-a", "rootpkg"] + + +@pytest.mark.slow +def test_deprecated_build_command_builds_only_current_package( + pixi: Path, tmp_pixi_workspace: Path +) -> None: + """`pixi build` keeps its historic single-package behavior. + + Unlike `pixi publish`, the deprecated `pixi build` command never operated + on the whole workspace: without `--path` it builds only the package at the + invocation directory. + """ + write_workspace( + tmp_pixi_workspace, + {"rootpkg": Pkg(), "member-a": Pkg()}, + root_package="rootpkg", + ) + target_dir = tmp_pixi_workspace.joinpath("dist") + output = verify_cli_command( + [pixi, "build", "--output-dir", str(target_dir)], + cwd=tmp_pixi_workspace, + ) + + assert built_package_names(target_dir) == ["rootpkg"] + # The suggested replacement must carry `--path`; a bare `pixi publish` + # would publish the whole workspace instead. + assert "--path" in output.stderr