From d8e718f3b760d8f2481f56426bb4b36a1097f786 Mon Sep 17 00:00:00 2001 From: Julian Hofer Date: Thu, 9 Jul 2026 11:13:53 +0000 Subject: [PATCH 1/4] feat(manifest): add the package publish flag Packages opt into workspace-wide publishing by setting `publish = true` in their `[package]` section. The flag defaults to false and is not inheritable from the workspace. --- crates/pixi_manifest/src/package.rs | 5 +++++ crates/pixi_manifest/src/toml/package.rs | 4 ++++ ...ixi_manifest__toml__package__test__invalid_extra_key.snap | 3 ++- docs/reference/pixi_manifest.md | 1 + schema/model.py | 5 +++++ schema/pyproject/partial-pixi.json | 5 +++++ schema/pyproject/schema.json | 5 +++++ schema/schema.json | 5 +++++ 8 files changed, 32 insertions(+), 1 deletion(-) diff --git a/crates/pixi_manifest/src/package.rs b/crates/pixi_manifest/src/package.rs index f2425847a7..3f71efed97 100644 --- a/crates/pixi_manifest/src/package.rs +++ b/crates/pixi_manifest/src/package.rs @@ -35,6 +35,10 @@ pub struct Package { /// URL of the project documentation pub documentation: Option, + + /// Whether a workspace-wide `pixi publish` publishes this package. + /// Packages that do not opt in are left out of the publish set. + pub publish: bool, } impl Package { @@ -51,6 +55,7 @@ impl Package { homepage: None, repository: None, documentation: None, + publish: false, } } } diff --git a/crates/pixi_manifest/src/toml/package.rs b/crates/pixi_manifest/src/toml/package.rs index e50244683b..ecf7b479a5 100644 --- a/crates/pixi_manifest/src/toml/package.rs +++ b/crates/pixi_manifest/src/toml/package.rs @@ -141,6 +141,7 @@ pub struct TomlPackage { pub documentation: Option>, // Fields that are package-specific and cannot be inherited + pub publish: Option, pub build: TomlPackageBuild, pub host_dependencies: Option>, pub build_dependencies: Option>, @@ -191,6 +192,7 @@ impl<'de> toml_span::Deserialize<'de> for TomlPackage { .map(TomlWith::into_inner) .unwrap_or_default(); let run_constraints = th.optional("run-constraints"); + let publish = th.optional("publish"); let build = th.required("build")?; let target = th .optional::>>("target") @@ -214,6 +216,7 @@ impl<'de> toml_span::Deserialize<'de> for TomlPackage { run_dependencies, extra_dependencies, run_constraints, + publish, build, target, span: value.span, @@ -612,6 +615,7 @@ impl TomlPackage { package_defaults.documentation, "documentation", )?, + publish: self.publish.unwrap_or(false), }, build: build_result.value, dependencies: default_package_target, diff --git a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__package__test__invalid_extra_key.snap b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__package__test__invalid_extra_key.snap index 63ed0126b1..b8ce7d889c 100644 --- a/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__package__test__invalid_extra_key.snap +++ b/crates/pixi_manifest/src/toml/snapshots/pixi_manifest__toml__package__test__invalid_extra_key.snap @@ -1,9 +1,10 @@ --- source: crates/pixi_manifest/src/toml/package.rs +assertion_line: 811 expression: "expect_parse_failure(r#\"\n foo = \"bar\"\n name = \"bla\"\n extra = \"key\"\n\n [build]\n backend = { name = \"bla\", version = \"1.0\" }\n \"#,)" --- × Unexpected keys, expected only 'name', 'version', 'description', 'authors', 'license', 'license-file', 'readme', 'homepage', 'repository', 'documentation', 'host-dependencies', 'build- - │ dependencies', 'run-dependencies', 'extra-dependencies', 'run-constraints', 'build', 'target' + │ dependencies', 'run-dependencies', 'extra-dependencies', 'run-constraints', 'publish', 'build', 'target' ╭─[pixi.toml:2:9] 1 │ 2 │ foo = "bar" diff --git a/docs/reference/pixi_manifest.md b/docs/reference/pixi_manifest.md index aa8352ba5e..d0d4409018 100644 --- a/docs/reference/pixi_manifest.md +++ b/docs/reference/pixi_manifest.md @@ -1377,6 +1377,7 @@ The package section is defined using the following fields: - `run-dependencies`: The run dependencies of the package. - `run-constraints`: Version constraints applied to the package's run environment. - `target`: The target table to configure target specific dependencies. (Similar to the [target](#the-target-table) table) +- `publish`: Whether a workspace-wide [`pixi publish`](cli/pixi/publish.md) publishes this package. Packages that do not opt in with `publish = true` are left out of the publish set. And to extend the basics, it can also contain the following fields: diff --git a/schema/model.py b/schema/model.py index 5d8f9f0129..5d0adde243 100644 --- a/schema/model.py +++ b/schema/model.py @@ -1093,6 +1093,11 @@ class Package(StrictBaseModel): description="The URL of the documentation of the project. Can be a URL or { workspace = true } to inherit from workspace", ) + publish: bool | None = Field( + None, + description="Whether a workspace-wide `pixi publish` publishes this package. Packages that do not opt in with `publish = true` are left out of the publish set.", + ) + build: Build = Field(..., description="The build configuration of the package") host_dependencies: ConditionalInheritableDependencies = HostDependenciesField diff --git a/schema/pyproject/partial-pixi.json b/schema/pyproject/partial-pixi.json index d3193ffe69..b34f566a9a 100644 --- a/schema/pyproject/partial-pixi.json +++ b/schema/pyproject/partial-pixi.json @@ -1738,6 +1738,11 @@ } ] }, + "publish": { + "title": "Publish", + "description": "Whether a workspace-wide `pixi publish` publishes this package. Packages that do not opt in with `publish = true` are left out of the publish set.", + "type": "boolean" + }, "readme": { "title": "Readme", "description": "The path to the readme file of the project. Can be a path or { workspace = true } to inherit from workspace", diff --git a/schema/pyproject/schema.json b/schema/pyproject/schema.json index 4840b1bcbb..001efabbac 100644 --- a/schema/pyproject/schema.json +++ b/schema/pyproject/schema.json @@ -1481,6 +1481,11 @@ } ] }, + "publish": { + "title": "Publish", + "description": "Whether a workspace-wide `pixi publish` publishes this package. Packages that do not opt in with `publish = true` are left out of the publish set.", + "type": "boolean" + }, "readme": { "title": "Readme", "description": "The path to the readme file of the project. Can be a path or { workspace = true } to inherit from workspace", diff --git a/schema/schema.json b/schema/schema.json index 5a11d49896..50df5d8821 100644 --- a/schema/schema.json +++ b/schema/schema.json @@ -1762,6 +1762,11 @@ } ] }, + "publish": { + "title": "Publish", + "description": "Whether a workspace-wide `pixi publish` publishes this package. Packages that do not opt in with `publish = true` are left out of the publish set.", + "type": "boolean" + }, "readme": { "title": "Readme", "description": "The path to the readme file of the project. Can be a path or { workspace = true } to inherit from workspace", From e33a034827b2a52edd6a416923fc3bf359fa059d Mon Sep 17 00:00:00 2001 From: Julian Hofer Date: Thu, 9 Jul 2026 11:16:07 +0000 Subject: [PATCH 2/4] fix(build): reuse workspace member builds as host and build dependencies Source dependencies recursed by the source build are handed to the prefix installer as pre-built binary records, so a package that is both built on its own and consumed as a host or build dependency of another source build is built only once. Records returned from the artifact cache are rewritten to point at the cached artifact instead of the mutable work directory, which may be cleaned before the record is consumed again. --- .../src/cache/artifact.rs | 33 ++++-- .../src/keys/source_build.rs | 112 ++++++++++++------ 2 files changed, 102 insertions(+), 43 deletions(-) diff --git a/crates/pixi_command_dispatcher/src/cache/artifact.rs b/crates/pixi_command_dispatcher/src/cache/artifact.rs index ad75744e71..96e56996a8 100644 --- a/crates/pixi_command_dispatcher/src/cache/artifact.rs +++ b/crates/pixi_command_dispatcher/src/cache/artifact.rs @@ -41,6 +41,7 @@ use rattler_digest::Sha256Hash; use serde::{Deserialize, Serialize}; use serde_with::serde_as; use thiserror::Error; +use url::Url; use xxhash_rust::xxh3::Xxh3; /// Opaque content-addressed handle for an artifact cache entry. @@ -332,10 +333,16 @@ impl ArtifactCache { return Ok(None); } + // Point the record at the artifact being returned. Sidecars written + // by older versions carry the work-directory output path, which does + // not survive workspace-cache cleanups. + let mut record = sidecar.record; + record.url = Url::from_file_path(&artifact_path).expect("cache entry paths are absolute"); + Ok(Some(CachedArtifact { artifact: artifact_path, sha256: sidecar.artifact_sha256, - record: sidecar.record, + record, })) } @@ -358,7 +365,7 @@ impl ArtifactCache { artifact_source: &Path, input_glob_sets: Vec, input_files: impl IntoIterator, - record: RepoDataRecord, + mut record: RepoDataRecord, ) -> Result { let entry_dir = self.entry_dir(package, key); let lock_file = self.open_lock_file(package, key).await?; @@ -386,6 +393,12 @@ impl ArtifactCache { ArtifactCacheError::io("copying artifact into cache", dest.clone(), err) })?; + // The record was synthesized from the freshly-built artifact and + // points at its work-directory location. Persist and return the + // cached copy instead; the work directory is mutable and may be + // cleaned before the record is consumed again. + record.url = Url::from_file_path(&dest).expect("cache entry paths are absolute"); + let sha256 = { let path = dest.clone(); tokio::task::spawn_blocking(move || { @@ -668,9 +681,10 @@ mod tests { #[tokio::test] async fn sidecar_preserves_record_fields_across_lookup() { // Verify that every RepoDataRecord field we care about - // (identifier, url, package_record.version, .build, .subdir, - // .depends) round-trips through the sidecar. A future refactor - // that accidentally drops a field will fail this test. + // (identifier, package_record.version, .build, .subdir, .depends) + // round-trips through the sidecar, and that the url is rewritten to + // the cached artifact. A future refactor that accidentally drops a + // field will fail this test. let tmp = TempDir::new().unwrap(); let cache = ArtifactCache::new(tmp.path().join("artifacts")); let engine = ComputeEngine::new(); @@ -722,8 +736,13 @@ mod tests { record.package_record.depends ); - // RepoDataRecord top-level fields. - assert_eq!(hit.record.url, record.url); + // RepoDataRecord top-level fields. The url always points at the + // cached artifact, replacing whatever location the record carried + // when it was stored. + assert_eq!( + hit.record.url, + url::Url::from_file_path(&hit.artifact).unwrap() + ); assert_eq!(hit.record.identifier, record.identifier); } diff --git a/crates/pixi_command_dispatcher/src/keys/source_build.rs b/crates/pixi_command_dispatcher/src/keys/source_build.rs index b76ff8f5fc..c721130e40 100644 --- a/crates/pixi_command_dispatcher/src/keys/source_build.rs +++ b/crates/pixi_command_dispatcher/src/keys/source_build.rs @@ -140,10 +140,20 @@ async fn compute_inner( ctx: &mut ComputeCtx, spec: Arc, ) -> Result { - // sha256s are collected in a stable (build, host) order so the - // artifact cache key stays deterministic across buckets. - let (build_source_dep_sha256s, host_source_dep_sha256s) = + // Results are collected in a stable (build, host) order so the + // artifact cache key stays deterministic across buckets. The full + // build results feed the prefix installs below so the source deps + // are not built a second time through the installer. + let (build_source_dep_results, host_source_dep_results) = recurse_source_deps(ctx, &spec).await?; + let build_source_dep_sha256s: Vec = build_source_dep_results + .iter() + .map(|result| result.artifact_sha256) + .collect(); + let host_source_dep_sha256s: Vec = host_source_dep_results + .iter() + .map(|result| result.artifact_sha256) + .collect(); let manifest_source = spec.record.manifest_source.clone(); let manifest_checkout = ctx @@ -277,11 +287,13 @@ async fn compute_inner( // could dedup. let output = fetch_matching_output(&backend, &spec, &work_directory).await?; - // install_prefix recurses into source entries via SourceBuildKey, - // so build_records / host_records are all binaries on disk. Build - // and host packages come pre-resolved on the input record (v7+ - // lock file), so the two installs are independent and can run - // concurrently. + // Source entries were already built by `recurse_source_deps`; hand + // their binary records to `install_prefix` so the installer links + // them instead of triggering another build. Build and host packages + // come pre-resolved on the input record (v7+ lock file), so the two + // installs are independent and can run concurrently. + let build_source_dep_records = records_by_name(&build_source_dep_results); + let host_source_dep_records = records_by_name(&host_source_dep_results); let directories = Directories::new(&work_directory, spec.build_environment.host_platform); let ((build_records, _build_install_result), (host_records, _host_install_result)) = ctx .try_compute2( @@ -292,6 +304,7 @@ async fn compute_inner( InstallTarget::Build, directories.build_prefix.clone(), spec.record.build_packages.clone(), + &build_source_dep_records, ) .await }, @@ -302,6 +315,7 @@ async fn compute_inner( InstallTarget::Host, directories.host_prefix.clone(), spec.record.host_packages.clone(), + &host_source_dep_records, ) .await }, @@ -475,14 +489,30 @@ async fn compute_inner( }) } +/// Index source-dependency build results by package name for the prefix +/// installs. +fn records_by_name( + results: &[Arc], +) -> std::collections::HashMap> { + results + .iter() + .map(|result| { + ( + result.record.package_record.name.clone(), + Arc::new(result.record.clone()), + ) + }) + .collect() +} + /// Fan out over every source entry in `build_packages` and /// `host_packages`, recursively build each via [`SourceBuildKey`], and -/// return their sha256s split by bucket. The two buckets feed into the +/// return the build results split by bucket. The buckets feed into the /// cache key separately so a dep moving build ↔ host invalidates. async fn recurse_source_deps( ctx: &mut ComputeCtx, spec: &Arc, -) -> Result<(Vec, Vec), SourceBuildError> { +) -> Result<(Vec>, Vec>), SourceBuildError> { // build_packages run on the build platform. The nested build's // HOST platform is therefore the outer's BUILD platform. let build = build_source_deps( @@ -510,7 +540,7 @@ async fn build_source_deps( spec: Arc, packages: Vec, nested_build_environment: BuildEnvironment, -) -> Result, SourceBuildError> { +) -> Result>, SourceBuildError> { let sources: Vec> = packages .into_iter() .filter_map(|r| match r { @@ -526,7 +556,7 @@ async fn build_source_deps( let build_env = nested_build_environment; async move |sub_ctx: &mut ComputeCtx, src: Arc| - -> Result { + -> Result, SourceBuildError> { let nested_spec = SourceBuildSpec { record: src, channels: spec.channels.clone(), @@ -540,15 +570,18 @@ async fn build_source_deps( // dependency closure builds against consistent values. build_string_prefix: spec.build_string_prefix.clone(), build_number: spec.build_number, - // Nested source deps are unpacked into the parent's - // prefix immediately; use the cheapest compression. - package_format: Some(CondaPackageFormat::fast()), + // The format is inherited too, so a package that is both + // built in its own right and consumed as a source + // dependency (e.g. two members of one workspace during + // `pixi publish`) shares a single cache entry instead of + // being built once per format. + package_format: spec.package_format, // A nested source dependency carries its own on-disk manifest; // inline definitions apply only to the consumer's direct deps. inline: None, }; let result = sub_ctx.compute(&SourceBuildKey::new(nested_spec)).await?; - Ok(result.artifact_sha256) + Ok(result) } }; ctx.try_compute_join(sources, mapper).await @@ -637,6 +670,10 @@ async fn install_prefix( target: InstallTarget, prefix_path: PathBuf, packages: Vec, + resolved_source_deps: &std::collections::HashMap< + rattler_conda_types::PackageName, + Arc, + >, ) -> Result< ( Vec, @@ -661,9 +698,31 @@ async fn install_prefix( InstallTarget::Build => format!("{} (build)", spec.record.name().as_source()), InstallTarget::Host => format!("{} (host)", spec.record.name().as_source()), }; + // Substitute every source entry with the binary record its build + // produced. Handing the installer the pre-built binaries keeps it + // from launching another build of the same package with different + // build parameters (a different package format, notably). + let mut records = Vec::with_capacity(packages.len()); + let install_records = packages + .into_iter() + .map(|record| match record { + UnresolvedPixiRecord::Binary(binary) => { + records.push((*binary).clone()); + UnresolvedPixiRecord::Binary(binary) + } + UnresolvedPixiRecord::Source(source) => { + let built = resolved_source_deps + .get(source.name()) + .expect("source dependency should have been built by recurse_source_deps") + .clone(); + records.push((*built).clone()); + UnresolvedPixiRecord::Binary(built) + } + }) + .collect(); let install_spec = InstallPixiEnvironmentSpec { name: label, - records: packages.clone(), + records: install_records, prefix, installed: None, ignore_packages: None, @@ -686,25 +745,6 @@ async fn install_prefix( }) .map_err(unwrap_dispatcher_err)?; - // Collect the RepoDataRecords that were installed: binaries pass - // through, sources come from the resolved_source_records map the - // ctx install just populated. - let mut records = Vec::with_capacity(packages.len()); - for r in packages { - match r { - UnresolvedPixiRecord::Binary(rec) => records.push((*rec).clone()), - UnresolvedPixiRecord::Source(src) => { - let built = result - .resolved_source_records - .get(src.name()) - .cloned() - .expect( - "source package should have been built by ctx.install_pixi_environment", - ); - records.push((*built).clone()); - } - } - } Ok((records, Some(result))) } From 5e96bf05c6d9c9ae17dbf1d33153c09593122e0e Mon Sep 17 00:00:00 2001 From: Julian Hofer Date: Thu, 9 Jul 2026 11:16:50 +0000 Subject: [PATCH 3/4] feat(publish): publish the workspace packages that opt in `pixi publish` without `--path` now operates on every package in the workspace whose manifest sets `publish = true`. Packages are discovered by walking the workspace directory tree with a gitignore-aware walker; subtrees that belong to a nested workspace are skipped, and packages that do not opt in are left out of the publish set. The set is validated to be self-contained: every source dependency of a published package must itself opt in, and output names must be unique across the set. Packages are built and uploaded in dependency order, with cycles forced apart and reported. Outputs of a multi-output package are ordered the same way, so `pin_subpackage` consumers upload after the outputs they depend on. Existing packages at the target are skipped by default; `--no-skip-existing` fails on them instead and `--force` overwrites them. The deprecated `pixi build` command keeps its historic single-package behavior by delegating to `pixi publish --path`. --- Cargo.lock | 2 + Cargo.toml | 1 + .../integration_rust/source_package_tests.rs | 392 +++++++++- crates/pixi_cli/Cargo.toml | 2 + crates/pixi_cli/src/build.rs | 14 +- crates/pixi_cli/src/publish/discovery.rs | 699 ++++++++++++++++++ .../src/{publish.rs => publish/mod.rs} | 351 ++++++--- crates/pixi_glob/Cargo.toml | 2 +- docs/build/workspace.md | 23 + docs/reference/cli/pixi.md | 2 +- docs/reference/cli/pixi/publish.md | 21 +- docs/reference/cli/pixi/publish_extender | 26 +- docs/switching_from/uv.md | 10 +- .../pixi_build/test_publish.py | 688 +++++++++++++++++ 14 files changed, 2120 insertions(+), 113 deletions(-) create mode 100644 crates/pixi_cli/src/publish/discovery.rs rename crates/pixi_cli/src/{publish.rs => publish/mod.rs} (82%) create mode 100644 tests/integration_python/pixi_build/test_publish.py diff --git a/Cargo.lock b/Cargo.lock index ae950ace3c..23293ff098 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6399,6 +6399,7 @@ dependencies = [ "futures", "hex", "human_bytes", + "ignore", "indexmap 2.14.0", "indicatif", "insta", @@ -6412,6 +6413,7 @@ dependencies = [ "pixi_api", "pixi_auth", "pixi_build_frontend", + "pixi_build_types", "pixi_command_dispatcher", "pixi_compute_reporters", "pixi_config", diff --git a/Cargo.toml b/Cargo.toml index f062a21b65..5f11f04dcf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,6 +66,7 @@ http = "1.3.1" http-cache-reqwest = "1.0.0-alpha.6" human_bytes = "0.4.3" humantime = "2.1.0" +ignore = "0.4" indexmap = "2.10.0" indicatif = "0.18.0" insta = "1.42.1" diff --git a/crates/pixi/tests/integration_rust/source_package_tests.rs b/crates/pixi/tests/integration_rust/source_package_tests.rs index 87e603d383..faab9d76e2 100644 --- a/crates/pixi/tests/integration_rust/source_package_tests.rs +++ b/crates/pixi/tests/integration_rust/source_package_tests.rs @@ -1005,7 +1005,7 @@ async fn test_publish_fails_before_build_or_upload_when_one_variant_is_unsatisfi target_channel: Some(target_url.to_string()), target_dir: None, force: false, - skip_existing: true, + no_skip_existing: false, generate_attestation: false, variant: Vec::new(), variant_config: Vec::new(), @@ -2829,7 +2829,7 @@ async fn test_publish_without_target_builds_but_does_not_upload() { target_channel: None, target_dir: None, force: false, - skip_existing: true, + no_skip_existing: false, generate_attestation: false, variant: Vec::new(), variant_config: Vec::new(), @@ -2844,6 +2844,390 @@ async fn test_publish_without_target_builds_but_does_not_upload() { ); } +/// Serializes tests that must change the process working directory: +/// workspace-wide publishing (no `--path`) anchors workspace discovery at the +/// current directory. +static PUBLISH_CWD_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +/// Run `pixi publish` with the process working directory set to +/// `workspace_root`, restoring the previous working directory afterwards. +async fn publish_from_directory( + workspace_root: &std::path::Path, + args: publish::Args, +) -> miette::Result<()> { + let _guard = PUBLISH_CWD_LOCK.lock().await; + let original_cwd = std::env::current_dir().unwrap(); + std::env::set_current_dir(workspace_root).unwrap(); + let result = publish::execute(args).await; + std::env::set_current_dir(original_cwd).unwrap(); + result +} + +fn publish_args_for_test( + backend_override: Option, + path: Option, + target_dir: Option, +) -> publish::Args { + publish::Args { + backend_override, + config_cli: Default::default(), + config_source: isolated_config_source(), + target_platform: Platform::current(), + build_platform: Platform::current(), + build_string_prefix: None, + build_number: None, + build_dir: None, + clean: false, + path, + target_channel: None, + target_dir, + force: false, + no_skip_existing: false, + generate_attestation: false, + variant: Vec::new(), + variant_config: Vec::new(), + package_format: None, + } +} + +/// The names (without version/build suffix) of the `.conda` artifacts below +/// `dir`, sorted. +fn conda_artifact_names(dir: &std::path::Path) -> Vec { + let mut found = Vec::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(current) = stack.pop() { + for entry in fs::read_dir(¤t).unwrap().filter_map(Result::ok) { + let path = entry.path(); + if path.is_dir() { + stack.push(path); + } else if path.extension().is_some_and(|ext| ext == "conda") { + let file_name = path.file_name().unwrap().to_string_lossy(); + let name = file_name.split('-').next().unwrap_or_default(); + found.push(name.to_string()); + } + } + } + found.sort(); + found +} + +/// Renders the `publish` line of a `[package]` section. +fn publish_flag(publish: Option) -> String { + match publish { + Some(value) => format!("publish = {value}\n"), + None => String::new(), + } +} + +/// Writes a three-package workspace: the root manifest holds the workspace +/// and a `kit` package that run-depends on member `cpp`, which in turn +/// run-depends on member `core`. The `publish` arguments set the `publish` +/// flag of the respective package, selecting which of them a workspace-wide +/// publish operates on. +fn write_three_package_workspace( + root: &std::path::Path, + kit_publish: Option, + cpp_publish: Option, + core_publish: Option, +) { + let platform = Platform::current(); + fs::write( + root.join("pixi.toml"), + format!( + r#" +[workspace] +channels = [] +platforms = ["{platform}"] +preview = ["pixi-build"] + +[package] +name = "kit" +version = "1.0.0" +{kit_flag} +[package.build] +backend = {{ name = "in-memory", version = "0.1.0" }} + +[package.run-dependencies] +cpp = {{ path = "./cpp" }} +"#, + kit_flag = publish_flag(kit_publish), + ), + ) + .unwrap(); + + let member_manifest = |name: &str, publish: Option, extra: &str| { + format!( + r#" +[package] +name = "{name}" +version = "1.0.0" +{flag} +[package.build] +backend = {{ name = "in-memory", version = "0.1.0" }} +{extra} +"#, + flag = publish_flag(publish), + ) + }; + + let cpp_dir = root.join("cpp"); + fs::create_dir_all(&cpp_dir).unwrap(); + fs::write( + cpp_dir.join("pixi.toml"), + member_manifest( + "cpp", + cpp_publish, + r#" +[package.run-dependencies] +core = { path = "../core" } +"#, + ), + ) + .unwrap(); + + let core_dir = root.join("core"); + fs::create_dir_all(&core_dir).unwrap(); + fs::write( + core_dir.join("pixi.toml"), + member_manifest("core", core_publish, ""), + ) + .unwrap(); +} + +/// `pixi publish` without `--path` must build every package that opts in +/// with `publish = true` - including the root package - and upload all of +/// them in dependency order. +#[tokio::test] +async fn test_publish_workspace_publishes_all_opted_in_packages() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + write_three_package_workspace(pixi.workspace_path(), Some(true), Some(true), Some(true)); + + let publish_dir = tempfile::tempdir().unwrap(); + publish_from_directory( + pixi.workspace_path(), + publish_args_for_test( + Some(BackendOverride::from_memory( + PassthroughBackend::instantiator(), + )), + None, + Some(publish_dir.path().to_path_buf()), + ), + ) + .await + .expect("workspace-wide publish should succeed"); + + assert_eq!( + conda_artifact_names(publish_dir.path()), + vec!["core", "cpp", "kit"], + "all three opted-in workspace packages should be published" + ); +} + +/// A published package whose source dependency does not opt into publishing +/// must fail the publish: uploading it would leave the target with an +/// unsatisfiable run dependency. +#[tokio::test] +async fn test_publish_workspace_rejects_unpublished_source_dependency() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + write_three_package_workspace(pixi.workspace_path(), Some(true), Some(true), None); + + let err = publish_from_directory( + pixi.workspace_path(), + publish_args_for_test( + Some(BackendOverride::from_memory( + PassthroughBackend::instantiator(), + )), + None, + None, + ), + ) + .await + .expect_err("publishing with a source dependency that does not opt in should fail"); + + let rendered = format_diagnostic(err.as_ref()); + assert!( + rendered.contains("not part of the publish set") && rendered.contains("core"), + "{rendered}" + ); +} + +/// An explicit `publish = false` behaves exactly like an absent flag: the +/// package is not part of the publish set, so depending on it fails. +#[tokio::test] +async fn test_publish_workspace_rejects_publish_false_source_dependency() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + write_three_package_workspace(pixi.workspace_path(), Some(true), Some(true), Some(false)); + + let err = publish_from_directory( + pixi.workspace_path(), + publish_args_for_test( + Some(BackendOverride::from_memory( + PassthroughBackend::instantiator(), + )), + None, + None, + ), + ) + .await + .expect_err("publishing with a `publish = false` source dependency should fail"); + + let rendered = format_diagnostic(err.as_ref()); + assert!( + rendered.contains("not part of the publish set") && rendered.contains("core"), + "{rendered}" + ); +} + +/// `pixi publish --path ` keeps the single-package behavior for +/// self-contained packages: only the addressed package is built and +/// uploaded, whether or not it sets `publish = true`. +#[tokio::test] +async fn test_publish_with_path_publishes_a_single_package() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + write_three_package_workspace(pixi.workspace_path(), Some(true), Some(true), None); + + let publish_dir = tempfile::tempdir().unwrap(); + publish::execute(publish_args_for_test( + Some(BackendOverride::from_memory( + PassthroughBackend::instantiator(), + )), + Some(pixi.workspace_path().join("core")), + Some(publish_dir.path().to_path_buf()), + )) + .await + .expect("single-package publish should succeed"); + + assert_eq!( + conda_artifact_names(publish_dir.path()), + vec!["core"], + "only the package addressed with --path should be published" + ); +} + +/// `pixi publish --path` is a batch of one: a package with source +/// dependencies cannot be published alone, because nothing in the batch +/// satisfies them on the target. +#[tokio::test] +async fn test_publish_with_path_rejects_source_dependencies() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + write_three_package_workspace(pixi.workspace_path(), Some(true), Some(true), Some(true)); + + let err = publish::execute(publish_args_for_test( + Some(BackendOverride::from_memory( + PassthroughBackend::instantiator(), + )), + Some(pixi.workspace_path().join("cpp")), + None, + )) + .await + .expect_err("--path on a package with source dependencies should fail"); + + let rendered = format_diagnostic(err.as_ref()); + assert!( + rendered.contains("cannot be published on its own") && rendered.contains("core"), + "{rendered}" + ); +} + +/// A package without `publish = true` is not published: a workspace whose +/// only package does not opt in must fail instead of falling back to the +/// closest package. +#[tokio::test] +async fn test_publish_without_opt_in_fails() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + let platform = Platform::current(); + fs::write( + pixi.manifest_path(), + format!( + r#" +[workspace] +channels = [] +platforms = ["{platform}"] +preview = ["pixi-build"] + +[package] +name = "solo" +version = "1.0.0" + +[package.build] +backend = {{ name = "in-memory", version = "0.1.0" }} +"# + ), + ) + .unwrap(); + + let err = publish_from_directory( + pixi.workspace_path(), + publish_args_for_test( + Some(BackendOverride::from_memory( + PassthroughBackend::instantiator(), + )), + None, + None, + ), + ) + .await + .expect_err("publishing a workspace whose packages do not opt in should fail"); + + let rendered = format_diagnostic(err.as_ref()); + assert!( + rendered.contains("no package in the workspace opts into publishing"), + "{rendered}" + ); +} + +/// A workspace without any package must fail with the same opt-in hint. +#[tokio::test] +async fn test_publish_workspace_without_packages_fails() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + fs::write( + pixi.manifest_path(), + format!( + r#" +[workspace] +channels = [] +platforms = ["{}"] +preview = ["pixi-build"] +"#, + Platform::current() + ), + ) + .unwrap(); + + let err = publish_from_directory( + pixi.workspace_path(), + publish_args_for_test( + Some(BackendOverride::from_memory( + PassthroughBackend::instantiator(), + )), + None, + None, + ), + ) + .await + .expect_err("publishing an empty workspace should fail"); + + let rendered = format_diagnostic(err.as_ref()); + assert!( + rendered.contains("no package in the workspace opts into publishing"), + "{rendered}" + ); +} + /// Regression test for #4761: `.pixi/.gitignore` must be created during /// `sanity_check_workspace`, even when the publish itself fails because the /// configured backend cannot be resolved. Without the gitignore, rattler-build @@ -2897,7 +3281,7 @@ backend.version = "0.1.0" target_channel: None, target_dir: None, force: false, - skip_existing: true, + no_skip_existing: false, generate_attestation: false, variant: Vec::new(), variant_config: Vec::new(), @@ -3021,7 +3405,7 @@ host-lib = "*" target_channel: None, target_dir: Some(target_dir.path().to_path_buf()), force: false, - skip_existing: true, + no_skip_existing: false, generate_attestation: false, variant: Vec::new(), variant_config: Vec::new(), diff --git a/crates/pixi_cli/Cargo.toml b/crates/pixi_cli/Cargo.toml index e4b25e6092..2117e438c1 100644 --- a/crates/pixi_cli/Cargo.toml +++ b/crates/pixi_cli/Cargo.toml @@ -35,6 +35,7 @@ fs-err = { workspace = true, features = ["tokio"] } futures = { workspace = true } hex = { workspace = true } human_bytes = { workspace = true } +ignore = { workspace = true } indexmap = { workspace = true, features = ["serde"] } indicatif = { workspace = true } is_executable = { workspace = true } @@ -46,6 +47,7 @@ pep508_rs = { workspace = true } pixi_api = { workspace = true } pixi_auth = { workspace = true } pixi_build_frontend = { workspace = true } +pixi_build_types = { workspace = true } pixi_command_dispatcher = { workspace = true } pixi_compute_reporters = { workspace = true } pixi_config = { workspace = true } diff --git a/crates/pixi_cli/src/build.rs b/crates/pixi_cli/src/build.rs index cdb856601a..c88294d771 100644 --- a/crates/pixi_cli/src/build.rs +++ b/crates/pixi_cli/src/build.rs @@ -60,6 +60,12 @@ pub struct Args { } pub async fn execute(args: Args) -> miette::Result<()> { + // `pixi build` predates workspace-wide publishing and always operated on + // a single package. Anchoring the publish invocation to the current + // directory when no `--path` is given keeps it that way instead of + // fanning out to every workspace member. + let path = args.path.unwrap_or_else(|| PathBuf::from(".")); + let mut cmd_parts = vec!["pixi publish".to_string()]; if args.target_platform != Platform::current() { @@ -74,9 +80,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { if args.clean { cmd_parts.push("--clean".to_string()); } - if let Some(ref path) = args.path { - cmd_parts.push(format!("--path {}", path.display())); - } + cmd_parts.push(format!("--path {}", path.display())); if args.output_dir != Path::new(".") { cmd_parts.push(format!("--target-dir {}", args.output_dir.display())); } @@ -102,11 +106,11 @@ pub async fn execute(args: Args) -> miette::Result<()> { build_number: None, build_dir: args.build_dir, clean: args.clean, - path: args.path, + path: Some(path), target_channel: None, target_dir: Some(args.output_dir), force: false, - skip_existing: true, + no_skip_existing: false, generate_attestation: false, variant: Vec::new(), variant_config: Vec::new(), diff --git a/crates/pixi_cli/src/publish/discovery.rs b/crates/pixi_cli/src/publish/discovery.rs new file mode 100644 index 0000000000..95474289a6 --- /dev/null +++ b/crates/pixi_cli/src/publish/discovery.rs @@ -0,0 +1,699 @@ +//! Discovery of the publish set for `pixi publish`. +//! +//! A workspace-wide `pixi publish` operates on every package in the workspace +//! that opts into publishing with `publish = true` in its `[package]` +//! section. Package manifests are discovered by walking the workspace +//! directory tree; the walk respects ignore files (such as `.gitignore`), +//! skips hidden directories, and skips subtrees that belong to a nested +//! workspace. +//! +//! Every publish is a closed batch: each source dependency (build, host, or +//! run) of every published package must itself be part of the batch. Source +//! dependencies that do not opt into publishing - whether they point at a +//! directory inside the workspace or at something external (git or url +//! sources, paths escaping the workspace root) - fail the publish. + +use std::{ + collections::{BTreeMap, BTreeSet}, + path::{Path, PathBuf}, +}; + +use miette::{Context, IntoDiagnostic}; +use pixi_build_types::{PackageSpec, SourcePackageSpec, procedures::conda_outputs::CondaOutput}; +use pixi_command_dispatcher::{ + BuildBackendMetadataSpec, CommandDispatcher, build::conversion::from_source_spec_v1, +}; +use pixi_consts::consts::{MOJOPROJECT_MANIFEST, PYPROJECT_MANIFEST, WORKSPACE_MANIFEST}; +use pixi_core::Workspace; +use pixi_record::{PinnedPathSpec, PinnedSourceSpec}; +use pixi_spec::{SourceAnchor, SourceLocationSpec}; +use typed_path::Utf8TypedPathBuf; + +/// The packages a workspace-wide `pixi publish` operates on, plus diagnostics +/// gathered while resolving them. +pub(crate) struct WorkspacePackageSet { + /// Workspace-relative package sources, ordered so that every package + /// appears after the packages it depends on (dependencies first). Upload + /// in this order never leaves the target channel with a package whose + /// run dependencies are missing. + pub packages: Vec, + + /// Members that had to be forced out of a dependency cycle. Uploads + /// involving these packages cannot be fully dependency-ordered. + pub cycle_members: Vec, +} + +/// Where a source dependency points, relative to the workspace. +enum SourceTarget { + /// A workspace-relative directory. + Member(String), + + /// A source that lives outside the workspace (git/url source or a path + /// escaping the workspace root). + External(String), +} + +/// Resolve the set of packages that opt into publishing with +/// `publish = true` and return them in dependency order (dependencies before +/// dependents). +/// +/// `make_metadata_spec` builds the per-package metadata request; the caller +/// owns the environment/channel/variant configuration that goes into it. The +/// metadata computed here is cached by the command dispatcher, so the +/// subsequent build phase does not pay for it twice. +pub(crate) async fn resolve_publish_set( + workspace: &Workspace, + command_dispatcher: &CommandDispatcher, + make_metadata_spec: impl Fn(PinnedSourceSpec) -> BuildBackendMetadataSpec, +) -> miette::Result { + let workspace_root = workspace.root(); + + let members = discover_opted_in_packages(workspace_root)?; + if members.is_empty() { + return Err(miette::diagnostic!( + help = "Set `publish = true` in the `[package]` section of every package that \ + `pixi publish` should publish. To publish a single package, pass \ + `--path `.", + "no package in the workspace opts into publishing", + ) + .into()); + } + + // Query each package's build backend metadata and validate the closure: + // every source dependency must itself opt into publishing. Output names + // must be unique across the set; the target cannot hold two different + // packages under the same name. + let mut member_dependencies: BTreeMap> = BTreeMap::new(); + let mut output_owners: BTreeMap = BTreeMap::new(); + let mut violations: BTreeSet = BTreeSet::new(); + for dir in &members { + let manifest_source: PinnedSourceSpec = PinnedPathSpec { + path: dir.clone().into(), + } + .into(); + let backend_metadata = command_dispatcher + .build_backend_metadata(make_metadata_spec(manifest_source.clone())) + .await?; + + // Relative path specs in the metadata are anchored to the package + // that declares them, and resolve to workspace-relative paths. + let anchor = SourceAnchor::from(SourceLocationSpec::from(manifest_source)); + + for output in &backend_metadata.metadata.outputs { + let output_name = output.metadata.name.as_normalized().to_string(); + if let Some(owner) = output_owners.get(&output_name) { + if owner != dir { + return Err(miette::diagnostic!( + help = "Packages in the publish set must have unique names. Remove \ + `publish = true` from one of the two packages.", + "packages '{owner}' and '{dir}' both produce an output named '{output_name}'", + ) + .into()); + } + } else { + output_owners.insert(output_name, dir.clone()); + } + + for (_name, source_spec) in output_source_dependencies(output) { + let resolved = anchor.resolve(from_source_spec_v1(source_spec.clone())); + match classify_source_location(workspace_root, resolved.location) { + SourceTarget::Member(dep_dir) => { + if members.contains(&dep_dir) { + if dep_dir != *dir { + member_dependencies + .entry(dir.clone()) + .or_default() + .insert(dep_dir); + } + } else { + violations.insert(format!( + "package '{dir}' depends on '{dep_dir}', which is not part of the publish set" + )); + } + } + SourceTarget::External(location) => { + violations.insert(format!( + "package '{dir}' depends on '{location}', which lives outside the workspace" + )); + } + } + } + } + } + + if !violations.is_empty() { + return Err(miette::diagnostic!( + help = "Every source dependency of a published package must itself be published \ + in the same batch. Set `publish = true` in the `[package]` section of \ + the missing packages, or replace the source dependencies with binary \ + dependencies.", + "the packages cannot be published as a self-contained set:\n {}", + violations.iter().cloned().collect::>().join("\n "), + ) + .into()); + } + + let ordering = dependency_order(&members, &member_dependencies); + let packages = ordering + .order + .into_iter() + .map(|dir| PinnedPathSpec { path: dir.into() }.into()) + .collect(); + + Ok(WorkspacePackageSet { + packages, + cycle_members: ordering.cycle_members, + }) +} + +/// All source dependencies of a backend output as `(name, spec)` pairs: +/// build, host, and run dependencies plus the dependencies of every extra +/// group. +pub(super) fn output_source_dependencies( + output: &CondaOutput, +) -> impl Iterator { + let dependency_sets = output + .build_dependencies + .iter() + .chain(output.host_dependencies.iter()) + .chain(std::iter::once(&output.run_dependencies)); + dependency_sets + .flat_map(|deps| deps.depends.iter()) + .chain(output.extra_dependencies.values().flatten()) + .filter_map(|named| match &named.spec { + PackageSpec::Source(source) => Some((named.name.as_str(), source)), + PackageSpec::Binary(_) | PackageSpec::PinCompatible(_) => None, + }) +} + +/// The publish-relevant contents of a manifest, read with a lightweight TOML +/// peek instead of a full manifest parse. +struct ManifestPeek { + /// The manifest declares a workspace section (`[workspace]`, or the + /// legacy `[project]` alias). + declares_workspace: bool, + + /// The value of `publish` in the package section, if the manifest + /// declares one. + publish: Option, +} + +/// A pixi manifest found while walking the workspace directory tree. +struct DiscoveredManifest { + /// Absolute directory containing the manifest. + dir: PathBuf, + peek: ManifestPeek, +} + +/// Read the publish-relevant keys of a manifest. For pyproject manifests the +/// pixi sections live under `[tool.pixi]`. +fn peek_manifest(manifest_path: &Path) -> miette::Result { + let source = fs_err::read_to_string(manifest_path).into_diagnostic()?; + let doc = source + .parse::() + .into_diagnostic() + .with_context(|| format!("failed to parse '{}'", manifest_path.display()))?; + + let is_pyproject = manifest_path + .file_name() + .is_some_and(|name| name == PYPROJECT_MANIFEST); + let root: Option<&toml_edit::Item> = if is_pyproject { + doc.get("tool").and_then(|tool| tool.get("pixi")) + } else { + Some(doc.as_item()) + }; + let Some(root) = root else { + return Ok(ManifestPeek { + declares_workspace: false, + publish: None, + }); + }; + + let declares_workspace = root.get("workspace").is_some() || root.get("project").is_some(); + let publish = match root + .get("package") + .and_then(|package| package.get("publish")) + { + None => None, + Some(item) => Some(item.as_bool().ok_or_else(|| { + miette::diagnostic!( + "the `publish` key of the package in '{}' must be a boolean", + manifest_path.display(), + ) + })?), + }; + + Ok(ManifestPeek { + declares_workspace, + publish, + }) +} + +/// Precedence of manifest files within one directory: a `pixi.toml` shadows a +/// `pyproject.toml`, which shadows a `mojoproject.toml`. +fn manifest_priority(file_name: &str) -> Option { + [WORKSPACE_MANIFEST, PYPROJECT_MANIFEST, MOJOPROJECT_MANIFEST] + .iter() + .position(|name| *name == file_name) +} + +/// Walk the workspace directory tree and collect the member directories of +/// every package that opts into publishing with `publish = true`. +/// +/// The walk respects ignore files (such as `.gitignore`) and skips hidden +/// directories. A manifest in a subdirectory that declares its own workspace +/// starts a nested, independent workspace: its entire subtree is skipped. +fn discover_opted_in_packages(workspace_root: &Path) -> miette::Result> { + // The highest-priority manifest of every visited directory. + let mut manifest_paths: BTreeMap = BTreeMap::new(); + for entry in ignore::WalkBuilder::new(workspace_root) + .require_git(false) + .build() + { + let entry = match entry { + Ok(entry) => entry, + Err(error) => { + tracing::debug!("skipping unreadable entry while discovering packages: {error}"); + continue; + } + }; + if !entry.file_type().is_some_and(|kind| kind.is_file()) { + continue; + } + let Some(priority) = entry.file_name().to_str().and_then(manifest_priority) else { + continue; + }; + let Some(dir) = entry.path().parent() else { + continue; + }; + match manifest_paths.entry(dir.to_path_buf()) { + std::collections::btree_map::Entry::Vacant(vacant) => { + vacant.insert((priority, entry.path().to_path_buf())); + } + std::collections::btree_map::Entry::Occupied(mut occupied) => { + if priority < occupied.get().0 { + occupied.insert((priority, entry.path().to_path_buf())); + } + } + } + } + + let mut discovered = Vec::with_capacity(manifest_paths.len()); + for (dir, (_priority, manifest_path)) in manifest_paths { + discovered.push(DiscoveredManifest { + dir, + peek: peek_manifest(&manifest_path)?, + }); + } + + // A workspace declaration below the root starts a nested workspace; its + // packages opt into publishing for that workspace, not for this one. + let nested_workspace_roots: Vec = discovered + .iter() + .filter(|manifest| manifest.peek.declares_workspace && manifest.dir != workspace_root) + .map(|manifest| manifest.dir.clone()) + .collect(); + + let mut members = BTreeSet::new(); + for manifest in discovered { + if manifest.peek.publish != Some(true) { + continue; + } + if nested_workspace_roots + .iter() + .any(|nested_root| manifest.dir.starts_with(nested_root)) + { + continue; + } + let Some(relative) = pathdiff::diff_paths(&manifest.dir, workspace_root) else { + continue; + }; + members.insert(member_dir_from_relative_path(&relative)); + } + + Ok(members) +} + +/// Canonical member-directory form of a workspace-root-relative filesystem +/// path. The workspace root itself is represented as `.`. +fn member_dir_from_relative_path(path: &Path) -> String { + normalize_member_dir(native_relative_path_to_member(path)) +} + +/// Convert a relative filesystem path into the forward-slash form used to +/// identify members, so the same directory referenced through a manifest path +/// spec and through a filesystem path (backslash-separated on Windows) +/// compares equal. +fn native_relative_path_to_member(path: &Path) -> Utf8TypedPathBuf { + Utf8TypedPathBuf::from( + path.to_string_lossy() + .replace(std::path::MAIN_SEPARATOR, "/"), + ) +} + +/// Determine whether a workspace-anchored source location refers to a +/// directory inside the workspace or to something external. +fn classify_source_location(workspace_root: &Path, location: SourceLocationSpec) -> SourceTarget { + let SourceLocationSpec::Path(path_spec) = &location else { + return SourceTarget::External(location.to_string()); + }; + + let path = path_spec.path.to_path(); + if path.is_absolute() || path.starts_with("~") { + // Absolute paths can still point into the workspace; try to re-anchor + // them. `~` cannot be resolved without the filesystem, treat it as + // external. + let std_path = Path::new(path_spec.path.as_str()); + let (Ok(canonical_path), Ok(canonical_root)) = ( + dunce::canonicalize(std_path), + dunce::canonicalize(workspace_root), + ) else { + return SourceTarget::External(location.to_string()); + }; + return match canonical_path.strip_prefix(&canonical_root) { + Ok(relative) => SourceTarget::Member(member_dir_for( + workspace_root, + native_relative_path_to_member(relative), + )), + Err(_) => SourceTarget::External(location.to_string()), + }; + } + + // Workspace-relative path. A normalized path that still starts with `..` + // escapes the workspace root. + let as_str = path_spec.path.as_str(); + if as_str == ".." || as_str.starts_with("../") || as_str.starts_with("..\\") { + return SourceTarget::External(location.to_string()); + } + + SourceTarget::Member(member_dir_for(workspace_root, path_spec.path.clone())) +} + +/// Reduce a workspace-relative source path to the directory that identifies +/// the member package: paths that point at a manifest file are replaced by +/// their parent directory so that `pkg/pixi.toml` and `pkg` name the same +/// member. +fn member_dir_for(workspace_root: &Path, path: Utf8TypedPathBuf) -> String { + let is_file = workspace_root.join(path.as_str()).is_file(); + let dir = if is_file { + path.to_path() + .parent() + .map(|parent| parent.to_path_buf()) + .unwrap_or(path) + } else { + path + }; + normalize_member_dir(dir) +} + +/// Canonical string form of a member directory. The workspace root itself is +/// represented as `.`. +fn normalize_member_dir(dir: Utf8TypedPathBuf) -> String { + let dir = dir.as_str().trim_end_matches(['/', '\\']); + if dir.is_empty() || dir == "." { + ".".to_string() + } else { + dir.to_string() + } +} + +/// The result of ordering members by their dependencies. +pub(super) struct DependencyOrdering { + /// Members with dependencies before their dependents. + pub order: Vec, + + /// Members that were forced out of a dependency cycle. When this is + /// non-empty the order is best-effort: a forced member is emitted before + /// its dependencies are complete. + pub cycle_members: Vec, +} + +/// Order members so that dependencies come before their dependents. Should a +/// dependency cycle exist, one member of the cycle is forced out instead of +/// failing; members that merely depend on the cycle still come after it. The +/// forced members are reported so the caller can warn the user. +/// +/// Also used by the publish flow to order the outputs of a single +/// multi-output package. +pub(super) fn dependency_order( + members: &BTreeSet, + dependencies: &BTreeMap>, +) -> DependencyOrdering { + let mut order = Vec::with_capacity(members.len()); + let mut cycle_members = Vec::new(); + let mut emitted: BTreeSet = BTreeSet::new(); + let mut remaining: BTreeSet = members.clone(); + + while !remaining.is_empty() { + let ready: Vec = remaining + .iter() + .filter(|member| { + dependencies.get(*member).is_none_or(|deps| { + deps.iter() + .all(|dep| emitted.contains(dep) || !members.contains(dep)) + }) + }) + .cloned() + .collect(); + + if ready.is_empty() { + let member = find_cycle_member(&remaining, dependencies); + remaining.remove(&member); + emitted.insert(member.clone()); + cycle_members.push(member.clone()); + order.push(member); + continue; + } + + for member in ready { + remaining.remove(&member); + emitted.insert(member.clone()); + order.push(member); + } + } + + DependencyOrdering { + order, + cycle_members, + } +} + +/// Pick a member that lies on a dependency cycle among `remaining`. +/// +/// Walks unmet dependencies (always the lexicographically first one, for +/// determinism) starting from the first remaining member; the first member +/// visited twice is on a cycle. Only called when no member is ready, which +/// guarantees every walk eventually revisits a member. +fn find_cycle_member( + remaining: &BTreeSet, + dependencies: &BTreeMap>, +) -> String { + let mut current = remaining + .first() + .expect("caller ensures remaining is not empty") + .clone(); + let mut visited: BTreeSet = BTreeSet::new(); + visited.insert(current.clone()); + loop { + let next = dependencies + .get(¤t) + .and_then(|deps| deps.iter().find(|dep| remaining.contains(*dep))) + .cloned(); + match next { + // No unmet dependency: the walk cannot continue, treat the + // current member as the one to force out. + None => return current, + Some(next) => { + if !visited.insert(next.clone()) { + return next; + } + current = next; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn set(items: &[&str]) -> BTreeSet { + items.iter().map(|s| s.to_string()).collect() + } + + fn write_manifest(root: &Path, dir: &str, contents: &str) { + let dir = root.join(dir); + fs_err::create_dir_all(&dir).unwrap(); + fs_err::write(dir.join("pixi.toml"), contents).unwrap(); + } + + #[test] + fn dependency_order_puts_dependencies_first() { + let members = set(&["kit", "core", "cpp"]); + let mut deps = BTreeMap::new(); + deps.insert("kit".to_string(), set(&["cpp"])); + deps.insert("cpp".to_string(), set(&["core"])); + + let ordering = dependency_order(&members, &deps); + assert_eq!(ordering.order, vec!["core", "cpp", "kit"]); + assert!(ordering.cycle_members.is_empty()); + } + + #[test] + fn dependency_order_ignores_non_member_dependencies() { + let members = set(&["a"]); + let mut deps = BTreeMap::new(); + deps.insert("a".to_string(), set(&["not-a-member"])); + + let ordering = dependency_order(&members, &deps); + assert_eq!(ordering.order, vec!["a"]); + assert!(ordering.cycle_members.is_empty()); + } + + #[test] + fn dependency_order_survives_cycles() { + let members = set(&["a", "b", "c"]); + let mut deps = BTreeMap::new(); + deps.insert("a".to_string(), set(&["b"])); + deps.insert("b".to_string(), set(&["a"])); + deps.insert("c".to_string(), set(&["a"])); + + let ordering = dependency_order(&members, &deps); + // One cycle member is forced out, then the rest resolves normally; + // `c` only depends on `a` and follows it. + assert_eq!(ordering.order, vec!["a", "b", "c"]); + assert_eq!(ordering.cycle_members, vec!["a"]); + } + + #[test] + fn dependency_order_emits_cycle_before_its_dependents() { + // `0-app` sorts before the cycle members but merely depends on the + // cycle; it must still be emitted after its dependency `a`. + let members = set(&["0-app", "a", "b"]); + let mut deps = BTreeMap::new(); + deps.insert("0-app".to_string(), set(&["a"])); + deps.insert("a".to_string(), set(&["b"])); + deps.insert("b".to_string(), set(&["a"])); + + let ordering = dependency_order(&members, &deps); + // `a` is forced out of the cycle first; `0-app` and `b` both become + // ready and follow in name order. + assert_eq!(ordering.order, vec!["a", "0-app", "b"]); + assert_eq!(ordering.cycle_members, vec!["a"]); + } + + #[test] + fn normalize_member_dir_maps_root_to_dot() { + assert_eq!(normalize_member_dir(Utf8TypedPathBuf::from("")), "."); + assert_eq!(normalize_member_dir(Utf8TypedPathBuf::from(".")), "."); + assert_eq!(normalize_member_dir(Utf8TypedPathBuf::from("pkg/")), "pkg"); + } + + #[test] + fn discovery_collects_only_packages_that_opt_in() { + let root = tempfile::tempdir().unwrap(); + let root_path = root.path(); + + write_manifest( + root_path, + ".", + "[workspace]\nchannels = []\n\n[package]\nname = \"root\"\npublish = true\n", + ); + write_manifest( + root_path, + "packages/foo", + "[package]\nname = \"foo\"\npublish = true\n", + ); + write_manifest( + root_path, + "packages/bar", + "[package]\nname = \"bar\"\npublish = false\n", + ); + write_manifest(root_path, "packages/baz", "[package]\nname = \"baz\"\n"); + + let members = discover_opted_in_packages(root_path).unwrap(); + assert_eq!(members, set(&[".", "packages/foo"])); + } + + #[test] + fn discovery_skips_nested_workspaces() { + let root = tempfile::tempdir().unwrap(); + let root_path = root.path(); + + write_manifest(root_path, ".", "[workspace]\nchannels = []\n"); + write_manifest( + root_path, + "packages/foo", + "[package]\nname = \"foo\"\npublish = true\n", + ); + write_manifest( + root_path, + "examples/demo", + "[workspace]\nchannels = []\n\n[package]\nname = \"demo\"\npublish = true\n", + ); + write_manifest( + root_path, + "examples/demo/member", + "[package]\nname = \"member\"\npublish = true\n", + ); + + let members = discover_opted_in_packages(root_path).unwrap(); + assert_eq!(members, set(&["packages/foo"])); + } + + #[test] + fn discovery_respects_ignore_files() { + let root = tempfile::tempdir().unwrap(); + let root_path = root.path(); + + write_manifest(root_path, ".", "[workspace]\nchannels = []\n"); + write_manifest( + root_path, + "packages/foo", + "[package]\nname = \"foo\"\npublish = true\n", + ); + write_manifest( + root_path, + "vendored/dep", + "[package]\nname = \"dep\"\npublish = true\n", + ); + fs_err::write(root_path.join(".gitignore"), "vendored/\n").unwrap(); + + let members = discover_opted_in_packages(root_path).unwrap(); + assert_eq!(members, set(&["packages/foo"])); + } + + #[test] + fn discovery_reads_pyproject_manifests() { + let root = tempfile::tempdir().unwrap(); + let root_path = root.path(); + + write_manifest(root_path, ".", "[workspace]\nchannels = []\n"); + let dir = root_path.join("packages/py"); + fs_err::create_dir_all(&dir).unwrap(); + fs_err::write( + dir.join("pyproject.toml"), + "[tool.pixi.package]\nname = \"py\"\npublish = true\n", + ) + .unwrap(); + + let members = discover_opted_in_packages(root_path).unwrap(); + assert_eq!(members, set(&["packages/py"])); + } + + #[test] + fn discovery_rejects_non_boolean_publish() { + let root = tempfile::tempdir().unwrap(); + let root_path = root.path(); + + write_manifest(root_path, ".", "[workspace]\nchannels = []\n"); + write_manifest( + root_path, + "packages/foo", + "[package]\nname = \"foo\"\npublish = \"yes\"\n", + ); + + let err = discover_opted_in_packages(root_path).unwrap_err(); + assert!(err.to_string().contains("must be a boolean")); + } +} diff --git a/crates/pixi_cli/src/publish.rs b/crates/pixi_cli/src/publish/mod.rs similarity index 82% rename from crates/pixi_cli/src/publish.rs rename to crates/pixi_cli/src/publish/mod.rs index 358f30188f..51e107c87f 100644 --- a/crates/pixi_cli/src/publish.rs +++ b/crates/pixi_cli/src/publish/mod.rs @@ -37,11 +37,18 @@ use rattler_conda_types::{GenericVirtualPackage, Platform}; use rattler_networking::{AuthenticationStorage, s3_middleware}; use rattler_package_streaming::seek::read_package_file; -/// Build a conda package and publish it to a channel. +mod discovery; + +/// Build the conda packages of a workspace and publish them to a channel. /// -/// Builds the package from your workspace and either uploads it to a channel -/// (`--target-channel`) or copies the artifact into a local directory -/// (`--target-dir`). +/// Builds every package in the workspace that opts into publishing with +/// `publish = true` in its `[package]` section - in dependency order, so +/// packages that depend on other workspace packages are built after them - +/// and either uploads the artifacts to a channel (`--target-channel`) or +/// copies them into a local directory (`--target-dir`). Every source +/// dependency of a published package must itself opt into publishing; the +/// publish fails otherwise. Use `--path` to build and publish a single +/// self-contained package instead. /// /// Supported destinations for `--target-channel` (alias `--to`): /// - prefix.dev: `https://prefix.dev/` @@ -94,6 +101,10 @@ pub struct Args { /// The path to a directory containing a package manifest, or to a specific manifest file. /// + /// When given, only that package is built and published - whether or not + /// it sets `publish = true`. The package must be self-contained: + /// publishing it alone fails if it has any source dependencies. + /// /// Supported manifest files: `package.xml`, `recipe.yaml`, `pixi.toml`, `pyproject.toml`, or `mojoproject.toml`. #[arg(long)] pub path: Option, @@ -114,10 +125,13 @@ pub struct Args { #[arg(long)] pub force: bool, - /// Skip uploading packages that already exist at the target. - /// This is enabled by default. Use `--no-skip-existing` to disable. - #[arg(long, default_value_t = true, action = clap::ArgAction::Set)] - pub skip_existing: bool, + /// Do not skip packages that already exist at the target. + /// + /// Packages that already exist at the target are skipped by default. + /// With this flag they fail the publish instead, unless `--force` is + /// also given to overwrite them. + #[arg(long)] + pub no_skip_existing: bool, /// Generate sigstore attestation (prefix.dev only) #[arg(long)] @@ -518,7 +532,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { let mut workspace = WorkspaceLocator::for_cli() .with_global_config_source(args.config_source.source()) .with_search_start(workspace_locator.clone()) - .with_closest_package(false) + .with_closest_package(true) .locate()? .with_cli_config(args.config_cli); if let Some(backend_override) = args.backend_override.clone() { @@ -527,10 +541,13 @@ pub async fn execute(args: Args) -> miette::Result<()> { sanity_check_workspace(&workspace).await?; + // `--force` requests an overwrite, so existing packages must not be + // skipped even though skipping is the default. + let skip_existing = !args.no_skip_existing && !args.force; let ctx = PublishContext::new( &workspace, args.force, - args.skip_existing, + skip_existing, args.generate_attestation, )?; @@ -604,70 +621,162 @@ pub async fn execute(args: Args) -> miette::Result<()> { host_virtual_packages, }; - let Ok(manifest_path) = workspace_locator.path() else { - miette::bail!("could not determine the current working directory to locate the workspace"); + let channel_config = workspace.channel_config(); + let channels = workspace + .default_environment() + .channel_urls(&channel_config) + .into_diagnostic()?; + + // When running `pixi publish`, the exclude_newer config is ignored; + // it only matters when using the package as a source dependency. + let make_env_ref = |manifest_source: &PinnedSourceSpec| { + EnvironmentRef::Ephemeral(EphemeralEnv::new( + manifest_source.to_string(), + EnvironmentSpec { + channels: channels.clone(), + build_environment: build_environment.clone(), + variants: pixi_utils::variants::VariantConfig { + variant_configuration: variant_configuration.clone(), + variant_files: variant_files.clone(), + }, + exclude_newer: None, + channel_priority: Default::default(), + }, + )) + }; + let make_metadata_spec = |manifest_source: PinnedSourceSpec| { + let env_ref = make_env_ref(&manifest_source); + BuildBackendMetadataSpec { + manifest_source, + preferred_build_source: None, + env_ref, + build_string_prefix: args.build_string_prefix.clone(), + build_number: args.build_number, + inline: None, + } }; - let package_manifest_path = match args.path { + // Determine which packages to publish: a single one when `--path` is + // given, otherwise every package in the workspace that opts in with + // `publish = true`, ordered so that dependencies are built and uploaded + // before their dependents. + let single_package_mode = args.path.is_some(); + let (package_sources, cycle_members) = match &args.path { Some(path) => { - validate_package_manifest(&path).await?; - path + validate_package_manifest(path).await?; + let package_manifest_path_canonical = dunce::canonicalize(path) + .into_diagnostic() + .with_context(|| { + format!("failed to canonicalize manifest path '{}'", path.display()) + })?; + let manifest_path_spec = + pathdiff::diff_paths(&package_manifest_path_canonical, workspace.root()) + .unwrap_or_else(|| package_manifest_path_canonical.clone()); + let manifest_source: PinnedSourceSpec = PinnedPathSpec { + path: manifest_path_spec.to_string_lossy().into_owned().into(), + } + .into(); + (vec![manifest_source], Vec::new()) + } + None => { + let resolved = discovery::resolve_publish_set( + &workspace, + &command_dispatcher, + &make_metadata_spec, + ) + .await?; + (resolved.packages, resolved.cycle_members) } - None => manifest_path.clone(), }; - let package_manifest_path_canonical = dunce::canonicalize(&package_manifest_path) - .into_diagnostic() - .with_context(|| { - format!( - "failed to canonicalize manifest path '{}'", - package_manifest_path.display() - ) - })?; + if !cycle_members.is_empty() { + pixi_progress::println!( + "{}dependency cycle among workspace packages involving {}; uploads cannot be fully \ + dependency-ordered, so the target may be inconsistent until every upload finishes", + console::style(console::Emoji("⚠️ ", "warning: ")).yellow(), + cycle_members.join(", "), + ); + } - let manifest_path_spec = - pathdiff::diff_paths(&package_manifest_path_canonical, workspace.root()) - .unwrap_or_else(|| package_manifest_path_canonical.to_path_buf()); + // Fetch the backend metadata of every package. For packages that were + // just resolved from the publish set this is a cache hit. Packages + // without any output for the target platform are skipped. + let mut package_plans = Vec::with_capacity(package_sources.len()); + for manifest_source in package_sources { + let backend_metadata = command_dispatcher + .build_backend_metadata(make_metadata_spec(manifest_source.clone())) + .await?; + if backend_metadata.metadata.outputs.is_empty() { + pixi_progress::println!( + "{}skipping '{}': no outputs for platform {}", + console::style(console::Emoji("ℹ️ ", "")).blue(), + manifest_source, + args.target_platform, + ); + continue; + } + package_plans.push((manifest_source, backend_metadata)); + } - let channel_config = workspace.channel_config(); - let channels = workspace - .default_environment() - .channel_urls(&channel_config) - .into_diagnostic()?; + if package_plans.is_empty() { + miette::bail!( + "no package produces outputs for platform {}", + args.target_platform + ); + } - let manifest_source: PinnedSourceSpec = PinnedPathSpec { - path: manifest_path_spec.to_string_lossy().into_owned().into(), + // A single-package publish is a batch of one, so the closure rule demands + // that it has no source dependencies beyond its own outputs: nothing else + // in the batch could satisfy them on the target. Dependencies on sibling + // outputs (e.g. through `pin_subpackage`) are published together with the + // package and are therefore fine. + if single_package_mode { + for (manifest_source, backend_metadata) in &package_plans { + let output_names: BTreeSet = backend_metadata + .metadata + .outputs + .iter() + .map(|output| output.metadata.name.as_normalized().to_string()) + .collect(); + let source_dependencies: BTreeSet<&str> = backend_metadata + .metadata + .outputs + .iter() + .flat_map(discovery::output_source_dependencies) + .map(|(name, _)| name) + .filter(|name| !output_names.contains(&name.to_lowercase())) + .collect(); + if !source_dependencies.is_empty() { + return Err(miette::diagnostic!( + help = "A single-package publish must be self-contained. Set \ + `publish = true` in the `[package]` section of the package and \ + its source dependencies, then run `pixi publish` without `--path` \ + to publish them together.", + "package '{}' has source dependencies ({}) and cannot be published on its own", + manifest_source, + source_dependencies + .iter() + .copied() + .collect::>() + .join(", "), + ) + .into()); + } + } } - .into(); - // When running `pixi publish`, the exclude_newer config is ignored; - // it only matters when using the package as a source dependency. - let env_ref = EnvironmentRef::Ephemeral(EphemeralEnv::new( - manifest_source.to_string(), - EnvironmentSpec { - channels: channels.clone(), - build_environment: build_environment.clone(), - variants: pixi_utils::variants::VariantConfig { - variant_configuration: variant_configuration.clone(), - variant_files: variant_files.clone(), - }, - exclude_newer: None, - channel_priority: Default::default(), - }, - )); - let backend_metadata_spec = BuildBackendMetadataSpec { - manifest_source: manifest_source.clone(), - preferred_build_source: None, - env_ref: env_ref.clone(), - build_string_prefix: args.build_string_prefix.clone(), - build_number: args.build_number, - inline: None, - }; - let backend_metadata = command_dispatcher - .build_backend_metadata(backend_metadata_spec.clone()) - .await?; + if package_plans.len() > 1 { + pixi_progress::println!( + "\n{}Publishing {} workspace packages that set `publish = true`", + console::style(console::Emoji("🔍 ", "")).cyan(), + package_plans.len(), + ); + } - let packages = &backend_metadata.metadata.outputs; + let packages: Vec<_> = package_plans + .iter() + .flat_map(|(_, backend_metadata)| backend_metadata.metadata.outputs.iter()) + .collect(); // The CondaOutput metadata uses `pixi_build_types::VariantValue`, while the // rest of the publish flow (and our helpers) work with the @@ -712,34 +821,95 @@ pub async fn execute(args: Args) -> miette::Result<()> { pixi_progress::println!(""); // Pre-resolve a SourceRecord per unique package name via RSP; each - // returned variant becomes a separate SourceBuildKey invocation. - let unique_names: BTreeSet<_> = packages.iter().map(|p| p.metadata.name.clone()).collect(); - let source_location: SourceLocationSpec = manifest_source.clone().into(); + // returned variant becomes a separate SourceBuildKey invocation. The + // records keep the dependency order of `package_plans`, so dependencies + // are built and uploaded before the packages that depend on them. let mut resolved_records = Vec::new(); - for name in unique_names { - let rsp = ResolveSourcePackageSpec { - package: name, - source_location: source_location.clone(), - preferred_build_source: Arc::new(BTreeMap::new()), - env_ref: env_ref.clone(), - inline: None, - installed_source_hints: Default::default(), - }; - let records = command_dispatcher - .engine() - .compute(&ResolveSourcePackageKey::new(rsp)) - .await - .map_err_into_dispatcher(std::convert::identity) - .into_diagnostic()?; - resolved_records.extend(records.iter().cloned()); + for (manifest_source, backend_metadata) in &package_plans { + let unique_names: BTreeSet<_> = backend_metadata + .metadata + .outputs + .iter() + .map(|p| p.metadata.name.clone()) + .collect(); + + // Order the outputs of this package so that an output consumed by a + // sibling output (e.g. through `pin_subpackage`) is built and + // uploaded before its dependents; iterating `unique_names` directly + // would order them alphabetically. + let output_names: BTreeSet = unique_names + .iter() + .map(|name| name.as_normalized().to_string()) + .collect(); + let mut output_dependencies: BTreeMap> = BTreeMap::new(); + for output in &backend_metadata.metadata.outputs { + let name = output.metadata.name.as_normalized().to_string(); + let sibling_deps: BTreeSet = output + .build_dependencies + .iter() + .chain(output.host_dependencies.iter()) + .chain(std::iter::once(&output.run_dependencies)) + .flat_map(|deps| deps.depends.iter()) + .map(|named| named.name.as_str().to_lowercase()) + .filter(|dep| *dep != name && output_names.contains(dep)) + .collect(); + if !sibling_deps.is_empty() { + output_dependencies + .entry(name) + .or_default() + .extend(sibling_deps); + } + } + let mut by_normalized_name: BTreeMap = unique_names + .into_iter() + .map(|name| (name.as_normalized().to_string(), name)) + .collect(); + + let source_location: SourceLocationSpec = manifest_source.clone().into(); + let env_ref = make_env_ref(manifest_source); + let output_ordering = discovery::dependency_order(&output_names, &output_dependencies); + if !output_ordering.cycle_members.is_empty() { + pixi_progress::println!( + "{}dependency cycle among the outputs of '{}' involving {}; uploads cannot be \ + fully dependency-ordered", + console::style(console::Emoji("⚠️ ", "warning: ")).yellow(), + manifest_source, + output_ordering.cycle_members.join(", "), + ); + } + for name in output_ordering.order { + let name = by_normalized_name + .remove(&name) + .expect("dependency_order returns exactly the members it was given"); + let rsp = ResolveSourcePackageSpec { + package: name, + source_location: source_location.clone(), + preferred_build_source: Arc::new(BTreeMap::new()), + env_ref: env_ref.clone(), + inline: None, + installed_source_hints: Default::default(), + }; + let records = command_dispatcher + .engine() + .compute(&ResolveSourcePackageKey::new(rsp)) + .await + .map_err_into_dispatcher(std::convert::identity) + .into_diagnostic()?; + resolved_records.extend(records.iter().cloned()); + } } // `--clean` nukes the per-package artifact + workspace caches so - // the upcoming SourceBuildKey calls rebuild from scratch. + // the upcoming SourceBuildKey calls rebuild from scratch. Clear before + // any build starts so a build never invalidates an earlier one. if args.clean { - for record in &resolved_records { + let clean_names: BTreeSet<_> = resolved_records + .iter() + .map(|record| record.data.package_record.name.clone()) + .collect(); + for name in clean_names { command_dispatcher - .clear_source_build_cache(&record.data.package_record.name) + .clear_source_build_cache(&name) .into_diagnostic()?; } } @@ -748,6 +918,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { // built with, so the publish summary can attribute every artifact back to // the variant that produced it. let mut built_packages: Vec<(PathBuf, BTreeMap)> = Vec::new(); + let mut seen_artifacts: BTreeSet = BTreeSet::new(); for record in resolved_records { let record = Arc::unwrap_or_clone(record); @@ -780,7 +951,9 @@ pub async fn execute(args: Args) -> miette::Result<()> { let package_path = dunce::canonicalize(&built.artifact) .expect("failed to canonicalize output file which must now exist"); - built_packages.push((package_path, variants)); + if seen_artifacts.insert(package_path.clone()) { + built_packages.push((package_path, variants)); + } } // Drop the dispatcher (and its repodata gateway) before indexing. The @@ -1309,7 +1482,9 @@ async fn upload_to_s3( write_shards: true, repodata_revisions: vec![], package_revision_assignment: Default::default(), - force: false, + // `--force` may have replaced an existing artifact under its old + // filename; only a forced index re-extracts its metadata. + force: ctx.force, max_parallel: std::thread::available_parallelism() .map(|p| p.get()) .unwrap_or(1), @@ -1386,7 +1561,9 @@ async fn upload_to_local_filesystem_channel( write_shards: true, repodata_revisions: vec![], package_revision_assignment: Default::default(), - force: false, + // `--force` may have replaced an existing artifact under its old + // filename; only a forced index re-extracts its metadata. + force: ctx.force, max_parallel: std::thread::available_parallelism() .map(|p| p.get()) .unwrap_or(1), diff --git a/crates/pixi_glob/Cargo.toml b/crates/pixi_glob/Cargo.toml index 9bbb8e3eda..1dd1d4510f 100644 --- a/crates/pixi_glob/Cargo.toml +++ b/crates/pixi_glob/Cargo.toml @@ -12,7 +12,7 @@ version = "0.1.0" [dependencies] dashmap = { workspace = true } fs-err = { workspace = true } -ignore = "0.4" +ignore = { workspace = true } itertools = { workspace = true } memchr = { workspace = true } parking_lot = { workspace = true } diff --git a/docs/build/workspace.md b/docs/build/workspace.md index 508471afcd..a6c24491bb 100644 --- a/docs/build/workspace.md +++ b/docs/build/workspace.md @@ -103,6 +103,29 @@ each member opt in per entry with `{ workspace = true }`. See [Workspace Dependencies](workspace_dependencies.md) for the syntax, override rules, and error semantics. +## Publishing the Workspace + +To publish a workspace's packages, opt each of them in with `publish = true` +in its `[package]` section: + +```toml title="packages/cpp_math/pixi.toml" +[package] +name = "cpp_math" +publish = true +``` + +`pixi publish` walks the workspace directory tree, finds every package that +opts in, and builds and uploads them in dependency order. +The discovery respects ignore files such as `.gitignore` and skips +subdirectories that contain their own workspace. +The set must be self-contained: every source dependency of a published +package has to opt in as well, and `pixi publish` fails otherwise. +This guarantees that the target channel never ends up with a package whose +dependencies were not uploaded. + +See [`pixi publish`](../reference/cli/pixi/publish.md) for the full +behavior, including single-package publishes with `--path`. + ## Conclusion In this tutorial, we created a Pixi workspace containing two packages. diff --git a/docs/reference/cli/pixi.md b/docs/reference/cli/pixi.md index e6b8c7bb9c..cb029113e0 100644 --- a/docs/reference/cli/pixi.md +++ b/docs/reference/cli/pixi.md @@ -28,7 +28,7 @@ pixi [OPTIONS] [COMMAND] | [`list`](pixi/list.md) | List the packages of the current workspace | | [`lock`](pixi/lock.md) | Solve environment and update the lock file without installing the environments | | [`reinstall`](pixi/reinstall.md) | Re-install an environment, both updating the lock file and re-installing the environment | -| [`publish`](pixi/publish.md) | Build a conda package and publish it to a channel. | +| [`publish`](pixi/publish.md) | Build the conda packages of a workspace and publish them to a channel. | | [`remove`](pixi/remove.md) | Removes dependencies from the workspace | | [`run`](pixi/run.md) | Runs task in the pixi environment | | [`search`](pixi/search.md) | Search a conda package | diff --git a/docs/reference/cli/pixi/publish.md b/docs/reference/cli/pixi/publish.md index 2e355363d0..8f7cf2e345 100644 --- a/docs/reference/cli/pixi/publish.md +++ b/docs/reference/cli/pixi/publish.md @@ -4,7 +4,7 @@ title: pixi publish # [pixi](../pixi.md) publish -Build a conda package and publish it to a channel. +Build the conda packages of a workspace and publish them to a channel. --8<-- "docs/reference/cli/pixi/publish_extender:description" @@ -37,10 +37,8 @@ pixi publish [OPTIONS] : The local filesystem path to copy the built package(s) into (no channel indexing) - `--force` : Force overwrite existing packages -- `--skip-existing ` -: Skip uploading packages that already exist at the target. This is enabled by default. Use `--no-skip-existing` to disable -
**default**: `true` -
**options**: `true`, `false` +- `--no-skip-existing` +: Do not skip packages that already exist at the target - `--generate-attestation` : Generate sigstore attestation (prefix.dev only) - `--variant ` @@ -92,11 +90,16 @@ pixi publish [OPTIONS] : Use environment activation cache (experimental) ## Description -Build a conda package and publish it to a channel. +Build the conda packages of a workspace and publish them to a channel. -Builds the package from your workspace and either uploads it to a channel -(`--target-channel`) or copies the artifact into a local directory -(`--target-dir`). +Builds every package in the workspace that opts into publishing with +`publish = true` in its `[package]` section - in dependency order, so +packages that depend on other workspace packages are built after them - +and either uploads the artifacts to a channel (`--target-channel`) or +copies them into a local directory (`--target-dir`). Every source +dependency of a published package must itself opt into publishing; the +publish fails otherwise. Use `--path` to build and publish a single +self-contained package instead. Supported destinations for `--target-channel` (alias `--to`): - prefix.dev: `https://prefix.dev/` diff --git a/docs/reference/cli/pixi/publish_extender b/docs/reference/cli/pixi/publish_extender index 7fedbae79f..a9b3b3a9ed 100644 --- a/docs/reference/cli/pixi/publish_extender +++ b/docs/reference/cli/pixi/publish_extender @@ -1,7 +1,21 @@ --8<-- [start:description] -`pixi publish` **builds** a conda package from your workspace and **uploads** it to a channel or copies it into a directory. +`pixi publish` **builds** the conda packages of your workspace and **uploads** them to a channel or copies them into a directory. + +By default every workspace package that opts into publishing with `publish = true` in its `[package]` section is published: + +```toml +[package] +name = "my-package" +publish = true +``` + +Packages are discovered by walking the workspace directory tree. The walk respects ignore files such as `.gitignore` and skips subdirectories that contain their own workspace. Packages that do not set `publish = true` are left out of the publish set; if no package opts in, the publish fails. + +Packages are built and uploaded in dependency order, so a package is never uploaded before the workspace packages it depends on. The published set must be closed: every source dependency (build, host, or run) of a published package must itself opt into publishing. Depending on a package outside the publish set, on a git or url source, or on a path outside the workspace root fails the publish - intentionally, so a publish can never leave the target channel referencing packages that were not uploaded. Replace external source dependencies with binary dependencies to publish. + +Use `--path ` to build and publish a single package instead - whether or not it sets `publish = true`. A `--path` publish is a batch of one, so the package must be self-contained: it may not have any source dependencies. - With `--target-channel ` (alias `--to`): builds and uploads to the specified channel. - With `--target-dir `: builds and copies the package(s) into the given directory (no channel indexing). @@ -116,14 +130,16 @@ pixi publish --target-dir /path/to/output/dir pixi publish --target-dir ../my-packages ``` -### Publishing from a specific manifest +### Publishing a single package + +When `--path` is given, only the addressed package is built and published, whether or not it sets `publish = true`. The package must not have any source dependencies. ```shell +# Publish a single package of the workspace +pixi publish --target-channel https://prefix.dev/my-channel --path ./my-package/ + # Publish a package from a specific recipe pixi publish --target-channel https://prefix.dev/my-channel --path ./my-recipe/recipe.yaml - -# Publish from a different workspace -pixi publish --target-channel https://prefix.dev/my-channel --path ./my-project/ ``` ### Clean rebuild and publish diff --git a/docs/switching_from/uv.md b/docs/switching_from/uv.md index b5f19228c6..06c7c50cce 100644 --- a/docs/switching_from/uv.md +++ b/docs/switching_from/uv.md @@ -157,7 +157,7 @@ Both tools support multi-package workspaces. uv defines workspace members with a members = ["packages/*"] ``` -Pixi takes a different approach: you reference local packages as path dependencies directly in the workspace manifest. Any subdirectory with its own `pixi.toml` (containing a `[package]` section) can be pulled in this way: +Pixi's environments work similarly: you reference local packages as path dependencies directly in the workspace manifest. Any subdirectory with its own `pixi.toml` (containing a `[package]` section) can be pulled in this way: ```toml title="pixi.toml" [workspace] @@ -168,6 +168,14 @@ platforms = ["linux-64", "osx-arm64", "win-64"] my_lib = { path = "packages/my_lib" } ``` +For publishing, each package opts in individually: a workspace-wide `pixi publish` builds and uploads every package that sets `publish = true` in its `[package]` section: + +```toml title="packages/my_lib/pixi.toml" +[package] +name = "my_lib" +publish = true +``` + Both tools share a single lock file across the workspace. See [Building Multiple Packages](../build/workspace.md). ### Standalone scripts diff --git a/tests/integration_python/pixi_build/test_publish.py b/tests/integration_python/pixi_build/test_publish.py new file mode 100644 index 0000000000..447a9d09ea --- /dev/null +++ b/tests/integration_python/pixi_build/test_publish.py @@ -0,0 +1,688 @@ +"""Integration tests for multi-package `pixi publish`. + +`pixi publish` without `--path` builds every workspace package that opts in +with `publish = true` in its `[package]` section, in dependency order, and +uploads them. Every source dependency of a published package must itself opt +in; the publish fails otherwise. These tests exercise the package discovery, +the closure validation, ordering and upload behavior end to end. They build +real packages and download backends from prefix.dev, so they are all marked +`slow`. +""" + +from __future__ import annotations + +import shutil +import tomllib +from dataclasses import dataclass, field +from pathlib import Path + +import pytest +import tomli_w +import yaml + +from .common import CURRENT_PLATFORM, ExitCode, git_test_repo, verify_cli_command + + +BACKEND_CHANNELS = [ + "https://prefix.dev/pixi-build-backends", + "https://prefix.dev/conda-forge", +] + + +@dataclass +class Pkg: + """A member package, its publish opt-in, and the members it depends on.""" + + host: list[str] = field(default_factory=list) + run: list[str] = field(default_factory=list) + build: list[str] = field(default_factory=list) + publish: bool = True + + +def _module(name: str) -> str: + return name.replace("-", "_") + + +def _package_table(name: str, pkg: Pkg, backend: str, *, dep_prefix: str = "../") -> dict: + backend_name = f"pixi-build-{backend}" + table: dict = { + "name": name, + "version": "1.0.0", + "publish": pkg.publish, + "build": { + "backend": { + "name": backend_name, + "channels": BACKEND_CHANNELS, + "version": "*", + }, + }, + } + host: dict = {} + if backend == "python": + host["hatchling"] = "*" + host.update({dep: {"path": f"{dep_prefix}{dep}"} for dep in pkg.host}) + if host: + table["host-dependencies"] = host + if pkg.run: + table["run-dependencies"] = {dep: {"path": f"{dep_prefix}{dep}"} for dep in pkg.run} + if pkg.build: + table["build-dependencies"] = {dep: {"path": f"{dep_prefix}{dep}"} for dep in pkg.build} + return table + + +def _write_package_sources(pkg_dir: Path, name: str, backend: str) -> None: + """Write the backend-specific source files a package needs to build.""" + if backend == "python": + module = _module(name) + pyproject = { + "project": { + "name": name, + "version": "1.0.0", + "requires-python": ">= 3.11", + "dependencies": [], + "scripts": {name: f"{module}:main"}, + }, + "build-system": { + "build-backend": "hatchling.build", + "requires": ["hatchling"], + }, + } + pkg_dir.joinpath("pyproject.toml").write_text(tomli_w.dumps(pyproject)) + src = pkg_dir.joinpath("src", module) + src.mkdir(parents=True, exist_ok=True) + src.joinpath("__init__.py").write_text( + f'def main() -> None:\n print("hello from {name}")\n' + ) + elif backend == "rattler-build": + recipe = {"package": {"name": name, "version": "1.0.0"}} + pkg_dir.joinpath("recipe.yaml").write_text(yaml.dump(recipe)) + else: + raise ValueError(f"unsupported backend {backend!r}") + + +def write_workspace( + root: Path, + packages: dict[str, Pkg], + *, + backend: str = "python", + root_package: str | None = None, + workspace_deps: list[str] | None = None, + channels: list[str] | None = None, +) -> None: + """Write a multi-package pixi workspace under `root`. + + Each entry in `packages` becomes a member directory `root/` with a + `pixi.toml` and the backend-specific source files; its `Pkg.publish` flag + becomes the `publish` key of the `[package]` section. `workspace_deps` + selects which members are referenced from the workspace `[dependencies]` + table (defaults to all of them except the root package). When + `root_package` is given, the workspace manifest also carries a `[package]` + section so the root itself is a publishable package. + """ + if workspace_deps is None: + workspace_deps = [name for name in packages if name != root_package] + + workspace_manifest: dict = { + "workspace": { + "channels": channels or ["https://prefix.dev/conda-forge"], + "preview": ["pixi-build"], + "platforms": [CURRENT_PLATFORM], + }, + "dependencies": {dep: {"path": dep} for dep in workspace_deps}, + } + + if root_package is not None: + workspace_manifest["package"] = _package_table( + root_package, packages.get(root_package, Pkg()), backend, dep_prefix="" + ) + _write_package_sources(root, root_package, backend) + + root.joinpath("pixi.toml").write_text(tomli_w.dumps(workspace_manifest)) + + for name, pkg in packages.items(): + if name == root_package: + continue + pkg_dir = root.joinpath(name) + pkg_dir.mkdir(parents=True, exist_ok=True) + manifest = {"package": _package_table(name, pkg, backend)} + pkg_dir.joinpath("pixi.toml").write_text(tomli_w.dumps(manifest)) + _write_package_sources(pkg_dir, name, backend) + + +def publish_to_dir(pixi: Path, workspace: Path, *, args: list[str] | None = None): + """Run `pixi publish` (whole-workspace) from `workspace` into `workspace/dist`.""" + target_dir = workspace.joinpath("dist") + output = verify_cli_command( + [pixi, "publish", "--target-dir", str(target_dir), *(args or [])], + cwd=workspace, + ) + return output, target_dir + + +def write_standalone_package(pkg_dir: Path, name: str, *, backend: str = "python") -> None: + """Write a package that lives outside any workspace under test.""" + pkg_dir.mkdir(parents=True, exist_ok=True) + manifest = {"package": _package_table(name, Pkg(), backend)} + pkg_dir.joinpath("pixi.toml").write_text(tomli_w.dumps(manifest)) + _write_package_sources(pkg_dir, name, backend) + + +def built_package_names(target_dir: Path) -> list[str]: + """Names of the packages in `target_dir`, derived from the artifact files.""" + return sorted(p.name.split("-1.0.0-")[0] for p in target_dir.glob("*.conda")) + + +def assert_first_occurrence_order(stderr: str, *names: str) -> None: + """Assert `names` first appear in `stderr` in the given order. + + All publish output is on stderr and the "Building N package(s):" list is + the first place the package names show up, so first occurrences reflect + the build order. + """ + indexes = [stderr.index(name) for name in names] + assert indexes == sorted(indexes), stderr + + +def _nest_workspace(tmp_pixi_workspace: Path, name: str) -> Path: + """Create a sub-workspace directory that keeps the fixture's pixi config.""" + nested = tmp_pixi_workspace.joinpath(name) + nested.joinpath(".pixi").mkdir(parents=True) + config = tmp_pixi_workspace.joinpath(".pixi", "config.toml") + if config.is_file(): + shutil.copy(config, nested.joinpath(".pixi", "config.toml")) + return nested + + +@pytest.mark.slow +def test_all_opted_in_packages_are_published(pixi: Path, tmp_pixi_workspace: Path) -> None: + """Two independent opted-in packages are both built and published.""" + write_workspace(tmp_pixi_workspace, {"alpha": Pkg(), "bravo": Pkg()}) + output, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + assert built_package_names(target_dir) == ["alpha", "bravo"] + assert "Publishing 2 workspace packages" in output.stderr + + +@pytest.mark.slow +def test_root_package_is_published_with_members(pixi: Path, tmp_pixi_workspace: Path) -> None: + """A `[package]` in the workspace manifest is published like any member. + + The root package also host-depends on a member, which exercises source + anchoring of relative paths declared in the root manifest. + """ + write_workspace( + tmp_pixi_workspace, + { + "rootpkg": Pkg(host=["member-a"]), + "member-a": Pkg(), + }, + root_package="rootpkg", + ) + output, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + assert built_package_names(target_dir) == ["member-a", "rootpkg"] + assert_first_occurrence_order(output.stderr, "member-a", "rootpkg") + + +@pytest.mark.slow +def test_non_opted_in_package_is_not_published(pixi: Path, tmp_pixi_workspace: Path) -> None: + """A package without `publish = true` is not published. + + Being referenced from the workspace `[dependencies]` table does not make a + package part of the publish set; only the `publish` flag does. + """ + write_workspace( + tmp_pixi_workspace, + {"alpha": Pkg(), "orphan": Pkg(publish=False)}, + ) + _, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + assert built_package_names(target_dir) == ["alpha"] + + +@pytest.mark.slow +def test_unpublished_source_dependency_fails(pixi: Path, tmp_pixi_workspace: Path) -> None: + """A source dependency on a package that does not opt in fails the publish. + + Uploading `alpha` while skipping `bravo` would leave the target channel + with an unsatisfiable dependency, so the whole publish is refused. + """ + write_workspace( + tmp_pixi_workspace, + {"alpha": Pkg(host=["bravo"]), "bravo": Pkg(publish=False)}, + ) + verify_cli_command( + [pixi, "publish", "--target-dir", str(tmp_pixi_workspace.joinpath("dist"))], + ExitCode.FAILURE, + cwd=tmp_pixi_workspace, + stderr_contains="not part of the publish set", + ) + + +@pytest.mark.slow +def test_transitive_chain_publishes_in_dependency_order( + pixi: Path, tmp_pixi_workspace: Path +) -> None: + """alpha -> bravo -> charlie (host deps) publishes charlie, bravo, alpha.""" + write_workspace( + tmp_pixi_workspace, + { + "alpha": Pkg(host=["bravo"]), + "bravo": Pkg(host=["charlie"]), + "charlie": Pkg(), + }, + ) + output, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + assert built_package_names(target_dir) == ["alpha", "bravo", "charlie"] + assert_first_occurrence_order(output.stderr, "charlie", "bravo", "alpha") + + +@pytest.mark.slow +def test_diamond_members_published_once_each(pixi: Path, tmp_pixi_workspace: Path) -> None: + """A diamond (alpha -> bravo+charlie -> delta) publishes each member once.""" + write_workspace( + tmp_pixi_workspace, + { + "alpha": Pkg(host=["bravo", "charlie"]), + "bravo": Pkg(host=["delta"]), + "charlie": Pkg(host=["delta"]), + "delta": Pkg(), + }, + ) + output, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + assert built_package_names(target_dir) == ["alpha", "bravo", "charlie", "delta"] + assert_first_occurrence_order(output.stderr, "delta", "bravo", "alpha") + assert_first_occurrence_order(output.stderr, "delta", "charlie", "alpha") + + +@pytest.mark.slow +def test_run_dependency_orders_publish(pixi: Path, tmp_pixi_workspace: Path) -> None: + """Run dependencies between members also order the publish.""" + write_workspace( + tmp_pixi_workspace, + {"alpha": Pkg(run=["bravo"]), "bravo": Pkg()}, + workspace_deps=["alpha"], + ) + output, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + assert built_package_names(target_dir) == ["alpha", "bravo"] + assert_first_occurrence_order(output.stderr, "bravo", "alpha") + + +@pytest.mark.slow +def test_path_option_publishes_only_that_package(pixi: Path, tmp_pixi_workspace: Path) -> None: + """`--path` restricts the publish to a single package.""" + write_workspace(tmp_pixi_workspace, {"alpha": Pkg(), "bravo": Pkg()}) + _, target_dir = publish_to_dir(pixi, tmp_pixi_workspace, args=["--path", "alpha"]) + + assert built_package_names(target_dir) == ["alpha"] + + +@pytest.mark.slow +def test_workspace_without_opted_in_packages_fails(pixi: Path, tmp_pixi_workspace: Path) -> None: + """A workspace where no package sets `publish = true` errors out.""" + write_workspace(tmp_pixi_workspace, {}) + verify_cli_command( + [pixi, "publish", "--target-dir", str(tmp_pixi_workspace.joinpath("dist"))], + ExitCode.FAILURE, + cwd=tmp_pixi_workspace, + stderr_contains="no package in the workspace opts into publishing", + ) + + +@pytest.mark.slow +def test_gitignored_package_is_not_published(pixi: Path, tmp_pixi_workspace: Path) -> None: + """Package discovery respects ignore files. + + A package that opts in with `publish = true` but sits in a gitignored + directory is not discovered and therefore not published. + """ + write_workspace( + tmp_pixi_workspace, + {"alpha": Pkg(), "vendored": Pkg()}, + workspace_deps=["alpha"], + ) + tmp_pixi_workspace.joinpath(".gitignore").write_text("vendored/\n") + + _, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + assert built_package_names(target_dir) == ["alpha"] + + +@pytest.mark.slow +def test_external_path_dependency_fails(pixi: Path, tmp_pixi_workspace: Path) -> None: + """A source dependency on a path outside the workspace root fails. + + An external source cannot be part of the published batch, so the closure + rule refuses the publish. + """ + workspace = _nest_workspace(tmp_pixi_workspace, "ws") + write_workspace(workspace, {"alpha": Pkg()}) + write_standalone_package(tmp_pixi_workspace.joinpath("external", "extpkg"), "extpkg") + + manifest_path = workspace.joinpath("alpha", "pixi.toml") + manifest = tomllib.loads(manifest_path.read_text()) + manifest["package"].setdefault("host-dependencies", {})["extpkg"] = { + "path": "../../external/extpkg" + } + manifest_path.write_text(tomli_w.dumps(manifest)) + + verify_cli_command( + [pixi, "publish", "--target-dir", str(workspace.joinpath("dist"))], + ExitCode.FAILURE, + cwd=workspace, + stderr_contains="outside the workspace", + ) + + +@pytest.mark.slow +def test_external_git_dependency_fails(pixi: Path, tmp_pixi_workspace: Path) -> None: + """A git source dependency fails the publish.""" + workspace = _nest_workspace(tmp_pixi_workspace, "ws") + write_workspace(workspace, {"alpha": Pkg()}) + pkg_src = tmp_pixi_workspace.joinpath("gitsrc") + write_standalone_package(pkg_src, "gitpkg") + repo_url = git_test_repo(pkg_src, "gitpkg-repo", tmp_pixi_workspace.joinpath("repos")) + + manifest_path = workspace.joinpath("alpha", "pixi.toml") + manifest = tomllib.loads(manifest_path.read_text()) + manifest["package"].setdefault("host-dependencies", {})["gitpkg"] = {"git": repo_url} + manifest_path.write_text(tomli_w.dumps(manifest)) + + verify_cli_command( + [pixi, "publish", "--target-dir", str(workspace.joinpath("dist"))], + ExitCode.FAILURE, + cwd=workspace, + stderr_contains="outside the workspace", + ) + + +@pytest.mark.slow +def test_source_dependency_on_manifest_path_matches_listed_directory( + pixi: Path, tmp_pixi_workspace: Path +) -> None: + """A `pkg/pixi.toml` source dependency matches the discovered `pkg` directory. + + The closure check must identify a dependency declared through the manifest + file path with the member directory of the opted-in package. + """ + write_workspace(tmp_pixi_workspace, {"alpha": Pkg(host=["bravo"]), "bravo": Pkg()}) + manifest_path = tmp_pixi_workspace.joinpath("alpha", "pixi.toml") + manifest = tomllib.loads(manifest_path.read_text()) + manifest["package"]["host-dependencies"]["bravo"] = {"path": "../bravo/pixi.toml"} + manifest_path.write_text(tomli_w.dumps(manifest)) + + output, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + assert built_package_names(target_dir) == ["alpha", "bravo"] + assert "Publishing 2 workspace packages" in output.stderr + + +@pytest.mark.slow +def test_multi_output_member_publishes_all_outputs(pixi: Path, tmp_pixi_workspace: Path) -> None: + """Every output of a multi-output recipe member is published.""" + write_workspace(tmp_pixi_workspace, {"multi": Pkg()}, backend="rattler-build") + recipe = { + "recipe": {"name": "multi", "version": "1.0.0"}, + "outputs": [ + {"package": {"name": "multi"}}, + {"package": {"name": "multi-tools"}}, + ], + } + tmp_pixi_workspace.joinpath("multi", "recipe.yaml").write_text(yaml.dump(recipe)) + + _, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + assert built_package_names(target_dir) == ["multi", "multi-tools"] + + +@pytest.mark.slow +def test_second_publish_skips_existing(pixi: Path, tmp_pixi_workspace: Path) -> None: + """Publishing into the same target twice skips the existing artifacts.""" + write_workspace(tmp_pixi_workspace, {"alpha": Pkg()}) + publish_to_dir(pixi, tmp_pixi_workspace) + output, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + assert built_package_names(target_dir) == ["alpha"] + assert "Skipping" in output.stderr, output.stderr + assert "already exists" in output.stderr, output.stderr + + +@pytest.mark.slow +def test_target_channel_file_url_installs_end_to_end(pixi: Path, tmp_pixi_workspace: Path) -> None: + """Packages published to a `file://` channel install and run elsewhere.""" + workspace = _nest_workspace(tmp_pixi_workspace, "ws") + write_workspace(workspace, {"alpha": Pkg()}) + channel = tmp_pixi_workspace.joinpath("channel") + verify_cli_command( + [pixi, "publish", "--target-channel", channel.as_uri()], + cwd=workspace, + ) + assert channel.joinpath("noarch", "repodata.json").is_file() + + consumer = _nest_workspace(tmp_pixi_workspace, "consumer") + consumer_manifest = { + "workspace": { + "channels": [channel.as_uri(), "https://prefix.dev/conda-forge"], + "platforms": [CURRENT_PLATFORM], + }, + "dependencies": {"alpha": "*"}, + } + consumer.joinpath("pixi.toml").write_text(tomli_w.dumps(consumer_manifest)) + + verify_cli_command( + [pixi, "run", "alpha"], + cwd=consumer, + stdout_contains="hello from alpha", + ) + + +@pytest.mark.slow +def test_smoke_host_dep_orders_and_builds_once(pixi: Path, tmp_pixi_workspace: Path) -> None: + """Validation smoke test: `alpha` host-depends on `bravo`. + + `bravo` must be built before `alpha` and exactly once, even though it is + both a standalone member and a host dependency of `alpha`. + """ + write_workspace( + tmp_pixi_workspace, + { + "alpha": Pkg(host=["bravo"]), + "bravo": Pkg(), + }, + ) + output, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + built = sorted(p.name for p in target_dir.glob("*.conda")) + assert len(built) == 2, built + + # bravo is built before alpha (dependency order). All publish output is on + # stderr; the "Building N package(s):" list is the first place both names + # appear, so first-occurrence order reflects build order. + assert output.stderr.index("bravo") < output.stderr.index("alpha"), output.stderr + + +@pytest.mark.slow +def test_host_dep_member_is_built_only_once(pixi: Path, tmp_pixi_workspace: Path) -> None: + """A member that is also a host dependency of another member builds once. + + `bravo` is published as a member and consumed as a host dependency of + `alpha`. Its member build artifact must be reused when assembling + `alpha`'s host environment instead of triggering a second build. + """ + write_workspace( + tmp_pixi_workspace, + { + "alpha": Pkg(host=["bravo"]), + "bravo": Pkg(), + }, + ) + output, _ = publish_to_dir(pixi, tmp_pixi_workspace) + + # The backend logs one "Running build for recipe:" line per package it + # actually builds; cache hits produce none. Two packages, two builds. + assert output.stderr.count("Running build for recipe") == 2, output.stderr + + +@pytest.mark.slow +def test_build_dep_member_is_built_only_once(pixi: Path, tmp_pixi_workspace: Path) -> None: + """A member that is also a build dependency of another member builds once. + + Build dependencies target the build platform. Since publish targets the + current platform by default, the build-dependency build of `bravo` must + reuse the member build instead of running again. + """ + write_workspace( + tmp_pixi_workspace, + { + "alpha": Pkg(build=["bravo"]), + "bravo": Pkg(), + }, + ) + output, _ = publish_to_dir(pixi, tmp_pixi_workspace) + + assert output.stderr.count("Running build for recipe") == 2, output.stderr + + +@pytest.mark.slow +def test_force_overwrites_existing_artifact(pixi: Path, tmp_pixi_workspace: Path) -> None: + """`--force` replaces an artifact that already exists at the target.""" + write_workspace(tmp_pixi_workspace, {"alpha": Pkg()}) + _, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + artifact = next(target_dir.glob("*.conda")) + first_bytes = artifact.read_bytes() + + # Change the package contents without changing name/version/build string, + # so the replacement artifact keeps the same filename. Comparing bytes + # instead of mtime: `fs::copy` preserves timestamps on Windows and macOS. + module = tmp_pixi_workspace.joinpath("alpha", "src", "alpha", "__init__.py") + module.write_text('def main() -> None:\n print("changed contents")\n') + + output, _ = publish_to_dir(pixi, tmp_pixi_workspace, args=["--force"]) + + assert "already exists" not in output.stderr, output.stderr + assert artifact.read_bytes() != first_bytes + + +@pytest.mark.slow +def test_no_skip_existing_fails_on_existing_artifact(pixi: Path, tmp_pixi_workspace: Path) -> None: + """`--no-skip-existing` without `--force` refuses to overwrite.""" + write_workspace(tmp_pixi_workspace, {"alpha": Pkg()}) + _, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + verify_cli_command( + [pixi, "publish", "--target-dir", str(target_dir), "--no-skip-existing"], + ExitCode.FAILURE, + cwd=tmp_pixi_workspace, + stderr_contains="--force", + ) + + +@pytest.mark.slow +def test_force_reindexes_local_channel(pixi: Path, tmp_pixi_workspace: Path) -> None: + """`--force` on a `file://` channel updates repodata for the new artifact. + + Overwriting the `.conda` file alone is not enough: the channel index must + pick up the replaced artifact's checksums, otherwise consumers fail with + checksum mismatches. + """ + workspace = _nest_workspace(tmp_pixi_workspace, "ws") + write_workspace(workspace, {"alpha": Pkg()}) + channel = tmp_pixi_workspace.joinpath("channel") + + verify_cli_command([pixi, "publish", "--target-channel", channel.as_uri()], cwd=workspace) + repodata_path = channel.joinpath("noarch", "repodata.json") + first_repodata = repodata_path.read_text() + + # Change the package contents without changing name/version/build string, + # so the replacement artifact keeps the same filename. + module = workspace.joinpath("alpha", "src", "alpha", "__init__.py") + module.write_text('def main() -> None:\n print("changed contents")\n') + + verify_cli_command( + [pixi, "publish", "--target-channel", channel.as_uri(), "--force"], + cwd=workspace, + ) + second_repodata = repodata_path.read_text() + + assert second_repodata != first_repodata, "repodata must reflect the replaced artifact" + + +@pytest.mark.slow +def test_multi_output_sibling_dependency_uploads_in_order( + pixi: Path, tmp_pixi_workspace: Path +) -> None: + """Outputs of one recipe upload dependencies-first, not alphabetically.""" + write_workspace(tmp_pixi_workspace, {"multi": Pkg()}, backend="rattler-build") + recipe = { + "recipe": {"name": "multi", "version": "1.0.0"}, + "outputs": [ + {"package": {"name": "zzz-lib"}}, + { + "package": {"name": "aaa-app"}, + "requirements": {"run": ['${{ pin_subpackage("zzz-lib", exact=True) }}']}, + }, + ], + } + tmp_pixi_workspace.joinpath("multi", "recipe.yaml").write_text(yaml.dump(recipe)) + + output, target_dir = publish_to_dir(pixi, tmp_pixi_workspace) + + assert built_package_names(target_dir) == ["aaa-app", "zzz-lib"] + # The upload listing reflects the build/upload order; the library that + # `aaa-app` run-depends on must come first. + published = output.stderr[output.stderr.index("Successfully published") :] + assert published.index("zzz-lib") < published.index("aaa-app"), output.stderr + + +@pytest.mark.slow +def test_publish_from_member_directory_includes_root_package( + pixi: Path, tmp_pixi_workspace: Path +) -> None: + """The published set does not depend on the invocation directory. + + Every opted-in package - including the root `[package]` - is published + even when the command runs inside a member directory. + """ + write_workspace( + tmp_pixi_workspace, + {"rootpkg": Pkg(), "member-a": Pkg()}, + root_package="rootpkg", + workspace_deps=["member-a"], + ) + target_dir = tmp_pixi_workspace.joinpath("dist") + verify_cli_command( + [pixi, "publish", "--target-dir", str(target_dir)], + cwd=tmp_pixi_workspace.joinpath("member-a"), + ) + + assert built_package_names(target_dir) == ["member-a", "rootpkg"] + + +@pytest.mark.slow +def test_deprecated_build_command_builds_only_current_package( + pixi: Path, tmp_pixi_workspace: Path +) -> None: + """`pixi build` keeps its historic single-package behavior. + + Unlike `pixi publish`, the deprecated `pixi build` command never operated + on the whole workspace: without `--path` it builds only the package at the + invocation directory. + """ + write_workspace( + tmp_pixi_workspace, + {"rootpkg": Pkg(), "member-a": Pkg()}, + root_package="rootpkg", + ) + target_dir = tmp_pixi_workspace.joinpath("dist") + output = verify_cli_command( + [pixi, "build", "--output-dir", str(target_dir)], + cwd=tmp_pixi_workspace, + ) + + assert built_package_names(target_dir) == ["rootpkg"] + # The suggested replacement must carry `--path`; a bare `pixi publish` + # would publish the whole workspace instead. + assert "--path" in output.stderr From 1a1fdff123defe1a07e8adb70be5c4f392547fac Mon Sep 17 00:00:00 2001 From: Julian Hofer Date: Thu, 9 Jul 2026 11:20:49 +0000 Subject: [PATCH 4/4] feat(publish): add a dry-run flag `pixi publish --dry-run` resolves the publish set, validates that it is self-contained, and prints the packages in build order without building or uploading anything. --- .../integration_rust/source_package_tests.rs | 34 +++++++++++++++++++ crates/pixi_cli/src/build.rs | 1 + crates/pixi_cli/src/publish/mod.rs | 24 ++++++++++++- docs/build/workspace.md | 2 ++ docs/reference/cli/pixi/publish.md | 2 ++ docs/reference/cli/pixi/publish_extender | 2 ++ .../pixi_build/test_publish.py | 15 ++++++++ 7 files changed, 79 insertions(+), 1 deletion(-) diff --git a/crates/pixi/tests/integration_rust/source_package_tests.rs b/crates/pixi/tests/integration_rust/source_package_tests.rs index faab9d76e2..930469f5dc 100644 --- a/crates/pixi/tests/integration_rust/source_package_tests.rs +++ b/crates/pixi/tests/integration_rust/source_package_tests.rs @@ -1006,6 +1006,7 @@ async fn test_publish_fails_before_build_or_upload_when_one_variant_is_unsatisfi target_dir: None, force: false, no_skip_existing: false, + dry_run: false, generate_attestation: false, variant: Vec::new(), variant_config: Vec::new(), @@ -2830,6 +2831,7 @@ async fn test_publish_without_target_builds_but_does_not_upload() { target_dir: None, force: false, no_skip_existing: false, + dry_run: false, generate_attestation: false, variant: Vec::new(), variant_config: Vec::new(), @@ -2883,6 +2885,7 @@ fn publish_args_for_test( target_dir, force: false, no_skip_existing: false, + dry_run: false, generate_attestation: false, variant: Vec::new(), variant_config: Vec::new(), @@ -3139,6 +3142,35 @@ async fn test_publish_with_path_rejects_source_dependencies() { ); } +/// `--dry-run` resolves and prints the publish set but must not build or +/// upload anything. +#[tokio::test] +async fn test_publish_dry_run_builds_and_uploads_nothing() { + setup_tracing(); + + let pixi = PixiControl::new().unwrap(); + write_three_package_workspace(pixi.workspace_path(), Some(true), Some(true), Some(true)); + + let publish_dir = tempfile::tempdir().unwrap(); + let mut args = publish_args_for_test( + Some(BackendOverride::from_memory( + PassthroughBackend::instantiator(), + )), + None, + Some(publish_dir.path().to_path_buf()), + ); + args.dry_run = true; + publish_from_directory(pixi.workspace_path(), args) + .await + .expect("a dry run of a valid publish set should succeed"); + + assert_eq!( + conda_artifact_names(publish_dir.path()), + Vec::::new(), + "a dry run must not upload any artifacts" + ); +} + /// A package without `publish = true` is not published: a workspace whose /// only package does not opt in must fail instead of falling back to the /// closest package. @@ -3282,6 +3314,7 @@ backend.version = "0.1.0" target_dir: None, force: false, no_skip_existing: false, + dry_run: false, generate_attestation: false, variant: Vec::new(), variant_config: Vec::new(), @@ -3406,6 +3439,7 @@ host-lib = "*" target_dir: Some(target_dir.path().to_path_buf()), force: false, no_skip_existing: false, + dry_run: false, generate_attestation: false, variant: Vec::new(), variant_config: Vec::new(), diff --git a/crates/pixi_cli/src/build.rs b/crates/pixi_cli/src/build.rs index c88294d771..4221f0a627 100644 --- a/crates/pixi_cli/src/build.rs +++ b/crates/pixi_cli/src/build.rs @@ -111,6 +111,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { target_dir: Some(args.output_dir), force: false, no_skip_existing: false, + dry_run: false, generate_attestation: false, variant: Vec::new(), variant_config: Vec::new(), diff --git a/crates/pixi_cli/src/publish/mod.rs b/crates/pixi_cli/src/publish/mod.rs index 51e107c87f..ab10cae592 100644 --- a/crates/pixi_cli/src/publish/mod.rs +++ b/crates/pixi_cli/src/publish/mod.rs @@ -133,6 +133,13 @@ pub struct Args { #[arg(long)] pub no_skip_existing: bool, + /// Resolve and print the publish set without building or uploading. + /// + /// Discovers the packages, validates that they form a self-contained + /// set, and prints them in the order they would be built and uploaded. + #[arg(long)] + pub dry_run: bool, + /// Generate sigstore attestation (prefix.dev only) #[arg(long)] pub generate_attestation: bool, @@ -804,8 +811,13 @@ pub async fn execute(args: Args) -> miette::Result<()> { // Print initial build summary pixi_progress::println!( - "\n{}Building {} package(s):", + "\n{}{} {} package(s):", console::style(console::Emoji("📋 ", "")).cyan(), + if args.dry_run { + "Would build" + } else { + "Building" + }, packages.len() ); for (pkg, variants) in packages.iter().zip(&pkg_variant_maps_owned) { @@ -820,6 +832,16 @@ pub async fn execute(args: Args) -> miette::Result<()> { } pixi_progress::println!(""); + // A dry run stops after the publish set has been resolved, validated, + // and printed in build order. + if args.dry_run { + pixi_progress::println!( + "{}Dry run: nothing was built or uploaded", + console::style(console::Emoji("ℹ️ ", "")).blue(), + ); + return Ok(()); + } + // Pre-resolve a SourceRecord per unique package name via RSP; each // returned variant becomes a separate SourceBuildKey invocation. The // records keep the dependency order of `package_plans`, so dependencies diff --git a/docs/build/workspace.md b/docs/build/workspace.md index a6c24491bb..981c97abf4 100644 --- a/docs/build/workspace.md +++ b/docs/build/workspace.md @@ -122,6 +122,8 @@ The set must be self-contained: every source dependency of a published package has to opt in as well, and `pixi publish` fails otherwise. This guarantees that the target channel never ends up with a package whose dependencies were not uploaded. +Use `pixi publish --dry-run` to see which packages would be published, in +which order, without building or uploading anything. See [`pixi publish`](../reference/cli/pixi/publish.md) for the full behavior, including single-package publishes with `--path`. diff --git a/docs/reference/cli/pixi/publish.md b/docs/reference/cli/pixi/publish.md index 8f7cf2e345..38351f3f6f 100644 --- a/docs/reference/cli/pixi/publish.md +++ b/docs/reference/cli/pixi/publish.md @@ -39,6 +39,8 @@ pixi publish [OPTIONS] : Force overwrite existing packages - `--no-skip-existing` : Do not skip packages that already exist at the target +- `--dry-run` +: Resolve and print the publish set without building or uploading - `--generate-attestation` : Generate sigstore attestation (prefix.dev only) - `--variant ` diff --git a/docs/reference/cli/pixi/publish_extender b/docs/reference/cli/pixi/publish_extender index a9b3b3a9ed..531db11dd2 100644 --- a/docs/reference/cli/pixi/publish_extender +++ b/docs/reference/cli/pixi/publish_extender @@ -15,6 +15,8 @@ Packages are discovered by walking the workspace directory tree. The walk respec Packages are built and uploaded in dependency order, so a package is never uploaded before the workspace packages it depends on. The published set must be closed: every source dependency (build, host, or run) of a published package must itself opt into publishing. Depending on a package outside the publish set, on a git or url source, or on a path outside the workspace root fails the publish - intentionally, so a publish can never leave the target channel referencing packages that were not uploaded. Replace external source dependencies with binary dependencies to publish. +Use `--dry-run` to resolve, validate, and print the publish set without building or uploading anything. + Use `--path ` to build and publish a single package instead - whether or not it sets `publish = true`. A `--path` publish is a batch of one, so the package must be self-contained: it may not have any source dependencies. - With `--target-channel ` (alias `--to`): builds and uploads to the specified channel. diff --git a/tests/integration_python/pixi_build/test_publish.py b/tests/integration_python/pixi_build/test_publish.py index 447a9d09ea..5d0f1a35c4 100644 --- a/tests/integration_python/pixi_build/test_publish.py +++ b/tests/integration_python/pixi_build/test_publish.py @@ -580,6 +580,21 @@ def test_no_skip_existing_fails_on_existing_artifact(pixi: Path, tmp_pixi_worksp ) +@pytest.mark.slow +def test_dry_run_prints_set_without_building(pixi: Path, tmp_pixi_workspace: Path) -> None: + """`--dry-run` prints the resolved publish set and uploads nothing.""" + write_workspace(tmp_pixi_workspace, {"alpha": Pkg(host=["bravo"]), "bravo": Pkg()}) + target_dir = tmp_pixi_workspace.joinpath("dist") + output = verify_cli_command( + [pixi, "publish", "--target-dir", str(target_dir), "--dry-run"], + cwd=tmp_pixi_workspace, + ) + + assert "Would build 2 package(s)" in output.stderr + assert_first_occurrence_order(output.stderr, "bravo", "alpha") + assert not target_dir.exists() or built_package_names(target_dir) == [] + + @pytest.mark.slow def test_force_reindexes_local_channel(pixi: Path, tmp_pixi_workspace: Path) -> None: """`--force` on a `file://` channel updates repodata for the new artifact.