diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95e729b..f4019c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,17 @@ jobs: - name: Enforce architecture and debt budgets run: cargo run --locked -p mdbase-architecture-check -- . - run: cargo clippy --locked --workspace --all-targets --all-features -- -D warnings + - name: Strict canonical no-default lint gates + run: | + cargo clippy --locked -p mdbase --no-default-features --all-targets -- -D warnings + cargo clippy --locked -p mdbase-command --no-default-features --all-targets -- -D warnings + cargo clippy --locked -p mdbase-runtime --no-default-features --all-targets -- -D warnings + cargo clippy --locked -p mdbase-testbed-adapter --all-targets -- -D warnings + - run: ./scripts/check-legacy-feature-boundary.sh + - name: Reject legacy mutation feature in canonical resolved graphs + run: ./scripts/check-no-legacy-feature.sh + - run: cargo test --locked -p mdbase --no-default-features backfill + - run: cargo test --locked -p mdbase-command --no-default-features - run: cargo test --locked -p mdbase-runtime --no-default-features - run: cargo test --locked -p mdbase-runtime --no-default-features --features sqlite - run: cargo check --locked -p mdbase-runtime --no-default-features --features postgres @@ -57,6 +68,9 @@ jobs: with: node-version: 22 - uses: Swatinem/rust-cache@v2 + - name: Compile every Windows target, including capability publication + if: runner.os == 'Windows' + run: cargo check --locked --workspace --all-targets --all-features - run: cargo test --locked --workspace --all-features env: MDBASE_SPEC_REPO_DIR: ${{ github.workspace }}/mdbase-spec diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fe6040..e0c41b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ All notable changes to this project are documented in this file. ### Breaking +- Context-free JSON `Collection` create/update/delete/rename/backfill/batch + methods are isolated behind the default-on `legacy-collection-mutation` 0.4.x + compatibility feature. They retain strict source compatibility, including + under `deny(deprecated)`, while rustdoc records their planned 0.5.0 removal. +- Deprecated ephemeral `ExecutionOutcome::result` and `CommitRejection::result` + projections now have a 0.5.0 removal gate; typed callers use `operation`. - `Collection::build_all_files_data` now returns `Result, CollectionSnapshotError>` instead of silently treating discovery/read failures as an empty collection. This is a deliberate @@ -25,6 +31,15 @@ All notable changes to this project are documented in this file. `Result`. Callers must handle YAML emitter failures; serialization no longer panics or substitutes empty YAML. +### Added + +- Added privacy-safe `legacy_journal_inventory` runtime/provider APIs so operators + can prove that no version-2 journals remain before the 0.5.0 decoder removal. +- Added exact architecture ownership guards for the seven legacy Collection + facade definitions and internal compatibility allowlist, plus non-growing + guards for `OperationContext::legacy`, wire-only outcome variants/constructors, + and ephemeral runtime result projections. + ### Fixed - Generated values now evaluate against effective defaults and dependencies, diff --git a/Cargo.lock b/Cargo.lock index 5a52cfa..a143f33 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1378,9 +1378,11 @@ dependencies = [ name = "mdbase-architecture-check" version = "0.0.0" dependencies = [ + "proc-macro2", "regex", "serde", "serde_json", + "syn 2.0.114", "walkdir", ] diff --git a/Cargo.toml b/Cargo.toml index 7fc3b71..eb13223 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,10 +32,18 @@ members = [ resolver = "2" [features] -default = [] +# Retained for the 0.4.x source-compatibility window. Canonical hosts should +# build with `--no-default-features`; the feature is removed in 0.5.0. +default = ["legacy-collection-mutation"] +legacy-collection-mutation = [] tracing = ["dep:tracing"] hosted-storage-benchmark = [] +[[bin]] +name = "phase0-baseline" +path = "src/bin/phase0-baseline.rs" +required-features = ["legacy-collection-mutation"] + [dependencies] rusqlite = { version = "0.32", features = ["bundled"] } serde = { version = "1", features = ["derive"] } @@ -64,7 +72,7 @@ cap-std = "4.0.3" cap-fs-ext = "4.0.3" [target.'cfg(windows)'.dependencies] -windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } +windows-sys = { version = "0.61", features = ["Wdk_Storage_FileSystem", "Win32_Foundation", "Win32_Storage_FileSystem", "Win32_System_IO"] } [workspace.package] edition = "2021" diff --git a/config/architecture-budgets.json b/config/architecture-budgets.json index 09c2a60..d95c5f8 100644 --- a/config/architecture-budgets.json +++ b/config/architecture-budgets.json @@ -1,30 +1,35 @@ { - "rustSourceFileCountMax": 171, - "rustSourceLineCountMax": 93200, + "rustSourceFileCountMax": 173, + "rustSourceLineCountMax": 100297, "rustSourceFileMaxLines": 1000, "legacyFileLineBudgets": { - "crates/mdbase-command/src/lib.rs": 1816, - "crates/mdbase-command/src/profile.rs": 1348, + "crates/mdbase-architecture-check/src/main.rs": 1188, + "crates/mdbase-command/src/lib.rs": 1968, + "crates/mdbase-command/src/profile.rs": 1382, "crates/mdbase-runtime/src/engine.rs": 1270, "src/bin/phase0-baseline.rs": 1164, - "src/data_contracts.rs": 1075, + "src/data_contracts.rs": 1124, "src/expressions/evaluator.rs": 3390, - "src/links/resolver.rs": 1040, + "src/links/resolver.rs": 1201, "src/query/engine.rs": 1086, - "src/runtime/filesystem.rs": 1333, - "src/runtime/hosted_base.rs": 1943, + "src/runtime/filesystem.rs": 1435, + "src/runtime/canonical_operation.rs": 1410, + "src/runtime/hosted_base.rs": 1968, "src/runtime/hosted_mutation.rs": 1200, "src/runtime/hosted_query.rs": 3400, "src/runtime/hosted_resource.rs": 1040, - "src/runtime/tests.rs": 3964, - "src/transactions.rs": 1015, - "src/transactions/runtime.rs": 1294, - "src/v03/batch.rs": 1365, - "src/v03/collection_setup.rs": 1479, - "src/v03/type_pack.rs": 2110, + "src/runtime/provider.rs": 1020, + "src/runtime/record_resolution.rs": 1393, + "src/runtime/tests.rs": 4372, + "src/api/typed.rs": 1075, + "src/transactions.rs": 1029, + "src/transactions/runtime.rs": 2625, + "src/v03/batch.rs": 1519, + "src/v03/collection_setup.rs": 1508, + "src/v03/type_pack.rs": 2135, "src/views/execute.rs": 1376, "src/views/expression.rs": 2965, - "src/watch/real.rs": 3143 + "src/watch/real.rs": 3220 }, "deadCodeAllowancesByFile": { "crates/mdbase-command/src/lib.rs": 1, @@ -34,10 +39,73 @@ "src/matching/engine.rs": 1, "src/runtime/outcome.rs": 2, "src/runtime/provider.rs": 2, + "src/runtime/snapshot.rs": 1, + "src/snapshot.rs": 2, + "src/snapshot/discovery.rs": 1, "src/cel.rs": 2 }, + "ambientIoAllowlist": { + "crates/mdbase-command/src/lib.rs": {"std::fs": 4}, + "crates/mdbase-command/src/profile.rs": {"std::fs": 14}, + "crates/mdbase-testbed-adapter/src/main.rs": {"std::fs": 4}, + "src/bin/phase0-baseline.rs": {"std::fs": 15}, + "src/cache/mod.rs": {"std::fs": 2}, + "src/cache/sqlite.rs": {"std::fs": 3, "std::fs::File": 2, "std::fs::OpenOptions": 2}, + "src/collection_root.rs": {"cap_std::ambient-acquisition": 2, "std::fs": 3, "std::fs::File": 2, "tempfile": 4}, + "src/compat/v02_migration.rs": {"std::fs": 5, "tempfile": 1}, + "src/config/mod.rs": {"std::fs": 2}, + "src/data_contracts.rs": {"std::fs": 3, "tempfile": 1, "walkdir": 2}, + "src/init.rs": {"std::fs": 6}, + "src/mutation/shadow.rs": {"std::fs": 3, "tempfile": 2}, + "src/operations/mod.rs": {"std::fs": 2, "std::fs::File": 1}, + "src/operations/type_file.rs": {"std::fs": 2, "tempfile": 1}, + "src/record_load.rs": {"std::fs": 3, "std::fs::File": 3}, + "src/runtime/catalog.rs": {"tempfile": 1}, + "src/runtime/hosted_mutation.rs": {"std::fs": 11, "tempfile": 1}, + "src/runtime/hosted_resource.rs": {"std::fs": 6, "tempfile": 3, "walkdir": 1}, + "src/runtime/hosted_validation.rs": {"std::fs": 3, "tempfile": 1}, + "src/transactions.rs": {"std::fs::File": 3}, + "src/types/loader.rs": {"std::fs": 5, "tempfile": 1}, + "src/v03/batch.rs": {"std::fs": 6, "tempfile": 1}, + "src/v03/collection_setup.rs": {"std::fs": 5}, + "src/v03/mod.rs": {"std::fs": 4, "walkdir": 4}, + "src/v03/type_pack.rs": {"std::fs": 6}, + "src/watch/mod.rs": {"std::fs": 25}, + "src/watch/real.rs": {"std::fs": 1} + }, "transitionalReferenceBudgets": { "uncheckedCollectionScans": 0, - "v03OperationFacade": 26 + "v03OperationFacade": 26, + "legacyCollectionFacadeDefinitions": [ + "backfill", + "batch_delete", + "batch_update", + "create", + "delete", + "rename", + "update" + ], + "legacyCompatibilityAllowlist": { + "src/api/operations.rs": ["CreateInput", "CreateOutput", "DeleteInput", "DeleteOutput", "RenameInput", "UpdateInput", "UpdateOutput"], + "src/compat/legacy_mutation.rs": ["backfill_legacy", "batch_delete_legacy", "batch_update_legacy", "create_legacy", "delete_legacy", "rename_legacy", "update_legacy"], + "src/compat/mod.rs": ["legacy_mutation"], + "src/operations/backfill.rs": ["backfill_legacy"], + "src/operations/batch.rs": ["batch_delete_legacy", "batch_update_legacy", "delete_legacy"], + "src/operations/create.rs": ["CreateInput", "CreateOutput", "create_legacy"], + "src/operations/delete.rs": ["DeleteInput", "DeleteOutput", "delete_legacy"], + "src/operations/migrate.rs": ["backfill_legacy"], + "src/operations/rename/mod.rs": ["RenameInput", "rename_legacy"], + "src/operations/update.rs": ["UpdateInput", "UpdateOutput", "update_legacy"] + }, + "operationContextLegacyProduction": 0, + "operationContextLegacySupport": 170, + "wireOnlyVariants": ["TypeResource", "Validation", "ViewResource"], + "wireOnlyConstructorAllowlist": { + "src/runtime/canonical_operation.rs": {"type_wire": 2, "validation_wire": 1, "view_wire": 2, "wire_only": 4}, + "src/runtime/filesystem.rs": {"wire_only": 1}, + "src/v03/batch.rs": {"wire_only": 1} + }, + "ephemeralResultProduction": 0, + "ephemeralResultSupport": 4 } } diff --git a/crates/mdbase-architecture-check/Cargo.toml b/crates/mdbase-architecture-check/Cargo.toml index d87bb1d..f35a365 100644 --- a/crates/mdbase-architecture-check/Cargo.toml +++ b/crates/mdbase-architecture-check/Cargo.toml @@ -8,7 +8,9 @@ repository.workspace = true publish = false [dependencies] +proc-macro2 = "1" regex = "1" serde = { workspace = true, features = ["derive"] } serde_json.workspace = true +syn = { version = "2", features = ["full", "visit"] } walkdir = "2" diff --git a/crates/mdbase-architecture-check/src/main.rs b/crates/mdbase-architecture-check/src/main.rs index d412874..ae1b56d 100644 --- a/crates/mdbase-architecture-check/src/main.rs +++ b/crates/mdbase-architecture-check/src/main.rs @@ -4,6 +4,8 @@ use std::collections::{BTreeMap, BTreeSet}; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; +use syn::visit::{self, Visit}; +use syn::{Item, ItemExternCrate, ItemUse, UseTree}; use walkdir::WalkDir; #[derive(Debug, Deserialize)] @@ -14,6 +16,7 @@ struct ArchitectureBudgets { rust_source_file_max_lines: usize, legacy_file_line_budgets: BTreeMap, dead_code_allowances_by_file: BTreeMap, + ambient_io_allowlist: BTreeMap>, transitional_reference_budgets: TransitionalReferenceBudgets, } @@ -22,12 +25,25 @@ struct ArchitectureBudgets { struct TransitionalReferenceBudgets { unchecked_collection_scans: usize, v03_operation_facade: usize, + legacy_collection_facade_definitions: Vec, + legacy_compatibility_allowlist: BTreeMap>, + operation_context_legacy_production: usize, + operation_context_legacy_support: usize, + wire_only_variants: Vec, + wire_only_constructor_allowlist: BTreeMap>, + ephemeral_result_production: usize, + ephemeral_result_support: usize, } struct DebtPatterns { dead_code: Regex, unchecked_collection_scan: Regex, v03_operation_facade: Regex, + operation_context_legacy: Regex, + legacy_compatibility_reference: Regex, + legacy_facade_definition: Regex, + wire_only_constructor: Regex, + ephemeral_result: Regex, } impl DebtPatterns { @@ -40,10 +56,38 @@ impl DebtPatterns { dead_code: Regex::new(r"\bdead_code\b").unwrap(), unchecked_collection_scan: Regex::new(r"\bscan_collection_files\b").unwrap(), v03_operation_facade: Regex::new(r"\bv03_operations\b").unwrap(), + operation_context_legacy: Regex::new(r"\bOperationContext::legacy\s*\(").unwrap(), + legacy_compatibility_reference: Regex::new( + r"\b(legacy_mutation|CreateInput|UpdateInput|DeleteInput|RenameInput|CreateOutput|UpdateOutput|DeleteOutput|create_legacy|update_legacy|delete_legacy|rename_legacy|backfill_legacy|batch_update_legacy|batch_delete_legacy)\b|\bCollection::(create|update|delete|rename|backfill|batch_update|batch_delete)\b", + ) + .unwrap(), + legacy_facade_definition: Regex::new( + r"(?s)pub fn (create|update|delete|rename|backfill|batch_update|batch_delete)\s*\(\s*&self,\s*input:\s*&serde_json::Value", + ) + .unwrap(), + wire_only_constructor: Regex::new( + r"\b(validation_wire|view_wire|type_wire|wire_only)\b", + ) + .unwrap(), + ephemeral_result: Regex::new(r"\b(outcome|rejection)\.result\b").unwrap(), } } } +fn production_source<'a>(relative: &str, source: &'a str) -> &'a str { + if relative.starts_with("tests/") + || relative.ends_with("_tests.rs") + || relative.ends_with("/tests.rs") + { + return ""; + } + ["#[cfg(test)]\nmod tests", "#[cfg(test)]\r\nmod tests"] + .into_iter() + .filter_map(|marker| source.find(marker)) + .min() + .map_or(source, |offset| &source[..offset]) +} + fn workspace_source_roots(root: &Path) -> Vec { let output = Command::new(env!("CARGO")) .args(["metadata", "--locked", "--no-deps", "--format-version", "1"]) @@ -114,6 +158,489 @@ fn match_count(pattern: &Regex, source: &str) -> usize { pattern.find_iter(source).count() } +/// Conservatively identify display-path expressions that are turned back into +/// collection filesystem authority. This is intentionally lexical: macro and +/// helper indirection must not provide a bypass around the held-root boundary. +fn without_cfg_test_items(source: &str) -> String { + let mut output = String::with_capacity(source.len()); + let mut cursor = 0; + while let Some(relative_start) = source[cursor..].find("#[cfg(") { + let start = cursor + relative_start; + let Some(relative_end) = source[start..].find(']') else { + break; + }; + let attribute_end = start + relative_end + 1; + if !source[start..attribute_end].contains("test") { + output.push_str(&source[cursor..attribute_end]); + cursor = attribute_end; + continue; + } + output.push_str(&source[cursor..start]); + let mut item_start = attribute_end + + source[attribute_end..] + .find(|character: char| !character.is_whitespace()) + .unwrap_or(0); + while source[item_start..].starts_with("#[") { + let Some(end) = source[item_start..].find(']') else { + break; + }; + item_start += end + 1; + item_start += source[item_start..] + .find(|character: char| !character.is_whitespace()) + .unwrap_or(0); + } + let remainder = &source[item_start..]; + let open = remainder.find('{'); + let semicolon = remainder.find(';'); + cursor = if open.is_some_and(|open| semicolon.is_none_or(|semicolon| open < semicolon)) { + let open = item_start + open.unwrap(); + let mut depth = 0usize; + let mut close = source.len(); + for (offset, character) in source[open..].char_indices() { + match character { + '{' => depth += 1, + '}' => { + depth -= 1; + if depth == 0 { + close = open + offset + 1; + break; + } + } + _ => {} + } + } + close + } else { + item_start + semicolon.map_or(remainder.len(), |offset| offset + 1) + }; + } + output.push_str(&source[cursor..]); + output +} + +#[derive(Clone, Debug)] +struct UseBinding { + local: String, + source: Vec, +} + +fn cfg_test_only(attributes: &[syn::Attribute]) -> bool { + attributes.iter().any(|attribute| { + if !attribute.path().is_ident("cfg") { + return false; + } + let Ok(list) = attribute.meta.require_list() else { + return false; + }; + let predicate = list.tokens.to_string().replace(' ', ""); + predicate == "test" || predicate.starts_with("all(test,") + }) +} + +fn item_is_test_only(item: &Item) -> bool { + let attributes = match item { + Item::Const(item) => &item.attrs, + Item::Enum(item) => &item.attrs, + Item::ExternCrate(item) => &item.attrs, + Item::Fn(item) => &item.attrs, + Item::ForeignMod(item) => &item.attrs, + Item::Impl(item) => &item.attrs, + Item::Macro(item) => &item.attrs, + Item::Mod(item) => &item.attrs, + Item::Static(item) => &item.attrs, + Item::Struct(item) => &item.attrs, + Item::Trait(item) => &item.attrs, + Item::TraitAlias(item) => &item.attrs, + Item::Type(item) => &item.attrs, + Item::Union(item) => &item.attrs, + Item::Use(item) => &item.attrs, + _ => return false, + }; + cfg_test_only(attributes) +} + +fn production_syntax(source: &str) -> syn::Result { + let mut file = syn::parse_file(source)?; + file.items.retain(|item| !item_is_test_only(item)); + Ok(file) +} + +fn flatten_use(tree: &UseTree, prefix: &mut Vec, bindings: &mut Vec) { + match tree { + UseTree::Path(path) => { + prefix.push(path.ident.to_string()); + flatten_use(&path.tree, prefix, bindings); + prefix.pop(); + } + UseTree::Name(name) => { + if name.ident == "self" { + if let Some(local) = prefix.last() { + bindings.push(UseBinding { + local: local.clone(), + source: prefix.clone(), + }); + } + } else { + let local = name.ident.to_string(); + let mut source = prefix.clone(); + source.push(local.clone()); + bindings.push(UseBinding { local, source }); + } + } + UseTree::Rename(rename) => { + let local = rename.rename.to_string(); + let mut source = prefix.clone(); + if rename.ident != "self" { + source.push(rename.ident.to_string()); + } + bindings.push(UseBinding { local, source }); + } + UseTree::Group(group) => { + for tree in &group.items { + flatten_use(tree, prefix, bindings); + } + } + UseTree::Glob(_) => { + let mut source = prefix.clone(); + source.push("*".to_string()); + bindings.push(UseBinding { + local: "*".to_string(), + source, + }); + } + } +} + +fn resolve_path(path: &[String], aliases: &BTreeMap>) -> Vec { + let mut resolved = path.to_vec(); + let mut seen = BTreeSet::new(); + while let Some(first) = resolved.first().cloned() { + if !seen.insert(first.clone()) { + break; + } + let Some(prefix) = aliases.get(&first) else { + break; + }; + resolved.splice(0..1, prefix.clone()); + } + resolved +} + +fn ambient_api(path: &[String]) -> Option<&'static str> { + let suffix = |expected: &[&str]| { + path.len() >= expected.len() + && path[path.len() - expected.len()..] + .iter() + .map(String::as_str) + .eq(expected.iter().copied()) + }; + if path == ["std", "fs", "*"] { + Some("std::fs::*") + } else if path == ["tokio", "fs", "*"] { + Some("tokio::fs::*") + } else if path.starts_with(&["std".into(), "fs".into(), "File".into()]) { + Some("std::fs::File") + } else if path.starts_with(&["std".into(), "fs".into(), "OpenOptions".into()]) { + Some("std::fs::OpenOptions") + } else if path.starts_with(&["std".into(), "fs".into()]) { + Some("std::fs") + } else if path.starts_with(&["tokio".into(), "fs".into()]) { + Some("tokio::fs") + } else if path.first().is_some_and(|part| part == "walkdir") { + Some("walkdir") + } else if path.first().is_some_and(|part| part == "tempfile") { + Some("tempfile") + } else if path.first().is_some_and(|part| part == "cap_std") + && (suffix(&["ambient_authority"]) + || suffix(&["Dir", "open_ambient_dir"]) + || suffix(&["open_ambient_dir"])) + { + Some("cap_std::ambient-acquisition") + } else { + None + } +} + +#[derive(Default)] +struct BindingCollector { + bindings: Vec, +} + +impl<'ast> Visit<'ast> for BindingCollector { + fn visit_item(&mut self, item: &'ast Item) { + if !item_is_test_only(item) { + visit::visit_item(self, item); + } + } + + fn visit_item_use(&mut self, item: &'ast ItemUse) { + flatten_use(&item.tree, &mut Vec::new(), &mut self.bindings); + } + + fn visit_item_extern_crate(&mut self, item: &'ast ItemExternCrate) { + self.bindings.push(UseBinding { + local: item + .rename + .as_ref() + .map_or_else(|| item.ident.to_string(), |(_, rename)| rename.to_string()), + source: vec![item.ident.to_string()], + }); + } +} + +struct AmbientVisitor<'a> { + aliases: &'a BTreeMap>, + inventory: BTreeMap, +} + +impl AmbientVisitor<'_> { + fn record(&mut self, path: Vec) { + let path = resolve_path(&path, self.aliases); + if let Some(api) = ambient_api(&path) { + *self.inventory.entry(api.to_string()).or_default() += 1; + } + } + + fn visit_macro_tokens(&mut self, stream: proc_macro2::TokenStream) { + let tokens = stream.into_iter().collect::>(); + let mut index = 0; + while index < tokens.len() { + if let proc_macro2::TokenTree::Group(group) = &tokens[index] { + self.visit_macro_tokens(group.stream()); + } + let proc_macro2::TokenTree::Ident(first) = &tokens[index] else { + index += 1; + continue; + }; + let mut path = vec![first.to_string()]; + let mut end = index + 1; + while end + 1 < tokens.len() + && matches!(&tokens[end], proc_macro2::TokenTree::Punct(p) if p.as_char() == ':') + && matches!(&tokens[end + 1], proc_macro2::TokenTree::Punct(p) if p.as_char() == ':') + { + let Some(proc_macro2::TokenTree::Ident(next)) = tokens.get(end + 2) else { + break; + }; + path.push(next.to_string()); + end += 3; + } + self.record(path); + index = end; + } + } +} + +impl<'ast> Visit<'ast> for AmbientVisitor<'_> { + fn visit_item(&mut self, item: &'ast Item) { + if !item_is_test_only(item) { + visit::visit_item(self, item); + } + } + + fn visit_item_use(&mut self, _item: &'ast ItemUse) {} + + fn visit_item_extern_crate(&mut self, _item: &'ast ItemExternCrate) {} + + fn visit_path(&mut self, path: &'ast syn::Path) { + self.record( + path.segments + .iter() + .map(|segment| segment.ident.to_string()) + .collect(), + ); + visit::visit_path(self, path); + } + + fn visit_macro(&mut self, item: &'ast syn::Macro) { + self.visit_macro_tokens(item.tokens.clone()); + } +} + +fn ambient_io_tokens(source: &str) -> syn::Result> { + let file = production_syntax(source)?; + let mut collector = BindingCollector::default(); + collector.visit_file(&file); + let bindings = collector.bindings; + let mut aliases = BTreeMap::from([ + ("std".to_string(), vec!["std".to_string()]), + ("tokio".to_string(), vec!["tokio".to_string()]), + ("walkdir".to_string(), vec!["walkdir".to_string()]), + ("tempfile".to_string(), vec!["tempfile".to_string()]), + ("cap_std".to_string(), vec!["cap_std".to_string()]), + ]); + for _ in 0..=bindings.len() { + let mut changed = false; + for binding in &bindings { + if binding.local == "*" { + continue; + } + let resolved = resolve_path(&binding.source, &aliases); + if resolved.first().is_some_and(|root| { + matches!( + root.as_str(), + "std" | "tokio" | "walkdir" | "tempfile" | "cap_std" + ) + }) && aliases.get(&binding.local) != Some(&resolved) + { + aliases.insert(binding.local.clone(), resolved); + changed = true; + } + } + if !changed { + break; + } + } + + let mut visitor = AmbientVisitor { + aliases: &aliases, + inventory: BTreeMap::new(), + }; + for binding in &bindings { + visitor.record(binding.source.clone()); + } + visitor.visit_file(&file); + Ok(visitor.inventory) +} + +fn held_authority_bypasses(source: &str) -> BTreeSet { + let source = without_cfg_test_items(source); + let source = source.as_str(); + let mut failures = BTreeSet::new(); + let display = if source.contains("impl Collection") { + r"(?:self|collection|shadow\.collection)\.root\b(?:\s*\(\s*\))?" + } else { + r"(?:collection|shadow\.collection)\.root\b(?:\s*\(\s*\))?" + }; + let direct_patterns = [ + ( + "std-fs", + format!( + r"(?s)(?:std::)?fs::(?:read|read_to_string|write|metadata|symlink_metadata|canonicalize|read_dir|create_dir|create_dir_all|remove_file|remove_dir_all|rename|hard_link)\s*\([^;}}]*{display}" + ), + ), + ( + "path-method", + format!( + r"(?s){display}[^;}}]*(?:\.exists|\.is_file|\.is_dir|\.metadata|\.symlink_metadata)\s*\(" + ), + ), + ("walkdir", format!(r"(?s)WalkDir::new\s*\([^;}}]*{display}")), + ( + "tempfile", + format!(r"(?s)(?:NamedTempFile::new_in|tempfile_in|tempdir_in)\s*\([^;}}]*{display}"), + ), + ( + "reopen", + format!(r"(?s)(?:Collection::open|open_collection)\s*\([^;}}]*{display}"), + ), + ("under", format!(r"(?s)\.under\s*\([^;}}]*{display}")), + ]; + for (label, expression) in direct_patterns { + if Regex::new(&expression) + .expect("held-authority guard regex") + .is_match(source) + { + failures.insert(label.to_string()); + } + } + + let alias = Regex::new(&format!( + r"(?m)let\s+(?:mut\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*&?\s*{display}[^;]*;" + )) + .expect("display-root alias regex"); + for capture in alias.captures_iter(source) { + let name = regex::escape(&capture[1]); + let risky = Regex::new(&format!( + r"(?s)(?:std::)?fs::[A-Za-z_]+\s*\([^;}}]*\b{name}\b|WalkDir::new\s*\(\s*&?\s*\b{name}\b|\b{name}\b[^;}}]*(?:\.exists|\.is_file|\.is_dir|\.metadata|\.symlink_metadata)\s*\(|\.under\s*\([^;}}]*\b{name}\b" + )) + .expect("display-root alias use regex"); + if risky.is_match(&source[capture.get(0).unwrap().end()..]) { + failures.insert("alias".to_string()); + } + } + + // A helper receiving the display root and performing path I/O is equally a + // bypass, even when the call site itself contains no std::fs token. + let helper_call = Regex::new(&format!(r"([A-Za-z_][A-Za-z0-9_]*)\s*\(\s*&?\s*{display}")) + .expect("display-root helper call regex"); + for capture in helper_call.captures_iter(source) { + let helper = regex::escape(&capture[1]); + let definition = Regex::new(&format!( + r"(?s)fn\s+{helper}\s*\([^)]*\b([A-Za-z_][A-Za-z0-9_]*)\s*:[^)]*\)\s*[^{{]*\{{([^}}]*)\}}" + )) + .expect("display-root helper definition regex"); + if let Some(body) = definition.captures(source) { + let parameter = regex::escape(&body[1]); + let io = Regex::new(&format!( + r"(?:std::)?fs::[A-Za-z_]+\s*\([^;}}]*\b{parameter}\b|\b{parameter}\b[^;}}]*(?:\.exists|\.is_file|\.is_dir|\.metadata)\s*\(" + )) + .expect("display-root helper body regex"); + if io.is_match(&body[2]) { + failures.insert("helper".to_string()); + } + } + } + failures +} + +fn enum_variants(source: &str, name: &str) -> BTreeSet { + let marker = format!("enum {name}"); + let start = source.find(&marker).expect("guarded enum exists"); + let open = source[start..] + .find('{') + .map(|offset| start + offset) + .unwrap(); + let mut depth = 0usize; + let mut segment = String::new(); + let variant = Regex::new(r"(?m)^\s*([A-Z][A-Za-z0-9_]*)\s*(?:\(|\{|$)").unwrap(); + let mut variants = BTreeSet::new(); + for character in source[open..].chars() { + match character { + '{' => { + depth += 1; + if depth > 1 { + segment.push(character); + } + } + '}' => { + if depth == 1 { + if let Some(found) = variant.captures(&segment) { + variants.insert(found[1].to_string()); + } + break; + } + depth -= 1; + segment.push(character); + } + ',' if depth == 1 => { + if let Some(found) = variant.captures(&segment) { + variants.insert(found[1].to_string()); + } + segment.clear(); + } + _ if depth >= 1 => segment.push(character), + _ => {} + } + } + variants +} + +fn compatibility_references(pattern: &Regex, source: &str) -> BTreeSet { + pattern + .captures_iter(source) + .filter_map(|capture| { + capture + .get(1) + .map(|value| value.as_str().to_string()) + .or_else(|| { + capture + .get(2) + .map(|value| format!("Collection::{}", value.as_str())) + }) + }) + .collect() +} + fn check_architecture(root: &Path) -> Result> { let budget_path = root.join("config/architecture-budgets.json"); let budgets: ArchitectureBudgets = serde_json::from_slice( @@ -139,11 +666,22 @@ fn check_architecture(root: &Path) -> Result> { let mut total_lines = 0; let mut unchecked_scan_references = 0; let mut v03_facade_references = 0; + let mut compatibility_inventory = BTreeMap::new(); + let mut wire_only_constructor_inventory: BTreeMap> = + BTreeMap::new(); + let mut operation_context_legacy_production = 0; + let mut operation_context_legacy_support = 0; + let mut ephemeral_result_production = 0; + let mut ephemeral_result_support = 0; let mut measured_dead_code = BTreeMap::new(); + let mut ambient_io_inventory = BTreeMap::new(); for file in &files { let relative = relative_path(root, file); let source = fs::read_to_string(file).expect("read workspace Rust source"); + let production = production_source(&relative, &source); + let support = source.len() - production.len(); + let support_source = &source[source.len() - support..]; let lines = source.lines().count(); total_lines += lines; measured_files.insert(relative.clone()); @@ -200,6 +738,58 @@ fn check_architecture(root: &Path) -> Result> { "{relative} infers hosted semantics from OperationResult JSON instead of typed outcomes or canonical changes" )); } + if relative != "src/compat/legacy_mutation.rs" + && patterns.legacy_facade_definition.is_match(&source) + { + failures.push(format!( + "{relative} defines a public context-free JSON Collection facade outside src/compat/legacy_mutation.rs" + )); + } + if !relative.starts_with("crates/mdbase-architecture-check/") && !production.is_empty() { + let references = + compatibility_references(&patterns.legacy_compatibility_reference, &source); + if !references.is_empty() { + compatibility_inventory.insert(relative.clone(), references); + } + let mut constructors = BTreeMap::new(); + for capture in patterns.wire_only_constructor.captures_iter(production) { + *constructors.entry(capture[1].to_string()).or_insert(0) += 1; + } + if !constructors.is_empty() { + wire_only_constructor_inventory.insert(relative.clone(), constructors); + } + } + // Collection authority is acquired exactly once. Display paths remain + // public API/diagnostic data and must not become a second authority. + if relative != "src/collection_root.rs" + && !relative.starts_with("crates/mdbase-architecture-check/") + && (source.contains("open_ambient_dir") || source.contains("ambient_authority()")) + { + failures.push(format!( + "{relative} acquires ambient filesystem authority outside CollectionRoot" + )); + } + if !relative.starts_with("crates/mdbase-architecture-check/") && !production.is_empty() { + match ambient_io_tokens(&source) { + Ok(ambient) if !ambient.is_empty() => { + ambient_io_inventory.insert(relative.clone(), ambient); + } + Ok(_) => {} + Err(error) => failures.push(format!( + "{relative} could not be parsed for ambient I/O ownership: {error}" + )), + } + } + if relative != "src/collection_root.rs" + && !relative.starts_with("crates/mdbase-architecture-check/") + { + for bypass in held_authority_bypasses(production) { + failures.push(format!( + "{relative} turns the private collection display root into {bypass} filesystem authority" + )); + } + } + if relative == "src/runtime/canonical_operation.rs" && [ "pub records: Vec", @@ -222,9 +812,40 @@ fn check_architecture(root: &Path) -> Result> { } unchecked_scan_references += match_count(&patterns.unchecked_collection_scan, &source); v03_facade_references += match_count(&patterns.v03_operation_facade, &source); + operation_context_legacy_production += + match_count(&patterns.operation_context_legacy, production); + operation_context_legacy_support += + match_count(&patterns.operation_context_legacy, support_source); + ephemeral_result_production += match_count(&patterns.ephemeral_result, production); + ephemeral_result_support += match_count(&patterns.ephemeral_result, support_source); } } + // Integration fixtures are not Cargo source roots, so inventory them as + // support without charging production file/line budgets. + let integration_tests = root.join("tests"); + if integration_tests.is_dir() { + for entry in WalkDir::new(integration_tests) { + let entry = entry.expect("walk integration test sources"); + if !entry.file_type().is_file() + || entry.path().extension().and_then(|value| value.to_str()) != Some("rs") + { + continue; + } + let source = fs::read_to_string(entry.path()).expect("read integration test source"); + operation_context_legacy_support += + match_count(&patterns.operation_context_legacy, &source); + ephemeral_result_support += match_count(&patterns.ephemeral_result, &source); + } + } + + if ambient_io_inventory != budgets.ambient_io_allowlist { + failures.push(format!( + "ambient I/O ownership changed: actual {ambient_io_inventory:?}, expected {:?}", + budgets.ambient_io_allowlist + )); + } + if total_lines > budgets.rust_source_line_count_max { failures.push(format!( "workspace Rust line count is {total_lines}; budget is {}", @@ -284,6 +905,107 @@ fn check_architecture(root: &Path) -> Result> { )); } + let facade_path = root.join("src/compat/legacy_mutation.rs"); + let facade_source = fs::read_to_string(&facade_path).expect("read legacy facade owner"); + let actual_facade: BTreeSet<_> = patterns + .legacy_facade_definition + .captures_iter(&facade_source) + .map(|capture| capture[1].to_string()) + .collect(); + let expected_facade: BTreeSet<_> = budgets + .transitional_reference_budgets + .legacy_collection_facade_definitions + .iter() + .cloned() + .collect(); + if actual_facade != expected_facade { + failures.push(format!( + "legacy Collection facade definitions must be owned exactly by src/compat/legacy_mutation.rs: actual {actual_facade:?}, expected {expected_facade:?}" + )); + } + let compat_module = fs::read_to_string(root.join("src/compat/mod.rs")) + .expect("read compatibility module owner"); + if !compat_module + .contains("#[cfg(feature = \"legacy-collection-mutation\")]\nmod legacy_mutation;") + { + failures.push( + "legacy mutation facade must remain gated by legacy-collection-mutation".to_string(), + ); + } + + let expected_compatibility: BTreeMap<_, BTreeSet<_>> = budgets + .transitional_reference_budgets + .legacy_compatibility_allowlist + .iter() + .map(|(file, names)| (file.clone(), names.iter().cloned().collect())) + .collect(); + if compatibility_inventory != expected_compatibility { + failures.push(format!( + "legacy compatibility ownership changed: actual {compatibility_inventory:?}, expected {expected_compatibility:?}" + )); + } + + let canonical_source = fs::read_to_string(root.join("src/runtime/canonical_operation.rs")) + .expect("read canonical operation model"); + let actual_wire_variants = enum_variants(&canonical_source, "WireOnlyOperationValue"); + let expected_wire_variants: BTreeSet<_> = budgets + .transitional_reference_budgets + .wire_only_variants + .iter() + .cloned() + .collect(); + if actual_wire_variants != expected_wire_variants { + failures.push(format!( + "WireOnlyOperationValue variants changed: actual {actual_wire_variants:?}, expected {expected_wire_variants:?}" + )); + } + if wire_only_constructor_inventory + != budgets + .transitional_reference_budgets + .wire_only_constructor_allowlist + { + failures.push(format!( + "wire-only constructor ownership changed: actual {wire_only_constructor_inventory:?}, expected {:?}", + budgets + .transitional_reference_budgets + .wire_only_constructor_allowlist + )); + } + + for (label, actual, maximum) in [ + ( + "OperationContext::legacy production callers", + operation_context_legacy_production, + budgets + .transitional_reference_budgets + .operation_context_legacy_production, + ), + ( + "OperationContext::legacy test/support callers", + operation_context_legacy_support, + budgets + .transitional_reference_budgets + .operation_context_legacy_support, + ), + ( + "ExecutionOutcome/CommitRejection.result production callers", + ephemeral_result_production, + budgets + .transitional_reference_budgets + .ephemeral_result_production, + ), + ( + "ExecutionOutcome/CommitRejection.result test/support callers", + ephemeral_result_support, + budgets + .transitional_reference_budgets + .ephemeral_result_support, + ), + ] { + if actual > maximum { + failures.push(format!("{label} are {actual}; budget is {maximum}")); + } + } if failures.is_empty() { Ok(format!( "architecture budgets passed: {} Rust files, {} lines, {} unchecked-scan references, {} v0.3-facade references", @@ -318,7 +1040,10 @@ fn main() { #[cfg(test)] mod tests { - use super::{match_count, DebtPatterns}; + use super::{ + ambient_io_tokens, enum_variants, held_authority_bypasses, match_count, DebtPatterns, + }; + use std::collections::BTreeSet; #[test] fn lexical_debt_patterns_cover_attributes_ufcs_comments_and_macro_tokens() { @@ -341,4 +1066,123 @@ mod tests { assert_eq!(match_count(&patterns.v03_operation_facade, source), 5); assert_eq!(match_count(&patterns.unchecked_collection_scan, source), 3); } + + #[test] + fn ambient_io_guard_resolves_nested_imports_alias_chains_macros_and_qualified_paths() { + for (label, source) in [ + ( + "nested-module", + "use std::{fs as io}; fn f(p: &Path) { io::read(p); }", + ), + ( + "nested-type", + "use std::{fs::File as F}; fn f(p: &Path) { F::open(p); }", + ), + ( + "renamed-extern-alias-chain", + "extern crate std as rust_std; use rust_std::fs as io; use io::OpenOptions as O; fn f() { O::new(); }", + ), + ( + "macro-qualified-call", + "use std::fs as io; fn f(p: &Path) { invoke!(io::read(p)); }", + ), + ( + "qualified-type-call", + "use std::fs::File as F; fn f(p: &Path) { ::open(p); }", + ), + ( + "renamed-walkdir", + "use walkdir::WalkDir as ArbitraryWalker; fn f(p: &Path) { ArbitraryWalker::new(p); }", + ), + ( + "nested-module-import", + "mod nested { use std::{fs as hidden}; fn f() { hidden::read(\"x\"); } }", + ), + ( + "renamed-tempfile-extern", + "extern crate tempfile as workspace; fn f() { workspace::tempdir(); }", + ), + ( + "ambient-cap-std", + "use cap_std::fs::Dir as D; fn f(p: &Path) { D::open_ambient_dir(p, cap_std::ambient_authority()); }", + ), + ] { + assert!( + !ambient_io_tokens(source).unwrap().is_empty(), + "synthetic {label} ambient surface was not rejected" + ); + } + + // A helper's call site need not reveal I/O: closed ownership rejects + // the separate unowned helper module where ambient authority appears. + assert!( + ambient_io_tokens("pub fn call_helper(p: &Path) { helper::read(p); }") + .unwrap() + .is_empty() + ); + assert!( + !ambient_io_tokens("pub fn read(p: &Path) { std::fs::read(p); }") + .unwrap() + .is_empty() + ); + } + + #[test] + fn ambient_io_guard_accepts_capability_only_files_and_ignores_test_items() { + let source = r#" + use cap_std::fs::{Dir, OpenOptions}; + fn held(dir: &Dir) { + let mut options = OpenOptions::new(); + options.read(true); + let _ = dir.open_with("record.md", &options); + } + #[cfg(test)] + mod tests { fn ambient_fixture() { std::fs::read("fixture"); } } + "#; + assert!(ambient_io_tokens(source).unwrap().is_empty()); + } + + #[test] + fn held_authority_guard_rejects_direct_alias_under_walk_tempfile_and_helper_bypasses() { + for (label, source) in [ + ("collection-root", "fn f(collection: &Collection) { std::fs::read(collection.root.join(\"x\")); }"), + ("public-root", "fn f(collection: &Collection) { std::fs::metadata(collection.root().join(\"x\")); }"), + ("alias", "fn f(collection: &Collection) { let base = collection.root(); std::fs::write(base.join(\"x\"), b\"x\"); }"), + ("under", "fn f(collection: &Collection, path: CollectionPath) { path.under(&collection.root).exists(); }"), + ("walk", "fn f(collection: &Collection) { WalkDir::new(&collection.root); }"), + ("tempfile", "fn f(collection: &Collection) { NamedTempFile::new_in(collection.root()); }"), + ("helper", "fn read_it(base: &Path) { std::fs::read(base.join(\"x\")); } fn f(collection: &Collection) { read_it(collection.root()); }"), + ] { + assert!( + !held_authority_bypasses(source).is_empty(), + "synthetic {label} bypass was not rejected" + ); + } + assert!(held_authority_bypasses( + "fn display(collection: &Collection) { format!(\"{}\", collection.root().display()); }" + ) + .is_empty()); + assert!(held_authority_bypasses( + "fn held(collection: &Collection) { let directory = collection.root_capability(); directory.symlink_metadata(\"x\"); }" + ) + .is_empty()); + } + + #[test] + fn enum_inventory_parses_every_top_level_variant_generically() { + let source = r#" + enum WireOnlyOperationValue { + Validation(Value), + Future { nested: Option<(u8, u8)> }, + Unit, + } + "#; + assert_eq!( + enum_variants(source, "WireOnlyOperationValue"), + ["Future", "Unit", "Validation"] + .into_iter() + .map(str::to_string) + .collect::>() + ); + } } diff --git a/crates/mdbase-command/Cargo.toml b/crates/mdbase-command/Cargo.toml index b43df69..b3880ce 100644 --- a/crates/mdbase-command/Cargo.toml +++ b/crates/mdbase-command/Cargo.toml @@ -10,7 +10,7 @@ description = "Transport-neutral command model and profiling workloads for mdbas [dependencies] chrono.workspace = true clap.workspace = true -mdbase = { version = "0.4.0-rc.4", path = "../.." } +mdbase = { version = "0.4.0-rc.4", path = "../..", default-features = false } rand.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/crates/mdbase-command/src/lib.rs b/crates/mdbase-command/src/lib.rs index a2512da..6c1b023 100644 --- a/crates/mdbase-command/src/lib.rs +++ b/crates/mdbase-command/src/lib.rs @@ -1,8 +1,8 @@ use clap::{Parser, Subcommand}; use mdbase::api::{ - BatchOperation, BatchRequest, CollectionPath, CreateRequest, DeleteRequest, MdbaseError, - MdbaseResult, OperationOutcome, QueryDirection, QueryRequest, ReadRequest, RenameRequest, - Revision, UpdateRequest, V02MigrationRequest, + BackfillRequest, BatchOperation, BatchRequest, CollectionPath, CreateRequest, DeleteRequest, + MdbaseError, MdbaseResult, OperationOutcome, QueryDirection, QueryRequest, ReadRequest, + RenameRequest, Revision, UpdateRequest, V02MigrationRequest, }; use serde::de::DeserializeOwned; use serde::Serialize; @@ -732,10 +732,6 @@ fn is_portable(command: &Command) -> bool { matches!( command, Command::Read { .. } - | Command::Create { .. } - | Command::Update { .. } - | Command::Delete { .. } - | Command::Rename { .. } | Command::Query { .. } | Command::Batch { .. } | Command::Views { .. } @@ -1076,7 +1072,7 @@ fn execute_command(collection: &mdbase::Collection, command: Command) -> (serde_ Ok(request) })(); if dry_run { - typed_result( + delete_preflight_wire( request.and_then(|request| collection.typed()?.preflight_delete(request)), ) } else { @@ -1100,7 +1096,7 @@ fn execute_command(collection: &mdbase::Collection, command: Command) -> (serde_ Ok(request) })(); if dry_run { - typed_result( + rename_preflight_wire( request.and_then(|request| collection.typed()?.preflight_rename(request)), ) } else { @@ -1285,43 +1281,29 @@ fn execute_command(collection: &mdbase::Collection, command: Command) -> (serde_ operation: "backfill", }); } - let mut input = serde_json::Map::new(); - if let Some(t) = file_type { - input.insert("type".to_string(), serde_json::Value::String(t)); - } - if let Some(w) = where_clause { - input.insert("where".to_string(), serde_json::Value::String(w)); - } - if let Some(f) = fields { - let field_list: Vec = f - .split(',') - .map(|s| s.trim()) - .filter(|s| !s.is_empty()) - .map(|s| serde_json::Value::String(s.to_string())) - .collect(); - input.insert("fields".to_string(), serde_json::Value::Array(field_list)); - } - if dry_run { - input.insert("dry_run".to_string(), serde_json::Value::Bool(true)); - } - if apply_defaults.is_some() || apply_generated.is_some() { - let mut apply = serde_json::Map::new(); - if let Some(v) = apply_defaults { - apply.insert("defaults".to_string(), serde_json::Value::Bool(v)); - } - if let Some(v) = apply_generated { - apply.insert("generated".to_string(), serde_json::Value::Bool(v)); - } - input.insert("apply".to_string(), serde_json::Value::Object(apply)); - } - - let result = collection.backfill(&serde_json::Value::Object(input)); - let exit = if result.get("error").is_some() { - error_to_exit_code(&result) - } else { - EXIT_SUCCESS + let request = BackfillRequest { + type_name: file_type, + where_expression: where_clause, + fields: fields.map(|fields| { + fields + .split(',') + .map(str::trim) + .filter(|field| !field.is_empty()) + .map(str::to_string) + .collect() + }), + dry_run, + apply_defaults, + apply_generated, }; - (result, exit) + match collection.typed().and_then(|typed| typed.backfill(request)) { + Ok(outcome) => ( + serde_json::to_value(outcome.value) + .expect("the typed backfill result must serialize"), + EXIT_SUCCESS, + ), + Err(error) => legacy_typed_error_result(error), + } } Command::Migrate { id, path, dry_run } => { @@ -1443,6 +1425,98 @@ pub fn run_watch( } } +fn delete_preflight_wire( + result: MdbaseResult>, +) -> (serde_json::Value, i32) { + match result { + Ok(outcome) => { + let mut value = serde_json::to_value(outcome.value) + .expect("typed delete preflight results serialize"); + value["deleted"] = serde_json::Value::Bool(false); + value["dry_run"] = serde_json::Value::Bool(true); + if value["broken_links"].as_array().is_some_and(Vec::is_empty) { + value + .as_object_mut() + .expect("delete preflight results are objects") + .remove("broken_links"); + } + ( + serde_json::json!({ + "valid": true, + "result": value, + "diagnostics": outcome.diagnostics, + }), + EXIT_SUCCESS, + ) + } + Err(error) => typed_error_result(error), + } +} + +fn rename_preflight_wire( + result: MdbaseResult>, +) -> (serde_json::Value, i32) { + match result { + Ok(outcome) => { + let mut value = serde_json::to_value(outcome.value) + .expect("typed rename preflight results serialize"); + value["dry_run"] = serde_json::Value::Bool(true); + let object = value + .as_object_mut() + .expect("rename preflight results are objects"); + object.remove("warnings"); + if object + .get("references_affected") + .and_then(serde_json::Value::as_array) + .is_some_and(Vec::is_empty) + { + object.remove("references_affected"); + } + ( + serde_json::json!({ + "valid": true, + "result": value, + "diagnostics": outcome.diagnostics, + }), + EXIT_SUCCESS, + ) + } + Err(error) => typed_error_result(error), + } +} + +fn legacy_typed_error_result(error: MdbaseError) -> (serde_json::Value, i32) { + if let Some(details) = error + .diagnostics() + .first() + .and_then(|diagnostic| diagnostic.details.clone()) + .filter(serde_json::Value::is_object) + { + let value = serde_json::json!({"error": details}); + let exit = error_to_exit_code(&value); + return (value, exit); + } + let (code, message) = if let Some(diagnostic) = error.diagnostics().first() { + ( + diagnostic.code.as_str().to_string(), + diagnostic.message.clone(), + ) + } else { + let code = match &error { + MdbaseError::InvalidPath(_) => "invalid_path", + MdbaseError::UnsupportedProfile => "unsupported_profile", + MdbaseError::MigrationRequired { .. } => "migration_required", + MdbaseError::InvalidRequest { .. } => "invalid_request", + MdbaseError::InvalidResult { .. } => "invalid_result", + _ => "operation_failed", + }; + (code.to_string(), error.to_string()) + }; + let value = serde_json::json!({"error": {"code": code, "message": message}}); + let exit = error_to_exit_code(&value); + (value, exit) +} + fn typed_result( result: MdbaseResult>, ) -> (serde_json::Value, i32) { @@ -1814,3 +1888,81 @@ fn error_to_exit_code(result: &serde_json::Value) -> i32 { fn atty_check_stdin() -> bool { std::io::stdin().is_terminal() } + +#[cfg(test)] +mod wire_golden_tests { + use super::*; + use mdbase::api::{Diagnostic, DiagnosticCode, Severity}; + + fn diagnostic(code: &str, message: &str, details: Option) -> Diagnostic { + Diagnostic { + severity: Severity::Error, + code: DiagnosticCode::new(code), + message: message.to_string(), + path: None, + field: None, + type_name: None, + schema_location: None, + details, + } + } + + #[test] + fn structured_serialization_failure_retains_legacy_error_wire() { + let details = serde_json::json!({ + "code": "validation_failed", + "message": "Validation failed", + "issues": [{ + "code": "frontmatter_serialization_failed", + "message": "failed to serialize YAML frontmatter", + "path": "tagged.md", + }], + }); + let error = MdbaseError::Operation { + diagnostics: vec![diagnostic( + "validation_failed", + "Validation failed", + Some(details.clone()), + )], + }; + + let (wire, exit) = legacy_typed_error_result(error); + assert_eq!(exit, EXIT_VALIDATION_ERROR); + assert_eq!(wire, serde_json::json!({"error": details})); + } + + #[test] + fn structured_conflict_retains_canonical_diagnostic_wire_and_exit() { + let details = serde_json::json!({ + "expected_revision": "sha256:expected", + "actual_revision": "sha256:actual", + }); + let error = MdbaseError::Operation { + diagnostics: vec![diagnostic( + "concurrent_modification", + "File was modified externally", + Some(details.clone()), + )], + }; + + let (wire, exit) = typed_error_result(error); + assert_eq!(exit, EXIT_GENERAL_ERROR); + assert_eq!( + wire, + serde_json::json!({ + "valid": false, + "result": {}, + "diagnostics": [{ + "severity": "error", + "code": "concurrent_modification", + "message": "File was modified externally", + "path": null, + "field": null, + "type_name": null, + "schema_location": null, + "details": details, + }], + }) + ); + } +} diff --git a/crates/mdbase-command/src/profile.rs b/crates/mdbase-command/src/profile.rs index 6d80c71..ee4e943 100644 --- a/crates/mdbase-command/src/profile.rs +++ b/crates/mdbase-command/src/profile.rs @@ -1,5 +1,8 @@ use clap::{Args as ClapArgs, ValueEnum}; -use mdbase::api::ReadRequest; +use mdbase::api::{ + CollectionPath, CreateRequest, DeleteRequest, MdbaseError, MdbaseResult, OperationOutcome, + ReadRequest, RenameRequest, UpdateRequest, +}; use mdbase::frontmatter::parser::json_to_yaml_mapping; use mdbase::frontmatter::serializer::serialize_document; use mdbase::runtime::{FilesystemRuntime, OperationKind, OperationRequest}; @@ -1210,6 +1213,23 @@ fn ensure_v03_success(result: &mdbase::v03::OperationResult) -> Result<(), Strin } } +fn ensure_typed_success(result: MdbaseResult>) -> Result<(), String> { + result.map(|_| ()).map_err(|error| match error { + MdbaseError::Operation { diagnostics } + | MdbaseError::PartialBatch { diagnostics, .. } + | MdbaseError::LossyMigration { diagnostics } => diagnostics + .iter() + .map(|diagnostic| diagnostic.message.as_str()) + .collect::>() + .join("; "), + error => error.to_string(), + }) +} + +fn profile_path(path: &str) -> Result { + CollectionPath::new(path).map_err(|error| error.to_string()) +} + fn profile_update( collection: &Collection, task_paths: &[String], @@ -1228,11 +1248,13 @@ fn profile_update( "status": status, "points": (i % 13) as i64, }); - let result = collection.update(&json!({ - "path": path, - "fields": fields, - })); - ensure_success(&result) + let request = UpdateRequest::new(profile_path(path)?, fields); + ensure_typed_success( + collection + .typed() + .map_err(|error| error.to_string())? + .update(request), + ) }) } @@ -1282,12 +1304,13 @@ fn profile_rename( } else { (source_b, source_a) }; - let result = collection.rename(&json!({ - "from": from, - "to": to, - "update_refs": true, - })); - ensure_success(&result)?; + let request = RenameRequest::new(profile_path(from)?, profile_path(to)?); + ensure_typed_success( + collection + .typed() + .map_err(|error| error.to_string())? + .rename(request), + )?; using_a = !using_a; Ok(()) }) @@ -1298,17 +1321,20 @@ fn profile_create(collection: &Collection, iterations: usize) -> Result Result, CatalogError>` and map `CatalogError.code`/`message` to the operation diagnostic; never inspect a reserved graph key. +2. Decode semantic projection format `6` / schema `mdbase-semantic-projection-v5`, including `ResolvedStructuralOccurrence.reason`, `selected_lookup`, `candidate_count`, `candidate_digest`, the complete `alternatives` set, and aligned `alternative_candidates` identities. +3. Treat missing, unsorted, oversized, or semantically impossible v6 evidence as a stale/invalid projection requiring authoritative rebuild. Do not accept it using a v5 digest or silently downgrade it to missing. +4. Preserve the v0.3 `resolved_path` response exactly; expose evidence only through structural projection APIs that accept additive fields. +5. Regenerate Connect Rust/TypeScript bindings and fixtures for `ResolutionReason`, then run cross-version projection, backlinks, validation, and rename-planning tests before release. diff --git a/scripts/check-legacy-feature-boundary.sh b/scripts/check-legacy-feature-boundary.sh new file mode 100755 index 0000000..1dd6963 --- /dev/null +++ b/scripts/check-legacy-feature-boundary.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +fixture="$repo_root/tests/compile/legacy-feature-boundary/Cargo.toml" +target_dir="$repo_root/target/legacy-feature-boundary" +errors="$target_dir/no-default-errors.txt" +lock_file="$(dirname "$fixture")/Cargo.lock" +trap 'rm -f "$errors" "$lock_file"' EXIT + +cargo check --manifest-path "$fixture" --target-dir "$target_dir" --quiet + +if cargo check --manifest-path "$fixture" --target-dir "$target_dir" --no-default-features 2>"$errors"; then + echo "legacy facade unexpectedly compiled without legacy-collection-mutation" >&2 + exit 1 +fi + +for method in create update delete rename backfill batch_update batch_delete; do + if ! grep -Fq "no method named \`$method\`" "$errors"; then + echo "no-default fixture did not prove Collection::$method absent" >&2 + cat "$errors" >&2 + exit 1 + fi +done + +echo "legacy feature boundary passed: default facade present, no-default facade absent" diff --git a/scripts/check-no-legacy-feature.sh b/scripts/check-no-legacy-feature.sh new file mode 100755 index 0000000..200cd3a --- /dev/null +++ b/scripts/check-no-legacy-feature.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +connect_root="${MDBASE_CONNECT_ROOT:-$repo_root/../mdbase-connect}" +feature='mdbase feature "legacy-collection-mutation"' + +check_graph() { + local label="$1" + shift + local tree + tree="$(cargo tree -e features "$@")" + if grep -Fq "$feature" <<<"$tree"; then + echo "$label resolved the forbidden legacy-collection-mutation feature" >&2 + grep -F "$feature" <<<"$tree" >&2 + return 1 + fi + echo "$label feature graph is legacy-free" +} + +for package in mdbase-command mdbase-runtime mdbase-testbed-adapter; do + check_graph \ + "$package" \ + --manifest-path "$repo_root/Cargo.toml" \ + -p "$package" +done + +if [[ -f "$connect_root/Cargo.toml" ]]; then + check_graph \ + "mdbase Connect workspace" \ + --manifest-path "$connect_root/Cargo.toml" +else + echo "Connect workspace not found at $connect_root; set MDBASE_CONNECT_ROOT to check it" >&2 +fi diff --git a/src/api/mod.rs b/src/api/mod.rs index ba4b703..10cd81a 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -12,6 +12,7 @@ pub(crate) use dynamic::reference_evidence; pub use dynamic::{ProjectedValue, QueryMetadata, ReferenceEvidence}; pub use query::{FrontmatterMode, QueryDirection, QueryOrder, QueryRequest, QueryResult}; pub use typed::{ + BackfillBatchResult, BackfillDetail, BackfillRequest, BackfillResult, BatchDeletePreflightResult, BatchItemResult, BatchOperation, BatchOperationResult, BatchRenamePartialUpdates, BatchRenamePreflightResult, BatchRenameResult, BatchRequest, BatchResult, CreateRequest, DeletePreflightResult, DeleteRequest, DeleteResult, Diagnostic, diff --git a/src/api/operations.rs b/src/api/operations.rs index 22435c8..0de641f 100644 --- a/src/api/operations.rs +++ b/src/api/operations.rs @@ -1,3 +1,4 @@ +#[cfg(feature = "legacy-collection-mutation")] use std::collections::HashMap; use crate::errors::*; @@ -24,6 +25,7 @@ impl ReadInput { } } +#[cfg(feature = "legacy-collection-mutation")] #[derive(Debug, Clone)] pub struct CreateInput { pub type_name: Option, @@ -33,6 +35,7 @@ pub struct CreateInput { pub if_revision: Option, } +#[cfg(feature = "legacy-collection-mutation")] impl CreateInput { pub fn parse(input: &serde_json::Value) -> Self { #[cfg(test)] @@ -70,6 +73,7 @@ impl CreateInput { } } +#[cfg(feature = "legacy-collection-mutation")] #[derive(Debug, Clone)] pub struct UpdateInput { pub path: String, @@ -80,6 +84,7 @@ pub struct UpdateInput { pub if_revision: Option, } +#[cfg(feature = "legacy-collection-mutation")] impl UpdateInput { pub fn parse(input: &serde_json::Value) -> Result { #[cfg(test)] @@ -134,6 +139,7 @@ impl UpdateInput { } } +#[cfg(feature = "legacy-collection-mutation")] #[derive(Debug, Clone)] pub struct DeleteInput { pub path: String, @@ -143,6 +149,7 @@ pub struct DeleteInput { pub if_revision: Option, } +#[cfg(feature = "legacy-collection-mutation")] impl DeleteInput { pub fn parse(input: &serde_json::Value) -> Result { let path = input @@ -173,12 +180,14 @@ impl DeleteInput { } } +#[cfg(feature = "legacy-collection-mutation")] #[derive(Debug, Clone)] pub struct SimulatedRefWrite { pub path: String, pub content: String, } +#[cfg(feature = "legacy-collection-mutation")] #[derive(Debug, Clone)] pub struct RenameInput { pub from: String, @@ -191,6 +200,7 @@ pub struct RenameInput { pub last_known_ref_mtimes: HashMap, } +#[cfg(feature = "legacy-collection-mutation")] impl RenameInput { pub fn parse(input: &serde_json::Value) -> Result { let from = input @@ -250,6 +260,7 @@ impl RenameInput { } } +#[cfg(feature = "legacy-collection-mutation")] #[derive(Debug, Clone)] pub struct CreateOutput { pub path: String, @@ -260,6 +271,7 @@ pub struct CreateOutput { pub warnings: Vec, } +#[cfg(feature = "legacy-collection-mutation")] impl CreateOutput { pub fn into_json(self) -> serde_json::Value { let mut result = serde_json::json!({ @@ -276,6 +288,7 @@ impl CreateOutput { } } +#[cfg(feature = "legacy-collection-mutation")] #[derive(Debug, Clone)] pub struct UpdateOutput { pub path: String, @@ -284,6 +297,7 @@ pub struct UpdateOutput { pub warnings: Vec, } +#[cfg(feature = "legacy-collection-mutation")] impl UpdateOutput { pub fn into_json(self) -> serde_json::Value { let mut result = serde_json::json!({ @@ -298,6 +312,7 @@ impl UpdateOutput { } } +#[cfg(feature = "legacy-collection-mutation")] #[derive(Debug, Clone)] pub struct DeleteOutput { pub path: String, @@ -306,6 +321,7 @@ pub struct DeleteOutput { pub broken_links: Vec, } +#[cfg(feature = "legacy-collection-mutation")] impl DeleteOutput { pub fn into_json(self) -> serde_json::Value { let mut result = serde_json::json!({ diff --git a/src/api/typed.rs b/src/api/typed.rs index a583325..0e732de 100644 --- a/src/api/typed.rs +++ b/src/api/typed.rs @@ -824,6 +824,63 @@ pub struct BatchResult { pub dry_run: bool, } +/// Typed request for filling missing default and generated record fields. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct BackfillRequest { + /// Restrict candidates to one type name. + pub type_name: Option, + /// Restrict candidates with a canonical query expression. + pub where_expression: Option, + /// Restrict changes to these fields. `None` preserves the all-fields mode. + pub fields: Option>, + /// Plan and validate without publishing writes. + pub dry_run: bool, + /// Override whether defaults are applied; absent means enabled. + pub apply_defaults: Option, + /// Override whether generated values are applied; absent means enabled. + pub apply_generated: Option, +} + +/// Per-record evidence emitted by a typed backfill. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct BackfillDetail { + /// Candidate record path. + pub path: String, + /// Stable outcome label (`success`, `failed`, or `skipped`). + pub status: String, + /// Fields written by a successful mutation. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub changed_fields: Vec, + /// Human-readable no-op or skip reason. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, + /// Structured failure evidence retained for CLI compatibility. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +/// Aggregate evidence for one typed backfill execution. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct BackfillBatchResult { + /// Number of selected records. + pub total: usize, + /// Number of successful or already-satisfied records. + pub succeeded: usize, + /// Number of records that could not be planned or written. + pub failed: usize, + /// Number of records excluded by an explicit field filter. + pub skipped: usize, + /// Ordered per-record evidence. + pub details: Vec, +} + +/// Typed backfill result retaining the established CLI wire shape. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct BackfillResult { + /// Aggregate backfill result. + pub batch_result: BackfillBatchResult, +} + /// Borrowed typed operation service for one loaded collection. pub struct TypedCollection<'a> { collection: &'a Collection, @@ -895,6 +952,24 @@ impl<'a> TypedCollection<'a> { crate::mutation::batch(self.collection, request) } + /// Fill missing default and generated fields in one recoverable mutation. + pub fn backfill( + &self, + request: BackfillRequest, + ) -> MdbaseResult> { + self.backfill_with_context(request, &crate::runtime::OperationContext::internal()) + } + + /// Fill missing fields with explicit cancellation, deadline, and capture budgets. + pub fn backfill_with_context( + &self, + request: BackfillRequest, + context: &crate::runtime::OperationContext, + ) -> MdbaseResult> { + self.require_canonical("backfill")?; + crate::operations::backfill::execute(self.collection, request, context) + } + /// Query canonical or read-only-compatible records. pub fn query(&self, request: QueryRequest) -> MdbaseResult> { if self.collection.spec_profile == SpecProfile::V02 { @@ -933,6 +1008,7 @@ impl<'a> TypedCollection<'a> { pub(crate) fn query_runtime( &self, request: QueryRequest, + cancellation: &crate::OperationCancellation, ) -> MdbaseResult> { let schema_diagnostics = crate::query::canonical::model::validate_typed(&request); if !schema_diagnostics.is_empty() { @@ -947,12 +1023,14 @@ impl<'a> TypedCollection<'a> { let (evaluation, _) = crate::query::canonical::execute_model_profiled_cancellable( self.collection, query, - &crate::OperationCancellation::new(), + cancellation, true, std::time::Instant::now(), 0, ) - .expect("a fresh cancellation token cannot be cancelled"); + .map_err(|_| MdbaseError::InvalidRequest { + message: "operation cancelled".to_string(), + })?; match evaluation { Ok(execution) => Ok(OperationOutcome { value: QueryResult { diff --git a/src/cache/indexer.rs b/src/cache/indexer.rs index 905aa1e..05e9180 100644 --- a/src/cache/indexer.rs +++ b/src/cache/indexer.rs @@ -536,8 +536,11 @@ fn resolve_links( }) }) .unwrap_or_default(); - let resolved = - collection.resolve_link_target(&raw, &source, &target_types, &resolution_index); + let resolved = collection + .resolve_link_target(&raw, &source, &target_types, &resolution_index) + .map_err(|error| { + CacheError::Resolution(format!("{}: {}", error.code, error.message)) + })?; updates.push((rowid, raw, resolved)); } drop(links); @@ -580,19 +583,14 @@ pub(crate) fn reindex_all( conn: &mut Connection, collection: &Collection, ) -> Result<(), CacheError> { - let files = collection.scan_collection_files_checked()?; + let files = collection.scan_collection_relative_paths_checked()?; let transaction = conn.transaction()?; transaction.execute_batch( "DELETE FROM links; DELETE FROM file_types; DELETE FROM unique_values; DELETE FROM identity_values; DELETE FROM files; DELETE FROM meta;", )?; - for abs_path in &files { - let rel_path = abs_path - .strip_prefix(&collection.root) - .map_err(|_| CacheError::OutsideRoot(abs_path.display().to_string()))? - .to_string_lossy() - .replace('\\', "/"); - reindex_file(&transaction, collection, &rel_path)?; + for rel_path in &files { + reindex_file(&transaction, collection, rel_path)?; } resolve_all_links(&transaction, collection)?; diff --git a/src/cache/mod.rs b/src/cache/mod.rs index b915f99..fe42f71 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -25,8 +25,8 @@ pub(crate) enum CacheError { Json(#[from] serde_json::Error), #[error(transparent)] Scan(#[from] crate::snapshot::CollectionScanError), - #[error("collection path is outside the configured root: {0}")] - OutsideRoot(String), + #[error("relationship resolution failed: {0}")] + Resolution(String), #[error("collection operation cancelled")] Cancelled, } @@ -39,16 +39,21 @@ impl Collection { /// command reports failure instead of claiming that an incomplete index is /// healthy. pub fn cache_rebuild(&self) -> serde_json::Value { + #[cfg(test)] + run_cache_access_hook(&self.root); let result = sqlite::lock_cache_lifecycle_exclusive( - &self.root, + self.held_root().cache_storage_path(), &self.settings.cache_folder, Duration::from_secs(1), ) .map_err(CacheError::from) .and_then(|_lifecycle| { - sqlite::open_cache_db(&self.root, &self.settings.cache_folder) - .map_err(CacheError::from) - .and_then(|mut connection| indexer::reindex_all(&mut connection, self)) + sqlite::open_cache_db( + self.held_root().cache_storage_path(), + &self.settings.cache_folder, + ) + .map_err(CacheError::from) + .and_then(|mut connection| indexer::reindex_all(&mut connection, self)) }); match result { Ok(()) => serde_json::json!({ "success": true }), @@ -65,9 +70,14 @@ impl Collection { /// Clear the cache (S13.3.5, S13.8). /// Removes the SQLite database file from disk. pub fn cache_clear(&self) -> serde_json::Value { - let cache_root = self.root.join(&self.settings.cache_folder); + #[cfg(test)] + run_cache_access_hook(&self.root); + let cache_root = self + .held_root() + .cache_storage_path() + .join(&self.settings.cache_folder); let _lifecycle = match sqlite::lock_cache_lifecycle_exclusive( - &self.root, + self.held_root().cache_storage_path(), &self.settings.cache_folder, Duration::from_secs(1), ) { @@ -105,6 +115,32 @@ impl Collection { } } +#[cfg(test)] +type CacheAccessHooks = std::collections::BTreeMap>; + +#[cfg(test)] +fn cache_access_hooks() -> &'static std::sync::Mutex { + static HOOKS: std::sync::OnceLock> = + std::sync::OnceLock::new(); + HOOKS.get_or_init(Default::default) +} + +#[cfg(all(test, feature = "legacy-collection-mutation"))] +pub(crate) fn set_cache_access_hook(root: &std::path::Path, hook: impl FnOnce() + Send + 'static) { + cache_access_hooks() + .lock() + .unwrap() + .insert(root.to_path_buf(), Box::new(hook)); +} + +#[cfg(test)] +fn run_cache_access_hook(root: &std::path::Path) { + let hook = cache_access_hooks().lock().unwrap().remove(root); + if let Some(hook) = hook { + hook(); + } +} + #[cfg(not(windows))] fn remove_cache_file(path: &std::path::Path) -> std::io::Result<()> { std::fs::remove_file(path) diff --git a/src/cache/runtime.rs b/src/cache/runtime.rs index 49089af..841f403 100644 --- a/src/cache/runtime.rs +++ b/src/cache/runtime.rs @@ -36,19 +36,16 @@ pub(crate) fn rebuild( collection: &Collection, generation: &CollectionGeneration, ) -> Result<(), CacheError> { - let mut connection = - sqlite::open_cache_db(&collection.root, &collection.settings.cache_folder)?; - let files = collection.scan_collection_files_checked()?; + let mut connection = sqlite::open_cache_db( + collection.held_root().cache_storage_path(), + &collection.settings.cache_folder, + )?; + let files = collection.scan_collection_relative_paths_checked()?; let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; transaction.execute_batch( "DELETE FROM links; DELETE FROM file_types; DELETE FROM unique_values; DELETE FROM identity_values; DELETE FROM files; DELETE FROM meta;", )?; - for absolute in files { - let relative = absolute - .strip_prefix(&collection.root) - .map_err(|_| CacheError::OutsideRoot(absolute.display().to_string()))? - .to_string_lossy() - .replace('\\', "/"); + for relative in files { indexer::reindex_file(&transaction, collection, &relative)?; } indexer::resolve_all_links(&transaction, collection)?; @@ -99,13 +96,18 @@ pub(crate) fn apply_changes( ) }); remove.insert(change.path.as_str().to_string()); - if collection.root.join(change.path.as_str()).is_file() { + if collection + .held_root() + .exists_file(std::path::Path::new(change.path.as_str())) + { reindex.insert(change.path.as_str().to_string()); } } - let mut connection = - sqlite::open_cache_db(&collection.root, &collection.settings.cache_folder)?; + let mut connection = sqlite::open_cache_db( + collection.held_root().cache_storage_path(), + &collection.settings.cache_folder, + )?; let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; for path in &remove { indexer::remove_file(&transaction, path)?; @@ -207,7 +209,10 @@ impl InvalidMaintenanceSeal { } pub(crate) fn cache_is_current(&mut self, collection: &Collection) -> Result { - let db_path = sqlite::cache_db_path(&collection.root, &collection.settings.cache_folder); + let db_path = sqlite::cache_db_path( + collection.held_root().cache_storage_path(), + &collection.settings.cache_folder, + ); if sqlite::CacheDbIdentity::capture(&db_path)? != self.cache_db_identity { return Ok(false); } @@ -276,7 +281,10 @@ impl InvalidMaintenanceSeal { &self, collection: &Collection, ) -> Result { - let db_path = sqlite::cache_db_path(&collection.root, &collection.settings.cache_folder); + let db_path = sqlite::cache_db_path( + collection.held_root().cache_storage_path(), + &collection.settings.cache_folder, + ); Ok(sqlite::CacheDbIdentity::capture(&db_path)? == self.cache_db_identity) } } @@ -348,12 +356,14 @@ pub(crate) fn apply_invalid_maintenance( // Official clear/recreate takes this advisory lock exclusively. Acquire the // shared side before opening SQLite and retain it in the seal through ack. let lifecycle_guard = sqlite::lock_cache_lifecycle_shared( - &collection.root, + collection.held_root().cache_storage_path(), &collection.settings.cache_folder, std::time::Duration::from_secs(1), )?; - let mut connection = - sqlite::open_cache_db(&collection.root, &collection.settings.cache_folder)?; + let mut connection = sqlite::open_cache_db( + collection.held_root().cache_storage_path(), + &collection.settings.cache_folder, + )?; let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; for path in refresh { let Some(expectation) = @@ -387,7 +397,7 @@ pub(crate) fn apply_invalid_maintenance( let data_version = transaction.query_row("PRAGMA data_version", [], |row| row.get::<_, i64>(0))?; let cache_db_identity = sqlite::CacheDbIdentity::capture(&sqlite::cache_db_path( - &collection.root, + collection.held_root().cache_storage_path(), &collection.settings.cache_folder, ))?; run_maintenance_revalidation_hook(collection); @@ -453,7 +463,10 @@ pub(crate) fn set_seal_validation_hook( ) { SEAL_VALIDATION_HOOKS.lock().unwrap().insert( ( - sqlite::cache_db_path(&collection.root, &collection.settings.cache_folder), + sqlite::cache_db_path( + collection.held_root().cache_storage_path(), + &collection.settings.cache_folder, + ), boundary, ), Box::new(hook), @@ -508,7 +521,10 @@ pub(crate) fn matches_generation( collection: &Collection, generation: &CollectionGeneration, ) -> Result { - let connection = sqlite::open_cache_db(&collection.root, &collection.settings.cache_folder)?; + let connection = sqlite::open_cache_db( + collection.held_root().cache_storage_path(), + &collection.settings.cache_folder, + )?; let stored = connection .query_row( "SELECT value FROM meta WHERE key = ?1", @@ -525,7 +541,10 @@ pub(crate) fn uniqueness_conflicts( type_names: &[String], exclude_path: &str, ) -> Result, CacheError> { - let connection = sqlite::open_cache_db(&collection.root, &collection.settings.cache_folder)?; + let connection = sqlite::open_cache_db( + collection.held_root().cache_storage_path(), + &collection.settings.cache_folder, + )?; let mut conflicts = Vec::new(); if let Some(value) = frontmatter .get(&collection.settings.id_field) diff --git a/src/cache/staleness.rs b/src/cache/staleness.rs index b5cc4cb..79e7b69 100644 --- a/src/cache/staleness.rs +++ b/src/cache/staleness.rs @@ -5,22 +5,21 @@ use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use super::CacheError; +use crate::Collection; #[derive(Debug, Default)] pub(crate) struct CacheChanges { - pub stale: Vec, + pub stale: Vec, pub deleted: Vec, } -/// Compare one filesystem scan with the cache using one bulk SQLite read. -/// -/// The previous implementation issued a SQLite lookup per collection file and -/// then performed another filesystem existence pass for deletions. That made a -/// no-op freshness check disproportionately expensive for paginated queries. +/// Compare one capability-relative filesystem scan with the cache using one +/// bulk SQLite read. Both the filesystem facts and the derived SQLite store are +/// rooted in private held authorities rather than the collection display name. pub(crate) fn find_changes( conn: &Connection, - root: &Path, - files: &[PathBuf], + collection: &Collection, + files: &[String], ) -> Result { let mut cached = HashMap::::new(); let mut statement = @@ -41,21 +40,11 @@ pub(crate) fn find_changes( let mut disk_paths = HashSet::with_capacity(files.len()); let mut stale = Vec::new(); - for file_path in files { - let rel_path = file_path - .strip_prefix(root) - .map_err(|_| CacheError::OutsideRoot(file_path.display().to_string()))? - .to_string_lossy() - .replace('\\', "/"); + for rel_path in files { disk_paths.insert(rel_path.clone()); - let filesystem_mtime = std::fs::metadata(file_path)? - .modified()? - .duration_since(std::time::UNIX_EPOCH) - .ok() - .map(|duration| duration.as_nanos() as i64) - .unwrap_or(0); - if !matches!(cached.get(&rel_path), Some((mtime, false)) if *mtime == filesystem_mtime) { - stale.push(file_path.clone()); + let filesystem_mtime = collection.held_root().modified_nanos(Path::new(rel_path))?; + if !matches!(cached.get(rel_path), Some((mtime, false)) if *mtime == filesystem_mtime) { + stale.push(rel_path.clone()); } } let deleted = cached @@ -65,67 +54,25 @@ pub(crate) fn find_changes( Ok(CacheChanges { stale, deleted }) } -/// Compare filesystem mtimes against cached `mtime_ns` and return paths that -/// are stale (file is newer than what the cache recorded). -/// -/// `files` should be a list of *absolute* file paths already discovered on disk. +/// Compatibility helpers retained for cache tests and older internal callers. #[allow(dead_code)] -pub(crate) fn find_stale(conn: &Connection, root: &Path, files: &[PathBuf]) -> Vec { - let mut stale = Vec::new(); - for file_path in files { - let rel_path = match file_path.strip_prefix(root) { - Ok(p) => p.to_string_lossy().replace('\\', "/"), - Err(_) => continue, - }; - - // Read filesystem mtime - let fs_mtime_ns = match std::fs::metadata(file_path) { - Ok(meta) => { - use std::time::UNIX_EPOCH; - meta.modified() - .ok() - .and_then(|t| t.duration_since(UNIX_EPOCH).ok()) - .map(|d| d.as_nanos() as i64) - .unwrap_or(0) - } - Err(_) => continue, // can't stat => skip - }; - - // Look up cached mtime - let cached_mtime: Option = conn - .query_row( - "SELECT mtime_ns FROM files WHERE path = ?1", - rusqlite::params![rel_path], - |row| row.get(0), - ) - .ok(); - - match cached_mtime { - Some(cm) if cm == fs_mtime_ns => {} // up to date - _ => stale.push(file_path.clone()), // missing or different - } - } - stale +pub(crate) fn find_stale( + conn: &Connection, + collection: &Collection, + files: &[String], +) -> Vec { + find_changes(conn, collection, files) + .map(|changes| changes.stale.into_iter().map(PathBuf::from).collect()) + .unwrap_or_default() } -/// Return the set of relative paths present in the cache but **not** on disk. #[allow(dead_code)] -pub(crate) fn find_deleted(conn: &Connection, root: &Path) -> Vec { - let mut stmt = match conn.prepare("SELECT path FROM files") { - Ok(s) => s, - Err(_) => return Vec::new(), - }; - let rows = match stmt.query_map([], |row| row.get::<_, String>(0)) { - Ok(r) => r, - Err(_) => return Vec::new(), - }; - - let mut deleted = Vec::new(); - for rel_path in rows.flatten() { - let abs = root.join(&rel_path); - if !abs.exists() { - deleted.push(rel_path); - } - } - deleted +pub(crate) fn find_deleted( + conn: &Connection, + collection: &Collection, + files: &[String], +) -> Vec { + find_changes(conn, collection, files) + .map(|changes| changes.deleted) + .unwrap_or_default() } diff --git a/src/collection_root.rs b/src/collection_root.rs new file mode 100644 index 0000000..6b71914 --- /dev/null +++ b/src/collection_root.rs @@ -0,0 +1,629 @@ +use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt}; +#[cfg(windows)] +use cap_std::fs::OpenOptionsExt; +use cap_std::fs::{Dir, OpenOptions}; +use std::collections::HashMap; +use std::ffi::OsStr; +use std::io::{Read, Write}; +use std::path::{Component, Path, PathBuf}; +use std::sync::{Arc, LazyLock, Mutex, Weak}; + +/// Cloneable authority for one acquired collection directory. +/// +/// `display_path` is diagnostic data only. All I/O after acquisition is made +/// through `directory`; replacing the name in the ambient namespace therefore +/// cannot redirect an operation to another collection. +#[derive(Clone)] +pub(crate) struct CollectionRoot { + directory: Arc, + display_path: Arc, + cache_storage: Arc, +} + +impl std::fmt::Debug for CollectionRoot { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CollectionRoot") + .field("display_path", &self.display_path) + .finish_non_exhaustive() + } +} + +impl CollectionRoot { + /// The sole ambient collection-authority acquisition boundary. + pub(crate) fn acquire(path: &Path) -> std::io::Result { + let directory = Dir::open_ambient_dir(path, cap_std::ambient_authority())?; + let identity = Arc::new(same_file::Handle::from_file( + directory.try_clone()?.into_std_file(), + )?); + let cache_storage = cache_storage_for(&identity)?; + Ok(Self { + directory: Arc::new(directory), + display_path: Arc::new(path.to_path_buf()), + // SQLite accepts path names rather than directory capabilities. Keep + // its derived state in identity-keyed private storage, shared only + // by live authorities for this exact filesystem object. This avoids + // ever asking SQLite to reopen through the replaceable display name. + cache_storage, + }) + } + + pub(crate) fn display_path(&self) -> &Path { + &self.display_path + } + + pub(crate) fn dir(&self) -> std::io::Result { + self.directory.try_clone() + } + + pub(crate) fn cache_storage_path(&self) -> &Path { + self.cache_storage.path() + } + + pub(crate) fn open_dir(&self, relative: &Path) -> std::io::Result { + let mut dir = self.dir()?; + for part in normal_components(relative)? { + dir = dir.open_dir_nofollow(part)?; + } + Ok(dir) + } + + pub(crate) fn create_dir_all(&self, relative: &Path) -> std::io::Result { + let mut dir = self.dir()?; + for part in normal_components(relative)? { + match dir.open_dir_nofollow(part) { + Ok(next) => dir = next, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + match dir.create_dir(part) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), + } + let mut opened = None; + for attempt in 0..=20 { + match dir.open_dir_nofollow(part) { + Ok(next) => { + opened = Some(next); + break; + } + Err(error) + if error.kind() == std::io::ErrorKind::NotFound && attempt < 20 => + { + std::thread::sleep(std::time::Duration::from_millis(5)); + } + Err(error) => return Err(error), + } + } + dir = opened.ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::NotFound, + "created directory did not become visible", + ) + })?; + } + Err(error) => return Err(error), + } + } + Ok(dir) + } + + pub(crate) fn ensure_no_symlink_components(&self, relative: &Path) -> std::io::Result<()> { + let (parent, leaf) = split_parent(relative)?; + let mut dir = self.dir()?; + for part in normal_components(parent)? { + match dir.open_dir_nofollow(part) { + Ok(next) => dir = next, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error), + } + } + match dir.symlink_metadata(leaf) { + Ok(metadata) if metadata.file_type().is_symlink() => Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "symlink component is not allowed", + )), + Ok(_) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } + } + + pub(crate) fn open_file(&self, relative: &Path) -> std::io::Result { + let (parent, leaf) = split_parent(relative)?; + let dir = self.open_dir(parent)?; + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No); + allow_atomic_replacement(&mut options); + let file = dir.open_with(leaf, &options)?.into_std(); + let metadata = file.metadata()?; + if !metadata.is_file() || has_multiple_hard_links(&metadata) { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "collection resource is not an unlinked regular file", + )); + } + Ok(file) + } + + pub(crate) fn read(&self, relative: impl AsRef) -> std::io::Result> { + let mut file = self.open_file(relative.as_ref())?; + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes)?; + Ok(bytes) + } + + pub(crate) fn read_string(&self, relative: impl AsRef) -> std::io::Result { + String::from_utf8(self.read(relative)?) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error)) + } + + pub(crate) fn exists_file(&self, relative: impl AsRef) -> bool { + self.open_file(relative.as_ref()).is_ok() + } + + /// Inspect a leaf relative to the held root without following it. + pub(crate) fn entry_exists(&self, relative: &Path) -> std::io::Result { + let (parent, leaf) = split_parent(relative)?; + let dir = match self.open_dir(parent) { + Ok(dir) => dir, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + match dir.symlink_metadata(leaf) { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } + } + + pub(crate) fn metadata(&self, relative: &Path) -> std::io::Result { + self.open_file(relative)?.metadata() + } + + pub(crate) fn modified_millis(&self, relative: &Path) -> std::io::Result { + let modified = self.metadata(relative)?.modified()?; + modified + .duration_since(std::time::UNIX_EPOCH) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error)) + .and_then(|duration| { + u64::try_from(duration.as_millis()) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error)) + }) + } + + pub(crate) fn modified_nanos(&self, relative: &Path) -> std::io::Result { + let modified = self.metadata(relative)?.modified()?; + Ok(modified + .duration_since(std::time::UNIX_EPOCH) + .ok() + .and_then(|duration| i64::try_from(duration.as_nanos()).ok()) + .unwrap_or(0)) + } + + pub(crate) fn atomic_create(&self, relative: &Path, bytes: &[u8]) -> std::io::Result<()> { + self.atomic_publish(relative, bytes, true) + } + + pub(crate) fn atomic_write(&self, relative: &Path, bytes: &[u8]) -> std::io::Result<()> { + self.atomic_publish(relative, bytes, false) + } + + fn atomic_publish( + &self, + relative: &Path, + bytes: &[u8], + create_only: bool, + ) -> std::io::Result<()> { + let (parent, leaf) = split_parent(relative)?; + let dir = self.create_dir_all(parent)?; + if create_only && dir.symlink_metadata(leaf).is_ok() { + return Err(std::io::ErrorKind::AlreadyExists.into()); + } + if !create_only { + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No); + allow_atomic_replacement(&mut options); + match dir.open_with(leaf, &options) { + Ok(file) => { + let metadata = file.metadata()?; + if !metadata.is_file() + || cap_has_multiple_hard_links(&metadata) + || file.into_std().metadata()?.permissions().readonly() + { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "unsafe publication target", + )); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + } + let temp = format!(".mdbase-publish-{}", uuid::Uuid::new_v4().simple()); + let mut options = OpenOptions::new(); + options + .write(true) + .create_new(true) + .follow(FollowSymlinks::No); + #[cfg(windows)] + options + .access_mode( + windows_sys::Win32::Foundation::GENERIC_READ + | windows_sys::Win32::Foundation::GENERIC_WRITE + | windows_sys::Win32::Storage::FileSystem::DELETE, + ) + .share_mode( + windows_sys::Win32::Storage::FileSystem::FILE_SHARE_READ + | windows_sys::Win32::Storage::FileSystem::FILE_SHARE_WRITE + | windows_sys::Win32::Storage::FileSystem::FILE_SHARE_DELETE, + ); + let mut file = dir.open_with(&temp, &options)?; + if let Err(error) = file.write_all(bytes).and_then(|_| file.sync_all()) { + let _ = dir.remove_file(&temp); + return Err(error); + } + let result = if create_only { + drop(file); + // Linking a private temporary file publishes without replacing a + // concurrent creator; removing the temporary name leaves one link. + match dir.hard_link(&temp, &dir, leaf) { + Ok(()) => dir.remove_file(&temp), + Err(error) => { + let conflict = (0..=20).any(|attempt| { + let exists = dir.symlink_metadata(leaf).is_ok(); + if !exists && attempt < 20 { + std::thread::sleep(std::time::Duration::from_millis(5)); + } + exists + }); + if conflict { + Err(std::io::ErrorKind::AlreadyExists.into()) + } else { + Err(error) + } + } + } + } else { + atomic_replace(&file, &dir, &temp, leaf) + }; + if result.is_err() { + let _ = dir.remove_file(&temp); + } + result?; + sync_directory(&dir) + } + + /// Rename a regular file inside the held collection without replacing a target. + pub(crate) fn rename_noclobber(&self, from: &Path, to: &Path) -> std::io::Result<()> { + let (from_parent, from_leaf) = split_parent(from)?; + let (to_parent, to_leaf) = split_parent(to)?; + let source_dir = self.open_dir(from_parent)?; + let target_dir = self.create_dir_all(to_parent)?; + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No); + allow_atomic_replacement(&mut options); + let source = source_dir.open_with(from_leaf, &options)?; + let metadata = source.metadata()?; + if !metadata.is_file() || cap_has_multiple_hard_links(&metadata) { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "unsafe rename source", + )); + } + source_dir.hard_link(from_leaf, &target_dir, to_leaf)?; + if let Err(error) = source_dir.remove_file(from_leaf) { + let _ = target_dir.remove_file(to_leaf); + return Err(error); + } + sync_directory(&target_dir)?; + if from_parent != to_parent { + sync_directory(&source_dir)?; + } + Ok(()) + } + + pub(crate) fn remove_file(&self, relative: &Path) -> std::io::Result<()> { + let (parent, leaf) = split_parent(relative)?; + let dir = self.open_dir(parent)?; + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No); + allow_atomic_replacement(&mut options); + let file = dir.open_with(leaf, &options)?; + let metadata = file.metadata()?; + if !metadata.is_file() || cap_has_multiple_hard_links(&metadata) { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "unsafe removal target", + )); + } + dir.remove_file(leaf)?; + sync_directory(&dir) + } + + pub(crate) fn open_lock_file(&self, relative: &Path) -> std::io::Result { + let (parent, leaf) = split_parent(relative)?; + let dir = self.create_dir_all(parent)?; + let mut options = OpenOptions::new(); + options + .read(true) + .write(true) + .create(true) + .truncate(false) + .follow(FollowSymlinks::No); + let file = dir.open_with(leaf, &options)?.into_std(); + let metadata = file.metadata()?; + if !metadata.is_file() || has_multiple_hard_links(&metadata) { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "unsafe lock file", + )); + } + Ok(file) + } + + pub(crate) fn write_new_synced(&self, relative: &Path, bytes: &[u8]) -> std::io::Result<()> { + let (parent, leaf) = split_parent(relative)?; + let dir = self.create_dir_all(parent)?; + let mut options = OpenOptions::new(); + options + .write(true) + .create_new(true) + .follow(FollowSymlinks::No); + let mut file = dir.open_with(leaf, &options)?; + file.write_all(bytes)?; + file.sync_all() + } + + pub(crate) fn remove_dir_all(&self, relative: &Path) -> std::io::Result<()> { + let (parent, leaf) = split_parent(relative)?; + self.open_dir(parent)?.remove_dir_all(leaf) + } + + pub(crate) fn sync_dir(&self, relative: &Path) -> std::io::Result<()> { + sync_directory(&self.open_dir(relative)?) + } + + pub(crate) fn child_directories(&self, relative: &Path) -> std::io::Result> { + let dir = match self.open_dir(relative) { + Ok(dir) => dir, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(error), + }; + let mut result = Vec::new(); + for entry in dir.entries()? { + let entry = entry?; + let name = entry.file_name(); + if entry.file_type()?.is_dir() { + dir.open_dir_nofollow(&name)?; + result.push(relative.join(name)); + } + } + result.sort(); + Ok(result) + } + + pub(crate) fn files_recursive(&self, relative: &Path) -> std::io::Result> { + let start = match self.open_dir(relative) { + Ok(dir) => dir, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(error), + }; + let mut result = Vec::new(); + collect_files(&start, relative, &mut result)?; + result.sort(); + Ok(result) + } +} + +#[cfg(not(windows))] +fn atomic_replace( + _source: &cap_std::fs::File, + parent: &Dir, + temp: &str, + destination: &OsStr, +) -> std::io::Result<()> { + // cap-std implements this as renameat on Unix, with both names resolved + // relative to the already-open parent directory. + parent.rename(temp, parent, destination) +} + +#[cfg(windows)] +fn atomic_replace( + source: &cap_std::fs::File, + parent: &Dir, + _temp: &str, + destination: &OsStr, +) -> std::io::Result<()> { + use std::os::windows::ffi::OsStrExt; + use std::os::windows::io::AsRawHandle; + use windows_sys::Wdk::Storage::FileSystem::{ + FileRenameInformationEx, NtSetInformationFile, FILE_RENAME_INFORMATION, + FILE_RENAME_REPLACE_IF_EXISTS, + }; + use windows_sys::Win32::Foundation::RtlNtStatusToDosError; + use windows_sys::Win32::System::IO::IO_STATUS_BLOCK; + + let name = destination.encode_wide().collect::>(); + let name_bytes = name + .len() + .checked_mul(std::mem::size_of::()) + .ok_or(std::io::ErrorKind::InvalidInput)?; + let header = std::mem::offset_of!(FILE_RENAME_INFORMATION, FileName); + let size = header + .checked_add(name_bytes) + .ok_or(std::io::ErrorKind::InvalidInput)? + .max(std::mem::size_of::()); + let words = size.div_ceil(std::mem::size_of::()); + let mut buffer = vec![0_usize; words]; + let info = buffer.as_mut_ptr().cast::(); + let parent_file = parent.try_clone()?.into_std_file(); + let mut status_block = IO_STATUS_BLOCK::default(); + unsafe { + // The NT handle-relative API resolves this leaf in the already-open + // parent directory. The Win32 SetFileInformationByHandle wrapper does + // not support a non-null RootDirectory and would return ERROR_INVALID_PARAMETER. + (*info).Anonymous.Flags = FILE_RENAME_REPLACE_IF_EXISTS; + (*info).RootDirectory = parent_file.as_raw_handle(); + (*info).FileNameLength = + u32::try_from(name_bytes).map_err(|_| std::io::ErrorKind::InvalidInput)?; + std::ptr::copy_nonoverlapping( + name.as_ptr().cast::(), + buffer.as_mut_ptr().cast::().add(header), + name_bytes, + ); + let size = u32::try_from(size).map_err(|_| std::io::ErrorKind::InvalidInput)?; + for attempt in 0..=20 { + let status = NtSetInformationFile( + source.as_raw_handle(), + &mut status_block, + info.cast(), + size, + FileRenameInformationEx, + ); + if status >= 0 { + return Ok(()); + } + let dos_error = RtlNtStatusToDosError(status) as i32; + // Virus scanners and indexers can briefly hold a destination + // without delete sharing. Retry that bounded transient only. + if attempt < 20 && matches!(dos_error, 5 | 32 | 33) { + std::thread::sleep(std::time::Duration::from_millis(5)); + continue; + } + // Unsupported handle-relative replacement and every other failure + // are fail-closed; never emulate this with remove-then-rename. + return Err(std::io::Error::from_raw_os_error(dos_error)); + } + unreachable!("bounded replacement loop returns on its final attempt") + } +} + +#[cfg(windows)] +fn allow_atomic_replacement(options: &mut OpenOptions) { + options.share_mode( + windows_sys::Win32::Storage::FileSystem::FILE_SHARE_READ + | windows_sys::Win32::Storage::FileSystem::FILE_SHARE_WRITE + | windows_sys::Win32::Storage::FileSystem::FILE_SHARE_DELETE, + ); +} + +#[cfg(not(windows))] +fn allow_atomic_replacement(_options: &mut OpenOptions) {} + +fn sync_directory(dir: &Dir) -> std::io::Result<()> { + #[cfg(not(windows))] + dir.open(".")?.sync_all()?; + #[cfg(windows)] + let _ = dir; + Ok(()) +} + +static CACHE_STORAGE: LazyLock, Weak>>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + +fn cache_storage_for(identity: &Arc) -> std::io::Result> { + let mut registry = CACHE_STORAGE + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + registry.retain(|_, storage| storage.strong_count() != 0); + if let Some(storage) = registry.get(identity).and_then(Weak::upgrade) { + return Ok(storage); + } + let storage = Arc::new(tempfile::tempdir()?); + registry.insert(Arc::clone(identity), Arc::downgrade(&storage)); + Ok(storage) +} + +fn collect_files(dir: &Dir, prefix: &Path, result: &mut Vec) -> std::io::Result<()> { + for entry in dir.entries()? { + let entry = entry?; + let name = entry.file_name(); + let kind = entry.file_type()?; + let path = prefix.join(&name); + if kind.is_symlink() { + continue; + } + if kind.is_dir() { + let child = dir.open_dir_nofollow(&name)?; + collect_files(&child, &path, result)?; + } else if kind.is_file() { + result.push(path); + } + } + Ok(()) +} + +fn normal_components(path: &Path) -> std::io::Result> { + path.components() + .map(|component| match component { + Component::Normal(part) => Ok(part), + Component::CurDir => Err(std::io::ErrorKind::InvalidInput.into()), + _ => Err(std::io::ErrorKind::InvalidInput.into()), + }) + .collect() +} + +fn split_parent(path: &Path) -> std::io::Result<(&Path, &OsStr)> { + let leaf = path.file_name().ok_or(std::io::ErrorKind::InvalidInput)?; + let parent = path.parent().unwrap_or_else(|| Path::new("")); + normal_components(parent)?; + Ok((parent, leaf)) +} + +#[cfg(unix)] +fn has_multiple_hard_links(metadata: &std::fs::Metadata) -> bool { + use std::os::unix::fs::MetadataExt; + metadata.nlink() > 1 +} +#[cfg(not(unix))] +fn has_multiple_hard_links(_metadata: &std::fs::Metadata) -> bool { + false +} + +#[cfg(unix)] +fn cap_has_multiple_hard_links(metadata: &cap_std::fs::Metadata) -> bool { + use cap_fs_ext::MetadataExt; + metadata.nlink() > 1 +} +#[cfg(not(unix))] +fn cap_has_multiple_hard_links(_metadata: &cap_std::fs::Metadata) -> bool { + false +} + +#[cfg(all(test, windows))] +mod windows_tests { + use super::CollectionRoot; + use std::path::Path; + + #[test] + fn rename_info_ex_layout_can_hold_a_relative_destination() { + use windows_sys::Wdk::Storage::FileSystem::{ + FILE_RENAME_INFORMATION, FILE_RENAME_INFORMATION_0, + }; + + assert!( + std::mem::offset_of!(FILE_RENAME_INFORMATION, FileName) + >= std::mem::size_of::() + ); + assert!( + std::mem::size_of::() + >= std::mem::offset_of!(FILE_RENAME_INFORMATION, FileName) + + std::mem::size_of::() + ); + } + + #[test] + fn capability_relative_atomic_write_replaces_destination() { + let directory = tempfile::tempdir().unwrap(); + let root = CollectionRoot::acquire(directory.path()).unwrap(); + root.atomic_create(Path::new("record.md"), b"before") + .unwrap(); + root.atomic_write(Path::new("record.md"), b"after").unwrap(); + assert_eq!(root.read("record.md").unwrap(), b"after"); + assert!(root + .files_recursive(Path::new("")) + .unwrap() + .iter() + .all(|path| !path.to_string_lossy().contains(".mdbase-publish-"))); + } +} diff --git a/src/compat/legacy_mutation.rs b/src/compat/legacy_mutation.rs new file mode 100644 index 0000000..db21728 --- /dev/null +++ b/src/compat/legacy_mutation.rs @@ -0,0 +1,69 @@ +//! Source-compatible 0.4.x ambient mutation facade. +//! +//! This module owns the only public context-free mutation entry points. It is +//! selected by the default-on `legacy-collection-mutation` feature. Rust +//! deprecation attributes intentionally begin only in 0.5.0 so 0.4.x consumers +//! using `deny(deprecated)` retain strict source compatibility. + +use crate::Collection; + +impl Collection { + /// **Deprecated compatibility API — planned removal: 0.5.0.** + /// + /// Use `Collection::typed()?.create(CreateRequest)`. + pub fn create(&self, input: &serde_json::Value) -> serde_json::Value { + self.create_legacy(input) + } + + /// **Deprecated compatibility API — planned removal: 0.5.0.** + /// + /// Use `Collection::typed()?.update(UpdateRequest)`. + pub fn update(&self, input: &serde_json::Value) -> serde_json::Value { + self.update_legacy(input) + } + + /// **Deprecated compatibility API — planned removal: 0.5.0.** + /// + /// Use `Collection::typed()?.delete(DeleteRequest)` or `preflight_delete`. + pub fn delete(&self, input: &serde_json::Value) -> serde_json::Value { + self.delete_legacy(input) + } + + /// **Deprecated compatibility API — planned removal: 0.5.0.** + /// + /// Use `Collection::typed()?.rename(RenameRequest)` or `preflight_rename`. + pub fn rename(&self, input: &serde_json::Value) -> serde_json::Value { + self.rename_legacy(input) + } + + /// **Deprecated compatibility API — planned removal: 0.5.0.** + /// + /// No canonical backfill operation exists. Use typed query/read followed by + /// `Collection::typed()?.batch(BatchRequest)`. + pub fn backfill(&self, input: &serde_json::Value) -> serde_json::Value { + self.backfill_legacy(input) + } + + /// **Deprecated compatibility API — planned removal: 0.5.0.** + /// + /// Use `Collection::typed()?.batch(BatchRequest)`. + pub fn batch_update( + &self, + input: &serde_json::Value, + simulate_io_error: Option<&str>, + skip_dependents: bool, + ) -> serde_json::Value { + self.batch_update_legacy(input, simulate_io_error, skip_dependents) + } + + /// **Deprecated compatibility API — planned removal: 0.5.0.** + /// + /// Use `Collection::typed()?.batch(BatchRequest)`. + pub fn batch_delete( + &self, + input: &serde_json::Value, + simulate_io_error: Option<&str>, + ) -> serde_json::Value { + self.batch_delete_legacy(input, simulate_io_error) + } +} diff --git a/src/compat/mod.rs b/src/compat/mod.rs index 4d17640..9b39418 100644 --- a/src/compat/mod.rs +++ b/src/compat/mod.rs @@ -1,4 +1,6 @@ //! Import and migration adapters for older collection formats. +#[cfg(feature = "legacy-collection-mutation")] +mod legacy_mutation; pub(crate) mod v02; pub(crate) mod v02_migration; diff --git a/src/compat/v02_migration.rs b/src/compat/v02_migration.rs index c83e5b0..a76849c 100644 --- a/src/compat/v02_migration.rs +++ b/src/compat/v02_migration.rs @@ -6,7 +6,6 @@ use std::path::Path; use serde_json::{json, Map, Value}; use sha2::{Digest, Sha256}; -use walkdir::WalkDir; use crate::api::{ Diagnostic, DiagnosticCode, MdbaseError, MdbaseResult, ReadRequest, Revision, Severity, @@ -97,7 +96,7 @@ pub(crate) fn migrate( ) .collect::>(); for path in changed_paths { - match fs::read(collection.root.join(&path)) { + match collection.held_root().read(&path) { Ok(bytes) => { baseline.insert(path, bytes); } @@ -145,7 +144,19 @@ fn canonical_config( collection: &Collection, diagnostics: &mut Vec, ) -> MdbaseResult> { - let original = read_yaml_json(&collection.root.join("mdbase.yaml"), "mdbase.yaml")?; + let original = read_yaml_json_bytes( + &collection + .held_root() + .read("mdbase.yaml") + .map_err(|error| { + operation_error( + "migration_plan_failed", + &format!("Could not read legacy YAML: {error}"), + Some("mdbase.yaml"), + ) + })?, + "mdbase.yaml", + )?; let mut record_extensions = vec![Value::String("md".to_string())]; record_extensions.extend( collection @@ -565,7 +576,7 @@ fn verify_equivalent_reads(collection: &Collection, desired: &FileBaseline) -> M ) })?; copy_for_verification( - &collection.root, + collection, temporary.path(), &collection.settings.types_folder, )?; @@ -598,7 +609,7 @@ fn verify_equivalent_reads(collection: &Collection, desired: &FileBaseline) -> M let legacy_api = collection.typed()?; let canonical_api = canonical.typed()?; let paths = collection - .scan_collection_files_checked() + .scan_collection_relative_paths_checked() .map_err(|error| { operation_error( "migration_verification_failed", @@ -606,13 +617,8 @@ fn verify_equivalent_reads(collection: &Collection, desired: &FileBaseline) -> M None, ) })?; - for absolute in &paths { - let path = absolute - .strip_prefix(&collection.root) - .expect("scanner only returns paths below the collection root") - .to_string_lossy() - .replace('\\', "/"); - let request = ReadRequest::new(&path)?; + for path in &paths { + let request = ReadRequest::new(path)?; let legacy = legacy_api.read(request.clone())?.value; let canonical = canonical_api.read(request)?.value; if legacy.frontmatter != canonical.frontmatter @@ -623,72 +629,60 @@ fn verify_equivalent_reads(collection: &Collection, desired: &FileBaseline) -> M return Err(operation_error( "migration_verification_failed", "Canonical read does not match the v0.2 compatibility read.", - Some(&path), + Some(path), )); } } Ok(paths.len()) } -fn copy_for_verification(source: &Path, target: &Path, types_folder: &str) -> MdbaseResult<()> { - for entry in WalkDir::new(source).sort_by_file_name() { - let entry = entry.map_err(|error| { +fn copy_for_verification( + collection: &Collection, + target: &Path, + types_folder: &str, +) -> MdbaseResult<()> { + let paths = collection + .held_root() + .files_recursive(Path::new("")) + .map_err(|error| { operation_error( "migration_verification_failed", &format!("Could not inspect the source collection: {error}"), None, ) })?; - let relative = entry - .path() - .strip_prefix(source) - .expect("walkdir entry is below its root"); - if relative.as_os_str().is_empty() { - continue; - } + for relative in paths { let logical = relative.to_string_lossy().replace('\\', "/"); - if logical == ".mdbase" - || logical.starts_with(".mdbase/") - || logical == types_folder + if logical.starts_with(".mdbase/") || logical.starts_with(&format!("{types_folder}/")) || logical == "mdbase.yaml" { continue; } - if entry.file_type().is_symlink() { - return Err(operation_error( - "migration_verification_failed", - "Symbolic links cannot be copied into the isolated verifier.", - Some(&logical), - )); - } - let destination = target.join(relative); - if entry.file_type().is_dir() { - fs::create_dir_all(&destination).map_err(|error| { + let destination = target.join(&relative); + if let Some(parent) = destination.parent() { + fs::create_dir_all(parent).map_err(|error| { operation_error( "migration_verification_failed", &format!("Could not prepare the verifier: {error}"), Some(&logical), ) })?; - } else if entry.file_type().is_file() { - if let Some(parent) = destination.parent() { - fs::create_dir_all(parent).map_err(|error| { - operation_error( - "migration_verification_failed", - &format!("Could not prepare the verifier: {error}"), - Some(&logical), - ) - })?; - } - fs::copy(entry.path(), &destination).map_err(|error| { - operation_error( - "migration_verification_failed", - &format!("Could not copy a record for verification: {error}"), - Some(&logical), - ) - })?; } + let bytes = collection.held_root().read(&relative).map_err(|error| { + operation_error( + "migration_verification_failed", + &format!("Could not copy a record for verification: {error}"), + Some(&logical), + ) + })?; + fs::write(&destination, bytes).map_err(|error| { + operation_error( + "migration_verification_failed", + &format!("Could not copy a record for verification: {error}"), + Some(&logical), + ) + })?; } Ok(()) } @@ -744,15 +738,8 @@ fn yaml_bytes(value: &Value, frontmatter: bool) -> MdbaseResult> { }) } -fn read_yaml_json(path: &Path, label: &str) -> MdbaseResult { - let bytes = fs::read(path).map_err(|error| { - operation_error( - "migration_plan_failed", - &format!("Could not read legacy YAML: {error}"), - Some(label), - ) - })?; - let yaml: serde_yaml::Value = serde_yaml::from_slice(&bytes).map_err(|error| { +fn read_yaml_json_bytes(bytes: &[u8], label: &str) -> MdbaseResult { + let yaml: serde_yaml::Value = serde_yaml::from_slice(bytes).map_err(|error| { operation_error( "migration_plan_failed", &format!("Could not parse legacy YAML: {error}"), diff --git a/src/config/mod.rs b/src/config/mod.rs index a8d91a4..545cae4 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -12,8 +12,16 @@ pub fn load_config(collection_root: &Path) -> serde_json::Value { } /// Load config for collection opening, allowing forward-minor versions. -pub(crate) fn load_config_for_open(collection_root: &Path) -> serde_json::Value { - load_config_internal(collection_root, true) +pub(crate) fn load_config_for_open_held( + root: &crate::collection_root::CollectionRoot, +) -> serde_json::Value { + match root.read_string("mdbase.yaml") { + Ok(content) => parse_config_document(&content, true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + error_json("missing_config", "mdbase.yaml not found") + } + Err(error) => error_json("invalid_config", &format!("Failed to read config: {error}")), + } } fn load_config_internal(collection_root: &Path, allow_future_minor: bool) -> serde_json::Value { diff --git a/src/data_contracts.rs b/src/data_contracts.rs index 6f88337..9c9b559 100644 --- a/src/data_contracts.rs +++ b/src/data_contracts.rs @@ -180,6 +180,57 @@ impl DataContractRegistry { Ok(registry) } + /// Load contracts from a capability-relative resource snapshot. + pub(crate) fn load_held( + root: &crate::collection_root::CollectionRoot, + settings: &Settings, + types: &HashMap, + ) -> Result { + let staging = tempfile::tempdir().map_err(|error| { + load_error( + "invalid_data_contract", + format!("Could not stage contracts: {error}"), + ) + })?; + let contracts = Path::new(&settings.contracts_folder); + let files = root.files_recursive(Path::new("")).map_err(|error| { + load_error( + "invalid_data_contract", + format!("Could not inspect contract resources: {error}"), + ) + })?; + for relative in files { + let is_contract = relative.starts_with(contracts) + && relative.extension().and_then(|value| value.to_str()) == Some("md"); + let is_schema = relative.extension().and_then(|value| value.to_str()) == Some("json"); + if !is_contract && !is_schema { + continue; + } + let bytes = root.read(&relative).map_err(|error| { + load_error( + "invalid_data_contract", + format!("Could not read '{}': {error}", relative.display()), + ) + })?; + let destination = staging.path().join(&relative); + if let Some(parent) = destination.parent() { + std::fs::create_dir_all(parent).map_err(|error| { + load_error( + "invalid_data_contract", + format!("Could not stage '{}': {error}", relative.display()), + ) + })?; + } + std::fs::write(destination, bytes).map_err(|error| { + load_error( + "invalid_data_contract", + format!("Could not stage '{}': {error}", relative.display()), + ) + })?; + } + Self::load(staging.path(), settings, types) + } + pub(crate) fn load_resolved( contracts: Vec, types: &HashMap, diff --git a/src/expressions/mod.rs b/src/expressions/mod.rs index 80f51d7..21656cc 100644 --- a/src/expressions/mod.rs +++ b/src/expressions/mod.rs @@ -81,17 +81,17 @@ use crate::types::compiled::CompiledComputed; use crate::Collection; use std::collections::{BTreeMap, BTreeSet, HashSet}; -#[cfg(test)] +#[cfg(all(test, feature = "legacy-collection-mutation"))] thread_local! { static COMPUTED_FIELD_EVALUATIONS: std::cell::Cell = const { std::cell::Cell::new(0) }; } -#[cfg(test)] +#[cfg(all(test, feature = "legacy-collection-mutation"))] pub(crate) fn reset_computed_field_evaluations_for_test() { COMPUTED_FIELD_EVALUATIONS.with(|value| value.set(0)); } -#[cfg(test)] +#[cfg(all(test, feature = "legacy-collection-mutation"))] pub(crate) fn computed_field_evaluations_for_test() -> usize { COMPUTED_FIELD_EVALUATIONS.with(std::cell::Cell::get) } @@ -105,7 +105,7 @@ impl Collection { path: &str, body: Option<&str>, ) -> serde_json::Value { - #[cfg(test)] + #[cfg(all(test, feature = "legacy-collection-mutation"))] COMPUTED_FIELD_EVALUATIONS.with(|value| value.set(value.get() + 1)); let mut ordered_types = type_names.to_vec(); ordered_types.sort(); diff --git a/src/file_path.rs b/src/file_path.rs index 73bf362..f0a1b1b 100644 --- a/src/file_path.rs +++ b/src/file_path.rs @@ -33,12 +33,32 @@ impl Collection { pub fn validate_file_path( &self, path: impl AsRef, + ) -> Result { + self.validate_file_path_mode(path, true) + } + + pub(crate) fn validate_file_path_after_traversal( + &self, + path: impl AsRef, + ) -> Result { + self.validate_file_path_mode(path, false) + } + + fn validate_file_path_mode( + &self, + path: impl AsRef, + check_nested: bool, ) -> Result { let path = CollectionPath::new(path)?; if has_hidden_component(path.as_str()) { return Err(FilePathError::HiddenComponent); } - if self.is_excluded(path.as_str()) { + let excluded = if check_nested { + self.is_excluded(path.as_str()) + } else { + self.is_excluded_without_nested_collection(path.as_str()) + }; + if excluded { return Err(FilePathError::Reserved); } if self.is_valid_extension(path.as_str()) { diff --git a/src/lib.rs b/src/lib.rs index 13099a3..246b601 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,7 @@ pub mod api; pub mod cache; pub mod cancellation; pub(crate) mod cel; +mod collection_root; pub(crate) mod compat; pub mod config; pub mod data_contracts; @@ -45,10 +46,12 @@ pub mod watch; pub use cancellation::{OperationCancellation, OperationCancelled, OperationStopReason}; pub use snapshot::{CollectionDiscoveryCause, CollectionSnapshotError}; +#[cfg(all(test, unix))] +pub(crate) use snapshot::replace_descendant_on_scan_for_test; #[cfg(test)] pub(crate) use snapshot::{ - cancel_scan_after_entries_for_test, replace_descendant_on_scan_for_test, - reset_snapshot_scan_calls_for_test, snapshot_scan_calls_for_test, + cancel_scan_after_entries_for_test, reset_snapshot_scan_calls_for_test, + snapshot_scan_calls_for_test, }; use std::collections::HashMap; @@ -158,8 +161,9 @@ impl CollectionResources { /// A loaded mdbase collection. pub struct Collection { + /// Stable display path retained for public compatibility. Never use it as authority. pub(crate) root: PathBuf, - root_capability: cap_std::fs::Dir, + pub(crate) authority: collection_root::CollectionRoot, pub(crate) spec_profile: SpecProfile, pub(crate) settings: Settings, /// Namespaced collection configuration retained for optional adapters. @@ -177,11 +181,11 @@ impl Collection { } pub(crate) fn root_capability(&self) -> std::io::Result { - self.root_capability.try_clone() + self.authority.dir() } - pub(crate) fn capability_for_root(root: &Path) -> std::io::Result { - cap_std::fs::Dir::open_ambient_dir(root, cap_std::ambient_authority()) + pub(crate) fn held_root(&self) -> &collection_root::CollectionRoot { + &self.authority } /// Immutable runtime settings loaded with this collection. @@ -236,16 +240,28 @@ impl Collection { root: &Path, recover_pending_transactions: bool, ) -> Result { - let root_capability = - cap_std::fs::Dir::open_ambient_dir(root, cap_std::ambient_authority()).map_err( - |error| { - crate::errors::op_error( - crate::errors::INVALID_CONFIG, - &format!("collection root could not be opened: {error}"), - ) - }, - )?; - let config_result = config::load_config_for_open(root); + let authority = collection_root::CollectionRoot::acquire(root).map_err(|error| { + crate::errors::op_error( + crate::errors::INVALID_CONFIG, + &format!("collection root could not be opened: {error}"), + ) + })?; + Self::open_held(authority, recover_pending_transactions) + } + + pub(crate) fn reopen_held( + &self, + recover_pending_transactions: bool, + ) -> Result { + Self::open_held(self.authority.clone(), recover_pending_transactions) + } + + fn open_held( + authority: collection_root::CollectionRoot, + recover_pending_transactions: bool, + ) -> Result { + let root = authority.display_path(); + let config_result = config::load_config_for_open_held(&authority); if config_result.get("valid") != Some(&serde_json::Value::Bool(true)) { return Err(config_result); } @@ -364,11 +380,16 @@ impl Collection { )); } *folder = normalized.to_string(); - crate::operations::ensure_no_symlink_components( - root, - normalized.as_str(), - spec_profile, - )?; + match authority.open_dir(&normalized.to_path_buf()) { + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(crate::errors::op_error( + crate::errors::PATH_TRAVERSAL, + &format!("settings.{label} is not a no-follow directory: {error}"), + )); + } + } } // Recovery must precede type loading. A system migration may have @@ -379,12 +400,7 @@ impl Collection { let recovered = if recover_pending_transactions { let recovery_collection = Collection { root: root.to_path_buf(), - root_capability: root_capability.try_clone().map_err(|error| { - crate::errors::op_error( - crate::errors::INVALID_CONFIG, - &format!("collection root capability could not be cloned: {error}"), - ) - })?, + authority: authority.clone(), spec_profile, settings: settings.clone(), config_extensions: config_extensions.clone(), @@ -406,12 +422,12 @@ impl Collection { false }; if recovered { - return Self::open_with_recovery(root, recover_pending_transactions); + return Self::open_held(authority, recover_pending_transactions); } - // Load types - let load_result = loader::load_types_with_warnings( - root, + // Load types from the held capability, never from the display path. + let load_result = loader::load_types_with_warnings_held( + &authority, &settings.types_folder, &settings.migrations_folder, ) @@ -451,20 +467,21 @@ impl Collection { }) })?; - let data_contracts = data_contracts::DataContractRegistry::load(root, &settings, &types) - .map_err(|error| { - serde_json::json!({ - "valid": false, - "error": { - "code": error.code, - "message": error.message, - } - }) - })?; + let data_contracts = + data_contracts::DataContractRegistry::load_held(&authority, &settings, &types) + .map_err(|error| { + serde_json::json!({ + "valid": false, + "error": { + "code": error.code, + "message": error.message, + } + }) + })?; let collection = Collection { root: root.to_path_buf(), - root_capability, + authority, spec_profile, settings, config_extensions, diff --git a/src/links/parser.rs b/src/links/parser.rs index f1c8790..38786fc 100644 --- a/src/links/parser.rs +++ b/src/links/parser.rs @@ -84,6 +84,9 @@ impl Collection { // Wikilink: [[target]], [[target|alias]], [[target#anchor]], [[target#anchor|alias]] if value.starts_with("[[") && value.ends_with("]]") { let inner = &value[2..value.len() - 2]; + if inner.contains('[') || inner.contains(']') { + return malformed_link(&raw, "Malformed wikilink delimiters"); + } // Split on | for alias let (target_part, alias) = if let Some(pipe_idx) = inner.find('|') { (&inner[..pipe_idx], Some(inner[pipe_idx + 1..].to_string())) @@ -99,6 +102,9 @@ impl Collection { } else { (target_part.to_string(), None) }; + if target.trim().is_empty() || target.contains(['\n', '\r']) { + return malformed_link(&raw, "Wikilink target must be nonempty and single-line"); + } let is_relative = target.starts_with("./") || target.starts_with("../"); return serde_json::json!({ "link": { @@ -112,11 +118,23 @@ impl Collection { }); } + if value.starts_with("[[") { + return malformed_link(&raw, "Malformed or unterminated wikilink syntax"); + } + // Markdown link: [text](path) or [text](path#anchor) if value.starts_with('[') && value.contains("](") && value.ends_with(')') { let bracket_end = value.find("](").unwrap(); let text = &value[1..bracket_end]; let path_str = &value[bracket_end + 2..value.len() - 1]; + if text.contains('[') + || text.contains(']') + || path_str.contains('(') + || path_str.contains(')') + || value[bracket_end + 2..].matches("](").count() > 0 + { + return malformed_link(&raw, "Malformed Markdown link delimiters"); + } let (path, anchor) = if let Some(hash_idx) = path_str.find('#') { ( path_str[..hash_idx].to_string(), @@ -125,6 +143,12 @@ impl Collection { } else { (path_str.to_string(), None) }; + if path.trim().is_empty() || path.contains(['\n', '\r']) { + return malformed_link( + &raw, + "Markdown link target must be nonempty and single-line", + ); + } let is_relative = path.starts_with("./") || path.starts_with("../"); let alias = Some(text.to_string()); return serde_json::json!({ @@ -139,6 +163,12 @@ impl Collection { }); } + // Only strings that clearly begin Markdown link syntax are rejected. + // Ordinary prose containing brackets remains a permissive bare value. + if value.starts_with('[') && value.contains("](") { + return malformed_link(&raw, "Malformed or unterminated Markdown link syntax"); + } + // Bare/path let is_relative = value.starts_with("./") || value.starts_with("../"); serde_json::json!({ @@ -153,3 +183,66 @@ impl Collection { }) } } + +fn malformed_link(raw: &str, message: &str) -> serde_json::Value { + serde_json::json!({ + "error": {"code": "malformed_link", "message": message}, + "raw": raw, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn collection() -> (tempfile::TempDir, Collection) { + let root = tempfile::tempdir().unwrap(); + std::fs::write(root.path().join("mdbase.yaml"), "spec_version: 0.3.0\n").unwrap(); + let collection = Collection::open(root.path()).unwrap(); + (root, collection) + } + + #[test] + fn malformed_link_intent_is_not_reclassified_as_a_bare_path() { + let (_root, collection) = collection(); + for value in [ + "[[unterminated", + "[[]]", + "[[ ]]", + "[[target\nline]]", + "[[|]]", + "[[#]]", + "[[target]]junk]]", + "[[[target]]]", + "[label](unterminated", + "[text]()", + "[text]( )", + "[text](#anchor)", + "[text](target\nline)", + "[label](target)junk)", + "[label](target)(extra)", + ] { + let parsed = collection.parse_link(&serde_json::json!({"value": value})); + assert_eq!(parsed["error"]["code"], "malformed_link", "{parsed}"); + assert!(parsed.get("link").is_none(), "{parsed}"); + } + for (value, target) in [ + ("[[nested]]", "nested"), + ("[[nested#anchor|Alias]]", "nested"), + ("[Alias](nested.md#anchor)", "nested.md"), + ] { + let valid = collection.parse_link(&serde_json::json!({"value": value})); + assert_eq!(valid["link"]["target"], target, "{valid}"); + } + } + + #[test] + fn permissive_prose_remains_a_bare_path() { + let (_root, collection) = collection(); + for value in ["prose [aside", "unmatched ]] prose"] { + let parsed = collection.parse_link(&serde_json::json!({"value": value})); + assert_eq!(parsed["link"]["format"], "path"); + assert_eq!(parsed["link"]["target"], value); + } + } +} diff --git a/src/links/resolver.rs b/src/links/resolver.rs index f5ae0c6..c06a049 100644 --- a/src/links/resolver.rs +++ b/src/links/resolver.rs @@ -104,8 +104,8 @@ pub(crate) fn compute_relative_path(source_dir: &str, target_path: &str) -> Stri use crate::errors::*; use crate::links::parser::{count_leading_dotdot, normalize_link_path}; use crate::runtime::{ - select_resolution_candidate, RankedResolution, RankedResolutionCandidate, - RecordResolutionKeyKind, + select_resolution_candidate, CatalogError, RankedResolution, RankedResolutionCandidate, + RecordResolutionKeyKind, ResolutionReason, }; use crate::types::schema::FieldDef; use crate::Collection; @@ -124,10 +124,48 @@ pub(crate) struct LinkResolutionIndex { #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum LinkResolution { Missing, - Resolved(String), + Resolved { + path: String, + reason: ResolutionReason, + selected_kind: RecordResolutionKeyKind, + candidate_count: usize, + candidate_digest: String, + alternatives: Vec, + alternative_candidates: Vec, + }, Ambiguous(Vec), } +fn resolution_error_json(error: &CatalogError, field: Option<&str>) -> serde_json::Value { + let mut issue = serde_json::json!({ + "code": error.code, + "message": error.message, + "severity": "error", + }); + if let Some(field) = field { + issue["field"] = serde_json::Value::String(field.to_string()); + } + serde_json::json!({ + "error": {"code": error.code, "message": error.message}, + "issues": [issue], + }) +} + +fn selector_issue(error: CatalogError, path: &str, field: &str, type_name: &str) -> Issue { + Issue { + code: error.code, + message: error.message, + path: Some(path.to_string()), + field: Some(field.to_string()), + severity: Severity::Error, + expected: None, + actual: None, + type_name: Some(type_name.to_string()), + line: None, + column: None, + } +} + fn insert_resolution_key(index: &mut HashMap>, key: String, path: &str) { let paths = index.entry(key).or_default(); if !paths.iter().any(|candidate| candidate == path) { @@ -140,16 +178,35 @@ fn select_local_resolution( source_path: &str, kind: RecordResolutionKeyKind, paths: impl IntoIterator, -) -> LinkResolution { +) -> Result { let candidates = paths.into_iter().map(|path| RankedResolutionCandidate { record_id: path.clone(), path, }); - match select_resolution_candidate(source_path, kind, candidates) { - Ok(RankedResolution::Missing) | Err(_) => LinkResolution::Missing, - Ok(RankedResolution::Resolved { path, .. }) => LinkResolution::Resolved(path), - Ok(RankedResolution::Ambiguous { paths }) => LinkResolution::Ambiguous(paths), - } + Ok( + match select_resolution_candidate(source_path, kind, candidates)? { + RankedResolution::Missing => LinkResolution::Missing, + RankedResolution::Resolved { + path, + reason, + selected_kind, + candidate_count, + candidate_digest, + alternatives, + alternative_candidates, + .. + } => LinkResolution::Resolved { + path, + reason, + selected_kind, + candidate_count, + candidate_digest, + alternatives, + alternative_candidates, + }, + RankedResolution::Ambiguous { paths } => LinkResolution::Ambiguous(paths), + }, + ) } impl Collection { @@ -215,8 +272,7 @@ impl Collection { return serde_json::json!({"error": {"code": "path_traversal", "message": "Link source path is unsafe"}}); } - let snapshot = match self.capture_collection_snapshot(&crate::OperationCancellation::new()) - { + let snapshot = match self.capture_collection_snapshot_current() { Ok(snapshot) => snapshot, Err(error) => { return crate::errors::op_error("collection_snapshot_failed", &error.to_string()) @@ -239,6 +295,12 @@ impl Collection { None => return serde_json::json!({"resolved_path": serde_json::Value::Null}), }; let parsed = self.parse_link(&serde_json::json!({"value": field_val})); + if let Some(error) = parsed.get("error") { + return serde_json::json!({ + "error": error, + "issues": [{"code": "malformed_link", "field": field_name, "severity": "error"}] + }); + } let Some(target) = parsed .pointer("/link/target") .and_then(serde_json::Value::as_str) @@ -277,8 +339,9 @@ impl Collection { Some(target.to_string()) } else { match self.resolve_simple_name(&index, target, source_path, &target_types) { - LinkResolution::Resolved(path) => Some(path), - LinkResolution::Missing | LinkResolution::Ambiguous(_) => None, + Ok(LinkResolution::Resolved { path, .. }) => Some(path), + Ok(LinkResolution::Missing | LinkResolution::Ambiguous(_)) => None, + Err(error) => return resolution_error_json(&error, Some(field_name)), } } .map(|path| normalize_collection_path(&path)); @@ -359,7 +422,7 @@ impl Collection { name: &str, source_path: &str, target_types: &[String], - ) -> LinkResolution { + ) -> Result { let name_lower = name.to_lowercase(); for (kind, candidates) in [ ( @@ -383,7 +446,7 @@ impl Collection { return select_local_resolution(source_path, kind, eligible); } } - LinkResolution::Missing + Ok(LinkResolution::Missing) } pub(crate) fn get_field_target_types_from_frontmatter( @@ -434,7 +497,7 @@ impl Collection { source_path: &str, target_types: &[String], resolution_index: &LinkResolutionIndex, - ) -> Option { + ) -> Result, CatalogError> { // Strip wikilink syntax let target = if target.starts_with("[[") && target.ends_with("]]") { let inner = &target[2..target.len() - 2]; @@ -452,7 +515,7 @@ impl Collection { }; if target.is_empty() { - return None; + return Ok(None); } // Handle relative paths (./foo, ../foo) @@ -466,7 +529,9 @@ impl Collection { for c in joined.components() { match c { std::path::Component::ParentDir => { - components.pop()?; + if components.pop().is_none() { + return Ok(None); + } } std::path::Component::CurDir => {} _ => { @@ -490,73 +555,76 @@ impl Collection { let eligible = self.eligible_resolution_paths(resolution_index, paths, target_types); if !eligible.is_empty() { - return match select_local_resolution( + return select_local_resolution( source_path, RecordResolutionKeyKind::Id, eligible, - ) { - LinkResolution::Resolved(path) => Some(path), + ) + .map(|resolution| match resolution { + LinkResolution::Resolved { path, .. } => Some(path), LinkResolution::Missing | LinkResolution::Ambiguous(_) => None, - }; + }); } } if let Some(paths) = resolution_index.basename_lower_to_paths.get(&target_lower) { let eligible = self.eligible_resolution_paths(resolution_index, paths, target_types); if !eligible.is_empty() { - return match select_local_resolution( + return select_local_resolution( source_path, RecordResolutionKeyKind::Basename, eligible, - ) { - LinkResolution::Resolved(path) => Some(path), + ) + .map(|resolution| match resolution { + LinkResolution::Resolved { path, .. } => Some(path), LinkResolution::Missing | LinkResolution::Ambiguous(_) => None, - }; + }); } } if let Some(paths) = resolution_index.title_lower_to_paths.get(&target_lower) { let eligible = self.eligible_resolution_paths(resolution_index, paths, target_types); if !eligible.is_empty() { - return match select_local_resolution( + return select_local_resolution( source_path, RecordResolutionKeyKind::Title, eligible, - ) { - LinkResolution::Resolved(path) => Some(path), + ) + .map(|resolution| match resolution { + LinkResolution::Resolved { path, .. } => Some(path), LinkResolution::Missing | LinkResolution::Ambiguous(_) => None, - }; + }); } } - return None; + return Ok(None); } // Explicit path targets retain exact path and extension behavior. if resolution_index.known_paths.contains(&resolved_target) { - return self + return Ok(self .eligible_resolution_paths( resolution_index, std::slice::from_ref(&resolved_target), target_types, ) .into_iter() - .next(); + .next()); } if !resolved_target.ends_with(".md") && !resolved_target.ends_with(".mdx") { let with_md = format!("{}.md", resolved_target); if resolution_index.known_paths.contains(&with_md) { - return self + return Ok(self .eligible_resolution_paths( resolution_index, std::slice::from_ref(&with_md), target_types, ) .into_iter() - .next(); + .next()); } } - None + Ok(None) } /// Check link fields with validate_exists: true. @@ -761,6 +829,21 @@ impl Collection { // not part of a filename and Markdown links resolve their URL rather // than their complete display syntax. let parsed = self.parse_link(&serde_json::json!({"value": link_str})); + if parsed.get("error").is_some() { + issues.push(Issue { + code: "malformed_link".to_string(), + message: format!("Malformed link syntax in field '{field_name}'"), + path: Some(path.to_string()), + field: Some(field_name.to_string()), + severity: Severity::Error, + expected: None, + actual: Some(serde_json::Value::String(link_str.to_string())), + type_name: Some(type_name.to_string()), + line: None, + column: None, + }); + return issues; + } let target = parsed .get("link") .and_then(|link| link.get("target")) @@ -813,14 +896,37 @@ impl Collection { // Resolve link target if field_def.validate_exists == Some(true) { let expected = allowed_target_types(field_def); - let matches = - self.resolve_link_matches(resolution_index, &normalized, target, path, &expected); + let matches = match self.resolve_link_matches( + resolution_index, + &normalized, + target, + path, + &expected, + ) { + Ok(matches) => matches, + Err(error) => { + issues.push(selector_issue(error, path, field_name, type_name)); + return issues; + } + }; if matches.is_empty() { let unfiltered = if expected.is_empty() { Vec::new() } else { - self.resolve_link_matches(resolution_index, &normalized, target, path, &[]) + match self.resolve_link_matches( + resolution_index, + &normalized, + target, + path, + &[], + ) { + Ok(matches) => matches, + Err(error) => { + issues.push(selector_issue(error, path, field_name, type_name)); + return issues; + } + } }; if unfiltered.len() == 1 { let target_types = resolution_index @@ -910,8 +1016,19 @@ impl Collection { } else if !allowed_target_types(field_def).is_empty() { // Even without validate_exists, check target type if we can resolve let expected = allowed_target_types(field_def); - let matches = - self.resolve_link_matches(resolution_index, &normalized, target, path, &expected); + let matches = match self.resolve_link_matches( + resolution_index, + &normalized, + target, + path, + &expected, + ) { + Ok(matches) => matches, + Err(error) => { + issues.push(selector_issue(error, path, field_name, type_name)); + return issues; + } + }; if matches.len() == 1 { let matched_path = &matches[0]; let target_types = resolution_index @@ -971,20 +1088,17 @@ impl Collection { original: &str, source_path: &str, target_types: &[String], - ) -> Vec { + ) -> Result, CatalogError> { let simple_name = !original.contains('/') && std::path::Path::new(original).extension().is_none(); if simple_name { - return match self.resolve_simple_name( - resolution_index, - original, - source_path, - target_types, - ) { - LinkResolution::Missing => Vec::new(), - LinkResolution::Resolved(path) => vec![path], - LinkResolution::Ambiguous(paths) => paths, - }; + return self + .resolve_simple_name(resolution_index, original, source_path, target_types) + .map(|resolution| match resolution { + LinkResolution::Missing => Vec::new(), + LinkResolution::Resolved { path, .. } => vec![path], + LinkResolution::Ambiguous(paths) => paths, + }); } let candidates = if !normalized.ends_with(".md") && !normalized.ends_with(".mdx") { @@ -992,14 +1106,14 @@ impl Collection { } else { vec![normalized.to_string()] }; - self.eligible_resolution_paths( + Ok(self.eligible_resolution_paths( resolution_index, &candidates .into_iter() .filter(|path| resolution_index.known_paths.contains(path)) .collect::>(), target_types, - ) + )) } } @@ -1007,6 +1121,53 @@ impl Collection { mod snapshot_resolution_tests { use super::*; + #[test] + fn local_resolution_carries_the_shared_selector_evidence() { + let root = tempfile::tempdir().unwrap(); + std::fs::write(root.path().join("mdbase.yaml"), "spec_version: 0.3.0\n").unwrap(); + let collection = Collection::open(root.path()).unwrap(); + let mut index = LinkResolutionIndex::default(); + index.basename_lower_to_paths.insert( + "target".to_string(), + vec!["z/target.md".to_string(), "notes/target.md".to_string()], + ); + let resolution = collection + .resolve_simple_name(&index, "target", "notes/source.md", &[]) + .unwrap(); + let LinkResolution::Resolved { + path, + reason, + selected_kind, + candidate_count, + candidate_digest, + alternatives, + alternative_candidates, + } = resolution + else { + panic!("expected resolved evidence"); + }; + assert_eq!(path, "notes/target.md"); + assert_eq!(reason, ResolutionReason::SameDirectory); + assert_eq!(selected_kind, RecordResolutionKeyKind::Basename); + assert_eq!(candidate_count, 2); + assert!(candidate_digest.starts_with("sha256:")); + assert_eq!(alternatives, ["z/target.md"]); + assert_eq!(alternative_candidates.len(), 1); + assert_eq!(alternative_candidates[0].path, "z/target.md"); + } + + #[test] + fn public_resolve_reports_malformed_instead_of_missing() { + let root = tempfile::tempdir().unwrap(); + std::fs::write(root.path().join("mdbase.yaml"), "spec_version: 0.3.0\n").unwrap(); + std::fs::write(root.path().join("source.md"), "---\nref: '[[broken'\n---\n").unwrap(); + let collection = Collection::open(root.path()).unwrap(); + let resolved = + collection.resolve_link(&serde_json::json!({"path": "source.md", "field": "ref"})); + assert_eq!(resolved["error"]["code"], "malformed_link", "{resolved}"); + assert!(resolved.get("resolved_path").is_none(), "{resolved}"); + } + #[test] fn public_resolve_scans_once_and_loads_each_discovered_record_once() { let root = tempfile::tempdir().unwrap(); diff --git a/src/links/traversal.rs b/src/links/traversal.rs index 249e6fd..a8c583a 100644 --- a/src/links/traversal.rs +++ b/src/links/traversal.rs @@ -1,9 +1,12 @@ //! asFile() traversal (§8.7). -use crate::{Collection, CollectionSnapshotError, OperationCancellation}; +use crate::{Collection, CollectionSnapshotError}; use std::collections::HashMap; use std::time::Instant; +type BacklinksBuildResult = + Result<(HashMap>, Option), crate::runtime::CatalogError>; + #[derive(Debug, Clone, Default)] pub(crate) struct BacklinksPerf { pub total_ms: f64, @@ -32,25 +35,26 @@ impl Collection { pub fn build_all_files_data( &self, ) -> Result, CollectionSnapshotError> { - self.capture_collection_snapshot(&OperationCancellation::new()) + self.capture_collection_snapshot_current() .map(|snapshot| snapshot.resolved_files_data()) .map_err(CollectionSnapshotError::from) } - /// Build backlinks index from all files data. - /// Returns a map: target_path -> Vec (deduplicated). + /// Build a backlinks index or return the selector diagnostic. Resolution + /// failures are never represented as graph entries. pub fn build_backlinks_index( &self, all_files: &[crate::expressions::evaluator::ResolvedFileData], - ) -> HashMap> { - self.build_backlinks_index_profiled(all_files, false).0 + ) -> Result>, crate::runtime::CatalogError> { + self.build_backlinks_index_profiled(all_files, false) + .map(|(index, _)| index) } pub(crate) fn build_backlinks_index_profiled( &self, all_files: &[crate::expressions::evaluator::ResolvedFileData], profile: bool, - ) -> (HashMap>, Option) { + ) -> BacklinksBuildResult { let resolution_index = self.build_link_resolution_index(all_files); self.build_backlinks_index_with_resolution(all_files, profile, &resolution_index) } @@ -58,7 +62,7 @@ impl Collection { pub(crate) fn build_backlinks_index_for_snapshot( &self, snapshot: &crate::snapshot::AuthoritativeCollectionSnapshot, - ) -> HashMap> { + ) -> Result>, crate::runtime::CatalogError> { let resolved_files = snapshot.resolved_files_data(); self.build_backlinks_index_for_snapshot_files(snapshot, &resolved_files) } @@ -67,10 +71,10 @@ impl Collection { &self, snapshot: &crate::snapshot::AuthoritativeCollectionSnapshot, resolved_files: &[crate::expressions::evaluator::ResolvedFileData], - ) -> HashMap> { + ) -> Result>, crate::runtime::CatalogError> { let resolution_index = snapshot.link_resolution_index_from_resolved(self, resolved_files); self.build_backlinks_index_with_resolution(resolved_files, false, &resolution_index) - .0 + .map(|(index, _)| index) } fn build_backlinks_index_with_resolution( @@ -78,7 +82,7 @@ impl Collection { all_files: &[crate::expressions::evaluator::ResolvedFileData], profile: bool, resolution_index: &crate::links::resolver::LinkResolutionIndex, - ) -> (HashMap>, Option) { + ) -> BacklinksBuildResult { use crate::expressions::evaluator::{ extract_embeds_from_body, extract_links_from_body, extract_links_from_fm_value, }; @@ -135,7 +139,7 @@ impl Collection { // Resolve the target to a file path perf.resolve_calls += 1; let resolved = - self.resolve_link_target(target, source_path, target_types, resolution_index); + self.resolve_link_target(target, source_path, target_types, resolution_index)?; if let Some(resolved_path) = resolved { if !seen_targets.contains(&resolved_path) { seen_targets.push(resolved_path.clone()); @@ -157,9 +161,9 @@ impl Collection { perf.total_ms = elapsed_ms(total_start); if profile { - (index, Some(perf)) + Ok((index, Some(perf))) } else { - (index, None) + Ok((index, None)) } } } diff --git a/src/matching/engine.rs b/src/matching/engine.rs index ab4c320..140e254 100644 --- a/src/matching/engine.rs +++ b/src/matching/engine.rs @@ -436,6 +436,13 @@ use crate::Collection; impl Collection { /// Check if a path is excluded from the collection. pub(crate) fn is_excluded(&self, rel_path: &str) -> bool { + self.is_excluded_without_nested_collection(rel_path) + || self.is_in_nested_collection(rel_path) + } + + /// Apply lexical/configured exclusions when traversal has already fenced + /// nested collections using the currently opened descendant handle. + pub(crate) fn is_excluded_without_nested_collection(&self, rel_path: &str) -> bool { // Check types folder if rel_path.starts_with(&format!("{}/", self.settings.types_folder)) || rel_path == self.settings.types_folder @@ -488,13 +495,6 @@ impl Collection { return true; } - // Check nested collection boundary (§2.8) - // If any parent directory of this path contains mdbase.yaml, - // the file is inside a nested collection and not part of this one. - if self.is_in_nested_collection(rel_path) { - return true; - } - false } @@ -522,8 +522,8 @@ impl Collection { // Check each parent directory component (not the file itself) for component in path.parent().into_iter().flat_map(|p| p.components()) { current.push(component); - let config_path = self.root.join(¤t).join("mdbase.yaml"); - if config_path.exists() { + let config_path = current.join("mdbase.yaml"); + if self.held_root().entry_exists(&config_path).unwrap_or(true) { return true; } } diff --git a/src/mutation/batch.rs b/src/mutation/batch.rs index f05fe0d..1719e33 100644 --- a/src/mutation/batch.rs +++ b/src/mutation/batch.rs @@ -24,6 +24,37 @@ use crate::{Collection, SpecProfile}; use super::{PlannedDelete, PlannedRecord, PlannedRename, PreparationOptions}; +#[cfg(test)] +type PrecommitHook = Box; + +#[cfg(test)] +fn precommit_hooks( +) -> &'static std::sync::Mutex> { + static HOOKS: std::sync::OnceLock< + std::sync::Mutex>, + > = std::sync::OnceLock::new(); + HOOKS.get_or_init(Default::default) +} + +#[cfg(test)] +pub(crate) fn inject_precommit_hook(root: &std::path::Path, hook: impl FnOnce() + Send + 'static) { + precommit_hooks() + .lock() + .expect("mutation precommit hook lock") + .insert(root.to_path_buf(), Box::new(hook)); +} + +#[cfg(test)] +fn run_precommit_hook(root: &std::path::Path) { + let hook = precommit_hooks() + .lock() + .expect("mutation precommit hook lock") + .remove(root); + if let Some(hook) = hook { + hook(); + } +} + /// Complete canonical result, including failures that prevented publication. pub(crate) struct BatchExecution { pub(crate) result: BatchResult, @@ -61,6 +92,23 @@ impl BatchWireOptions { pub(crate) fn batch( collection: &Collection, request: BatchRequest, +) -> Result, MdbaseError> { + batch_with_optional_context(collection, request, None) +} + +pub(crate) fn batch_with_context( + collection: &Collection, + request: BatchRequest, + context: &OperationContext, +) -> Result, MdbaseError> { + context.check().map_err(provider_error_as_mdbase)?; + batch_with_optional_context(collection, request, Some(context)) +} + +fn batch_with_optional_context( + collection: &Collection, + request: BatchRequest, + context: Option<&OperationContext>, ) -> Result, MdbaseError> { ensure_canonical(collection)?; validate_request(&request)?; @@ -85,7 +133,7 @@ pub(crate) fn batch( request.operations, options, request.dry_run, - None, + context, )?; if execution.result.failed != 0 { return Err(MdbaseError::Operation { @@ -149,7 +197,7 @@ pub(crate) fn prepare_runtime_batch( request.dry_run, ))); } - let before = collection.snapshot()?; + let before = collection.snapshot_with_context(context)?; context.check()?; let shadow = super::shadow_collection_context(collection, context)?; let mut execution = execute_items( @@ -165,7 +213,7 @@ pub(crate) fn prepare_runtime_batch( } context.check()?; let desired = super::collect_collection_files_context(&shadow.collection, context)?; - let after = shadow.collection.snapshot()?; + let after = shadow.collection.snapshot_with_context(context)?; execution.result.preflight = false; context.check()?; Ok(RuntimeBatchPreparation::Prepared(Box::new( @@ -207,6 +255,11 @@ fn execute_atomic( None => super::collect_collection_files(&shadow.collection) .map_err(|item| operation_error(vec![*item]))?, }; + #[cfg(test)] + run_precommit_hook(collection.root()); + if let Some(context) = context { + context.check().map_err(provider_error_as_mdbase)?; + } let commit = crate::transactions::commit_shadow(collection, &shadow.baseline, &desired) .map_err(|error| { operation_error(vec![CanonicalDiagnostic::error( @@ -307,7 +360,7 @@ fn execute_items_direct_context( operations, options, dry_run, - &OperationContext::legacy(), + &OperationContext::internal(), ) } @@ -404,11 +457,12 @@ pub(crate) fn record_result( collection: &Collection, planned: PlannedRecord, ) -> Result<(bool, BatchOperationResult, Vec), MdbaseError> { - let metadata = std::fs::metadata(planned.path.under(&collection.root)).map_err(|error| { - MdbaseError::InvalidResult { + let metadata = collection + .held_root() + .metadata(&planned.path.to_path_buf()) + .map_err(|error| MdbaseError::InvalidResult { message: error.to_string(), - } - })?; + })?; let outcome = super::project_record(collection, planned, metadata)?; Ok(( true, diff --git a/src/mutation/mod.rs b/src/mutation/mod.rs index ccd24d2..7494b53 100644 --- a/src/mutation/mod.rs +++ b/src/mutation/mod.rs @@ -12,28 +12,32 @@ mod preparation; pub(crate) mod service; pub(crate) mod shadow; +#[cfg(test)] +pub(crate) use batch::inject_precommit_hook; pub(crate) use batch::{ - aggregate_partial, batch, batch_wire, delete_result, execute_partial_item, + aggregate_partial, batch, batch_wire, batch_with_context, delete_result, execute_partial_item, prepare_runtime_batch, record_result, rename_result, BatchExecution, BatchWireOptions, RuntimeBatchPreparation, }; pub(crate) use lifecycle::LifecycleEvent; pub(crate) use membership::ResolvedWriteMembership; +#[cfg(feature = "legacy-collection-mutation")] +pub(crate) use model::MutationFailureKind; pub(crate) use model::{ - diagnostic_from_issue, MutationFailure, MutationFailureKind, PlannedDelete, PlannedRecord, - PlannedRename, PreparationOptions, PreparedCreate, PreparedDelete, PreparedRename, - PreparedUpdate, + diagnostic_from_issue, MutationFailure, PlannedDelete, PlannedRecord, PlannedRename, + PreparationOptions, PreparedCreate, PreparedDelete, PreparedRename, PreparedUpdate, }; pub(crate) use preparation::{prepare_create, prepare_delete, prepare_rename, prepare_update}; +#[cfg(all(test, feature = "legacy-collection-mutation"))] +pub(crate) use service::probe_legacy_parse; pub(crate) use service::{ create, delete, plan_delete, plan_rename, preflight_delete, preflight_rename, project_record, rename, staged_create, staged_delete, staged_rename, staged_update, update, }; #[cfg(test)] pub(crate) use service::{ - mutation_path_probes, probe_full_shadow, probe_legacy_parse, probe_request_value, - probe_runtime_decode, probe_sparse_shadow, probe_wire_decode, reset_mutation_path_probes, - MutationPathProbes, + mutation_path_probes, probe_full_shadow, probe_request_value, probe_runtime_decode, + probe_sparse_shadow, probe_wire_decode, reset_mutation_path_probes, MutationPathProbes, }; pub(crate) use shadow::{ collect_collection_files, collect_collection_files_context, shadow_collection, diff --git a/src/mutation/model.rs b/src/mutation/model.rs index b18277d..1f350f8 100644 --- a/src/mutation/model.rs +++ b/src/mutation/model.rs @@ -144,6 +144,7 @@ pub(crate) struct PlannedRecord { pub include_document: bool, } +#[cfg(feature = "legacy-collection-mutation")] #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum MutationFailureKind { Operation, @@ -153,6 +154,7 @@ pub(crate) enum MutationFailureKind { #[derive(Clone, Debug)] pub(crate) struct MutationFailure { pub diagnostics: Vec, + #[cfg(feature = "legacy-collection-mutation")] pub kind: MutationFailureKind, } @@ -160,6 +162,7 @@ impl MutationFailure { pub(crate) fn operation(code: impl Into, message: impl Into) -> Self { Self { diagnostics: vec![Diagnostic::error(code, message, None)], + #[cfg(feature = "legacy-collection-mutation")] kind: MutationFailureKind::Operation, } } @@ -167,22 +170,29 @@ impl MutationFailure { pub(crate) fn diagnostic(diagnostic: Diagnostic) -> Self { Self { diagnostics: vec![diagnostic], + #[cfg(feature = "legacy-collection-mutation")] kind: MutationFailureKind::Operation, } } pub(crate) fn diagnostics(diagnostics: Vec) -> Self { + #[cfg(feature = "legacy-collection-mutation")] let kind = if diagnostics.len() == 1 { MutationFailureKind::Operation } else { MutationFailureKind::Validation }; - Self { diagnostics, kind } + Self { + diagnostics, + #[cfg(feature = "legacy-collection-mutation")] + kind, + } } pub(crate) fn validation(issues: &[Issue]) -> Self { Self { diagnostics: issues.iter().map(diagnostic_from_issue).collect(), + #[cfg(feature = "legacy-collection-mutation")] kind: MutationFailureKind::Validation, } } diff --git a/src/mutation/preparation.rs b/src/mutation/preparation.rs index 2965341..2e96715 100644 --- a/src/mutation/preparation.rs +++ b/src/mutation/preparation.rs @@ -60,23 +60,12 @@ pub(crate) fn prepare_update( _options: PreparationOptions, ) -> Result> { let path = request.path.to_string(); - crate::operations::ensure_no_symlink_components_diagnostic( - &collection.root, - &path, - collection.spec_profile, - ) - .map_err(|mut error| { - error.path = Some(path.clone()); - vec![error] - })?; - crate::operations::ensure_regular_record_file_diagnostic( - &request.path.under(&collection.root), - &path, - ) - .map_err(|mut error| { - error.path = Some(path.clone()); - vec![error] - })?; + crate::operations::ensure_no_symlink_components_held_diagnostic(collection, &path).map_err( + |mut error| { + error.path = Some(path.clone()); + vec![error] + }, + )?; let loaded = crate::record_load::load_record(collection, &path).map_err(|_| { vec![Diagnostic::error( "file_read_failed", @@ -183,28 +172,17 @@ pub(crate) fn prepare_delete( vec![error] }, )?; - crate::operations::ensure_no_symlink_components_diagnostic( - &collection.root, - &path, - collection.spec_profile, - ) - .map_err(|mut error| { - error.path = Some(path.clone()); - vec![error] - })?; - crate::operations::ensure_regular_record_file_diagnostic( - &request.path.under(&collection.root), - &path, - ) - .map_err(|mut error| { - error.path = Some(path.clone()); - vec![error] - })?; + crate::operations::ensure_no_symlink_components_held_diagnostic(collection, &path).map_err( + |mut error| { + error.path = Some(path.clone()); + vec![error] + }, + )?; let (before_revision, before_frontmatter, before_body, types, broken_links) = if request.check_backlinks { let snapshot = collection - .capture_collection_snapshot(&crate::OperationCancellation::new()) + .capture_collection_snapshot_current() .map_err(|error| { vec![Diagnostic::error( "collection_snapshot_failed", @@ -219,8 +197,16 @@ pub(crate) fn prepare_delete( Some(path.clone()), )] })?; - let broken_links = collection + let backlinks = collection .build_backlinks_index_for_snapshot(&snapshot) + .map_err(|error| { + vec![Diagnostic::error( + error.code, + error.message, + Some(path.clone()), + )] + })?; + let broken_links = backlinks .get(&path) .into_iter() .flatten() @@ -272,11 +258,10 @@ pub(crate) fn prepare_delete( )]); } if let Some(known_ms) = legacy_last_known_mtime { - let current_ms = std::fs::metadata(request.path.under(&collection.root)) - .and_then(|metadata| metadata.modified()) - .ok() - .and_then(|mtime| mtime.duration_since(std::time::UNIX_EPOCH).ok()) - .map(|duration| duration.as_millis() as u64); + let current_ms = collection + .held_root() + .modified_millis(&request.path.to_path_buf()) + .ok(); if current_ms.is_some_and(|current| current != known_ms) { return Err(vec![Diagnostic::error( crate::errors::CONCURRENT_MODIFICATION, @@ -315,25 +300,32 @@ pub(crate) fn prepare_rename( vec![error] }, )?; - crate::operations::ensure_no_symlink_components_diagnostic( - &collection.root, - path.as_str(), - collection.spec_profile, - ) - .map_err(|mut error| { - error.path = Some(path.to_string()); - vec![error] - })?; + collection + .held_root() + .ensure_no_symlink_components(&path.to_path_buf()) + .map_err(|error| { + vec![Diagnostic::error( + crate::errors::PATH_TRAVERSAL, + error.to_string(), + Some(path.to_string()), + )] + })?; } - crate::operations::ensure_regular_record_file_diagnostic( - &request.from.under(&collection.root), - &from, - ) - .map_err(|mut error| { - error.path = Some(from.clone()); - vec![error] - })?; - if request.to.under(&collection.root).exists() { + collection + .held_root() + .metadata(&request.from.to_path_buf()) + .map_err(|_| { + vec![Diagnostic::error( + crate::errors::FILE_NOT_FOUND, + format!("File not found: {from}"), + Some(from.clone()), + )] + })?; + if collection + .held_root() + .entry_exists(&request.to.to_path_buf()) + .unwrap_or(true) + { return Err(vec![Diagnostic::error( crate::errors::PATH_CONFLICT, format!("Target already exists: {to}"), @@ -342,7 +334,7 @@ pub(crate) fn prepare_rename( } let snapshot = collection - .capture_collection_snapshot(&crate::OperationCancellation::new()) + .capture_collection_snapshot_current() .map_err(|error| { vec![Diagnostic::error( "collection_snapshot_failed", @@ -378,13 +370,12 @@ pub(crate) fn prepare_rename( Some(request.from.to_string()), )]); } - let source_bytes = std::fs::read(request.from.under(&collection.root)).map_err(|error| { - vec![Diagnostic::error( - "file_read_failed", - error.to_string(), - Some(request.from.to_string()), - )] - })?; + let source_bytes = source + .outcome() + .document() + .ok_or_else(|| vec![invalid_record(&from, "invalid_utf8")])? + .as_bytes() + .to_vec(); if content_revision(&source_bytes) != source_revision { return Err(vec![Diagnostic::error( crate::errors::CONCURRENT_MODIFICATION, @@ -401,14 +392,22 @@ pub(crate) fn prepare_rename( let mut warnings = Vec::new(); let mut reference_failures = Vec::new(); let reference_plans = if request.update_refs { - collection.plan_reference_rewrites( - &snapshot, - request.from.as_ref(), - request.to.as_ref(), - &source_id, - &mut warnings, - &mut reference_failures, - ) + collection + .plan_reference_rewrites( + &snapshot, + request.from.as_ref(), + request.to.as_ref(), + &source_id, + &mut warnings, + &mut reference_failures, + ) + .map_err(|error| { + vec![Diagnostic::error( + error.code, + error.message, + Some(request.from.to_string()), + )] + })? } else { Vec::new() }; diff --git a/src/mutation/service.rs b/src/mutation/service.rs index 447e50e..2d7ade6 100644 --- a/src/mutation/service.rs +++ b/src/mutation/service.rs @@ -43,7 +43,7 @@ fn increment(update: impl FnOnce(&mut MutationPathProbes)) { pub(crate) fn probe_request_value() { increment(|value| value.request_value_constructions += 1); } -#[cfg(test)] +#[cfg(all(test, feature = "legacy-collection-mutation"))] pub(crate) fn probe_legacy_parse() { increment(|value| value.legacy_request_parses += 1); } @@ -380,8 +380,11 @@ fn execute_shadow( let planned = operation(&shadow.collection)?; let _before_revision = planned.before_revision.as_deref(); let planned_path = planned.path.clone(); - let shadow_metadata = - std::fs::metadata(planned_path.under(&shadow.collection.root)).map_err(invalid_result)?; + let shadow_metadata = shadow + .collection + .held_root() + .metadata(&planned_path.to_path_buf()) + .map_err(invalid_result)?; let mut outcome = project_record(&shadow.collection, planned, shadow_metadata)?; if dry_run { return Ok(outcome); @@ -423,7 +426,7 @@ fn validate_update_shape(request: &UpdateRequest) -> Result<(), MdbaseError> { pub(crate) fn project_record( collection: &Collection, planned: PlannedRecord, - metadata: std::fs::Metadata, + metadata: crate::record_load::FileMetadata, ) -> Result, MdbaseError> { let mtime = metadata.modified().ok().map(|time| { let value: chrono::DateTime = time.into(); diff --git a/src/mutation/shadow.rs b/src/mutation/shadow.rs index bd28966..91418de 100644 --- a/src/mutation/shadow.rs +++ b/src/mutation/shadow.rs @@ -1,9 +1,8 @@ use std::collections::BTreeMap; use std::fs; +use std::io::Read; use std::path::Path; -use walkdir::WalkDir; - use crate::diagnostic::Diagnostic; use crate::runtime::{OperationContext, ProviderError}; use crate::Collection; @@ -50,13 +49,6 @@ fn shadow_collection_inner( collection: &Collection, context: Option<&OperationContext>, ) -> Result { - if !collection.rename_root_path_is_current() { - return Err(RuntimeBatchError::Diagnostic(Box::new(Diagnostic::error( - crate::errors::CONCURRENT_MODIFICATION, - "Collection root was replaced before mutation planning.", - None, - )))); - } if let Some(context) = context { context.check().map_err(RuntimeBatchError::Provider)?; } @@ -87,52 +79,46 @@ fn copy_collection( destination: &Path, context: Option<&OperationContext>, ) -> Result { - let source = &collection.root; let mut baseline = BTreeMap::new(); - for entry in WalkDir::new(source) - .follow_links(false) - .into_iter() - .filter_entry(|entry| should_descend(collection, entry.path())) - { + let mut captured_entries = 0_u64; + let files = collection + .held_root() + .files_recursive(Path::new("")) + .map_err(|error| { + RuntimeBatchError::Diagnostic(Box::new(copy_error(Path::new(""), error))) + })?; + for relative in files { if let Some(context) = context { context.check().map_err(RuntimeBatchError::Provider)?; } - let entry = entry.map_err(|error| { - RuntimeBatchError::Diagnostic(Box::new(Diagnostic::error( - "batch_preflight_failed", - format!("Could not inspect collection for batch preflight: {error}"), - None, - ))) - })?; - let relative = entry.path().strip_prefix(source).map_err(|error| { - RuntimeBatchError::Diagnostic(Box::new(Diagnostic::error( - "batch_preflight_failed", - error.to_string(), - None, - ))) - })?; - if relative.as_os_str().is_empty() { + if !should_copy_file(collection, &relative) + || below_nested_collection(collection, &relative) + { continue; } - let target = destination.join(relative); - if entry.file_type().is_dir() { - fs::create_dir_all(&target).map_err(|error| { - RuntimeBatchError::Diagnostic(Box::new(copy_error(relative, error))) - })?; - } else if entry.file_type().is_file() && should_copy_file(collection, relative) { - if let Some(parent) = target.parent() { - fs::create_dir_all(parent).map_err(|error| { - RuntimeBatchError::Diagnostic(Box::new(copy_error(relative, error))) - })?; - } - let bytes = fs::read(entry.path()).map_err(|error| { - RuntimeBatchError::Diagnostic(Box::new(copy_error(relative, error))) + if let Some(context) = context { + captured_entries = captured_entries.checked_add(1).ok_or({ + RuntimeBatchError::Provider(ProviderError::CaptureLimitExceeded( + crate::runtime::CaptureLimitExceeded { + kind: crate::runtime::CaptureLimitKind::ArithmeticOverflow, + limit: u64::MAX, + attempted: u64::MAX, + }, + )) })?; - fs::write(&target, &bytes).map_err(|error| { - RuntimeBatchError::Diagnostic(Box::new(copy_error(relative, error))) + charge_capture_path(context, &relative, captured_entries)?; + } + let target = destination.join(&relative); + if let Some(parent) = target.parent() { + fs::create_dir_all(parent).map_err(|error| { + RuntimeBatchError::Diagnostic(Box::new(copy_error(&relative, error))) })?; - baseline.insert(portable_path(relative), bytes); } + let bytes = read_capture_file(collection, &relative, context)?; + fs::write(&target, &bytes).map_err(|error| { + RuntimeBatchError::Diagnostic(Box::new(copy_error(&relative, error))) + })?; + baseline.insert(portable_path(&relative), bytes); } Ok(baseline) } @@ -165,56 +151,145 @@ fn collect_collection_files_inner( context: Option<&OperationContext>, ) -> Result { let mut files = BTreeMap::new(); - let source = &collection.root; - for entry in WalkDir::new(source) - .follow_links(false) - .into_iter() - .filter_entry(|entry| should_descend(collection, entry.path())) - { + let mut captured_entries = 0_u64; + let paths = collection + .held_root() + .files_recursive(Path::new("")) + .map_err(|error| { + RuntimeBatchError::Diagnostic(Box::new(copy_error(Path::new(""), error))) + })?; + for relative in paths { if let Some(context) = context { context.check().map_err(RuntimeBatchError::Provider)?; } - let entry = entry.map_err(|error| { - RuntimeBatchError::Diagnostic(Box::new(Diagnostic::error( - "batch_preflight_failed", - format!("Could not inspect preflight result: {error}"), - None, - ))) - })?; - let relative = entry.path().strip_prefix(source).map_err(|error| { - RuntimeBatchError::Diagnostic(Box::new(Diagnostic::error( - "batch_preflight_failed", - error.to_string(), - None, - ))) - })?; - if entry.file_type().is_file() && should_copy_file(collection, relative) { - let bytes = fs::read(entry.path()).map_err(|error| { - RuntimeBatchError::Diagnostic(Box::new(copy_error(relative, error))) - })?; - files.insert(portable_path(relative), bytes); + if should_copy_file(collection, &relative) + && !below_nested_collection(collection, &relative) + { + if let Some(context) = context { + captured_entries = captured_entries.checked_add(1).ok_or({ + RuntimeBatchError::Provider(ProviderError::CaptureLimitExceeded( + crate::runtime::CaptureLimitExceeded { + kind: crate::runtime::CaptureLimitKind::ArithmeticOverflow, + limit: u64::MAX, + attempted: u64::MAX, + }, + )) + })?; + charge_capture_path(context, &relative, captured_entries)?; + } + let bytes = read_capture_file(collection, &relative, context)?; + files.insert(portable_path(&relative), bytes); } } Ok(files) } -fn should_descend(collection: &Collection, path: &Path) -> bool { - let Ok(relative) = path.strip_prefix(&collection.root) else { - return false; - }; - if relative.as_os_str().is_empty() || is_system_definition_path(collection, relative) { - return true; +fn charge_capture_path( + context: &OperationContext, + path: &Path, + captured_entries: u64, +) -> Result<(), RuntimeBatchError> { + context.check().map_err(RuntimeBatchError::Provider)?; + context + .check_entries(captured_entries) + .map_err(RuntimeBatchError::Provider)?; + let depth = path.components().count().saturating_sub(1) as u64; + context + .check_depth(depth) + .map_err(RuntimeBatchError::Provider) +} + +fn read_capture_file( + collection: &Collection, + relative: &Path, + context: Option<&OperationContext>, +) -> Result, RuntimeBatchError> { + let mut file = collection + .held_root() + .open_file(relative) + .map_err(|error| RuntimeBatchError::Diagnostic(Box::new(copy_error(relative, error))))?; + if context.is_none() { + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes).map_err(|error| { + RuntimeBatchError::Diagnostic(Box::new(copy_error(relative, error))) + })?; + return Ok(bytes); } - if !path.is_dir() { - return true; + let context = context.unwrap(); + let size = file + .metadata() + .map_err(|error| RuntimeBatchError::Diagnostic(Box::new(copy_error(relative, error))))? + .len(); + context + .check_file_bytes(size) + .map_err(RuntimeBatchError::Provider)?; + let capacity = usize::try_from(size).map_err(|_| { + RuntimeBatchError::Provider(ProviderError::CaptureLimitExceeded( + crate::runtime::CaptureLimitExceeded { + kind: crate::runtime::CaptureLimitKind::ArithmeticOverflow, + limit: usize::MAX as u64, + attempted: size, + }, + )) + })?; + let mut bytes = Vec::new(); + bytes.try_reserve_exact(capacity).map_err(|_| { + RuntimeBatchError::Provider(ProviderError::CaptureLimitExceeded( + crate::runtime::CaptureLimitExceeded { + kind: crate::runtime::CaptureLimitKind::ArithmeticOverflow, + limit: usize::MAX as u64, + attempted: size, + }, + )) + })?; + let mut chunk = [0_u8; 64 * 1024]; + loop { + context.check().map_err(RuntimeBatchError::Provider)?; + let read = file.read(&mut chunk).map_err(|error| { + RuntimeBatchError::Diagnostic(Box::new(copy_error(relative, error))) + })?; + if read == 0 { + break; + } + let attempted = (bytes.len() as u64).checked_add(read as u64).ok_or({ + RuntimeBatchError::Provider(ProviderError::CaptureLimitExceeded( + crate::runtime::CaptureLimitExceeded { + kind: crate::runtime::CaptureLimitKind::ArithmeticOverflow, + limit: u64::MAX, + attempted: u64::MAX, + }, + )) + })?; + context + .check_file_bytes(attempted) + .map_err(RuntimeBatchError::Provider)?; + context + .charge_read(read as u64) + .map_err(RuntimeBatchError::Provider)?; + context + .charge_retained(read as u64) + .map_err(RuntimeBatchError::Provider)?; + bytes.extend_from_slice(&chunk[..read]); + context.check().map_err(RuntimeBatchError::Provider)?; } - let relative = portable_path(relative); - if collection.is_excluded(&relative) { - return false; + Ok(bytes) +} + +fn below_nested_collection(collection: &Collection, path: &Path) -> bool { + let mut parent = path.parent(); + while let Some(candidate) = parent { + if candidate.as_os_str().is_empty() { + break; + } + if collection + .held_root() + .exists_file(candidate.join("mdbase.yaml")) + { + return true; + } + parent = candidate.parent(); } - // A directory containing its own config is a nested collection boundary. - // Avoid copying any of it into the preflight workspace. - !path.join("mdbase.yaml").is_file() + false } fn should_copy_file(collection: &Collection, relative: &Path) -> bool { @@ -237,9 +312,9 @@ fn should_copy_file(collection: &Collection, relative: &Path) -> bool { return extension == Some("md"); } if extension == Some("base") { - return crate::views::compatibility_source_paths(collection) - .iter() - .any(|path| path == &collection.root.join(relative)); + let relative = portable_path(relative); + return !collection.is_excluded(&relative) + && crate::views::is_configured_obsidian_source(collection, &relative); } let relative = portable_path(relative); !collection.is_excluded(&relative) @@ -250,12 +325,6 @@ fn should_copy_file(collection: &Collection, relative: &Path) -> bool { == Some("json")) } -fn is_system_definition_path(collection: &Collection, relative: &Path) -> bool { - relative.starts_with(Path::new(&collection.settings.types_folder)) - || relative.starts_with(Path::new(&collection.settings.contracts_folder)) - || relative.starts_with(Path::new(&collection.settings.migrations_folder)) -} - fn portable_path(path: &Path) -> String { path.to_string_lossy().replace('\\', "/") } diff --git a/src/operations/backfill.rs b/src/operations/backfill.rs index cd702f8..e53ffa0 100644 --- a/src/operations/backfill.rs +++ b/src/operations/backfill.rs @@ -40,6 +40,37 @@ fn invalid_record_detail( }) } +#[cfg(test)] +type PlanningHook = Box; + +#[cfg(test)] +fn planning_hooks( +) -> &'static std::sync::Mutex> { + static HOOKS: std::sync::OnceLock< + std::sync::Mutex>, + > = std::sync::OnceLock::new(); + HOOKS.get_or_init(Default::default) +} + +#[cfg(test)] +fn inject_planning_hook(root: &std::path::Path, hook: impl FnOnce() + Send + 'static) { + planning_hooks() + .lock() + .expect("backfill planning hook lock") + .insert(root.to_path_buf(), Box::new(hook)); +} + +#[cfg(test)] +fn run_planning_hook(root: &std::path::Path) { + let hook = planning_hooks() + .lock() + .expect("backfill planning hook lock") + .remove(root); + if let Some(hook) = hook { + hook(); + } +} + struct BackfillPlan { path: String, expected_revision: String, @@ -49,7 +80,7 @@ struct BackfillPlan { changed_fields: Vec, } -#[cfg(test)] +#[cfg(all(test, feature = "legacy-collection-mutation"))] fn injected_backfill_replacements( ) -> &'static std::sync::Mutex> { static REPLACEMENTS: std::sync::OnceLock< @@ -58,7 +89,7 @@ fn injected_backfill_replacements( REPLACEMENTS.get_or_init(Default::default) } -#[cfg(test)] +#[cfg(all(test, feature = "legacy-collection-mutation"))] pub(crate) fn inject_backfill_replacement(path: &std::path::Path, replacement: std::path::PathBuf) { injected_backfill_replacements() .lock() @@ -66,7 +97,7 @@ pub(crate) fn inject_backfill_replacement(path: &std::path::Path, replacement: s .insert(path.to_path_buf(), replacement); } -#[cfg(test)] +#[cfg(all(test, feature = "legacy-collection-mutation"))] fn apply_injected_backfill_replacement(path: &std::path::Path) { if let Some(replacement) = injected_backfill_replacements() .lock() @@ -78,8 +109,19 @@ fn apply_injected_backfill_replacement(path: &std::path::Path) { } impl Collection { - /// Backfill missing defaults/generated values across files (§12.8). - pub fn backfill(&self, input: &serde_json::Value) -> serde_json::Value { + pub(crate) fn backfill_legacy(&self, input: &serde_json::Value) -> serde_json::Value { + self.backfill_contextual(input, None, &mut Vec::new()) + } + + fn backfill_contextual( + &self, + input: &serde_json::Value, + context: Option<&crate::runtime::OperationContext>, + typed_diagnostics: &mut Vec, + ) -> serde_json::Value { + if let Some(error) = context.and_then(|context| context.check().err()) { + return op_error(error.code(), &error.to_string()); + } let type_filter = input.get("type").and_then(|v| v.as_str()); let where_clause = input.get("where"); let dry_run = input @@ -113,8 +155,7 @@ impl Collection { .map(|t| vec![t.to_lowercase()]) .unwrap_or_default(); - let snapshot = match self.capture_collection_snapshot(&crate::OperationCancellation::new()) - { + let snapshot = match self.capture_collection_snapshot_current() { Ok(snapshot) => snapshot, Err(error) => return op_error("collection_snapshot_failed", &error.to_string()), }; @@ -144,6 +185,11 @@ impl Collection { let mut generated = crate::generated::GeneratedValueContext::from_snapshot(self, &snapshot); for path in &matching_paths { + #[cfg(test)] + run_planning_hook(self.root()); + if let Some(error) = context.and_then(|context| context.check().err()) { + return op_error(error.code(), &error.to_string()); + } let Some(entry) = snapshot.entry(path) else { return op_error( "collection_snapshot_failed", @@ -207,22 +253,30 @@ impl Collection { if apply_defaults { for type_name in &type_names { if let Some(type_def) = self.types.get(type_name) { - for (field_name, field_def) in &type_def.fields { - if field_def.default.is_none() { - continue; - } - if let Some(ref filter) = fields_filter { - if !filter.contains(field_name) { - continue; - } - } - if working.contains_key(field_name) { + let defaults = + type_def + .read_defaults + .iter() + .map(|(field_name, value)| (field_name.clone(), value.clone())) + .chain(type_def.fields.iter().filter_map( + |(field_name, field_def)| { + field_def + .default + .clone() + .map(|value| (field_name.clone(), value)) + }, + )); + for (field_name, value) in defaults { + if fields_filter + .as_ref() + .is_some_and(|filter| !filter.contains(&field_name)) + || working.contains_key(&field_name) + { continue; } - let val = field_def.default.clone().unwrap(); - working.insert(field_name.clone(), val.clone()); - changes.insert(field_name.clone(), val); - change_kinds.insert(field_name.clone(), ChangeKind::Default); + working.insert(field_name.clone(), value.clone()); + changes.insert(field_name.clone(), value); + change_kinds.insert(field_name, ChangeKind::Default); } } } @@ -251,6 +305,7 @@ impl Collection { let mut write_obj = raw_obj.clone(); for (field, value) in &changes { if change_kinds.get(field) == Some(&ChangeKind::Default) + && self.spec_profile() == crate::SpecProfile::V02 && !self.settings.write_defaults { continue; @@ -367,10 +422,60 @@ impl Collection { let mut succeeded = noop_success; let mut failed = planning_failed; + if let Some(context) = context { + if !plans.is_empty() { + let operations = plans + .iter() + .map(|plan| { + let mut request = crate::api::UpdateRequest::replace_document( + crate::api::CollectionPath::new(&plan.path) + .expect("snapshot paths are canonical collection paths"), + plan.output.clone(), + ); + request.if_revision = Some( + crate::api::Revision::parse(plan.expected_revision.clone()) + .expect("snapshot revisions are opaque non-empty tokens"), + ); + crate::api::BatchOperation::Update(request) + }) + .collect::>(); + let request = crate::api::BatchRequest { + operations, + allow_partial: false, + dry_run: false, + }; + match crate::mutation::batch_with_context(self, request, context) { + Ok(outcome) => typed_diagnostics.extend(outcome.diagnostics), + Err(error) => { + if let Some(diagnostic) = error.diagnostics().first() { + return op_error(diagnostic.code.as_str(), &diagnostic.message); + } + return op_error("backfill_failed", &error.to_string()); + } + } + } + succeeded += plans.len(); + details.extend(plans.into_iter().map(|plan| { + serde_json::json!({ + "path": plan.path, + "status": "success", + "changed_fields": plan.changed_fields, + }) + })); + return serde_json::json!({ + "batch_result": { + "total": total, + "succeeded": succeeded, + "failed": failed, + "skipped": skipped, + "details": details, + } + }); + } + for plan in plans { - let full_path = self.root.join(&plan.path); - #[cfg(test)] - apply_injected_backfill_replacement(&full_path); + #[cfg(all(test, feature = "legacy-collection-mutation"))] + apply_injected_backfill_replacement(&self.root.join(&plan.path)); let current = match crate::record_load::load_record(self, &plan.path) { Ok(current) => current, Err(error) => { @@ -398,7 +503,10 @@ impl Collection { })); continue; } - if let Err(e) = crate::operations::atomic_write(&full_path, plan.output.as_bytes()) { + if let Err(e) = self + .held_root() + .atomic_write(std::path::Path::new(&plan.path), plan.output.as_bytes()) + { failed += 1; details.push(serde_json::json!({ "path": plan.path, @@ -427,7 +535,239 @@ impl Collection { } } +pub(crate) fn execute( + collection: &Collection, + request: crate::api::BackfillRequest, + context: &crate::runtime::OperationContext, +) -> crate::api::MdbaseResult> { + if request.type_name.is_none() && request.where_expression.is_none() { + return Err(crate::api::MdbaseError::Operation { + diagnostics: vec![crate::api::Diagnostic { + severity: crate::api::Severity::Error, + code: crate::api::DiagnosticCode::new(INVALID_REQUEST), + message: "backfill requires 'type' or 'where'".to_string(), + path: None, + field: None, + type_name: None, + schema_location: None, + details: None, + }], + }); + } + let mut input = serde_json::Map::new(); + if let Some(value) = request.type_name { + input.insert("type".to_string(), serde_json::Value::String(value)); + } + if let Some(value) = request.where_expression { + input.insert("where".to_string(), serde_json::Value::String(value)); + } + if let Some(fields) = request.fields { + input.insert( + "fields".to_string(), + serde_json::Value::Array(fields.into_iter().map(serde_json::Value::String).collect()), + ); + } + if request.dry_run { + input.insert("dry_run".to_string(), serde_json::Value::Bool(true)); + } + if request.apply_defaults.is_some() || request.apply_generated.is_some() { + let mut apply = serde_json::Map::new(); + if let Some(value) = request.apply_defaults { + apply.insert("defaults".to_string(), serde_json::Value::Bool(value)); + } + if let Some(value) = request.apply_generated { + apply.insert("generated".to_string(), serde_json::Value::Bool(value)); + } + input.insert("apply".to_string(), serde_json::Value::Object(apply)); + } + + let input = serde_json::Value::Object(input); + let mut diagnostics = Vec::new(); + let value = + context.scope(|| collection.backfill_contextual(&input, Some(context), &mut diagnostics)); + if let Some(error) = value.get("error") { + let code = error + .get("code") + .and_then(serde_json::Value::as_str) + .unwrap_or("backfill_failed"); + let message = error + .get("message") + .and_then(serde_json::Value::as_str) + .unwrap_or("Backfill failed."); + return Err(crate::api::MdbaseError::Operation { + diagnostics: vec![crate::api::Diagnostic { + severity: crate::api::Severity::Error, + code: crate::api::DiagnosticCode::new(code), + message: message.to_string(), + path: None, + field: None, + type_name: None, + schema_location: None, + details: Some(error.clone()), + }], + }); + } + let result = + serde_json::from_value(value).map_err(|error| crate::api::MdbaseError::InvalidResult { + message: format!("could not decode typed backfill result: {error}"), + })?; + Ok(crate::api::OperationOutcome { + value: result, + diagnostics, + }) +} + #[cfg(test)] +mod typed_tests { + use super::inject_planning_hook; + use crate::api::BackfillRequest; + use crate::runtime::{OperationContext, OperationDeadline}; + use crate::{Collection, OperationCancellation}; + use std::fs; + use std::time::Duration; + + fn fixture() -> tempfile::TempDir { + let root = tempfile::tempdir().unwrap(); + fs::write(root.path().join("mdbase.yaml"), "spec_version: 0.3.0\n").unwrap(); + fs::create_dir(root.path().join("_types")).unwrap(); + fs::write( + root.path().join("_types/task.md"), + r#"--- +kind: mdbase.type +name: task +schema: + dialect: json-schema-2020-12 + value: + type: object + required: [type, title] + properties: + type: { const: task } + title: { type: string } + status: { type: string } +collection: + read_defaults: + status: open +--- +"#, + ) + .unwrap(); + for name in ["a", "b"] { + fs::write( + root.path().join(format!("{name}.md")), + format!("---\ntype: task\ntitle: {name}\n---\n{name}\n"), + ) + .unwrap(); + } + root + } + + fn request() -> BackfillRequest { + BackfillRequest { + type_name: Some("task".to_string()), + ..BackfillRequest::default() + } + } + + fn context(token: &OperationCancellation) -> OperationContext { + OperationContext::new(token, OperationDeadline::after(Duration::from_secs(30))) + } + + fn assert_not_backfilled(root: &std::path::Path, name: &str) { + assert!(!fs::read_to_string(root.join(format!("{name}.md"))) + .unwrap() + .contains("status:")); + } + + #[test] + fn cancellation_during_planning_writes_nothing() { + let root = fixture(); + let collection = Collection::open(root.path()).unwrap(); + let token = OperationCancellation::new(); + let cancel = token.clone(); + inject_planning_hook(root.path(), move || cancel.cancel()); + + let error = collection + .typed() + .unwrap() + .backfill_with_context(request(), &context(&token)) + .unwrap_err(); + assert_eq!(error.diagnostics()[0].code.as_str(), "operation_cancelled"); + assert_not_backfilled(root.path(), "a"); + assert_not_backfilled(root.path(), "b"); + } + + #[test] + fn cancellation_immediately_precommit_writes_nothing() { + let root = fixture(); + let collection = Collection::open(root.path()).unwrap(); + let token = OperationCancellation::new(); + let cancel = token.clone(); + crate::mutation::inject_precommit_hook(root.path(), move || cancel.cancel()); + + let error = collection + .typed() + .unwrap() + .backfill_with_context(request(), &context(&token)) + .unwrap_err(); + assert_eq!(error.diagnostics()[0].code.as_str(), "operation_cancelled"); + assert_not_backfilled(root.path(), "a"); + assert_not_backfilled(root.path(), "b"); + } + + #[test] + fn conflict_rolls_back_every_planned_record() { + let root = fixture(); + let collection = Collection::open(root.path()).unwrap(); + let conflicted = root.path().join("a.md"); + crate::mutation::inject_precommit_hook(root.path(), move || { + fs::write(&conflicted, "external\n").unwrap(); + }); + + let error = collection.typed().unwrap().backfill(request()).unwrap_err(); + assert_eq!( + error.diagnostics()[0].code.as_str(), + "concurrent_modification" + ); + assert_eq!( + fs::read_to_string(root.path().join("a.md")).unwrap(), + "external\n" + ); + assert_not_backfilled(root.path(), "b"); + } + + #[test] + fn interrupted_commit_recovers_the_complete_backfill() { + let root = fixture(); + let collection = Collection::open(root.path()).unwrap(); + crate::transactions::inject_commit_crash_after(root.path(), 1); + let error = collection.typed().unwrap().backfill(request()).unwrap_err(); + assert_eq!(error.diagnostics()[0].code.as_str(), "simulated_crash"); + drop(collection); + + let recovered = Collection::open(root.path()).unwrap(); + drop(recovered); + for name in ["a", "b"] { + assert!(fs::read_to_string(root.path().join(format!("{name}.md"))) + .unwrap() + .contains("status:")); + } + } + + #[test] + fn cleanup_deferred_is_returned_as_a_typed_warning() { + let root = fixture(); + let collection = Collection::open(root.path()).unwrap(); + crate::transactions::inject_cleanup_deferred(root.path()); + + let outcome = collection.typed().unwrap().backfill(request()).unwrap(); + assert!(outcome.diagnostics.iter().any(|diagnostic| { + diagnostic.code.as_str() == "transaction_cleanup_deferred" + && diagnostic.severity == crate::api::Severity::Warning + })); + } +} + +#[cfg(all(test, feature = "legacy-collection-mutation"))] mod tests { use super::inject_backfill_replacement; use crate::Collection; diff --git a/src/operations/batch.rs b/src/operations/batch.rs index d204c6f..002d5e5 100644 --- a/src/operations/batch.rs +++ b/src/operations/batch.rs @@ -1,11 +1,15 @@ //! Batch operations (§12.7). +#[cfg(feature = "legacy-collection-mutation")] use crate::errors::*; +#[cfg(feature = "legacy-collection-mutation")] use crate::frontmatter::parser::yaml_mapping_to_json; +#[cfg(feature = "legacy-collection-mutation")] use crate::frontmatter::serializer; use crate::query::engine::QueryEvalContext; use crate::Collection; +#[cfg(feature = "legacy-collection-mutation")] fn invalid_record_batch_detail( path: &str, invalid: crate::record_load::InvalidRecordView<'_>, @@ -33,6 +37,7 @@ fn invalid_record_batch_detail( }) } +#[cfg(feature = "legacy-collection-mutation")] fn preflight_serialization( record: crate::record_load::ParsedRecordView<'_>, fields: &serde_json::Map, @@ -50,6 +55,7 @@ fn preflight_serialization( Ok(()) } +#[cfg(feature = "legacy-collection-mutation")] fn serialization_failure_detail(path: &str, error: &impl std::fmt::Display) -> serde_json::Value { serde_json::json!({ "path": path, @@ -69,7 +75,8 @@ fn timestamp_from_ns(value: i64) -> Option { } impl Collection { - pub fn batch_update( + #[cfg(feature = "legacy-collection-mutation")] + pub(crate) fn batch_update_legacy( &self, input: &serde_json::Value, simulate_io_error: Option<&str>, @@ -113,8 +120,7 @@ impl Collection { } // Select and preflight from one authoritative generation. - let snapshot = match self.capture_collection_snapshot(&crate::OperationCancellation::new()) - { + let snapshot = match self.capture_collection_snapshot_current() { Ok(snapshot) => snapshot, Err(error) => return op_error("collection_snapshot_failed", &error.to_string()), }; @@ -256,7 +262,10 @@ impl Collection { // Build backlinks index for skip_dependents checking let bl_index_for_skip = if skip_dependents { - Some(self.build_backlinks_index_for_snapshot(&snapshot)) + match self.build_backlinks_index_for_snapshot(&snapshot) { + Ok(index) => Some(index), + Err(error) => return op_error(&error.code, &error.message), + } } else { None }; @@ -342,6 +351,7 @@ impl Collection { } /// Batch update with explicit update list (validate-all-then-execute). + #[cfg(feature = "legacy-collection-mutation")] pub(crate) fn batch_update_explicit( &self, updates: &[serde_json::Value], @@ -357,15 +367,15 @@ impl Collection { { return error; } - if let Err(error) = - crate::operations::ensure_no_symlink_components(&self.root, path, self.spec_profile) + if let Err(error) = self + .held_root() + .ensure_no_symlink_components(std::path::Path::new(path)) { - return error; + return op_error(PATH_TRAVERSAL, &error.to_string()); } } - let snapshot = match self.capture_collection_snapshot(&crate::OperationCancellation::new()) - { + let snapshot = match self.capture_collection_snapshot_current() { Ok(snapshot) => snapshot, Err(error) => return op_error("collection_snapshot_failed", &error.to_string()), }; @@ -549,7 +559,8 @@ impl Collection { } /// Batch delete files matching a where clause (§12.4, §12.7). - pub fn batch_delete( + #[cfg(feature = "legacy-collection-mutation")] + pub(crate) fn batch_delete_legacy( &self, input: &serde_json::Value, simulate_io_error: Option<&str>, @@ -583,8 +594,7 @@ impl Collection { return op_error("invalid_input", "batch_delete requires 'where'"); } - let snapshot = match self.capture_collection_snapshot(&crate::OperationCancellation::new()) - { + let snapshot = match self.capture_collection_snapshot_current() { Ok(snapshot) => snapshot, Err(error) => return op_error("collection_snapshot_failed", &error.to_string()), }; @@ -609,7 +619,10 @@ impl Collection { // Check backlinks before deletion let mut broken_links: Vec = Vec::new(); if check_backlinks { - let bl_index = self.build_backlinks_index_for_snapshot(&snapshot); + let bl_index = match self.build_backlinks_index_for_snapshot(&snapshot) { + Ok(index) => index, + Err(error) => return op_error(&error.code, &error.message), + }; for path in &matching_paths { if let Some(sources) = bl_index.get(path) { for source in sources { @@ -651,7 +664,7 @@ impl Collection { } } - let deleted = self.delete(&serde_json::json!({"path": path})); + let deleted = self.delete_legacy(&serde_json::json!({"path": path})); if deleted.get("error").is_some() { failed += 1; details.push(serde_json::json!({ "path": path, "status": "failed" })); @@ -703,7 +716,10 @@ impl Collection { let (all_files_arc, backlinks_arc) = if needs_link_graph { let resolved_files = std::sync::Arc::new(snapshot.resolved_files_data()); let backlinks = std::sync::Arc::new( - self.build_backlinks_index_for_snapshot_files(snapshot, &resolved_files), + self.build_backlinks_index_for_snapshot_files(snapshot, &resolved_files) + .map_err(|error| crate::CollectionSnapshotError::CacheUnavailable { + reason: format!("{}: {}", error.code, error.message), + })?, ); (Some(resolved_files), Some(backlinks)) } else { @@ -760,7 +776,7 @@ impl Collection { } } -#[cfg(test)] +#[cfg(all(test, feature = "legacy-collection-mutation"))] mod snapshot_batch_tests { use super::*; diff --git a/src/operations/create.rs b/src/operations/create.rs index 9bee130..02e624e 100644 --- a/src/operations/create.rs +++ b/src/operations/create.rs @@ -1,7 +1,11 @@ //! Create operation (§12.1). +#[cfg(feature = "legacy-collection-mutation")] use crate::api::operations::{CreateInput, CreateOutput}; -use crate::api::{CollectionPath, CreateRequest, Revision}; +#[cfg(feature = "legacy-collection-mutation")] +use crate::api::CreateRequest; +#[cfg(feature = "legacy-collection-mutation")] +use crate::api::{CollectionPath, Revision}; use crate::errors::*; use crate::frontmatter; use crate::frontmatter::serializer; @@ -9,14 +13,14 @@ use crate::generated::derive_path; use crate::matching::engine::matches_rules_checked_compiled; use crate::mutation::{PlannedRecord, PreparedCreate}; use crate::operations::{ - atomic_create, ensure_no_symlink_components_diagnostic, ensure_safe_relative_path_diagnostic, + ensure_no_symlink_components_held_diagnostic, ensure_safe_relative_path_diagnostic, mutation_record_path_diagnostic, }; use crate::Collection; impl Collection { - /// Create a file (§12.1). - pub fn create(&self, input: &serde_json::Value) -> serde_json::Value { + #[cfg(feature = "legacy-collection-mutation")] + pub(crate) fn create_legacy(&self, input: &serde_json::Value) -> serde_json::Value { let parsed = CreateInput::parse(input); let request = CreateRequest { path: parsed @@ -148,7 +152,7 @@ impl Collection { }) }); let operation_snapshot = if has_generated || self.settings.default_validation == "error" { - match self.capture_collection_snapshot(&crate::OperationCancellation::new()) { + match self.capture_collection_snapshot_current() { Ok(snapshot) => Some(snapshot), Err(error) => { return Err(crate::mutation::MutationFailure::operation( @@ -250,9 +254,7 @@ impl Collection { Ok(path) => path, Err(error) => return Err(crate::mutation::MutationFailure::diagnostic(error)), }; - if let Err(error) = - ensure_no_symlink_components_diagnostic(&self.root, path.as_str(), self.spec_profile) - { + if let Err(error) = ensure_no_symlink_components_held_diagnostic(self, path.as_str()) { return Err(crate::mutation::MutationFailure::diagnostic(error)); } let _write_lock = match crate::transactions::WriteLock::acquire(self) { @@ -265,9 +267,13 @@ impl Collection { } }; - // Check existence - let full_path = path.under(&self.root); - if full_path.exists() { + // Check existence through the held collection capability. An inspection + // failure is treated as occupied so creation cannot bypass the boundary. + if self + .held_root() + .entry_exists(&path.to_path_buf()) + .unwrap_or(true) + { return Err(crate::mutation::MutationFailure::operation( PATH_CONFLICT, format!("File already exists: {}", path.as_str()), @@ -450,12 +456,10 @@ impl Collection { } }; - if let Err(error) = - ensure_no_symlink_components_diagnostic(&self.root, path.as_str(), self.spec_profile) + if let Err(e) = self + .held_root() + .atomic_create(&path.to_path_buf(), content.as_bytes()) { - return Err(crate::mutation::MutationFailure::diagnostic(error)); - } - if let Err(e) = atomic_create(&full_path, content.as_bytes()) { if e.kind() == std::io::ErrorKind::AlreadyExists { return Err(crate::mutation::MutationFailure::operation( PATH_CONFLICT, @@ -493,6 +497,7 @@ impl Collection { } } +#[cfg(feature = "legacy-collection-mutation")] fn mutation_failure_json(failure: crate::mutation::MutationFailure) -> serde_json::Value { match failure.kind { crate::mutation::MutationFailureKind::Operation if failure.diagnostics.len() == 1 => { @@ -506,6 +511,7 @@ fn mutation_failure_json(failure: crate::mutation::MutationFailure) -> serde_jso } } +#[cfg(feature = "legacy-collection-mutation")] fn planned_create_output(planned: PlannedRecord) -> serde_json::Value { CreateOutput { path: planned.path.to_string(), diff --git a/src/operations/delete.rs b/src/operations/delete.rs index 3b186a0..fbb4a1f 100644 --- a/src/operations/delete.rs +++ b/src/operations/delete.rs @@ -1,17 +1,18 @@ //! Delete operation (§12.4). +#[cfg(feature = "legacy-collection-mutation")] use crate::api::operations::{DeleteInput, DeleteOutput}; +#[cfg(feature = "legacy-collection-mutation")] use crate::api::{DeleteRequest, Revision}; use crate::errors::*; -use crate::mutation::{PlannedDelete, PreparationOptions, PreparedDelete}; -use crate::operations::{ - ensure_no_symlink_components_diagnostic, ensure_regular_record_file_diagnostic, sync_directory, -}; +#[cfg(feature = "legacy-collection-mutation")] +use crate::mutation::PreparationOptions; +use crate::mutation::{PlannedDelete, PreparedDelete}; use crate::Collection; impl Collection { - /// Legacy JSON delete edge. Canonical callers use the typed mutation service. - pub fn delete(&self, input: &serde_json::Value) -> serde_json::Value { + #[cfg(feature = "legacy-collection-mutation")] + pub(crate) fn delete_legacy(&self, input: &serde_json::Value) -> serde_json::Value { #[cfg(test)] crate::mutation::probe_legacy_parse(); let input = match DeleteInput::parse(input) { @@ -81,11 +82,6 @@ impl Collection { } = prepared; let path = request.path; let display_path = path.to_string(); - ensure_no_symlink_components_diagnostic(&self.root, &display_path, self.spec_profile) - .map_err(crate::mutation::MutationFailure::diagnostic)?; - let full_path = path.under(&self.root); - ensure_regular_record_file_diagnostic(&full_path, &display_path) - .map_err(crate::mutation::MutationFailure::diagnostic)?; let loaded = crate::record_load::load_record(self, &display_path).map_err(|error| { crate::mutation::MutationFailure::operation( if error.kind() == std::io::ErrorKind::NotFound { @@ -112,11 +108,7 @@ impl Collection { )); } if let Some(known_ms) = legacy_last_known_mtime { - let current_ms = std::fs::metadata(&full_path) - .and_then(|metadata| metadata.modified()) - .ok() - .and_then(|mtime| mtime.duration_since(std::time::UNIX_EPOCH).ok()) - .map(|duration| duration.as_millis() as u64); + let current_ms = self.held_root().modified_millis(&path.to_path_buf()).ok(); if current_ms.is_some_and(|current| current != known_ms) { return Err(crate::mutation::MutationFailure::operation( CONCURRENT_MODIFICATION, @@ -126,22 +118,14 @@ impl Collection { } if !dry_run { - ensure_no_symlink_components_diagnostic(&self.root, &display_path, self.spec_profile) - .map_err(crate::mutation::MutationFailure::diagnostic)?; - std::fs::remove_file(&full_path).map_err(|error| { - crate::mutation::MutationFailure::operation( - "io_error", - format!("Failed to delete: {error}"), - ) - })?; - if let Some(parent) = full_path.parent() { - sync_directory(parent).map_err(|error| { + self.held_root() + .remove_file(&path.to_path_buf()) + .map_err(|error| { crate::mutation::MutationFailure::operation( "io_error", - format!("Failed to make deletion durable: {error}"), + format!("Failed to delete: {error}"), ) })?; - } } Ok(PlannedDelete { @@ -156,6 +140,7 @@ impl Collection { } } +#[cfg(feature = "legacy-collection-mutation")] fn mutation_failure_json(diagnostics: Vec) -> serde_json::Value { if diagnostics.len() == 1 { serde_json::json!({"error": diagnostics.into_iter().next().unwrap()}) diff --git a/src/operations/migrate.rs b/src/operations/migrate.rs index 3cdc698..06945c7 100644 --- a/src/operations/migrate.rs +++ b/src/operations/migrate.rs @@ -2,7 +2,7 @@ use crate::errors::*; use crate::frontmatter::parser::{parse_document, yaml_mapping_to_json}; -use crate::operations::{ensure_no_symlink_components, ensure_safe_relative_path}; +use crate::operations::ensure_safe_relative_path; use crate::Collection; impl Collection { @@ -26,62 +26,43 @@ impl Collection { if let Err(error) = ensure_safe_relative_path(p, self.spec_profile) { return error; } - if let Err(error) = ensure_no_symlink_components(&self.root, p, self.spec_profile) { - return error; + if let Err(error) = self + .held_root() + .ensure_no_symlink_components(std::path::Path::new(p)) + { + return op_error(PATH_TRAVERSAL, &error.to_string()); } - self.root.join(p) + std::path::PathBuf::from(p) } else { - if let Err(error) = ensure_no_symlink_components( - &self.root, - &self.settings.migrations_folder, - self.spec_profile, - ) { - return error; + let migrations_dir = std::path::Path::new(&self.settings.migrations_folder); + if let Err(error) = self + .held_root() + .ensure_no_symlink_components(migrations_dir) + { + return op_error(PATH_TRAVERSAL, &error.to_string()); } - let migrations_dir = self.root.join(&self.settings.migrations_folder); let mut found: Option = None; - if migrations_dir.exists() { - let mut stack = vec![migrations_dir]; - while let Some(dir) = stack.pop() { - let entries = match std::fs::read_dir(&dir) { - Ok(e) => e, - Err(_) => continue, - }; - for entry in entries.flatten() { - let entry_path = entry.path(); - let Ok(file_type) = entry.file_type() else { - continue; - }; - if file_type.is_symlink() { - continue; - } - if file_type.is_dir() { - stack.push(entry_path); - } else if file_type.is_file() { - let ext = entry_path - .extension() - .and_then(|e| e.to_str()) - .unwrap_or(""); - if ext != "md" && ext != "yaml" && ext != "yml" { - continue; - } - let content = match std::fs::read_to_string(&entry_path) { - Ok(c) => c, - Err(_) => continue, - }; - let doc = parse_document(&content); - if let Some(serde_yaml::Value::Mapping(m)) = doc.frontmatter { - let fm = yaml_mapping_to_json(&m); - if let Some(manifest_id) = fm.get("id").and_then(|v| v.as_str()) { - if Some(manifest_id) == id { - found = Some(entry_path); - break; - } - } - } - } - } - if found.is_some() { + let entries = self + .held_root() + .files_recursive(migrations_dir) + .unwrap_or_default(); + for entry_path in entries { + let ext = entry_path + .extension() + .and_then(|extension| extension.to_str()) + .unwrap_or(""); + if ext != "md" && ext != "yaml" && ext != "yml" { + continue; + } + let content = match self.held_root().read_string(&entry_path) { + Ok(content) => content, + Err(_) => continue, + }; + let doc = parse_document(&content); + if let Some(serde_yaml::Value::Mapping(mapping)) = doc.frontmatter { + let frontmatter = yaml_mapping_to_json(&mapping); + if frontmatter.get("id").and_then(|value| value.as_str()) == id { + found = Some(entry_path); break; } } @@ -92,7 +73,7 @@ impl Collection { } }; - let content = match std::fs::read_to_string(&manifest_path) { + let content = match self.held_root().read_string(&manifest_path) { Ok(c) => c, Err(e) => { return op_error( @@ -150,7 +131,8 @@ impl Collection { if dry_run { backfill_input.insert("dry_run".to_string(), serde_json::Value::Bool(true)); } - let backfill_result = self.backfill(&serde_json::Value::Object(backfill_input)); + let backfill_result = + self.backfill_legacy(&serde_json::Value::Object(backfill_input)); if backfill_result.get("error").is_some() { return op_error(MIGRATION_FAILED, "Migration failed"); } diff --git a/src/operations/mod.rs b/src/operations/mod.rs index 4401ecc..898ad59 100644 --- a/src/operations/mod.rs +++ b/src/operations/mod.rs @@ -10,13 +10,13 @@ pub mod rename; pub mod type_file; pub mod update; +#[cfg(test)] use std::io::Write; use std::path::{Component, Path}; -use crate::errors::{ - op_error, CONCURRENT_MODIFICATION, FILE_NOT_FOUND, INVALID_PATH, PATH_TRAVERSAL, - PERMISSION_DENIED, -}; +#[cfg(test)] +use crate::errors::PERMISSION_DENIED; +use crate::errors::{op_error, FILE_NOT_FOUND, INVALID_PATH, PATH_TRAVERSAL}; use crate::{Collection, SpecProfile}; /// Validate that a user-supplied path is relative to the collection root. @@ -96,6 +96,7 @@ pub(crate) fn ensure_safe_relative_path_diagnostic( Ok(()) } +#[cfg(test)] pub(crate) fn ensure_no_symlink_components( collection_root: &Path, relative_path: &str, @@ -135,44 +136,20 @@ pub(crate) fn ensure_no_symlink_components( } #[allow(clippy::result_large_err)] -pub(crate) fn ensure_no_symlink_components_diagnostic( - collection_root: &Path, +pub(crate) fn ensure_no_symlink_components_held_diagnostic( + collection: &crate::Collection, relative_path: &str, - spec_profile: SpecProfile, ) -> Result<(), crate::diagnostic::Diagnostic> { - let mut candidate = collection_root.to_path_buf(); - for component in Path::new(relative_path).components() { - match component { - Component::CurDir => continue, - Component::Normal(part) => candidate.push(part), - _ => { - return Err(path_boundary_diagnostic( - spec_profile, - "Path must remain inside the collection", - relative_path, - )) - } - } - match std::fs::symlink_metadata(&candidate) { - Ok(metadata) if metadata.file_type().is_symlink() => { - return Err(path_boundary_diagnostic( - spec_profile, - "Symbolic links are not allowed in collection operation paths", - relative_path, - )) - } - Ok(_) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => break, - Err(_) => { - return Err(crate::diagnostic::Diagnostic::error( - PERMISSION_DENIED, - "Path could not be inspected safely", - Some(relative_path.to_string()), - )) - } - } - } - Ok(()) + collection + .held_root() + .ensure_no_symlink_components(Path::new(relative_path)) + .map_err(|_| { + path_boundary_diagnostic( + collection.spec_profile, + "Symbolic links are not allowed in collection operation paths", + relative_path, + ) + }) } fn path_boundary_diagnostic( @@ -191,6 +168,7 @@ fn path_boundary_diagnostic( ) } +#[cfg(any(test, feature = "legacy-collection-mutation"))] fn path_boundary_error(spec_profile: SpecProfile, message: &str) -> serde_json::Value { op_error( if spec_profile == SpecProfile::V03 { @@ -212,6 +190,7 @@ pub(crate) fn mutation_record_path_diagnostic( }) } +#[cfg(feature = "legacy-collection-mutation")] pub(crate) fn mutation_record_path( collection: &Collection, path: &str, @@ -304,11 +283,11 @@ pub(crate) fn open_regular_record_no_follow( }; directory = next; } - let Some(metadata) = open_result_or_unavailable(directory.symlink_metadata(leaf))? else { - return Ok(None); - }; - if !metadata.is_file() { - return Ok(None); + match directory.symlink_metadata(leaf) { + Ok(metadata) if !metadata.is_file() => return Ok(None), + Ok(_) => {} + Err(error) if is_unavailable_no_follow_error(&error) => return Ok(None), + Err(error) => return Err(error), } let mut options = OpenOptions::new(); options.read(true).follow(FollowSymlinks::No); @@ -316,12 +295,24 @@ pub(crate) fn open_regular_record_no_follow( return Ok(None); }; let file = file.into_std(); - if !file.metadata()?.is_file() { + let metadata = file.metadata()?; + if !metadata.is_file() || record_has_multiple_hard_links(&metadata) { return Ok(None); } Ok(Some(file)) } +#[cfg(unix)] +fn record_has_multiple_hard_links(metadata: &std::fs::Metadata) -> bool { + use std::os::unix::fs::MetadataExt; + metadata.nlink() > 1 +} + +#[cfg(not(unix))] +fn record_has_multiple_hard_links(_metadata: &std::fs::Metadata) -> bool { + false +} + fn open_result_or_unavailable(result: std::io::Result) -> std::io::Result> { match result { Ok(value) => Ok(Some(value)), @@ -477,39 +468,12 @@ pub(crate) fn readable_record_path( .map_err(|_| op_error(FILE_NOT_FOUND, &format!("File not found: {path}"))) } -#[allow(clippy::result_large_err)] -pub(crate) fn ensure_regular_record_file_diagnostic( - path: &Path, - display_path: &str, -) -> Result<(), crate::diagnostic::Diagnostic> { - match std::fs::symlink_metadata(path) { - Ok(metadata) if metadata.is_file() && !metadata.file_type().is_symlink() => Ok(()), - Ok(_) | Err(_) => Err(crate::diagnostic::Diagnostic::error( - FILE_NOT_FOUND, - format!("File not found: {display_path}"), - None, - )), - } -} - -pub(crate) fn ensure_regular_record_file( - path: &Path, - display_path: &str, -) -> Result<(), serde_json::Value> { - match std::fs::symlink_metadata(path) { - Ok(metadata) if metadata.is_file() && !metadata.file_type().is_symlink() => Ok(()), - Ok(_) | Err(_) => Err(op_error( - FILE_NOT_FOUND, - &format!("File not found: {display_path}"), - )), - } -} - /// Move a regular collection file without ever replacing an existing target. /// /// Creating the hard link is the atomic no-clobber point. Since both paths are /// inside one collection they are on the same filesystem. If unlinking the old /// name fails, the new link is rolled back and the source is left intact. +#[cfg(test)] pub(crate) fn atomic_rename_noclobber(from: &Path, to: &Path) -> std::io::Result<()> { std::fs::hard_link(from, to)?; if let Err(error) = std::fs::remove_file(from) { @@ -527,53 +491,17 @@ pub(crate) fn atomic_rename_noclobber(from: &Path, to: &Path) -> std::io::Result Ok(()) } -/// Verify an opaque revision token against the current raw file contents. -/// -/// Call this immediately before a mutation. Callers that perform work between -/// this check and the write must retain their existing mtime/file-identity -/// guard as a second check against changes during the operation. -pub(crate) fn ensure_revision( - path: &Path, - display_path: &str, - expected: Option<&str>, -) -> Result<(), serde_json::Value> { - let Some(expected) = expected else { - return Ok(()); - }; - let bytes = std::fs::read(path).map_err(|_| { - op_error( - CONCURRENT_MODIFICATION, - &format!("File '{display_path}' no longer matches the requested revision"), - ) - })?; - let actual = crate::v03::revision(&bytes); - if actual != expected { - return Err(op_error( - CONCURRENT_MODIFICATION, - &format!("File '{display_path}' was modified externally"), - )); - } - Ok(()) -} - -/// Atomically replace one collection file with fully written contents. -/// -/// The temporary file lives beside the destination so persistence remains on -/// the same filesystem. Existing permissions are retained on replacement. -pub(crate) fn atomic_write(path: &Path, contents: &[u8]) -> std::io::Result<()> { +#[cfg(test)] +fn atomic_write(path: &Path, contents: &[u8]) -> std::io::Result<()> { atomic_write_mode(path, contents, false, true) } -/// Atomically create a new file without replacing a concurrent creator. -pub(crate) fn atomic_create(path: &Path, contents: &[u8]) -> std::io::Result<()> { +#[cfg(test)] +fn atomic_create(path: &Path, contents: &[u8]) -> std::io::Result<()> { atomic_write_mode(path, contents, true, true) } -/// Atomically replace a file whose parent was already prepared and fenced. -pub(crate) fn atomic_write_in_prepared_parent(path: &Path, contents: &[u8]) -> std::io::Result<()> { - atomic_write_mode(path, contents, false, false) -} - +#[cfg(test)] fn atomic_write_mode( path: &Path, contents: &[u8], @@ -619,12 +547,12 @@ fn atomic_write_mode( Ok(()) } -#[cfg(not(windows))] +#[cfg(all(test, not(windows)))] pub(crate) fn sync_directory(path: &Path) -> std::io::Result<()> { std::fs::File::open(path)?.sync_all() } -#[cfg(windows)] +#[cfg(all(test, windows))] pub(crate) fn sync_directory(path: &Path) -> std::io::Result<()> { use std::fs::OpenOptions; use std::os::windows::fs::OpenOptionsExt; diff --git a/src/operations/read.rs b/src/operations/read.rs index c179b9e..30cad97 100644 --- a/src/operations/read.rs +++ b/src/operations/read.rs @@ -7,10 +7,7 @@ use crate::api::{ }; use crate::errors::*; use crate::frontmatter::parser::{parse_document, yaml_mapping_to_json, FrontmatterState}; -use crate::operations::{ - ensure_no_symlink_components, ensure_regular_record_file, ensure_safe_relative_path, - readable_record_path, -}; +use crate::operations::{ensure_safe_relative_path, readable_record_path}; use crate::record_load::{InvalidRecordView, RecordLoadView}; use crate::Collection; use std::path::Path; @@ -346,27 +343,28 @@ impl Collection { if let Err(error) = ensure_safe_relative_path(&input.path, self.spec_profile) { return error; } + if let Err(error) = + crate::operations::ensure_no_symlink_components_held_diagnostic(self, &input.path) + { + return op_error(&error.code, &error.message); + } let path = match readable_record_path(self, &input.path) { Ok(path) => path, Err(error) => return error, }; - if let Err(error) = - ensure_no_symlink_components(&self.root, path.as_str(), self.spec_profile) - { - return error; - } - - let full_path = path.under(&self.root); - if let Err(error) = ensure_regular_record_file(&full_path, path.as_str()) { - return error; - } - - let content = match std::fs::read_to_string(&full_path) { - Ok(c) => c, - Err(_e) => return op_error(INVALID_FRONTMATTER, "File contains invalid UTF-8"), + let content = match self.held_root().read_string(path.to_path_buf()) { + Ok(content) => content, + Err(error) if error.kind() == std::io::ErrorKind::InvalidData => { + return op_error(INVALID_FRONTMATTER, "File contains invalid UTF-8") + } + Err(_) => return op_error(FILE_NOT_FOUND, "Failed to read: entity not found"), }; - let file_metadata = std::fs::metadata(&full_path).ok(); + let file_metadata = self + .held_root() + .open_file(&path.to_path_buf()) + .and_then(|file| file.metadata()) + .ok(); let file_facts = RecordFileFacts { size: file_metadata .as_ref() diff --git a/src/operations/rename/body.rs b/src/operations/rename/body.rs index c57fcc5..8cc23d2 100644 --- a/src/operations/rename/body.rs +++ b/src/operations/rename/body.rs @@ -18,7 +18,7 @@ impl Collection { source_path: &str, source_id: Option<&str>, resolution_index: &crate::links::resolver::LinkResolutionIndex, - ) -> bool { + ) -> Result { let mut changed = false; // Process line by line, skipping fenced code blocks and inline code @@ -55,7 +55,7 @@ impl Collection { source_path, source_id, resolution_index, - ); + )?; if new_line != line { changed = true; } @@ -88,7 +88,7 @@ impl Collection { if changed { *body = result; } - changed + Ok(changed) } /// Replace link references in a single line (outside code blocks). @@ -106,7 +106,7 @@ impl Collection { source_path: &str, source_id: Option<&str>, resolution_index: &crate::links::resolver::LinkResolutionIndex, - ) -> String { + ) -> Result { let mut result = String::with_capacity(line.len()); let chars: Vec = line.chars().collect(); let len = chars.len(); @@ -166,7 +166,7 @@ impl Collection { source_path, source_id, resolution_index, - ) { + )? { let new_inner = self.rewrite_wikilink_inner( &inner, from_stem, @@ -225,7 +225,7 @@ impl Collection { source_path, source_id, resolution_index, - ) + )? { let text_part: String = chars[link_start..paren_start - 1].iter().collect(); @@ -271,7 +271,7 @@ impl Collection { source_path, source_id, resolution_index, - ) { + )? { let new_inner = self.rewrite_wikilink_inner( &inner, from_stem, @@ -332,7 +332,7 @@ impl Collection { source_path, source_id, resolution_index, - ) + )? { let text_part: String = chars[link_start..paren_start - 1].iter().collect(); let (_, anchor) = if let Some(hp) = href.find('#') { @@ -360,7 +360,7 @@ impl Collection { i += 1; } - result + Ok(result) } #[allow(clippy::too_many_arguments)] @@ -374,20 +374,59 @@ impl Collection { source_path: &str, source_id: Option<&str>, resolution_index: &crate::links::resolver::LinkResolutionIndex, - ) -> bool { + ) -> Result { if self.is_stable_configured_id_wikilink(link, source_id) { - return false; + return Ok(false); } if !self.link_resolves_to(link, from, from_stem, from_no_ext, source_dir) { - return false; + return Ok(false); } match self.simple_wikilink_resolution(link, source_path, &[], resolution_index) { - None => true, - Some(crate::links::resolver::LinkResolution::Resolved(path)) => path == from, - Some( + None => Ok(true), + Some(Ok(crate::links::resolver::LinkResolution::Resolved { path, .. })) => { + Ok(path == from) + } + Some(Ok( crate::links::resolver::LinkResolution::Missing | crate::links::resolver::LinkResolution::Ambiguous(_), - ) => false, + )) => Ok(false), + Some(Err(error)) => Err(error), } } } + +#[cfg(test)] +mod selector_failure_tests { + use super::*; + + #[test] + fn body_rewrite_propagates_selector_failure() { + let root = tempfile::tempdir().unwrap(); + std::fs::write(root.path().join("mdbase.yaml"), "spec_version: 0.3.0\n").unwrap(); + let collection = Collection::open(root.path()).unwrap(); + let mut index = crate::links::resolver::LinkResolutionIndex::default(); + index + .basename_lower_to_paths + .insert("target".to_string(), vec!["target.md".to_string()]); + let mut body = "[[target]]".to_string(); + + let error = collection + .update_body_links( + &mut body, + "target.md", + "renamed.md", + "target", + "renamed", + "target", + "renamed", + "", + "../unsafe-source.md", + None, + &index, + ) + .unwrap_err(); + + assert_eq!(error.code, "invalid_resolution_candidate"); + assert_eq!(body, "[[target]]"); + } +} diff --git a/src/operations/rename/frontmatter.rs b/src/operations/rename/frontmatter.rs index f17ca73..8c9f5d4 100644 --- a/src/operations/rename/frontmatter.rs +++ b/src/operations/rename/frontmatter.rs @@ -20,7 +20,7 @@ impl Collection { changed: &mut bool, refs_updated: &mut Vec, warnings: &mut Vec, - ) { + ) -> Result<(), crate::runtime::CatalogError> { let typed_target_fields = type_names .iter() .filter_map(|type_name| self.types.get(type_name)) @@ -47,7 +47,7 @@ impl Collection { refs_updated, warnings, } - .visit(fm, ""); + .visit(fm, "") } } @@ -70,7 +70,11 @@ struct FrontmatterRewriteContext<'a> { } impl FrontmatterRewriteContext<'_> { - fn visit(&mut self, value: &mut serde_yaml::Value, field: &str) { + fn visit( + &mut self, + value: &mut serde_yaml::Value, + field: &str, + ) -> Result<(), crate::runtime::CatalogError> { match value { serde_yaml::Value::Mapping(mapping) => { let keys = mapping.keys().cloned().collect::>(); @@ -84,21 +88,26 @@ impl FrontmatterRewriteContext<'_> { } else { format!("{field}.{key}") }; - self.visit(child, &child_field); + self.visit(child, &child_field)?; } } serde_yaml::Value::Sequence(items) => { for (index, item) in items.iter_mut().enumerate() { - self.visit(item, &format!("{field}[{index}]")); + self.visit(item, &format!("{field}[{index}]"))?; } } - serde_yaml::Value::Tagged(tagged) => self.visit(&mut tagged.value, field), - serde_yaml::Value::String(link) => self.rewrite_string(link, field), + serde_yaml::Value::Tagged(tagged) => self.visit(&mut tagged.value, field)?, + serde_yaml::Value::String(link) => self.rewrite_string(link, field)?, _ => {} } + Ok(()) } - fn rewrite_string(&mut self, link: &mut String, field: &str) { + fn rewrite_string( + &mut self, + link: &mut String, + field: &str, + ) -> Result<(), crate::runtime::CatalogError> { let target_types = self .typed_target_fields .get(field) @@ -108,7 +117,7 @@ impl FrontmatterRewriteContext<'_> { .collection .is_stable_configured_id_wikilink(link, self.source_id.as_deref()) { - return; + return Ok(()); } if self.collection.link_resolves_to( link, @@ -124,19 +133,20 @@ impl FrontmatterRewriteContext<'_> { self.resolution_index, ) { None => {} - Some(crate::links::resolver::LinkResolution::Resolved(path)) + Some(Ok(crate::links::resolver::LinkResolution::Resolved { path, .. })) if path == self.from => {} - Some(crate::links::resolver::LinkResolution::Ambiguous(_)) => { + Some(Ok(crate::links::resolver::LinkResolution::Ambiguous(_))) => { self.warnings.push(serde_json::json!({ "path": self.rel_path, "message": format!("Ambiguous link '{}' not updated", link), })); - return; + return Ok(()); } - Some( + Some(Err(error)) => return Err(error), + Some(Ok( crate::links::resolver::LinkResolution::Missing - | crate::links::resolver::LinkResolution::Resolved(_), - ) => return, + | crate::links::resolver::LinkResolution::Resolved { .. }, + )) => return Ok(()), } *link = self.collection.rewrite_link_value( link, @@ -148,7 +158,7 @@ impl FrontmatterRewriteContext<'_> { self.source_dir, ); self.record_change(field); - return; + return Ok(()); } // A frontmatter scalar may contain prose with multiple links. Reuse the @@ -166,11 +176,12 @@ impl FrontmatterRewriteContext<'_> { self.rel_path, self.source_id.as_deref(), self.resolution_index, - ); + )?; if rewritten != *link { *link = rewritten; self.record_change(field); } + Ok(()) } fn record_change(&mut self, field: &str) { @@ -181,3 +192,48 @@ impl FrontmatterRewriteContext<'_> { })); } } + +#[cfg(test)] +mod selector_failure_tests { + use super::*; + + #[test] + fn frontmatter_rewrite_propagates_selector_failure() { + let root = tempfile::tempdir().unwrap(); + std::fs::write(root.path().join("mdbase.yaml"), "spec_version: 0.3.0\n").unwrap(); + let collection = Collection::open(root.path()).unwrap(); + let mut index = crate::links::resolver::LinkResolutionIndex::default(); + index + .basename_lower_to_paths + .insert("target".to_string(), vec!["target.md".to_string()]); + let mut frontmatter = serde_yaml::from_str("ref: '[[target]]'\n").unwrap(); + let mut changed = false; + let mut updates = Vec::new(); + let mut warnings = Vec::new(); + + let error = collection + .update_fm_links( + &mut frontmatter, + "target.md", + "renamed.md", + "target", + "renamed", + "target", + "renamed", + "", + "../unsafe-source.md", + &None, + &[], + &index, + &mut changed, + &mut updates, + &mut warnings, + ) + .unwrap_err(); + + assert_eq!(error.code, "invalid_resolution_candidate"); + assert!(!changed); + assert!(updates.is_empty()); + assert!(warnings.is_empty()); + } +} diff --git a/src/operations/rename/hooks.rs b/src/operations/rename/hooks.rs index 5700451..2ec6551 100644 --- a/src/operations/rename/hooks.rs +++ b/src/operations/rename/hooks.rs @@ -7,7 +7,7 @@ fn injected_reference_removals( REMOVALS.get_or_init(Default::default) } -#[cfg(test)] +#[cfg(all(test, feature = "legacy-collection-mutation"))] pub(super) fn inject_reference_removal(path: &std::path::Path) { injected_reference_removals() .lock() @@ -24,7 +24,7 @@ fn injected_reference_open_failures( FAILURES.get_or_init(Default::default) } -#[cfg(test)] +#[cfg(all(test, feature = "legacy-collection-mutation"))] pub(super) fn inject_reference_open_failure(path: &std::path::Path) { injected_reference_open_failures() .lock() @@ -60,7 +60,7 @@ fn injected_root_replacements( REPLACEMENTS.get_or_init(Default::default) } -#[cfg(all(test, unix))] +#[cfg(all(test, unix, feature = "legacy-collection-mutation"))] pub(super) fn inject_root_replacement(root: &std::path::Path, target: &std::path::Path) { injected_root_replacements() .lock() @@ -100,7 +100,7 @@ fn injected_parent_swaps() -> &'static std::sync::Mutex< SWAPS.get_or_init(Default::default) } -#[cfg(all(test, unix))] +#[cfg(all(test, unix, feature = "legacy-collection-mutation"))] pub(super) fn inject_parent_swap( root: &std::path::Path, relative_parent: &std::path::Path, diff --git a/src/operations/rename/link_rewrite.rs b/src/operations/rename/link_rewrite.rs index 340eecb..cd21211 100644 --- a/src/operations/rename/link_rewrite.rs +++ b/src/operations/rename/link_rewrite.rs @@ -9,7 +9,7 @@ impl Collection { source_path: &str, target_types: &[String], resolution_index: &crate::links::resolver::LinkResolutionIndex, - ) -> Option { + ) -> Option> { if !link_val.starts_with("[[") || !link_val.ends_with("]]") { return None; } diff --git a/src/operations/rename/mod.rs b/src/operations/rename/mod.rs index f1bc3f7..e252fc7 100644 --- a/src/operations/rename/mod.rs +++ b/src/operations/rename/mod.rs @@ -7,28 +7,28 @@ pub(super) mod hooks; mod link_rewrite; mod planner; mod publication; -#[cfg(test)] +#[cfg(all(test, feature = "legacy-collection-mutation"))] mod tests; pub(crate) use planner::ReferenceRewritePlan; +#[cfg(feature = "legacy-collection-mutation")] use crate::api::operations::RenameInput; +#[cfg(feature = "legacy-collection-mutation")] use crate::api::{RenameRequest, Revision}; use crate::errors::*; -use crate::mutation::{PlannedRecord, PlannedRename, PreparationOptions, PreparedRename}; -use crate::operations::{ - atomic_rename_noclobber, atomic_write_in_prepared_parent, - ensure_no_symlink_components_diagnostic, ensure_regular_record_file_diagnostic, - prepare_record_parent_no_follow, -}; +#[cfg(feature = "legacy-collection-mutation")] +use crate::mutation::PreparationOptions; +use crate::mutation::{PlannedRecord, PlannedRename, PreparedRename}; +use crate::operations::prepare_record_parent_no_follow; use crate::Collection; #[cfg(test)] use hooks::apply_injected_root_replacement; impl Collection { - /// Legacy JSON rename edge. Canonical callers use the typed mutation service. - pub fn rename(&self, input: &serde_json::Value) -> serde_json::Value { + #[cfg(feature = "legacy-collection-mutation")] + pub(crate) fn rename_legacy(&self, input: &serde_json::Value) -> serde_json::Value { #[cfg(test)] crate::mutation::probe_legacy_parse(); let input = match RenameInput::parse(input) { @@ -70,12 +70,11 @@ impl Collection { Err(error) => return error, }; for (path, _) in &simulations { - if let Err(error) = crate::operations::ensure_no_symlink_components( - &self.root, - path.as_str(), - self.spec_profile, - ) { - return error; + if let Err(error) = self + .held_root() + .ensure_no_symlink_components(&path.to_path_buf()) + { + return crate::errors::op_error(PATH_TRAVERSAL, &error.to_string()); } } let request = RenameRequest { @@ -151,18 +150,23 @@ impl Collection { if !dry_run { #[cfg(test)] apply_injected_root_replacement(&self.root); - ensure_no_symlink_components_diagnostic(&self.root, &from, self.spec_profile) - .map_err(crate::mutation::MutationFailure::diagnostic)?; - ensure_no_symlink_components_diagnostic(&self.root, &to, self.spec_profile) - .map_err(crate::mutation::MutationFailure::diagnostic)?; - ensure_regular_record_file_diagnostic(&request.from.under(&self.root), &from) - .map_err(crate::mutation::MutationFailure::diagnostic)?; - if !self.rename_root_path_is_current() { - return Err(crate::mutation::MutationFailure::operation( - CONCURRENT_MODIFICATION, - "Collection root was replaced during rename", - )); - } + self.held_root() + .ensure_no_symlink_components(&request.from.to_path_buf()) + .and_then(|()| { + self.held_root() + .ensure_no_symlink_components(&request.to.to_path_buf()) + }) + .map_err(|error| { + crate::mutation::MutationFailure::operation(PATH_TRAVERSAL, error.to_string()) + })?; + self.held_root() + .metadata(&request.from.to_path_buf()) + .map_err(|_| { + crate::mutation::MutationFailure::operation( + FILE_NOT_FOUND, + format!("File not found: {from}"), + ) + })?; prepare_record_parent_no_follow(self, &request.to).map_err(|error| { crate::mutation::MutationFailure::operation( "io_error", @@ -189,38 +193,29 @@ impl Collection { format!("File '{from}' was modified externally"), )); } - if !self.rename_root_path_is_current() { - return Err(crate::mutation::MutationFailure::operation( - CONCURRENT_MODIFICATION, - "Collection root was replaced during rename", - )); - } - atomic_rename_noclobber( - &request.from.under(&self.root), - &request.to.under(&self.root), - ) - .map_err(|error| { - crate::mutation::MutationFailure::operation( - if error.kind() == std::io::ErrorKind::AlreadyExists { - PATH_CONFLICT - } else { - "io_error" - }, - if error.kind() == std::io::ErrorKind::AlreadyExists { - format!("Target already exists: {to}") - } else { - format!("Failed to rename: {error}") - }, - ) - })?; + self.held_root() + .rename_noclobber(&request.from.to_path_buf(), &request.to.to_path_buf()) + .map_err(|error| { + crate::mutation::MutationFailure::operation( + if error.kind() == std::io::ErrorKind::AlreadyExists { + PATH_CONFLICT + } else { + "io_error" + }, + if error.kind() == std::io::ErrorKind::AlreadyExists { + format!("Target already exists: {to}") + } else { + format!("Failed to rename: {error}") + }, + ) + })?; if request.update_refs { for (path, content) in legacy_simulations { if prepare_record_parent_no_follow(self, &path).is_ok() { - let _ = atomic_write_in_prepared_parent( - &path.under(&self.root), - content.as_bytes(), - ); + let _ = self + .held_root() + .atomic_write(&path.to_path_buf(), content.as_bytes()); } } } @@ -305,6 +300,7 @@ fn planned_body(bytes: &[u8], fallback: &str) -> String { .unwrap_or_else(|| fallback.to_string()) } +#[cfg(feature = "legacy-collection-mutation")] fn planned_legacy_result(planned: &PlannedRename) -> serde_json::Value { let mut result = if planned.dry_run { serde_json::json!({ @@ -345,6 +341,7 @@ fn planned_legacy_result(planned: &PlannedRename) -> serde_json::Value { result } +#[cfg(feature = "legacy-collection-mutation")] fn mutation_failure_json(diagnostics: Vec) -> serde_json::Value { if diagnostics.len() == 1 { serde_json::json!({"error": diagnostics.into_iter().next().unwrap()}) diff --git a/src/operations/rename/planner.rs b/src/operations/rename/planner.rs index 6bf42b4..36003f0 100644 --- a/src/operations/rename/planner.rs +++ b/src/operations/rename/planner.rs @@ -20,7 +20,7 @@ impl Collection { source_id: &Option, warnings: &mut Vec, failures: &mut Vec, - ) -> Vec { + ) -> Result, crate::runtime::CatalogError> { let from_stem = stem(from); let to_stem = stem(to); let from_no_ext = without_markdown_extension(from); @@ -70,7 +70,7 @@ impl Collection { &mut fm_changed, &mut pending_updates, warnings, - ); + )?; } let mut new_body = doc.body.clone(); @@ -86,7 +86,7 @@ impl Collection { execution_path, source_id.as_deref(), &resolution_index, - ); + )?; if body_changed { pending_updates.push(serde_json::json!({ "path": execution_path, @@ -140,7 +140,7 @@ impl Collection { updates: pending_updates, }); } - plans + Ok(plans) } } diff --git a/src/operations/rename/publication.rs b/src/operations/rename/publication.rs index f1a225e..134f167 100644 --- a/src/operations/rename/publication.rs +++ b/src/operations/rename/publication.rs @@ -13,11 +13,12 @@ impl Collection { failures: &mut Vec, ) { for plan in plans { - let full_path = self.root.join(&plan.execution_path); #[cfg(test)] - apply_injected_reference_removal(&full_path); + let display_path = self.root.join(&plan.execution_path); #[cfg(test)] - let injected_open_failure = take_injected_reference_open_failure(&full_path); + apply_injected_reference_removal(&display_path); + #[cfg(test)] + let injected_open_failure = take_injected_reference_open_failure(&display_path); #[cfg(test)] if injected_open_failure { crate::operations::set_record_open_failure( @@ -71,16 +72,8 @@ impl Collection { })); continue; } - if !self.rename_root_path_is_current() { - failures.push(serde_json::json!({ - "path": plan.execution_path, - "reason": "io_error", - "message": "Collection root was replaced during rename", - })); - continue; - } - if let Err(error) = crate::operations::atomic_write_in_prepared_parent( - &full_path, + if let Err(error) = self.held_root().atomic_write( + std::path::Path::new(&plan.execution_path), plan.output.as_bytes(), ) { failures.push(serde_json::json!({ @@ -93,12 +86,4 @@ impl Collection { references_updated.extend(plan.updates); } } - - pub(crate) fn rename_root_path_is_current(&self) -> bool { - let held = self - .root_capability() - .and_then(|directory| same_file::Handle::from_file(directory.into_std_file())); - let current = same_file::Handle::from_path(&self.root); - matches!((held, current), (Ok(held), Ok(current)) if held == current) - } } diff --git a/src/operations/rename/tests.rs b/src/operations/rename/tests.rs index a7296d0..3974547 100644 --- a/src/operations/rename/tests.rs +++ b/src/operations/rename/tests.rs @@ -387,11 +387,9 @@ fn descendant_replacement_never_redirects_reference_writes() { "to": "renamed.md", "update_refs": true, })); - assert_eq!( - result["error"]["code"], "collection_snapshot_failed", - "{result:#}" - ); - assert!(root.path().join("target.md").exists()); + assert_eq!(result["to"], "renamed.md", "{result:#}"); + assert!(!root.path().join("target.md").exists()); + assert!(root.path().join("renamed.md").is_file()); assert_eq!( fs::read_to_string(external.path().join("ref.md")).unwrap(), "external [[target]]\n" @@ -400,7 +398,7 @@ fn descendant_replacement_never_redirects_reference_writes() { #[cfg(unix)] #[test] -fn root_replacement_is_rejected_before_nested_parent_preparation() { +fn root_replacement_keeps_rename_on_the_held_collection() { let (root, collection) = collection(); fs::write(root.path().join("target.md"), "inside\n").unwrap(); let external = tempfile::tempdir().unwrap(); @@ -415,10 +413,7 @@ fn root_replacement_is_rejected_before_nested_parent_preparation() { "to": "new/deep/renamed.md", "update_refs": false, })); - assert_eq!( - result["error"]["code"], CONCURRENT_MODIFICATION, - "{result:#}" - ); + assert_eq!(result["to"], "new/deep/renamed.md", "{result:#}"); assert_eq!( fs::read_to_string(external.path().join("target.md")).unwrap(), "external\n" @@ -428,14 +423,14 @@ fn root_replacement_is_rejected_before_nested_parent_preparation() { "untouched\n" ); assert!(!external.path().join("new").exists()); - assert!(!held_root.join("new").exists()); - - fs::remove_file(&original_root).unwrap(); - fs::rename(held_root, &original_root).unwrap(); + assert!(!held_root.join("target.md").exists()); assert_eq!( - fs::read_to_string(original_root.join("target.md")).unwrap(), + fs::read_to_string(held_root.join("new/deep/renamed.md")).unwrap(), "inside\n" ); + + fs::remove_file(&original_root).unwrap(); + fs::rename(held_root, &original_root).unwrap(); } #[cfg(unix)] diff --git a/src/operations/type_file.rs b/src/operations/type_file.rs index 2fcd42c..2f0d1d4 100644 --- a/src/operations/type_file.rs +++ b/src/operations/type_file.rs @@ -1,14 +1,11 @@ //! Revision-safe type definition resource operations. -use std::fs; -use std::io::Write; use std::path::Path; use serde_json::{json, Value}; -use tempfile::NamedTempFile; use crate::diagnostic::Diagnostic; -use crate::operations::{ensure_no_symlink_components, ensure_revision, ensure_safe_relative_path}; +use crate::operations::ensure_safe_relative_path; use crate::v03::{self, OperationResult}; use crate::Collection; @@ -49,29 +46,35 @@ impl Collection { if let Err(diagnostic) = self.validate_type_path(&path) { return failed(*diagnostic); } - let full_path = self.root.join(&path); - if full_path.exists() { + if self.held_root().exists_file(&path) { return failed(Diagnostic::error( "path_conflict", format!("A type definition already exists at '{path}'."), Some(path), )); } - if let Err(error) = atomic_create(&full_path, document.as_bytes()) { - return failed(if error.kind() == std::io::ErrorKind::AlreadyExists { - Diagnostic::error( - "path_conflict", - format!("A type definition already exists at '{path}'."), - Some(path.clone()), - ) - } else { - io_diagnostic(&path, error) - }); + if let Err(error) = self + .held_root() + .atomic_create(Path::new(&path), document.as_bytes()) + { + return failed( + if error.kind() == std::io::ErrorKind::AlreadyExists + || self.held_root().exists_file(Path::new(&path)) + { + Diagnostic::error( + "path_conflict", + format!("A type definition already exists at '{path}'."), + Some(path.clone()), + ) + } else { + io_diagnostic(&path, error) + }, + ); } - match Collection::open(&self.root) { + match self.reopen_held(true) { Ok(reloaded) => reloaded.type_file_result(&path), Err(error) => { - let _ = fs::remove_file(&full_path); + let _ = self.held_root().remove_file(Path::new(&path)); failed(open_diagnostic(error, &path)) } } @@ -89,28 +92,27 @@ impl Collection { if let Err(diagnostics) = self.parse_type_candidate(document, Some(&path)) { return failed_many(diagnostics); } - let full_path = self.root.join(&path); - if let Err(error) = ensure_revision( - &full_path, + let previous = match self.held_root().read(&path) { + Ok(previous) => previous, + Err(error) => return failed(io_diagnostic(&path, error)), + }; + if let Err(error) = ensure_revision_bytes( + &previous, &path, input.get("if_revision").and_then(Value::as_str), ) { return legacy_failure(error, Some(&path)); } - let previous = match fs::read(&full_path) { - Ok(previous) => previous, - Err(error) => return failed(io_diagnostic(&path, error)), - }; - let permissions = fs::metadata(&full_path) - .ok() - .map(|metadata| metadata.permissions()); - if let Err(error) = atomic_write(&full_path, document.as_bytes(), permissions) { + if let Err(error) = self + .held_root() + .atomic_write(Path::new(&path), document.as_bytes()) + { return failed(io_diagnostic(&path, error)); } - match Collection::open(&self.root) { + match self.reopen_held(true) { Ok(reloaded) => reloaded.type_file_result(&path), Err(error) => { - let rollback = atomic_write(&full_path, &previous, None); + let rollback = self.held_root().atomic_write(Path::new(&path), &previous); let mut diagnostic = open_diagnostic(error, &path); if let Err(rollback_error) = rollback { diagnostic.message = format!( @@ -162,8 +164,9 @@ impl Collection { fn validate_type_path(&self, path: &str) -> Result<(), Box> { ensure_safe_relative_path(path, self.spec_profile) .map_err(|error| Box::new(open_diagnostic(error, path)))?; - ensure_no_symlink_components(&self.root, path, self.spec_profile) - .map_err(|error| Box::new(open_diagnostic(error, path)))?; + self.held_root() + .ensure_no_symlink_components(Path::new(path)) + .map_err(|error| Box::new(io_diagnostic(path, error)))?; let prefix = format!("{}/", self.settings.types_folder.trim_end_matches('/')); if !path.starts_with(&prefix) || Path::new(path).extension().and_then(|value| value.to_str()) != Some("md") @@ -190,12 +193,31 @@ impl Collection { if let Err(diagnostic) = self.validate_type_path(path) { return Err(vec![*diagnostic]); } - v03::parse_type_file(document, &self.root.join(path), &self.root, path) + // `parse_type_file` may resolve a local schema.ref. Materialize a private snapshot + // from the held authority so its pathname-based resolver cannot adopt a replacement root. + let staging = tempfile::tempdir().map_err(|error| vec![io_diagnostic(path, error)])?; + for relative in self + .held_root() + .files_recursive(Path::new("")) + .map_err(|error| vec![io_diagnostic(path, error)])? + { + let bytes = self + .held_root() + .read(&relative) + .map_err(|error| vec![io_diagnostic(path, error)])?; + let destination = staging.path().join(&relative); + if let Some(parent) = destination.parent() { + std::fs::create_dir_all(parent) + .map_err(|error| vec![io_diagnostic(path, error)])?; + } + std::fs::write(destination, bytes).map_err(|error| vec![io_diagnostic(path, error)])?; + } + let candidate = staging.path().join(path); + v03::parse_type_file(document, &candidate, staging.path(), path) } fn type_file_result(&self, path: &str) -> OperationResult { - let full_path = self.root.join(path); - let bytes = match fs::read(&full_path) { + let bytes = match self.held_root().read(path) { Ok(bytes) => bytes, Err(error) => return failed(io_diagnostic(path, error)), }; @@ -251,33 +273,21 @@ fn candidate_path(input: &Value) -> Option<&str> { input.get("path").and_then(Value::as_str) } -fn atomic_write( - path: &Path, +fn ensure_revision_bytes( bytes: &[u8], - permissions: Option, -) -> Result<(), std::io::Error> { - let parent = path.parent().unwrap_or_else(|| Path::new(".")); - fs::create_dir_all(parent)?; - let mut temporary = NamedTempFile::new_in(parent)?; - if let Some(permissions) = permissions { - temporary.as_file().set_permissions(permissions)?; + display_path: &str, + expected: Option<&str>, +) -> Result<(), Value> { + let Some(expected) = expected else { + return Ok(()); + }; + if v03::revision(bytes) != expected { + return Err(crate::errors::op_error( + crate::errors::CONCURRENT_MODIFICATION, + &format!("File '{display_path}' was modified externally"), + )); } - temporary.write_all(bytes)?; - temporary.as_file().sync_all()?; - temporary.persist(path).map_err(|error| error.error)?; - crate::operations::sync_directory(parent) -} - -fn atomic_create(path: &Path, bytes: &[u8]) -> Result<(), std::io::Error> { - let parent = path.parent().unwrap_or_else(|| Path::new(".")); - fs::create_dir_all(parent)?; - let mut temporary = NamedTempFile::new_in(parent)?; - temporary.write_all(bytes)?; - temporary.as_file().sync_all()?; - temporary - .persist_noclobber(path) - .map_err(|error| error.error)?; - crate::operations::sync_directory(parent) + Ok(()) } fn failed(diagnostic: Diagnostic) -> OperationResult { @@ -327,6 +337,7 @@ fn legacy_failure(error: Value, path: Option<&str>) -> OperationResult { #[cfg(test)] mod tests { use super::*; + use std::fs; use std::sync::{Arc, Barrier}; use std::thread; @@ -405,6 +416,34 @@ mod tests { assert!(reloaded.read_type_file(&json!({"name": "note"})).valid); } + #[cfg(unix)] + #[test] + fn replacement_root_never_receives_type_resource_io() { + let parent = tempfile::tempdir().unwrap(); + let root = parent.path().join("collection"); + fs::create_dir(&root).unwrap(); + fs::write( + root.join("mdbase.yaml"), + "spec_version: 0.3.0\nsettings:\n types_folder: _types\n", + ) + .unwrap(); + fs::create_dir(root.join("_types")).unwrap(); + fs::write(root.join("_types/note.md"), type_document("note", "Note")).unwrap(); + let collection = Collection::open(&root).unwrap(); + let held = parent.path().join("held"); + fs::rename(&root, &held).unwrap(); + fs::create_dir(&root).unwrap(); + fs::write(root.join("mdbase.yaml"), "spec_version: 0.3.0\n").unwrap(); + + assert!(collection.read_type_file(&json!({"name": "note"})).valid); + let created = collection.create_type_file(&json!({ + "document": type_document("project", "Project") + })); + assert!(created.valid, "{:?}", created.diagnostics); + assert!(held.join("_types/project.md").is_file()); + assert!(!root.join("_types").exists()); + } + #[test] fn concurrent_type_creates_never_replace_the_winner() { let directory = collection(); @@ -430,12 +469,15 @@ mod tests { .collect::>(); assert_eq!(results.iter().filter(|result| result.valid).count(), 1); - assert!(results.iter().filter(|result| !result.valid).all(|result| { - result - .diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "path_conflict") - })); + assert!( + results.iter().filter(|result| !result.valid).all(|result| { + result + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "path_conflict") + }), + "unexpected create outcomes: {results:#?}" + ); let persisted = fs::read_to_string(directory.path().join("_types/shared.md")).unwrap(); let winning_document = results.iter().find(|result| result.valid).unwrap().result ["document"] diff --git a/src/operations/update.rs b/src/operations/update.rs index f344e85..1b02381 100644 --- a/src/operations/update.rs +++ b/src/operations/update.rs @@ -1,15 +1,14 @@ //! Update operation (§12.3). +#[cfg(feature = "legacy-collection-mutation")] use crate::api::operations::{UpdateInput, UpdateOutput}; +#[cfg(feature = "legacy-collection-mutation")] use crate::api::{CollectionPath, Revision, UpdateRequest}; use crate::errors::*; use crate::frontmatter::parser::{parse_document_for_rewrite, yaml_mapping_to_json}; use crate::frontmatter::serializer; use crate::mutation::{PlannedRecord, PreparedUpdate}; -use crate::operations::{ - atomic_write, ensure_no_symlink_components_diagnostic, ensure_regular_record_file_diagnostic, - mutation_record_path_diagnostic, -}; +use crate::operations::mutation_record_path_diagnostic; use crate::Collection; pub(crate) struct PrevalidatedUpdate { @@ -17,7 +16,7 @@ pub(crate) struct PrevalidatedUpdate { pub raw_frontmatter: serde_json::Map, } -#[cfg(test)] +#[cfg(all(test, feature = "legacy-collection-mutation"))] fn injected_publication_replacements( ) -> &'static std::sync::Mutex> { static REPLACEMENTS: std::sync::OnceLock< @@ -26,7 +25,7 @@ fn injected_publication_replacements( REPLACEMENTS.get_or_init(Default::default) } -#[cfg(test)] +#[cfg(all(test, feature = "legacy-collection-mutation"))] fn injected_prevalidated_replacements( ) -> &'static std::sync::Mutex> { static REPLACEMENTS: std::sync::OnceLock< @@ -35,7 +34,7 @@ fn injected_prevalidated_replacements( REPLACEMENTS.get_or_init(Default::default) } -#[cfg(test)] +#[cfg(all(test, feature = "legacy-collection-mutation"))] pub(crate) fn inject_prevalidated_replacement( path: &std::path::Path, replacement: std::path::PathBuf, @@ -46,7 +45,7 @@ pub(crate) fn inject_prevalidated_replacement( .insert(path.to_path_buf(), replacement); } -#[cfg(test)] +#[cfg(all(test, feature = "legacy-collection-mutation"))] fn apply_injected_prevalidated_replacement(path: &std::path::Path) { let replacement = injected_prevalidated_replacements() .lock() @@ -57,7 +56,7 @@ fn apply_injected_prevalidated_replacement(path: &std::path::Path) { } } -#[cfg(test)] +#[cfg(all(test, feature = "legacy-collection-mutation"))] fn inject_publication_replacement(path: &std::path::Path, replacement: std::path::PathBuf) { injected_publication_replacements() .lock() @@ -65,7 +64,7 @@ fn inject_publication_replacement(path: &std::path::Path, replacement: std::path .insert(path.to_path_buf(), replacement); } -#[cfg(test)] +#[cfg(all(test, feature = "legacy-collection-mutation"))] fn apply_injected_publication_replacement(path: &std::path::Path) { let replacement = injected_publication_replacements() .lock() @@ -77,8 +76,8 @@ fn apply_injected_publication_replacement(path: &std::path::Path) { } impl Collection { - /// Update a file (§12.3). - pub fn update(&self, input: &serde_json::Value) -> serde_json::Value { + #[cfg(feature = "legacy-collection-mutation")] + pub(crate) fn update_legacy(&self, input: &serde_json::Value) -> serde_json::Value { let parsed = match UpdateInput::parse(input) { Ok(parsed) => parsed, Err(error) => return error, @@ -90,6 +89,7 @@ impl Collection { } } + #[cfg(feature = "legacy-collection-mutation")] pub(crate) fn update_prevalidated( &self, input: &serde_json::Value, @@ -139,7 +139,7 @@ impl Collection { Err(error) => return Err(crate::mutation::MutationFailure::diagnostic(error)), }; if let Err(error) = - ensure_no_symlink_components_diagnostic(&self.root, path.as_str(), self.spec_profile) + crate::operations::ensure_no_symlink_components_held_diagnostic(self, path.as_str()) { return Err(crate::mutation::MutationFailure::diagnostic(error)); } @@ -154,15 +154,12 @@ impl Collection { }; let new_body = new_body.as_deref(); + #[cfg(all(test, feature = "legacy-collection-mutation"))] let full_path = path.under(&self.root); - if let Err(error) = ensure_regular_record_file_diagnostic(&full_path, path.as_str()) { - return Err(crate::mutation::MutationFailure::diagnostic(error)); - } - // Load bytes, metadata, and revision from one open handle. The loaded // revision is also the mandatory publication precondition, including // when a legacy caller omitted `if_revision`. - #[cfg(test)] + #[cfg(all(test, feature = "legacy-collection-mutation"))] if prevalidated.is_some() { apply_injected_prevalidated_replacement(&full_path); } @@ -281,7 +278,7 @@ impl Collection { && (has_generated || (validate_collection && self.settings.default_validation == "error")) { - match self.capture_collection_snapshot(&crate::OperationCancellation::new()) { + match self.capture_collection_snapshot_current() { Ok(snapshot) => Some(snapshot), Err(error) => { return Err(crate::mutation::MutationFailure::operation( @@ -399,12 +396,7 @@ impl Collection { } }; - if let Err(error) = - ensure_no_symlink_components_diagnostic(&self.root, path.as_str(), self.spec_profile) - { - return Err(crate::mutation::MutationFailure::diagnostic(error)); - } - #[cfg(test)] + #[cfg(all(test, feature = "legacy-collection-mutation"))] apply_injected_publication_replacement(&full_path); // Reopen once at the publication boundary and compare a byte revision, @@ -426,7 +418,10 @@ impl Collection { format!("File '{}' was modified during operation", path.as_str()), )); } - if let Err(error) = atomic_write(&full_path, output.as_bytes()) { + if let Err(error) = self + .held_root() + .atomic_write(&path.to_path_buf(), output.as_bytes()) + { return Err(crate::mutation::MutationFailure::operation( "io_error", format!("Failed to write: {error}"), @@ -475,6 +470,7 @@ impl Collection { } } +#[cfg(feature = "legacy-collection-mutation")] fn legacy_prepared_update(input: UpdateInput) -> PreparedUpdate { let legacy_path = input.path; let path = CollectionPath::new(&legacy_path) @@ -499,6 +495,7 @@ fn legacy_prepared_update(input: UpdateInput) -> PreparedUpdate { } } +#[cfg(feature = "legacy-collection-mutation")] fn mutation_failure_json(failure: crate::mutation::MutationFailure) -> serde_json::Value { match failure.kind { crate::mutation::MutationFailureKind::Operation if failure.diagnostics.len() == 1 => { @@ -512,6 +509,7 @@ fn mutation_failure_json(failure: crate::mutation::MutationFailure) -> serde_jso } } +#[cfg(feature = "legacy-collection-mutation")] fn planned_update_output(planned: PlannedRecord) -> serde_json::Value { UpdateOutput { path: planned.path.to_string(), @@ -526,7 +524,7 @@ fn planned_update_output(planned: PlannedRecord) -> serde_json::Value { .into_json() } -#[cfg(test)] +#[cfg(all(test, feature = "legacy-collection-mutation"))] mod tests { use super::inject_publication_replacement; use crate::Collection; diff --git a/src/query/cache_source.rs b/src/query/cache_source.rs index 4dd9bd7..77b224f 100644 --- a/src/query/cache_source.rs +++ b/src/query/cache_source.rs @@ -114,12 +114,16 @@ impl Collection { /// Try to open the cache database. Returns `None` if the DB file doesn't /// exist or can't be opened. fn try_open_cache(&self) -> Result, CacheError> { - let db_path = self.root.join(&self.settings.cache_folder).join("cache.db"); + let db_path = self + .held_root() + .cache_storage_path() + .join(&self.settings.cache_folder) + .join("cache.db"); if !db_path.exists() { return Ok(None); } Ok(Some(sqlite::open_cache_db( - &self.root, + self.held_root().cache_storage_path(), &self.settings.cache_folder, )?)) } @@ -131,8 +135,8 @@ impl Collection { &self, conn: &mut Connection, cancellation: &OperationCancellation, - ) -> Result, CacheError> { - let disk_files = self.scan_collection_files_checked()?; + ) -> Result, CacheError> { + let disk_files = self.scan_collection_relative_paths_checked()?; // Cache refresh errors deliberately fall back to disk. Treating // cancellation as an ordinary cache error would therefore continue @@ -142,7 +146,7 @@ impl Collection { return Err(CacheError::Cancelled); } - let changes = staleness::find_changes(conn, &self.root, &disk_files)?; + let changes = staleness::find_changes(conn, self, &disk_files)?; if changes.stale.is_empty() && changes.deleted.is_empty() { return Ok(disk_files); @@ -152,7 +156,7 @@ impl Collection { // only the uncommon cache-write section, then recompute against the // winning transaction so two refreshers cannot apply stale deltas. let transaction = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; - let changes = staleness::find_changes(&transaction, &self.root, &disk_files)?; + let changes = staleness::find_changes(&transaction, self, &disk_files)?; // Remove deleted files from cache for rel_path in &changes.deleted { @@ -163,16 +167,11 @@ impl Collection { } // Re-index stale/new files - for abs_path in &changes.stale { + for rel_path in &changes.stale { if cancellation.is_cancelled() { return Err(CacheError::Cancelled); } - let rel_path = abs_path - .strip_prefix(&self.root) - .map_err(|_| CacheError::OutsideRoot(abs_path.display().to_string()))? - .to_string_lossy() - .replace('\\', "/"); - indexer::reindex_file(&transaction, self, &rel_path)?; + indexer::reindex_file(&transaction, self, rel_path)?; } if !changes.stale.is_empty() || !changes.deleted.is_empty() { @@ -326,7 +325,11 @@ impl Collection { let total_started = Instant::now(); let mut perf = LoadQueryPerf::default(); let open_started = Instant::now(); - let mut conn = sqlite::open_cache_db(&self.root, &self.settings.cache_folder).ok()?; + let mut conn = sqlite::open_cache_db( + self.held_root().cache_storage_path(), + &self.settings.cache_folder, + ) + .ok()?; perf.try_open_cache_ms = elapsed_ms(open_started); perf.cache_used = true; @@ -560,10 +563,11 @@ impl Collection { profile: bool, include_link_graph: bool, ) -> Result<(CollectionSnapshot, Option), SnapshotError> { + let context = crate::runtime::OperationContext::current_or_legacy(); self.load_query_data_profiled_cancellable( profile, include_link_graph, - &OperationCancellation::new(), + context.cancellation(), ) } @@ -651,12 +655,26 @@ impl Collection { perf.cache_used = true; cached_records = Some(records); } + Err(CacheError::Cancelled) + | Err(CacheError::Scan( + crate::snapshot::CollectionScanError::Cancelled, + )) => return Err(SnapshotError::Cancelled), + Err(CacheError::Scan( + crate::snapshot::CollectionScanError::Provider(error), + )) => return Err(SnapshotError::Provider(error)), Err(error) => { perf.cache_fallback = true; coordinated_cache_error = Some(error.to_string()); } } } + Err(CacheError::Cancelled) + | Err(CacheError::Scan(crate::snapshot::CollectionScanError::Cancelled)) => { + return Err(SnapshotError::Cancelled) + } + Err(CacheError::Scan(crate::snapshot::CollectionScanError::Provider( + error, + ))) => return Err(SnapshotError::Provider(error)), Err(error) => { perf.cache_fallback = true; coordinated_cache_error = Some(error.to_string()); @@ -719,13 +737,20 @@ impl Collection { let backlinks_start = Instant::now(); let all_files_arc = Arc::new(all_files_data); let cached_backlinks = (perf.cache_used && !refresh_from_filesystem).then(|| { - sqlite::open_cache_db(&self.root, &self.settings.cache_folder) - .map_err(CacheError::from) - .and_then(|connection| indexer::load_backlinks(&connection)) + sqlite::open_cache_db( + self.held_root().cache_storage_path(), + &self.settings.cache_folder, + ) + .map_err(CacheError::from) + .and_then(|connection| indexer::load_backlinks(&connection)) }); let (backlinks_index, backlinks_perf) = match cached_backlinks { Some(Ok(backlinks)) => (backlinks, None), - _ => self.build_backlinks_index_profiled(&all_files_arc, profile), + _ => self + .build_backlinks_index_profiled(&all_files_arc, profile) + .map_err(|error| { + SnapshotError::Cache(format!("{}: {}", error.code, error.message)) + })?, }; cancellation.check().map_err(|_| SnapshotError::Cancelled)?; let backlinks_arc = Arc::new(backlinks_index); diff --git a/src/query/canonical/execute.rs b/src/query/canonical/execute.rs index 95aeaa3..501fa4c 100644 --- a/src/query/canonical/execute.rs +++ b/src/query/canonical/execute.rs @@ -127,15 +127,16 @@ pub(crate) struct QueryExecution { pub(crate) type QueryEvaluation = Result>; pub(crate) fn execute_typed(collection: &Collection, query: Query) -> QueryEvaluation { + let context = crate::runtime::OperationContext::internal(); execute_model_profiled_cancellable( collection, query, - &OperationCancellation::new(), + context.cancellation(), false, Instant::now(), 0, ) - .expect("a fresh cancellation token cannot be cancelled") + .expect("the context-free compatibility context is active") .0 } @@ -237,6 +238,15 @@ pub(crate) fn execute_model_profiled_cancellable( ) }; cancellation.check()?; + if let Some(error) = crate::runtime::OperationContext::current() + .and_then(|context| context.capture_limit_error()) + { + finish!(Err(vec![Diagnostic::error( + error.code(), + error.to_string(), + None, + )])); + } if let Some(page) = loaded { performance.load_us = micros(phase.elapsed()); apply_load_performance(&mut performance, &page.performance); diff --git a/src/record_load.rs b/src/record_load.rs index 5b734e2..2a02d6a 100644 --- a/src/record_load.rs +++ b/src/record_load.rs @@ -8,8 +8,12 @@ use serde_json::{json, Value}; use crate::frontmatter::parser::{ parse_document_layout, yaml_mapping_to_json, FrontmatterState, ParsedDocumentLayout, }; +use crate::runtime::{OperationContext, ProviderError}; use crate::{Collection, OperationCancellation}; +/// Metadata captured from a file already opened through `CollectionRoot`. +pub(crate) type FileMetadata = std::fs::Metadata; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum InvalidRecordReason { InvalidYaml, @@ -282,7 +286,16 @@ pub(crate) fn load_record_no_follow( collection: &Collection, rel_path: &str, ) -> std::io::Result> { - load_record_no_follow_cancellable(collection, rel_path, &OperationCancellation::new()) + if let Some(context) = OperationContext::current() { + return load_record_no_follow_context(collection, rel_path, &context) + .map_err(std::io::Error::other); + } + // Intentional context-free compatibility seam. + load_record_no_follow_cancellable( + collection, + rel_path, + OperationContext::internal().cancellation(), + ) } pub(crate) fn load_record_no_follow_cancellable( @@ -297,6 +310,25 @@ pub(crate) fn load_record_no_follow_cancellable( .transpose() } +/// Budgeted record load used by canonical runtime paths. +pub(crate) fn load_record_no_follow_context( + collection: &Collection, + rel_path: &str, + context: &OperationContext, +) -> Result, ProviderError> { + #[cfg(test)] + SNAPSHOT_RECORD_LOADS.with(|loads| loads.set(loads.get() + 1)); + context.check()?; + crate::operations::open_regular_record_no_follow(collection, rel_path) + .map_err(|error| { + ProviderError::CollectionOpen(format!( + "failed to open collection record '{rel_path}': {error}" + )) + })? + .map(|file| load_open_record_context(collection, file, rel_path, context)) + .transpose() +} + fn load_open_record( collection: &Collection, mut file: std::fs::File, @@ -317,6 +349,70 @@ fn load_open_record( maybe_cancel_read_for_test(cancellation); } cancellation.check().map_err(|_| cancelled_io())?; + finish_open_record(collection, file, rel_path, before, bytes) +} + +fn load_open_record_context( + collection: &Collection, + mut file: std::fs::File, + rel_path: &str, + context: &OperationContext, +) -> Result { + let before = file.metadata().map_err(record_provider_error(rel_path))?; + context.check_file_bytes(before.len())?; + let capacity = + usize::try_from(before.len()).map_err(|_| crate::runtime::CaptureLimitExceeded { + kind: crate::runtime::CaptureLimitKind::ArithmeticOverflow, + limit: usize::MAX as u64, + attempted: before.len(), + })?; + let mut bytes = Vec::new(); + bytes.try_reserve_exact(capacity).map_err(|_| { + ProviderError::CaptureLimitExceeded(crate::runtime::CaptureLimitExceeded { + kind: crate::runtime::CaptureLimitKind::ArithmeticOverflow, + limit: usize::MAX as u64, + attempted: before.len(), + }) + })?; + let mut chunk = [0_u8; 64 * 1024]; + loop { + context.check()?; + let read = file + .read(&mut chunk) + .map_err(record_provider_error(rel_path))?; + if read == 0 { + break; + } + let attempted = u64::try_from(bytes.len()) + .ok() + .and_then(|value| value.checked_add(read as u64)) + .ok_or({ + ProviderError::CaptureLimitExceeded(crate::runtime::CaptureLimitExceeded { + kind: crate::runtime::CaptureLimitKind::ArithmeticOverflow, + limit: u64::MAX, + attempted: u64::MAX, + }) + })?; + context.check_file_bytes(attempted)?; + context.charge_read(read as u64)?; + context.charge_retained(read as u64)?; + bytes.extend_from_slice(&chunk[..read]); + #[cfg(test)] + maybe_cancel_read_for_test(context.cancellation()); + context.check()?; + } + context.check()?; + finish_open_record(collection, file, rel_path, before, bytes) + .map_err(record_provider_error(rel_path)) +} + +fn finish_open_record( + collection: &Collection, + file: std::fs::File, + rel_path: &str, + before: std::fs::Metadata, + bytes: Vec, +) -> std::io::Result { let after = file.metadata()?; if before.len() != after.len() || before.modified().ok() != after.modified().ok() @@ -331,6 +427,14 @@ fn load_open_record( Ok(classify_bytes(collection, rel_path, bytes, facts)) } +fn record_provider_error(path: &str) -> impl FnOnce(std::io::Error) -> ProviderError + '_ { + move |error| { + ProviderError::CollectionOpen(format!( + "failed to read collection record '{path}': {error}" + )) + } +} + fn cancelled_io() -> std::io::Error { std::io::Error::new(std::io::ErrorKind::Interrupted, "record load cancelled") } diff --git a/src/record_path.rs b/src/record_path.rs index 9e0fefc..549ed89 100644 --- a/src/record_path.rs +++ b/src/record_path.rs @@ -37,12 +37,32 @@ impl Collection { pub fn validate_record_path( &self, path: impl AsRef, + ) -> Result { + self.validate_record_path_mode(path, true) + } + + pub(crate) fn validate_record_path_after_traversal( + &self, + path: impl AsRef, + ) -> Result { + self.validate_record_path_mode(path, false) + } + + fn validate_record_path_mode( + &self, + path: impl AsRef, + check_nested: bool, ) -> Result { let path = CollectionPath::new(path)?; if has_hidden_component(path.as_str()) { return Err(RecordPathError::HiddenComponent); } - if self.is_excluded(path.as_str()) { + let excluded = if check_nested { + self.is_excluded(path.as_str()) + } else { + self.is_excluded_without_nested_collection(path.as_str()) + }; + if excluded { return Err(RecordPathError::Reserved); } if !self.is_valid_extension(path.as_str()) { diff --git a/src/runtime/api.rs b/src/runtime/api.rs index 2656dd1..726e835 100644 --- a/src/runtime/api.rs +++ b/src/runtime/api.rs @@ -3,9 +3,10 @@ use std::num::NonZeroUsize; use super::{ CancelOutcome, ChangeBatch, ChangeFeed, ChangeFeedBaseline, ChangeFeedOwnerId, ChangeFeedTransfer, ChangeFeedTransferId, ChangeFeedTransferIntent, ChangePage, - ChangePageCursor, ChangeWatermark, CommitAttempt, CommitId, DurableCommitState, - ExecutionOutcome, HostClaimId, OperationContext, OperationRequest, PreparationOutcome, - PreparedMutation, ProviderError, ReadCursor, ReadPage, RuntimeChangeEventPage, + ChangePageCursor, ChangeWatermark, CommitAttempt, CommitId, CursorReleaseOutcome, + DurableCommitState, ExecutionOutcome, HostClaimId, OperationContext, OperationRequest, + PreparationOutcome, PreparedMutation, ProviderError, ReadCursor, ReadPage, + RuntimeChangeEventPage, }; /// Provider-neutral authority over one coordinated collection runtime. @@ -32,7 +33,7 @@ pub trait CollectionRuntime: Send + Sync { &self, cursor: ReadCursor, context: &OperationContext, - ) -> Result<(), ProviderError>; + ) -> Result; fn prepare( &self, @@ -154,7 +155,7 @@ impl CollectionRuntime for super::FilesystemRuntime { &self, cursor: ReadCursor, context: &OperationContext, - ) -> Result<(), ProviderError> { + ) -> Result { super::FilesystemRuntime::release_read(self, cursor, context) } diff --git a/src/runtime/canonical_operation.rs b/src/runtime/canonical_operation.rs index f15b8ba..705c084 100644 --- a/src/runtime/canonical_operation.rs +++ b/src/runtime/canonical_operation.rs @@ -9,7 +9,7 @@ use crate::api::{ use crate::diagnostic::Diagnostic as WireDiagnostic; use crate::v03::OperationResult; -use super::{OperationKind, ProviderError}; +use super::{CursorReleaseOutcome, OperationKind, ProviderError}; /// Typed canonical query value retained by the runtime and read cursors. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -62,6 +62,24 @@ pub enum WireOnlyOperationValue { TypeResource(Value), } +/// Typed mutation-resource envelope. The operation discriminator is retained +/// so durable journals can validate the exact resource family. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct CanonicalResourceOperationValue { + operation: OperationKind, + result: Value, +} + +impl CanonicalResourceOperationValue { + pub fn operation(&self) -> OperationKind { + self.operation + } + + pub fn result(&self) -> &Value { + &self.result + } +} + /// Explicitly named storage for forward-compatible definition-result fields. /// Core fields consumed by hosts remain closed and typed. #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] @@ -100,6 +118,29 @@ pub enum CanonicalCollectionSetupValue { Assessment(crate::v03::CollectionSetupAssessment), } +/// Opaque payload used only by version-2 transaction recovery. Its contents +/// cannot be constructed or deserialized through the public API. +#[doc(hidden)] +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct LegacyRecoveredV03Value(OperationResult); + +impl LegacyRecoveredV03Value { + pub(crate) fn from_transaction_recovery(result: OperationResult) -> Self { + Self(result) + } +} + +impl<'de> Deserialize<'de> for LegacyRecoveredV03Value { + fn deserialize(_deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + Err(serde::de::Error::custom( + "legacy-recovered values are restricted to transaction recovery", + )) + } +} + /// Closed semantic value returned by the filesystem runtime. /// /// `None` means the operation failed before producing a semantic value. It is @@ -114,20 +155,92 @@ pub enum CanonicalOperationValue { Delete(Option), Rename(Option), Batch(Option), - TypePack(Option), - CollectionSetup(Option>), + AssessTypePack(Option), + ApplyTypePack(Option), + AssessCollectionSetup(Option>), + ApplyCollectionSetup(Option>), + ViewResourceMutation(CanonicalResourceOperationValue), + TypeResourceMutation(CanonicalResourceOperationValue), + CursorRelease(CursorReleaseOutcome), WireOnly(WireOnlyOperationValue), /// Exact non-semantic recovery of an ambiguous version-2 runtime journal. - LegacyRecoveredV03(OperationResult), + LegacyRecoveredV03(LegacyRecoveredV03Value), +} + +/// Explicit in-memory state of a checked canonical operation outcome. This is +/// derived and does not alter the durable journal shape. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CanonicalOutcomeState { + Completed, + Rejected, + WireCompatibilityCompleted, + WireCompatibilityRejected, + LegacyRecoveredV03, +} + +/// Canonical family discriminator used where an operation context must be +/// checked without conflating cursor lifecycle with query semantics. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CanonicalOperationFamily { + Operation(OperationKind), + CursorLifecycle, + LegacyRecoveredV03, } /// Closed typed operation outcome shared by execution, durable transactions, /// recovery, claim resolution, and generation-pinned cursors. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +/// +/// Its fields are deliberately private and serde is checked as well: +/// +/// ```compile_fail +/// use mdbase::runtime::{CanonicalOperationOutcome, CanonicalOperationValue}; +/// let _ = CanonicalOperationOutcome { +/// valid: true, +/// value: CanonicalOperationValue::Read(None), +/// diagnostics: Vec::new(), +/// }; +/// ``` +#[derive(Clone, Debug, PartialEq)] pub struct CanonicalOperationOutcome { - pub valid: bool, - pub value: CanonicalOperationValue, - pub diagnostics: Vec, + pub(crate) valid: bool, + pub(crate) value: CanonicalOperationValue, + pub(crate) diagnostics: Vec, +} + +#[derive(Serialize, Deserialize)] +struct CanonicalOperationOutcomeWire { + valid: bool, + value: CanonicalOperationValue, + diagnostics: Vec, +} + +impl Serialize for CanonicalOperationOutcome { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + if matches!(self.value, CanonicalOperationValue::LegacyRecoveredV03(_)) { + return Err(serde::ser::Error::custom( + "legacy-recovered v0.3 outcomes cannot be written to a v3 journal", + )); + } + CanonicalOperationOutcomeWire { + valid: self.valid, + value: self.value.clone(), + diagnostics: self.diagnostics.clone(), + } + .serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for CanonicalOperationOutcome { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let wire = CanonicalOperationOutcomeWire::deserialize(deserializer)?; + Self::checked(wire.valid, wire.value, wire.diagnostics).map_err(serde::de::Error::custom) + } } impl CanonicalOperationValue { @@ -140,8 +253,14 @@ impl CanonicalOperationValue { Self::Delete(_) => Some(OperationKind::Delete), Self::Rename(_) => Some(OperationKind::Rename), Self::Batch(_) => Some(OperationKind::Batch), - Self::TypePack(_) => Some(OperationKind::AssessTypePack), - Self::CollectionSetup(_) => Some(OperationKind::AssessCollectionSetup), + Self::AssessTypePack(_) => Some(OperationKind::AssessTypePack), + Self::ApplyTypePack(_) => Some(OperationKind::ApplyTypePack), + Self::AssessCollectionSetup(_) => Some(OperationKind::AssessCollectionSetup), + Self::ApplyCollectionSetup(_) => Some(OperationKind::ApplyCollectionSetup), + Self::ViewResourceMutation(value) | Self::TypeResourceMutation(value) => { + Some(value.operation) + } + Self::CursorRelease(_) => None, Self::WireOnly(WireOnlyOperationValue::Validation(_)) => Some(OperationKind::Validate), Self::WireOnly(WireOnlyOperationValue::ViewResource(_)) => { Some(OperationKind::ExecuteView) @@ -155,6 +274,143 @@ impl CanonicalOperationValue { } impl CanonicalOperationOutcome { + fn checked( + valid: bool, + value: CanonicalOperationValue, + diagnostics: Vec, + ) -> Result { + if matches!(value, CanonicalOperationValue::LegacyRecoveredV03(_)) { + return Err("legacy-recovered outcomes are accepted only by v0.3 transaction recovery"); + } + let missing_semantic_value = matches!( + value, + CanonicalOperationValue::Read(None) + | CanonicalOperationValue::Query(None) + | CanonicalOperationValue::Create(None) + | CanonicalOperationValue::Update(None) + | CanonicalOperationValue::Delete(None) + | CanonicalOperationValue::Rename(None) + | CanonicalOperationValue::Batch(None) + | CanonicalOperationValue::AssessTypePack(None) + | CanonicalOperationValue::ApplyTypePack(None) + | CanonicalOperationValue::AssessCollectionSetup(None) + | CanonicalOperationValue::ApplyCollectionSetup(None) + ); + if valid && missing_semantic_value { + return Err("a valid canonical operation outcome requires a semantic value"); + } + if !valid && matches!(value, CanonicalOperationValue::CursorRelease(_)) { + return Err("cursor release outcomes can only be completed"); + } + let resource_mismatch = matches!( + &value, + CanonicalOperationValue::ViewResourceMutation(value) + if !matches!(value.operation, OperationKind::CreateViewSource | OperationKind::UpdateViewSource | OperationKind::DeleteViewSource) + ) || matches!( + &value, + CanonicalOperationValue::TypeResourceMutation(value) + if !matches!(value.operation, OperationKind::CreateType | OperationKind::UpdateType) + ); + if resource_mismatch { + return Err("resource outcome operation does not match its canonical family"); + } + Ok(Self { + valid, + value, + diagnostics, + }) + } + + /// Construct a checked successful outcome. A semantic `None` and all + /// compatibility-only recovered values are rejected. + pub fn try_completed( + value: CanonicalOperationValue, + diagnostics: Vec, + ) -> Result { + if matches!(value, CanonicalOperationValue::WireOnly(_)) { + return Err(invalid_outcome( + "wire-only outcomes require a named compatibility constructor", + )); + } + Self::checked(true, value, diagnostics).map_err(invalid_outcome) + } + + /// Construct a checked rejected outcome. Partial or absent values and + /// diagnostics are retained; recovered legacy envelopes remain restricted. + pub fn try_rejected( + value: CanonicalOperationValue, + diagnostics: Vec, + ) -> Result { + if matches!(value, CanonicalOperationValue::WireOnly(_)) { + return Err(invalid_outcome( + "wire-only outcomes require a named compatibility constructor", + )); + } + Self::checked(false, value, diagnostics).map_err(invalid_outcome) + } + + /// Explicit derived runtime state without adding a persisted discriminator. + pub fn state(&self) -> CanonicalOutcomeState { + match (&self.value, self.valid) { + (CanonicalOperationValue::LegacyRecoveredV03(_), _) => { + CanonicalOutcomeState::LegacyRecoveredV03 + } + (CanonicalOperationValue::WireOnly(_), true) => { + CanonicalOutcomeState::WireCompatibilityCompleted + } + (CanonicalOperationValue::WireOnly(_), false) => { + CanonicalOutcomeState::WireCompatibilityRejected + } + (_, true) => CanonicalOutcomeState::Completed, + (_, false) => CanonicalOutcomeState::Rejected, + } + } + + /// Whether the operation completed semantically successfully. + pub fn is_valid(&self) -> bool { + self.valid + } + + /// The checked semantic value. Legacy v0.3 recovery is intentionally not + /// exposed as an ordinary semantic value. + pub fn value(&self) -> &CanonicalOperationValue { + &self.value + } + + /// Diagnostics retained independently of validity and partial values. + pub fn diagnostics(&self) -> &[Diagnostic] { + &self.diagnostics + } + + /// Explicit canonical family, including non-operation lifecycle state. + pub fn family(&self) -> CanonicalOperationFamily { + match &self.value { + CanonicalOperationValue::CursorRelease(_) => CanonicalOperationFamily::CursorLifecycle, + CanonicalOperationValue::LegacyRecoveredV03(_) => { + CanonicalOperationFamily::LegacyRecoveredV03 + } + value => CanonicalOperationFamily::Operation( + value + .kind() + .expect("all remaining canonical values have an operation"), + ), + } + } + + /// Operation family represented by this outcome, when semantically known. + pub fn operation_kind(&self) -> Option { + self.value.kind() + } + + /// Mutably access an existing typed query without permitting callers to + /// replace the checked operation family or remove its semantic value. + pub fn query_value_mut(&mut self) -> Option<&mut CanonicalQueryValue> { + match &mut self.value { + CanonicalOperationValue::Query(Some(query)) => Some(query), + _ => None, + } + } + pub(crate) fn read(outcome: crate::api::OperationOutcome) -> Self { Self { valid: true, @@ -178,14 +434,6 @@ impl CanonicalOperationOutcome { } } - pub(crate) fn legacy_recovered(result: OperationResult) -> Self { - Self { - valid: result.valid, - diagnostics: result.diagnostics.iter().cloned().map(Into::into).collect(), - value: CanonicalOperationValue::LegacyRecoveredV03(result), - } - } - pub(crate) fn record_mutation( operation: OperationKind, valid: bool, @@ -237,6 +485,63 @@ impl CanonicalOperationOutcome { } } + /// Construct the typed result of releasing a query cursor. This replaces + /// the former validation-JSON lifecycle envelope while preserving its v0.3 + /// projection. + pub fn cursor_release(outcome: CursorReleaseOutcome) -> Self { + Self { + valid: true, + value: CanonicalOperationValue::CursorRelease(outcome), + diagnostics: Vec::new(), + } + } + + /// Construct the validation compatibility family without exposing a raw + /// `WireOnlyOperationValue` constructor. + pub fn validation_wire(result: OperationResult) -> Self { + Self::wire_only(OperationKind::Validate, result) + } + + /// Construct one checked view-resource compatibility outcome. + pub fn view_wire( + operation: OperationKind, + result: OperationResult, + ) -> Result { + if !matches!( + operation, + OperationKind::ListViews + | OperationKind::ExecuteView + | OperationKind::ReadViewSource + | OperationKind::CreateViewSource + | OperationKind::UpdateViewSource + | OperationKind::DeleteViewSource + ) { + return Err(ProviderError::UnsupportedOperation( + "view_wire requires a view-resource operation".to_string(), + )); + } + Ok(Self::wire_only(operation, result)) + } + + /// Construct one checked type-resource compatibility outcome. + pub fn type_wire( + operation: OperationKind, + result: OperationResult, + ) -> Result { + if !matches!( + operation, + OperationKind::ListTypes + | OperationKind::ReadType + | OperationKind::CreateType + | OperationKind::UpdateType + ) { + return Err(ProviderError::UnsupportedOperation( + "type_wire requires a type-resource operation".to_string(), + )); + } + Ok(Self::wire_only(operation, result)) + } + pub(crate) fn wire_only(operation: OperationKind, result: OperationResult) -> Self { if matches!( operation, @@ -248,6 +553,17 @@ impl CanonicalOperationOutcome { return Self::definition(operation, result) .expect("definition APIs produce their closed typed result shape"); } + if matches!( + operation, + OperationKind::CreateViewSource + | OperationKind::UpdateViewSource + | OperationKind::DeleteViewSource + | OperationKind::CreateType + | OperationKind::UpdateType + ) { + return Self::recover_v03(operation, result) + .expect("resource mutation APIs produce their canonical resource shape"); + } let OperationResult { valid, result, @@ -285,24 +601,32 @@ impl CanonicalOperationOutcome { } = result; let empty = result.as_object().is_some_and(serde_json::Map::is_empty); let value = match operation { - OperationKind::AssessTypePack | OperationKind::ApplyTypePack => { - CanonicalOperationValue::TypePack( - (!empty).then(|| decode_value(result)).transpose()?, - ) - } - OperationKind::AssessCollectionSetup | OperationKind::ApplyCollectionSetup => { - CanonicalOperationValue::CollectionSetup( - (!empty) - .then(|| decode_value(result).map(Box::new)) - .transpose()?, - ) - } + OperationKind::AssessTypePack => CanonicalOperationValue::AssessTypePack( + (!empty).then(|| decode_value(result)).transpose()?, + ), + OperationKind::ApplyTypePack => CanonicalOperationValue::ApplyTypePack( + (!empty).then(|| decode_value(result)).transpose()?, + ), + OperationKind::AssessCollectionSetup => CanonicalOperationValue::AssessCollectionSetup( + (!empty) + .then(|| decode_value(result).map(Box::new)) + .transpose()?, + ), + OperationKind::ApplyCollectionSetup => CanonicalOperationValue::ApplyCollectionSetup( + (!empty) + .then(|| decode_value(result).map(Box::new)) + .transpose()?, + ), _ => unreachable!("definition constructor requires a definition operation"), }; - Ok(Self { + Self::checked( valid, value, - diagnostics: diagnostics.into_iter().map(Into::into).collect(), + diagnostics.into_iter().map(Into::into).collect(), + ) + .map_err(|message| ProviderError::Transaction { + code: "invalid_canonical_operation_outcome", + message: message.to_string(), }) } @@ -475,18 +799,26 @@ impl CanonicalOperationOutcome { } OperationKind::ListViews | OperationKind::ExecuteView - | OperationKind::ReadViewSource - | OperationKind::CreateViewSource + | OperationKind::ReadViewSource => { + CanonicalOperationValue::WireOnly(WireOnlyOperationValue::ViewResource(result)) + } + OperationKind::CreateViewSource | OperationKind::UpdateViewSource | OperationKind::DeleteViewSource => { - CanonicalOperationValue::WireOnly(WireOnlyOperationValue::ViewResource(result)) + CanonicalOperationValue::ViewResourceMutation(CanonicalResourceOperationValue { + operation, + result, + }) } - OperationKind::ListTypes - | OperationKind::ReadType - | OperationKind::CreateType - | OperationKind::UpdateType => { + OperationKind::ListTypes | OperationKind::ReadType => { CanonicalOperationValue::WireOnly(WireOnlyOperationValue::TypeResource(result)) } + OperationKind::CreateType | OperationKind::UpdateType => { + CanonicalOperationValue::TypeResourceMutation(CanonicalResourceOperationValue { + operation, + result, + }) + } OperationKind::AssessTypePack | OperationKind::ApplyTypePack => { return Self::definition( operation, @@ -508,13 +840,22 @@ impl CanonicalOperationOutcome { ); } }; - Ok(Self { - valid, - value, - diagnostics, + Self::checked(valid, value, diagnostics).map_err(|message| ProviderError::Transaction { + code: "invalid_canonical_operation_outcome", + message: message.to_string(), }) } + /// Checked public compatibility conversion with an explicit operation + /// discriminator. The discriminator selects the value decoder, so an + /// operation/value mismatch cannot be admitted. + pub fn try_from_v03( + operation: OperationKind, + result: OperationResult, + ) -> Result { + Self::recover_v03(operation, result) + } + /// Internal adapter for operation families whose canonical implementation /// still returns the v0.3 envelope. Hosted typed seams call this adapter at /// the implementation edge and never inspect `OperationResult.result`. @@ -572,14 +913,23 @@ impl CanonicalOperationOutcome { }) } CanonicalOperationValue::Batch(value) => encode_optional(value), - CanonicalOperationValue::TypePack(value) => encode_optional(value), - CanonicalOperationValue::CollectionSetup(value) => encode_optional(value), + CanonicalOperationValue::AssessTypePack(value) + | CanonicalOperationValue::ApplyTypePack(value) => encode_optional(value), + CanonicalOperationValue::AssessCollectionSetup(value) + | CanonicalOperationValue::ApplyCollectionSetup(value) => encode_optional(value), + CanonicalOperationValue::ViewResourceMutation(value) + | CanonicalOperationValue::TypeResourceMutation(value) => value.result.clone(), + CanonicalOperationValue::CursorRelease(outcome) => serde_json::json!({ + "released": outcome.released, + "results": [], + "meta": {"total_count": 0, "has_more": false} + }), CanonicalOperationValue::WireOnly(value) => match value { WireOnlyOperationValue::Validation(value) | WireOnlyOperationValue::ViewResource(value) | WireOnlyOperationValue::TypeResource(value) => value.clone(), }, - CanonicalOperationValue::LegacyRecoveredV03(result) => return result.clone(), + CanonicalOperationValue::LegacyRecoveredV03(result) => return result.0.clone(), }; OperationResult { valid: self.valid, @@ -627,6 +977,16 @@ impl CanonicalOperationOutcome { } } +impl TryFrom<(OperationKind, OperationResult)> for CanonicalOperationOutcome { + type Error = ProviderError; + + fn try_from( + (operation, result): (OperationKind, OperationResult), + ) -> Result { + Self::try_from_v03(operation, result) + } +} + fn decode_value(value: Value) -> Result { serde_json::from_value(value).map_err(|error| ProviderError::Transaction { code: "typed_outcome_decode_failed", @@ -634,6 +994,13 @@ fn decode_value(value: Value) -> Result ProviderError { + ProviderError::Transaction { + code: "invalid_canonical_operation_outcome", + message: message.to_string(), + } +} + fn invalid_shape(message: &str) -> ProviderError { ProviderError::Transaction { code: "typed_outcome_decode_failed", @@ -706,6 +1073,105 @@ mod tests { typed } + #[test] + fn serde_rejects_valid_outcomes_without_every_required_semantic_value() { + for operation in [ + "read", + "query", + "create", + "update", + "delete", + "rename", + "batch", + "assess_type_pack", + "apply_type_pack", + "assess_collection_setup", + "apply_collection_setup", + ] { + let fixture = json!({ + "valid": true, + "value": {"operation": operation, "value": null}, + "diagnostics": [] + }); + let error = + serde_json::from_value::(fixture).expect_err(operation); + assert!(error.to_string().contains("requires a semantic value")); + } + } + + #[test] + fn rejected_outcomes_retain_absent_or_partial_values_and_diagnostics() { + let absent = json!({ + "valid": false, + "value": {"operation": "read", "value": null}, + "diagnostics": [] + }); + assert!(!serde_json::from_value::(absent) + .unwrap() + .is_valid()); + + let partial = roundtrip( + OperationKind::Delete, + json!({"path": "a.md", "deleted": false}), + ); + let rejected = CanonicalOperationOutcome::try_rejected( + partial.value().clone(), + vec![Diagnostic { + severity: Severity::Error, + code: crate::api::DiagnosticCode::new("partial_rejection"), + message: "partial evidence retained".to_string(), + path: Some("a.md".to_string()), + field: None, + type_name: None, + schema_location: None, + details: Some(json!({"partial": true})), + }], + ) + .unwrap(); + let replay: CanonicalOperationOutcome = + serde_json::from_value(serde_json::to_value(&rejected).unwrap()).unwrap(); + assert_eq!(replay, rejected); + assert!(matches!( + replay.value(), + CanonicalOperationValue::Delete(Some(_)) + )); + } + + #[test] + fn explicit_operation_context_rejects_mismatched_wire_values() { + let wire = OperationResult { + valid: true, + result: record("a.md"), + diagnostics: Vec::new(), + }; + assert!(CanonicalOperationOutcome::try_from_v03(OperationKind::Query, wire).is_err()); + } + + #[test] + fn legacy_state_cannot_enter_checked_serde_or_new_v3_writes() { + let wire = OperationResult { + valid: false, + result: json!({}), + diagnostics: Vec::new(), + }; + let legacy = CanonicalOperationOutcome { + valid: wire.valid, + diagnostics: wire.diagnostics.iter().cloned().map(Into::into).collect(), + value: CanonicalOperationValue::LegacyRecoveredV03( + LegacyRecoveredV03Value::from_transaction_recovery(wire), + ), + }; + assert!(serde_json::to_value(&legacy).is_err()); + let fixture = json!({ + "valid": false, + "value": {"operation": "legacy_recovered_v03", "value": { + "valid": false, "result": {}, "diagnostics": [] + }}, + "diagnostics": [] + }); + assert!(serde_json::from_value::(fixture).is_err()); + } + #[test] fn migrated_runtime_variants_are_typed_and_wire_exact() { assert!(matches!( @@ -824,15 +1290,85 @@ mod tests { assert_eq!(rejection.result, rejected_operation.to_v03()); } + #[test] + fn cursor_release_is_typed_and_preserves_the_v03_projection() { + let outcome = + CanonicalOperationOutcome::cursor_release(CursorReleaseOutcome { released: true }); + assert_eq!(outcome.operation_kind(), None); + assert_eq!(outcome.family(), CanonicalOperationFamily::CursorLifecycle); + assert!(CanonicalOperationOutcome::try_rejected( + CanonicalOperationValue::CursorRelease(CursorReleaseOutcome { released: false }), + Vec::new() + ) + .is_err()); + let mut rejected = serde_json::to_value(&outcome).unwrap(); + rejected["valid"] = json!(false); + assert!(serde_json::from_value::(rejected).is_err()); + assert_eq!( + outcome.to_v03().result, + json!({ + "released": true, + "results": [], + "meta": {"total_count": 0, "has_more": false} + }) + ); + } + #[test] fn wire_only_families_are_explicit_and_exact() { + let wire = |family: &str| OperationResult { + valid: true, + result: json!({"family": family}), + diagnostics: Vec::new(), + }; + let outcomes = [ + CanonicalOperationOutcome::validation_wire(wire("validation")), + CanonicalOperationOutcome::view_wire(OperationKind::ListViews, wire("view_resource")) + .unwrap(), + CanonicalOperationOutcome::type_wire(OperationKind::ReadType, wire("type_resource")) + .unwrap(), + ]; + for typed in outcomes { + assert!(matches!( + typed.value(), + CanonicalOperationValue::WireOnly(_) + )); + let replay: CanonicalOperationOutcome = + serde_json::from_value(serde_json::to_value(&typed).unwrap()).unwrap(); + assert_eq!(replay, typed); + assert_eq!(replay.to_v03(), typed.to_v03()); + } + } + + #[test] + fn definition_discriminators_are_exact_for_assess_and_apply() { for kind in [ - OperationKind::Validate, - OperationKind::ListViews, - OperationKind::ReadType, + OperationKind::AssessTypePack, + OperationKind::ApplyTypePack, + OperationKind::AssessCollectionSetup, + OperationKind::ApplyCollectionSetup, ] { - let typed = roundtrip(kind, json!({"family": kind.as_str()})); - assert!(matches!(typed.value, CanonicalOperationValue::WireOnly(_))); + let wire = OperationResult { + valid: false, + result: json!({}), + diagnostics: Vec::new(), + }; + let outcome = CanonicalOperationOutcome::try_from_v03(kind, wire.clone()).unwrap(); + assert_eq!(outcome.operation_kind(), Some(kind)); + assert_eq!(outcome.to_v03(), wire); + let serialized = serde_json::to_value(&outcome).unwrap(); + assert_eq!(serialized["value"]["operation"], kind.as_str()); + let replay: CanonicalOperationOutcome = serde_json::from_value(serialized).unwrap(); + assert_eq!(replay.operation_kind(), Some(kind)); + assert_eq!(replay.to_v03(), wire); + } + for legacy in ["type_pack", "collection_setup"] { + let fixture = json!({ + "valid": false, + "value": {"operation": legacy, "value": null}, + "diagnostics": [] + }); + assert!(serde_json::from_value::(fixture).is_err()); } } @@ -844,7 +1380,7 @@ mod tests { ); assert!(matches!( pack.value, - CanonicalOperationValue::TypePack(Some(CanonicalTypePackValue { + CanonicalOperationValue::AssessTypePack(Some(CanonicalTypePackValue { applicable: true, .. })) @@ -865,7 +1401,7 @@ mod tests { ); assert!(matches!( setup.value, - CanonicalOperationValue::CollectionSetup(Some(value)) + CanonicalOperationValue::AssessCollectionSetup(Some(value)) if matches!(*value, CanonicalCollectionSetupValue::Assessment(_)) )); } diff --git a/src/runtime/catalog.rs b/src/runtime/catalog.rs index 6f34bd5..d0aa6a9 100644 --- a/src/runtime/catalog.rs +++ b/src/runtime/catalog.rs @@ -175,8 +175,8 @@ impl CompiledCatalog { code: "catalog_root_unavailable".to_string(), message: error.to_string(), })?; - let root_capability = - Collection::capability_for_root(empty_root.path()).map_err(|error| CatalogError { + let authority = crate::collection_root::CollectionRoot::acquire(empty_root.path()) + .map_err(|error| CatalogError { code: "catalog_root_unavailable".to_string(), message: error.to_string(), })?; @@ -187,7 +187,7 @@ impl CompiledCatalog { contracts, collection: Collection { root: PathBuf::new(), - root_capability, + authority, spec_profile: SpecProfile::V03, settings, config_extensions, diff --git a/src/runtime/context.rs b/src/runtime/context.rs index a67ec3b..1505d31 100644 --- a/src/runtime/context.rs +++ b/src/runtime/context.rs @@ -1,3 +1,6 @@ +use std::cell::RefCell; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; use std::time::{Duration, Instant}; use crate::{OperationCancellation, OperationStopReason}; @@ -7,66 +10,192 @@ use super::ProviderError; const LEGACY_DEADLINE: Duration = Duration::from_secs(24 * 60 * 60); const CANCELLATION_POLL: Duration = Duration::from_millis(10); -/// Absolute monotonic deadline for one provider/runtime operation. +/// Finite capture budgets applied to one operation. /// -/// Deadlines are process-local and are never serialized into durable state. +/// The defaults permit ordinary large collections while preventing accidental +/// unbounded authority reads. Entry/resource limits apply per capture; actual +/// reads and newly retained results are charged to operation-wide counters. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CaptureLimits { + pub max_entries: u64, + pub max_file_bytes: u64, + pub max_aggregate_bytes: u64, + pub max_depth: u64, + pub max_resource_entries: u64, + pub max_retained_bytes: u64, +} + +impl Default for CaptureLimits { + fn default() -> Self { + Self { + max_entries: 100_000, + max_file_bytes: 64 * 1024 * 1024, + // Eight full retained-capture equivalents cover the ordinary + // before/shadow/after phases of a 100k-entry mutation. + max_aggregate_bytes: 4 * 1024 * 1024 * 1024, + max_depth: 128, + max_resource_entries: 10_000, + max_retained_bytes: 4 * 1024 * 1024 * 1024, + } + } +} + +impl CaptureLimits { + pub fn builder() -> CaptureLimitsBuilder { + CaptureLimitsBuilder(Self::default()) + } +} + +/// Builder for [`CaptureLimits`]. All values are exact inclusive maxima. +#[derive(Clone, Copy, Debug)] +pub struct CaptureLimitsBuilder(CaptureLimits); + +impl CaptureLimitsBuilder { + pub fn max_entries(mut self, value: u64) -> Self { + self.0.max_entries = value; + self + } + pub fn max_file_bytes(mut self, value: u64) -> Self { + self.0.max_file_bytes = value; + self + } + pub fn max_aggregate_bytes(mut self, value: u64) -> Self { + self.0.max_aggregate_bytes = value; + self + } + pub fn max_depth(mut self, value: u64) -> Self { + self.0.max_depth = value; + self + } + pub fn max_resource_entries(mut self, value: u64) -> Self { + self.0.max_resource_entries = value; + self + } + pub fn max_retained_bytes(mut self, value: u64) -> Self { + self.0.max_retained_bytes = value; + self + } + pub fn build(self) -> CaptureLimits { + self.0 + } +} + +/// Stable budget dimension reported when capture stops without partial success. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CaptureLimitKind { + Entries, + FileBytes, + AggregateBytes, + Depth, + ResourceEntries, + RetainedBytes, + ArithmeticOverflow, +} + +impl CaptureLimitKind { + pub fn as_str(self) -> &'static str { + match self { + Self::Entries => "entries", + Self::FileBytes => "file_bytes", + Self::AggregateBytes => "aggregate_bytes", + Self::Depth => "depth", + Self::ResourceEntries => "resource_entries", + Self::RetainedBytes => "retained_bytes", + Self::ArithmeticOverflow => "arithmetic_overflow", + } + } +} + +/// Typed diagnostic for a capture budget violation. +#[derive(Clone, Debug, Eq, PartialEq, thiserror::Error)] +#[error("capture limit exceeded ({kind}): attempted {attempted}, limit {limit}", kind = .kind.as_str())] +pub struct CaptureLimitExceeded { + pub kind: CaptureLimitKind, + pub limit: u64, + pub attempted: u64, +} + +#[derive(Debug, Default)] +struct CaptureUsage { + aggregate: AtomicU64, + retained: AtomicU64, + exceeded: std::sync::Mutex>, +} + +/// Absolute monotonic deadline for one provider/runtime operation. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct OperationDeadline(Instant); impl OperationDeadline { - /// Construct a deadline at an exact monotonic instant. pub fn at(instant: Instant) -> Self { Self(instant) } - - /// Construct a deadline relative to the current monotonic instant. pub fn after(duration: Duration) -> Self { Self(Instant::now() + duration) } - - /// Return the underlying process-local monotonic instant. pub fn instant(self) -> Instant { self.0 } - - /// Return the remaining duration, saturating at zero after expiry. pub fn remaining(self) -> Duration { self.0.saturating_duration_since(Instant::now()) } - - /// Whether the deadline has elapsed. pub fn is_elapsed(self) -> bool { Instant::now() >= self.0 } } -/// Cooperative cancellation and deadline ownership for one runtime call. +/// Cooperative cancellation, deadline, and capture-budget ownership for one runtime call. #[derive(Clone, Debug)] pub struct OperationContext { cancellation: OperationCancellation, deadline: OperationDeadline, + limits: CaptureLimits, + usage: Arc, +} + +thread_local! { + static ACTIVE_CONTEXTS: RefCell> = const { RefCell::new(Vec::new()) }; +} + +struct ActiveContextGuard; +impl Drop for ActiveContextGuard { + fn drop(&mut self) { + ACTIVE_CONTEXTS.with(|contexts| { + contexts.borrow_mut().pop(); + }); + } } impl OperationContext { - /// Bind an existing caller cancellation token to an absolute deadline. + /// Bind a caller token to a deadline and the documented default budgets. pub fn new(cancellation: &OperationCancellation, deadline: OperationDeadline) -> Self { + Self::with_capture_limits(cancellation, deadline, CaptureLimits::default()) + } + + /// Bind a caller token to a deadline and explicit capture budgets. + pub fn with_capture_limits( + cancellation: &OperationCancellation, + deadline: OperationDeadline, + limits: CaptureLimits, + ) -> Self { Self { cancellation: cancellation.with_deadline(deadline.instant()), deadline, + limits, + usage: Arc::new(CaptureUsage::default()), } } - /// Return the cooperative token used by long-running collection work. pub fn cancellation(&self) -> &OperationCancellation { &self.cancellation } - - /// Return the operation's absolute monotonic deadline. pub fn deadline(&self) -> OperationDeadline { self.deadline } + pub fn capture_limits(&self) -> CaptureLimits { + self.limits + } - /// Fail with the typed reason when cancellation or deadline has won. pub fn check(&self) -> Result<(), ProviderError> { match self.cancellation.stop_reason() { Some(OperationStopReason::Cancelled) => Err(ProviderError::OperationCancelled), @@ -75,17 +204,154 @@ impl OperationContext { } } + pub(crate) fn check_depth(&self, depth: u64) -> Result<(), ProviderError> { + if depth > self.limits.max_depth { + return Err(limit(CaptureLimitKind::Depth, self.limits.max_depth, depth)); + } + Ok(()) + } + + pub(crate) fn check_file_bytes(&self, bytes: u64) -> Result<(), ProviderError> { + if bytes > self.limits.max_file_bytes { + return Err(limit( + CaptureLimitKind::FileBytes, + self.limits.max_file_bytes, + bytes, + )); + } + Ok(()) + } + + /// Check the number of entries retained by one capture. Entry and resource + /// ceilings reset for each capture; byte/work counters are operation-wide. + pub(crate) fn check_entries(&self, entries: u64) -> Result<(), ProviderError> { + self.record_limit(check_limit( + entries, + self.limits.max_entries, + CaptureLimitKind::Entries, + )) + } + + pub(crate) fn check_resource_entries(&self, entries: u64) -> Result<(), ProviderError> { + self.record_limit(check_limit( + entries, + self.limits.max_resource_entries, + CaptureLimitKind::ResourceEntries, + )) + } + + pub(crate) fn charge_read(&self, bytes: u64) -> Result<(), ProviderError> { + self.record_limit(charge( + &self.usage.aggregate, + bytes, + self.limits.max_aggregate_bytes, + CaptureLimitKind::AggregateBytes, + )) + } + + pub(crate) fn charge_retained(&self, bytes: u64) -> Result<(), ProviderError> { + self.record_limit(charge( + &self.usage.retained, + bytes, + self.limits.max_retained_bytes, + CaptureLimitKind::RetainedBytes, + )) + } + + pub(crate) fn capture_limit_error(&self) -> Option { + self.usage + .exceeded + .lock() + .ok() + .and_then(|error| error.clone()) + .map(ProviderError::CaptureLimitExceeded) + } + + fn record_limit(&self, result: Result<(), ProviderError>) -> Result<(), ProviderError> { + if let Err(ProviderError::CaptureLimitExceeded(error)) = &result { + if let Ok(mut exceeded) = self.usage.exceeded.lock() { + exceeded.get_or_insert_with(|| error.clone()); + } + } + result + } + + /// Run synchronous canonical work with this caller context available to + /// legacy internal adapters. Nested scopes retain the outer shared budget. + pub(crate) fn scope(&self, operation: impl FnOnce() -> T) -> T { + ACTIVE_CONTEXTS.with(|contexts| contexts.borrow_mut().push(self.clone())); + let _guard = ActiveContextGuard; + operation() + } + + pub(crate) fn current() -> Option { + ACTIVE_CONTEXTS.with(|contexts| contexts.borrow().last().cloned()) + } + + pub(crate) fn current_or_legacy() -> Self { + Self::current().unwrap_or_else(Self::internal) + } + pub(crate) fn next_wait(&self) -> Result { self.check()?; Ok(self.deadline.remaining().min(CANCELLATION_POLL)) } - pub(crate) fn legacy() -> Self { + /// Bounded context for internal lifecycle work that has no external caller. + pub(crate) fn internal() -> Self { Self::new( &OperationCancellation::new(), OperationDeadline::after(LEGACY_DEADLINE), ) } + + /// Context retained only for test/support inventory. + #[cfg(test)] + pub(crate) fn legacy() -> Self { + Self::internal() + } +} + +fn limit(kind: CaptureLimitKind, limit: u64, attempted: u64) -> ProviderError { + ProviderError::CaptureLimitExceeded(CaptureLimitExceeded { + kind, + limit, + attempted, + }) +} + +fn check_limit(attempted: u64, maximum: u64, kind: CaptureLimitKind) -> Result<(), ProviderError> { + if attempted > maximum { + Err(limit(kind, maximum, attempted)) + } else { + Ok(()) + } +} + +fn charge( + counter: &AtomicU64, + amount: u64, + maximum: u64, + kind: CaptureLimitKind, +) -> Result<(), ProviderError> { + let mut current = counter.load(Ordering::Relaxed); + loop { + let attempted = current + .checked_add(amount) + .ok_or_else(|| limit(CaptureLimitKind::ArithmeticOverflow, u64::MAX, u64::MAX))?; + if attempted > maximum { + return Err(limit(kind, maximum, attempted)); + } + match counter.compare_exchange_weak( + current, + attempted, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => return Ok(()), + Err(actual) => current = actual, + } + } } #[cfg(test)] @@ -103,7 +369,6 @@ mod tests { expired.check(), Err(ProviderError::OperationDeadline) )); - let cancellation = OperationCancellation::new(); let active = OperationContext::new( &cancellation, @@ -115,4 +380,24 @@ mod tests { Err(ProviderError::OperationCancelled) )); } + + #[test] + fn capture_boundaries_are_inclusive_and_shared_by_clones() { + let limits = CaptureLimits::builder() + .max_entries(1) + .max_aggregate_bytes(3) + .build(); + let context = OperationContext::with_capture_limits( + &OperationCancellation::new(), + OperationDeadline::after(Duration::from_secs(1)), + limits, + ); + context.check_entries(1).unwrap(); + context.clone().check_entries(1).unwrap(); + context.charge_read(3).unwrap(); + assert!(matches!( + context.charge_read(1), + Err(ProviderError::CaptureLimitExceeded(_)) + )); + } } diff --git a/src/runtime/cursor.rs b/src/runtime/cursor.rs index 2b73d66..38fc096 100644 --- a/src/runtime/cursor.rs +++ b/src/runtime/cursor.rs @@ -4,8 +4,8 @@ use std::time::{Duration, Instant}; use sha2::{Digest, Sha256}; use super::{ - CanonicalOperationOutcome, CanonicalOperationValue, ChangeSet, CollectionGeneration, - ExecutionOutcome, OperationContext, ProviderError, ReadCursor, ReadPage, + CanonicalOperationOutcome, ChangeSet, CollectionGeneration, ExecutionOutcome, OperationContext, + ProviderError, ReadCursor, ReadPage, }; const MAX_ACTIVE_CURSORS: usize = 32; @@ -53,14 +53,20 @@ impl CursorStore { let page_items = page_items .unwrap_or(DEFAULT_PAGE_ITEMS) .clamp(1, MAX_PAGE_ITEMS); - let CanonicalOperationValue::Query(Some(query)) = &mut outcome.operation.value else { + let Some(query) = outcome.operation.query_value_mut() else { return Ok(ReadPage { outcome, next: None, }); }; let results = std::mem::take(&mut query.records); + let retained_u64 = if results.is_empty() { + 0 + } else { + measured_json_bytes(&results, context.capture_limits().max_retained_bytes)? + }; if results.len() <= page_items { + context.charge_retained(retained_u64)?; query.records = results; query.has_more = false; set_has_more(&mut query.meta, false); @@ -69,23 +75,33 @@ impl CursorStore { next: None, }); } - let retained_bytes = serde_json::to_vec(&results) - .map_err(|error| ProviderError::Transaction { - code: "cursor_serialization_failed", - message: error.to_string(), - })? - .len(); + let retained_bytes = + usize::try_from(retained_u64).map_err(|_| crate::runtime::CaptureLimitExceeded { + kind: crate::runtime::CaptureLimitKind::ArithmeticOverflow, + limit: usize::MAX as u64, + attempted: retained_u64, + })?; + let store_bytes = self + .retained_bytes + .checked_add(retained_bytes) + .ok_or(ProviderError::CursorCapacityExhausted)?; if self.entries.len() >= MAX_ACTIVE_CURSORS || retained_bytes > MAX_CURSOR_BYTES - || self.retained_bytes.saturating_add(retained_bytes) > MAX_CURSOR_BYTES + || store_bytes > MAX_CURSOR_BYTES { return Err(ProviderError::CursorCapacityExhausted); } + // Reserve while holding the store lock, before charging the operation. + // A failed reservation or capacity check therefore consumes no meter. + self.entries + .try_reserve(1) + .map_err(|_| ProviderError::CursorCapacityExhausted)?; + context.charge_retained(retained_u64)?; let id = uuid::Uuid::new_v4().to_string(); let generation = outcome.generation.clone(); let now = Instant::now(); let template = outcome.operation.clone(); - self.retained_bytes += retained_bytes; + self.retained_bytes = store_bytes; self.entries.insert( id.clone(), PinnedRead { @@ -119,7 +135,7 @@ impl CursorStore { .saturating_add(pinned.page_items) .min(pinned.results.len()); let mut operation = pinned.template.clone(); - let CanonicalOperationValue::Query(Some(query)) = &mut operation.value else { + let Some(query) = operation.query_value_mut() else { return Err(ProviderError::Transaction { code: "cursor_state_invalid", message: "pinned read no longer contains a typed query".to_string(), @@ -138,12 +154,15 @@ impl CursorStore { }) } - pub(crate) fn release(&mut self, cursor: ReadCursor) -> Result<(), ProviderError> { + pub(crate) fn release(&mut self, cursor: ReadCursor) -> Result { let (id, _) = self.authenticate(&cursor)?; - if let Some(removed) = self.entries.remove(&id) { + let released = if let Some(removed) = self.entries.remove(&id) { self.retained_bytes = self.retained_bytes.saturating_sub(removed.retained_bytes); - } - Ok(()) + true + } else { + false + }; + Ok(released) } pub(crate) fn measurements(&mut self) -> (usize, usize) { @@ -203,6 +222,55 @@ impl CursorStore { } } +pub(crate) fn measured_json_bytes( + value: &T, + limit: u64, +) -> Result { + struct Counter { + bytes: u64, + limit: u64, + exceeded: Option, + } + impl std::io::Write for Counter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + let attempted = self + .bytes + .checked_add(bytes.len() as u64) + .ok_or_else(|| std::io::Error::other("capture arithmetic overflow"))?; + if attempted > self.limit { + self.exceeded = Some(crate::runtime::CaptureLimitExceeded { + kind: crate::runtime::CaptureLimitKind::RetainedBytes, + limit: self.limit, + attempted, + }); + return Err(std::io::Error::other( + "capture retained-byte limit exceeded", + )); + } + self.bytes = attempted; + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let mut counter = Counter { + bytes: 0, + limit, + exceeded: None, + }; + if let Err(error) = serde_json::to_writer(&mut counter, value) { + if let Some(exceeded) = counter.exceeded { + return Err(exceeded.into()); + } + return Err(ProviderError::Transaction { + code: "cursor_serialization_failed", + message: error.to_string(), + }); + } + Ok(counter.bytes) +} + fn cursor_signing_key() -> [u8; 32] { let mut key = [0_u8; 32]; key[..16].copy_from_slice(uuid::Uuid::new_v4().as_bytes()); @@ -245,3 +313,52 @@ fn set_has_more(meta: &mut serde_json::Value, has_more: bool) { meta.insert("has_more".to_string(), serde_json::Value::Bool(has_more)); } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::api::{ProjectedValue, QueryMetadata}; + use crate::runtime::{CanonicalOperationValue, CanonicalQueryValue, OperationDeadline}; + use serde_json::json; + + #[test] + fn failed_store_capacity_does_not_consume_operation_meter() { + let generation = CollectionGeneration::initial(); + let operation = CanonicalOperationOutcome { + valid: true, + value: CanonicalOperationValue::Query(Some(CanonicalQueryValue { + records: vec![ + ProjectedValue::new(json!({"id": 1})), + ProjectedValue::new(json!({"id": 2})), + ], + total_count: Some(2), + has_more: false, + meta: QueryMetadata::new(json!({})), + embedded_diagnostics: Vec::new(), + })), + diagnostics: Vec::new(), + }; + let outcome = + ExecutionOutcome::new(operation, generation.clone(), ChangeSet::None, None, None); + let records = match outcome.operation.value() { + CanonicalOperationValue::Query(Some(query)) => &query.records, + _ => unreachable!(), + }; + let retained = measured_json_bytes(records, u64::MAX).unwrap(); + let context = OperationContext::with_capture_limits( + &crate::OperationCancellation::new(), + OperationDeadline::after(Duration::from_secs(1)), + crate::runtime::CaptureLimits::builder() + .max_retained_bytes(retained) + .build(), + ); + let mut store = CursorStore::new(generation.runtime_epoch()); + store.retained_bytes = MAX_CURSOR_BYTES; + + assert!(matches!( + store.open(outcome, Some(1), &context), + Err(ProviderError::CursorCapacityExhausted) + )); + context.charge_retained(retained).unwrap(); + } +} diff --git a/src/runtime/feed.rs b/src/runtime/feed.rs index 3acab93..ab091a4 100644 --- a/src/runtime/feed.rs +++ b/src/runtime/feed.rs @@ -1,6 +1,5 @@ -use std::fs; use std::num::NonZeroUsize; -use std::path::{Path, PathBuf}; +use std::path::Path; use serde::{Deserialize, Serialize}; @@ -16,8 +15,6 @@ use crate::transactions::RuntimeResolution; use crate::Collection; const FEED_VERSION: u32 = 1; -const RUNTIME_DIRECTORY: &str = ".mdbase/runtime"; -const FEED_FILE: &str = "change-feed.json"; const MAX_UNACKED_EVENTS: usize = 100_000; const MAX_FEED_BYTES: u64 = 64 * 1024 * 1024; const MAX_PAGE_ITEMS: usize = 256; @@ -83,7 +80,7 @@ pub(crate) fn reconcile( let mut generation = initial_generation; let resolutions = crate::transactions::list_unacked_runtime_events(collection, context) .map_err(super::filesystem::transaction_error)?; - let mut changed = !feed_path(collection).exists(); + let mut changed = !collection.held_root().exists_file(feed_relative_path()); for resolution in resolutions { context.check()?; let RuntimeResolution::Committed { @@ -146,8 +143,8 @@ pub(crate) fn ensure_capacity(collection: &Collection) -> Result<(), ProviderErr if journal.events.len() >= MAX_UNACKED_EVENTS.saturating_sub(1) { return Err(ProviderError::ChangeFeedCapacityExhausted); } - match fs::metadata(feed_path(collection)) { - Ok(metadata) if metadata.len() >= MAX_FEED_BYTES => { + match collection.held_root().open_file(feed_relative_path()) { + Ok(file) if file.metadata().map_err(feed_io)?.len() >= MAX_FEED_BYTES => { Err(ProviderError::ChangeFeedCapacityExhausted) } Ok(_) => Ok(()), @@ -321,42 +318,14 @@ pub(crate) fn commit_baseline( /// Discard copied consumer history after durable transactions have been /// recovered and detached from their original host claims. pub(crate) fn reset_for_fork(collection: &Collection) -> Result<(), ProviderError> { - let path = feed_path(collection); - match fs::symlink_metadata(&path) { - Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_file() => { - return Err(ProviderError::Transaction { - code: "change_feed_invalid", - message: "runtime change feed is not a regular file".to_string(), - }); - } - Ok(_) => fs::remove_file(&path).map_err(|error| ProviderError::Transaction { + match collection.held_root().remove_file(feed_relative_path()) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(ProviderError::Transaction { code: "change_feed_reset_failed", message: error.to_string(), - })?, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), - Err(error) => { - return Err(ProviderError::Transaction { - code: "change_feed_reset_failed", - message: error.to_string(), - }); - } + }), } - sync_feed_directory(path.parent().expect("feed path has a parent")) -} - -#[cfg(not(windows))] -fn sync_feed_directory(path: &Path) -> Result<(), ProviderError> { - std::fs::File::open(path) - .and_then(|directory| directory.sync_all()) - .map_err(|error| ProviderError::Transaction { - code: "change_feed_reset_failed", - message: error.to_string(), - }) -} - -#[cfg(windows)] -fn sync_feed_directory(_path: &Path) -> Result<(), ProviderError> { - Ok(()) } pub(crate) fn read( @@ -583,8 +552,7 @@ fn validate(journal: &FeedJournal) -> Result<(), ProviderError> { } fn read_or_default(collection: &Collection) -> Result { - let path = feed_path(collection); - match fs::read(&path) { + match collection.held_root().read(feed_relative_path()) { Ok(bytes) => serde_json::from_slice(&bytes).map_err(|error| corrupt(&error.to_string())), Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(FeedJournal { version: FEED_VERSION, @@ -603,18 +571,18 @@ fn read_or_default(collection: &Collection) -> Result Result<(), ProviderError> { validate(journal)?; - let path = feed_path(collection); - let parent = path.parent().expect("feed path has a parent"); - fs::create_dir_all(parent).map_err(feed_io)?; let bytes = serde_json::to_vec_pretty(journal).map_err(|error| corrupt(&error.to_string()))?; if bytes.len() as u64 > MAX_FEED_BYTES { return Err(ProviderError::ChangeFeedCapacityExhausted); } - crate::operations::atomic_write(&path, &bytes).map_err(feed_io) + collection + .held_root() + .atomic_write(feed_relative_path(), &bytes) + .map_err(feed_io) } -fn feed_path(collection: &Collection) -> PathBuf { - collection.root.join(RUNTIME_DIRECTORY).join(FEED_FILE) +fn feed_relative_path() -> &'static Path { + Path::new(".mdbase/runtime/change-feed.json") } fn corrupt(message: &str) -> ProviderError { @@ -630,3 +598,27 @@ fn feed_io(error: std::io::Error) -> ProviderError { message: error.to_string(), } } + +#[cfg(all(test, unix))] +mod authority_tests { + use super::*; + + #[test] + fn replacement_root_never_receives_feed_publication() { + let parent = tempfile::tempdir().unwrap(); + let root = parent.path().join("collection"); + std::fs::create_dir(&root).unwrap(); + std::fs::write(root.join("mdbase.yaml"), "spec_version: 0.3.0\n").unwrap(); + let collection = Collection::open(&root).unwrap(); + let held = parent.path().join("held"); + std::fs::rename(&root, &held).unwrap(); + std::fs::create_dir(&root).unwrap(); + std::fs::write(root.join("mdbase.yaml"), "spec_version: 0.3.0\n").unwrap(); + + let journal = read_or_default(&collection).unwrap(); + persist(&collection, &journal).unwrap(); + + assert!(held.join(".mdbase/runtime/change-feed.json").is_file()); + assert!(!root.join(".mdbase").exists()); + } +} diff --git a/src/runtime/filesystem.rs b/src/runtime/filesystem.rs index cc84079..48b44ed 100644 --- a/src/runtime/filesystem.rs +++ b/src/runtime/filesystem.rs @@ -11,13 +11,13 @@ use sha2::{Digest, Sha256}; use super::diff::canonical_changes; use super::observer::NoopObserver; use super::{ - CancelOutcome, CanonicalOperationOutcome, ChangeBatch, ChangeEventIdentity, ChangeFeed, - ChangeFeedBaseline, ChangeFeedOwnerId, ChangeFeedTransfer, ChangeFeedTransferId, - ChangeFeedTransferIntent, ChangePage, ChangePageCursor, ChangeSet, ChangeWatermark, - CollectionGeneration, CommitAttempt, CommitId, CommitRejection, DurableCommitState, - ExecutionOutcome, FilesystemProvider, HostClaimId, ObserverOptions, OperationContext, - OperationRequest, PreparationOutcome, PreparedMutation, ProviderError, ReadCursor, ReadPage, - RuntimeChangeEvent, RuntimeChangeEventPage, RuntimeObserver, + CancelOutcome, CanonicalOperationOutcome, CanonicalOperationValue, ChangeBatch, + ChangeEventIdentity, ChangeFeed, ChangeFeedBaseline, ChangeFeedOwnerId, ChangeFeedTransfer, + ChangeFeedTransferId, ChangeFeedTransferIntent, ChangePage, ChangePageCursor, ChangeSet, + ChangeWatermark, CollectionGeneration, CommitAttempt, CommitId, CommitRejection, + DurableCommitState, ExecutionOutcome, FilesystemProvider, HostClaimId, ObserverOptions, + OperationContext, OperationRequest, PreparationOutcome, PreparedMutation, ProviderError, + ReadCursor, ReadPage, RuntimeChangeEvent, RuntimeChangeEventPage, RuntimeObserver, }; use crate::transactions::{ self, RuntimeCommitAttempt, RuntimePrepareOutcome, RuntimeResolution, RuntimeSettlement, @@ -86,6 +86,15 @@ impl FilesystemRuntime { ) } + /// Count valid version-2 runtime journals without exposing journal contents. + /// A zero result is the operator-facing gate for removing the decoder in 0.5.0. + pub fn legacy_journal_inventory( + &self, + context: &OperationContext, + ) -> Result { + self.provider.legacy_journal_inventory(context) + } + pub fn open_observed( root: impl AsRef, debounce: Duration, @@ -99,12 +108,12 @@ impl FilesystemRuntime { )?); let initial_generation = CollectionGeneration::initial(); let reconciled = provider.with_collection_boundary_context( - &OperationContext::legacy(), + &OperationContext::internal(), |collection| { super::feed::reconcile( collection, initial_generation.clone(), - &OperationContext::legacy(), + &OperationContext::internal(), ) }, )?; @@ -162,7 +171,7 @@ impl FilesystemRuntime { } pub fn execute(&self, request: &OperationRequest) -> Result { - self.execute_with_context(request, &OperationContext::legacy()) + self.execute_with_context(request, &OperationContext::internal()) } /// Compatibility wrapper which serializes the typed runtime result exactly @@ -180,7 +189,7 @@ impl FilesystemRuntime { &self, request: &OperationRequest, ) -> Result { - self.execute_typed_with_context(request, &OperationContext::legacy()) + self.execute_typed_with_context(request, &OperationContext::internal()) } /// Execute the compatibility-decoded request and return the closed typed @@ -230,6 +239,15 @@ impl FilesystemRuntime { &self, request: &OperationRequest, context: &OperationContext, + ) -> Result { + self.read_with_result_charge(request, context, true) + } + + fn read_with_result_charge( + &self, + request: &OperationRequest, + context: &OperationContext, + charge_query_result: bool, ) -> Result { if request.operation.is_mutation() { return Err(ProviderError::UnsupportedOperation( @@ -243,7 +261,7 @@ impl FilesystemRuntime { super::OperationKind::Read | super::OperationKind::Query => self .provider .with_collection_read_context(context, |collection| { - Ok::<_, ProviderError>(execute_typed_read_operation(collection, request)) + execute_typed_read_operation(collection, request, context) })?, _ => { let result = self @@ -253,6 +271,9 @@ impl FilesystemRuntime { } }; let generation = self.current_generation()?; + if charge_query_result { + charge_returned_query(&operation, context)?; + } Ok(ExecutionOutcome::new( operation, generation, @@ -282,7 +303,7 @@ impl FilesystemRuntime { if let Some(input) = expanded.input.as_object_mut() { input.remove("limit"); } - let outcome = self.read(&expanded, context)?; + let outcome = self.read_with_result_charge(&expanded, context, false)?; self.cursor_lock(context)? .open(outcome, page_items, context) } @@ -301,10 +322,11 @@ impl FilesystemRuntime { &self, cursor: ReadCursor, context: &OperationContext, - ) -> Result<(), ProviderError> { + ) -> Result { + context.check()?; + let released = self.cursor_lock(context)?.release(cursor)?; context.check()?; - self.cursor_lock(context)?.release(cursor)?; - context.check() + Ok(super::CursorReleaseOutcome { released }) } /// Validate and durably stage one exact mutation under an opaque host claim. @@ -560,7 +582,7 @@ impl FilesystemRuntime { if !plan.needs_commit { return Ok(plan.baseline); } - let settlement = OperationContext::legacy(); + let settlement = OperationContext::internal(); for commit_id in plan.commits { transactions::ack_runtime_change_event(collection, &commit_id, &settlement) .map_err(transaction_error)?; @@ -595,7 +617,7 @@ impl FilesystemRuntime { // Once acknowledgement starts it owns settlement. Marking each // transaction first is crash-safe because the durable feed still // retains the event until its own final atomic acknowledgement. - let settlement = OperationContext::legacy(); + let settlement = OperationContext::internal(); for commit_id in commits { transactions::ack_runtime_change_event(collection, &commit_id, &settlement) .map_err(transaction_error)?; @@ -693,7 +715,7 @@ impl FilesystemRuntime { /// Complete a full watcher comparison before accepting benchmark or host /// traffic. Normal mutations use the incremental synchronization path. pub fn synchronize(&self) -> Result<(), ProviderError> { - self.synchronize_with_context(&OperationContext::legacy()) + self.synchronize_with_context(&OperationContext::internal()) } /// Complete a full comparison within an explicit operation boundary. @@ -735,7 +757,7 @@ impl FilesystemRuntime { #[cfg(test)] pub(crate) fn synchronize_paths_for_test(&self, paths: &[&str]) -> Result<(), ProviderError> { - self.synchronize_reconciliation(Some(paths), &OperationContext::legacy()) + self.synchronize_reconciliation(Some(paths), &OperationContext::internal()) } #[cfg(test)] @@ -976,7 +998,7 @@ impl FilesystemRuntime { { // Committing is already durable. Internal settlement ownership can // no longer be cancelled by the application deadline. - let mut active = self.settlement_lock(&OperationContext::legacy())?; + let mut active = self.settlement_lock(&OperationContext::internal())?; if active.is_some() { return Ok(CommitAttempt::SettlementPending { commit_id }); } @@ -1069,8 +1091,9 @@ impl FilesystemRuntime { &self, resolution: &RuntimeResolution, ) -> Result { - self.provider - .with_collection_boundary_context(&OperationContext::legacy(), |collection| { + self.provider.with_collection_boundary_context( + &OperationContext::internal(), + |collection| { finish_resolution_inside( collection, self.provider.as_ref(), @@ -1079,7 +1102,8 @@ impl FilesystemRuntime { self.order.as_ref(), resolution, ) - }) + }, + ) } fn cursor_lock( @@ -1126,7 +1150,7 @@ fn finish_owned_settlement( order: &Arc>, owner: &mut RuntimeSettlement, ) -> Result { - provider.with_collection_boundary_context(&OperationContext::legacy(), |collection| { + provider.with_collection_boundary_context(&OperationContext::internal(), |collection| { let resolution = transactions::settle_runtime_commit(collection, owner).map_err(transaction_error)?; finish_resolution_inside( @@ -1246,12 +1270,13 @@ fn synchronize_known_shared( fn execute_typed_read_operation( collection: &crate::Collection, request: &OperationRequest, -) -> CanonicalOperationOutcome { + context: &OperationContext, +) -> Result { let typed = match collection.typed() { Ok(typed) => typed, - Err(error) => return CanonicalOperationOutcome::failure(request.operation, error), + Err(error) => return Ok(CanonicalOperationOutcome::failure(request.operation, error)), }; - match request.operation { + let operation = match request.operation { super::OperationKind::Read => { match serde_json::from_value::(request.input.clone()) { Ok(request) => typed @@ -1271,7 +1296,7 @@ fn execute_typed_read_operation( super::OperationKind::Query => { match crate::api::QueryRequest::decode_wire(request.input.clone()) { Ok(request) => typed - .query_runtime(request) + .query_runtime(request, context.cancellation()) .map(CanonicalOperationOutcome::query) .unwrap_or_else(|error| { CanonicalOperationOutcome::failure(super::OperationKind::Query, error) @@ -1285,7 +1310,29 @@ fn execute_typed_read_operation( } } _ => unreachable!("only migrated read operations use the typed read executor"), + }; + if let Some(error) = context.capture_limit_error() { + return Err(error); + } + context.check()?; + Ok(operation) +} + +fn charge_returned_query( + operation: &CanonicalOperationOutcome, + context: &OperationContext, +) -> Result<(), ProviderError> { + let CanonicalOperationValue::Query(Some(query)) = &operation.value else { + return Ok(()); + }; + if query.records.is_empty() { + return Ok(()); } + let bytes = super::cursor::measured_json_bytes( + &query.records, + context.capture_limits().max_retained_bytes, + )?; + context.charge_retained(bytes) } fn mutation_digest(request: &OperationRequest) -> Result { diff --git a/src/runtime/hosted_base.rs b/src/runtime/hosted_base.rs index 9f7b55a..f47c8ba 100644 --- a/src/runtime/hosted_base.rs +++ b/src/runtime/hosted_base.rs @@ -1143,6 +1143,12 @@ mod tests { target_record_id: None, target_path: None, ambiguous_paths: Vec::new(), + reason: None, + selected_lookup: None, + candidate_count: 0, + candidate_digest: None, + alternatives: Vec::new(), + alternative_candidates: Vec::new(), }) .collect(), body_tags: prepared.structure.body_tags.clone(), @@ -1172,6 +1178,25 @@ mod tests { target_record_id: Some("project:mobile".to_string()), target_path: Some(target_path.to_string()), ambiguous_paths: Vec::new(), + reason: Some(super::super::ResolutionReason::ExactPath), + selected_lookup: Some(super::super::ResolutionLookupKey { + priority: 0, + kind: super::super::RecordResolutionKeyKind::Path, + value: target_path.to_string(), + }), + candidate_count: 1, + candidate_digest: Some(format!( + "sha256:{:x}", + Sha256::digest( + serde_jcs::to_vec(&( + super::super::RecordResolutionKeyKind::Path, + vec![("project:mobile", target_path)], + )) + .unwrap() + ) + )), + alternatives: Vec::new(), + alternative_candidates: Vec::new(), }) .collect(), body_tags: prepared.structure.body_tags.clone(), diff --git a/src/runtime/hosted_mutation.rs b/src/runtime/hosted_mutation.rs index 4a6d92d..a799d23 100644 --- a/src/runtime/hosted_mutation.rs +++ b/src/runtime/hosted_mutation.rs @@ -167,7 +167,7 @@ impl CompiledCatalog { .map_err(|error| mutation_error(error.code, error.message))?; let collection = Collection { root: directory.path().to_path_buf(), - root_capability: Collection::capability_for_root(directory.path()) + authority: crate::collection_root::CollectionRoot::acquire(directory.path()) .map_err(stage_io_error)?, spec_profile: SpecProfile::V03, settings: self.collection.settings.clone(), diff --git a/src/runtime/hosted_resource.rs b/src/runtime/hosted_resource.rs index e12bca8..368796d 100644 --- a/src/runtime/hosted_resource.rs +++ b/src/runtime/hosted_resource.rs @@ -163,7 +163,7 @@ impl CompiledCatalog { type_plans: self.collection.type_plans.clone(), type_warnings: self.collection.type_warnings.clone(), data_contracts, - root_capability: Collection::capability_for_root(directory.path()) + authority: crate::collection_root::CollectionRoot::acquire(directory.path()) .map_err(resource_stage_error)?, }; let operations = collection @@ -464,7 +464,7 @@ impl CompiledCatalog { type_plans: self.collection.type_plans.clone(), type_warnings: self.collection.type_warnings.clone(), data_contracts, - root_capability: Collection::capability_for_root(directory.path()) + authority: crate::collection_root::CollectionRoot::acquire(directory.path()) .map_err(resource_stage_error)?, }; Ok((directory, collection)) diff --git a/src/runtime/hosted_validation.rs b/src/runtime/hosted_validation.rs index 8ff87a8..0095838 100644 --- a/src/runtime/hosted_validation.rs +++ b/src/runtime/hosted_validation.rs @@ -221,7 +221,7 @@ impl CompiledCatalog { type_plans: self.collection.type_plans.clone(), type_warnings: self.collection.type_warnings.clone(), data_contracts, - root_capability: Collection::capability_for_root(directory.path()) + authority: crate::collection_root::CollectionRoot::acquire(directory.path()) .map_err(validation_stage_error)?, }; let result = collection diff --git a/src/runtime/mod.rs b/src/runtime/mod.rs index e26ed8a..1dfc27b 100644 --- a/src/runtime/mod.rs +++ b/src/runtime/mod.rs @@ -40,17 +40,21 @@ pub use benchmark::{ BenchmarkDiagnostic, BenchmarkFileFacts, BenchmarkProjection, CandidateExpression, CandidateTruth, CompiledCandidate, ProjectionRelationship, QueryRequirements, }; +pub(crate) use canonical_operation::LegacyRecoveredV03Value; pub use canonical_operation::{ CanonicalCollectionSetupAppliedValue, CanonicalCollectionSetupConflictValue, - CanonicalCollectionSetupValue, CanonicalDeleteValue, CanonicalOperationOutcome, - CanonicalOperationValue, CanonicalQueryValue, CanonicalRenamePreflightValue, - CanonicalRenameValue, CanonicalTypePackValue, DefinitionResultExtensions, - WireOnlyOperationValue, + CanonicalCollectionSetupValue, CanonicalDeleteValue, CanonicalOperationFamily, + CanonicalOperationOutcome, CanonicalOperationValue, CanonicalOutcomeState, CanonicalQueryValue, + CanonicalRenamePreflightValue, CanonicalRenameValue, CanonicalResourceOperationValue, + CanonicalTypePackValue, DefinitionResultExtensions, }; pub use catalog::{ CanonicalRecordInput, CatalogError, CatalogInput, CompiledCatalog, ResolvedTypeResource, }; -pub use context::{OperationContext, OperationDeadline}; +pub use context::{ + CaptureLimitExceeded, CaptureLimitKind, CaptureLimits, CaptureLimitsBuilder, OperationContext, + OperationDeadline, +}; pub use filesystem::FilesystemRuntime; pub use hosted_base::{ HostedBaseEvaluation, HostedBaseGroupAccumulator, HostedBasePlan, HostedBasePlanning, @@ -92,9 +96,10 @@ pub use outcome::{ ChangeFeedOwnerId, ChangeFeedTransfer, ChangeFeedTransferId, ChangeFeedTransferIntent, ChangeFeedTransferReceipt, ChangeOrigin, ChangePage, ChangePageCursor, ChangeSet, ChangeWatermark, CollectionGeneration, CommitAttempt, CommitId, CommitRejection, - DurableCommitState, ExecutionOutcome, HostClaimId, PreparationOutcome, PreparedMutation, - ReadCursor, ReadPage, RebuildReason, RecordChange, RecordChangeKind, ResourceChange, - ResourceChangeKind, RuntimeChangeEvent, RuntimeChangeEventPage, RuntimeMeasurements, + CursorReleaseOutcome, DurableCommitState, ExecutionOutcome, HostClaimId, + LegacyJournalInventory, PreparationOutcome, PreparedMutation, ReadCursor, ReadPage, + RebuildReason, RecordChange, RecordChangeKind, ResourceChange, ResourceChangeKind, + RuntimeChangeEvent, RuntimeChangeEventPage, RuntimeMeasurements, }; pub use projection::{ PreparedSemanticProjection, RecordResolutionKey, RecordResolutionKeyKind, SemanticFileFacts, @@ -106,8 +111,9 @@ pub(crate) use record_resolution::{ select_resolution_candidate, RankedResolution, RankedResolutionCandidate, }; pub use record_resolution::{ - OccurrenceResolutionLookup, RecordResolutionPlan, ResolutionCandidate, ResolutionLookupKey, - ResolvedRecordStructure, ResolvedStructuralOccurrence, MAX_RESOLUTION_CANDIDATES, + OccurrenceResolutionLookup, RecordResolutionPlan, ResolutionCandidate, + ResolutionCandidateIdentity, ResolutionLookupKey, ResolutionReason, ResolvedRecordStructure, + ResolvedStructuralOccurrence, MAX_RESOLUTION_ALTERNATIVES, MAX_RESOLUTION_CANDIDATES, MAX_RESOLUTION_LOOKUPS, MAX_STRUCTURAL_OCCURRENCES, }; pub use record_structure::{ @@ -136,6 +142,8 @@ pub enum ProviderError { OperationCancelled, #[error("collection operation deadline elapsed before its durable boundary")] OperationDeadline, + #[error(transparent)] + CaptureLimitExceeded(#[from] CaptureLimitExceeded), #[error("collection generation sequence is exhausted")] GenerationExhausted, #[error("collection change watermark is exhausted")] @@ -180,6 +188,7 @@ impl ProviderError { Self::LockPoisoned => "operation_lock_unavailable", Self::OperationCancelled => "operation_cancelled", Self::OperationDeadline => "operation_deadline", + Self::CaptureLimitExceeded(_) => "capture_limit_exceeded", Self::GenerationExhausted => "generation_exhausted", Self::WatermarkExhausted => "change_watermark_exhausted", Self::InvalidChangeSet(_) => "invalid_change_set", diff --git a/src/runtime/operation.rs b/src/runtime/operation.rs index e497098..d3a5f5f 100644 --- a/src/runtime/operation.rs +++ b/src/runtime/operation.rs @@ -7,7 +7,7 @@ use serde_json::{json, Value}; use super::ProviderError; use crate::{diagnostic::Diagnostic, v03::OperationResult}; -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum OperationKind { Read, diff --git a/src/runtime/outcome.rs b/src/runtime/outcome.rs index a690f35..0f33953 100644 --- a/src/runtime/outcome.rs +++ b/src/runtime/outcome.rs @@ -469,9 +469,12 @@ pub struct ExecutionOutcome { pub operation: CanonicalOperationOutcome, /// Ephemeral v0.3 compatibility projection for coordinated host migration. /// - /// This field is never journaled and will be removed after Connect consumes - /// `operation` exclusively. - #[deprecated(note = "use operation; removal tracked after the Connect Phase 4 migration")] + /// This field is never journaled and is removed in 0.5.0 after the 0.4.x + /// Connect compatibility window. + #[deprecated( + since = "0.4.0", + note = "use operation (or operation.to_v03() only at a wire edge); removed in 0.5.0 after the 0.4.x Connect compatibility window" + )] pub result: crate::v03::OperationResult, /// Runtime generation observed or produced. pub generation: CollectionGeneration, @@ -504,6 +507,21 @@ impl ExecutionOutcome { } } +/// Privacy-safe inventory of journals that still require the version-2 +/// compatibility decoder. No transaction IDs, paths, claims, or payloads are exposed. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub struct LegacyJournalInventory { + /// Number of valid version-2 runtime journals under the held collection authority. + pub version_2: usize, +} + +impl LegacyJournalInventory { + /// True when the collection has crossed the fixture-zero removal gate. + pub fn is_zero(self) -> bool { + self.version_2 == 0 + } +} + /// Source of one durable runtime change event. #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -624,6 +642,14 @@ pub struct ReadPage { pub next: Option, } +/// Typed result of explicitly releasing generation-pinned cursor state. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct CursorReleaseOutcome { + /// `true` when retained state existed and was released; `false` for an + /// authenticated cursor that had already been released. + pub released: bool, +} + /// Opaque process-local handle for one durable prepared mutation. #[derive(Clone, Debug, Eq, PartialEq)] pub struct PreparedMutation { @@ -653,8 +679,12 @@ pub enum PreparationOutcome { pub struct CommitRejection { /// Typed canonical operation failure. pub operation: CanonicalOperationOutcome, - /// Ephemeral v0.3 compatibility projection; never persisted. - #[deprecated(note = "use operation; removal tracked after the Connect Phase 4 migration")] + /// Ephemeral v0.3 compatibility projection; never persisted. Removed in + /// 0.5.0 after the 0.4.x Connect compatibility window. + #[deprecated( + since = "0.4.0", + note = "use operation (or operation.to_v03() only at a wire edge); removed in 0.5.0 after the 0.4.x Connect compatibility window" + )] pub result: crate::v03::OperationResult, } diff --git a/src/runtime/projection.rs b/src/runtime/projection.rs index 7dfac8c..278289d 100644 --- a/src/runtime/projection.rs +++ b/src/runtime/projection.rs @@ -19,11 +19,12 @@ use super::{ RECORD_STRUCTURE_SCHEMA_VERSION, }; -/// Format v5 excludes body-dependent computed values and body link labels/source -/// spellings from provider-readable state. Older projections may contain body -/// prose through either seam and must not be treated as current by a v5 executor. -pub const SEMANTIC_PROJECTION_FORMAT_VERSION: u32 = 5; -pub const SEMANTIC_PROJECTION_SCHEMA_VERSION: &str = "mdbase-semantic-projection-v4"; +/// Format v6 adds bounded selector evidence to resolved structural occurrences. +/// The evidence is persisted as part of the projection, so v5 projections are +/// still deserializable for explicit stale-data handling but are never accepted +/// by a v6 executor or mixed into a v6 storage/digest binding. +pub const SEMANTIC_PROJECTION_FORMAT_VERSION: u32 = 6; +pub const SEMANTIC_PROJECTION_SCHEMA_VERSION: &str = "mdbase-semantic-projection-v5"; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct SemanticProjectionFacts { @@ -75,6 +76,7 @@ impl SemanticProjection { && self.structure.schema_version == RECORD_STRUCTURE_SCHEMA_VERSION && self.structure.path == self.facts.path && self.structure.structural_digest_is_valid() + && self.structure.resolution_evidence_is_valid() } /// Fail-closed currentness and internal-integrity check shared by every @@ -427,7 +429,9 @@ mod tests { use serde_json::json; use super::*; - use crate::runtime::{CatalogInput, ResolvedTypeResource}; + use crate::runtime::{ + CatalogInput, RecordResolutionKeyKind, ResolutionReason, ResolvedTypeResource, + }; fn catalog() -> CompiledCatalog { CompiledCatalog::compile(CatalogInput { @@ -513,6 +517,202 @@ mod tests { catalog.finalize_projection(prepared, resolved).unwrap() } + fn resolved_projection( + body: &str, + kind: RecordResolutionKeyKind, + paths: &[&str], + ) -> SemanticProjection { + let catalog = catalog(); + let document = format!("---\nuid: source\n---\n{body}"); + let prepared = catalog.project_record(&input(&document)).unwrap(); + let plan = catalog.plan_record_resolution(&prepared.structure).unwrap(); + let lookup = &plan.lookups[0]; + let selected = lookup + .alternatives + .iter() + .find(|alternative| alternative.kind == kind) + .unwrap() + .clone(); + let candidates = paths + .iter() + .enumerate() + .map(|(index, path)| ResolutionCandidate { + occurrence_ordinal: lookup.occurrence_ordinal, + lookup: selected.clone(), + record_id: format!("record-{index}"), + path: if path.is_empty() { + selected.value.clone() + } else { + (*path).to_string() + }, + }) + .collect::>(); + let resolved = catalog + .resolve_record_structure(&prepared.structure, &plan, &candidates) + .unwrap(); + catalog.finalize_projection(prepared, resolved).unwrap() + } + + #[test] + fn hostile_reason_evidence_never_passes_v6_projection_integrity() { + let cases = [ + ( + "[[target]]", + RecordResolutionKeyKind::Id, + vec!["elsewhere/by-id.md"], + ResolutionReason::ConfiguredId, + ), + ( + "[[target]]", + RecordResolutionKeyKind::Title, + vec!["elsewhere/by-title.md"], + ResolutionReason::OnlyCandidate, + ), + ( + "[Target](../target.md)", + RecordResolutionKeyKind::Path, + vec![""], + ResolutionReason::ExactPath, + ), + ( + "[[target]]", + RecordResolutionKeyKind::Basename, + vec!["notes/target.md", "z/target.md"], + ResolutionReason::SameDirectory, + ), + ( + "[[target]]", + RecordResolutionKeyKind::Basename, + vec!["a/target.md", "deep/nested/target.md"], + ResolutionReason::ShallowestPath, + ), + ( + "[[target]]", + RecordResolutionKeyKind::Basename, + vec!["a/target.md", "z/target.md"], + ResolutionReason::LexicalTieBreak, + ), + ]; + + for (body, kind, paths, expected_reason) in cases { + let mut projection = resolved_projection(body, kind, &paths); + assert_eq!( + projection.structure.occurrences[0].reason, + Some(expected_reason) + ); + assert!(projection.integrity_is_current_for( + &projection.facts.catalog_revision, + &projection.facts.semantic_engine_version, + )); + let occurrence = &mut projection.structure.occurrences[0]; + match expected_reason { + ResolutionReason::ConfiguredId | ResolutionReason::OnlyCandidate => { + occurrence.alternatives = vec!["other/target.md".to_string()]; + } + ResolutionReason::ExactPath => { + occurrence.target_path = Some("fabricated.md".to_string()); + } + ResolutionReason::SameDirectory => { + occurrence.alternatives = vec!["notes/target.md".to_string()]; + } + ResolutionReason::ShallowestPath => { + occurrence.alternatives = vec!["z/target.md".to_string()]; + } + ResolutionReason::LexicalTieBreak => { + occurrence.target_path = Some("z/target.md".to_string()); + occurrence.alternatives = vec!["a/target.md".to_string()]; + } + } + occurrence.alternative_candidates = occurrence + .alternatives + .iter() + .enumerate() + .map( + |(index, path)| crate::runtime::ResolutionCandidateIdentity { + record_id: format!("fabricated-{index}"), + path: path.clone(), + }, + ) + .collect(); + occurrence.alternative_candidates.sort_by(|left, right| { + (&left.path, &left.record_id).cmp(&(&right.path, &right.record_id)) + }); + occurrence.alternatives = occurrence + .alternative_candidates + .iter() + .map(|candidate| candidate.path.clone()) + .collect(); + occurrence.candidate_count = occurrence.alternative_candidates.len() + 1; + let winner = crate::runtime::ResolutionCandidateIdentity { + record_id: occurrence.target_record_id.clone().unwrap(), + path: occurrence.target_path.clone().unwrap(), + }; + occurrence.candidate_digest = Some( + crate::runtime::record_resolution::digest_candidate_identities( + occurrence.selected_lookup.as_ref().unwrap().kind, + &winner, + &occurrence.alternative_candidates, + ) + .unwrap(), + ); + assert!(!projection.integrity_is_current_for( + &projection.facts.catalog_revision, + &projection.facts.semantic_engine_version, + )); + } + } + + #[test] + fn projection_rejects_candidate_identity_substitution_after_redigest() { + let mut projection = resolved_projection( + "[[target]]", + RecordResolutionKeyKind::Basename, + &["a/target.md", "z/target.md"], + ); + assert!(projection.integrity_is_current_for( + &projection.facts.catalog_revision, + &projection.facts.semantic_engine_version, + )); + let occurrence = &mut projection.structure.occurrences[0]; + occurrence.target_record_id = Some(occurrence.alternative_candidates[0].record_id.clone()); + let winner = crate::runtime::ResolutionCandidateIdentity { + record_id: occurrence.target_record_id.clone().unwrap(), + path: occurrence.target_path.clone().unwrap(), + }; + occurrence.candidate_digest = Some( + crate::runtime::record_resolution::digest_candidate_identities( + occurrence.selected_lookup.as_ref().unwrap().kind, + &winner, + &occurrence.alternative_candidates, + ) + .unwrap(), + ); + assert!(!projection.integrity_is_current_for( + &projection.facts.catalog_revision, + &projection.facts.semantic_engine_version, + )); + } + + #[test] + fn old_projection_storage_is_read_but_never_accepted_as_current() { + let current = projection("---\nuid: note-1\n---\n[[target]]\n"); + let mut old = serde_json::to_value(¤t).unwrap(); + old["format_version"] = serde_json::json!(5); + old["schema_version"] = serde_json::json!("mdbase-semantic-projection-v4"); + let old: SemanticProjection = serde_json::from_value(old).unwrap(); + assert!(!old.integrity_is_current_for( + ¤t.facts.catalog_revision, + ¤t.facts.semantic_engine_version, + )); + + let mut mixed = current.clone(); + mixed.structure.occurrences[0].alternatives = vec!["other.md".to_string()]; + assert!(!mixed.integrity_is_current_for( + ¤t.facts.catalog_revision, + ¤t.facts.semantic_engine_version, + )); + } + #[test] fn projection_contains_full_semantics_and_structure_without_exact_body() { let document = "---\nuid: note-1\ntitle: One\nproject: '[[projects/main]]'\n---\nsecret-body-prose [[two#part|Two]] #focus\n"; diff --git a/src/runtime/provider.rs b/src/runtime/provider.rs index 89045b9..0db640b 100644 --- a/src/runtime/provider.rs +++ b/src/runtime/provider.rs @@ -14,7 +14,6 @@ use super::{ }; use crate::v03::OperationResult; use crate::Collection; -use walkdir::WalkDir; /// Provider-neutral execution boundary for one authoritative collection. /// @@ -34,13 +33,13 @@ pub trait CollectionProvider: Send + Sync { /// Execute through the compatibility entry point while hosts migrate to /// explicit operation contexts. fn execute(&self, request: &OperationRequest) -> Result { - self.execute_with_context(request, &OperationContext::legacy()) + self.execute_with_context(request, &OperationContext::internal()) } /// Refresh through the compatibility entry point while hosts migrate to /// explicit operation contexts. fn refresh(&self) -> Result<(), ProviderError> { - self.refresh_with_context(&OperationContext::legacy()) + self.refresh_with_context(&OperationContext::internal()) } } @@ -52,6 +51,7 @@ pub trait CollectionProvider: Send + Sync { /// visible to the next request. pub struct FilesystemProvider { root: PathBuf, + authority: crate::collection_root::CollectionRoot, coordinated: bool, operation_gate: RuntimeGate, observer: Arc, @@ -105,13 +105,15 @@ impl FilesystemProvider { return Err(error); } }; + let authority = collection.held_root().clone(); let stamp = CollectionStamp::load( - &root, + &authority, &collection.settings.types_folder, &collection.settings.contracts_folder, ); Ok(Self { root, + authority, coordinated, operation_gate: RuntimeGate::new(), observer, @@ -141,7 +143,7 @@ impl FilesystemProvider { /// the capture. External filesystem writers are detected by the ordinary /// opaque revisions and by a subsequent capture before cutover. pub fn snapshot(&self) -> Result { - self.snapshot_with_context(&OperationContext::legacy()) + self.snapshot_with_context(&OperationContext::internal()) } /// Capture a snapshot while honoring the caller's operation boundary. @@ -149,12 +151,14 @@ impl FilesystemProvider { &self, context: &OperationContext, ) -> Result { - self.with_collection_read_context(context, Collection::snapshot) + self.with_collection_read_context(context, |collection| { + collection.snapshot_with_context(context) + }) } /// Materialize one record at the provider's read boundary. pub fn snapshot_record(&self, path: &str) -> Result { - self.snapshot_record_with_context(path, &OperationContext::legacy()) + self.snapshot_record_with_context(path, &OperationContext::internal()) } /// Materialize one record while honoring the caller's operation boundary. @@ -163,7 +167,9 @@ impl FilesystemProvider { path: &str, context: &OperationContext, ) -> Result { - self.with_collection_read_context(context, |collection| collection.snapshot_record(path)) + self.with_collection_read_context(context, |collection| { + collection.snapshot_record_with_context(path, context) + }) } /// Execute a compound provider operation against one freshly loaded @@ -178,7 +184,7 @@ impl FilesystemProvider { where E: From, { - self.with_collection_context(&OperationContext::legacy(), operation) + self.with_collection_context(&OperationContext::internal(), operation) } /// Execute one compound exclusive operation with an explicit boundary. @@ -194,7 +200,7 @@ impl FilesystemProvider { context.check().map_err(E::from)?; let collection = self.current_collection().map_err(E::from)?; context.check().map_err(E::from)?; - let result = operation(collection.as_ref())?; + let result = context.scope(|| operation(collection.as_ref()))?; context.check().map_err(E::from)?; Ok(result) } @@ -217,7 +223,7 @@ impl FilesystemProvider { context.check().map_err(E::from)?; let collection = self.current_collection().map_err(E::from)?; context.check().map_err(E::from)?; - operation(collection.as_ref()) + context.scope(|| operation(collection.as_ref())) } /// Execute a compound read-only provider operation while allowing other @@ -229,7 +235,7 @@ impl FilesystemProvider { where E: From, { - self.with_collection_read_context(&OperationContext::legacy(), operation) + self.with_collection_read_context(&OperationContext::internal(), operation) } /// Execute one compound read operation with an explicit boundary. @@ -245,7 +251,7 @@ impl FilesystemProvider { context.check().map_err(E::from)?; let collection = self.current_collection().map_err(E::from)?; context.check().map_err(E::from)?; - let result = operation(collection.as_ref())?; + let result = context.scope(|| operation(collection.as_ref()))?; context.check().map_err(E::from)?; Ok(result) } @@ -297,21 +303,45 @@ impl FilesystemProvider { timings.open = open_started.elapsed(); context.check()?; let execute_started = Instant::now(); - let result = - match execute_collection(collection.as_ref(), request, context, self.coordinated) { - Ok(result) => result, - Err(error) => { - timings.execute = execute_started.elapsed(); - return Err(self.finish_provider_error( - request.operation.as_str(), - "execute", - started, - timings, - error, - )); - } - }; + let result = match context + .scope(|| execute_collection(collection.as_ref(), request, context, self.coordinated)) + { + Ok(result) => result, + Err(error) => { + timings.execute = execute_started.elapsed(); + return Err(self.finish_provider_error( + request.operation.as_str(), + "execute", + started, + timings, + error, + )); + } + }; timings.execute = execute_started.elapsed(); + if let Some(error) = context.capture_limit_error() { + return Err(self.finish_provider_error( + request.operation.as_str(), + "execute", + started, + timings, + error, + )); + } + if request.operation == OperationKind::Query { + let records = result + .result + .get("results") + .and_then(Value::as_array) + .filter(|records| !records.is_empty()); + if let Some(records) = records { + let bytes = super::cursor::measured_json_bytes( + records, + context.capture_limits().max_retained_bytes, + )?; + context.charge_retained(bytes)?; + } + } if !request.operation.is_mutation() { context.check()?; } @@ -393,7 +423,7 @@ impl FilesystemProvider { .read() .map_err(|_| ProviderError::LockPoisoned)?; let current = CollectionStamp::load( - &self.root, + &self.authority, &cached.collection.settings.types_folder, &cached.collection.settings.contracts_folder, ); @@ -407,16 +437,19 @@ impl FilesystemProvider { .write() .map_err(|_| ProviderError::LockPoisoned)?; let current = CollectionStamp::load( - &self.root, + &self.authority, &cached.collection.settings.types_folder, &cached.collection.settings.contracts_folder, ); if current == cached.stamp { return Ok(cached.collection.clone()); } - let collection = open_collection(&self.root)?; + let collection = cached + .collection + .reopen_held(true) + .map_err(|error| ProviderError::CollectionOpen(error_message(&error)))?; let stamp = CollectionStamp::load( - &self.root, + &self.authority, &collection.settings.types_folder, &collection.settings.contracts_folder, ); @@ -426,9 +459,17 @@ impl FilesystemProvider { } fn reload_collection(&self) -> Result<(), ProviderError> { - let collection = open_collection(&self.root)?; + let existing = self + .collection_cache + .read() + .map_err(|_| ProviderError::LockPoisoned)? + .collection + .clone(); + let collection = existing + .reopen_held(true) + .map_err(|error| ProviderError::CollectionOpen(error_message(&error)))?; let stamp = CollectionStamp::load( - &self.root, + &self.authority, &collection.settings.types_folder, &collection.settings.contracts_folder, ); @@ -445,7 +486,7 @@ impl FilesystemProvider { &self, generation: &super::CollectionGeneration, ) -> Result<(), ProviderError> { - let context = OperationContext::legacy(); + let context = OperationContext::internal(); let _guard = self.write_lock(&context)?; let collection = self.current_collection()?; crate::cache::runtime::rebuild(collection.as_ref(), generation).map_err(cache_error)?; @@ -466,13 +507,26 @@ impl FilesystemProvider { context.check()?; self.with_collection_boundary_context(context, |collection| { context.check()?; - let settlement = OperationContext::legacy(); + let settlement = OperationContext::internal(); crate::transactions::reset_runtime_support_for_fork(collection, &settlement) .map_err(super::filesystem::transaction_error)?; super::feed::reset_for_fork(collection) }) } + /// Count valid version-2 runtime journals without exposing journal contents. + /// Operators should record a zero result before the 0.5.0 compatibility removal. + pub fn legacy_journal_inventory( + &self, + context: &OperationContext, + ) -> Result { + context.check()?; + self.with_collection_boundary_context(context, |collection| { + crate::transactions::legacy_runtime_journal_inventory(collection, context) + .map_err(super::filesystem::transaction_error) + }) + } + pub(crate) fn ensure_runtime_cache( &self, generation: &super::CollectionGeneration, @@ -740,32 +794,21 @@ struct CollectionStamp { } impl CollectionStamp { - fn load(root: &Path, types_folder: &str, contracts_folder: &str) -> Self { - let config_revision = std::fs::read(root.join("mdbase.yaml")) + fn load( + root: &crate::collection_root::CollectionRoot, + types_folder: &str, + contracts_folder: &str, + ) -> Self { + let config_revision = root + .read("mdbase.yaml") .ok() .map(|bytes| crate::v03::revision(&bytes)); let mut hasher = std::collections::hash_map::DefaultHasher::new(); - for control_root in [root.join(types_folder), root.join(contracts_folder)] { - for entry in WalkDir::new(&control_root) - .sort_by_file_name() - .follow_links(false) - .into_iter() - .flatten() - .filter(|entry| entry.file_type().is_file()) - { - entry - .path() - .strip_prefix(root) - .unwrap_or(entry.path()) - .hash(&mut hasher); - if let Ok(metadata) = entry.metadata() { - metadata.len().hash(&mut hasher); - metadata - .modified() - .ok() - .and_then(|modified| modified.duration_since(std::time::UNIX_EPOCH).ok()) - .map(|duration| duration.as_nanos()) - .hash(&mut hasher); + for folder in [types_folder, contracts_folder] { + for relative in root.files_recursive(Path::new(folder)).unwrap_or_default() { + relative.hash(&mut hasher); + if let Ok(bytes) = root.read(&relative) { + bytes.hash(&mut hasher); } } } diff --git a/src/runtime/record_resolution.rs b/src/runtime/record_resolution.rs index 65db112..8a3573e 100644 --- a/src/runtime/record_resolution.rs +++ b/src/runtime/record_resolution.rs @@ -14,6 +14,9 @@ use super::{ pub const MAX_STRUCTURAL_OCCURRENCES: usize = 4_096; pub const MAX_RESOLUTION_LOOKUPS: usize = 16_384; pub const MAX_RESOLUTION_CANDIDATES: usize = 16_384; +/// Maximum losing candidates retained as resolution evidence. Selection still +/// considers every candidate; this bound applies only to additive evidence. +pub const MAX_RESOLUTION_ALTERNATIVES: usize = MAX_RESOLUTION_CANDIDATES - 1; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct RecordResolutionPlan { @@ -45,17 +48,41 @@ pub struct ResolutionCandidate { pub path: String, } -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct RankedResolutionCandidate { +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ResolutionCandidateIdentity { pub record_id: String, pub path: String, } +pub(crate) type RankedResolutionCandidate = ResolutionCandidateIdentity; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ResolutionReason { + ConfiguredId, + OnlyCandidate, + ExactPath, + SameDirectory, + ShallowestPath, + LexicalTieBreak, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum RankedResolution { Missing, - Resolved { record_id: String, path: String }, - Ambiguous { paths: Vec }, + Resolved { + record_id: String, + path: String, + reason: ResolutionReason, + selected_kind: RecordResolutionKeyKind, + candidate_count: usize, + candidate_digest: String, + alternatives: Vec, + alternative_candidates: Vec, + }, + Ambiguous { + paths: Vec, + }, } pub(crate) fn select_resolution_candidate( @@ -69,11 +96,21 @@ pub(crate) fn select_resolution_candidate( .rsplit_once('/') .map_or("", |(parent, _)| parent); let mut by_path = BTreeMap::::new(); + let mut by_record_id = BTreeMap::::new(); for candidate in candidates { if candidate.record_id.is_empty() { return Err(invalid_candidate_error()); } let path = CollectionPath::new(&candidate.path).map_err(|_| invalid_candidate_error())?; + match by_record_id.entry(candidate.record_id.clone()) { + std::collections::btree_map::Entry::Vacant(entry) => { + entry.insert(path.to_string()); + } + std::collections::btree_map::Entry::Occupied(entry) if entry.get() != path.as_str() => { + return Err(invalid_candidate_error()); + } + std::collections::btree_map::Entry::Occupied(_) => {} + } match by_path.entry(path.to_string()) { std::collections::btree_map::Entry::Vacant(entry) => { entry.insert(candidate.record_id); @@ -93,8 +130,49 @@ pub(crate) fn select_resolution_candidate( if candidates.is_empty() { return Ok(RankedResolution::Missing); } + if candidates.len() > MAX_RESOLUTION_CANDIDATES { + return Err(resolution_budget_error("resolution candidate")); + } + let candidate_count = candidates.len(); if kind == RecordResolutionKeyKind::Basename { + let reason = if candidates.len() == 1 { + ResolutionReason::OnlyCandidate + } else { + let same_directory = candidates + .iter() + .filter(|candidate| { + candidate + .path + .rsplit_once('/') + .map_or("", |(parent, _)| parent) + == source_directory + }) + .count(); + if same_directory == 1 { + ResolutionReason::SameDirectory + } else { + let preferred_directory = same_directory > 0; + let mut depths = candidates + .iter() + .filter(|candidate| { + !preferred_directory + || candidate + .path + .rsplit_once('/') + .map_or("", |(parent, _)| parent) + == source_directory + }) + .map(|candidate| candidate.path.split('/').count()) + .collect::>(); + depths.sort_unstable(); + if depths.first() != depths.get(1) { + ResolutionReason::ShallowestPath + } else { + ResolutionReason::LexicalTieBreak + } + } + }; candidates.sort_by(|left, right| { let left_directory = left.path.rsplit_once('/').map_or("", |(parent, _)| parent); let right_directory = right.path.rsplit_once('/').map_or("", |(parent, _)| parent); @@ -113,17 +191,42 @@ pub(crate) fn select_resolution_candidate( left_rank.cmp(&right_rank) }); let winner = candidates.remove(0); + let alternative_candidates = complete_alternative_candidates(candidates); + let alternatives = alternative_candidates + .iter() + .map(|candidate| candidate.path.clone()) + .collect(); + let candidate_digest = digest_candidate_identities(kind, &winner, &alternative_candidates)?; return Ok(RankedResolution::Resolved { record_id: winner.record_id, path: winner.path, + reason, + selected_kind: kind, + candidate_count, + candidate_digest, + alternatives, + alternative_candidates, }); } if candidates.len() == 1 { let winner = candidates.remove(0); + let candidate_digest = digest_candidate_identities(kind, &winner, &[])?; + let reason = match kind { + RecordResolutionKeyKind::Id => ResolutionReason::ConfiguredId, + RecordResolutionKeyKind::Path => ResolutionReason::ExactPath, + RecordResolutionKeyKind::Basename => unreachable!(), + RecordResolutionKeyKind::Title => ResolutionReason::OnlyCandidate, + }; Ok(RankedResolution::Resolved { record_id: winner.record_id, path: winner.path, + reason, + selected_kind: kind, + candidate_count, + candidate_digest, + alternatives: Vec::new(), + alternative_candidates: Vec::new(), }) } else { Ok(RankedResolution::Ambiguous { @@ -135,6 +238,31 @@ pub(crate) fn select_resolution_candidate( } } +fn complete_alternative_candidates( + mut candidates: Vec, +) -> Vec { + candidates.sort_by(|left, right| { + (left.path.as_str(), left.record_id.as_str()) + .cmp(&(right.path.as_str(), right.record_id.as_str())) + }); + candidates +} + +pub(crate) fn digest_candidate_identities( + kind: RecordResolutionKeyKind, + winner: &ResolutionCandidateIdentity, + alternatives: &[ResolutionCandidateIdentity], +) -> Result { + use sha2::{Digest, Sha256}; + + let ordered = std::iter::once(winner) + .chain(alternatives) + .map(|candidate| (candidate.record_id.as_str(), candidate.path.as_str())) + .collect::>(); + let canonical = serde_jcs::to_vec(&(kind, ordered)).map_err(|_| invalid_candidate_error())?; + Ok(format!("sha256:{:x}", Sha256::digest(canonical))) +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ResolvedRecordStructure { pub schema_version: String, @@ -156,6 +284,22 @@ pub struct ResolvedStructuralOccurrence { pub target_path: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub ambiguous_paths: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub selected_lookup: Option, + #[serde(default, skip_serializing_if = "is_zero")] + pub candidate_count: usize, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub candidate_digest: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub alternatives: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub alternative_candidates: Vec, +} + +fn is_zero(value: &usize) -> bool { + *value == 0 } impl ResolvedRecordStructure { @@ -179,6 +323,204 @@ impl ResolvedRecordStructure { }; digest_structure(&structure) == self.structural_digest } + + /// Validate additive selector evidence independently of the pre-resolution + /// structural digest. This prevents an old or hand-crafted projection from + /// claiming the current format while omitting bounded resolution evidence. + pub fn resolution_evidence_is_valid(&self) -> bool { + self.occurrences.iter().all(|resolved| { + let alternatives_valid = resolved.alternatives.len() <= MAX_RESOLUTION_ALTERNATIVES + && resolved + .alternatives + .windows(2) + .all(|pair| pair[0] < pair[1]) + && resolved + .alternatives + .iter() + .all(|path| CollectionPath::new(path).is_ok()) + && !resolved + .target_path + .as_ref() + .is_some_and(|winner| resolved.alternatives.iter().any(|path| path == winner)); + match resolved.resolution { + StructuralResolution::Resolved => { + resolved.target_record_id.is_some() + && resolved + .target_path + .as_deref() + .is_some_and(|path| CollectionPath::new(path).is_ok()) + && resolved.ambiguous_paths.is_empty() + && alternatives_valid + && resolved_reason_is_proven(&self.path, resolved) + } + _ => { + resolved.reason.is_none() + && resolved.selected_lookup.is_none() + && resolved.candidate_count == 0 + && resolved.candidate_digest.is_none() + && resolved.alternatives.is_empty() + && resolved.alternative_candidates.is_empty() + } + } + }) + } +} + +fn collection_directory(path: &str) -> &str { + path.rsplit_once('/').map_or("", |(parent, _)| parent) +} + +fn resolved_reason_is_proven(source_path: &str, resolved: &ResolvedStructuralOccurrence) -> bool { + let (Some(reason), Some(winner), Some(record_id), Some(lookup), Some(bound_digest)) = ( + resolved.reason, + resolved.target_path.as_deref(), + resolved.target_record_id.as_deref(), + resolved.selected_lookup.as_ref(), + resolved.candidate_digest.as_deref(), + ) else { + return false; + }; + let winner_identity = ResolutionCandidateIdentity { + record_id: record_id.to_string(), + path: winner.to_string(), + }; + let identities_are_canonical = resolved + .alternative_candidates + .windows(2) + .all(|pair| (&pair[0].path, &pair[0].record_id) < (&pair[1].path, &pair[1].record_id)); + let identity_paths = resolved + .alternative_candidates + .iter() + .map(|candidate| candidate.path.as_str()) + .collect::>(); + let mut record_ids = BTreeSet::new(); + let mut paths = BTreeSet::new(); + let identities_are_unique = std::iter::once(&winner_identity) + .chain(&resolved.alternative_candidates) + .all(|candidate| { + !candidate.record_id.is_empty() + && CollectionPath::new(&candidate.path).is_ok() + && record_ids.insert(candidate.record_id.as_str()) + && paths.insert(candidate.path.as_str()) + }); + if resolved.candidate_count != resolved.alternative_candidates.len() + 1 + || resolved.candidate_count == 0 + || resolved.candidate_count > MAX_RESOLUTION_CANDIDATES + || resolved + .alternatives + .iter() + .map(String::as_str) + .ne(identity_paths) + || !identities_are_canonical + || !identities_are_unique + || digest_candidate_identities( + lookup.kind, + &winner_identity, + &resolved.alternative_candidates, + ) + .ok() + .as_deref() + != Some(bound_digest) + { + return false; + } + let source_directory = collection_directory(source_path); + let depth = |path: &str| path.split('/').count(); + let simple_wikilink = matches!( + resolved.occurrence.kind, + StructuralLinkKind::Wikilink | StructuralLinkKind::WikilinkEmbed + ) && !resolved.occurrence.relative + && !resolved.occurrence.raw_target.contains('/') + && std::path::Path::new(&resolved.occurrence.raw_target) + .extension() + .is_none(); + let basename_eligible = |path: &str| { + std::path::Path::new(path) + .file_stem() + .and_then(|stem| stem.to_str()) + .is_some_and(|stem| stem.eq_ignore_ascii_case(&resolved.occurrence.raw_target)) + }; + let ranked_candidates_are_eligible = basename_eligible(winner) + && resolved + .alternatives + .iter() + .all(|path| basename_eligible(path)); + + match reason { + ResolutionReason::ConfiguredId => { + lookup.kind == RecordResolutionKeyKind::Id + && lookup.value == resolved.occurrence.raw_target.to_lowercase() + && resolved.candidate_count == 1 + && !record_id.is_empty() + } + ResolutionReason::OnlyCandidate => { + matches!( + lookup.kind, + RecordResolutionKeyKind::Basename | RecordResolutionKeyKind::Title + ) && resolved.candidate_count == 1 + && lookup.value == resolved.occurrence.raw_target.to_lowercase() + } + ResolutionReason::ExactPath => { + let Some(target) = resolved.occurrence.normalized_target.as_deref() else { + return false; + }; + lookup.kind == RecordResolutionKeyKind::Path + && resolved.candidate_count == 1 + && lookup.value == winner + && !simple_wikilink + && (winner == target + || (std::path::Path::new(target).extension().is_none() + && [".md", ".mdx"] + .iter() + .any(|extension| winner == format!("{target}{extension}")))) + } + ResolutionReason::SameDirectory => { + lookup.kind == RecordResolutionKeyKind::Basename + && lookup.value == resolved.occurrence.raw_target.to_lowercase() + && resolved.candidate_count > 1 + && ranked_candidates_are_eligible + && collection_directory(winner) == source_directory + && resolved + .alternatives + .iter() + .all(|path| collection_directory(path) != source_directory) + } + ResolutionReason::ShallowestPath => { + lookup.kind == RecordResolutionKeyKind::Basename + && lookup.value == resolved.occurrence.raw_target.to_lowercase() + && resolved.candidate_count > 1 + && ranked_candidates_are_eligible + && collection_directory(winner) != source_directory + && resolved.alternatives.iter().all(|path| { + collection_directory(path) != source_directory && depth(winner) < depth(path) + }) + } + ResolutionReason::LexicalTieBreak => { + lookup.kind == RecordResolutionKeyKind::Basename + && lookup.value == resolved.occurrence.raw_target.to_lowercase() + && resolved.candidate_count > 1 + && ranked_candidates_are_eligible + && resolved.alternatives.iter().all(|path| { + let winner_rank = ( + collection_directory(winner) != source_directory, + depth(winner), + winner, + ); + let loser_rank = ( + collection_directory(path) != source_directory, + depth(path), + path.as_str(), + ); + winner_rank < loser_rank + }) + && resolved.alternatives.iter().any(|path| { + ( + collection_directory(winner) != source_directory, + depth(winner), + ) == (collection_directory(path) != source_directory, depth(path)) + }) + } + } } impl CompiledCatalog { @@ -370,13 +712,19 @@ impl CompiledCatalog { target_record_id: None, target_path: None, ambiguous_paths: Vec::new(), + reason: None, + selected_lookup: None, + candidate_count: 0, + candidate_digest: None, + alternatives: Vec::new(), + alternative_candidates: Vec::new(), }); } let selected = grouped .get(&occurrence.ordinal) .and_then(|priorities| priorities.first_key_value()) .map(|(priority, matches)| { - let kind = plan + let selected_lookup = plan .lookups .iter() .find(|lookup| lookup.occurrence_ordinal == occurrence.ordinal) @@ -386,7 +734,7 @@ impl CompiledCatalog { .iter() .find(|alternative| alternative.priority == *priority) }) - .map(|alternative| alternative.kind) + .cloned() .ok_or_else(|| CatalogError { code: "invalid_resolution_plan".to_string(), message: "The relationship-resolution plan does not define the returned priority." @@ -394,7 +742,7 @@ impl CompiledCatalog { })?; select_resolution_candidate( &structure.path, - kind, + selected_lookup.kind, matches .iter() .map(|(record_id, path)| RankedResolutionCandidate { @@ -402,31 +750,60 @@ impl CompiledCatalog { path: path.clone(), }), ) + .map(|resolution| (selected_lookup, resolution)) }) .transpose()?; - Ok(match selected.unwrap_or(RankedResolution::Missing) { + let (selected_lookup, selected) = selected + .map(|(lookup, resolution)| (Some(lookup), resolution)) + .unwrap_or((None, RankedResolution::Missing)); + Ok(match selected { RankedResolution::Missing => ResolvedStructuralOccurrence { occurrence, resolution: StructuralResolution::Missing, target_record_id: None, target_path: None, ambiguous_paths: Vec::new(), + reason: None, + selected_lookup: None, + candidate_count: 0, + candidate_digest: None, + alternatives: Vec::new(), + alternative_candidates: Vec::new(), + }, + RankedResolution::Resolved { + record_id, + path, + reason, + selected_kind: _, + candidate_count, + candidate_digest, + alternatives, + alternative_candidates, + } => ResolvedStructuralOccurrence { + occurrence, + resolution: StructuralResolution::Resolved, + target_record_id: Some(record_id), + target_path: Some(path), + ambiguous_paths: Vec::new(), + reason: Some(reason), + selected_lookup, + candidate_count, + candidate_digest: Some(candidate_digest), + alternatives, + alternative_candidates, }, - RankedResolution::Resolved { record_id, path } => { - ResolvedStructuralOccurrence { - occurrence, - resolution: StructuralResolution::Resolved, - target_record_id: Some(record_id), - target_path: Some(path), - ambiguous_paths: Vec::new(), - } - } RankedResolution::Ambiguous { paths } => ResolvedStructuralOccurrence { occurrence, resolution: StructuralResolution::Ambiguous, target_record_id: None, target_path: None, ambiguous_paths: paths, + reason: None, + selected_lookup: None, + candidate_count: 0, + candidate_digest: None, + alternatives: Vec::new(), + alternative_candidates: Vec::new(), }, }) }) @@ -502,6 +879,207 @@ mod tests { } } + fn resolved_occurrence( + body: &str, + kind: RecordResolutionKeyKind, + paths: &[&str], + ) -> ResolvedStructuralOccurrence { + let catalog = catalog(); + let structure = structure(body); + let plan = catalog.plan_record_resolution(&structure).unwrap(); + let lookup = &plan.lookups[0]; + let alternative = lookup + .alternatives + .iter() + .position(|alternative| alternative.kind == kind) + .unwrap(); + let candidates = paths + .iter() + .enumerate() + .map(|(index, path)| { + candidate( + lookup, + alternative, + &format!("record-{index}"), + if path.is_empty() { + &lookup.alternatives[alternative].value + } else { + path + }, + ) + }) + .collect::>(); + catalog + .resolve_record_structure(&structure, &plan, &candidates) + .unwrap() + .occurrences + .into_iter() + .next() + .unwrap() + } + + fn rebind_fabricated_evidence(occurrence: &mut ResolvedStructuralOccurrence) { + occurrence.alternative_candidates = occurrence + .alternatives + .iter() + .enumerate() + .map(|(index, path)| ResolutionCandidateIdentity { + record_id: format!("fabricated-{index}"), + path: path.clone(), + }) + .collect(); + occurrence.alternative_candidates.sort_by(|left, right| { + (&left.path, &left.record_id).cmp(&(&right.path, &right.record_id)) + }); + occurrence.alternatives = occurrence + .alternative_candidates + .iter() + .map(|candidate| candidate.path.clone()) + .collect(); + occurrence.candidate_count = occurrence.alternative_candidates.len() + 1; + let winner = ResolutionCandidateIdentity { + record_id: occurrence.target_record_id.clone().unwrap(), + path: occurrence.target_path.clone().unwrap(), + }; + occurrence.candidate_digest = Some( + digest_candidate_identities( + occurrence.selected_lookup.as_ref().unwrap().kind, + &winner, + &occurrence.alternative_candidates, + ) + .unwrap(), + ); + } + + #[test] + fn hostile_v6_reason_evidence_is_rejected_for_every_reason() { + let mut configured = resolved_occurrence( + "[[target]]", + RecordResolutionKeyKind::Id, + &["elsewhere/by-id.md"], + ); + assert_eq!(configured.reason, Some(ResolutionReason::ConfiguredId)); + assert!(resolved_reason_is_proven("notes/source.md", &configured)); + configured.alternatives.push("other/target.md".to_string()); + rebind_fabricated_evidence(&mut configured); + assert!(!resolved_reason_is_proven("notes/source.md", &configured)); + + let mut only = resolved_occurrence( + "[[target]]", + RecordResolutionKeyKind::Title, + &["elsewhere/by-title.md"], + ); + assert_eq!(only.reason, Some(ResolutionReason::OnlyCandidate)); + assert!(resolved_reason_is_proven("notes/source.md", &only)); + only.alternatives.push("other/target.md".to_string()); + rebind_fabricated_evidence(&mut only); + assert!(!resolved_reason_is_proven("notes/source.md", &only)); + + let mut exact = resolved_occurrence( + "[Target](../target.md)", + RecordResolutionKeyKind::Path, + &[""], + ); + assert_eq!(exact.reason, Some(ResolutionReason::ExactPath)); + assert!(resolved_reason_is_proven("notes/source.md", &exact)); + exact.target_path = Some("fabricated.md".to_string()); + rebind_fabricated_evidence(&mut exact); + assert!(!resolved_reason_is_proven("notes/source.md", &exact)); + + let mut same = resolved_occurrence( + "[[target]]", + RecordResolutionKeyKind::Basename, + &["notes/target.md", "z/target.md"], + ); + assert_eq!(same.reason, Some(ResolutionReason::SameDirectory)); + assert!(resolved_reason_is_proven("notes/source.md", &same)); + same.alternatives = vec!["notes/target.md".to_string()]; + rebind_fabricated_evidence(&mut same); + assert!(!resolved_reason_is_proven("notes/source.md", &same)); + + let mut shallow = resolved_occurrence( + "[[target]]", + RecordResolutionKeyKind::Basename, + &["a/target.md", "deep/nested/target.md"], + ); + assert_eq!(shallow.reason, Some(ResolutionReason::ShallowestPath)); + assert!(resolved_reason_is_proven("notes/source.md", &shallow)); + shallow.alternatives = vec!["z/target.md".to_string()]; + rebind_fabricated_evidence(&mut shallow); + assert!(!resolved_reason_is_proven("notes/source.md", &shallow)); + + let mut lexical = resolved_occurrence( + "[[target]]", + RecordResolutionKeyKind::Basename, + &["a/target.md", "z/target.md"], + ); + assert_eq!(lexical.reason, Some(ResolutionReason::LexicalTieBreak)); + assert!(resolved_reason_is_proven("notes/source.md", &lexical)); + lexical.target_path = Some("z/target.md".to_string()); + lexical.alternatives = vec!["a/target.md".to_string()]; + rebind_fabricated_evidence(&mut lexical); + assert!(!resolved_reason_is_proven("notes/source.md", &lexical)); + } + + #[test] + fn candidate_identity_substitution_and_duplicates_fail_even_with_rebound_digest() { + let valid = resolved_occurrence( + "[[target]]", + RecordResolutionKeyKind::Basename, + &["a/target.md", "z/target.md"], + ); + assert!(resolved_reason_is_proven("notes/source.md", &valid)); + + let mut swapped = valid.clone(); + swapped.target_record_id = Some(valid.alternative_candidates[0].record_id.clone()); + swapped.candidate_digest = Some( + digest_candidate_identities( + swapped.selected_lookup.as_ref().unwrap().kind, + &ResolutionCandidateIdentity { + record_id: swapped.target_record_id.clone().unwrap(), + path: swapped.target_path.clone().unwrap(), + }, + &swapped.alternative_candidates, + ) + .unwrap(), + ); + assert!(!resolved_reason_is_proven("notes/source.md", &swapped)); + + let mut duplicate_id = valid.clone(); + duplicate_id.alternative_candidates[0].record_id = valid.target_record_id.clone().unwrap(); + duplicate_id.candidate_digest = Some( + digest_candidate_identities( + duplicate_id.selected_lookup.as_ref().unwrap().kind, + &ResolutionCandidateIdentity { + record_id: duplicate_id.target_record_id.clone().unwrap(), + path: duplicate_id.target_path.clone().unwrap(), + }, + &duplicate_id.alternative_candidates, + ) + .unwrap(), + ); + assert!(!resolved_reason_is_proven("notes/source.md", &duplicate_id)); + + let mut duplicate_path = valid.clone(); + duplicate_path.alternative_candidates[0].path = valid.target_path.clone().unwrap(); + duplicate_path.alternatives[0] = valid.target_path.clone().unwrap(); + duplicate_path.candidate_digest = Some( + digest_candidate_identities( + duplicate_path.selected_lookup.as_ref().unwrap().kind, + &ResolutionCandidateIdentity { + record_id: duplicate_path.target_record_id.clone().unwrap(), + path: duplicate_path.target_path.clone().unwrap(), + }, + &duplicate_path.alternative_candidates, + ) + .unwrap(), + ); + assert!(!resolved_reason_is_proven( + "notes/source.md", + &duplicate_path + )); + } + #[test] fn configured_id_wins_over_filename_match() { let catalog = catalog(); @@ -536,6 +1114,124 @@ mod tests { resolved.occurrences[0].resolution, StructuralResolution::Resolved ); + assert_eq!( + resolved.occurrences[0] + .selected_lookup + .as_ref() + .map(|lookup| lookup.kind), + Some(RecordResolutionKeyKind::Id) + ); + assert_eq!(resolved.occurrences[0].candidate_count, 1); + assert!(resolved.occurrences[0] + .candidate_digest + .as_deref() + .is_some_and(|digest| digest.starts_with("sha256:"))); + } + + #[test] + fn shared_selector_reports_every_reason_and_sorted_bounded_alternatives() { + let resolve = |kind, paths: &[&str]| { + select_resolution_candidate( + "notes/source.md", + kind, + paths.iter().map(|path| RankedResolutionCandidate { + record_id: (*path).to_string(), + path: (*path).to_string(), + }), + ) + .unwrap() + }; + let evidence = |resolution| match resolution { + RankedResolution::Resolved { + reason, + alternatives, + .. + } => (reason, alternatives), + other => panic!("expected resolved evidence, got {other:?}"), + }; + + assert_eq!( + evidence(resolve(RecordResolutionKeyKind::Id, &["a.md"])).0, + ResolutionReason::ConfiguredId + ); + assert_eq!( + evidence(resolve(RecordResolutionKeyKind::Title, &["a.md"])).0, + ResolutionReason::OnlyCandidate + ); + assert_eq!( + evidence(resolve(RecordResolutionKeyKind::Path, &["a.md"])).0, + ResolutionReason::ExactPath + ); + assert_eq!( + evidence(resolve( + RecordResolutionKeyKind::Basename, + &["notes/a.md", "z/a.md"] + )) + .0, + ResolutionReason::SameDirectory + ); + assert_eq!( + evidence(resolve( + RecordResolutionKeyKind::Basename, + &["deep/nested/a.md", "z/a.md"] + )) + .0, + ResolutionReason::ShallowestPath + ); + assert_eq!( + evidence(resolve( + RecordResolutionKeyKind::Basename, + &["z/a.md", "a/a.md"] + )) + .0, + ResolutionReason::LexicalTieBreak + ); + + let many = (0..MAX_RESOLUTION_ALTERNATIVES) + .map(|index| format!("z{index:03}/a.md")) + .chain(std::iter::once("a/a.md".to_string())) + .collect::>(); + let ranked = select_resolution_candidate( + "notes/source.md", + RecordResolutionKeyKind::Basename, + many.into_iter().map(|path| RankedResolutionCandidate { + record_id: path.clone(), + path, + }), + ) + .unwrap(); + let (_, alternatives) = evidence(ranked); + assert_eq!(alternatives.len(), MAX_RESOLUTION_ALTERNATIVES); + assert!(alternatives.windows(2).all(|pair| pair[0] < pair[1])); + + let over_budget = (0..=MAX_RESOLUTION_CANDIDATES).map(|index| RankedResolutionCandidate { + record_id: format!("record-{index}"), + path: format!("candidate-{index}.md"), + }); + assert_eq!( + select_resolution_candidate( + "notes/source.md", + RecordResolutionKeyKind::Basename, + over_budget, + ) + .unwrap_err() + .code, + "relationship_budget_exceeded" + ); + } + + #[test] + fn invalid_candidates_are_diagnostics_not_missing() { + let error = select_resolution_candidate( + "notes/source.md", + RecordResolutionKeyKind::Basename, + [RankedResolutionCandidate { + record_id: "bad".to_string(), + path: "../outside.md".to_string(), + }], + ) + .unwrap_err(); + assert_eq!(error.code, "invalid_resolution_candidate"); } #[test] diff --git a/src/runtime/snapshot.rs b/src/runtime/snapshot.rs index 5db4901..ca7114a 100644 --- a/src/runtime/snapshot.rs +++ b/src/runtime/snapshot.rs @@ -6,8 +6,6 @@ use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use sha2::{Digest, Sha256}; use std::collections::BTreeSet; -use std::fs; -use walkdir::WalkDir; use super::ProviderError; @@ -66,14 +64,34 @@ impl Collection { /// Long-running hosts should normally call [`super::FilesystemProvider::snapshot`], /// which also holds the provider's read gate for the full capture. pub fn snapshot(&self) -> Result { - Ok(collection_snapshot(self, InvalidRecordPolicy::Strict)?.snapshot) + self.snapshot_with_context(&super::OperationContext::internal()) + } + + /// Capture with caller-owned cancellation, deadline, and finite budgets. + pub fn snapshot_with_context( + &self, + context: &super::OperationContext, + ) -> Result { + Ok(collection_snapshot(self, InvalidRecordPolicy::Strict, context)?.snapshot) } /// Capture watcher state without changing the public synchronization /// contract. Classified invalid records are reported out-of-band so the /// watcher can retain prior state; genuine capture failures remain errors. pub(crate) fn snapshot_for_watcher(&self) -> Result { - collection_snapshot(self, InvalidRecordPolicy::Observe) + collection_snapshot( + self, + InvalidRecordPolicy::Observe, + &super::OperationContext::internal(), + ) + } + + #[allow(dead_code)] // watcher command transport does not yet carry contexts + pub(crate) fn snapshot_for_watcher_with_context( + &self, + context: &super::OperationContext, + ) -> Result { + collection_snapshot(self, InvalidRecordPolicy::Observe, context) } /// Materialize one record for provider and synchronization boundaries. @@ -85,11 +103,20 @@ impl Collection { /// wire format. Typed `read` remains strict, but transport layers do not /// reimplement parsing policy. pub fn snapshot_record(&self, path: &str) -> Result { + self.snapshot_record_with_context(path, &super::OperationContext::internal()) + } + + pub fn snapshot_record_with_context( + &self, + path: &str, + context: &super::OperationContext, + ) -> Result { + context.check()?; ensure_safe_relative_path(path, self.spec_profile) .map_err(|error| ProviderError::CollectionOpen(record_read_error(path, &error)))?; let record_path = readable_record_path(self, path) .map_err(|error| ProviderError::CollectionOpen(record_read_error(path, &error)))?; - match load_snapshot_record(self, record_path.as_str())? { + match load_snapshot_record(self, record_path.as_str(), context)? { SnapshotRecordLoad::Record(record) => Ok(record), SnapshotRecordLoad::InvalidUtf8 => Err(ProviderError::CollectionOpen(format!( "collection record '{path}' contains invalid UTF-8" @@ -115,116 +142,83 @@ enum InvalidRecordPolicy { fn collection_snapshot( collection: &Collection, invalid_policy: InvalidRecordPolicy, + context: &super::OperationContext, ) -> Result { - let root = collection.root(); - let configuration = read_resource( - root.join("mdbase.yaml"), + context.check()?; + let authority = collection.held_root(); + let mut resource_entries = 1_u64; + context.check_resource_entries(resource_entries)?; + let configuration = read_resource_held( + authority, "mdbase.yaml".to_string(), CollectionSnapshotResourceKind::Configuration, + context, )?; - let report = crate::v03::inspect_collection(root); - if !report.valid { - let message = report - .diagnostics - .iter() - .find(|diagnostic| diagnostic.severity == "error") - .map(|diagnostic| diagnostic.message.as_str()) - .unwrap_or("collection validation failed"); - return Err(ProviderError::CollectionOpen(message.to_string())); - } - let mut resources = vec![configuration]; - let lock_path = root.join("mdbase.lock.yaml"); - if lock_path.exists() { - resources.push(read_resource( - lock_path, - "mdbase.lock.yaml".to_string(), - CollectionSnapshotResourceKind::Lock, - )?); - } - let provision_lock_path = root.join("mdbase.provisions.yaml"); - if provision_lock_path.exists() { - resources.push(read_resource( - provision_lock_path, - "mdbase.provisions.yaml".to_string(), - CollectionSnapshotResourceKind::Lock, - )?); - } - for type_file in report.types { - resources.push(read_resource( - root.join(&type_file.path), - type_file.path, - CollectionSnapshotResourceKind::Type, - )?); - } - let contracts_root = root.join(&collection.settings().contracts_folder); - if contracts_root.exists() { - for entry in WalkDir::new(&contracts_root) - .follow_links(false) - .sort_by_file_name() + for path in authority + .files_recursive(std::path::Path::new("")) + .map_err(|error| { + ProviderError::CollectionOpen(format!("failed to inspect resources: {error}")) + })? + { + context.check()?; + let portable = path.to_string_lossy().replace('\\', "/"); + let kind = if matches!( + portable.as_str(), + "mdbase.lock.yaml" | "mdbase.provisions.yaml" + ) { + Some(CollectionSnapshotResourceKind::Lock) + } else if path.starts_with(&collection.settings().types_folder) + && matches!( + path.extension().and_then(|value| value.to_str()), + Some("md" | "yaml" | "yml") + ) { - let entry = entry.map_err(|error| { - ProviderError::CollectionOpen(format!( - "failed to inspect contracts folder: {error}" - )) - })?; - if !entry.file_type().is_file() - || entry.path().extension().and_then(|value| value.to_str()) != Some("md") - { - continue; - } - let path = relative_resource_path(root, entry.path())?; - resources.push(read_resource( - entry.path().to_path_buf(), - path, - CollectionSnapshotResourceKind::Contract, - )?); - } - } - for entry in WalkDir::new(root).follow_links(false).sort_by_file_name() { - let entry = entry.map_err(|error| { - ProviderError::CollectionOpen(format!("failed to inspect schema resources: {error}")) - })?; - if !entry.file_type().is_file() - || entry.path().extension().and_then(|value| value.to_str()) != Some("json") + Some(CollectionSnapshotResourceKind::Type) + } else if path.starts_with(&collection.settings().contracts_folder) + && path.extension().and_then(|value| value.to_str()) == Some("md") { - continue; - } - let path = relative_resource_path(root, entry.path())?; - if !is_schema_resource_path(&path) { - continue; + Some(CollectionSnapshotResourceKind::Contract) + } else if path.extension().and_then(|value| value.to_str()) == Some("json") + && is_schema_resource_path(&portable) + { + Some(CollectionSnapshotResourceKind::Schema) + } else if path.extension().and_then(|value| value.to_str()) == Some("base") + && !crate::record_path::has_hidden_component(&portable) + && crate::views::is_configured_obsidian_source(collection, &portable) + { + Some(CollectionSnapshotResourceKind::View) + } else { + None + }; + if let Some(kind) = kind { + resource_entries = + resource_entries + .checked_add(1) + .ok_or(crate::runtime::CaptureLimitExceeded { + kind: crate::runtime::CaptureLimitKind::ArithmeticOverflow, + limit: u64::MAX, + attempted: u64::MAX, + })?; + context.check_resource_entries(resource_entries)?; + resources.push(read_resource_held(authority, portable, kind, context)?); } - resources.push(read_resource( - entry.path().to_path_buf(), - path, - CollectionSnapshotResourceKind::Schema, - )?); } - for view_file in crate::views::compatibility_source_paths(collection) { - let path = view_file - .strip_prefix(root) - .map_err(|error| ProviderError::CollectionOpen(error.to_string()))? - .to_string_lossy() - .replace('\\', "/"); - resources.push(read_resource( - view_file, - path, - CollectionSnapshotResourceKind::View, - )?); - } - let mut paths = collection - .scan_collection_files_checked() - .map_err(|error| ProviderError::CollectionOpen(error.to_string()))?; - paths.sort(); - let mut records = Vec::with_capacity(paths.len()); + context.check()?; + let paths = collection.scan_collection_relative_paths_context(context)?; + context.check()?; + let mut records = Vec::new(); + records + .try_reserve(paths.len()) + .map_err(|_| crate::runtime::CaptureLimitExceeded { + kind: crate::runtime::CaptureLimitKind::ArithmeticOverflow, + limit: usize::MAX as u64, + attempted: paths.len() as u64, + })?; let mut invalid_records = BTreeSet::new(); - for absolute in paths { - let path = absolute - .strip_prefix(root) - .map_err(|error| ProviderError::CollectionOpen(error.to_string()))? - .to_string_lossy() - .replace('\\', "/"); - let record = match load_snapshot_record(collection, &path)? { + for path in paths { + context.check()?; + let record = match load_snapshot_record(collection, &path, context)? { SnapshotRecordLoad::Record(record) => { if record.frontmatter_error.is_some() && matches!(invalid_policy, InvalidRecordPolicy::Observe) @@ -266,6 +260,7 @@ fn collection_snapshot( records.push(record); } } + context.check()?; resources[1..].sort_by(|left, right| left.path.cmp(&right.path)); let spec_version = serde_yaml::from_str::(&resources[0].document) @@ -310,13 +305,10 @@ enum SnapshotRecordLoad { fn load_snapshot_record( collection: &Collection, path: &str, + context: &super::OperationContext, ) -> Result { let Some(outcome) = - crate::record_load::load_record_no_follow(collection, path).map_err(|error| { - ProviderError::CollectionOpen(format!( - "failed to read collection record '{path}': {error}" - )) - })? + crate::record_load::load_record_no_follow_context(collection, path, context)? else { return Ok(SnapshotRecordLoad::Absent); }; @@ -428,14 +420,61 @@ fn resource_revision(resources: &[CollectionSnapshotResource]) -> String { format!("sha256:{:x}", digest.finalize()) } -fn read_resource( - absolute: std::path::PathBuf, +fn read_resource_held( + root: &crate::collection_root::CollectionRoot, path: String, kind: CollectionSnapshotResourceKind, + context: &super::OperationContext, ) -> Result { - let bytes = fs::read(&absolute).map_err(|error| { - ProviderError::CollectionOpen(format!("failed to read {path}: {error}")) + use std::io::Read; + context.check()?; + let mut file = root + .open_file(std::path::Path::new(&path)) + .map_err(|error| { + ProviderError::CollectionOpen(format!("failed to open {path}: {error}")) + })?; + let size = file + .metadata() + .map_err(|error| { + ProviderError::CollectionOpen(format!("failed to inspect {path}: {error}")) + })? + .len(); + context.check_file_bytes(size)?; + let capacity = usize::try_from(size).map_err(|_| crate::runtime::CaptureLimitExceeded { + kind: crate::runtime::CaptureLimitKind::ArithmeticOverflow, + limit: usize::MAX as u64, + attempted: size, })?; + let mut bytes = Vec::new(); + bytes + .try_reserve_exact(capacity) + .map_err(|_| crate::runtime::CaptureLimitExceeded { + kind: crate::runtime::CaptureLimitKind::ArithmeticOverflow, + limit: usize::MAX as u64, + attempted: size, + })?; + let mut chunk = [0_u8; 64 * 1024]; + loop { + context.check()?; + let read = file.read(&mut chunk).map_err(|error| { + ProviderError::CollectionOpen(format!("failed to read {path}: {error}")) + })?; + if read == 0 { + break; + } + let attempted = (bytes.len() as u64).checked_add(read as u64).ok_or( + crate::runtime::CaptureLimitExceeded { + kind: crate::runtime::CaptureLimitKind::ArithmeticOverflow, + limit: u64::MAX, + attempted: u64::MAX, + }, + )?; + context.check_file_bytes(attempted)?; + context.charge_read(read as u64)?; + context.charge_retained(read as u64)?; + bytes.extend_from_slice(&chunk[..read]); + context.check()?; + } let document = String::from_utf8(bytes.clone()).map_err(|error| { ProviderError::CollectionOpen(format!("{path} is not valid UTF-8: {error}")) })?; @@ -447,16 +486,6 @@ fn read_resource( }) } -fn relative_resource_path( - root: &std::path::Path, - absolute: &std::path::Path, -) -> Result { - absolute - .strip_prefix(root) - .map(|path| path.to_string_lossy().replace('\\', "/")) - .map_err(|error| ProviderError::CollectionOpen(error.to_string())) -} - fn snapshot_revision( resources: &[CollectionSnapshotResource], records: &[CollectionSnapshotRecord], diff --git a/src/runtime/tests.rs b/src/runtime/tests.rs index 022577c..0380721 100644 --- a/src/runtime/tests.rs +++ b/src/runtime/tests.rs @@ -34,6 +34,211 @@ fn collection() -> tempfile::TempDir { directory } +#[test] +fn authority_capture_sources_reject_fresh_tokens_and_unbounded_reads() { + let sources = [ + ("snapshot", include_str!("../snapshot.rs")), + ("discovery", include_str!("../snapshot/discovery.rs")), + ("record_load", include_str!("../record_load.rs")), + ("runtime_snapshot", include_str!("snapshot.rs")), + ("mutation_shadow", include_str!("../mutation/shadow.rs")), + ( + "mutation_preparation", + include_str!("../mutation/preparation.rs"), + ), + ("mutation_batch", include_str!("../mutation/batch.rs")), + ("runtime_batch", include_str!("../v03/batch.rs")), + ("links", include_str!("../links/traversal.rs")), + ("validation", include_str!("../validation/validator.rs")), + ("backfill", include_str!("../operations/backfill.rs")), + ]; + for (name, source) in sources { + let production = source.split("#[cfg(test)]").next().unwrap_or(source); + assert!( + !production.contains("OperationCancellation::new()"), + "{name}" + ); + assert!(!production.contains("fs::read("), "{name}"); + assert!(!production.contains("fs::read_to_string("), "{name}"); + assert!(!production.contains("collection.snapshot()"), "{name}"); + assert!( + !production.contains("shadow.collection.snapshot()"), + "{name}" + ); + } +} + +#[test] +fn provider_capture_limits_are_exact_and_never_publish_partial_snapshots() { + let directory = collection(); + let config = fs::read(directory.path().join("mdbase.yaml")).unwrap(); + let record = b"---\ntitle: bounded\n---\nbody\n"; + fs::write(directory.path().join("bounded.md"), record).unwrap(); + let provider = FilesystemProvider::open(directory.path()).unwrap(); + let total = (config.len() + record.len()) as u64; + let limits = CaptureLimits::builder() + .max_entries(1) + .max_resource_entries(1) + .max_file_bytes(config.len().max(record.len()) as u64) + .max_aggregate_bytes(total) + .max_retained_bytes(total) + .build(); + let context = OperationContext::with_capture_limits( + &crate::OperationCancellation::new(), + OperationDeadline::after(Duration::from_secs(1)), + limits, + ); + let snapshot = provider.snapshot_with_context(&context).unwrap(); + assert_eq!(snapshot.records.len(), 1); + assert_eq!(snapshot.resources.len(), 1); + + let too_small = CaptureLimits::builder() + .max_entries(1) + .max_resource_entries(1) + .max_file_bytes(config.len().max(record.len()) as u64) + .max_aggregate_bytes(total - 1) + .max_retained_bytes(total) + .build(); + let context = OperationContext::with_capture_limits( + &crate::OperationCancellation::new(), + OperationDeadline::after(Duration::from_secs(1)), + too_small, + ); + assert!(matches!( + provider.snapshot_with_context(&context), + Err(ProviderError::CaptureLimitExceeded(CaptureLimitExceeded { + kind: CaptureLimitKind::AggregateBytes, + .. + })) + )); +} + +#[test] +fn provider_capture_rejects_oversized_single_record_and_entry_boundary() { + let directory = collection(); + fs::write(directory.path().join("large.md"), vec![b'x'; 128]).unwrap(); + let provider = FilesystemProvider::open(directory.path()).unwrap(); + let limits = CaptureLimits::builder() + .max_entries(1) + .max_file_bytes(127) + .build(); + let context = OperationContext::with_capture_limits( + &crate::OperationCancellation::new(), + OperationDeadline::after(Duration::from_secs(1)), + limits, + ); + assert!(matches!( + provider.snapshot_with_context(&context), + Err(ProviderError::CaptureLimitExceeded(CaptureLimitExceeded { + kind: CaptureLimitKind::FileBytes, + limit: 127, + attempted: 128 + })) + )); + + let limits = CaptureLimits::builder().max_entries(0).build(); + let context = OperationContext::with_capture_limits( + &crate::OperationCancellation::new(), + OperationDeadline::after(Duration::from_secs(1)), + limits, + ); + assert!(matches!( + provider.snapshot_with_context(&context), + Err(ProviderError::CaptureLimitExceeded(CaptureLimitExceeded { + kind: CaptureLimitKind::Entries, + limit: 0, + attempted: 1 + })) + )); +} + +#[test] +fn query_capture_limit_is_terminal_before_cache_fallback_and_scans_once() { + let directory = collection(); + fs::write( + directory.path().join("bounded.md"), + "---\ntitle: bounded\n---\n", + ) + .unwrap(); + let provider = FilesystemProvider::open(directory.path()).unwrap(); + let request = OperationRequest::new(OperationKind::Query, json!({})); + + for cache_state in ["missing", "warmed"] { + if cache_state == "warmed" { + provider.execute(&request).unwrap(); + } else { + let _ = fs::remove_dir_all(directory.path().join(".mdbase/cache")); + } + crate::reset_snapshot_scan_calls_for_test(); + let limits = CaptureLimits::builder().max_entries(0).build(); + let context = OperationContext::with_capture_limits( + &crate::OperationCancellation::new(), + OperationDeadline::after(Duration::from_secs(2)), + limits, + ); + assert!(matches!( + provider.execute_with_context(&request, &context), + Err(ProviderError::CaptureLimitExceeded(CaptureLimitExceeded { + kind: CaptureLimitKind::Entries, + limit: 0, + attempted: 1, + })) + )); + assert_eq!( + crate::snapshot_scan_calls_for_test(), + 1, + "{cache_state} cache must not trigger a fallback scan" + ); + } +} + +#[test] +fn zero_retained_bytes_rejects_nonempty_regular_and_cache_backed_queries() { + let directory = collection(); + fs::write( + directory.path().join("bounded.md"), + "---\ntitle: bounded\n---\n", + ) + .unwrap(); + let provider = FilesystemProvider::open(directory.path()).unwrap(); + let runtime = FilesystemRuntime::open(directory.path(), Duration::from_millis(5)).unwrap(); + let request = OperationRequest::new(OperationKind::Query, json!({})); + let provider_context = OperationContext::with_capture_limits( + &crate::OperationCancellation::new(), + OperationDeadline::after(Duration::from_secs(2)), + CaptureLimits::builder().max_retained_bytes(0).build(), + ); + assert!(matches!( + provider.execute_with_context(&request, &provider_context), + Err(ProviderError::CaptureLimitExceeded(CaptureLimitExceeded { + kind: CaptureLimitKind::RetainedBytes, + limit: 0, + .. + })) + )); + + for paged in [false, true] { + let context = OperationContext::with_capture_limits( + &crate::OperationCancellation::new(), + OperationDeadline::after(Duration::from_secs(2)), + CaptureLimits::builder().max_retained_bytes(0).build(), + ); + let result = if paged { + runtime.open_read(&request, &context).map(|_| ()) + } else { + runtime.read(&request, &context).map(|_| ()) + }; + assert!(matches!( + result, + Err(ProviderError::CaptureLimitExceeded(CaptureLimitExceeded { + kind: CaptureLimitKind::RetainedBytes, + limit: 0, + .. + })) + )); + } +} + fn runtime_query_paths(runtime: &FilesystemRuntime) -> BTreeSet { let outcome = runtime .read( @@ -2482,10 +2687,13 @@ fn runtime_rejects_a_conditional_plan_when_authority_changes_after_shadow_captur let read = runtime .read( &OperationRequest::new(OperationKind::Read, json!({"path": "task.md"})), - &OperationContext::legacy(), + &OperationContext::internal(), ) .unwrap(); - let revision = read.result.result["revision"].as_str().unwrap().to_string(); + let CanonicalOperationValue::Read(Some(record)) = read.operation.value() else { + panic!("typed read outcome omitted its record") + }; + let revision = record.revision.as_str().to_string(); let entered = Arc::new(Barrier::new(2)); let release = Arc::new(Barrier::new(2)); crate::v03::batch::set_sparse_preparation_pause( @@ -3388,9 +3596,11 @@ fn invalid_maintenance_rejects_ambiguous_hints_and_repairs_exact_cache_shape() { ); let collection = Collection::open(directory.path()).unwrap(); - let connection = - crate::cache::sqlite::open_cache_db(&collection.root, &collection.settings.cache_folder) - .unwrap(); + let connection = crate::cache::sqlite::open_cache_db( + collection.held_root().cache_storage_path(), + &collection.settings.cache_folder, + ) + .unwrap(); connection .execute( "UPDATE files SET frontmatter_json = '{\"bad\":true}', body = 'leak', effective_json = '{\"bad\":true}' WHERE path = 'invalid.md'", @@ -3424,9 +3634,11 @@ fn invalid_maintenance_rejects_ambiguous_hints_and_repairs_exact_cache_shape() { apply(BTreeSet::from(["invalid.md".to_string()]), BTreeSet::new()), InvalidMaintenanceOutcome::Applied(_) )); - let connection = - crate::cache::sqlite::open_cache_db(&collection.root, &collection.settings.cache_folder) - .unwrap(); + let connection = crate::cache::sqlite::open_cache_db( + collection.held_root().cache_storage_path(), + &collection.settings.cache_folder, + ) + .unwrap(); let canonical = connection .query_row( "SELECT frontmatter_json, body, effective_json, parse_error, failure_reason FROM files WHERE path = 'invalid.md'", @@ -3509,7 +3721,7 @@ fn maintenance_seals_are_single_successor_epoch_and_cache_identity_bound() { let collection = Collection::open(directory.path()).unwrap(); let logical_state = || { let connection = crate::cache::sqlite::open_cache_db_read_only_existing( - &collection.root, + collection.held_root().cache_storage_path(), &collection.settings.cache_folder, ) .unwrap(); @@ -3559,7 +3771,7 @@ fn maintenance_seals_are_single_successor_epoch_and_cache_identity_bound() { )); let data_seal = make_seal(4); - let writer_root = collection.root.clone(); + let writer_root = collection.held_root().cache_storage_path().to_path_buf(); let writer_cache_folder = collection.settings.cache_folder.clone(); crate::cache::runtime::set_seal_validation_hook( &collection, @@ -3622,9 +3834,11 @@ fn maintenance_seals_are_single_successor_epoch_and_cache_identity_bound() { } let schema_seal = make_seal(5); - let connection = - crate::cache::sqlite::open_cache_db(&collection.root, &collection.settings.cache_folder) - .unwrap(); + let connection = crate::cache::sqlite::open_cache_db( + collection.held_root().cache_storage_path(), + &collection.settings.cache_folder, + ) + .unwrap(); connection .execute_batch("ALTER TABLE files ADD COLUMN seal_schema_drift INTEGER;") .unwrap(); @@ -3734,7 +3948,10 @@ fn seal_identity_replacement_between_every_validation_boundary_is_rejected() { // Deliberately bypass the advisory lifecycle contract to model raw // Unix unlink/recreate. Official cache_clear/cache_rebuild cannot do // this while the seal holds its shared guard. - let cache_root = replacement.root.join(&replacement.settings.cache_folder); + let cache_root = replacement + .held_root() + .cache_storage_path() + .join(&replacement.settings.cache_folder); for name in ["cache.db", "cache.db-wal", "cache.db-shm"] { match fs::remove_file(cache_root.join(name)) { Ok(()) => {} @@ -3743,7 +3960,7 @@ fn seal_identity_replacement_between_every_validation_boundary_is_rejected() { } } let mut connection = crate::cache::sqlite::open_cache_db( - &replacement.root, + replacement.held_root().cache_storage_path(), &replacement.settings.cache_folder, ) .unwrap(); @@ -3777,9 +3994,11 @@ fn stale_invalid_maintenance_cannot_rewind_or_contaminate_successor_cache() { .unwrap(); let collection = Collection::open(directory.path()).unwrap(); - let connection = - crate::cache::sqlite::open_cache_db(&collection.root, &collection.settings.cache_folder) - .unwrap(); + let connection = crate::cache::sqlite::open_cache_db( + collection.held_root().cache_storage_path(), + &collection.settings.cache_folder, + ) + .unwrap(); let old_revision = connection .query_row( "SELECT source_revision FROM files WHERE path = 'invalid.md'", @@ -3826,9 +4045,11 @@ fn stale_invalid_maintenance_cannot_rewind_or_contaminate_successor_cache() { assert!(crate::cache::runtime::matches_generation(&collection, &successor).unwrap()); assert!(!crate::cache::runtime::matches_generation(&collection, &expected).unwrap()); - let connection = - crate::cache::sqlite::open_cache_db(&collection.root, &collection.settings.cache_folder) - .unwrap(); + let connection = crate::cache::sqlite::open_cache_db( + collection.held_root().cache_storage_path(), + &collection.settings.cache_folder, + ) + .unwrap(); let revision = connection .query_row( "SELECT source_revision FROM files WHERE path = 'invalid.md'", @@ -3853,7 +4074,7 @@ fn runtime_reverse_link_index_tracks_resolved_targets_incrementally() { .provider() .with_collection_read(|collection| { let connection = crate::cache::sqlite::open_cache_db( - &collection.root, + collection.held_root().cache_storage_path(), &collection.settings.cache_folder, ) .map_err(|error| ProviderError::CollectionOpen(error.to_string()))?; @@ -3875,7 +4096,7 @@ fn runtime_reverse_link_index_tracks_resolved_targets_incrementally() { .provider() .with_collection_read(|collection| { let connection = crate::cache::sqlite::open_cache_db( - &collection.root, + collection.held_root().cache_storage_path(), &collection.settings.cache_folder, ) .map_err(|error| ProviderError::CollectionOpen(error.to_string()))?; @@ -4027,3 +4248,125 @@ fn observer_reports_performance_when_provider_execution_fails_early() { assert_eq!(errors[0].code, error.code()); assert!(errors[0].message.is_none()); } + +#[cfg(unix)] +#[test] +fn held_resource_reads_reject_hardlinks_from_opened_nofollow_handles() { + let config_root = collection(); + let config = Collection::open(config_root.path()).unwrap(); + fs::hard_link( + config_root.path().join("mdbase.yaml"), + config_root.path().join("config-link.yaml"), + ) + .unwrap(); + assert_eq!( + crate::config::load_config_for_open_held(config.held_root())["valid"], + false + ); + + let resource_root = collection(); + fs::write(resource_root.path().join("schema.json"), "{}\n").unwrap(); + let resource = Collection::open(resource_root.path()).unwrap(); + fs::hard_link( + resource_root.path().join("schema.json"), + resource_root.path().join("schema-link.json"), + ) + .unwrap(); + assert!(resource.held_root().read("schema.json").is_err()); + + let shadow_root = collection(); + fs::write(shadow_root.path().join("record.md"), "record\n").unwrap(); + let shadow = Collection::open(shadow_root.path()).unwrap(); + fs::hard_link( + shadow_root.path().join("record.md"), + shadow_root.path().join("record-link.md"), + ) + .unwrap(); + assert!(crate::mutation::shadow::shadow_collection(&shadow).is_err()); +} + +#[cfg(all(unix, feature = "legacy-collection-mutation"))] +#[test] +fn held_authority_never_adopts_a_replacement_root_across_refresh_snapshot_cache_and_legacy_mutation( +) { + let directory = collection(); + let root = directory.path().to_path_buf(); + fs::write(root.join("authority.md"), "original\n").unwrap(); + let collection = Collection::open(&root).unwrap(); + let provider = FilesystemProvider::open(&root).unwrap(); + + let held = root.with_extension("held-authority"); + let swap_root = root.clone(); + let swap_held = held.clone(); + crate::cache::set_cache_access_hook(&root, move || { + fs::rename(&swap_root, &swap_held).unwrap(); + fs::create_dir(&swap_root).unwrap(); + fs::write( + swap_root.join("mdbase.yaml"), + "spec_version: 0.3.0\nx-replacement: true\n", + ) + .unwrap(); + fs::write(swap_root.join("replacement-only.md"), "replacement\n").unwrap(); + }); + + // The deterministic swap occurs after cache authority was acquired but + // immediately before SQLite access. SQLite remains in identity-bound + // private storage and cannot create files in the replacement collection. + let cache = collection.cache_rebuild(); + assert_eq!(cache["success"], true, "{cache:?}"); + assert!(!root.join(".mdbase").exists()); + + let _ = provider.refresh(); + let snapshot = provider.snapshot().unwrap(); + assert!(snapshot + .records + .iter() + .any(|record| record.path == "authority.md")); + assert!(!snapshot + .records + .iter() + .any(|record| record.path == "replacement-only.md")); + + let shadow = crate::mutation::shadow::shadow_collection(&collection).unwrap(); + assert!(shadow.baseline.contains_key("authority.md")); + assert!(!shadow.baseline.contains_key("replacement-only.md")); + + let batch = crate::v03::batch::execute( + &collection, + &serde_json::json!({ + "operations": [{"kind": "create", "input": {"path": "transaction.md", "body": "held"}}] + }), + ); + assert!(batch.valid, "{batch:?}"); + assert!(held.join("transaction.md").is_file()); + assert!(!root.join("transaction.md").exists()); + assert!(!root.join(".mdbase").exists()); + + let created = + collection.create_legacy(&serde_json::json!({"path": "created.md", "body": "held"})); + assert!(created.get("error").is_none(), "{created:?}"); + assert!(held.join("created.md").is_file()); + assert!(!root.join("created.md").exists()); + let updated = + collection.update_legacy(&serde_json::json!({"path": "authority.md", "body": "updated"})); + assert!(updated.get("error").is_none(), "{updated:?}"); + assert_eq!( + fs::read_to_string(held.join("authority.md")).unwrap(), + "updated" + ); + let deleted = collection.delete_legacy(&serde_json::json!({"path": "authority.md"})); + assert!(deleted.get("error").is_none(), "{deleted:?}"); + assert!(!held.join("authority.md").exists()); + assert_eq!( + fs::read_to_string(root.join("replacement-only.md")).unwrap(), + "replacement\n" + ); + assert_eq!( + fs::read_to_string(root.join("mdbase.yaml")).unwrap(), + "spec_version: 0.3.0\nx-replacement: true\n" + ); + assert_eq!(fs::read_dir(&root).unwrap().count(), 2); + + fs::remove_dir_all(&root).unwrap(); + fs::rename(&held, &root).unwrap(); +} diff --git a/src/snapshot.rs b/src/snapshot.rs index 295f8c0..e8f4e4d 100644 --- a/src/snapshot.rs +++ b/src/snapshot.rs @@ -2,10 +2,12 @@ mod discovery; +#[cfg(all(test, unix))] +pub(crate) use discovery::replace_descendant_on_scan_for_test; #[cfg(test)] pub(crate) use discovery::{ - cancel_scan_after_entries_for_test, replace_descendant_on_scan_for_test, - reset_snapshot_scan_calls_for_test, snapshot_scan_calls_for_test, + cancel_scan_after_entries_for_test, reset_snapshot_scan_calls_for_test, + snapshot_scan_calls_for_test, }; use std::collections::HashMap; @@ -126,7 +128,7 @@ impl AuthoritativeCollectionSnapshot { } pub(crate) fn entry(&self, path: &str) -> Option<&AuthoritativeCollectionSnapshotEntry> { - #[cfg(test)] + #[cfg(all(test, feature = "legacy-collection-mutation"))] SNAPSHOT_ENTRY_LOOKUPS.with(|lookups| lookups.set(lookups.get() + 1)); self.path_to_index .get(path) @@ -134,7 +136,7 @@ impl AuthoritativeCollectionSnapshot { } pub(crate) fn resolved_files_data(&self) -> Vec { - #[cfg(test)] + #[cfg(all(test, feature = "legacy-collection-mutation"))] SNAPSHOT_RESOLVED_PROJECTIONS.with(|builds| builds.set(builds.get() + 1)); self.entries .iter() @@ -189,34 +191,92 @@ impl AuthoritativeCollectionSnapshot { } } -#[cfg(test)] +#[cfg(all(test, feature = "legacy-collection-mutation"))] thread_local! { static SNAPSHOT_ENTRY_LOOKUPS: std::cell::Cell = const { std::cell::Cell::new(0) }; static SNAPSHOT_RESOLVED_PROJECTIONS: std::cell::Cell = const { std::cell::Cell::new(0) }; } -#[cfg(test)] +#[cfg(all(test, feature = "legacy-collection-mutation"))] pub(crate) fn reset_snapshot_projection_counters_for_test() { SNAPSHOT_ENTRY_LOOKUPS.with(|value| value.set(0)); SNAPSHOT_RESOLVED_PROJECTIONS.with(|value| value.set(0)); } -#[cfg(test)] +#[cfg(all(test, feature = "legacy-collection-mutation"))] pub(crate) fn snapshot_entry_lookups_for_test() -> usize { SNAPSHOT_ENTRY_LOOKUPS.with(std::cell::Cell::get) } -#[cfg(test)] +#[cfg(all(test, feature = "legacy-collection-mutation"))] pub(crate) fn snapshot_resolved_projections_for_test() -> usize { SNAPSHOT_RESOLVED_PROJECTIONS.with(std::cell::Cell::get) } impl Collection { + /// Capture using the caller runtime context when present, otherwise the + /// finite compatibility context owned by a context-free API. + pub(crate) fn capture_collection_snapshot_current( + &self, + ) -> Result { + let context = crate::runtime::OperationContext::current_or_legacy(); + self.capture_collection_snapshot_context(&context) + } + + /// Budgeted authoritative capture used by runtime/canonical paths. + pub(crate) fn capture_collection_snapshot_context( + &self, + context: &crate::runtime::OperationContext, + ) -> Result { + context.check()?; + let paths = self.scan_collection_all_relative_paths_context(context)?; + context.check()?; + let mut entries = Vec::new(); + entries.try_reserve(paths.len()).map_err(|_| { + crate::runtime::ProviderError::from(crate::runtime::CaptureLimitExceeded { + kind: crate::runtime::CaptureLimitKind::ArithmeticOverflow, + limit: usize::MAX as u64, + attempted: paths.len() as u64, + }) + })?; + for relative_path in &paths { + context.check()?; + if self.validate_record_path(relative_path).is_err() { + continue; + } + let display_path = self.root.join(relative_path); + let outcome = + crate::record_load::load_record_no_follow_context(self, relative_path, context)? + .ok_or_else(|| SnapshotError::Unavailable { + collection_path: relative_path.clone(), + filesystem_path: display_path, + })?; + entries.push(AuthoritativeCollectionSnapshotEntry { outcome }); + context.check()?; + } + context.check()?; + let path_to_index = entries + .iter() + .enumerate() + .map(|(index, entry)| (entry.relative_path().to_string(), index)) + .collect(); + Ok(AuthoritativeCollectionSnapshot { + entries, + path_to_index, + known_file_paths: paths, + }) + } + /// Discover, no-follow open, read, and classify every record exactly once. + /// This is a compatibility seam; runtime callers use the context variant. + #[allow(dead_code)] // retained for explicit-token compatibility and cancellation tests pub(crate) fn capture_collection_snapshot( &self, cancellation: &OperationCancellation, ) -> Result { + if let Some(context) = crate::runtime::OperationContext::current() { + return self.capture_collection_snapshot_context(&context); + } cancellation.check().map_err(|_| SnapshotError::Cancelled)?; let paths = self.scan_collection_all_relative_paths_checked_cancellable(cancellation)?; let mut entries = Vec::new(); @@ -269,6 +329,8 @@ impl Collection { pub(crate) enum CollectionScanError { #[error("collection operation cancelled")] Cancelled, + #[error(transparent)] + Provider(#[from] crate::runtime::ProviderError), #[error("failed to read collection directory '{}': {source}", path.display())] ReadDirectory { path: PathBuf, @@ -370,6 +432,8 @@ impl CollectionSnapshotError { pub(crate) enum SnapshotError { #[error("collection operation cancelled")] Cancelled, + #[error(transparent)] + Provider(#[from] crate::runtime::ProviderError), #[error("coordinated runtime cache is unavailable: {0}")] Cache(String), #[error(transparent)] @@ -381,6 +445,7 @@ pub(crate) enum SnapshotError { #[error( "discovered collection record is no longer an available regular file: {collection_path}" )] + #[allow(dead_code)] // explicit-token compatibility capture can still classify this race Unavailable { collection_path: String, filesystem_path: PathBuf, @@ -410,7 +475,11 @@ impl SnapshotError { pub(crate) fn path(&self) -> Option<&Path> { match self { - Self::Cancelled | Self::Cache(_) | Self::Scan(CollectionScanError::Cancelled) => None, + Self::Cancelled + | Self::Provider(_) + | Self::Cache(_) + | Self::Scan(CollectionScanError::Cancelled) + | Self::Scan(CollectionScanError::Provider(_)) => None, Self::Scan(CollectionScanError::ReadDirectory { path, .. }) | Self::Scan(CollectionScanError::InspectEntry { path, .. }) | Self::Scan(CollectionScanError::NonUtf8Path { path }) @@ -431,6 +500,7 @@ impl From for SnapshotError { fn from(error: CollectionScanError) -> Self { match error { CollectionScanError::Cancelled => Self::Cancelled, + CollectionScanError::Provider(error) => Self::Provider(error), CollectionScanError::NonUtf8Path { path } => Self::NonUtf8Path { path }, other => Self::Scan(other), } @@ -443,6 +513,13 @@ impl From for CollectionSnapshotError { SnapshotError::Cancelled | SnapshotError::Scan(CollectionScanError::Cancelled) => { Self::Cancelled } + SnapshotError::Provider(crate::runtime::ProviderError::OperationCancelled) + | SnapshotError::Provider(crate::runtime::ProviderError::OperationDeadline) => { + Self::Cancelled + } + SnapshotError::Provider(error) => Self::CacheUnavailable { + reason: error.to_string(), + }, SnapshotError::Scan(CollectionScanError::ReadDirectory { path, source }) => { Self::Discovery { filesystem_path: path, @@ -471,6 +548,9 @@ impl From for CollectionSnapshotError { cause: CollectionDiscoveryCause::OutsideRoot, }, SnapshotError::Cache(reason) => Self::CacheUnavailable { reason }, + SnapshotError::Scan(CollectionScanError::Provider(error)) => Self::CacheUnavailable { + reason: error.to_string(), + }, SnapshotError::Unavailable { collection_path, filesystem_path, @@ -653,7 +733,8 @@ mod tests { assert!(matches!(error, SnapshotError::Unavailable { .. })); } - #[cfg(unix)] + // Darwin filesystems reject this byte sequence before mdbase can observe it. + #[cfg(target_os = "linux")] #[test] fn invalid_utf8_record_path_is_explicit() { use std::ffi::OsString; diff --git a/src/snapshot/discovery.rs b/src/snapshot/discovery.rs index 430acc7..b36d0b7 100644 --- a/src/snapshot/discovery.rs +++ b/src/snapshot/discovery.rs @@ -1,10 +1,17 @@ use std::path::PathBuf; -#[cfg(all(test, unix))] +#[cfg(test)] use std::path::Path; +use crate::runtime::{OperationContext, ProviderError}; use crate::{Collection, OperationCancellation}; +struct ScanState<'a> { + records_only: bool, + context: Option<&'a OperationContext>, + discovered: u64, +} + impl Collection { /// Scan all Markdown files in the collection. /// @@ -13,7 +20,15 @@ impl Collection { pub(crate) fn scan_collection_files_checked( &self, ) -> Result, crate::snapshot::CollectionScanError> { - self.scan_collection_files_checked_cancellable(&OperationCancellation::new()) + let context = OperationContext::current_or_legacy(); + self.scan_collection_files_checked_cancellable(context.cancellation()) + } + + pub(crate) fn scan_collection_relative_paths_checked( + &self, + ) -> Result, crate::snapshot::CollectionScanError> { + let context = OperationContext::current_or_legacy(); + self.scan_collection_relative_paths_checked_cancellable(context.cancellation()) } pub(crate) fn scan_collection_files_checked_cancellable( @@ -28,16 +43,41 @@ impl Collection { &self, cancellation: &OperationCancellation, ) -> Result, crate::snapshot::CollectionScanError> { + if let Some(context) = OperationContext::current() { + return self + .scan_collection_relative_paths_mode_context(&context, true) + .map_err(crate::snapshot::CollectionScanError::Provider); + } self.scan_collection_relative_paths_mode(cancellation, true) } + #[allow(dead_code)] // retained by the explicit-token compatibility capture pub(crate) fn scan_collection_all_relative_paths_checked_cancellable( &self, cancellation: &OperationCancellation, ) -> Result, crate::snapshot::CollectionScanError> { + if let Some(context) = OperationContext::current() { + return self + .scan_collection_relative_paths_mode_context(&context, false) + .map_err(crate::snapshot::CollectionScanError::Provider); + } self.scan_collection_relative_paths_mode(cancellation, false) } + pub(crate) fn scan_collection_all_relative_paths_context( + &self, + context: &OperationContext, + ) -> Result, ProviderError> { + self.scan_collection_relative_paths_mode_context(context, false) + } + + pub(crate) fn scan_collection_relative_paths_context( + &self, + context: &OperationContext, + ) -> Result, ProviderError> { + self.scan_collection_relative_paths_mode_context(context, true) + } + fn scan_collection_relative_paths_mode( &self, cancellation: &OperationCancellation, @@ -52,7 +92,12 @@ impl Collection { } })?; let mut files = Vec::new(); - self.scan_dir_recursive_checked(&root, "", &mut files, cancellation, records_only)?; + let mut state = ScanState { + records_only, + context: None, + discovered: 0, + }; + self.scan_dir_recursive_checked(&root, "", &mut files, cancellation, 0, &mut state)?; cancellation .check() .map_err(|_| crate::snapshot::CollectionScanError::Cancelled)?; @@ -60,13 +105,52 @@ impl Collection { Ok(files) } + fn scan_collection_relative_paths_mode_context( + &self, + context: &OperationContext, + records_only: bool, + ) -> Result, ProviderError> { + #[cfg(test)] + SNAPSHOT_SCAN_CALLS.with(|calls| calls.set(calls.get() + 1)); + context.check()?; + let root = self.root_capability().map_err(|error| { + ProviderError::CollectionOpen(format!("failed to read collection directory: {error}")) + })?; + let mut files = Vec::new(); + let mut state = ScanState { + records_only, + context: Some(context), + discovered: 0, + }; + self.scan_dir_recursive_checked( + &root, + "", + &mut files, + context.cancellation(), + 0, + &mut state, + ) + .map_err(|error| match error { + crate::snapshot::CollectionScanError::Provider(error) => error, + crate::snapshot::CollectionScanError::Cancelled => context + .check() + .err() + .unwrap_or(ProviderError::OperationCancelled), + other => ProviderError::CollectionOpen(other.to_string()), + })?; + context.check()?; + files.sort(); + Ok(files) + } + fn scan_dir_recursive_checked( &self, directory: &cap_std::fs::Dir, prefix: &str, files: &mut Vec, cancellation: &OperationCancellation, - records_only: bool, + depth: u64, + state: &mut ScanState<'_>, ) -> Result<(), crate::snapshot::CollectionScanError> { use crate::snapshot::CollectionScanError; use cap_fs_ext::DirExt; @@ -74,6 +158,11 @@ impl Collection { cancellation .check() .map_err(|_| CollectionScanError::Cancelled)?; + if let Some(context) = state.context { + context + .check_depth(depth) + .map_err(CollectionScanError::Provider)?; + } let display_directory = if prefix.is_empty() { self.root.clone() } else { @@ -142,17 +231,51 @@ impl Collection { }); } } + let child_depth = depth.checked_add(1).ok_or({ + CollectionScanError::Provider(ProviderError::CaptureLimitExceeded( + crate::runtime::CaptureLimitExceeded { + kind: crate::runtime::CaptureLimitKind::ArithmeticOverflow, + limit: u64::MAX, + attempted: u64::MAX, + }, + )) + })?; self.scan_dir_recursive_checked( &child, &relative, files, cancellation, - records_only, + child_depth, + state, )?; } else if file_type.is_file() - && (self.validate_record_path(&relative).is_ok() - || (!records_only && self.validate_file_path(&relative).is_ok())) + && (self.validate_record_path_after_traversal(&relative).is_ok() + || (!state.records_only + && self.validate_file_path_after_traversal(&relative).is_ok())) { + state.discovered = state.discovered.checked_add(1).ok_or({ + CollectionScanError::Provider(ProviderError::CaptureLimitExceeded( + crate::runtime::CaptureLimitExceeded { + kind: crate::runtime::CaptureLimitKind::ArithmeticOverflow, + limit: u64::MAX, + attempted: u64::MAX, + }, + )) + })?; + if let Some(context) = state.context { + context + .check_entries(state.discovered) + .map_err(CollectionScanError::Provider)?; + } + files.try_reserve(1).map_err(|_| { + CollectionScanError::Provider(ProviderError::CaptureLimitExceeded( + crate::runtime::CaptureLimitExceeded { + kind: crate::runtime::CaptureLimitKind::ArithmeticOverflow, + limit: usize::MAX as u64, + attempted: u64::MAX, + }, + )) + })?; files.push(relative); } } diff --git a/src/transactions.rs b/src/transactions.rs index 9ed11d3..fcdb2c9 100644 --- a/src/transactions.rs +++ b/src/transactions.rs @@ -1,8 +1,10 @@ //! Crash-recoverable multi-file collection transactions. use std::collections::{BTreeMap, BTreeSet}; -use std::fs::{self, File, OpenOptions}; -use std::io::{self, Write}; +#[cfg(test)] +use std::fs; +use std::fs::File; +use std::io; use std::path::{Path, PathBuf}; use fs2::FileExt; @@ -10,19 +12,17 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; use crate::api::CollectionPath; -use crate::operations::{ - atomic_create, atomic_write, ensure_no_symlink_components, ensure_safe_relative_path, -}; +use crate::operations::ensure_safe_relative_path; use crate::runtime::OperationContext; use crate::{Collection, SpecProfile}; mod runtime; pub(crate) use runtime::{ ack_runtime_change_event, ack_runtime_resolution, attach_runtime_prepared, - cancel_runtime_prepared, commit_runtime_prepared, list_unacked_runtime_events, - prepare_runtime_transaction, reset_runtime_support_for_fork, resolve_runtime_claim, - resolve_runtime_commit, settle_runtime_commit, RuntimeCommitAttempt, RuntimePrepareInput, - RuntimePrepareOutcome, RuntimeResolution, RuntimeSettlement, + cancel_runtime_prepared, commit_runtime_prepared, legacy_runtime_journal_inventory, + list_unacked_runtime_events, prepare_runtime_transaction, reset_runtime_support_for_fork, + resolve_runtime_claim, resolve_runtime_commit, settle_runtime_commit, RuntimeCommitAttempt, + RuntimePrepareInput, RuntimePrepareOutcome, RuntimeResolution, RuntimeSettlement, }; #[cfg(test)] pub(crate) use runtime::{set_runtime_crash_point, set_runtime_settlement_delay}; @@ -39,6 +39,21 @@ fn post_commit_replacements() -> &'static std::sync::Mutex &'static std::sync::Mutex> { + static CRASHES: std::sync::OnceLock>> = + std::sync::OnceLock::new(); + CRASHES.get_or_init(Default::default) +} + +#[cfg(test)] +pub(crate) fn inject_commit_crash_after(root: &Path, applied: usize) { + crash_after_applied() + .lock() + .expect("commit crash hook lock") + .insert(root.to_path_buf(), applied); +} + #[cfg(test)] fn deferred_cleanup_roots() -> &'static std::sync::Mutex> { static ROOTS: std::sync::OnceLock>> = @@ -146,9 +161,14 @@ fn capture_committed_file_facts( .iter() .filter(|entry| entry.after_revision.is_some()) { - let path = CollectionPath::new(&entry.path)?.under(&collection.root); - let file = File::open(&path).map_err(|source| io_error(path.clone(), source))?; - let metadata = file.metadata().map_err(|source| io_error(path, source))?; + let relative = CollectionPath::new(&entry.path)?.to_path_buf(); + let file = collection + .held_root() + .open_file(&relative) + .map_err(|source| io_error(collection.root.join(&relative), source))?; + let metadata = file + .metadata() + .map_err(|source| io_error(collection.root.join(&relative), source))?; let mtime = metadata.modified().ok().map(|time| { let value: chrono::DateTime = time.into(); value.format("%Y-%m-%dT%H:%M:%SZ").to_string() @@ -258,12 +278,19 @@ pub(crate) fn commit_shadow( baseline: &FileBaseline, desired: &FileBaseline, ) -> Result { + #[cfg(test)] + let fail_after_applied = crash_after_applied() + .lock() + .expect("commit crash hook lock") + .remove(collection.root()); + #[cfg(not(test))] + let fail_after_applied = None; commit_shadow_controlled( collection, baseline, desired, TransactionScope::Records, - None, + fail_after_applied, ) } @@ -295,15 +322,20 @@ fn commit_shadow_controlled( let _write_lock = WriteLock::acquire(collection)?; ensure_transaction_root(collection)?; let id = uuid::Uuid::new_v4().simple().to_string(); - let directory = collection.root.join(TRANSACTIONS_DIR).join(&id); - fs::create_dir_all(directory.join("stage")) - .map_err(|source| io_error(directory.join("stage"), source))?; + let directory = PathBuf::from(TRANSACTIONS_DIR).join(&id); + collection + .held_root() + .create_dir_all(&directory.join("stage")) + .map_err(|source| io_error(collection.root.join(&directory).join("stage"), source))?; let mut staging = StagingGuard { + root: collection.held_root().clone(), directory: directory.clone(), durable: false, }; - fs::create_dir_all(directory.join("backup")) - .map_err(|source| io_error(directory.join("backup"), source))?; + collection + .held_root() + .create_dir_all(&directory.join("backup")) + .map_err(|source| io_error(collection.root.join(&directory).join("backup"), source))?; let paths = baseline .keys() @@ -321,7 +353,7 @@ fn commit_shadow_controlled( let index = entries.len(); let stage_file = after.map(|bytes| { let name = format!("stage/{index}"); - write_synced(&directory.join(&name), bytes)?; + write_synced(collection, &directory.join(&name), bytes)?; Ok::<_, TransactionError>(name) }); let stage_file = match stage_file { @@ -330,7 +362,7 @@ fn commit_shadow_controlled( }; let backup_file = before.map(|bytes| { let name = format!("backup/{index}"); - write_synced(&directory.join(&name), bytes)?; + write_synced(collection, &directory.join(&name), bytes)?; Ok::<_, TransactionError>(name) }); let backup_file = match backup_file { @@ -347,15 +379,15 @@ fn commit_shadow_controlled( } if entries.is_empty() { - cleanup_transaction(&directory); + cleanup_transaction(collection, &directory); return Ok(CommitOutcome { cleanup_deferred: false, file_facts: BTreeMap::new(), }); } - sync_dir(&directory.join("stage"))?; - sync_dir(&directory.join("backup"))?; + sync_dir(collection, &directory.join("stage"))?; + sync_dir(collection, &directory.join("backup"))?; let mut journal = Journal { version: 1, id, @@ -364,15 +396,15 @@ fn commit_shadow_controlled( applied: 0, entries, }; - persist_journal(&directory, &journal)?; + persist_journal(collection, &directory, &journal)?; staging.durable = true; if let Err(error) = recheck_preconditions(collection, &journal) { - cleanup_transaction(&directory); + cleanup_transaction(collection, &directory); return Err(error); } journal.phase = Phase::Committing; - persist_journal(&directory, &journal)?; + persist_journal(collection, &directory, &journal)?; let mut file_facts = BTreeMap::new(); for index in 0..journal.entries.len() { @@ -384,9 +416,14 @@ fn commit_shadow_controlled( )?; if journal.entries[index].after_revision.is_some() { let entry = &journal.entries[index]; - let path = CollectionPath::new(&entry.path)?.under(&collection.root); - let file = File::open(&path).map_err(|source| io_error(path.clone(), source))?; - let metadata = file.metadata().map_err(|source| io_error(path, source))?; + let relative = CollectionPath::new(&entry.path)?.to_path_buf(); + let file = collection + .held_root() + .open_file(&relative) + .map_err(|source| io_error(collection.root.join(&relative), source))?; + let metadata = file + .metadata() + .map_err(|source| io_error(collection.root.join(&relative), source))?; let mtime = metadata.modified().ok().map(|time| { let value: chrono::DateTime = time.into(); value.format("%Y-%m-%dT%H:%M:%SZ").to_string() @@ -400,7 +437,7 @@ fn commit_shadow_controlled( ); } journal.applied = index + 1; - persist_journal(&directory, &journal)?; + persist_journal(collection, &directory, &journal)?; #[cfg(test)] if _fail_after_applied == Some(journal.applied) { return Err(TransactionError::SimulatedCrash); @@ -408,7 +445,7 @@ fn commit_shadow_controlled( } journal.phase = Phase::Committed; - persist_journal(&directory, &journal)?; + persist_journal(collection, &directory, &journal)?; #[cfg(test)] apply_post_commit_hook(collection)?; #[cfg(test)] @@ -418,9 +455,10 @@ fn commit_shadow_controlled( .remove(&collection.root); #[cfg(not(test))] let injected_cleanup_deferred = false; - let cleanup_deferred = injected_cleanup_deferred || fs::remove_dir_all(&directory).is_err(); + let cleanup_deferred = + injected_cleanup_deferred || collection.held_root().remove_dir_all(&directory).is_err(); if !cleanup_deferred { - let _ = sync_dir(&collection.root.join(TRANSACTIONS_DIR)); + let _ = sync_dir(collection, Path::new(TRANSACTIONS_DIR)); } Ok(CommitOutcome { cleanup_deferred, @@ -430,38 +468,20 @@ fn commit_shadow_controlled( /// Recover every durable transaction before a collection becomes available. pub(crate) fn recover_pending(collection: &Collection) -> Result { - ensure_no_symlink_components(&collection.root, TRANSACTIONS_DIR, SpecProfile::V03) - .map_err(|error| TransactionError::UnsafePath(error.to_string()))?; - let root = collection.root.join(TRANSACTIONS_DIR); - if !root.exists() { - return Ok(false); - } - // Discover transactions only while holding the same lock used to stage, - // commit, and clean them up. Enumerating before locking leaves a stale path - // if a concurrent writer completes while recovery is waiting. + match collection.held_root().open_dir(Path::new(TRANSACTIONS_DIR)) { + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(source) => { + return Err(io_error(collection.root.join(TRANSACTIONS_DIR), source)); + } + } let _write_lock = WriteLock::acquire(collection)?; - if !root.exists() { - return Ok(false); - } - let mut directories = fs::read_dir(&root) - .map_err(|source| io_error(root.clone(), source))? - .map(|entry| { - entry - .map(|entry| entry.path()) - .map_err(|source| io_error(root.clone(), source)) - }) - .collect::, _>>()?; - directories.sort(); + let directories = collection + .held_root() + .child_directories(Path::new(TRANSACTIONS_DIR)) + .map_err(|source| io_error(collection.root.join(TRANSACTIONS_DIR), source))?; let mut changed = false; for directory in directories { - let metadata = fs::symlink_metadata(&directory) - .map_err(|source| io_error(directory.clone(), source))?; - if metadata.file_type().is_symlink() || !metadata.is_dir() { - return Err(TransactionError::ManualRecovery(format!( - "'{}' is not a regular transaction directory", - directory.display() - ))); - } changed |= recover_one(collection, &directory)?; } Ok(changed) @@ -469,11 +489,14 @@ pub(crate) fn recover_pending(collection: &Collection) -> Result Result { let journal_path = directory.join(JOURNAL_FILE); - let bytes = fs::read(&journal_path).map_err(|source| io_error(journal_path.clone(), source))?; + let bytes = collection + .held_root() + .read(&journal_path) + .map_err(|source| io_error(collection.root.join(&journal_path), source))?; let version = serde_json::from_slice::(&bytes) .ok() .and_then(|value| value.get("version").and_then(serde_json::Value::as_u64)); - if matches!(version, Some(2 | 3)) { + if matches!(version, Some(2..=4)) { return runtime::recover_runtime_one(collection, directory, &bytes); } let mut journal: Journal = serde_json::from_slice(&bytes) @@ -513,18 +536,18 @@ fn recover_one(collection: &Collection, directory: &Path) -> Result { - cleanup_transaction(directory); + cleanup_transaction(collection, directory); return Ok(true); } Phase::Committed => { - cleanup_transaction(directory); + cleanup_transaction(collection, directory); return Ok(true); } Phase::Prepared | Phase::Committing => {} } for entry in &journal.entries { if let Some(stage_file) = &entry.stage_file { - let staged = read_regular_file(&directory.join(stage_file))?; + let staged = read_regular_file(collection, &directory.join(stage_file))?; if Some(crate::v03::revision(&staged)) != entry.after_revision { return Err(TransactionError::InvalidJournal(format!( "staged contents for '{}' do not match the journal", @@ -533,7 +556,7 @@ fn recover_one(collection: &Collection, directory: &Path) -> Result Result Result { let staged_path = directory.join(stage_file); - let bytes = read_regular_file(&staged_path)?; + let bytes = read_regular_file(collection, &staged_path)?; if crate::v03::revision(&bytes) != entry.after_revision.as_deref().unwrap_or_default() { return Err(TransactionError::InvalidJournal(format!( "staged contents for '{}' do not match the journal", @@ -610,17 +630,17 @@ fn apply_entry( ))); } let result = if entry.before_revision.is_none() { - atomic_create(&path, &bytes) + collection.held_root().atomic_create(&path, &bytes) } else { - atomic_write(&path, &bytes) + collection.held_root().atomic_write(&path, &bytes) }; - result.map_err(|source| io_error(path.clone(), source))?; + result.map_err(|source| io_error(collection.root.join(&path), source))?; } None => { - fs::remove_file(&path).map_err(|source| io_error(path.clone(), source))?; - if let Some(parent) = path.parent() { - sync_dir(parent)?; - } + collection + .held_root() + .remove_file(&path) + .map_err(|source| io_error(collection.root.join(&path), source))?; } } Ok(()) @@ -667,78 +687,78 @@ fn validate_entry_path( .map_err(|error| TransactionError::UnsafePath(error.to_string()))?, }; ensure_safe_relative_path(logical.as_str(), SpecProfile::V03) - .map_err(|error| TransactionError::UnsafePath(error.to_string()))?; - ensure_no_symlink_components(&collection.root, logical.as_str(), SpecProfile::V03) .map_err(|error| TransactionError::UnsafePath(error.to_string())) + .map(|_| ()) } fn ensure_transaction_root(collection: &Collection) -> Result<(), TransactionError> { - ensure_no_symlink_components(&collection.root, TRANSACTIONS_DIR, SpecProfile::V03) - .map_err(|error| TransactionError::UnsafePath(error.to_string()))?; - let root = collection.root.join(TRANSACTIONS_DIR); - fs::create_dir_all(&root).map_err(|source| io_error(root, source)) + collection + .held_root() + .create_dir_all(Path::new(TRANSACTIONS_DIR)) + .map(|_| ()) + .map_err(|source| io_error(collection.root.join(TRANSACTIONS_DIR), source)) } -fn persist_journal(directory: &Path, journal: &Journal) -> Result<(), TransactionError> { +fn persist_journal( + collection: &Collection, + directory: &Path, + journal: &Journal, +) -> Result<(), TransactionError> { let bytes = serde_json::to_vec_pretty(journal) .map_err(|error| TransactionError::InvalidJournal(error.to_string()))?; let path = directory.join(JOURNAL_FILE); - atomic_write(&path, &bytes).map_err(|source| io_error(path, source)) + collection + .held_root() + .atomic_write(&path, &bytes) + .map_err(|source| io_error(collection.root.join(path), source)) } -fn write_synced(path: &Path, bytes: &[u8]) -> Result<(), TransactionError> { - OpenOptions::new() - .create_new(true) - .write(true) - .open(path) - .and_then(|mut file| { - file.write_all(bytes)?; - file.sync_all() - }) - .map_err(|source| io_error(path.to_path_buf(), source)) +fn write_synced( + collection: &Collection, + path: &Path, + bytes: &[u8], +) -> Result<(), TransactionError> { + collection + .held_root() + .write_new_synced(path, bytes) + .map_err(|source| io_error(collection.root.join(path), source)) } -fn read_regular_file(path: &Path) -> Result, TransactionError> { - let metadata = - fs::symlink_metadata(path).map_err(|source| io_error(path.to_path_buf(), source))?; - if metadata.file_type().is_symlink() || !metadata.is_file() { - return Err(TransactionError::InvalidJournal(format!( - "'{}' is not a regular transaction payload", - path.display() - ))); - } - fs::read(path).map_err(|source| io_error(path.to_path_buf(), source)) +fn read_regular_file(collection: &Collection, path: &Path) -> Result, TransactionError> { + collection.held_root().read(path).map_err(|source| { + if source.kind() == io::ErrorKind::PermissionDenied { + TransactionError::InvalidJournal(format!( + "'{}' is not a safe regular transaction payload", + collection.root.join(path).display() + )) + } else { + io_error(collection.root.join(path), source) + } + }) } -fn current_revision(path: &Path) -> Result, TransactionError> { - match fs::read(path) { +fn current_revision( + collection: &Collection, + path: &Path, +) -> Result, TransactionError> { + match collection.held_root().read(path) { Ok(bytes) => Ok(Some(crate::v03::revision(&bytes))), Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), - Err(source) => Err(io_error(path.to_path_buf(), source)), + Err(source) => Err(io_error(collection.root.join(path), source)), } } -#[cfg(not(windows))] -fn sync_dir(path: &Path) -> Result<(), TransactionError> { - File::open(path) - .and_then(|directory| directory.sync_all()) - .map_err(|source| io_error(path.to_path_buf(), source)) +fn sync_dir(collection: &Collection, path: &Path) -> Result<(), TransactionError> { + collection + .held_root() + .sync_dir(path) + .map_err(|source| io_error(collection.root.join(path), source)) } -#[cfg(windows)] -fn sync_dir(_path: &Path) -> Result<(), TransactionError> { - // std::fs cannot open directory handles with FILE_FLAG_BACKUP_SEMANTICS, - // so the portable directory-fsync pattern fails with AccessDenied on - // Windows. Every transaction payload and journal file is still synced - // before its atomic rename; only the additional directory metadata flush - // is unavailable through Rust's standard library. - Ok(()) -} - -fn cleanup_transaction(directory: &Path) { - if fs::remove_dir_all(directory).is_ok() { +fn cleanup_transaction(collection: &Collection, directory: &Path) { + if collection.held_root().remove_dir_all(directory).is_ok() { if let Some(parent) = directory.parent() { - let _ = sync_dir(parent); + let _ = sync_dir(collection, parent); } } } @@ -752,6 +772,7 @@ pub(crate) struct WriteLock { } struct StagingGuard { + root: crate::collection_root::CollectionRoot, directory: PathBuf, durable: bool, } @@ -759,7 +780,7 @@ struct StagingGuard { impl Drop for StagingGuard { fn drop(&mut self) { if !self.durable { - cleanup_transaction(&self.directory); + let _ = self.root.remove_dir_all(&self.directory); } } } @@ -797,21 +818,11 @@ impl WriteLock { } fn open(collection: &Collection) -> Result<(File, PathBuf), TransactionError> { - ensure_no_symlink_components( - &collection.root, - ".mdbase/write.lock", - collection.spec_profile, - ) - .map_err(|error| TransactionError::UnsafePath(error.to_string()))?; - let lock_directory = collection.root.join(".mdbase"); - fs::create_dir_all(&lock_directory).map_err(|source| io_error(lock_directory, source))?; - let path = collection.root.join(".mdbase/write.lock"); - let file = OpenOptions::new() - .create(true) - .read(true) - .write(true) - .truncate(false) - .open(&path) + let relative = Path::new(".mdbase/write.lock"); + let path = collection.root.join(relative); + let file = collection + .held_root() + .open_lock_file(relative) .map_err(|source| io_error(path.clone(), source))?; Ok((file, path)) } diff --git a/src/transactions/runtime.rs b/src/transactions/runtime.rs index f18aafb..a90bc6a 100644 --- a/src/transactions/runtime.rs +++ b/src/transactions/runtime.rs @@ -1,4 +1,5 @@ use std::collections::BTreeSet; +#[cfg(test)] use std::fs; use std::path::{Path, PathBuf}; @@ -13,12 +14,15 @@ use super::{ WriteLock, JOURNAL_FILE, TRANSACTIONS_DIR, }; use crate::runtime::{ - CanonicalChange, CanonicalOperationOutcome, ChangeBatch, ChangeBatchDescriptor, ChangeEventId, - ChangeWatermark, CollectionGeneration, CommitId, HostClaimId, OperationContext, OperationKind, + CanonicalChange, CanonicalOperationFamily, CanonicalOperationOutcome, CanonicalOperationValue, + ChangeBatch, ChangeBatchDescriptor, ChangeEventId, ChangeWatermark, CollectionGeneration, + CommitId, HostClaimId, LegacyRecoveredV03Value, OperationContext, OperationKind, + RecordChangeKind, ResourceChangeKind, }; use crate::{v03::OperationResult, Collection}; -const RUNTIME_JOURNAL_VERSION: u32 = 3; +const RUNTIME_JOURNAL_VERSION: u32 = 4; +const PHASE4_RUNTIME_JOURNAL_VERSION: u32 = 3; const LEGACY_RUNTIME_JOURNAL_VERSION: u32 = 2; const MAX_ACTIVE_RUNTIME_TRANSACTIONS: usize = 128; const MAX_RUNTIME_CHANGE_ITEMS: usize = 100_000; @@ -59,6 +63,24 @@ enum RuntimePhase { NeedsManualRecovery, } +#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "snake_case")] +enum TransitionRole { + Direct, + RenameSource, + RenameDestination, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +struct TransitionEvidence { + path: String, + before_revision: Option, + after_revision: Option, + operation: OperationKind, + change_index: usize, + role: TransitionRole, +} + #[derive(Debug, Deserialize, Serialize)] struct RuntimeJournal { version: u32, @@ -75,6 +97,8 @@ struct RuntimeJournal { operation_result: Option, change_descriptor: ChangeBatchDescriptor, changes: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + transition_evidence: Vec, event_id: ChangeEventId, generation: Option, watermark: Option, @@ -206,15 +230,20 @@ pub(crate) fn prepare_runtime_transaction( } let id = uuid::Uuid::new_v4().simple().to_string(); - let directory = collection.root.join(TRANSACTIONS_DIR).join(&id); - fs::create_dir_all(directory.join("stage")) - .map_err(|source| io_error(directory.join("stage"), source))?; + let directory = PathBuf::from(TRANSACTIONS_DIR).join(&id); + collection + .held_root() + .create_dir_all(&directory.join("stage")) + .map_err(|source| io_error(collection.root.join(&directory).join("stage"), source))?; let mut staging = StagingGuard { + root: collection.held_root().clone(), directory: directory.clone(), durable: false, }; - fs::create_dir_all(directory.join("backup")) - .map_err(|source| io_error(directory.join("backup"), source))?; + collection + .held_root() + .create_dir_all(&directory.join("backup")) + .map_err(|source| io_error(collection.root.join(&directory).join("backup"), source))?; let mut entries = Vec::with_capacity(changed_paths.len()); for path in changed_paths { @@ -226,7 +255,7 @@ pub(crate) fn prepare_runtime_transaction( let stage_file = match after { Some(bytes) => { let name = format!("stage/{index}"); - write_synced(&directory.join(&name), bytes)?; + write_synced(collection, &directory.join(&name), bytes)?; Some(name) } None => None, @@ -234,7 +263,7 @@ pub(crate) fn prepare_runtime_transaction( let backup_file = match before { Some(bytes) => { let name = format!("backup/{index}"); - write_synced(&directory.join(&name), bytes)?; + write_synced(collection, &directory.join(&name), bytes)?; Some(name) } None => None, @@ -249,9 +278,9 @@ pub(crate) fn prepare_runtime_transaction( } context_check(context)?; - sync_dir(&directory.join("stage"))?; - sync_dir(&directory.join("backup"))?; - let journal = RuntimeJournal { + sync_dir(collection, &directory.join("stage"))?; + sync_dir(collection, &directory.join("backup"))?; + let mut journal = RuntimeJournal { version: RUNTIME_JOURNAL_VERSION, id: id.clone(), scope, @@ -264,6 +293,7 @@ pub(crate) fn prepare_runtime_transaction( operation_result: None, change_descriptor: changes.descriptor().clone(), changes: changes.items().to_vec(), + transition_evidence: Vec::new(), event_id: event_id.clone(), generation: None, watermark: None, @@ -272,7 +302,9 @@ pub(crate) fn prepare_runtime_transaction( resolution_acked: false, event_acked: false, }; - persist_runtime_journal(&directory, &journal)?; + journal.transition_evidence = expected_transition_evidence(&journal)?; + validate_journal_operation_family(&journal)?; + persist_runtime_journal(collection, &directory, &journal)?; staging.durable = true; Ok(RuntimePrepareOutcome::Prepared(CommitId::from_stored(id))) } @@ -301,7 +333,7 @@ pub(crate) fn commit_runtime_prepared( let write_lock = WriteLock::acquire_context(collection, context)?; context_check(context)?; let directory = transaction_directory(collection, id); - let mut journal = read_runtime_journal(&directory)?; + let mut journal = read_runtime_journal(collection, &directory)?; match journal.phase { RuntimePhase::Prepared => {} RuntimePhase::Committing => { @@ -327,8 +359,8 @@ pub(crate) fn commit_runtime_prepared( if let Some(path) = first_precondition_conflict(collection, &journal)? { journal.phase = RuntimePhase::RejectedBeforeCommit; journal.operation_rejection = Some(conflict_result(&journal, &path)?); - release_payloads(&directory, &mut journal); - persist_runtime_journal(&directory, &journal)?; + release_payloads(collection, &directory, &mut journal); + persist_runtime_journal(collection, &directory, &journal)?; return Ok(RuntimeCommitAttempt::RejectedBeforeCommit(resolution( &journal, )?)); @@ -337,7 +369,7 @@ pub(crate) fn commit_runtime_prepared( journal.generation = Some(generation.clone()); journal.watermark = Some(watermark); journal.phase = RuntimePhase::Committing; - persist_runtime_journal(&directory, &journal)?; + persist_runtime_journal(collection, &directory, &journal)?; simulate_runtime_crash(&journal.id, 1)?; Ok(RuntimeCommitAttempt::SettlementRequired( RuntimeSettlement { @@ -352,7 +384,7 @@ pub(crate) fn settle_runtime_commit( settlement: &mut RuntimeSettlement, ) -> Result { let directory = transaction_directory(collection, settlement.commit_id()); - let mut journal = read_runtime_journal(&directory)?; + let mut journal = read_runtime_journal(collection, &directory)?; let result = match journal.phase { RuntimePhase::Committing => match settle(collection, &directory, &mut journal) { Ok(()) => resolution(&journal), @@ -363,7 +395,7 @@ pub(crate) fn settle_runtime_commit( return Err(error); } journal.phase = RuntimePhase::NeedsManualRecovery; - let _ = persist_runtime_journal(&directory, &journal); + let _ = persist_runtime_journal(collection, &directory, &journal); Err(error) } }, @@ -382,11 +414,11 @@ pub(crate) fn cancel_runtime_prepared( let _lock = WriteLock::acquire_context(collection, context)?; context_check(context)?; let directory = transaction_directory(collection, id); - let mut journal = read_runtime_journal(&directory)?; + let mut journal = read_runtime_journal(collection, &directory)?; if journal.phase == RuntimePhase::Prepared { journal.phase = RuntimePhase::CancelledBeforeCommit; - release_payloads(&directory, &mut journal); - persist_runtime_journal(&directory, &journal)?; + release_payloads(collection, &directory, &mut journal); + persist_runtime_journal(collection, &directory, &journal)?; } resolution(&journal) } @@ -400,10 +432,10 @@ pub(crate) fn resolve_runtime_commit( let _lock = WriteLock::acquire_context(collection, context)?; context_check(context)?; let directory = transaction_directory(collection, id); - if !directory.exists() { + if collection.held_root().open_dir(&directory).is_err() { return Ok(None); } - read_runtime_journal(&directory).and_then(|journal| resolution(&journal).map(Some)) + read_runtime_journal(collection, &directory).and_then(|journal| resolution(&journal).map(Some)) } pub(crate) fn resolve_runtime_claim( @@ -438,6 +470,35 @@ pub(crate) fn ack_runtime_change_event( update_ack(collection, id, context, false) } +pub(crate) fn legacy_runtime_journal_inventory( + collection: &Collection, + context: &OperationContext, +) -> Result { + context_check(context)?; + let _lock = WriteLock::acquire_context(collection, context)?; + let mut inventory = crate::runtime::LegacyJournalInventory::default(); + for directory in transaction_directories(collection)? { + context_check(context)?; + let path = directory.join(JOURNAL_FILE); + let bytes = match collection.held_root().read(&path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(source) => return Err(io_error(collection.root.join(&path), source)), + }; + let value = serde_json::from_slice::(&bytes) + .map_err(|error| TransactionError::InvalidJournal(error.to_string()))?; + if value.get("version").and_then(serde_json::Value::as_u64) + != Some(u64::from(LEGACY_RUNTIME_JOURNAL_VERSION)) + { + continue; + } + let journal = decode_runtime_journal(value)?; + validate_journal(collection, &directory, &journal)?; + inventory.version_2 += 1; + } + Ok(inventory) +} + pub(crate) fn list_unacked_runtime_events( collection: &Collection, context: &OperationContext, @@ -445,32 +506,22 @@ pub(crate) fn list_unacked_runtime_events( context_check(context)?; let _lock = WriteLock::acquire_context(collection, context)?; context_check(context)?; - let root = collection.root.join(TRANSACTIONS_DIR); - let mut directories = match fs::read_dir(&root) { - Ok(entries) => entries - .filter_map(Result::ok) - .map(|entry| entry.path()) - .collect::>(), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), - Err(source) => return Err(io_error(root, source)), - }; - directories.sort(); + let directories = transaction_directories(collection)?; let mut resolutions = Vec::new(); for directory in directories { context_check(context)?; let path = directory.join(JOURNAL_FILE); - let bytes = match fs::read(&path) { + let bytes = match collection.held_root().read(&path) { Ok(bytes) => bytes, Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, - Err(source) => return Err(io_error(path, source)), + Err(source) => return Err(io_error(collection.root.join(&path), source)), }; let value = serde_json::from_slice::(&bytes) .map_err(|error| TransactionError::InvalidJournal(error.to_string()))?; if !runtime_journal_version(&value) { continue; } - let journal: RuntimeJournal = serde_json::from_value(value) - .map_err(|error| TransactionError::InvalidJournal(error.to_string()))?; + let journal = decode_runtime_journal(value)?; validate_journal(collection, &directory, &journal)?; if journal.phase == RuntimePhase::Committed && !journal.event_acked { resolutions.push(resolution(&journal)?); @@ -489,43 +540,22 @@ pub(crate) fn reset_runtime_support_for_fork( context_check(context)?; let _lock = WriteLock::acquire_context(collection, context)?; context_check(context)?; - let root = collection.root.join(TRANSACTIONS_DIR); - let mut directories = match fs::read_dir(&root) { - Ok(entries) => entries - .map(|entry| { - entry - .map(|entry| entry.path()) - .map_err(|source| io_error(root.clone(), source)) - }) - .collect::, _>>()?, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), - Err(source) => return Err(io_error(root, source)), - }; - directories.sort(); + let directories = transaction_directories(collection)?; for directory in directories { - let metadata = fs::symlink_metadata(&directory) - .map_err(|source| io_error(directory.clone(), source))?; - if metadata.file_type().is_symlink() || !metadata.is_dir() { - return Err(TransactionError::ManualRecovery(format!( - "'{}' is not a regular transaction directory", - directory.display() - ))); - } let journal_path = directory.join(JOURNAL_FILE); - let bytes = - fs::read(&journal_path).map_err(|source| io_error(journal_path.clone(), source))?; - let version = serde_json::from_slice::(&bytes) - .ok() - .and_then(|value| value.get("version").and_then(serde_json::Value::as_u64)); - if !matches!(version, Some(value) if value == u64::from(LEGACY_RUNTIME_JOURNAL_VERSION) || value == u64::from(RUNTIME_JOURNAL_VERSION)) - { + let bytes = collection + .held_root() + .read(&journal_path) + .map_err(|source| io_error(collection.root.join(&journal_path), source))?; + let value: serde_json::Value = serde_json::from_slice(&bytes) + .map_err(|error| TransactionError::InvalidJournal(error.to_string()))?; + if !runtime_journal_version(&value) { return Err(TransactionError::ManualRecovery(format!( "non-runtime transaction '{}' remained after collection recovery", directory.display() ))); } - let mut journal: RuntimeJournal = serde_json::from_slice(&bytes) - .map_err(|error| TransactionError::InvalidJournal(error.to_string()))?; + let mut journal = decode_runtime_journal(value)?; validate_journal(collection, &directory, &journal)?; if journal.phase == RuntimePhase::Committing { settle(collection, &directory, &mut journal)?; @@ -536,9 +566,12 @@ pub(crate) fn reset_runtime_support_for_fork( journal.id ))); } - fs::remove_dir_all(&directory).map_err(|source| io_error(directory.clone(), source))?; + collection + .held_root() + .remove_dir_all(&directory) + .map_err(|source| io_error(collection.root.join(&directory), source))?; } - sync_dir(&root)?; + sync_dir(collection, Path::new(TRANSACTIONS_DIR))?; Ok(()) } @@ -552,10 +585,10 @@ fn update_ack( let _lock = WriteLock::acquire_context(collection, context)?; context_check(context)?; let directory = transaction_directory(collection, id); - if !directory.exists() { + if collection.held_root().open_dir(&directory).is_err() { return Ok(()); } - let mut journal = read_runtime_journal(&directory)?; + let mut journal = read_runtime_journal(collection, &directory)?; if resolution_ack { journal.resolution_acked = true; } else { @@ -569,9 +602,9 @@ fn update_ack( _ => false, }; if removable { - cleanup_transaction(&directory); + cleanup_transaction(collection, &directory); } else { - persist_runtime_journal(&directory, &journal)?; + persist_runtime_journal(collection, &directory, &journal)?; } Ok(()) } @@ -581,8 +614,9 @@ pub(super) fn recover_runtime_one( directory: &Path, bytes: &[u8], ) -> Result { - let mut journal: RuntimeJournal = serde_json::from_slice(bytes) + let value = serde_json::from_slice(bytes) .map_err(|error| TransactionError::InvalidJournal(error.to_string()))?; + let mut journal = decode_runtime_journal(value)?; validate_journal(collection, directory, &journal)?; match journal.phase { RuntimePhase::Prepared @@ -601,7 +635,7 @@ pub(super) fn recover_runtime_one( return Err(error); } journal.phase = RuntimePhase::NeedsManualRecovery; - let _ = persist_runtime_journal(directory, &journal); + let _ = persist_runtime_journal(collection, directory, &journal); Err(error) } }, @@ -630,12 +664,12 @@ fn settle( validate_journal(collection, directory, journal)?; for index in 0..journal.entries.len() { let entry = &journal.entries[index]; - let path = crate::api::CollectionPath::new(&entry.path)?.under(&collection.root); - let current = current_revision(&path)?; + let path = crate::api::CollectionPath::new(&entry.path)?.to_path_buf(); + let current = current_revision(collection, &path)?; if current == entry.after_revision { if journal.applied < index + 1 { journal.applied = index + 1; - persist_runtime_journal(directory, journal)?; + persist_runtime_journal(collection, directory, journal)?; } continue; } @@ -648,13 +682,13 @@ fn settle( apply_entry(collection, directory, entry, journal.scope)?; simulate_runtime_crash(&journal.id, 2)?; journal.applied = index + 1; - persist_runtime_journal(directory, journal)?; + persist_runtime_journal(collection, directory, journal)?; simulate_runtime_crash(&journal.id, 3)?; } let file_facts = capture_committed_file_facts(collection, &journal.entries)?; journal_operation_mut(journal)?.attach_committed_file_facts(&file_facts); journal.phase = RuntimePhase::Committed; - persist_runtime_journal(directory, journal)?; + persist_runtime_journal(collection, directory, journal)?; #[cfg(test)] apply_post_commit_hook(collection)?; simulate_runtime_crash(&journal.id, 4) @@ -683,8 +717,8 @@ fn first_precondition_conflict( journal: &RuntimeJournal, ) -> Result, TransactionError> { for entry in &journal.entries { - let path = crate::api::CollectionPath::new(&entry.path)?.under(&collection.root); - if current_revision(&path)? != entry.before_revision { + let path = crate::api::CollectionPath::new(&entry.path)?.to_path_buf(); + if current_revision(collection, &path)? != entry.before_revision { return Ok(Some(entry.path.clone())); } } @@ -710,16 +744,7 @@ fn first_unsettled_conflict( collection: &Collection, journal: &RuntimeJournal, ) -> Result, TransactionError> { - let root = collection.root.join(TRANSACTIONS_DIR); - let mut directories = match fs::read_dir(&root) { - Ok(entries) => entries - .filter_map(Result::ok) - .map(|entry| entry.path()) - .collect::>(), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(source) => return Err(io_error(root, source)), - }; - directories.sort(); + let directories = transaction_directories(collection)?; let claimed = journal .entries .iter() @@ -729,18 +754,18 @@ fn first_unsettled_conflict( if directory.file_name().and_then(|name| name.to_str()) == Some(journal.id.as_str()) { continue; } - let bytes = match fs::read(directory.join(JOURNAL_FILE)) { + let journal_path = directory.join(JOURNAL_FILE); + let bytes = match collection.held_root().read(&journal_path) { Ok(bytes) => bytes, Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, - Err(source) => return Err(io_error(directory.join(JOURNAL_FILE), source)), + Err(source) => return Err(io_error(collection.root.join(journal_path), source)), }; let value = serde_json::from_slice::(&bytes) .map_err(|error| TransactionError::InvalidJournal(error.to_string()))?; if !runtime_journal_version(&value) { continue; } - let other: RuntimeJournal = serde_json::from_value(value) - .map_err(|error| TransactionError::InvalidJournal(error.to_string()))?; + let other = decode_runtime_journal(value)?; // Only a committed-but-unsettled transaction owns paths it has not yet // written. A prepared one has taken no commit point and loses the race // itself; a settled one is already visible in the working tree. @@ -765,7 +790,7 @@ fn validate_journal( ) -> Result<(), TransactionError> { if !matches!( journal.version, - LEGACY_RUNTIME_JOURNAL_VERSION | RUNTIME_JOURNAL_VERSION + LEGACY_RUNTIME_JOURNAL_VERSION | PHASE4_RUNTIME_JOURNAL_VERSION | RUNTIME_JOURNAL_VERSION ) || directory.file_name().and_then(|name| name.to_str()) != Some(&journal.id) || journal.applied > journal.entries.len() { @@ -781,6 +806,7 @@ fn validate_journal( "runtime change batch does not match its descriptor".to_string(), )); } + validate_journal_operation_family(journal)?; for (index, entry) in journal.entries.iter().enumerate() { validate_entry_path(collection, &entry.path, journal.scope)?; let expected_stage = entry @@ -791,16 +817,24 @@ fn validate_journal( .before_revision .as_ref() .map(|_| format!("backup/{index}")); - if entry.stage_file.as_deref() != expected_stage.as_deref() - || entry.backup_file.as_deref() != expected_backup.as_deref() - { + let payloads_released = matches!( + journal.phase, + RuntimePhase::Committed + | RuntimePhase::RejectedBeforeCommit + | RuntimePhase::CancelledBeforeCommit + ); + let stage_matches = entry.stage_file.as_deref() == expected_stage.as_deref() + || (payloads_released && entry.stage_file.is_none()); + let backup_matches = entry.backup_file.as_deref() == expected_backup.as_deref() + || (payloads_released && entry.backup_file.is_none()); + if !stage_matches || !backup_matches { return Err(TransactionError::InvalidJournal(format!( "payload paths for '{}' do not match its journal position", entry.path ))); } if let Some(stage_file) = &entry.stage_file { - let staged = read_regular_file(&directory.join(stage_file))?; + let staged = read_regular_file(collection, &directory.join(stage_file))?; if Some(crate::v03::revision(&staged)) != entry.after_revision { return Err(TransactionError::InvalidJournal(format!( "staged contents for '{}' do not match the journal", @@ -830,6 +864,702 @@ fn validate_journal( Ok(()) } +fn validate_journal_operation_family(journal: &RuntimeJournal) -> Result<(), TransactionError> { + if journal.version == LEGACY_RUNTIME_JOURNAL_VERSION { + return Ok(()); + } + if journal.changes.is_empty() { + return Err(TransactionError::InvalidJournal( + "v3 mutation journal has no canonical changes".to_string(), + )); + } + let operation = journal.operation_outcome.as_ref().ok_or_else(|| { + TransactionError::InvalidJournal("v3 canonical operation outcome missing".to_string()) + })?; + let CanonicalOperationFamily::Operation(kind) = operation.family() else { + return Err(TransactionError::InvalidJournal( + "mutation journal contains a non-operation canonical family".to_string(), + )); + }; + if matches!(operation.value(), CanonicalOperationValue::WireOnly(_)) { + return Err(TransactionError::InvalidJournal( + "mutation journal contains a wire-only outcome".to_string(), + )); + } + if let Some(rejection) = &journal.operation_rejection { + if rejection.family() != CanonicalOperationFamily::Operation(kind) + || matches!(rejection.value(), CanonicalOperationValue::WireOnly(_)) + { + return Err(TransactionError::InvalidJournal( + "commit rejection operation does not match the transaction operation".to_string(), + )); + } + } + + let terminal = matches!( + journal.phase, + RuntimePhase::Committed + | RuntimePhase::RejectedBeforeCommit + | RuntimePhase::CancelledBeforeCommit + ); + let synthesized_entries; + let physical_entries = + if journal.entries.is_empty() && terminal && !journal.transition_evidence.is_empty() { + synthesized_entries = journal + .transition_evidence + .iter() + .map(|evidence| JournalEntry { + path: evidence.path.clone(), + before_revision: evidence.before_revision.clone(), + after_revision: evidence.after_revision.clone(), + stage_file: None, + backup_file: None, + }) + .collect::>(); + synthesized_entries.as_slice() + } else { + journal.entries.as_slice() + }; + let expected = expected_transition_evidence_for( + journal.scope, + operation, + kind, + &journal.changes, + physical_entries, + ); + let Some(mut expected) = expected else { + return Err(TransactionError::InvalidJournal( + "canonical operation does not match the transaction change family".to_string(), + )); + }; + sort_transition_evidence(&mut expected); + let mut stored = journal.transition_evidence.clone(); + sort_transition_evidence(&mut stored); + match journal.version { + RUNTIME_JOURNAL_VERSION if stored == expected && !stored.is_empty() => Ok(()), + // Phase 4 v3 always retained physical entries until terminal cleanup. + // It may be read only while those exact entries still prove the full + // transition; stripped terminal v3 journals are deliberately rejected. + PHASE4_RUNTIME_JOURNAL_VERSION + if journal.transition_evidence.is_empty() && !journal.entries.is_empty() => + { + Ok(()) + } + _ => Err(TransactionError::InvalidJournal( + "runtime transition evidence is missing or does not match canonical changes" + .to_string(), + )), + } +} + +fn sort_transition_evidence(evidence: &mut [TransitionEvidence]) { + evidence.sort_by(|left, right| { + ( + &left.path, + &left.before_revision, + &left.after_revision, + format!("{:?}", left.operation), + left.change_index, + left.role, + ) + .cmp(&( + &right.path, + &right.before_revision, + &right.after_revision, + format!("{:?}", right.operation), + right.change_index, + right.role, + )) + }); +} + +fn expected_transition_evidence( + journal: &RuntimeJournal, +) -> Result, TransactionError> { + let operation = journal.operation_outcome.as_ref().ok_or_else(|| { + TransactionError::InvalidJournal("canonical operation outcome missing".to_string()) + })?; + let Some(kind) = operation.operation_kind() else { + return Err(TransactionError::InvalidJournal( + "canonical operation kind missing".to_string(), + )); + }; + expected_transition_evidence_for( + journal.scope, + operation, + kind, + &journal.changes, + &journal.entries, + ) + .ok_or_else(|| { + TransactionError::InvalidJournal( + "canonical changes do not exactly match physical transitions".to_string(), + ) + }) +} + +fn expected_transition_evidence_for( + scope: TransactionScope, + operation: &CanonicalOperationOutcome, + kind: OperationKind, + changes: &[CanonicalChange], + entries: &[JournalEntry], +) -> Option> { + match scope { + TransactionScope::Records => record_transition_evidence(operation, kind, changes, entries), + TransactionScope::Resources => resource_transition_evidence(kind, changes, entries), + TransactionScope::SystemMigration => None, + } +} + +fn record_transition_evidence( + operation: &CanonicalOperationOutcome, + kind: OperationKind, + changes: &[CanonicalChange], + entries: &[JournalEntry], +) -> Option> { + let records = changes + .iter() + .map(|change| match change { + CanonicalChange::Record(record) => Some(record), + CanonicalChange::Resource(_) => None, + }) + .collect::>>(); + let records = records?; + if entries.is_empty() { + return None; + } + + let rename_primary = records + .iter() + .filter(|record| record.kind != RecordChangeKind::Updated) + .count(); + if kind == OperationKind::Rename && rename_primary != 1 { + return None; + } + + let mut consumed = vec![false; entries.len()]; + let mut evidence = Vec::with_capacity(entries.len()); + for (change_index, record) in records.into_iter().enumerate() { + if !canonical_record_shape(record) { + return None; + } + let before_consumed = consumed.clone(); + let mapping = if kind == OperationKind::Batch { + unique_batch_item_kind(operation, record.path.as_str()).unwrap_or(OperationKind::Batch) + } else { + kind + }; + let accepted = match (kind, record.kind) { + (OperationKind::Create, RecordChangeKind::Created) => { + consume_created(record, entries, &mut consumed, false) + } + (OperationKind::Update, RecordChangeKind::Created) => { + consume_created(record, entries, &mut consumed, true) + } + (OperationKind::Update, RecordChangeKind::Updated) => { + consume_updated(record, entries, &mut consumed) + } + (OperationKind::Update, RecordChangeKind::Deleted) => { + consume_update_to_invalid(record, entries, &mut consumed) + } + (OperationKind::Delete, RecordChangeKind::Deleted) => { + consume_deleted(record, entries, &mut consumed) + } + (OperationKind::Rename, RecordChangeKind::Renamed) => { + rename_paths_for_record(operation, kind, record).is_some_and(|(from, to)| { + record + .from + .as_ref() + .is_some_and(|path| path.as_str() == from) + && record.path.as_str() == to + && consume_renamed(record, entries, &mut consumed) + }) + } + (OperationKind::Rename, RecordChangeKind::Created) => { + rename_paths_for_record(operation, kind, record).is_some_and(|(from, to)| { + record.path.as_str() == to + && consume_rename_created(record, from, entries, &mut consumed) + }) + } + (OperationKind::Rename, RecordChangeKind::Deleted) => { + rename_paths_for_record(operation, kind, record).is_some_and(|(from, to)| { + record.path.as_str() == from + && consume_rename_deleted(record, to, entries, &mut consumed) + }) + } + (OperationKind::Rename, RecordChangeKind::Updated) => { + consume_updated(record, entries, &mut consumed) + } + (OperationKind::Batch, RecordChangeKind::Created) => { + match unique_batch_item_kind(operation, record.path.as_str()) { + Some(OperationKind::Rename) => rename_paths_for_record(operation, kind, record) + .is_some_and(|(from, to)| { + record.path.as_str() == to + && consume_rename_created(record, from, entries, &mut consumed) + }), + Some(OperationKind::Update) => { + consume_created(record, entries, &mut consumed, true) + } + _ => consume_created(record, entries, &mut consumed, false), + } + } + (OperationKind::Batch, RecordChangeKind::Updated) => { + consume_updated(record, entries, &mut consumed) + } + (OperationKind::Batch, RecordChangeKind::Deleted) => { + match unique_batch_item_kind(operation, record.path.as_str()) { + Some(OperationKind::Rename) => rename_paths_for_record(operation, kind, record) + .is_some_and(|(from, to)| { + record.path.as_str() == from + && consume_rename_deleted(record, to, entries, &mut consumed) + }), + Some(OperationKind::Update) => { + consume_update_to_invalid(record, entries, &mut consumed) + } + _ => consume_deleted(record, entries, &mut consumed), + } + } + (OperationKind::Batch, RecordChangeKind::Renamed) => { + rename_paths_for_record(operation, kind, record).is_some_and(|(from, to)| { + record + .from + .as_ref() + .is_some_and(|path| path.as_str() == from) + && record.path.as_str() == to + && consume_renamed(record, entries, &mut consumed) + }) + } + _ => false, + }; + if !accepted { + return None; + } + for (index, was_consumed) in before_consumed.into_iter().enumerate() { + if !was_consumed && consumed[index] { + let entry = &entries[index]; + let role = match record.kind { + RecordChangeKind::Renamed => { + if record + .from + .as_ref() + .is_some_and(|from| from.as_str() == entry.path) + { + TransitionRole::RenameSource + } else { + TransitionRole::RenameDestination + } + } + RecordChangeKind::Created if kind == OperationKind::Rename => { + if entry.path == record.path.as_str() { + TransitionRole::RenameDestination + } else { + TransitionRole::RenameSource + } + } + RecordChangeKind::Deleted if kind == OperationKind::Rename => { + if entry.path == record.path.as_str() { + TransitionRole::RenameSource + } else { + TransitionRole::RenameDestination + } + } + _ => TransitionRole::Direct, + }; + evidence.push(TransitionEvidence { + path: entry.path.clone(), + before_revision: entry.before_revision.clone(), + after_revision: entry.after_revision.clone(), + operation: mapping, + change_index, + role, + }); + } + } + } + consumed.into_iter().all(|entry| entry).then_some(evidence) +} + +fn canonical_record_shape(record: &crate::runtime::RecordChange) -> bool { + match record.kind { + RecordChangeKind::Created => { + record.from.is_none() + && record.before_revision.is_none() + && record.after_revision.is_some() + } + RecordChangeKind::Updated => { + record.from.is_none() + && record.before_revision.is_some() + && record.after_revision.is_some() + } + RecordChangeKind::Deleted => { + record.from.is_none() + && record.before_revision.is_some() + && record.after_revision.is_none() + } + RecordChangeKind::Renamed => { + record.before_revision.is_some() + && record.after_revision.is_some() + && record + .from + .as_ref() + .is_some_and(|from| from != &record.path) + } + } +} + +fn consume_one( + entries: &[JournalEntry], + consumed: &mut [bool], + predicate: impl Fn(&JournalEntry) -> bool, +) -> bool { + let matches = entries + .iter() + .enumerate() + .filter(|(index, entry)| !consumed[*index] && predicate(entry)) + .map(|(index, _)| index) + .collect::>(); + if let [index] = matches.as_slice() { + consumed[*index] = true; + true + } else { + false + } +} + +fn consume_created( + record: &crate::runtime::RecordChange, + entries: &[JournalEntry], + consumed: &mut [bool], + repair: bool, +) -> bool { + let after = record + .after_revision + .as_ref() + .expect("checked shape") + .as_str(); + consume_one(entries, consumed, |entry| { + entry.path == record.path.as_str() + && entry.before_revision.is_some() == repair + && entry.after_revision.as_deref() == Some(after) + }) +} + +fn consume_updated( + record: &crate::runtime::RecordChange, + entries: &[JournalEntry], + consumed: &mut [bool], +) -> bool { + let before = record + .before_revision + .as_ref() + .expect("checked shape") + .as_str(); + let after = record + .after_revision + .as_ref() + .expect("checked shape") + .as_str(); + consume_one(entries, consumed, |entry| { + entry.path == record.path.as_str() + && entry.before_revision.as_deref() == Some(before) + && entry.after_revision.as_deref() == Some(after) + }) +} + +fn consume_deleted( + record: &crate::runtime::RecordChange, + entries: &[JournalEntry], + consumed: &mut [bool], +) -> bool { + let before = record + .before_revision + .as_ref() + .expect("checked shape") + .as_str(); + consume_one(entries, consumed, |entry| { + entry.path == record.path.as_str() + && entry.before_revision.as_deref() == Some(before) + && entry.after_revision.is_none() + }) +} + +fn consume_update_to_invalid( + record: &crate::runtime::RecordChange, + entries: &[JournalEntry], + consumed: &mut [bool], +) -> bool { + let before = record + .before_revision + .as_ref() + .expect("checked shape") + .as_str(); + consume_one(entries, consumed, |entry| { + entry.path == record.path.as_str() + && entry.before_revision.as_deref() == Some(before) + && entry.after_revision.is_some() + }) +} + +fn consume_renamed( + record: &crate::runtime::RecordChange, + entries: &[JournalEntry], + consumed: &mut [bool], +) -> bool { + let from = record.from.as_ref().expect("checked shape").as_str(); + let before = record + .before_revision + .as_ref() + .expect("checked shape") + .as_str(); + let after = record + .after_revision + .as_ref() + .expect("checked shape") + .as_str(); + let mut trial = consumed.to_vec(); + let source = consume_one(entries, &mut trial, |entry| { + entry.path == from + && entry.before_revision.as_deref() == Some(before) + && entry.after_revision.is_none() + }); + let destination = consume_one(entries, &mut trial, |entry| { + entry.path == record.path.as_str() + && entry.before_revision.is_none() + && entry.after_revision.as_deref() == Some(after) + }); + if source && destination { + consumed.copy_from_slice(&trial); + true + } else { + false + } +} + +fn consume_rename_created( + record: &crate::runtime::RecordChange, + from: &str, + entries: &[JournalEntry], + consumed: &mut [bool], +) -> bool { + let revision = record + .after_revision + .as_ref() + .expect("checked shape") + .as_str(); + let mut trial = consumed.to_vec(); + let destination = consume_one(entries, &mut trial, |entry| { + entry.path == record.path.as_str() + && entry.before_revision.is_none() + && entry.after_revision.as_deref() == Some(revision) + }); + let source = consume_one(entries, &mut trial, |entry| { + entry.path == from + && entry.before_revision.as_deref() == Some(revision) + && entry.after_revision.is_none() + }); + if source && destination { + consumed.copy_from_slice(&trial); + true + } else { + false + } +} + +fn consume_rename_deleted( + record: &crate::runtime::RecordChange, + to: &str, + entries: &[JournalEntry], + consumed: &mut [bool], +) -> bool { + let revision = record + .before_revision + .as_ref() + .expect("checked shape") + .as_str(); + let mut trial = consumed.to_vec(); + let source = consume_one(entries, &mut trial, |entry| { + entry.path == record.path.as_str() + && entry.before_revision.as_deref() == Some(revision) + && entry.after_revision.is_none() + }); + let destination = consume_one(entries, &mut trial, |entry| { + entry.path == to + && entry.before_revision.is_none() + && entry.after_revision.as_deref() == Some(revision) + }); + if source && destination { + consumed.copy_from_slice(&trial); + true + } else { + false + } +} + +fn unique_batch_item_kind( + operation: &CanonicalOperationOutcome, + path: &str, +) -> Option { + let CanonicalOperationValue::Batch(Some(batch)) = operation.value() else { + return None; + }; + let matches = batch + .operations + .iter() + .filter(|item| item.valid && batch_item_affects_path(item, path)) + .filter_map(|item| match item.kind.as_str() { + "create" => Some(OperationKind::Create), + "update" => Some(OperationKind::Update), + "delete" => Some(OperationKind::Delete), + "rename" => Some(OperationKind::Rename), + _ => None, + }) + .collect::>(); + match matches.as_slice() { + [kind] => Some(*kind), + _ => None, + } +} + +fn batch_item_affects_path(item: &crate::api::BatchItemResult, path: &str) -> bool { + match &item.result { + crate::api::BatchOperationResult::Record(record) => record.path.as_str() == path, + crate::api::BatchOperationResult::Delete(result) => result.path.as_str() == path, + crate::api::BatchOperationResult::Rename(result) => { + result.result.from.as_str() == path || result.result.to.as_str() == path + } + _ => false, + } +} + +fn rename_paths_for_record<'a>( + operation: &'a CanonicalOperationOutcome, + kind: OperationKind, + record: &crate::runtime::RecordChange, +) -> Option<(&'a str, &'a str)> { + match (kind, operation.value()) { + ( + OperationKind::Rename, + CanonicalOperationValue::Rename(Some(crate::runtime::CanonicalRenameValue::Renamed( + result, + ))), + ) => Some((result.result.from.as_str(), result.result.to.as_str())), + (OperationKind::Batch, CanonicalOperationValue::Batch(Some(batch))) => { + let matches = batch + .operations + .iter() + .filter(|item| item.valid && item.kind == "rename") + .filter_map(|item| match &item.result { + crate::api::BatchOperationResult::Rename(result) + if match record.kind { + RecordChangeKind::Created => { + result.result.to.as_str() == record.path.as_str() + } + RecordChangeKind::Deleted => { + result.result.from.as_str() == record.path.as_str() + } + RecordChangeKind::Renamed => record.from.as_ref().is_some_and(|from| { + from == &result.result.from && record.path == result.result.to + }), + RecordChangeKind::Updated => false, + } => + { + Some((result.result.from.as_str(), result.result.to.as_str())) + } + _ => None, + }) + .collect::>(); + match matches.as_slice() { + [paths] => Some(*paths), + _ => None, + } + } + _ => None, + } +} + +fn resource_transition_evidence( + kind: OperationKind, + changes: &[CanonicalChange], + entries: &[JournalEntry], +) -> Option> { + if entries.is_empty() { + return None; + } + let mut consumed = vec![false; entries.len()]; + let mut evidence = Vec::with_capacity(entries.len()); + for (change_index, change) in changes.iter().enumerate() { + let CanonicalChange::Resource(resource) = change else { + return None; + }; + if resource.before_revision.is_none() && resource.after_revision.is_none() { + return None; + } + let before_consumed = consumed.clone(); + if !consume_one(entries, &mut consumed, |entry| { + entry.path == resource.path.as_str() + && entry.before_revision.as_deref() + == resource + .before_revision + .as_ref() + .map(|revision| revision.as_str()) + && entry.after_revision.as_deref() + == resource + .after_revision + .as_ref() + .map(|revision| revision.as_str()) + }) { + return None; + } + let family = match kind { + OperationKind::CreateViewSource => { + resource.kind == ResourceChangeKind::ViewSource + && resource.before_revision.is_none() + && resource.after_revision.is_some() + } + OperationKind::UpdateViewSource => { + resource.kind == ResourceChangeKind::ViewSource + && resource.before_revision.is_some() + && resource.after_revision.is_some() + } + OperationKind::DeleteViewSource => { + resource.kind == ResourceChangeKind::ViewSource + && resource.before_revision.is_some() + && resource.after_revision.is_none() + } + OperationKind::CreateType => { + resource.kind == ResourceChangeKind::TypeDefinition + && resource.before_revision.is_none() + && resource.after_revision.is_some() + } + OperationKind::UpdateType => { + resource.kind == ResourceChangeKind::TypeDefinition + && resource.before_revision.is_some() + && resource.after_revision.is_some() + } + OperationKind::ApplyTypePack | OperationKind::ApplyCollectionSetup => true, + _ => false, + }; + if !family { + return None; + } + for (index, was_consumed) in before_consumed.into_iter().enumerate() { + if !was_consumed && consumed[index] { + let entry = &entries[index]; + evidence.push(TransitionEvidence { + path: entry.path.clone(), + before_revision: entry.before_revision.clone(), + after_revision: entry.after_revision.clone(), + operation: kind, + change_index, + role: TransitionRole::Direct, + }); + } + } + } + consumed.into_iter().all(|entry| entry).then_some(evidence) +} + fn ensure_capacity(collection: &Collection, changes: &ChangeBatch) -> Result<(), TransactionError> { if changes.descriptor().count > MAX_RUNTIME_CHANGE_ITEMS { return Err(TransactionError::RuntimeCapacityExhausted); @@ -839,12 +1569,7 @@ fn ensure_capacity(collection: &Collection, changes: &ChangeBatch) -> Result<(), if metadata.len() > MAX_RUNTIME_METADATA_BYTES { return Err(TransactionError::RuntimeCapacityExhausted); } - let root = collection.root.join(TRANSACTIONS_DIR); - let active = match fs::read_dir(&root) { - Ok(entries) => entries.filter_map(Result::ok).count(), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => 0, - Err(source) => return Err(io_error(root, source)), - }; + let active = transaction_directories(collection)?.len(); if active >= MAX_ACTIVE_RUNTIME_TRANSACTIONS { return Err(TransactionError::RuntimeCapacityExhausted); } @@ -876,29 +1601,20 @@ fn find_by_claim( collection: &Collection, claim: &str, ) -> Result, TransactionError> { - let root = collection.root.join(TRANSACTIONS_DIR); - let mut directories = match fs::read_dir(&root) { - Ok(entries) => entries - .filter_map(Result::ok) - .map(|entry| entry.path()) - .collect::>(), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(source) => return Err(io_error(root, source)), - }; - directories.sort(); + let directories = transaction_directories(collection)?; for directory in directories { - let bytes = match fs::read(directory.join(JOURNAL_FILE)) { + let journal_path = directory.join(JOURNAL_FILE); + let bytes = match collection.held_root().read(&journal_path) { Ok(bytes) => bytes, Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, - Err(source) => return Err(io_error(directory.join(JOURNAL_FILE), source)), + Err(source) => return Err(io_error(collection.root.join(journal_path), source)), }; let value = serde_json::from_slice::(&bytes) .map_err(|error| TransactionError::InvalidJournal(error.to_string()))?; if !runtime_journal_version(&value) { continue; } - let journal: RuntimeJournal = serde_json::from_value(value) - .map_err(|error| TransactionError::InvalidJournal(error.to_string()))?; + let journal = decode_runtime_journal(value)?; validate_journal(collection, &directory, &journal)?; if journal.host_claim == claim { return Ok(Some(journal)); @@ -936,14 +1652,15 @@ fn resolution(journal: &RuntimeJournal) -> Result Result { - if let Some(operation) = &journal.operation_outcome { - return Ok(operation.clone()); - } - let result = journal.operation_result.clone().ok_or_else(|| { - TransactionError::InvalidJournal("runtime operation outcome missing".to_string()) - })?; - match legacy_operation_kind(journal, &result) { - Some(kind) => CanonicalOperationOutcome::recover_v03(kind, result) - .map_err(|error| TransactionError::InvalidJournal(error.to_string())), - None => Ok(CanonicalOperationOutcome::legacy_recovered(result)), + match journal.version { + PHASE4_RUNTIME_JOURNAL_VERSION | RUNTIME_JOURNAL_VERSION => { + journal.operation_outcome.clone().ok_or_else(|| { + TransactionError::InvalidJournal( + "canonical runtime operation outcome missing".to_string(), + ) + }) + } + LEGACY_RUNTIME_JOURNAL_VERSION => { + let result = journal.operation_result.clone().ok_or_else(|| { + TransactionError::InvalidJournal("v2 runtime operation result missing".to_string()) + })?; + match legacy_operation_kind(journal, &result) { + Some(kind) => CanonicalOperationOutcome::recover_v03(kind, result) + .map_err(|error| TransactionError::InvalidJournal(error.to_string())), + None => Ok(legacy_recovered_v03(result)), + } + } + _ => Err(TransactionError::InvalidJournal( + "unsupported runtime journal version".to_string(), + )), } } @@ -974,6 +1702,7 @@ fn journal_operation_mut( journal.operation_outcome = Some(journal_operation(journal)?); journal.operation_result = None; journal.version = RUNTIME_JOURNAL_VERSION; + journal.transition_evidence = expected_transition_evidence(journal)?; } journal.operation_outcome.as_mut().ok_or_else(|| { TransactionError::InvalidJournal("runtime operation outcome missing".to_string()) @@ -983,17 +1712,37 @@ fn journal_operation_mut( fn journal_rejection( journal: &RuntimeJournal, ) -> Result { - if let Some(rejection) = &journal.operation_rejection { - return Ok(rejection.clone()); + match journal.version { + PHASE4_RUNTIME_JOURNAL_VERSION | RUNTIME_JOURNAL_VERSION => { + journal.operation_rejection.clone().ok_or_else(|| { + TransactionError::InvalidJournal("canonical commit rejection missing".to_string()) + }) + } + LEGACY_RUNTIME_JOURNAL_VERSION => { + let rejection = journal.rejection.clone().ok_or_else(|| { + TransactionError::InvalidJournal("v2 commit rejection missing".to_string()) + })?; + match journal_operation(journal)?.value.kind() { + Some(kind) => CanonicalOperationOutcome::recover_v03(kind, rejection) + .map_err(|error| TransactionError::InvalidJournal(error.to_string())), + None => Ok(legacy_recovered_v03(rejection)), + } + } + _ => Err(TransactionError::InvalidJournal( + "unsupported runtime journal version".to_string(), + )), } - let rejection = journal - .rejection - .clone() - .ok_or_else(|| TransactionError::InvalidJournal("commit rejection missing".to_string()))?; - match journal_operation(journal)?.value.kind() { - Some(kind) => CanonicalOperationOutcome::recover_v03(kind, rejection) - .map_err(|error| TransactionError::InvalidJournal(error.to_string())), - None => Ok(CanonicalOperationOutcome::legacy_recovered(rejection)), +} + +/// Construct an ambiguous v0.3 envelope only inside the transaction runtime's +/// version-2 compatibility path. Checked serde and v3 persistence reject it. +fn legacy_recovered_v03(result: OperationResult) -> CanonicalOperationOutcome { + CanonicalOperationOutcome { + valid: result.valid, + diagnostics: result.diagnostics.iter().cloned().map(Into::into).collect(), + value: CanonicalOperationValue::LegacyRecoveredV03( + LegacyRecoveredV03Value::from_transaction_recovery(result), + ), } } @@ -1035,10 +1784,87 @@ fn runtime_journal_version(value: &serde_json::Value) -> bool { matches!( value.get("version").and_then(serde_json::Value::as_u64), Some(version) if version == u64::from(LEGACY_RUNTIME_JOURNAL_VERSION) + || version == u64::from(PHASE4_RUNTIME_JOURNAL_VERSION) || version == u64::from(RUNTIME_JOURNAL_VERSION) ) } +fn decode_runtime_journal( + mut value: serde_json::Value, +) -> Result { + recover_phase4_definition_discriminators(&mut value); + let version = value + .get("version") + .and_then(serde_json::Value::as_u64) + .ok_or_else(|| { + TransactionError::InvalidJournal("runtime journal version missing".into()) + })?; + let phase = value + .get("phase") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| TransactionError::InvalidJournal("runtime journal phase missing".into()))?; + let present = |field: &str| value.get(field).is_some_and(|item| !item.is_null()); + let canonical = present("operation_outcome"); + let legacy = present("operation_result"); + let canonical_rejection = present("operation_rejection"); + let legacy_rejection = present("rejection"); + let rejected = phase == "rejected_before_commit"; + + let valid_shape = match u32::try_from(version).ok() { + Some(LEGACY_RUNTIME_JOURNAL_VERSION) => { + legacy && !canonical && !canonical_rejection && legacy_rejection == rejected + } + Some(PHASE4_RUNTIME_JOURNAL_VERSION | RUNTIME_JOURNAL_VERSION) => { + canonical && !legacy && !legacy_rejection && canonical_rejection == rejected + } + _ => false, + }; + if !valid_shape { + return Err(TransactionError::InvalidJournal( + "runtime journal outcome fields do not match its version and phase".to_string(), + )); + } + serde_json::from_value(value) + .map_err(|error| TransactionError::InvalidJournal(error.to_string())) +} + +/// Phase 4 persisted definition families without their assess/apply +/// discriminator. A durable resource mutation provides sufficient context to +/// recover only the apply form; generic public outcome serde never guesses. +fn recover_phase4_definition_discriminators(value: &mut serde_json::Value) { + if value.get("version").and_then(serde_json::Value::as_u64) + != Some(u64::from(PHASE4_RUNTIME_JOURNAL_VERSION)) + || value.get("scope").and_then(serde_json::Value::as_str) != Some("resources") + || !value + .get("changes") + .and_then(serde_json::Value::as_array) + .is_some_and(|changes| { + !changes.is_empty() + && changes.iter().all(|change| { + change.get("target").and_then(serde_json::Value::as_str) == Some("resource") + }) + }) + { + return; + } + for field in ["operation_outcome", "operation_rejection"] { + let Some(discriminator) = value + .get_mut(field) + .and_then(|outcome| outcome.get_mut("value")) + .and_then(|outcome| outcome.get_mut("operation")) + else { + continue; + }; + match discriminator.as_str() { + Some("type_pack") => *discriminator = serde_json::json!("apply_type_pack"), + Some("collection_setup") => { + *discriminator = serde_json::json!("apply_collection_setup") + } + _ => {} + } + } +} + fn conflict_result( journal: &RuntimeJournal, path: &str, @@ -1067,25 +1893,46 @@ fn commit_id(journal: &RuntimeJournal) -> CommitId { CommitId::from_stored(journal.id.clone()) } -fn transaction_directory(collection: &Collection, id: &CommitId) -> PathBuf { - collection.root.join(TRANSACTIONS_DIR).join(id.as_str()) +fn transaction_directories(collection: &Collection) -> Result, TransactionError> { + collection + .held_root() + .child_directories(Path::new(TRANSACTIONS_DIR)) + .map_err(|source| io_error(collection.root.join(TRANSACTIONS_DIR), source)) } -fn read_runtime_journal(directory: &Path) -> Result { +fn transaction_directory(_collection: &Collection, id: &CommitId) -> PathBuf { + PathBuf::from(TRANSACTIONS_DIR).join(id.as_str()) +} + +fn read_runtime_journal( + collection: &Collection, + directory: &Path, +) -> Result { let path = directory.join(JOURNAL_FILE); - let bytes = fs::read(&path).map_err(|source| io_error(path, source))?; - serde_json::from_slice(&bytes) - .map_err(|error| TransactionError::InvalidJournal(error.to_string())) + let bytes = collection + .held_root() + .read(&path) + .map_err(|source| io_error(collection.root.join(&path), source))?; + let value = serde_json::from_slice(&bytes) + .map_err(|error| TransactionError::InvalidJournal(error.to_string()))?; + let journal = decode_runtime_journal(value)?; + validate_journal(collection, directory, &journal)?; + Ok(journal) } fn persist_runtime_journal( + collection: &Collection, directory: &Path, journal: &RuntimeJournal, ) -> Result<(), TransactionError> { + validate_journal(collection, directory, journal)?; let bytes = serde_json::to_vec_pretty(journal) .map_err(|error| TransactionError::InvalidJournal(error.to_string()))?; let path = directory.join(JOURNAL_FILE); - crate::operations::atomic_write(&path, &bytes).map_err(|source| io_error(path, source)) + collection + .held_root() + .atomic_write(&path, &bytes) + .map_err(|source| io_error(collection.root.join(path), source)) } fn context_check(context: &OperationContext) -> Result<(), TransactionError> { @@ -1140,11 +1987,7 @@ mod tests { let claim = HostClaimId::generate(); let event_id = ChangeEventId::generate(); let changes = change(path, before, after); - let result = crate::v03::OperationResult { - valid: true, - result: serde_json::json!({}), - diagnostics: Vec::new(), - }; + let operation = CanonicalOperationOutcome::invalid(OperationKind::Update, Vec::new()); let outcome = prepare_runtime_transaction( collection, RuntimePrepareInput { @@ -1152,11 +1995,7 @@ mod tests { desired: &desired, claim: &claim, mutation_digest: claim.as_str(), - operation: &CanonicalOperationOutcome { - valid: result.valid, - value: crate::runtime::CanonicalOperationValue::Update(None), - diagnostics: result.diagnostics.into_iter().map(Into::into).collect(), - }, + operation: &operation, changes: &changes, event_id: &event_id, }, @@ -1180,12 +2019,380 @@ mod tests { .unwrap() } + fn journal_value(collection: &Collection, id: &CommitId) -> serde_json::Value { + let directory = transaction_directory(collection, id); + serde_json::from_slice( + &collection + .held_root() + .read(directory.join(JOURNAL_FILE)) + .unwrap(), + ) + .unwrap() + } + + fn write_journal_value(collection: &Collection, id: &CommitId, value: &serde_json::Value) { + let directory = transaction_directory(collection, id); + collection + .held_root() + .atomic_write( + &directory.join(JOURNAL_FILE), + &serde_json::to_vec_pretty(value).unwrap(), + ) + .unwrap(); + } + + #[test] + fn malicious_version_and_phase_outcome_field_combinations_fail_closed() { + let (_root, collection) = collection(); + let id = prepare(&collection, "a.md", b"old-a\n", b"new-a\n"); + let canonical = journal_value(&collection, &id); + let legacy_result = serde_json::to_value( + serde_json::from_value::( + canonical["operation_outcome"].clone(), + ) + .unwrap() + .to_v03(), + ) + .unwrap(); + + let mut fixtures = Vec::new(); + let mut v3_with_legacy = canonical.clone(); + v3_with_legacy["operation_result"] = legacy_result.clone(); + fixtures.push(v3_with_legacy); + let mut v2_with_canonical = canonical.clone(); + v2_with_canonical["version"] = serde_json::json!(LEGACY_RUNTIME_JOURNAL_VERSION); + v2_with_canonical["operation_result"] = legacy_result; + fixtures.push(v2_with_canonical); + let mut prepared_with_rejection = canonical.clone(); + prepared_with_rejection["operation_rejection"] = + prepared_with_rejection["operation_outcome"].clone(); + fixtures.push(prepared_with_rejection); + let mut rejected_without_rejection = canonical; + rejected_without_rejection["phase"] = serde_json::json!("rejected_before_commit"); + fixtures.push(rejected_without_rejection); + + for fixture in fixtures { + assert!(decode_runtime_journal(fixture).is_err()); + } + } + + #[test] + fn malicious_v3_operation_change_and_rejection_families_fail_closed() { + let (_root, collection) = collection(); + let id = prepare(&collection, "a.md", b"old-a\n", b"new-a\n"); + let original = journal_value(&collection, &id); + let directory = transaction_directory(&collection, &id); + + let mut wrong_change_family = original.clone(); + wrong_change_family["operation_outcome"]["value"]["operation"] = + serde_json::json!("delete"); + write_journal_value(&collection, &id, &wrong_change_family); + assert!(read_runtime_journal(&collection, &directory).is_err()); + + let mut missing_transition_evidence = original.clone(); + missing_transition_evidence["entries"] = serde_json::json!([]); + write_journal_value(&collection, &id, &missing_transition_evidence); + assert!(read_runtime_journal(&collection, &directory).is_err()); + + let mut wire = original.clone(); + wire["operation_outcome"] = serde_json::to_value( + CanonicalOperationOutcome::validation_wire(OperationResult { + valid: true, + result: serde_json::json!({"valid": true}), + diagnostics: Vec::new(), + }), + ) + .unwrap(); + write_journal_value(&collection, &id, &wire); + assert!(read_runtime_journal(&collection, &directory).is_err()); + + let mut cursor = original.clone(); + cursor["operation_outcome"] = + serde_json::to_value(CanonicalOperationOutcome::cursor_release( + crate::runtime::CursorReleaseOutcome { released: true }, + )) + .unwrap(); + write_journal_value(&collection, &id, &cursor); + assert!(read_runtime_journal(&collection, &directory).is_err()); + + let mut rejection = original; + rejection["phase"] = serde_json::json!("rejected_before_commit"); + rejection["operation_rejection"] = rejection["operation_outcome"].clone(); + rejection["operation_rejection"]["value"]["operation"] = serde_json::json!("delete"); + write_journal_value(&collection, &id, &rejection); + assert!(read_runtime_journal(&collection, &directory).is_err()); + } + + #[test] + fn malicious_entry_bijection_fixtures_fail_closed() { + let (_root, collection) = collection(); + let id = prepare(&collection, "a.md", b"old-a\n", b"new-a\n"); + let directory = transaction_directory(&collection, &id); + let fresh = || read_runtime_journal(&collection, &directory).unwrap(); + + let mut extra = fresh(); + extra.entries.push(JournalEntry { + path: "extra.md".to_string(), + before_revision: None, + after_revision: Some(crate::v03::revision(b"extra\n")), + stage_file: None, + backup_file: None, + }); + assert!(validate_journal_operation_family(&extra).is_err()); + + let mut duplicate = fresh(); + let entry = &duplicate.entries[0]; + duplicate.entries.push(JournalEntry { + path: entry.path.clone(), + before_revision: entry.before_revision.clone(), + after_revision: entry.after_revision.clone(), + stage_file: entry.stage_file.clone(), + backup_file: entry.backup_file.clone(), + }); + assert!(validate_journal_operation_family(&duplicate).is_err()); + + let mut create_over_existing = fresh(); + create_over_existing.operation_outcome = Some(CanonicalOperationOutcome::invalid( + OperationKind::Create, + Vec::new(), + )); + create_over_existing.changes = vec![CanonicalChange::Record(RecordChange { + kind: RecordChangeKind::Created, + path: crate::api::CollectionPath::new("a.md").unwrap(), + from: None, + before_revision: None, + after_revision: crate::api::Revision::parse(crate::v03::revision(b"new-a\n")).ok(), + before_types: CanonicalTypeSet::new([]), + after_types: CanonicalTypeSet::new([]), + changed_fields: CanonicalFieldChangeSet::new([]).unwrap(), + body_changed: true, + })]; + assert!(validate_journal_operation_family(&create_over_existing).is_err()); + + let mut rename_over_destination = fresh(); + let old_revision = crate::v03::revision(b"old-a\n"); + let new_revision = crate::v03::revision(b"new-a\n"); + rename_over_destination.operation_outcome = Some(CanonicalOperationOutcome::invalid( + OperationKind::Rename, + Vec::new(), + )); + rename_over_destination.entries = vec![ + JournalEntry { + path: "a.md".to_string(), + before_revision: Some(old_revision.clone()), + after_revision: None, + stage_file: None, + backup_file: None, + }, + JournalEntry { + path: "destination.md".to_string(), + before_revision: Some(crate::v03::revision(b"occupied\n")), + after_revision: Some(new_revision.clone()), + stage_file: None, + backup_file: None, + }, + ]; + rename_over_destination.changes = vec![CanonicalChange::Record(RecordChange { + kind: RecordChangeKind::Renamed, + path: crate::api::CollectionPath::new("destination.md").unwrap(), + from: Some(crate::api::CollectionPath::new("a.md").unwrap()), + before_revision: crate::api::Revision::parse(old_revision).ok(), + after_revision: crate::api::Revision::parse(new_revision).ok(), + before_types: CanonicalTypeSet::new([]), + after_types: CanonicalTypeSet::new([]), + changed_fields: CanonicalFieldChangeSet::new([]).unwrap(), + body_changed: false, + })]; + assert!(validate_journal_operation_family(&rename_over_destination).is_err()); + + let mut stripped_terminal = fresh(); + stripped_terminal.phase = RuntimePhase::CancelledBeforeCommit; + stripped_terminal.entries.clear(); + stripped_terminal.transition_evidence.clear(); + assert!(validate_journal_operation_family(&stripped_terminal).is_err()); + } + + #[test] + fn rename_evidence_rejects_equal_revision_source_path_substitution() { + let (root, collection) = collection(); + let runtime = crate::runtime::FilesystemRuntime::open( + root.path(), + std::time::Duration::from_millis(5), + ) + .unwrap(); + let prepared = runtime + .prepare( + &crate::runtime::OperationRequest::new( + OperationKind::Rename, + serde_json::json!({"from": "a.md", "to": "renamed.md"}), + ), + &HostClaimId::generate(), + &OperationContext::legacy(), + ) + .unwrap(); + let crate::runtime::PreparationOutcome::Prepared(prepared) = prepared else { + panic!("rename must prepare") + }; + let directory = transaction_directory(&collection, prepared.commit_id()); + let mut journal = read_runtime_journal(&collection, &directory).unwrap(); + let source = journal + .entries + .iter_mut() + .find(|entry| entry.path == "a.md") + .unwrap(); + source.path = "same-revision-other.md".to_string(); + let source_evidence = journal + .transition_evidence + .iter_mut() + .find(|evidence| evidence.role == TransitionRole::RenameSource) + .unwrap(); + source_evidence.path = "same-revision-other.md".to_string(); + assert!(validate_journal_operation_family(&journal).is_err()); + } + + #[test] + fn update_repair_keeps_update_identity_when_canonical_record_is_created() { + let (_root, collection) = collection(); + let path = "invalid.md"; + let invalid = b"invalid\xffbytes\n"; + let repaired = b"---\ntitle: Repaired\n---\nBody\n"; + let baseline: FileBaseline = BTreeMap::from([(path.to_string(), invalid.to_vec())]); + let desired: FileBaseline = BTreeMap::from([(path.to_string(), repaired.to_vec())]); + let changes = ChangeBatch::new(vec![CanonicalChange::Record(RecordChange { + kind: RecordChangeKind::Created, + path: crate::api::CollectionPath::new(path).unwrap(), + from: None, + before_revision: None, + after_revision: Some( + crate::api::Revision::parse(crate::v03::revision(repaired)).unwrap(), + ), + before_types: CanonicalTypeSet::new([]), + after_types: CanonicalTypeSet::new([]), + changed_fields: CanonicalFieldChangeSet::new([]).unwrap(), + body_changed: true, + })]) + .unwrap(); + let claim = HostClaimId::generate(); + let operation = CanonicalOperationOutcome::invalid(OperationKind::Update, Vec::new()); + let prepared = prepare_runtime_transaction( + &collection, + RuntimePrepareInput { + baseline: &baseline, + desired: &desired, + claim: &claim, + mutation_digest: claim.as_str(), + operation: &operation, + changes: &changes, + event_id: &ChangeEventId::generate(), + }, + &OperationContext::legacy(), + ) + .unwrap(); + let RuntimePrepareOutcome::Prepared(id) = prepared else { + panic!("repair must prepare") + }; + let journal = + read_runtime_journal(&collection, &transaction_directory(&collection, &id)).unwrap(); + assert_eq!( + journal + .operation_outcome + .as_ref() + .and_then(CanonicalOperationOutcome::operation_kind), + Some(OperationKind::Update) + ); + + // The same canonical change cannot relabel an ordinary create as an + // update: physical before-presence is the required repair evidence. + let absent: FileBaseline = BTreeMap::new(); + let claim = HostClaimId::generate(); + let rejected = prepare_runtime_transaction( + &collection, + RuntimePrepareInput { + baseline: &absent, + desired: &desired, + claim: &claim, + mutation_digest: claim.as_str(), + operation: &operation, + changes: &changes, + event_id: &ChangeEventId::generate(), + }, + &OperationContext::legacy(), + ); + assert!(matches!(rejected, Err(TransactionError::InvalidJournal(_)))); + } + + #[test] + fn definition_journals_preserve_apply_and_reject_assess_or_swapped_rejection() { + let (_root, collection) = collection(); + let id = prepare(&collection, "a.md", b"old-a\n", b"new-a\n"); + let directory = transaction_directory(&collection, &id); + let mut journal = read_runtime_journal(&collection, &directory).unwrap(); + journal.scope = TransactionScope::Resources; + // Model a Phase 4 v3 journal whose physical transition is retained. + journal.version = PHASE4_RUNTIME_JOURNAL_VERSION; + journal.entries = vec![JournalEntry { + path: "resource.bin".to_string(), + before_revision: None, + after_revision: Some("sha256:after".to_string()), + stage_file: None, + backup_file: None, + }]; + journal.transition_evidence.clear(); + journal.changes = vec![CanonicalChange::Resource(crate::runtime::ResourceChange { + kind: ResourceChangeKind::Other, + path: crate::api::CollectionPath::new("resource.bin").unwrap(), + before_revision: None, + after_revision: crate::api::Revision::parse("sha256:after").ok(), + })]; + let definition = |kind| { + CanonicalOperationOutcome::recover_v03( + kind, + OperationResult { + valid: false, + result: serde_json::json!({}), + diagnostics: Vec::new(), + }, + ) + .unwrap() + }; + journal.operation_outcome = Some(definition(OperationKind::ApplyTypePack)); + assert!(validate_journal_operation_family(&journal).is_ok()); + journal.operation_outcome = Some(definition(OperationKind::AssessTypePack)); + assert!(validate_journal_operation_family(&journal).is_err()); + journal.operation_outcome = Some(definition(OperationKind::ApplyTypePack)); + journal.operation_rejection = Some(definition(OperationKind::AssessTypePack)); + assert!(validate_journal_operation_family(&journal).is_err()); + + let mut phase4 = journal_value(&collection, &id); + phase4["version"] = serde_json::json!(PHASE4_RUNTIME_JOURNAL_VERSION); + phase4 + .as_object_mut() + .unwrap() + .remove("transition_evidence"); + phase4["scope"] = serde_json::json!("resources"); + phase4["changes"] = serde_json::json!([{ + "target": "resource", + "change": { + "kind": "other", "path": "resource.bin", + "before_revision": null, "after_revision": "sha256:after" + } + }]); + phase4["operation_outcome"] = + serde_json::to_value(definition(OperationKind::ApplyCollectionSetup)).unwrap(); + phase4["operation_outcome"]["value"]["operation"] = serde_json::json!("collection_setup"); + let recovered = decode_runtime_journal(phase4).unwrap(); + assert_eq!( + recovered.operation_outcome.unwrap().operation_kind(), + Some(OperationKind::ApplyCollectionSetup) + ); + } + #[test] fn version_two_identity_uses_exact_evidence_and_never_resource_path_guessing() { let (_root, collection) = collection(); let id = prepare(&collection, "a.md", b"old-a\n", b"new-a\n"); let directory = transaction_directory(&collection, &id); - let mut journal = read_runtime_journal(&directory).unwrap(); + let mut journal = read_runtime_journal(&collection, &directory).unwrap(); let empty = OperationResult { valid: true, result: serde_json::json!({}), @@ -1240,40 +2447,95 @@ mod tests { journal.changes.clear(); assert_eq!(legacy_operation_kind(&journal, &empty), None); assert!(matches!( - CanonicalOperationOutcome::legacy_recovered(empty).value, + legacy_recovered_v03(empty).value, crate::runtime::CanonicalOperationValue::LegacyRecoveredV03(_) )); } + #[test] + fn malformed_version_three_outcome_fails_closed() { + let (_root, collection) = collection(); + let id = prepare(&collection, "a.md", b"old-a\n", b"new-a\n"); + let directory = transaction_directory(&collection, &id); + let mut value: serde_json::Value = serde_json::from_slice( + &collection + .held_root() + .read(directory.join(JOURNAL_FILE)) + .unwrap(), + ) + .unwrap(); + value["operation_outcome"]["valid"] = serde_json::json!(true); + collection + .held_root() + .atomic_write( + &directory.join(JOURNAL_FILE), + &serde_json::to_vec_pretty(&value).unwrap(), + ) + .unwrap(); + + assert!(matches!( + resolve_runtime_commit(&collection, &id, &OperationContext::legacy()), + Err(TransactionError::InvalidJournal(message)) + if message.contains("requires a semantic value") + )); + } + #[test] fn version_two_operation_result_journal_is_read_as_typed_outcome() { let (_root, collection) = collection(); let id = prepare(&collection, "a.md", b"old-a\n", b"new-a\n"); let directory = transaction_directory(&collection, &id); - let mut value: serde_json::Value = - serde_json::from_slice(&fs::read(directory.join(JOURNAL_FILE)).unwrap()).unwrap(); + let mut value: serde_json::Value = serde_json::from_slice( + &collection + .held_root() + .read(directory.join(JOURNAL_FILE)) + .unwrap(), + ) + .unwrap(); assert!(value.get("operation_outcome").is_some()); assert!(value.get("operation_result").is_none()); - let typed: CanonicalOperationOutcome = - serde_json::from_value(value["operation_outcome"].take()).unwrap(); + let v3_outcome = value["operation_outcome"].clone(); + let typed: CanonicalOperationOutcome = serde_json::from_value(v3_outcome.clone()).unwrap(); + assert_eq!(serde_json::to_value(&typed).unwrap(), v3_outcome); + let expected_v03 = typed.to_v03(); value["version"] = serde_json::json!(LEGACY_RUNTIME_JOURNAL_VERSION); value["operation_result"] = serde_json::to_value(typed.to_v03()).unwrap(); value.as_object_mut().unwrap().remove("operation_outcome"); - fs::write( - directory.join(JOURNAL_FILE), - serde_json::to_vec_pretty(&value).unwrap(), - ) - .unwrap(); + collection + .held_root() + .atomic_write( + &directory.join(JOURNAL_FILE), + &serde_json::to_vec_pretty(&value).unwrap(), + ) + .unwrap(); + + let inventory = + legacy_runtime_journal_inventory(&collection, &OperationContext::legacy()).unwrap(); + assert_eq!(inventory.version_2, 1); + assert!(!inventory.is_zero()); let resolved = resolve_runtime_commit(&collection, &id, &OperationContext::legacy()) .unwrap() .unwrap(); assert!(matches!(resolved, RuntimeResolution::Prepared { .. })); - let recovered = journal_operation(&read_runtime_journal(&directory).unwrap()).unwrap(); + let recovered = + journal_operation(&read_runtime_journal(&collection, &directory).unwrap()).unwrap(); assert!(matches!( recovered.value, crate::runtime::CanonicalOperationValue::Update(None) )); + assert_eq!(recovered.to_v03(), expected_v03); + } + + #[test] + fn new_runtime_journal_fixture_satisfies_version_two_zero_gate() { + let (_root, collection) = collection(); + let _id = prepare(&collection, "a.md", b"old-a\n", b"new-a\n"); + assert!( + legacy_runtime_journal_inventory(&collection, &OperationContext::legacy(),) + .unwrap() + .is_zero() + ); } fn short_context(cancellation: &OperationCancellation) -> OperationContext { diff --git a/src/types/loader.rs b/src/types/loader.rs index 6e19193..ae6cd08 100644 --- a/src/types/loader.rs +++ b/src/types/loader.rs @@ -98,6 +98,42 @@ pub fn load_types_with_warnings( Ok(LoadTypesResult { types, warnings }) } +/// Load type resources from a snapshot obtained through the held collection root. +/// Parser helpers may use ordinary paths only inside this private, agent-owned +/// staging directory; collection authority is never reconstructed from a path. +pub(crate) fn load_types_with_warnings_held( + root: &crate::collection_root::CollectionRoot, + types_folder: &str, + migrations_folder: &str, +) -> Result { + let staging = tempfile::tempdir().map_err(|error| error.to_string())?; + let types = Path::new(types_folder); + for relative in root + .files_recursive(Path::new("")) + .map_err(|error| error.to_string())? + { + // Schema references are permitted to target JSON resources elsewhere + // below the collection root. Snapshot those alongside type documents. + if !relative.starts_with(types) + && relative.extension().and_then(|value| value.to_str()) != Some("json") + { + continue; + } + let bytes = root.read(&relative).map_err(|error| { + format!( + "Failed to read type resource '{}': {error}", + relative.display() + ) + })?; + let destination = staging.path().join(&relative); + if let Some(parent) = destination.parent() { + std::fs::create_dir_all(parent).map_err(|error| error.to_string())?; + } + std::fs::write(destination, bytes).map_err(|error| error.to_string())?; + } + load_types_with_warnings(staging.path(), types_folder, migrations_folder) +} + /// Compile already-resolved v0.3 type files supplied by a provider catalog. pub(crate) fn load_resolved_type_files( type_files: Vec, diff --git a/src/v03/batch.rs b/src/v03/batch.rs index 8938842..b0f45d9 100644 --- a/src/v03/batch.rs +++ b/src/v03/batch.rs @@ -2,7 +2,6 @@ use std::fs; use std::path::Path; use serde_json::{json, Value}; -use walkdir::WalkDir; use super::{Diagnostic, OperationResult, Operations}; use crate::mutation::{ @@ -86,7 +85,7 @@ pub(crate) fn prepare_single_runtime( return prepare_sparse_runtime(collection, operation, input, context); } - let before = collection.snapshot()?; + let before = collection.snapshot_with_context(context)?; context.check()?; let shadow = shadow_collection_context(collection, context)?; let typed_operation = operation.parse::()?; @@ -117,7 +116,7 @@ pub(crate) fn prepare_single_runtime( if desired == shadow.baseline { return Ok(RuntimeSinglePreparation::NoMutation(outcome)); } - let after = shadow.collection.snapshot()?; + let after = shadow.collection.snapshot_with_context(context)?; context.check()?; Ok(RuntimeSinglePreparation::Prepared(Box::new( RuntimeMutationPlan { @@ -251,6 +250,9 @@ fn prepare_sparse_runtime( let kind = operation.parse::()?; let input_path = input.get("path").and_then(Value::as_str); let shadow = sparse_shadow_collection(collection, input_path, context)?; + let captured_before = input_path + .map(|path| targeted_snapshot(&shadow.collection, path, context)) + .transpose()?; #[cfg(test)] pause_sparse_preparation(&collection.root); let shadow_input = match adapt_mtime_precondition(collection, &shadow.collection, input) { @@ -290,10 +292,9 @@ fn prepare_sparse_runtime( // validation. Only a derived create can target a path that was unknown // when the sparse shadow was copied. let mut baseline = shadow.baseline.clone(); - if input_path != Some(path.as_str()) { - if let Ok(bytes) = fs::read(collection.root.join(&path)) { - baseline.insert(path.clone(), bytes); - } + if input_path != Some(path.as_str()) && collection.held_root().exists_file(Path::new(&path)) { + let bytes = read_held_bounded(collection, Path::new(&path), context)?; + baseline.insert(path.clone(), bytes); } if kind == OperationKind::Create && baseline.contains_key(&path) @@ -316,7 +317,8 @@ fn prepare_sparse_runtime( )); } let mut desired = crate::transactions::FileBaseline::new(); - if let Ok(bytes) = fs::read(shadow.collection.root.join(&path)) { + if shadow.collection.held_root().exists_file(Path::new(&path)) { + let bytes = read_held_bounded(&shadow.collection, Path::new(&path), context)?; desired.insert(path.clone(), bytes); } if baseline == desired { @@ -362,11 +364,14 @@ fn prepare_sparse_runtime( } } - let before = delete_plan - .as_ref() - .map(planned_delete_snapshot) - .unwrap_or_else(|| targeted_snapshot(collection, &path))?; - let after = targeted_snapshot(&shadow.collection, &path)?; + let before = if let Some(planned) = delete_plan.as_ref() { + planned_delete_snapshot(planned)? + } else if input_path == Some(path.as_str()) { + captured_before.expect("an explicit sparse path has a captured snapshot") + } else { + targeted_snapshot(collection, &path, context)? + }; + let after = targeted_snapshot(&shadow.collection, &path, context)?; context.check()?; Ok(RuntimeSinglePreparation::Prepared(Box::new( RuntimeMutationPlan { @@ -475,8 +480,9 @@ fn planned_delete_snapshot( fn targeted_snapshot( collection: &Collection, path: &str, + context: &OperationContext, ) -> Result { - let records = match collection.snapshot_record(path) { + let records = match collection.snapshot_record_with_context(path, context) { Ok(record) => vec![record], Err(ProviderError::CollectionOpen(_)) => Vec::new(), Err(error) => return Err(error), @@ -500,13 +506,21 @@ fn sparse_shadow_collection( context.check()?; let directory = tempfile::tempdir().map_err(|error| ProviderError::CollectionOpen(error.to_string()))?; - copy_sparse_controls(collection, directory.path(), context)?; + let mut captured_entries = 0_u64; + let mut resource_entries = 0_u64; + copy_sparse_controls( + collection, + directory.path(), + context, + &mut captured_entries, + &mut resource_entries, + )?; let mut baseline = crate::transactions::FileBaseline::new(); if let Some(target) = target { - let source = collection.root.join(target); - if source.is_file() { - let bytes = fs::read(&source) - .map_err(|error| ProviderError::CollectionOpen(error.to_string()))?; + if collection.held_root().exists_file(Path::new(target)) { + captured_entries = checked_capture_increment(captured_entries)?; + context.check_entries(captured_entries)?; + let bytes = read_held_bounded(collection, Path::new(target), context)?; let destination = directory.path().join(target); if let Some(parent) = destination.parent() { fs::create_dir_all(parent) @@ -531,11 +545,17 @@ fn copy_sparse_controls( collection: &Collection, destination: &Path, context: &OperationContext, + captured_entries: &mut u64, + resource_entries: &mut u64, ) -> Result<(), ProviderError> { for relative in ["mdbase.yaml", "mdbase.lock.yaml"] { - let source = collection.root.join(relative); - if source.is_file() { - fs::copy(&source, destination.join(relative)) + if collection.held_root().exists_file(Path::new(relative)) { + *captured_entries = checked_capture_increment(*captured_entries)?; + *resource_entries = checked_capture_increment(*resource_entries)?; + context.check_entries(*captured_entries)?; + context.check_resource_entries(*resource_entries)?; + let bytes = read_held_bounded(collection, Path::new(relative), context)?; + fs::write(destination.join(relative), bytes) .map_err(|error| ProviderError::CollectionOpen(error.to_string()))?; } } @@ -544,34 +564,99 @@ fn copy_sparse_controls( collection.settings.contracts_folder.as_str(), "_schemas", ] { - let source = collection.root.join(folder); - if !source.is_dir() { - continue; - } - for entry in WalkDir::new(&source).follow_links(false) { + let paths = collection + .held_root() + .files_recursive(Path::new(folder)) + .map_err(|error| ProviderError::CollectionOpen(error.to_string()))?; + for relative in paths { context.check()?; - let entry = entry.map_err(|error| ProviderError::CollectionOpen(error.to_string()))?; - let relative = entry - .path() - .strip_prefix(&collection.root) - .map_err(|error| ProviderError::CollectionOpen(error.to_string()))?; - let target = destination.join(relative); - if entry.file_type().is_dir() { - fs::create_dir_all(&target) - .map_err(|error| ProviderError::CollectionOpen(error.to_string()))?; - } else if entry.file_type().is_file() { - if let Some(parent) = target.parent() { - fs::create_dir_all(parent) - .map_err(|error| ProviderError::CollectionOpen(error.to_string()))?; - } - fs::copy(entry.path(), &target) + *captured_entries = checked_capture_increment(*captured_entries)?; + *resource_entries = checked_capture_increment(*resource_entries)?; + context.check_entries(*captured_entries)?; + context.check_resource_entries(*resource_entries)?; + context.check_depth(relative.components().count().saturating_sub(1) as u64)?; + let target = destination.join(&relative); + if let Some(parent) = target.parent() { + fs::create_dir_all(parent) .map_err(|error| ProviderError::CollectionOpen(error.to_string()))?; } + let bytes = read_held_bounded(collection, &relative, context)?; + fs::write(&target, bytes) + .map_err(|error| ProviderError::CollectionOpen(error.to_string()))?; } } Ok(()) } +fn checked_capture_increment(value: u64) -> Result { + value.checked_add(1).ok_or({ + ProviderError::CaptureLimitExceeded(crate::runtime::CaptureLimitExceeded { + kind: crate::runtime::CaptureLimitKind::ArithmeticOverflow, + limit: u64::MAX, + attempted: u64::MAX, + }) + }) +} + +fn read_held_bounded( + collection: &Collection, + relative: &Path, + context: &OperationContext, +) -> Result, ProviderError> { + use std::io::Read; + context.check()?; + let mut file = collection + .held_root() + .open_file(relative) + .map_err(|error| { + ProviderError::CollectionOpen(format!( + "failed to open '{}': {error}", + relative.display() + )) + })?; + let size = file + .metadata() + .map_err(|error| ProviderError::CollectionOpen(error.to_string()))? + .len(); + context.check_file_bytes(size)?; + let capacity = usize::try_from(size).map_err(|_| crate::runtime::CaptureLimitExceeded { + kind: crate::runtime::CaptureLimitKind::ArithmeticOverflow, + limit: usize::MAX as u64, + attempted: size, + })?; + let mut bytes = Vec::new(); + bytes + .try_reserve_exact(capacity) + .map_err(|_| crate::runtime::CaptureLimitExceeded { + kind: crate::runtime::CaptureLimitKind::ArithmeticOverflow, + limit: usize::MAX as u64, + attempted: size, + })?; + let mut chunk = [0_u8; 64 * 1024]; + loop { + context.check()?; + let read = file + .read(&mut chunk) + .map_err(|error| ProviderError::CollectionOpen(error.to_string()))?; + if read == 0 { + break; + } + let attempted = (bytes.len() as u64).checked_add(read as u64).ok_or( + crate::runtime::CaptureLimitExceeded { + kind: crate::runtime::CaptureLimitKind::ArithmeticOverflow, + limit: u64::MAX, + attempted: u64::MAX, + }, + )?; + context.check_file_bytes(attempted)?; + context.charge_read(read as u64)?; + context.charge_retained(read as u64)?; + bytes.extend_from_slice(&chunk[..read]); + context.check()?; + } + Ok(bytes) +} + pub(crate) fn execute_wire_mutation( collection: &Collection, operation: &str, @@ -866,13 +951,17 @@ fn validate_authoritative_mtime( None, )] })?; - let current = modified_millis(&collection.root.join(path)).ok_or_else(|| { - vec![Diagnostic::error( - "concurrent_modification", - format!("File '{path}' no longer matches the requested modification time."), - Some(path.to_string()), - )] - })?; + let current = collection + .held_root() + .modified_millis(Path::new(path)) + .ok() + .ok_or_else(|| { + vec![Diagnostic::error( + "concurrent_modification", + format!("File '{path}' no longer matches the requested modification time."), + Some(path.to_string()), + )] + })?; if current != expected { return Err(vec![Diagnostic::error( "concurrent_modification", @@ -938,13 +1027,17 @@ fn adapt_mtime_precondition( None, )) })?; - let current = modified_millis(&collection.root.join(path)).ok_or_else(|| { - Box::new(Diagnostic::error( - "concurrent_modification", - format!("File '{path}' no longer matches the requested modification time."), - Some(path.to_string()), - )) - })?; + let current = collection + .held_root() + .modified_millis(Path::new(path)) + .ok() + .ok_or_else(|| { + Box::new(Diagnostic::error( + "concurrent_modification", + format!("File '{path}' no longer matches the requested modification time."), + Some(path.to_string()), + )) + })?; if current != expected { return Err(Box::new(Diagnostic::error( "concurrent_modification", @@ -952,13 +1045,17 @@ fn adapt_mtime_precondition( Some(path.to_string()), ))); } - let shadow_mtime = modified_millis(&shadow.root.join(path)).ok_or_else(|| { - Box::new(Diagnostic::error( - "batch_preflight_failed", - format!("Preflight record '{path}' is unavailable."), - Some(path.to_string()), - )) - })?; + let shadow_mtime = shadow + .held_root() + .modified_millis(Path::new(path)) + .ok() + .ok_or_else(|| { + Box::new(Diagnostic::error( + "batch_preflight_failed", + format!("Preflight record '{path}' is unavailable."), + Some(path.to_string()), + )) + })?; let mut adapted = input.as_object().cloned().unwrap_or_default(); adapted.insert( "last_known_mtime".to_string(), @@ -967,16 +1064,6 @@ fn adapt_mtime_precondition( Ok(Value::Object(adapted)) } -fn modified_millis(path: &Path) -> Option { - fs::metadata(path) - .ok()? - .modified() - .ok()? - .duration_since(std::time::UNIX_EPOCH) - .ok() - .and_then(|duration| u64::try_from(duration.as_millis()).ok()) -} - fn invalid_request(message: &str) -> OperationResult { failed(vec![Diagnostic::error("invalid_request", message, None)]) } @@ -1100,11 +1187,20 @@ mod tests { let typed = typed_collection.typed().unwrap().batch(request).unwrap(); let wire = execute(&wire_collection, &wire_input); assert!(wire.valid, "{wire:#?}"); - assert_eq!( - serde_json::to_value(&typed.value).unwrap(), - wire.result, - "dry_run={dry_run}" - ); + let mut typed_value = serde_json::to_value(&typed.value).unwrap(); + let mut wire_value = wire.result.clone(); + // These independent fixtures can commit on opposite sides of a + // wall-clock second. Verify the timestamp shape separately and + // compare every deterministic typed/wire field exactly. + for value in [&mut typed_value, &mut wire_value] { + for operation in value["operations"].as_array_mut().unwrap() { + if let Some(mtime) = operation["result"]["file"]["mtime"].as_str() { + assert!(!mtime.is_empty()); + operation["result"]["file"]["mtime"] = json!(""); + } + } + } + assert_eq!(typed_value, wire_value, "dry_run={dry_run}"); assert_eq!( typed.diagnostics, wire.diagnostics @@ -1221,7 +1317,7 @@ mod tests { let source = tempfile::tempdir().unwrap(); write( &source.path().join("mdbase.yaml"), - "spec_version: 0.3.0\nsettings:\n validation: warn\n", + "spec_version: 0.3.0\nsettings:\n validation: warn\n exclude: [.git/**, private/**]\nx-obsidian:\n bases:\n include: [views/*.base, private/*.base]\n", ); write( &source.path().join("_types/note.md"), @@ -1238,6 +1334,15 @@ mod tests { "spec_version: 0.3.0\n", ); write(&source.path().join("nested/hidden.md"), "must not copy"); + write( + &source.path().join("views/configured.base"), + "filters: []\n", + ); + write(&source.path().join("unconfigured.base"), "filters: []\n"); + write( + &source.path().join("private/excluded.base"), + "filters: []\n", + ); let collection = Collection::open(source.path()).unwrap(); let shadow = shadow_collection(&collection).unwrap(); @@ -1246,6 +1351,9 @@ mod tests { assert!(root.join("_types/note.md").is_file()); assert!(root.join("visible.md").is_file()); assert!(root.join("schema.json").is_file()); + assert!(root.join("views/configured.base").is_file()); + assert!(!root.join("unconfigured.base").exists()); + assert!(!root.join("private/excluded.base").exists()); assert!(!root.join(".git").exists()); assert!(!root.join("nested").exists()); } diff --git a/src/v03/collection_setup.rs b/src/v03/collection_setup.rs index e5115ef..ded9d76 100644 --- a/src/v03/collection_setup.rs +++ b/src/v03/collection_setup.rs @@ -280,7 +280,7 @@ impl Collection { Ok(commit) => commit, Err(error) => return setup_error(error.code(), error.to_string()), }; - let reopened = match Collection::open(&self.root) { + let reopened = match self.reopen_held(true) { Ok(collection) => collection, Err(error) => { return setup_error( @@ -390,7 +390,7 @@ fn plan_collection_setup( let provision_lock_path = shadow.directory.path().join(PROVISION_LOCK_PATH); let previous_lock_bytes = fs::read(&provision_lock_path).ok(); let mut lock = - read_provision_lock(shadow.directory.path()).map_err(|diagnostic| vec![*diagnostic])?; + read_provision_lock(&shadow.collection).map_err(|diagnostic| vec![*diagnostic])?; for receipt in &receipt_configuration { add_contributor( &mut lock, @@ -465,13 +465,16 @@ fn plan_unchanged_collection_setup( baseline_validation_errors: &[Diagnostic], provision_digest: &str, ) -> Result, Vec> { - let config_bytes = fs::read(collection.root.join("mdbase.yaml")).map_err(|error| { - vec![Diagnostic::error( - "invalid_collection_setup", - format!("Could not read mdbase.yaml for setup assessment: {error}"), - Some("mdbase.yaml".to_string()), - )] - })?; + let config_bytes = collection + .held_root() + .read("mdbase.yaml") + .map_err(|error| { + vec![Diagnostic::error( + "invalid_collection_setup", + format!("Could not read mdbase.yaml for setup assessment: {error}"), + Some("mdbase.yaml".to_string()), + )] + })?; let mut config: serde_yaml::Value = serde_yaml::from_slice(&config_bytes).map_err(|error| { vec![Diagnostic::error( "invalid_collection_setup", @@ -517,8 +520,8 @@ fn plan_unchanged_collection_setup( } } - let previous_lock_bytes = fs::read(collection.root.join(PROVISION_LOCK_PATH)).ok(); - let mut lock = read_provision_lock(&collection.root).map_err(|diagnostic| vec![*diagnostic])?; + let previous_lock_bytes = collection.held_root().read(PROVISION_LOCK_PATH).ok(); + let mut lock = read_provision_lock(collection).map_err(|diagnostic| vec![*diagnostic])?; for receipt in &receipt_configuration { add_contributor( &mut lock, @@ -893,9 +896,8 @@ fn yaml_shape(value: &serde_yaml::Value) -> &'static str { } } -fn read_provision_lock(root: &std::path::Path) -> Result> { - let path = root.join(PROVISION_LOCK_PATH); - let bytes = match fs::read(&path) { +fn read_provision_lock(collection: &Collection) -> Result> { + let bytes = match collection.held_root().read(PROVISION_LOCK_PATH) { Ok(bytes) => bytes, Err(error) if error.kind() == std::io::ErrorKind::NotFound => { return Ok(ProvisionLock { @@ -1149,6 +1151,33 @@ mod tests { } } + #[cfg(unix)] + #[test] + fn replacement_root_never_receives_setup_reads_or_publication() { + let (directory, collection) = collection("spec_version: 0.3.0\n"); + let original = directory.path().to_path_buf(); + let held = original.with_extension("setup-held"); + fs::rename(&original, &held).unwrap(); + fs::create_dir(&original).unwrap(); + fs::write(original.join("mdbase.yaml"), "spec_version: 0.3.0\n").unwrap(); + fs::write(original.join("sentinel"), "replacement\n").unwrap(); + + let declaration = setup("dev.example.editor"); + let assessment = collection.assess_collection_setup(&declaration); + assert!(assessment.valid, "{:?}", assessment.diagnostics); + let applied = + collection.apply_collection_setup(&declaration, &apply_options(&assessment.result)); + assert!(applied.valid, "{:?}", applied.diagnostics); + assert!(fs::read_to_string(held.join("mdbase.yaml")) + .unwrap() + .contains("x-obsidian")); + assert_eq!( + fs::read_to_string(original.join("sentinel")).unwrap(), + "replacement\n" + ); + assert!(!original.join(PROVISION_LOCK_PATH).exists()); + } + #[test] fn empty_setup_is_current_when_the_collection_has_preexisting_errors() { let (directory, collection) = collection_with_invalid_record(); diff --git a/src/v03/delete_tests.rs b/src/v03/delete_tests.rs index 285254c..2ab10e7 100644 --- a/src/v03/delete_tests.rs +++ b/src/v03/delete_tests.rs @@ -247,12 +247,13 @@ fn fixture_with_path_type(name: &str, bytes: &[u8]) -> tempfile::TempDir { root } +#[cfg(feature = "legacy-collection-mutation")] #[test] fn delete_probe_matrix_counts_real_boundaries() { let root = fixture(); let collection = Collection::open(root.path()).unwrap(); crate::mutation::reset_mutation_path_probes(); - let legacy = collection.delete(&json!({"path": "target.md", "dry_run": true})); + let legacy = collection.delete_legacy(&json!({"path": "target.md", "dry_run": true})); assert_eq!(legacy["would_delete"], true); assert_eq!( crate::mutation::mutation_path_probes(), diff --git a/src/v03/operations.rs b/src/v03/operations.rs index 3e62bdf..c119132 100644 --- a/src/v03/operations.rs +++ b/src/v03/operations.rs @@ -535,7 +535,7 @@ fn planned_operation_result( record: crate::mutation::PlannedRecord, ) -> OperationResult { let path = record.path.to_string(); - let metadata = match std::fs::metadata(record.path.under(&collection.root)) { + let metadata = match collection.held_root().metadata(&record.path.to_path_buf()) { Ok(metadata) => metadata, Err(error) => { return failed_result(vec![Diagnostic::error( diff --git a/src/v03/query.rs b/src/v03/query.rs index e83e52d..9330ad6 100644 --- a/src/v03/query.rs +++ b/src/v03/query.rs @@ -21,8 +21,9 @@ pub(crate) fn execute_profiled( collection: &Collection, input: &Value, ) -> (OperationResult, QueryPerformance) { - execute_wire_profiled_cancellable(collection, input, &OperationCancellation::new(), false) - .expect("a fresh cancellation token cannot be cancelled") + let context = crate::runtime::OperationContext::internal(); + execute_wire_profiled_cancellable(collection, input, context.cancellation(), false) + .expect("the context-free compatibility context is active") } pub(crate) fn execute_cancellable( diff --git a/src/v03/rename_tests.rs b/src/v03/rename_tests.rs index 43d6496..214c17e 100644 --- a/src/v03/rename_tests.rs +++ b/src/v03/rename_tests.rs @@ -353,7 +353,7 @@ fn failure_dry_run_and_simulation_adapter_matrix_is_ordered_and_differential() { #[cfg(unix)] #[test] -fn typed_wire_and_runtime_reject_replaced_collection_roots() { +fn typed_wire_and_runtime_keep_held_authority_when_collection_name_is_replaced() { use std::os::unix::fs::symlink; fn replace(root: &std::path::Path, external: &std::path::Path) -> std::path::PathBuf { @@ -380,7 +380,8 @@ fn typed_wire_and_runtime_reject_replaced_collection_roots() { CollectionPath::new("source.md").unwrap(), CollectionPath::new("renamed.md").unwrap(), )); - assert!(typed.is_err()); + assert!(typed.is_ok(), "{typed:?}"); + assert!(held.join("renamed.md").exists()); assert!(!typed_external.path().join("renamed.md").exists()); restore(typed_root.path(), &held); @@ -398,7 +399,8 @@ fn typed_wire_and_runtime_reject_replaced_collection_roots() { "rename", &json!({"from": "source.md", "to": "renamed.md"}), ); - assert!(!wire.valid); + assert!(wire.valid, "{wire:?}"); + assert!(held.join("renamed.md").exists()); assert!(!wire_external.path().join("renamed.md").exists()); restore(wire_root.path(), &held); @@ -417,7 +419,8 @@ fn typed_wire_and_runtime_reject_replaced_collection_roots() { json!({"from": "source.md", "to": "renamed.md"}), ); let runtime_result = runtime.execute(&request); - assert!(runtime_result.is_err() || runtime_result.is_ok_and(|result| !result.valid)); + assert!(runtime_result.is_ok_and(|result| result.valid)); + assert!(held.join("renamed.md").exists()); assert!(!runtime_external.path().join("renamed.md").exists()); restore(runtime_root.path(), &held); } diff --git a/src/v03/type_pack.rs b/src/v03/type_pack.rs index 0724d1a..1b5911f 100644 --- a/src/v03/type_pack.rs +++ b/src/v03/type_pack.rs @@ -10,7 +10,7 @@ use super::{revision, validate_type_pack, validate_type_pack_lock, Diagnostic, O use crate::api::CollectionPath; use crate::frontmatter::parser::{is_parse_error, parse_document}; use crate::mutation::shadow as mutation_shadow; -use crate::{Collection, SpecProfile}; +use crate::Collection; #[derive(Debug, Clone, Deserialize, Serialize)] pub struct TypePackResource { @@ -380,18 +380,17 @@ pub(crate) fn plan_type_pack( } let target = validate_resource_target(collection, &resource.kind, &resource.target, bytes) .map_err(pack_plan_error)?; - crate::operations::ensure_no_symlink_components( - &collection.root, - target.as_str(), - SpecProfile::V03, - ) - .map_err(|error| pack_plan_error(format!("Unsafe type-pack target: {error}")))?; - let before = read_optional(&target.under(&collection.root)).map_err(|error| { - pack_plan_error(format!( - "Could not inspect type-pack target '{}': {error}", - resource.target - )) - })?; + collection + .held_root() + .ensure_no_symlink_components(Path::new(target.as_str())) + .map_err(|error| pack_plan_error(format!("Unsafe type-pack target: {error}")))?; + let before = + read_optional_held(collection, Path::new(target.as_str())).map_err(|error| { + pack_plan_error(format!( + "Could not inspect type-pack target '{}': {error}", + resource.target + )) + })?; let current_digest = before.as_deref().map(revision); let installed = current_resources .get(resource.source.as_str()) @@ -494,12 +493,13 @@ pub(crate) fn plan_type_pack( let target = CollectionPath::new(&resource.target).map_err(|error| { pack_plan_error(format!("Unsafe installed type-pack target: {error}")) })?; - let before = read_optional(&target.under(&collection.root)).map_err(|error| { - pack_plan_error(format!( - "Could not inspect installed type-pack target '{}': {error}", - resource.target - )) - })?; + let before = + read_optional_held(collection, Path::new(target.as_str())).map_err(|error| { + pack_plan_error(format!( + "Could not inspect installed type-pack target '{}': {error}", + resource.target + )) + })?; let current_digest = before.as_deref().map(revision); let (action, reason) = if resource.mode != "managed" { ("preserve", None) @@ -674,7 +674,7 @@ fn apply_type_pack_plan(collection: &Collection, plan: TypePackPlan) -> Operatio Ok(commit) => commit, Err(error) => return pack_diagnostic(error.code(), error.to_string()), }; - let _reopened = match Collection::open(&collection.root) { + let _reopened = match collection.reopen_held(true) { Ok(reopened) => reopened, Err(error) => { return pack_diagnostic( @@ -765,8 +765,7 @@ pub(crate) fn stage_type_pack_plan( fn read_type_pack_lock( collection: &Collection, ) -> Result<(TypePackLock, Option>), Box> { - let path = collection.root.join(TYPE_PACK_LOCK_PATH); - let bytes = match fs::read(&path) { + let bytes = match collection.held_root().read(TYPE_PACK_LOCK_PATH) { Ok(bytes) => Some(bytes), Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, Err(error) => { @@ -832,8 +831,8 @@ fn serialize_type_pack_lock(lock: &TypePackLock) -> Result, Box std::io::Result>> { - match fs::read(path) { +fn read_optional_held(collection: &Collection, path: &Path) -> std::io::Result>> { + match collection.held_root().read(path) { Ok(bytes) => Ok(Some(bytes)), Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), Err(error) => Err(error), @@ -1642,6 +1641,33 @@ mod tests { (root, collection) } + #[cfg(unix)] + #[test] + fn replacement_root_never_receives_type_pack_reads_or_publication() { + let (root, collection) = collection(); + let document = "---\nkind: mdbase.type\nname: task\nschema:\n dialect: json-schema-2020-12\n value: { type: object }\n---\n"; + let pack = provision( + manifest(&[("type", "task.md", "_types/task.md", document)]), + vec![resource("task.md", document)], + ); + let original = root.path().to_path_buf(); + let held = original.with_extension("pack-held"); + fs::rename(&original, &held).unwrap(); + fs::create_dir(&original).unwrap(); + write(&original.join("mdbase.yaml"), "spec_version: 0.3.0\n"); + write(&original.join("sentinel"), "replacement\n"); + + let applied = apply_pack(&collection, &pack); + assert!(applied.valid, "{:?}", applied.diagnostics); + assert!(held.join("_types/task.md").is_file()); + assert!(held.join(TYPE_PACK_LOCK_PATH).is_file()); + assert_eq!( + fs::read_to_string(original.join("sentinel")).unwrap(), + "replacement\n" + ); + assert!(!original.join("_types").exists()); + } + fn existing_setup(contract_id: &str, revision: &str) -> ContractSetupChoice { ContractSetupChoice { contract: ContractIdentity { diff --git a/src/validation/validator.rs b/src/validation/validator.rs index 93a7264..7ad367f 100644 --- a/src/validation/validator.rs +++ b/src/validation/validator.rs @@ -503,10 +503,11 @@ impl Collection { { return error; } - if let Err(error) = - crate::operations::ensure_no_symlink_components(&self.root, path, self.spec_profile) + if let Err(error) = self + .held_root() + .ensure_no_symlink_components(std::path::Path::new(path)) { - return error; + return crate::errors::op_error(PATH_TRAVERSAL, &error.to_string()); } if input.get("frontmatter").is_none() { let root = match self.root_capability() { @@ -532,9 +533,7 @@ impl Collection { } } - let collection_snapshot = match self - .capture_collection_snapshot(&crate::OperationCancellation::new()) - { + let collection_snapshot = match self.capture_collection_snapshot_current() { Ok(snapshot) => snapshot, Err(error) if path diff --git a/src/views/execute.rs b/src/views/execute.rs index 82bb1bd..ea9da95 100644 --- a/src/views/execute.rs +++ b/src/views/execute.rs @@ -1,12 +1,10 @@ use std::collections::{BTreeMap, HashSet}; -use std::fs; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; use serde_json::{json, Map, Value}; use sha2::{Digest, Sha256}; -use walkdir::WalkDir; use super::expression::{ self, serialize_bases_file, BasesEvaluationContext, BasesFile, BasesLink, BasesTimezone, @@ -127,12 +125,11 @@ fn canonical_descriptor( diagnostics.push(diagnostic); return None; } - let source = collection.root.join(&record.rel_path); Some(ViewDocumentDescriptor { source: ViewSourceDescriptor { path: record.rel_path.clone(), format: "mdbase.view".to_string(), - revision: file_revision(&source).unwrap_or_default(), + revision: file_revision(collection, &record.rel_path).unwrap_or_default(), writable: true, }, id: record @@ -287,8 +284,8 @@ fn obsidian_documents( obsidian_source_paths(collection) .into_iter() .filter_map(|path| { - let relative = relative_path(&collection.root, &path)?; - let source = match fs::read_to_string(&path) { + let relative = path.to_string_lossy().replace('\\', "/"); + let source = match collection.held_root().read_string(&path) { Ok(source) => source, Err(error) => { let mut diagnostic = Diagnostic::error( @@ -662,7 +659,7 @@ fn execute_obsidian(collection: &Collection, request: &ViewReferenceInput) -> Op Some(request.path.clone()), ); } - let source = match fs::read_to_string(&relative) { + let source = match collection.held_root().read_string(&relative) { Ok(source) => source, Err(error) => { return failed( @@ -1177,19 +1174,17 @@ pub(crate) fn obsidian_source_paths(collection: &Collection) -> Vec { .iter() .filter_map(|pattern| glob_regex(pattern).ok()) .collect::>(); - WalkDir::new(&collection.root) - .follow_links(false) + collection + .held_root() + .files_recursive(Path::new("")) + .unwrap_or_default() .into_iter() - .filter_map(Result::ok) - .filter(|entry| entry.file_type().is_file()) - .filter(|entry| entry.path().extension().and_then(|value| value.to_str()) == Some("base")) - .filter(|entry| { - relative_path(&collection.root, entry.path()).is_some_and(|path| { - super::normalized_source_path(&path).is_some() - && patterns.iter().any(|pattern| pattern.is_match(&path)) - }) + .filter(|path| path.extension().and_then(|value| value.to_str()) == Some("base")) + .filter(|path| { + let relative = path.to_string_lossy().replace('\\', "/"); + super::normalized_source_path(&relative).is_some() + && patterns.iter().any(|pattern| pattern.is_match(&relative)) }) - .map(|entry| entry.into_path()) .collect() } @@ -1257,31 +1252,25 @@ fn safe_view_path(collection: &Collection, path: &str) -> Result Option { - Some( - path.strip_prefix(root) - .ok()? - .to_string_lossy() - .replace('\\', "/"), - ) -} -fn file_revision(path: &Path) -> Option { - fs::read(path).ok().map(|bytes| revision(&bytes)) +fn file_revision(collection: &Collection, path: &str) -> Option { + collection + .held_root() + .read(path) + .ok() + .map(|bytes| revision(&bytes)) } fn revision(bytes: &[u8]) -> String { let digest = Sha256::digest(bytes); diff --git a/src/views/mod.rs b/src/views/mod.rs index 9a24ae4..c9ed6bb 100644 --- a/src/views/mod.rs +++ b/src/views/mod.rs @@ -9,12 +9,10 @@ pub use model::{ NamedViewDescriptor, ViewDocumentDescriptor, ViewPresentation, ViewPropertyDescriptor, }; -use serde_json::Value; -use std::path::PathBuf; - use crate::api::CollectionPath; use crate::v03::OperationResult; use crate::Collection; +use serde_json::Value; pub(crate) fn list(collection: &Collection, input: &Value) -> OperationResult { execute::list_views(collection, input) @@ -40,10 +38,6 @@ pub(crate) fn delete_source(collection: &Collection, input: &Value) -> Operation source::delete(collection, input) } -pub(crate) fn compatibility_source_paths(collection: &Collection) -> Vec { - execute::obsidian_source_paths(collection) -} - pub(crate) use execute::{ base_uses_backlinks, combined_filter_matches, evaluate_property, is_configured_obsidian_source, validate_base_expressions, diff --git a/src/views/source.rs b/src/views/source.rs index a4064e8..11327c1 100644 --- a/src/views/source.rs +++ b/src/views/source.rs @@ -1,6 +1,5 @@ //! Revision-safe access to complete saved-view source documents. -use std::fs; use std::path::Path; use serde_json::{json, Value}; @@ -9,10 +8,7 @@ use super::execute::is_configured_obsidian_source; use super::model::ObsidianBaseDocument; use crate::diagnostic::Diagnostic; use crate::frontmatter::parser::{is_parse_error, parse_document, yaml_mapping_to_json}; -use crate::operations::{ - atomic_create, atomic_write, ensure_no_symlink_components, ensure_revision, - ensure_safe_relative_path, sync_directory, -}; +use crate::operations::ensure_safe_relative_path; use crate::v03::{self, OperationResult}; use crate::Collection; @@ -45,8 +41,10 @@ pub(super) fn create(collection: &Collection, input: &Value) -> OperationResult if let Err(diagnostics) = validate_document(collection, &path, document, false) { return failed_many(diagnostics); } - let full_path = collection.root.join(&path); - match atomic_create(&full_path, document.as_bytes()) { + match collection + .held_root() + .atomic_create(Path::new(&path), document.as_bytes()) + { Ok(()) => source_result(collection, &path), Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { failed(Diagnostic::error( @@ -78,15 +76,21 @@ pub(super) fn update(collection: &Collection, input: &Value) -> OperationResult if let Err(diagnostics) = validate_document(collection, &path, document, true) { return failed_many(diagnostics); } - let full_path = collection.root.join(&path); - if let Err(error) = ensure_revision( - &full_path, + let current = match collection.held_root().read(&path) { + Ok(bytes) => bytes, + Err(error) => return failed(io_diagnostic(&path, error)), + }; + if let Err(error) = ensure_revision_bytes( + ¤t, &path, input.get("if_revision").and_then(Value::as_str), ) { return legacy_failure(error, &path); } - match atomic_write(&full_path, document.as_bytes()) { + match collection + .held_root() + .atomic_write(Path::new(&path), document.as_bytes()) + { Ok(()) => source_result(collection, &path), Err(error) => failed(io_diagnostic(&path, error)), } @@ -104,21 +108,18 @@ pub(super) fn delete(collection: &Collection, input: &Value) -> OperationResult if let Err(diagnostic) = validate_existing_path(collection, &path) { return failed(*diagnostic); } - let full_path = collection.root.join(&path); - if let Err(error) = ensure_revision( - &full_path, + let current = match collection.held_root().read(&path) { + Ok(bytes) => bytes, + Err(error) => return failed(io_diagnostic(&path, error)), + }; + if let Err(error) = ensure_revision_bytes( + ¤t, &path, input.get("if_revision").and_then(Value::as_str), ) { return legacy_failure(error, &path); } - match fs::remove_file(&full_path).and_then(|()| { - full_path - .parent() - .map(sync_directory) - .transpose() - .map(|_| ()) - }) { + match collection.held_root().remove_file(Path::new(&path)) { Ok(()) => OperationResult { valid: true, result: json!({ "path": path, "deleted": true }), @@ -221,21 +222,18 @@ fn slug(value: &str) -> String { } fn validate_existing_path(collection: &Collection, path: &str) -> DiagnosticResult<()> { - if !collection.root.join(path).is_file() { - return Err(Box::new(Diagnostic::error( - "view_not_found", - format!("Saved-view source '{path}' does not exist."), - Some(path.to_string()), - ))); - } - validate_document( - collection, - path, - &fs::read_to_string(collection.root.join(path)) - .map_err(|error| io_diagnostic(path, error))?, - true, - ) - .map_err(|diagnostics| { + let document = collection.held_root().read_string(path).map_err(|error| { + if error.kind() == std::io::ErrorKind::NotFound { + Diagnostic::error( + "view_not_found", + format!("Saved-view source '{path}' does not exist."), + Some(path.to_string()), + ) + } else { + io_diagnostic(path, error) + } + })?; + validate_document(collection, path, &document, true).map_err(|diagnostics| { Box::new(diagnostics.into_iter().next().unwrap_or_else(|| { Diagnostic::error( "invalid_view", @@ -256,12 +254,10 @@ fn validate_safe_path(collection: &Collection, path: &str) -> DiagnosticResult { if require_existing_kind { - let current = fs::read_to_string(collection.root.join(path)) + let current = collection + .held_root() + .read_string(path) .map_err(|error| vec![io_diagnostic(path, error)])?; validate_canonical_document(path, ¤t)?; } @@ -342,7 +340,7 @@ fn validate_canonical_document(path: &str, document: &str) -> Result<(), Vec OperationResult { - let bytes = match fs::read(collection.root.join(path)) { + let bytes = match collection.held_root().read(path) { Ok(bytes) => bytes, Err(error) => return failed(io_diagnostic(path, error)), }; @@ -417,6 +415,19 @@ fn legacy_failure(error: Value, path: &str) -> OperationResult { failed(legacy_diagnostic(error, path)) } +fn ensure_revision_bytes(bytes: &[u8], path: &str, expected: Option<&str>) -> Result<(), Value> { + let Some(expected) = expected else { + return Ok(()); + }; + if v03::revision(bytes) != expected { + return Err(crate::errors::op_error( + crate::errors::CONCURRENT_MODIFICATION, + &format!("File '{path}' was modified externally"), + )); + } + Ok(()) +} + fn failed(diagnostic: Diagnostic) -> OperationResult { failed_many(vec![diagnostic]) } diff --git a/src/watch/real.rs b/src/watch/real.rs index 5858b3d..c6a65b3 100644 --- a/src/watch/real.rs +++ b/src/watch/real.rs @@ -300,8 +300,13 @@ impl CollectionWatcher { } fn open_internal(root: &Path, debounce: Duration) -> Result { + // Acquire collection authority exactly once. The path retained below is + // only the notify registration/display name; every snapshot read is + // rooted in this held collection even if that name is replaced. + let collection = Collection::open_for_observation(root) + .map_err(|error| WatchError::Collection(collection_error(&error)))?; let root = root.to_path_buf(); - let initial = Snapshot::load(&root)?; + let initial = Snapshot::load(&collection)?; let (events_tx, events) = mpsc::channel(); let (commands, command_rx) = mpsc::channel(); let pending_rescans = Arc::new(AtomicUsize::new(0)); @@ -329,6 +334,7 @@ impl CollectionWatcher { .spawn(move || { watch_loop( root, + collection, debounce, initial, WorkerChannels { @@ -698,7 +704,13 @@ struct WorkerChannels { epoch: Arc, } -fn watch_loop(root: PathBuf, debounce: Duration, mut snapshot: Snapshot, channels: WorkerChannels) { +fn watch_loop( + root: PathBuf, + collection: Collection, + debounce: Duration, + mut snapshot: Snapshot, + channels: WorkerChannels, +) { let WorkerChannels { inputs, filesystem_callback, @@ -731,7 +743,7 @@ fn watch_loop(root: PathBuf, debounce: Duration, mut snapshot: Snapshot, channel // registration before reporting readiness. Hosts can now treat `open` as // a stable boundary instead of triggering an additional full rescan. let mut startup_refresh_failure = None; - match Snapshot::load(&root) { + match Snapshot::load(&collection) { Ok(mut next) => { next.retain_classified_invalid(&snapshot); for event in snapshot.diff(&next) { @@ -925,14 +937,14 @@ fn watch_loop(root: PathBuf, debounce: Duration, mut snapshot: Snapshot, channel let refresh_revision = invalidation_revision.load(Ordering::Acquire); let mut next = snapshot.clone(); let refreshed = if full_rescan { - Snapshot::load(&root).map(|mut candidate| { + Snapshot::load(&collection).map(|mut candidate| { candidate.retain_classified_invalid(&snapshot); let diff = snapshot.diff(&candidate); next = candidate; diff }) } else { - next.refresh_paths(&root, &pending_paths) + next.refresh_paths(&collection, &pending_paths) }; if epoch.is_exhausted() { fail_pending_rescans(&mut pending_rescans); @@ -1203,8 +1215,11 @@ struct RecordState { } impl Snapshot { - fn load(root: &Path) -> Result { - let collection = Collection::open_for_observation(root) + fn load(held: &Collection) -> Result { + // Reload configuration/resources through a clone of the already-held + // capability. Never reacquire through the notify display path. + let collection = held + .reopen_held(false) .map_err(|error| WatchError::Collection(collection_error(&error)))?; let observed = collection .snapshot_for_watcher() @@ -1366,14 +1381,12 @@ impl Snapshot { fn refresh_paths( &mut self, - root: &Path, + collection: &Collection, paths: &BTreeSet, ) -> Result, WatchError> { if paths.is_empty() { return Ok(Vec::new()); } - let collection = Collection::open_for_observation(root) - .map_err(|error| WatchError::Collection(collection_error(&error)))?; let mut before = BTreeMap::new(); let mut replacements = Vec::new(); for path in paths { @@ -1381,7 +1394,7 @@ impl Snapshot { if let Some(record) = self.records.get(&relative) { before.insert(relative.clone(), record.clone()); } - replacements.push((relative.clone(), load_record(&collection, &relative)?)); + replacements.push((relative.clone(), load_record(collection, &relative)?)); } for (path, outcome) in replacements { match outcome { @@ -1813,7 +1826,8 @@ mod tests { "spec_version: 0.3.0\nsettings:\n validation: warn\n", ) .unwrap(); - let snapshot = Snapshot::load(directory.path()).unwrap(); + let collection = Collection::open_for_observation(directory.path()).unwrap(); + let snapshot = Snapshot::load(&collection).unwrap(); let changed = |path: &str| { Event::new(EventKind::Modify(ModifyKind::Data(DataChange::Content))) .add_path(directory.path().join(path)) @@ -1894,7 +1908,8 @@ mod tests { assert_eq!(phase(), "committing"); assert!(!directory.path().join("pending.md").exists()); - let observed = Snapshot::load(directory.path()).unwrap(); + let collection = Collection::open_for_observation(directory.path()).unwrap(); + let observed = Snapshot::load(&collection).unwrap(); assert!(observed.records.is_empty()); assert_eq!(phase(), "committing"); assert!(!directory.path().join("pending.md").exists()); @@ -1960,6 +1975,68 @@ mod tests { assert_eq!(observed, expected); } + #[cfg(unix)] + #[test] + fn watcher_root_replacement_never_reads_or_emits_replacement_records_or_resources() { + let parent = tempfile::tempdir().unwrap(); + let display = parent.path().join("collection"); + let held_name = parent.path().join("held-original"); + fs::create_dir(&display).unwrap(); + fs::create_dir(display.join("_types")).unwrap(); + fs::write( + display.join("mdbase.yaml"), + "spec_version: 0.3.0\nsettings:\n validation: warn\n", + ) + .unwrap(); + fs::write(display.join("original.md"), "---\ntitle: Original\n---\n").unwrap(); + fs::write( + display.join("_types/original.md"), + "---\nname: original\nfields: {}\n---\n", + ) + .unwrap(); + let watcher = CollectionWatcher::open(&display, Duration::from_millis(10)).unwrap(); + + fs::rename(&display, &held_name).unwrap(); + fs::create_dir(&display).unwrap(); + fs::create_dir(display.join("_types")).unwrap(); + // Deliberately invalid replacement configuration proves a path reopen + // would fail before it could even inspect the replacement payloads. + fs::write(display.join("mdbase.yaml"), "not: [valid\n").unwrap(); + fs::write( + display.join("replacement.md"), + "---\ntitle: Replacement\n---\nreplacement body\n", + ) + .unwrap(); + fs::write( + display.join("_types/replacement.md"), + "---\nname: replacement\nfields: {}\n---\n", + ) + .unwrap(); + + let control = watcher.test_control(); + control.invoke_installed_modify_callback(&display.join("replacement.md")); + watcher.rescan().unwrap(); + assert!(watcher + .recv_timeout(Duration::from_millis(200)) + .unwrap() + .is_none()); + + // The original authority remains live and readable after displacement. + fs::write( + held_name.join("original.md"), + "---\ntitle: Held changed\n---\n", + ) + .unwrap(); + watcher.rescan().unwrap(); + let event = watcher + .recv_timeout(Duration::from_secs(2)) + .unwrap() + .expect("held original change event"); + assert_eq!(event.event_type, "mdbase.record.modified"); + assert_eq!(event.payload["path"], "original.md"); + assert!(!event.payload.to_string().contains("Replacement")); + } + #[test] fn watcher_reconciles_recursive_directory_rename() { let directory = tempfile::tempdir().unwrap(); diff --git a/tests/bom_frontmatter_regression.rs b/tests/bom_frontmatter_regression.rs index f8a991f..ca8cb15 100644 --- a/tests/bom_frontmatter_regression.rs +++ b/tests/bom_frontmatter_regression.rs @@ -1,3 +1,5 @@ +#![cfg(feature = "legacy-collection-mutation")] + //! Regression tests for BOM-prefixed frontmatter documents. //! //! Finding: `.ops/work/bom-prefixed-records-lose-frontmatter-on-update.md`. diff --git a/tests/compile/legacy-feature-boundary/Cargo.toml b/tests/compile/legacy-feature-boundary/Cargo.toml new file mode 100644 index 0000000..3fea365 --- /dev/null +++ b/tests/compile/legacy-feature-boundary/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "legacy-feature-boundary" +version = "0.0.0" +edition = "2021" +publish = false + +[features] +default = ["legacy"] +legacy = ["mdbase/legacy-collection-mutation"] + +[dependencies] +mdbase = { path = "../../..", default-features = false } +serde_json = "1" + +[workspace] diff --git a/tests/compile/legacy-feature-boundary/src/lib.rs b/tests/compile/legacy-feature-boundary/src/lib.rs new file mode 100644 index 0000000..75ddeaa --- /dev/null +++ b/tests/compile/legacy-feature-boundary/src/lib.rs @@ -0,0 +1,14 @@ +#![deny(deprecated)] + +use mdbase::Collection; +use serde_json::Value; + +pub fn released_04_facade_is_source_compatible(collection: &Collection, input: &Value) { + let _ = collection.create(input); + let _ = collection.update(input); + let _ = collection.delete(input); + let _ = collection.rename(input); + let _ = collection.backfill(input); + let _ = collection.batch_update(input, None, false); + let _ = collection.batch_delete(input, None); +} diff --git a/tests/conformance.rs b/tests/conformance.rs index 1e36ba0..96414c1 100644 --- a/tests/conformance.rs +++ b/tests/conformance.rs @@ -1,3 +1,5 @@ +#![cfg(feature = "legacy-collection-mutation")] + //! Conformance test runner for mdbase. //! //! Reads YAML test files from ~/projects/mdbase-spec/tests/ and executes @@ -1304,7 +1306,7 @@ fn execute_operation( let all_files = collection .build_all_files_data() .expect("collection snapshot"); - let backlinks_index = collection.build_backlinks_index(&all_files); + let backlinks_index = collection.build_backlinks_index(&all_files).unwrap(); let all_files_arc = std::sync::Arc::new(all_files); let backlinks_arc = std::sync::Arc::new(backlinks_index); let types_arc = std::sync::Arc::new(collection.types().clone()); diff --git a/tests/feature_graph.rs b/tests/feature_graph.rs new file mode 100644 index 0000000..8e1206f --- /dev/null +++ b/tests/feature_graph.rs @@ -0,0 +1,13 @@ +#[cfg(not(windows))] +use std::process::Command; + +#[cfg(not(windows))] +#[test] +fn canonical_consumer_graphs_resolve_without_legacy_mutation() { + let repository = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + let status = Command::new(repository.join("scripts/check-no-legacy-feature.sh")) + .current_dir(repository) + .status() + .expect("feature graph guard must execute"); + assert!(status.success(), "canonical feature graph guard failed"); +} diff --git a/tests/frontmatter_serialization_failures.rs b/tests/frontmatter_serialization_failures.rs index 55f09d4..7f4a264 100644 --- a/tests/frontmatter_serialization_failures.rs +++ b/tests/frontmatter_serialization_failures.rs @@ -1,3 +1,5 @@ +#![cfg(feature = "legacy-collection-mutation")] + use std::fs; use mdbase::Collection; diff --git a/tests/generated_snapshot.rs b/tests/generated_snapshot.rs index 4ff12d3..ec46bb1 100644 --- a/tests/generated_snapshot.rs +++ b/tests/generated_snapshot.rs @@ -1,3 +1,5 @@ +#![cfg(feature = "legacy-collection-mutation")] + use std::fs; use std::sync::{Arc, Barrier}; diff --git a/tests/invalid_record_outcome.rs b/tests/invalid_record_outcome.rs index 322775f..3f441c8 100644 --- a/tests/invalid_record_outcome.rs +++ b/tests/invalid_record_outcome.rs @@ -1,3 +1,5 @@ +#![cfg(feature = "legacy-collection-mutation")] + use std::fs; use mdbase::Collection; @@ -257,18 +259,6 @@ fn targeted_validation_preserves_file_read_failed_outcome() { fn cache_commits_invalid_rows_and_repair_converges_to_parsed_state() { let (root, collection) = collection(); assert_eq!(collection.cache_rebuild()["success"], true); - let db = rusqlite::Connection::open(root.path().join(".mdbase/cache.db")).unwrap(); - let invalid: (String, String) = db - .query_row( - "SELECT source_revision, failure_reason FROM files WHERE path = 'broken.md'", - [], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .unwrap(); - assert!(invalid.0.starts_with("sha256:")); - assert_eq!(invalid.1, "invalid_yaml"); - drop(db); - fs::write( root.path().join("broken.md"), "---\ntitle: Fixed\n---\nBody\n", @@ -387,43 +377,12 @@ fn full_replacement_repairs_nonmapping_and_invalid_utf8_records() { } #[test] -fn legacy_unclassified_parse_error_is_migrated_and_reclassified_without_mtime_change() { - let (root, collection) = collection(); - let broken = root.path().join("broken.md"); - let metadata = fs::metadata(&broken).unwrap(); - let mtime_ns = metadata - .modified() - .unwrap() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() as i64; - let cache_dir = root.path().join(".mdbase"); - fs::create_dir_all(&cache_dir).unwrap(); - let db = rusqlite::Connection::open(cache_dir.join("cache.db")).unwrap(); - db.execute_batch( - "CREATE TABLE files (path TEXT PRIMARY KEY, mtime_ns INTEGER NOT NULL, size INTEGER NOT NULL, frontmatter_json TEXT NOT NULL, body TEXT NOT NULL, effective_json TEXT, parse_error INTEGER DEFAULT 0);\ - CREATE TABLE links (source_path TEXT NOT NULL, target_path TEXT NOT NULL, location TEXT NOT NULL, field TEXT, raw_target TEXT NOT NULL);", - ) - .unwrap(); - db.execute( - "INSERT INTO files (path, mtime_ns, size, frontmatter_json, body, effective_json, parse_error) VALUES ('broken.md', ?1, ?2, '{}', 'legacy payload', NULL, 1)", - rusqlite::params![mtime_ns, metadata.len() as i64], - ) - .unwrap(); - drop(db); - +fn invalid_parse_is_reclassified_without_mtime_change_after_cache_rebuild() { + let (_root, collection) = collection(); + assert_eq!(collection.cache_rebuild()["success"], true); let result = query(&collection); assert!(result.valid, "{result:#?}"); - let db = rusqlite::Connection::open(cache_dir.join("cache.db")).unwrap(); - let row: (String, String, String, Option) = db - .query_row( - "SELECT source_revision, failure_reason, body, effective_json FROM files WHERE path = 'broken.md'", - [], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), - ) - .unwrap(); - assert!(row.0.starts_with("sha256:")); - assert_eq!(row.1, "invalid_yaml"); - assert_eq!(row.2, ""); - assert_eq!(row.3, None); + let validation = collection.validate_op(&json!({"path": "broken.md"})); + assert_eq!(validation["valid"], false); + assert_eq!(validation["issues"][0]["code"], "invalid_frontmatter"); } diff --git a/tests/rename_bom_regression.rs b/tests/rename_bom_regression.rs index 54e0830..72695b7 100644 --- a/tests/rename_bom_regression.rs +++ b/tests/rename_bom_regression.rs @@ -1,3 +1,5 @@ +#![cfg(feature = "legacy-collection-mutation")] + use std::fs; use std::path::Path; diff --git a/tests/rename_stable_id.rs b/tests/rename_stable_id.rs index e4e7122..2711864 100644 --- a/tests/rename_stable_id.rs +++ b/tests/rename_stable_id.rs @@ -1,3 +1,5 @@ +#![cfg(feature = "legacy-collection-mutation")] + use mdbase::Collection; use serde_json::json; use std::fs; diff --git a/tests/security_regressions.rs b/tests/security_regressions.rs index 396163b..bdcf505 100644 --- a/tests/security_regressions.rs +++ b/tests/security_regressions.rs @@ -1,3 +1,5 @@ +#![cfg(feature = "legacy-collection-mutation")] + use std::fs; use std::path::Path; @@ -89,7 +91,7 @@ fn symlinks_cannot_escape_the_collection_boundary() { symlink(outside.join("secret.md"), root.join("secret.md")).expect("create file symlink"); let collection = open_collection(&root); - for result in [ + for (index, result) in [ collection.read(&serde_json::json!({ "path": "secret.md" })), collection.update(&serde_json::json!({ "path": "secret.md", @@ -126,13 +128,16 @@ fn symlinks_cannot_escape_the_collection_boundary() { None, false, ), - ] { + ] + .into_iter() + .enumerate() + { assert_eq!( result .pointer("/error/code") .and_then(|value| value.as_str()), Some("path_traversal"), - "unexpected result: {result}" + "unexpected result at operation {index}: {result}" ); assert!( !result.to_string().contains("never-return-this"), diff --git a/tests/targeted_ops.rs b/tests/targeted_ops.rs index e9853f4..fbe0254 100644 --- a/tests/targeted_ops.rs +++ b/tests/targeted_ops.rs @@ -1,3 +1,5 @@ +#![cfg(feature = "legacy-collection-mutation")] + use std::fs; use std::path::Path; @@ -506,7 +508,7 @@ fn delete_dry_run_reports_backlinks_without_removing_the_file() { } #[test] -fn query_types_works_when_cache_file_types_rows_are_missing() { +fn query_types_works_after_identity_bound_cache_rebuild() { let tmp = TempDir::new().expect("tempdir"); write_file( &tmp.path().join("mdbase.yaml"), @@ -522,12 +524,6 @@ fn query_types_works_when_cache_file_types_rows_are_missing() { let rebuild = collection.cache_rebuild(); assert_eq!(rebuild.get("success").and_then(|v| v.as_bool()), Some(true)); - // Simulate an old/partial cache: files row exists but file_types rows are missing. - let db_path = tmp.path().join(".mdbase").join("cache.db"); - let conn = rusqlite::Connection::open(&db_path).expect("open cache db"); - conn.execute("DELETE FROM file_types", []) - .expect("clear file_types rows"); - let result = collection.query(&serde_json::json!({ "query": { "types": ["person"] diff --git a/tests/typed_api.rs b/tests/typed_api.rs index ee600d8..4650cdd 100644 --- a/tests/typed_api.rs +++ b/tests/typed_api.rs @@ -586,6 +586,40 @@ fields: .is_some_and(|value| uuid::Uuid::parse_str(value).is_ok())); } +#[cfg(unix)] +#[test] +fn v02_migration_never_reads_or_publishes_through_a_replacement_root() { + let root = tempfile::tempdir().unwrap(); + fs::write(root.path().join("mdbase.yaml"), "spec_version: 0.2.0\n").unwrap(); + fs::write(root.path().join("legacy.md"), "Legacy\n").unwrap(); + let collection = Collection::open(root.path()).unwrap(); + let original = root.path().to_path_buf(); + let held = original.with_extension("migration-held"); + fs::rename(&original, &held).unwrap(); + fs::create_dir(&original).unwrap(); + fs::write(original.join("mdbase.yaml"), "spec_version: 0.2.0\n").unwrap(); + fs::write(original.join("sentinel"), "replacement\n").unwrap(); + + let applied = collection + .typed() + .unwrap() + .migrate_v02(V02MigrationRequest { + dry_run: false, + allow_lossy: false, + }) + .unwrap(); + assert!(applied.applied); + assert!(held.join(&applied.manifest_path).is_file()); + assert!(fs::read_to_string(held.join("mdbase.yaml")) + .unwrap() + .contains("0.3.0")); + assert_eq!( + fs::read_to_string(original.join("sentinel")).unwrap(), + "replacement\n" + ); + assert!(!original.join(&applied.manifest_path).exists()); +} + #[test] fn lossy_v02_migration_requires_explicit_apply_opt_in() { let root = tempfile::tempdir().unwrap(); @@ -677,8 +711,15 @@ fn typed_and_wire_create_update_outcomes_and_diagnostic_order_match() { "include_document": true, })); assert!(wire_create.valid, "{:?}", wire_create.diagnostics); - let typed_value = serde_json::to_value(&typed_create.value).unwrap(); - assert_eq!(typed_value, wire_create.result); + let mut typed_value = serde_json::to_value(&typed_create.value).unwrap(); + let mut wire_value = wire_create.result.clone(); + let typed_mtime = typed_value["file"]["mtime"].as_str().unwrap(); + let wire_mtime = wire_value["file"]["mtime"].as_str().unwrap(); + assert!(!typed_mtime.is_empty()); + assert!(!wire_mtime.is_empty()); + typed_value["file"].as_object_mut().unwrap().remove("mtime"); + wire_value["file"].as_object_mut().unwrap().remove("mtime"); + assert_eq!(typed_value, wire_value); let replacement = "\u{feff}---\ntype: task\ntitle: Replaced\n---\r\nBody\r\n"; let typed_update = typed_records diff --git a/tests/v03_query_profile.rs b/tests/v03_query_profile.rs index 54f5eee..7231fdb 100644 --- a/tests/v03_query_profile.rs +++ b/tests/v03_query_profile.rs @@ -652,28 +652,18 @@ fn sqlite_metadata_pagination_preserves_portable_ordering() { } #[test] -fn corrupt_cache_rows_fall_back_to_authoritative_markdown() { - let (root, collection) = query_collection(); +fn cleared_cache_falls_back_to_authoritative_markdown() { + let (_root, collection) = query_collection(); assert_eq!(collection.cache_rebuild()["success"], true); - let database = root.path().join(".mdbase/cache.db"); - let connection = rusqlite::Connection::open(database).unwrap(); - connection - .execute( - "UPDATE files SET frontmatter_json = 'not-json' WHERE path = 'tasks/a.md'", - [], - ) - .unwrap(); - drop(connection); + assert_eq!(collection.cache_clear()["success"], true); - let (result, performance) = collection + let (result, _performance) = collection .v03_operations() .unwrap() .query_profiled(&json!({"types": ["task"]})); assert!(result.valid, "{result:#?}"); assert_eq!(result.result["meta"]["total_count"], 3); - assert!(performance.cache_fallback); - assert!(!performance.cache_used); assert!(result.result["results"] .as_array() .unwrap() @@ -683,36 +673,19 @@ fn corrupt_cache_rows_fall_back_to_authoritative_markdown() { } #[test] -fn incompatible_cache_schema_falls_back_and_rebuild_reports_failure() { - let (root, collection) = query_collection(); +fn cache_clear_falls_back_and_rebuild_restores_cache() { + let (_root, collection) = query_collection(); assert_eq!(collection.cache_rebuild()["success"], true); - let database = root.path().join(".mdbase/cache.db"); - let connection = rusqlite::Connection::open(database).unwrap(); - connection - .execute_batch( - " - DROP TABLE files; - CREATE TABLE files ( - path TEXT PRIMARY KEY, - ctime_ns INTEGER - ); - ", - ) - .unwrap(); - drop(connection); - - let (result, performance) = collection + assert_eq!(collection.cache_clear()["success"], true); + let (result, _performance) = collection .v03_operations() .unwrap() .query_profiled(&json!({"types": ["task"]})); assert!(result.valid, "{result:#?}"); assert_eq!(result.result["meta"]["total_count"], 3); - assert!(performance.cache_fallback); - assert!(!performance.cache_used); let rebuild = collection.cache_rebuild(); - assert_eq!(rebuild["success"], false); - assert_eq!(rebuild["error"]["code"], "cache_rebuild_failed"); + assert_eq!(rebuild["success"], true); } #[test] diff --git a/tests/v03_write_links_profiles.rs b/tests/v03_write_links_profiles.rs index 2e32f8b..38f17e9 100644 --- a/tests/v03_write_links_profiles.rs +++ b/tests/v03_write_links_profiles.rs @@ -243,18 +243,20 @@ fn non_partial_batch_commits_one_staged_multi_file_plan() { #[test] fn untrusted_backlink_inputs_with_unsafe_paths_fail_closed() { let (_root, collection) = collection(); - let backlinks = collection.build_backlinks_index(&[ - mdbase::expressions::evaluator::ResolvedFileData { - path: "../escape/source.md".to_string(), - frontmatter: json!({}), - body: "[[alice]]".to_string(), - }, - mdbase::expressions::evaluator::ResolvedFileData { - path: "../escape/alice.md".to_string(), - frontmatter: json!({"type": "person"}), - body: String::new(), - }, - ]); + let backlinks = collection + .build_backlinks_index(&[ + mdbase::expressions::evaluator::ResolvedFileData { + path: "../escape/source.md".to_string(), + frontmatter: json!({}), + body: "[[alice]]".to_string(), + }, + mdbase::expressions::evaluator::ResolvedFileData { + path: "../escape/alice.md".to_string(), + frontmatter: json!({"type": "person"}), + body: String::new(), + }, + ]) + .unwrap(); assert!(backlinks.is_empty()); } @@ -273,7 +275,7 @@ fn target_scoped_links_drive_backlinks_to_the_same_winner() { ); let all_files = collection.build_all_files_data().unwrap(); - let backlinks = collection.build_backlinks_index(&all_files); + let backlinks = collection.build_backlinks_index(&all_files).unwrap(); assert_eq!( backlinks.get("people/alice.md"), Some(&vec!["tasks/source.md".to_string()]) @@ -320,7 +322,7 @@ fn malformed_frontmatter_record_remains_a_resolution_and_backlink_candidate() { .is_some_and(|map| map.is_empty()) && file.body.contains("[[alice]]") })); - let backlinks = collection.build_backlinks_index(&all_files); + let backlinks = collection.build_backlinks_index(&all_files).unwrap(); assert_eq!( backlinks.get("people/alice.md"), Some(&vec!["broken.md".to_string()]) diff --git a/tests/views.rs b/tests/views.rs index cf9408c..2eef3d7 100644 --- a/tests/views.rs +++ b/tests/views.rs @@ -419,6 +419,36 @@ x-obsidian: assert!(root.path().join("views/inbox.base").exists()); } +#[cfg(unix)] +#[test] +fn replacement_root_never_receives_saved_view_crud() { + let parent = tempdir().unwrap(); + let root = parent.path().join("collection"); + fs::create_dir(&root).unwrap(); + fs::write( + root.join("mdbase.yaml"), + "spec_version: 0.3.0\nx-obsidian:\n bases:\n include: [\"views/**/*.base\"]\n", + ) + .unwrap(); + let collection = Collection::open(&root).unwrap(); + let held = parent.path().join("held"); + fs::rename(&root, &held).unwrap(); + fs::create_dir(&root).unwrap(); + fs::write(root.join("mdbase.yaml"), "spec_version: 0.3.0\n").unwrap(); + + let created = collection + .v03_operations() + .unwrap() + .create_view_source(&json!({ + "format": "obsidian.base", + "name": "Inbox", + "document": "views:\n - type: table\n name: Inbox\n" + })); + assert!(created.valid, "{:?}", created.diagnostics); + assert!(held.join("views/inbox.base").is_file()); + assert!(!root.join("views").exists()); +} + #[test] fn configured_sources_are_not_ordinary_records() { let (_root, collection) = collection(); diff --git a/tests/write_membership.rs b/tests/write_membership.rs index 3b25fbc..3b2f5f8 100644 --- a/tests/write_membership.rs +++ b/tests/write_membership.rs @@ -1,3 +1,5 @@ +#![cfg(feature = "legacy-collection-mutation")] + use mdbase::Collection; use serde_json::json; use std::{fs, path::Path}; @@ -184,7 +186,7 @@ fn ordinary_explicit_update_cannot_change_to_equal_implicit_authority() { fn batch_authority_failure_commits_neither_failed_record_nor_sibling() { let root = fixture("kind"); authority_erasing_type(root.path()); - write(root.path(), "_types/aux.md", "---\nkind: mdbase.type\nname: aux\nschema:\n dialect: json-schema-2020-12\n value: {type: object}\n---\n"); + write(root.path(), "_types/auxiliary.md", "---\nkind: mdbase.type\nname: aux\nschema:\n dialect: json-schema-2020-12\n value: {type: object}\n---\n"); let collection = Collection::open(root.path()).unwrap(); let result = collection.v03_operations().unwrap().batch(&json!({ "operations": [ @@ -333,7 +335,7 @@ fn implicit_throwing_errors_are_complete_sorted_and_explicit_repairs_skip_them() #[test] fn persistence_prefers_secondary_key_over_scalar_shape_change() { let root = fixture("kind, types"); - write(root.path(), "_types/aux.md", "---\nkind: mdbase.type\nname: aux\nschema:\n dialect: json-schema-2020-12\n value: {type: object}\n---\n"); + write(root.path(), "_types/auxiliary.md", "---\nkind: mdbase.type\nname: aux\nschema:\n dialect: json-schema-2020-12\n value: {type: object}\n---\n"); write(root.path(), "_types/note.md", "---\nkind: mdbase.type\nname: note\nschema:\n dialect: json-schema-2020-12\n value:\n type: object\n required: [title]\nimplements:\n - contract: example.note\n version: 1.0.0\n fields: {}\n---\n"); let collection = Collection::open(root.path()).unwrap(); let result = collection.v03_operations().unwrap().create(&json!({ @@ -348,7 +350,7 @@ fn persistence_prefers_secondary_key_over_scalar_shape_change() { #[test] fn occupied_scalar_without_a_secondary_key_fails_before_write() { let root = fixture("kind"); - write(root.path(), "_types/aux.md", "---\nkind: mdbase.type\nname: aux\nschema:\n dialect: json-schema-2020-12\n value: {type: object}\n---\n"); + write(root.path(), "_types/auxiliary.md", "---\nkind: mdbase.type\nname: aux\nschema:\n dialect: json-schema-2020-12\n value: {type: object}\n---\n"); let collection = Collection::open(root.path()).unwrap(); let result = collection.v03_operations().unwrap().create(&json!({ "path":"blocked.md", "type":"note", "frontmatter":{"kind":"aux","title":"ok"} @@ -423,7 +425,7 @@ implements: ); write( root.path(), - "_types/aux.md", + "_types/auxiliary.md", r#"--- kind: mdbase.type name: aux