diff --git a/Cargo.lock b/Cargo.lock index 13a38aff3..4f80b2d76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -542,6 +542,12 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -569,13 +575,17 @@ dependencies = [ "ignore", "notify", "notify-debouncer-full", + "pulldown-cmark", "rand 0.8.7", + "regex", "reqwest", + "rusqlite", "serde", "serde_json", "sha2", "socket2", "tungstenite", + "unicode-normalization", ] [[package]] @@ -770,6 +780,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -843,6 +862,33 @@ dependencies = [ "subtle", ] +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" +dependencies = [ + "hashbrown 0.17.1", +] + [[package]] name = "heck" version = "0.5.0" @@ -1445,6 +1491,25 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pulldown-cmark" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "278e9268c93b9e498a2da46c34d90ba93a3255eb99009646398f34bc73ab36eb" +dependencies = [ + "bitflags", + "getopts", + "memchr", + "pulldown-cmark-escape", + "unicase", +] + +[[package]] +name = "pulldown-cmark-escape" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" + [[package]] name = "quinn" version = "0.11.11" @@ -1618,6 +1683,18 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + [[package]] name = "regex-automata" version = "0.4.18" @@ -1700,6 +1777,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror 2.0.20", +] + [[package]] name = "rusqlite" version = "0.40.2" @@ -1709,8 +1796,10 @@ dependencies = [ "bitflags", "fallible-iterator", "fallible-streaming-iterator", + "hashlink", "libsqlite3-sys", "smallvec", + "sqlite-wasm-rs", ] [[package]] @@ -1984,6 +2073,18 @@ dependencies = [ "der", ] +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -2239,12 +2340,33 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unicase" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75cd3fb03d5288b3ba9a1ee103ac7e9cbb59f0b03d64f8ddfcc04dd42758a85d" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-width" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca718471876992059f251e33aafc334346ada08042b7661dd1fe313a9bced557" + [[package]] name = "universal-hash" version = "0.5.1" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 90eeea513..da8ffc138 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -32,6 +32,10 @@ ignore = "=0.4.25" notify = "=8.2.0" notify-debouncer-full = "=0.7.0" fs2 = "=0.4.3" +rusqlite = { version = "=0.40.2", features = ["bundled"] } +pulldown-cmark = "0.13" +unicode-normalization = "0.1" +regex = "1" tungstenite = { version = "=0.29.0", features = ["rustls-tls-webpki-roots", "rustls-tls-native-roots"] } diff --git a/cli/GUIDE.md b/cli/GUIDE.md index 4436a3158..053de1e28 100644 --- a/cli/GUIDE.md +++ b/cli/GUIDE.md @@ -938,6 +938,7 @@ fragment invites # the invites waiting for your yes fragment invites accept|decline # an agent: its owner's, acting for them fragment sync [--dir D] [--watch] [--mode M [--prune] | --live] [--apply-mass-delete] fragment verify [--dir D] fragment secret list|rm ... +fragment search [--dir D] [--limit N] fragment secret set --to HOST [--to HOST] # the value from $KEY, else stdin fragment deploy [--dir D] [--note N] fragment write --text T | --from FILE|- [--message M] @@ -1010,3 +1011,24 @@ it. A failed `fragment call` names its id (in the message, and as up); when its outcome is unknown, call it again with that `--id`, which replays it and never runs it twice. A request may take 30 s plus a second for every 32 KiB it uploads. + +### Search a local brain + +`fragment search '' --dir [--limit 1..50]` searches +current Markdown sections offline, including unsynced edits and new files. +It requires no key, server, watcher or prior sync. `--json` returns +`{q,words,results,skipped}` in the usual success envelope; each ranked result +names its fragment, path, title, heading, ancestry, snippet and state. +`state` is `synced` (matches the last local journal), `local_only` (new or +edited), or `conflicted` (recorded divergence, conflict copy or merge markers). +`Synced` is not a fresh check of the remote or published content. Deleted +files and remote conflict copies are not hits. The name must match a valid +existing sync journal; with no journal it labels the folder's source. + +Each invocation rebuilds an in-memory FTS5 index from actual files. Words are +all required prefixes, ranked by weighted BM25. Queries take 1–256 characters +and 1–16 words; results default to 10. Sync's ignores and symlink exclusions +apply, with brain machinery and instruction files excluded too. Oversized +(1 MiB), non-UTF-8 and truncated pages are reported in `skipped`. See the +brain template README for the complete output contract, parsing limits and +conflict semantics. Search never syncs or writes an index into the folder. diff --git a/cli/src/brain_search.rs b/cli/src/brain_search.rs new file mode 100644 index 000000000..01c8638f3 --- /dev/null +++ b/cli/src/brain_search.rs @@ -0,0 +1,389 @@ +//! Local-current search. This interface takes a folder and a fragment identity, +//! never a client. The FTS index lives only in memory and is rebuilt per call: +//! no watcher, stat cache, or persisted index can hide an edit or deletion. +use crate::sync; +use anyhow::{Context, Result}; +use fragment_core::blob::sha256_hex; +use fragment_proto::RepoPath; +use pulldown_cmark::{Event, Parser, Tag, TagEnd}; +use regex::Regex; +use rusqlite::{params, Connection}; +use serde::Serialize; +use std::{collections::BTreeSet, fs, io::Read, path::Path, sync::LazyLock}; +use unicode_normalization::UnicodeNormalization; + +const PAGE_BYTES: u64 = 1024 * 1024; +static WORD: LazyLock = LazyLock::new(|| Regex::new(r"[\p{L}\p{N}\p{M}]+").unwrap()); + +pub fn validate_query(q: &str) -> std::result::Result, String> { + if q.chars().count() > 256 { + return Err("search needs 1–256 characters and 1–16 words".into()); + } + let normalized: String = q.nfkc().collect::().to_lowercase(); + let words: Vec = WORD + .find_iter(&normalized) + .map(|m| m.as_str().to_owned()) + .collect(); + if words.is_empty() || words.len() > 16 { + return Err("search needs 1–256 characters and 1–16 words".into()); + } + Ok(words) +} + +#[derive(Serialize)] +pub struct Answer { + pub q: String, + pub words: Vec, + pub results: Vec, + pub skipped: Vec, +} +#[derive(Serialize)] +pub struct Hit { + pub rank: usize, + pub fragment: String, + pub path: String, + pub title: String, + pub heading: Option, + pub ancestry: Vec, + /// synced means equal to the last local sync journal, not a remote check. + pub state: String, + pub snippet: String, +} +#[derive(Serialize)] +pub struct Skipped { + pub path: String, + pub reason: String, +} + +fn indexable(path: &str) -> bool { + let lower = path.to_lowercase(); + (lower.ends_with(".md") || lower.ends_with(".markdown")) + && !path.starts_with("site/") + && !path.starts_with("applib/") + && !matches!(lower.rsplit('/').next(), Some("agents.md" | "_index.md")) +} + +// Sync splits a conflict name before the final extension. Keep its sibling +// marked until the copy is removed, even if sync has since adopted our hash. +fn conflict_original(path: &str) -> Option { + let (stem, suffix) = path.rsplit_once(".conflict-")?; + let (stamp, writer_ext) = suffix.split_once('-')?; + if stamp.parse::().is_err() { + return None; + } + let (writer, ext) = writer_ext + .split_once('.') + .map_or((writer_ext, ""), |(w, e)| (w, e)); + if writer.is_empty() { + return None; + } + Some(if ext.is_empty() { + stem.into() + } else { + format!("{stem}.{ext}") + }) +} + +pub fn search(name: &str, dir: &Path, q: &str, limit: u32) -> Result { + let words = validate_query(q).map_err(anyhow::Error::msg)?; + anyhow::ensure!((1..=50).contains(&limit), "limit must be 1–50"); + anyhow::ensure!(dir.is_dir(), "{} is not a directory", dir.display()); + let state = sync::load_state(dir, name)?; + let root = dir.to_path_buf(); + // Same ignore rules as sync, except conflict copies must be inventoried. + // They are evidence, never indexed as ordinary published content. + let walker = ignore::WalkBuilder::new(dir) + .hidden(false) + .filter_entry(move |e| { + let rel = sync::relative(&root, e.path()); + e.depth() == 0 || sync::syncable(&conflict_original(&rel).unwrap_or(rel)) + }) + .build(); + let mut paths = BTreeSet::new(); + let mut conflicts = BTreeSet::new(); + for entry in walker { + let entry = entry?; + if !entry.file_type().is_some_and(|t| t.is_file()) { + continue; + } + if entry.path().to_str().is_none() { + continue; + } + let path = sync::relative(dir, entry.path()); + if let Some(original) = conflict_original(&path) { + conflicts.insert(original); + } else if RepoPath::parse(&path).is_ok() && indexable(&path) { + paths.insert(path); + } + } + let mut db = Connection::open_in_memory()?; + let tx = db.transaction()?; + tx.execute_batch("CREATE VIRTUAL TABLE sections USING fts5(path UNINDEXED, state UNINDEXED, title, location, heading, ancestry, body, tokenize='unicode61 remove_diacritics 2');")?; + let mut skipped = Vec::new(); + for path in paths { + let mut bytes = Vec::new(); + fs::File::open(dir.join(&path)) + .with_context(|| format!("read {path}"))? + .take(PAGE_BYTES) + .read_to_end(&mut bytes)?; + if bytes.len() as u64 >= PAGE_BYTES { + skipped.push(Skipped { + path, + reason: "page is 1 MiB or larger".into(), + }); + continue; + } + let Ok(text) = std::str::from_utf8(&bytes) else { + skipped.push(Skipped { + path, + reason: "not UTF-8".into(), + }); + continue; + }; + let status = if state.files.get(path.as_str()).is_some_and(|s| s.conflicted) + || conflicts.contains(&path) + || text + .lines() + .any(|l| l.starts_with("<<<<<<< ") || l == "=======" || l.starts_with(">>>>>>> ")) + { + "conflicted" + } else if state + .files + .get(path.as_str()) + .is_some_and(|s| s.sha256 == sha256_hex(&bytes)) + { + "synced" + } else { + "local_only" + }; + let (title, sections, truncated) = sections(&path, text); + if truncated { + skipped.push(Skipped { + path: path.clone(), + reason: "sections beyond the first 500 omitted".into(), + }); + } + for section in sections { + tx.execute( + "INSERT INTO sections VALUES (?1,?2,?3,?1,?4,?5,?6)", + params![ + path, + status, + title, + section.heading, + serde_json::to_string(§ion.ancestry)?, + section.body + ], + )?; + } + } + tx.commit()?; + // Parameters contain only quoted prefix words: no FTS operators from q. + let expression = words + .iter() + .map(|w| format!("\"{}\"*", w.replace('"', "\"\""))) + .collect::>() + .join(" "); + let mut stmt = db.prepare("SELECT path,state,title,heading,ancestry,snippet(sections,6,'','','…',24) FROM sections WHERE sections MATCH ?1 ORDER BY bm25(sections,0,0,4,2,3,2,1),path,rowid LIMIT ?2")?; + let rows = stmt.query_map(params![expression, limit], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, String>(4)?, + row.get::<_, String>(5)?, + )) + })?; + let mut results = Vec::new(); + for row in rows { + let (path, state, title, heading, ancestry, snippet) = row?; + let snippet = snippet.split_whitespace().collect::>().join(" "); + results.push(Hit { + rank: results.len() + 1, + fragment: name.into(), + path, + state, + title, + heading, + ancestry: serde_json::from_str(&ancestry)?, + snippet: clip(&snippet, 300), + }); + } + Ok(Answer { + q: q.into(), + words, + results, + skipped, + }) +} + +fn clip(s: &str, max: usize) -> String { + if s.len() <= max { + return s.into(); + } + let mut end = max - '…'.len_utf8(); + while !s.is_char_boundary(end) { + end -= 1; + } + format!("{}…", &s[..end]) +} + +struct Section { + heading: Option, + ancestry: Vec, + body: String, +} + +// CommonMark supplies heading boundaries and rendered heading text, ignoring +// fenced code. Frontmatter's title overrides H1, as in the brain template. +fn sections(path: &str, text: &str) -> (String, Vec
, bool) { + let text = text.trim_start_matches('\u{feff}'); + let mut body = text; + let mut title = None; + let mut lead = String::new(); + if let Some(rest) = text + .strip_prefix("---\n") + .or_else(|| text.strip_prefix("---\r\n")) + { + let mut offset = text.len() - rest.len(); + for line in rest.split_inclusive('\n') { + offset += line.len(); + if matches!(line.trim(), "---" | "...") { + body = &text[offset..]; + break; + } + if let Some((key, value)) = line.split_once(':') { + let value = value.trim().trim_matches(['\'', '"']); + match key { + "title" if !value.is_empty() => title = Some(value.to_string()), + "summary" | "description" | "tags" => { + lead.push_str(value); + lead.push('\n'); + } + _ => {} + } + } + } + if body.len() == text.len() { + // An unclosed block is ordinary Markdown, not metadata. + title = None; + lead.clear(); + } + } + let mut heads = Vec::new(); + let mut current = None; + for (event, range) in Parser::new(body).into_offset_iter() { + match event { + Event::Start(Tag::Heading { level, .. }) => { + current = Some((level as usize, range.start, range.end, String::new())) + } + Event::Text(s) | Event::Code(s) => { + if let Some(h) = &mut current { + h.3.push_str(&s); + } + } + Event::SoftBreak | Event::HardBreak => { + if let Some(h) = &mut current { + h.3.push(' '); + } + } + Event::End(TagEnd::Heading(_)) => { + if let Some(h) = current.take() { + heads.push(h); + } + } + _ => {} + } + } + let title = title + .or_else(|| heads.iter().find(|h| h.0 == 1).map(|h| h.3.clone())) + .unwrap_or_else(|| { + Path::new(path) + .file_stem() + .unwrap() + .to_string_lossy() + .replace(['-', '_'], " ") + }); + let mut out = Vec::new(); + let mut push = |heading: Option, ancestry: Vec, text: String| { + if text.trim().is_empty() && heading.is_none() { + return; + } + let chars: Vec = text.trim().chars().collect(); + if chars.is_empty() { + out.push(Section { + heading, + ancestry, + body: String::new(), + }); + } else { + for chunk in chars.chunks(8000) { + out.push(Section { + heading: heading.clone(), + ancestry: ancestry.clone(), + body: chunk.iter().collect(), + }); + } + } + }; + push( + None, + vec![], + format!( + "{lead}\n{}", + &body[..heads.first().map_or(body.len(), |h| h.1)] + ), + ); + let mut above: Vec<(usize, String)> = Vec::new(); + for (i, h) in heads.iter().enumerate() { + while above.last().is_some_and(|a| a.0 >= h.0) { + above.pop(); + } + push( + Some(h.3.clone()), + above.iter().map(|a| a.1.clone()).collect(), + body[h.2..heads.get(i + 1).map_or(body.len(), |h| h.1)].into(), + ); + above.push((h.0, h.3.clone())); + } + let truncated = out.len() > 500; + out.truncate(500); + (title.chars().take(300).collect(), out, truncated) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn section_limits_and_unclosed_metadata() { + let (_, parts, truncated) = sections("a.md", &format!("# Title\n{}", "é".repeat(8001))); + assert!(!truncated); + assert_eq!(parts.len(), 2); + assert_eq!(parts[0].body.chars().count(), 8000); + let (_, parts, truncated) = sections("a.md", &"## Leaf\ntext\n".repeat(501)); + assert!(truncated); + assert_eq!(parts.len(), 500); + let (title, _, _) = sections("a.md", "---\ntitle: not metadata\n# Real title"); + assert_eq!(title, "Real title"); + } + #[test] + fn commonmark_sections_keep_ancestry_and_fences() { + let (title, parts, _) = sections("wiki/a.md", "---\ntitle: 'Garden'\n---\n# Root\nintro\n## **Leaf**\nfind this\n```md\n# fake\n```\n"); + assert_eq!(title, "Garden"); + assert_eq!(parts.len(), 2); + assert_eq!(parts[1].heading.as_deref(), Some("Leaf")); + assert_eq!(parts[1].ancestry, vec!["Root"]); + assert!(parts[1].body.contains("# fake")); + } + #[test] + fn query_is_words_and_clips_utf8() { + assert_eq!( + validate_query("café OR title:hello*").unwrap(), + vec!["café", "or", "title", "hello"] + ); + assert!(validate_query("***").is_err()); + assert!(validate_query(&"word ".repeat(17)).is_err()); + assert!(clip(&"é".repeat(200), 300).len() <= 300); + } +} diff --git a/cli/src/main.rs b/cli/src/main.rs index eb0e9fb7a..47d5da164 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,5 +1,6 @@ mod api; mod ask; +mod brain_search; mod auth; mod blobs; mod codestorage; @@ -56,6 +57,15 @@ struct Cli { #[derive(Subcommand)] enum Cmd { + /// Search a brain's current local Markdown sections, offline (never syncs) + Search { + name: String, + q: String, + #[arg(long, default_value = ".")] + dir: PathBuf, + #[arg(long, default_value = "10", value_parser = clap::value_parser!(u32).range(1..=50))] + limit: u32, + }, /// Make a nostr key (or use the one you have) and add it to you: it /// prints a code, which you type at the host's /cli page, signed in, in /// a browser (--force makes a new key) @@ -1087,6 +1097,19 @@ fn run(cli: Cli) -> Result<()> { let j = cli.json || json_env_flag(); match cli.cmd { + Cmd::Search { name, q, dir, limit } => { + brain_search::validate_query(&q).map_err(usage)?; + let answer = brain_search::search(&name, &dir, &q, limit)?; + json_exit(j, &answer); + for hit in &answer.results { + println!("{}. {}:{} [{}] {}", hit.rank, hit.fragment, hit.path, hit.state, hit.ancestry.iter().chain(hit.heading.iter()).cloned().collect::>().join(" > ")); + println!(" {}", hit.snippet); + } + for skipped in &answer.skipped { + eprintln!("skipped {}: {}", skipped.path, skipped.reason); + } + return Ok(()); + } Cmd::Login { force, no_wait, no_browser } => { // an agent has no key to log in with: its computer signs for it if let Some(mode) = agent_mode()? { @@ -2244,7 +2267,7 @@ fn run(cli: Cli) -> Result<()> { json_exit(j, &json!({ "visibility": visibility })); println!("{name}: {}", visibility.as_str()); } - Cmd::Login { .. } | Cmd::Host { .. } | Cmd::Guide | Cmd::Skill | Cmd::New { .. } | Cmd::Operator { .. } => unreachable!(), + Cmd::Search { .. } | Cmd::Login { .. } | Cmd::Host { .. } | Cmd::Guide | Cmd::Skill | Cmd::New { .. } | Cmd::Operator { .. } => unreachable!(), } Ok(()) } diff --git a/cli/src/sync.rs b/cli/src/sync.rs index 0410f6e86..b87764f97 100644 --- a/cli/src/sync.rs +++ b/cli/src/sync.rs @@ -124,6 +124,9 @@ pub struct SyncState { #[derive(Serialize, Deserialize, Clone)] pub struct FileState { + /// Divergence observed by sync, even when saving a remote copy failed. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub conflicted: bool, pub sha256: String, pub size: u64, pub mtime_ns: i128, @@ -352,7 +355,7 @@ pub fn syncable(rel: &str) -> bool { /// `path` relative to `root`, its segments joined by `/` ("" for the root /// itself). A segment keeps its name whole: a Unix file named `a\b` is /// one segment with a backslash (no repo path), never `a/b`. -fn relative(root: &Path, path: &Path) -> String { +pub(crate) fn relative(root: &Path, path: &Path) -> String { let Ok(rel) = path.strip_prefix(root) else { return String::new() }; let segments: Vec> = rel.components().map(|c| c.as_os_str().to_string_lossy()).collect(); segments.join("/") @@ -629,7 +632,7 @@ pub fn pass_over(client: &Client, storage: &CodeStorage, name: &str, dir: &Path, if let Some(lf) = local.get(p) { state.files.insert( p.clone(), - FileState { sha256: lf.sha256.clone(), size: lf.size, mtime_ns: lf.mtime_ns, commit: tip.clone() }, + FileState { conflicted: false, sha256: lf.sha256.clone(), size: lf.size, mtime_ns: lf.mtime_ns, commit: tip.clone() }, ); } report.pushed.push(p.to_string()); @@ -821,7 +824,7 @@ fn adopt_identical( if crate::blobs::content_sha(&bytes) == lf.sha256 { state.files.insert( p.clone(), - FileState { sha256: lf.sha256.clone(), size: lf.size, mtime_ns: lf.mtime_ns, commit: rf.last_commit_sha.clone() }, + FileState { conflicted: false, sha256: lf.sha256.clone(), size: lf.size, mtime_ns: lf.mtime_ns, commit: rf.last_commit_sha.clone() }, ); } } @@ -884,9 +887,9 @@ fn record_conflicts( } } let entry = match ctx.local.get(path) { - Some(lf) => FileState { sha256: lf.sha256.clone(), size: lf.size, mtime_ns: lf.mtime_ns, commit: rf_commit }, + Some(lf) => FileState { conflicted: true, sha256: lf.sha256.clone(), size: lf.size, mtime_ns: lf.mtime_ns, commit: rf_commit }, None => match ctx.state.files.get(path) { - Some(st) => FileState { commit: rf_commit, ..st.clone() }, + Some(st) => FileState { conflicted: true, commit: rf_commit, ..st.clone() }, None => continue, }, }; @@ -957,7 +960,7 @@ fn pull_file(storage: &CodeStorage, blobs: &crate::blobs::Blobs<'_>, dir: &Path, let mtime_ns = md.modified().map(|m| m.duration_since(std::time::UNIX_EPOCH).map(|d| d.as_nanos() as i128).unwrap_or(0)).unwrap_or(0); state.files.insert( path.clone(), - FileState { sha256: sha, size: md.len(), mtime_ns, commit: commit.to_string() }, + FileState { conflicted: false, sha256: sha, size: md.len(), mtime_ns, commit: commit.to_string() }, ); report.pulled.push(path.to_string()); Ok(()) @@ -1025,7 +1028,7 @@ mod tests { assert_eq!(s1.hashed, 1); let mut st = SyncState { schema_version: 3, name: "x".into(), host: None, repo: None, files: BTreeMap::new() }; let lf = l1.get("site/index.html").unwrap(); - st.files.insert(rp("site/index.html"), FileState { sha256: lf.sha256.clone(), size: lf.size, mtime_ns: lf.mtime_ns, commit: "c".into() }); + st.files.insert(rp("site/index.html"), FileState { conflicted: false, sha256: lf.sha256.clone(), size: lf.size, mtime_ns: lf.mtime_ns, commit: "c".into() }); let s2 = scan_local(&dir, Some(&st), false).unwrap().stats; assert_eq!(s2.hashed, 0); fs::remove_dir_all(&dir).ok(); @@ -1562,6 +1565,8 @@ mod tests { assert_eq!(report.conflicts.len(), 1); assert_eq!(report.exit_code(), 3); assert_eq!(fs::read(dir.join("shared.txt")).unwrap(), b"ours", "local keeps ours"); + assert!(load_state(&dir, "t--k3x9").unwrap().files["shared.txt"].conflicted, + "a conflict survives even when no remote copy could be saved"); // the copy name splits stem/ext like the old engine: shared.conflict--.txt let copy = fs::read_dir(&dir).unwrap().filter_map(|e| e.ok()).map(|e| e.file_name().to_string_lossy().to_string()).find(|n| n.starts_with("shared.conflict-")); assert!(copy.is_some(), "remote copy saved beside it"); diff --git a/cli/tests/brain_search.rs b/cli/tests/brain_search.rs new file mode 100644 index 000000000..703c8fae7 --- /dev/null +++ b/cli/tests/brain_search.rs @@ -0,0 +1,173 @@ +//! Run the real CLI without credentials or a reachable host. Mutations happen +//! between independent processes: no watcher or cached import can help them. +use fragment_core::blob::sha256_hex; +use serde_json::{json, Value}; +use std::{ + fs, + path::{Path, PathBuf}, + process::Command, +}; + +struct Folder(PathBuf); +impl Folder { + fn new(tag: &str) -> Self { + let p = std::env::temp_dir().join(format!("fragment-search-{tag}-{}", std::process::id())); + fs::create_dir_all(p.join("wiki")).unwrap(); + fs::create_dir_all(p.join(".fragment")).unwrap(); + Self(p) + } + fn write(&self, path: &str, body: &str) { + fs::write(self.0.join(path), body).unwrap(); + } + fn journal(&self, body: &str, conflicted: bool) { + fs::create_dir_all(self.0.join(".fragment")).unwrap(); + self.write(".fragment/state.json", &json!({"schemaVersion":3,"name":"garden--test","host":"http://127.0.0.1:1","files":{"wiki/a.md":{"sha256":sha256_hex(body.as_bytes()),"size":body.len(),"mtime_ns":0,"commit":"base","conflicted":conflicted}}}).to_string()); + } + fn find(&self, q: &str) -> Value { + search(&self.0, q) + } +} +impl Drop for Folder { + fn drop(&mut self) { + fs::remove_dir_all(&self.0).unwrap(); + } +} +fn command(dir: &Path, q: &str) -> Command { + let mut c = Command::new(env!("CARGO_BIN_EXE_fragment")); + c.args(["--json", "search", "garden--test", q, "--dir"]) + .arg(dir) + .env("HOME", dir) + .env("XDG_CONFIG_HOME", dir.join("empty-config")) + .env("FRAGMENT_HOST", "http://127.0.0.1:1") + .env_remove("FRAGMENT_AS_AGENT") + .env_remove("FRAGMENT_ACTING_FOR"); + c +} +fn search(dir: &Path, q: &str) -> Value { + let out = command(dir, q).output().unwrap(); + assert!( + out.status.success(), + "{} {}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + let v: Value = serde_json::from_slice(&out.stdout).unwrap(); + assert_eq!(v["ok"], true); + v["data"].clone() +} + +#[test] +fn current_edits_new_files_deletions_and_restarts_offline() { + let f = Folder::new("current"); + let base = "# Root\n## Leaf\noldword\n"; + f.write("wiki/a.md", base); + f.journal(base, false); + let hit = &f.find("oldword")["results"][0]; + assert_eq!(hit["state"], "synced"); + assert_eq!(hit["fragment"], "garden--test"); + assert_eq!(hit["path"], "wiki/a.md"); + assert_eq!(hit["heading"], "Leaf"); + assert_eq!(hit["ancestry"], json!(["Root"])); + // Same size; a stat cache must not hide this change. + f.write("wiki/a.md", "# Root\n## Leaf\nnewword\n"); + assert!(f.find("oldword")["results"].as_array().unwrap().is_empty()); + let hit = &f.find("newword")["results"][0]; + assert_eq!(hit["state"], "local_only"); + assert!(hit["snippet"].as_str().unwrap().contains("newword")); + f.write("wiki/new.md", "# Discovery\nlocalword"); + assert_eq!(f.find("localword")["results"][0]["state"], "local_only"); + fs::remove_file(f.0.join("wiki/a.md")).unwrap(); + assert!(f.find("newword")["results"].as_array().unwrap().is_empty()); + // Search makes no persistent index, journal changes, or synced files. + assert_eq!(fs::read_dir(f.0.join(".fragment")).unwrap().count(), 1); +} + +#[test] +fn conflicts_are_evidence_not_published_truth() { + let f = Folder::new("conflict"); + let body = "# Local\nconflictword"; + f.write("wiki/a.md", body); + f.journal(body, true); // copy could not be saved + assert_eq!(f.find("conflictword")["results"][0]["state"], "conflicted"); + f.journal(body, false); // old journal, no conflict flag + f.write( + "wiki/a.conflict-1700000000-deadbeef.md", + "# Remote\nremoteword", + ); + assert_eq!(f.find("conflictword")["results"][0]["state"], "conflicted"); + assert!(f.find("remoteword")["results"] + .as_array() + .unwrap() + .is_empty()); + fs::remove_file(f.0.join("wiki/a.conflict-1700000000-deadbeef.md")).unwrap(); + f.write( + "wiki/a.md", + "<<<<<<< ours\nconflictword\n=======\ntheirs\n>>>>>>> remote", + ); + assert_eq!(f.find("conflictword")["results"][0]["state"], "conflicted"); +} + +#[test] +fn ranked_prefix_words_limits_skips_and_invalid_input() { + let f = Folder::new("ranking"); + f.write("wiki/a.md", "# Orchards\napple"); + f.write("wiki/b.md", "# Other\nOrchards apple"); + let answer = f.find("orch app"); + assert_eq!(answer["results"][0]["path"], "wiki/a.md"); + assert_eq!(answer["results"].as_array().unwrap().len(), 2); + assert!(f.find("apple OR absent")["results"] + .as_array() + .unwrap() + .is_empty()); + f.write("wiki/AGENTS.md", "instructionword"); + f.write("wiki/_index.md", "instructionword"); + f.write("wiki/.secret.md", "instructionword"); + assert!(f.find("instructionword")["results"] + .as_array() + .unwrap() + .is_empty()); + f.write("wiki/large.md", &"x".repeat(1024 * 1024)); + fs::write(f.0.join("wiki/binary.md"), [255]).unwrap(); + assert_eq!(f.find("apple")["skipped"].as_array().unwrap().len(), 2); + for q in ["***".to_string(), "x ".repeat(17), "x".repeat(257)] { + let out = command(&f.0, &q).output().unwrap(); + assert_eq!(out.status.code(), Some(2)); + let v: Value = serde_json::from_slice(&out.stdout).unwrap(); + assert_eq!(v["error"]["code"], "invalid_usage"); + } + let out = command(&f.0, "apple") + .args(["--limit", "1"]) + .output() + .unwrap(); + let v: Value = serde_json::from_slice(&out.stdout).unwrap(); + assert_eq!(v["data"]["results"].as_array().unwrap().len(), 1); + f.write(".fragment/state.json", "broken"); // disposable journal: all local + assert_eq!(f.find("apple")["results"][0]["state"], "local_only"); + f.journal("# Orchards\napple", false); + let out = command(&f.0, "apple") + .args(["--limit", "0"]) + .output() + .unwrap(); + assert_eq!(out.status.code(), Some(2)); + let out = Command::new(env!("CARGO_BIN_EXE_fragment")) + .args(["search", "other--test", "apple", "--dir"]) + .arg(&f.0) + .output() + .unwrap(); + assert!(!out.status.success()); + assert!(String::from_utf8_lossy(&out.stderr).contains("garden--test")); +} + +#[cfg(unix)] +#[test] +fn symlinks_are_not_content() { + let f = Folder::new("symlinks"); + let outside = Folder::new("outside"); + outside.write("wiki/a.md", "secretword"); + std::os::unix::fs::symlink(outside.0.join("wiki"), f.0.join("linked")).unwrap(); + std::os::unix::fs::symlink(outside.0.join("wiki/a.md"), f.0.join("wiki/link.md")).unwrap(); + assert!(f.find("secretword")["results"] + .as_array() + .unwrap() + .is_empty()); +} diff --git a/templates/brain/README.md b/templates/brain/README.md index 1e73f8576..cb4cec9b9 100644 --- a/templates/brain/README.md +++ b/templates/brain/README.md @@ -86,3 +86,53 @@ only, so a note's HTML shows as text). `api/tree` lists the brain's files and two it makes: `_index.md` (its wikis, the landing) and `AGENTS.md` (the guide), each unless the brain holds one of its own. An asset's bytes come from the platform's `__file`. + +## Search the current local folder + + fragment search garden--k3x9 'orchard pruning' --dir ./garden + fragment search garden--k3x9 'orchard pruning' --dir ./garden --limit 5 --json + +This CLI search reads the actual working files without syncing or contacting +any server, and needs no login or watcher. Each invocation rebuilds a disposable +SQLite FTS5 index **in memory**, outside the synced content. Unsynced edits and +new Markdown files are included; deleted files disappear immediately. There is +no disk index to invalidate or repair. A missing, old or corrupt sync journal +makes files `local_only`; a valid journal naming another fragment is refused. +The supplied name identifies the source; without a journal it is a label, not +proof that the folder has ever synced to that fragment. + +`--json` uses the CLI's usual `{ok:true,data:…}` envelope. Data is +`{q, words, results, skipped}`; each result has `{rank, fragment, path, title, +heading, ancestry, state, snippet}`. Rank starts at 1, ancestry is an array of +parent headings, heading is null for a page's introductory text, and snippet +is a plain excerpt of at most 300 UTF-8 bytes. States are: + +- `synced`: bytes match the **last local sync journal**. This does not verify + the server's current version or a deployed release. +- `local_only`: a new local file or bytes changed since the last sync. +- `conflicted`: sync recorded divergence, a sibling `.conflict-…` copy remains, + or the working file contains merge conflict markers. Search returns the + working file's text with this warning; remote conflict copies are evidence, + not separate search hits. Sync clears its journal flag when it next adopts, + pushes or pulls agreed content; retained copies keep warning until removed. + Older journals cannot report a conflict whose copy was never saved. + +Query words use the server search's NFKC/lowercase, all-required prefix +semantics, with no FTS operators: 1–256 characters, 1–16 words, and `--limit` +1–50 (default 10). Ranking uses BM25 with title 4, heading 3, path and ancestry +2, body 1, with stable path/section tie breaks. Local files obey sync's ignore +rules and never follow symlinks. Brain machinery, `AGENTS.md` and `_index.md` +are excluded, as on the server. Pages at least 1 MiB and non-UTF-8 pages are +reported in `skipped`; only the first 500 sections per page are retained +(also reported). Other read failures fail the command rather than silently +returning an incomplete answer. + +The local parser uses CommonMark (including Setext headings), ignoring headings +inside code fences. It recognizes scalar frontmatter title, summary, +description and inline tags; unlike the server parser it does not interpret +multiline YAML tags. Long sections split at 8000 Unicode characters, including +inside paragraphs. Local and server rank values are not interchangeable. +Rebuilding means each search reads every eligible page: this favors freshness +and a simple rebuildable interface over incremental performance. A search is +not an atomic filesystem snapshot; files edited during a scan can reflect +different moments. It never changes working files or the sync journal.