From 8f1506a2604a23535837eebc574bb93d4b9dad43 Mon Sep 17 00:00:00 2001 From: Minoru Osuka Date: Thu, 10 Sep 2026 21:33:35 -0400 Subject: [PATCH 01/10] fix(lexical): fall back to the stored document when a DocValues read misses InvertedIndexReader::has_doc_values is an index-wide any(...) across segments, not a per-document guarantee. TopFieldCollector::get_field_value and FacetCollector::collect_doc both treated a get_doc_value miss under has_dv == true as "the value is Null" / "no contribution" instead of falling back to the stored document. Unreachable via the "no segment has the column at all" case (already fixed in #1053), but real whenever segments disagree on whether they have the column -- which mixed old/new segments, or a doc_values: false field, will do routinely. Fixing this is a prerequisite for the doc_values opt-out flag (#1047). Also adds the missing document_fields override on PerSegmentReaderView, which was falling through to the trait default (document() + full clone). Refs #1047 --- .../index/inverted/per_segment_view.rs | 14 ++ laurus/src/lexical/index/inverted/reader.rs | 18 ++ laurus/src/lexical/query/collector.rs | 144 ++++++++++++-- laurus/src/lexical/search/features/facet.rs | 115 ++++++++--- .../lexical_doc_values_mixed_segments_test.rs | 179 ++++++++++++++++++ 5 files changed, 420 insertions(+), 50 deletions(-) create mode 100644 laurus/tests/lexical_doc_values_mixed_segments_test.rs diff --git a/laurus/src/lexical/index/inverted/per_segment_view.rs b/laurus/src/lexical/index/inverted/per_segment_view.rs index 57fa374a..1ade251b 100644 --- a/laurus/src/lexical/index/inverted/per_segment_view.rs +++ b/laurus/src/lexical/index/inverted/per_segment_view.rs @@ -161,6 +161,20 @@ impl LexicalIndexReader for PerSegmentReaderView { seg.document(doc_id) } + fn document_fields( + &self, + doc_id: u64, + field_names: &[&str], + ) -> Result>> { + // Without this override, the trait default forwards to `document` + // above and clones every field just to filter it back down -- + // defeating the whole point of `document_fields` (Issue #1047: a + // DocValues-miss fallback on this per-segment view would otherwise + // clone the full document per hit). + let seg = self.segment.read().unwrap(); + seg.document_fields(doc_id, field_names) + } + fn doc_ids(&self) -> Result> { // Segment-local ids (global id values, but only those present in // this segment) so stored-document scans stay segment-bounded diff --git a/laurus/src/lexical/index/inverted/reader.rs b/laurus/src/lexical/index/inverted/reader.rs index da96759a..532a787e 100644 --- a/laurus/src/lexical/index/inverted/reader.rs +++ b/laurus/src/lexical/index/inverted/reader.rs @@ -2126,6 +2126,18 @@ impl crate::lexical::reader::LexicalIndexReader for InvertedIndexReader { self } + /// Searches every segment for `doc_id`'s value, returning the first + /// hit. `Ok(None)` means either no segment has a DocValues column for + /// `field`, or every segment that does simply lacks a value for this + /// particular doc — the two cases are indistinguishable from this + /// return value alone. Callers that also consult + /// [`Self::has_doc_values`] to decide whether to read DocValues at + /// all must still treat `Ok(None)` from this method as "fall back to + /// the stored document", not as "the value is absent" (Issue #1047): + /// segments can disagree on whether they have the column (mixed + /// old/new segments, or a field with `doc_values: false`), so + /// `has_doc_values() == true` index-wide does not guarantee this + /// specific document's segment has it. fn get_doc_value(&self, field: &str, doc_id: u64) -> Result> { // Search across all segments for segment_lock in &self.segment_readers { @@ -2137,6 +2149,12 @@ impl crate::lexical::reader::LexicalIndexReader for InvertedIndexReader { Ok(None) } + /// Returns whether ANY segment has a DocValues column for `field` — + /// an index-wide, not per-document, answer. `true` does not mean + /// every document has a value via [`Self::get_doc_value`]: segments + /// can disagree (Issue #1047), so a caller must still fall back to + /// the stored document on an `Ok(None)` miss from `get_doc_value` + /// rather than treating this method's `true` as a per-doc guarantee. fn has_doc_values(&self, field: &str) -> bool { // Check if any segment has DocValues for this field self.segment_readers.iter().any(|seg_lock| { diff --git a/laurus/src/lexical/query/collector.rs b/laurus/src/lexical/query/collector.rs index 86d35c43..422cc757 100644 --- a/laurus/src/lexical/query/collector.rs +++ b/laurus/src/lexical/query/collector.rs @@ -117,13 +117,20 @@ pub struct TopFieldCollector<'a> { total_hits: u64, /// Reference to the index reader for accessing field values. reader: &'a dyn crate::lexical::reader::LexicalIndexReader, - /// Whether `field_name` has a DocValues column, resolved once at - /// construction (Issue #1053). `field_name` is fixed for the whole - /// collector's lifetime, so caching this here -- rather than - /// re-probing per document -- avoids paying a lock-guarded + /// Whether ANY segment has a DocValues column for `field_name`, + /// resolved once at construction (Issue #1053). `field_name` is fixed + /// for the whole collector's lifetime, so caching this here -- rather + /// than re-probing per document -- avoids paying a lock-guarded /// `has_doc_values` check on every hit for no benefit; mirrors the /// same per-field caching `FacetCollector::collect_doc` does for the /// same reason (#597). + /// + /// This is index-wide, not per-document: `InvertedIndexReader:: + /// has_doc_values` is `any(...)` across segments (Issue #1047), so + /// `true` does not guarantee `get_doc_value` will find a value for + /// every doc -- a segment lacking the column entirely (mixed old/new + /// segments, or a field with `doc_values: false`) still needs the + /// stored-document fallback per miss. See [`Self::get_field_value`]. has_dv: bool, } @@ -171,26 +178,28 @@ impl<'a> TopFieldCollector<'a> { /// Get the field value for a document, preferring DocValues. /// - /// When `field_name` has no DocValues column, falls back to the - /// stored document (Issue #1053) instead of yielding `Null` -- - /// otherwise every value compares equal and sorting silently - /// degrades to doc-id order. Mirrors `FacetCollector::collect_doc`'s - /// stored-document fallback for the same absent-column case. + /// When `field_name` has no DocValues column at all, or a `Some` + /// column exists index-wide but this particular document's segment + /// doesn't have it (Issue #1047: mixed segments, e.g. straddling a + /// `doc_values: false` change or a #1052-era segment boundary), falls + /// back to the stored document instead of yielding `Null` -- + /// otherwise every such value compares equal and sorting silently + /// degrades to doc-id order for exactly the documents that lack the + /// column. Mirrors `FacetCollector::collect_doc`'s stored-document + /// fallback for the same absent-column case. Uses `document_fields` + /// rather than `document` to avoid cloning every field of a + /// wide-schema document just to read one. fn get_field_value(&self, doc_id: u64) -> crate::lexical::core::field::FieldValue { use crate::lexical::core::field::FieldValue; - if self.has_dv { - return match self.reader.get_doc_value(&self.field_name, doc_id) { - Ok(Some(value)) => value, - _ => FieldValue::Null, - }; + if self.has_dv + && let Ok(Some(value)) = self.reader.get_doc_value(&self.field_name, doc_id) + { + return value; } - match self.reader.document(doc_id) { - Ok(Some(document)) => document - .get(&self.field_name) - .cloned() - .unwrap_or(FieldValue::Null), + match self.reader.document_fields(doc_id, &[&self.field_name]) { + Ok(Some(mut fields)) => fields.remove(&self.field_name).unwrap_or(FieldValue::Null), _ => FieldValue::Null, } } @@ -1362,4 +1371,101 @@ mod tests { "Vector values always tie on rank, so order stays doc-id order" ); } + + /// Reader where `has_doc_values` reports `true` (simulating "some + /// segment has this column") but `get_doc_value` always misses + /// (simulating THIS document's segment lacking the column) — the + /// mixed-segment case Issue #1047 identified as unreachable via + /// `DocFallbackMockReader` (which reports `has_doc_values = false` + /// index-wide, the "no column anywhere" case #1053 already covered). + #[derive(Debug)] + struct DocValuesMissMockReader { + docs: Vec, + } + + impl DocValuesMissMockReader { + fn new(docs: Vec) -> Self { + Self { docs } + } + } + + impl crate::lexical::reader::LexicalIndexReader for DocValuesMissMockReader { + fn doc_count(&self) -> u64 { + self.docs.len() as u64 + } + fn max_doc(&self) -> u64 { + self.docs.len() as u64 + } + fn is_deleted(&self, _doc_id: u64) -> bool { + false + } + fn document(&self, doc_id: u64) -> Result> { + Ok(self.docs.get(doc_id as usize).cloned()) + } + fn term_info( + &self, + _field: &str, + _term: &str, + ) -> Result> { + Ok(None) + } + fn postings( + &self, + _field: &str, + _term: &str, + ) -> Result>> { + Ok(None) + } + fn field_stats(&self, _field: &str) -> Result> { + Ok(None) + } + fn close(&mut self) -> Result<()> { + Ok(()) + } + fn is_closed(&self) -> bool { + false + } + fn as_any(&self) -> &dyn std::any::Any { + self + } + fn has_doc_values(&self, _field: &str) -> bool { + true + } + // `get_doc_value` deliberately left at the trait's default + // (`Ok(None)`): every lookup misses, as if this doc's segment + // never wrote the column despite another segment having it. + } + + /// Issue #1047 regression: when `has_doc_values` is `true` index-wide + /// but this document's segment misses (`get_doc_value` returns + /// `Ok(None)`), `get_field_value` must fall back to the stored + /// document instead of collapsing to `Null`. Before the fix every + /// value resolved to `Null` (doc-id order `[0, 1, 2]`); after, content + /// order (`[1] < [2] < [3]`, i.e. `[2, 0, 1]`). + #[test] + fn test_top_field_collector_falls_back_when_has_dv_is_true_but_the_value_is_missing() { + let docs = vec![ + crate::Document::builder() + .add_bytes("blob", vec![2]) + .build(), // doc_id 0 + crate::Document::builder() + .add_bytes("blob", vec![3]) + .build(), // doc_id 1 + crate::Document::builder() + .add_bytes("blob", vec![1]) + .build(), // doc_id 2 + ]; + let reader = DocValuesMissMockReader::new(docs); + let mut collector = TopFieldCollector::new(3, "blob".to_string(), true, &reader); + for doc_id in 0..3 { + collector.collect(doc_id, 0.0).unwrap(); + } + let ids: Vec = collector.results().iter().map(|h| h.doc_id).collect(); + assert_eq!( + ids, + vec![2, 0, 1], + "a DocValues miss with has_dv=true must still fall back to the \ + stored document, ordering by content ([1] < [2] < [3]) not doc id" + ); + } } diff --git a/laurus/src/lexical/search/features/facet.rs b/laurus/src/lexical/search/features/facet.rs index c34ce9f8..d58ec9fa 100644 --- a/laurus/src/lexical/search/features/facet.rs +++ b/laurus/src/lexical/search/features/facet.rs @@ -282,7 +282,12 @@ impl FacetCollector { pub fn collect_doc(&mut self, doc_id: u64, reader: &dyn LexicalIndexReader) -> Result<()> { // Resolve per-field DocValues availability once (Issue #597). It is // doc-independent, so caching it here keeps `collect_doc` free of a - // lock-guarded `has_doc_values` probe per hit. + // lock-guarded `has_doc_values` probe per hit. NOTE: `true` here + // means "some segment has this column" (`InvertedIndexReader:: + // has_doc_values` is `any(...)` across segments) -- it does not + // guarantee `get_doc_value` finds a value for THIS doc, since + // segments can disagree on whether a field has a column (Issue + // #1047: mixed old/new segments, or a `doc_values: false` field). if self.field_has_dv.len() != self.facet_fields.len() { self.field_has_dv = self .facet_fields @@ -291,17 +296,17 @@ impl FacetCollector { .collect(); } - // Only decode the stored-fields blob (`reader.document`) when at - // least one facet field lacks a DocValues column. When every facet - // field has DocValues we read the per-field values directly and skip - // the whole-document decode + `Document::clone()` entirely (#597). - // The document, when needed, is still fetched once per call (#409). - let needs_document = self.field_has_dv.iter().any(|&has| !has); - let doc_result = if needs_document { - Some(reader.document(doc_id)) - } else { - None - }; + // Fetched lazily and cached for the rest of this call (#409: at + // most once per `collect_doc`) the first time a field actually + // needs it -- either because it has no DocValues column at all, + // or because this document's segment misses despite the field + // having DocValues elsewhere (#1047). Fields that always hit + // DocValues never pay for this. Uses `document_fields` (only the + // facet fields), not `document`, to avoid cloning every field of + // a wide-schema document. + let mut doc_fields: Option< + Result>>, + > = None; // Reusable scratch buffers — allocated once per call, cleared at // each field iteration. Avoids per-field `Vec` reallocations that @@ -321,35 +326,38 @@ impl FacetCollector { path_components.clear(); { let field_name: &str = &self.facet_fields[field_idx]; - if has_dv { - // DocValues fast path (#597). `FieldValue` is - // `DataValue`, so the value maps to facet path - // components exactly as the stored document would; a - // read error or absent value yields no contribution. - if let Ok(Some(value)) = reader.get_doc_value(field_name, doc_id) { - push_path_components(&value, &mut path_components); - } + // DocValues fast path (#597). `FieldValue` is `DataValue`, + // so the value maps to facet path components exactly as + // the stored document would. + let dv_hit = has_dv + .then(|| reader.get_doc_value(field_name, doc_id).ok().flatten()) + .flatten(); + + if let Some(value) = dv_hit { + push_path_components(&value, &mut path_components); } else { - match &doc_result { - Some(Ok(Some(document))) => { - if let Some(val) = document.get(field_name) { + // No DocValues column, or a miss despite `has_dv` + // (#1047) -- fall back to the stored document. + let result = doc_fields.get_or_insert_with(|| { + let field_refs: Vec<&str> = + self.facet_fields.iter().map(String::as_str).collect(); + reader.document_fields(doc_id, &field_refs) + }); + match result { + Ok(Some(fields)) => { + if let Some(val) = fields.get(field_name) { push_path_components(val, &mut path_components); } } - Some(Ok(None)) => { + Ok(None) => { // Document not found — no facet contribution. } - Some(Err(_)) => { + Err(_) => { // Synthetic fallback preserved from the pre-#409 // implementation: 5 distinct values stratified // by `doc_id`. path_components.push(format!("value_{}", doc_id % 5)); } - None => { - // Unreachable: a field without DocValues forces - // `needs_document = true`, so `doc_result` is - // `Some`. Guard defensively rather than panic. - } } } } @@ -1069,6 +1077,11 @@ mod tests { docs: Vec, dv_fields: HashSet, panic_on_document: bool, + /// Doc ids for which `get_doc_value` reports `Ok(None)` even though + /// the field is listed in `dv_fields` -- simulating a segment that + /// has the DocValues column but lacks this particular document's + /// value (Issue #1047 mixed-segment case). + dv_miss_doc_ids: HashSet, } impl DvMockReader { @@ -1077,6 +1090,18 @@ mod tests { docs, dv_fields: dv_fields.iter().map(|s| (*s).to_string()).collect(), panic_on_document, + dv_miss_doc_ids: HashSet::new(), + } + } + + /// Like `new`, but `get_doc_value` misses for every doc id in + /// `miss_doc_ids` while `has_doc_values` still reports `true`. + fn with_dv_miss(docs: Vec, dv_fields: &[&str], miss_doc_ids: &[u64]) -> Self { + Self { + docs, + dv_fields: dv_fields.iter().map(|s| (*s).to_string()).collect(), + panic_on_document: false, + dv_miss_doc_ids: miss_doc_ids.iter().copied().collect(), } } } @@ -1123,7 +1148,7 @@ mod tests { self.dv_fields.contains(field) } fn get_doc_value(&self, field: &str, doc_id: u64) -> Result> { - if !self.dv_fields.contains(field) { + if !self.dv_fields.contains(field) || self.dv_miss_doc_ids.contains(&doc_id) { return Ok(None); } Ok(self @@ -1241,6 +1266,34 @@ mod tests { assert_eq!(flatten(&results, "cat"), vec![(vec!["a".to_string()], 2)]); } + #[test] + fn facet_falls_back_when_has_dv_is_true_but_this_docs_value_is_missing() { + // Issue #1047 regression: `has_doc_values("cat")` reports `true` + // (another segment has the column), but THIS doc's DocValues lookup + // misses (`Ok(None)`). Before the fix, `collect_doc` treated any + // non-`Ok(Some(_))` DV read under `has_dv == true` as "no + // contribution" and never fell back to the stored document, so + // doc 1's `cat` facet would be silently dropped. + let docs = vec![ + text_doc(&[("cat", "a")]), // doc_id 0: DV hit + text_doc(&[("cat", "b")]), // doc_id 1: DV miss -> must fall back + ]; + let reader = DvMockReader::with_dv_miss(docs, &["cat"], &[1]); + let mut collector = FacetCollector::new(FacetConfig::default(), vec!["cat".to_string()]); + collector + .collect_doc(0, &reader) + .expect("collect_doc must not error"); + collector + .collect_doc(1, &reader) + .expect("collect_doc must not error"); + let results = collector.finalize().expect("finalize must not error"); + assert_eq!( + flatten(&results, "cat"), + vec![(vec!["a".to_string()], 1), (vec!["b".to_string()], 1)], + "a DocValues miss with has_dv=true must still fall back to the stored document" + ); + } + #[test] fn test_facet_path_creation() { let path = FacetPath::new( diff --git a/laurus/tests/lexical_doc_values_mixed_segments_test.rs b/laurus/tests/lexical_doc_values_mixed_segments_test.rs new file mode 100644 index 00000000..b2ad798f --- /dev/null +++ b/laurus/tests/lexical_doc_values_mixed_segments_test.rs @@ -0,0 +1,179 @@ +//! Index-backed end-to-end tests for Issue #1047's mixed-segment DocValues +//! bug — the sort and facet paths both cached `has_doc_values` per field +//! across the whole (possibly multi-segment) index, then treated any +//! `get_doc_value` miss under a `true` cache as "the value is `Null`" / +//! "no contribution" instead of falling back to the stored document. That +//! collapses correctly whenever *no* segment has the column (already +//! covered by #1053's stored-document fallback), but a segment that lacks +//! the column while a sibling segment has it was unreachable by any +//! existing test. +//! +//! These tests reproduce that exact situation on a real, on-disk, +//! multi-segment index (not a mock reader): two segments are committed +//! with `use_compound: false` so each segment's `.dv` file exists +//! standalone, then one segment's `.dv` file is deleted outright via +//! `Storage::delete_file`. `has_doc_values` still reports `true` +//! index-wide (the surviving segment has the column), but every document +//! in the deleted segment now misses on `get_doc_value` — exactly the +//! `has_dv == true, get_doc_value == Ok(None)` case `TopFieldCollector` +//! and `FacetCollector` must fall back on. + +use std::sync::Arc; + +use laurus::Document; +use laurus::lexical::index::LexicalIndex; +use laurus::lexical::index::config::InvertedIndexConfig; +use laurus::lexical::index::inverted::InvertedIndex; +use laurus::lexical::query::Query; +use laurus::lexical::search::features::facet::{FacetCollector, FacetConfig}; +use laurus::lexical::writer::LexicalIndexWriter; +use laurus::lexical::{LexicalIndexConfig, LexicalSearchRequest, LexicalStore, TermQuery}; +use laurus::storage::Storage; +use laurus::storage::memory::{MemoryStorage, MemoryStorageConfig}; + +/// `.dv` files present in `storage`, sorted by name. Segment names are +/// `{prefix}_{:06}`, assigned in increasing order as segments are +/// flushed, so a lexicographic sort is also commit order. +fn dv_files_sorted(storage: &Arc) -> Vec { + let mut files: Vec = storage + .list_files() + .unwrap() + .into_iter() + .filter(|f| f.ends_with(".dv")) + .collect(); + files.sort(); + files +} + +fn doc_with_score(score: i64) -> Document { + Document::builder() + .add_text("body", "alpha") + .add_integer("score", score) + .build() +} + +fn doc_with_brand(brand: &str) -> Document { + Document::builder().add_text("brand", brand).build() +} + +/// Loose (non-compound) config so each segment's DocValues live in a +/// standalone `{segment}.dv` file that this test can delete directly. A +/// high `max_segments` keeps auto-merge from folding the two segments +/// back into one before the deletion takes effect. +fn loose_config() -> InvertedIndexConfig { + InvertedIndexConfig { + use_compound: false, + max_segments: 1000, + ..Default::default() + } +} + +#[test] +fn field_sort_falls_back_when_a_segments_dv_file_is_missing() { + let storage: Arc = Arc::new(MemoryStorage::new(MemoryStorageConfig::default())); + let config = LexicalIndexConfig::Inverted(loose_config()); + let store = LexicalStore::new(storage.clone(), config).unwrap(); + + // Segment 0: doc 1 -> 30, doc 2 -> 10, doc 3 -> 20. + for (doc_id, score) in [(1u64, 30i64), (2, 10), (3, 20)] { + store + .upsert_document(doc_id, doc_with_score(score)) + .unwrap(); + } + store.commit().unwrap(); + + // Segment 1: doc 4 -> 5, doc 5 -> 60, doc 6 -> 40. + for (doc_id, score) in [(4u64, 5i64), (5, 60), (6, 40)] { + store + .upsert_document(doc_id, doc_with_score(score)) + .unwrap(); + } + store.commit().unwrap(); + + let dv_files = dv_files_sorted(&storage); + assert_eq!( + dv_files.len(), + 2, + "expected one standalone .dv file per segment, found {dv_files:?}" + ); + // Delete segment 0's DocValues column entirely -- its docs (1, 2, 3) + // now have no "score" column even though segment 1 still does. + storage.delete_file(&dv_files[0]).unwrap(); + + let query: Box = Box::new(TermQuery::new("body", "alpha")); + let results = store + .search( + LexicalSearchRequest::new(query) + .limit(6) + .sort_by_field_asc("score"), + ) + .unwrap(); + + // True ascending order by score: 4(5), 2(10), 3(20), 1(30), 6(40), 5(60). + // Before the #1047 fix, docs 1-3 would collapse to `Null` (sorts last + // under both directions) instead of falling back to their stored + // "score" value, producing a different order. + assert_eq!( + results.hits.iter().map(|h| h.doc_id).collect::>(), + vec![4, 2, 3, 1, 6, 5], + "docs in the segment with the deleted .dv file must still sort by \ + their true stored score, not collapse to Null" + ); + assert_eq!(results.total_hits, 6); +} + +#[test] +fn facet_falls_back_when_a_segments_dv_file_is_missing() { + let storage: Arc = Arc::new(MemoryStorage::new(MemoryStorageConfig::default())); + let index = InvertedIndex::create(storage.clone(), loose_config()).unwrap(); + let mut writer: Box = index.writer().unwrap(); + + // Segment 0: apple, apple, dell. + let mut doc_ids = Vec::new(); + for brand in ["apple", "apple", "dell"] { + doc_ids.push(writer.add_document(doc_with_brand(brand)).unwrap()); + } + writer.commit().unwrap(); + + // Segment 1: apple, dell, dell. + for brand in ["apple", "dell", "dell"] { + doc_ids.push(writer.add_document(doc_with_brand(brand)).unwrap()); + } + writer.commit().unwrap(); + + let dv_files = dv_files_sorted(&storage); + assert_eq!( + dv_files.len(), + 2, + "expected one standalone .dv file per segment, found {dv_files:?}" + ); + // Delete segment 0's DocValues column entirely. + storage.delete_file(&dv_files[0]).unwrap(); + + let reader = writer.build_reader().unwrap(); + assert!( + reader.has_doc_values("brand"), + "segment 1 still has the brand column, so this stays true index-wide" + ); + + let mut collector = FacetCollector::new(FacetConfig::default(), vec!["brand".to_string()]); + for doc_id in &doc_ids { + collector.collect_doc(*doc_id, reader.as_ref()).unwrap(); + } + let results = collector.finalize().unwrap(); + + let counts: std::collections::HashMap = results + .get_field_facets("brand") + .expect("brand facets must be present") + .iter() + .map(|c| (c.path.path[0].clone(), c.count)) + .collect(); + + // True counts across both segments: apple = 3 (2 from segment 0 + 1 + // from segment 1), dell = 3 (1 from segment 0 + 2 from segment 1). + // Before the #1047 fix, segment 0's docs would silently contribute + // nothing (has_dv == true but get_doc_value misses, and the old code + // never fell back), undercounting both to apple = 1, dell = 2. + assert_eq!(counts.get("apple").copied(), Some(3)); + assert_eq!(counts.get("dell").copied(), Some(3)); +} From 01fcddf564e541e68e631db5c0b874eede18ae0f Mon Sep 17 00:00:00 2001 From: Minoru Osuka Date: Thu, 10 Sep 2026 23:26:44 -0400 Subject: [PATCH 02/10] refactor(index): move alloc_bounds to crate::util so lexical loaders can use it alloc_bounds's checked_capacity/checked_len bound header-declared sizes against the true file length before allocating -- useful for any segment loader parsing unverified on-disk headers, not just vector segments. Moves it from vector::index (pub(crate)) to util (pub(crate)) and generalizes the "vector segment is corrupted" wording to "segment is corrupted" so the upcoming DocValuesReader bounds check (#1047) can reuse it without a vector-specific error message. No behavior change; existing tests only assert on the "corrupted" substring. Refs #1047 --- laurus/src/storage.rs | 2 +- laurus/src/util.rs | 1 + .../{vector/index => util}/alloc_bounds.rs | 29 ++++++++++--------- laurus/src/vector/index.rs | 1 - laurus/src/vector/index/flat/reader.rs | 2 +- laurus/src/vector/index/flat/writer.rs | 2 +- laurus/src/vector/index/format.rs | 6 ++-- laurus/src/vector/index/hnsw/reader.rs | 6 ++-- laurus/src/vector/index/hnsw/writer.rs | 2 +- laurus/src/vector/index/ivf/reader.rs | 2 +- laurus/src/vector/index/ivf/writer.rs | 2 +- laurus/src/vector/index/quantized_segment.rs | 6 +--- 12 files changed, 30 insertions(+), 31 deletions(-) rename laurus/src/{vector/index => util}/alloc_bounds.rs (83%) diff --git a/laurus/src/storage.rs b/laurus/src/storage.rs index 8439914b..14b146a2 100644 --- a/laurus/src/storage.rs +++ b/laurus/src/storage.rs @@ -594,7 +594,7 @@ pub trait StorageInput: Read + Seek + Send + Sync + std::fmt::Debug { /// rejected as corruption *before* the allocation, so a flipped byte /// cannot drive a multi-GiB `with_capacity` that aborts the process via /// `handle_alloc_error` (Issues #791 and #806; see - /// [`crate::vector::index::alloc_bounds`]). A backend that returns a + /// [`crate::util::alloc_bounds`]). A backend that returns a /// truncated or inflated size would weaken that guard, so new backends /// must preserve the invariant. fn size(&self) -> Result; diff --git a/laurus/src/util.rs b/laurus/src/util.rs index 02b4ba3f..6d0fdc3d 100644 --- a/laurus/src/util.rs +++ b/laurus/src/util.rs @@ -1,5 +1,6 @@ //! Shared utility modules used across Laurus components. +pub(crate) mod alloc_bounds; pub mod ecef; pub mod id; pub mod levenshtein; diff --git a/laurus/src/vector/index/alloc_bounds.rs b/laurus/src/util/alloc_bounds.rs similarity index 83% rename from laurus/src/vector/index/alloc_bounds.rs rename to laurus/src/util/alloc_bounds.rs index 25259667..8b2879b1 100644 --- a/laurus/src/vector/index/alloc_bounds.rs +++ b/laurus/src/util/alloc_bounds.rs @@ -1,15 +1,18 @@ -//! Allocation bounds for on-disk vector segment parsing (Issue #806). +//! Allocation bounds for on-disk segment parsing (Issue #806, moved here +//! from `vector::index` in Issue #1047 so lexical loaders can share it). //! -//! Generalizes the Issue #791 technique. A vector segment reader/loader -//! takes element counts and byte lengths straight from an on-disk header -//! that has **not** yet been integrity-checked. The `.hnsw` CRC footer -//! (Issue #786) is verified before the reader's structural parse, but -//! *legacy footer-less segments* — and every writer load path, which does -//! not run the footer verification at all — reach these counts unverified. -//! A single flipped byte can turn a small `num_vectors` / `node_count` / -//! `field_name_len` into a multi-GiB allocation request that aborts the -//! process through `handle_alloc_error` (OOM) instead of surfacing a clean -//! "corrupted segment" error. +//! Generalizes the Issue #791 technique. A segment reader/loader takes +//! element counts and byte lengths straight from an on-disk header that +//! has **not** yet been integrity-checked. For vector segments, the +//! `.hnsw` CRC footer (Issue #786) is verified before the reader's +//! structural parse, but *legacy footer-less segments* — and every writer +//! load path, which does not run the footer verification at all — reach +//! these counts unverified. Lexical `.dv` (DocValues) segments have no +//! footer verification at all. A single flipped byte can turn a small +//! `num_vectors` / `node_count` / `field_name_len` / `num_fields` into a +//! multi-GiB allocation request that aborts the process through +//! `handle_alloc_error` (OOM) instead of surfacing a clean "corrupted +//! segment" error. //! //! These helpers reject impossible sizes up front by comparing the //! header-declared size against ground truth: the true byte length of the @@ -63,7 +66,7 @@ pub(crate) fn checked_capacity( if count as u64 > max_elements { return Err(LaurusError::index(format!( "{what}: header declares {count} elements but at most {max_elements} can fit in the \ - {available} bytes left in the file — vector segment is corrupted" + {available} bytes left in the file — segment is corrupted" ))); } Ok(count) @@ -96,7 +99,7 @@ pub(crate) fn checked_len(len: usize, available: u64, what: &str) -> Result available { return Err(LaurusError::index(format!( "{what}: header declares {len} bytes but only {available} bytes are left in the file \ - — vector segment is corrupted" + — segment is corrupted" ))); } Ok(len) diff --git a/laurus/src/vector/index.rs b/laurus/src/vector/index.rs index 003b9a43..1e2b9346 100644 --- a/laurus/src/vector/index.rs +++ b/laurus/src/vector/index.rs @@ -6,7 +6,6 @@ //! - Vector quantization and compression //! - Index optimization and maintenance -pub(crate) mod alloc_bounds; pub mod config; pub mod factory; pub mod field; diff --git a/laurus/src/vector/index/flat/reader.rs b/laurus/src/vector/index/flat/reader.rs index 6bfd0cde..4b90ba60 100644 --- a/laurus/src/vector/index/flat/reader.rs +++ b/laurus/src/vector/index/flat/reader.rs @@ -87,7 +87,7 @@ impl FlatVectorIndexReader { path: &str, distance_metric: DistanceMetric, ) -> Result { - use crate::vector::index::alloc_bounds::checked_capacity; + use crate::util::alloc_bounds::checked_capacity; use std::io::{Read, Seek}; // Open the index file diff --git a/laurus/src/vector/index/flat/writer.rs b/laurus/src/vector/index/flat/writer.rs index 78aee619..3287c26a 100644 --- a/laurus/src/vector/index/flat/writer.rs +++ b/laurus/src/vector/index/flat/writer.rs @@ -7,10 +7,10 @@ use rayon::prelude::*; use crate::error::{LaurusError, Result}; use crate::storage::Storage; +use crate::util::alloc_bounds::checked_capacity; use crate::vector::core::quantization::ScalarQuantParams; use crate::vector::core::vector::Vector; use crate::vector::index::FlatIndexConfig; -use crate::vector::index::alloc_bounds::checked_capacity; use crate::vector::index::field::LegacyVectorFieldWriter; use crate::vector::index::format::{ QuantHeader, VERSION_FIELD_DICT, VectorSegmentHeader, build_field_dict, record_prefix_size, diff --git a/laurus/src/vector/index/format.rs b/laurus/src/vector/index/format.rs index 9e7c4920..6795da65 100644 --- a/laurus/src/vector/index/format.rs +++ b/laurus/src/vector/index/format.rs @@ -510,7 +510,7 @@ impl VectorSegmentHeader { // bytes the file actually has left (the fixed prefix + // PQ params block consumed so far is PQ_PREFIX_SIZE) // before reserving anything. - crate::vector::index::alloc_bounds::checked_capacity( + crate::util::alloc_bounds::checked_capacity( codebook_len, 4, available.saturating_sub(PQ_PREFIX_SIZE), @@ -545,7 +545,7 @@ impl VectorSegmentHeader { } let codebook_len = params.codebook_len(); // Issue #921: same allocation bound as the PQ branch above. - crate::vector::index::alloc_bounds::checked_capacity( + crate::util::alloc_bounds::checked_capacity( codebook_len, 4, available.saturating_sub(PQ_PREFIX_SIZE), @@ -650,7 +650,7 @@ impl VectorSegmentHeader { )) }) } else { - use crate::vector::index::alloc_bounds::checked_len; + use crate::util::alloc_bounds::checked_len; let mut len_bytes = [0u8; 4]; reader.read_exact(&mut len_bytes)?; diff --git a/laurus/src/vector/index/hnsw/reader.rs b/laurus/src/vector/index/hnsw/reader.rs index 50e9c817..9e1c1213 100644 --- a/laurus/src/vector/index/hnsw/reader.rs +++ b/laurus/src/vector/index/hnsw/reader.rs @@ -254,7 +254,7 @@ impl HnswIndexReader { file_size: u64, doc_ids: Arc<[u64]>, ) -> Result>> { - use crate::vector::index::alloc_bounds::checked_capacity; + use crate::util::alloc_bounds::checked_capacity; use ahash::AHashMap; let mut has_graph_buf = [0u8; 1]; @@ -388,7 +388,7 @@ impl HnswIndexReader { file_size: u64, doc_ids: Arc<[u64]>, ) -> Result>> { - use crate::vector::index::alloc_bounds::checked_capacity; + use crate::util::alloc_bounds::checked_capacity; let mut has_graph_buf = [0u8; 1]; if input.read_exact(&mut has_graph_buf).is_err() || has_graph_buf[0] != 1 { @@ -480,7 +480,7 @@ impl HnswIndexReader { path: &str, distance_metric: DistanceMetric, ) -> Result { - use crate::vector::index::alloc_bounds::{checked_capacity, checked_len}; + use crate::util::alloc_bounds::{checked_capacity, checked_len}; use std::io::{Read, Seek}; // Open the index file diff --git a/laurus/src/vector/index/hnsw/writer.rs b/laurus/src/vector/index/hnsw/writer.rs index 47c894b0..13bb2d2f 100644 --- a/laurus/src/vector/index/hnsw/writer.rs +++ b/laurus/src/vector/index/hnsw/writer.rs @@ -4,10 +4,10 @@ use std::sync::Arc; use crate::error::{LaurusError, Result}; use crate::storage::Storage; +use crate::util::alloc_bounds::checked_capacity; use crate::vector::core::rerank::RerankStorageKind; use crate::vector::core::vector::Vector; use crate::vector::index::HnswIndexConfig; -use crate::vector::index::alloc_bounds::checked_capacity; use crate::vector::index::field::LegacyVectorFieldWriter; use crate::vector::index::format::{ QuantHeader, VERSION_FIELD_DICT, VERSION_ORDINAL_GRAPH, VectorSegmentHeader, build_field_dict, diff --git a/laurus/src/vector/index/ivf/reader.rs b/laurus/src/vector/index/ivf/reader.rs index d6e119d1..21b04708 100644 --- a/laurus/src/vector/index/ivf/reader.rs +++ b/laurus/src/vector/index/ivf/reader.rs @@ -96,7 +96,7 @@ impl IvfIndexReader { path: &str, distance_metric: DistanceMetric, ) -> Result { - use crate::vector::index::alloc_bounds::checked_capacity; + use crate::util::alloc_bounds::checked_capacity; use std::io::{Read, Seek}; // Open the index file diff --git a/laurus/src/vector/index/ivf/writer.rs b/laurus/src/vector/index/ivf/writer.rs index fef9c915..63198aa7 100644 --- a/laurus/src/vector/index/ivf/writer.rs +++ b/laurus/src/vector/index/ivf/writer.rs @@ -7,10 +7,10 @@ use rayon::prelude::*; use crate::error::{LaurusError, Result}; use crate::storage::Storage; +use crate::util::alloc_bounds::checked_capacity; use crate::vector::core::quantization::ScalarQuantParams; use crate::vector::core::vector::Vector; use crate::vector::index::IvfIndexConfig; -use crate::vector::index::alloc_bounds::checked_capacity; use crate::vector::index::field::LegacyVectorFieldWriter; use crate::vector::index::format::{ QuantHeader, VERSION_FIELD_DICT, VectorSegmentHeader, build_field_dict, record_prefix_size, diff --git a/laurus/src/vector/index/quantized_segment.rs b/laurus/src/vector/index/quantized_segment.rs index c2f899a4..720479b7 100644 --- a/laurus/src/vector/index/quantized_segment.rs +++ b/laurus/src/vector/index/quantized_segment.rs @@ -203,11 +203,7 @@ impl QuantizedSegmentVectors { // (the header + prefix already consumed can only make the true // remainder smaller, so `available` is a safe upper bound). let total = vector_count.saturating_mul(Self::record_size(dim)); - crate::vector::index::alloc_bounds::checked_len( - total, - available, - "quantized segment data size", - )?; + crate::util::alloc_bounds::checked_len(total, available, "quantized segment data size")?; let mut data = vec![0u8; total]; reader.read_exact(&mut data)?; From 1c11600ade8c14b4304db207c51e1dec54b432f9 Mon Sep 17 00:00:00 2001 From: Minoru Osuka Date: Fri, 11 Sep 2026 17:02:04 -0400 Subject: [PATCH 03/10] perf(lexical): load DocValues columns lazily per field, with bounds checking DocValuesReader::load used to deserialize every field's doc_id -> value payload into memory on first access, even when a query only touched one field. It now reads just the header and each field's directory entry (name, on-disk offset, payload length), skipping past payloads via seek; a field's map is deserialized and cached on its first get_value()/materialize() call. has_field/field_names are pure directory lookups that never touch a payload. Mirrors BKDReader's storage + path pattern (re-open per access) rather than holding a handle across the segment's lifetime. Also, as part of the same pass: - Bound every header-declared size (num_fields, name length, payload length) against the file's true byte length via crate::util::alloc_bounds, so a flipped header byte surfaces a clean "corrupted" error instead of an OOM abort or an out-of-bounds read. - DocValuesWriter.fields: HashMap -> BTreeMap, and each field's values_vec is now sorted by doc_id before serializing, so identical content always produces byte-identical .dv files regardless of insertion order. - Removed the unused DocValuesReader::get_field (zero callers). - get_value's signature changed from Option<&FieldValue> to Result> since materialization is now fallible I/O; updated its one caller (InvertedIndexReader::get_doc_value). Rewrote lexical_field_sort_test.rs::doc_values_load_once_per_segment, which pinned a literal "2 opens" tied to the old eager whole-file load; it now asserts the real invariant -- a repeated identical search adds zero further .dv opens -- since the exact open count changed shape (directory open + one payload open per touched field, per segment). No behavior change: same values, same .dv format, only when/how the payload is read changes. Refs #1047 --- laurus/src/lexical/index/inverted/reader.rs | 2 +- .../lexical/index/structures/doc_values.rs | 520 ++++++++++++++++-- laurus/tests/lexical_field_sort_test.rs | 33 +- 3 files changed, 492 insertions(+), 63 deletions(-) diff --git a/laurus/src/lexical/index/inverted/reader.rs b/laurus/src/lexical/index/inverted/reader.rs index 532a787e..31d24350 100644 --- a/laurus/src/lexical/index/inverted/reader.rs +++ b/laurus/src/lexical/index/inverted/reader.rs @@ -925,7 +925,7 @@ impl SegmentReader { let doc_values = self.doc_values.read().unwrap(); if let Some(reader) = doc_values.as_ref() { - Ok(reader.get_value(field, doc_id).cloned()) + reader.get_value(field, doc_id) } else { Ok(None) } diff --git a/laurus/src/lexical/index/structures/doc_values.rs b/laurus/src/lexical/index/structures/doc_values.rs index a9d546b4..14c30bd0 100644 --- a/laurus/src/lexical/index/structures/doc_values.rs +++ b/laurus/src/lexical/index/structures/doc_values.rs @@ -7,19 +7,42 @@ //! //! Unlike stored fields (row-oriented), DocValues store values in a columnar format //! where accessing all values of a single field is very efficient. +//! +//! ## On-disk layout +//! +//! ```text +//! "DVFF"(4B) | version(2B) | num_fields(u32 LE) | { +//! name_len(u32 LE) | name | num_values(u64 LE) | data_len(u64 LE) | rkyv payload +//! } x num_fields +//! ``` +//! +//! [`DocValuesReader::load`] (Issue #1047 Phase 2) reads only the header and +//! each field's directory entry (name, offset, length); the rkyv payload +//! itself is skipped over via `seek` and deserialized lazily, on first +//! access, by [`DocValuesReader::materialize`]. A query that sorts or +//! facets on one field no longer pays to deserialize every other stored +//! field's DocValues column. -use std::collections::HashMap; -use std::io::{Read, Write}; -use std::sync::Arc; +use std::collections::{BTreeMap, HashMap}; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::sync::{Arc, RwLock}; use crate::error::{LaurusError, Result}; use crate::lexical::core::field::FieldValue; use crate::storage::Storage; +use crate::util::alloc_bounds::{checked_capacity, checked_len}; /// DocValues file extension const DOC_VALUES_EXTENSION: &str = ".dv"; -/// DocValues format for a single field. +/// Minimum bytes one field's directory record can occupy on disk: a 4-byte +/// name length + a (possibly empty) name + an 8-byte value count + an +/// 8-byte payload length. Used to bound a header-declared `num_fields` +/// against the file's true size before it drives a `BTreeMap` build-out +/// (Issue #1047). +const MIN_FIELD_RECORD_SIZE: u64 = 4 + 8 + 8; + +/// DocValues format for a single field, materialized in memory. /// Stores a mapping from document ID to field value. #[derive(Debug, Clone)] pub struct FieldDocValues { @@ -65,8 +88,13 @@ pub struct DocValuesWriter { storage: Arc, /// Segment name segment_name: String, - /// Field DocValues being built (field_name -> FieldDocValues) - fields: HashMap, + /// Field DocValues being built (field_name -> FieldDocValues). + /// + /// `BTreeMap`, not `HashMap`: [`Self::write_to_output`] iterates this + /// in field-name order, so two writers fed the same documents in a + /// different field-insertion order still produce byte-identical `.dv` + /// files for identical content (Issue #1047). + fields: BTreeMap, } impl DocValuesWriter { @@ -75,7 +103,7 @@ impl DocValuesWriter { DocValuesWriter { storage, segment_name, - fields: HashMap::new(), + fields: BTreeMap::new(), } } @@ -132,7 +160,8 @@ impl DocValuesWriter { let num_fields = self.fields.len() as u32; output.write_all(&num_fields.to_le_bytes())?; - // Write each field's DocValues + // Write each field's DocValues, in field-name order (`fields` is a + // `BTreeMap`). for (field_name, field_dv) in &self.fields { // Write field name length and name let name_bytes = field_name.as_bytes(); @@ -143,14 +172,17 @@ impl DocValuesWriter { let num_values = field_dv.values.len() as u64; output.write_all(&num_values.to_le_bytes())?; - // Write values using rkyv - // Convert HashMap to Vec for easier serialization if preferred, - // but rkyv handles AHashMap if configured. Let's use Vec of pairs for stability. - let values_vec: Vec<(u64, FieldValue)> = field_dv + // `values` is an `AHashMap`, whose iteration order is not + // stable across insertion patterns -- sort by doc_id so + // identical content always serializes to identical bytes + // (Issue #1047), not just identical field order. + let mut values_vec: Vec<(u64, FieldValue)> = field_dv .values .iter() .map(|(k, v)| (*k, v.clone())) .collect(); + values_vec.sort_by_key(|(doc_id, _)| *doc_id); + let serialized = rkyv::to_bytes::(&values_vec) .map_err(|e| LaurusError::Index(format!("Failed to serialize DocValues: {}", e)))?; @@ -162,29 +194,73 @@ impl DocValuesWriter { } } -/// Reader for DocValues +/// One field's on-disk location within a `.dv` file's directory, resolved +/// once at [`DocValuesReader::load`] time. The payload itself is read +/// lazily, on first access, via [`DocValuesReader::materialize`]. +#[derive(Debug, Clone, Copy)] +struct FieldRecord { + /// Byte offset of the field's rkyv payload from the start of the file. + offset: u64, + /// Byte length of the field's rkyv payload. + len: u64, +} + +/// Reader for DocValues. +/// +/// [`Self::load`] reads only the file's directory (each field's name and +/// on-disk offset/length); no field's `doc_id -> value` payload is +/// deserialized until [`Self::get_value`] or [`Self::materialize`] first +/// asks for it, and the result is cached for later calls (Issue #1047 +/// Phase 2). [`Self::has_field`] and [`Self::field_names`] never touch the +/// payload at all -- they answer straight from the directory. +/// +/// Holds `storage` + `file_name` rather than an open file handle or mmap, +/// mirroring +/// [`BKDReader`](crate::lexical::index::structures::bkd_tree::BKDReader): +/// each access re-opens the file, so nothing here holds a handle across +/// the segment's lifetime that would block a concurrent merge's +/// `delete_segment_files` (notably on Windows). #[derive(Debug)] pub struct DocValuesReader { - /// Field DocValues (field_name -> FieldDocValues) - fields: HashMap, + storage: Arc, + file_name: String, + /// field_name -> on-disk location, in file order. + directory: BTreeMap, + /// Fields materialized so far. + cache: RwLock>>, } impl DocValuesReader { - /// Load DocValues from storage + /// Load a `.dv` file's directory from storage. + /// + /// Only the header and each field's directory entry are read here; + /// see [`Self::materialize`] for the lazy payload read. A missing + /// `.dv` file is not an error -- it loads as an empty reader (no + /// segment has ever needed DocValues), so a miss on this field stays + /// O(1). pub fn load(storage: Arc, segment_name: &str) -> Result { - let dv_filename = format!("{}{}", segment_name, DOC_VALUES_EXTENSION); + let file_name = format!("{}{}", segment_name, DOC_VALUES_EXTENSION); // Try to open the DocValues file - let mut input = match storage.open_input(&dv_filename) { + let mut input = match storage.open_input(&file_name) { Ok(input) => input, Err(_) => { - // If DocValues file doesn't exist, return empty reader + // If DocValues file doesn't exist, return an empty reader. return Ok(DocValuesReader { - fields: HashMap::new(), + storage, + file_name, + directory: BTreeMap::new(), + cache: RwLock::new(HashMap::new()), }); } }; + // Ground truth for the bounds checks below: a header-declared + // count or length that this file cannot physically back is + // corruption, not a value to allocate for (Issue #1047, same + // technique as Issue #806's vector-segment bounds checks). + let file_size = input.size()?; + // Read and verify magic number let mut magic = [0u8; 4]; input.read_exact(&mut magic)?; @@ -209,63 +285,136 @@ impl DocValuesReader { input.read_exact(&mut num_fields_bytes)?; let num_fields = u32::from_le_bytes(num_fields_bytes); - let mut fields = HashMap::new(); + let available = file_size.saturating_sub(input.stream_position()?); + let num_fields = checked_capacity( + num_fields as usize, + MIN_FIELD_RECORD_SIZE, + available, + "num_fields", + )?; + + let mut directory = BTreeMap::new(); - // Read each field's DocValues + // Read each field's directory entry for _ in 0..num_fields { // Read field name let mut name_len_bytes = [0u8; 4]; input.read_exact(&mut name_len_bytes)?; let name_len = u32::from_le_bytes(name_len_bytes) as usize; + let available = file_size.saturating_sub(input.stream_position()?); + let name_len = checked_len(name_len, available, "field name length")?; + let mut name_bytes = vec![0u8; name_len]; input.read_exact(&mut name_bytes)?; let field_name = String::from_utf8(name_bytes) .map_err(|e| LaurusError::Index(format!("Invalid field name: {}", e)))?; - // Read number of values + // Number of values is informational only (kept for on-disk + // format stability) -- the payload length below is what + // actually bounds the read. let mut num_values_bytes = [0u8; 8]; input.read_exact(&mut num_values_bytes)?; - let _num_values = u64::from_le_bytes(num_values_bytes); - // Read serialized values + // Read serialized values' length let mut data_len_bytes = [0u8; 8]; input.read_exact(&mut data_len_bytes)?; - let data_len = u64::from_le_bytes(data_len_bytes) as usize; - - let mut data = vec![0u8; data_len]; - input.read_exact(&mut data)?; - - let values_vec: Vec<(u64, FieldValue)> = - rkyv::from_bytes::, rkyv::rancor::Error>(&data).map_err( - |e| LaurusError::Index(format!("Failed to deserialize DocValues: {}", e)), - )?; + let data_len = u64::from_le_bytes(data_len_bytes); + + let available = file_size.saturating_sub(input.stream_position()?); + let data_len = checked_len(data_len as usize, available, "field data length")? as u64; + + let offset = input.stream_position()?; + directory.insert( + field_name, + FieldRecord { + offset, + len: data_len, + }, + ); - let values = values_vec.into_iter().collect(); - fields.insert(field_name.clone(), FieldDocValues { field_name, values }); + // Skip the payload -- deserialized lazily on first access. + let skip = i64::try_from(data_len).map_err(|_| { + LaurusError::Index( + "field data length overflows a seek offset — segment is corrupted".to_string(), + ) + })?; + input.seek(SeekFrom::Current(skip))?; } - Ok(DocValuesReader { fields }) + Ok(DocValuesReader { + storage, + file_name, + directory, + cache: RwLock::new(HashMap::new()), + }) } - /// Get DocValues for a field - pub fn get_field(&self, field_name: &str) -> Option<&FieldDocValues> { - self.fields.get(field_name) + /// Materialize `field_name`'s `doc_id -> value` map, reading and + /// deserializing its payload on first access and caching the result + /// for later calls. `Ok(None)` means this segment's `.dv` file has no + /// column for `field_name` at all; a `Err` means the column exists + /// but its payload could not be read or deserialized. + fn materialize(&self, field_name: &str) -> Result>> { + let record = match self.directory.get(field_name) { + Some(record) => *record, + None => return Ok(None), + }; + + if let Some(fdv) = self.cache.read().unwrap().get(field_name) { + return Ok(Some(fdv.clone())); + } + + // Re-open per access rather than holding a handle for the + // reader's lifetime -- mirrors `BKDReader` (see the struct doc + // comment above). + let mut input = self.storage.open_input(&self.file_name)?; + input.seek(SeekFrom::Start(record.offset))?; + let mut data = vec![0u8; record.len as usize]; + input.read_exact(&mut data)?; + + let values_vec: Vec<(u64, FieldValue)> = + rkyv::from_bytes::, rkyv::rancor::Error>(&data).map_err( + |e| LaurusError::Index(format!("Failed to deserialize DocValues: {}", e)), + )?; + let values = values_vec.into_iter().collect(); + let fdv = Arc::new(FieldDocValues { + field_name: field_name.to_string(), + values, + }); + + self.cache + .write() + .unwrap() + .insert(field_name.to_string(), fdv.clone()); + Ok(Some(fdv)) } - /// Get a value for a document and field - pub fn get_value(&self, field_name: &str, doc_id: u64) -> Option<&FieldValue> { - self.fields.get(field_name).and_then(|dv| dv.get(doc_id)) + /// Get a value for a document and field, materializing the field's + /// column on first access. + /// + /// `Ok(None)` covers two distinct cases callers must not conflate: + /// this segment has no DocValues column for `field_name` at all, or + /// the column exists but has no value for this particular `doc_id`. + /// Use [`Self::has_field`] first if the distinction matters (Issue + /// #1047 — an index-wide "some segment has this column" does not + /// guarantee this segment does). + pub fn get_value(&self, field_name: &str, doc_id: u64) -> Result> { + Ok(self + .materialize(field_name)? + .and_then(|fdv| fdv.get(doc_id).cloned())) } - /// Check if a field has DocValues + /// Check if a field has a DocValues column in this segment. A pure + /// directory lookup -- never materializes the payload. pub fn has_field(&self, field_name: &str) -> bool { - self.fields.contains_key(field_name) + self.directory.contains_key(field_name) } - /// Get all field names with DocValues + /// Get all field names with a DocValues column in this segment. A + /// pure directory lookup -- never materializes any payload. pub fn field_names(&self) -> Vec { - self.fields.keys().cloned().collect() + self.directory.keys().cloned().collect() } } @@ -318,21 +467,282 @@ mod tests { assert!(!reader.has_field("unknown")); assert_eq!( - reader.get_value("year", 0), - Some(&crate::data::DataValue::Int64(2023)) + reader.get_value("year", 0).unwrap(), + Some(crate::data::DataValue::Int64(2023)) ); assert_eq!( - reader.get_value("year", 1), - Some(&crate::data::DataValue::Int64(2024)) + reader.get_value("year", 1).unwrap(), + Some(crate::data::DataValue::Int64(2024)) ); assert_eq!( - reader.get_value("rating", 0), - Some(&crate::data::DataValue::Float64(4.5)) + reader.get_value("rating", 0).unwrap(), + Some(crate::data::DataValue::Float64(4.5)) ); assert_eq!( - reader.get_value("rating", 1), - Some(&crate::data::DataValue::Float64(5.0)) + reader.get_value("rating", 1).unwrap(), + Some(crate::data::DataValue::Float64(5.0)) ); } } + + #[test] + fn field_names_and_has_field_are_directory_only_and_need_no_materialization() { + // A corrupted payload for a field that is never queried through + // `get_value` must not matter to `has_field`/`field_names` -- both + // stop at the directory. + let storage = Arc::new(MemoryStorage::new(MemoryStorageConfig::default())); + let segment_name = "segment_dir_only".to_string(); + { + let mut writer = DocValuesWriter::new(storage.clone(), segment_name.clone()); + writer.add_value(0, "a", crate::data::DataValue::Int64(1)); + writer.add_value(0, "z", crate::data::DataValue::Int64(2)); + writer.write().unwrap(); + } + + let reader = DocValuesReader::load(storage.clone(), &segment_name).unwrap(); + assert_eq!(reader.field_names(), vec!["a".to_string(), "z".to_string()]); + assert!(reader.has_field("a")); + assert!(reader.has_field("z")); + assert!(!reader.has_field("missing")); + } + + #[test] + fn get_value_on_a_missing_field_is_ok_none_not_an_error() { + let storage = Arc::new(MemoryStorage::new(MemoryStorageConfig::default())); + let segment_name = "segment_missing_field".to_string(); + { + let mut writer = DocValuesWriter::new(storage.clone(), segment_name.clone()); + writer.add_value(0, "present", crate::data::DataValue::Int64(1)); + writer.write().unwrap(); + } + + let reader = DocValuesReader::load(storage.clone(), &segment_name).unwrap(); + assert_eq!(reader.get_value("absent", 0).unwrap(), None); + } + + #[test] + fn written_bytes_are_deterministic_regardless_of_insertion_order() { + // Same (doc_id, field, value) triples fed in two different orders + // must produce byte-identical `.dv` files (Issue #1047): the + // `BTreeMap` field ordering and the doc_id-sorted `values_vec` + // together remove every source of nondeterminism `AHashMap` + // iteration would otherwise introduce. + let storage = Arc::new(MemoryStorage::new(MemoryStorageConfig::default())); + + let mut forward = DocValuesWriter::new(storage.clone(), "fwd".to_string()); + forward.add_value(0, "alpha", crate::data::DataValue::Int64(1)); + forward.add_value(1, "alpha", crate::data::DataValue::Int64(2)); + forward.add_value(0, "beta", crate::data::DataValue::Int64(3)); + forward.add_value(1, "beta", crate::data::DataValue::Int64(4)); + + let mut reverse = DocValuesWriter::new(storage.clone(), "rev".to_string()); + reverse.add_value(1, "beta", crate::data::DataValue::Int64(4)); + reverse.add_value(0, "beta", crate::data::DataValue::Int64(3)); + reverse.add_value(1, "alpha", crate::data::DataValue::Int64(2)); + reverse.add_value(0, "alpha", crate::data::DataValue::Int64(1)); + + let mut forward_bytes = Vec::new(); + forward.write_to_output(&mut forward_bytes).unwrap(); + let mut reverse_bytes = Vec::new(); + reverse.write_to_output(&mut reverse_bytes).unwrap(); + + assert_eq!(forward_bytes, reverse_bytes); + } + + #[test] + fn load_rejects_a_num_fields_the_file_cannot_back() { + // A flipped `num_fields` byte must surface a clean "corrupted" + // error before any per-field allocation, not read past EOF or + // abort the process (Issue #1047, same technique as #806). + let storage = Arc::new(MemoryStorage::new(MemoryStorageConfig::default())); + let file_name = "corrupt.dv"; + { + let mut output = storage.create_output(file_name).unwrap(); + output.write_all(b"DVFF").unwrap(); + output.write_all(&[1u8, 0u8]).unwrap(); + // Declare an impossible number of fields for a file this + // short. + output.write_all(&u32::MAX.to_le_bytes()).unwrap(); + output.flush().unwrap(); + } + + let err = DocValuesReader::load(storage, "corrupt").unwrap_err(); + match err { + LaurusError::Index(msg) => assert!( + msg.contains("corrupted"), + "expected a corruption message, got: {msg}" + ), + other => panic!("expected Index error, got {other:?}"), + } + } + + #[test] + fn load_rejects_a_field_data_len_the_file_cannot_back() { + // A valid, small `num_fields` (1) paired with a field whose + // declared payload length exceeds what remains in the file must + // also be rejected as corruption. + let storage = Arc::new(MemoryStorage::new(MemoryStorageConfig::default())); + let file_name = "corrupt_len.dv"; + { + let mut output = storage.create_output(file_name).unwrap(); + output.write_all(b"DVFF").unwrap(); + output.write_all(&[1u8, 0u8]).unwrap(); + output.write_all(&1u32.to_le_bytes()).unwrap(); // num_fields = 1 + let name = b"f"; + output + .write_all(&(name.len() as u32).to_le_bytes()) + .unwrap(); + output.write_all(name).unwrap(); + output.write_all(&1u64.to_le_bytes()).unwrap(); // num_values (unused) + // Declare a payload far larger than any bytes that follow. + output.write_all(&(1u64 << 40).to_le_bytes()).unwrap(); + output.flush().unwrap(); + } + + let err = DocValuesReader::load(storage, "corrupt_len").unwrap_err(); + match err { + LaurusError::Index(msg) => assert!( + msg.contains("corrupted"), + "expected a corruption message, got: {msg}" + ), + other => panic!("expected Index error, got {other:?}"), + } + } + + /// Issue #1047 Phase 2: querying one field must not read another + /// field's payload bytes off disk. Wraps storage in a byte-counting + /// shim and proves that materializing a small field reads far fewer + /// bytes than a much larger sibling field's payload occupies. + #[test] + fn materializing_one_field_does_not_read_another_fields_payload() { + use std::sync::atomic::{AtomicU64, Ordering}; + + #[derive(Debug)] + struct CountingInput { + inner: Box, + bytes_read: Arc, + } + impl Read for CountingInput { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let n = self.inner.read(buf)?; + self.bytes_read.fetch_add(n as u64, Ordering::Relaxed); + Ok(n) + } + } + impl Seek for CountingInput { + fn seek(&mut self, pos: SeekFrom) -> std::io::Result { + self.inner.seek(pos) + } + } + impl crate::storage::StorageInput for CountingInput { + fn size(&self) -> Result { + self.inner.size() + } + fn clone_input(&self) -> Result> { + self.inner.clone_input() + } + fn close(&mut self) -> Result<()> { + self.inner.close() + } + } + + #[derive(Debug)] + struct CountingStorage { + inner: Arc, + bytes_read: Arc, + } + impl Storage for CountingStorage { + fn open_input(&self, name: &str) -> Result> { + Ok(Box::new(CountingInput { + inner: self.inner.open_input(name)?, + bytes_read: self.bytes_read.clone(), + })) + } + fn create_output(&self, name: &str) -> Result> { + self.inner.create_output(name) + } + fn create_output_append( + &self, + name: &str, + ) -> Result> { + self.inner.create_output_append(name) + } + fn delete_file(&self, name: &str) -> Result<()> { + self.inner.delete_file(name) + } + fn file_exists(&self, name: &str) -> bool { + self.inner.file_exists(name) + } + fn list_files(&self) -> Result> { + self.inner.list_files() + } + fn file_size(&self, name: &str) -> Result { + self.inner.file_size(name) + } + fn rename_file(&self, from: &str, to: &str) -> Result<()> { + self.inner.rename_file(from, to) + } + fn metadata(&self, name: &str) -> Result { + self.inner.metadata(name) + } + fn create_temp_output( + &self, + prefix: &str, + ) -> Result<(String, Box)> { + self.inner.create_temp_output(prefix) + } + fn sync(&self) -> Result<()> { + self.inner.sync() + } + fn close(&mut self) -> Result<()> { + Ok(()) + } + } + + let bytes_read = Arc::new(AtomicU64::new(0)); + let inner: Arc = Arc::new(MemoryStorage::new(MemoryStorageConfig::default())); + let storage: Arc = Arc::new(CountingStorage { + inner: inner.clone(), + bytes_read: bytes_read.clone(), + }); + + let segment_name = "segment_lazy".to_string(); + let huge_text: String = "x".repeat(1_000_000); + { + let mut writer = DocValuesWriter::new(inner.clone(), segment_name.clone()); + writer.add_value(0, "small", crate::data::DataValue::Int64(42)); + writer.add_value(0, "huge", crate::data::DataValue::Text(huge_text)); + writer.write().unwrap(); + } + + let reader = DocValuesReader::load(storage, &segment_name).unwrap(); + let bytes_for_load = bytes_read.load(Ordering::Relaxed); + assert!( + bytes_for_load < 1_000, + "load() read {bytes_for_load} bytes -- it must read only the header \ + and directory, not the huge field's ~1MB payload" + ); + + // Reset so what follows isolates the cost of materializing "small". + bytes_read.store(0, Ordering::Relaxed); + let value = reader.get_value("small", 0).unwrap(); + assert_eq!(value, Some(crate::data::DataValue::Int64(42))); + let bytes_for_small_field = bytes_read.load(Ordering::Relaxed); + assert!( + bytes_for_small_field < 1_000, + "materializing the small field read {bytes_for_small_field} bytes -- \ + the huge field's ~1MB payload must not have been touched" + ); + + // The huge field's payload is read once it is actually requested + // -- laziness defers the cost, it does not eliminate it. + let huge_value = reader.get_value("huge", 0).unwrap(); + assert!(matches!(huge_value, Some(crate::data::DataValue::Text(_)))); + let bytes_for_huge_field = bytes_read.load(Ordering::Relaxed) - bytes_for_small_field; + assert!( + bytes_for_huge_field > 500_000, + "materializing the huge field only read {bytes_for_huge_field} bytes -- \ + expected roughly its ~1MB payload" + ); + } } diff --git a/laurus/tests/lexical_field_sort_test.rs b/laurus/tests/lexical_field_sort_test.rs index 59608b2b..4369f279 100644 --- a/laurus/tests/lexical_field_sort_test.rs +++ b/laurus/tests/lexical_field_sort_test.rs @@ -311,8 +311,18 @@ impl Storage for DvOpenCountingStorage { } } -/// #943: repeated field-sorted searches must load each segment's `.dv` -/// file once — not re-parse it on every per-hit `get_doc_value` call. +/// #943 / #1047 Phase 2: repeated field-sorted searches must not re-parse +/// a segment's `.dv` file per call. +/// +/// Lazy per-field materialization (#1047 Phase 2) changed the *exact* +/// open count from "one open per segment" (the old eager, whole-file +/// load) to "one directory-only open, plus one payload open per field +/// the query actually touches, per segment" -- so this no longer pins a +/// literal `2`. What must still hold is the actual invariant behind the +/// old assertion: once a segment's directory is read and a field's +/// column is materialized, neither is fetched again by a later call. +/// This re-runs the identical search twice and asserts the second run +/// adds zero further `.dv` opens. #[test] fn doc_values_load_once_per_segment() { let dv_opens = Arc::new(std::sync::atomic::AtomicU64::new(0)); @@ -337,21 +347,30 @@ fn doc_values_load_once_per_segment() { store.upsert_document(4, doc(20)).unwrap(); store.commit().unwrap(); - for _ in 0..2 { + let run = |store: &LexicalStore| { let query: Box = Box::new(TermQuery::new("body", "alpha")); let ids = field_sorted_ids( - &store, + store, LexicalSearchRequest::new(query) .limit(4) .sort_by_field_desc("popularity"), ); assert_eq!(ids, vec![3, 2, 4, 1]); - } + }; + + run(&store); + let opens_after_first_search = dv_opens.load(std::sync::atomic::Ordering::Relaxed); + assert!( + opens_after_first_search > 0, + "the .dv file must have been opened at least once to serve the sort" + ); + run(&store); assert_eq!( dv_opens.load(std::sync::atomic::Ordering::Relaxed), - 2, - ".dv must be loaded once per segment, not per get_doc_value call" + opens_after_first_search, + "a repeated search must not re-open .dv files whose directory and \ + materialized fields are already cached" ); } From 60717e648b4bda136d81d171b70f997f79e758b9 Mon Sep 17 00:00:00 2001 From: Minoru Osuka Date: Fri, 11 Sep 2026 21:54:37 -0400 Subject: [PATCH 04/10] feat(lexical): add a per-field doc_values option to the schema Adds doc_values: bool (default true) to the seven field-option structs that can have a DocValues column -- TextOption, IntegerOption, FloatOption, BooleanOption, DateTimeOption, GeoOption, Geo3dOption. Not added to BytesOption: is_doc_values_candidate already excludes Bytes/Vector from DocValues unconditionally (#1052/#1053), so a flag there would always be dead. Effective rule (documented on each field): a DocValues column is written only when stored && doc_values are both true; doc_values: true with stored: false is silently ignored, the same precedent as indexed: false + term_vectors: true. This commit only adds the field and its accessors -- it does not yet gate the write path (that lands in the next commit). Every other exhaustive struct literal of these seven types across the workspace (laurus tests, the five bindings' schema.rs, laurus-cli's create wizard, laurus-server's proto<->domain conversion) is updated mechanically with doc_values: true to preserve exact prior behavior; none of them expose the new flag yet. Refs #1047 --- laurus-cli/src/commands/create.rs | 28 +++- laurus-nodejs/src/schema.rs | 7 + laurus-php/src/schema.rs | 39 +++-- laurus-python/src/schema.rs | 27 ++- laurus-ruby/src/schema.rs | 7 + laurus-server/src/convert/schema.rs | 10 ++ laurus-wasm/src/schema.rs | 7 + laurus/src/lexical/core/field.rs | 154 ++++++++++++++++++ laurus/tests/geo_multi_segment_test.rs | 3 + laurus/tests/lexical_merge_bkd_test.rs | 1 + .../tests/numeric_range_multi_segment_test.rs | 2 + 11 files changed, 267 insertions(+), 18 deletions(-) diff --git a/laurus-cli/src/commands/create.rs b/laurus-cli/src/commands/create.rs index bd89caf9..318e59dd 100644 --- a/laurus-cli/src/commands/create.rs +++ b/laurus-cli/src/commands/create.rs @@ -393,6 +393,7 @@ fn prompt_text_option() -> Result { indexed, stored, term_vectors, + doc_values: true, analyzer, })) } @@ -423,16 +424,34 @@ fn prompt_indexed_stored_option(type_name: &str) -> Result { indexed, stored, multi_valued, + doc_values: true, }), "Float" => FieldOption::Float(FloatOption { indexed, stored, multi_valued, + doc_values: true, + }), + "Boolean" => FieldOption::Boolean(BooleanOption { + indexed, + stored, + doc_values: true, + }), + "DateTime" => FieldOption::DateTime(DateTimeOption { + indexed, + stored, + doc_values: true, + }), + "Geo" => FieldOption::Geo(GeoOption { + indexed, + stored, + doc_values: true, + }), + "Geo3d" => FieldOption::Geo3d(Geo3dOption { + indexed, + stored, + doc_values: true, }), - "Boolean" => FieldOption::Boolean(BooleanOption { indexed, stored }), - "DateTime" => FieldOption::DateTime(DateTimeOption { indexed, stored }), - "Geo" => FieldOption::Geo(GeoOption { indexed, stored }), - "Geo3d" => FieldOption::Geo3d(Geo3dOption { indexed, stored }), _ => unreachable!(), }) } @@ -706,6 +725,7 @@ mod tests { let opt = FieldOption::Geo3d(Geo3dOption { indexed: true, stored: true, + doc_values: true, }); assert_eq!(field_type_label(&opt), "Geo3d"); assert!(is_lexical_field(&opt)); diff --git a/laurus-nodejs/src/schema.rs b/laurus-nodejs/src/schema.rs index 687c92b2..a098dc44 100644 --- a/laurus-nodejs/src/schema.rs +++ b/laurus-nodejs/src/schema.rs @@ -141,6 +141,7 @@ impl JsSchema { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), term_vectors: term_vectors.unwrap_or(true), + doc_values: true, analyzer: analyzer.map(laurus::AnalyzerSpec::Named), }), ); @@ -170,6 +171,7 @@ impl JsSchema { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), multi_valued: multi_valued.unwrap_or(false), + doc_values: true, }), ); } @@ -198,6 +200,7 @@ impl JsSchema { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), multi_valued: multi_valued.unwrap_or(false), + doc_values: true, }), ); } @@ -216,6 +219,7 @@ impl JsSchema { FieldOption::Boolean(BooleanOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), + doc_values: true, }), ); } @@ -239,6 +243,7 @@ impl JsSchema { FieldOption::DateTime(DateTimeOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), + doc_values: true, }), ); } @@ -257,6 +262,7 @@ impl JsSchema { FieldOption::Geo(GeoOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), + doc_values: true, }), ); } @@ -280,6 +286,7 @@ impl JsSchema { FieldOption::Geo3d(Geo3dOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), + doc_values: true, }), ); } diff --git a/laurus-php/src/schema.rs b/laurus-php/src/schema.rs index e874180e..53d3cfa9 100644 --- a/laurus-php/src/schema.rs +++ b/laurus-php/src/schema.rs @@ -138,6 +138,7 @@ impl PhpSchema { indexed, stored, term_vectors, + doc_values: true, analyzer: analyzer.map(laurus::AnalyzerSpec::Named), }), ); @@ -161,6 +162,7 @@ impl PhpSchema { indexed, stored, multi_valued, + doc_values: true, }), ); } @@ -183,6 +185,7 @@ impl PhpSchema { indexed, stored, multi_valued, + doc_values: true, }), ); } @@ -198,7 +201,11 @@ impl PhpSchema { pub fn add_boolean_field(&self, name: String, stored: bool, indexed: bool) { self.inner.borrow_mut().fields.insert( name, - FieldOption::Boolean(BooleanOption { indexed, stored }), + FieldOption::Boolean(BooleanOption { + indexed, + stored, + doc_values: true, + }), ); } @@ -213,7 +220,11 @@ impl PhpSchema { pub fn add_datetime_field(&self, name: String, stored: bool, indexed: bool) { self.inner.borrow_mut().fields.insert( name, - FieldOption::DateTime(DateTimeOption { indexed, stored }), + FieldOption::DateTime(DateTimeOption { + indexed, + stored, + doc_values: true, + }), ); } @@ -226,10 +237,14 @@ impl PhpSchema { /// * `indexed` - Whether the field is searchable (default: true). #[php(defaults(stored = true, indexed = true))] pub fn add_geo_field(&self, name: String, stored: bool, indexed: bool) { - self.inner - .borrow_mut() - .fields - .insert(name, FieldOption::Geo(GeoOption { indexed, stored })); + self.inner.borrow_mut().fields.insert( + name, + FieldOption::Geo(GeoOption { + indexed, + stored, + doc_values: true, + }), + ); } /// Add a 3D ECEF Cartesian point field (x, y, z in meters). @@ -246,10 +261,14 @@ impl PhpSchema { /// * `indexed` - Whether the field is searchable (default: true). #[php(defaults(stored = true, indexed = true))] pub fn add_geo3d_field(&self, name: String, stored: bool, indexed: bool) { - self.inner - .borrow_mut() - .fields - .insert(name, FieldOption::Geo3d(Geo3dOption { indexed, stored })); + self.inner.borrow_mut().fields.insert( + name, + FieldOption::Geo3d(Geo3dOption { + indexed, + stored, + doc_values: true, + }), + ); } /// Add a binary data field. diff --git a/laurus-python/src/schema.rs b/laurus-python/src/schema.rs index bddbb967..6e5da69c 100644 --- a/laurus-python/src/schema.rs +++ b/laurus-python/src/schema.rs @@ -244,6 +244,7 @@ impl PySchema { indexed, stored, term_vectors, + doc_values: true, analyzer, }), ); @@ -274,6 +275,7 @@ impl PySchema { indexed, stored, multi_valued, + doc_values: true, }), ); } @@ -296,6 +298,7 @@ impl PySchema { indexed, stored, multi_valued, + doc_values: true, }), ); } @@ -305,7 +308,11 @@ impl PySchema { pub fn add_boolean_field(&mut self, name: &str, stored: bool, indexed: bool) { self.inner.fields.insert( name.to_string(), - FieldOption::Boolean(BooleanOption { indexed, stored }), + FieldOption::Boolean(BooleanOption { + indexed, + stored, + doc_values: true, + }), ); } @@ -314,7 +321,11 @@ impl PySchema { pub fn add_datetime_field(&mut self, name: &str, stored: bool, indexed: bool) { self.inner.fields.insert( name.to_string(), - FieldOption::DateTime(DateTimeOption { indexed, stored }), + FieldOption::DateTime(DateTimeOption { + indexed, + stored, + doc_values: true, + }), ); } @@ -323,7 +334,11 @@ impl PySchema { pub fn add_geo_field(&mut self, name: &str, stored: bool, indexed: bool) { self.inner.fields.insert( name.to_string(), - FieldOption::Geo(GeoOption { indexed, stored }), + FieldOption::Geo(GeoOption { + indexed, + stored, + doc_values: true, + }), ); } @@ -337,7 +352,11 @@ impl PySchema { pub fn add_geo3d_field(&mut self, name: &str, stored: bool, indexed: bool) { self.inner.fields.insert( name.to_string(), - FieldOption::Geo3d(Geo3dOption { indexed, stored }), + FieldOption::Geo3d(Geo3dOption { + indexed, + stored, + doc_values: true, + }), ); } diff --git a/laurus-ruby/src/schema.rs b/laurus-ruby/src/schema.rs index 1946d9a2..2fca003c 100644 --- a/laurus-ruby/src/schema.rs +++ b/laurus-ruby/src/schema.rs @@ -167,6 +167,7 @@ impl RbSchema { indexed, stored, term_vectors, + doc_values: true, analyzer, }), ); @@ -199,6 +200,7 @@ impl RbSchema { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), multi_valued: multi_valued.unwrap_or(false), + doc_values: true, }), ); Ok(()) @@ -230,6 +232,7 @@ impl RbSchema { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), multi_valued: multi_valued.unwrap_or(false), + doc_values: true, }), ); Ok(()) @@ -257,6 +260,7 @@ impl RbSchema { FieldOption::Boolean(BooleanOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), + doc_values: true, }), ); Ok(()) @@ -284,6 +288,7 @@ impl RbSchema { FieldOption::DateTime(DateTimeOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), + doc_values: true, }), ); Ok(()) @@ -311,6 +316,7 @@ impl RbSchema { FieldOption::Geo(GeoOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), + doc_values: true, }), ); Ok(()) @@ -343,6 +349,7 @@ impl RbSchema { FieldOption::Geo3d(Geo3dOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), + doc_values: true, }), ); Ok(()) diff --git a/laurus-server/src/convert/schema.rs b/laurus-server/src/convert/schema.rs index c5ed11c6..06dff058 100644 --- a/laurus-server/src/convert/schema.rs +++ b/laurus-server/src/convert/schema.rs @@ -205,33 +205,42 @@ pub fn field_option_from_proto(fo: &v1::FieldOption) -> Option { // Unset means "use the engine's default", matching // `TextOption::default()` (#1083). term_vectors: o.term_vectors.unwrap_or(true), + // The proto message has no `doc_values` field yet; pin to the + // pre-#1047 implicit default until the protocol is wired up. + doc_values: true, analyzer: o.analyzer.as_ref().and_then(analyzer_spec_from_proto), })), Some(Opt::Integer(o)) => Some(FieldOption::Integer(IntegerOption { indexed: o.indexed, stored: o.stored, multi_valued: o.multi_valued, + doc_values: true, })), Some(Opt::Float(o)) => Some(FieldOption::Float(FloatOption { indexed: o.indexed, stored: o.stored, multi_valued: o.multi_valued, + doc_values: true, })), Some(Opt::Boolean(o)) => Some(FieldOption::Boolean(BooleanOption { indexed: o.indexed, stored: o.stored, + doc_values: true, })), Some(Opt::DateTime(o)) => Some(FieldOption::DateTime(DateTimeOption { indexed: o.indexed, stored: o.stored, + doc_values: true, })), Some(Opt::Geo(o)) => Some(FieldOption::Geo(GeoOption { indexed: o.indexed, stored: o.stored, + doc_values: true, })), Some(Opt::Geo3d(o)) => Some(FieldOption::Geo3d(Geo3dOption { indexed: o.indexed, stored: o.stored, + doc_values: true, })), Some(Opt::Bytes(o)) => Some(FieldOption::Bytes(BytesOption { stored: o.stored })), Some(Opt::Hnsw(o)) => Some(FieldOption::Hnsw(HnswOption { @@ -1177,6 +1186,7 @@ mod tests { Geo3dOption { indexed: true, stored: false, + doc_values: true, }, ) .build(); diff --git a/laurus-wasm/src/schema.rs b/laurus-wasm/src/schema.rs index 59769f10..ea4af439 100644 --- a/laurus-wasm/src/schema.rs +++ b/laurus-wasm/src/schema.rs @@ -155,6 +155,7 @@ impl WasmSchema { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), term_vectors: term_vectors.unwrap_or(true), + doc_values: true, analyzer: analyzer.map(laurus::AnalyzerSpec::Named), }), ); @@ -175,6 +176,7 @@ impl WasmSchema { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), multi_valued: multi_valued.unwrap_or(false), + doc_values: true, }), ); } @@ -194,6 +196,7 @@ impl WasmSchema { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), multi_valued: multi_valued.unwrap_or(false), + doc_values: true, }), ); } @@ -206,6 +209,7 @@ impl WasmSchema { FieldOption::Boolean(BooleanOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), + doc_values: true, }), ); } @@ -223,6 +227,7 @@ impl WasmSchema { FieldOption::DateTime(DateTimeOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), + doc_values: true, }), ); } @@ -235,6 +240,7 @@ impl WasmSchema { FieldOption::Geo(GeoOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), + doc_values: true, }), ); } @@ -252,6 +258,7 @@ impl WasmSchema { FieldOption::Geo3d(Geo3dOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), + doc_values: true, }), ); } diff --git a/laurus/src/lexical/core/field.rs b/laurus/src/lexical/core/field.rs index f34e7338..34854766 100644 --- a/laurus/src/lexical/core/field.rs +++ b/laurus/src/lexical/core/field.rs @@ -108,6 +108,7 @@ impl /// indexed: true, /// stored: true, /// term_vectors: true, +/// doc_values: true, /// analyzer: None, /// }), /// }; @@ -201,6 +202,21 @@ pub struct TextOption { #[serde(default = "default_true")] pub term_vectors: bool, + /// Whether this field's value is also copied into DocValues, the + /// column-oriented store `SortField::Field`, faceting, and + /// aggregations read from (Issue #1047). + /// + /// Effective rule: a DocValues column is written only when `stored + /// && doc_values` are both `true`. `doc_values: true` with `stored: + /// false` is silently ignored (no error) rather than rejected, the + /// same precedent as an `indexed: false` + `term_vectors: true` + /// combination. Turning this off for a large field that is never + /// sorted or faceted on shrinks the segment at the cost of losing + /// that field from sort/facet/aggregation until the field is + /// rebuilt or the segment merges. + #[serde(default = "default_true")] + pub doc_values: bool, + /// Analyzer reference for this field. Either a bare name /// (e.g. `"standard"`, `"english"`) or a parameterized built-in /// preset (e.g. `{"language": "japanese", "dict": "/path/to/ipadic"}`). @@ -259,6 +275,23 @@ impl TextOption { self } + /// Sets whether this field's value is also copied into DocValues. + /// + /// Takes effect only when `stored` is also `true` (Issue #1047); see + /// the field's doc comment for the full effective rule. + /// + /// # Arguments + /// + /// * `doc_values` - `true` to write a DocValues column, `false` otherwise. + /// + /// # Returns + /// + /// The modified `TextOption` for method chaining. + pub fn doc_values(mut self, doc_values: bool) -> Self { + self.doc_values = doc_values; + self + } + /// Sets the analyzer for this field. /// /// Accepts either a bare name (`&str` / `String`) for analyzers that @@ -292,12 +325,19 @@ impl Default for TextOption { indexed: true, stored: true, term_vectors: true, + doc_values: true, analyzer: None, } } } /// Option for Bytes field. +/// +/// Has no `doc_values` flag, unlike the other six field options: a +/// `Bytes` (or `Vector`) value is never written to DocValues regardless +/// of any flag (`is_doc_values_candidate` excludes both types +/// unconditionally, Issue #1052/#1053), so a `doc_values` field here +/// would always be a dead setting. #[derive( Debug, Clone, PartialEq, Serialize, Deserialize, Archive, RkyvSerialize, RkyvDeserialize, )] @@ -350,6 +390,16 @@ pub struct IntegerOption { /// with constant scoring). Defaults to `false`. #[serde(default)] pub multi_valued: bool, + + /// Whether this field's value is also copied into DocValues, the + /// column-oriented store `SortField::Field`, faceting, and + /// aggregations read from (Issue #1047). + /// + /// Effective rule: a DocValues column is written only when `stored + /// && doc_values` are both `true`; `doc_values: true` with `stored: + /// false` is silently ignored (no error). + #[serde(default = "default_true")] + pub doc_values: bool, } impl IntegerOption { @@ -364,6 +414,14 @@ impl IntegerOption { self.stored = stored; self } + + /// Set whether this field's value is also copied into DocValues. + /// + /// Takes effect only when `stored` is also `true` (Issue #1047). + pub fn doc_values(mut self, doc_values: bool) -> Self { + self.doc_values = doc_values; + self + } } impl Default for IntegerOption { @@ -372,6 +430,7 @@ impl Default for IntegerOption { indexed: true, stored: true, multi_valued: false, + doc_values: true, } } } @@ -397,6 +456,16 @@ pub struct FloatOption { /// with constant scoring). Defaults to `false`. #[serde(default)] pub multi_valued: bool, + + /// Whether this field's value is also copied into DocValues, the + /// column-oriented store `SortField::Field`, faceting, and + /// aggregations read from (Issue #1047). + /// + /// Effective rule: a DocValues column is written only when `stored + /// && doc_values` are both `true`; `doc_values: true` with `stored: + /// false` is silently ignored (no error). + #[serde(default = "default_true")] + pub doc_values: bool, } impl FloatOption { @@ -411,6 +480,14 @@ impl FloatOption { self.stored = stored; self } + + /// Set whether this field's value is also copied into DocValues. + /// + /// Takes effect only when `stored` is also `true` (Issue #1047). + pub fn doc_values(mut self, doc_values: bool) -> Self { + self.doc_values = doc_values; + self + } } impl Default for FloatOption { @@ -419,6 +496,7 @@ impl Default for FloatOption { indexed: true, stored: true, multi_valued: false, + doc_values: true, } } } @@ -435,6 +513,16 @@ pub struct BooleanOption { /// Whether to store the original value. #[serde(default = "default_true")] pub stored: bool, + + /// Whether this field's value is also copied into DocValues, the + /// column-oriented store `SortField::Field`, faceting, and + /// aggregations read from (Issue #1047). + /// + /// Effective rule: a DocValues column is written only when `stored + /// && doc_values` are both `true`; `doc_values: true` with `stored: + /// false` is silently ignored (no error). + #[serde(default = "default_true")] + pub doc_values: bool, } impl BooleanOption { @@ -449,6 +537,14 @@ impl BooleanOption { self.stored = stored; self } + + /// Set whether this field's value is also copied into DocValues. + /// + /// Takes effect only when `stored` is also `true` (Issue #1047). + pub fn doc_values(mut self, doc_values: bool) -> Self { + self.doc_values = doc_values; + self + } } impl Default for BooleanOption { @@ -456,6 +552,7 @@ impl Default for BooleanOption { Self { indexed: true, stored: true, + doc_values: true, } } } @@ -472,6 +569,16 @@ pub struct DateTimeOption { /// Whether to store the original value. #[serde(default = "default_true")] pub stored: bool, + + /// Whether this field's value is also copied into DocValues, the + /// column-oriented store `SortField::Field`, faceting, and + /// aggregations read from (Issue #1047). + /// + /// Effective rule: a DocValues column is written only when `stored + /// && doc_values` are both `true`; `doc_values: true` with `stored: + /// false` is silently ignored (no error). + #[serde(default = "default_true")] + pub doc_values: bool, } impl DateTimeOption { @@ -486,6 +593,14 @@ impl DateTimeOption { self.stored = stored; self } + + /// Set whether this field's value is also copied into DocValues. + /// + /// Takes effect only when `stored` is also `true` (Issue #1047). + pub fn doc_values(mut self, doc_values: bool) -> Self { + self.doc_values = doc_values; + self + } } impl Default for DateTimeOption { @@ -493,6 +608,7 @@ impl Default for DateTimeOption { Self { indexed: true, stored: true, + doc_values: true, } } } @@ -509,6 +625,16 @@ pub struct GeoOption { /// Whether to store the original value. #[serde(default = "default_true")] pub stored: bool, + + /// Whether this field's value is also copied into DocValues, the + /// column-oriented store `SortField::Field`, faceting, and + /// aggregations read from (Issue #1047). + /// + /// Effective rule: a DocValues column is written only when `stored + /// && doc_values` are both `true`; `doc_values: true` with `stored: + /// false` is silently ignored (no error). + #[serde(default = "default_true")] + pub doc_values: bool, } impl GeoOption { @@ -523,6 +649,14 @@ impl GeoOption { self.stored = stored; self } + + /// Set whether this field's value is also copied into DocValues. + /// + /// Takes effect only when `stored` is also `true` (Issue #1047). + pub fn doc_values(mut self, doc_values: bool) -> Self { + self.doc_values = doc_values; + self + } } /// Options for 3D Earth-Centered Earth-Fixed (ECEF) geographic point fields. @@ -543,6 +677,16 @@ pub struct Geo3dOption { /// Whether to store the original value. #[serde(default = "default_true")] pub stored: bool, + + /// Whether this field's value is also copied into DocValues, the + /// column-oriented store `SortField::Field`, faceting, and + /// aggregations read from (Issue #1047). + /// + /// Effective rule: a DocValues column is written only when `stored + /// && doc_values` are both `true`; `doc_values: true` with `stored: + /// false` is silently ignored (no error). + #[serde(default = "default_true")] + pub doc_values: bool, } impl Geo3dOption { @@ -557,6 +701,14 @@ impl Geo3dOption { self.stored = stored; self } + + /// Set whether this field's value is also copied into DocValues. + /// + /// Takes effect only when `stored` is also `true` (Issue #1047). + pub fn doc_values(mut self, doc_values: bool) -> Self { + self.doc_values = doc_values; + self + } } impl Default for Geo3dOption { @@ -564,6 +716,7 @@ impl Default for Geo3dOption { Self { indexed: true, stored: true, + doc_values: true, } } } @@ -660,6 +813,7 @@ impl Default for GeoOption { Self { indexed: true, stored: true, + doc_values: true, } } } diff --git a/laurus/tests/geo_multi_segment_test.rs b/laurus/tests/geo_multi_segment_test.rs index 836bf375..df95e22c 100644 --- a/laurus/tests/geo_multi_segment_test.rs +++ b/laurus/tests/geo_multi_segment_test.rs @@ -59,6 +59,7 @@ fn stored_only_geo_field_matches_via_fallback_across_segments() { FieldOption::Geo(GeoOption { indexed: false, stored: true, + doc_values: true, }), ) .build(); @@ -106,6 +107,7 @@ fn index_only_geo_field_matches_via_bkd() { FieldOption::Geo(GeoOption { indexed: true, stored: false, + doc_values: true, }), ) .build(); @@ -149,6 +151,7 @@ fn geo_query_on_sparse_field_across_segments() { FieldOption::Geo(GeoOption { indexed: true, stored: true, + doc_values: true, }), ) .add_field("body", FieldOption::Text(TextOption::default())) diff --git a/laurus/tests/lexical_merge_bkd_test.rs b/laurus/tests/lexical_merge_bkd_test.rs index ddbda736..00499e23 100644 --- a/laurus/tests/lexical_merge_bkd_test.rs +++ b/laurus/tests/lexical_merge_bkd_test.rs @@ -47,6 +47,7 @@ fn merge_preserves_bkd_points_for_index_only_numeric_field() { indexed: true, stored: false, multi_valued: false, + doc_values: true, }), ) .build(); diff --git a/laurus/tests/numeric_range_multi_segment_test.rs b/laurus/tests/numeric_range_multi_segment_test.rs index c1ca7f31..b9ad4e66 100644 --- a/laurus/tests/numeric_range_multi_segment_test.rs +++ b/laurus/tests/numeric_range_multi_segment_test.rs @@ -38,6 +38,7 @@ fn store_config() -> LexicalIndexConfig { indexed: true, stored: true, multi_valued: false, + doc_values: true, }), ) .add_field("body", FieldOption::Text(TextOption::default())) @@ -101,6 +102,7 @@ fn range_query_on_stored_only_field_matches_via_fallback() { indexed: false, stored: true, multi_valued: false, + doc_values: true, }), ) .build(); From 36273153965a3f985fc22686e7454eb8eb953a32 Mon Sep 17 00:00:00 2001 From: Minoru Osuka Date: Sat, 12 Sep 2026 08:49:40 -0400 Subject: [PATCH 05/10] feat(lexical): wire doc_values into the write, merge, and field-rebuild paths Core write-path wiring (Phase 3): - InvertedIndexConfig::store_doc_values / LexicalIndexConfigBuilder:: store_doc_values: index-wide default, mirrors store_term_vectors. - InvertedIndexWriterConfig::{field_doc_values, store_doc_values} and stores_doc_values(field_name), a 3-level resolution (override > field's own schema setting > index-wide default) identical in shape to stores_term_positions (#1083). - Both DocValues feed sites (upsert_analyzed_document, rebuild_in_memory_index) now gate on `is_doc_values_candidate(value) && stores_doc_values(field_name)`, not just the type check. - FieldOption::doc_values(&self) -> Option centralizes the 7-arm match (None only for Bytes) so this and the merge path below don't each duplicate it. - InvertedIndex::writer() passes config.store_doc_values through. Merge / field-rebuild wiring (Phase 4), deliberately NOT mirroring how #1083 handled term_vectors: a discarded term position is unrecoverable, so #1083 could only detect what a source segment already had. A DocValues column can always be regenerated from stored_fields, so here the CURRENT schema wins over whatever is on disk: - MergeConfig::{field_doc_values, default_doc_values}: the owning index's current per-field settings and index-wide default, populated by InvertedIndex::merge_segment_set / rebuild_field via the new current_field_doc_values() helper. - reconstruct_segment / reconstruct_segment_with_field_override thread a doc_values_by_field map alongside the existing positions_by_field: schema-declared fields are pre-seeded (so detection's .entry().or_insert_with() never overrides them); anything left undeclared falls back to detecting the source segment's actual column. rebuild_field_across_segments gained a target_doc_values parameter, seeded the same way target_term_vectors already is. - The replay InvertedIndexWriterConfig gets the resolved field_doc_values map and store_doc_values default in both merge paths. Tests: stores_doc_values resolution order, a doc_values: false field excluded from a real segment's DocValues (unit + on-disk), a large field's .dv payload measurably shrinking, three merge-path tests (independent per-field state across a merge, current schema winning over a stale on-disk column, and a value-less field not being misdetected as opted out). All new assertions were RED-proven by reverting the relevant line and confirming the expected failure. Refs #1047 --- laurus/src/lexical/core/field.rs | 21 ++ laurus/src/lexical/index/config.rs | 25 ++ laurus/src/lexical/index/inverted.rs | 35 ++ laurus/src/lexical/index/inverted/reader.rs | 59 ++++ .../index/inverted/segment/merge_engine.rs | 316 +++++++++++++++++- laurus/src/lexical/index/inverted/writer.rs | 212 +++++++++++- laurus/src/lexical/store/config.rs | 17 + 7 files changed, 679 insertions(+), 6 deletions(-) diff --git a/laurus/src/lexical/core/field.rs b/laurus/src/lexical/core/field.rs index 34854766..737bce82 100644 --- a/laurus/src/lexical/core/field.rs +++ b/laurus/src/lexical/core/field.rs @@ -806,6 +806,27 @@ impl FieldOption { }), } } + + /// This option's own `doc_values` setting, or `None` for `Bytes` (the + /// one field option with no such flag -- Issue #1047). + /// + /// Centralizes the 7-arm match so callers that need a field's raw + /// schema-level `doc_values` setting (as opposed to a fully resolved + /// default-and-override decision, like + /// [`InvertedIndexWriterConfig::stores_doc_values`](crate::lexical::index::inverted::writer::InvertedIndexWriterConfig::stores_doc_values)) + /// don't each duplicate it. + pub(crate) fn doc_values(&self) -> Option { + match self { + FieldOption::Text(opt) => Some(opt.doc_values), + FieldOption::Integer(opt) => Some(opt.doc_values), + FieldOption::Float(opt) => Some(opt.doc_values), + FieldOption::Boolean(opt) => Some(opt.doc_values), + FieldOption::DateTime(opt) => Some(opt.doc_values), + FieldOption::Geo(opt) => Some(opt.doc_values), + FieldOption::Geo3d(opt) => Some(opt.doc_values), + FieldOption::Bytes(_) => None, + } + } } impl Default for GeoOption { diff --git a/laurus/src/lexical/index/config.rs b/laurus/src/lexical/index/config.rs index e18923c7..1edac6b1 100644 --- a/laurus/src/lexical/index/config.rs +++ b/laurus/src/lexical/index/config.rs @@ -20,6 +20,13 @@ fn default_use_compound() -> bool { crate::lexical::index::inverted::compound::default_use_compound() } +/// serde default for [`InvertedIndexConfig::store_doc_values`] -- a +/// config missing the field (every index created before Issue #1047) +/// must behave exactly as it did before the flag existed. +fn default_true() -> bool { + true +} + #[derive(Clone, Serialize, Deserialize)] pub struct InvertedIndexConfig { /// Write flushed segments as one compound `.cfs` container instead of @@ -68,6 +75,22 @@ pub struct InvertedIndexConfig { /// queries over the fields it applies to. pub store_term_vectors: bool, + /// Index-wide default for whether a field's value is also copied into + /// DocValues (Issue #1047). + /// + /// Like [`Self::store_term_vectors`], this is only the **default**: a + /// field's own [`TextOption::doc_values`](crate::lexical::core::field::TextOption::doc_values) + /// (or the equivalent on the other six field options that carry the + /// flag) overrides it. This value governs schema-less fields, + /// reserved `_`-prefixed fields, and any lexical field type declared + /// without an explicit `doc_values` setting. + /// + /// Disabling it index-wide shrinks every segment at the cost of + /// losing sort/facet/aggregation support on every field that does not + /// explicitly opt back in via its own `doc_values: true`. + #[serde(default = "default_true")] + pub store_doc_values: bool, + /// Merge factor for segment merging. /// /// Controls how many segments are merged at once. Higher values reduce @@ -140,6 +163,7 @@ impl Default for InvertedIndexConfig { write_buffer_size: 1024 * 1024, // 1MB compress_stored_fields: false, store_term_vectors: true, + store_doc_values: true, merge_factor: 10, max_segments: 100, analyzer: std::sync::Arc::new( @@ -162,6 +186,7 @@ impl std::fmt::Debug for InvertedIndexConfig { .field("write_buffer_size", &self.write_buffer_size) .field("compress_stored_fields", &self.compress_stored_fields) .field("store_term_vectors", &self.store_term_vectors) + .field("store_doc_values", &self.store_doc_values) .field("merge_factor", &self.merge_factor) .field("max_segments", &self.max_segments) .field("analyzer", &self.analyzer.name()) diff --git a/laurus/src/lexical/index/inverted.rs b/laurus/src/lexical/index/inverted.rs index c916bb7f..4e6648a2 100644 --- a/laurus/src/lexical/index/inverted.rs +++ b/laurus/src/lexical/index/inverted.rs @@ -552,6 +552,31 @@ impl InvertedIndex { /// files are removed — minimizing any window in which a document could be /// seen in both a source and the merged segment). A no-op for fewer than /// two sources. + /// Every currently-declared field's `doc_values` setting, merging the + /// initial schema (`config.fields`) with fields added at runtime + /// (`extra_fields`) -- the same union [`Self::writer`] builds for + /// `InvertedIndexWriterConfig::fields`. Feeds + /// [`MergeConfig::field_doc_values`](self::segment::merge_engine::MergeConfig::field_doc_values) + /// so a merge or field rebuild resolves DocValues from the CURRENT + /// schema first (Issue #1047), not from whatever a source segment + /// happens to have on disk. + fn current_field_doc_values(&self) -> HashMap { + let mut out = HashMap::new(); + for (name, option) in &self.config.fields { + if let Some(dv) = option.doc_values() { + out.insert(name.clone(), dv); + } + } + // Runtime-added fields win over the initial schema on a name + // clash, mirroring `writer()`'s `fields.extend(extra_fields)`. + for (name, option) in self.extra_fields.read().iter() { + if let Some(dv) = option.doc_values() { + out.insert(name.clone(), dv); + } + } + out + } + fn merge_segment_set(&self, sources: &[SegmentInfo], next_generation: u64) -> Result<()> { use self::segment::merge_engine::{MergeConfig, MergeEngine}; use self::segment::{ManagedSegmentInfo, MergeCandidate, MergeStrategy}; @@ -578,6 +603,8 @@ impl InvertedIndex { let engine = MergeEngine::new( MergeConfig { use_compound: self.config.use_compound, + field_doc_values: self.current_field_doc_values(), + default_doc_values: self.config.store_doc_values, ..MergeConfig::default() }, self.storage.clone(), @@ -739,6 +766,7 @@ impl LexicalIndex for InvertedIndex { fields, use_compound: self.config.use_compound, store_term_positions: self.config.store_term_vectors, + store_doc_values: self.config.store_doc_values, ..Default::default() }; // Hand the writer the shared metadata and manifest handles @@ -863,6 +891,10 @@ impl LexicalIndex for InvertedIndex { FieldOption::Text(text_option) => text_option.term_vectors, _ => self.config.store_term_vectors, }; + // #1047: same idea for DocValues -- `option.doc_values()` is + // `None` only for `Bytes` (which has no such flag), so this falls + // back to the index-wide default in exactly that one case. + let target_doc_values = option.doc_values().unwrap_or(self.config.store_doc_values); let segments = self.load_segments()?; if !segments.is_empty() { @@ -891,6 +923,8 @@ impl LexicalIndex for InvertedIndex { let engine = MergeEngine::new( MergeConfig { use_compound: self.config.use_compound, + field_doc_values: self.current_field_doc_values(), + default_doc_values: self.config.store_doc_values, ..MergeConfig::default() }, self.storage.clone(), @@ -906,6 +940,7 @@ impl LexicalIndex for InvertedIndex { name, analyzer.as_ref(), target_term_vectors, + target_doc_values, &new_segment_ids, )?; diff --git a/laurus/src/lexical/index/inverted/reader.rs b/laurus/src/lexical/index/inverted/reader.rs index 31d24350..d93f2623 100644 --- a/laurus/src/lexical/index/inverted/reader.rs +++ b/laurus/src/lexical/index/inverted/reader.rs @@ -2516,6 +2516,65 @@ mod tests { use super::*; use crate::lexical::reader::PostingIterator; + /// #1047: `has_doc_values` must reflect the schema's per-field + /// `doc_values` flag on a real, on-disk segment -- a field declared + /// `doc_values: false` gets no column, one left at the default does. + #[test] + fn has_doc_values_reflects_the_schemas_doc_values_flag() { + use crate::lexical::core::field::{FieldOption, TextOption}; + use crate::lexical::index::LexicalIndex; + use crate::lexical::index::inverted::{InvertedIndex, InvertedIndexConfig}; + use crate::storage::memory::{MemoryStorage, MemoryStorageConfig}; + + let mut fields = std::collections::HashMap::new(); + fields.insert( + "title".to_string(), + FieldOption::Text(TextOption::default()), + ); + fields.insert( + "internal_note".to_string(), + FieldOption::Text(TextOption { + doc_values: false, + ..Default::default() + }), + ); + let config = InvertedIndexConfig { + fields, + ..Default::default() + }; + + let storage: Arc = + Arc::new(MemoryStorage::new(MemoryStorageConfig::default())); + let index = InvertedIndex::create(storage, config).unwrap(); + let mut writer = index.writer().unwrap(); + writer + .add_document( + crate::Document::builder() + .add_text("title", "hello") + .add_text("internal_note", "shh") + .build(), + ) + .unwrap(); + writer.commit().unwrap(); + + let reader = writer.build_reader().unwrap(); + let inverted = reader + .as_any() + .downcast_ref::() + .unwrap(); + let segment = inverted.segment_readers()[0].read().unwrap(); + + assert!( + segment.has_doc_values("title"), + "a field without doc_values: false must get a column" + ); + assert!( + !segment.has_doc_values("internal_note"), + "doc_values: false must keep the field out of the segment's \ + DocValues directory entirely" + ); + } + /// #541 — `SegmentReader::postings` must never yield a deleted /// document, on either of its paths. /// diff --git a/laurus/src/lexical/index/inverted/segment/merge_engine.rs b/laurus/src/lexical/index/inverted/segment/merge_engine.rs index 75a04d3c..9c644819 100644 --- a/laurus/src/lexical/index/inverted/segment/merge_engine.rs +++ b/laurus/src/lexical/index/inverted/segment/merge_engine.rs @@ -49,6 +49,34 @@ pub struct MergeConfig { /// Verify integrity after merge. pub verify_after_merge: bool, + + /// The current schema's per-field `doc_values` setting, keyed by + /// field name (Issue #1047). + /// + /// Unlike term positions (which are detected from what a source + /// segment already has on disk, since a discarded position cannot be + /// recovered -- see [`Self::default_doc_values`]'s doc comment for + /// why this is deliberately different), DocValues columns can always + /// be regenerated from `stored_fields`. So the CURRENT schema wins + /// here: a field this map declares is written (or not) according to + /// the schema regardless of what any source segment happened to have. + /// Only a field this map does *not* mention falls back to detecting + /// each source segment's existing column. + /// + /// Populated by the caller from the owning index's `config.fields` + + /// `extra_fields` (`InvertedIndex::merge_segment_set` / + /// `InvertedIndex::rebuild_field`); empty for a bare + /// `MergeConfig::default()`, which makes every field fall through to + /// pure detection, then [`Self::default_doc_values`]. + pub field_doc_values: HashMap, + + /// Index-wide default for whether a field's value is written to + /// DocValues, used only for a field neither `field_doc_values` nor + /// per-segment detection resolves (i.e. no source segment ever had a + /// candidate value for it at all -- so this is rarely, if ever, + /// actually consulted). Mirrors + /// [`InvertedIndexConfig::store_doc_values`](crate::lexical::index::config::InvertedIndexConfig::store_doc_values). + pub default_doc_values: bool, } impl Default for MergeConfig { @@ -61,6 +89,8 @@ impl Default for MergeConfig { remove_deleted_docs: true, sort_by_doc_id: true, verify_after_merge: true, + field_doc_values: HashMap::new(), + default_doc_values: true, } } } @@ -299,12 +329,23 @@ impl MergeEngine { // segment reproduces every field's positions state independently // (#1083). let mut positions_by_field: HashMap = HashMap::new(); + // Whether the merged segment should write a DocValues column, per + // field (#1047). Seeded from the CURRENT schema so it wins over + // whatever a source segment happens to have on disk; a field the + // schema does not mention falls back to detection inside + // `reconstruct_segment` (`.entry().or_insert_with()` only fills + // gaps `field_doc_values` left open). + let mut doc_values_by_field: HashMap = self.config.field_doc_values.clone(); for segment in segments { let reader = SegmentReader::open(segment.segment_info.clone(), self.storage.clone())?; let deleted = self.load_deleted_docs(&segment.segment_info)?; - let reconstructed = - self.reconstruct_segment(&reader, &deleted, &mut positions_by_field)?; + let reconstructed = self.reconstruct_segment( + &reader, + &deleted, + &mut positions_by_field, + &mut doc_values_by_field, + )?; stats.deleted_docs_removed += deleted.len(); for (doc_id, analyzed) in reconstructed { if docs.insert(doc_id, analyzed).is_none() { @@ -327,6 +368,8 @@ impl MergeEngine { // unbounded so the merge produces exactly one output segment. let writer_config = InvertedIndexWriterConfig { field_term_positions: positions_by_field, + field_doc_values: doc_values_by_field, + store_doc_values: self.config.default_doc_values, shard_id: stats.shard_id, max_buffered_docs: usize::MAX, max_buffer_memory: usize::MAX, @@ -424,6 +467,12 @@ impl MergeEngine { /// positions state it already had, detected the same way /// [`Self::perform_merge`] does. /// + /// `target_doc_values` is `target_field`'s new `doc_values` setting + /// (#1047), seeded the same way into the per-field DocValues map for + /// the same reason. Every other field resolves from + /// [`MergeConfig::field_doc_values`] first, detection second, exactly + /// as [`Self::perform_merge`] does. + /// /// # Errors /// /// Returns the first error encountered and aborts the writer that hit @@ -441,6 +490,7 @@ impl MergeEngine { target_field: &str, analyzer: Option<&Arc>, target_term_vectors: bool, + target_doc_values: bool, new_segment_ids: &[String], ) -> Result> { assert_eq!( @@ -457,6 +507,12 @@ impl MergeEngine { // never overrides the new schema's setting. let mut positions_by_field: HashMap = HashMap::new(); positions_by_field.insert(target_field.to_string(), target_term_vectors); + // Same idea for DocValues (#1047): seed the CURRENT schema (minus + // `target_field`, whose stale on-disk column must not leak + // through), then pin `target_field` to its NEW setting so + // detection can never override either. + let mut doc_values_by_field: HashMap = self.config.field_doc_values.clone(); + doc_values_by_field.insert(target_field.to_string(), target_doc_values); let mut results = Vec::with_capacity(segments.len()); for (segment, new_segment_id) in segments.iter().zip(new_segment_ids) { @@ -466,6 +522,7 @@ impl MergeEngine { &reader, &deleted, &mut positions_by_field, + &mut doc_values_by_field, target_field, analyzer, )?; @@ -476,6 +533,8 @@ impl MergeEngine { let writer_config = InvertedIndexWriterConfig { field_term_positions: positions_by_field.clone(), + field_doc_values: doc_values_by_field.clone(), + store_doc_values: self.config.default_doc_values, shard_id: segment.segment_info.shard_id, max_buffered_docs: usize::MAX, max_buffer_memory: usize::MAX, @@ -594,11 +653,19 @@ impl MergeEngine { /// the merged segment reproduces each field's positions state /// independently — fields can disagree, e.g. one `term_vectors: true` /// and one `false` (#1083). + /// + /// `doc_values_by_field` (#1047) works the same way but only fills + /// gaps the caller's schema-derived seed left open (see + /// [`MergeConfig::field_doc_values`]): for a field this map does not + /// already mention, the first document in *this* segment with a + /// DocValues-candidate value for it records whether this segment's + /// `.dv` currently has a column for that field. fn reconstruct_segment( &self, reader: &SegmentReader, deleted: &RoaringTreemap, positions_by_field: &mut HashMap, + doc_values_by_field: &mut HashMap, ) -> Result> { // Pass 1: bucket postings into per-doc analyzed terms. let mut field_terms: AHashMap>> = AHashMap::new(); @@ -707,6 +774,15 @@ impl MergeEngine { analyzed.point_values = points.remove(&doc_id).unwrap_or_default(); for (field_name, value) in &stored.fields { + if InvertedIndexWriter::is_doc_values_candidate(value) { + // Only fills a gap the schema-derived seed left open + // (#1047) -- a field `doc_values_by_field` already + // covers (the current schema decided it) is never + // touched here. + doc_values_by_field + .entry(field_name.clone()) + .or_insert_with(|| reader.has_doc_values(field_name)); + } analyzed .stored_fields .insert(field_name.clone(), value.clone()); @@ -754,6 +830,7 @@ impl MergeEngine { reader: &SegmentReader, deleted: &RoaringTreemap, positions_by_field: &mut HashMap, + doc_values_by_field: &mut HashMap, target_field: &str, analyzer: Option<&Arc>, ) -> Result> { @@ -851,6 +928,17 @@ impl MergeEngine { analyzed.point_values = points.remove(&doc_id).unwrap_or_default(); for (field_name, value) in &stored.fields { + if InvertedIndexWriter::is_doc_values_candidate(value) { + // `target_field` is pre-seeded by + // `rebuild_field_across_segments` with the field's NEW + // `doc_values` setting, so this never overrides it + // with the stale on-disk state (#1047, mirrors + // `positions_by_field`'s identical pre-seeding for + // `target_term_vectors`). + doc_values_by_field + .entry(field_name.clone()) + .or_insert_with(|| reader.has_doc_values(field_name)); + } analyzed .stored_fields .insert(field_name.clone(), value.clone()); @@ -1320,4 +1408,228 @@ mod tests { run("a_vec", "b_novec"); run("a_novec", "b_vec"); } + + /// #1047: mirrors [`merge_preserves_per_field_term_vectors_independently`] + /// for DocValues -- after merging two segments, each field must keep its + /// OWN `doc_values` state independently, detected per field from what + /// each source segment actually has on disk (`MergeConfig::default()`, + /// no schema override). Run with both field-name orderings. + #[test] + fn merge_preserves_per_field_doc_values_independently() { + use crate::lexical::core::field::{FieldOption, TextOption}; + + let run = |dv_field: &str, nodv_field: &str| { + let storage: Arc = + Arc::new(MemoryStorage::new(MemoryStorageConfig::default())); + + let mut fields = std::collections::HashMap::new(); + fields.insert( + dv_field.to_string(), + FieldOption::Text(TextOption { + doc_values: true, + ..Default::default() + }), + ); + fields.insert( + nodv_field.to_string(), + FieldOption::Text(TextOption { + doc_values: false, + ..Default::default() + }), + ); + let config = InvertedIndexWriterConfig { + fields, + ..Default::default() + }; + + let mut writer = InvertedIndexWriter::new(storage.clone(), config).unwrap(); + let d0 = writer + .add_document( + Document::builder() + .add_field(dv_field, DataValue::Text("alpha".to_string())) + .add_field(nodv_field, DataValue::Text("alpha".to_string())) + .build(), + ) + .unwrap(); + writer.commit().unwrap(); // segment_000000 + let d1 = writer + .add_document( + Document::builder() + .add_field(dv_field, DataValue::Text("bravo".to_string())) + .add_field(nodv_field, DataValue::Text("bravo".to_string())) + .build(), + ) + .unwrap(); + writer.commit().unwrap(); // segment_000001 + drop(writer); + + let si0 = segment_info("segment_000000", 1, d0, d0, 0); + let si1 = segment_info("segment_000001", 1, d1, d1, 1); + let candidate = MergeCandidate { + segments: vec![si0.segment_id.clone(), si1.segment_id.clone()], + priority: 1.0, + estimated_size: 0, + strategy: MergeStrategy::SizeBased, + }; + let engine = MergeEngine::new(MergeConfig::default(), storage.clone()); + let result = engine + .merge_segments( + &candidate, + &[ManagedSegmentInfo::new(si0), ManagedSegmentInfo::new(si1)], + 1, + ) + .unwrap(); + + let merged = + SegmentReader::open(result.new_segment.segment_info.clone(), storage.clone()) + .unwrap(); + assert!( + merged.has_doc_values(dv_field), + "{dv_field} must keep its DocValues column after merge" + ); + assert!( + !merged.has_doc_values(nodv_field), + "{nodv_field} must not have a DocValues column after merge" + ); + }; + + // Both orderings, so the result cannot depend on field-name sort order. + run("a_dv", "b_nodv"); + run("a_nodv", "b_dv"); + } + + /// #1047: the merge's CURRENT schema (`MergeConfig::field_doc_values`) + /// must win over a source segment's stale on-disk DocValues state -- + /// the deliberate deviation from `term_vectors`' detection-only + /// resolution (Issue #1047's design rationale: a DocValues column can + /// always be regenerated from `stored_fields`, so there is no harm in + /// re-deriving it from the current schema on every merge). + #[test] + fn merge_resolves_doc_values_from_the_current_schema_over_stale_segments() { + let storage: Arc = + Arc::new(MemoryStorage::new(MemoryStorageConfig::default())); + + // Written with a bare (pre-flag-change) writer, so "notes" gets a + // DocValues column the ordinary way (doc_values defaults to true). + let mut writer = + InvertedIndexWriter::new(storage.clone(), InvertedIndexWriterConfig::default()) + .unwrap(); + let d0 = writer + .add_document( + Document::builder() + .add_field("title", DataValue::Text("alpha".to_string())) + .add_field("notes", DataValue::Text("legacy note".to_string())) + .build(), + ) + .unwrap(); + writer.commit().unwrap(); // segment_000000 + drop(writer); + + let si0 = segment_info("segment_000000", 1, d0, d0, 0); + assert!( + SegmentReader::open(si0.clone(), storage.clone()) + .unwrap() + .has_doc_values("notes"), + "sanity: the source segment must have the column before the merge" + ); + + // The schema has SINCE changed "notes" to `doc_values: false`. Even + // though the only source segment still has the column on disk, the + // merge must honor the new schema. + let mut field_doc_values = std::collections::HashMap::new(); + field_doc_values.insert("notes".to_string(), false); + let config = MergeConfig { + field_doc_values, + ..MergeConfig::default() + }; + let candidate = MergeCandidate { + segments: vec![si0.segment_id.clone()], + priority: 1.0, + estimated_size: 0, + strategy: MergeStrategy::SizeBased, + }; + let engine = MergeEngine::new(config, storage.clone()); + let result = engine + .merge_segments(&candidate, &[ManagedSegmentInfo::new(si0)], 1) + .unwrap(); + + let merged = + SegmentReader::open(result.new_segment.segment_info.clone(), storage.clone()).unwrap(); + assert!( + !merged.has_doc_values("notes"), + "the current schema's doc_values: false must win over the \ + source segment's on-disk column" + ); + assert!( + merged.has_doc_values("title"), + "a field the schema doesn't mention still detects from the source segment" + ); + } + + /// #1047: a source segment with no value at all for some field must + /// not be misread as that field having opted out of DocValues -- + /// another segment's real column, and its values, must still survive + /// the merge intact. Deliberately processes the value-less segment + /// FIRST so a naive "first segment seen decides" bug (as opposed to + /// "first segment that actually HAS the field decides") would be + /// caught. + #[test] + fn merge_does_not_misdetect_a_valueless_field_as_opted_out() { + let storage: Arc = + Arc::new(MemoryStorage::new(MemoryStorageConfig::default())); + + let mut writer = + InvertedIndexWriter::new(storage.clone(), InvertedIndexWriterConfig::default()) + .unwrap(); + // segment_000000: no "extra" field at all. + let d0 = writer + .add_document( + Document::builder() + .add_field("title", DataValue::Text("alpha".to_string())) + .build(), + ) + .unwrap(); + writer.commit().unwrap(); + // segment_000001: "extra" is present, with its own real column. + let d1 = writer + .add_document( + Document::builder() + .add_field("title", DataValue::Text("bravo".to_string())) + .add_field("extra", DataValue::Text("present".to_string())) + .build(), + ) + .unwrap(); + writer.commit().unwrap(); + drop(writer); + + let si0 = segment_info("segment_000000", 1, d0, d0, 0); + let si1 = segment_info("segment_000001", 1, d1, d1, 1); + let candidate = MergeCandidate { + segments: vec![si0.segment_id.clone(), si1.segment_id.clone()], + priority: 1.0, + estimated_size: 0, + strategy: MergeStrategy::SizeBased, + }; + let engine = MergeEngine::new(MergeConfig::default(), storage.clone()); + let result = engine + .merge_segments( + &candidate, + &[ManagedSegmentInfo::new(si0), ManagedSegmentInfo::new(si1)], + 1, + ) + .unwrap(); + + let merged = + SegmentReader::open(result.new_segment.segment_info.clone(), storage.clone()).unwrap(); + assert!( + merged.has_doc_values("extra"), + "segment_000000 having no value for \"extra\" at all must not be \ + misread as an opt-out; segment_000001's real column must still win" + ); + assert_eq!( + merged.get_doc_value("extra", d1).unwrap(), + Some(DataValue::Text("present".to_string())), + "the merged value itself must be intact" + ); + } } diff --git a/laurus/src/lexical/index/inverted/writer.rs b/laurus/src/lexical/index/inverted/writer.rs index 265ecbd2..13265eda 100644 --- a/laurus/src/lexical/index/inverted/writer.rs +++ b/laurus/src/lexical/index/inverted/writer.rs @@ -58,6 +58,10 @@ pub struct InvertedIndexWriterConfig { /// Whether to store term positions for phrase queries. pub store_term_positions: bool, + /// Index-wide default for whether a field's value is also copied into + /// DocValues (Issue #1047). See [`Self::stores_doc_values`]. + pub store_doc_values: bool, + /// Whether to optimize segments after writing. pub optimize_segments: bool, @@ -79,6 +83,16 @@ pub struct InvertedIndexWriterConfig { /// document-indexing path, where [`Self::stores_term_positions`] /// instead resolves from `fields` and `store_term_positions`. pub field_term_positions: HashMap, + + /// Per-field override for whether to write a DocValues column, keyed + /// by field name. + /// + /// Set only when replaying documents during a segment merge or a + /// field rebuild, to reproduce the DocValues state each field already + /// had on disk regardless of the current schema. Empty in the normal + /// document-indexing path, where [`Self::stores_doc_values`] instead + /// resolves from `fields` and `store_doc_values`. + pub field_doc_values: HashMap, } impl std::fmt::Debug for InvertedIndexWriterConfig { @@ -88,6 +102,7 @@ impl std::fmt::Debug for InvertedIndexWriterConfig { .field("max_buffer_memory", &self.max_buffer_memory) .field("segment_prefix", &self.segment_prefix) .field("store_term_positions", &self.store_term_positions) + .field("store_doc_values", &self.store_doc_values) .field("optimize_segments", &self.optimize_segments) .field("analyzer", &self.analyzer.name()) .finish() @@ -102,11 +117,13 @@ impl Default for InvertedIndexWriterConfig { max_buffer_memory: 64 * 1024 * 1024, // 64MB segment_prefix: "segment".to_string(), store_term_positions: true, + store_doc_values: true, optimize_segments: false, analyzer: Arc::new(StandardAnalyzer::new().unwrap()), shard_id: 0, fields: HashMap::new(), field_term_positions: HashMap::new(), + field_doc_values: HashMap::new(), } } } @@ -132,6 +149,34 @@ impl InvertedIndexWriterConfig { } self.store_term_positions } + + /// Resolves whether `field_name`'s value should also be written to + /// DocValues. + /// + /// Resolution order (most specific wins): + /// 1. `field_doc_values[field_name]` — set only when replaying + /// documents during a segment merge or field rebuild, to preserve + /// each field's on-disk DocValues state. + /// 2. `fields[field_name]`'s own `doc_values` setting — every field + /// option except `BytesOption` carries one. + /// 3. `store_doc_values` — the index-wide default, used for + /// schema-less fields, reserved fields (names starting with `_`), + /// and any field declared `Bytes` (moot in practice: a `Bytes` + /// value is excluded from DocValues by [`Self`]'s caller + /// regardless of this result, via `is_doc_values_candidate`). + /// + /// Note this only decides *whether* a candidate value is written -- + /// [`InvertedIndexWriter::is_doc_values_candidate`] separately + /// excludes `Bytes`/`Vector` values by type unconditionally. + pub(crate) fn stores_doc_values(&self, field_name: &str) -> bool { + if let Some(&override_value) = self.field_doc_values.get(field_name) { + return override_value; + } + self.fields + .get(field_name) + .and_then(FieldOption::doc_values) + .unwrap_or(self.store_doc_values) + } } /// Statistics about the writing process. @@ -697,9 +742,10 @@ impl InvertedIndexWriter { } // Add field values to DocValues, skipping payloads no consumer of - // DocValues can use (#1047). + // DocValues can use (#1047) and fields opted out of DocValues via + // `doc_values: false`. for (field_name, value) in &analyzed_doc.stored_fields { - if Self::is_doc_values_candidate(value) { + if Self::is_doc_values_candidate(value) && self.config.stores_doc_values(field_name) { self.doc_values_writer .add_value(doc_id, field_name, value.clone()); } @@ -999,7 +1045,7 @@ impl InvertedIndexWriter { /// /// [`sort_type_rank`]: crate::lexical::query::collector /// [`TopFieldCollector::get_field_value`]: crate::lexical::query::collector::TopFieldCollector - fn is_doc_values_candidate(value: &crate::data::DataValue) -> bool { + pub(crate) fn is_doc_values_candidate(value: &crate::data::DataValue) -> bool { !matches!( value, crate::data::DataValue::Bytes(_, _) | crate::data::DataValue::Vector(_) @@ -2024,7 +2070,8 @@ impl InvertedIndexWriter { // Re-add stored fields to DocValues, under the same filter as // the ingest path (#1047) so a rebuild cannot reintroduce them. for (field_name, value) in &analyzed_doc.stored_fields { - if Self::is_doc_values_candidate(value) { + if Self::is_doc_values_candidate(value) && self.config.stores_doc_values(field_name) + { self.doc_values_writer .add_value(id, field_name, value.clone()); } @@ -2708,6 +2755,163 @@ mod tests { assert!(config.stores_term_positions("schemaless_field")); } + /// #1047: `stores_doc_values` must resolve `field_doc_values` before + /// the field's own schema setting, and the field's schema setting + /// before the index-wide default -- for every field option type that + /// carries `doc_values`, not just `Text`. + #[test] + fn stores_doc_values_resolution_order() { + use crate::lexical::core::field::{FieldOption, IntegerOption, TextOption}; + + let mut config = InvertedIndexWriterConfig { + store_doc_values: false, + ..Default::default() + }; + config.fields.insert( + "with_dv".to_string(), + FieldOption::Text(TextOption { + doc_values: true, + ..Default::default() + }), + ); + config.fields.insert( + "without_dv".to_string(), + FieldOption::Text(TextOption { + doc_values: false, + ..Default::default() + }), + ); + config.fields.insert( + "integer_without_dv".to_string(), + FieldOption::Integer(IntegerOption { + doc_values: false, + ..Default::default() + }), + ); + + // Level 3: no field entry, no override -> index-wide default. + assert!(!config.stores_doc_values("schemaless_field")); + + // Level 2: the field's own `doc_values` setting overrides the + // index-wide default in both directions, for Text and non-Text + // field options alike. + assert!(config.stores_doc_values("with_dv")); + assert!(!config.stores_doc_values("without_dv")); + assert!(!config.stores_doc_values("integer_without_dv")); + + // Level 1: `field_doc_values` overrides everything, including a + // field with an opposite schema setting. + config.field_doc_values.insert("with_dv".to_string(), false); + config + .field_doc_values + .insert("schemaless_field".to_string(), true); + assert!(!config.stores_doc_values("with_dv")); + assert!(config.stores_doc_values("schemaless_field")); + } + + /// #1047: a `Text` field declared `doc_values: false` must not get a + /// DocValues column, even though its value is a perfectly ordinary + /// sortable type (unlike the type-based exclusion `Bytes`/`Vector` + /// values already get). Asserted on the serialized payload, mirroring + /// [`binary_payloads_are_kept_out_of_doc_values`]. + #[test] + fn doc_values_false_excludes_a_sortable_field() { + use crate::lexical::core::field::{FieldOption, TextOption}; + + let storage = Arc::new(crate::storage::memory::MemoryStorage::new( + crate::storage::memory::MemoryStorageConfig::default(), + )); + let mut config = InvertedIndexWriterConfig::default(); + // Declaring any field switches `analyze_document` out of + // schema-less mode (undeclared, non-`_`-prefixed fields are then + // skipped entirely) -- so "title" must be declared too, with the + // default `doc_values: true`, to stay a fair comparison. + config.fields.insert( + "title".to_string(), + FieldOption::Text(TextOption::default()), + ); + config.fields.insert( + "internal_note".to_string(), + FieldOption::Text(TextOption { + doc_values: false, + ..Default::default() + }), + ); + let mut writer = InvertedIndexWriter::new(storage, config).unwrap(); + + let doc = Document::builder() + .add_field("title", crate::data::DataValue::Text("sortable".into())) + .add_field( + "internal_note", + crate::data::DataValue::Text("opted out via schema".into()), + ) + .build(); + writer.add_document(doc).unwrap(); + + let mut serialized: Vec = Vec::new(); + writer + .doc_values_writer + .write_to_output(&mut serialized) + .unwrap(); + let names = String::from_utf8_lossy(&serialized).to_string(); + + assert!( + names.contains("title"), + "a field without doc_values: false must still get a column" + ); + assert!( + !names.contains("internal_note"), + "doc_values: false must keep the field out of DocValues even \ + though its value type (Text) is otherwise a DocValues candidate" + ); + } + + /// #1047: `doc_values: false` on a large field must measurably shrink + /// the `.dv` payload, mirroring + /// [`doc_values_payload_does_not_scale_with_binary_fields`] but for + /// the schema flag rather than a type-based exclusion. + #[test] + fn doc_values_false_shrinks_the_dv_payload_for_a_large_field() { + use crate::lexical::core::field::{FieldOption, TextOption}; + + let build = |doc_values: bool| -> usize { + let storage = Arc::new(crate::storage::memory::MemoryStorage::new( + crate::storage::memory::MemoryStorageConfig::default(), + )); + let mut config = InvertedIndexWriterConfig::default(); + config.fields.insert( + "title".to_string(), + FieldOption::Text(TextOption::default()), + ); + config.fields.insert( + "notes".to_string(), + FieldOption::Text(TextOption { + doc_values, + ..Default::default() + }), + ); + let mut writer = InvertedIndexWriter::new(storage, config).unwrap(); + for i in 0..20u64 { + let doc = Document::builder() + .add_field("title", crate::data::DataValue::Text(format!("doc {i}"))) + .add_field("notes", crate::data::DataValue::Text("x".repeat(4096))) + .build(); + writer.upsert_document(i, doc).unwrap(); + } + let mut out: Vec = Vec::new(); + writer.doc_values_writer.write_to_output(&mut out).unwrap(); + out.len() + }; + + let with_dv = build(true); + let without_dv = build(false); + assert!( + without_dv < with_dv, + "doc_values: false on a large field must shrink the .dv payload: \ + {without_dv} (false) vs {with_dv} (true)" + ); + } + /// #1083: repeated occurrences of the same term in one document must /// aggregate into a single posting whose frequency equals the actual /// occurrence count, in both the positions-enabled and diff --git a/laurus/src/lexical/store/config.rs b/laurus/src/lexical/store/config.rs index ddd0b5c3..713f7b8d 100644 --- a/laurus/src/lexical/store/config.rs +++ b/laurus/src/lexical/store/config.rs @@ -135,6 +135,7 @@ pub struct LexicalIndexConfigBuilder { write_buffer_size: Option, compress_stored_fields: Option, store_term_vectors: Option, + store_doc_values: Option, merge_factor: Option, max_segments: Option, default_fields: Vec, @@ -160,6 +161,7 @@ impl LexicalIndexConfigBuilder { write_buffer_size: None, compress_stored_fields: None, store_term_vectors: None, + store_doc_values: None, merge_factor: None, max_segments: None, default_fields: Vec::new(), @@ -240,6 +242,18 @@ impl LexicalIndexConfigBuilder { self } + /// Enable or disable the index-wide default for writing a field's + /// value into DocValues (Issue #1047). + /// + /// A field's own `doc_values` option (present on every field option + /// except `BytesOption`) overrides this default. See + /// [`InvertedIndexConfig::store_doc_values`]. + /// Default: true + pub fn store_doc_values(mut self, store: bool) -> Self { + self.store_doc_values = Some(store); + self + } + /// Set the merge factor for segment merging. /// /// Controls how many segments are merged at once. Higher values reduce @@ -333,6 +347,9 @@ impl LexicalIndexConfigBuilder { if let Some(store) = self.store_term_vectors { config.store_term_vectors = store; } + if let Some(store) = self.store_doc_values { + config.store_doc_values = store; + } if let Some(factor) = self.merge_factor { config.merge_factor = factor; } From 4f8deebe19ee5e8b2a80b864fa13d429251bf3d1 Mon Sep 17 00:00:00 2001 From: Minoru Osuka Date: Sat, 12 Sep 2026 08:49:52 -0400 Subject: [PATCH 06/10] feat(engine): classify doc_values changes in update_field Adds classify_doc_values(old, new, old_stored), folded via .max(...) into every lexical arm of classify_change except Bytes (which has no doc_values flag). Reuses classify_indexed_only's exact logic: turning DocValues on needs the column regenerated from stored_fields (Reindex when old_stored, Destructive otherwise, mirroring #1083's term_vectors); turning it off is MetadataOnly, since InvertedIndexWriterConfig::stores_doc_values gates every future write and a stale on-disk column is simply never read once the schema says false. Engine::update_field itself needs no changes: its Reindex/Destructive dispatch already calls InvertedIndex::rebuild_field generically, which was updated in the previous commit to derive target_doc_values from the field's new option. Extends classify_change_table with doc_values cases for Text (both directions, plus the stored:false destructive variant) and one representative non-Text case each for Integer and Boolean, proving the fold applies uniformly across arms. Refs #1047 --- laurus/src/engine/schema.rs | 94 ++++++++++++++++++++++++++++++------- 1 file changed, 78 insertions(+), 16 deletions(-) diff --git a/laurus/src/engine/schema.rs b/laurus/src/engine/schema.rs index f0b33474..1604340b 100644 --- a/laurus/src/engine/schema.rs +++ b/laurus/src/engine/schema.rs @@ -363,36 +363,46 @@ pub enum FieldChangeKind { /// stored as raw vectors; lexical fields honor `stored` independently). pub fn classify_change(old: &FieldOption, new: &FieldOption) -> FieldChangeKind { match (old, new) { - (FieldOption::Text(o), FieldOption::Text(n)) => classify_text(o, n), + (FieldOption::Text(o), FieldOption::Text(n)) => { + classify_text(o, n).max(classify_doc_values(o.doc_values, n.doc_values, o.stored)) + } (FieldOption::Integer(o), FieldOption::Integer(n)) => classify_numeric_lexical( o.indexed, n.indexed, o.stored, o.multi_valued, n.multi_valued, - ), + ) + .max(classify_doc_values(o.doc_values, n.doc_values, o.stored)), (FieldOption::Float(o), FieldOption::Float(n)) => classify_numeric_lexical( o.indexed, n.indexed, o.stored, o.multi_valued, n.multi_valued, - ), - (FieldOption::Boolean(o), FieldOption::Boolean(n)) => { - classify_indexed_only(o.indexed, n.indexed, o.stored) - } - (FieldOption::DateTime(o), FieldOption::DateTime(n)) => { - classify_indexed_only(o.indexed, n.indexed, o.stored) - } - (FieldOption::Geo(o), FieldOption::Geo(n)) => { - classify_indexed_only(o.indexed, n.indexed, o.stored) - } - (FieldOption::Geo3d(o), FieldOption::Geo3d(n)) => { - classify_indexed_only(o.indexed, n.indexed, o.stored) - } + ) + .max(classify_doc_values(o.doc_values, n.doc_values, o.stored)), + (FieldOption::Boolean(o), FieldOption::Boolean(n)) => classify_indexed_only( + o.indexed, n.indexed, o.stored, + ) + .max(classify_doc_values(o.doc_values, n.doc_values, o.stored)), + (FieldOption::DateTime(o), FieldOption::DateTime(n)) => classify_indexed_only( + o.indexed, n.indexed, o.stored, + ) + .max(classify_doc_values(o.doc_values, n.doc_values, o.stored)), + (FieldOption::Geo(o), FieldOption::Geo(n)) => classify_indexed_only( + o.indexed, n.indexed, o.stored, + ) + .max(classify_doc_values(o.doc_values, n.doc_values, o.stored)), + (FieldOption::Geo3d(o), FieldOption::Geo3d(n)) => classify_indexed_only( + o.indexed, n.indexed, o.stored, + ) + .max(classify_doc_values(o.doc_values, n.doc_values, o.stored)), // `stored` changes (for every lexical variant, including Bytes) are // always metadata-only: they only affect documents ingested after - // the change, never data already on disk. + // the change, never data already on disk. `Bytes` has no + // `doc_values` flag (Issue #1047: is_doc_values_candidate excludes + // it unconditionally), so there is no fold here. (FieldOption::Bytes(_), FieldOption::Bytes(_)) => FieldChangeKind::MetadataOnly, (FieldOption::Hnsw(o), FieldOption::Hnsw(n)) => { @@ -449,6 +459,28 @@ fn classify_indexed_only( } } +/// Classification for a `doc_values` change (Issue #1047), shared by +/// every lexical field option except `Bytes` (which has no such flag). +/// +/// Identical reasoning to [`classify_indexed_only`], so it delegates to +/// the same function: turning DocValues ON needs the column regenerated +/// from the field's original values (existing segments simply have no +/// column to read), so `false -> true` is `Reindex` when `old_stored` +/// (the merge/field-rebuild path detects the gap and regenerates the +/// column from `stored_fields`, Issue #1047 Phase 4) or `Destructive` +/// otherwise. `true -> false` is `MetadataOnly`: an existing column is +/// simply never consulted once every reader treats the field as +/// `doc_values: false` (`InvertedIndexWriterConfig::stores_doc_values` +/// gates every future write), so there is no correctness risk in leaving +/// stale columns on disk until the next merge reclaims the space. +fn classify_doc_values( + old_doc_values: bool, + new_doc_values: bool, + old_stored: bool, +) -> FieldChangeKind { + classify_indexed_only(old_doc_values, new_doc_values, old_stored) +} + /// Classification for `TextOption`: `indexed` follows /// [`classify_indexed_only`]; an `analyzer` change always requires /// rebuilding from the field's original values, since existing postings @@ -964,6 +996,24 @@ mod tests { text(|o| o.stored(false).term_vectors(true)), Destructive, ), + ( + "text: doc_values false->true requires reindex", + text(|o| o.doc_values(false)), + text(|o| o.doc_values(true)), + Reindex, + ), + ( + "text: doc_values true->false is metadata-only", + text(|o| o.doc_values(true)), + text(|o| o.doc_values(false)), + MetadataOnly, + ), + ( + "text: doc_values false->true on a stored:false field is destructive (no original value to regenerate the column from)", + text(|o| o.stored(false).doc_values(false)), + text(|o| o.stored(false).doc_values(true)), + Destructive, + ), // ---- Integer ---- ( "integer: stored toggle is metadata-only", @@ -1011,6 +1061,12 @@ mod tests { integer(|o| o.stored(false)), Reindex, ), + ( + "integer: doc_values false->true requires reindex", + integer(|o| o.doc_values(false)), + integer(|o| o.doc_values(true)), + Reindex, + ), // ---- Float ---- ( "float: stored toggle is metadata-only", @@ -1046,6 +1102,12 @@ mod tests { boolean(|o| o.indexed(true)), Reindex, ), + ( + "boolean: doc_values true->false is metadata-only", + boolean(|o| o.doc_values(true)), + boolean(|o| o.doc_values(false)), + MetadataOnly, + ), // ---- DateTime ---- ( "datetime: stored toggle is metadata-only", From 2f63629afbf5c2955a3fe56ffb982dc2b47d5710 Mon Sep 17 00:00:00 2001 From: Minoru Osuka Date: Sat, 12 Sep 2026 09:50:09 -0400 Subject: [PATCH 07/10] feat(server): expose doc_values over gRPC and the HTTP gateway Adds optional bool doc_values to the 7 proto field-option messages that carry the flag (not BytesOption, which has none). Tri-state for the same reason as term_vectors (#1083): a plain bool cannot distinguish "the client omitted this" from "the client explicitly set it to false", and the engine default is true. - convert/schema.rs: field_option_to_proto / field_option_from_proto carry doc_values through both directions, unset defaulting to true. - gateway/convert.rs: json_to_proto_field_option reads an optional doc_values key; proto_field_option_to_json's six non-Text arms (previously fixed-shape json! literals) are rewritten to conditionally-mutated objects, matching the Text arm's existing pattern, so doc_values is only emitted when explicitly set. Tests: proto round-trip (explicit true/false, and unset defaulting to true) for Text and Integer; JSON round-trip with the same tri-state contract, mirroring the existing base_weight tests. Refs #1047 --- laurus-server/proto/laurus/v1/index.proto | 29 ++++++ laurus-server/src/convert/schema.rs | 105 +++++++++++++++++-- laurus-server/src/gateway/convert.rs | 121 ++++++++++++++++++---- 3 files changed, 229 insertions(+), 26 deletions(-) diff --git a/laurus-server/proto/laurus/v1/index.proto b/laurus-server/proto/laurus/v1/index.proto index 66ebc827..050a5813 100644 --- a/laurus-server/proto/laurus/v1/index.proto +++ b/laurus-server/proto/laurus/v1/index.proto @@ -114,6 +114,11 @@ message TextOption { optional bool term_vectors = 3; // Analyzer reference. Unset means use the engine default analyzer. AnalyzerSpec analyzer = 4; + // Whether this field's value is also copied into DocValues (the + // column-oriented store sort/facet/aggregation read from). Unset means + // "use the engine's default" (currently `true`), for the same reason + // as `term_vectors` above. + optional bool doc_values = 5; } // Reference to an analyzer for a text field. @@ -157,6 +162,10 @@ message IntegerOption { // When true, the field accepts arrays of integers and range queries // match a document if any value satisfies the predicate (Lucene-style). bool multi_valued = 3; + // Whether this field's value is also copied into DocValues. Unset + // means "use the engine's default" (currently `true`); see + // `TextOption.doc_values`. + optional bool doc_values = 4; } message FloatOption { @@ -165,21 +174,37 @@ message FloatOption { // When true, the field accepts arrays of floats and range queries // match a document if any value satisfies the predicate (Lucene-style). bool multi_valued = 3; + // Whether this field's value is also copied into DocValues. Unset + // means "use the engine's default" (currently `true`); see + // `TextOption.doc_values`. + optional bool doc_values = 4; } message BooleanOption { bool indexed = 1; bool stored = 2; + // Whether this field's value is also copied into DocValues. Unset + // means "use the engine's default" (currently `true`); see + // `TextOption.doc_values`. + optional bool doc_values = 3; } message DateTimeOption { bool indexed = 1; bool stored = 2; + // Whether this field's value is also copied into DocValues. Unset + // means "use the engine's default" (currently `true`); see + // `TextOption.doc_values`. + optional bool doc_values = 3; } message GeoOption { bool indexed = 1; bool stored = 2; + // Whether this field's value is also copied into DocValues. Unset + // means "use the engine's default" (currently `true`); see + // `TextOption.doc_values`. + optional bool doc_values = 3; } // Options for 3D ECEF (Earth-Centered Earth-Fixed) geographic point fields. @@ -188,6 +213,10 @@ message GeoOption { message Geo3dOption { bool indexed = 1; bool stored = 2; + // Whether this field's value is also copied into DocValues. Unset + // means "use the engine's default" (currently `true`); see + // `TextOption.doc_values`. + optional bool doc_values = 3; } message BytesOption { diff --git a/laurus-server/src/convert/schema.rs b/laurus-server/src/convert/schema.rs index 06dff058..b56fe44f 100644 --- a/laurus-server/src/convert/schema.rs +++ b/laurus-server/src/convert/schema.rs @@ -127,32 +127,39 @@ pub fn field_option_to_proto(fo: &FieldOption) -> v1::FieldOption { stored: o.stored, term_vectors: Some(o.term_vectors), analyzer: o.analyzer.as_ref().map(analyzer_spec_to_proto), + doc_values: Some(o.doc_values), })), FieldOption::Integer(o) => Some(Opt::Integer(v1::IntegerOption { indexed: o.indexed, stored: o.stored, multi_valued: o.multi_valued, + doc_values: Some(o.doc_values), })), FieldOption::Float(o) => Some(Opt::Float(v1::FloatOption { indexed: o.indexed, stored: o.stored, multi_valued: o.multi_valued, + doc_values: Some(o.doc_values), })), FieldOption::Boolean(o) => Some(Opt::Boolean(v1::BooleanOption { indexed: o.indexed, stored: o.stored, + doc_values: Some(o.doc_values), })), FieldOption::DateTime(o) => Some(Opt::DateTime(v1::DateTimeOption { indexed: o.indexed, stored: o.stored, + doc_values: Some(o.doc_values), })), FieldOption::Geo(o) => Some(Opt::Geo(v1::GeoOption { indexed: o.indexed, stored: o.stored, + doc_values: Some(o.doc_values), })), FieldOption::Geo3d(o) => Some(Opt::Geo3d(v1::Geo3dOption { indexed: o.indexed, stored: o.stored, + doc_values: Some(o.doc_values), })), FieldOption::Bytes(o) => Some(Opt::Bytes(v1::BytesOption { stored: o.stored })), FieldOption::Hnsw(o) => Some(Opt::Hnsw(v1::HnswOption { @@ -205,37 +212,36 @@ pub fn field_option_from_proto(fo: &v1::FieldOption) -> Option { // Unset means "use the engine's default", matching // `TextOption::default()` (#1083). term_vectors: o.term_vectors.unwrap_or(true), - // The proto message has no `doc_values` field yet; pin to the - // pre-#1047 implicit default until the protocol is wired up. - doc_values: true, + // Same tri-state treatment for the same reason (#1047). + doc_values: o.doc_values.unwrap_or(true), analyzer: o.analyzer.as_ref().and_then(analyzer_spec_from_proto), })), Some(Opt::Integer(o)) => Some(FieldOption::Integer(IntegerOption { indexed: o.indexed, stored: o.stored, multi_valued: o.multi_valued, - doc_values: true, + doc_values: o.doc_values.unwrap_or(true), })), Some(Opt::Float(o)) => Some(FieldOption::Float(FloatOption { indexed: o.indexed, stored: o.stored, multi_valued: o.multi_valued, - doc_values: true, + doc_values: o.doc_values.unwrap_or(true), })), Some(Opt::Boolean(o)) => Some(FieldOption::Boolean(BooleanOption { indexed: o.indexed, stored: o.stored, - doc_values: true, + doc_values: o.doc_values.unwrap_or(true), })), Some(Opt::DateTime(o)) => Some(FieldOption::DateTime(DateTimeOption { indexed: o.indexed, stored: o.stored, - doc_values: true, + doc_values: o.doc_values.unwrap_or(true), })), Some(Opt::Geo(o)) => Some(FieldOption::Geo(GeoOption { indexed: o.indexed, stored: o.stored, - doc_values: true, + doc_values: o.doc_values.unwrap_or(true), })), Some(Opt::Geo3d(o)) => Some(FieldOption::Geo3d(Geo3dOption { indexed: o.indexed, @@ -1202,4 +1208,87 @@ mod tests { other => panic!("expected FieldOption::Geo3d, got {other:?}"), } } + + /// #1047: `doc_values` follows the same tri-state contract as + /// `term_vectors` (#1083) -- `to_proto` carries an explicit setting as + /// `Some`, and `from_proto` restores an unset value to the engine's + /// default (`true`), not the proto3 zero-value (`false`). Covers + /// `Text` (which also carries `term_vectors`, to prove the two flags + /// don't interfere) and `Integer` (a non-`Text` arm). + #[test] + fn doc_values_round_trips_through_proto_and_unset_defaults_to_true() { + let schema = Schema::builder() + .add_field( + "text_explicit_false", + FieldOption::Text(TextOption { + doc_values: false, + ..Default::default() + }), + ) + .add_field( + "integer_explicit_false", + FieldOption::Integer(IntegerOption { + doc_values: false, + ..Default::default() + }), + ) + .build(); + + let proto = to_proto(&schema); + for (name, opt_matcher) in [ + ( + "text_explicit_false", + &(|o: &v1::FieldOption| match o.option.as_ref() { + Some(v1::field_option::Option::Text(t)) => t.doc_values, + other => panic!("unexpected proto option: {other:?}"), + }) as &dyn Fn(&v1::FieldOption) -> Option, + ), + ( + "integer_explicit_false", + &(|o: &v1::FieldOption| match o.option.as_ref() { + Some(v1::field_option::Option::Integer(i)) => i.doc_values, + other => panic!("unexpected proto option: {other:?}"), + }), + ), + ] { + let doc_values = opt_matcher(proto.fields.get(name).expect("field must exist")); + assert_eq!( + doc_values, + Some(false), + "to_proto must carry an explicit doc_values: false as Some for {name}" + ); + } + + let back = from_proto(&proto).expect("from_proto must succeed"); + match back.fields.get("text_explicit_false") { + Some(FieldOption::Text(o)) => assert!(!o.doc_values), + other => panic!("expected FieldOption::Text, got {other:?}"), + } + match back.fields.get("integer_explicit_false") { + Some(FieldOption::Integer(o)) => assert!(!o.doc_values), + other => panic!("expected FieldOption::Integer, got {other:?}"), + } + + // A client omitting doc_values entirely (proto `None`) must + // restore to the engine default (`true`), not the proto3 + // zero-value (`false`). + let mut unset = proto.clone(); + if let Some(v1::field_option::Option::Text(t)) = unset + .fields + .get_mut("text_explicit_false") + .and_then(|f| f.option.as_mut()) + { + t.doc_values = None; + } else { + panic!("text_explicit_false must be a Text proto option"); + } + let back = from_proto(&unset).expect("from_proto must succeed"); + match back.fields.get("text_explicit_false") { + Some(FieldOption::Text(o)) => assert!( + o.doc_values, + "unset doc_values must default to true, not the proto3 zero-value" + ), + other => panic!("expected FieldOption::Text, got {other:?}"), + } + } } diff --git a/laurus-server/src/gateway/convert.rs b/laurus-server/src/gateway/convert.rs index 88b25730..cd756d8d 100644 --- a/laurus-server/src/gateway/convert.rs +++ b/laurus-server/src/gateway/convert.rs @@ -424,6 +424,8 @@ pub fn json_to_proto_field_option(json: &Value) -> Result Result Result Value { if let Some(term_vectors) = v.term_vectors { text_obj["term_vectors"] = json!(term_vectors); } + // Same tri-state treatment for the same reason (#1047). + if let Some(doc_values) = v.doc_values { + text_obj["doc_values"] = json!(doc_values); + } if let Some(spec) = v.analyzer.as_ref().and_then(analyzer_spec_to_json) { text_obj["analyzer"] = spec; } json!({ "text": text_obj }) } - Some(Opt::Integer(v)) => json!({ - "integer": { "indexed": v.indexed, "stored": v.stored } - }), - Some(Opt::Float(v)) => json!({ - "float": { "indexed": v.indexed, "stored": v.stored } - }), - Some(Opt::Boolean(v)) => json!({ - "boolean": { "indexed": v.indexed, "stored": v.stored } - }), - Some(Opt::DateTime(v)) => json!({ - "date_time": { "indexed": v.indexed, "stored": v.stored } - }), - Some(Opt::Geo(v)) => json!({ - "geo": { "indexed": v.indexed, "stored": v.stored } - }), - Some(Opt::Geo3d(v)) => json!({ - "geo3d": { "indexed": v.indexed, "stored": v.stored } - }), + Some(Opt::Integer(v)) => { + let mut obj = json!({ "indexed": v.indexed, "stored": v.stored }); + if let Some(doc_values) = v.doc_values { + obj["doc_values"] = json!(doc_values); + } + json!({ "integer": obj }) + } + Some(Opt::Float(v)) => { + let mut obj = json!({ "indexed": v.indexed, "stored": v.stored }); + if let Some(doc_values) = v.doc_values { + obj["doc_values"] = json!(doc_values); + } + json!({ "float": obj }) + } + Some(Opt::Boolean(v)) => { + let mut obj = json!({ "indexed": v.indexed, "stored": v.stored }); + if let Some(doc_values) = v.doc_values { + obj["doc_values"] = json!(doc_values); + } + json!({ "boolean": obj }) + } + Some(Opt::DateTime(v)) => { + let mut obj = json!({ "indexed": v.indexed, "stored": v.stored }); + if let Some(doc_values) = v.doc_values { + obj["doc_values"] = json!(doc_values); + } + json!({ "date_time": obj }) + } + Some(Opt::Geo(v)) => { + let mut obj = json!({ "indexed": v.indexed, "stored": v.stored }); + if let Some(doc_values) = v.doc_values { + obj["doc_values"] = json!(doc_values); + } + json!({ "geo": obj }) + } + Some(Opt::Geo3d(v)) => { + let mut obj = json!({ "indexed": v.indexed, "stored": v.stored }); + if let Some(doc_values) = v.doc_values { + obj["doc_values"] = json!(doc_values); + } + json!({ "geo3d": obj }) + } Some(Opt::Bytes(v)) => json!({ "bytes": { "stored": v.stored } }), @@ -1425,6 +1461,55 @@ mod tests { assert!(ivf_option_to_json(&ivf).get("base_weight").is_none()); } + /// #1047: `doc_values` follows the same tri-state contract as + /// `base_weight`/`term_vectors` above -- an absent key must round-trip + /// as absent (not a `false`/zero-value standing in for "unset"), for + /// both `Text` (which also carries `term_vectors`) and a non-`Text` + /// variant. + #[test] + fn test_doc_values_round_trips_through_json_and_absent_key_stays_absent() { + let with_dv = json!({ "text": { "indexed": true, "stored": true, "doc_values": false } }); + let proto = json_to_proto_field_option(&with_dv).unwrap(); + match &proto.option { + Some(v1::field_option::Option::Text(t)) => assert_eq!(t.doc_values, Some(false)), + other => panic!("expected Text option, got {other:?}"), + } + let back = proto_field_option_to_json(&proto); + assert_eq!( + back["text"].get("doc_values").and_then(|v| v.as_bool()), + Some(false) + ); + + let without_dv = json!({ "text": { "indexed": true, "stored": true } }); + let proto = json_to_proto_field_option(&without_dv).unwrap(); + match &proto.option { + Some(v1::field_option::Option::Text(t)) => assert_eq!(t.doc_values, None), + other => panic!("expected Text option, got {other:?}"), + } + let back = proto_field_option_to_json(&proto); + assert!( + back["text"].get("doc_values").is_none(), + "an unset doc_values must not emit a key: {back:?}" + ); + + // Non-Text variant: same contract. + let integer_dv = + json!({ "integer": { "indexed": true, "stored": true, "doc_values": true } }); + let proto = json_to_proto_field_option(&integer_dv).unwrap(); + match &proto.option { + Some(v1::field_option::Option::Integer(i)) => assert_eq!(i.doc_values, Some(true)), + other => panic!("expected Integer option, got {other:?}"), + } + let integer_unset = json!({ "integer": { "indexed": true, "stored": true } }); + let proto = json_to_proto_field_option(&integer_unset).unwrap(); + match &proto.option { + Some(v1::field_option::Option::Integer(i)) => assert_eq!(i.doc_values, None), + other => panic!("expected Integer option, got {other:?}"), + } + let back = proto_field_option_to_json(&proto); + assert!(back["integer"].get("doc_values").is_none()); + } + #[test] fn test_json_to_proto_search_request() { let json = json!({ From 876500aeb0339f9f87c232e7f6da2f1f480a2adb Mon Sep 17 00:00:00 2001 From: Minoru Osuka Date: Sat, 12 Sep 2026 10:12:50 -0400 Subject: [PATCH 08/10] feat(bindings,cli): expose doc_values in the five bindings and the CLI wizard Adds a doc_values parameter (default true) to the seven add_*_field schema-builder methods across laurus-python, laurus-ruby, laurus-nodejs, laurus-wasm, and laurus-php, mirroring each binding's existing term_vectors parameter: - Python: keyword-only pyo3 signature (doc_values=true), inserted before analyzer on add_text_field. - Node.js / WASM: positional Option, inserted before the trailing analyzer param on add_text_field (same precedent term_vectors already set there). - PHP: #[php(defaults(...))] entry plus a plain bool parameter. - Ruby: extended each add_*_field's get_kwargs type tuple and name list by one Option slot; add_text_field's tuple grows to 5, still well under magnus's 9-element ScanArgsOpt limit. add_text_field's extra parameter on Python pushed it to 8 arguments, tripping clippy::too_many_arguments; added the same #[allow(...)] the vector field builders already carry for the same reason. laurus-cli's create wizard gets a "Doc values?" Confirm prompt (default true) in both prompt_text_option and prompt_indexed_stored_option. Tests: new laurus-python/tests/test_doc_values.py and laurus-ruby/test/test_doc_values.rb -- doc_values: false keeps a stored field retrievable, doesn't affect searchability, and every field option that carries the flag accepts it. Neither binding exposes field-sorted search or faceting yet, so these can't observe the DocValues column itself disappearing on disk; that's covered at the Rust level (Phases 3-4). Full workspace check/clippy/fmt clean; laurus-python's 107 pytest cases and laurus-ruby's 81 minitest cases all pass. Refs #1047 --- laurus-cli/src/commands/create.rs | 25 ++++--- laurus-nodejs/src/schema.rs | 57 ++++++++++++--- laurus-php/src/schema.rs | 70 ++++++++++++------ laurus-python/src/schema.rs | 84 +++++++++++++++++----- laurus-python/tests/test_doc_values.py | 60 ++++++++++++++++ laurus-ruby/src/schema.rs | 98 +++++++++++++++++--------- laurus-ruby/test/test_doc_values.rb | 59 ++++++++++++++++ laurus-wasm/src/schema.rs | 63 ++++++++++++++--- 8 files changed, 416 insertions(+), 100 deletions(-) create mode 100644 laurus-python/tests/test_doc_values.py create mode 100644 laurus-ruby/test/test_doc_values.rb diff --git a/laurus-cli/src/commands/create.rs b/laurus-cli/src/commands/create.rs index 318e59dd..0f80518f 100644 --- a/laurus-cli/src/commands/create.rs +++ b/laurus-cli/src/commands/create.rs @@ -333,7 +333,7 @@ fn prompt_field_type_and_options() -> Result { } } -/// Prompt for TextOption (indexed, stored, term_vectors, analyzer). +/// Prompt for TextOption (indexed, stored, term_vectors, doc_values, analyzer). fn prompt_text_option() -> Result { let indexed = Confirm::new() .with_prompt("Indexed?") @@ -347,6 +347,10 @@ fn prompt_text_option() -> Result { .with_prompt("Term vectors?") .default(true) .interact()?; + let doc_values = Confirm::new() + .with_prompt("Doc values? (needed for sorting/faceting/aggregation)") + .default(true) + .interact()?; let analyzer_choices = [ "standard", "keyword", "english", "japanese", "simple", "noop", @@ -393,7 +397,7 @@ fn prompt_text_option() -> Result { indexed, stored, term_vectors, - doc_values: true, + doc_values, analyzer, })) } @@ -419,38 +423,43 @@ fn prompt_indexed_stored_option(type_name: &str) -> Result { false }; + let doc_values = Confirm::new() + .with_prompt("Doc values? (needed for sorting/faceting/aggregation)") + .default(true) + .interact()?; + Ok(match type_name { "Integer" => FieldOption::Integer(IntegerOption { indexed, stored, multi_valued, - doc_values: true, + doc_values, }), "Float" => FieldOption::Float(FloatOption { indexed, stored, multi_valued, - doc_values: true, + doc_values, }), "Boolean" => FieldOption::Boolean(BooleanOption { indexed, stored, - doc_values: true, + doc_values, }), "DateTime" => FieldOption::DateTime(DateTimeOption { indexed, stored, - doc_values: true, + doc_values, }), "Geo" => FieldOption::Geo(GeoOption { indexed, stored, - doc_values: true, + doc_values, }), "Geo3d" => FieldOption::Geo3d(Geo3dOption { indexed, stored, - doc_values: true, + doc_values, }), _ => unreachable!(), }) diff --git a/laurus-nodejs/src/schema.rs b/laurus-nodejs/src/schema.rs index a098dc44..c4593cae 100644 --- a/laurus-nodejs/src/schema.rs +++ b/laurus-nodejs/src/schema.rs @@ -120,6 +120,9 @@ impl JsSchema { /// * `indexed` - Whether the field is searchable (default `true`). /// * `term_vectors` - Whether term positions are stored, required by /// phrase and span queries over this field (default `true`). + /// * `docValues` - Whether the value is also copied into DocValues, + /// the column-oriented store sort/facet/aggregation read from + /// (default `true`). Takes effect only when `stored` is also `true`. /// * `analyzer` - Optional analyzer name. For parameter-less built-ins /// (`"standard"`, `"english"`, `"keyword"`, `"simple"`, `"noop"`) /// pass the name directly. For parameterized presets such as the @@ -133,6 +136,7 @@ impl JsSchema { stored: Option, indexed: Option, term_vectors: Option, + doc_values: Option, analyzer: Option, ) { self.inner.fields.insert( @@ -141,7 +145,7 @@ impl JsSchema { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), term_vectors: term_vectors.unwrap_or(true), - doc_values: true, + doc_values: doc_values.unwrap_or(true), analyzer: analyzer.map(laurus::AnalyzerSpec::Named), }), ); @@ -157,6 +161,8 @@ impl JsSchema { /// * `multi_valued` - When `true`, the field accepts arrays of integers /// and range queries match if any value satisfies the predicate /// (Lucene-style "any match"). Default `false`. + /// * `docValues` - Whether the value is also copied into DocValues + /// (default `true`). Takes effect only when `stored` is also `true`. #[napi] pub fn add_integer_field( &mut self, @@ -164,6 +170,7 @@ impl JsSchema { stored: Option, indexed: Option, multi_valued: Option, + doc_values: Option, ) { self.inner.fields.insert( name, @@ -171,7 +178,7 @@ impl JsSchema { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), multi_valued: multi_valued.unwrap_or(false), - doc_values: true, + doc_values: doc_values.unwrap_or(true), }), ); } @@ -186,6 +193,8 @@ impl JsSchema { /// * `multi_valued` - When `true`, the field accepts arrays of floats /// and range queries match if any value satisfies the predicate /// (Lucene-style "any match"). Default `false`. + /// * `docValues` - Whether the value is also copied into DocValues + /// (default `true`). Takes effect only when `stored` is also `true`. #[napi] pub fn add_float_field( &mut self, @@ -193,6 +202,7 @@ impl JsSchema { stored: Option, indexed: Option, multi_valued: Option, + doc_values: Option, ) { self.inner.fields.insert( name, @@ -200,7 +210,7 @@ impl JsSchema { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), multi_valued: multi_valued.unwrap_or(false), - doc_values: true, + doc_values: doc_values.unwrap_or(true), }), ); } @@ -212,14 +222,22 @@ impl JsSchema { /// * `name` - Field name. /// * `stored` - Whether the value is retrievable (default `true`). /// * `indexed` - Whether the field is searchable (default `true`). + /// * `docValues` - Whether the value is also copied into DocValues + /// (default `true`). Takes effect only when `stored` is also `true`. #[napi] - pub fn add_boolean_field(&mut self, name: String, stored: Option, indexed: Option) { + pub fn add_boolean_field( + &mut self, + name: String, + stored: Option, + indexed: Option, + doc_values: Option, + ) { self.inner.fields.insert( name, FieldOption::Boolean(BooleanOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), - doc_values: true, + doc_values: doc_values.unwrap_or(true), }), ); } @@ -231,19 +249,22 @@ impl JsSchema { /// * `name` - Field name. /// * `stored` - Whether the value is retrievable (default `true`). /// * `indexed` - Whether the field is searchable (default `true`). + /// * `docValues` - Whether the value is also copied into DocValues + /// (default `true`). Takes effect only when `stored` is also `true`. #[napi] pub fn add_datetime_field( &mut self, name: String, stored: Option, indexed: Option, + doc_values: Option, ) { self.inner.fields.insert( name, FieldOption::DateTime(DateTimeOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), - doc_values: true, + doc_values: doc_values.unwrap_or(true), }), ); } @@ -255,14 +276,22 @@ impl JsSchema { /// * `name` - Field name. /// * `stored` - Whether the value is retrievable (default `true`). /// * `indexed` - Whether the field is searchable (default `true`). + /// * `docValues` - Whether the value is also copied into DocValues + /// (default `true`). Takes effect only when `stored` is also `true`. #[napi] - pub fn add_geo_field(&mut self, name: String, stored: Option, indexed: Option) { + pub fn add_geo_field( + &mut self, + name: String, + stored: Option, + indexed: Option, + doc_values: Option, + ) { self.inner.fields.insert( name, FieldOption::Geo(GeoOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), - doc_values: true, + doc_values: doc_values.unwrap_or(true), }), ); } @@ -279,14 +308,22 @@ impl JsSchema { /// * `name` - Field name. /// * `stored` - Whether the value is retrievable (default `true`). /// * `indexed` - Whether the field is searchable (default `true`). + /// * `docValues` - Whether the value is also copied into DocValues + /// (default `true`). Takes effect only when `stored` is also `true`. #[napi(js_name = "addGeo3dField")] - pub fn add_geo3d_field(&mut self, name: String, stored: Option, indexed: Option) { + pub fn add_geo3d_field( + &mut self, + name: String, + stored: Option, + indexed: Option, + doc_values: Option, + ) { self.inner.fields.insert( name, FieldOption::Geo3d(Geo3dOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), - doc_values: true, + doc_values: doc_values.unwrap_or(true), }), ); } diff --git a/laurus-php/src/schema.rs b/laurus-php/src/schema.rs index 53d3cfa9..d92baf88 100644 --- a/laurus-php/src/schema.rs +++ b/laurus-php/src/schema.rs @@ -118,18 +118,22 @@ impl PhpSchema { /// * `indexed` - Whether the field is searchable (default: true). /// * `term_vectors` - Whether term positions are stored, required by /// phrase and span queries over this field (default: true). + /// * `doc_values` - Whether the value is also copied into DocValues, + /// the column-oriented store sort/facet/aggregation read from + /// (default: true). Takes effect only when `stored` is also true. /// * `analyzer` - Optional analyzer name. For parameter-less built-in /// analyzers (`"standard"`, `"english"`, `"keyword"`, `"simple"`, /// `"noop"`) pass the name directly. Parameterized presets such as /// the Japanese analyzer (which needs a Lindera dictionary path) /// should be registered via `addAnalyzer` and referenced by name. - #[php(defaults(stored = true, indexed = true, term_vectors = true))] + #[php(defaults(stored = true, indexed = true, term_vectors = true, doc_values = true))] pub fn add_text_field( &self, name: String, stored: bool, indexed: bool, term_vectors: bool, + doc_values: bool, analyzer: Option, ) { self.inner.borrow_mut().fields.insert( @@ -138,7 +142,7 @@ impl PhpSchema { indexed, stored, term_vectors, - doc_values: true, + doc_values, analyzer: analyzer.map(laurus::AnalyzerSpec::Named), }), ); @@ -154,15 +158,24 @@ impl PhpSchema { /// * `multi_valued` - When true, the field accepts arrays of integers /// and range queries match if any value satisfies the predicate /// (Lucene-style "any match"). Default: false. - #[php(defaults(stored = true, indexed = true, multi_valued = false))] - pub fn add_integer_field(&self, name: String, stored: bool, indexed: bool, multi_valued: bool) { + /// * `doc_values` - Whether the value is also copied into DocValues + /// (default: true). Takes effect only when `stored` is also true. + #[php(defaults(stored = true, indexed = true, multi_valued = false, doc_values = true))] + pub fn add_integer_field( + &self, + name: String, + stored: bool, + indexed: bool, + multi_valued: bool, + doc_values: bool, + ) { self.inner.borrow_mut().fields.insert( name, FieldOption::Integer(IntegerOption { indexed, stored, multi_valued, - doc_values: true, + doc_values, }), ); } @@ -177,15 +190,24 @@ impl PhpSchema { /// * `multi_valued` - When true, the field accepts arrays of floats /// and range queries match if any value satisfies the predicate /// (Lucene-style "any match"). Default: false. - #[php(defaults(stored = true, indexed = true, multi_valued = false))] - pub fn add_float_field(&self, name: String, stored: bool, indexed: bool, multi_valued: bool) { + /// * `doc_values` - Whether the value is also copied into DocValues + /// (default: true). Takes effect only when `stored` is also true. + #[php(defaults(stored = true, indexed = true, multi_valued = false, doc_values = true))] + pub fn add_float_field( + &self, + name: String, + stored: bool, + indexed: bool, + multi_valued: bool, + doc_values: bool, + ) { self.inner.borrow_mut().fields.insert( name, FieldOption::Float(FloatOption { indexed, stored, multi_valued, - doc_values: true, + doc_values, }), ); } @@ -197,14 +219,16 @@ impl PhpSchema { /// * `name` - Field name. /// * `stored` - Whether the value is retrievable (default: true). /// * `indexed` - Whether the field is searchable (default: true). - #[php(defaults(stored = true, indexed = true))] - pub fn add_boolean_field(&self, name: String, stored: bool, indexed: bool) { + /// * `doc_values` - Whether the value is also copied into DocValues + /// (default: true). Takes effect only when `stored` is also true. + #[php(defaults(stored = true, indexed = true, doc_values = true))] + pub fn add_boolean_field(&self, name: String, stored: bool, indexed: bool, doc_values: bool) { self.inner.borrow_mut().fields.insert( name, FieldOption::Boolean(BooleanOption { indexed, stored, - doc_values: true, + doc_values, }), ); } @@ -216,14 +240,16 @@ impl PhpSchema { /// * `name` - Field name. /// * `stored` - Whether the value is retrievable (default: true). /// * `indexed` - Whether the field is searchable (default: true). - #[php(defaults(stored = true, indexed = true))] - pub fn add_datetime_field(&self, name: String, stored: bool, indexed: bool) { + /// * `doc_values` - Whether the value is also copied into DocValues + /// (default: true). Takes effect only when `stored` is also true. + #[php(defaults(stored = true, indexed = true, doc_values = true))] + pub fn add_datetime_field(&self, name: String, stored: bool, indexed: bool, doc_values: bool) { self.inner.borrow_mut().fields.insert( name, FieldOption::DateTime(DateTimeOption { indexed, stored, - doc_values: true, + doc_values, }), ); } @@ -235,14 +261,16 @@ impl PhpSchema { /// * `name` - Field name. /// * `stored` - Whether the value is retrievable (default: true). /// * `indexed` - Whether the field is searchable (default: true). - #[php(defaults(stored = true, indexed = true))] - pub fn add_geo_field(&self, name: String, stored: bool, indexed: bool) { + /// * `doc_values` - Whether the value is also copied into DocValues + /// (default: true). Takes effect only when `stored` is also true. + #[php(defaults(stored = true, indexed = true, doc_values = true))] + pub fn add_geo_field(&self, name: String, stored: bool, indexed: bool, doc_values: bool) { self.inner.borrow_mut().fields.insert( name, FieldOption::Geo(GeoOption { indexed, stored, - doc_values: true, + doc_values, }), ); } @@ -259,14 +287,16 @@ impl PhpSchema { /// * `name` - Field name. /// * `stored` - Whether the value is retrievable (default: true). /// * `indexed` - Whether the field is searchable (default: true). - #[php(defaults(stored = true, indexed = true))] - pub fn add_geo3d_field(&self, name: String, stored: bool, indexed: bool) { + /// * `doc_values` - Whether the value is also copied into DocValues + /// (default: true). Takes effect only when `stored` is also true. + #[php(defaults(stored = true, indexed = true, doc_values = true))] + pub fn add_geo3d_field(&self, name: String, stored: bool, indexed: bool, doc_values: bool) { self.inner.borrow_mut().fields.insert( name, FieldOption::Geo3d(Geo3dOption { indexed, stored, - doc_values: true, + doc_values, }), ); } diff --git a/laurus-python/src/schema.rs b/laurus-python/src/schema.rs index 6e5da69c..f6381bcb 100644 --- a/laurus-python/src/schema.rs +++ b/laurus-python/src/schema.rs @@ -220,12 +220,17 @@ impl PySchema { /// indexed: Whether the field is searchable (default True). /// term_vectors: Whether term positions are stored, required by /// phrase and span queries over this field (default True). + /// doc_values: Whether the value is also copied into DocValues, + /// the column-oriented store sort/facet/aggregation read + /// from (default True). Takes effect only when ``stored`` is + /// also True. /// analyzer: Either a string analyzer name (``"standard"``, /// ``"english"``, ``"keyword"``, ``"simple"``, ``"noop"``, or /// a custom name registered via ``add_analyzer``), or a dict /// configuring a parameterized built-in preset such as /// ``{"language": "japanese", "dict": "/var/lib/lindera/ipadic"}``. - #[pyo3(signature = (name, *, stored=true, indexed=true, term_vectors=true, analyzer=None))] + #[pyo3(signature = (name, *, stored=true, indexed=true, term_vectors=true, doc_values=true, analyzer=None))] + #[allow(clippy::too_many_arguments)] pub fn add_text_field( &mut self, py: Python<'_>, @@ -233,6 +238,7 @@ impl PySchema { stored: bool, indexed: bool, term_vectors: bool, + doc_values: bool, analyzer: Option>, ) -> PyResult<()> { let analyzer = analyzer @@ -244,7 +250,7 @@ impl PySchema { indexed, stored, term_vectors, - doc_values: true, + doc_values, analyzer, }), ); @@ -261,13 +267,17 @@ impl PySchema { /// multi_valued: When True, the field accepts arrays of integers /// and range queries match if any value satisfies the /// predicate (Lucene-style "any match"). Default False. - #[pyo3(signature = (name, *, stored=true, indexed=true, multi_valued=false))] + /// doc_values: Whether the value is also copied into DocValues + /// (default True). Takes effect only when ``stored`` is also + /// True. + #[pyo3(signature = (name, *, stored=true, indexed=true, multi_valued=false, doc_values=true))] pub fn add_integer_field( &mut self, name: &str, stored: bool, indexed: bool, multi_valued: bool, + doc_values: bool, ) { self.inner.fields.insert( name.to_string(), @@ -275,7 +285,7 @@ impl PySchema { indexed, stored, multi_valued, - doc_values: true, + doc_values, }), ); } @@ -290,54 +300,85 @@ impl PySchema { /// multi_valued: When True, the field accepts arrays of floats /// and range queries match if any value satisfies the /// predicate (Lucene-style "any match"). Default False. - #[pyo3(signature = (name, *, stored=true, indexed=true, multi_valued=false))] - pub fn add_float_field(&mut self, name: &str, stored: bool, indexed: bool, multi_valued: bool) { + /// doc_values: Whether the value is also copied into DocValues + /// (default True). Takes effect only when ``stored`` is also + /// True. + #[pyo3(signature = (name, *, stored=true, indexed=true, multi_valued=false, doc_values=true))] + pub fn add_float_field( + &mut self, + name: &str, + stored: bool, + indexed: bool, + multi_valued: bool, + doc_values: bool, + ) { self.inner.fields.insert( name.to_string(), FieldOption::Float(FloatOption { indexed, stored, multi_valued, - doc_values: true, + doc_values, }), ); } /// Add a boolean field. - #[pyo3(signature = (name, *, stored=true, indexed=true))] - pub fn add_boolean_field(&mut self, name: &str, stored: bool, indexed: bool) { + /// + /// Args: + /// doc_values: Whether the value is also copied into DocValues + /// (default True). Takes effect only when ``stored`` is also + /// True. + #[pyo3(signature = (name, *, stored=true, indexed=true, doc_values=true))] + pub fn add_boolean_field(&mut self, name: &str, stored: bool, indexed: bool, doc_values: bool) { self.inner.fields.insert( name.to_string(), FieldOption::Boolean(BooleanOption { indexed, stored, - doc_values: true, + doc_values, }), ); } /// Add a date/time field. - #[pyo3(signature = (name, *, stored=true, indexed=true))] - pub fn add_datetime_field(&mut self, name: &str, stored: bool, indexed: bool) { + /// + /// Args: + /// doc_values: Whether the value is also copied into DocValues + /// (default True). Takes effect only when ``stored`` is also + /// True. + #[pyo3(signature = (name, *, stored=true, indexed=true, doc_values=true))] + pub fn add_datetime_field( + &mut self, + name: &str, + stored: bool, + indexed: bool, + doc_values: bool, + ) { self.inner.fields.insert( name.to_string(), FieldOption::DateTime(DateTimeOption { indexed, stored, - doc_values: true, + doc_values, }), ); } /// Add a geographic coordinate field (latitude, longitude). - #[pyo3(signature = (name, *, stored=true, indexed=true))] - pub fn add_geo_field(&mut self, name: &str, stored: bool, indexed: bool) { + /// + /// Args: + /// doc_values: Whether the value is also copied into DocValues + /// (default True). Takes effect only when ``stored`` is also + /// True. + #[pyo3(signature = (name, *, stored=true, indexed=true, doc_values=true))] + pub fn add_geo_field(&mut self, name: &str, stored: bool, indexed: bool, doc_values: bool) { self.inner.fields.insert( name.to_string(), FieldOption::Geo(GeoOption { indexed, stored, - doc_values: true, + doc_values, }), ); } @@ -348,14 +389,19 @@ impl PySchema { /// queryable via `Geo3dDistanceQuery`, `Geo3dBoundingBoxQuery`, and /// `Geo3dNearestQuery`. See the conceptual docs at /// `docs/src/concepts/geo3d.md` for the coordinate system. - #[pyo3(signature = (name, *, stored=true, indexed=true))] - pub fn add_geo3d_field(&mut self, name: &str, stored: bool, indexed: bool) { + /// + /// Args: + /// doc_values: Whether the value is also copied into DocValues + /// (default True). Takes effect only when ``stored`` is also + /// True. + #[pyo3(signature = (name, *, stored=true, indexed=true, doc_values=true))] + pub fn add_geo3d_field(&mut self, name: &str, stored: bool, indexed: bool, doc_values: bool) { self.inner.fields.insert( name.to_string(), FieldOption::Geo3d(Geo3dOption { indexed, stored, - doc_values: true, + doc_values, }), ); } diff --git a/laurus-python/tests/test_doc_values.py b/laurus-python/tests/test_doc_values.py new file mode 100644 index 00000000..43ebc5c9 --- /dev/null +++ b/laurus-python/tests/test_doc_values.py @@ -0,0 +1,60 @@ +"""Integration tests for the per-field `doc_values` schema option (Issue #1047). + +The Python binding does not currently expose field-sorted search or +faceting, so these tests cannot observe the DocValues column itself +disappearing on disk. What they cover is the acceptance criterion that +matters at this layer: `doc_values=False` must not lose stored-field +retrieval, and the schema builder must accept the flag on every field +type that carries it without raising. +""" + +import laurus + + +def test_doc_values_false_keeps_stored_field_retrievable(): + schema = laurus.Schema() + schema.add_text_field("title") + schema.add_text_field("internal_note", doc_values=False) + idx = laurus.Index(schema=schema) + idx.put_document( + "doc1", + {"title": "Hello", "internal_note": "opted out of doc values"}, + ) + idx.commit() + + docs = idx.get_documents("doc1") + assert len(docs) == 1 + assert docs[0]["title"] == "Hello" + assert docs[0]["internal_note"] == "opted out of doc values" + + +def test_doc_values_false_field_is_still_searchable(): + """`doc_values` only controls the column-oriented store; it must not + affect whether the field is indexed for search.""" + schema = laurus.Schema() + schema.add_text_field("title", doc_values=False) + idx = laurus.Index(schema=schema) + idx.put_document("doc1", {"title": "Rust programming"}) + idx.commit() + + results = idx.search(laurus.TermQuery("title", "rust"), limit=5) + ids = [hit.id for hit in results] + assert "doc1" in ids + + +def test_add_field_methods_accept_doc_values_on_every_carrying_type(): + """Every field option except Bytes carries `doc_values`; this must not + raise for any of them, in either direction.""" + schema = laurus.Schema() + schema.add_text_field("t", doc_values=False) + schema.add_integer_field("i", doc_values=False) + schema.add_float_field("f", doc_values=False) + schema.add_boolean_field("b", doc_values=False) + schema.add_datetime_field("d", doc_values=False) + schema.add_geo_field("g", doc_values=False) + schema.add_geo3d_field("g3", doc_values=False) + # And the default (True) must still be accepted explicitly too. + schema.add_text_field("t2", doc_values=True) + + idx = laurus.Index(schema=schema) + assert idx is not None diff --git a/laurus-ruby/src/schema.rs b/laurus-ruby/src/schema.rs index 2fca003c..fe95727c 100644 --- a/laurus-ruby/src/schema.rs +++ b/laurus-ruby/src/schema.rs @@ -132,6 +132,10 @@ impl RbSchema { /// - `indexed:` (bool, default true): Whether the field is searchable. /// - `term_vectors:` (bool, default true): Whether term positions are /// stored, required by phrase and span queries over this field. + /// - `doc_values:` (bool, default true): Whether the value is also + /// copied into DocValues, the column-oriented store + /// sort/facet/aggregation read from. Takes effect only when + /// `stored:` is also true. /// - `analyzer:` (String, optional): Analyzer name. For /// parameter-less built-ins (`"standard"`, `"english"`, /// `"keyword"`, `"simple"`, `"noop"`) pass the name directly. @@ -148,18 +152,26 @@ impl RbSchema { Option, Option, Option, + Option, Option>, ), (), >( args.keywords, &[], - &["stored", "indexed", "term_vectors", "analyzer"], + &[ + "stored", + "indexed", + "term_vectors", + "doc_values", + "analyzer", + ], )?; - let (stored, indexed, term_vectors, analyzer) = kwargs.optional; + let (stored, indexed, term_vectors, doc_values, analyzer) = kwargs.optional; let stored = stored.unwrap_or(true); let indexed = indexed.unwrap_or(true); let term_vectors = term_vectors.unwrap_or(true); + let doc_values = doc_values.unwrap_or(true); let analyzer = analyzer.flatten().map(laurus::AnalyzerSpec::Named); self.inner.borrow_mut().fields.insert( name, @@ -167,7 +179,7 @@ impl RbSchema { indexed, stored, term_vectors, - doc_values: true, + doc_values, analyzer, }), ); @@ -185,22 +197,26 @@ impl RbSchema { /// - `multi_valued:` (bool, default false): When true, the field /// accepts arrays of integers and range queries match if any value /// satisfies the predicate (Lucene-style "any match"). + /// - `doc_values:` (bool, default true): Whether the value is also + /// copied into DocValues. Takes effect only when `stored:` is + /// also true. fn add_integer_field(&self, args: &[Value]) -> Result<(), Error> { let args = scan_args::<(String,), (), (), (), RHash, ()>(args)?; let (name,) = args.required; - let kwargs = get_kwargs::<_, (), (Option, Option, Option), ()>( - args.keywords, - &[], - &["stored", "indexed", "multi_valued"], - )?; - let (stored, indexed, multi_valued) = kwargs.optional; + let kwargs = + get_kwargs::<_, (), (Option, Option, Option, Option), ()>( + args.keywords, + &[], + &["stored", "indexed", "multi_valued", "doc_values"], + )?; + let (stored, indexed, multi_valued, doc_values) = kwargs.optional; self.inner.borrow_mut().fields.insert( name, FieldOption::Integer(IntegerOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), multi_valued: multi_valued.unwrap_or(false), - doc_values: true, + doc_values: doc_values.unwrap_or(true), }), ); Ok(()) @@ -217,22 +233,26 @@ impl RbSchema { /// - `multi_valued:` (bool, default false): When true, the field /// accepts arrays of floats and range queries match if any value /// satisfies the predicate (Lucene-style "any match"). + /// - `doc_values:` (bool, default true): Whether the value is also + /// copied into DocValues. Takes effect only when `stored:` is + /// also true. fn add_float_field(&self, args: &[Value]) -> Result<(), Error> { let args = scan_args::<(String,), (), (), (), RHash, ()>(args)?; let (name,) = args.required; - let kwargs = get_kwargs::<_, (), (Option, Option, Option), ()>( - args.keywords, - &[], - &["stored", "indexed", "multi_valued"], - )?; - let (stored, indexed, multi_valued) = kwargs.optional; + let kwargs = + get_kwargs::<_, (), (Option, Option, Option, Option), ()>( + args.keywords, + &[], + &["stored", "indexed", "multi_valued", "doc_values"], + )?; + let (stored, indexed, multi_valued, doc_values) = kwargs.optional; self.inner.borrow_mut().fields.insert( name, FieldOption::Float(FloatOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), multi_valued: multi_valued.unwrap_or(false), - doc_values: true, + doc_values: doc_values.unwrap_or(true), }), ); Ok(()) @@ -246,21 +266,24 @@ impl RbSchema { /// - `name` (String): Field name. /// - `stored:` (bool, default true): Whether the value is retrievable. /// - `indexed:` (bool, default true): Whether the field is searchable. + /// - `doc_values:` (bool, default true): Whether the value is also + /// copied into DocValues. Takes effect only when `stored:` is + /// also true. fn add_boolean_field(&self, args: &[Value]) -> Result<(), Error> { let args = scan_args::<(String,), (), (), (), RHash, ()>(args)?; let (name,) = args.required; - let kwargs = get_kwargs::<_, (), (Option, Option), ()>( + let kwargs = get_kwargs::<_, (), (Option, Option, Option), ()>( args.keywords, &[], - &["stored", "indexed"], + &["stored", "indexed", "doc_values"], )?; - let (stored, indexed) = kwargs.optional; + let (stored, indexed, doc_values) = kwargs.optional; self.inner.borrow_mut().fields.insert( name, FieldOption::Boolean(BooleanOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), - doc_values: true, + doc_values: doc_values.unwrap_or(true), }), ); Ok(()) @@ -274,21 +297,24 @@ impl RbSchema { /// - `name` (String): Field name. /// - `stored:` (bool, default true): Whether the value is retrievable. /// - `indexed:` (bool, default true): Whether the field is searchable. + /// - `doc_values:` (bool, default true): Whether the value is also + /// copied into DocValues. Takes effect only when `stored:` is + /// also true. fn add_datetime_field(&self, args: &[Value]) -> Result<(), Error> { let args = scan_args::<(String,), (), (), (), RHash, ()>(args)?; let (name,) = args.required; - let kwargs = get_kwargs::<_, (), (Option, Option), ()>( + let kwargs = get_kwargs::<_, (), (Option, Option, Option), ()>( args.keywords, &[], - &["stored", "indexed"], + &["stored", "indexed", "doc_values"], )?; - let (stored, indexed) = kwargs.optional; + let (stored, indexed, doc_values) = kwargs.optional; self.inner.borrow_mut().fields.insert( name, FieldOption::DateTime(DateTimeOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), - doc_values: true, + doc_values: doc_values.unwrap_or(true), }), ); Ok(()) @@ -302,21 +328,24 @@ impl RbSchema { /// - `name` (String): Field name. /// - `stored:` (bool, default true): Whether the value is retrievable. /// - `indexed:` (bool, default true): Whether the field is searchable. + /// - `doc_values:` (bool, default true): Whether the value is also + /// copied into DocValues. Takes effect only when `stored:` is + /// also true. fn add_geo_field(&self, args: &[Value]) -> Result<(), Error> { let args = scan_args::<(String,), (), (), (), RHash, ()>(args)?; let (name,) = args.required; - let kwargs = get_kwargs::<_, (), (Option, Option), ()>( + let kwargs = get_kwargs::<_, (), (Option, Option, Option), ()>( args.keywords, &[], - &["stored", "indexed"], + &["stored", "indexed", "doc_values"], )?; - let (stored, indexed) = kwargs.optional; + let (stored, indexed, doc_values) = kwargs.optional; self.inner.borrow_mut().fields.insert( name, FieldOption::Geo(GeoOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), - doc_values: true, + doc_values: doc_values.unwrap_or(true), }), ); Ok(()) @@ -335,21 +364,24 @@ impl RbSchema { /// - `name` (String): Field name. /// - `stored:` (bool, default true): Whether the value is retrievable. /// - `indexed:` (bool, default true): Whether the field is searchable. + /// - `doc_values:` (bool, default true): Whether the value is also + /// copied into DocValues. Takes effect only when `stored:` is + /// also true. fn add_geo3d_field(&self, args: &[Value]) -> Result<(), Error> { let args = scan_args::<(String,), (), (), (), RHash, ()>(args)?; let (name,) = args.required; - let kwargs = get_kwargs::<_, (), (Option, Option), ()>( + let kwargs = get_kwargs::<_, (), (Option, Option, Option), ()>( args.keywords, &[], - &["stored", "indexed"], + &["stored", "indexed", "doc_values"], )?; - let (stored, indexed) = kwargs.optional; + let (stored, indexed, doc_values) = kwargs.optional; self.inner.borrow_mut().fields.insert( name, FieldOption::Geo3d(Geo3dOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), - doc_values: true, + doc_values: doc_values.unwrap_or(true), }), ); Ok(()) diff --git a/laurus-ruby/test/test_doc_values.rb b/laurus-ruby/test/test_doc_values.rb new file mode 100644 index 00000000..87ae92ec --- /dev/null +++ b/laurus-ruby/test/test_doc_values.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +require_relative "test_helper" + +# Integration tests for the per-field `doc_values` schema option (Issue +# #1047). +# +# The Ruby binding does not currently expose field-sorted search or +# faceting, so these tests cannot observe the DocValues column itself +# disappearing on disk. What they cover is the acceptance criterion that +# matters at this layer: `doc_values: false` must not lose stored-field +# retrieval, and the schema builder must accept the flag on every field +# type that carries it without raising. +class TestDocValues < Minitest::Test + def test_doc_values_false_keeps_stored_field_retrievable + schema = Laurus::Schema.new + schema.add_text_field("title") + schema.add_text_field("internal_note", doc_values: false) + idx = Laurus::Index.new(schema: schema) + idx.put_document("doc1", { "title" => "Hello", "internal_note" => "opted out of doc values" }) + idx.commit + + docs = idx.get_documents("doc1") + assert_equal 1, docs.length + assert_equal "Hello", docs.first["title"] + assert_equal "opted out of doc values", docs.first["internal_note"] + end + + # `doc_values` only controls the column-oriented store; it must not + # affect whether the field is indexed for search. + def test_doc_values_false_field_is_still_searchable + schema = Laurus::Schema.new + schema.add_text_field("title", doc_values: false) + idx = Laurus::Index.new(schema: schema) + idx.put_document("doc1", { "title" => "Rust programming" }) + idx.commit + + results = idx.search(Laurus::TermQuery.new("title", "rust"), limit: 5) + assert results.any? { |r| r.id == "doc1" } + end + + # Every field option except Bytes carries `doc_values`; this must not + # raise for any of them, in either direction. + def test_add_field_methods_accept_doc_values_on_every_carrying_type + schema = Laurus::Schema.new + schema.add_text_field("t", doc_values: false) + schema.add_integer_field("i", doc_values: false) + schema.add_float_field("f", doc_values: false) + schema.add_boolean_field("b", doc_values: false) + schema.add_datetime_field("d", doc_values: false) + schema.add_geo_field("g", doc_values: false) + schema.add_geo3d_field("g3", doc_values: false) + # And the default (true) must still be accepted explicitly too. + schema.add_text_field("t2", doc_values: true) + + idx = Laurus::Index.new(schema: schema) + refute_nil idx + end +end diff --git a/laurus-wasm/src/schema.rs b/laurus-wasm/src/schema.rs index ea4af439..5314e175 100644 --- a/laurus-wasm/src/schema.rs +++ b/laurus-wasm/src/schema.rs @@ -131,6 +131,9 @@ impl WasmSchema { /// * `indexed` - Whether the field is searchable (default `true`). /// * `term_vectors` - Whether term positions are stored, required by /// phrase and span queries over this field (default `true`). + /// * `doc_values` - Whether the value is also copied into DocValues, + /// the column-oriented store sort/facet/aggregation read from + /// (default `true`). Takes effect only when `stored` is also `true`. /// * `analyzer` - Optional analyzer name. Pass a parameter-less /// built-in directly: `"standard"`, `"english"`, `"keyword"`, /// `"simple"`, `"noop"`. For the Japanese analyzer, build it @@ -147,6 +150,7 @@ impl WasmSchema { stored: Option, indexed: Option, term_vectors: Option, + doc_values: Option, analyzer: Option, ) { self.inner.fields.insert( @@ -155,13 +159,16 @@ impl WasmSchema { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), term_vectors: term_vectors.unwrap_or(true), - doc_values: true, + doc_values: doc_values.unwrap_or(true), analyzer: analyzer.map(laurus::AnalyzerSpec::Named), }), ); } /// Add an integer (i64) field. + /// + /// * `doc_values` - Whether the value is also copied into DocValues + /// (default `true`). Takes effect only when `stored` is also `true`. #[wasm_bindgen(js_name = "addIntegerField")] pub fn add_integer_field( &mut self, @@ -169,6 +176,7 @@ impl WasmSchema { stored: Option, indexed: Option, multi_valued: Option, + doc_values: Option, ) { self.inner.fields.insert( name, @@ -176,12 +184,15 @@ impl WasmSchema { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), multi_valued: multi_valued.unwrap_or(false), - doc_values: true, + doc_values: doc_values.unwrap_or(true), }), ); } /// Add a float (f64) field. + /// + /// * `doc_values` - Whether the value is also copied into DocValues + /// (default `true`). Takes effect only when `stored` is also `true`. #[wasm_bindgen(js_name = "addFloatField")] pub fn add_float_field( &mut self, @@ -189,6 +200,7 @@ impl WasmSchema { stored: Option, indexed: Option, multi_valued: Option, + doc_values: Option, ) { self.inner.fields.insert( name, @@ -196,51 +208,73 @@ impl WasmSchema { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), multi_valued: multi_valued.unwrap_or(false), - doc_values: true, + doc_values: doc_values.unwrap_or(true), }), ); } /// Add a boolean field. + /// + /// * `doc_values` - Whether the value is also copied into DocValues + /// (default `true`). Takes effect only when `stored` is also `true`. #[wasm_bindgen(js_name = "addBooleanField")] - pub fn add_boolean_field(&mut self, name: String, stored: Option, indexed: Option) { + pub fn add_boolean_field( + &mut self, + name: String, + stored: Option, + indexed: Option, + doc_values: Option, + ) { self.inner.fields.insert( name, FieldOption::Boolean(BooleanOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), - doc_values: true, + doc_values: doc_values.unwrap_or(true), }), ); } /// Add a date/time field. + /// + /// * `doc_values` - Whether the value is also copied into DocValues + /// (default `true`). Takes effect only when `stored` is also `true`. #[wasm_bindgen(js_name = "addDatetimeField")] pub fn add_datetime_field( &mut self, name: String, stored: Option, indexed: Option, + doc_values: Option, ) { self.inner.fields.insert( name, FieldOption::DateTime(DateTimeOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), - doc_values: true, + doc_values: doc_values.unwrap_or(true), }), ); } /// Add a geographic coordinate field (latitude, longitude). + /// + /// * `doc_values` - Whether the value is also copied into DocValues + /// (default `true`). Takes effect only when `stored` is also `true`. #[wasm_bindgen(js_name = "addGeoField")] - pub fn add_geo_field(&mut self, name: String, stored: Option, indexed: Option) { + pub fn add_geo_field( + &mut self, + name: String, + stored: Option, + indexed: Option, + doc_values: Option, + ) { self.inner.fields.insert( name, FieldOption::Geo(GeoOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), - doc_values: true, + doc_values: doc_values.unwrap_or(true), }), ); } @@ -251,14 +285,23 @@ impl WasmSchema { /// queryable via `searchGeo3dDistance`, `searchGeo3dBoundingBox`, /// and `searchGeo3dNearest` on `Index`. See the conceptual docs at /// `docs/src/concepts/geo3d.md`. + /// + /// * `doc_values` - Whether the value is also copied into DocValues + /// (default `true`). Takes effect only when `stored` is also `true`. #[wasm_bindgen(js_name = "addGeo3dField")] - pub fn add_geo3d_field(&mut self, name: String, stored: Option, indexed: Option) { + pub fn add_geo3d_field( + &mut self, + name: String, + stored: Option, + indexed: Option, + doc_values: Option, + ) { self.inner.fields.insert( name, FieldOption::Geo3d(Geo3dOption { indexed: indexed.unwrap_or(true), stored: stored.unwrap_or(true), - doc_values: true, + doc_values: doc_values.unwrap_or(true), }), ); } From 4f274d04d83e4d47d2e133718989b1f41439b6a7 Mon Sep 17 00:00:00 2001 From: Minoru Osuka Date: Sat, 12 Sep 2026 15:53:08 -0400 Subject: [PATCH 09/10] docs: document the doc_values field option Documents the doc_values: bool schema option (default true, Issue #1047) across both the English and Japanese docs trees: - concepts/schema_and_fields.md: added to the TextOption builder example and option table, plus a shared note explaining it applies to every lexical field option except BytesOption. - laurus/faceting.md: corrected the "every stored field is written to DocValues" statement, which the opt-out makes inaccurate -- it's now "every stored field not excluded by type or by doc_values: false". - laurus-cli/schema_format.md: added to all 7 carrying field types' TOML snippets and option tables, plus a new "Common option: doc_values" section spelling out the effective rule (stored && doc_values) and BytesOption's exclusion. - laurus-server/grpc_api.md: added to the FieldOption oneof summary table, plus a note on the tri-state (optional bool) wire contract, mirroring term_vectors. - All 5 bindings' api_reference.md (laurus-python, laurus-ruby, laurus-nodejs, laurus-wasm, laurus-php): added doc_values to all 7 add_*_field signatures. Along the way, fixed a pre-existing wrong default in laurus-php's docs (addTextField's $termVectors was documented as defaulting to false; the actual default is true). laurus-server/http_gateway.md, concepts/indexing/lexical_indexing.md, and laurus/api_reference.md were left unchanged: the first has no option table (only bare JSON examples), the second never mentioned term_vectors either (segment-format overview, not a field-option reference), and the third documents SchemaBuilder methods by option struct name only, without enumerating struct fields. `mdbook build` clean for both docs/ and docs/ja/. Refs #1047 --- docs/ja/src/concepts/schema_and_fields.md | 15 +++++++-- docs/ja/src/laurus-cli/schema_format.md | 29 +++++++++++++++++ docs/ja/src/laurus-nodejs/api_reference.md | 16 ++++----- docs/ja/src/laurus-php/api_reference.md | 16 ++++----- docs/ja/src/laurus-python/api_reference.md | 16 ++++----- docs/ja/src/laurus-ruby/api_reference.md | 16 ++++----- docs/ja/src/laurus-server/grpc_api.md | 16 +++++---- docs/ja/src/laurus-wasm/api_reference.md | 37 ++++++++++++--------- docs/ja/src/laurus/faceting.md | 13 ++++++-- docs/src/concepts/schema_and_fields.md | 16 +++++++-- docs/src/laurus-cli/schema_format.md | 31 ++++++++++++++++++ docs/src/laurus-nodejs/api_reference.md | 16 ++++----- docs/src/laurus-php/api_reference.md | 16 ++++----- docs/src/laurus-python/api_reference.md | 16 ++++----- docs/src/laurus-ruby/api_reference.md | 16 ++++----- docs/src/laurus-server/grpc_api.md | 16 +++++---- docs/src/laurus-wasm/api_reference.md | 38 ++++++++++++---------- docs/src/laurus/faceting.md | 13 ++++++-- 18 files changed, 231 insertions(+), 121 deletions(-) diff --git a/docs/ja/src/concepts/schema_and_fields.md b/docs/ja/src/concepts/schema_and_fields.md index 1c81d86d..d415ca27 100644 --- a/docs/ja/src/concepts/schema_and_fields.md +++ b/docs/ja/src/concepts/schema_and_fields.md @@ -69,14 +69,15 @@ Lexical フィールドは転置インデックス（Inverted Index）を使用 ```rust use laurus::lexical::TextOption; -// Default: indexed + stored + term vectors (all true) +// Default: indexed + stored + term vectors + doc values (all true) let opt = TextOption::default(); // Customize let opt = TextOption::default() .indexed(true) .stored(true) - .term_vectors(true); + .term_vectors(true) + .doc_values(true); ``` | オプション | デフォルト | 説明 | @@ -84,6 +85,16 @@ let opt = TextOption::default() | `indexed` | `true` | フィールドが検索可能かどうか | | `stored` | `true` | 元の値が取得用に保存されるかどうか | | `term_vectors` | `true` | ターム位置が保存されるかどうか（フレーズクエリ・スパンクエリで使用。ハイライトは常に保存済みテキストを再トークナイズするため使用しない） | +| `doc_values` | `true` | 値を DocValues（[ソート](../laurus/faceting.md)・ファセット・集計が読み取る列指向ストア）にもコピーするかどうか | + +`doc_values` は `TextOption` 専用ではありません。`BytesOption` を除く全ての lexical +フィールドオプション（`IntegerOption`, `FloatOption`, `BooleanOption`, `DateTimeOption`, +`GeoOption`, `Geo3dOption`）が同じ設定を持ちます。`BytesOption` にはこの設定がありません +―― `Bytes` の値はソートにもファセットにも使えないため、設定にかかわらず DocValues には +一切書き込まれないからです。実効ルールは次のとおりです: DocValues 列が書き込まれるのは +`stored` と `doc_values` の両方が `true`（かつ値の型が `Bytes` でない）場合のみです。 +ソートにもファセットにも使わないフィールドで `doc_values: false` を設定すると、二重目の +コピーを省略できるため、セグメントの使用容量が削減されます。 ### Vector フィールド diff --git a/docs/ja/src/laurus-cli/schema_format.md b/docs/ja/src/laurus-cli/schema_format.md index 7bb50e6f..8d129058 100644 --- a/docs/ja/src/laurus-cli/schema_format.md +++ b/docs/ja/src/laurus-cli/schema_format.md @@ -43,6 +43,7 @@ default_fields = ["title", "body"] indexed = true # このフィールドを検索用にインデックスするかどうか stored = true # 取得用に元の値を保存するかどうか term_vectors = true # タームの位置を保存するかどうか（フレーズクエリ・スパンクエリ用） +doc_values = true # 値を DocValues にもコピーするかどうか（ソート・ファセット用） ``` | オプション | 型 | デフォルト | 説明 | @@ -50,6 +51,7 @@ term_vectors = true # タームの位置を保存するかどうか（フレー | `indexed` | `bool` | `true` | このフィールドの検索を有効にする | | `stored` | `bool` | `true` | 結果に返せるよう元の値を保存する | | `term_vectors` | `bool` | `true` | フレーズクエリ・スパンクエリが読み取るタームの位置を保存する。ハイライトは常に保存済みテキストを再トークナイズするため使用しない | +| `doc_values` | `bool` | `true` | 値を DocValues（[ソート](../laurus/faceting.md)・ファセット・集計が読み取る列指向ストア）にもコピーする。`stored` も `true` の場合のみ有効 —— 詳細は後述の [共通オプション: `doc_values`](#共通オプション-doc_values) を参照 | #### Integer @@ -60,6 +62,7 @@ term_vectors = true # タームの位置を保存するかどうか（フレー indexed = true stored = true multi_valued = false +doc_values = true ``` | オプション | 型 | デフォルト | 説明 | @@ -67,6 +70,7 @@ multi_valued = false | `indexed` | `bool` | `true` | 範囲クエリおよび完全一致クエリを有効にする | | `stored` | `bool` | `true` | 元の値を保存する | | `multi_valued` | `bool` | `false` | 整数の配列を受け付け、範囲クエリは**いずれかの値**が条件を満たせばマッチ（Lucene 流の "any match"、constant スコア） | +| `doc_values` | `bool` | `true` | 詳細は後述の [共通オプション: `doc_values`](#共通オプション-doc_values) を参照 | #### Float @@ -77,6 +81,7 @@ multi_valued = false indexed = true stored = true multi_valued = false +doc_values = true ``` | オプション | 型 | デフォルト | 説明 | @@ -84,6 +89,7 @@ multi_valued = false | `indexed` | `bool` | `true` | 範囲クエリを有効にする | | `stored` | `bool` | `true` | 元の値を保存する | | `multi_valued` | `bool` | `false` | 浮動小数点の配列を受け付け、範囲クエリは**いずれかの値**が条件を満たせばマッチ（Lucene 流の "any match"、constant スコア） | +| `doc_values` | `bool` | `true` | 詳細は後述の [共通オプション: `doc_values`](#共通オプション-doc_values) を参照 | #### Boolean @@ -93,12 +99,14 @@ multi_valued = false [fields.published.Boolean] indexed = true stored = true +doc_values = true ``` | オプション | 型 | デフォルト | 説明 | | :--- | :--- | :--- | :--- | | `indexed` | `bool` | `true` | ブーリアン値によるフィルタリングを有効にする | | `stored` | `bool` | `true` | 元の値を保存する | +| `doc_values` | `bool` | `true` | 詳細は後述の [共通オプション: `doc_values`](#共通オプション-doc_values) を参照 | #### DateTime @@ -108,12 +116,14 @@ UTC タイムスタンプフィールド。範囲クエリをサポートしま [fields.created_at.DateTime] indexed = true stored = true +doc_values = true ``` | オプション | 型 | デフォルト | 説明 | | :--- | :--- | :--- | :--- | | `indexed` | `bool` | `true` | 日時の範囲クエリを有効にする | | `stored` | `bool` | `true` | 元の値を保存する | +| `doc_values` | `bool` | `true` | 詳細は後述の [共通オプション: `doc_values`](#共通オプション-doc_values) を参照 | #### Geo @@ -123,12 +133,14 @@ stored = true [fields.location.Geo] indexed = true stored = true +doc_values = true ``` | オプション | 型 | デフォルト | 説明 | | :--- | :--- | :--- | :--- | | `indexed` | `bool` | `true` | Geo クエリ（半径、バウンディングボックス）を有効にする | | `stored` | `bool` | `true` | 元の値を保存する | +| `doc_values` | `bool` | `true` | 詳細は後述の [共通オプション: `doc_values`](#共通オプション-doc_values) を参照 | #### Geo3d @@ -138,12 +150,14 @@ stored = true [fields.position.Geo3d] indexed = true stored = true +doc_values = true ``` | オプション | 型 | デフォルト | 説明 | | :--- | :--- | :--- | :--- | | `indexed` | `bool` | `true` | 3D 地理クエリ（`geo3d_distance`、`geo3d_bbox`、`geo3d_nearest`）を有効にする | | `stored` | `bool` | `true` | 元の `(x, y, z)` 値を保存する | +| `doc_values` | `bool` | `true` | 詳細は後述の [共通オプション: `doc_values`](#共通オプション-doc_values) を参照 | #### Bytes @@ -158,6 +172,21 @@ stored = true | :--- | :--- | :--- | :--- | | `stored` | `bool` | `true` | バイナリデータを保存する | +`BytesOption` に `doc_values` 設定はありません。`Bytes` の値はソートにもファセットにも +使えないため、設定にかかわらず DocValues には一切書き込まれないからです。 + +#### 共通オプション: `doc_values` + +上記の lexical フィールドオプションのうち `BytesOption` を除く全てが `doc_values` オプションを +持ち、値を DocValues ―― [ソート](../laurus/faceting.md)・ファセット・集計が読み取る列指向ストア +―― にもコピーするかどうかを制御します。実効ルールは次のとおりです: DocValues 列が書き込まれる +のは `stored` と `doc_values` の両方が `true` の場合のみです。`doc_values: false` と +`stored: false` の組み合わせは（エラーにせず）黙って無視されます。ソートにもファセットにも +使わないフィールドで `doc_values` を無効にすると、値が二重（stored document と DocValues） +ではなく一度（stored document のみ）しか書き込まれなくなるため、セグメントの使用容量が +削減されます。フィールド自体は引き続き完全に検索・取得可能で、ソートやファセットを行う際は +単に stored document へフォールバックします。 + ### Vector フィールド Vector フィールドは近似最近傍探索（ANN: Approximate Nearest Neighbor）用にインデックスされます。`dimension`（各ベクトルの長さ）と `distance` メトリクスの指定が必要です。 diff --git a/docs/ja/src/laurus-nodejs/api_reference.md b/docs/ja/src/laurus-nodejs/api_reference.md index d313f3c3..83255dd8 100644 --- a/docs/ja/src/laurus-nodejs/api_reference.md +++ b/docs/ja/src/laurus-nodejs/api_reference.md @@ -196,14 +196,14 @@ class Schema { | メソッド | 説明 | | :--- | :--- | -| `addTextField(name, stored?, indexed?, termVectors?, analyzer?)` | 全文検索フィールド（転置インデックス、BM25）。`analyzer` にはパラメータ不要の組込名（`"standard"` / `"english"` / `"keyword"` / `"simple"` / `"noop"`、または `addAnalyzer` で登録したカスタム名）を指定します。Lindera 辞書パスが必要な Japanese プリセットを使う場合は、`lindera` tokenizer を含むカスタム analyzer を登録して、その名前を参照してください。 | -| `addIntegerField(name, stored?, indexed?, multiValued?)` | 64 ビット整数フィールド。`multiValued: true` で整数配列を受け付け（範囲クエリは "any match"）。 | -| `addFloatField(name, stored?, indexed?, multiValued?)` | 64 ビット浮動小数点フィールド。`multiValued: true` で浮動小数点配列を受け付け（範囲クエリは "any match"）。 | -| `addBooleanField(name, stored?, indexed?)` | 真偽値フィールド。 | -| `addBytesField(name, stored?)` | バイナリデータフィールド。 | -| `addGeoField(name, stored?, indexed?)` | 地理座標フィールド。 | -| `addGeo3dField(name, stored?, indexed?)` | 3D ECEF カルテシアン座標フィールド（x, y, z はメートル）。詳細は [Geo3d の概念](../concepts/geo3d.md)。 | -| `addDatetimeField(name, stored?, indexed?)` | UTC 日時フィールド。 | +| `addTextField(name, stored?, indexed?, termVectors?, docValues?, analyzer?)` | 全文検索フィールド（転置インデックス、BM25）。`docValues` は値を DocValues（ソート・ファセット・集計が読み取る列指向ストア）にもコピーするかどうかを制御します（Issue #1047、デフォルト `true`）。`stored` も `true` の場合のみ有効です。`analyzer` にはパラメータ不要の組込名（`"standard"` / `"english"` / `"keyword"` / `"simple"` / `"noop"`、または `addAnalyzer` で登録したカスタム名）を指定します。Lindera 辞書パスが必要な Japanese プリセットを使う場合は、`lindera` tokenizer を含むカスタム analyzer を登録して、その名前を参照してください。 | +| `addIntegerField(name, stored?, indexed?, multiValued?, docValues?)` | 64 ビット整数フィールド。`multiValued: true` で整数配列を受け付け（範囲クエリは "any match"）。`docValues` は上記を参照。 | +| `addFloatField(name, stored?, indexed?, multiValued?, docValues?)` | 64 ビット浮動小数点フィールド。`multiValued: true` で浮動小数点配列を受け付け（範囲クエリは "any match"）。`docValues` は上記を参照。 | +| `addBooleanField(name, stored?, indexed?, docValues?)` | 真偽値フィールド。`docValues` は上記を参照。 | +| `addBytesField(name, stored?)` | バイナリデータフィールド。`docValues` オプションはありません —— `Bytes` の値は設定にかかわらず DocValues に一切書き込まれないためです。 | +| `addGeoField(name, stored?, indexed?, docValues?)` | 地理座標フィールド。`docValues` は上記を参照。 | +| `addGeo3dField(name, stored?, indexed?, docValues?)` | 3D ECEF カルテシアン座標フィールド（x, y, z はメートル）。詳細は [Geo3d の概念](../concepts/geo3d.md)。`docValues` は上記を参照。 | +| `addDatetimeField(name, stored?, indexed?, docValues?)` | UTC 日時フィールド。`docValues` は上記を参照。 | | `addHnswField(name, dimension, distance?, m?, efConstruction?, defaultEfSearch?, embedder?, quantizer?, subvectorCount?, rerankStorage?, pqCodebookPath?, baseWeight?)` | HNSW ベクトルフィールド。`baseWeight` は他の vector フィールドと同時に検索されたときの相対的なスコアリング優先度（Issue #1084）。[ウェイト](../concepts/search/vector_search.md#ウェイト)を参照。 | | `addFlatField(name, dimension, distance?, embedder?, baseWeight?)` | Flat（全探索）ベクトルフィールド。 | | `addIvfField(name, dimension, distance?, nClusters?, nProbe?, embedder?, baseWeight?)` | IVF ベクトルフィールド。 | diff --git a/docs/ja/src/laurus-php/api_reference.md b/docs/ja/src/laurus-php/api_reference.md index da68aebf..52bbd01f 100644 --- a/docs/ja/src/laurus-php/api_reference.md +++ b/docs/ja/src/laurus-php/api_reference.md @@ -163,14 +163,14 @@ new \Laurus\Schema() | メソッド | 説明 | | :--- | :--- | -| `addTextField(string $name, bool $stored = true, bool $indexed = true, bool $termVectors = false, ?string $analyzer = null): void` | 全文フィールド（転置インデックス、BM25）。`$analyzer` にはパラメータ不要の組込名（`"standard"` / `"english"` / `"keyword"` / `"simple"` / `"noop"`、または `addAnalyzer` で登録したカスタム名）を指定します。Lindera 辞書パスが必要な Japanese プリセットは、`lindera` tokenizer を含むカスタム analyzer として登録し、名前で参照してください。 | -| `addIntegerField(string $name, bool $stored = true, bool $indexed = true, bool $multiValued = false): void` | 64 ビット整数フィールド。`$multiValued = true` で整数配列を受け付け（範囲クエリは "any match"）。 | -| `addFloatField(string $name, bool $stored = true, bool $indexed = true, bool $multiValued = false): void` | 64 ビット浮動小数点フィールド。`$multiValued = true` で浮動小数点配列を受け付け（範囲クエリは "any match"）。 | -| `addBooleanField(string $name, bool $stored = true, bool $indexed = true): void` | ブールフィールド。 | -| `addBytesField(string $name, bool $stored = true): void` | 生バイトフィールド。 | -| `addGeoField(string $name, bool $stored = true, bool $indexed = true): void` | 地理座標フィールド（緯度/経度）。 | -| `addGeo3dField(string $name, bool $stored = true, bool $indexed = true): void` | 3D ECEF カルテシアン座標フィールド（x, y, z はメートル）。詳細は [Geo3d の概念](../concepts/geo3d.md)。 | -| `addDatetimeField(string $name, bool $stored = true, bool $indexed = true): void` | UTC 日時フィールド。 | +| `addTextField(string $name, bool $stored = true, bool $indexed = true, bool $termVectors = true, bool $docValues = true, ?string $analyzer = null): void` | 全文フィールド（転置インデックス、BM25）。`$docValues` は値を DocValues（ソート・ファセット・集計が読み取る列指向ストア）にもコピーするかどうかを制御します（Issue #1047）。`$stored` も `true` の場合のみ有効です。`$analyzer` にはパラメータ不要の組込名（`"standard"` / `"english"` / `"keyword"` / `"simple"` / `"noop"`、または `addAnalyzer` で登録したカスタム名）を指定します。Lindera 辞書パスが必要な Japanese プリセットは、`lindera` tokenizer を含むカスタム analyzer として登録し、名前で参照してください。 | +| `addIntegerField(string $name, bool $stored = true, bool $indexed = true, bool $multiValued = false, bool $docValues = true): void` | 64 ビット整数フィールド。`$multiValued = true` で整数配列を受け付け（範囲クエリは "any match"）。`$docValues` は上記を参照。 | +| `addFloatField(string $name, bool $stored = true, bool $indexed = true, bool $multiValued = false, bool $docValues = true): void` | 64 ビット浮動小数点フィールド。`$multiValued = true` で浮動小数点配列を受け付け（範囲クエリは "any match"）。`$docValues` は上記を参照。 | +| `addBooleanField(string $name, bool $stored = true, bool $indexed = true, bool $docValues = true): void` | ブールフィールド。`$docValues` は上記を参照。 | +| `addBytesField(string $name, bool $stored = true): void` | 生バイトフィールド。`$docValues` オプションはありません —— `Bytes` の値は設定にかかわらず DocValues に一切書き込まれないためです。 | +| `addGeoField(string $name, bool $stored = true, bool $indexed = true, bool $docValues = true): void` | 地理座標フィールド（緯度/経度）。`$docValues` は上記を参照。 | +| `addGeo3dField(string $name, bool $stored = true, bool $indexed = true, bool $docValues = true): void` | 3D ECEF カルテシアン座標フィールド（x, y, z はメートル）。詳細は [Geo3d の概念](../concepts/geo3d.md)。`$docValues` は上記を参照。 | +| `addDatetimeField(string $name, bool $stored = true, bool $indexed = true, bool $docValues = true): void` | UTC 日時フィールド。`$docValues` は上記を参照。 | | `addHnswField(string $name, int $dimension, ?string $distance = "cosine", int $m = 16, int $efConstruction = 200, ?int $defaultEfSearch = null, ?string $embedder = null, ?string $quantizer = null, ?int $subvectorCount = null, ?string $rerankStorage = null, ?string $pqCodebookPath = null, float $baseWeight = 1.0): void` | HNSW 近似最近傍ベクトルフィールド。`$baseWeight` は他の vector フィールドと同時に検索されたときの相対的なスコアリング優先度（Issue #1084）。[ウェイト](../concepts/search/vector_search.md#ウェイト)を参照。 | | `addFlatField(string $name, int $dimension, ?string $distance = "cosine", ?string $embedder = null, float $baseWeight = 1.0): void` | Flat（総当たり）ベクトルフィールド。 | | `addIvfField(string $name, int $dimension, ?string $distance = "cosine", int $nClusters = 100, int $nProbe = 1, ?string $embedder = null, float $baseWeight = 1.0): void` | IVF 近似最近傍ベクトルフィールド。 | diff --git a/docs/ja/src/laurus-python/api_reference.md b/docs/ja/src/laurus-python/api_reference.md index 1adb4f5c..fc043246 100644 --- a/docs/ja/src/laurus-python/api_reference.md +++ b/docs/ja/src/laurus-python/api_reference.md @@ -192,14 +192,14 @@ class Schema: | メソッド | 説明 | | :--- | :--- | -| `add_text_field(name, *, stored=True, indexed=True, term_vectors=True, analyzer=None)` | 全文フィールド（転置インデックス、BM25）。`term_vectors` はタームの位置を保存するかどうかを制御し、フレーズクエリ・スパンクエリが読み取ります。`analyzer` には組込名（`"standard"` / `"english"` / `"keyword"` / `"simple"` / `"noop"`、または `add_analyzer` で登録したカスタム名）か、`{"language": "japanese", "mode": "normal", "dict": "/var/lib/lindera/ipadic"}` のようなパラメータ付きプリセットの dict を渡せます。文字列単独の `"japanese"` は Lindera 辞書パスが必須なため拒否されます。 | -| `add_integer_field(name, *, stored=True, indexed=True, multi_valued=False)` | 64 ビット整数フィールド。`multi_valued=True` で整数配列を受け付け（範囲クエリは "any match"）。 | -| `add_float_field(name, *, stored=True, indexed=True, multi_valued=False)` | 64 ビット浮動小数点フィールド。`multi_valued=True` で浮動小数点配列を受け付け（範囲クエリは "any match"）。 | -| `add_boolean_field(name, *, stored=True, indexed=True)` | ブールフィールド。 | -| `add_bytes_field(name, *, stored=True)` | 生バイトフィールド。 | -| `add_geo_field(name, *, stored=True, indexed=True)` | 地理座標フィールド（緯度/経度）。 | -| `add_geo3d_field(name, *, stored=True, indexed=True)` | 3D ECEF カルテシアン座標フィールド（x, y, z はメートル）。詳細は [Geo3d の概念](../concepts/geo3d.md)。 | -| `add_datetime_field(name, *, stored=True, indexed=True)` | UTC 日時フィールド。 | +| `add_text_field(name, *, stored=True, indexed=True, term_vectors=True, doc_values=True, analyzer=None)` | 全文フィールド（転置インデックス、BM25）。`term_vectors` はタームの位置を保存するかどうかを制御し、フレーズクエリ・スパンクエリが読み取ります。`doc_values` は値を DocValues（ソート・ファセット・集計が読み取る列指向ストア）にもコピーするかどうかを制御します（Issue #1047）。`stored=True` の場合のみ有効です。`analyzer` には組込名（`"standard"` / `"english"` / `"keyword"` / `"simple"` / `"noop"`、または `add_analyzer` で登録したカスタム名）か、`{"language": "japanese", "mode": "normal", "dict": "/var/lib/lindera/ipadic"}` のようなパラメータ付きプリセットの dict を渡せます。文字列単独の `"japanese"` は Lindera 辞書パスが必須なため拒否されます。 | +| `add_integer_field(name, *, stored=True, indexed=True, multi_valued=False, doc_values=True)` | 64 ビット整数フィールド。`multi_valued=True` で整数配列を受け付け（範囲クエリは "any match"）。`doc_values` は上記を参照。 | +| `add_float_field(name, *, stored=True, indexed=True, multi_valued=False, doc_values=True)` | 64 ビット浮動小数点フィールド。`multi_valued=True` で浮動小数点配列を受け付け（範囲クエリは "any match"）。`doc_values` は上記を参照。 | +| `add_boolean_field(name, *, stored=True, indexed=True, doc_values=True)` | ブールフィールド。`doc_values` は上記を参照。 | +| `add_bytes_field(name, *, stored=True)` | 生バイトフィールド。`doc_values` オプションはありません —— `Bytes` の値は設定にかかわらず DocValues に一切書き込まれないためです。 | +| `add_geo_field(name, *, stored=True, indexed=True, doc_values=True)` | 地理座標フィールド（緯度/経度）。`doc_values` は上記を参照。 | +| `add_geo3d_field(name, *, stored=True, indexed=True, doc_values=True)` | 3D ECEF カルテシアン座標フィールド（x, y, z はメートル）。詳細は [Geo3d の概念](../concepts/geo3d.md)。`doc_values` は上記を参照。 | +| `add_datetime_field(name, *, stored=True, indexed=True, doc_values=True)` | UTC 日時フィールド。`doc_values` は上記を参照。 | | `add_hnsw_field(name, dimension, *, distance="cosine", m=16, ef_construction=200, quantizer=None, subvector_count=None, rerank_storage=None, embedder=None, pq_codebook_path=None, base_weight=1.0)` | HNSW 近似最近傍ベクトルフィールド。`base_weight` は他の vector フィールドと同時に検索されたときの相対的なスコアリング優先度（Issue #1084）。[ウェイト](../concepts/search/vector_search.md#ウェイト)を参照。 | | `add_flat_field(name, dimension, *, distance="cosine", embedder=None, base_weight=1.0)` | Flat（総当たり）ベクトルフィールド。 | | `add_ivf_field(name, dimension, *, distance="cosine", n_clusters=100, n_probe=1, embedder=None, base_weight=1.0)` | IVF 近似最近傍ベクトルフィールド。 | diff --git a/docs/ja/src/laurus-ruby/api_reference.md b/docs/ja/src/laurus-ruby/api_reference.md index 3a62311d..40ec36fd 100644 --- a/docs/ja/src/laurus-ruby/api_reference.md +++ b/docs/ja/src/laurus-ruby/api_reference.md @@ -155,14 +155,14 @@ Laurus::Schema.new | メソッド | 説明 | | :--- | :--- | -| `add_text_field(name, stored: true, indexed: true, term_vectors: true, analyzer: nil)` | 全文フィールド（転置インデックス、BM25）。`term_vectors:` はタームの位置を保存するかどうかを制御し、フレーズクエリ・スパンクエリが読み取ります。`analyzer:` にはパラメータ不要の組込名（`"standard"` / `"english"` / `"keyword"` / `"simple"` / `"noop"`、または `add_analyzer` で登録したカスタム名）を指定します。Lindera 辞書パスが必要な Japanese プリセットは、`lindera` tokenizer を含むカスタム analyzer として登録し、名前で参照してください。 | -| `add_integer_field(name, stored: true, indexed: true, multi_valued: false)` | 64 ビット整数フィールド。`multi_valued: true` で整数配列を受け付け（範囲クエリは "any match"）。 | -| `add_float_field(name, stored: true, indexed: true, multi_valued: false)` | 64 ビット浮動小数点フィールド。`multi_valued: true` で浮動小数点配列を受け付け（範囲クエリは "any match"）。 | -| `add_boolean_field(name, stored: true, indexed: true)` | ブールフィールド。 | -| `add_bytes_field(name, stored: true)` | 生バイトフィールド。 | -| `add_geo_field(name, stored: true, indexed: true)` | 地理座標フィールド（緯度/経度）。 | -| `add_geo3d_field(name, stored: true, indexed: true)` | 3D ECEF カルテシアン座標フィールド（x, y, z はメートル）。詳細は [Geo3d の概念](../concepts/geo3d.md)。 | -| `add_datetime_field(name, stored: true, indexed: true)` | UTC 日時フィールド。 | +| `add_text_field(name, stored: true, indexed: true, term_vectors: true, doc_values: true, analyzer: nil)` | 全文フィールド（転置インデックス、BM25）。`term_vectors:` はタームの位置を保存するかどうかを制御し、フレーズクエリ・スパンクエリが読み取ります。`doc_values:` は値を DocValues（ソート・ファセット・集計が読み取る列指向ストア）にもコピーするかどうかを制御します（Issue #1047）。`stored: true` の場合のみ有効です。`analyzer:` にはパラメータ不要の組込名（`"standard"` / `"english"` / `"keyword"` / `"simple"` / `"noop"`、または `add_analyzer` で登録したカスタム名）を指定します。Lindera 辞書パスが必要な Japanese プリセットは、`lindera` tokenizer を含むカスタム analyzer として登録し、名前で参照してください。 | +| `add_integer_field(name, stored: true, indexed: true, multi_valued: false, doc_values: true)` | 64 ビット整数フィールド。`multi_valued: true` で整数配列を受け付け（範囲クエリは "any match"）。`doc_values:` は上記を参照。 | +| `add_float_field(name, stored: true, indexed: true, multi_valued: false, doc_values: true)` | 64 ビット浮動小数点フィールド。`multi_valued: true` で浮動小数点配列を受け付け（範囲クエリは "any match"）。`doc_values:` は上記を参照。 | +| `add_boolean_field(name, stored: true, indexed: true, doc_values: true)` | ブールフィールド。`doc_values:` は上記を参照。 | +| `add_bytes_field(name, stored: true)` | 生バイトフィールド。`doc_values:` オプションはありません —— `Bytes` の値は設定にかかわらず DocValues に一切書き込まれないためです。 | +| `add_geo_field(name, stored: true, indexed: true, doc_values: true)` | 地理座標フィールド（緯度/経度）。`doc_values:` は上記を参照。 | +| `add_geo3d_field(name, stored: true, indexed: true, doc_values: true)` | 3D ECEF カルテシアン座標フィールド（x, y, z はメートル）。詳細は [Geo3d の概念](../concepts/geo3d.md)。`doc_values:` は上記を参照。 | +| `add_datetime_field(name, stored: true, indexed: true, doc_values: true)` | UTC 日時フィールド。`doc_values:` は上記を参照。 | | `add_hnsw_field(name, dimension, distance: "cosine", m: 16, ef_construction: 200, quantizer: nil, subvector_count: nil, rerank_storage: nil, embedder: nil, pq_codebook_path: nil, base_weight: 1.0)` | HNSW 近似最近傍ベクトルフィールド。`base_weight` は他の vector フィールドと同時に検索されたときの相対的なスコアリング優先度（Issue #1084）。[ウェイト](../concepts/search/vector_search.md#ウェイト)を参照。 | | `add_flat_field(name, dimension, distance: "cosine", embedder: nil, base_weight: 1.0)` | Flat（総当たり）ベクトルフィールド。 | | `add_ivf_field(name, dimension, distance: "cosine", n_clusters: 100, n_probe: 1, embedder: nil, base_weight: 1.0)` | IVF 近似最近傍ベクトルフィールド。 | diff --git a/docs/ja/src/laurus-server/grpc_api.md b/docs/ja/src/laurus-server/grpc_api.md index b37e1377..10b9088c 100644 --- a/docs/ja/src/laurus-server/grpc_api.md +++ b/docs/ja/src/laurus-server/grpc_api.md @@ -100,17 +100,19 @@ message AnalyzerDefinition { | Lexical フィールド | Vector フィールド | | :--- | :--- | -| `TextOption` (`indexed`, `stored`, `term_vectors`, `analyzer`) | `HnswOption` (`dimension`, `distance`, `m`, `ef_construction`, `base_weight`, `quantizer`, `embedder`, `rerank_storage`, `pq_codebook_path`) | -| `IntegerOption` (`indexed`, `stored`, `multi_valued`) | `FlatOption` (`dimension`, `distance`, `base_weight`, `quantizer`, `embedder`, `rerank_storage`) | -| `FloatOption` (`indexed`, `stored`, `multi_valued`) | `IvfOption` (`dimension`, `distance`, `n_clusters`, `n_probe`, `base_weight`, `quantizer`, `embedder`, `rerank_storage`) | -| `BooleanOption` (`indexed`, `stored`) | | -| `DateTimeOption` (`indexed`, `stored`) | | -| `GeoOption` (`indexed`, `stored`) | | -| `Geo3dOption` (`indexed`, `stored`) | | +| `TextOption` (`indexed`, `stored`, `term_vectors`, `doc_values`, `analyzer`) | `HnswOption` (`dimension`, `distance`, `m`, `ef_construction`, `base_weight`, `quantizer`, `embedder`, `rerank_storage`, `pq_codebook_path`) | +| `IntegerOption` (`indexed`, `stored`, `multi_valued`, `doc_values`) | `FlatOption` (`dimension`, `distance`, `base_weight`, `quantizer`, `embedder`, `rerank_storage`) | +| `FloatOption` (`indexed`, `stored`, `multi_valued`, `doc_values`) | `IvfOption` (`dimension`, `distance`, `n_clusters`, `n_probe`, `base_weight`, `quantizer`, `embedder`, `rerank_storage`) | +| `BooleanOption` (`indexed`, `stored`, `doc_values`) | | +| `DateTimeOption` (`indexed`, `stored`, `doc_values`) | | +| `GeoOption` (`indexed`, `stored`, `doc_values`) | | +| `Geo3dOption` (`indexed`, `stored`, `doc_values`) | | | `BytesOption` (`stored`) | | ベクトルフィールドオプションの `embedder` フィールドには、`Schema.embedders` で定義したエンベッダー名を指定します。設定すると、インデックス時にドキュメントのテキストフィールドからベクトルを自動生成します。事前計算済みのベクトルを直接供給する場合は空のままにします。 +**Doc values:** `doc_values`（Issue #1047）は、上記の `BytesOption` を除く全ての lexical オプションで `optional bool` であり、`term_vectors` と同じ tri-state の契約に従います。クライアントが省略するとエンジンのデフォルト（`true`）になり、明示的な `false` とは区別されます。フィールドの値を DocValues（ソート・ファセット・集計が読み取る列指向ストア）にもコピーするかどうかを制御し、DocValues 列が書き込まれるのは `stored` と `doc_values` の両方が `true` の場合のみです。`BytesOption` にはこのフィールドがありません —— `Bytes` の値は設定にかかわらず DocValues に一切書き込まれないためです。ソートにもファセットにも使わないフィールドで `doc_values` を無効にするとセグメントの使用容量が削減されます。フィールド自体は引き続き完全に検索・取得可能です。 + **距離メトリクス:** `COSINE`, `EUCLIDEAN`, `MANHATTAN`, `DOT_PRODUCT`, `ANGULAR` **量子化手法:** `SCALAR_8BIT`（デフォルト）, `PRODUCT_QUANTIZATION`（Issue #481 Stage 3。HNSW インデックスがサポート — Flat / IVF は書き込み時に拒否） diff --git a/docs/ja/src/laurus-wasm/api_reference.md b/docs/ja/src/laurus-wasm/api_reference.md index 833cbae0..750eaed8 100644 --- a/docs/ja/src/laurus-wasm/api_reference.md +++ b/docs/ja/src/laurus-wasm/api_reference.md @@ -356,45 +356,49 @@ for (let i = 0; i < 10000; i++) { ### メソッド -#### `addTextField(name, stored?, indexed?, termVectors?, analyzer?)` +#### `addTextField(name, stored?, indexed?, termVectors?, docValues?, analyzer?)` -全文検索テキストフィールドを追加します。`analyzer` にはパラメータ不要の -組込名（`"standard"` / `"english"` / `"keyword"` / `"simple"` / -`"noop"`）または `addAnalyzer()` で登録したランタイム analyzer 名を -指定します。 +全文検索テキストフィールドを追加します。`docValues` は値を DocValues +（ソート・ファセット・集計が読み取る列指向ストア）にもコピーするかどうかを +制御します（Issue #1047、デフォルト `true`）。`stored` も `true` の場合のみ +有効です。`analyzer` にはパラメータ不要の組込名（`"standard"` / +`"english"` / `"keyword"` / `"simple"` / `"noop"`）または `addAnalyzer()` +で登録したランタイム analyzer 名を指定します。 日本語の形態素解析を行う場合は、まず `JapaneseAnalyzer` を IPADIC の バイト列から構築し、`addAnalyzer()` で登録してください。 [`JapaneseAnalyzer.fromBytes`](#japaneseanalyzerfrombytesmetadata-dicttrie--mode) と [`addAnalyzer`](#addanalyzername-analyzer) を参照。 -#### `addIntegerField(name, stored?, indexed?, multiValued?)` +#### `addIntegerField(name, stored?, indexed?, multiValued?, docValues?)` 64 ビット整数フィールドを追加します。`multiValued: true` を指定すると整数配列を受け付け、 範囲クエリは**いずれかの値**が条件を満たせばマッチ（Lucene 流の "any match"、constant スコア）します。 +`docValues` は上記を参照。 -#### `addFloatField(name, stored?, indexed?, multiValued?)` +#### `addFloatField(name, stored?, indexed?, multiValued?, docValues?)` 64 ビット浮動小数点フィールドを追加します。`multiValued: true` を指定すると浮動小数点配列を受け付け、 範囲クエリは**いずれかの値**が条件を満たせばマッチ（Lucene 流の "any match"、constant スコア）します。 +`docValues` は上記を参照。 -#### `addBooleanField(name, stored?, indexed?)` +#### `addBooleanField(name, stored?, indexed?, docValues?)` -真偽値フィールドを追加します。 +真偽値フィールドを追加します。`docValues` は上記を参照。 -#### `addDatetimeField(name, stored?, indexed?)` +#### `addDatetimeField(name, stored?, indexed?, docValues?)` -日時フィールドを追加します。 +日時フィールドを追加します。`docValues` は上記を参照。 -#### `addGeoField(name, stored?, indexed?)` +#### `addGeoField(name, stored?, indexed?, docValues?)` -地理座標フィールドを追加します。 +地理座標フィールドを追加します。`docValues` は上記を参照。 -#### `addGeo3dField(name, stored?, indexed?)` +#### `addGeo3dField(name, stored?, indexed?, docValues?)` 3D ECEF カルテシアン座標フィールド（x, y, z はメートル）を追加します。値は `{ x, y, z }` オブジェクトで投入します。詳細は -[Geo3d の概念](../concepts/geo3d.md) を参照。 +[Geo3d の概念](../concepts/geo3d.md) を参照。`docValues` は上記を参照。 WASM バインディングは `Geo3dDistanceQuery` / `Geo3dBoundingBoxQuery` / `Geo3dNearestQuery` を JS クラスとして公開していません（wasm-bindgen は @@ -404,7 +408,8 @@ WASM バインディングは `Geo3dDistanceQuery` / `Geo3dBoundingBoxQuery` / #### `addBytesField(name, stored?)` -バイナリデータフィールドを追加します。 +バイナリデータフィールドを追加します。`docValues` オプションはありません +—— `Bytes` の値は設定にかかわらず DocValues に一切書き込まれないためです。 #### `addHnswField(name, dimension, distance?, m?, efConstruction?, defaultEfSearch?, embedder?, quantizer?, subvectorCount?, rerankStorage?, pqCodebookPath?, baseWeight?)` diff --git a/docs/ja/src/laurus/faceting.md b/docs/ja/src/laurus/faceting.md index fe2c6026..9c4ffe1f 100644 --- a/docs/ja/src/laurus/faceting.md +++ b/docs/ja/src/laurus/faceting.md @@ -82,7 +82,14 @@ Category ファセットカウントは stored document ではなく、各フィールドの **DocValues** 列から読み取られます。 収集された各ヒットについて、コレクターはファセットフィールドの値だけを per-field の DocValues -ルックアップで読むため、ファセット対象の全フィールドが DocValues 列を持つ場合（既定では、index 時に -全 stored field が DocValues に書かれるため常に成立）、stored fields blob 全体を decode / clone しません。 -DocValues を持たないフィールドは透過的に stored document へフォールバックするため、結果はどちらの経路でも +ルックアップで読むため、ファセット対象の全フィールドが DocValues 列を持つ場合（`stored: true` な +フィールドは既定でこれに該当します。ただし後述のとおり型によって除外される場合や、`doc_values` +オプションが明示的に `false` に設定されている場合を除きます）、stored fields blob 全体を +decode / clone しません。DocValues を持たないフィールド ―― オプトアウトしている、`stored` +ではない、あるいは `Bytes`/`Vector` の値（DocValues には設定にかかわらず一切格納されません） +であるため ―― は透過的に stored document へフォールバックするため、結果はどちらの経路でも 同一で、変わるのは読み取り経路だけです。 + +ソートにもファセットにも使わないフィールドで `doc_values: false` を設定すると、値が二重（stored +document と DocValues）ではなく一度（stored document のみ）しか書き込まれなくなるため、 +セグメントの使用容量が削減されます。 diff --git a/docs/src/concepts/schema_and_fields.md b/docs/src/concepts/schema_and_fields.md index 2a92f6f6..c902ef60 100644 --- a/docs/src/concepts/schema_and_fields.md +++ b/docs/src/concepts/schema_and_fields.md @@ -69,14 +69,15 @@ Lexical fields are indexed using an inverted index and support keyword-based que ```rust use laurus::lexical::TextOption; -// Default: indexed + stored + term vectors (all true) +// Default: indexed + stored + term vectors + doc values (all true) let opt = TextOption::default(); // Customize let opt = TextOption::default() .indexed(true) .stored(true) - .term_vectors(true); + .term_vectors(true) + .doc_values(true); ``` | Option | Default | Description | @@ -84,6 +85,17 @@ let opt = TextOption::default() | `indexed` | `true` | Whether the field is searchable | | `stored` | `true` | Whether the original value is stored for retrieval | | `term_vectors` | `true` | Whether term positions are stored (needed for phrase and span queries; highlighting always re-tokenizes the stored text and does not use them) | +| `doc_values` | `true` | Whether the value is also copied into DocValues, the column-oriented store [sorting](../laurus/faceting.md) and faceting/aggregation read from | + +`doc_values` is not unique to `TextOption` — every lexical field option except +`BytesOption` carries it (`IntegerOption`, `FloatOption`, `BooleanOption`, +`DateTimeOption`, `GeoOption`, `Geo3dOption`). `BytesOption` has no such +setting: a `Bytes` value is never written to DocValues regardless, since +neither sorting nor faceting can do anything with it. The effective rule is: +a DocValues column is written only when `stored` and `doc_values` are both +`true` (and the value's type isn't `Bytes`); setting `doc_values: false` on a +field that is never sorted or faceted on shrinks its segment footprint by +skipping the second copy. ### Vector Fields diff --git a/docs/src/laurus-cli/schema_format.md b/docs/src/laurus-cli/schema_format.md index ea3dd5ec..26f82669 100644 --- a/docs/src/laurus-cli/schema_format.md +++ b/docs/src/laurus-cli/schema_format.md @@ -43,6 +43,7 @@ Full-text searchable field. Text is processed by the analysis pipeline (tokeniza indexed = true # Whether to index this field for search stored = true # Whether to store the original value for retrieval term_vectors = true # Whether to store term positions (for phrase and span queries) +doc_values = true # Whether to also copy the value into DocValues (for sorting/faceting) ``` | Option | Type | Default | Description | @@ -50,6 +51,7 @@ term_vectors = true # Whether to store term positions (for phrase and span queri | `indexed` | `bool` | `true` | Enables searching this field | | `stored` | `bool` | `true` | Stores the original value so it can be returned in results | | `term_vectors` | `bool` | `true` | Stores term positions, read by phrase and span queries; highlighting always re-tokenizes the stored text and does not use them | +| `doc_values` | `bool` | `true` | Copies the value into DocValues, the column-oriented store [sorting](../laurus/faceting.md) and faceting/aggregation read from. Takes effect only when `stored` is also `true` — see [Common option: `doc_values`](#common-option-doc_values) below | #### Integer @@ -60,6 +62,7 @@ term_vectors = true # Whether to store term positions (for phrase and span queri indexed = true stored = true multi_valued = false +doc_values = true ``` | Option | Type | Default | Description | @@ -67,6 +70,7 @@ multi_valued = false | `indexed` | `bool` | `true` | Enables range and exact-match queries | | `stored` | `bool` | `true` | Stores the original value | | `multi_valued` | `bool` | `false` | Accept arrays of integers; range queries match if **any** value satisfies the predicate (Lucene-style "any match" with constant scoring) | +| `doc_values` | `bool` | `true` | See [Common option: `doc_values`](#common-option-doc_values) below | #### Float @@ -77,6 +81,7 @@ multi_valued = false indexed = true stored = true multi_valued = false +doc_values = true ``` | Option | Type | Default | Description | @@ -84,6 +89,7 @@ multi_valued = false | `indexed` | `bool` | `true` | Enables range queries | | `stored` | `bool` | `true` | Stores the original value | | `multi_valued` | `bool` | `false` | Accept arrays of floats; range queries match if **any** value satisfies the predicate (Lucene-style "any match" with constant scoring) | +| `doc_values` | `bool` | `true` | See [Common option: `doc_values`](#common-option-doc_values) below | #### Boolean @@ -93,12 +99,14 @@ Boolean field (`true` / `false`). [fields.published.Boolean] indexed = true stored = true +doc_values = true ``` | Option | Type | Default | Description | | :--- | :--- | :--- | :--- | | `indexed` | `bool` | `true` | Enables filtering by boolean value | | `stored` | `bool` | `true` | Stores the original value | +| `doc_values` | `bool` | `true` | See [Common option: `doc_values`](#common-option-doc_values) below | #### DateTime @@ -108,12 +116,14 @@ UTC timestamp field. Supports range queries. [fields.created_at.DateTime] indexed = true stored = true +doc_values = true ``` | Option | Type | Default | Description | | :--- | :--- | :--- | :--- | | `indexed` | `bool` | `true` | Enables range queries on date/time | | `stored` | `bool` | `true` | Stores the original value | +| `doc_values` | `bool` | `true` | See [Common option: `doc_values`](#common-option-doc_values) below | #### Geo @@ -123,12 +133,14 @@ Geographic point field (latitude/longitude). Supports radius and bounding box qu [fields.location.Geo] indexed = true stored = true +doc_values = true ``` | Option | Type | Default | Description | | :--- | :--- | :--- | :--- | | `indexed` | `bool` | `true` | Enables geo queries (radius, bounding box) | | `stored` | `bool` | `true` | Stores the original value | +| `doc_values` | `bool` | `true` | See [Common option: `doc_values`](#common-option-doc_values) below | #### Geo3d @@ -138,12 +150,14 @@ stored = true [fields.position.Geo3d] indexed = true stored = true +doc_values = true ``` | Option | Type | Default | Description | | :--- | :--- | :--- | :--- | | `indexed` | `bool` | `true` | Enables 3D geo queries (`geo3d_distance`, `geo3d_bbox`, `geo3d_nearest`) | | `stored` | `bool` | `true` | Stores the original `(x, y, z)` value | +| `doc_values` | `bool` | `true` | See [Common option: `doc_values`](#common-option-doc_values) below | #### Bytes @@ -158,6 +172,23 @@ stored = true | :--- | :--- | :--- | :--- | | `stored` | `bool` | `true` | Stores the binary data | +`BytesOption` has no `doc_values` setting: a `Bytes` value is never written to +DocValues regardless, since neither sorting nor faceting can do anything with +it. + +#### Common option: `doc_values` + +Every lexical field option above except `BytesOption` carries a `doc_values` +option, controlling whether the value is also copied into DocValues — the +column-oriented store [sorting](../laurus/faceting.md) and faceting/aggregation +read from. The effective rule: a DocValues column is written only when +`stored` and `doc_values` are both `true`. Setting `doc_values: false` with +`stored: false` is silently ignored (not an error). Turning `doc_values` off +for a field that is never sorted or faceted on shrinks its segment footprint, +since the value is then written once (to the stored document) instead of +twice; the field remains fully searchable and retrievable either way — sorting +and faceting on it simply fall back to the stored document. + ### Vector Fields Vector fields are indexed for approximate nearest neighbor (ANN) search. They require a `dimension` (the length of each vector) and a `distance` metric. diff --git a/docs/src/laurus-nodejs/api_reference.md b/docs/src/laurus-nodejs/api_reference.md index e316a37a..7719f1d8 100644 --- a/docs/src/laurus-nodejs/api_reference.md +++ b/docs/src/laurus-nodejs/api_reference.md @@ -194,14 +194,14 @@ class Schema { | Method | Description | | :--- | :--- | -| `addTextField(name, stored?, indexed?, termVectors?, analyzer?)` | Full-text field (inverted index, BM25). `analyzer` is the name of a parameter-less built-in (`"standard"`, `"english"`, `"keyword"`, `"simple"`, `"noop"`) or any custom name registered via `addAnalyzer`. For the parameterised Japanese preset (which requires a Lindera dictionary path), register a custom analyzer with a `lindera` tokenizer and reference it by name. | -| `addIntegerField(name, stored?, indexed?, multiValued?)` | 64-bit integer field. Pass `multiValued: true` to accept arrays of integers (range queries match if any value satisfies the predicate). | -| `addFloatField(name, stored?, indexed?, multiValued?)` | 64-bit float field. Pass `multiValued: true` to accept arrays of floats (range queries match if any value satisfies the predicate). | -| `addBooleanField(name, stored?, indexed?)` | Boolean field. | -| `addBytesField(name, stored?)` | Raw bytes field. | -| `addGeoField(name, stored?, indexed?)` | Geographic coordinate field. | -| `addGeo3dField(name, stored?, indexed?)` | 3D ECEF Cartesian point field (x, y, z in metres). See [Geo3d concepts](../concepts/geo3d.md). | -| `addDatetimeField(name, stored?, indexed?)` | UTC datetime field. | +| `addTextField(name, stored?, indexed?, termVectors?, docValues?, analyzer?)` | Full-text field (inverted index, BM25). `docValues` controls whether the value is also copied into DocValues, the column-oriented store sort/facet/aggregation read from (Issue #1047, default `true`); takes effect only when `stored` is also `true`. `analyzer` is the name of a parameter-less built-in (`"standard"`, `"english"`, `"keyword"`, `"simple"`, `"noop"`) or any custom name registered via `addAnalyzer`. For the parameterised Japanese preset (which requires a Lindera dictionary path), register a custom analyzer with a `lindera` tokenizer and reference it by name. | +| `addIntegerField(name, stored?, indexed?, multiValued?, docValues?)` | 64-bit integer field. Pass `multiValued: true` to accept arrays of integers (range queries match if any value satisfies the predicate). See `docValues` above. | +| `addFloatField(name, stored?, indexed?, multiValued?, docValues?)` | 64-bit float field. Pass `multiValued: true` to accept arrays of floats (range queries match if any value satisfies the predicate). See `docValues` above. | +| `addBooleanField(name, stored?, indexed?, docValues?)` | Boolean field. See `docValues` above. | +| `addBytesField(name, stored?)` | Raw bytes field. No `docValues` option: a `Bytes` value is never written to DocValues regardless. | +| `addGeoField(name, stored?, indexed?, docValues?)` | Geographic coordinate field. See `docValues` above. | +| `addGeo3dField(name, stored?, indexed?, docValues?)` | 3D ECEF Cartesian point field (x, y, z in metres). See [Geo3d concepts](../concepts/geo3d.md) and `docValues` above. | +| `addDatetimeField(name, stored?, indexed?, docValues?)` | UTC datetime field. See `docValues` above. | | `addHnswField(name, dimension, distance?, m?, efConstruction?, defaultEfSearch?, embedder?, quantizer?, subvectorCount?, rerankStorage?, pqCodebookPath?, baseWeight?)` | HNSW vector field. `baseWeight` sets this field's relative scoring priority when searched alongside other vector fields (Issue #1084); see [Vector Search → Weights](../concepts/search/vector_search.md#weights). | | `addFlatField(name, dimension, distance?, embedder?, baseWeight?)` | Flat (brute-force) vector field. | | `addIvfField(name, dimension, distance?, nClusters?, nProbe?, embedder?, baseWeight?)` | IVF vector field. | diff --git a/docs/src/laurus-php/api_reference.md b/docs/src/laurus-php/api_reference.md index e2403217..3865e0d2 100644 --- a/docs/src/laurus-php/api_reference.md +++ b/docs/src/laurus-php/api_reference.md @@ -161,14 +161,14 @@ new \Laurus\Schema() | Method | Description | | :--- | :--- | -| `addTextField(string $name, bool $stored = true, bool $indexed = true, bool $termVectors = false, ?string $analyzer = null): void` | Full-text field (inverted index, BM25). `$analyzer` is the name of a parameter-less built-in (`"standard"`, `"english"`, `"keyword"`, `"simple"`, `"noop"`) or a custom name registered via `addAnalyzer`. The Japanese preset requires a Lindera dictionary path, so register it as a custom analyzer with a `lindera` tokenizer and reference it by name. | -| `addIntegerField(string $name, bool $stored = true, bool $indexed = true, bool $multiValued = false): void` | 64-bit integer field. Pass `$multiValued = true` to accept arrays of integers (range queries match if any value satisfies the predicate). | -| `addFloatField(string $name, bool $stored = true, bool $indexed = true, bool $multiValued = false): void` | 64-bit float field. Pass `$multiValued = true` to accept arrays of floats (range queries match if any value satisfies the predicate). | -| `addBooleanField(string $name, bool $stored = true, bool $indexed = true): void` | Boolean field. | -| `addBytesField(string $name, bool $stored = true): void` | Raw bytes field. | -| `addGeoField(string $name, bool $stored = true, bool $indexed = true): void` | Geographic coordinate field (lat/lon). | -| `addGeo3dField(string $name, bool $stored = true, bool $indexed = true): void` | 3D ECEF Cartesian point field (x, y, z in metres). See [Geo3d concepts](../concepts/geo3d.md). | -| `addDatetimeField(string $name, bool $stored = true, bool $indexed = true): void` | UTC datetime field. | +| `addTextField(string $name, bool $stored = true, bool $indexed = true, bool $termVectors = true, bool $docValues = true, ?string $analyzer = null): void` | Full-text field (inverted index, BM25). `$docValues` controls whether the value is also copied into DocValues, the column-oriented store sort/facet/aggregation read from (Issue #1047); takes effect only when `$stored` is also `true`. `$analyzer` is the name of a parameter-less built-in (`"standard"`, `"english"`, `"keyword"`, `"simple"`, `"noop"`) or a custom name registered via `addAnalyzer`. The Japanese preset requires a Lindera dictionary path, so register it as a custom analyzer with a `lindera` tokenizer and reference it by name. | +| `addIntegerField(string $name, bool $stored = true, bool $indexed = true, bool $multiValued = false, bool $docValues = true): void` | 64-bit integer field. Pass `$multiValued = true` to accept arrays of integers (range queries match if any value satisfies the predicate). See `$docValues` above. | +| `addFloatField(string $name, bool $stored = true, bool $indexed = true, bool $multiValued = false, bool $docValues = true): void` | 64-bit float field. Pass `$multiValued = true` to accept arrays of floats (range queries match if any value satisfies the predicate). See `$docValues` above. | +| `addBooleanField(string $name, bool $stored = true, bool $indexed = true, bool $docValues = true): void` | Boolean field. See `$docValues` above. | +| `addBytesField(string $name, bool $stored = true): void` | Raw bytes field. No `$docValues` option: a `Bytes` value is never written to DocValues regardless. | +| `addGeoField(string $name, bool $stored = true, bool $indexed = true, bool $docValues = true): void` | Geographic coordinate field (lat/lon). See `$docValues` above. | +| `addGeo3dField(string $name, bool $stored = true, bool $indexed = true, bool $docValues = true): void` | 3D ECEF Cartesian point field (x, y, z in metres). See [Geo3d concepts](../concepts/geo3d.md) and `$docValues` above. | +| `addDatetimeField(string $name, bool $stored = true, bool $indexed = true, bool $docValues = true): void` | UTC datetime field. See `$docValues` above. | | `addHnswField(string $name, int $dimension, ?string $distance = "cosine", int $m = 16, int $efConstruction = 200, ?int $defaultEfSearch = null, ?string $embedder = null, ?string $quantizer = null, ?int $subvectorCount = null, ?string $rerankStorage = null, ?string $pqCodebookPath = null, float $baseWeight = 1.0): void` | HNSW approximate nearest-neighbor vector field. `$baseWeight` sets this field's relative scoring priority when searched alongside other vector fields (Issue #1084); see [Vector Search → Weights](../concepts/search/vector_search.md#weights). | | `addFlatField(string $name, int $dimension, ?string $distance = "cosine", ?string $embedder = null, float $baseWeight = 1.0): void` | Flat (brute-force) vector field. | | `addIvfField(string $name, int $dimension, ?string $distance = "cosine", int $nClusters = 100, int $nProbe = 1, ?string $embedder = null, float $baseWeight = 1.0): void` | IVF approximate nearest-neighbor vector field. | diff --git a/docs/src/laurus-python/api_reference.md b/docs/src/laurus-python/api_reference.md index bca938d4..b05dd649 100644 --- a/docs/src/laurus-python/api_reference.md +++ b/docs/src/laurus-python/api_reference.md @@ -189,14 +189,14 @@ class Schema: | Method | Description | | :--- | :--- | -| `add_text_field(name, *, stored=True, indexed=True, term_vectors=True, analyzer=None)` | Full-text field (inverted index, BM25). `term_vectors` controls whether term positions are stored, read by phrase and span queries. `analyzer` accepts a built-in name (`"standard"`, `"english"`, `"keyword"`, `"simple"`, `"noop"`, or any custom name registered via `add_analyzer`) or a dict configuring a parameterised preset such as `{"language": "japanese", "mode": "normal", "dict": "/var/lib/lindera/ipadic"}`. The bare string `"japanese"` is rejected because the preset requires a Lindera dictionary path. | -| `add_integer_field(name, *, stored=True, indexed=True, multi_valued=False)` | 64-bit integer field. Set `multi_valued=True` to accept arrays of integers (range queries match if any value satisfies the predicate). | -| `add_float_field(name, *, stored=True, indexed=True, multi_valued=False)` | 64-bit float field. Set `multi_valued=True` to accept arrays of floats (range queries match if any value satisfies the predicate). | -| `add_boolean_field(name, *, stored=True, indexed=True)` | Boolean field. | -| `add_bytes_field(name, *, stored=True)` | Raw bytes field. | -| `add_geo_field(name, *, stored=True, indexed=True)` | Geographic coordinate field (lat/lon). | -| `add_geo3d_field(name, *, stored=True, indexed=True)` | 3D ECEF Cartesian point field (x, y, z in metres). See [Geo3d concepts](../concepts/geo3d.md). | -| `add_datetime_field(name, *, stored=True, indexed=True)` | UTC datetime field. | +| `add_text_field(name, *, stored=True, indexed=True, term_vectors=True, doc_values=True, analyzer=None)` | Full-text field (inverted index, BM25). `term_vectors` controls whether term positions are stored, read by phrase and span queries. `doc_values` controls whether the value is also copied into DocValues, the column-oriented store sort/facet/aggregation read from (Issue #1047); takes effect only when `stored=True`. `analyzer` accepts a built-in name (`"standard"`, `"english"`, `"keyword"`, `"simple"`, `"noop"`, or any custom name registered via `add_analyzer`) or a dict configuring a parameterised preset such as `{"language": "japanese", "mode": "normal", "dict": "/var/lib/lindera/ipadic"}`. The bare string `"japanese"` is rejected because the preset requires a Lindera dictionary path. | +| `add_integer_field(name, *, stored=True, indexed=True, multi_valued=False, doc_values=True)` | 64-bit integer field. Set `multi_valued=True` to accept arrays of integers (range queries match if any value satisfies the predicate). See `doc_values` above. | +| `add_float_field(name, *, stored=True, indexed=True, multi_valued=False, doc_values=True)` | 64-bit float field. Set `multi_valued=True` to accept arrays of floats (range queries match if any value satisfies the predicate). See `doc_values` above. | +| `add_boolean_field(name, *, stored=True, indexed=True, doc_values=True)` | Boolean field. See `doc_values` above. | +| `add_bytes_field(name, *, stored=True)` | Raw bytes field. No `doc_values` option: a `Bytes` value is never written to DocValues regardless. | +| `add_geo_field(name, *, stored=True, indexed=True, doc_values=True)` | Geographic coordinate field (lat/lon). See `doc_values` above. | +| `add_geo3d_field(name, *, stored=True, indexed=True, doc_values=True)` | 3D ECEF Cartesian point field (x, y, z in metres). See [Geo3d concepts](../concepts/geo3d.md) and `doc_values` above. | +| `add_datetime_field(name, *, stored=True, indexed=True, doc_values=True)` | UTC datetime field. See `doc_values` above. | | `add_hnsw_field(name, dimension, *, distance="cosine", m=16, ef_construction=200, quantizer=None, subvector_count=None, rerank_storage=None, embedder=None, pq_codebook_path=None, base_weight=1.0)` | HNSW approximate nearest-neighbor vector field. `base_weight` sets this field's relative scoring priority when searched alongside other vector fields (Issue #1084); see [Vector Search → Weights](../concepts/search/vector_search.md#weights). | | `add_flat_field(name, dimension, *, distance="cosine", embedder=None, base_weight=1.0)` | Flat (brute-force) vector field. | | `add_ivf_field(name, dimension, *, distance="cosine", n_clusters=100, n_probe=1, embedder=None, base_weight=1.0)` | IVF approximate nearest-neighbor vector field. | diff --git a/docs/src/laurus-ruby/api_reference.md b/docs/src/laurus-ruby/api_reference.md index 1ab3ff67..cea179c5 100644 --- a/docs/src/laurus-ruby/api_reference.md +++ b/docs/src/laurus-ruby/api_reference.md @@ -153,14 +153,14 @@ Laurus::Schema.new | Method | Description | | :--- | :--- | -| `add_text_field(name, stored: true, indexed: true, term_vectors: true, analyzer: nil)` | Full-text field (inverted index, BM25). `term_vectors:` controls whether term positions are stored, read by phrase and span queries. `analyzer:` is the name of a parameter-less built-in (`"standard"`, `"english"`, `"keyword"`, `"simple"`, `"noop"`) or a custom name registered via `add_analyzer`. The Japanese preset requires a Lindera dictionary path, so register it as a custom analyzer with a `lindera` tokenizer and reference it by name. | -| `add_integer_field(name, stored: true, indexed: true, multi_valued: false)` | 64-bit integer field. Pass `multi_valued: true` to accept arrays of integers (range queries match if any value satisfies the predicate). | -| `add_float_field(name, stored: true, indexed: true, multi_valued: false)` | 64-bit float field. Pass `multi_valued: true` to accept arrays of floats (range queries match if any value satisfies the predicate). | -| `add_boolean_field(name, stored: true, indexed: true)` | Boolean field. | -| `add_bytes_field(name, stored: true)` | Raw bytes field. | -| `add_geo_field(name, stored: true, indexed: true)` | Geographic coordinate field (lat/lon). | -| `add_geo3d_field(name, stored: true, indexed: true)` | 3D ECEF Cartesian point field (x, y, z in metres). See [Geo3d concepts](../concepts/geo3d.md). | -| `add_datetime_field(name, stored: true, indexed: true)` | UTC datetime field. | +| `add_text_field(name, stored: true, indexed: true, term_vectors: true, doc_values: true, analyzer: nil)` | Full-text field (inverted index, BM25). `term_vectors:` controls whether term positions are stored, read by phrase and span queries. `doc_values:` controls whether the value is also copied into DocValues, the column-oriented store sort/facet/aggregation read from (Issue #1047); takes effect only when `stored: true`. `analyzer:` is the name of a parameter-less built-in (`"standard"`, `"english"`, `"keyword"`, `"simple"`, `"noop"`) or a custom name registered via `add_analyzer`. The Japanese preset requires a Lindera dictionary path, so register it as a custom analyzer with a `lindera` tokenizer and reference it by name. | +| `add_integer_field(name, stored: true, indexed: true, multi_valued: false, doc_values: true)` | 64-bit integer field. Pass `multi_valued: true` to accept arrays of integers (range queries match if any value satisfies the predicate). See `doc_values:` above. | +| `add_float_field(name, stored: true, indexed: true, multi_valued: false, doc_values: true)` | 64-bit float field. Pass `multi_valued: true` to accept arrays of floats (range queries match if any value satisfies the predicate). See `doc_values:` above. | +| `add_boolean_field(name, stored: true, indexed: true, doc_values: true)` | Boolean field. See `doc_values:` above. | +| `add_bytes_field(name, stored: true)` | Raw bytes field. No `doc_values:` option: a `Bytes` value is never written to DocValues regardless. | +| `add_geo_field(name, stored: true, indexed: true, doc_values: true)` | Geographic coordinate field (lat/lon). See `doc_values:` above. | +| `add_geo3d_field(name, stored: true, indexed: true, doc_values: true)` | 3D ECEF Cartesian point field (x, y, z in metres). See [Geo3d concepts](../concepts/geo3d.md) and `doc_values:` above. | +| `add_datetime_field(name, stored: true, indexed: true, doc_values: true)` | UTC datetime field. See `doc_values:` above. | | `add_hnsw_field(name, dimension, distance: "cosine", m: 16, ef_construction: 200, quantizer: nil, subvector_count: nil, rerank_storage: nil, embedder: nil, pq_codebook_path: nil, base_weight: 1.0)` | HNSW approximate nearest-neighbor vector field. `base_weight` sets this field's relative scoring priority when searched alongside other vector fields (Issue #1084); see [Vector Search → Weights](../concepts/search/vector_search.md#weights). | | `add_flat_field(name, dimension, distance: "cosine", embedder: nil, base_weight: 1.0)` | Flat (brute-force) vector field. | | `add_ivf_field(name, dimension, distance: "cosine", n_clusters: 100, n_probe: 1, embedder: nil, base_weight: 1.0)` | IVF approximate nearest-neighbor vector field. | diff --git a/docs/src/laurus-server/grpc_api.md b/docs/src/laurus-server/grpc_api.md index 3bc754ec..697b894b 100644 --- a/docs/src/laurus-server/grpc_api.md +++ b/docs/src/laurus-server/grpc_api.md @@ -100,17 +100,19 @@ Each `FieldOption` is a `oneof` with one of the following field types: | Lexical Fields | Vector Fields | | :--- | :--- | -| `TextOption` (`indexed`, `stored`, `term_vectors`, `analyzer`) | `HnswOption` (`dimension`, `distance`, `m`, `ef_construction`, `base_weight`, `quantizer`, `embedder`, `rerank_storage`, `pq_codebook_path`) | -| `IntegerOption` (`indexed`, `stored`, `multi_valued`) | `FlatOption` (`dimension`, `distance`, `base_weight`, `quantizer`, `embedder`, `rerank_storage`) | -| `FloatOption` (`indexed`, `stored`, `multi_valued`) | `IvfOption` (`dimension`, `distance`, `n_clusters`, `n_probe`, `base_weight`, `quantizer`, `embedder`, `rerank_storage`) | -| `BooleanOption` (`indexed`, `stored`) | | -| `DateTimeOption` (`indexed`, `stored`) | | -| `GeoOption` (`indexed`, `stored`) | | -| `Geo3dOption` (`indexed`, `stored`) | | +| `TextOption` (`indexed`, `stored`, `term_vectors`, `doc_values`, `analyzer`) | `HnswOption` (`dimension`, `distance`, `m`, `ef_construction`, `base_weight`, `quantizer`, `embedder`, `rerank_storage`, `pq_codebook_path`) | +| `IntegerOption` (`indexed`, `stored`, `multi_valued`, `doc_values`) | `FlatOption` (`dimension`, `distance`, `base_weight`, `quantizer`, `embedder`, `rerank_storage`) | +| `FloatOption` (`indexed`, `stored`, `multi_valued`, `doc_values`) | `IvfOption` (`dimension`, `distance`, `n_clusters`, `n_probe`, `base_weight`, `quantizer`, `embedder`, `rerank_storage`) | +| `BooleanOption` (`indexed`, `stored`, `doc_values`) | | +| `DateTimeOption` (`indexed`, `stored`, `doc_values`) | | +| `GeoOption` (`indexed`, `stored`, `doc_values`) | | +| `Geo3dOption` (`indexed`, `stored`, `doc_values`) | | | `BytesOption` (`stored`) | | The `embedder` field in vector options specifies the name of an embedder defined in `Schema.embedders`. When set, the server automatically generates vectors from document text fields at index time. Leave empty to supply pre-computed vectors directly. +**Doc values:** `doc_values` (Issue #1047) is `optional bool` on every lexical option above except `BytesOption`, following the same tri-state contract as `term_vectors`: a client that omits it gets the engine's default (`true`), distinguishable from an explicit `false`. It controls whether the field's value is also copied into DocValues, the column-oriented store sorting and faceting/aggregation read from — a DocValues column is written only when `stored` and `doc_values` are both `true`. `BytesOption` carries no such field: a `Bytes` value is never written to DocValues regardless. Turning `doc_values` off for a field that is never sorted or faceted on shrinks its segment footprint; the field remains fully searchable and retrievable either way. + **Distance metrics:** `COSINE`, `EUCLIDEAN`, `MANHATTAN`, `DOT_PRODUCT`, `ANGULAR` **Quantization methods:** `SCALAR_8BIT` (default), `PRODUCT_QUANTIZATION` (Issue #481 Stage 3; supported by the HNSW index — Flat / IVF reject it at write time). diff --git a/docs/src/laurus-wasm/api_reference.md b/docs/src/laurus-wasm/api_reference.md index cbff3bd7..558aa7db 100644 --- a/docs/src/laurus-wasm/api_reference.md +++ b/docs/src/laurus-wasm/api_reference.md @@ -355,46 +355,49 @@ Create an empty schema. ### Methods -#### `addTextField(name, stored?, indexed?, termVectors?, analyzer?)` +#### `addTextField(name, stored?, indexed?, termVectors?, docValues?, analyzer?)` -Add a full-text field. `analyzer` is the name of a parameter-less -built-in (`"standard"`, `"english"`, `"keyword"`, `"simple"`, `"noop"`) -or the name of a runtime analyzer registered via `addAnalyzer()`. +Add a full-text field. `docValues` controls whether the value is also +copied into DocValues, the column-oriented store sort/facet/aggregation +read from (Issue #1047, default `true`); takes effect only when `stored` +is also `true`. `analyzer` is the name of a parameter-less built-in +(`"standard"`, `"english"`, `"keyword"`, `"simple"`, `"noop"`) or the +name of a runtime analyzer registered via `addAnalyzer()`. For Japanese morphological analysis, build a `JapaneseAnalyzer` from raw IPADIC bytes and register it with `addAnalyzer()` first; see [`JapaneseAnalyzer.fromBytes`](#japaneseanalyzerfrombytesmetadata-dicttrie--mode) and [`addAnalyzer`](#addanalyzername-analyzer) below. -#### `addIntegerField(name, stored?, indexed?, multiValued?)` +#### `addIntegerField(name, stored?, indexed?, multiValued?, docValues?)` Add a 64-bit integer field. Pass `multiValued: true` to accept arrays of integers; range queries then match if any value satisfies the predicate -(Lucene-style "any match" with constant scoring). +(Lucene-style "any match" with constant scoring). See `docValues` above. -#### `addFloatField(name, stored?, indexed?, multiValued?)` +#### `addFloatField(name, stored?, indexed?, multiValued?, docValues?)` Add a 64-bit float field. Pass `multiValued: true` to accept arrays of floats; range queries then match if any value satisfies the predicate -(Lucene-style "any match" with constant scoring). +(Lucene-style "any match" with constant scoring). See `docValues` above. -#### `addBooleanField(name, stored?, indexed?)` +#### `addBooleanField(name, stored?, indexed?, docValues?)` -Add a boolean field. +Add a boolean field. See `docValues` above. -#### `addDatetimeField(name, stored?, indexed?)` +#### `addDatetimeField(name, stored?, indexed?, docValues?)` -Add a date/time field. +Add a date/time field. See `docValues` above. -#### `addGeoField(name, stored?, indexed?)` +#### `addGeoField(name, stored?, indexed?, docValues?)` -Add a geographic coordinate field. +Add a geographic coordinate field. See `docValues` above. -#### `addGeo3dField(name, stored?, indexed?)` +#### `addGeo3dField(name, stored?, indexed?, docValues?)` Add a 3D ECEF Cartesian point field. Values are submitted as a `{ x, y, z }` object with metres units. See [Geo3d concepts](../concepts/geo3d.md) for -ECEF theory. +ECEF theory, and `docValues` above. The WASM binding does not expose `Geo3dDistanceQuery` / `Geo3dBoundingBoxQuery` / `Geo3dNearestQuery` as JS classes (wasm-bindgen cannot expose `dyn Query` @@ -404,7 +407,8 @@ above. #### `addBytesField(name, stored?)` -Add a binary data field. +Add a binary data field. No `docValues` option: a `Bytes` value is never +written to DocValues regardless. #### `addHnswField(name, dimension, distance?, m?, efConstruction?, defaultEfSearch?, embedder?, quantizer?, subvectorCount?, rerankStorage?, pqCodebookPath?, baseWeight?)` diff --git a/docs/src/laurus/faceting.md b/docs/src/laurus/faceting.md index 0d986b53..202ad67f 100644 --- a/docs/src/laurus/faceting.md +++ b/docs/src/laurus/faceting.md @@ -84,6 +84,13 @@ Facet counts are read from each field's **DocValues** column, not from the stored document. For every collected hit the collector reads only the facet field's value via the per-field DocValues lookup, so it never decodes or clones the whole stored-fields blob when every faceted field has a DocValues column -(which is the default — every stored field is written to DocValues at index -time). A field that lacks DocValues transparently falls back to the stored -document, so results are identical either way; only the read path changes. +(the default for any `stored: true` field, unless its type is excluded — see +below — or its `doc_values` option is explicitly set to `false`). A field that +lacks DocValues — because it opted out, isn't stored, or is a `Bytes`/`Vector` +value, which DocValues never carries regardless of the setting — transparently +falls back to the stored document, so results are identical either way; only +the read path changes. + +Setting `doc_values: false` on a field that is never sorted or faceted on +shrinks its segment footprint, since the value is then written once (to the +stored document) instead of twice. From 2557427e7124e7e56e8091ca8ae7d3d78b94d8c0 Mon Sep 17 00:00:00 2001 From: Minoru Osuka Date: Sat, 12 Sep 2026 16:27:21 -0400 Subject: [PATCH 10/10] fix(lexical): add doc_values to the FieldOption enum's doctest example A second TextOption struct literal in a doc comment (the FieldOption enum's own example, distinct from the Field struct's example fixed in an earlier commit) was missed when doc_values was added to the struct, failing `cargo test -p laurus --doc` under --features embeddings-all. Refs #1047 --- laurus/src/lexical/core/field.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/laurus/src/lexical/core/field.rs b/laurus/src/lexical/core/field.rs index 737bce82..4e72eb26 100644 --- a/laurus/src/lexical/core/field.rs +++ b/laurus/src/lexical/core/field.rs @@ -736,6 +736,7 @@ impl Default for Geo3dOption { /// indexed: true, /// stored: true, /// term_vectors: true, +/// doc_values: true, /// analyzer: None, /// }); ///