New release

.gitignore

@@ -1,5 +1,10 @@
 /target
 zebo_test
 zebo_data_dir
+zebo_data_dir_perf
+zebo_data_dir_reload
+zebo_data_dir_simple
+my-folder
+
 zebo-*.tar.gz
 .data

Cargo.toml

@@ -21,8 +21,11 @@ name = "zebo-inspect"
 path = "src/bin/zebo-inspect.rs"
 
 [dependencies]
-clap = { version = "4.0", features = ["derive"] }
+clap = { version = "4", features = ["derive"] }
 tracing = "0.1.41"
 
 [dev-dependencies]
 tracing-subscriber = "0.3.20"
+
+[lints.rust]
+unexpected_cfgs = { level = "warn", check-cfg = ['cfg(coverage_nightly)'] }

README.md

@@ -30,24 +30,48 @@ fn main() {
     .expect("Failed to create Zebo instance");
 
     // Batch insertions
-    zebo.add_documents_batch(vec![
-        (1, b"Document 1".to_vec()),
-        (2, b"Document 2".to_vec()),
-        (3, b"Document 3".to_vec()),
-    ], 200, 1024)
-    .expect("Failed to add documents");
-
-    // Simple insertions
-    zebo.add_documents(vec![
-        (4, b"Document 4".repeat(100)),
-        (5, b"Document 5".to_vec()),
-    ])
-    .expect("Failed to add documents");
-
-    let mut docs = zebo.get_documents(vec![1, 3, 5])
-        .unwrap();
+    let space = zebo.reserve_space_for(
+        &[
+            (1, "Document 1"),
+            (2, "Document 2"),
+            (3, "Document 3"),
+        ],
+    )
+    .expect("Failed to reserve space for documents");
+    space.write_all().expect("Failed to write documents");
+
+    let mut docs = zebo
+        // We found 1 and 3, but not 5 (does not exist)
+        .get_documents(vec![1, 3, 5])
+        .expect("Failed to get documents");
     while let Some(Ok((doc_id, doc))) = docs.next() {
         println!("Document ({doc_id}): {:?}", String::from_utf8(doc));
     }
 }
 ```
+
+## Design choice
+
+You may wonder why there's no "insert_document" method.
+Instead, we have "reserve_space_for" method that reserves space for multiple documents at once.
+This is because reserving space for multiple documents at once allows Zebo to optimize file usage and minimize fragmentation.
+
+But, there's another important reason.
+Consider you have `RwLock<Zebo>` and you want to insert a lot of documents.
+Based on the current design, you can do:
+
+```rust
+use zebo::Zebo;
+use std::sync::RwLock;
+
+let zebo = Zebo::<50, 1024, u32>::try_new("./my-folder").unwrap();
+let zebo = RwLock::new(zebo);
+
+let mut lock = zebo.write().unwrap();
+let space = lock.reserve_space_for(&[(1_u32, "my content")]).unwrap();
+drop(lock); // Release the lock ASAP
+space.write_all().unwrap(); // Write the content outside the lock
+```
+
+So you can write documents without holding the lock.
+This because `space` points to a specific location in a specific file, and writing to that location does not require access to the `Zebo` instance itself.

examples/perf.rs

@@ -28,6 +28,10 @@ impl zebo::Document for Document {
         v.extend(ZERO);
         v.extend(self.data.as_bytes());
     }
+
+    fn len(&self) -> usize {
+        self.id.len() + 1 + self.data.len()
+    }
 }
 
 // 1 GB
@@ -38,7 +42,7 @@ fn main() {
     const LEN: usize = 64_000;
 
     // Prepare docs
-    let docs_single: Vec<_> = (0..LEN)
+    let docs: Vec<_> = (0..LEN)
         .map(|i| {
             (
                 DocumentId(1),
@@ -49,34 +53,21 @@ fn main() {
             )
         })
         .collect();
-    let docs_multi = docs_single.clone();
 
     // Run single
 
     println!("Run single");
-    let data_dir = "./zebo_data_dir/perf/1/single";
+    let data_dir = "./zebo_data_dir_perf";
     let _ = std::fs::remove_dir_all(data_dir);
     std::fs::create_dir_all(data_dir).unwrap();
     let mut zebo = Zebo::<MAX_DOC_PER_PAGE, PAGE_SIZE, DocumentId>::try_new(data_dir)
         .expect("Failed to create Zebo instance");
     let start = Instant::now();
 
-    zebo.add_documents(docs_single)
-        .expect("Failed to add documents");
-    println!("Elapsed {:?}", start.elapsed());
-    drop(zebo);
-
-    // Run multi
-
-    println!("Run multi");
-    let data_dir = "./zebo_data_dir/perf/1/multi";
-    let _ = std::fs::remove_dir_all(data_dir);
-    std::fs::create_dir_all(data_dir).unwrap();
-    let mut zebo = Zebo::<MAX_DOC_PER_PAGE, PAGE_SIZE, DocumentId>::try_new(data_dir)
-        .expect("Failed to create Zebo instance");
-    let start = Instant::now();
-    zebo.add_documents_batch(docs_multi, 200, 512)
-        .expect("Failed to add documents");
+    zebo.reserve_space_for(&docs)
+        .expect("Failed to add documents")
+        .write_all()
+        .expect("Failed to write all documents");
     println!("Elapsed {:?}", start.elapsed());
     drop(zebo);
 }

examples/reload.rs

@@ -25,24 +25,30 @@ impl zebo::Document for Document {
         v.extend(ZERO);
         v.extend(self.data.as_bytes());
     }
+
+    fn len(&self) -> usize {
+        self.id.len() + 1 + self.data.len()
+    }
 }
 
 // 1 GB
 static PAGE_SIZE: u64 = 1024 * 5;
 
 fn main() {
-    let data_dir = "./zebo_data_dir";
+    let data_dir = "./zebo_data_dir_reload";
     let mut zebo = Zebo::<5, PAGE_SIZE, DocumentId>::try_new(data_dir)
         .expect("Failed to create Zebo instance");
 
-    zebo.add_documents(vec![(
+    zebo.reserve_space_for(&[(
         DocumentId(1),
         Document {
             id: "Document 1".to_string(),
             data: "This is the content of document 1.".to_string(),
         },
     )])
-    .expect("Failed to add documents");
+    .expect("Failed to add documents")
+    .write_all()
+    .expect("Failed to write documents");
 
     let info_before = zebo.get_info().unwrap();
     drop(zebo);
@@ -53,14 +59,16 @@ fn main() {
     let info_after = zebo.get_info().unwrap();
     assert_eq!(info_before, info_after);
 
-    zebo.add_documents(vec![(
+    zebo.reserve_space_for(&[(
         DocumentId(4),
         Document {
             id: "Document 4".to_string(),
             data: "This is the content of document 4.".to_string(),
         },
     )])
-    .expect("Failed to add documents");
+    .expect("Failed to add documents")
+    .write_all()
+    .expect("Failed to write documents");
 
     let info = zebo.get_info();
     println!("Zebo Info: {info:#?}");
@@ -69,14 +77,16 @@ fn main() {
     let mut zebo = Zebo::<5, PAGE_SIZE, DocumentId>::try_new(data_dir)
         .expect("Failed to create Zebo instance");
 
-    zebo.add_documents(vec![(
+    zebo.reserve_space_for(&[(
         DocumentId(5),
         Document {
             id: "Document 5".to_string(),
             data: "This is the content of document 5.".to_string(),
         },
     )])
-    .expect("Failed to add documents");
+    .expect("Failed to add documents")
+    .write_all()
+    .expect("Failed to write documents");
 
     let info = zebo.get_info();
     println!("Zebo Info: {info:#?}");

examples/simple.rs

@@ -1,7 +1,7 @@
 use zebo::Zebo;
 
 fn main() {
-    let data_dir = "./zebo_data_dir";
+    let data_dir = "./zebo_data_dir_simple";
     let mut zebo = Zebo::<
         // Max number of documents per file
         10,
@@ -13,24 +13,13 @@ fn main() {
     .expect("Failed to create Zebo instance");
 
     // Batch insertions
-    zebo.add_documents_batch(
-        vec![
-            (1, b"Document 1".to_vec()),
-            (2, b"Document 2".to_vec()),
-            (3, b"Document 3".to_vec()),
-        ],
-        200,
-        1024,
-    )
-    .expect("Failed to add documents");
-
-    zebo.add_documents(vec![
-        (4, b"Document 4".repeat(100)),
-        (5, b"Document 5".to_vec()),
-    ])
-    .expect("Failed to add documents");
+    let space = zebo
+        .reserve_space_for(&[(1, "Document 1"), (2, "Document 2"), (3, "Document 3")])
+        .expect("Failed to reserve space for documents");
+    space.write_all().expect("Failed to write documents");
 
     let mut docs = zebo
+        // We found 1 and 3, but not 5 (does not exist)
         .get_documents(vec![1, 3, 5])
         .expect("Failed to get documents");
     while let Some(Ok((doc_id, doc))) = docs.next() {

src/error.rs

@@ -31,13 +31,20 @@ pub enum ZeboError {
     },
     OperationError(std::io::Error),
     UnexpectedPageId,
+    TooManyDocuments {
+        max: u32,
+        got: u32,
+    },
 }
 impl Display for ZeboError {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             ZeboError::UnsupportedVersion { version, wanted } => {
                 write!(f, "Unsupported version: {version}. Wanted: {wanted}")
             }
+            ZeboError::TooManyDocuments { max, got } => {
+                write!(f, "Too many documents: got {got}, max is {max}")
+            }
             ZeboError::CannotCreateBaseDir {
                 inner_error,
                 base_dir,