itdove/lore#87: DocRepoIngester: avoid double file read in process_file

src/lore/ingest/chunker.py

@@ -228,9 +228,14 @@ def _split_large_chunks(
 
 
 def chunk_document(
-    file_path: Path, *, min_words: int = 100, max_words: int = 1000
+    file_path: Path,
+    *,
+    text: str | None = None,
+    min_words: int = 100,
+    max_words: int = 1000,
 ) -> list[DocumentChunk]:
-    text = file_path.read_text(encoding="utf-8", errors="replace")
+    if text is None:
+        text = file_path.read_text(encoding="utf-8", errors="replace")
     _, body = _strip_frontmatter(text)
 
     if not body.strip():

src/lore/ingest/doc.py

@@ -14,9 +14,9 @@
 log = logging.getLogger("lore.ingest")
 
 
-def extract_doc_chunks(provider: LLMProvider, file_path: Path):
+def extract_doc_chunks(provider: LLMProvider, file_path: Path, text: str | None = None):
     """Chunk file, LLM extract, validate keys. Yields (ext, tags, chunk)."""
-    chunks = chunk_document(file_path)
+    chunks = chunk_document(file_path, text=text)
     for chunk in chunks:
         extractions = provider.extract_from_chunk(
             chunk.text, chunk.heading, chunk.source_file

src/lore/ingest/doc_repo.py

@@ -66,10 +66,13 @@ def process_file(
     ) -> list[ParsedFile]:
         raw = file_path.read_bytes()
         content_hash = compute_content_hash(raw)
+        text = raw.decode("utf-8", errors="replace")
         rel = file_path.relative_to(repo_path).as_posix()
 
         parsed_files: list[ParsedFile] = []
-        for ext, tags, _chunk in extract_doc_chunks(self._provider, file_path):
+        for ext, tags, _chunk in extract_doc_chunks(
+            self._provider, file_path, text=text
+        ):
             parsed_files.append(
                 ParsedFile(
                     key=ext.key,

tests/unit/test_ingest_chunker.py

@@ -176,6 +176,16 @@ def test_chunk_document_end_to_end(tmp_path):
     assert chunks[0].word_count > 0
 
 
+def test_chunk_document_uses_supplied_text(tmp_path):
+    doc = tmp_path / "not-on-disk.md"
+    text = "# Supplied content\n" + "word " * 100
+
+    chunks = chunk_document(doc, text=text)
+
+    assert chunks[0].heading == "Supplied content"
+    assert chunks[0].text.startswith("word")
+
+
 def test_chunk_document_empty_file(tmp_path):
     doc = tmp_path / "empty.md"
     doc.write_text("")

tests/unit/test_ingest_doc_repo.py

@@ -125,6 +125,40 @@ def test_process_file_returns_parsed_files(tmp_path):
     assert result[0].locked is False
 
 
+def test_process_file_reads_file_once(tmp_path, monkeypatch):
+    doc = _write_doc(tmp_path / "guide.md")
+    extractions = [
+        DocChunkExtraction(
+            key="guide:test:intro",
+            summary="Test content.",
+        )
+    ]
+    ingester = DocRepoIngester(_MockProvider(extractions))
+
+    original_read_bytes = Path.read_bytes
+    original_read_text = Path.read_text
+    read_bytes_count = 0
+
+    def count_read_bytes(path, *args, **kwargs):
+        nonlocal read_bytes_count
+        if path == doc:
+            read_bytes_count += 1
+        return original_read_bytes(path, *args, **kwargs)
+
+    def fail_read_text(path, *args, **kwargs):
+        if path == doc:
+            raise AssertionError("process_file should pass decoded text to chunker")
+        return original_read_text(path, *args, **kwargs)
+
+    monkeypatch.setattr(Path, "read_bytes", count_read_bytes)
+    monkeypatch.setattr(Path, "read_text", fail_read_text)
+
+    result = ingester.process_file(doc, tmp_path)
+
+    assert len(result) == 1
+    assert read_bytes_count == 1
+
+
 def test_process_file_multiple_extractions(tmp_path):
     doc = _write_doc(tmp_path / "multi.md")
     extractions = [