itdove/lore#89: Capture: smart existing entry selection for LLM prompt

.github/pull_request_template.md

@@ -0,0 +1,49 @@
+## Description
+
+<!-- Summarize the change and the problem it solves. -->
+
+## Related issue (if applicable)
+
+<!-- If this PR addresses an existing issue, link it here using a closing keyword, for example: Closes #123. If no issue exists, write N/A. -->
+
+## Type of change
+
+- [ ] Bug fix
+- [ ] New feature
+- [ ] Security fix or detection change
+- [ ] Documentation or workflow change
+- [ ] Refactoring or maintenance
+
+## Security impact
+
+Select exactly one of the following mutually exclusive options:
+
+- [ ] This change has no security impact.
+- [ ] This change affects security behavior; describe the threat, protection, and user-visible behavior below.
+
+<!-- Include any relevant threat model, compatibility considerations, or UX contract coverage. -->
+
+## Testing
+
+<!-- List the related tests and any manual verification performed. -->
+
+- [ ] Tests added or updated where needed.
+- [ ] Related tests pass locally.
+- [ ] Manual verification completed, if applicable.
+
+## Commands run
+
+<!-- Record the exact commands run and relevant output or other evidence below. -->
+
+```text
+# Record command evidence here
+```
+
+## Checklist
+
+- [ ] The change is limited to this PR's purpose.
+- [ ] Documentation is updated where needed.
+- [ ] `CHANGELOG.md` is updated for notable changes, or no update is needed.
+- [ ] Backward compatibility was considered for the public SDK and configuration.
+- [ ] New dependencies have compatible licenses, if applicable.
+- [ ] UX contract tests are included when user-visible security behavior changes.

src/lore/capture.py

@@ -1,13 +1,16 @@
 from __future__ import annotations
 
 import logging
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
 
 from lore.llm.base import KnowledgeCandidate, LLMProvider
-from lore.store.base import StoreBackend
+from lore.store.base import KnowledgeEntry, StoreBackend
 
 logger = logging.getLogger(__name__)
 
+_RELEVANT_EXISTING_LIMIT = 20
+_FALLBACK_EXISTING_LIMIT = 50
+
 
 @dataclass
 class CaptureAction:
@@ -29,6 +32,49 @@ class CaptureResult:
     skipped: list[CaptureAction] = field(default_factory=list)
 
 
+def _keys_only(entries: list[KnowledgeEntry]) -> list[KnowledgeEntry]:
+    """Return prompt-safe copies that retain keys but omit entry values."""
+
+    return [replace(entry, value="") for entry in entries]
+
+
+def _select_existing_for_prompt(
+    transcript: str,
+    existing: list[KnowledgeEntry],
+    store: StoreBackend,
+    emb_provider,
+) -> list[KnowledgeEntry]:
+    """Select the bounded existing-knowledge context for capture prompts.
+
+    Exact duplicate and update checks still use the complete ``existing`` list.
+    This separate prompt context is intentionally bounded so a large knowledge
+    base does not consume the capture model's context window.
+    """
+
+    fallback = _keys_only(existing[-_FALLBACK_EXISTING_LIMIT:])
+    if not existing or emb_provider is None:
+        return fallback
+
+    try:
+        embedding = emb_provider.embed(transcript)
+        if not embedding:
+            return fallback
+
+        matches = store.query_vector(
+            embedding,
+            limit=_RELEVANT_EXISTING_LIMIT,
+            filter_levels=None,
+        )
+        relevant = [entry for entry, _distance in matches][:_RELEVANT_EXISTING_LIMIT]
+        return relevant or fallback
+    except Exception:
+        logger.debug(
+            "Existing-entry relevance search failed; using keys-only fallback",
+            exc_info=True,
+        )
+        return fallback
+
+
 def capture_knowledge(
     transcript: str,
     store: StoreBackend,
@@ -37,8 +83,29 @@ def capture_knowledge(
     capture_config=None,
 ) -> CaptureResult:
     existing = store.list_entries()
+    emb_provider = None
+    dedup_threshold = 0.20
+    try:
+        from lore.config.manager import get_global_config
+
+        cfg = get_global_config()
+        search_cfg = getattr(cfg, "search", None)
+        dedup_threshold = getattr(search_cfg, "dedup_threshold", dedup_threshold)
+        if getattr(search_cfg, "embedding_provider", "none") != "none":
+            from lore.embedding import get_embedding_provider
+
+            emb_provider = get_embedding_provider()
+    except Exception:
+        logger.debug("Could not initialize capture search configuration", exc_info=True)
+
+    prompt_existing = _select_existing_for_prompt(
+        transcript,
+        existing,
+        store,
+        emb_provider,
+    )
     candidates = provider.extract_knowledge(
-        transcript, existing, project_config=project_config
+        transcript, prompt_existing, project_config=project_config
     )
     result = CaptureResult(candidates=list(candidates))
 
@@ -55,20 +122,6 @@ def capture_knowledge(
 
     existing_by_key = {e.key: e for e in existing}
 
-    emb_provider = None
-    dedup_threshold = 0.20
-    try:
-        from lore.config.manager import get_global_config
-
-        cfg = get_global_config()
-        dedup_threshold = cfg.search.dedup_threshold
-        if cfg.search.embedding_provider != "none":
-            from lore.embedding import get_embedding_provider
-
-            emb_provider = get_embedding_provider()
-    except Exception:
-        pass
-
     for c in result.candidates:
         if c.negate_key:
             result.negations.append(c)

src/lore/llm/base.py

@@ -101,6 +101,9 @@ def extract_from_chunk(
     "Also identify entries from the existing knowledge list that are\n"
     "contradicted by this session's findings. Return as negations with\n"
     "negate_key and negate_reason fields.\n"
+    "Existing knowledge may be provided as relevant key/value snippets, or as\n"
+    "keys only when semantic search is unavailable. Use keys-only entries for\n"
+    "duplicate and negation checks without guessing their missing values.\n"
     "\n"
     "Skip: ephemeral task state, git history, conversation filler.\n"
     "Output as JSON array."

src/lore/llm/ollama.py

@@ -18,6 +18,8 @@
 log = logging.getLogger("lore.llm")
 
 _SMALL_MODEL_PATTERNS = ("phi", "qwen", ":1b", ":3b", ":7b", "gemma:2b")
+_CAPTURE_VALUE_SNIPPET_CHARS = 240
+_CAPTURE_FALLBACK_ENTRY_LIMIT = 50
 
 
 class OllamaProvider(LLMProvider):
@@ -72,7 +74,7 @@ def extract_knowledge(
     ) -> list[KnowledgeCandidate]:
         from lore.llm.base import build_capture_prompt
 
-        existing_text = "\n".join(f"- {e.key}" for e in existing[-50:])
+        existing_text = _format_capture_existing(existing)
         capture_prompt = build_capture_prompt(project_config)
         prompt = (
             f"{capture_prompt}\n\n"
@@ -102,6 +104,27 @@ def extract_from_chunk(
         return _parse_doc_extractions(raw)
 
 
+def _format_capture_existing(existing: list[KnowledgeEntry]) -> str:
+    """Format bounded capture context with optional short value snippets.
+
+    Capture passes value-less entries when semantic search is unavailable. In
+    that case, keeping only the key preserves duplicate and negation hints
+    without adding unbounded or low-quality context to the prompt.
+    """
+
+    lines = []
+    for entry in existing[-_CAPTURE_FALLBACK_ENTRY_LIMIT:]:
+        if not entry.value:
+            lines.append(f"- {entry.key}")
+            continue
+
+        value = " ".join(entry.value.split())
+        if len(value) > _CAPTURE_VALUE_SNIPPET_CHARS:
+            value = value[:_CAPTURE_VALUE_SNIPPET_CHARS].rstrip() + "..."
+        lines.append(f"- {entry.key}: {value}")
+    return "\n".join(lines)
+
+
 def _parse_doc_extractions(raw: str) -> list[DocChunkExtraction]:
     text = re.sub(r"```(?:json)?\s*", "", raw).strip()
     match = re.search(r"\[.*\]", text, re.DOTALL)

tests/unit/test_capture.py

@@ -1,5 +1,8 @@
 from __future__ import annotations
 
+from types import SimpleNamespace
+from unittest import mock
+
 import pytest
 
 from lore.capture import CaptureResult, capture_knowledge
@@ -65,6 +68,132 @@ def extract_from_chunk(self, chunk_text, heading, source_file):
     assert received["existing"][0].key == "k1"
 
 
+def test_capture_passes_relevant_existing_entries_to_provider(store, monkeypatch):
+    relevant = _make_entry(key="relevant:api:timeout", value="Use a 30 second timeout.")
+    unrelated = _make_entry(key="unrelated:ui:theme", value="Use the dark theme.")
+    store.store(relevant)
+    store.store(unrelated)
+    received = {}
+
+    class _SpyProvider(LLMProvider):
+        def synthesize(self, topic, candidates):
+            return ""
+
+        def extract_knowledge(self, transcript, existing, project_config=None):
+            received["existing"] = existing
+            return []
+
+        def extract_from_chunk(self, chunk_text, heading, source_file):
+            return []
+
+    embedding_provider = mock.Mock()
+    embedding_provider.embed.return_value = [0.1, 0.2]
+    config = SimpleNamespace(
+        search=SimpleNamespace(
+            embedding_provider="ollama",
+            dedup_threshold=0.2,
+        )
+    )
+    query_vector = mock.Mock(return_value=[(relevant, 0.05)])
+    monkeypatch.setattr("lore.config.manager.get_global_config", lambda: config)
+    monkeypatch.setattr(
+        "lore.embedding.get_embedding_provider",
+        lambda: embedding_provider,
+    )
+    monkeypatch.setattr(store, "query_vector", query_vector)
+
+    capture_knowledge("transcript about timeouts", store, _SpyProvider())
+
+    assert [entry.key for entry in received["existing"]] == [relevant.key]
+    assert received["existing"][0].value == relevant.value
+    embedding_provider.embed.assert_called_once_with("transcript about timeouts")
+    query_vector.assert_called_once_with(
+        [0.1, 0.2],
+        limit=20,
+        filter_levels=None,
+    )
+
+
+def test_capture_falls_back_to_keys_only_when_embedding_unavailable(store, monkeypatch):
+    first = _make_entry(key="first:key", value="first value")
+    second = _make_entry(key="second:key", value="second value")
+    store.store(first)
+    store.store(second)
+    received = {}
+
+    class _SpyProvider(LLMProvider):
+        def synthesize(self, topic, candidates):
+            return ""
+
+        def extract_knowledge(self, transcript, existing, project_config=None):
+            received["existing"] = existing
+            return []
+
+        def extract_from_chunk(self, chunk_text, heading, source_file):
+            return []
+
+    embedding_provider = mock.Mock()
+    embedding_provider.embed.return_value = []
+    config = SimpleNamespace(
+        search=SimpleNamespace(
+            embedding_provider="ollama",
+            dedup_threshold=0.2,
+        )
+    )
+    query_vector = mock.Mock()
+    monkeypatch.setattr("lore.config.manager.get_global_config", lambda: config)
+    monkeypatch.setattr(
+        "lore.embedding.get_embedding_provider",
+        lambda: embedding_provider,
+    )
+    monkeypatch.setattr(store, "query_vector", query_vector)
+
+    capture_knowledge("transcript", store, _SpyProvider())
+
+    assert [(entry.key, entry.value) for entry in received["existing"]] == [
+        (first.key, ""),
+        (second.key, ""),
+    ]
+    query_vector.assert_not_called()
+
+
+def test_capture_exact_duplicate_check_uses_all_existing_entries(store, monkeypatch):
+    exact_match = _make_entry(key="existing:exact", value="same value")
+    relevant = _make_entry(key="relevant:other", value="other value")
+    store.store(exact_match)
+    store.store(relevant)
+
+    config = SimpleNamespace(
+        search=SimpleNamespace(
+            embedding_provider="ollama",
+            dedup_threshold=0.2,
+        )
+    )
+    embedding_provider = mock.Mock()
+    embedding_provider.embed.return_value = [0.1, 0.2]
+    monkeypatch.setattr("lore.config.manager.get_global_config", lambda: config)
+    monkeypatch.setattr(
+        "lore.embedding.get_embedding_provider",
+        lambda: embedding_provider,
+    )
+    monkeypatch.setattr(
+        store,
+        "query_vector",
+        mock.Mock(return_value=[(relevant, 0.05)]),
+    )
+
+    result = capture_knowledge(
+        "transcript",
+        store,
+        _MockProvider(
+            [KnowledgeCandidate(key=exact_match.key, value=exact_match.value)]
+        ),
+    )
+
+    assert result.duplicates == [result.candidates[0]]
+    assert result.new == []
+
+
 # =====================================================================
 # AC5: Candidates include suggested_level
 # =====================================================================