itdove/lore#16: Ingester framework: LoreIngester ABC + adapters

src/lore/cli.py

@@ -965,43 +965,38 @@ def _cmd_hook_capture(args: argparse.Namespace) -> int:
             file=out,
         )
 
-    return 0
-
-
-def _cmd_ingest(args: argparse.Namespace) -> int:
-    from lore.ingest.chunker import SUPPORTED_EXTENSIONS
-    from lore.ingest.doc import ingest_file
-    from lore.llm import get_llm_provider
+    # --- Auto-detect and run hook-triggered ingesters ---
+    try:
+        from lore.ingest.registry import detect_ingesters
 
-    file_path = Path(args.file)
-    if not file_path.exists():
-        print(f"File not found: {file_path}", file=sys.stderr)
-        return 1
+        ingest_cfg = cfg.ingest
+        if ingest_cfg.enabled:
+            project_dir = Path.cwd()
+            detected = detect_ingesters(
+                project_dir, store, disabled_sources=ingest_cfg.disabled_sources
+            )
+            hook_ingesters = [ing for ing in detected if "hook" in ing.triggers]
 
-    if file_path.suffix.lower() not in SUPPORTED_EXTENSIONS:
-        print(
-            f"Unsupported format: {file_path.suffix}. "
-            f"Supported: {', '.join(sorted(SUPPORTED_EXTENSIONS))}",
-            file=sys.stderr,
-        )
-        return 1
+            for ingester in hook_ingesters:
+                try:
+                    results = ingester.run()
+                    if results:
+                        print(
+                            f"  [INGEST] {ingester.name}: {len(results)} entries",
+                            file=out,
+                        )
+                except Exception as exc:
+                    print(
+                        f"  [INGEST] {ingester.name}: failed — {exc}",
+                        file=out,
+                    )
+    except Exception:
+        pass
 
-    try:
-        provider = get_llm_provider()
-    except Exception as exc:
-        print(f"LLM provider error: {exc}", file=sys.stderr)
-        return 1
+    return 0
 
-    store = _get_store()
-    entries = ingest_file(
-        file_path,
-        provider,
-        store,
-        level=args.level,
-        level_name=args.level_name,
-    )
 
-    print(f"Ingested {len(entries)} entries from {file_path}")
+def _print_entries(entries: list) -> None:
     for entry in entries:
         tags = f" [{entry.tags}]" if entry.tags else ""
         print(f"  {entry.key}{tags}")
@@ -1010,7 +1005,105 @@ def _cmd_ingest(args: argparse.Namespace) -> int:
             snippet += "..."
         print(f"    {snippet}")
 
-    return 0
+
+def _cmd_ingest(args: argparse.Namespace) -> int:
+    from lore.ingest.registry import detect_ingesters, get_ingester, list_ingesters
+
+    if getattr(args, "list_sources", False):
+        for name in list_ingesters():
+            print(f"  {name}")
+        return 0
+
+    if getattr(args, "detect_sources", False):
+        from lore.config.manager import get_global_config
+
+        store = _get_store()
+        cfg = get_global_config()
+        detected = detect_ingesters(
+            Path.cwd(), store, disabled_sources=cfg.ingest.disabled_sources
+        )
+        if not detected:
+            print("No ingestable sources detected in current directory.")
+        else:
+            print("Detected sources:")
+            for ing in detected:
+                print(f"  {ing.name}")
+        return 0
+
+    since = None
+    if getattr(args, "since", None):
+        from datetime import datetime
+
+        since = datetime.fromisoformat(args.since)
+
+    file_path = getattr(args, "file", None)
+    source_name = getattr(args, "source", None)
+
+    if file_path:
+        from lore.ingest.chunker import SUPPORTED_EXTENSIONS
+        from lore.ingest.doc import ingest_file
+        from lore.llm import get_llm_provider
+
+        file_path = Path(file_path)
+        if not file_path.exists():
+            print(f"File not found: {file_path}", file=sys.stderr)
+            return 1
+
+        if file_path.suffix.lower() not in SUPPORTED_EXTENSIONS:
+            print(
+                f"Unsupported format: {file_path.suffix}. "
+                f"Supported: {', '.join(sorted(SUPPORTED_EXTENSIONS))}",
+                file=sys.stderr,
+            )
+            return 1
+
+        try:
+            provider = get_llm_provider()
+        except Exception as exc:
+            print(f"LLM provider error: {exc}", file=sys.stderr)
+            return 1
+
+        store = _get_store()
+        entries = ingest_file(
+            file_path,
+            provider,
+            store,
+            level=args.level,
+            level_name=args.level_name,
+        )
+
+        print(f"Ingested {len(entries)} entries from {file_path}")
+        _print_entries(entries)
+        return 0
+
+    elif source_name:
+        store = _get_store()
+        project_dir = Path.cwd()
+
+        try:
+            ingester = get_ingester(source_name, store, project_dir)
+        except ValueError as exc:
+            print(str(exc), file=sys.stderr)
+            return 1
+
+        if not ingester.detect(project_dir):
+            print(
+                f"Source '{source_name}' not detected in {project_dir}",
+                file=sys.stderr,
+            )
+            return 1
+
+        entries = ingester.run(since=since)
+        print(f"Ingested {len(entries)} entries from {source_name}")
+        _print_entries(entries)
+        return 0
+
+    else:
+        print(
+            "Specify --file or --source. Use --list or --detect for options.",
+            file=sys.stderr,
+        )
+        return 1
 
 
 def main(argv: list[str] | None = None) -> None:
@@ -1101,16 +1194,33 @@ def main(argv: list[str] | None = None) -> None:
     hook_sub.add_parser("nudge", help="Mid-session nudge (PostToolUse)")
     hook_sub.add_parser("capture", help="Capture knowledge (SessionEnd)")
 
-    ingest_parser = sub.add_parser("ingest", help="Ingest document into knowledge base")
+    ingest_parser = sub.add_parser("ingest", help="Ingest knowledge from sources")
+    ingest_source = ingest_parser.add_mutually_exclusive_group()
+    ingest_source.add_argument("--file", type=Path, help="Path to document file")
+    ingest_source.add_argument(
+        "--source", help="Ingester name (e.g., reasonsforge, openwolf)"
+    )
     ingest_parser.add_argument(
-        "--file", required=True, type=Path, help="Path to document file"
+        "--since", default=None, help="Only ingest entries since date (ISO format)"
     )
     ingest_parser.add_argument(
         "--level", type=int, default=0, help="Knowledge level (default: 0/individual)"
     )
     ingest_parser.add_argument(
         "--level-name", default=None, help="Level name (e.g., team, org)"
     )
+    ingest_parser.add_argument(
+        "--list",
+        dest="list_sources",
+        action="store_true",
+        help="List available ingesters",
+    )
+    ingest_parser.add_argument(
+        "--detect",
+        dest="detect_sources",
+        action="store_true",
+        help="Auto-detect ingestable sources in current directory",
+    )
 
     args = parser.parse_args(argv)
 

src/lore/config/manager.py

@@ -14,6 +14,7 @@
     GitConfig,
     GlobalConfig,
     HierarchyLevel,
+    IngestConfig,
     KeyStructure,
     LLMConfig,
     ProjectConfig,
@@ -96,6 +97,7 @@ def get_global_config(project_dir: Path | None = None) -> GlobalConfig:
         git=_parse_sub_config(GitConfig, lore.get("git")),
         sync=_parse_sub_config(SyncConfig, lore.get("sync")),
         capture=capture,
+        ingest=_parse_sub_config(IngestConfig, lore.get("ingest")),
     )
 
 

src/lore/config/models.py

@@ -67,6 +67,12 @@ class CaptureConfig:
     auto_pr_shared: bool = False
 
 
+@dataclass
+class IngestConfig:
+    enabled: bool = True
+    disabled_sources: list[str] = field(default_factory=list)
+
+
 @dataclass
 class GlobalConfig:
     projects: list[str] = field(default_factory=list)
@@ -76,6 +82,7 @@ class GlobalConfig:
     git: GitConfig = field(default_factory=GitConfig)
     sync: SyncConfig = field(default_factory=SyncConfig)
     capture: CaptureConfig = field(default_factory=CaptureConfig)
+    ingest: IngestConfig = field(default_factory=IngestConfig)
 
 
 @dataclass

src/lore/ingest/__init__.py

@@ -1,10 +1,24 @@
 from lore.ingest.base import LoreIngester
 from lore.ingest.chunker import DocumentChunk, chunk_document
 from lore.ingest.doc import DocIngester
+from lore.ingest.openwolf import OpenWolfIngester
+from lore.ingest.reasonsforge import ReasonsForgeIngester
+from lore.ingest.registry import (
+    detect_ingesters,
+    get_ingester,
+    list_ingesters,
+    register,
+)
 
 __all__ = [
     "DocIngester",
     "DocumentChunk",
     "LoreIngester",
+    "OpenWolfIngester",
+    "ReasonsForgeIngester",
     "chunk_document",
+    "detect_ingesters",
+    "get_ingester",
+    "list_ingesters",
+    "register",
 ]

src/lore/ingest/base.py

@@ -1,25 +1,83 @@
 from __future__ import annotations
 
+import logging
 from abc import ABC, abstractmethod
 from datetime import datetime
 from pathlib import Path
-from typing import Literal
+from typing import ClassVar
 
-from lore.store.base import KnowledgeEntry
+from lore.store.base import KnowledgeEntry, StoreBackend
+
+log = logging.getLogger("lore.ingest")
 
 
 class LoreIngester(ABC):
-    trigger: Literal["hook", "scheduled", "manual"] = "manual"
-    review_policy: Literal["immediate", "pr_based"] = "immediate"
+    name: ClassVar[str] = "base"
+    triggers: ClassVar[frozenset[str]] = frozenset({"manual"})
+    review_policy: ClassVar[str] = "immediate"
+    auto_detect: ClassVar[bool] = True
+
+    def __init__(self, store: StoreBackend, project_dir: Path) -> None:
+        self._store = store
+        self._project_dir = project_dir
 
     @abstractmethod
     def detect(self, project_dir: Path) -> bool: ...
 
     @abstractmethod
     def extract_delta(self, since: datetime | None = None) -> list[KnowledgeEntry]: ...
 
-    @abstractmethod
-    def transform(self, entries: list[KnowledgeEntry]) -> list[KnowledgeEntry]: ...
+    def transform(self, entries: list[KnowledgeEntry]) -> list[KnowledgeEntry]:
+        return entries
 
-    @abstractmethod
-    def load(self, entries: list[KnowledgeEntry]) -> None: ...
+    def load(self, entries: list[KnowledgeEntry]) -> None:
+        for entry in entries:
+            self._store.store(entry)
+        self._store.commit()
+
+    def run(self, since: datetime | None = None) -> list[KnowledgeEntry]:
+        entries = self.extract_delta(since)
+        entries = self.transform(entries)
+        entries = self._dedup(entries)
+        self.load(entries)
+        return entries
+
+    def _dedup(self, entries: list[KnowledgeEntry]) -> list[KnowledgeEntry]:
+        try:
+            from lore.config.manager import get_global_config
+            from lore.embedding import get_embedding_provider
+            from lore.embedding.base import embed_to_blob
+
+            cfg = get_global_config()
+            if cfg.search.embedding_provider == "none":
+                return entries
+            provider = get_embedding_provider()
+            threshold = cfg.search.dedup_threshold
+        except Exception:
+            return entries
+
+        kept: list[KnowledgeEntry] = []
+        for entry in entries:
+            try:
+                emb = provider.embed(f"{entry.key} {entry.value}")
+                if not emb:
+                    kept.append(entry)
+                    continue
+                dupes = self._store.query_vector(
+                    emb, limit=1, min_similarity=1.0 - threshold
+                )
+                if dupes:
+                    closest, dist = dupes[0]
+                    if dist < threshold and closest.level == entry.level:
+                        log.info(
+                            "Dedup skip: %s matches %s (dist=%.4f)",
+                            entry.key,
+                            closest.key,
+                            dist,
+                        )
+                        continue
+                entry.embedding = embed_to_blob(emb)
+                kept.append(entry)
+            except Exception:
+                kept.append(entry)
+        return kept