From c0b756e12360aa6055611267c57fe1b812cd26fe Mon Sep 17 00:00:00 2001 From: itdove Date: Sun, 26 Jul 2026 10:28:11 -0400 Subject: [PATCH] feat: add ingester registry with auto-detection and dedup support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add ingester registry with detect/list/get pattern and plugin architecture - Add OpenWolf and ReasonsForge ingesters with hook-triggered capture - Integrate hook-triggered ingesters into capture command flow - Add dedup logic to base ingester to skip already-stored entries - Expand CLI ingest command with --source, --since, --list, --detect flags - Add config model for ingest settings (enabled, disabled_sources) 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- src/lore/cli.py | 180 +++++++++++++++---- src/lore/config/manager.py | 2 + src/lore/config/models.py | 7 + src/lore/ingest/__init__.py | 14 ++ src/lore/ingest/base.py | 74 +++++++- src/lore/ingest/doc.py | 18 +- src/lore/ingest/openwolf.py | 227 ++++++++++++++++++++++++ src/lore/ingest/reasonsforge.py | 131 ++++++++++++++ src/lore/ingest/registry.py | 52 ++++++ tests/unit/test_ingest_dedup.py | 171 ++++++++++++++++++ tests/unit/test_ingest_doc.py | 2 +- tests/unit/test_ingest_openwolf.py | 236 +++++++++++++++++++++++++ tests/unit/test_ingest_reasonsforge.py | 193 ++++++++++++++++++++ tests/unit/test_ingest_registry.py | 134 ++++++++++++++ 14 files changed, 1386 insertions(+), 55 deletions(-) create mode 100644 src/lore/ingest/openwolf.py create mode 100644 src/lore/ingest/reasonsforge.py create mode 100644 src/lore/ingest/registry.py create mode 100644 tests/unit/test_ingest_dedup.py create mode 100644 tests/unit/test_ingest_openwolf.py create mode 100644 tests/unit/test_ingest_reasonsforge.py create mode 100644 tests/unit/test_ingest_registry.py diff --git a/src/lore/cli.py b/src/lore/cli.py index cf19338..de7205a 100644 --- a/src/lore/cli.py +++ b/src/lore/cli.py @@ -965,43 +965,38 @@ def _cmd_hook_capture(args: argparse.Namespace) -> int: file=out, ) - return 0 - - -def _cmd_ingest(args: argparse.Namespace) -> int: - from lore.ingest.chunker import SUPPORTED_EXTENSIONS - from lore.ingest.doc import ingest_file - from lore.llm import get_llm_provider + # --- Auto-detect and run hook-triggered ingesters --- + try: + from lore.ingest.registry import detect_ingesters - file_path = Path(args.file) - if not file_path.exists(): - print(f"File not found: {file_path}", file=sys.stderr) - return 1 + ingest_cfg = cfg.ingest + if ingest_cfg.enabled: + project_dir = Path.cwd() + detected = detect_ingesters( + project_dir, store, disabled_sources=ingest_cfg.disabled_sources + ) + hook_ingesters = [ing for ing in detected if "hook" in ing.triggers] - if file_path.suffix.lower() not in SUPPORTED_EXTENSIONS: - print( - f"Unsupported format: {file_path.suffix}. " - f"Supported: {', '.join(sorted(SUPPORTED_EXTENSIONS))}", - file=sys.stderr, - ) - return 1 + for ingester in hook_ingesters: + try: + results = ingester.run() + if results: + print( + f" [INGEST] {ingester.name}: {len(results)} entries", + file=out, + ) + except Exception as exc: + print( + f" [INGEST] {ingester.name}: failed — {exc}", + file=out, + ) + except Exception: + pass - try: - provider = get_llm_provider() - except Exception as exc: - print(f"LLM provider error: {exc}", file=sys.stderr) - return 1 + return 0 - store = _get_store() - entries = ingest_file( - file_path, - provider, - store, - level=args.level, - level_name=args.level_name, - ) - print(f"Ingested {len(entries)} entries from {file_path}") +def _print_entries(entries: list) -> None: for entry in entries: tags = f" [{entry.tags}]" if entry.tags else "" print(f" {entry.key}{tags}") @@ -1010,7 +1005,105 @@ def _cmd_ingest(args: argparse.Namespace) -> int: snippet += "..." print(f" {snippet}") - return 0 + +def _cmd_ingest(args: argparse.Namespace) -> int: + from lore.ingest.registry import detect_ingesters, get_ingester, list_ingesters + + if getattr(args, "list_sources", False): + for name in list_ingesters(): + print(f" {name}") + return 0 + + if getattr(args, "detect_sources", False): + from lore.config.manager import get_global_config + + store = _get_store() + cfg = get_global_config() + detected = detect_ingesters( + Path.cwd(), store, disabled_sources=cfg.ingest.disabled_sources + ) + if not detected: + print("No ingestable sources detected in current directory.") + else: + print("Detected sources:") + for ing in detected: + print(f" {ing.name}") + return 0 + + since = None + if getattr(args, "since", None): + from datetime import datetime + + since = datetime.fromisoformat(args.since) + + file_path = getattr(args, "file", None) + source_name = getattr(args, "source", None) + + if file_path: + from lore.ingest.chunker import SUPPORTED_EXTENSIONS + from lore.ingest.doc import ingest_file + from lore.llm import get_llm_provider + + file_path = Path(file_path) + if not file_path.exists(): + print(f"File not found: {file_path}", file=sys.stderr) + return 1 + + if file_path.suffix.lower() not in SUPPORTED_EXTENSIONS: + print( + f"Unsupported format: {file_path.suffix}. " + f"Supported: {', '.join(sorted(SUPPORTED_EXTENSIONS))}", + file=sys.stderr, + ) + return 1 + + try: + provider = get_llm_provider() + except Exception as exc: + print(f"LLM provider error: {exc}", file=sys.stderr) + return 1 + + store = _get_store() + entries = ingest_file( + file_path, + provider, + store, + level=args.level, + level_name=args.level_name, + ) + + print(f"Ingested {len(entries)} entries from {file_path}") + _print_entries(entries) + return 0 + + elif source_name: + store = _get_store() + project_dir = Path.cwd() + + try: + ingester = get_ingester(source_name, store, project_dir) + except ValueError as exc: + print(str(exc), file=sys.stderr) + return 1 + + if not ingester.detect(project_dir): + print( + f"Source '{source_name}' not detected in {project_dir}", + file=sys.stderr, + ) + return 1 + + entries = ingester.run(since=since) + print(f"Ingested {len(entries)} entries from {source_name}") + _print_entries(entries) + return 0 + + else: + print( + "Specify --file or --source. Use --list or --detect for options.", + file=sys.stderr, + ) + return 1 def main(argv: list[str] | None = None) -> None: @@ -1101,9 +1194,14 @@ def main(argv: list[str] | None = None) -> None: hook_sub.add_parser("nudge", help="Mid-session nudge (PostToolUse)") hook_sub.add_parser("capture", help="Capture knowledge (SessionEnd)") - ingest_parser = sub.add_parser("ingest", help="Ingest document into knowledge base") + ingest_parser = sub.add_parser("ingest", help="Ingest knowledge from sources") + ingest_source = ingest_parser.add_mutually_exclusive_group() + ingest_source.add_argument("--file", type=Path, help="Path to document file") + ingest_source.add_argument( + "--source", help="Ingester name (e.g., reasonsforge, openwolf)" + ) ingest_parser.add_argument( - "--file", required=True, type=Path, help="Path to document file" + "--since", default=None, help="Only ingest entries since date (ISO format)" ) ingest_parser.add_argument( "--level", type=int, default=0, help="Knowledge level (default: 0/individual)" @@ -1111,6 +1209,18 @@ def main(argv: list[str] | None = None) -> None: ingest_parser.add_argument( "--level-name", default=None, help="Level name (e.g., team, org)" ) + ingest_parser.add_argument( + "--list", + dest="list_sources", + action="store_true", + help="List available ingesters", + ) + ingest_parser.add_argument( + "--detect", + dest="detect_sources", + action="store_true", + help="Auto-detect ingestable sources in current directory", + ) args = parser.parse_args(argv) diff --git a/src/lore/config/manager.py b/src/lore/config/manager.py index 49dd680..8e63c0e 100644 --- a/src/lore/config/manager.py +++ b/src/lore/config/manager.py @@ -14,6 +14,7 @@ GitConfig, GlobalConfig, HierarchyLevel, + IngestConfig, KeyStructure, LLMConfig, ProjectConfig, @@ -96,6 +97,7 @@ def get_global_config(project_dir: Path | None = None) -> GlobalConfig: git=_parse_sub_config(GitConfig, lore.get("git")), sync=_parse_sub_config(SyncConfig, lore.get("sync")), capture=capture, + ingest=_parse_sub_config(IngestConfig, lore.get("ingest")), ) diff --git a/src/lore/config/models.py b/src/lore/config/models.py index ed51c0a..f6b4154 100644 --- a/src/lore/config/models.py +++ b/src/lore/config/models.py @@ -67,6 +67,12 @@ class CaptureConfig: auto_pr_shared: bool = False +@dataclass +class IngestConfig: + enabled: bool = True + disabled_sources: list[str] = field(default_factory=list) + + @dataclass class GlobalConfig: projects: list[str] = field(default_factory=list) @@ -76,6 +82,7 @@ class GlobalConfig: git: GitConfig = field(default_factory=GitConfig) sync: SyncConfig = field(default_factory=SyncConfig) capture: CaptureConfig = field(default_factory=CaptureConfig) + ingest: IngestConfig = field(default_factory=IngestConfig) @dataclass diff --git a/src/lore/ingest/__init__.py b/src/lore/ingest/__init__.py index c88f3c0..6467e18 100644 --- a/src/lore/ingest/__init__.py +++ b/src/lore/ingest/__init__.py @@ -1,10 +1,24 @@ from lore.ingest.base import LoreIngester from lore.ingest.chunker import DocumentChunk, chunk_document from lore.ingest.doc import DocIngester +from lore.ingest.openwolf import OpenWolfIngester +from lore.ingest.reasonsforge import ReasonsForgeIngester +from lore.ingest.registry import ( + detect_ingesters, + get_ingester, + list_ingesters, + register, +) __all__ = [ "DocIngester", "DocumentChunk", "LoreIngester", + "OpenWolfIngester", + "ReasonsForgeIngester", "chunk_document", + "detect_ingesters", + "get_ingester", + "list_ingesters", + "register", ] diff --git a/src/lore/ingest/base.py b/src/lore/ingest/base.py index e507ef9..8ffd89a 100644 --- a/src/lore/ingest/base.py +++ b/src/lore/ingest/base.py @@ -1,16 +1,25 @@ from __future__ import annotations +import logging from abc import ABC, abstractmethod from datetime import datetime from pathlib import Path -from typing import Literal +from typing import ClassVar -from lore.store.base import KnowledgeEntry +from lore.store.base import KnowledgeEntry, StoreBackend + +log = logging.getLogger("lore.ingest") class LoreIngester(ABC): - trigger: Literal["hook", "scheduled", "manual"] = "manual" - review_policy: Literal["immediate", "pr_based"] = "immediate" + name: ClassVar[str] = "base" + triggers: ClassVar[frozenset[str]] = frozenset({"manual"}) + review_policy: ClassVar[str] = "immediate" + auto_detect: ClassVar[bool] = True + + def __init__(self, store: StoreBackend, project_dir: Path) -> None: + self._store = store + self._project_dir = project_dir @abstractmethod def detect(self, project_dir: Path) -> bool: ... @@ -18,8 +27,57 @@ def detect(self, project_dir: Path) -> bool: ... @abstractmethod def extract_delta(self, since: datetime | None = None) -> list[KnowledgeEntry]: ... - @abstractmethod - def transform(self, entries: list[KnowledgeEntry]) -> list[KnowledgeEntry]: ... + def transform(self, entries: list[KnowledgeEntry]) -> list[KnowledgeEntry]: + return entries - @abstractmethod - def load(self, entries: list[KnowledgeEntry]) -> None: ... + def load(self, entries: list[KnowledgeEntry]) -> None: + for entry in entries: + self._store.store(entry) + self._store.commit() + + def run(self, since: datetime | None = None) -> list[KnowledgeEntry]: + entries = self.extract_delta(since) + entries = self.transform(entries) + entries = self._dedup(entries) + self.load(entries) + return entries + + def _dedup(self, entries: list[KnowledgeEntry]) -> list[KnowledgeEntry]: + try: + from lore.config.manager import get_global_config + from lore.embedding import get_embedding_provider + from lore.embedding.base import embed_to_blob + + cfg = get_global_config() + if cfg.search.embedding_provider == "none": + return entries + provider = get_embedding_provider() + threshold = cfg.search.dedup_threshold + except Exception: + return entries + + kept: list[KnowledgeEntry] = [] + for entry in entries: + try: + emb = provider.embed(f"{entry.key} {entry.value}") + if not emb: + kept.append(entry) + continue + dupes = self._store.query_vector( + emb, limit=1, min_similarity=1.0 - threshold + ) + if dupes: + closest, dist = dupes[0] + if dist < threshold and closest.level == entry.level: + log.info( + "Dedup skip: %s matches %s (dist=%.4f)", + entry.key, + closest.key, + dist, + ) + continue + entry.embedding = embed_to_blob(emb) + kept.append(entry) + except Exception: + kept.append(entry) + return kept diff --git a/src/lore/ingest/doc.py b/src/lore/ingest/doc.py index 9a4eae0..cb65bb7 100644 --- a/src/lore/ingest/doc.py +++ b/src/lore/ingest/doc.py @@ -7,15 +7,19 @@ from lore.ingest.base import LoreIngester from lore.ingest.chunker import SUPPORTED_EXTENSIONS, chunk_document +from lore.ingest.registry import register from lore.llm.base import LLMProvider from lore.store.base import KnowledgeEntry, StoreBackend, validate_key log = logging.getLogger("lore.ingest") +@register class DocIngester(LoreIngester): - trigger = "manual" + name = "doc" + triggers = frozenset({"manual"}) review_policy = "pr_based" + auto_detect = False def __init__( self, @@ -25,9 +29,9 @@ def __init__( level: int = 0, level_name: str | None = None, ) -> None: + super().__init__(store, file_path.parent) self._file_path = file_path self._provider = provider - self._store = store self._level = level self._level_name = level_name @@ -66,21 +70,13 @@ def extract_delta(self, since: datetime | None = None) -> list[KnowledgeEntry]: level=self._level, level_name=self._level_name, tags=tags, - ingested_from="doc-ingester", + ingested_from="doc", provenance=provenance, ) ) return entries - def transform(self, entries: list[KnowledgeEntry]) -> list[KnowledgeEntry]: - return entries - - def load(self, entries: list[KnowledgeEntry]) -> None: - for entry in entries: - self._store.store(entry) - self._store.commit() - def ingest_file( file_path: Path, diff --git a/src/lore/ingest/openwolf.py b/src/lore/ingest/openwolf.py new file mode 100644 index 0000000..1a8478c --- /dev/null +++ b/src/lore/ingest/openwolf.py @@ -0,0 +1,227 @@ +from __future__ import annotations + +import json +import logging +import re +from datetime import datetime +from pathlib import Path + +from lore.ingest.base import LoreIngester +from lore.ingest.registry import register +from lore.store.base import KnowledgeEntry, StoreBackend, validate_key + +log = logging.getLogger("lore.ingest") + +_SECTION_MAP = { + "User Preferences": "preference", + "Key Learnings": "learning", + "Do-Not-Repeat": "do-not-repeat", + "Decision Log": "decision", +} + +_HTML_COMMENT = re.compile(r"", re.DOTALL) +_DATE_PREFIX = re.compile(r"^\[(\d{4}-\d{2}-\d{2})\]\s*") + + +@register +class OpenWolfIngester(LoreIngester): + """Ingest knowledge from an OpenWolf .wolf/ directory. + + Reads cerebrum.md (structured learnings/preferences/decisions) + and buglog.json (bug records with root cause and fix). + """ + + name = "openwolf" + triggers = frozenset({"hook"}) + review_policy = "pr_based" + + def __init__(self, store: StoreBackend, project_dir: Path) -> None: + super().__init__(store, project_dir) + self._wolf_dir = project_dir / ".wolf" + + def detect(self, project_dir: Path) -> bool: + wolf = project_dir / ".wolf" + return wolf.is_dir() and (wolf / "cerebrum.md").is_file() + + def extract_delta(self, since: datetime | None = None) -> list[KnowledgeEntry]: + entries: list[KnowledgeEntry] = [] + entries.extend(self._extract_cerebrum(since)) + entries.extend(self._extract_buglog(since)) + return entries + + def _extract_cerebrum(self, since: datetime | None = None) -> list[KnowledgeEntry]: + path = self._wolf_dir / "cerebrum.md" + if not path.exists(): + return [] + + text = path.read_text(encoding="utf-8", errors="replace") + text = _HTML_COMMENT.sub("", text) + sections = self._parse_sections(text) + entries: list[KnowledgeEntry] = [] + + for section_name, items in sections.items(): + key_prefix = _SECTION_MAP.get(section_name) + if not key_prefix: + continue + + for i, item in enumerate(items): + if not item.strip(): + continue + + date_match = _DATE_PREFIX.match(item) + if since and date_match: + try: + item_date = datetime.fromisoformat(date_match.group(1)) + if item_date < since: + continue + except ValueError: + pass + + slug = self._slugify(item) + if not slug: + continue + + key = f"{key_prefix}:{slug}" + if validate_key(key) is not None: + log.warning("Skipping invalid key: %s", key) + continue + + provenance = json.dumps( + { + "source": "cerebrum.md", + "section": section_name, + "item_index": i, + } + ) + + entries.append( + KnowledgeEntry( + key=key, + value=item.strip(), + level=0, + tags=key_prefix, + ingested_from="openwolf", + provenance=provenance, + ) + ) + + return entries + + def _extract_buglog(self, since: datetime | None = None) -> list[KnowledgeEntry]: + path = self._wolf_dir / "buglog.json" + if not path.exists(): + return [] + + try: + data = json.loads(path.read_text(encoding="utf-8", errors="replace")) + except (json.JSONDecodeError, OSError): + log.warning("Failed to parse buglog.json") + return [] + + bugs = data.get("bugs", []) + if not isinstance(bugs, list): + return [] + + entries: list[KnowledgeEntry] = [] + for bug in bugs: + if not isinstance(bug, dict): + continue + + bug_id = bug.get("id", "") + if since and bug.get("timestamp"): + try: + bug_date = datetime.fromisoformat(bug.get("timestamp")) + if bug_date < since: + continue + except ValueError: + pass + + slug = ( + str(bug_id).strip() + if bug_id + else self._slugify( + bug.get("error_message", bug.get("root_cause", "unknown")) + ) + ) + key = f"bug:{slug}" + if validate_key(key) is not None: + log.warning("Skipping invalid bug key: %s", key) + continue + + parts = [] + if bug.get("error_message"): + parts.append(f"Error: {bug['error_message']}") + if bug.get("root_cause"): + parts.append(f"Root cause: {bug['root_cause']}") + if bug.get("fix"): + parts.append(f"Fix: {bug['fix']}") + if bug.get("file"): + parts.append(f"File: {bug['file']}") + + value = "\n".join(parts) if parts else str(bug) + if not value.strip(): + continue + + tags_list = bug.get("tags", []) + tags = ( + ",".join(tags_list) + if isinstance(tags_list, list) and tags_list + else "bug" + ) + + provenance = json.dumps( + { + "source": "buglog.json", + "bug_id": str(bug_id), + } + ) + + entries.append( + KnowledgeEntry( + key=key, + value=value, + level=0, + tags=tags, + ingested_from="openwolf", + provenance=provenance, + ) + ) + + return entries + + @staticmethod + def _parse_sections(text: str) -> dict[str, list[str]]: + sections: dict[str, list[str]] = {} + current_section: str | None = None + current_items: list[str] = [] + + for line in text.split("\n"): + heading_match = re.match(r"^##\s+(.+)$", line) + if heading_match: + if current_section is not None: + sections[current_section] = current_items + current_section = heading_match.group(1).strip() + current_items = [] + continue + + if current_section is None: + continue + + if line.startswith("- "): + current_items.append(line[2:].strip()) + elif current_items and line.startswith(" "): + current_items[-1] += " " + line.strip() + + if current_section is not None: + sections[current_section] = current_items + + return sections + + @staticmethod + def _slugify(text: str) -> str: + text = _DATE_PREFIX.sub("", text) + text = re.sub(r"\*\*([^*]+)\*\*", r"\1", text) + text = re.sub(r"`([^`]+)`", r"\1", text) + words = re.findall(r"[a-zA-Z0-9]+", text) + slug = "-".join(words[:5]).lower() + return slug[:60] diff --git a/src/lore/ingest/reasonsforge.py b/src/lore/ingest/reasonsforge.py new file mode 100644 index 0000000..06abae4 --- /dev/null +++ b/src/lore/ingest/reasonsforge.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import json +import logging +import sqlite3 +from datetime import datetime +from pathlib import Path + +from lore.ingest.base import LoreIngester +from lore.ingest.registry import register +from lore.store.base import KnowledgeEntry, StoreBackend, validate_key + +log = logging.getLogger("lore.ingest") + +_KEY_COLUMNS = ("key", "name", "title") +_VALUE_COLUMNS = ("summary", "description", "value", "content") +_TIME_COLUMNS = ("created_at", "updated_at") +_TAG_COLUMN = "tags" + + +@register +class ReasonsForgeIngester(LoreIngester): + """Ingest from a ReasonsForge reasons.db SQLite file. + + Schema discovery is flexible — any table with a key-like column + (key/name/title) and a value-like column (summary/description/value/content) + will be read. Typical tables: decisions, patterns, conventions. + """ + + name = "reasonsforge" + triggers = frozenset({"hook", "manual"}) + review_policy = "immediate" + + def __init__(self, store: StoreBackend, project_dir: Path) -> None: + super().__init__(store, project_dir) + self._db_path = project_dir / "reasons.db" + + def detect(self, project_dir: Path) -> bool: + return (project_dir / "reasons.db").is_file() + + def extract_delta(self, since: datetime | None = None) -> list[KnowledgeEntry]: + if not self._db_path.is_file(): + return [] + + conn = sqlite3.connect(str(self._db_path)) + conn.row_factory = sqlite3.Row + try: + tables = self._discover_tables(conn) + entries: list[KnowledgeEntry] = [] + for table, key_col, val_col, tag_col, time_col in tables: + entries.extend( + self._extract_table( + conn, table, key_col, val_col, tag_col, time_col, since + ) + ) + return entries + finally: + conn.close() + + def _discover_tables( + self, conn: sqlite3.Connection + ) -> list[tuple[str, str, str, str | None, str | None]]: + tables = [] + cursor = conn.execute("SELECT name FROM sqlite_master WHERE type='table'") + for (name,) in cursor: + cols = {r[1] for r in conn.execute(f"PRAGMA table_info([{name}])")} + key_col = next((c for c in _KEY_COLUMNS if c in cols), None) + val_col = next((c for c in _VALUE_COLUMNS if c in cols), None) + if key_col and val_col: + tag_col = _TAG_COLUMN if _TAG_COLUMN in cols else None + time_col = next((c for c in _TIME_COLUMNS if c in cols), None) + tables.append((name, key_col, val_col, tag_col, time_col)) + return tables + + def _extract_table( + self, + conn: sqlite3.Connection, + table: str, + key_col: str, + val_col: str, + tag_col: str | None, + time_col: str | None, + since: datetime | None, + ) -> list[KnowledgeEntry]: + query = f"SELECT * FROM [{table}]" + params: list[str] = [] + if since and time_col: + query += f" WHERE [{time_col}] >= ?" + params.append(since.isoformat()) + + entries: list[KnowledgeEntry] = [] + for row in conn.execute(query, params): + raw_key = row[key_col] + if not raw_key: + continue + + key = str(raw_key).strip().lower().replace(" ", "-") + if ":" not in key: + key = f"{table}:{key}" + + if validate_key(key) is not None: + log.warning("Skipping invalid key: %s (table=%s)", raw_key, table) + continue + + value = str(row[val_col] or "") + if not value.strip(): + continue + + tags = str(row[tag_col]) if tag_col and row[tag_col] else None + row_id = row["id"] if "id" in row.keys() else None + + provenance = json.dumps( + { + "source": "reasons.db", + "table": table, + "row_id": str(row_id) if row_id else None, + } + ) + + entries.append( + KnowledgeEntry( + key=key, + value=value, + level=0, + tags=tags, + ingested_from="reasonsforge", + provenance=provenance, + ) + ) + + return entries diff --git a/src/lore/ingest/registry.py b/src/lore/ingest/registry.py new file mode 100644 index 0000000..51cd403 --- /dev/null +++ b/src/lore/ingest/registry.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Type + +from lore.ingest.base import LoreIngester +from lore.store.base import StoreBackend + +log = logging.getLogger("lore.ingest") + +_REGISTRY: dict[str, Type[LoreIngester]] = {} + + +def register(cls: Type[LoreIngester]) -> Type[LoreIngester]: + _REGISTRY[cls.name] = cls + return cls + + +def get_ingester(name: str, store: StoreBackend, project_dir: Path) -> LoreIngester: + cls = _REGISTRY.get(name) + if cls is None: + raise ValueError( + f"Unknown ingester: {name!r}. " f"Available: {', '.join(sorted(_REGISTRY))}" + ) + return cls(store=store, project_dir=project_dir) + + +def detect_ingesters( + project_dir: Path, + store: StoreBackend, + disabled_sources: list[str] | None = None, +) -> list[LoreIngester]: + detected = [] + disabled = set(disabled_sources or []) + for name, cls in sorted(_REGISTRY.items()): + if not cls.auto_detect: + continue + if name in disabled: + log.debug("Ingester %s disabled by config", name) + continue + try: + ingester = cls(store=store, project_dir=project_dir) + if ingester.detect(project_dir): + detected.append(ingester) + except Exception: + log.debug("Ingester %s failed detection", name, exc_info=True) + return detected + + +def list_ingesters() -> list[str]: + return sorted(_REGISTRY) diff --git a/tests/unit/test_ingest_dedup.py b/tests/unit/test_ingest_dedup.py new file mode 100644 index 0000000..e325377 --- /dev/null +++ b/tests/unit/test_ingest_dedup.py @@ -0,0 +1,171 @@ +from __future__ import annotations + +from datetime import datetime +from pathlib import Path +from unittest.mock import MagicMock, patch + +from lore.ingest.base import LoreIngester +from lore.store.base import KnowledgeEntry +from lore.store.sqlite import SQLiteStore, create_schema + + +def _make_store(): + return SQLiteStore(create_schema(":memory:")) + + +class _SimpleIngester(LoreIngester): + name = "_test_simple" + triggers = frozenset({"manual"}) + + def __init__(self, store, project_dir, entries=None): + super().__init__(store, project_dir) + self._entries = entries or [] + + def detect(self, project_dir: Path) -> bool: + return True + + def extract_delta(self, since: datetime | None = None) -> list[KnowledgeEntry]: + return list(self._entries) + + +def _mock_cfg(provider="none", threshold=0.20): + cfg = MagicMock() + cfg.search.embedding_provider = provider + cfg.search.dedup_threshold = threshold + return cfg + + +def test_dedup_no_embedding_passthrough(tmp_path): + store = _make_store() + entries = [KnowledgeEntry(key="a", value="hello", level=0)] + ing = _SimpleIngester(store, tmp_path, entries) + + with patch("lore.config.manager.get_global_config", return_value=_mock_cfg("none")): + result = ing._dedup(entries) + assert len(result) == 1 + + +def test_dedup_removes_near_duplicates(tmp_path): + store = _make_store() + existing = KnowledgeEntry(key="existing:entry", value="hello world", level=0) + store.store(existing) + store.commit() + + new_entry = KnowledgeEntry(key="new:entry", value="hello world again", level=0) + ing = _SimpleIngester(store, tmp_path, [new_entry]) + + mock_provider = MagicMock() + mock_provider.embed.return_value = [0.1] * 10 + + with ( + patch( + "lore.config.manager.get_global_config", + return_value=_mock_cfg("ollama"), + ), + patch("lore.embedding.get_embedding_provider", return_value=mock_provider), + patch.object(store, "query_vector", return_value=[(existing, 0.05)]), + ): + result = ing._dedup([new_entry]) + assert len(result) == 0 + + +def test_dedup_keeps_distant_entries(tmp_path): + store = _make_store() + new_entry = KnowledgeEntry(key="new:entry", value="totally different", level=0) + ing = _SimpleIngester(store, tmp_path, [new_entry]) + + mock_provider = MagicMock() + mock_provider.embed.return_value = [0.1] * 10 + + with ( + patch( + "lore.config.manager.get_global_config", + return_value=_mock_cfg("ollama"), + ), + patch("lore.embedding.get_embedding_provider", return_value=mock_provider), + patch.object(store, "query_vector", return_value=[]), + ): + result = ing._dedup([new_entry]) + assert len(result) == 1 + + +def test_dedup_sets_embedding_blob(tmp_path): + store = _make_store() + new_entry = KnowledgeEntry(key="new:entry", value="test blob", level=0) + ing = _SimpleIngester(store, tmp_path, [new_entry]) + + mock_provider = MagicMock() + mock_provider.embed.return_value = [0.1, 0.2, 0.3] + + with ( + patch( + "lore.config.manager.get_global_config", + return_value=_mock_cfg("ollama"), + ), + patch("lore.embedding.get_embedding_provider", return_value=mock_provider), + patch.object(store, "query_vector", return_value=[]), + ): + result = ing._dedup([new_entry]) + assert len(result) == 1 + assert result[0].embedding is not None + + +def test_dedup_embedding_failure_passthrough(tmp_path): + store = _make_store() + new_entry = KnowledgeEntry(key="new:entry", value="test", level=0) + ing = _SimpleIngester(store, tmp_path, [new_entry]) + + mock_provider = MagicMock() + mock_provider.embed.side_effect = Exception("Ollama down") + + with ( + patch( + "lore.config.manager.get_global_config", + return_value=_mock_cfg("ollama"), + ), + patch("lore.embedding.get_embedding_provider", return_value=mock_provider), + ): + result = ing._dedup([new_entry]) + assert len(result) == 1 + + +def test_dedup_empty_entries(tmp_path): + store = _make_store() + ing = _SimpleIngester(store, tmp_path) + result = ing._dedup([]) + assert result == [] + + +def test_dedup_respects_level(tmp_path): + store = _make_store() + existing = KnowledgeEntry(key="existing:entry", value="hello", level=1) + store.store(existing) + + new_entry = KnowledgeEntry(key="new:entry", value="hello", level=0) + ing = _SimpleIngester(store, tmp_path, [new_entry]) + + mock_provider = MagicMock() + mock_provider.embed.return_value = [0.1] * 10 + + with ( + patch( + "lore.config.manager.get_global_config", + return_value=_mock_cfg("ollama"), + ), + patch("lore.embedding.get_embedding_provider", return_value=mock_provider), + patch.object(store, "query_vector", return_value=[(existing, 0.05)]), + ): + result = ing._dedup([new_entry]) + assert len(result) == 1 + + +def test_run_orchestrates_full_pipeline(tmp_path): + store = _make_store() + entries = [KnowledgeEntry(key="run:test", value="pipeline", level=0)] + ing = _SimpleIngester(store, tmp_path, entries) + + result = ing.run() + assert len(result) == 1 + stored = store.get("run:test") + assert stored is not None + assert stored.value == "pipeline" diff --git a/tests/unit/test_ingest_doc.py b/tests/unit/test_ingest_doc.py index b198a5a..d826a49 100644 --- a/tests/unit/test_ingest_doc.py +++ b/tests/unit/test_ingest_doc.py @@ -89,7 +89,7 @@ def test_extract_delta_sets_provenance(tmp_path): ingester = DocIngester(p, provider, _make_store()) entries = ingester.extract_delta() - assert entries[0].ingested_from == "doc-ingester" + assert entries[0].ingested_from == "doc" prov = json.loads(entries[0].provenance) assert prov["source_file"] == str(p) assert "chunk_index" in prov diff --git a/tests/unit/test_ingest_openwolf.py b/tests/unit/test_ingest_openwolf.py new file mode 100644 index 0000000..b65706b --- /dev/null +++ b/tests/unit/test_ingest_openwolf.py @@ -0,0 +1,236 @@ +from __future__ import annotations + +import json +from datetime import datetime + +from lore.ingest.base import LoreIngester +from lore.ingest.openwolf import OpenWolfIngester +from lore.store.sqlite import SQLiteStore, create_schema + + +def _make_store(): + return SQLiteStore(create_schema(":memory:")) + + +def _write_cerebrum(wolf_dir, content): + wolf_dir.mkdir(parents=True, exist_ok=True) + (wolf_dir / "cerebrum.md").write_text(content) + + +def _write_buglog(wolf_dir, bugs): + wolf_dir.mkdir(parents=True, exist_ok=True) + (wolf_dir / "buglog.json").write_text(json.dumps({"version": 1, "bugs": bugs})) + + +def test_is_lore_ingester(tmp_path): + ing = OpenWolfIngester(_make_store(), tmp_path) + assert isinstance(ing, LoreIngester) + + +def test_detect_with_wolf_dir(tmp_path): + _write_cerebrum(tmp_path / ".wolf", "# Cerebrum\n## User Preferences\n- pref1") + ing = OpenWolfIngester(_make_store(), tmp_path) + assert ing.detect(tmp_path) is True + + +def test_detect_without_wolf_dir(tmp_path): + ing = OpenWolfIngester(_make_store(), tmp_path) + assert ing.detect(tmp_path) is False + + +def test_detect_wolf_dir_no_cerebrum(tmp_path): + (tmp_path / ".wolf").mkdir() + ing = OpenWolfIngester(_make_store(), tmp_path) + assert ing.detect(tmp_path) is False + + +def test_extract_preferences(tmp_path): + _write_cerebrum( + tmp_path / ".wolf", + "## User Preferences\n\n- Prefer dark mode\n- Use tabs not spaces\n", + ) + ing = OpenWolfIngester(_make_store(), tmp_path) + entries = ing.extract_delta() + assert len(entries) == 2 + assert all(e.key.startswith("preference:") for e in entries) + assert entries[0].tags == "preference" + + +def test_extract_learnings(tmp_path): + _write_cerebrum( + tmp_path / ".wolf", + "## Key Learnings\n\n- **Project:** uses FastMCP for tools\n", + ) + ing = OpenWolfIngester(_make_store(), tmp_path) + entries = ing.extract_delta() + assert len(entries) == 1 + assert entries[0].key.startswith("learning:") + + +def test_extract_do_not_repeat(tmp_path): + _write_cerebrum( + tmp_path / ".wolf", + "## Do-Not-Repeat\n\n- [2026-07-24] Don't mock the database\n", + ) + ing = OpenWolfIngester(_make_store(), tmp_path) + entries = ing.extract_delta() + assert len(entries) == 1 + assert entries[0].key.startswith("do-not-repeat:") + + +def test_extract_decisions(tmp_path): + _write_cerebrum( + tmp_path / ".wolf", + "## Decision Log\n\n" + "- [2026-07-24] **Use git CLI not GitPython.** Simpler.\n", + ) + ing = OpenWolfIngester(_make_store(), tmp_path) + entries = ing.extract_delta() + assert len(entries) == 1 + assert entries[0].key.startswith("decision:") + + +def test_extract_empty_section(tmp_path): + _write_cerebrum(tmp_path / ".wolf", "## User Preferences\n\n## Key Learnings\n") + ing = OpenWolfIngester(_make_store(), tmp_path) + entries = ing.extract_delta() + assert entries == [] + + +def test_html_comments_skipped(tmp_path): + _write_cerebrum( + tmp_path / ".wolf", + "## User Preferences\n\n\n- Real preference\n", + ) + ing = OpenWolfIngester(_make_store(), tmp_path) + entries = ing.extract_delta() + assert len(entries) == 1 + assert "comment" not in entries[0].value + + +def test_multi_line_bullet(tmp_path): + _write_cerebrum( + tmp_path / ".wolf", + "## User Preferences\n\n- First line of pref\n continued on next line\n", + ) + ing = OpenWolfIngester(_make_store(), tmp_path) + entries = ing.extract_delta() + assert len(entries) == 1 + assert "continued" in entries[0].value + + +def test_since_filter_cerebrum(tmp_path): + _write_cerebrum( + tmp_path / ".wolf", + "## Do-Not-Repeat\n\n" + "- [2026-01-01] Old mistake\n" + "- [2026-07-24] Recent mistake\n", + ) + ing = OpenWolfIngester(_make_store(), tmp_path) + entries = ing.extract_delta(since=datetime(2026, 6, 1)) + assert len(entries) == 1 + assert "Recent" in entries[0].value + + +def test_extract_buglog(tmp_path): + _write_buglog( + tmp_path / ".wolf", + [ + { + "id": "bug-001", + "error_message": "ImportError: no module named foo", + "root_cause": "Missing dependency", + "fix": "pip install foo", + "file": "app.py", + "tags": ["import", "dependency"], + "timestamp": "2026-07-24T10:00:00", + } + ], + ) + ing = OpenWolfIngester(_make_store(), tmp_path) + entries = ing.extract_delta() + assert len(entries) == 1 + assert entries[0].key == "bug:bug-001" + assert "ImportError" in entries[0].value + assert "pip install foo" in entries[0].value + assert entries[0].tags == "import,dependency" + assert entries[0].ingested_from == "openwolf" + + +def test_buglog_empty(tmp_path): + _write_buglog(tmp_path / ".wolf", []) + ing = OpenWolfIngester(_make_store(), tmp_path) + entries = ing.extract_delta() + assert entries == [] + + +def test_buglog_missing_file(tmp_path): + _write_cerebrum(tmp_path / ".wolf", "## User Preferences\n") + ing = OpenWolfIngester(_make_store(), tmp_path) + entries = ing._extract_buglog() + assert entries == [] + + +def test_buglog_since_filter(tmp_path): + _write_buglog( + tmp_path / ".wolf", + [ + {"id": "old", "root_cause": "Old bug", "timestamp": "2026-01-01T00:00:00"}, + {"id": "new", "root_cause": "New bug", "timestamp": "2026-07-24T00:00:00"}, + ], + ) + ing = OpenWolfIngester(_make_store(), tmp_path) + entries = ing._extract_buglog(since=datetime(2026, 6, 1)) + assert len(entries) == 1 + assert entries[0].key == "bug:new" + + +def test_provenance_cerebrum(tmp_path): + _write_cerebrum(tmp_path / ".wolf", "## User Preferences\n\n- Test provenance\n") + ing = OpenWolfIngester(_make_store(), tmp_path) + entries = ing.extract_delta() + prov = json.loads(entries[0].provenance) + assert prov["source"] == "cerebrum.md" + assert prov["section"] == "User Preferences" + assert prov["item_index"] == 0 + + +def test_provenance_buglog(tmp_path): + _write_buglog( + tmp_path / ".wolf", + [{"id": "bug-042", "root_cause": "Test provenance"}], + ) + ing = OpenWolfIngester(_make_store(), tmp_path) + entries = ing.extract_delta() + prov = json.loads(entries[0].provenance) + assert prov["source"] == "buglog.json" + assert prov["bug_id"] == "bug-042" + + +def test_slugify_strips_markdown(): + slug = OpenWolfIngester._slugify("**Bold text** with `code`") + assert "bold" in slug.lower() + assert "code" in slug.lower() + assert "*" not in slug + assert "`" not in slug + + +def test_slugify_strips_date(): + slug = OpenWolfIngester._slugify("[2026-07-24] Some text here") + assert "2026" not in slug + assert "some" in slug.lower() + + +def test_full_pipeline(tmp_path): + _write_cerebrum( + tmp_path / ".wolf", + "## Key Learnings\n\n- SQLite needs fresh connections in threads\n", + ) + store = _make_store() + ing = OpenWolfIngester(store, tmp_path) + entries = ing.extract_delta() + ing.load(entries) + + all_entries = store.list_entries() + assert len(all_entries) == 1 + assert "SQLite" in all_entries[0].value diff --git a/tests/unit/test_ingest_reasonsforge.py b/tests/unit/test_ingest_reasonsforge.py new file mode 100644 index 0000000..045a842 --- /dev/null +++ b/tests/unit/test_ingest_reasonsforge.py @@ -0,0 +1,193 @@ +from __future__ import annotations + +import json +import sqlite3 +from datetime import datetime + +from lore.ingest.base import LoreIngester +from lore.ingest.reasonsforge import ReasonsForgeIngester +from lore.store.sqlite import SQLiteStore, create_schema + + +def _make_store(): + return SQLiteStore(create_schema(":memory:")) + + +def _create_reasons_db(path, tables=None): + db_path = path / "reasons.db" + conn = sqlite3.connect(str(db_path)) + if tables is None: + tables = { + "decisions": [ + ("id", "TEXT PRIMARY KEY"), + ("key", "TEXT"), + ("summary", "TEXT"), + ("tags", "TEXT"), + ("created_at", "TIMESTAMP"), + ] + } + for table_name, columns in tables.items(): + cols_sql = ", ".join(f"{n} {t}" for n, t in columns) + conn.execute(f"CREATE TABLE {table_name} ({cols_sql})") + conn.commit() + return conn, db_path + + +def test_is_lore_ingester(tmp_path): + ing = ReasonsForgeIngester(_make_store(), tmp_path) + assert isinstance(ing, LoreIngester) + + +def test_detect_with_db(tmp_path): + _create_reasons_db(tmp_path) + ing = ReasonsForgeIngester(_make_store(), tmp_path) + assert ing.detect(tmp_path) is True + + +def test_detect_without_db(tmp_path): + ing = ReasonsForgeIngester(_make_store(), tmp_path) + assert ing.detect(tmp_path) is False + + +def test_extract_decisions(tmp_path): + conn, _ = _create_reasons_db(tmp_path) + conn.execute( + "INSERT INTO decisions VALUES (?, ?, ?, ?, ?)", + ("d1", "auth:jwt", "Use JWT for auth", "auth,security", "2026-07-01T00:00:00"), + ) + conn.commit() + conn.close() + + ing = ReasonsForgeIngester(_make_store(), tmp_path) + entries = ing.extract_delta() + assert len(entries) == 1 + assert entries[0].key == "auth:jwt" + assert entries[0].value == "Use JWT for auth" + assert entries[0].tags == "auth,security" + assert entries[0].ingested_from == "reasonsforge" + + +def test_extract_with_since(tmp_path): + conn, _ = _create_reasons_db(tmp_path) + conn.execute( + "INSERT INTO decisions VALUES (?, ?, ?, ?, ?)", + ("d1", "old-entry", "Old decision", None, "2026-01-01T00:00:00"), + ) + conn.execute( + "INSERT INTO decisions VALUES (?, ?, ?, ?, ?)", + ("d2", "new-entry", "New decision", None, "2026-07-01T00:00:00"), + ) + conn.commit() + conn.close() + + ing = ReasonsForgeIngester(_make_store(), tmp_path) + entries = ing.extract_delta(since=datetime(2026, 6, 1)) + assert len(entries) == 1 + assert entries[0].key == "decisions:new-entry" + + +def test_extract_empty_db(tmp_path): + _create_reasons_db(tmp_path) + ing = ReasonsForgeIngester(_make_store(), tmp_path) + entries = ing.extract_delta() + assert entries == [] + + +def test_provenance_set(tmp_path): + conn, _ = _create_reasons_db(tmp_path) + conn.execute( + "INSERT INTO decisions VALUES (?, ?, ?, ?, ?)", + ("d1", "test:prov", "Provenance test", None, None), + ) + conn.commit() + conn.close() + + ing = ReasonsForgeIngester(_make_store(), tmp_path) + entries = ing.extract_delta() + prov = json.loads(entries[0].provenance) + assert prov["source"] == "reasons.db" + assert prov["table"] == "decisions" + assert prov["row_id"] == "d1" + + +def test_flexible_schema_discovery(tmp_path): + tables = { + "patterns": [ + ("id", "TEXT"), + ("title", "TEXT"), + ("description", "TEXT"), + ] + } + conn, _ = _create_reasons_db(tmp_path, tables) + conn.execute( + "INSERT INTO patterns VALUES (?, ?, ?)", + ("p1", "singleton", "Use singleton pattern for DB connections"), + ) + conn.commit() + conn.close() + + ing = ReasonsForgeIngester(_make_store(), tmp_path) + entries = ing.extract_delta() + assert len(entries) == 1 + assert "singleton" in entries[0].key + + +def test_skips_invalid_keys(tmp_path): + conn, _ = _create_reasons_db(tmp_path) + conn.execute( + "INSERT INTO decisions VALUES (?, ?, ?, ?, ?)", + ("d1", "has spaces invalid!", "Bad key", None, None), + ) + conn.commit() + conn.close() + + ing = ReasonsForgeIngester(_make_store(), tmp_path) + entries = ing.extract_delta() + assert entries == [] + + +def test_skips_empty_values(tmp_path): + conn, _ = _create_reasons_db(tmp_path) + conn.execute( + "INSERT INTO decisions VALUES (?, ?, ?, ?, ?)", + ("d1", "empty:val", "", None, None), + ) + conn.commit() + conn.close() + + ing = ReasonsForgeIngester(_make_store(), tmp_path) + entries = ing.extract_delta() + assert entries == [] + + +def test_key_gets_table_prefix(tmp_path): + conn, _ = _create_reasons_db(tmp_path) + conn.execute( + "INSERT INTO decisions VALUES (?, ?, ?, ?, ?)", + ("d1", "simple-key", "A simple key without colon", None, None), + ) + conn.commit() + conn.close() + + ing = ReasonsForgeIngester(_make_store(), tmp_path) + entries = ing.extract_delta() + assert entries[0].key == "decisions:simple-key" + + +def test_full_pipeline(tmp_path): + conn, _ = _create_reasons_db(tmp_path) + conn.execute( + "INSERT INTO decisions VALUES (?, ?, ?, ?, ?)", + ("d1", "test:pipeline", "Pipeline test", "test", None), + ) + conn.commit() + conn.close() + + store = _make_store() + ing = ReasonsForgeIngester(store, tmp_path) + entries = ing.extract_delta() + ing.load(entries) + + stored = store.get("test:pipeline") + assert stored is not None + assert stored.value == "Pipeline test" diff --git a/tests/unit/test_ingest_registry.py b/tests/unit/test_ingest_registry.py new file mode 100644 index 0000000..807917e --- /dev/null +++ b/tests/unit/test_ingest_registry.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +from datetime import datetime +from pathlib import Path + +from lore.ingest.base import LoreIngester +from lore.ingest.registry import ( + _REGISTRY, + detect_ingesters, + get_ingester, + list_ingesters, + register, +) +from lore.store.base import KnowledgeEntry +from lore.store.sqlite import SQLiteStore, create_schema + + +def _make_store(): + return SQLiteStore(create_schema(":memory:")) + + +class _DummyIngester(LoreIngester): + name = "_test_dummy" + triggers = frozenset({"manual"}) + auto_detect = True + + def detect(self, project_dir: Path) -> bool: + return (project_dir / "dummy.txt").exists() + + def extract_delta(self, since: datetime | None = None) -> list[KnowledgeEntry]: + return [KnowledgeEntry(key="dummy:entry", value="dummy", level=0)] + + +class _NoAutoIngester(LoreIngester): + name = "_test_noauto" + triggers = frozenset({"manual"}) + auto_detect = False + + def detect(self, project_dir: Path) -> bool: + return True + + def extract_delta(self, since: datetime | None = None) -> list[KnowledgeEntry]: + return [] + + +def _with_clean_registry(fn): + import functools + + @functools.wraps(fn) + def wrapper(*a, **kw): + saved = dict(_REGISTRY) + try: + return fn(*a, **kw) + finally: + _REGISTRY.clear() + _REGISTRY.update(saved) + + return wrapper + + +@_with_clean_registry +def test_register_decorator(): + @register + class _Reg(LoreIngester): + name = "_test_reg" + triggers = frozenset({"manual"}) + + def detect(self, project_dir): + return False + + def extract_delta(self, since=None): + return [] + + assert "_test_reg" in _REGISTRY + assert _REGISTRY["_test_reg"] is _Reg + + +@_with_clean_registry +def test_get_ingester_known(tmp_path): + register(_DummyIngester) + ing = get_ingester("_test_dummy", _make_store(), tmp_path) + assert isinstance(ing, _DummyIngester) + + +@_with_clean_registry +def test_get_ingester_unknown(): + import pytest + + with pytest.raises(ValueError, match="Unknown ingester"): + get_ingester("nonexistent", _make_store(), Path("/tmp")) + + +@_with_clean_registry +def test_detect_ingesters_finds_match(tmp_path): + register(_DummyIngester) + (tmp_path / "dummy.txt").write_text("x") + detected = detect_ingesters(tmp_path, _make_store()) + assert any(isinstance(d, _DummyIngester) for d in detected) + + +@_with_clean_registry +def test_detect_ingesters_no_match(tmp_path): + register(_DummyIngester) + detected = detect_ingesters(tmp_path, _make_store()) + assert not any(isinstance(d, _DummyIngester) for d in detected) + + +@_with_clean_registry +def test_detect_skips_auto_detect_false(tmp_path): + register(_NoAutoIngester) + detected = detect_ingesters(tmp_path, _make_store()) + assert not any(isinstance(d, _NoAutoIngester) for d in detected) + + +@_with_clean_registry +def test_detect_respects_disabled_sources(tmp_path): + register(_DummyIngester) + (tmp_path / "dummy.txt").write_text("x") + detected = detect_ingesters( + tmp_path, _make_store(), disabled_sources=["_test_dummy"] + ) + assert not any(isinstance(d, _DummyIngester) for d in detected) + + +def test_list_ingesters_returns_sorted(): + names = list_ingesters() + assert names == sorted(names) + + +def test_builtin_ingesters_registered(): + names = list_ingesters() + assert "doc" in names + assert "openwolf" in names + assert "reasonsforge" in names