From 2a02542e644781d0de9e71f7c1d40113cbfcdf69 Mon Sep 17 00:00:00 2001 From: callumalpass Date: Mon, 20 Jul 2026 23:26:02 +1000 Subject: [PATCH 1/2] add safe v0.2 collection migrator --- scripts/migrate_v02_collection.py | 953 ++++++++++++++++++++++++++++++ 1 file changed, 953 insertions(+) create mode 100644 scripts/migrate_v02_collection.py diff --git a/scripts/migrate_v02_collection.py b/scripts/migrate_v02_collection.py new file mode 100644 index 0000000..a54b8db --- /dev/null +++ b/scripts/migrate_v02_collection.py @@ -0,0 +1,953 @@ +#!/usr/bin/env python3 +"""Stage or apply a complete mdbase v0.2 collection metadata migration. + +The migrator rewrites only ``mdbase.yaml`` and Markdown files in the configured +types folder. Record files are never modified. It flattens v0.2 inheritance, +maps field definitions to JSON Schema, and moves collection behavior into the +v0.3 ``collection`` and ``lifecycle`` sections. + +TaskNotes-generated types receive the ``tasknotes.task`` contract wrapper used +by current TaskNotes releases. Source features without a portable v0.3 mapping +are retained under ``x-legacy-v0.2`` and listed in the report. +""" + +from __future__ import annotations + +import argparse +import copy +import datetime as dt +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml +from jsonschema import Draft202012Validator + + +REPO_ROOT = Path(__file__).resolve().parent.parent +CONFIG_SCHEMA = REPO_ROOT / "schemas/v0.3/config.schema.json" +TYPE_FILE_SCHEMA = REPO_ROOT / "schemas/v0.3/type-file.schema.json" +FRONTMATTER = re.compile(r"\A---(?:\r?\n)(.*?)(?:\r?\n)---(?=\r?\n|\Z)", re.S) +CORE_CONFIG_SETTINGS = { + "types_folder", + "record_extensions", + "validation", + "explicit_type_keys", + "id_field", + "include_subfolders", + "exclude", +} + + +@dataclass(frozen=True) +class SourceType: + path: Path + frontmatter: dict[str, Any] + body: str + + @property + def name(self) -> str: + return str(self.frontmatter["name"]) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Stage or apply an mdbase v0.2 collection metadata migration" + ) + parser.add_argument("collection", type=Path) + parser.add_argument("--output", type=Path, required=True, help="analysis/report directory") + parser.add_argument("--apply", action="store_true", help="replace live metadata after analysis") + parser.add_argument("--backup-dir", type=Path, help="backup destination used with --apply") + parser.add_argument("--mdb-bin", type=Path, help="mdb binary used for full staged validation") + parser.add_argument( + "--allow-unsupported", + action="store_true", + help="apply while retaining unsupported source behavior under x-legacy-v0.2", + ) + args = parser.parse_args() + + collection = args.collection.resolve() + output = args.output.resolve() + if not collection.is_dir(): + parser.error(f"collection is not a directory: {collection}") + if output == collection or collection in output.parents: + parser.error("--output must be outside the collection") + if args.apply and not args.backup_dir: + parser.error("--backup-dir is required with --apply") + + result = analyze(collection, output, args.mdb_bin) + report_path = output / "migration-report.json" + report_path.write_text(json.dumps(result["report"], indent=2, sort_keys=True) + "\n") + + report = result["report"] + if report["unsupported"] and not args.allow_unsupported: + print(json.dumps({"ok": False, "report": str(report_path), "reason": "unsupported"})) + return 2 + if report["target_validation"]["status"] == "failed": + print(json.dumps({"ok": False, "report": str(report_path), "reason": "invalid-target"})) + return 3 + if report["record_validation"]["regressions"]: + print(json.dumps({"ok": False, "report": str(report_path), "reason": "record-regression"})) + return 4 + + backup = None + if args.apply: + backup = apply_migration( + collection, + output / "proposed", + args.backup_dir.resolve(), + report, + ) + report["apply"] = {"status": "complete", "backup": str(backup)} + report_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + + print( + json.dumps( + { + "ok": True, + "applied": args.apply, + "report": str(report_path), + "backup": str(backup) if backup else None, + "types": report["summary"]["types_migrated"], + "unsupported": len(report["unsupported"]), + "record_regressions": len(report["record_validation"]["regressions"]), + } + ) + ) + return 0 + + +def analyze(collection: Path, output: Path, mdb_bin: Path | None) -> dict[str, Any]: + if output.exists(): + raise SystemExit(f"output already exists: {output}") + proposed = output / "proposed" + proposed.mkdir(parents=True) + + config_path = collection / "mdbase.yaml" + source_config = load_yaml(config_path) + if not isinstance(source_config, dict): + raise SystemExit("mdbase.yaml must contain a mapping") + if not re.fullmatch(r"0\.2\.\d+(?:[-+].*)?", str(source_config.get("spec_version", ""))): + raise SystemExit("collection is not an mdbase v0.2 collection") + + types_folder = str((source_config.get("settings") or {}).get("types_folder") or "_types") + type_root = collection / types_folder + source_types = read_source_types(type_root) + by_name = {source.name.casefold(): source for source in source_types} + if len(by_name) != len(source_types): + raise SystemExit("type names are not unique case-insensitively") + + unsupported: list[dict[str, Any]] = [] + proposed_config = migrate_config(source_config) + migrated_types: dict[Path, str] = {} + type_summaries: list[dict[str, Any]] = [] + + for source in source_types: + effective_fields, strict, inheritance = resolve_effective_fields(source, by_name) + migrated, type_unsupported = migrate_type( + source, + effective_fields, + strict, + inheritance, + ) + unsupported.extend(type_unsupported) + validate_schema(TYPE_FILE_SCHEMA, migrated, f"type {source.name}") + rendered = render_type(migrated, migrated_body(source, migrated)) + relative = source.path.relative_to(collection) + migrated_types[relative] = rendered + type_summaries.append( + { + "name": source.name, + "path": relative.as_posix(), + "source_sha256": sha256(source.path.read_bytes()), + "target_sha256": sha256(rendered.encode()), + "flattened_inheritance": inheritance, + "tasknotes_contract": migrated.get("x-tasknotes", {}).get("contract"), + } + ) + + validate_schema(CONFIG_SCHEMA, proposed_config, "config") + (proposed / types_folder).mkdir(parents=True, exist_ok=True) + (proposed / "mdbase.yaml").write_text(yaml_dump(proposed_config)) + for relative, rendered in migrated_types.items(): + target = proposed / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(rendered) + + target_validation = {"status": "passed", "errors": []} + record_validation: dict[str, Any] = { + "status": "not-run", + "baseline": None, + "target": None, + "regressions": [], + } + if mdb_bin: + mdb = mdb_bin.resolve() + if not mdb.is_file(): + raise SystemExit(f"mdb binary not found: {mdb}") + target_validation, record_validation = validate_staged_collection( + collection, + proposed, + mdb, + ) + + report = { + "source": str(collection), + "source_version": str(source_config["spec_version"]), + "target_version": "0.3.0", + "summary": { + "types_migrated": len(migrated_types), + "tasknotes_contracts": sum( + 1 for item in type_summaries if item["tasknotes_contract"] == "tasknotes.task" + ), + }, + "config": { + "source_sha256": sha256(config_path.read_bytes()), + "target_sha256": sha256((proposed / "mdbase.yaml").read_bytes()), + }, + "types": sorted(type_summaries, key=lambda item: item["name"].casefold()), + "unsupported": unsupported, + "target_validation": target_validation, + "record_validation": record_validation, + "proposed": str(proposed), + "apply": {"status": "not-requested"}, + } + return {"report": report} + + +def read_source_types(type_root: Path) -> list[SourceType]: + if not type_root.is_dir(): + raise SystemExit(f"types folder not found: {type_root}") + result = [] + for path in sorted(type_root.rglob("*.md")): + if ".bak-" in path.name: + continue + text = path.read_text() + match = FRONTMATTER.match(text) + if not match: + raise SystemExit(f"type file lacks frontmatter: {path}") + value = normalize_yaml(yaml.safe_load(match.group(1)) or {}) + if not isinstance(value, dict) or not isinstance(value.get("name"), str): + raise SystemExit(f"type file has no name: {path}") + body = text[match.end() :] + result.append(SourceType(path=path, frontmatter=value, body=body)) + return result + + +def resolve_effective_fields( + source: SourceType, + by_name: dict[str, SourceType], + chain: tuple[str, ...] = (), +) -> tuple[dict[str, Any], bool, list[str]]: + name = source.name.casefold() + if name in chain: + raise SystemExit(f"circular inheritance: {' -> '.join((*chain, name))}") + own_fields = copy.deepcopy(source.frontmatter.get("fields") or {}) + strict_value = source.frontmatter.get("strict") + strict = strict_value is True or strict_value == "error" + inheritance: list[str] = [] + parent_name = source.frontmatter.get("extends") + if parent_name is not None: + if not isinstance(parent_name, str) or parent_name.casefold() not in by_name: + raise SystemExit(f"type {source.name} has an invalid parent: {parent_name!r}") + parent = by_name[parent_name.casefold()] + parent_fields, parent_strict, ancestors = resolve_effective_fields( + parent, by_name, (*chain, name) + ) + parent_fields.update(own_fields) + own_fields = parent_fields + if strict_value is None: + strict = parent_strict + inheritance = [*ancestors, parent.name] + return own_fields, strict, inheritance + + +def migrate_config(source: dict[str, Any]) -> dict[str, Any]: + settings = copy.deepcopy(source.get("settings") or {}) + target_settings = {key: value for key, value in settings.items() if key in CORE_CONFIG_SETTINGS} + if "extensions" in settings and "record_extensions" not in target_settings: + target_settings["record_extensions"] = [str(value).lstrip(".") for value in settings["extensions"]] + target_settings.setdefault("record_extensions", ["md"]) + target_settings.setdefault("validation", "warn") + # Preserve an explicitly empty list: this collection uses CSL's `type` + # field as data and therefore cannot use the default explicit type keys. + target_settings.setdefault("explicit_type_keys", ["type", "types"]) + target_settings.setdefault("id_field", "id") + + target: dict[str, Any] = { + "spec_version": "0.3.0", + "settings": target_settings, + } + for key in ("name", "description", "runtime"): + if key in source: + target[key] = copy.deepcopy(source[key]) + legacy_settings = {key: value for key, value in settings.items() if key not in CORE_CONFIG_SETTINGS} + legacy_root = { + key: value + for key, value in source.items() + if key not in {"spec_version", "settings", "name", "description", "runtime"} + } + if legacy_settings or legacy_root: + target["x-legacy-v0.2"] = { + **({"settings": legacy_settings} if legacy_settings else {}), + **({"root": legacy_root} if legacy_root else {}), + } + return target + + +def type_matches_types_folder(match: Any) -> bool: + if not isinstance(match, dict): + return False + pattern = match.get("path_glob") + patterns = pattern if isinstance(pattern, list) else [pattern] + return any(isinstance(value, str) and "_types/" in value for value in patterns) + + +def migrate_meta_type(source: SourceType) -> dict[str, Any]: + return { + "kind": "mdbase.type", + "name": "meta", + "version": 1, + "description": "mdbase v0.3 type-file schema.", + "match": migrate_match(source.frontmatter.get("match")) + or {"path_glob": "_types/**/*.md"}, + "schema": { + "dialect": "json-schema-2020-12", + "value": json.loads(TYPE_FILE_SCHEMA.read_text()), + }, + "x-mdbase": {"materialized": True, "authoritative": "built-in"}, + "x-legacy-v0.2": {"replaced_meta_schema": True}, + } + + +def migrate_type( + source: SourceType, + fields: dict[str, Any], + strict: bool, + inheritance: list[str], +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + if source.name.casefold() == "meta" and type_matches_types_folder(source.frontmatter.get("match")): + return migrate_meta_type(source), [] + + required: list[str] = [] + properties: dict[str, Any] = {} + read_defaults: dict[str, Any] = {} + links: dict[str, Any] = {} + unique: list[dict[str, Any]] = [] + projections: dict[str, Any] = {} + lifecycle: dict[str, Any] = {} + unsupported: list[dict[str, Any]] = [] + tasknotes_roles: dict[str, str] = {} + completed_values: list[Any] = [] + is_tasknotes = any( + isinstance(definition, dict) and isinstance(definition.get("tn_role"), str) + for definition in fields.values() + ) + + for field_name, definition in fields.items(): + if not isinstance(definition, dict): + continue + computed = definition.get("computed") + if isinstance(computed, str): + projections[field_name] = {"expr": computed} + if isinstance(definition.get("description"), str): + projections[field_name]["description"] = definition["description"] + continue + role = definition.get("tn_role") if isinstance(definition.get("tn_role"), str) else None + schema, field_links = convert_field( + field_name, + definition, + strict=strict, + tasknotes_role=role if is_tasknotes else None, + allow_null=definition.get("required") is not True, + ) + properties[field_name] = schema + links.update(field_links) + if definition.get("required") is True: + required.append(field_name) + if "default" in definition: + properties[field_name]["default"] = copy.deepcopy(definition["default"]) + read_defaults[field_name] = copy.deepcopy(definition["default"]) + if definition.get("unique") is True: + unique.append({"field": field_name, "scope": "type"}) + if role: + tasknotes_roles[role] = field_name + if isinstance(definition.get("tn_completed_values"), list): + completed_values = copy.deepcopy(definition["tn_completed_values"]) + generated = definition.get("generated") + if generated is not None: + mapped = add_lifecycle(lifecycle, field_name, generated) + own_fields = source.frontmatter.get("fields") or {} + if not mapped and field_name in own_fields: + unsupported.append( + { + "type": source.name, + "path": source.path.name, + "feature": f"fields.{field_name}.generated", + "value": generated, + "preserved_at": f"x-legacy-v0.2.generated.{field_name}", + } + ) + + schema: dict[str, Any] = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": not strict, + "properties": properties, + } + if required: + schema["required"] = required + + target: dict[str, Any] = { + "kind": "mdbase.type", + "name": source.name, + "version": int(source.frontmatter.get("version") or 1), + "description": str(source.frontmatter.get("description") or f"{source.name} record."), + "schema": {"dialect": "json-schema-2020-12", "value": schema}, + } + match = migrate_match(source.frontmatter.get("match")) + if match: + target["match"] = match + + collection: dict[str, Any] = {} + display = source.frontmatter.get("display_name_key") + if isinstance(display, str) and is_field_path(display): + collection["display"] = {"name_field": display} + if read_defaults: + collection["read_defaults"] = read_defaults + if links: + collection["links"] = links + if unique: + collection["unique"] = unique + path_pattern = source.frontmatter.get("path_pattern") or source.frontmatter.get("filename_pattern") + if isinstance(path_pattern, str): + collection["path"] = {"pattern": path_pattern} + if projections: + collection["projections"] = projections + + if is_tasknotes: + title_field = tasknotes_roles.get("title") + if title_field and is_field_path(title_field): + properties[title_field]["minLength"] = 1 + collection["display"] = {"name_field": title_field} + if isinstance(path_pattern, str): + folder, template = tasknotes_path(path_pattern) + collection["path"] = { + "runtime": "tasknotes", + "template": template, + "folder": folder, + "generated_by": "tasknotes.filename.create", + } + status_field = tasknotes_roles.get("status") + priority_field = tasknotes_roles.get("priority") + target["x-tasknotes"] = { + "contract": "tasknotes.task", + "version": 1, + "field_roles": tasknotes_roles, + "status": { + "completed_values": completed_values, + **( + {"default": read_defaults[status_field]} + if status_field in read_defaults + else {} + ), + }, + "priority": ( + {"default": read_defaults[priority_field]} + if priority_field in read_defaults + else {} + ), + "archive": { + "tags_field": tasknotes_roles.get("tags", "tags"), + "archived_tag": "archived", + }, + } + + if collection: + target["collection"] = collection + if lifecycle: + target["lifecycle"] = lifecycle + + known = { + "name", + "version", + "description", + "display_name_key", + "extends", + "strict", + "match", + "path_pattern", + "filename_pattern", + "fields", + } + unknown_top = { + key: copy.deepcopy(value) + for key, value in source.frontmatter.items() + if key not in known + } + source_fields = source.frontmatter.get("fields") or {} + unsupported_generated = { + field_name: copy.deepcopy(definition["generated"]) + for field_name, definition in source_fields.items() + if isinstance(definition, dict) + and "generated" in definition + and not lifecycle_has_field(lifecycle, field_name) + } + legacy: dict[str, Any] = {} + if inheritance: + legacy["flattened_inheritance"] = inheritance + if unsupported_generated: + legacy["generated"] = unsupported_generated + if unknown_top: + legacy["source_metadata"] = unknown_top + if legacy: + target["x-legacy-v0.2"] = legacy + target.setdefault("x-legacy-v0.2", {})["coercion_compatible_schema"] = True + return target, unsupported + + +def convert_field( + selector: str, + definition: dict[str, Any], + *, + strict: bool, + tasknotes_role: str | None = None, + allow_null: bool = False, +) -> tuple[dict[str, Any], dict[str, Any]]: + field_type = definition.get("type") + links: dict[str, Any] = {} + if field_type == "string": + # v0.2 reads coerced scalar values before validation. A migrated schema + # must therefore accept the raw YAML scalar forms that v0.2 accepted. + schema = {"type": ["string", "number", "boolean"]} + elif field_type == "integer": + schema = { + "anyOf": [ + {"type": "integer"}, + {"type": "number", "multipleOf": 1}, + {"type": "string", "pattern": r"^-?(?:0|[1-9][0-9]*)(?:\.0+)?$"}, + ] + } + elif field_type == "number": + schema = { + "anyOf": [ + {"type": "number"}, + { + "type": "string", + "pattern": r"^-?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?$", + }, + ] + } + elif field_type == "boolean": + schema = { + "anyOf": [ + {"type": "boolean"}, + {"enum": ["true", "false", "yes", "no", "on", "off"]}, + ] + } + elif field_type == "date": + schema = {"type": "string", "format": "date"} + elif field_type == "datetime": + schema = { + "type": "string", + "pattern": ( + r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}" + r"(?:\.[0-9]+)?(?:Z|[+-][0-9]{2}:[0-9]{2})?$" + ), + } + elif field_type == "time": + schema = {"type": "string", "pattern": r"^[0-9]{2}:[0-9]{2}(?::[0-9]{2})?$"} + elif field_type == "enum": + schema = {"enum": copy.deepcopy(definition.get("values") or [])} + elif field_type == "link": + schema = {"type": "string"} + target = definition.get("target") or "any" + if tasknotes_role in {"recurrenceParent", "blockedBy"} and selector.endswith(("recurrence_parent", ".uid")): + target = "task" + links[selector] = { + "target_type": target, + "validate_exists": bool(definition.get("validate_exists", False)), + } + elif field_type == "list": + item_definition = definition.get("items") if isinstance(definition.get("items"), dict) else {} + item_schema, item_links = convert_field( + f"{selector}[]", + item_definition, + strict=strict, + tasknotes_role=tasknotes_role, + # v0.2 treats a null list item as empty for optional item schemas. + # Retain that accepted persisted shape during metadata-only migration. + allow_null=True, + ) + schema = {"type": "array", "items": item_schema} + links.update(item_links) + elif field_type == "object": + child_properties: dict[str, Any] = {} + child_required: list[str] = [] + for child_name, child_definition in (definition.get("fields") or {}).items(): + if not isinstance(child_definition, dict): + continue + child_schema, child_links = convert_field( + f"{selector}.{child_name}", + child_definition, + strict=strict, + tasknotes_role=tasknotes_role, + allow_null=child_definition.get("required") is not True, + ) + child_properties[child_name] = child_schema + links.update(child_links) + if child_definition.get("required") is True: + child_required.append(child_name) + schema = { + "type": "object", + "additionalProperties": not strict if child_properties else True, + "properties": child_properties, + } + if child_required: + schema["required"] = child_required + else: + schema = {} + + if isinstance(definition.get("description"), str): + schema["description"] = definition["description"] + if isinstance(definition.get("pattern"), str): + schema["pattern"] = definition["pattern"] + if definition.get("deprecated"): + schema["deprecated"] = True + if isinstance(definition.get("min"), (int, float)): + if field_type == "string": + schema["minLength"] = definition["min"] + elif field_type == "list": + schema["minItems"] = definition["min"] + else: + schema["minimum"] = definition["min"] + if isinstance(definition.get("max"), (int, float)): + if field_type == "string": + schema["maxLength"] = definition["max"] + elif field_type == "list": + schema["maxItems"] = definition["max"] + else: + schema["maximum"] = definition["max"] + if isinstance(definition.get("min_length"), int): + schema["minLength"] = definition["min_length"] + if isinstance(definition.get("max_length"), int): + schema["maxLength"] = definition["max_length"] + if isinstance(definition.get("min_items"), int): + schema["minItems"] = definition["min_items"] + if isinstance(definition.get("max_items"), int): + schema["maxItems"] = definition["max_items"] + if definition.get("unique") is True and field_type == "list": + schema["uniqueItems"] = True + if allow_null: + schema = {"anyOf": [schema, {"type": "null"}]} + return schema, links + + +def add_lifecycle(lifecycle: dict[str, Any], field: str, generated: Any) -> bool: + on_create: dict[str, Any] | None = None + on_update: dict[str, Any] | None = None + if generated == "now": + on_create = {"now": True} + elif generated == "now_on_write": + on_create = {"now": True} + on_update = {"now": True} + elif generated == "uuid": + on_create = {"uuid": True} + elif generated == "ulid": + on_create = {"ulid": True} + elif isinstance(generated, dict) and isinstance(generated.get("from"), str): + transform = generated.get("transform") + if transform in (None, "copy"): + on_create = {"copy": generated["from"]} + elif transform == "slugify": + on_create = {"slugify": generated["from"]} + else: + return False + else: + return False + if on_create: + lifecycle.setdefault("on_create", {}).setdefault("set", {})[field] = on_create + if on_update: + lifecycle.setdefault("on_update", {}).setdefault("set", {})[field] = on_update + return True + + +def lifecycle_has_field(lifecycle: dict[str, Any], field: str) -> bool: + return any(field in event.get("set", {}) for event in lifecycle.values()) + + +def migrate_match(value: Any) -> dict[str, Any] | None: + if not isinstance(value, dict): + return None + target = copy.deepcopy(value) + where = target.get("where") + if isinstance(where, dict): + normalized: dict[str, Any] = {} + for key, predicate in where.items(): + if isinstance(key, str) and "." in key: + field, operator = key.rsplit(".", 1) + if operator in { + "eq", + "neq", + "contains", + "containsAll", + "containsAny", + "exists", + "startsWith", + "endsWith", + "matches", + "gt", + "gte", + "lt", + "lte", + }: + normalized.setdefault(field, {})[operator] = predicate + continue + normalized[key] = predicate + target["where"] = normalized + return target or None + + +def tasknotes_path(pattern: str) -> tuple[str, str]: + normalized = pattern.replace("\\", "/") + folder, _, filename = normalized.rpartition("/") + template = re.sub(r"\{([A-Za-z_][A-Za-z0-9_]*)\}", r"{{\1}}", filename) + if template.endswith(".md"): + template = template[:-3] + return folder, template or "{{title}}" + + +def migrated_body(source: SourceType, target: dict[str, Any]) -> str: + if target.get("name") == "meta" and target.get("x-mdbase", {}).get("materialized") is True: + return ( + "\n\n# Meta\n\n" + "This materialized meta type mirrors the canonical mdbase v0.3 type-file schema.\n" + "The implementation's built-in schema remains authoritative during collection bootstrap.\n" + ) + if target.get("x-tasknotes", {}).get("contract") == "tasknotes.task": + return ( + "\n\n# Task\n\n" + "This type definition is generated from TaskNotes settings for mdbase v0.3.\n" + "Its JSON Schema describes persisted task frontmatter; collection and lifecycle\n" + "metadata describe generic mdbase behavior; `x-tasknotes` records the optional\n" + "TaskNotes task contract.\n\n" + "This file is automatically generated and should not be edited manually.\n" + ) + return source.body + + +def render_type(frontmatter: dict[str, Any], body: str) -> str: + return f"---\n{yaml_dump(frontmatter).rstrip()}\n---{body}" + + +def validate_staged_collection( + collection: Path, + proposed: Path, + mdb: Path, +) -> tuple[dict[str, Any], dict[str, Any]]: + baseline = run_mdb_validate(mdb, collection) + with tempfile.TemporaryDirectory(prefix="mdbase-v03-stage-") as temporary: + stage = Path(temporary) + copy_markdown_collection(collection, stage) + shutil.copy2(proposed / "mdbase.yaml", stage / "mdbase.yaml") + target_types = load_yaml(proposed / "mdbase.yaml")["settings"]["types_folder"] + staged_types = stage / target_types + if staged_types.exists(): + shutil.rmtree(staged_types) + shutil.copytree(proposed / target_types, staged_types) + target = run_mdb_validate(mdb, stage) + + baseline_keys = diagnostic_keys(baseline) + target_keys = diagnostic_keys(target) + regressions = sorted(target_keys - baseline_keys) + validation_errors = [ + item for item in target.get("issues", []) if item.get("path", "").startswith("_types/") + ] + if target.get("error"): + validation_errors.append(target["error"]) + target_validation = { + "status": "failed" if validation_errors else "passed", + "errors": summarize_diagnostics(validation_errors), + } + records = { + "status": "passed" if not regressions else "failed", + "baseline": summarize_validation(baseline), + "target": summarize_validation(target), + "regressions": [ + {"path": path, "field": field, "code": code} + for path, field, code in regressions + ], + } + return target_validation, records + + +def copy_markdown_collection(source: Path, target: Path) -> None: + ignored_roots = {".git", ".obsidian", ".mdbase", "node_modules"} + for root, directories, files in os.walk(source): + directories[:] = [name for name in directories if name not in ignored_roots] + root_path = Path(root) + relative_root = root_path.relative_to(source) + for filename in files: + if filename == "mdbase.yaml" or filename.endswith(".md"): + source_file = root_path / filename + target_file = target / relative_root / filename + target_file.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source_file, target_file) + + +def run_mdb_validate(mdb: Path, collection: Path) -> dict[str, Any]: + completed = subprocess.run( + [str(mdb), "-C", str(collection), "validate"], + check=False, + capture_output=True, + text=True, + ) + payload = completed.stdout.strip() or completed.stderr.strip() + if not payload: + raise SystemExit("mdb validation produced no JSON") + try: + return json.loads(payload) + except json.JSONDecodeError as error: + raise SystemExit(f"mdb validation returned invalid JSON: {error}") from error + + +def diagnostic_keys(result: dict[str, Any]) -> set[tuple[str, str, str]]: + return { + ( + str(item.get("path") or ""), + str(item.get("field") or ""), + diagnostic_family(str(item.get("code") or "")), + ) + for item in result.get("issues", []) + if not str(item.get("path") or "").startswith("_types/") + } + + +def diagnostic_family(code: str) -> str: + if code in { + "invalid_datetime", + "invalid_enum", + "list_item_invalid", + "missing_required", + "type_mismatch", + "schema_validation_error", + "schema_required", + "schema_type", + "schema_enum", + "schema_one_of", + "schema_any_of", + "schema_additional_properties", + "format_invalid", + }: + return "schema" + return code + + +def summarize_validation(result: dict[str, Any]) -> dict[str, Any]: + issues = result.get("issues", []) + return { + "valid": bool(result.get("valid")), + "issue_count": len(issues), + "codes": summarize_diagnostics(issues), + } + + +def summarize_diagnostics(issues: list[dict[str, Any]]) -> list[dict[str, Any]]: + counts: dict[str, int] = {} + for item in issues: + code = str(item.get("code") or "unknown") + counts[code] = counts.get(code, 0) + 1 + return [{"code": code, "count": counts[code]} for code in sorted(counts)] + + +def apply_migration( + collection: Path, + proposed: Path, + backup: Path, + report: dict[str, Any], +) -> Path: + if backup.exists(): + raise SystemExit(f"backup already exists: {backup}") + backup.mkdir(parents=True) + paths = [Path("mdbase.yaml"), *(Path(item["path"]) for item in report["types"])] + manifest = {"collection": str(collection), "files": []} + for relative in paths: + source = collection / relative + backup_file = backup / relative + backup_file.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, backup_file) + manifest["files"].append( + { + "path": relative.as_posix(), + "sha256": sha256(source.read_bytes()), + } + ) + (backup / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + + for relative in paths: + target = collection / relative + staged = proposed / relative + temporary = target.with_name(f".{target.name}.mdbase-v03.tmp") + shutil.copy2(staged, temporary) + os.replace(temporary, target) + return backup + + +def validate_schema(schema_path: Path, value: Any, label: str) -> None: + schema = json.loads(schema_path.read_text()) + errors = sorted(Draft202012Validator(schema).iter_errors(value), key=lambda item: list(item.path)) + if errors: + details = "; ".join( + f"{'/'.join(str(part) for part in error.path) or ''}: {error.message}" + for error in errors[:10] + ) + raise SystemExit(f"invalid migrated {label}: {details}") + + +def load_yaml(path: Path) -> Any: + return normalize_yaml(yaml.safe_load(path.read_text())) + + +def normalize_yaml(value: Any) -> Any: + if isinstance(value, dict): + return {str(key): normalize_yaml(child) for key, child in value.items()} + if isinstance(value, list): + return [normalize_yaml(child) for child in value] + if isinstance(value, (dt.datetime, dt.date, dt.time)): + return value.isoformat() + return value + + +def yaml_dump(value: Any) -> str: + return yaml.safe_dump( + value, + sort_keys=False, + allow_unicode=True, + default_flow_style=False, + width=1000, + ) + + +def is_field_path(value: str) -> bool: + return bool( + re.fullmatch( + r"[A-Za-z_][A-Za-z0-9_:-]*(?:\[\])?(?:\.[A-Za-z_][A-Za-z0-9_:-]*(?:\[\])?)*", + value, + ) + ) + + +def sha256(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +if __name__ == "__main__": + sys.exit(main()) From bedd6c9b4567b301a4a1bd88ae8d8272247d8b17 Mon Sep 17 00:00:00 2001 From: callumalpass Date: Wed, 22 Jul 2026 06:56:00 +1000 Subject: [PATCH 2/2] feat: specify portable query and view records --- 00-overview.md | 22 +- 01-concepts.md | 15 + 02-collection-layout.md | 5 + 04-configuration.md | 5 + 05-type-files.md | 13 + 07-collection-semantics.md | 5 + 10-cel-profile.md | 40 +- 11-querying.md | 333 ++++++++++++-- 15-migrations-and-compatibility.md | 36 ++ 16-conformance.md | 35 +- CHANGELOG.md | 22 + README.md | 3 +- _types/view.md | 28 ++ schemas/v0.3/README.md | 3 + schemas/v0.3/query-result.schema.json | 59 ++- schemas/v0.3/query.schema.json | 171 ++++++++ schemas/v0.3/view.schema.json | 260 +++++++++++ scripts/check_v03_tests.py | 4 +- tests/v0.3/README.md | 16 +- tests/v0.3/cel/cel-profile.yaml | 1 + .../fixtures/views/invalid-query-context.yml | 4 + .../views/invalid-view-unknown-member.md | 12 + tests/v0.3/fixtures/views/valid-query.yml | 31 ++ tests/v0.3/fixtures/views/valid-task-views.md | 68 +++ tests/v0.3/lifecycle/lifecycle.yaml | 6 +- tests/v0.3/manifest.yaml | 8 + tests/v0.3/schema/schema-artifacts.yaml | 47 +- tests/v0.3/views/view-records.yaml | 406 ++++++++++++++++++ 28 files changed, 1603 insertions(+), 55 deletions(-) create mode 100644 _types/view.md create mode 100644 schemas/v0.3/query.schema.json create mode 100644 schemas/v0.3/view.schema.json create mode 100644 tests/v0.3/fixtures/views/invalid-query-context.yml create mode 100644 tests/v0.3/fixtures/views/invalid-view-unknown-member.md create mode 100644 tests/v0.3/fixtures/views/valid-query.yml create mode 100644 tests/v0.3/fixtures/views/valid-task-views.md create mode 100644 tests/v0.3/views/view-records.yaml diff --git a/00-overview.md b/00-overview.md index 0766332..1167f30 100644 --- a/00-overview.md +++ b/00-overview.md @@ -4,8 +4,9 @@ This specification defines the behavior of tools that treat folders of Markdown files as typed, queryable, link-aware data collections. It covers -collection discovery, JSON Schema types, validation, links, CEL queries, record -operations, lifecycle policy, and optional runtime workflows. +collection discovery, JSON Schema types, validation, links, CEL queries, +ordinary records that save named views, record operations, lifecycle policy, +and optional runtime workflows. ## Motivation @@ -44,7 +45,8 @@ workflows. ## What a conforming tool does -Depending on its conformance profiles, a tool implementing this specification: +Depending on its conformance profiles and optional features, a tool implementing +this specification: 1. **Recognizes collections** by the presence of an `mdbase.yaml` configuration file. @@ -56,9 +58,11 @@ Depending on its conformance profiles, a tool implementing this specification: 5. **Resolves links** between records and exposes link-aware metadata. 6. **Executes queries** using CEL expressions for filtering, ordering, and projection. -7. **Performs record operations** with validation, reference handling, and +7. **Executes saved view records** when it advertises the optional + `view_records` feature. +8. **Performs record operations** with validation, reference handling, and lifecycle-managed values. -8. **Loads runtime contracts and workflows** when it supports active behavior. +9. **Loads runtime contracts and workflows** when it supports active behavior. Conformance profiles define the expected behavior for each capability and its dependencies. @@ -208,6 +212,14 @@ due_date < today() assignee.asFile().team == "engineering" ``` +### View records save reusable queries + +A collection can define the ordinary `view` type and store one or more named +queries in a Markdown record. Shared query scope, named-view filters, +projections, ordering, grouping, and summaries remain machine-readable, while +the Markdown body documents the view for people. Optional presentation metadata +can select a renderer without changing query results. + ### Validation is progressive Files in a collection can remain untyped records. Types can be added diff --git a/01-concepts.md b/01-concepts.md index e68fbdc..65a8b71 100644 --- a/01-concepts.md +++ b/01-concepts.md @@ -82,6 +82,21 @@ lifecycle guards. `match.where` uses the standalone structured predicate language defined in Chapter 07. +## View + +A view is an ordinary Markdown record whose matched type is `view`. It stores +shared query scope and one or more named queries, with optional advisory +presentation metadata. + +Views do not introduce a second query engine. A view-aware tool resolves a +named view to the query model from Chapter 11 and executes it through the Query +profile. Tools that do not support view execution continue to read and validate +view files as ordinary typed records. + +View records are passive collection data. Rendering a view, registering a +renderer, or connecting user interaction to actions may be tool- or +runtime-specific, but the record itself is not a runtime contract. + ## Link A link is a frontmatter value or body reference that can resolve to another diff --git a/02-collection-layout.md b/02-collection-layout.md index 6b19310..9e8b8ab 100644 --- a/02-collection-layout.md +++ b/02-collection-layout.md @@ -37,6 +37,11 @@ collection/ Only `mdbase.yaml` is required. Untyped records form a valid collection. +View records are ordinary records and require no reserved folder. A collection +MAY organize them under `Views/`, `_views/`, or any other non-excluded path. +Unlike the configured types folder, such a folder remains part of the normal +record scan unless explicitly excluded. + ## Reserved Paths The following paths are reserved by default: diff --git a/04-configuration.md b/04-configuration.md index 75d9f1d..4f7cb79 100644 --- a/04-configuration.md +++ b/04-configuration.md @@ -98,6 +98,11 @@ Tools MAY support non-portable UI expression dialects. Portable stored v0.3 files MUST use the mdbase CEL profile unless a feature declares a different extension namespace. +View records use CEL for portable filters, projections, selections, and custom +summaries. Compatibility tools MAY read another view or expression format, but +alternate source and round-trip metadata belong under an `x-*` extension and do +not change the meaning of the portable CEL fields. + ## Version Compatibility Patch versions within the same stable minor version MUST be backward compatible. diff --git a/05-type-files.md b/05-type-files.md index 946a562..a2b5673 100644 --- a/05-type-files.md +++ b/05-type-files.md @@ -170,6 +170,19 @@ type-file frontmatter against the v0.3 type-file JSON Schema. The built-in schema is authoritative during bootstrap. A materialized `_types/meta.md` mirrors and documents that behavior. +## View Type + +Saved views use the ordinary `view` type defined by +`schemas/v0.3/view.schema.json`. A collection that stores portable view records +SHOULD materialize `_types/view.md` with `match.where.type: view` and a local +reference to that schema. The repository's `_types/view.md` is the canonical +materialization. + +Unlike the meta type, the view type is not required for bootstrap and is not a +built-in control-file category. A view file remains an ordinary Markdown record +and participates in normal reads, validation, links, writes, and type matching. +View-aware execution is the optional behavior defined in Chapter 11. + ## Type Membership Records may select types explicitly or through inferred matching. Chapter 07 diff --git a/07-collection-semantics.md b/07-collection-semantics.md index 233ded6..9550eeb 100644 --- a/07-collection-semantics.md +++ b/07-collection-semantics.md @@ -236,6 +236,11 @@ collection: Projection values are available to queries. Persistence occurs only through an explicit write operation or runtime workflow. +Collection projections enter the effective record and remain available through +their declared field names. Query- and view-local named projections are +separate, live under the `projection` CEL namespace, and never replace a +collection projection or persisted field with the same name. + ## Domain Namespaces Domain annotations use namespaced extension sections: diff --git a/10-cel-profile.md b/10-cel-profile.md index 25e76d6..dcd5976 100644 --- a/10-cel-profile.md +++ b/10-cel-profile.md @@ -44,7 +44,8 @@ record fields. | Context | Available values | | --- | --- | | inferred match | candidate raw fields at top level; `record`, `raw`, `present`, `file`, `note` | -| query or projection | effective fields at top level; `record`, `raw`, `present`, `file`, `note`; `this` for an embedded query | +| query or projection | effective candidate fields at top level; `record`, `raw`, `present`, `file`, `note`; `projection`; `this` as the invocation-context record or null | +| query summary | `values`; the fixed operation time and timezone | | lifecycle guard | current draft fields at top level; `record`, `raw`, `present`, `old`, `file`, `operation` | | workflow variable, trigger, or workflow condition | `event`, `workflow`, `trigger`, `vars` | | workflow step condition or input | `event`, `workflow`, `trigger`, `steps`, `vars`; `item` during iteration | @@ -53,10 +54,10 @@ record fields. An unavailable system binding is a compile or preflight diagnostic. For example, `steps` is unavailable to a trigger condition because no step has run. -The system names `record`, `raw`, `present`, `file`, `note`, `this`, `old`, -`operation`, `event`, `workflow`, `trigger`, `steps`, `vars`, and `item` are -reserved. A frontmatter field with one of those names remains available through -`record.` and `raw.`. +The system names `record`, `raw`, `present`, `file`, `note`, `projection`, +`this`, `values`, `old`, `operation`, `event`, `workflow`, `trigger`, `steps`, +`vars`, and `item` are reserved. A frontmatter field with one of those names +remains available through `record.` and `raw.`. ### Query Context @@ -67,9 +68,32 @@ contains persisted frontmatter. `note` is an alias for `record`. present.raw.status == false && record.status == "open" ``` -`file` supplies the metadata and helpers defined by the collection, query, and -link profiles. An embedded query may receive `this`, which refers to its -containing record. +`file` supplies the candidate metadata and helpers defined by the collection, +query, and link profiles. `projection` contains named query projections after +their dependency-ordered evaluation. + +`this` is reserved for the query invocation-context record. It is null when no +context is bound. A non-null context mirrors the candidate query namespaces: + +- `this.`, `this.record.`, and `this.note.` expose + effective context values +- `this.raw.` exposes persisted context frontmatter +- `this.present.record.` and `this.present.raw.` expose presence +- `this.file` exposes context-file metadata and the helpers available to the + active profiles + +When an effective context field conflicts with the reserved members `record`, +`note`, `raw`, `present`, or `file`, it remains available through +`this.record.` and `this.raw.`. + +The host resolves and snapshots the context once before evaluating candidates. +All candidates see the same immutable context, operation time, and timezone. +Context link values and `this.file` helpers resolve relative to the context +record; candidate values and `file` helpers resolve relative to the candidate. +`this` is record-only and MUST NOT be repurposed as an arbitrary caller +parameter map. A feature that adds parameters uses a distinct binding and +declares its own context contract. +Chapter 11 defines portable context binding and saved-view invocation. ### Matching Context diff --git a/11-querying.md b/11-querying.md index 3fcbe7c..d0af532 100644 --- a/11-querying.md +++ b/11-querying.md @@ -2,11 +2,22 @@ ## Query Object -A query selects records from a collection. +A query selects records from a collection. The canonical query-object schema is +`schemas/v0.3/query.schema.json`. ```yaml types: [task] +context: + this: + path: projects/alpha.md +projections: + is_overdue: + expr: 'present.record.due && due < today() && status != "done"' where: 'status != "done" && priority >= 3' +select: + - title + - due + - projection.is_overdue order_by: - field: due direction: asc @@ -15,6 +26,13 @@ offset: 0 include_body: false ``` +Unknown query members are invalid. `x-*` members MAY carry adapter metadata but +MUST NOT change the meaning of portable members. + +Schema violations and semantic preflight failures such as cyclic projection +dependencies or duplicate result names produce `invalid_query` and abort before +candidate evaluation. + ## Types `types` is an OR filter. A record is included if it matches at least one listed @@ -22,40 +40,100 @@ type. If `types` is omitted, all records are candidates. +## Invocation Context And `this` + +A query MAY bind one collection record as its invocation context: + +```yaml +context: + this: + path: projects/alpha.md +``` + +The path is collection-relative and MUST resolve to an ordinary record in the +same collection. The implementation reads the context through the same raw and +effective-value pipeline as any other record, snapshots it once before +candidate evaluation, and binds it to `this` as defined in Chapter 10. + +When no context is supplied, `this` is null. Supplying an unresolved context +produces `context_not_found` and aborts the query before candidate evaluation. +An invalid context is handled according to the collection validation level. + +Query time, timezone, and collection state are fixed for the context and all +candidates in one execution. A caller MUST NOT replace or mutate the context +between candidate evaluations. + +## Named Projections + +Queries MAY define named projections evaluated for every candidate before the +filter: + +```yaml +projections: + is_overdue: + expr: 'present.record.due && due < today() && status != "done"' + urgency: + expr: 'priority + (projection.is_overdue ? 10 : 0)' +``` + +Projection names are available under `projection.` in subsequent +projection expressions, `where`, selection expressions, ordering, grouping, +summaries, and presentation mappings. + +Implementations MUST resolve projection dependencies deterministically and +reject direct or indirect cycles before evaluating candidates. A projection +evaluation error produces null for that candidate and a per-record diagnostic; +it does not abort evaluation of other candidates. + +Named query projections are effective query values. They are not persisted and +do not replace raw or effective frontmatter fields with the same name. + ## Where -`where` is a CEL expression evaluated against the effective record. The -expression result is truthy only when it evaluates to boolean true. +`where` is a CEL expression evaluated against the effective candidate and its +named projections. The expression result includes the record only when it is +boolean true. -Evaluation errors MUST produce null for that record and report diagnostics -according to query options. +Evaluation errors MUST produce null for that record, exclude it, and report a +diagnostic according to query options. -## Projections +## Selection -Queries MAY request projections: +Queries MAY request a logical result shape: ```yaml select: - title - file.path - - name: is_overdue - expr: 'present.record.due && due < today() && status != "done"' + - projection.is_overdue + - name: display_due + expr: 'due == null ? "Unscheduled" : string(due)' + label: Due ``` -Projection expressions use CEL. +A string selects an effective field, file value, or named projection. An object +defines a named CEL result value. Selection expressions run after filtering and +may use the candidate, named projections, and `this`. + +For a string selector, the output name is the effective-field name or the final +member of a `file.` or `projection.` selector. An object uses its +required `name`. Two selections that produce the same output name make the +query invalid; callers use an object selection to assign an unambiguous name. -Computed projection values belong to the query result. Persistence requires an -explicit write operation. +Computed selection values belong to the query result. Persistence requires an +explicit write operation. Every result includes `file.path` in its `file` +object even when `file.path` is not selected. ## Ordering -`order_by` sorts by one or more fields or projection names. +`order_by` sorts by one or more effective fields, file values, named +projections, or named selection values. ```yaml order_by: - field: due direction: asc - - field: priority + - field: projection.urgency direction: desc ``` @@ -64,12 +142,64 @@ Null values sort last in ascending order and first in descending order. When all ordering fields compare equal, tools MUST tie-break by ascending `file.path` for deterministic results. +## Grouping And Summaries + +Queries MAY group the complete filtered and ordered result set by one or more +values: + +```yaml +group_by: + - field: status + direction: asc +``` + +Each distinct ordered tuple of grouping values creates one group. Null and +missing values form the same null group. Group tuple ordering applies the same +direction and null rules as `order_by`. Results within a group retain query +order. + +Queries MAY define custom summary functions and apply built-in or custom +functions to values: + +```yaml +summary_functions: + completion_rate: + expr: 'values.size() == 0 ? 0 : values.filter(v, v).size() * 100 / values.size()' + +summaries: + - field: estimate + function: sum + name: total_estimate + - field: projection.is_completed + function: completion_rate + name: completion_rate +``` + +A custom summary expression receives `values`, an ordered list containing one +value per matching result in result order. Missing fields contribute null. +When grouping is active, the function is evaluated independently for each +group; otherwise it is evaluated once for the complete match set. + +A summary's result key is its explicit `name`, or its `function` identifier when +`name` is omitted. Duplicate result keys make the query invalid. + +The portable built-in summary identifiers are `count`, `sum`, `average`, +`minimum`, `maximum`, `earliest`, `latest`, `empty`, and `filled`. `count` +counts all values. `empty` and `filled` count empty and non-empty values using +the CEL missing/null contract. Other built-ins ignore null values and produce +null when no compatible non-null value exists. Incompatible non-null values +produce a summary diagnostic and a null summary result. + +Grouping and summaries are calculated before pagination and therefore describe +the complete filtered match set. Group metadata does not replace the flat +`results` page. + ## Pagination `limit` and `offset` apply after filtering and sorting. -`limit: 0` returns an empty result page. Total count metadata still describes -the complete match set. +`limit: 0` returns an empty result page. Total count, grouping, and summary +metadata still describe the complete match set. ## Body Search @@ -79,6 +209,19 @@ case the body is available for filtering but not returned in results. Tools MAY report that body filtering requires a profile or index when they cannot read bodies on demand. +## Frontmatter Mode + +`frontmatter` controls which record frontmatter appears in each result: + +- `effective` (the default) returns effective values in `frontmatter` +- `raw` returns raw persisted values in `frontmatter` +- `both` returns effective values in `frontmatter` and raw persisted values in + `raw_frontmatter` + +The mode changes result serialization only. Filtering, projections, selection, +ordering, grouping, and summaries continue to use effective values unless an +expression explicitly reads the raw namespace. + ## Result Envelope Query results MUST use this envelope: @@ -90,9 +233,20 @@ results: frontmatter: title: Fix login status: open + values: + title: Fix login + is_overdue: true meta: total_count: 1 has_more: false + context: + path: projects/alpha.md + groups: + - values: + status: open + count: 1 + summaries: + total_estimate: 30 diagnostics: [] ``` @@ -100,15 +254,146 @@ Each result MUST include `file.path`. `meta.total_count` is the count before pagination, and `meta.has_more` is true when additional matching records remain. Diagnostics use the canonical diagnostic envelope from the conformance chapter. -`frontmatter` contains effective values unless the query asks for raw persisted -frontmatter. +Read defaults are included in effective frontmatter. `values` contains +requested selection values and is omitted when `select` is omitted. + +`meta.context.path` identifies the bound invocation context and is omitted when +`this` is null. `meta.groups` is present only when grouping or summaries were +requested. Without grouping, summaries appear in one group whose `values` is an +empty object. + +## Saved View Records + +A saved view is an ordinary Markdown record matched by the `view` type. It is +not a runtime contract and does not introduce a second query language or query +engine. The specification does not define a separate persisted query-record +kind: a query is the reusable value object above, and a view record is its +portable persisted container. The canonical record schema is +`schemas/v0.3/view.schema.json`; a collection can materialize the corresponding +`_types/view.md` type file. + +One view record contains a shared canonical query fragment and one or more +named views. The shared fragment is nested under `query` so its `types` member +cannot be mistaken for the record-level explicit type declaration: + +```markdown +--- +type: view +id: tasknotes.tasks +version: 1 +name: Task views + +query: + types: [task] + projections: + urgency: + expr: 'priority + (due < today() ? 10 : 0)' + +views: + - id: today + name: Today + where: 'due == today() || scheduled == today()' + select: [title, due, projection.urgency] + order_by: + - field: projection.urgency + direction: desc + presentation: + type: tasknotes.task-list + fallback: mdbase.table +--- + +# Task views + +Reusable task views for editors, CLIs, and agents. +``` + +### Named-view resolution + +A named view is addressed by the view record path or stable record ID plus the +named-view ID. Human-readable names are not identifiers. Duplicate named-view +IDs make the view record invalid. + +To derive an executable query: -Read defaults are included in effective frontmatter. Projections are included -only when requested. +1. inherit `query.types` unless the named view supplies `types` +2. combine `query.where` and named-view `where` with AND +3. merge `query.projections` and named-view projections; the same name may appear in both + only when its parsed definition is structurally equal, ignoring mapping key + order +4. inherit `query.context` unless the named view supplies context +5. copy the named view's selection, ordering, grouping, summaries, pagination, + and body options +6. bind the invocation context using the rules below -## Embedded Queries +Record-level property metadata and summary functions remain available to every +named view. Resolving an unknown view record or named-view ID produces +`view_not_found`. -Embedded query contexts may bind `this` to the containing record. +View execution returns the ordinary query envelope and adds +`meta.view: { path, id }`, using the resolved view-record path and named-view +ID. It does not introduce a second result format. + +Property-metadata keys MAY name effective fields, `file.*` values, +`projection.*` values, or selection outputs. They provide labels, descriptions, +format hints, and visibility hints only; they do not add values to `select`. + +### View invocation context + +View records declare what to do when the caller supplies no context: + +```yaml +query: + context: + this: + on_missing: view + types: [project] +``` -Tools that support embedded queries MUST define how `this` is supplied and how -collection-relative links resolve from the embedding file. +`on_missing` values are: + +| Value | Behavior | +| --- | --- | +| `view` | bind the view-definition record; default | +| `null` | bind `this` to null | +| `error` | abort with `context_required` | + +An explicitly supplied context always wins. If `types` is present, a non-null +context must match at least one listed type or execution aborts with +`context_type_mismatch`. A named-view context declaration replaces, rather than +partially merges with, the shared `query.context` declaration. + +An embedding host supplies the embedding record. A headless caller supplies a +collection-relative record path. An editor may map an active record to the +explicit invocation context, but ambient concepts such as workspace leaves or +sidebars are not part of portable execution. + +### Presentation + +`presentation.type` and `presentation.fallback` are open renderer identifiers, +not a closed enumeration. `select` defines the portable logical result shape; +presentation mappings and options describe how a supporting tool may render +that result. + +Each `presentation.mappings` key is a renderer-defined role and its value names +an output produced by `select`. Selected named projections are therefore +available to presentation without giving presentation its own projection +language. A named view with no `presentation` is a complete headless saved +query. + +Presentation metadata MUST NOT affect filtering, projection, ordering, +grouping, summaries, pagination, or the headless result envelope. An unknown +renderer does not prevent headless execution. A request that specifically asks +for unavailable rendering reports `unsupported_presentation` and may use the +declared fallback. + +Tool-specific source syntax, renderer configuration, and round-trip data use +`x-*` extensions. An Obsidian Bases adapter, for example, may translate Bases +filters and formulas to portable CEL while preserving untranslatable source +under `x-obsidian`. + +### Optional support + +All Core Read implementations treat a view file as an ordinary typed record. +A tool advertises `view_records` in its existing `optional_features` claim only +when it resolves and executes named views with the semantics in this chapter. +No runtime profile is required. diff --git a/15-migrations-and-compatibility.md b/15-migrations-and-compatibility.md index e1c4d3d..da92add 100644 --- a/15-migrations-and-compatibility.md +++ b/15-migrations-and-compatibility.md @@ -85,6 +85,42 @@ Tool-specific expression dialects are adapter concerns. Tools may translate them to CEL for portable storage and translate them back for user interfaces or exports. +## Obsidian Bases Views + +Obsidian `.base` files are compatibility inputs, not mdbase records required by +Core Read. An adapter may import, execute, or export them through the ordinary +view-record model from Chapter 11. + +The structural mapping is: + +| Obsidian Bases | mdbase view record | +| --- | --- | +| global `filters` | `query.where` | +| view `filters` | named-view `where`, combined with the shared filter | +| `formulas` | `query.projections` | +| `formula.name` | `projection.name` | +| `properties` | property metadata | +| view `order` | `select` order | +| view `sort` | `order_by` | +| `groupBy` | `group_by` | +| custom and property summaries | `summary_functions` and `summaries` | +| view `type` | `presentation.type` | +| plugin view keys | presentation options or `x-*` extension data | + +An adapter SHOULD parse the source dialect into an inspectable syntax tree and +translate it to CEL only when behavior can be preserved. A partial translation +MUST report unsupported expressions, functions, value coercions, or renderer +features. It MUST NOT silently label a behavior-changing translation as +portable. Lossless source and round-trip metadata may be retained under +`x-obsidian`. + +Obsidian placement state maps to the portable invocation context rather than to +query semantics: opening a Base directly supplies the view definition, an +embed supplies its embedding record, and an active-file interface supplies its +active record. The adapter resolves that host state before calling the query or +view executor. Portable mdbase execution does not inspect editor workspace +state. + ## Runtime Workflows Current generated-field and tool-conforming behavior that causes mutation diff --git a/16-conformance.md b/16-conformance.md index 447ba62..b5324fa 100644 --- a/16-conformance.md +++ b/16-conformance.md @@ -12,7 +12,7 @@ queries, writes, runtime preflight, workflow execution, and watching. | Collection Semantics | apply defaults, uniqueness, path policy, multi-type composition, and collection diagnostics | | CEL | compile and evaluate the shared mdbase CEL language and host contract | | CEL Match | evaluate `match.expr` against raw candidate records | -| Query | evaluate CEL filters and projections and return query envelopes | +| Query | evaluate contextual CEL filters, projections, grouping, summaries, and query envelopes | | Links | parse, resolve, validate, and traverse links | | Core Write | create, update, delete, rename, and batch records | | Lifecycle | apply standard managed-field policy during writes | @@ -86,7 +86,9 @@ frontmatter selector. Implementations MAY add fields under `x-*`. The v0.3 core codes include `unsupported_profile`, `type_conflict`, `type_membership_changed`, `path_value_missing`, `schema_ref_forbidden`, `schema_ref_unresolved`, `schema_ref_cycle`, `format_invalid`, -`lifecycle_expression_error`, and `concurrent_modification`. Runtime profile +`lifecycle_expression_error`, `concurrent_modification`, `invalid_query`, +`context_not_found`, `context_required`, `context_type_mismatch`, +`view_not_found`, `invalid_view`, and `unsupported_presentation`. Runtime profile 0.1 additionally defines `contract_conflict`, `contract_version_mismatch`, `event_provider_mismatch`, `provider_version_mismatch`, `capability_denied`, `policy_not_selected`, `executor_not_selected`, and @@ -146,14 +148,43 @@ CEL Match implementations MUST: Query implementations MUST: +- validate portable query objects against the canonical query schema +- resolve and snapshot an optional same-collection invocation context +- expose the complete `this` context contract, binding it to null when absent +- evaluate named query projections in dependency order before filtering +- reject cyclic projection dependencies and duplicate result names with + `invalid_query` - evaluate `where` filters against the effective query context - evaluate requested CEL projections - support OR-based type filtering - support deterministic ordering and pagination +- support deterministic grouping and built-in and custom summaries - return total-count and has-more metadata +- return context, grouping, and summary metadata when requested - expose raw and effective frontmatter when requested - report per-record evaluation errors and continue evaluating remaining records +## View Record Optional Feature + +View records do not define a separate conformance profile. They are ordinary +typed records and require no runtime profile. An implementation advertises +`view_records` through the existing `optional_features` member only when it: + +- validates view frontmatter against the canonical view schema +- resolves a stable view-record ID or path plus a stable named-view ID +- rejects duplicate named-view IDs with `invalid_view` +- derives the executable query using the inheritance and merge rules in + Chapter 11 +- applies `context.this.on_missing` and context type constraints before query + execution +- reports the selected view and resolved context in query result metadata +- treats presentation metadata as advisory and preserves headless results when + a renderer is unavailable +- keeps alternate dialect and renderer-specific data under `x-*` extensions + +A tool MAY advertise supported presentation identifiers separately in +`optional_features`. Presentation support is not required for `view_records`. + ## Links Requirements Links implementations MUST: diff --git a/CHANGELOG.md b/CHANGELOG.md index 81cf43f..0ed08c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,28 @@ All notable changes to this specification and conformance suite are documented here. +## 2026-07-21 (v0.3.0 draft) + +### Added + +- Canonical query-object and ordinary `type: view` record schemas. +- A materialized `_types/view.md` definition for Markdown view records. +- Named query projections, grouping, built-in/custom summaries, and selected + result values. +- Explicit query invocation context with a complete, immutable `this` record + namespace. +- Saved-view inheritance, context fallback/type constraints, open presentation + identifiers, and the optional `view_records` feature declaration. +- Obsidian Bases structural mapping guidance and view/context conformance + fixtures. + +### Changed + +- Replaced implementation-defined embedded-query context with deterministic + same-collection context binding and result metadata. +- Clarified that view records are passive ordinary records, not runtime + contracts, and that presentation metadata cannot change headless results. + ## 2026-02-03 (v0.2.0) ### Added diff --git a/README.md b/README.md index 9b28d08..953df55 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ The current specification is **v0.3.0**. - collection-aware defaults, uniqueness rules, and validation - portable links between Markdown records - CEL expressions for filtering, ordering, and projections +- ordinary Markdown view records for saved queries and advisory presentation - consistent create, read, update, delete, rename, and batch operations - lifecycle policies for IDs, timestamps, slugs, and managed values - optional runtime contracts for events, actions, capabilities, and workflows @@ -107,7 +108,7 @@ order_by: | --- | --- | | Understand the model | [Overview](./00-overview.md) and [Concepts](./01-concepts.md) | | Create a collection | [Collection Layout](./02-collection-layout.md), [Configuration](./04-configuration.md), and [Type Files](./05-type-files.md) | -| Validate or query records | [JSON Schema Profile](./06-json-schema-profile.md), [CEL Profile](./10-cel-profile.md), and [Querying](./11-querying.md) | +| Validate, query, or save views over records | [JSON Schema Profile](./06-json-schema-profile.md), [CEL Profile](./10-cel-profile.md), and [Querying](./11-querying.md) | | Add links or managed fields | [Links](./08-links.md) and [Lifecycle](./09-lifecycle.md) | | Define automation | [Runtime Contracts](./13-runtime-contracts.md) and [Workflows](./14-workflows.md) | | Migrate a v0.2 collection | [Migrations And Compatibility](./15-migrations-and-compatibility.md) | diff --git a/_types/view.md b/_types/view.md new file mode 100644 index 0000000..100eef6 --- /dev/null +++ b/_types/view.md @@ -0,0 +1,28 @@ +--- +kind: mdbase.type +name: view +version: 1 +description: A portable named collection of executable mdbase views. + +match: + where: + type: view + +schema: + dialect: json-schema-2020-12 + ref: "../schemas/v0.3/view.schema.json" + +collection: + display: + name_field: name +--- + +# View + +A view record stores shared query scope and one or more stable named views. +Each named view resolves to the query model in Chapter 11. Optional +`presentation` metadata is advisory and does not alter headless query results. + +View records are ordinary Markdown records, not runtime contracts. Collections +may copy or materialize this type definition when they want portable saved +views. diff --git a/schemas/v0.3/README.md b/schemas/v0.3/README.md index 12187de..f080728 100644 --- a/schemas/v0.3/README.md +++ b/schemas/v0.3/README.md @@ -10,6 +10,9 @@ yet published package artifacts. | Schema | Purpose | | --- | --- | | `type-file.schema.json` | frontmatter of `_types/*.md` v0.3 type files | +| `query.schema.json` | portable query input objects | +| `query-result.schema.json` | query results plus optional context, view, grouping, and summary metadata | +| `view.schema.json` | ordinary `type: view` record frontmatter | | `conformance-claim.schema.json` | machine-readable implementation profile claims and evidence | | `runtime/provider.schema.json` | provider contract records | | `runtime/workflow.schema.json` | workflow records | diff --git a/schemas/v0.3/query-result.schema.json b/schemas/v0.3/query-result.schema.json index 79f165c..7b87f94 100644 --- a/schemas/v0.3/query-result.schema.json +++ b/schemas/v0.3/query-result.schema.json @@ -25,6 +25,12 @@ "frontmatter": { "type": "object" }, + "raw_frontmatter": { + "type": "object" + }, + "values": { + "type": "object" + }, "body": { "type": "string" } @@ -42,6 +48,38 @@ }, "has_more": { "type": "boolean" + }, + "context": { + "type": "object", + "required": ["path"], + "properties": { + "path": { + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false + }, + "view": { + "type": "object", + "required": ["path", "id"], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "id": { + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false + }, + "groups": { + "type": "array", + "items": { + "$ref": "#/$defs/group" + } } }, "additionalProperties": true @@ -53,5 +91,24 @@ } } }, - "additionalProperties": false + "additionalProperties": false, + "$defs": { + "group": { + "type": "object", + "required": ["values", "count", "summaries"], + "properties": { + "values": { + "type": "object" + }, + "count": { + "type": "integer", + "minimum": 0 + }, + "summaries": { + "type": "object" + } + }, + "additionalProperties": false + } + } } diff --git a/schemas/v0.3/query.schema.json b/schemas/v0.3/query.schema.json new file mode 100644 index 0000000..ae5a71c --- /dev/null +++ b/schemas/v0.3/query.schema.json @@ -0,0 +1,171 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://mdbase.dev/schemas/v0.3/query.schema.json", + "title": "mdbase v0.3 query object", + "type": "object", + "properties": { + "types": { "$ref": "#/$defs/typeList" }, + "context": { "$ref": "#/$defs/queryContext" }, + "projections": { "$ref": "#/$defs/projectionSet" }, + "where": { "$ref": "#/$defs/expression" }, + "select": { "$ref": "#/$defs/select" }, + "order_by": { "$ref": "#/$defs/orderBy" }, + "group_by": { "$ref": "#/$defs/groupBy" }, + "summary_functions": { "$ref": "#/$defs/summaryFunctionSet" }, + "summaries": { "$ref": "#/$defs/summaries" }, + "limit": { "type": "integer", "minimum": 0 }, + "offset": { "type": "integer", "minimum": 0 }, + "include_body": { "type": "boolean", "default": false }, + "frontmatter": { + "enum": ["effective", "raw", "both"], + "default": "effective" + } + }, + "patternProperties": { + "^x-[A-Za-z][A-Za-z0-9._:-]{0,127}$": { "$ref": "#/$defs/extension" } + }, + "additionalProperties": false, + "$defs": { + "identifier": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9._:-]*$" + }, + "typeName": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9_-]{0,127}$" + }, + "fieldName": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_:-]*$" + }, + "expression": { + "type": "string", + "minLength": 1 + }, + "extension": { + "type": "object", + "additionalProperties": true + }, + "typeList": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "$ref": "#/$defs/typeName" } + }, + "queryContext": { + "type": "object", + "required": ["this"], + "properties": { + "this": { + "type": "object", + "required": ["path"], + "properties": { + "path": { "type": "string", "minLength": 1 } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "projectionSet": { + "type": "object", + "propertyNames": { "$ref": "#/$defs/fieldName" }, + "additionalProperties": { "$ref": "#/$defs/projection" } + }, + "projection": { + "type": "object", + "required": ["expr"], + "properties": { + "expr": { "$ref": "#/$defs/expression" }, + "description": { "type": "string" } + }, + "patternProperties": { + "^x-[A-Za-z][A-Za-z0-9._:-]{0,127}$": { "$ref": "#/$defs/extension" } + }, + "additionalProperties": false + }, + "select": { + "type": "array", + "minItems": 1, + "items": { + "oneOf": [ + { "type": "string", "minLength": 1 }, + { "$ref": "#/$defs/selectExpression" } + ] + } + }, + "selectExpression": { + "type": "object", + "required": ["name", "expr"], + "properties": { + "name": { "$ref": "#/$defs/fieldName" }, + "expr": { "$ref": "#/$defs/expression" }, + "label": { "type": "string" }, + "description": { "type": "string" } + }, + "patternProperties": { + "^x-[A-Za-z][A-Za-z0-9._:-]{0,127}$": { "$ref": "#/$defs/extension" } + }, + "additionalProperties": false + }, + "orderBy": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["field"], + "properties": { + "field": { "type": "string", "minLength": 1 }, + "direction": { "enum": ["asc", "desc"], "default": "asc" } + }, + "additionalProperties": false + } + }, + "groupBy": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["field"], + "properties": { + "field": { "type": "string", "minLength": 1 }, + "direction": { "enum": ["asc", "desc"], "default": "asc" } + }, + "additionalProperties": false + } + }, + "summaryFunctionSet": { + "type": "object", + "propertyNames": { "$ref": "#/$defs/fieldName" }, + "additionalProperties": { "$ref": "#/$defs/summaryFunction" } + }, + "summaryFunction": { + "type": "object", + "required": ["expr"], + "properties": { + "expr": { "$ref": "#/$defs/expression" }, + "description": { "type": "string" } + }, + "patternProperties": { + "^x-[A-Za-z][A-Za-z0-9._:-]{0,127}$": { "$ref": "#/$defs/extension" } + }, + "additionalProperties": false + }, + "summaries": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/summary" } + }, + "summary": { + "type": "object", + "required": ["field", "function"], + "properties": { + "field": { "type": "string", "minLength": 1 }, + "function": { "$ref": "#/$defs/identifier" }, + "name": { "$ref": "#/$defs/fieldName" }, + "label": { "type": "string" } + }, + "additionalProperties": false + } + } +} diff --git a/schemas/v0.3/view.schema.json b/schemas/v0.3/view.schema.json new file mode 100644 index 0000000..706d45f --- /dev/null +++ b/schemas/v0.3/view.schema.json @@ -0,0 +1,260 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://mdbase.dev/schemas/v0.3/view.schema.json", + "title": "mdbase v0.3 view record", + "type": "object", + "required": ["type", "id", "version", "name", "views"], + "properties": { + "type": { "const": "view" }, + "id": { "$ref": "#/$defs/identifier" }, + "version": { "type": "integer", "minimum": 1 }, + "name": { "type": "string", "minLength": 1 }, + "description": { "type": "string" }, + "query": { "$ref": "#/$defs/sharedQuery" }, + "properties": { "$ref": "#/$defs/propertyMetadataSet" }, + "summary_functions": { "$ref": "#/$defs/summaryFunctionSet" }, + "views": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/view" } + } + }, + "patternProperties": { + "^x-[A-Za-z][A-Za-z0-9._:-]{0,127}$": { "$ref": "#/$defs/extension" } + }, + "additionalProperties": false, + "$defs": { + "identifier": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9._:-]*$" + }, + "typeName": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9_-]{0,127}$" + }, + "fieldName": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_:-]*$" + }, + "expression": { + "type": "string", + "minLength": 1 + }, + "extension": { + "type": "object", + "additionalProperties": true + }, + "typeList": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "$ref": "#/$defs/typeName" } + }, + "sharedQuery": { + "type": "object", + "properties": { + "types": { "$ref": "#/$defs/typeList" }, + "where": { "$ref": "#/$defs/expression" }, + "context": { "$ref": "#/$defs/viewContext" }, + "projections": { "$ref": "#/$defs/projectionSet" } + }, + "patternProperties": { + "^x-[A-Za-z][A-Za-z0-9._:-]{0,127}$": { "$ref": "#/$defs/extension" } + }, + "additionalProperties": false + }, + "viewContext": { + "type": "object", + "required": ["this"], + "properties": { + "this": { "$ref": "#/$defs/thisContext" } + }, + "patternProperties": { + "^x-[A-Za-z][A-Za-z0-9._:-]{0,127}$": { "$ref": "#/$defs/extension" } + }, + "additionalProperties": false + }, + "thisContext": { + "type": "object", + "properties": { + "on_missing": { + "enum": ["view", "null", "error"], + "default": "view" + }, + "types": { "$ref": "#/$defs/typeList" } + }, + "patternProperties": { + "^x-[A-Za-z][A-Za-z0-9._:-]{0,127}$": { "$ref": "#/$defs/extension" } + }, + "additionalProperties": false + }, + "projectionSet": { + "type": "object", + "propertyNames": { "$ref": "#/$defs/fieldName" }, + "additionalProperties": { "$ref": "#/$defs/projection" } + }, + "projection": { + "type": "object", + "required": ["expr"], + "properties": { + "expr": { "$ref": "#/$defs/expression" }, + "description": { "type": "string" } + }, + "patternProperties": { + "^x-[A-Za-z][A-Za-z0-9._:-]{0,127}$": { "$ref": "#/$defs/extension" } + }, + "additionalProperties": false + }, + "propertyMetadataSet": { + "type": "object", + "propertyNames": { "type": "string", "minLength": 1 }, + "additionalProperties": { "$ref": "#/$defs/propertyMetadata" } + }, + "propertyMetadata": { + "type": "object", + "properties": { + "label": { "type": "string" }, + "description": { "type": "string" }, + "format": { "type": "string" }, + "hidden": { "type": "boolean" } + }, + "patternProperties": { + "^x-[A-Za-z][A-Za-z0-9._:-]{0,127}$": { "$ref": "#/$defs/extension" } + }, + "additionalProperties": false + }, + "summaryFunctionSet": { + "type": "object", + "propertyNames": { "$ref": "#/$defs/fieldName" }, + "additionalProperties": { "$ref": "#/$defs/summaryFunction" } + }, + "summaryFunction": { + "type": "object", + "required": ["expr"], + "properties": { + "expr": { "$ref": "#/$defs/expression" }, + "description": { "type": "string" } + }, + "patternProperties": { + "^x-[A-Za-z][A-Za-z0-9._:-]{0,127}$": { "$ref": "#/$defs/extension" } + }, + "additionalProperties": false + }, + "view": { + "type": "object", + "required": ["id", "name"], + "properties": { + "id": { "$ref": "#/$defs/identifier" }, + "name": { "type": "string", "minLength": 1 }, + "description": { "type": "string" }, + "types": { "$ref": "#/$defs/typeList" }, + "where": { "$ref": "#/$defs/expression" }, + "context": { "$ref": "#/$defs/viewContext" }, + "projections": { "$ref": "#/$defs/projectionSet" }, + "select": { "$ref": "#/$defs/select" }, + "order_by": { "$ref": "#/$defs/orderBy" }, + "group_by": { "$ref": "#/$defs/groupBy" }, + "summaries": { "$ref": "#/$defs/summaries" }, + "limit": { "type": "integer", "minimum": 0 }, + "offset": { "type": "integer", "minimum": 0 }, + "include_body": { "type": "boolean", "default": false }, + "frontmatter": { + "enum": ["effective", "raw", "both"], + "default": "effective" + }, + "presentation": { "$ref": "#/$defs/presentation" } + }, + "patternProperties": { + "^x-[A-Za-z][A-Za-z0-9._:-]{0,127}$": { "$ref": "#/$defs/extension" } + }, + "additionalProperties": false + }, + "select": { + "type": "array", + "minItems": 1, + "items": { + "oneOf": [ + { "type": "string", "minLength": 1 }, + { "$ref": "#/$defs/selectExpression" } + ] + } + }, + "selectExpression": { + "type": "object", + "required": ["name", "expr"], + "properties": { + "name": { "$ref": "#/$defs/fieldName" }, + "expr": { "$ref": "#/$defs/expression" }, + "label": { "type": "string" }, + "description": { "type": "string" } + }, + "patternProperties": { + "^x-[A-Za-z][A-Za-z0-9._:-]{0,127}$": { "$ref": "#/$defs/extension" } + }, + "additionalProperties": false + }, + "orderBy": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["field"], + "properties": { + "field": { "type": "string", "minLength": 1 }, + "direction": { "enum": ["asc", "desc"], "default": "asc" } + }, + "additionalProperties": false + } + }, + "groupBy": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["field"], + "properties": { + "field": { "type": "string", "minLength": 1 }, + "direction": { "enum": ["asc", "desc"], "default": "asc" } + }, + "additionalProperties": false + } + }, + "summaries": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/summary" } + }, + "summary": { + "type": "object", + "required": ["field", "function"], + "properties": { + "field": { "type": "string", "minLength": 1 }, + "function": { "$ref": "#/$defs/identifier" }, + "name": { "$ref": "#/$defs/fieldName" }, + "label": { "type": "string" } + }, + "additionalProperties": false + }, + "presentation": { + "type": "object", + "required": ["type"], + "properties": { + "type": { "$ref": "#/$defs/identifier" }, + "fallback": { "$ref": "#/$defs/identifier" }, + "mappings": { + "type": "object", + "propertyNames": { "$ref": "#/$defs/fieldName" }, + "additionalProperties": { "type": "string", "minLength": 1 } + }, + "options": { + "type": "object", + "additionalProperties": true + } + }, + "patternProperties": { + "^x-[A-Za-z][A-Za-z0-9._:-]{0,127}$": { "$ref": "#/$defs/extension" } + }, + "additionalProperties": false + } + } +} diff --git a/scripts/check_v03_tests.py b/scripts/check_v03_tests.py index 47c526a..dd2c7d0 100755 --- a/scripts/check_v03_tests.py +++ b/scripts/check_v03_tests.py @@ -331,9 +331,7 @@ def run_executable_test(test: dict[str, Any], setup: dict[str, Any] | None = Non validator = Draft202012Validator(schema) for path in expand_paths(input_data.get("paths", [])): errors = list(validator.iter_errors(load_markdown_frontmatter(path))) - if errors: - raise AssertionError(format_schema_errors(path, errors)) - assert_valid(expect) + assert_document_schema_result(path, errors, expect) return if operation == "embedded_json_schema_validate": diff --git a/tests/v0.3/README.md b/tests/v0.3/README.md index 00beee3..ea7c9aa 100644 --- a/tests/v0.3/README.md +++ b/tests/v0.3/README.md @@ -4,21 +4,22 @@ This directory is the parallel v0.3 conformance suite. It does not replace the existing `tests/level-*` v0.2.x suite. v0.3 conformance claims use the atomic profiles defined by the specification. -The tests below are grouped into six fixture sets for the rollout plan: +The tests below are grouped into seven fixture sets for the rollout plan: 1. `schema_artifacts` 2. `migration` 3. `core_collection` 4. `lifecycle` 5. `cel` -6. `runtime_contracts` +6. `views` +7. `runtime_contracts` The suite covers JSON Schema artifacts, type wrappers, collection semantics, -CEL host bindings, lifecycle operations, runtime contract registries, workflow -preflight, and execution cases for available adapters. Compatible v0.2 fixtures -remain useful for frontmatter parsing, missing/null semantics, links, operation -safety, and watch ordering. Tests tied to the earlier custom field grammar are -migrated into the v0.3 fixture sets. +CEL host bindings, saved views, lifecycle operations, runtime contract +registries, workflow preflight, and execution cases for available adapters. +Compatible v0.2 fixtures remain useful for frontmatter parsing, missing/null +semantics, links, operation safety, and watch ordering. Tests tied to the +earlier custom field grammar are migrated into the v0.3 fixture sets. ## Format @@ -81,6 +82,7 @@ Future v0.3 adapters should support these operations: - `read` - `validate` - `query` +- `execute_view` - `evaluate_cel` - `create` - `update` diff --git a/tests/v0.3/cel/cel-profile.yaml b/tests/v0.3/cel/cel-profile.yaml index ae33c68..8d10c14 100644 --- a/tests/v0.3/cel/cel-profile.yaml +++ b/tests/v0.3/cel/cel-profile.yaml @@ -96,6 +96,7 @@ groups: - cel_query.query_filtering operation: query input: + types: [task] where: 'status == "open"' order_by: - field: file.path diff --git a/tests/v0.3/fixtures/views/invalid-query-context.yml b/tests/v0.3/fixtures/views/invalid-query-context.yml new file mode 100644 index 0000000..631c1d9 --- /dev/null +++ b/tests/v0.3/fixtures/views/invalid-query-context.yml @@ -0,0 +1,4 @@ +where: 'this.status == "active"' +context: + this: + path: "" diff --git a/tests/v0.3/fixtures/views/invalid-view-unknown-member.md b/tests/v0.3/fixtures/views/invalid-view-unknown-member.md new file mode 100644 index 0000000..b5ad1c9 --- /dev/null +++ b/tests/v0.3/fixtures/views/invalid-view-unknown-member.md @@ -0,0 +1,12 @@ +--- +type: view +id: invalid.example +version: 1 +name: Invalid view +views: + - id: all + name: All +unsupported_top_level_key: true +--- + +# Invalid view diff --git a/tests/v0.3/fixtures/views/valid-query.yml b/tests/v0.3/fixtures/views/valid-query.yml new file mode 100644 index 0000000..a595928 --- /dev/null +++ b/tests/v0.3/fixtures/views/valid-query.yml @@ -0,0 +1,31 @@ +types: [task] +context: + this: + path: projects/alpha.md +projections: + is_overdue: + expr: 'present.record.due && due < today() && status != "done"' +where: 'status != "done"' +select: + - title + - projection.is_overdue + - name: context_title + expr: 'this.title' + label: Project +order_by: + - field: projection.is_overdue + direction: desc +group_by: + - field: status + direction: asc +summary_functions: + completion_rate: + expr: 'values.size() == 0 ? 0 : values.filter(v, v).size() * 100 / values.size()' +summaries: + - field: projection.is_overdue + function: completion_rate + name: completion_rate +limit: 20 +offset: 0 +include_body: false +frontmatter: both diff --git a/tests/v0.3/fixtures/views/valid-task-views.md b/tests/v0.3/fixtures/views/valid-task-views.md new file mode 100644 index 0000000..6468ebd --- /dev/null +++ b/tests/v0.3/fixtures/views/valid-task-views.md @@ -0,0 +1,68 @@ +--- +type: view +id: tasknotes.tasks +version: 1 +name: Task views +description: Portable task queries with optional TaskNotes presentation. + +query: + types: [task] + context: + this: + on_missing: view + projections: + urgency: + expr: 'priority + (present.record.due && due < today() ? 10 : 0)' + +properties: + title: + label: Task + projection.urgency: + label: Urgency + +summary_functions: + completion_rate: + expr: 'values.size() == 0 ? 0 : values.filter(v, v).size() * 100 / values.size()' + +views: + - id: all + name: All tasks + select: [title, status, due, projection.urgency] + order_by: + - field: due + direction: asc + presentation: + type: tasknotes.task-list + fallback: mdbase.table + options: + show_actions: true + + - id: subtasks + name: Subtasks + context: + this: + on_missing: error + types: [project] + where: 'projects.exists(p, p == this.file.asLink())' + select: [title, status, projection.urgency] + group_by: + - field: status + direction: asc + summaries: + - field: title + function: count + name: task_count + presentation: + type: tasknotes.kanban + fallback: mdbase.table + mappings: + title: title + group: status + +x-obsidian: + source_format: base +--- + +# Task views + +Shared task views for query tools and supporting renderers. diff --git a/tests/v0.3/lifecycle/lifecycle.yaml b/tests/v0.3/lifecycle/lifecycle.yaml index 8ebbfae..6241877 100644 --- a/tests/v0.3/lifecycle/lifecycle.yaml +++ b/tests/v0.3/lifecycle/lifecycle.yaml @@ -101,10 +101,10 @@ groups: frontmatter_not_contains: - status - - name: "effective read still exposes read defaults after create" + - name: "effective read exposes read defaults without materializing them" operation: read input: - path: "tasks/no-status.md" + path: "tasks/existing.md" effective: true expect: valid: true @@ -219,5 +219,5 @@ groups: expect: valid: false issues: - - code: lifecycle_conflict + - code: type_conflict field: stamp diff --git a/tests/v0.3/manifest.yaml b/tests/v0.3/manifest.yaml index d2384c9..c681c1f 100644 --- a/tests/v0.3/manifest.yaml +++ b/tests/v0.3/manifest.yaml @@ -62,8 +62,11 @@ claim_profiles: requires: [core_read, cel] requirements: - query_context + - invocation_context - query_filtering - query_projections + - named_projections + - grouping_and_summaries - query_evaluation_diagnostics - id: links status: draft @@ -126,6 +129,11 @@ fixture_sets: coverage_targets: [cel, cel_match, cel_query, links] files: - cel/cel-profile.yaml + - id: views + description: Ordinary view records resolve named queries, invocation context, and advisory presentation. + coverage_targets: [cel_query] + files: + - views/view-records.yaml - id: runtime_contracts description: runtime registries, event/action validation, implicit contracts, and materialization. coverage_targets: [runtime_contracts/0.1] diff --git a/tests/v0.3/schema/schema-artifacts.yaml b/tests/v0.3/schema/schema-artifacts.yaml index 049fb03..9724089 100644 --- a/tests/v0.3/schema/schema-artifacts.yaml +++ b/tests/v0.3/schema/schema-artifacts.yaml @@ -2,7 +2,7 @@ name: "v0.3 schema artifact conformance" spec_version: "0.3.0" fixture_set: schema_artifacts category: schema_artifacts -spec_ref: "v0.3/04, v0.3/05, v0.3/06, v0.3/13, v0.3/14, v0.3/16" +spec_ref: "v0.3/04, v0.3/05, v0.3/06, v0.3/11, v0.3/13, v0.3/14, v0.3/16" groups: - name: "canonical schemas" @@ -78,6 +78,51 @@ groups: expect: valid: true + - name: "type-file schema validates the canonical view type" + operation: markdown_frontmatter_schema_validate + input: + schema: "schemas/v0.3/type-file.schema.json" + paths: + - "_types/view.md" + expect: + valid: true + + - name: "query schema validates a contextual grouped query" + operation: yaml_document_schema_validate + input: + schema: "schemas/v0.3/query.schema.json" + paths: + - "tests/v0.3/fixtures/views/valid-query.yml" + expect: + valid: true + + - name: "query schema rejects an empty context path" + operation: yaml_document_schema_validate + input: + schema: "schemas/v0.3/query.schema.json" + paths: + - "tests/v0.3/fixtures/views/invalid-query-context.yml" + expect: + valid: false + + - name: "view schema validates a portable task view record" + operation: markdown_frontmatter_schema_validate + input: + schema: "schemas/v0.3/view.schema.json" + paths: + - "tests/v0.3/fixtures/views/valid-task-views.md" + expect: + valid: true + + - name: "view schema rejects unknown portable members" + operation: markdown_frontmatter_schema_validate + input: + schema: "schemas/v0.3/view.schema.json" + paths: + - "tests/v0.3/fixtures/views/invalid-view-unknown-member.md" + expect: + valid: false + - name: "embedded task schemas are themselves valid JSON Schemas" operation: embedded_json_schema_validate input: diff --git a/tests/v0.3/views/view-records.yaml b/tests/v0.3/views/view-records.yaml new file mode 100644 index 0000000..302887e --- /dev/null +++ b/tests/v0.3/views/view-records.yaml @@ -0,0 +1,406 @@ +name: "v0.3 saved view record semantics" +spec_version: "0.3.0" +fixture_set: views +category: views +spec_ref: "v0.3/10, v0.3/11, v0.3/16" + +groups: + - name: "ordinary view records and invocation context" + setup: + config: | + spec_version: "0.3.0" + types: + task.md: | + --- + kind: mdbase.type + name: task + version: 1 + match: + where: + type: task + schema: + dialect: json-schema-2020-12 + value: + type: object + properties: + type: { const: task } + title: { type: string } + status: { type: string } + project_id: { type: string } + priority: { type: integer } + --- + project.md: | + --- + kind: mdbase.type + name: project + version: 1 + match: + where: + type: project + schema: + dialect: json-schema-2020-12 + value: + type: object + properties: + type: { const: project } + id: { type: string } + title: { type: string } + category: { type: string } + missing: { type: string } + record: { type: string } + collection: + read_defaults: + category: defaulted + --- + view.md: | + --- + kind: mdbase.type + name: view + version: 1 + match: + where: + type: view + schema: + dialect: json-schema-2020-12 + value: + type: object + --- + files: + projects/alpha.md: | + --- + type: project + id: alpha + title: Alpha + record: project-record + --- + projects/beta.md: | + --- + type: project + id: beta + title: Beta + --- + tasks/alpha-open.md: | + --- + type: task + title: Alpha open + status: open + project_id: alpha + priority: 2 + --- + tasks/alpha-done.md: | + --- + type: task + title: Alpha done + status: done + project_id: alpha + priority: 1 + --- + tasks/beta-open.md: | + --- + type: task + title: Beta open + status: open + project_id: beta + priority: 3 + --- + tasks/archived.md: | + --- + type: task + title: Archived + status: archived + project_id: alpha + priority: 4 + --- + views/tasks.md: | + --- + type: view + id: task.views + version: 1 + name: Task views + query: + types: [task] + where: 'status != "archived"' + context: + this: + on_missing: view + views: + - id: project-open + name: Open tasks for project + context: + this: + on_missing: error + types: [project] + projections: + contextual_title: + expr: 'this.title + ": " + title' + where: 'status == "open" && project_id == this.id' + select: [title, projection.contextual_title] + order_by: + - field: priority + direction: desc + group_by: + - field: status + direction: asc + summaries: + - field: title + function: count + name: task_count + presentation: + type: tasknotes.task-list + - id: direct + name: Direct invocation + where: 'this.type == "view"' + select: [title] + - id: optional + name: Optional context + context: + this: + on_missing: "null" + where: 'this == null && status == "open"' + select: [title] + - id: context-shape + name: Complete context shape + context: + this: + on_missing: error + types: [project] + where: 'project_id == this.id' + select: + - name: direct + expr: 'this.title' + - name: record_alias + expr: 'this.record.title' + - name: note_alias + expr: 'this.note.title' + - name: raw_value + expr: 'this.raw.title' + - name: default_value + expr: 'this.category' + - name: effective_present + expr: 'this.present.record.category' + - name: raw_present + expr: 'this.present.raw.category' + - name: missing_is_null + expr: 'this.missing == null' + - name: context_path + expr: 'this.file.path' + - name: reserved_collision + expr: 'this.record.record' + --- + views/duplicate.md: | + --- + type: view + id: duplicate.views + version: 1 + name: Duplicate views + views: + - id: same + name: First + - id: same + name: Second + --- + tests: + - name: "explicit context is fixed while candidates change" + id: views.explicit_context + covers: + - cel.context_bindings + - cel_query.invocation_context + - cel_query.named_projections + operation: execute_view + input: + path: views/tasks.md + view: project-open + context: + path: projects/alpha.md + render: false + expect: + valid: true + results: + - path: tasks/alpha-open.md + values: + title: Alpha open + contextual_title: "Alpha: Alpha open" + meta: + view: + path: views/tasks.md + id: project-open + context: + path: projects/alpha.md + groups: + - values: + status: open + count: 1 + summaries: + task_count: 1 + + - name: "grouping and summaries describe the complete match set" + id: views.grouping_summaries + covers: + - cel_query.grouping_and_summaries + operation: execute_view + input: + path: views/tasks.md + view: project-open + context: + path: projects/alpha.md + limit: 0 + expect: + valid: true + results: [] + meta: + total_count: 1 + has_more: true + groups: + - values: + status: open + count: 1 + summaries: + task_count: 1 + + - name: "record and named-view filters combine with AND" + operation: execute_view + input: + path: views/tasks.md + view: project-open + context: + path: projects/alpha.md + expect: + valid: true + paths: [tasks/alpha-open.md] + + - name: "direct invocation defaults this to the view record" + operation: execute_view + input: + path: views/tasks.md + view: direct + expect: + valid: true + context: + path: views/tasks.md + paths: + - tasks/alpha-done.md + - tasks/alpha-open.md + - tasks/beta-open.md + + - name: "optional missing context binds this to null" + operation: execute_view + input: + path: views/tasks.md + view: optional + expect: + valid: true + context: null + paths: + - tasks/alpha-open.md + - tasks/beta-open.md + + - name: "invocation context exposes effective raw presence and file namespaces" + id: views.context_shape + covers: + - cel.context_bindings + - cel_query.invocation_context + operation: execute_view + input: + path: views/tasks.md + view: context-shape + context: + path: projects/alpha.md + expect: + valid: true + results: + - path: tasks/alpha-done.md + values: + direct: Alpha + record_alias: Alpha + note_alias: Alpha + raw_value: Alpha + default_value: defaulted + effective_present: true + raw_present: false + missing_is_null: true + context_path: projects/alpha.md + reserved_collision: project-record + - path: tasks/alpha-open.md + values: + direct: Alpha + record_alias: Alpha + note_alias: Alpha + raw_value: Alpha + default_value: defaulted + effective_present: true + raw_present: false + missing_is_null: true + context_path: projects/alpha.md + reserved_collision: project-record + + - name: "required invocation context fails before candidate evaluation" + operation: execute_view + input: + path: views/tasks.md + view: project-open + expect: + valid: false + diagnostics: + - code: context_required + + - name: "context type constraints are enforced" + operation: execute_view + input: + path: views/tasks.md + view: project-open + context: + path: tasks/alpha-open.md + expect: + valid: false + diagnostics: + - code: context_type_mismatch + + - name: "unknown presentation does not block headless execution" + operation: execute_view + input: + path: views/tasks.md + view: project-open + context: + path: projects/beta.md + render: false + expect: + valid: true + paths: [tasks/beta-open.md] + + - name: "duplicate stable named-view ids invalidate the record" + operation: execute_view + input: + path: views/duplicate.md + view: same + expect: + valid: false + diagnostics: + - code: invalid_view + + - name: "cyclic named projections fail query preflight" + operation: query + input: + types: [task] + projections: + first: + expr: 'projection.second' + second: + expr: 'projection.first' + select: [projection.first] + expect: + valid: false + diagnostics: + - code: invalid_query + + - name: "selection output names must be unique" + operation: query + input: + types: [task] + projections: + title: + expr: 'title + "!"' + select: [title, projection.title] + expect: + valid: false + diagnostics: + - code: invalid_query