feat(python): complete runtime parity harness

runtime/python/prompty/.env.example

@@ -9,7 +9,7 @@ OPENAI_API_KEY=
 OPENAI_BASE_URL=
 OPENAI_MODEL=gpt-4o-mini
 OPENAI_EMBEDDING_MODEL=text-embedding-3-small
-OPENAI_IMAGE_MODEL=dall-e-2
+OPENAI_IMAGE_MODEL=gpt-image-1
 
 # Direct OpenAI (api.openai.com — no proxy/compat layer)
 DIRECT_OPENAI_API_KEY=

runtime/python/prompty/prompty/__init__.py

@@ -117,6 +117,7 @@
     "InMemoryCheckpointStore",
     "JsonlEventJournalWriter",
     "ReferenceReplayVerifier",
+    "ReferenceTurnEngine",
     "ReferenceTurnRunner",
     "RunTurnRequest",
     "RunTurnResult",
@@ -127,6 +128,8 @@
     "cast",
     "bind_tools",
     "tool",
+    "load_engine_checkpoint",
+    "save_engine_checkpoint",
     # Backward-compat aliases
     "AzureExecutor",
     "AzureProcessor",
@@ -174,11 +177,14 @@
     InMemoryCheckpointStore,
     JsonlEventJournalWriter,
     ReferenceReplayVerifier,
+    ReferenceTurnEngine,
     ReferenceTurnRunner,
     RunTurnRequest,
     RunTurnResult,
     TurnModelRequest,
     TurnModelResponse,
+    load_engine_checkpoint,
+    save_engine_checkpoint,
 )
 
 # Pipeline (via backward-compat shim)

runtime/python/prompty/prompty/core/__init__.py

@@ -16,6 +16,7 @@
 )
 from .guardrails import GuardrailError, GuardrailResult, Guardrails
 from .loader import default_save_context, load, load_async
+from .model_capabilities import ModelCapabilities, enrich, lookup
 from .pipeline import (
     ExecuteError,
     invoke,

runtime/python/prompty/prompty/core/model_capabilities.py

@@ -0,0 +1,190 @@
+"""Cross-runtime model-capability enrichment for provider discovery.
+
+Provider ``/models`` endpoints vary in richness: some (Anthropic, Foundry)
+return capability fields directly, while others (OpenAI) return only ids. To
+keep discovery results consistent across providers *and* across runtimes,
+Prompty ships a single shared, provider-keyed capability dataset
+(``spec/data/model_capabilities.json``) and applies it with one rule:
+
+    Provider-supplied fields always win. Dataset entries only fill fields
+    the provider left empty (fill-only-missing). Matching is by longest
+    prefix on the model id, applied only at token boundaries.
+
+**Canonical source vs. vendored copy.** The cross-runtime source of truth is
+``spec/data/model_capabilities.json``. This package vendors a byte-identical
+copy at ``prompty/data/model_capabilities.json`` so the installed package has
+no filesystem dependency on the repo's ``spec/`` directory (mirrors
+``runtime/rust/prompty/src/discovery.rs``, which embeds the same dataset via
+``include_str!``). ``tests/test_model_capabilities.py`` guards against drift
+between the two copies whenever the repo layout is available.
+
+To refresh: edit ``spec/data/model_capabilities.json``, then copy it to
+``runtime/python/prompty/prompty/data/model_capabilities.json`` (the drift
+guard test will fail until you do).
+
+This dataset is intentionally **not** emitted from TypeSpec/Typra: it is
+volatile provider data (context windows, modalities, new model families)
+refreshed as a snapshot, whereas TypeSpec/Typra owns the structural
+:class:`~prompty.model.ModelInfo` contract consumed here.
+
+**Emitted-model caveat (Python-specific).** The Typra-generated
+:class:`~prompty.model.ModelInfo` declares ``input_modalities`` and
+``output_modalities`` as ``list[str] = field(default_factory=list)`` rather
+than ``list[str] | None = None`` (unlike the Rust ``Option<Vec<String>>`` and
+C# ``IList<string>?`` emissions for the same field). That default makes a
+freshly constructed ``ModelInfo()`` indistinguishable from one where a
+provider explicitly reported an empty modality list — which breaks the
+fill-only-missing contract's tri-state requirement (absent vs.
+explicitly-empty vs. non-empty). Since generated files under ``prompty/model``
+must not be hand-edited, callers that build a ``ModelInfo`` for use with
+:func:`enrich` MUST explicitly pass ``input_modalities=None`` /
+``output_modalities=None`` (not rely on the constructor default) when the
+provider payload does not include that field, and MUST NOT construct the
+object via ``ModelInfo.load(data)`` for this purpose (its presence-check
+logic leaves the buggy ``[]`` default when a key is absent). Every provider
+mapping function in ``prompty.providers.*.models`` follows this pattern.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+__all__ = ["ModelCapabilities", "enrich", "lookup"]
+
+_DATA_PATH = Path(__file__).resolve().parent.parent / "data" / "model_capabilities.json"
+
+
+@dataclass(frozen=True)
+class ModelCapabilities:
+    """Fallback capability fields for a single model, as looked up from the shared dataset.
+
+    All fields are optional; ``None`` means "the dataset does not supply this field" and the
+    caller should leave whatever the provider returned.
+    """
+
+    context_window: int | None = None
+    input_modalities: list[str] | None = None
+    output_modalities: list[str] | None = None
+
+
+@dataclass(frozen=True)
+class _CapabilityEntry:
+    prefix: str
+    capabilities: ModelCapabilities
+
+
+def _parse_modalities(value: Any) -> list[str] | None:
+    """Parse a modality array, distinguishing "absent" (None) from "present but empty" ([])."""
+    if not isinstance(value, list):
+        return None
+    return [item for item in value if isinstance(item, str)]
+
+
+def _entry_from_value(value: Any) -> _CapabilityEntry | None:
+    if not isinstance(value, dict):
+        return None
+    prefix = value.get("prefix")
+    if not isinstance(prefix, str):
+        return None
+    context_window = value.get("contextWindow")
+    return _CapabilityEntry(
+        prefix=prefix,
+        capabilities=ModelCapabilities(
+            context_window=context_window
+            if isinstance(context_window, int) and not isinstance(context_window, bool)
+            else None,
+            input_modalities=_parse_modalities(value.get("inputModalities")),
+            output_modalities=_parse_modalities(value.get("outputModalities")),
+        ),
+    )
+
+
+class _CapabilityTable:
+    """Provider-keyed capability lookup, parsed once from the vendored dataset."""
+
+    def __init__(self, providers: dict[str, list[_CapabilityEntry]]) -> None:
+        self._providers = providers
+
+    @staticmethod
+    def from_value(value: Any) -> _CapabilityTable:
+        providers: dict[str, list[_CapabilityEntry]] = {}
+        raw_providers = value.get("providers") if isinstance(value, dict) else None
+        if isinstance(raw_providers, dict):
+            for provider, entries in raw_providers.items():
+                if not isinstance(entries, list):
+                    continue
+                parsed = [e for e in (_entry_from_value(item) for item in entries) if e is not None]
+                # Longest prefix first, so the first match is the most specific.
+                parsed.sort(key=lambda e: len(e.prefix), reverse=True)
+                providers[provider] = parsed
+        return _CapabilityTable(providers)
+
+    def lookup(self, provider: str, model_id: str) -> ModelCapabilities | None:
+        for entry in self._providers.get(provider, []):
+            if _prefix_matches(model_id, entry.prefix):
+                return entry.capabilities
+        return None
+
+
+def _prefix_matches(model_id: str, prefix: str) -> bool:
+    """Whether `model_id` is matched by dataset `prefix` under the cross-runtime rule.
+
+    A prefix matches only at a token boundary: `model_id` must either equal `prefix`
+    exactly, or the character immediately following the prefix must be a separator
+    (any non-ASCII-alphanumeric character, e.g. ``-``, ``.``, ``:``). This keeps real
+    ids matching (``gpt-4`` -> ``gpt-4-0613``, ``gpt-4o`` -> ``gpt-4o-2024-05-13``)
+    while rejecting accidental substring hits (``gpt-4`` must NOT match a future
+    ``gpt-45``). Every runtime MUST implement this same boundary rule so the shared
+    enrichment vectors converge.
+    """
+    if not model_id.startswith(prefix):
+        return False
+    rest = model_id[len(prefix) :]
+    if not rest:
+        return True
+    ch = rest[0]
+    return not (ch.isascii() and ch.isalnum())
+
+
+@lru_cache(maxsize=1)
+def _table() -> _CapabilityTable:
+    with open(_DATA_PATH, encoding="utf-8") as f:
+        value = json.load(f)
+    return _CapabilityTable.from_value(value)
+
+
+def lookup(provider: str, model_id: str) -> ModelCapabilities | None:
+    """Look up fallback capabilities for a model id within a provider's dataset.
+
+    Returns ``None`` when the provider has no entry matching ``model_id``. Matching is by
+    longest prefix, applied only at token boundaries (see :func:`_prefix_matches`).
+    """
+    return _table().lookup(provider, model_id)
+
+
+def enrich(provider: str, info: Any) -> None:
+    """Enrich a :class:`~prompty.model.ModelInfo` in place using the shared capability dataset.
+
+    Applies the cross-runtime fill-only-missing rule: a dataset field is written only when the
+    corresponding ``ModelInfo`` field is still empty (``context_window`` is ``None``; a modality
+    list is ``None``). Provider-supplied values are never overwritten. A dataset modality of
+    ``[]`` (e.g. embeddings) is a valid fill and will replace a ``None`` list.
+
+    Callers MUST have constructed ``info`` with explicit ``None`` for
+    ``input_modalities``/``output_modalities`` when the provider did not supply them — see the
+    module docstring for why the generated ``ModelInfo`` default cannot be relied on for this.
+    """
+    caps = lookup(provider, info.id)
+    if caps is None:
+        return
+
+    if info.context_window is None and caps.context_window is not None:
+        info.context_window = caps.context_window
+    if info.input_modalities is None and caps.input_modalities is not None:
+        info.input_modalities = caps.input_modalities
+    if info.output_modalities is None and caps.output_modalities is not None:
+        info.output_modalities = caps.output_modalities

runtime/python/prompty/prompty/data/model_capabilities.json

@@ -0,0 +1,55 @@
+{
+  "description": "Cross-runtime fallback capability data for provider model discovery. Some provider /models endpoints (Anthropic, Foundry) return capability fields directly; others (OpenAI) return only ids. To keep discovery results consistent across providers AND across runtimes, every Prompty runtime embeds THIS file and applies one shared rule: provider-supplied fields always win; entries here only fill fields the provider left empty (fill-only-missing). Model ids are matched by longest prefix within a provider's list. This dataset is deliberately NOT emitted from TypeSpec: it is volatile provider data (context windows, modalities, new model families) refreshed as a snapshot, whereas TypeSpec owns the structural ModelInfo contract. Fields use the canonical camelCase ModelInfo names. A missing 'contextWindow' means unknown; a present empty modality list (e.g. []) is intentional (e.g. embeddings produce no textual/image output modality).",
+  "match": "longest_prefix",
+  "providers": {
+    "openai": [
+      {
+        "prefix": "gpt-4o-mini",
+        "contextWindow": 128000,
+        "inputModalities": ["text", "image"],
+        "outputModalities": ["text"]
+      },
+      {
+        "prefix": "gpt-4o",
+        "contextWindow": 128000,
+        "inputModalities": ["text", "image"],
+        "outputModalities": ["text"]
+      },
+      {
+        "prefix": "gpt-4-turbo",
+        "contextWindow": 128000,
+        "inputModalities": ["text", "image"],
+        "outputModalities": ["text"]
+      },
+      {
+        "prefix": "gpt-4",
+        "contextWindow": 8192,
+        "inputModalities": ["text"],
+        "outputModalities": ["text"]
+      },
+      {
+        "prefix": "gpt-3.5-turbo",
+        "contextWindow": 16385,
+        "inputModalities": ["text"],
+        "outputModalities": ["text"]
+      },
+      {
+        "prefix": "text-embedding-3-small",
+        "contextWindow": 8191,
+        "inputModalities": ["text"],
+        "outputModalities": []
+      },
+      {
+        "prefix": "text-embedding-3-large",
+        "contextWindow": 8191,
+        "inputModalities": ["text"],
+        "outputModalities": []
+      },
+      {
+        "prefix": "dall-e-3",
+        "inputModalities": ["text"],
+        "outputModalities": ["image"]
+      }
+    ]
+  }
+}

runtime/python/prompty/prompty/harness/__init__.py

@@ -8,6 +8,7 @@
     InMemoryCheckpointStore,
     JsonlEventJournalWriter,
 )
+from .engine import ReferenceTurnEngine, load_engine_checkpoint, save_engine_checkpoint
 from .replay_verifier import ReferenceReplayVerifier
 from .turn_runner import (
     ReferenceTurnRunner,
@@ -25,9 +26,12 @@
     "InMemoryCheckpointStore",
     "JsonlEventJournalWriter",
     "ReferenceReplayVerifier",
+    "ReferenceTurnEngine",
     "ReferenceTurnRunner",
     "RunTurnRequest",
     "RunTurnResult",
     "TurnModelRequest",
     "TurnModelResponse",
+    "load_engine_checkpoint",
+    "save_engine_checkpoint",
 ]