[Eval track] E1 — acceptance-length eval harness: EvalConfig + EvalCache + gates

examples/disagg/run_disagg_eagle3.py

@@ -61,6 +61,7 @@
 )
 
 from specforge.distributed import destroy_distributed, init_distributed
+from specforge.launch import build_disagg_offline_runtime, build_offline_runtime
 from specforge.optimizer import BF16Optimizer
 from specforge.runtime.data_plane.disagg_ingest import (
     ingest_offline_features,
@@ -70,7 +71,6 @@
 from specforge.runtime.data_plane.disaggregated import AuthPolicy, SharedDirFeatureStore
 from specforge.runtime.data_plane.feature_store import FeatureStore
 from specforge.runtime.data_plane.mooncake_store import MooncakeFeatureStore
-from specforge.runtime.launch import build_disagg_offline_runtime, build_offline_runtime
 
 RUN_ID = "eagle3-disagg"
 

scripts/train_dflash.py

@@ -28,11 +28,11 @@
 from specforge.core.dflash import OnlineDFlashModel
 from specforge.data import build_eagle3_dataset, prepare_dp_dataloaders
 from specforge.distributed import destroy_distributed, get_dp_group, init_distributed
-from specforge.modeling.draft.dflash import DFlashDraftModel
-from specforge.modeling.target.dflash_target_model import (
+from specforge.inference.target_engine.dflash_target_model import (
     DFlashTargetModel,
     get_dflash_target_model,
 )
+from specforge.modeling.draft.dflash import DFlashDraftModel
 from specforge.modeling.target.target_utils import TargetEmbeddingsAndHead
 from specforge.optimizer import BF16Optimizer
 from specforge.tracker import create_tracker

scripts/train_domino.py

@@ -25,11 +25,11 @@
 from specforge.core.domino import OnlineDominoModel
 from specforge.data import build_eagle3_dataset, prepare_dp_dataloaders
 from specforge.distributed import destroy_distributed, get_dp_group, init_distributed
-from specforge.modeling.draft.dflash import DFlashDraftModel
-from specforge.modeling.target.dflash_target_model import (
+from specforge.inference.target_engine.dflash_target_model import (
     DFlashTargetModel,
     get_dflash_target_model,
 )
+from specforge.modeling.draft.dflash import DFlashDraftModel
 from specforge.modeling.target.target_utils import TargetEmbeddingsAndHead
 from specforge.optimizer import BF16Optimizer
 from specforge.tracker import create_tracker
@@ -431,7 +431,7 @@ def get_lambda_base(
 ) -> float:
     # Delegates to the runtime's single source of the Domino lambda schedule so the
     # standalone script and DominoTrainStrategy cannot drift.
-    from specforge.runtime.training.strategy import linear_lambda_base
+    from specforge.training.strategies.base import linear_lambda_base
 
     return linear_lambda_base(global_step, total_steps, lambda_start, decay_ratio)
 

scripts/train_eagle3_dataflow.py

@@ -134,7 +134,7 @@ def optimizer_factory(draft_module):
     if online:
         # `strategy=` selects the draft model (here eagle3); the runtime resolves
         # its StrategySpec. The topology is the builder; the model is a parameter.
-        from specforge.runtime.launch import build_online_runtime
+        from specforge.launch import build_online_runtime
 
         # Online target produces features in-loop (any backend exposing
         # generate_eagle3_data — HF or SGLang). is_online=True returns the model.
@@ -171,7 +171,7 @@ def optimizer_factory(draft_module):
         print(f"[online] rollout produced {produced} samples", flush=True)
         trainer.fit(loader)
     else:
-        from specforge.runtime.launch import build_offline_runtime
+        from specforge.launch import build_offline_runtime
 
         target_head, _ = build_target_model(args, draft_config, is_online=False)
         trainer, loader = build_offline_runtime(

specforge/eval/__init__.py

@@ -0,0 +1,9 @@
+# coding=utf-8
+"""Evaluation: correct acceptance-length / accuracy metrics for draft training,
+plus the eval identity (:class:`EvalConfig`) and the on-disk feature cache
+(:class:`EvalCache`) that keeps repeat evals from recomputing hidden states."""
+
+from specforge.eval.cache import EvalCache
+from specforge.eval.evaluator import EvalConfig, Evaluator
+
+__all__ = ["EvalCache", "EvalConfig", "Evaluator"]

specforge/eval/cache.py

@@ -0,0 +1,124 @@
+# coding=utf-8
+# Copyright 2024 The SpecForge team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+"""EvalCache: the on-disk cache for materialized eval feature sets (E1).
+
+Repeat evals over an unchanged *(eval-data, target, revision, tokenizer,
+template, aux-layers, seqlen)* tuple must not recompute target hidden states.
+The cache maps an :class:`~specforge.eval.evaluator.EvalConfig` identity to a
+directory of materialized eval features — whatever the producer writes there
+(offline feature files a ``spec.make_offline_reader`` reads back into
+``SampleRef``s for a ``FeatureDataLoader``).
+
+``get_or_produce`` is the whole consumption surface: a hit returns the cached
+directory; a miss runs the producer into a staging directory that is renamed
+into place atomically — the rename IS the completeness marker, so a crashed
+producer leaves only a stale ``*.tmp-*`` staging dir, never a half-visible
+entry. A ``eval_cache_meta.json`` with the plain identity fields is written
+beside the features for humans debugging a cache.
+
+Deliberately **separate** from the tokenization cache under ``specforge/data``
+— different key fields, different lifecycle; do not merge (plan §4.4). The
+cache is process-local: multi-rank callers should produce on rank 0 and
+barrier before reading (it does no cross-rank coordination itself).
+
+Scope note: the launch builders currently take a pre-produced
+``hidden_states_path`` directly, so nothing in the run surface constructs an
+``EvalCache`` yet — it is consumed programmatically (``EvalCache.for_config``
++ ``get_or_produce``) until Phase E's typed config + CLI own eval-data
+production and wire the cache in front of it (roadmap E1 note).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import shutil
+import uuid
+from typing import Callable, Optional
+
+from specforge.eval.evaluator import EvalConfig
+
+
+class EvalCache:
+    def __init__(self, cache_dir: str) -> None:
+        self.root = os.path.join(cache_dir, "eval_cache")
+
+    @classmethod
+    def for_config(cls, config: EvalConfig) -> "EvalCache":
+        """The cache rooted at ``config.cache_dir`` (the §4.4 Evaluator wiring)."""
+        if not config.cache_dir:
+            raise ValueError("EvalConfig.cache_dir is not set")
+        return cls(config.cache_dir)
+
+    # -- identity ------------------------------------------------------------
+    def key(self, config: EvalConfig) -> str:
+        """Digest of the eval identity — flipping ANY identity field changes it,
+        and execution knobs (micro-batch size) deliberately do not. The fields
+        are JSON-encoded before hashing so the encoding is injective — a
+        delimiter character inside one field can never realign the boundaries
+        into another identity's digest."""
+        content = json.dumps(config.identity_fields(), ensure_ascii=True)
+        return hashlib.sha256(content.encode("utf-8")).hexdigest()[:16]
+
+    def dir_for(self, config: EvalConfig) -> str:
+        return os.path.join(self.root, self.key(config))
+
+    # -- lookup / produce ------------------------------------------------------
+    def try_get(self, config: EvalConfig) -> Optional[str]:
+        """The cached feature directory for this identity, or ``None`` on a miss."""
+        path = self.dir_for(config)
+        return path if os.path.isdir(path) else None
+
+    def get_or_produce(self, config: EvalConfig, produce: Callable[[str], None]) -> str:
+        """Return the eval feature directory, producing it once on a miss.
+
+        ``produce(staging_dir)`` materializes the eval set (e.g. runs the
+        target capture over the eval prompts) into an empty staging directory;
+        on return the staging is atomically renamed into place. If a concurrent
+        producer won the race, its entry is used and this one's staging is
+        discarded — either way exactly one complete entry exists afterwards.
+        """
+        hit = self.try_get(config)
+        if hit is not None:
+            return hit
+        final = self.dir_for(config)
+        os.makedirs(self.root, exist_ok=True)
+        staging = f"{final}.tmp-{uuid.uuid4().hex[:8]}"
+        os.makedirs(staging)
+        try:
+            produce(staging)
+            with open(os.path.join(staging, "eval_cache_meta.json"), "w") as fh:
+                json.dump(
+                    {"key": self.key(config), "identity": config.identity_fields()},
+                    fh,
+                    indent=2,
+                )
+            try:
+                os.rename(staging, final)
+            except OSError:
+                # Lost a produce race: another process published first — theirs
+                # is complete, use it. Anything else is a real failure.
+                if not os.path.isdir(final):
+                    raise
+        finally:
+            if os.path.isdir(staging):
+                shutil.rmtree(staging, ignore_errors=True)
+        return final
+
+    def invalidate(self, config: EvalConfig) -> bool:
+        """Drop the entry for this identity; ``True`` if one existed."""
+        path = self.try_get(config)
+        if path is None:
+            return False
+        shutil.rmtree(path)
+        return True
+
+
+__all__ = ["EvalCache"]