Record ordinary judge call provenance

tests/v1/test_e2e.py

@@ -322,7 +322,7 @@ async def test_rubric_judge(run_v1, tmp_path):
     assert trace.ok
     assert trace.rewards["rubric"].score > 0  # the judge's verdict landed in the reward
     assert trace.metrics["rubric/always_yes"] == 1.0
-    assert trace.info["judge"]  # the call was recorded onto the trace
+    assert trace.judge_calls  # the call was recorded onto the trace
 
 
 @pytest.mark.e2e
@@ -560,9 +560,11 @@ async def test_replay_round_trip(run_v1, tmp_path):
     import tomllib
     from pathlib import Path
 
-    from verifiers.v1.cli.output import CONFIG_FILE
+    from verifiers.v1.cli.output import CONFIG_FILE, TRACES_FILE, write_episode
     from verifiers.v1.cli.replay import run_replay
     from verifiers.v1.configs.cli.replay import ReplayConfig
+    from verifiers.v1.episode import Episode
+    from verifiers.v1.trace import Error, JudgeCall
 
     run_dir = tmp_path / "run"
     (source,) = await run_v1(
@@ -595,3 +597,51 @@ async def replay(source_dir: Path, out: Path):
     # The wire task keeps its taskset-specific fields in the replay's own output.
     raw = (tmp_path / "replay2" / "traces.jsonl").read_text()
     assert '"answer"' in raw
+
+    # A judge/parser failure happens after generation and is recoverable by re-scoring.
+    failed_dir = tmp_path / "failed-score"
+    failed_dir.mkdir()
+    (failed_dir / CONFIG_FILE).write_text((run_dir / CONFIG_FILE).read_text())
+    (failed_dir / TRACES_FILE).write_text("")
+    failed = source.model_copy(deep=True)
+    failed.ok = False
+    failed.stop_condition = "error"
+    judge_call = JudgeCall(
+        judge="test.Judge",
+        config_digest="config",
+        request_digest="request",
+        outcome="provider_error",
+        error=Error(type="ProviderError", message="judge failed"),
+    )
+    failed.errors.append(Error(type="ProviderError", message="judge failed"))
+    failed.judge_calls.append(judge_call)
+    write_episode(failed_dir, Episode.of(failed))
+    recovered = await replay(failed_dir, tmp_path / "recovered-score")
+    assert recovered.ok and recovered.stop_condition == "done"
+    assert recovered.errors == source.errors
+
+    # Ordinary task scoring failures are also replayable without judge evidence.
+    failed.judge_calls = []
+    failed.errors[-1] = Error(type="TaskError", message="task scoring failed")
+    (failed_dir / TRACES_FILE).write_text("")
+    write_episode(failed_dir, Episode.of(failed))
+    recovered = await replay(failed_dir, tmp_path / "recovered-task-score")
+    assert recovered.ok and recovered.stop_condition == "done"
+
+    # Harness scoring is runtime-dependent and replay only reruns task scoring.
+    failed.errors[-1] = Error(type="HarnessError", message="harness metric failed")
+    (failed_dir / TRACES_FILE).write_text("")
+    write_episode(failed_dir, Episode.of(failed))
+    skipped = await replay(failed_dir, tmp_path / "skipped-harness-score")
+    assert not skipped.ok
+    assert skipped.errors[-1].message == "harness metric failed"
+
+    # A judge called during finalization cannot be recovered by task scoring.
+    failed.judge_calls = [judge_call]
+    failed.errors[-1] = Error(type="ProviderError", message="judge failed")
+    failed.timing.scoring.start = 0.0
+    (failed_dir / TRACES_FILE).write_text("")
+    write_episode(failed_dir, Episode.of(failed))
+    skipped = await replay(failed_dir, tmp_path / "skipped-finalize-judge")
+    assert not skipped.ok
+    assert skipped.errors[-1].message == "judge failed"

tests/v1/test_judges.py

@@ -1,15 +1,15 @@
-"""Pluggable judges: plugin resolution, base-`TaskConfig.judges` narrowing, the built-in
-`reference` / `rubric` judges, and `Task.score` running plugged judges after the decorated
-rewards. Judge model calls are faked at `Judge.complete` — no network."""
+"""Pluggable judges, scoring integration, and the typed provider-call lifecycle."""
 
 import json
 import re
 
 import pytest
+from pydantic import BaseModel
 
 import verifiers.v1 as vf
+from verifiers.v1.clients.eval import EvalClient
 from verifiers.v1.graph import MessageNode
-from verifiers.v1.judge import Judge, JudgeResponse
+from verifiers.v1.judge import JudgeResponse
 from verifiers.v1.loaders import judge_class, judge_config_type, load_judge
 from verifiers.v1.types import AssistantMessage, UserMessage
 
@@ -52,20 +52,87 @@ def make_trace(
     )
 
 
+def model_response(text: str) -> vf.Response:
+    return vf.Response(
+        id="fake",
+        created=0,
+        model="fake",
+        message=AssistantMessage(content=text),
+        finish_reason="stop",
+        raw={"choices": [{"message": {"content": text, "refusal": None}}]},
+    )
+
+
+async def test_complete_uses_sdk_strict_schema(monkeypatch):
+    request = {}
+
+    class Verdict(BaseModel):
+        score: int | None = None
+
+    async def fake_response(self, dialect, body, model, sampling_args):
+        request.update(body)
+        return model_response('{"score": 1}')
+
+    monkeypatch.setattr(EvalClient, "get_response", fake_response)
+    response = await vf.Judge().complete("grade this", schema=Verdict)
+    schema = request["response_format"]["json_schema"]["schema"]
+    assert schema["additionalProperties"] is False
+    assert schema["required"] == ["score"]
+    assert response.parsed == Verdict(score=1)
+
+
+async def test_complete_honors_provider_retry_headers(monkeypatch):
+    attempts = 0
+    delays = []
+
+    async def retry_once(self, dialect, body, model, sampling_args):
+        nonlocal attempts
+        attempts += 1
+        if attempts == 1:
+            raise vf.ProviderError(
+                "retry",
+                status_code=400,
+                headers={"retry-after-ms": "250", "x-should-retry": "true"},
+            )
+        return model_response("yes")
+
+    async def record_sleep(delay):
+        delays.append(delay)
+
+    monkeypatch.setattr(EvalClient, "get_response", retry_once)
+    monkeypatch.setattr("verifiers.v1.judge.asyncio.sleep", record_sleep)
+    response = await vf.Judge(vf.JudgeConfig(max_retries=1)).complete("grade this")
+    assert attempts == 2
+    assert delays == [0.25]
+    assert len(response.call.calls) == 2
+
+    async def refuse_retry(self, dialect, body, model, sampling_args):
+        nonlocal attempts
+        attempts += 1
+        raise vf.ProviderError(
+            "do not retry", status_code=503, headers={"x-should-retry": "false"}
+        )
+
+    attempts = 0
+    monkeypatch.setattr(EvalClient, "get_response", refuse_retry)
+    with pytest.raises(vf.ProviderError, match="do not retry"):
+        await vf.Judge(vf.JudgeConfig(max_retries=1)).complete("grade this")
+    assert attempts == 1
+
+
 @pytest.fixture
 def fake_judge_model(monkeypatch):
     """Fake the judge's model call, recording each prompt for assertions. Rubric calls (a JSON
     `verdicts` instruction or a `schema`) reply one reasoned verdict per `- name: text` criterion,
     "yes" iff it mentions Paris; other judges reply plain "yes"/"no" by the response block."""
     prompts: list[str] = []
 
-    async def fake_complete(
-        self, messages, *, trace=None, schema=None, parse=None, **sampling
-    ):
+    async def fake_response(self, dialect, body, model, sampling_args):
+        messages = body["messages"][0]["content"]
         prompts.append(messages)
         # Rubric calls carry criteria lines + a JSON `verdicts` instruction; reply one verdict per
         # criterion (yes iff its text mentions Paris) with a reason. Other judges get plain yes/no.
-        if schema is not None or '"verdicts"' in messages:
+        if "response_format" in body or '"verdicts"' in messages:
             verdicts = [
                 {
                     "name": name,
@@ -74,23 +141,12 @@ async def fake_complete(
                 }
                 for name, text in re.findall(r"^- ([^:]+): (.+)$", messages, re.M)
             ]
-            response = JudgeResponse(
-                text=json.dumps({"verdicts": verdicts}),
-                parsed=schema.model_validate({"verdicts": verdicts})
-                if schema
-                else None,
-            )
+            text = json.dumps({"verdicts": verdicts})
         else:
-            response = JudgeResponse(
-                text="yes" if "Paris" in messages.split("Response:")[-1] else "no"
-            )
-        if parse is not None:
-            response.parsed = parse(response)
-        if trace is not None:
-            trace.record_judge(response)
-        return response
+            text = "yes" if "Paris" in messages.split("Response:")[-1] else "no"
+        return model_response(text)
 
-    monkeypatch.setattr(Judge, "complete", fake_complete)
+    monkeypatch.setattr(EvalClient, "get_response", fake_response)
     return prompts
 
 
@@ -215,10 +271,12 @@ async def test_reference_score(fake_judge_model):
     assert (
         "Capital of France?" in fake_judge_model[0]
     )  # the task prompt is in the judge prompt
-    assert len(trace.info["judge"]) == 1  # the call is recorded onto the trace
+    assert len(trace.judge_calls) == 1  # the call is recorded onto the trace
+    assert trace.judge_calls[0].outcome == "success"
 
     trace = make_trace(reply="It is Rome.")
     assert await vf.ReferenceJudge().score(trace.task.data, trace) == 0.0
+    assert trace.judge_calls[0].outcome == "success"
 
     judge = vf.ReferenceJudge(vf.ReferenceJudgeConfig(answer_field="gold"))
     with pytest.raises(ValueError, match="no 'gold' field"):  # misconfig raises, not 0
@@ -432,21 +490,12 @@ async def test_reference_choices(fake_judge_model):
 
 async def test_error_attribution(monkeypatch, tmp_path):
     # The policy: a MODEL failure scores 0.0; a JUDGE failure errors the rollout (raises
-    # out of Task.score as a TaskError) so training skips the sample instead of
+    # out of Task.score) so training skips the sample instead of
     # punishing the model for a broken judge.
-    async def gibberish_judge(
-        self, messages, *, trace=None, schema=None, parse=None, **s
-    ):
-        response = JudgeResponse(text="as an AI language model I cannot grade this")
-        try:
-            if parse is not None:
-                response.parsed = parse(response)
-            return response
-        finally:
-            if trace is not None:
-                trace.record_judge(response)
-
-    monkeypatch.setattr(Judge, "complete", gibberish_judge)
+    async def gibberish_judge(self, dialect, body, model, sampling_args):
+        return model_response("as an AI language model I cannot grade this")
+
+    monkeypatch.setattr(EvalClient, "get_response", gibberish_judge)
     taskset = JudgedTaskset(
         JudgedConfig.model_validate({"task": {"judges": [{"id": "reference"}]}})
     )
@@ -461,7 +510,8 @@ async def gibberish_judge(
             trace, runtime=None
         )
     assert "reference" not in trace.rewards
-    assert len(trace.info["judge"]) == 1  # the billed call is still recorded
+    assert trace.judge_calls[0].outcome == "parse_error"
+    assert trace.judge_calls[0].error is not None
 
 
 # --- rubric --------------------------------------------------------------------------------
@@ -521,33 +571,34 @@ async def test_rubric_score(tmp_path, fake_judge_model):
     trace = make_trace()
     assert await judge.score(trace.task.data, trace) == 0.75
     assert trace.metrics == {"rubric/mentions_paris": 1.0, "rubric/is_polite": 0.0}
-    assert len(trace.info["judge"]) == 1  # one call for the whole rubric
+    assert len(trace.judge_calls) == 1  # one call for the whole rubric
 
 
 async def test_rubric_verdict_mismatch_raises(tmp_path, monkeypatch):
     # A reply that doesn't verdict exactly the rubric's criteria is a judge failure: raise
     # (-> rollout error), don't guess or silently score 0.
-    async def wrong_names(self, messages, *, trace=None, schema=None, parse=None, **s):
+    async def wrong_names(self, dialect, body, model, sampling_args):
         verdicts = {"verdicts": [{"name": "typo", "reason": "x", "verdict": "yes"}]}
-        return JudgeResponse(text=json.dumps(verdicts), parsed=None)
+        return model_response(json.dumps(verdicts))
 
-    monkeypatch.setattr(Judge, "complete", wrong_names)
+    monkeypatch.setattr(EvalClient, "get_response", wrong_names)
     judge = rubric_judge(tmp_path)
     trace = make_trace()
     with pytest.raises(ValueError, match="expected the batch"):
         await judge.score(trace.task.data, trace)
+    assert trace.judge_calls[0].outcome == "parse_error"
 
 
 CHOICES_TOML = '[[criteria]]\nname = "depth"\ntext = "How thorough?"\nchoices = ["none", "partial", "good"]\n'
 
 
 async def test_rubric_choices_normalize(tmp_path, monkeypatch):
     # Ordered choices (worst→best) score by rank: "partial" of ["none","partial","good"] -> 0.5.
-    async def graded(self, messages, *, trace=None, schema=None, parse=None, **s):
+    async def graded(self, dialect, body, model, sampling_args):
         v = {"verdicts": [{"name": "depth", "reason": "r", "verdict": "partial"}]}
-        return JudgeResponse(text=json.dumps(v), parsed=None)
+        return model_response(json.dumps(v))
 
-    monkeypatch.setattr(Judge, "complete", graded)
+    monkeypatch.setattr(EvalClient, "get_response", graded)
     judge = rubric_judge(tmp_path, body=CHOICES_TOML, name="q")
     trace = make_trace()
     assert await judge.score(trace.task.data, trace) == 0.5
@@ -556,11 +607,11 @@ async def graded(self, messages, *, trace=None, schema=None, parse=None, **s):
 
 async def test_rubric_off_menu_answer_raises(tmp_path, monkeypatch):
     # A verdict that isn't one of the criterion's choices is a judge failure, not a 0.
-    async def off_menu(self, messages, *, trace=None, schema=None, parse=None, **s):
+    async def off_menu(self, dialect, body, model, sampling_args):
         v = {"verdicts": [{"name": "depth", "reason": "r", "verdict": "maybe"}]}
-        return JudgeResponse(text=json.dumps(v), parsed=None)
+        return model_response(json.dumps(v))
 
-    monkeypatch.setattr(Judge, "complete", off_menu)
+    monkeypatch.setattr(EvalClient, "get_response", off_menu)
     judge = rubric_judge(tmp_path, body=CHOICES_TOML)
     trace = make_trace()
     with pytest.raises(ValueError, match="expected one of"):
@@ -630,9 +681,7 @@ async def test_task_score_runs_plugged_judges(tmp_path, fake_judge_model):
     assert (
         trace.rewards["quality"].score == 0.75
     )  # the rubric's aggregate, under its `name`
-    assert (
-        len(trace.info["judge"]) == 2
-    )  # every judge call recorded (rubric = one call)
+    assert len(trace.judge_calls) == 2  # every judge call recorded (rubric = one call)
 
 
 async def test_task_without_judges_scores_as_before():

verifiers/v1/__init__.py

@@ -106,6 +106,7 @@
     Error,
     EvalRunInfo,
     GenerationSpan,
+    JudgeCall,
     ModelCall,
     Reward,
     RunInfo,
@@ -179,6 +180,7 @@
     "AgentInfo",
     "RunInfo",
     "EvalRunInfo",
+    "JudgeCall",
     "ModelCall",
     "TrainRunInfo",
     "VersionInfo",

verifiers/v1/cli/dashboard/eval.py

@@ -19,7 +19,6 @@
 from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.env import RunSlot
 from verifiers.v1.trace import Trace
-from verifiers.v1.types import Usage
 from verifiers.v1.utils.format import (
     format_count,
     format_mean,
@@ -405,8 +404,8 @@ def _breakdown(scored: list[Trace], done: list[Trace]) -> Table | None:
         if trace.usage is not None and trace.usage.cost is not None:
             total_cost += trace.usage.cost
             have_cost = True
-        # Judge / auxiliary scoring calls (off the message graph) shown separately from the agent's.
-        judge = Usage.aggregate(trace.extra_usage)
+        # Judge calls (off the message graph) shown separately from the agent's.
+        judge = trace.judge_usage
         if judge is not None:
             total_judge_in += judge.input_tokens
             total_judge_out += judge.completion_tokens