fix(eval): re-run infra-retryable verifier-errored tasks on resume

src/benchflow/evaluation.py

@@ -1084,6 +1084,10 @@ def _get_task_dirs(self) -> list[Path]:
     def _get_completed_tasks(self) -> dict[str, dict]:
         """Load tasks that already have results with rewards or verifier errors.
 
+        Scoreless results whose verifier error is infra-retryable (per
+        ``RetryConfig.should_retry_verifier_error``) are not reused: the
+        verifier never scored the frozen workspace, so the task re-runs.
+
         Scoped to the current job directory (``_jobs_dir / _job_name``) to
         prevent cross-job contamination.  When multiple result.json files
         exist for the same task (retry artifacts), the newest by mtime wins.
@@ -1109,6 +1113,17 @@ def _get_completed_tasks(self) -> dict[str, dict]:
         completed: dict[str, dict] = {}
         for task, (_mt, r) in best.items():
             if r.get("verifier_error"):
+                # A scoreless result whose verifier error is infra-retryable
+                # (same taxonomy as the within-run retry) records no signal
+                # about the task; reusing it pins a lost score forever.
+                if r.get("rewards") is None and (
+                    self._config.retry.should_retry_verifier_error(r["verifier_error"])
+                ):
+                    logger.info(
+                        f"Re-running verifier-errored task on resume: {task} "
+                        f"({truncate_end(r['verifier_error'], 80)})"
+                    )
+                    continue
                 logger.info(
                     f"Reusing completed verifier-errored task on resume: {task} "
                     f"({truncate_end(r['verifier_error'], 80)})"

tests/test_verify.py

@@ -489,8 +489,8 @@ async def test_agent_error_still_retries(self, job_factory):
 
 
 class TestResume:
-    def test_verifier_errored_is_complete(self, tmp_path, caplog):
-        """Guards the PR #819 fix for issue #542's misleading resume log."""
+    def test_infra_verifier_errored_reruns(self, tmp_path, caplog):
+        """A scoreless infra-retryable verifier error must not pin a lost score."""
         task_dir = tmp_path / "task1" / "trial-1"
         task_dir.mkdir(parents=True)
         (task_dir / "result.json").write_text(
@@ -505,6 +505,32 @@ def test_verifier_errored_is_complete(self, tmp_path, caplog):
         )
         from benchflow.evaluation import Evaluation, EvaluationConfig
 
+        job = Evaluation(
+            tasks_dir=tmp_path, jobs_dir=tmp_path, config=EvaluationConfig()
+        )
+        with caplog.at_level(logging.INFO):
+            completed = job._get_completed_tasks()
+        assert "task1" not in completed
+        assert any(
+            "Re-running verifier-errored task" in m for m in caplog.messages
+        )
+
+    def test_contract_verifier_errored_is_complete(self, tmp_path, caplog):
+        """Guards the PR #819 fix for issue #542's misleading resume log."""
+        task_dir = tmp_path / "task1" / "trial-1"
+        task_dir.mkdir(parents=True)
+        (task_dir / "result.json").write_text(
+            json.dumps(
+                {
+                    "task_name": "task1",
+                    "rewards": None,
+                    "error": None,
+                    "verifier_error": "verifier crashed: No reward file found",
+                }
+            )
+        )
+        from benchflow.evaluation import Evaluation, EvaluationConfig
+
         job = Evaluation(
             tasks_dir=tmp_path, jobs_dir=tmp_path, config=EvaluationConfig()
         )