diff --git a/src/benchflow/eval_plan.py b/src/benchflow/eval_plan.py index f2cd6b39c..af5fd9f44 100644 --- a/src/benchflow/eval_plan.py +++ b/src/benchflow/eval_plan.py @@ -264,6 +264,14 @@ def build_eval_plan(request: EvalCreateRequest) -> EvalPlan: raise EvalPlanError("--matrix currently requires --tasks-dir") if request.trials < 1: raise EvalPlanError("--trials must be >= 1") + if request.trials > 1 and request.matrix is None: + # Only the matrix expansion consumes trials; a plain run would silently + # do one trial per task while the caller believes it ran N. Per-trial + # requests created by run_matrix_eval bypass this planner by design. + raise EvalPlanError( + "--trials > 1 requires --matrix; to repeat one model, use a " + 'single-entry matrix such as "models: {default: }"' + ) if request.expected_tasks is not None and request.expected_tasks < 1: raise EvalPlanError("--expected-tasks must be >= 1") if request.canonicalize not in {"none", "one-healthy-per-task"}: diff --git a/tests/test_cli_arg_validation.py b/tests/test_cli_arg_validation.py index 44cf1d382..e8d32128b 100644 --- a/tests/test_cli_arg_validation.py +++ b/tests/test_cli_arg_validation.py @@ -65,6 +65,14 @@ def test_build_concurrency_zero_rejected(tmp_path: Path): assert "--build-concurrency must be >= 1" in result.stderr +def test_trials_without_matrix_rejected(tmp_path: Path): + """Guards PR #1064: plain runs must not silently ignore --trials.""" + result = _invoke(tmp_path, "--sandbox", "docker", "--trials", "3") + assert result.exit_code == 1 + assert "--trials > 1 requires --matrix" in result.stderr + assert "single-entry matrix" in result.stderr + + def test_skill_mode_bogus_clean_error(tmp_path: Path): result = _invoke(tmp_path, "--sandbox", "docker", "--skill-mode", "bogus") assert result.exit_code == 1