From 66994ca409abf1f486ed65a01cca184351aab88e Mon Sep 17 00:00:00 2001 From: Bo Liu Date: Sat, 29 Aug 2026 16:37:42 +0900 Subject: [PATCH 1/2] fix(cli): reject --trials > 1 without --matrix instead of silently running one trial --- src/benchflow/eval_plan.py | 4 ++++ tests/test_cli_arg_validation.py | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/src/benchflow/eval_plan.py b/src/benchflow/eval_plan.py index f2cd6b39c..84d307b22 100644 --- a/src/benchflow/eval_plan.py +++ b/src/benchflow/eval_plan.py @@ -264,6 +264,10 @@ def build_eval_plan(request: EvalCreateRequest) -> EvalPlan: raise EvalPlanError("--matrix currently requires --tasks-dir") if request.trials < 1: raise EvalPlanError("--trials must be >= 1") + if request.trials > 1 and request.matrix is None: + # Only the matrix expansion consumes trials; a plain run would silently + # do one trial per task while the caller believes it ran N. + raise EvalPlanError("--trials > 1 requires --matrix") if request.expected_tasks is not None and request.expected_tasks < 1: raise EvalPlanError("--expected-tasks must be >= 1") if request.canonicalize not in {"none", "one-healthy-per-task"}: diff --git a/tests/test_cli_arg_validation.py b/tests/test_cli_arg_validation.py index 44cf1d382..2d7b0ca44 100644 --- a/tests/test_cli_arg_validation.py +++ b/tests/test_cli_arg_validation.py @@ -65,6 +65,13 @@ def test_build_concurrency_zero_rejected(tmp_path: Path): assert "--build-concurrency must be >= 1" in result.stderr +def test_trials_without_matrix_rejected(tmp_path: Path): + # Only --matrix consumes trials; a plain run silently does one trial/task. + result = _invoke(tmp_path, "--sandbox", "docker", "--trials", "3") + assert result.exit_code == 1 + assert "--trials > 1 requires --matrix" in result.stderr + + def test_skill_mode_bogus_clean_error(tmp_path: Path): result = _invoke(tmp_path, "--sandbox", "docker", "--skill-mode", "bogus") assert result.exit_code == 1 From b186cfb5f547127d5670635d0c085b9aaa44a47d Mon Sep 17 00:00:00 2001 From: Bingran You Date: Tue, 1 Sep 2026 16:53:55 -0700 Subject: [PATCH 2/2] fix(cli): explain the matrix escape hatch --- src/benchflow/eval_plan.py | 8 ++++++-- tests/test_cli_arg_validation.py | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/benchflow/eval_plan.py b/src/benchflow/eval_plan.py index 84d307b22..af5fd9f44 100644 --- a/src/benchflow/eval_plan.py +++ b/src/benchflow/eval_plan.py @@ -266,8 +266,12 @@ def build_eval_plan(request: EvalCreateRequest) -> EvalPlan: raise EvalPlanError("--trials must be >= 1") if request.trials > 1 and request.matrix is None: # Only the matrix expansion consumes trials; a plain run would silently - # do one trial per task while the caller believes it ran N. - raise EvalPlanError("--trials > 1 requires --matrix") + # do one trial per task while the caller believes it ran N. Per-trial + # requests created by run_matrix_eval bypass this planner by design. + raise EvalPlanError( + "--trials > 1 requires --matrix; to repeat one model, use a " + 'single-entry matrix such as "models: {default: }"' + ) if request.expected_tasks is not None and request.expected_tasks < 1: raise EvalPlanError("--expected-tasks must be >= 1") if request.canonicalize not in {"none", "one-healthy-per-task"}: diff --git a/tests/test_cli_arg_validation.py b/tests/test_cli_arg_validation.py index 2d7b0ca44..e8d32128b 100644 --- a/tests/test_cli_arg_validation.py +++ b/tests/test_cli_arg_validation.py @@ -66,10 +66,11 @@ def test_build_concurrency_zero_rejected(tmp_path: Path): def test_trials_without_matrix_rejected(tmp_path: Path): - # Only --matrix consumes trials; a plain run silently does one trial/task. + """Guards PR #1064: plain runs must not silently ignore --trials.""" result = _invoke(tmp_path, "--sandbox", "docker", "--trials", "3") assert result.exit_code == 1 assert "--trials > 1 requires --matrix" in result.stderr + assert "single-entry matrix" in result.stderr def test_skill_mode_bogus_clean_error(tmp_path: Path):