From 33f7611f546dc0be538316d80c81af673f0f3cda Mon Sep 17 00:00:00 2001 From: jiafatom Date: Tue, 14 Jul 2026 17:40:45 +0000 Subject: [PATCH 1/7] Allow overriding past_present_share_buffer in LMEvaluator ortgenai backend The `ortgenai` lm-eval backend always derived `past_present_share_buffer` from the exported `genai_config.json`, with no way to override it from a recipe config. Some models (e.g. Gemma 4) export with shared buffers enabled but require it disabled for correct KV-cache handling during evaluation, forcing users to write a standalone eval script instead of a declarative Olive config. - `LMEvalORTGenAIEvaluator` now accepts an explicit `past_present_share_buffer` argument that takes precedence over the exported value (extracted into a small `_resolve_past_present_share_buffer` helper for testability). - `LMEvaluator` gains a `model_args` dict that is forwarded verbatim to the lm-eval model backend constructor, so genai-specific options can be set from the evaluator config. User-provided `model_args` override the Olive-derived defaults (batch_size, max_length, ep, ...). This lets recipes evaluate such models with a standard `LMEvaluator` config, e.g. `"model_args": {"past_present_share_buffer": false}`. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 37949b3b-0064-4033-8d7a-8127271bcf39 --- olive/evaluator/lmeval_ort.py | 20 +++++++++++++++- olive/evaluator/olive_evaluator.py | 13 +++++++++- test/evaluator/test_lmeval_ort.py | 29 ++++++++++++++++++++++ test/evaluator/test_olive_evaluator.py | 33 ++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 2 deletions(-) diff --git a/olive/evaluator/lmeval_ort.py b/olive/evaluator/lmeval_ort.py index ca4a0f4539..39fc8c8f9b 100644 --- a/olive/evaluator/lmeval_ort.py +++ b/olive/evaluator/lmeval_ort.py @@ -545,6 +545,17 @@ def _normalize_provider_name(cls, ep: str) -> tuple[str, str]: normalized_ep = str(ep).lower().replace("executionprovider", "") return normalized_ep, cls._ORT_GENAI_PROVIDER_NAMES.get(normalized_ep, normalized_ep) + @staticmethod + def _resolve_past_present_share_buffer(override: bool | None, genai_config: dict) -> bool: + """Resolve the ``past_present_share_buffer`` search option. + + An explicit ``override`` takes precedence; otherwise fall back to the exported + ``genai_config.json`` value (defaulting to ``False`` when absent). + """ + if override is not None: + return override + return genai_config.get("search", {}).get("past_present_share_buffer", False) + def __init__( self, pretrained: str, @@ -553,6 +564,7 @@ def __init__( ep: str = "follow_config", ep_options: dict | None = None, device: str = "cpu", + past_present_share_buffer: bool | None = None, **kwargs, ): """Initialize the evaluator. @@ -563,6 +575,9 @@ def __init__( :param ep: The execution provider to use. "follow_config" will use the provider specified in the genai_config file :param ep_options: The options to use for the execution provider. Only applicable if ep is not "follow_config" :param device: The device to run log likelihood calculations on + :param past_present_share_buffer: Override the exported ``search.past_present_share_buffer`` setting. When + ``None`` (default) the value from ``genai_config.json`` is used. Some models (e.g. Gemma 4) export with + shared buffers enabled but require it disabled for correct KV-cache handling during evaluation. """ if og is None: raise ImportError("onnxruntime-genai is not installed.") @@ -602,7 +617,10 @@ def __init__( self._eot_token_id = self._eos_token_ids[0] # Mirror the exported GenAI cache-sharing setting when creating GeneratorParams. # Artifacts with shared past/present buffers require the same search option at runtime. - self._past_present_share_buffer = genai_config["search"].get("past_present_share_buffer", False) + # An explicit override (e.g. from a recipe config) takes precedence over the exported value. + self._past_present_share_buffer = self._resolve_past_present_share_buffer( + past_present_share_buffer, genai_config + ) self.params = og.GeneratorParams(self.model) self.params.set_search_options( max_length=self.max_length, diff --git a/olive/evaluator/olive_evaluator.py b/olive/evaluator/olive_evaluator.py index 9b25e5c268..a9002e38eb 100644 --- a/olive/evaluator/olive_evaluator.py +++ b/olive/evaluator/olive_evaluator.py @@ -2056,6 +2056,10 @@ def __init__(self, tasks: list[str], **kwargs): self.ep = kwargs.get("execution_provider") self.ep_options = kwargs.get("provider_options") self.device = kwargs.get("device") + # Extra keyword arguments forwarded verbatim to the lm-eval model backend constructor + # (e.g. ``past_present_share_buffer`` for the ``ortgenai`` backend). Backend-specific; + # values here override the defaults Olive derives (batch_size, max_length, ep, ...). + self.model_args = kwargs.get("model_args") or {} def evaluate( self, @@ -2125,7 +2129,14 @@ def evaluate( ) if self.tasks: - lmmodel = get_model(self.model_class)(**init_args, batch_size=self.batch_size, max_length=self.max_length) + model_init_args = { + **init_args, + "batch_size": self.batch_size, + "max_length": self.max_length, + # User-provided model_args win over the Olive-derived defaults above. + **self.model_args, + } + lmmodel = get_model(self.model_class)(**model_init_args) results = simple_evaluate( model=lmmodel, diff --git a/test/evaluator/test_lmeval_ort.py b/test/evaluator/test_lmeval_ort.py index aa1b9ede91..e0541d8311 100644 --- a/test/evaluator/test_lmeval_ort.py +++ b/test/evaluator/test_lmeval_ort.py @@ -40,3 +40,32 @@ class _Holder: device_prop.fset(holder, "cpu") assert device_prop.fget(holder) == "cpu" + + +class TestResolvePastPresentShareBuffer: + """The exported genai_config value is the default; an explicit override wins.""" + + @staticmethod + def _resolve(override, genai_config): + from olive.evaluator.lmeval_ort import LMEvalORTGenAIEvaluator + + # pylint: disable=protected-access + return LMEvalORTGenAIEvaluator._resolve_past_present_share_buffer(override, genai_config) + + @pytest.mark.parametrize("config_value", [True, False]) + def test_uses_config_value_when_no_override(self, config_value): + genai_config = {"search": {"past_present_share_buffer": config_value}} + assert self._resolve(None, genai_config) is config_value + + def test_defaults_to_false_when_absent(self): + assert self._resolve(None, {"search": {}}) is False + assert self._resolve(None, {}) is False + + @pytest.mark.parametrize( + ("override", "config_value"), + [(False, True), (True, False)], + ) + def test_override_takes_precedence_over_config(self, override, config_value): + # Gemma 4 exports with shared buffers enabled but requires it disabled for evaluation. + genai_config = {"search": {"past_present_share_buffer": config_value}} + assert self._resolve(override, genai_config) is override diff --git a/test/evaluator/test_olive_evaluator.py b/test/evaluator/test_olive_evaluator.py index 1b5528e79c..c7c85b9a7e 100644 --- a/test/evaluator/test_olive_evaluator.py +++ b/test/evaluator/test_olive_evaluator.py @@ -513,6 +513,39 @@ def test_lm_evaluator_dispatches_to_requested_backend( get_model_mock.assert_called_once_with(model_class) + @patch("lm_eval.utils.setup_logging") + @patch("lm_eval.tasks.TaskManager") + @patch("lm_eval.simple_evaluate") + @patch("lm_eval.api.registry.get_model") + def test_lm_evaluator_forwards_model_args_to_backend( + self, get_model_mock, simple_evaluate_mock, _task_manager_mock, _setup_logging_mock + ): + from olive.evaluator.olive_evaluator import LMEvaluator + from olive.model.handler.onnx import ONNXModelHandler + + simple_evaluate_mock.return_value = {"results": {}} + backend_ctor = MagicMock(return_value=MagicMock()) + get_model_mock.return_value = backend_ctor + + # model_args should reach the backend constructor and override Olive-derived defaults. + evaluator = LMEvaluator( + tasks=["arc_easy"], + model_class="ortgenai", + batch_size=1, + max_length=128, + model_args={"past_present_share_buffer": False, "batch_size": 4}, + ) + + model = MagicMock(spec=ONNXModelHandler) + model.model_path = "/tmp/model.onnx" + + evaluator.evaluate(model, metrics=[], device=Device.CPU, execution_providers=["CPUExecutionProvider"]) + + _, call_kwargs = backend_ctor.call_args + assert call_kwargs["past_present_share_buffer"] is False + assert call_kwargs["batch_size"] == 4 + assert call_kwargs["max_length"] == 128 + @pytest.mark.skipif( importlib.util.find_spec("lm_eval") is None, From 02d78188ad5b26f0f926e3cf11c34529e87e41cd Mon Sep 17 00:00:00 2001 From: jiafatom Date: Tue, 14 Jul 2026 17:53:03 +0000 Subject: [PATCH 2/7] Address review: validate model_args and keep batch_size consistent - Validate that model_args is a dict, failing early with a clear error instead of a confusing unpacking error later. - Use the effective batch size (after model_args overrides) for the lm-eval simple_evaluate call so backend batching and lm-eval batching stay consistent. - Assert simple_evaluate receives the overridden batch size, and add a test that a non-dict model_args is rejected. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 37949b3b-0064-4033-8d7a-8127271bcf39 --- olive/evaluator/olive_evaluator.py | 8 +++++++- test/evaluator/test_olive_evaluator.py | 10 ++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/olive/evaluator/olive_evaluator.py b/olive/evaluator/olive_evaluator.py index a9002e38eb..5a7865ffbe 100644 --- a/olive/evaluator/olive_evaluator.py +++ b/olive/evaluator/olive_evaluator.py @@ -2060,6 +2060,8 @@ def __init__(self, tasks: list[str], **kwargs): # (e.g. ``past_present_share_buffer`` for the ``ortgenai`` backend). Backend-specific; # values here override the defaults Olive derives (batch_size, max_length, ep, ...). self.model_args = kwargs.get("model_args") or {} + if not isinstance(self.model_args, dict): + raise ValueError(f"model_args must be a dict, got {type(self.model_args).__name__}.") def evaluate( self, @@ -2138,12 +2140,16 @@ def evaluate( } lmmodel = get_model(self.model_class)(**model_init_args) + # Keep lm-eval's batching consistent with the backend's effective batch size, + # which model_args may have overridden. + effective_batch_size = model_init_args["batch_size"] + results = simple_evaluate( model=lmmodel, tasks=self.tasks, task_manager=TaskManager(), log_samples=False, - batch_size=self.batch_size, + batch_size=effective_batch_size, device=device, limit=self.limit, # Forward the configured value instead of letting lm-eval silently use its default. diff --git a/test/evaluator/test_olive_evaluator.py b/test/evaluator/test_olive_evaluator.py index c7c85b9a7e..5d65e90361 100644 --- a/test/evaluator/test_olive_evaluator.py +++ b/test/evaluator/test_olive_evaluator.py @@ -546,6 +546,16 @@ def test_lm_evaluator_forwards_model_args_to_backend( assert call_kwargs["batch_size"] == 4 assert call_kwargs["max_length"] == 128 + # lm-eval's simple_evaluate must use the same effective batch size as the backend. + _, eval_kwargs = simple_evaluate_mock.call_args + assert eval_kwargs["batch_size"] == 4 + + def test_lm_evaluator_rejects_non_dict_model_args(self): + from olive.evaluator.olive_evaluator import LMEvaluator + + with pytest.raises(ValueError, match="model_args must be a dict"): + LMEvaluator(tasks=["arc_easy"], model_class="ortgenai", model_args=["not", "a", "dict"]) + @pytest.mark.skipif( importlib.util.find_spec("lm_eval") is None, From 7df7ef0f081e8903abbfe4dde3a1e01e4ff5c463 Mon Sep 17 00:00:00 2001 From: jiafatom Date: Tue, 14 Jul 2026 18:15:38 +0000 Subject: [PATCH 3/7] Add optional sample logging to LMEvaluator When sample_log_num > 0, LMEvaluator now enables lm-eval log_samples and writes the first N per-task predictions (question, options, resolved prediction, target, acc) to _samples.jsonl, mirroring the accuracy evaluators' sample logging. This makes it possible to inspect why a task scores low. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 37949b3b-0064-4033-8d7a-8127271bcf39 --- olive/evaluator/olive_evaluator.py | 66 +++++++++++++++++++++++++- test/evaluator/test_olive_evaluator.py | 39 +++++++++++++++ 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/olive/evaluator/olive_evaluator.py b/olive/evaluator/olive_evaluator.py index 5a7865ffbe..f103a4cba0 100644 --- a/olive/evaluator/olive_evaluator.py +++ b/olive/evaluator/olive_evaluator.py @@ -2062,6 +2062,65 @@ def __init__(self, tasks: list[str], **kwargs): self.model_args = kwargs.get("model_args") or {} if not isinstance(self.model_args, dict): raise ValueError(f"model_args must be a dict, got {type(self.model_args).__name__}.") + # When > 0, log the first N per-task sample predictions (question, prediction, target) to a + # JSONL file for debugging, mirroring the accuracy evaluators' ``sample_log`` behavior. + self.sample_log_num = kwargs.get("sample_log_num", 0) + self.sample_log_dir = kwargs.get("sample_log_dir") + + @staticmethod + def _extract_prediction(filtered_resps): + """Best-effort recovery of the model's prediction from an lm-eval sample record. + + For multiple-choice tasks ``filtered_resps`` is a list of ``[loglikelihood, is_greedy]`` + pairs (one per choice); the prediction is the argmax choice index. For generation tasks it + is a list of response strings, which are returned as-is. + """ + if not filtered_resps: + return filtered_resps + first = filtered_resps[0] + if isinstance(first, (list, tuple)) and first and isinstance(first[0], (int, float, bool)): + logliks = [r[0] for r in filtered_resps] + return int(max(range(len(logliks)), key=logliks.__getitem__)) + return filtered_resps + + @classmethod + def _save_lmeval_samples(cls, task_name: str, samples: list, num_samples: int, output_dir: Path) -> None: + """Save the first ``num_samples`` lm-eval sample records for ``task_name`` to a JSONL file. + + Each line captures the question, options, resolved prediction, and target so a low score can + be inspected. Best-effort: filesystem or serialization errors are logged as warnings. + """ + if num_samples <= 0 or not samples: + return + try: + output_dir.mkdir(parents=True, exist_ok=True) + safe_name = task_name.replace("/", "_").replace("\\", "_") or "task" + log_path = output_dir / f"{safe_name}_samples.jsonl" + n = min(num_samples, len(samples)) + with log_path.open("w", encoding="utf-8") as f: + for s in samples[:n]: + doc = s.get("doc") or {} + record = {"index": s.get("doc_id")} + for qk in ("question", "query", "input", "problem"): + if qk in doc: + record["question"] = doc[qk] + break + options = doc.get("options") or doc.get("choices") + if options is not None: + record["options"] = options + pred = cls._extract_prediction(s.get("filtered_resps")) + if isinstance(pred, int) and isinstance(options, list) and 0 <= pred < len(options): + record["prediction_index"] = pred + record["prediction"] = f"{chr(ord('A') + pred)}. {options[pred]}" + else: + record["prediction"] = pred + record["target"] = s.get("target") + if "acc" in s: + record["acc"] = s["acc"] + f.write(json.dumps(record, ensure_ascii=False, default=str) + "\n") + logger.info("Saved %d %s sample predictions to %s", n, task_name, log_path) + except Exception as e: + logger.warning("Failed to save lm-eval sample log for task '%s': %s", task_name, e) def evaluate( self, @@ -2148,7 +2207,7 @@ def evaluate( model=lmmodel, tasks=self.tasks, task_manager=TaskManager(), - log_samples=False, + log_samples=self.sample_log_num > 0, batch_size=effective_batch_size, device=device, limit=self.limit, @@ -2156,6 +2215,11 @@ def evaluate( bootstrap_iters=self.bootstrap_iters, ) + if self.sample_log_num > 0: + sample_dir = Path(self.sample_log_dir) if self.sample_log_dir else Path.cwd() / "sample_logs" + for task_name, task_samples in (results.get("samples") or {}).items(): + self._save_lmeval_samples(task_name, task_samples, self.sample_log_num, sample_dir) + for task_name in sorted(results["results"].keys()): metric_items = sorted(results["results"][task_name].items()) diff --git a/test/evaluator/test_olive_evaluator.py b/test/evaluator/test_olive_evaluator.py index 5d65e90361..0261bed98a 100644 --- a/test/evaluator/test_olive_evaluator.py +++ b/test/evaluator/test_olive_evaluator.py @@ -3,6 +3,7 @@ # Licensed under the MIT License. # -------------------------------------------------------------------------- import importlib.util +import json from functools import partial from types import FunctionType from typing import ClassVar @@ -556,6 +557,44 @@ def test_lm_evaluator_rejects_non_dict_model_args(self): with pytest.raises(ValueError, match="model_args must be a dict"): LMEvaluator(tasks=["arc_easy"], model_class="ortgenai", model_args=["not", "a", "dict"]) + def test_lm_evaluator_extract_prediction_multiple_choice(self): + from olive.evaluator.olive_evaluator import LMEvaluator + + # Multiple-choice: list of [loglikelihood, is_greedy] pairs -> argmax index. + filtered_resps = [[-5.5, False], [-0.05, True], [-7.3, False]] + assert LMEvaluator._extract_prediction(filtered_resps) == 1 + + def test_lm_evaluator_extract_prediction_generation(self): + from olive.evaluator.olive_evaluator import LMEvaluator + + # Generation: list of response strings returned as-is. + assert LMEvaluator._extract_prediction(["Paris"]) == ["Paris"] + assert LMEvaluator._extract_prediction([]) == [] + + def test_lm_evaluator_save_lmeval_samples(self, tmp_path): + from olive.evaluator.olive_evaluator import LMEvaluator + + samples = [ + { + "doc_id": 0, + "doc": {"question": "2+2?", "options": ["3", "4", "5"]}, + "filtered_resps": [[-5.0, False], [-0.1, True], [-6.0, False]], + "target": "B", + "acc": 1.0, + } + ] + LMEvaluator._save_lmeval_samples("my_task", samples, num_samples=5, output_dir=tmp_path) + + log_path = tmp_path / "my_task_samples.jsonl" + assert log_path.exists() + record = json.loads(log_path.read_text().strip()) + assert record["index"] == 0 + assert record["question"] == "2+2?" + assert record["prediction_index"] == 1 + assert record["prediction"] == "B. 4" + assert record["target"] == "B" + assert record["acc"] == 1.0 + @pytest.mark.skipif( importlib.util.find_spec("lm_eval") is None, From 2084b1078d65b0a3ef87c1e71e6b517d8b30149f Mon Sep 17 00:00:00 2001 From: jiafatom Date: Tue, 14 Jul 2026 18:24:09 +0000 Subject: [PATCH 4/7] Silence pylint protected-access in LMEvaluator sample-log tests Add # pylint: disable=protected-access to the tests that call the _extract_prediction / _save_lmeval_samples helpers directly, matching the existing pattern in this file. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 37949b3b-0064-4033-8d7a-8127271bcf39 --- test/evaluator/test_olive_evaluator.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/evaluator/test_olive_evaluator.py b/test/evaluator/test_olive_evaluator.py index 0261bed98a..64a7349e39 100644 --- a/test/evaluator/test_olive_evaluator.py +++ b/test/evaluator/test_olive_evaluator.py @@ -558,6 +558,7 @@ def test_lm_evaluator_rejects_non_dict_model_args(self): LMEvaluator(tasks=["arc_easy"], model_class="ortgenai", model_args=["not", "a", "dict"]) def test_lm_evaluator_extract_prediction_multiple_choice(self): + # pylint: disable=protected-access from olive.evaluator.olive_evaluator import LMEvaluator # Multiple-choice: list of [loglikelihood, is_greedy] pairs -> argmax index. @@ -565,6 +566,7 @@ def test_lm_evaluator_extract_prediction_multiple_choice(self): assert LMEvaluator._extract_prediction(filtered_resps) == 1 def test_lm_evaluator_extract_prediction_generation(self): + # pylint: disable=protected-access from olive.evaluator.olive_evaluator import LMEvaluator # Generation: list of response strings returned as-is. @@ -572,6 +574,7 @@ def test_lm_evaluator_extract_prediction_generation(self): assert LMEvaluator._extract_prediction([]) == [] def test_lm_evaluator_save_lmeval_samples(self, tmp_path): + # pylint: disable=protected-access from olive.evaluator.olive_evaluator import LMEvaluator samples = [ From a2d9efbafcca1c78caf486ab8e663910c7c3d274 Mon Sep 17 00:00:00 2001 From: jiafatom Date: Tue, 14 Jul 2026 19:23:21 +0000 Subject: [PATCH 5/7] Fix CI: pylint reimport warnings and torch-version-brittle tuning test - Remove redundant function-local `import json` in test_olive_evaluator.py (json is imported at module scope), fixing pylint W0621/W0404 that failed the Python format lint job. - Relax the dynamic-shapes tuning assertion to match torch.ones' message across torch versions ("ones() received an invalid combination..." vs "ones(): argument 'size' ... must be tuple of ints"), fixing the Linux/ Windows unit test failure. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 770571fd-fbfe-46d2-8c21-67c2941eede4 --- test/evaluator/test_olive_evaluator.py | 14 -------------- test/passes/onnx/test_session_params_tuning.py | 5 ++++- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/test/evaluator/test_olive_evaluator.py b/test/evaluator/test_olive_evaluator.py index 64a7349e39..b7486fb985 100644 --- a/test/evaluator/test_olive_evaluator.py +++ b/test/evaluator/test_olive_evaluator.py @@ -712,8 +712,6 @@ class TestOnnxEvaluatorGenaiVisionDetection: def _make_model_with_genai_config(self, tmp_path, genai_config_content): """Create a mock ONNXModelHandler with a genai_config.json in its directory.""" - import json - from olive.model.handler.onnx import ONNXModelHandler model_dir = tmp_path / "model" @@ -878,8 +876,6 @@ class TestFindGenaiConfig: def test_find_genai_config_same_directory(self, tmp_path): """Find genai_config.json in the same directory as the ONNX file.""" - import json - from olive.evaluator.olive_evaluator import _find_genai_config from olive.model.handler.onnx import ONNXModelHandler @@ -898,8 +894,6 @@ def test_find_genai_config_same_directory(self, tmp_path): def test_find_genai_config_parent_directory(self, tmp_path): """Find genai_config.json one level up (nested model layout).""" - import json - from olive.evaluator.olive_evaluator import _find_genai_config from olive.model.handler.onnx import ONNXModelHandler @@ -984,8 +978,6 @@ def test_save_sample_log_disabled_when_zero(self, tmp_path): def test_save_sample_log_with_tensor_data(self, tmp_path): """Should write a JSONL file with tensor preds/targets converted to Python values.""" - import json - import torch from olive.evaluator.olive_evaluator import OliveModelOutput @@ -1011,8 +1003,6 @@ def test_save_sample_log_with_tensor_data(self, tmp_path): def test_save_sample_log_with_string_data(self, tmp_path): """Should handle string predictions and targets (text-based metrics).""" - import json - from olive.evaluator.olive_evaluator import OliveModelOutput metric = self._make_metric(sample_log_num=2, sample_log_dir=str(tmp_path), name="wer") @@ -1055,8 +1045,6 @@ def test_save_sample_log_caps_at_available_samples(self, tmp_path): def test_save_sample_log_merges_extras(self, tmp_path): """Per-sample extras (e.g. prompt and image/audio file name) should be merged into records.""" - import json - from olive.evaluator.olive_evaluator import OliveModelOutput metric = self._make_metric(sample_log_num=2, sample_log_dir=str(tmp_path), name="vision_accuracy") @@ -1087,8 +1075,6 @@ def test_save_sample_log_merges_extras(self, tmp_path): def test_save_sample_log_without_extras_is_unchanged(self, tmp_path): """When extras is None, records should only contain index/prediction/target.""" - import json - from olive.evaluator.olive_evaluator import OliveModelOutput metric = self._make_metric(sample_log_num=1, sample_log_dir=str(tmp_path), name="acc") diff --git a/test/passes/onnx/test_session_params_tuning.py b/test/passes/onnx/test_session_params_tuning.py index 4eed0653a9..82ae798782 100644 --- a/test/passes/onnx/test_session_params_tuning.py +++ b/test/passes/onnx/test_session_params_tuning.py @@ -171,4 +171,7 @@ def test_ort_session_params_tuning_pass_with_dynamic_shapes(mock_get_io_config, with pytest.raises(TypeError) as e: # execute p.run(input_model, output_folder) - assert "ones() received an invalid combination of arguments" in str(e.value) + # torch.ones rejects the dynamic (string) dimensions. The exact message differs across torch + # versions ("ones() received an invalid combination of arguments" vs. "ones(): argument 'size' + # ... must be tuple of ints, but found element of type str"), so match on the common signal. + assert "ones()" in str(e.value) From 924355db35a76779255fbf7da4d9cfa3992cd271 Mon Sep 17 00:00:00 2001 From: jiafatom Date: Tue, 14 Jul 2026 20:56:05 +0000 Subject: [PATCH 6/7] Reuse save_sample_log for lm-eval sample logging Decouple OliveEvaluator.save_sample_log to take (name, sample_log_dir) instead of a Metric, so LMEvaluator can reuse it. Replace _save_lmeval_samples with _lmeval_samples_to_output, which maps lm-eval sample records into an OliveModelOutput (question/options/prediction_index/acc carried via extras) fed to the shared writer. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 770571fd-fbfe-46d2-8c21-67c2941eede4 --- olive/evaluator/olive_evaluator.py | 107 +++++++++++++------------ test/evaluator/test_olive_evaluator.py | 15 ++-- 2 files changed, 65 insertions(+), 57 deletions(-) diff --git a/olive/evaluator/olive_evaluator.py b/olive/evaluator/olive_evaluator.py index f103a4cba0..7a3fd29dfd 100644 --- a/olive/evaluator/olive_evaluator.py +++ b/olive/evaluator/olive_evaluator.py @@ -260,14 +260,18 @@ def compute_accuracy(metric: Metric, model_outputs: Union[tuple, NamedTuple], ta return evaluate_backend_cls().measure(model_outputs, targets, metric) @staticmethod - def save_sample_log(metric: Metric, inference_output: "OliveModelOutput", targets: Any, num_samples: int) -> None: + def save_sample_log( + name: str, sample_log_dir: Optional[str], inference_output: "OliveModelOutput", targets: Any, num_samples: int + ) -> None: """Save top N sample predictions and ground truth to a JSONL file. Each line in the output file is a JSON object with 'index', 'prediction', and 'target' fields. When the inference output carries per-sample ``extras`` (e.g. the input prompt and - the vision/audio file name), those key/value pairs are merged into each record as well. - For tensor data, values are converted to Python scalars or lists. - This is best-effort: filesystem or serialization errors are logged as warnings. + the vision/audio file name, or lm-eval question/options metadata), those key/value pairs are + merged into each record as well. For tensor data, values are converted to Python scalars or + lists. The log is written to ``/_samples.jsonl`` (current working + directory when ``sample_log_dir`` is not set). This is best-effort: filesystem or + serialization errors are logged as warnings. """ if num_samples <= 0: return @@ -275,9 +279,9 @@ def save_sample_log(metric: Metric, inference_output: "OliveModelOutput", target try: preds = inference_output.preds extras = getattr(inference_output, "extras", None) - output_dir = Path(metric.sample_log_dir) if metric.sample_log_dir else Path.cwd() - # Sanitize metric name to prevent path traversal - safe_name = Path(metric.name).name.replace("/", "_").replace("\\", "_") or "metric" + output_dir = Path(sample_log_dir) if sample_log_dir else Path.cwd() + # Sanitize name to prevent path traversal + safe_name = Path(name).name.replace("/", "_").replace("\\", "_") or "metric" output_dir.mkdir(parents=True, exist_ok=True) log_path = output_dir / f"{safe_name}_samples.jsonl" @@ -311,11 +315,11 @@ def _to_serializable(val): record[key] = _to_serializable(value) record["prediction"] = _to_serializable(pred_val) record["target"] = _to_serializable(target_val) - f.write(json.dumps(record, ensure_ascii=False) + "\n") + f.write(json.dumps(record, ensure_ascii=False, default=str) + "\n") logger.info("Saved %d sample predictions to %s", n, log_path) except Exception as e: - logger.warning("Failed to save sample log for metric '%s': %s", metric.name, e) + logger.warning("Failed to save sample log for '%s': %s", name, e) @staticmethod def latency_helper(latencies) -> dict: @@ -793,7 +797,9 @@ def _evaluate_onnx_accuracy( inference_output, targets = self._inference( model, metric, dataloader, post_func, device, execution_providers ) - OliveEvaluator.save_sample_log(metric, inference_output, targets, metric.sample_log_num) + OliveEvaluator.save_sample_log( + metric.name, metric.sample_log_dir, inference_output, targets, metric.sample_log_num + ) return OliveEvaluator.compute_accuracy(metric, inference_output, targets) def _inference_text( @@ -1490,7 +1496,7 @@ def _evaluate_distributed_accuracy( targets = [x for _, t, _ in results for x in t] logits = [x for _, _, logit in results for x in logit] model_output = OliveModelOutput(preds, logits) - OliveEvaluator.save_sample_log(metric, model_output, targets, metric.sample_log_num) + OliveEvaluator.save_sample_log(metric.name, metric.sample_log_dir, model_output, targets, metric.sample_log_num) return OliveEvaluator.compute_accuracy(metric, model_output, targets) @staticmethod @@ -1694,7 +1700,9 @@ def _evaluate_accuracy( inference_output, targets = self._inference( model, metric, dataloader, post_func, device, execution_providers ) - OliveEvaluator.save_sample_log(metric, inference_output, targets, metric.sample_log_num) + OliveEvaluator.save_sample_log( + metric.name, metric.sample_log_dir, inference_output, targets, metric.sample_log_num + ) return OliveEvaluator.compute_accuracy(metric, inference_output, targets) @torch.no_grad() @@ -1929,7 +1937,9 @@ def _evaluate_accuracy( execution_providers: Optional[Union[str, list[str]]] = None, ) -> MetricResult: inference_output, targets = self._inference(model, metric, dataloader, post_func, device, execution_providers) - OliveEvaluator.save_sample_log(metric, inference_output, targets, metric.sample_log_num) + OliveEvaluator.save_sample_log( + metric.name, metric.sample_log_dir, inference_output, targets, metric.sample_log_num + ) return OliveEvaluator.compute_accuracy(metric, inference_output, targets) def _evaluate_raw_latency( @@ -2004,7 +2014,9 @@ def _evaluate_accuracy( execution_providers: Optional[Union[str, list[str]]] = None, ) -> MetricResult: inference_output, targets = self._inference(model, metric, dataloader, post_func, device, execution_providers) - OliveEvaluator.save_sample_log(metric, inference_output, targets, metric.sample_log_num) + OliveEvaluator.save_sample_log( + metric.name, metric.sample_log_dir, inference_output, targets, metric.sample_log_num + ) return OliveEvaluator.compute_accuracy(metric, inference_output, targets) def _evaluate_raw_latency( @@ -2084,43 +2096,35 @@ def _extract_prediction(filtered_resps): return filtered_resps @classmethod - def _save_lmeval_samples(cls, task_name: str, samples: list, num_samples: int, output_dir: Path) -> None: - """Save the first ``num_samples`` lm-eval sample records for ``task_name`` to a JSONL file. + def _lmeval_samples_to_output(cls, samples: list, num_samples: int) -> tuple["OliveModelOutput", list]: + """Convert lm-eval per-task sample records into an ``OliveModelOutput`` + targets. - Each line captures the question, options, resolved prediction, and target so a low score can - be inspected. Best-effort: filesystem or serialization errors are logged as warnings. + Extracts the question, options and resolved prediction from each record and packs the + per-sample metadata (``question``, ``options``, ``prediction_index``, ``acc``) into + ``extras`` so the shared :meth:`OliveEvaluator.save_sample_log` writer can persist them. """ - if num_samples <= 0 or not samples: - return - try: - output_dir.mkdir(parents=True, exist_ok=True) - safe_name = task_name.replace("/", "_").replace("\\", "_") or "task" - log_path = output_dir / f"{safe_name}_samples.jsonl" - n = min(num_samples, len(samples)) - with log_path.open("w", encoding="utf-8") as f: - for s in samples[:n]: - doc = s.get("doc") or {} - record = {"index": s.get("doc_id")} - for qk in ("question", "query", "input", "problem"): - if qk in doc: - record["question"] = doc[qk] - break - options = doc.get("options") or doc.get("choices") - if options is not None: - record["options"] = options - pred = cls._extract_prediction(s.get("filtered_resps")) - if isinstance(pred, int) and isinstance(options, list) and 0 <= pred < len(options): - record["prediction_index"] = pred - record["prediction"] = f"{chr(ord('A') + pred)}. {options[pred]}" - else: - record["prediction"] = pred - record["target"] = s.get("target") - if "acc" in s: - record["acc"] = s["acc"] - f.write(json.dumps(record, ensure_ascii=False, default=str) + "\n") - logger.info("Saved %d %s sample predictions to %s", n, task_name, log_path) - except Exception as e: - logger.warning("Failed to save lm-eval sample log for task '%s': %s", task_name, e) + n = min(num_samples, len(samples)) + preds, targets, extras = [], [], [] + for s in samples[:n]: + doc = s.get("doc") or {} + extra = {} + for qk in ("question", "query", "input", "problem"): + if qk in doc: + extra["question"] = doc[qk] + break + options = doc.get("options") or doc.get("choices") + if options is not None: + extra["options"] = options + pred = cls._extract_prediction(s.get("filtered_resps")) + if isinstance(pred, int) and isinstance(options, list) and 0 <= pred < len(options): + extra["prediction_index"] = pred + pred = f"{chr(ord('A') + pred)}. {options[pred]}" + if "acc" in s: + extra["acc"] = s["acc"] + preds.append(pred) + targets.append(s.get("target")) + extras.append(extra) + return OliveModelOutput(preds=preds, logits=None, extras=extras), targets def evaluate( self, @@ -2218,7 +2222,10 @@ def evaluate( if self.sample_log_num > 0: sample_dir = Path(self.sample_log_dir) if self.sample_log_dir else Path.cwd() / "sample_logs" for task_name, task_samples in (results.get("samples") or {}).items(): - self._save_lmeval_samples(task_name, task_samples, self.sample_log_num, sample_dir) + if not task_samples: + continue + output, targets = self._lmeval_samples_to_output(task_samples, self.sample_log_num) + OliveEvaluator.save_sample_log(task_name, sample_dir, output, targets, self.sample_log_num) for task_name in sorted(results["results"].keys()): metric_items = sorted(results["results"][task_name].items()) diff --git a/test/evaluator/test_olive_evaluator.py b/test/evaluator/test_olive_evaluator.py index b7486fb985..7617b69eaa 100644 --- a/test/evaluator/test_olive_evaluator.py +++ b/test/evaluator/test_olive_evaluator.py @@ -586,7 +586,8 @@ def test_lm_evaluator_save_lmeval_samples(self, tmp_path): "acc": 1.0, } ] - LMEvaluator._save_lmeval_samples("my_task", samples, num_samples=5, output_dir=tmp_path) + output, targets = LMEvaluator._lmeval_samples_to_output(samples, num_samples=5) + OliveEvaluator.save_sample_log("my_task", str(tmp_path), output, targets, 5) log_path = tmp_path / "my_task_samples.jsonl" assert log_path.exists() @@ -973,7 +974,7 @@ def test_save_sample_log_disabled_when_zero(self, tmp_path): output = OliveModelOutput(preds=torch.tensor([1, 2, 3]), logits=None) targets = torch.tensor([1, 2, 3]) - OliveEvaluator.save_sample_log(metric, output, targets, 0) + OliveEvaluator.save_sample_log(metric.name, metric.sample_log_dir, output, targets, 0) assert not list(tmp_path.iterdir()) def test_save_sample_log_with_tensor_data(self, tmp_path): @@ -987,7 +988,7 @@ def test_save_sample_log_with_tensor_data(self, tmp_path): targets = torch.tensor([0, 1, 0, 0, 1]) output = OliveModelOutput(preds=preds, logits=None) - OliveEvaluator.save_sample_log(metric, output, targets, 3) + OliveEvaluator.save_sample_log(metric.name, metric.sample_log_dir, output, targets, 3) log_path = tmp_path / "accuracy_samples.jsonl" assert log_path.exists() @@ -1010,7 +1011,7 @@ def test_save_sample_log_with_string_data(self, tmp_path): targets = ["hello world", "foo baz"] output = OliveModelOutput(preds=preds, logits=None) - OliveEvaluator.save_sample_log(metric, output, targets, 2) + OliveEvaluator.save_sample_log(metric.name, metric.sample_log_dir, output, targets, 2) log_path = tmp_path / "wer_samples.jsonl" assert log_path.exists() @@ -1037,7 +1038,7 @@ def test_save_sample_log_caps_at_available_samples(self, tmp_path): targets = torch.tensor([1, 0]) output = OliveModelOutput(preds=preds, logits=None) - OliveEvaluator.save_sample_log(metric, output, targets, 100) + OliveEvaluator.save_sample_log(metric.name, metric.sample_log_dir, output, targets, 100) log_path = tmp_path / "acc_samples.jsonl" lines = log_path.read_text().strip().split("\n") @@ -1056,7 +1057,7 @@ def test_save_sample_log_merges_extras(self, tmp_path): ] output = OliveModelOutput(preds=preds, logits=None, extras=extras) - OliveEvaluator.save_sample_log(metric, output, targets, 2) + OliveEvaluator.save_sample_log(metric.name, metric.sample_log_dir, output, targets, 2) log_path = tmp_path / "vision_accuracy_samples.jsonl" lines = log_path.read_text().strip().split("\n") @@ -1080,7 +1081,7 @@ def test_save_sample_log_without_extras_is_unchanged(self, tmp_path): metric = self._make_metric(sample_log_num=1, sample_log_dir=str(tmp_path), name="acc") output = OliveModelOutput(preds=["a"], logits=None) - OliveEvaluator.save_sample_log(metric, output, ["a"], 1) + OliveEvaluator.save_sample_log(metric.name, metric.sample_log_dir, output, ["a"], 1) record = json.loads((tmp_path / "acc_samples.jsonl").read_text().strip()) assert list(record.keys()) == ["index", "prediction", "target"] From b39ddb6da0d3de11d8cf5b266f18310e2c144d81 Mon Sep 17 00:00:00 2001 From: jiafatom Date: Tue, 14 Jul 2026 22:09:24 +0000 Subject: [PATCH 7/7] Add chat-template / few-shot prompting options to LMEvaluator Forward apply_chat_template, system_instruction, fewshot_as_multiturn and num_fewshot from the evaluator config to lm-eval's simple_evaluate. Instruction- tuned models (e.g. Gemma) score near-random on multiple-choice tasks when evaluated 0-shot without their chat template; these options let recipes match a model card's reported protocol. Defaults preserve lm-eval's legacy behavior, so existing configs are unaffected. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 770571fd-fbfe-46d2-8c21-67c2941eede4 --- olive/evaluator/olive_evaluator.py | 19 ++++++++++++ test/evaluator/test_olive_evaluator.py | 43 ++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/olive/evaluator/olive_evaluator.py b/olive/evaluator/olive_evaluator.py index 7a3fd29dfd..75d743602f 100644 --- a/olive/evaluator/olive_evaluator.py +++ b/olive/evaluator/olive_evaluator.py @@ -2078,6 +2078,19 @@ def __init__(self, tasks: list[str], **kwargs): # JSONL file for debugging, mirroring the accuracy evaluators' ``sample_log`` behavior. self.sample_log_num = kwargs.get("sample_log_num", 0) self.sample_log_dir = kwargs.get("sample_log_dir") + # Chat-template / prompting controls forwarded to lm-eval's ``simple_evaluate``. Instruction- + # tuned models (e.g. Gemma) score near-random on multiple-choice tasks unless the request is + # wrapped in their chat template, so allow recipes to opt in. Defaults preserve lm-eval's + # legacy behavior (no chat template, no system prompt) so existing configs are unaffected. + # ``apply_chat_template`` may be a bool or, for models with multiple templates, a template name. + self.apply_chat_template = kwargs.get("apply_chat_template", False) + self.system_instruction = kwargs.get("system_instruction") + # Render few-shot examples as separate chat turns instead of one flattened block; only takes + # effect when ``apply_chat_template`` is enabled (lm-eval rejects it otherwise). + self.fewshot_as_multiturn = kwargs.get("fewshot_as_multiturn", False) + # Number of in-context few-shot examples. ``None`` keeps each task's own default (0 for many + # tasks); set it to match a model card's reported protocol (e.g. 5-shot MMLU). + self.num_fewshot = kwargs.get("num_fewshot") @staticmethod def _extract_prediction(filtered_resps): @@ -2217,6 +2230,12 @@ def evaluate( limit=self.limit, # Forward the configured value instead of letting lm-eval silently use its default. bootstrap_iters=self.bootstrap_iters, + # Wrap requests in the model's chat template / system prompt when requested so + # instruction-tuned models are evaluated the way they are served. + apply_chat_template=self.apply_chat_template, + system_instruction=self.system_instruction, + fewshot_as_multiturn=self.fewshot_as_multiturn, + num_fewshot=self.num_fewshot, ) if self.sample_log_num > 0: diff --git a/test/evaluator/test_olive_evaluator.py b/test/evaluator/test_olive_evaluator.py index 7617b69eaa..cca73a8fd4 100644 --- a/test/evaluator/test_olive_evaluator.py +++ b/test/evaluator/test_olive_evaluator.py @@ -551,6 +551,49 @@ def test_lm_evaluator_forwards_model_args_to_backend( _, eval_kwargs = simple_evaluate_mock.call_args assert eval_kwargs["batch_size"] == 4 + @patch("lm_eval.utils.setup_logging") + @patch("lm_eval.tasks.TaskManager") + @patch("lm_eval.simple_evaluate") + @patch("lm_eval.api.registry.get_model") + def test_lm_evaluator_forwards_chat_template_args( + self, get_model_mock, simple_evaluate_mock, _task_manager_mock, _setup_logging_mock + ): + from olive.evaluator.olive_evaluator import LMEvaluator + from olive.model.handler.onnx import ONNXModelHandler + + simple_evaluate_mock.return_value = {"results": {}} + get_model_mock.return_value = MagicMock(return_value=MagicMock()) + + model = MagicMock(spec=ONNXModelHandler) + model.model_path = "/tmp/model.onnx" + + # Defaults preserve lm-eval's legacy behavior: no chat template, no system prompt. + default_eval = LMEvaluator(tasks=["arc_easy"], model_class="ortgenai", batch_size=1, max_length=128) + default_eval.evaluate(model, metrics=[], device=Device.CPU, execution_providers=["CPUExecutionProvider"]) + _, eval_kwargs = simple_evaluate_mock.call_args + assert eval_kwargs["apply_chat_template"] is False + assert eval_kwargs["system_instruction"] is None + assert eval_kwargs["fewshot_as_multiturn"] is False + assert eval_kwargs["num_fewshot"] is None + + # Explicit opt-in must be forwarded to simple_evaluate. + chat_eval = LMEvaluator( + tasks=["arc_easy"], + model_class="ortgenai", + batch_size=1, + max_length=128, + apply_chat_template=True, + system_instruction="You are a helpful assistant.", + fewshot_as_multiturn=True, + num_fewshot=5, + ) + chat_eval.evaluate(model, metrics=[], device=Device.CPU, execution_providers=["CPUExecutionProvider"]) + _, eval_kwargs = simple_evaluate_mock.call_args + assert eval_kwargs["apply_chat_template"] is True + assert eval_kwargs["system_instruction"] == "You are a helpful assistant." + assert eval_kwargs["fewshot_as_multiturn"] is True + assert eval_kwargs["num_fewshot"] == 5 + def test_lm_evaluator_rejects_non_dict_model_args(self): from olive.evaluator.olive_evaluator import LMEvaluator