Fix prompts for audiollm for misrepresenting_tool_result and information_loss

src/eva/metrics/accuracy/faithfulness.py

@@ -35,9 +35,7 @@
     "audio perception errors — mishearing letters, numbers, names, or codes is common with spoken input. "
     "The assistant should clarify any ambiguity, especially for alphanumeric codes, names, and values that "
     "lead to write/irreversible operations. It's not needed to clarify if the tools called are simple "
-    "lookups, but if the lookups fail, the assistant is expected to clarify the user's intent. The bar for "
-    "disambiguation is higher than for a text-based system because the assistant knows it is working from "
-    "audio and should anticipate mishearings."
+    "lookups, but if the lookups fail, the assistant is expected to clarify the user's intent."
 )
 
 
@@ -54,7 +52,7 @@ class FaithfulnessJudgeMetric(ConversationTextJudgeMetric):
     """
 
     name = "faithfulness"
-    version = "v0.2"
+    version = "v0.3"
     description = (
         "LLM judge evaluation of whether the assistant remains faithful to information, policies, and instructions"
     )
@@ -77,8 +75,8 @@ def get_prompt_variables(self, context: MetricContext, transcript_text: str) ->
             "conversation_trace": transcript_text,
             "current_date_time": context.current_date_time,
             "user_turns_disclaimer": get_user_turns_disclaimer(context.is_audio_native),
-            "assistant_turns_disclaimer": get_assistant_turns_disclaimer(context.is_audio_native),
-            "misrepresentation_pipeline_note": get_misrepresentation_pipeline_note(context.is_audio_native),
+            "assistant_turns_disclaimer": get_assistant_turns_disclaimer(context.is_s2s),
+            "misrepresentation_pipeline_note": get_misrepresentation_pipeline_note(context.is_s2s),
             "disambiguation_context": disambiguation_context,
         }
 

src/eva/metrics/base.py

@@ -147,8 +147,28 @@ def __init__(
 
     @property
     def is_audio_native(self) -> bool:
+        """True when the assistant hears user audio directly (S2S or AudioLLMs).
+
+        Use for judge fragments about *audio perception* (e.g. disambiguation
+        expectations): both S2S and AUDIO_LLM take raw audio in, so both own their
+        mishearings. Do NOT use for fragments that reason about how assistant turns
+        were captured in the trace — see ``is_s2s``.
+        """
         return self.pipeline_type in (PipelineType.S2S, PipelineType.AUDIO_LLM)
 
+    @property
+    def is_s2s(self) -> bool:
+        """True only for speech-to-speech models not AudioLLMs.
+
+        AUDIO_LLM emits text before a separate TTS step, so its assistant turns are
+        the model's literal output — character-level slips there are real, not
+        transcription artifacts. Judge fragments that excuse token-level
+        discrepancies as TTS/STT artifacts (misrepresentation / information-loss
+        carve-outs, the assistant-turns disclaimer) must gate on this, not on
+        ``is_audio_native``.
+        """
+        return self.pipeline_type == PipelineType.S2S
+
     def to_dict(self) -> dict[str, Any]:
         """Convert MetricContext to a serializable dictionary."""
         return {key: str(value) if isinstance(value, Path) else value for key, value in self.__dict__.items()}

src/eva/metrics/experience/conversation_progression.py

@@ -32,7 +32,7 @@ class ConversationProgressionJudgeMetric(ConversationTextJudgeMetric):
     """
 
     name = "conversation_progression"
-    version = "v0.1"
+    version = "v0.2"
     description = "LLM judge evaluation of whether the assistant moved the conversation forward productively"
     category = "experience"
     rating_scale = (1, 3)
@@ -42,8 +42,8 @@ def get_prompt_variables(self, context: MetricContext, transcript_text: str) ->
         return {
             "conversation_trace": transcript_text,
             "user_turns_disclaimer": get_user_turns_disclaimer(context.is_audio_native),
-            "assistant_turns_disclaimer": get_assistant_turns_disclaimer(context.is_audio_native),
-            "information_loss_pipeline_note": get_information_loss_pipeline_note(context.is_audio_native),
+            "assistant_turns_disclaimer": get_assistant_turns_disclaimer(context.is_s2s),
+            "information_loss_pipeline_note": get_information_loss_pipeline_note(context.is_s2s),
         }
 
     def build_metric_score(

src/eva/metrics/pipeline_prompts.py

@@ -82,20 +82,39 @@
 
 
 def get_user_turns_disclaimer(is_audio_native: bool) -> str:
-    """Return the user-turns disclaimer matching the pipeline type."""
+    """Return the user-turns disclaimer matching how the assistant received user speech.
+
+    Keys on audio-native: both S2S and AUDIO_LLM take raw user audio, so user turns
+    in the trace are *intended* text and the assistant owns any mishearing. Cascade
+    receives an STT transcript instead.
+    """
     return S2S_USER_TURNS_DISCLAIMER if is_audio_native else CASCADE_USER_TURNS_DISCLAIMER
 
 
-def get_assistant_turns_disclaimer(is_audio_native: bool) -> str:
-    """Return the assistant-turns disclaimer matching the pipeline type."""
-    return S2S_ASSISTANT_TURNS_DISCLAIMER if is_audio_native else CASCADE_ASSISTANT_TURNS_DISCLAIMER
+def get_assistant_turns_disclaimer(is_s2s: bool) -> str:
+    """Return the assistant-turns disclaimer matching how assistant turns were captured.
+
+    Only S2S traces show STT transcriptions of the agent's audio, so only S2S gets the
+    "don't penalize TTS/STT artifacts" disclaimer. Cascade and AUDIO_LLM both surface
+    the LLM's *intended* text in the trace, so their turns are the model's literal output.
+    """
+    return S2S_ASSISTANT_TURNS_DISCLAIMER if is_s2s else CASCADE_ASSISTANT_TURNS_DISCLAIMER
+
+
+def get_misrepresentation_pipeline_note(is_s2s: bool) -> str:
+    """Return the scoping note for the misrepresenting_tool_result dimension.
 
+    S2S only: the carve-out excuses token-level slips as TTS/STT artifacts, which is
+    valid only when the assistant turn is an STT transcription of audio. AUDIO_LLM
+    emits text directly, so its character-level slips are real model output and must
+    not be excused.
+    """
+    return S2S_MISREPRESENTATION_NOTE if is_s2s else ""
 
-def get_misrepresentation_pipeline_note(is_audio_native: bool) -> str:
-    """Return the pipeline-specific scoping note for the misrepresenting_tool_result dimension."""
-    return S2S_MISREPRESENTATION_NOTE if is_audio_native else ""
 
+def get_information_loss_pipeline_note(is_s2s: bool) -> str:
+    """Return the scoping note for the information_loss dimension.
 
-def get_information_loss_pipeline_note(is_audio_native: bool) -> str:
-    """Return the pipeline-specific scoping note for the information_loss dimension."""
-    return S2S_INFORMATION_LOSS_NOTE if is_audio_native else ""
+    S2S only, for the same reason as ``get_misrepresentation_pipeline_note``.
+    """
+    return S2S_INFORMATION_LOSS_NOTE if is_s2s else ""

tests/fixtures/metric_signatures.json

@@ -20,8 +20,8 @@
   "ConversationProgressionJudgeMetric": {
     "name": "conversation_progression",
     "prompt_hash": "174cdf93b398",
-    "source_hash": "4e5f59c32946",
-    "version": "v0.1"
+    "source_hash": "84e43143d00d",
+    "version": "v0.2"
   },
   "ConversationTimeLimitExceededMetric": {
     "name": "conversation_completed_on_time",
@@ -38,8 +38,8 @@
   "FaithfulnessJudgeMetric": {
     "name": "faithfulness",
     "prompt_hash": "060e3f1f9e7a",
-    "source_hash": "93a559d0a028",
-    "version": "v0.2"
+    "source_hash": "ee9b8ea530a4",
+    "version": "v0.3"
   },
   "ResponseSpeedMetric": {
     "name": "response_speed",

tests/unit/metrics/test_faithfulness.py

@@ -37,6 +37,32 @@ def test_get_prompt_variables_s2s(self):
         variables = self.metric.get_prompt_variables(ctx, "transcript")
         assert "speech-to-speech" in variables["user_turns_disclaimer"]
         assert "raw audio" in variables["disambiguation_context"]
+        # S2S traces show STT-of-audio for assistant turns, so the artifact carve-out applies.
+        assert variables["misrepresentation_pipeline_note"] != ""
+        assert "STT transcriptions of the agent" in variables["assistant_turns_disclaimer"]
+
+    def test_get_prompt_variables_audio_llm(self):
+        # AUDIO_LLM hears user audio directly (so disambiguation is audio-native), but its
+        # assistant turns in the trace are the LLM's literal *text* output, not STT-of-audio.
+        # Character-level slips there are real, so the misrepresentation/info-loss carve-outs
+        # must NOT apply and the assistant-turns disclaimer must be the intended-text one.
+        ctx = make_metric_context(pipeline_type="audio_llm")
+        variables = self.metric.get_prompt_variables(ctx, "transcript")
+        # Audio-native perception → S2S-style user/disambiguation framing.
+        assert "raw audio" in variables["disambiguation_context"]
+        assert "speech-to-speech" in variables["user_turns_disclaimer"]
+        # Trace provenance is text, not audio → no free pass on character-level slips.
+        assert variables["misrepresentation_pipeline_note"] == ""
+        assert "STT transcriptions of the agent" not in variables["assistant_turns_disclaimer"]
+        assert "intended text" in variables["assistant_turns_disclaimer"]
+
+    def test_disambiguation_context_drops_comparative_bar(self):
+        # The bare "bar is higher than a text-based system" clause was removed: it added
+        # severity pressure without an operational criterion. The concrete expectation stays.
+        ctx = make_metric_context(pipeline_type="s2s")
+        variables = self.metric.get_prompt_variables(ctx, "transcript")
+        assert "higher than for a text-based system" not in variables["disambiguation_context"]
+        assert "alphanumeric codes" in variables["disambiguation_context"]
 
     def test_build_metric_score(self):
         ctx = make_metric_context(conversation_trace=[{"role": "user"}, {"role": "assistant"}])