diff --git a/src/eva/metrics/accuracy/faithfulness.py b/src/eva/metrics/accuracy/faithfulness.py index 250eb36b..0a9cf9c5 100644 --- a/src/eva/metrics/accuracy/faithfulness.py +++ b/src/eva/metrics/accuracy/faithfulness.py @@ -35,9 +35,7 @@ "audio perception errors — mishearing letters, numbers, names, or codes is common with spoken input. " "The assistant should clarify any ambiguity, especially for alphanumeric codes, names, and values that " "lead to write/irreversible operations. It's not needed to clarify if the tools called are simple " - "lookups, but if the lookups fail, the assistant is expected to clarify the user's intent. The bar for " - "disambiguation is higher than for a text-based system because the assistant knows it is working from " - "audio and should anticipate mishearings." + "lookups, but if the lookups fail, the assistant is expected to clarify the user's intent." ) @@ -54,7 +52,7 @@ class FaithfulnessJudgeMetric(ConversationTextJudgeMetric): """ name = "faithfulness" - version = "v0.2" + version = "v0.3" description = ( "LLM judge evaluation of whether the assistant remains faithful to information, policies, and instructions" ) @@ -77,8 +75,8 @@ def get_prompt_variables(self, context: MetricContext, transcript_text: str) -> "conversation_trace": transcript_text, "current_date_time": context.current_date_time, "user_turns_disclaimer": get_user_turns_disclaimer(context.is_audio_native), - "assistant_turns_disclaimer": get_assistant_turns_disclaimer(context.is_audio_native), - "misrepresentation_pipeline_note": get_misrepresentation_pipeline_note(context.is_audio_native), + "assistant_turns_disclaimer": get_assistant_turns_disclaimer(context.is_s2s), + "misrepresentation_pipeline_note": get_misrepresentation_pipeline_note(context.is_s2s), "disambiguation_context": disambiguation_context, } diff --git a/src/eva/metrics/base.py b/src/eva/metrics/base.py index 1f8d6ce7..68e7b4b8 100644 --- a/src/eva/metrics/base.py +++ b/src/eva/metrics/base.py @@ -147,8 +147,28 @@ def __init__( @property def is_audio_native(self) -> bool: + """True when the assistant hears user audio directly (S2S or AudioLLMs). + + Use for judge fragments about *audio perception* (e.g. disambiguation + expectations): both S2S and AUDIO_LLM take raw audio in, so both own their + mishearings. Do NOT use for fragments that reason about how assistant turns + were captured in the trace — see ``is_s2s``. + """ return self.pipeline_type in (PipelineType.S2S, PipelineType.AUDIO_LLM) + @property + def is_s2s(self) -> bool: + """True only for speech-to-speech models not AudioLLMs. + + AUDIO_LLM emits text before a separate TTS step, so its assistant turns are + the model's literal output — character-level slips there are real, not + transcription artifacts. Judge fragments that excuse token-level + discrepancies as TTS/STT artifacts (misrepresentation / information-loss + carve-outs, the assistant-turns disclaimer) must gate on this, not on + ``is_audio_native``. + """ + return self.pipeline_type == PipelineType.S2S + def to_dict(self) -> dict[str, Any]: """Convert MetricContext to a serializable dictionary.""" return {key: str(value) if isinstance(value, Path) else value for key, value in self.__dict__.items()} diff --git a/src/eva/metrics/experience/conversation_progression.py b/src/eva/metrics/experience/conversation_progression.py index f5abfa80..77412c10 100644 --- a/src/eva/metrics/experience/conversation_progression.py +++ b/src/eva/metrics/experience/conversation_progression.py @@ -32,7 +32,7 @@ class ConversationProgressionJudgeMetric(ConversationTextJudgeMetric): """ name = "conversation_progression" - version = "v0.1" + version = "v0.2" description = "LLM judge evaluation of whether the assistant moved the conversation forward productively" category = "experience" rating_scale = (1, 3) @@ -42,8 +42,8 @@ def get_prompt_variables(self, context: MetricContext, transcript_text: str) -> return { "conversation_trace": transcript_text, "user_turns_disclaimer": get_user_turns_disclaimer(context.is_audio_native), - "assistant_turns_disclaimer": get_assistant_turns_disclaimer(context.is_audio_native), - "information_loss_pipeline_note": get_information_loss_pipeline_note(context.is_audio_native), + "assistant_turns_disclaimer": get_assistant_turns_disclaimer(context.is_s2s), + "information_loss_pipeline_note": get_information_loss_pipeline_note(context.is_s2s), } def build_metric_score( diff --git a/src/eva/metrics/pipeline_prompts.py b/src/eva/metrics/pipeline_prompts.py index 083985de..b01540af 100644 --- a/src/eva/metrics/pipeline_prompts.py +++ b/src/eva/metrics/pipeline_prompts.py @@ -82,20 +82,39 @@ def get_user_turns_disclaimer(is_audio_native: bool) -> str: - """Return the user-turns disclaimer matching the pipeline type.""" + """Return the user-turns disclaimer matching how the assistant received user speech. + + Keys on audio-native: both S2S and AUDIO_LLM take raw user audio, so user turns + in the trace are *intended* text and the assistant owns any mishearing. Cascade + receives an STT transcript instead. + """ return S2S_USER_TURNS_DISCLAIMER if is_audio_native else CASCADE_USER_TURNS_DISCLAIMER -def get_assistant_turns_disclaimer(is_audio_native: bool) -> str: - """Return the assistant-turns disclaimer matching the pipeline type.""" - return S2S_ASSISTANT_TURNS_DISCLAIMER if is_audio_native else CASCADE_ASSISTANT_TURNS_DISCLAIMER +def get_assistant_turns_disclaimer(is_s2s: bool) -> str: + """Return the assistant-turns disclaimer matching how assistant turns were captured. + + Only S2S traces show STT transcriptions of the agent's audio, so only S2S gets the + "don't penalize TTS/STT artifacts" disclaimer. Cascade and AUDIO_LLM both surface + the LLM's *intended* text in the trace, so their turns are the model's literal output. + """ + return S2S_ASSISTANT_TURNS_DISCLAIMER if is_s2s else CASCADE_ASSISTANT_TURNS_DISCLAIMER + + +def get_misrepresentation_pipeline_note(is_s2s: bool) -> str: + """Return the scoping note for the misrepresenting_tool_result dimension. + S2S only: the carve-out excuses token-level slips as TTS/STT artifacts, which is + valid only when the assistant turn is an STT transcription of audio. AUDIO_LLM + emits text directly, so its character-level slips are real model output and must + not be excused. + """ + return S2S_MISREPRESENTATION_NOTE if is_s2s else "" -def get_misrepresentation_pipeline_note(is_audio_native: bool) -> str: - """Return the pipeline-specific scoping note for the misrepresenting_tool_result dimension.""" - return S2S_MISREPRESENTATION_NOTE if is_audio_native else "" +def get_information_loss_pipeline_note(is_s2s: bool) -> str: + """Return the scoping note for the information_loss dimension. -def get_information_loss_pipeline_note(is_audio_native: bool) -> str: - """Return the pipeline-specific scoping note for the information_loss dimension.""" - return S2S_INFORMATION_LOSS_NOTE if is_audio_native else "" + S2S only, for the same reason as ``get_misrepresentation_pipeline_note``. + """ + return S2S_INFORMATION_LOSS_NOTE if is_s2s else "" diff --git a/tests/fixtures/metric_signatures.json b/tests/fixtures/metric_signatures.json index 1d589929..4dc4639f 100644 --- a/tests/fixtures/metric_signatures.json +++ b/tests/fixtures/metric_signatures.json @@ -20,8 +20,8 @@ "ConversationProgressionJudgeMetric": { "name": "conversation_progression", "prompt_hash": "174cdf93b398", - "source_hash": "4e5f59c32946", - "version": "v0.1" + "source_hash": "84e43143d00d", + "version": "v0.2" }, "ConversationTimeLimitExceededMetric": { "name": "conversation_completed_on_time", @@ -38,8 +38,8 @@ "FaithfulnessJudgeMetric": { "name": "faithfulness", "prompt_hash": "060e3f1f9e7a", - "source_hash": "93a559d0a028", - "version": "v0.2" + "source_hash": "ee9b8ea530a4", + "version": "v0.3" }, "ResponseSpeedMetric": { "name": "response_speed", diff --git a/tests/unit/metrics/test_faithfulness.py b/tests/unit/metrics/test_faithfulness.py index 9eb10d91..7e8ca2fc 100644 --- a/tests/unit/metrics/test_faithfulness.py +++ b/tests/unit/metrics/test_faithfulness.py @@ -37,6 +37,24 @@ def test_get_prompt_variables_s2s(self): variables = self.metric.get_prompt_variables(ctx, "transcript") assert "speech-to-speech" in variables["user_turns_disclaimer"] assert "raw audio" in variables["disambiguation_context"] + # S2S traces show STT-of-audio for assistant turns, so the artifact carve-out applies. + assert variables["misrepresentation_pipeline_note"] != "" + assert "STT transcriptions of the agent" in variables["assistant_turns_disclaimer"] + + def test_get_prompt_variables_audio_llm(self): + # AUDIO_LLM hears user audio directly (so disambiguation is audio-native), but its + # assistant turns in the trace are the LLM's literal *text* output, not STT-of-audio. + # Character-level slips there are real, so the misrepresentation/info-loss carve-outs + # must NOT apply and the assistant-turns disclaimer must be the intended-text one. + ctx = make_metric_context(pipeline_type="audio_llm") + variables = self.metric.get_prompt_variables(ctx, "transcript") + # Audio-native perception → S2S-style user/disambiguation framing. + assert "raw audio" in variables["disambiguation_context"] + assert "speech-to-speech" in variables["user_turns_disclaimer"] + # Trace provenance is text, not audio → no free pass on character-level slips. + assert variables["misrepresentation_pipeline_note"] == "" + assert "STT transcriptions of the agent" not in variables["assistant_turns_disclaimer"] + assert "intended text" in variables["assistant_turns_disclaimer"] def test_build_metric_score(self): ctx = make_metric_context(conversation_trace=[{"role": "user"}, {"role": "assistant"}])