From f73ce6bae8219067d5d5c48f7c18c18a1856f7a0 Mon Sep 17 00:00:00 2001 From: Rick Gao Date: Wed, 29 Jul 2026 10:36:52 -0700 Subject: [PATCH 1/3] fix(weave): exclude media payloads from agent chat text --- tests/trace_server/test_genai_chat_view.py | 34 ++++++++++++++++++++++ weave/trace_server/agents/chat_view.py | 7 ++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/tests/trace_server/test_genai_chat_view.py b/tests/trace_server/test_genai_chat_view.py index e9789988ed76..4320caede74a 100644 --- a/tests/trace_server/test_genai_chat_view.py +++ b/tests/trace_server/test_genai_chat_view.py @@ -1386,6 +1386,40 @@ def test_inline_internal_ref_surfaces_without_span_content_refs() -> None: assert _assistant_payload(assistant).content_refs == [] +def test_blob_content_is_media_not_user_message_text() -> None: + """A Weave blob's `content` is base64/ref data, not displayable prose.""" + audio_internal = "weave-trace-internal:///PID/object/Content:AUDIODIGEST" + spans = [ + _span( + span_id="agent", + operation_name="invoke_agent", + agent_name="audio-inspector", + input_messages=[ + { + "role": "user", + "content": _parts( + { + "type": "blob", + "content": audio_internal, + "mimeType": "audio/mpeg", + "modality": "audio", + }, + _text_part("Inspect this audio."), + ), + } + ], + ) + ] + + messages = build_chat_messages(spans) + user = next(m for m in messages if m.type == "user_message") + + assert _user_payload(user) == AgentChatUserMessage( + text="Inspect this audio.", + content_refs=[audio_internal], + ) + + @pytest.mark.xfail( reason=( "Known limitation (PR #7489 discussion): the inline-ref sweep " diff --git a/weave/trace_server/agents/chat_view.py b/weave/trace_server/agents/chat_view.py index e66662b79c30..22b2e553ee2d 100644 --- a/weave/trace_server/agents/chat_view.py +++ b/weave/trace_server/agents/chat_view.py @@ -709,9 +709,14 @@ def _display_text(content: str) -> str: # concatenating it here would duplicate it in the message body. if p.get("type") == "reasoning": continue + # Weave blob parts carry their base64 payload (or the Content ref + # that replaces it at ingest) in `content`. That field is media, + # not prose, and must not leak into the chat bubble's text. + if p.get("type") in _MEDIA_PART_TYPES: + continue # Support both the weave parts model (``content``) and the # OpenAI-style multimodal shape (``text``). Non-text parts (e.g. - # images) carry neither and are skipped for display. + # images) are skipped for display. if isinstance(p.get("content"), str): texts.append(p["content"]) elif isinstance(p.get("text"), str): From a42d3ddc72d752f2eb902b2def8828cbdec4d98d Mon Sep 17 00:00:00 2001 From: Rick Gao Date: Wed, 29 Jul 2026 11:35:20 -0700 Subject: [PATCH 2/3] test(weave): cover image blob chat projection --- AGENTS.md | 3 +++ tests/trace_server/test_genai_chat_view.py | 16 ++++++++-------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 332176e446c9..95a0218ea6d4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -562,4 +562,7 @@ Think of this as the reverse-task assignment - a place where you can communicate - [ ] Repair the existing `pnpm run typecheck:examples` failures caused by OpenAI type drift in `examples/agent.ts`, `classesWithOps.ts`, `imageGeneration.ts`, `quickstart*.ts`, and `streamFunctionCalls.ts`. +- [ ] Exclude `.git` from the Ruff pre-push scan. A remote branch whose name + ends in `.py` currently makes `nox --no-install -e lint` parse its ref and + reflog files as Python source. - [ ] ... diff --git a/tests/trace_server/test_genai_chat_view.py b/tests/trace_server/test_genai_chat_view.py index 4320caede74a..c427d7a495ad 100644 --- a/tests/trace_server/test_genai_chat_view.py +++ b/tests/trace_server/test_genai_chat_view.py @@ -1388,23 +1388,23 @@ def test_inline_internal_ref_surfaces_without_span_content_refs() -> None: def test_blob_content_is_media_not_user_message_text() -> None: """A Weave blob's `content` is base64/ref data, not displayable prose.""" - audio_internal = "weave-trace-internal:///PID/object/Content:AUDIODIGEST" + image_internal = "weave-trace-internal:///PID/object/Content:IMAGEDIGEST" spans = [ _span( span_id="agent", operation_name="invoke_agent", - agent_name="audio-inspector", + agent_name="image-analyzer", input_messages=[ { "role": "user", "content": _parts( { "type": "blob", - "content": audio_internal, - "mimeType": "audio/mpeg", - "modality": "audio", + "content": image_internal, + "mimeType": "image/png", + "modality": "image", }, - _text_part("Inspect this audio."), + _text_part("Describe this image."), ), } ], @@ -1415,8 +1415,8 @@ def test_blob_content_is_media_not_user_message_text() -> None: user = next(m for m in messages if m.type == "user_message") assert _user_payload(user) == AgentChatUserMessage( - text="Inspect this audio.", - content_refs=[audio_internal], + text="Describe this image.", + content_refs=[image_internal], ) From 9090605147378fce32456ff326a5d1caff402628 Mon Sep 17 00:00:00 2001 From: Rick Gao Date: Thu, 30 Jul 2026 12:34:04 -0700 Subject: [PATCH 3/3] chore(weave): address review feedback --- AGENTS.md | 3 --- weave/trace_server/agents/chat_view.py | 10 ++++------ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 95a0218ea6d4..332176e446c9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -562,7 +562,4 @@ Think of this as the reverse-task assignment - a place where you can communicate - [ ] Repair the existing `pnpm run typecheck:examples` failures caused by OpenAI type drift in `examples/agent.ts`, `classesWithOps.ts`, `imageGeneration.ts`, `quickstart*.ts`, and `streamFunctionCalls.ts`. -- [ ] Exclude `.git` from the Ruff pre-push scan. A remote branch whose name - ends in `.py` currently makes `nox --no-install -e lint` parse its ref and - reflog files as Python source. - [ ] ... diff --git a/weave/trace_server/agents/chat_view.py b/weave/trace_server/agents/chat_view.py index 22b2e553ee2d..d4c2ffccd4d0 100644 --- a/weave/trace_server/agents/chat_view.py +++ b/weave/trace_server/agents/chat_view.py @@ -690,9 +690,9 @@ def _display_text(content: str) -> str: """Extract human-readable text from a message content field. Content is either plain text (legacy) or a JSON-serialized parts array - (multimodal messages). For parts arrays, concatenate the text parts; - reasoning parts are excluded (they surface separately via - `reasoning_content`). For plain text, return as-is. + (multimodal messages). For parts arrays, concatenate text parts. Reasoning + surfaces through ``reasoning_content``; media surfaces through + ``content_refs``. For plain text, return as-is. """ if not content or not content.startswith("["): return content @@ -709,9 +709,7 @@ def _display_text(content: str) -> str: # concatenating it here would duplicate it in the message body. if p.get("type") == "reasoning": continue - # Weave blob parts carry their base64 payload (or the Content ref - # that replaces it at ingest) in `content`. That field is media, - # not prose, and must not leak into the chat bubble's text. + # Media belongs in content_refs, never chat text. if p.get("type") in _MEDIA_PART_TYPES: continue # Support both the weave parts model (``content``) and the