Skip to content

Commit d80d395

Browse files
alexander-alderman-webbalexander-alderman-webb
committed
simplify tests
1 parent a9d5e15 commit d80d395

1 file changed

Lines changed: 0 additions & 204 deletions

File tree

‎tests/integrations/langchain/test_langchain.py‎

Lines changed: 0 additions & 204 deletions
Original file line numberDiff line numberDiff line change
@@ -853,110 +853,8 @@ def test_langchain_openai_tools_agent_with_config(
853853
assert tx["contexts"]["trace"]["origin"] == "manual"
854854

855855
invoke_agent_span = next(x for x in tx["spans"] if x["op"] == "gen_ai.invoke_agent")
856-
chat_spans = list(x for x in tx["spans"] if x["op"] == "gen_ai.chat")
857-
tool_exec_span = next(x for x in tx["spans"] if x["op"] == "gen_ai.execute_tool")
858-
859-
assert len(chat_spans) == 2
860-
861-
assert invoke_agent_span["origin"] == "auto.ai.langchain"
862-
assert chat_spans[0]["origin"] == "auto.ai.langchain"
863-
assert chat_spans[1]["origin"] == "auto.ai.langchain"
864-
assert tool_exec_span["origin"] == "auto.ai.langchain"
865-
866856
assert invoke_agent_span["data"]["gen_ai.function_id"] == "my-snazzy-pipeline"
867857

868-
# We can't guarantee anything about the "shape" of the langchain execution graph
869-
assert len(list(x for x in tx["spans"] if x["op"] == "gen_ai.chat")) > 0
870-
871-
# Token usage is only available in newer versions of langchain (v0.2+)
872-
# where usage_metadata is supported on AIMessageChunk
873-
if "gen_ai.usage.input_tokens" in chat_spans[0]["data"]:
874-
assert chat_spans[0]["data"]["gen_ai.usage.input_tokens"] == 142
875-
assert chat_spans[0]["data"]["gen_ai.usage.output_tokens"] == 50
876-
assert chat_spans[0]["data"]["gen_ai.usage.total_tokens"] == 192
877-
878-
if "gen_ai.usage.input_tokens" in chat_spans[1]["data"]:
879-
assert chat_spans[1]["data"]["gen_ai.usage.input_tokens"] == 89
880-
assert chat_spans[1]["data"]["gen_ai.usage.output_tokens"] == 28
881-
assert chat_spans[1]["data"]["gen_ai.usage.total_tokens"] == 117
882-
883-
if send_default_pii and include_prompts:
884-
assert "5" in chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_TEXT]
885-
assert "word" in tool_exec_span["data"][SPANDATA.GEN_AI_TOOL_INPUT]
886-
assert 5 == int(tool_exec_span["data"][SPANDATA.GEN_AI_TOOL_OUTPUT])
887-
888-
param_id = request.node.callspec.id
889-
if "string" in param_id:
890-
assert [
891-
{
892-
"type": "text",
893-
"content": "You are very powerful assistant, but don't know current events",
894-
}
895-
] == json.loads(chat_spans[0]["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS])
896-
else:
897-
assert [
898-
{
899-
"type": "text",
900-
"content": "You are a helpful assistant.",
901-
},
902-
{
903-
"type": "text",
904-
"content": "Be concise and clear.",
905-
},
906-
] == json.loads(chat_spans[0]["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS])
907-
908-
assert "5" in chat_spans[1]["data"][SPANDATA.GEN_AI_RESPONSE_TEXT]
909-
910-
# Verify tool calls are recorded when PII is enabled
911-
assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get("data", {}), (
912-
"Tool calls should be recorded when send_default_pii=True and include_prompts=True"
913-
)
914-
tool_calls_data = chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]
915-
assert isinstance(tool_calls_data, (list, str)) # Could be serialized
916-
if isinstance(tool_calls_data, str):
917-
assert "get_word_length" in tool_calls_data
918-
elif isinstance(tool_calls_data, list) and len(tool_calls_data) > 0:
919-
# Check if tool calls contain expected function name
920-
tool_call_str = str(tool_calls_data)
921-
assert "get_word_length" in tool_call_str
922-
else:
923-
assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get("data", {})
924-
assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get("data", {})
925-
assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("data", {})
926-
assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[1].get("data", {})
927-
assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[1].get("data", {})
928-
assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[1].get("data", {})
929-
assert SPANDATA.GEN_AI_TOOL_INPUT not in tool_exec_span.get("data", {})
930-
assert SPANDATA.GEN_AI_TOOL_OUTPUT not in tool_exec_span.get("data", {})
931-
932-
# Verify tool calls are NOT recorded when PII is disabled
933-
assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[0].get(
934-
"data", {}
935-
), (
936-
f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} "
937-
f"and include_prompts={include_prompts}"
938-
)
939-
assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[1].get(
940-
"data", {}
941-
), (
942-
f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} "
943-
f"and include_prompts={include_prompts}"
944-
)
945-
946-
# Verify finish_reasons is always an array of strings
947-
assert chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [
948-
"function_call"
949-
]
950-
assert chat_spans[1]["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["stop"]
951-
952-
# Verify that available tools are always recorded regardless of PII settings
953-
for chat_span in chat_spans:
954-
tools_data = chat_span["data"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS]
955-
assert tools_data is not None, (
956-
"Available tools should always be recorded regardless of PII settings"
957-
)
958-
assert "get_word_length" in tools_data
959-
960858

961859
@pytest.mark.parametrize(
962860
"send_default_pii, include_prompts",
@@ -1258,110 +1156,8 @@ def test_langchain_openai_tools_agent_stream_with_config(
12581156
assert tx["contexts"]["trace"]["origin"] == "manual"
12591157

12601158
invoke_agent_span = next(x for x in tx["spans"] if x["op"] == "gen_ai.invoke_agent")
1261-
chat_spans = list(x for x in tx["spans"] if x["op"] == "gen_ai.chat")
1262-
tool_exec_span = next(x for x in tx["spans"] if x["op"] == "gen_ai.execute_tool")
1263-
1264-
assert len(chat_spans) == 2
1265-
1266-
assert invoke_agent_span["origin"] == "auto.ai.langchain"
1267-
assert chat_spans[0]["origin"] == "auto.ai.langchain"
1268-
assert chat_spans[1]["origin"] == "auto.ai.langchain"
1269-
assert tool_exec_span["origin"] == "auto.ai.langchain"
1270-
12711159
assert invoke_agent_span["data"]["gen_ai.function_id"] == "my-snazzy-pipeline"
12721160

1273-
# We can't guarantee anything about the "shape" of the langchain execution graph
1274-
assert len(list(x for x in tx["spans"] if x["op"] == "gen_ai.chat")) > 0
1275-
1276-
# Token usage is only available in newer versions of langchain (v0.2+)
1277-
# where usage_metadata is supported on AIMessageChunk
1278-
if "gen_ai.usage.input_tokens" in chat_spans[0]["data"]:
1279-
assert chat_spans[0]["data"]["gen_ai.usage.input_tokens"] == 142
1280-
assert chat_spans[0]["data"]["gen_ai.usage.output_tokens"] == 50
1281-
assert chat_spans[0]["data"]["gen_ai.usage.total_tokens"] == 192
1282-
1283-
if "gen_ai.usage.input_tokens" in chat_spans[1]["data"]:
1284-
assert chat_spans[1]["data"]["gen_ai.usage.input_tokens"] == 89
1285-
assert chat_spans[1]["data"]["gen_ai.usage.output_tokens"] == 28
1286-
assert chat_spans[1]["data"]["gen_ai.usage.total_tokens"] == 117
1287-
1288-
if send_default_pii and include_prompts:
1289-
assert "5" in chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_TEXT]
1290-
assert "word" in tool_exec_span["data"][SPANDATA.GEN_AI_TOOL_INPUT]
1291-
assert 5 == int(tool_exec_span["data"][SPANDATA.GEN_AI_TOOL_OUTPUT])
1292-
1293-
param_id = request.node.callspec.id
1294-
if "string" in param_id:
1295-
assert [
1296-
{
1297-
"type": "text",
1298-
"content": "You are very powerful assistant, but don't know current events",
1299-
}
1300-
] == json.loads(chat_spans[0]["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS])
1301-
else:
1302-
assert [
1303-
{
1304-
"type": "text",
1305-
"content": "You are a helpful assistant.",
1306-
},
1307-
{
1308-
"type": "text",
1309-
"content": "Be concise and clear.",
1310-
},
1311-
] == json.loads(chat_spans[0]["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS])
1312-
1313-
assert "5" in chat_spans[1]["data"][SPANDATA.GEN_AI_RESPONSE_TEXT]
1314-
1315-
# Verify tool calls are recorded when PII is enabled
1316-
assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get("data", {}), (
1317-
"Tool calls should be recorded when send_default_pii=True and include_prompts=True"
1318-
)
1319-
tool_calls_data = chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]
1320-
assert isinstance(tool_calls_data, (list, str)) # Could be serialized
1321-
if isinstance(tool_calls_data, str):
1322-
assert "get_word_length" in tool_calls_data
1323-
elif isinstance(tool_calls_data, list) and len(tool_calls_data) > 0:
1324-
# Check if tool calls contain expected function name
1325-
tool_call_str = str(tool_calls_data)
1326-
assert "get_word_length" in tool_call_str
1327-
else:
1328-
assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get("data", {})
1329-
assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get("data", {})
1330-
assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("data", {})
1331-
assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[1].get("data", {})
1332-
assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[1].get("data", {})
1333-
assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[1].get("data", {})
1334-
assert SPANDATA.GEN_AI_TOOL_INPUT not in tool_exec_span.get("data", {})
1335-
assert SPANDATA.GEN_AI_TOOL_OUTPUT not in tool_exec_span.get("data", {})
1336-
1337-
# Verify tool calls are NOT recorded when PII is disabled
1338-
assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[0].get(
1339-
"data", {}
1340-
), (
1341-
f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} "
1342-
f"and include_prompts={include_prompts}"
1343-
)
1344-
assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[1].get(
1345-
"data", {}
1346-
), (
1347-
f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} "
1348-
f"and include_prompts={include_prompts}"
1349-
)
1350-
1351-
# Verify finish_reasons is always an array of strings
1352-
assert chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [
1353-
"function_call"
1354-
]
1355-
assert chat_spans[1]["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["stop"]
1356-
1357-
# Verify that available tools are always recorded regardless of PII settings
1358-
for chat_span in chat_spans:
1359-
tools_data = chat_span["data"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS]
1360-
assert tools_data is not None, (
1361-
"Available tools should always be recorded regardless of PII settings"
1362-
)
1363-
assert "get_word_length" in tools_data
1364-
13651161

13661162
def test_langchain_error(sentry_init, capture_events):
13671163
global llm_type

0 commit comments

Comments
 (0)