diff --git a/README.md b/README.md index 832a2df..329ac5b 100644 --- a/README.md +++ b/README.md @@ -26,12 +26,12 @@ offline with no provider keys. | Metric | Value | |---|---| -| Reproduced issue folders | 46 | +| Reproduced issue folders | 47 | | Gateways under test | LiteLLM, NVIDIA Switchyard, Bifrost, GoModel, AxonHub, and any-llm | -| Harness tests | 124 (110 conformance checks against recorded transcripts, 14 unit) | +| Harness tests | 130 (114 conformance checks against recorded transcripts, 16 unit) | -The 46 folders cover 50 distinct defects confirmed on the wire, 49 of them on +The 47 folders cover 51 distinct defects confirmed on the wire, 50 of them on current releases at the time of capture. The full ledger, including the cited bugs that did not reproduce, is [`issues/SCOREBOARD.md`](issues/SCOREBOARD.md). One finding is filed upstream as @@ -49,7 +49,7 @@ reproduction commands. |---|---|---|---|---|---|---| | Terminal reason survives translation (`tool_use`, `content_filter`, `max_tokens`, refusal) | [001](issues/001-anthropic-stream-toolcall-translation), [002](issues/002-litellm-ollama-toolcall-loss) | [010](issues/010-switchyard-content-filter-and-reorder) | [030](issues/030-bifrost-anthropic-stream-stop-reason), [034](issues/034-bifrost-erases-content-filter), [035](issues/035-bifrost-erases-truncation), [036](issues/036-bifrost-drops-refusal-content) | | | | | Request constraints survive (`disable_parallel_tool_use`, `stop_sequences`, `tools[].strict`, `output_format`) | [017](issues/017-parallel-tool-flag-dropped), [041](issues/041-litellm-drops-stop-sequences), [064](issues/064-litellm-drops-tool-strict) | [006](issues/006-switchyard-crossformat-losses), [017](issues/017-parallel-tool-flag-dropped), [040](issues/040-switchyard-drops-output-format), [065](issues/065-switchyard-responses-instruction-loss), [066](issues/066-switchyard-drops-tool-strict) | [031](issues/031-bifrost-drops-parallel-tool-flag), [032](issues/032-bifrost-drops-stop-sequences) | [042](issues/042-gomodel-drops-output-format), [043](issues/043-gomodel-drops-parallel-tool-flag) | [051](issues/051-axonhub-drops-output-format) | [058](issues/058-any-llm-drops-parallel-tool-flag), [062](issues/062-any-llm-empty-schema-shell) | -| Content blocks survive (refusal, `is_error`, image and document blocks in tool results and user turns) | [006](issues/006-switchyard-crossformat-losses), [007](issues/007-switchyard-toolresult-multimodal-stringified), [018](issues/018-user-document-dropped), [067](issues/067-litellm-drops-refusal-content) | [006](issues/006-switchyard-crossformat-losses), [007](issues/007-switchyard-toolresult-multimodal-stringified), [018](issues/018-user-document-dropped), [068](issues/068-switchyard-drops-refusal-content) | | | | [059](issues/059-any-llm-drops-is-error), [060](issues/060-any-llm-drops-toolresult-image), [061](issues/061-any-llm-drops-toolresult-document) | +| Content blocks survive (refusal, `is_error`, image and document blocks in tool results and user turns) | [006](issues/006-switchyard-crossformat-losses), [007](issues/007-switchyard-toolresult-multimodal-stringified), [018](issues/018-user-document-dropped), [067](issues/067-litellm-drops-refusal-content) | [006](issues/006-switchyard-crossformat-losses), [007](issues/007-switchyard-toolresult-multimodal-stringified), [018](issues/018-user-document-dropped), [068](issues/068-switchyard-drops-refusal-content), [069](issues/069-switchyard-responses-refusal) | | | | [059](issues/059-any-llm-drops-is-error), [060](issues/060-any-llm-drops-toolresult-image), [061](issues/061-any-llm-drops-toolresult-document) | | Assistant history survives replay (`thinking` blocks and signatures) | [016](issues/016-thinking-history-lost) (leaked as visible text) | [016](issues/016-thinking-history-lost) (dropped) | [033](issues/033-bifrost-drops-thinking-history) | | | [057](issues/057-any-llm-drops-thinking-history) | | Tool-call ids round-trip | [004](issues/004-gemini-thought-signature) | [005](issues/005-switchyard-toolid-sanitizer) | [037](issues/037-bifrost-toolid-not-restored) | | | | | Nothing is invented (empty text blocks, phantom message items, `cache_control`) | [001](issues/001-anthropic-stream-toolcall-translation), [009](issues/009-litellm-responses-phantom-message) | [019](issues/019-switchyard-invents-prompt-cache), [045](issues/045-switchyard-empty-text-before-tooluse), [068](issues/068-switchyard-drops-refusal-content) | | | | | diff --git a/crates/harness/src/checks.rs b/crates/harness/src/checks.rs index 7755e27..c2362cb 100644 --- a/crates/harness/src/checks.rs +++ b/crates/harness/src/checks.rs @@ -838,6 +838,175 @@ pub fn refusal_text_preserved(upstream_response_json: &str, client_response_json Verdict::Conformant } +fn refusal_strings(response: &str) -> Result, String> { + if response.lines().any(|line| line.starts_with("data:")) { + let events = sse_data_json(response); + if events.is_empty() { + return Err("upstream response has no parseable SSE data events".to_string()); + } + let chat_refusal = events + .iter() + .flat_map(|event| { + event + .get("choices") + .and_then(Value::as_array) + .into_iter() + .flatten() + }) + .filter_map(|choice| choice.pointer("/delta/refusal").and_then(Value::as_str)) + .collect::(); + if !chat_refusal.is_empty() { + return Ok(vec![chat_refusal]); + } + let response_done_refusals = events + .iter() + .filter(|event| { + event.get("type").and_then(Value::as_str) == Some("response.refusal.done") + }) + .filter_map(|event| event.get("refusal").and_then(Value::as_str)) + .map(str::to_string) + .collect::>(); + if !response_done_refusals.is_empty() { + return Ok(response_done_refusals); + } + let mut refusals = Vec::new(); + for event in &events { + collect_keyed_strings(event, "refusal", &mut refusals); + } + return Ok(refusals); + } + + let body = serde_json::from_str::(response) + .map_err(|_| "upstream response body is not valid JSON".to_string())?; + let mut refusals = Vec::new(); + collect_keyed_strings(&body, "refusal", &mut refusals); + Ok(refusals) +} + +fn responses_event_key(event: &Value) -> Option<(&str, u64, u64)> { + Some(( + event.get("item_id")?.as_str()?, + event.get("output_index")?.as_u64()?, + event.get("content_index")?.as_u64()?, + )) +} + +fn refusal_part_event_matches( + event: &Value, + event_type: &str, + key: (&str, u64, u64), + expected_refusal: Option<&str>, +) -> bool { + if event.get("type").and_then(Value::as_str) != Some(event_type) + || responses_event_key(event) != Some(key) + || event.pointer("/part/type").and_then(Value::as_str) != Some("refusal") + { + return false; + } + expected_refusal.is_none_or(|expected| { + event.pointer("/part/refusal").and_then(Value::as_str) == Some(expected) + }) +} + +/// Invariant (bug 069): a structured refusal returned by an upstream dialect +/// MUST remain machine-identifiable when encoded as an OpenAI Responses result. +/// Buffered Responses use a `refusal` content part. Streams use refusal delta +/// and done events. Merely copying the explanation into `output_text` preserves +/// bytes but destroys the semantic signal used by refusal-aware consumers. +pub fn responses_refusal_semantics_preserved( + upstream_response: &str, + client_response: &str, +) -> Verdict { + let upstream_refusals = match refusal_strings(upstream_response) { + Ok(refusals) if !refusals.is_empty() => refusals, + Ok(_) => { + return Verdict::Violation( + "upstream response contains no structured refusal signal".to_string(), + ); + } + Err(message) => return Verdict::Violation(message), + }; + + if client_response + .lines() + .any(|line| line.starts_with("data:")) + { + let events = sse_data_json(client_response); + if events.is_empty() { + return Verdict::Violation( + "client response has no parseable Responses SSE data events".to_string(), + ); + } + for refusal in upstream_refusals { + let represented = events + .iter() + .filter(|event| { + event.get("type").and_then(Value::as_str) == Some("response.refusal.done") + && event.get("refusal").and_then(Value::as_str) == Some(refusal.as_str()) + }) + .filter_map(responses_event_key) + .any(|key| { + let delta_text = events + .iter() + .filter(|event| { + event.get("type").and_then(Value::as_str) + == Some("response.refusal.delta") + && responses_event_key(event) == Some(key) + }) + .filter_map(|event| event.get("delta").and_then(Value::as_str)) + .collect::(); + delta_text == refusal + && events.iter().any(|event| { + refusal_part_event_matches( + event, + "response.content_part.added", + key, + None, + ) + }) + && events.iter().any(|event| { + refusal_part_event_matches( + event, + "response.content_part.done", + key, + Some(refusal.as_str()), + ) + }) + }); + if !represented { + return Verdict::Violation(format!( + "upstream refusal {refusal:?} is not represented by one correlated Responses refusal content-part lifecycle" + )); + } + } + return Verdict::Conformant; + } + + let Ok(client) = serde_json::from_str::(client_response) else { + return Verdict::Violation("client response body is not valid JSON".to_string()); + }; + let typed_refusals = client + .get("output") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter(|item| item.get("type").and_then(Value::as_str) == Some("message")) + .filter(|item| item.get("role").and_then(Value::as_str) == Some("assistant")) + .filter_map(|item| item.get("content").and_then(Value::as_array)) + .flatten() + .filter(|part| part.get("type").and_then(Value::as_str) == Some("refusal")) + .filter_map(|part| part.get("refusal").and_then(Value::as_str)) + .collect::>(); + for refusal in upstream_refusals { + if !typed_refusals.contains(&refusal.as_str()) { + return Verdict::Violation(format!( + "upstream refusal {refusal:?} is not a typed Responses refusal content part" + )); + } + } + Verdict::Conformant +} + /// Invariant (bug 037): when a gateway rewrites an upstream tool-call id to satisfy /// a client-side charset contract, it MUST reverse the rewrite before sending the /// id back upstream. The upstream never issued the sanitized id; echoing it breaks @@ -937,6 +1106,120 @@ mod tests { ); } + #[test] + fn responses_refusal_checker_requires_typed_buffered_content() { + let upstream = r#"{"choices":[{"message":{"content":null,"refusal":"cannot help"}}]}"#; + let conformant = r#"{"output":[{"type":"message","role":"assistant","content":[{"type":"refusal","refusal":"cannot help"}]}]}"#; + assert_eq!( + responses_refusal_semantics_preserved(upstream, conformant), + Verdict::Conformant + ); + + let flattened = r#"{"output":[{"type":"message","content":[{"type":"output_text","text":"cannot help"}]}]}"#; + assert!(matches!( + responses_refusal_semantics_preserved(upstream, flattened), + Verdict::Violation(_) + )); + assert!(matches!( + responses_refusal_semantics_preserved(r#"{"choices":[]}"#, conformant), + Verdict::Violation(_) + )); + assert!(matches!( + responses_refusal_semantics_preserved(upstream, "not-json"), + Verdict::Violation(_) + )); + let wrong_output_item = r#"{"output":[{"type":"function_call","role":"assistant","content":[{"type":"refusal","refusal":"cannot help"}]}]}"#; + assert!(matches!( + responses_refusal_semantics_preserved(upstream, wrong_output_item), + Verdict::Violation(_) + )); + assert!(matches!( + responses_refusal_semantics_preserved("not-json", conformant), + Verdict::Violation(_) + )); + let two_upstream = + r#"{"choices":[{"message":{"refusal":"first"}},{"message":{"refusal":"second"}}]}"#; + let two_client = r#"{"output":[{"type":"message","role":"assistant","content":[{"type":"refusal","refusal":"first"},{"type":"refusal","refusal":"second"}]}]}"#; + assert_eq!( + responses_refusal_semantics_preserved(two_upstream, two_client), + Verdict::Conformant + ); + assert!(matches!( + responses_refusal_semantics_preserved(two_upstream, conformant), + Verdict::Violation(_) + )); + } + + #[test] + fn responses_refusal_checker_requires_stream_delta_and_done() { + let upstream = "data: {\"choices\":[{\"delta\":{\"refusal\":\"cannot \"}}]}\n\n\ + data: {\"choices\":[{\"delta\":{\"refusal\":\"help\"}}]}\n\n"; + let conformant = "event: response.content_part.added\n\ + data: {\"type\":\"response.content_part.added\",\"item_id\":\"msg_1\",\"output_index\":0,\"content_index\":0,\"part\":{\"type\":\"refusal\",\"refusal\":\"\"}}\n\n\ + event: response.refusal.delta\n\ + data: {\"type\":\"response.refusal.delta\",\"item_id\":\"msg_1\",\"output_index\":0,\"content_index\":0,\"delta\":\"cannot help\"}\n\n\ + event: response.refusal.done\n\ + data: {\"type\":\"response.refusal.done\",\"item_id\":\"msg_1\",\"output_index\":0,\"content_index\":0,\"refusal\":\"cannot help\"}\n\n\ + event: response.content_part.done\n\ + data: {\"type\":\"response.content_part.done\",\"item_id\":\"msg_1\",\"output_index\":0,\"content_index\":0,\"part\":{\"type\":\"refusal\",\"refusal\":\"cannot help\"}}\n\n"; + assert_eq!( + responses_refusal_semantics_preserved(upstream, conformant), + Verdict::Conformant + ); + + let flattened = "event: response.output_text.delta\n\ + data: {\"type\":\"response.output_text.delta\",\"delta\":\"cannot help\"}\n\n"; + assert!(matches!( + responses_refusal_semantics_preserved(upstream, flattened), + Verdict::Violation(_) + )); + let missing_done = "event: response.refusal.delta\n\ + data: {\"type\":\"response.refusal.delta\",\"delta\":\"cannot help\"}\n\n"; + assert!(matches!( + responses_refusal_semantics_preserved(upstream, missing_done), + Verdict::Violation(_) + )); + let crossed_items = conformant.replace( + "\"type\":\"response.refusal.delta\",\"item_id\":\"msg_1\"", + "\"type\":\"response.refusal.delta\",\"item_id\":\"msg_2\"", + ); + assert!(matches!( + responses_refusal_semantics_preserved(upstream, &crossed_items), + Verdict::Violation(_) + )); + assert!(matches!( + responses_refusal_semantics_preserved("data: not-json\n\n", conformant), + Verdict::Violation(_) + )); + assert!(matches!( + responses_refusal_semantics_preserved(upstream, "data: not-json\n\n"), + Verdict::Violation(_) + )); + let responses_upstream = + "data: {\"type\":\"response.refusal.delta\",\"delta\":\"cannot help\"}\n\n\ + data: {\"type\":\"response.refusal.done\",\"refusal\":\"cannot help\"}\n\n"; + assert_eq!( + responses_refusal_semantics_preserved(responses_upstream, conformant), + Verdict::Conformant + ); + let two_upstream = + "data: {\"type\":\"response.refusal.done\",\"refusal\":\"cannot help\"}\n\n\ + data: {\"type\":\"response.refusal.done\",\"refusal\":\"second refusal\"}\n\n"; + let second_lifecycle = conformant + .replace("msg_1", "msg_2") + .replace("cannot help", "second refusal") + .replace("\"output_index\":0", "\"output_index\":1"); + let two_client = format!("{conformant}{second_lifecycle}"); + assert_eq!( + responses_refusal_semantics_preserved(two_upstream, &two_client), + Verdict::Conformant + ); + assert!(matches!( + responses_refusal_semantics_preserved(two_upstream, conformant), + Verdict::Violation(_) + )); + } + #[test] fn json_schema_forwarded_needs_the_wire_field() { let present = r#"{"body":{"text":{"format":{"type":"json_schema"}}}}"#; diff --git a/crates/harness/tests/conformance.rs b/crates/harness/tests/conformance.rs index 45b525c..7638eef 100644 --- a/crates/harness/tests/conformance.rs +++ b/crates/harness/tests/conformance.rs @@ -13,8 +13,9 @@ use kairo::checks::{ no_phantom_null_output_text, non_text_block_not_json_dumped, openai_stream_finish_reason, openai_toolcall_id_charset, parallel_tool_disable_preserved, reasoning_text_order_preserved, refusal_text_preserved, response_content_not_empty, response_omits_secret, - stop_sequence_forwarded, thinking_not_leaked_as_visible_text, thinking_text_forwarded, - tool_strict_forwarded, toolcall_id_restored_upstream, truncation_preserved, upstream_bearer_is, + responses_refusal_semantics_preserved, stop_sequence_forwarded, + thinking_not_leaked_as_visible_text, thinking_text_forwarded, tool_strict_forwarded, + toolcall_id_restored_upstream, truncation_preserved, upstream_bearer_is, upstream_omits_header_value, FunctionToolFormat, Verdict, EMPTY_TEXT_ALONGSIDE_TOOL_USE, JSON_SCHEMA_ABSENT, JSON_SCHEMA_PROPERTY_ABSENT, }; @@ -1966,3 +1967,318 @@ fn switchyard_redirect_live_scoreboard_5_of_5() { assert_eq!(row["sink_has_authorization"], false); } } + +// ---- bug 069: Switchyard loses refusal typing on OpenAI Responses output ---- + +struct Issue069Case { + commit: &'static str, + scenario: &'static str, + client_path: &'static str, + streaming: bool, + first_exchange: usize, +} + +fn issue_069_case(rel: &str) -> Issue069Case { + let commit = if rel.contains("switchyard-main-") { + "7a23989cbe18f1c6c67ee03684ce76bd5901a27d" + } else if rel.contains("switchyard-pr623-") { + "2765f46972bf89a96beb5b2158b0fc56a3a72288" + } else { + panic!("unexpected issue 069 target: {rel}"); + }; + let (scenario, client_path, streaming, first_exchange) = + if rel.ends_with("responses-buffered.jsonl") { + ("responses-buffered", "/v1/responses", false, 1) + } else if rel.ends_with("responses-stream.jsonl") { + ("responses-stream", "/v1/responses", true, 6) + } else if rel.ends_with("chat-buffered-control.jsonl") { + ("chat-buffered-control", "/v1/chat/completions", false, 11) + } else if rel.ends_with("chat-stream-control.jsonl") { + ("chat-stream-control", "/v1/chat/completions", true, 16) + } else { + panic!("unexpected issue 069 scenario: {rel}"); + }; + Issue069Case { + commit, + scenario, + client_path, + streaming, + first_exchange, + } +} + +fn assert_issue_069_envelope( + rel: &str, + record: &serde_json::Value, + index: usize, + case: &Issue069Case, + binary_sha: &str, +) { + let line = index + 1; + assert_eq!( + record["target"]["repository"], + "https://github.com/NVIDIA-NeMo/Switchyard" + ); + assert_eq!(record["target"]["commit"], case.commit, "{rel} line {line}"); + assert_eq!(record["target"]["binary_version"], "0.2.0"); + assert_eq!(record["target"]["binary_sha256"], binary_sha); + assert!( + record["target"]["rustc_version"] + .as_str() + .is_some_and(|version| version.starts_with("rustc 1.96.1 ")), + "{rel} line {line} must bind the compiler version" + ); + assert_eq!( + record["target"]["configuration"], + "openai_chat backend, passthrough route, max_retries=0" + ); + assert_eq!(record["trial"], line); + assert_eq!(record["scenario"], case.scenario); + assert_eq!(record["client_request"]["method"], "POST"); + assert_eq!(record["client_request"]["path"], case.client_path); + assert_eq!(record["client_request"]["content_type"], "application/json"); + assert_eq!( + record["upstream_exchange"]["exchange_index"], + case.first_exchange + index + ); + assert_eq!(record["upstream_exchange"]["method"], "POST"); + assert_eq!(record["upstream_exchange"]["path"], "/v1/chat/completions"); + assert_eq!( + record["upstream_exchange"]["content_type"], + "application/json" + ); + assert_eq!(record["upstream_exchange"]["response_status"], 200); + assert_eq!(record["client_response"]["status"], 200); +} + +fn assert_issue_069_requests(rel: &str, record: &serde_json::Value, case: &Issue069Case) { + let client_body: serde_json::Value = + serde_json::from_str(record["client_request"]["body_raw"].as_str().unwrap()).unwrap(); + let mut expected_client = if case.client_path == "/v1/responses" { + serde_json::json!({ + "model": "main", "input": "REFUSALPROBE trigger", "max_output_tokens": 32 + }) + } else { + serde_json::json!({ + "model": "main", + "messages": [{"role": "user", "content": "REFUSALPROBE trigger"}], + "max_tokens": 32 + }) + }; + if case.streaming { + expected_client["stream"] = serde_json::json!(true); + } + assert_eq!(client_body, expected_client, "{rel} client request"); + + let upstream_body: serde_json::Value = + serde_json::from_str(record["upstream_exchange"]["body_raw"].as_str().unwrap()).unwrap(); + let mut expected_upstream = serde_json::json!({ + "model": "captured-model", + "messages": [{"role": "user", "content": "REFUSALPROBE trigger"}] + }); + let token_field = if case.client_path == "/v1/responses" { + "max_completion_tokens" + } else { + "max_tokens" + }; + expected_upstream[token_field] = serde_json::json!(32); + if case.streaming { + expected_upstream["stream"] = serde_json::json!(true); + expected_upstream["stream_options"] = serde_json::json!({"include_usage": true}); + } + assert_eq!(upstream_body, expected_upstream, "{rel} upstream request"); +} + +fn assert_issue_069_upstream_response(rel: &str, record: &serde_json::Value, case: &Issue069Case) { + let upstream_raw = record["upstream_exchange"]["response_body_raw"] + .as_str() + .unwrap(); + if case.streaming { + assert_eq!( + record["upstream_exchange"]["response_content_type"], + "text/event-stream; charset=utf-8" + ); + assert_eq!( + record["client_response"]["content_type"], + "text/event-stream" + ); + assert!(upstream_raw.ends_with("data: [DONE]\n\n")); + let events = upstream_raw + .lines() + .filter_map(|line| line.strip_prefix("data: ")) + .filter(|data| *data != "[DONE]") + .map(|data| serde_json::from_str::(data).unwrap()) + .collect::>(); + assert_eq!(events.len(), 3, "{rel} upstream SSE event count"); + assert!(events.iter().all(|event| { + event["object"] == "chat.completion.chunk" && event["model"] == "captured-model" + })); + let refusal = events + .iter() + .filter_map(|event| { + event + .pointer("/choices/0/delta/refusal") + .and_then(serde_json::Value::as_str) + }) + .collect::(); + assert_eq!(refusal, "REFUSALPROBE cannot help"); + assert_eq!(events[2]["choices"][0]["finish_reason"], "stop"); + return; + } + + assert_eq!( + record["upstream_exchange"]["response_content_type"], + "application/json" + ); + assert_eq!( + record["client_response"]["content_type"], + "application/json" + ); + let upstream: serde_json::Value = serde_json::from_str(upstream_raw).unwrap(); + assert_eq!(upstream["object"], "chat.completion"); + assert_eq!(upstream["model"], "captured-model"); + assert_eq!(upstream["choices"].as_array().unwrap().len(), 1); + assert_eq!( + upstream["choices"][0]["message"]["content"], + serde_json::Value::Null + ); + assert_eq!( + upstream["choices"][0]["message"]["refusal"], + "REFUSALPROBE cannot help" + ); + assert_eq!(upstream["choices"][0]["finish_reason"], "stop"); +} + +fn issue_069_records(rel: &str) -> Vec { + let records = fixture(rel) + .lines() + .filter(|line| !line.trim().is_empty()) + .map(|line| { + serde_json::from_str::(line).unwrap_or_else(|e| panic!("{rel}: {e}")) + }) + .collect::>(); + assert_eq!(records.len(), 5, "{rel} must remain the five-run capture"); + let case = issue_069_case(rel); + let binary_sha = records[0]["target"]["binary_sha256"] + .as_str() + .unwrap_or_else(|| panic!("{rel} target binary SHA-256")); + assert_eq!(binary_sha.len(), 64, "{rel} binary SHA-256 length"); + assert!(binary_sha.bytes().all(|byte| byte.is_ascii_hexdigit())); + for (index, record) in records.iter().enumerate() { + assert_issue_069_envelope(rel, record, index, &case, binary_sha); + assert_issue_069_requests(rel, record, &case); + assert_issue_069_upstream_response(rel, record, &case); + } + records +} + +#[test] +fn switchyard_issue_069_expected_responses_shapes_are_conformant() { + let buffered_upstream = + r#"{"choices":[{"message":{"content":null,"refusal":"REFUSALPROBE cannot help"}}]}"#; + assert_eq!( + responses_refusal_semantics_preserved( + buffered_upstream, + &fixture("transcripts/069/expected-responses-buffered.json") + ), + Verdict::Conformant + ); + + let stream_upstream = + "data: {\"choices\":[{\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"}}]}\n\n"; + assert_eq!( + responses_refusal_semantics_preserved( + stream_upstream, + &fixture("transcripts/069/expected-responses-stream.sse") + ), + Verdict::Conformant + ); +} + +#[test] +fn switchyard_main_drops_responses_refusal_semantics() { + for rel in [ + "transcripts/069/switchyard-main-responses-buffered.jsonl", + "transcripts/069/switchyard-main-responses-stream.jsonl", + ] { + let records = issue_069_records(rel); + for (index, record) in records.iter().enumerate() { + assert_eq!(record["client_request"]["path"], "/v1/responses"); + assert_eq!(record["client_response"]["status"], 200); + assert_eq!(record["consumer"]["classified_as_refusal"], false); + let upstream = record["upstream_exchange"]["response_body_raw"] + .as_str() + .unwrap_or_else(|| panic!("{rel} line {} upstream raw body", index + 1)); + let client = record["client_response"]["body_raw"] + .as_str() + .unwrap_or_else(|| panic!("{rel} line {} client raw body", index + 1)); + assert!(upstream.contains("REFUSALPROBE cannot help")); + assert!(matches!( + responses_refusal_semantics_preserved(upstream, client), + Verdict::Violation(_) + )); + } + } +} + +#[test] +fn switchyard_pr623_flattens_responses_refusal_to_output_text() { + for rel in [ + "transcripts/069/switchyard-pr623-responses-buffered.jsonl", + "transcripts/069/switchyard-pr623-responses-stream.jsonl", + ] { + let records = issue_069_records(rel); + for (index, record) in records.iter().enumerate() { + assert_eq!(record["client_request"]["path"], "/v1/responses"); + assert_eq!(record["client_response"]["status"], 200); + assert_eq!(record["consumer"]["classified_as_refusal"], false); + assert_eq!( + record["consumer"]["ordinary_output_text"], + "REFUSALPROBE cannot help" + ); + let upstream = record["upstream_exchange"]["response_body_raw"] + .as_str() + .unwrap_or_else(|| panic!("{rel} line {} upstream raw body", index + 1)); + let client = record["client_response"]["body_raw"] + .as_str() + .unwrap_or_else(|| panic!("{rel} line {} client raw body", index + 1)); + assert!(matches!( + responses_refusal_semantics_preserved(upstream, client), + Verdict::Violation(_) + )); + assert!( + !client.contains("response.refusal") && !client.contains("\"type\":\"refusal\""), + "{rel} line {} must remain semantically flattened", + index + 1 + ); + } + } +} + +#[test] +fn switchyard_chat_route_preserves_the_same_refusal_control() { + for rel in [ + "transcripts/069/switchyard-main-chat-buffered-control.jsonl", + "transcripts/069/switchyard-main-chat-stream-control.jsonl", + "transcripts/069/switchyard-pr623-chat-buffered-control.jsonl", + "transcripts/069/switchyard-pr623-chat-stream-control.jsonl", + ] { + let records = issue_069_records(rel); + for (index, record) in records.iter().enumerate() { + assert_eq!(record["client_request"]["path"], "/v1/chat/completions"); + assert_eq!(record["upstream_exchange"]["path"], "/v1/chat/completions"); + assert_eq!(record["client_response"]["status"], 200); + assert_eq!(record["consumer"]["classified_as_refusal"], true); + assert_eq!( + record["consumer"]["refusal_text"], + "REFUSALPROBE cannot help" + ); + assert_eq!( + record["upstream_exchange"]["response_body_raw"], + record["client_response"]["body_raw"], + "{rel} line {} must preserve the same raw refusal response", + index + 1 + ); + } + } +} diff --git a/issues/069-switchyard-responses-refusal/README.md b/issues/069-switchyard-responses-refusal/README.md new file mode 100644 index 0000000..121ebe3 --- /dev/null +++ b/issues/069-switchyard-responses-refusal/README.md @@ -0,0 +1,209 @@ +# 069, Switchyard loses structured refusal semantics on Responses output + +- **Upstream**: [NVIDIA-NeMo/Switchyard#622](https://github.com/NVIDIA-NeMo/Switchyard/issues/622) + is open and [pull request #623](https://github.com/NVIDIA-NeMo/Switchyard/pull/623) + is a draft as of 2026-09-05. They cover refusal-text loss on Anthropic output, + not the OpenAI Responses type loss recorded here. +- **Tool under test**: Switchyard `main` commit **`7a23989`** and pull request + #623 head **`2765f46`**, both reporting `switchyard-server` **0.2.0**. +- **Reproduced**: 2026-09-04 on macOS arm64, Rust 1.96.1, with an + `openai_chat` backend and a keyless deterministic capture upstream. + +## What breaks + +An OpenAI Responses client calls Switchyard's `/v1/responses` endpoint. The +configured OpenAI Chat backend returns the documented structured-refusal shape: + +```json +{"message":{"role":"assistant","content":null,"refusal":"REFUSALPROBE cannot help"}} +``` + +Current Switchyard `main` returns a completed Responses object with an empty +`output_text` part in buffered mode and no output item in streaming mode. Pull +request #623 preserves the words, but still returns them as ordinary +`output_text` and `response.output_text.delta` data. Neither revision emits a +Responses `refusal` content part or `response.refusal.delta` and +`response.refusal.done` events. + +This distinction is part of the Responses wire contract. A client that detects +refusals by their documented content type or stream events classifies every +translated refusal as an ordinary answer. The consumer check recorded in each +trial returned `classified_as_refusal: false` for all Responses calls and true +for the same-dialect Chat controls. + +This is separate from issue 068. Issue 068 freezes the complete loss of refusal +text on OpenAI Chat to Anthropic translation. Issue 069 freezes the loss of the +machine-readable refusal type on OpenAI Chat to OpenAI Responses translation, +including the state that remains after the issue 068 fix in pull request #623. + +## Wire evidence + +Every JSONL line is one complete trial. It contains the raw client request body, +the raw request Switchyard sent upstream, the raw upstream response, the raw +client response, status and content type, and the consumer classification. No +credential was used or recorded. + +- `transcripts/069/switchyard-main-responses-buffered.jsonl`: five current-main + buffered violations. The refusal becomes empty `output_text` 5/5. +- `transcripts/069/switchyard-main-responses-stream.jsonl`: five current-main + stream violations. Only `response.created` and `response.completed` appear + 5/5. +- `transcripts/069/switchyard-pr623-responses-buffered.jsonl`: five pull request + #623 buffered violations. The refusal text becomes ordinary `output_text` + 5/5. +- `transcripts/069/switchyard-pr623-responses-stream.jsonl`: five pull request + #623 stream violations. The refusal text becomes + `response.output_text.delta` 5/5. +- `transcripts/069/switchyard-{main,pr623}-chat-{buffered,stream}-control.jsonl`: + twenty same-process controls. The same upstream response remains byte-equal + at the client boundary and retains `message.refusal` or `delta.refusal` 20/20. +- `transcripts/069/expected-responses-buffered.json` and + `expected-responses-stream.sse`: minimal conformant Responses representations + based on the OpenAI Responses schema. + +| Target | Client path | Mode | Typed refusal detected | Result | +|---|---|---|---:|---| +| `7a23989` current main | `/v1/responses` | buffered | 0/5 | violation | +| `7a23989` current main | `/v1/responses` | streaming | 0/5 | violation | +| `2765f46` pull request #623 | `/v1/responses` | buffered | 0/5 | violation | +| `2765f46` pull request #623 | `/v1/responses` | streaming | 0/5 | violation | +| `7a23989` current main | `/v1/chat/completions` | buffered and streaming | 10/10 | control passes | +| `2765f46` pull request #623 | `/v1/chat/completions` | buffered and streaming | 10/10 | control passes | + +## Root cause (if found) + +On pull request #623, the buffered Chat decoder correctly creates +`ContentBlock::Refusal`, but `encode_responses_output` in +`crates/switchyard-translation/src/codecs/responses/buffered.rs` folds refusal +blocks into `text_from_blocks` and always writes the result as +`{"type":"output_text"}` at lines 1336 and 1355-1358. + +The streaming path loses the distinction earlier. The pull request's Chat +decoder maps `delta.refusal` into the generic `LlmResponseChunk::TextDelta` at +`codecs/openai_chat/stream.rs:137-144`. The provider-neutral stream enum has no +refusal delta variant, and the Responses encoder maps every `TextDelta` to +`response.output_text` at `codecs/responses/stream.rs:238`. + +Current `main` also omits the sibling Chat refusal during decoding, which +explains its empty output. Pull request #623 fixes that first loss but leaves the +Responses output type loss intact. + +## Test + +`responses_refusal_semantics_preserved` requires non-vacuous upstream refusal +evidence. It then checks the client dialect's protocol-level representation: + +- buffered Responses must contain a matching `type: "refusal"` content part; +- streamed Responses must reconstruct the same text from + `response.refusal.delta` and finish it with `response.refusal.done`. + +The checker intentionally rejects a byte-for-byte refusal explanation carried +only as `output_text`. It tests whether the semantic signal survives, not a +Switchyard-specific implementation detail. + +Replay the frozen evidence offline: + +```bash +cargo test -p kairo responses_refusal +cargo test -p kairo --test conformance switchyard_main_drops_responses_refusal_semantics +cargo test -p kairo --test conformance switchyard_pr623_flattens_responses_refusal_to_output_text +cargo test -p kairo --test conformance switchyard_chat_route_preserves_the_same_refusal_control +``` + +Rebuild and rerun the real server path: + +```bash +git clone --filter=blob:none https://github.com/NVIDIA-NeMo/Switchyard.git /tmp/switchyard-069 +rustup toolchain install 1.96.1 +cd /tmp/switchyard-069 +git checkout --detach 7a23989cbe18f1c6c67ee03684ce76bd5901a27d +cd /path/to/kairo +python3 transcripts/069/reproduce.py \ + --switchyard-source /tmp/switchyard-069 \ + --label main \ + --expected-commit 7a23989cbe18f1c6c67ee03684ce76bd5901a27d + +cd /tmp/switchyard-069 +git checkout --detach 2765f46972bf89a96beb5b2158b0fc56a3a72288 +cd /path/to/kairo +python3 transcripts/069/reproduce.py \ + --switchyard-source /tmp/switchyard-069 \ + --label pr623 \ + --expected-commit 2765f46972bf89a96beb5b2158b0fc56a3a72288 +``` + +The reproducer requires a clean tracked checkout, resolves the Cargo and Rust +compiler binaries for toolchain 1.96.1 through `rustup`, builds with `--locked`, +checks the exact commit and reported version, and records the binary SHA-256 and +compiler version in every trial. + +## Three-gate review + +### Gate 1: correctness + +The exact OpenAI Responses and OpenAI Chat endpoints were exercised through the +real `switchyard-server` public HTTP entry point. The model, prompt, local +upstream, and returned refusal bytes were identical. Only the client endpoint +changed. Every upstream request reached `/v1/chat/completions` and every +upstream response contained the same structured refusal. + +The input request is valid because Switchyard translated and forwarded it and +returned HTTP 200. Model nondeterminism is irrelevant because the upstream bytes +are fixed. The capture is not inventing a nonstandard provider shape: OpenAI's +Chat schema defines `message.refusal`, OpenAI's Responses schema defines typed +refusal output, and pull request #623 independently records live +`gpt-4o-2024-08-06` responses with the same `content: null` and populated +`message.refusal` shape. + +Result: **PASS**. + +### Gate 2: usefulness + +- **Affected user**: an application, evaluator, or guardrail using the OpenAI + Responses API through Switchyard with an OpenAI Chat backend. +- **Workflow**: send a Responses request, receive an upstream policy refusal, + and branch on the documented refusal content type or refusal stream events. +- **Observable consequence**: the recorded consumer sees zero typed refusals and + classifies all translated refusals as ordinary output or empty success. +- **Measured impact**: 20/20 Responses trials missed the refusal type across the + two pinned revisions and modes. All 20 Chat controls detected it. +- **Inferred impact**: refusal analytics, evaluation scoring, retry policy, and + safety handling can record false negatives whenever this exact backend shape + occurs. Real-world refusal frequency was not measured. + +Result: **PASS**. + +### Gate 3: upstream status + +Checked 2026-09-05 against Switchyard `main` `7a23989`, release 0.2.0, and pull +request #623 head `2765f46`. + +Searches covered open and closed issues and pull requests for +`message.refusal`, `delta.refusal`, `response.refusal.delta`, +`response.refusal.done`, `output_text`, `Responses refusal`, and `refusal`. +The repository code, `CHANGELOG.md`, releases, relevant commits, pull request +review comments, and official OpenAI Chat and Responses API documentation were +also checked. + +Relevant links: + +- [Switchyard issue #622](https://github.com/NVIDIA-NeMo/Switchyard/issues/622) +- [Switchyard pull request #623](https://github.com/NVIDIA-NeMo/Switchyard/pull/623) +- [Switchyard pull request #370](https://github.com/NVIDIA-NeMo/Switchyard/pull/370) +- [OpenAI Chat Completions schema](https://developers.openai.com/api/reference/cli/resources/chat/subresources/completions) +- [OpenAI Responses schema](https://developers.openai.com/api/reference/ruby/resources/beta/subresources/responses) +- [OpenAI Responses refusal stream events](https://platform.openai.com/docs/api-reference/responses-streaming/response/refusal?lang=python) + +No dedicated issue or pull request for Chat to Responses refusal typing was +found. Issue #622 and pull request #623 discuss the adjacent Chat to Anthropic +loss. Pull request #370 covers Anthropic refusal stop metadata. The exact claim +is classified **discussed upstream without a dedicated ticket**. + +Result: **PASS**. + +## Verdict + +- Correctness: **PASS** +- Usefulness: **PASS** +- Upstream status: **PASS** +- Overall: **ACCEPT** diff --git a/issues/SCOREBOARD.md b/issues/SCOREBOARD.md index f5146e1..fbe17f5 100644 --- a/issues/SCOREBOARD.md +++ b/issues/SCOREBOARD.md @@ -64,10 +64,11 @@ bytes on the stated version. Each folder has a writeup + transcripts. | 066 | Switchyard `/v1/messages` drops Anthropic `tools[].strict` while translating to OpenAI Chat; schema and name survive but the strict constraint does not | Switchyard main `27fc1ce` (keyless capture rig) | 064 family, distinct gateway and target format | ✅ capture 5/5, client HTTP 200 5/5. Same-proxy OpenAI Chat ingress preserves `function.strict: true` 5/5. | | 067 | LiteLLM `/v1/messages` erases structured refusal text while translating an OpenAI Responses refusal; client receives `content: []` with `end_turn` | LiteLLM 1.99.0 (keyless capture rig) | no matching upstream issue | ✅ capture 5/5, client HTTP 200 5/5. OpenAI Chat control preserves structured refusal 5/5. | | 068 | Switchyard `/v1/messages` erases structured refusal text from an OpenAI Chat response and invents an empty Anthropic text block | Switchyard main `9523023`, 0.2.0 (keyless capture rig) | no matching upstream issue; distinct from 036 and 045 | ✅ capture 5/5, client HTTP 200 5/5. Same-proxy OpenAI response path preserves the same refusal 5/5. | +| 069 | Switchyard `/v1/responses` loses the machine-readable refusal type from an OpenAI Chat response; current main erases it and PR #623 flattens it to `output_text` | Switchyard main `7a23989` and PR #623 `2765f46`, 0.2.0 (keyless capture rig) | discussed by Switchyard #622/#623 without a dedicated Responses ticket | ✅ buffered and stream violations 5/5 per mode on both revisions. Same-process Chat controls preserve the identical refusal 20/20. | Numbers 046-050 are reserved for unpublished GoModel round-2 findings (one bug per PR). Issues 052-056 (AxonHub round 2) land on sibling branches, not missing rows here. -**Coverage**: 46 documented issue folders covering 50 distinct defects confirmed on the wire (49 on current releases) +**Coverage**: 47 documented issue folders covering 51 distinct defects confirmed on the wire (50 on current releases) across LiteLLM, Switchyard, Bifrost, GoModel, AxonHub, and any-llm, counting 006 as its 4 independent field losses plus the LiteLLM copy of that class. LiteLLM confirmed: 001 (stop_reason, 1.82), 002a (finish_reason), 002b (route drop), 004a (id smuggle), 004b (Responses @@ -82,7 +83,7 @@ confirmed: 005 (id sanitizer), 006 (4 field losses), 007 (multimodal stringified), 016 (thinking dropped), 017 (parallel flag), 018 (document dumped), 019 (invented cache breakpoint), 023 (`api-key` and OpenAI org/project header forward), 025 (transport 502 echoes `?key=`), 027 -(`x-goog-api-key` header forward), 040 (Anthropic `output_format` dropped), 045 (empty text block before non-stream `tool_use`), 063 (307 follow keeps `x-api-key` / `x-goog-api-key`), 066 (Anthropic function-tool `strict` dropped on the OpenAI Chat hop), 068 (structured refusal text erased and an empty Anthropic text block invented). Bifrost confirmed: 030 (Anthropic streaming +(`x-goog-api-key` header forward), 040 (Anthropic `output_format` dropped), 045 (empty text block before non-stream `tool_use`), 063 (307 follow keeps `x-api-key` / `x-goog-api-key`), 066 (Anthropic function-tool `strict` dropped on the OpenAI Chat hop), 068 (structured refusal text erased and an empty Anthropic text block invented), 069 (Responses refusal type lost). Bifrost confirmed: 030 (Anthropic streaming ends a tool-call turn as `end_turn`, a regression of their own fixed #3638, caught by the bug-001 checker unchanged), 031 (parallel flag dropped), 032 (`stop_sequences` dropped), 033 (thinking history dropped), 034 (`content_filter` diff --git a/testing/069-switchyard-responses-refusal.md b/testing/069-switchyard-responses-refusal.md new file mode 100644 index 0000000..0e25cae --- /dev/null +++ b/testing/069-switchyard-responses-refusal.md @@ -0,0 +1,61 @@ +# 069 Switchyard Responses refusal semantics: Test Contract + +## Functional Behavior + +- A valid OpenAI Chat completion whose assistant message contains a non-empty + `message.refusal` must remain machine-identifiable as a refusal after + Switchyard translates it to an OpenAI Responses client response. +- A buffered `/v1/responses` result must expose the refusal as a content part + with `type: "refusal"` and the original refusal string, not as ordinary + `output_text` and not as an empty successful turn. +- A streamed `/v1/responses` result must emit `response.refusal.delta` and + `response.refusal.done` for the original refusal string, not only + `response.output_text.*` events. +- The same upstream refusal sent through Switchyard's same-dialect + `/v1/chat/completions` route must preserve `message.refusal` as the control. +- Evidence must cover current Switchyard `main` and pull request #623's pinned + head so the report distinguishes the existing erasure from the semantic loss + that remains after the proposed refusal-text fix. + +## Unit Tests + +- `responses_refusal_semantics_preserved` rejects a buffered Responses payload + with no typed refusal content. +- `responses_refusal_semantics_preserved` rejects a Responses SSE stream with no + refusal events. +- The checker accepts conformant buffered and streamed Responses fixtures and + rejects malformed or vacuous upstream evidence. + +## Integration / Functional Tests + +- Build and start the real `switchyard-server` at pinned current `main` and + drive its public `/v1/responses` and `/v1/chat/completions` endpoints against + a deterministic local OpenAI Chat capture upstream. +- Repeat buffered violation, streamed violation, and same-dialect control five + times per pinned revision and save sanitized raw request, upstream response, + and client response bytes under `transcripts/069/`. +- Demonstrate at the consumer boundary that a detector keyed to the documented + Responses refusal content type and stream events records a false negative. + +## Smoke Tests + +- The reproduction script exits successfully only after all expected trial + counts and wire shapes are observed. +- `cargo test --workspace` passes with the new transcript replay coverage. +- Formatting, clippy, and README count checks pass. + +## E2E Tests + +- The real Switchyard HTTP server receives an OpenAI Responses request, + contacts the deterministic Chat Completions backend, and returns the + translated client response. No provider credential is required because the + claim concerns gateway translation of captured provider bytes. + +## Manual / cURL Tests + +- Follow the exact build and reproduction commands documented in + `issues/069-switchyard-responses-refusal/README.md` from a clean checkout. +- Inspect each JSONL record and confirm the upstream refusal text and type, + translated Responses result, route, status, and consumer classification. +- Run the same-dialect Chat route control with the same prompt, model, backend, + and upstream response. diff --git a/transcripts/069/expected-responses-buffered.json b/transcripts/069/expected-responses-buffered.json new file mode 100644 index 0000000..808544a --- /dev/null +++ b/transcripts/069/expected-responses-buffered.json @@ -0,0 +1 @@ +{"id":"resp_refusal_069","object":"response","created_at":1788541200,"model":"captured-model","status":"completed","output":[{"type":"message","id":"msg_refusal_069","status":"completed","role":"assistant","content":[{"type":"refusal","refusal":"REFUSALPROBE cannot help"}]}]} diff --git a/transcripts/069/expected-responses-stream.sse b/transcripts/069/expected-responses-stream.sse new file mode 100644 index 0000000..d2acb38 --- /dev/null +++ b/transcripts/069/expected-responses-stream.sse @@ -0,0 +1,11 @@ +event: response.content_part.added +data: {"type":"response.content_part.added","output_index":0,"content_index":0,"item_id":"msg_refusal_069","part":{"type":"refusal","refusal":""},"sequence_number":2} + +event: response.refusal.delta +data: {"type":"response.refusal.delta","output_index":0,"content_index":0,"item_id":"msg_refusal_069","delta":"REFUSALPROBE cannot help","sequence_number":3} + +event: response.refusal.done +data: {"type":"response.refusal.done","output_index":0,"content_index":0,"item_id":"msg_refusal_069","refusal":"REFUSALPROBE cannot help","sequence_number":4} + +event: response.content_part.done +data: {"type":"response.content_part.done","output_index":0,"content_index":0,"item_id":"msg_refusal_069","part":{"type":"refusal","refusal":"REFUSALPROBE cannot help"},"sequence_number":5} diff --git a/transcripts/069/reproduce.py b/transcripts/069/reproduce.py new file mode 100644 index 0000000..606fd9c --- /dev/null +++ b/transcripts/069/reproduce.py @@ -0,0 +1,542 @@ +#!/usr/bin/env python3 +"""Reproduce Switchyard OpenAI Chat to Responses refusal type loss. + +The script runs a real switchyard-server binary against a deterministic local +OpenAI Chat upstream. It saves raw request and response bodies for five runs of +each buffered, streaming, and same-dialect control path. No provider credential +is read or required. +""" + +from __future__ import annotations + +import argparse +import hashlib +import http.client +import json +import os +import socket +import subprocess +import tempfile +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any + +MODEL = "captured-model" +REFUSAL = "REFUSALPROBE cannot help" +PROMPT = "REFUSALPROBE trigger" +MAIN_COMMIT = "7a23989cbe18f1c6c67ee03684ce76bd5901a27d" +PR623_COMMIT = "2765f46972bf89a96beb5b2158b0fc56a3a72288" +RUST_TOOLCHAIN = "1.96.1" + + +def json_bytes(value: Any) -> bytes: + return json.dumps(value, separators=(",", ":")).encode() + + +def chat_buffered() -> bytes: + return json_bytes( + { + "id": "chatcmpl-refusal-069", + "object": "chat.completion", + "created": 1788541200, + "model": MODEL, + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "refusal": REFUSAL, + "annotations": [], + }, + "logprobs": None, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 4, + "completion_tokens": 4, + "total_tokens": 8, + }, + } + ) + + +def chat_stream() -> bytes: + base = { + "id": "chatcmpl-refusal-069", + "object": "chat.completion.chunk", + "created": 1788541200, + "model": MODEL, + } + chunks = [ + {**base, "choices": [{"index": 0, "delta": {"role": "assistant", "content": None, "refusal": ""}, "finish_reason": None}]}, + {**base, "choices": [{"index": 0, "delta": {"refusal": REFUSAL}, "finish_reason": None}]}, + {**base, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + ] + return ("\n\n".join(f"data: {json.dumps(chunk, separators=(',', ':'))}" for chunk in chunks) + "\n\ndata: [DONE]\n\n").encode() + + +class CaptureServer(ThreadingHTTPServer): + def __init__(self) -> None: + super().__init__(("127.0.0.1", 0), CaptureHandler) + self.daemon_threads = True + self.exchanges: list[dict[str, Any]] = [] + self.lock = threading.Lock() + + def add_exchange(self, exchange: dict[str, Any]) -> None: + with self.lock: + exchange["exchange_index"] = len(self.exchanges) + 1 + self.exchanges.append(exchange) + + +class CaptureHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *_args: Any) -> None: + return + + def do_POST(self) -> None: + server: CaptureServer = self.server # type: ignore[assignment] + request_raw = self.rfile.read(int(self.headers.get("content-length", "0"))) + request = json.loads(request_raw) + streaming = bool(request.get("stream")) + valid_path = self.path == "/v1/chat/completions" + response_status = 200 if valid_path else 404 + response_raw = ( + chat_stream() + if valid_path and streaming + else chat_buffered() + if valid_path + else json_bytes({"error": "unexpected upstream path"}) + ) + server.add_exchange( + { + "method": "POST", + "path": self.path, + "content_type": self.headers.get("content-type"), + "body_raw": request_raw.decode(), + "response_status": response_status, + "response_content_type": ( + "text/event-stream; charset=utf-8" + if valid_path and streaming + else "application/json" + ), + "response_body_raw": response_raw.decode(), + } + ) + self.send_response(response_status) + self.send_header( + "content-type", + "text/event-stream; charset=utf-8" + if valid_path and streaming + else "application/json", + ) + if valid_path and streaming: + self.send_header("connection", "close") + else: + self.send_header("content-length", str(len(response_raw))) + self.end_headers() + self.wfile.write(response_raw) + self.wfile.flush() + if valid_path and streaming: + self.close_connection = True + + +def free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def wait_for_port(port: int, timeout_seconds: float = 30.0) -> None: + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + with socket.socket() as sock: + sock.settimeout(0.2) + if sock.connect_ex(("127.0.0.1", port)) == 0: + return + time.sleep(0.05) + raise RuntimeError(f"Switchyard did not listen on port {port}") + + +def sse_events(raw: str) -> list[dict[str, Any]]: + events = [] + for line in raw.splitlines(): + if not line.startswith("data:"): + continue + data = line.removeprefix("data:").strip() + if not data or data == "[DONE]": + continue + events.append(json.loads(data)) + return events + + +def responses_consumer(raw: str, streaming: bool) -> dict[str, Any]: + if streaming: + events = sse_events(raw) + event_types = [event.get("type") for event in events] + refusal = "".join( + event.get("delta", "") + for event in events + if event.get("type") == "response.refusal.delta" + ) + done_refusals = [ + event.get("refusal") + for event in events + if event.get("type") == "response.refusal.done" + ] + output_text = "".join( + event.get("delta", "") + for event in events + if event.get("type") == "response.output_text.delta" + ) + return { + "classified_as_refusal": bool(refusal) and REFUSAL in done_refusals, + "refusal_text": refusal, + "ordinary_output_text": output_text, + "event_types": event_types, + } + + body = json.loads(raw) + parts = [ + part + for item in body.get("output", []) + if item.get("type") == "message" + for part in item.get("content", []) + ] + typed = [part.get("refusal") for part in parts if part.get("type") == "refusal"] + ordinary = [part.get("text") for part in parts if part.get("type") == "output_text"] + return { + "classified_as_refusal": REFUSAL in typed, + "refusal_text": "".join(value for value in typed if isinstance(value, str)), + "ordinary_output_text": "".join(value for value in ordinary if isinstance(value, str)), + "content_types": [part.get("type") for part in parts], + } + + +def chat_consumer(raw: str, streaming: bool) -> dict[str, Any]: + if streaming: + events = sse_events(raw) + refusal = "".join( + choice.get("delta", {}).get("refusal", "") + for event in events + for choice in event.get("choices", []) + ) + return { + "classified_as_refusal": refusal == REFUSAL, + "refusal_text": refusal, + } + body = json.loads(raw) + refusal = body["choices"][0]["message"].get("refusal") + return { + "classified_as_refusal": refusal == REFUSAL, + "refusal_text": refusal, + } + + +def post(port: int, path: str, payload: dict[str, Any]) -> tuple[int, str, str]: + request_raw = json.dumps(payload, separators=(",", ":")) + connection = http.client.HTTPConnection("127.0.0.1", port, timeout=30) + try: + connection.request( + "POST", + path, + body=request_raw, + headers={"content-type": "application/json"}, + ) + response = connection.getresponse() + return response.status, response.getheader("content-type", ""), response.read().decode() + finally: + connection.close() + + +def request_for(path: str, streaming: bool) -> dict[str, Any]: + if path == "/v1/responses": + request: dict[str, Any] = { + "model": "main", + "input": PROMPT, + "max_output_tokens": 32, + } + else: + request = { + "model": "main", + "messages": [{"role": "user", "content": PROMPT}], + "max_tokens": 32, + } + if streaming: + request["stream"] = True + return request + + +def command_output(*args: str, cwd: Path | None = None) -> str: + return subprocess.check_output(args, cwd=cwd, text=True).strip() + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def build_pinned_server(source: Path, expected_commit: str) -> tuple[Path, dict[str, str]]: + source = source.resolve() + actual_commit = command_output("git", "rev-parse", "HEAD", cwd=source) + if actual_commit != expected_commit: + raise SystemExit( + f"source commit {actual_commit} does not match --expected-commit {expected_commit}" + ) + dirty = command_output( + "git", "status", "--porcelain", "--untracked-files=no", cwd=source + ) + if dirty: + raise SystemExit("Switchyard source has tracked modifications; use a clean checkout") + cargo = command_output("rustup", "which", "--toolchain", RUST_TOOLCHAIN, "cargo") + rustc = command_output("rustup", "which", "--toolchain", RUST_TOOLCHAIN, "rustc") + build_environment = os.environ.copy() + build_environment.update({"CARGO": cargo, "RUSTC": rustc}) + subprocess.run( + [ + cargo, + "build", + "--release", + "--locked", + "-p", + "switchyard-server", + ], + cwd=source, + env=build_environment, + check=True, + ) + binary = source / "target/release/switchyard-server" + version = command_output(str(binary), "--version") + if version != "switchyard-server 0.2.0": + raise SystemExit(f"unexpected binary version: {version}") + return binary, { + "commit": actual_commit, + "binary_version": version.removeprefix("switchyard-server "), + "binary_sha256": sha256(binary), + "rustc_version": command_output(rustc, "--version"), + } + + +def assert_upstream_refusal( + exchange: dict[str, Any], streaming: bool, client_path: str +) -> None: + if exchange["path"] != "/v1/chat/completions": + raise AssertionError(f"unexpected upstream path: {exchange['path']}") + if exchange["content_type"] != "application/json": + raise AssertionError(f"unexpected upstream content type: {exchange['content_type']}") + expected_request: dict[str, Any] = { + "model": MODEL, + "messages": [{"role": "user", "content": PROMPT}], + "max_completion_tokens" if client_path == "/v1/responses" else "max_tokens": 32, + } + if streaming: + expected_request.update( + {"stream": True, "stream_options": {"include_usage": True}} + ) + if json.loads(exchange["body_raw"]) != expected_request: + raise AssertionError("upstream request is not the exact expected translation") + if exchange["response_status"] != 200: + raise AssertionError("capture upstream did not return HTTP 200") + + raw = exchange["response_body_raw"] + if streaming: + if exchange["response_content_type"] != "text/event-stream; charset=utf-8": + raise AssertionError("upstream SSE content type changed") + if raw != chat_stream().decode(): + raise AssertionError("upstream SSE is not the exact canned structured refusal") + return + + if exchange["response_content_type"] != "application/json": + raise AssertionError("upstream JSON content type changed") + if raw != chat_buffered().decode(): + raise AssertionError("upstream JSON is not the exact canned structured refusal") + + +def assert_revision_result( + commit: str, + path: str, + streaming: bool, + exchange: dict[str, Any], + client_raw: str, + consumer: dict[str, Any], +) -> None: + if path == "/v1/chat/completions": + if client_raw != exchange["response_body_raw"]: + raise AssertionError("Chat control is not byte-equal to the upstream response") + if not consumer["classified_as_refusal"]: + raise AssertionError("Chat control did not preserve the structured refusal") + return + + if consumer["classified_as_refusal"]: + raise AssertionError("typed Responses refusal unexpectedly survived") + if "response.refusal" in client_raw or '"type":"refusal"' in client_raw: + raise AssertionError("Responses result unexpectedly contains a typed refusal") + if commit == MAIN_COMMIT: + if consumer["ordinary_output_text"]: + raise AssertionError("current main unexpectedly preserved refusal text") + expected_types = ["response.created", "response.completed"] + if streaming and consumer["event_types"] != expected_types: + raise AssertionError("current-main stream shape changed") + if not streaming and consumer["content_types"] != ["output_text"]: + raise AssertionError("current-main buffered shape changed") + elif commit == PR623_COMMIT: + if consumer["ordinary_output_text"] != REFUSAL: + raise AssertionError("PR 623 did not flatten the refusal to output_text") + if streaming and "response.output_text.delta" not in consumer["event_types"]: + raise AssertionError("PR 623 stream has no output_text delta") + if not streaming and consumer["content_types"] != ["output_text"]: + raise AssertionError("PR 623 buffered shape changed") + else: + raise AssertionError(f"no frozen expectation for commit {commit}") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--switchyard-source", type=Path, required=True) + parser.add_argument("--label", required=True) + parser.add_argument("--expected-commit", required=True) + parser.add_argument("--output-dir", type=Path, default=Path(__file__).parent) + parser.add_argument("--runs", type=int, default=5) + args = parser.parse_args() + + expected_labels = {MAIN_COMMIT: "main", PR623_COMMIT: "pr623"} + if expected_labels.get(args.expected_commit) != args.label: + raise SystemExit("--label must match the frozen --expected-commit") + args.output_dir.mkdir(parents=True, exist_ok=True) + binary, target = build_pinned_server(args.switchyard_source, args.expected_commit) + + upstream = CaptureServer() + upstream_port = int(upstream.server_address[1]) + threading.Thread(target=upstream.serve_forever, daemon=True).start() + switchyard_port = free_port() + + with tempfile.TemporaryDirectory(prefix="kairo-069-") as temp_dir: + config = Path(temp_dir) / "switchyard.toml" + config.write_text( + "schema_version = 1\n" + "[llm_clients.openai]\n" + "format = \"openai_chat\"\n" + f"base_url = \"http://127.0.0.1:{upstream_port}/v1\"\n" + "max_retries = 0\n" + "[targets.gpt]\n" + f"id = \"{MODEL}\"\n" + "llm_client = \"openai\"\n" + "[routes.main]\n" + "id = \"main\"\n" + "type = \"passthrough\"\n" + "target = \"gpt\"\n" + ) + process = subprocess.Popen( + [ + str(binary), + "--config", + str(config), + "--host", + "127.0.0.1", + "--port", + str(switchyard_port), + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + ) + try: + wait_for_port(switchyard_port) + scenarios = [ + ("responses-buffered", "/v1/responses", False), + ("responses-stream", "/v1/responses", True), + ("chat-buffered-control", "/v1/chat/completions", False), + ("chat-stream-control", "/v1/chat/completions", True), + ] + summaries: dict[str, dict[str, int]] = {} + for name, path, streaming in scenarios: + output = args.output_dir / f"switchyard-{args.label}-{name}.jsonl" + records = [] + for trial in range(1, args.runs + 1): + before = len(upstream.exchanges) + request = request_for(path, streaming) + status, content_type, client_raw = post(switchyard_port, path, request) + if len(upstream.exchanges) != before + 1: + raise AssertionError("expected exactly one upstream exchange") + exchange = upstream.exchanges[-1] + assert_upstream_refusal(exchange, streaming, path) + consumer = ( + responses_consumer(client_raw, streaming) + if path == "/v1/responses" + else chat_consumer(client_raw, streaming) + ) + record = { + "target": { + "repository": "https://github.com/NVIDIA-NeMo/Switchyard", + **target, + "configuration": "openai_chat backend, passthrough route, max_retries=0", + }, + "trial": trial, + "scenario": name, + "client_request": { + "method": "POST", + "path": path, + "content_type": "application/json", + "body_raw": json.dumps(request, separators=(",", ":")), + }, + "upstream_exchange": exchange, + "client_response": { + "status": status, + "content_type": content_type, + "body_raw": client_raw, + }, + "consumer": consumer, + } + if status != 200: + raise AssertionError(f"{name} trial {trial}: HTTP {status}") + assert_revision_result( + target["commit"], path, streaming, exchange, client_raw, consumer + ) + records.append(record) + + output.write_text("".join(json.dumps(record) + "\n" for record in records)) + classified = sum( + 1 for record in records if record["consumer"]["classified_as_refusal"] + ) + summaries[name] = { + "runs": args.runs, + "typed_refusal_detected": classified, + "typed_refusal_missed": args.runs - classified, + } + + summary_path = args.output_dir / f"switchyard-{args.label}-summary.json" + summary_path.write_text( + json.dumps( + { + "target_commit": target["commit"], + "binary_sha256": target["binary_sha256"], + "label": args.label, + "model": MODEL, + "results": summaries, + }, + indent=2, + ) + + "\n" + ) + print(summary_path.read_text(), end="") + finally: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + upstream.shutdown() + if process.returncode not in (None, 0, -15): + stderr = process.stderr.read() if process.stderr else "" + raise RuntimeError(f"switchyard-server exited {process.returncode}: {stderr}") + + +if __name__ == "__main__": + main() diff --git a/transcripts/069/switchyard-main-chat-buffered-control.jsonl b/transcripts/069/switchyard-main-chat-buffered-control.jsonl new file mode 100644 index 0000000..b366294 --- /dev/null +++ b/transcripts/069/switchyard-main-chat-buffered-control.jsonl @@ -0,0 +1,5 @@ +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "7a23989cbe18f1c6c67ee03684ce76bd5901a27d", "binary_version": "0.2.0", "binary_sha256": "cdf1ff53f411b5bd5a6dadbcd85e4883e986d310f2a94ee26693b4911eff0e8e", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 1, "scenario": "chat-buffered-control", "client_request": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32}", "response_status": 200, "response_content_type": "application/json", "response_body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}", "exchange_index": 11}, "client_response": {"status": 200, "content_type": "application/json", "body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}"}, "consumer": {"classified_as_refusal": true, "refusal_text": "REFUSALPROBE cannot help"}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "7a23989cbe18f1c6c67ee03684ce76bd5901a27d", "binary_version": "0.2.0", "binary_sha256": "cdf1ff53f411b5bd5a6dadbcd85e4883e986d310f2a94ee26693b4911eff0e8e", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 2, "scenario": "chat-buffered-control", "client_request": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32}", "response_status": 200, "response_content_type": "application/json", "response_body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}", "exchange_index": 12}, "client_response": {"status": 200, "content_type": "application/json", "body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}"}, "consumer": {"classified_as_refusal": true, "refusal_text": "REFUSALPROBE cannot help"}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "7a23989cbe18f1c6c67ee03684ce76bd5901a27d", "binary_version": "0.2.0", "binary_sha256": "cdf1ff53f411b5bd5a6dadbcd85e4883e986d310f2a94ee26693b4911eff0e8e", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 3, "scenario": "chat-buffered-control", "client_request": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32}", "response_status": 200, "response_content_type": "application/json", "response_body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}", "exchange_index": 13}, "client_response": {"status": 200, "content_type": "application/json", "body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}"}, "consumer": {"classified_as_refusal": true, "refusal_text": "REFUSALPROBE cannot help"}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "7a23989cbe18f1c6c67ee03684ce76bd5901a27d", "binary_version": "0.2.0", "binary_sha256": "cdf1ff53f411b5bd5a6dadbcd85e4883e986d310f2a94ee26693b4911eff0e8e", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 4, "scenario": "chat-buffered-control", "client_request": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32}", "response_status": 200, "response_content_type": "application/json", "response_body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}", "exchange_index": 14}, "client_response": {"status": 200, "content_type": "application/json", "body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}"}, "consumer": {"classified_as_refusal": true, "refusal_text": "REFUSALPROBE cannot help"}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "7a23989cbe18f1c6c67ee03684ce76bd5901a27d", "binary_version": "0.2.0", "binary_sha256": "cdf1ff53f411b5bd5a6dadbcd85e4883e986d310f2a94ee26693b4911eff0e8e", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 5, "scenario": "chat-buffered-control", "client_request": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32}", "response_status": 200, "response_content_type": "application/json", "response_body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}", "exchange_index": 15}, "client_response": {"status": 200, "content_type": "application/json", "body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}"}, "consumer": {"classified_as_refusal": true, "refusal_text": "REFUSALPROBE cannot help"}} diff --git a/transcripts/069/switchyard-main-chat-stream-control.jsonl b/transcripts/069/switchyard-main-chat-stream-control.jsonl new file mode 100644 index 0000000..0aee0ae --- /dev/null +++ b/transcripts/069/switchyard-main-chat-stream-control.jsonl @@ -0,0 +1,5 @@ +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "7a23989cbe18f1c6c67ee03684ce76bd5901a27d", "binary_version": "0.2.0", "binary_sha256": "cdf1ff53f411b5bd5a6dadbcd85e4883e986d310f2a94ee26693b4911eff0e8e", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 1, "scenario": "chat-stream-control", "client_request": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32,\"stream\":true}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32,\"stream\":true,\"stream_options\":{\"include_usage\":true}}", "response_status": 200, "response_content_type": "text/event-stream; charset=utf-8", "response_body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "exchange_index": 16}, "client_response": {"status": 200, "content_type": "text/event-stream", "body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n"}, "consumer": {"classified_as_refusal": true, "refusal_text": "REFUSALPROBE cannot help"}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "7a23989cbe18f1c6c67ee03684ce76bd5901a27d", "binary_version": "0.2.0", "binary_sha256": "cdf1ff53f411b5bd5a6dadbcd85e4883e986d310f2a94ee26693b4911eff0e8e", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 2, "scenario": "chat-stream-control", "client_request": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32,\"stream\":true}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32,\"stream\":true,\"stream_options\":{\"include_usage\":true}}", "response_status": 200, "response_content_type": "text/event-stream; charset=utf-8", "response_body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "exchange_index": 17}, "client_response": {"status": 200, "content_type": "text/event-stream", "body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n"}, "consumer": {"classified_as_refusal": true, "refusal_text": "REFUSALPROBE cannot help"}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "7a23989cbe18f1c6c67ee03684ce76bd5901a27d", "binary_version": "0.2.0", "binary_sha256": "cdf1ff53f411b5bd5a6dadbcd85e4883e986d310f2a94ee26693b4911eff0e8e", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 3, "scenario": "chat-stream-control", "client_request": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32,\"stream\":true}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32,\"stream\":true,\"stream_options\":{\"include_usage\":true}}", "response_status": 200, "response_content_type": "text/event-stream; charset=utf-8", "response_body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "exchange_index": 18}, "client_response": {"status": 200, "content_type": "text/event-stream", "body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n"}, "consumer": {"classified_as_refusal": true, "refusal_text": "REFUSALPROBE cannot help"}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "7a23989cbe18f1c6c67ee03684ce76bd5901a27d", "binary_version": "0.2.0", "binary_sha256": "cdf1ff53f411b5bd5a6dadbcd85e4883e986d310f2a94ee26693b4911eff0e8e", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 4, "scenario": "chat-stream-control", "client_request": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32,\"stream\":true}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32,\"stream\":true,\"stream_options\":{\"include_usage\":true}}", "response_status": 200, "response_content_type": "text/event-stream; charset=utf-8", "response_body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "exchange_index": 19}, "client_response": {"status": 200, "content_type": "text/event-stream", "body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n"}, "consumer": {"classified_as_refusal": true, "refusal_text": "REFUSALPROBE cannot help"}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "7a23989cbe18f1c6c67ee03684ce76bd5901a27d", "binary_version": "0.2.0", "binary_sha256": "cdf1ff53f411b5bd5a6dadbcd85e4883e986d310f2a94ee26693b4911eff0e8e", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 5, "scenario": "chat-stream-control", "client_request": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32,\"stream\":true}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32,\"stream\":true,\"stream_options\":{\"include_usage\":true}}", "response_status": 200, "response_content_type": "text/event-stream; charset=utf-8", "response_body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "exchange_index": 20}, "client_response": {"status": 200, "content_type": "text/event-stream", "body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n"}, "consumer": {"classified_as_refusal": true, "refusal_text": "REFUSALPROBE cannot help"}} diff --git a/transcripts/069/switchyard-main-responses-buffered.jsonl b/transcripts/069/switchyard-main-responses-buffered.jsonl new file mode 100644 index 0000000..c215c9b --- /dev/null +++ b/transcripts/069/switchyard-main-responses-buffered.jsonl @@ -0,0 +1,5 @@ +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "7a23989cbe18f1c6c67ee03684ce76bd5901a27d", "binary_version": "0.2.0", "binary_sha256": "cdf1ff53f411b5bd5a6dadbcd85e4883e986d310f2a94ee26693b4911eff0e8e", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 1, "scenario": "responses-buffered", "client_request": {"method": "POST", "path": "/v1/responses", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"input\":\"REFUSALPROBE trigger\",\"max_output_tokens\":32}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_completion_tokens\":32}", "response_status": 200, "response_content_type": "application/json", "response_body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}", "exchange_index": 1}, "client_response": {"status": 200, "content_type": "application/json", "body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"model\":\"captured-model\",\"status\":\"completed\",\"incomplete_details\":null,\"output\":[{\"type\":\"message\",\"id\":\"msg_switchyard\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"\",\"annotations\":[]}]}],\"usage\":{\"input_tokens\":4,\"output_tokens\":4,\"total_tokens\":8,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}},\"parallel_tool_calls\":true,\"tool_choice\":\"auto\",\"tools\":[]}"}, "consumer": {"classified_as_refusal": false, "refusal_text": "", "ordinary_output_text": "", "content_types": ["output_text"]}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "7a23989cbe18f1c6c67ee03684ce76bd5901a27d", "binary_version": "0.2.0", "binary_sha256": "cdf1ff53f411b5bd5a6dadbcd85e4883e986d310f2a94ee26693b4911eff0e8e", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 2, "scenario": "responses-buffered", "client_request": {"method": "POST", "path": "/v1/responses", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"input\":\"REFUSALPROBE trigger\",\"max_output_tokens\":32}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_completion_tokens\":32}", "response_status": 200, "response_content_type": "application/json", "response_body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}", "exchange_index": 2}, "client_response": {"status": 200, "content_type": "application/json", "body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"model\":\"captured-model\",\"status\":\"completed\",\"incomplete_details\":null,\"output\":[{\"type\":\"message\",\"id\":\"msg_switchyard\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"\",\"annotations\":[]}]}],\"usage\":{\"input_tokens\":4,\"output_tokens\":4,\"total_tokens\":8,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}},\"parallel_tool_calls\":true,\"tool_choice\":\"auto\",\"tools\":[]}"}, "consumer": {"classified_as_refusal": false, "refusal_text": "", "ordinary_output_text": "", "content_types": ["output_text"]}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "7a23989cbe18f1c6c67ee03684ce76bd5901a27d", "binary_version": "0.2.0", "binary_sha256": "cdf1ff53f411b5bd5a6dadbcd85e4883e986d310f2a94ee26693b4911eff0e8e", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 3, "scenario": "responses-buffered", "client_request": {"method": "POST", "path": "/v1/responses", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"input\":\"REFUSALPROBE trigger\",\"max_output_tokens\":32}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_completion_tokens\":32}", "response_status": 200, "response_content_type": "application/json", "response_body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}", "exchange_index": 3}, "client_response": {"status": 200, "content_type": "application/json", "body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"model\":\"captured-model\",\"status\":\"completed\",\"incomplete_details\":null,\"output\":[{\"type\":\"message\",\"id\":\"msg_switchyard\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"\",\"annotations\":[]}]}],\"usage\":{\"input_tokens\":4,\"output_tokens\":4,\"total_tokens\":8,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}},\"parallel_tool_calls\":true,\"tool_choice\":\"auto\",\"tools\":[]}"}, "consumer": {"classified_as_refusal": false, "refusal_text": "", "ordinary_output_text": "", "content_types": ["output_text"]}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "7a23989cbe18f1c6c67ee03684ce76bd5901a27d", "binary_version": "0.2.0", "binary_sha256": "cdf1ff53f411b5bd5a6dadbcd85e4883e986d310f2a94ee26693b4911eff0e8e", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 4, "scenario": "responses-buffered", "client_request": {"method": "POST", "path": "/v1/responses", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"input\":\"REFUSALPROBE trigger\",\"max_output_tokens\":32}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_completion_tokens\":32}", "response_status": 200, "response_content_type": "application/json", "response_body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}", "exchange_index": 4}, "client_response": {"status": 200, "content_type": "application/json", "body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"model\":\"captured-model\",\"status\":\"completed\",\"incomplete_details\":null,\"output\":[{\"type\":\"message\",\"id\":\"msg_switchyard\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"\",\"annotations\":[]}]}],\"usage\":{\"input_tokens\":4,\"output_tokens\":4,\"total_tokens\":8,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}},\"parallel_tool_calls\":true,\"tool_choice\":\"auto\",\"tools\":[]}"}, "consumer": {"classified_as_refusal": false, "refusal_text": "", "ordinary_output_text": "", "content_types": ["output_text"]}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "7a23989cbe18f1c6c67ee03684ce76bd5901a27d", "binary_version": "0.2.0", "binary_sha256": "cdf1ff53f411b5bd5a6dadbcd85e4883e986d310f2a94ee26693b4911eff0e8e", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 5, "scenario": "responses-buffered", "client_request": {"method": "POST", "path": "/v1/responses", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"input\":\"REFUSALPROBE trigger\",\"max_output_tokens\":32}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_completion_tokens\":32}", "response_status": 200, "response_content_type": "application/json", "response_body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}", "exchange_index": 5}, "client_response": {"status": 200, "content_type": "application/json", "body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"model\":\"captured-model\",\"status\":\"completed\",\"incomplete_details\":null,\"output\":[{\"type\":\"message\",\"id\":\"msg_switchyard\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"\",\"annotations\":[]}]}],\"usage\":{\"input_tokens\":4,\"output_tokens\":4,\"total_tokens\":8,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}},\"parallel_tool_calls\":true,\"tool_choice\":\"auto\",\"tools\":[]}"}, "consumer": {"classified_as_refusal": false, "refusal_text": "", "ordinary_output_text": "", "content_types": ["output_text"]}} diff --git a/transcripts/069/switchyard-main-responses-stream.jsonl b/transcripts/069/switchyard-main-responses-stream.jsonl new file mode 100644 index 0000000..3a85011 --- /dev/null +++ b/transcripts/069/switchyard-main-responses-stream.jsonl @@ -0,0 +1,5 @@ +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "7a23989cbe18f1c6c67ee03684ce76bd5901a27d", "binary_version": "0.2.0", "binary_sha256": "cdf1ff53f411b5bd5a6dadbcd85e4883e986d310f2a94ee26693b4911eff0e8e", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 1, "scenario": "responses-stream", "client_request": {"method": "POST", "path": "/v1/responses", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"input\":\"REFUSALPROBE trigger\",\"max_output_tokens\":32,\"stream\":true}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_completion_tokens\":32,\"stream\":true,\"stream_options\":{\"include_usage\":true}}", "response_status": 200, "response_content_type": "text/event-stream; charset=utf-8", "response_body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "exchange_index": 6}, "client_response": {"status": 200, "content_type": "text/event-stream", "body_raw": "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"completed_at\":null,\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"metadata\":null,\"model\":\"captured-model\",\"output\":[],\"parallel_tool_calls\":true,\"frequency_penalty\":null,\"presence_penalty\":null,\"status\":\"in_progress\",\"temperature\":null,\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0,\"total_tokens\":0,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}}},\"sequence_number\":0}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"completed_at\":null,\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"metadata\":null,\"model\":\"captured-model\",\"output\":[],\"parallel_tool_calls\":true,\"frequency_penalty\":null,\"presence_penalty\":null,\"status\":\"completed\",\"temperature\":null,\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0,\"total_tokens\":0,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}}},\"sequence_number\":1}\n\n"}, "consumer": {"classified_as_refusal": false, "refusal_text": "", "ordinary_output_text": "", "event_types": ["response.created", "response.completed"]}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "7a23989cbe18f1c6c67ee03684ce76bd5901a27d", "binary_version": "0.2.0", "binary_sha256": "cdf1ff53f411b5bd5a6dadbcd85e4883e986d310f2a94ee26693b4911eff0e8e", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 2, "scenario": "responses-stream", "client_request": {"method": "POST", "path": "/v1/responses", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"input\":\"REFUSALPROBE trigger\",\"max_output_tokens\":32,\"stream\":true}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_completion_tokens\":32,\"stream\":true,\"stream_options\":{\"include_usage\":true}}", "response_status": 200, "response_content_type": "text/event-stream; charset=utf-8", "response_body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "exchange_index": 7}, "client_response": {"status": 200, "content_type": "text/event-stream", "body_raw": "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"completed_at\":null,\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"metadata\":null,\"model\":\"captured-model\",\"output\":[],\"parallel_tool_calls\":true,\"frequency_penalty\":null,\"presence_penalty\":null,\"status\":\"in_progress\",\"temperature\":null,\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0,\"total_tokens\":0,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}}},\"sequence_number\":0}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"completed_at\":null,\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"metadata\":null,\"model\":\"captured-model\",\"output\":[],\"parallel_tool_calls\":true,\"frequency_penalty\":null,\"presence_penalty\":null,\"status\":\"completed\",\"temperature\":null,\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0,\"total_tokens\":0,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}}},\"sequence_number\":1}\n\n"}, "consumer": {"classified_as_refusal": false, "refusal_text": "", "ordinary_output_text": "", "event_types": ["response.created", "response.completed"]}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "7a23989cbe18f1c6c67ee03684ce76bd5901a27d", "binary_version": "0.2.0", "binary_sha256": "cdf1ff53f411b5bd5a6dadbcd85e4883e986d310f2a94ee26693b4911eff0e8e", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 3, "scenario": "responses-stream", "client_request": {"method": "POST", "path": "/v1/responses", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"input\":\"REFUSALPROBE trigger\",\"max_output_tokens\":32,\"stream\":true}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_completion_tokens\":32,\"stream\":true,\"stream_options\":{\"include_usage\":true}}", "response_status": 200, "response_content_type": "text/event-stream; charset=utf-8", "response_body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "exchange_index": 8}, "client_response": {"status": 200, "content_type": "text/event-stream", "body_raw": "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"completed_at\":null,\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"metadata\":null,\"model\":\"captured-model\",\"output\":[],\"parallel_tool_calls\":true,\"frequency_penalty\":null,\"presence_penalty\":null,\"status\":\"in_progress\",\"temperature\":null,\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0,\"total_tokens\":0,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}}},\"sequence_number\":0}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"completed_at\":null,\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"metadata\":null,\"model\":\"captured-model\",\"output\":[],\"parallel_tool_calls\":true,\"frequency_penalty\":null,\"presence_penalty\":null,\"status\":\"completed\",\"temperature\":null,\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0,\"total_tokens\":0,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}}},\"sequence_number\":1}\n\n"}, "consumer": {"classified_as_refusal": false, "refusal_text": "", "ordinary_output_text": "", "event_types": ["response.created", "response.completed"]}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "7a23989cbe18f1c6c67ee03684ce76bd5901a27d", "binary_version": "0.2.0", "binary_sha256": "cdf1ff53f411b5bd5a6dadbcd85e4883e986d310f2a94ee26693b4911eff0e8e", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 4, "scenario": "responses-stream", "client_request": {"method": "POST", "path": "/v1/responses", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"input\":\"REFUSALPROBE trigger\",\"max_output_tokens\":32,\"stream\":true}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_completion_tokens\":32,\"stream\":true,\"stream_options\":{\"include_usage\":true}}", "response_status": 200, "response_content_type": "text/event-stream; charset=utf-8", "response_body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "exchange_index": 9}, "client_response": {"status": 200, "content_type": "text/event-stream", "body_raw": "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"completed_at\":null,\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"metadata\":null,\"model\":\"captured-model\",\"output\":[],\"parallel_tool_calls\":true,\"frequency_penalty\":null,\"presence_penalty\":null,\"status\":\"in_progress\",\"temperature\":null,\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0,\"total_tokens\":0,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}}},\"sequence_number\":0}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"completed_at\":null,\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"metadata\":null,\"model\":\"captured-model\",\"output\":[],\"parallel_tool_calls\":true,\"frequency_penalty\":null,\"presence_penalty\":null,\"status\":\"completed\",\"temperature\":null,\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0,\"total_tokens\":0,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}}},\"sequence_number\":1}\n\n"}, "consumer": {"classified_as_refusal": false, "refusal_text": "", "ordinary_output_text": "", "event_types": ["response.created", "response.completed"]}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "7a23989cbe18f1c6c67ee03684ce76bd5901a27d", "binary_version": "0.2.0", "binary_sha256": "cdf1ff53f411b5bd5a6dadbcd85e4883e986d310f2a94ee26693b4911eff0e8e", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 5, "scenario": "responses-stream", "client_request": {"method": "POST", "path": "/v1/responses", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"input\":\"REFUSALPROBE trigger\",\"max_output_tokens\":32,\"stream\":true}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_completion_tokens\":32,\"stream\":true,\"stream_options\":{\"include_usage\":true}}", "response_status": 200, "response_content_type": "text/event-stream; charset=utf-8", "response_body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "exchange_index": 10}, "client_response": {"status": 200, "content_type": "text/event-stream", "body_raw": "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"completed_at\":null,\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"metadata\":null,\"model\":\"captured-model\",\"output\":[],\"parallel_tool_calls\":true,\"frequency_penalty\":null,\"presence_penalty\":null,\"status\":\"in_progress\",\"temperature\":null,\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0,\"total_tokens\":0,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}}},\"sequence_number\":0}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"completed_at\":null,\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"metadata\":null,\"model\":\"captured-model\",\"output\":[],\"parallel_tool_calls\":true,\"frequency_penalty\":null,\"presence_penalty\":null,\"status\":\"completed\",\"temperature\":null,\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0,\"total_tokens\":0,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}}},\"sequence_number\":1}\n\n"}, "consumer": {"classified_as_refusal": false, "refusal_text": "", "ordinary_output_text": "", "event_types": ["response.created", "response.completed"]}} diff --git a/transcripts/069/switchyard-main-summary.json b/transcripts/069/switchyard-main-summary.json new file mode 100644 index 0000000..b806879 --- /dev/null +++ b/transcripts/069/switchyard-main-summary.json @@ -0,0 +1,28 @@ +{ + "target_commit": "7a23989cbe18f1c6c67ee03684ce76bd5901a27d", + "binary_sha256": "cdf1ff53f411b5bd5a6dadbcd85e4883e986d310f2a94ee26693b4911eff0e8e", + "label": "main", + "model": "captured-model", + "results": { + "responses-buffered": { + "runs": 5, + "typed_refusal_detected": 0, + "typed_refusal_missed": 5 + }, + "responses-stream": { + "runs": 5, + "typed_refusal_detected": 0, + "typed_refusal_missed": 5 + }, + "chat-buffered-control": { + "runs": 5, + "typed_refusal_detected": 5, + "typed_refusal_missed": 0 + }, + "chat-stream-control": { + "runs": 5, + "typed_refusal_detected": 5, + "typed_refusal_missed": 0 + } + } +} diff --git a/transcripts/069/switchyard-pr623-chat-buffered-control.jsonl b/transcripts/069/switchyard-pr623-chat-buffered-control.jsonl new file mode 100644 index 0000000..9751a06 --- /dev/null +++ b/transcripts/069/switchyard-pr623-chat-buffered-control.jsonl @@ -0,0 +1,5 @@ +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "2765f46972bf89a96beb5b2158b0fc56a3a72288", "binary_version": "0.2.0", "binary_sha256": "749029d3e404deb7077ea76e00fdc73cd3e8f0d6756b9609755c1b6176c0a63f", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 1, "scenario": "chat-buffered-control", "client_request": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32}", "response_status": 200, "response_content_type": "application/json", "response_body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}", "exchange_index": 11}, "client_response": {"status": 200, "content_type": "application/json", "body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}"}, "consumer": {"classified_as_refusal": true, "refusal_text": "REFUSALPROBE cannot help"}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "2765f46972bf89a96beb5b2158b0fc56a3a72288", "binary_version": "0.2.0", "binary_sha256": "749029d3e404deb7077ea76e00fdc73cd3e8f0d6756b9609755c1b6176c0a63f", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 2, "scenario": "chat-buffered-control", "client_request": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32}", "response_status": 200, "response_content_type": "application/json", "response_body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}", "exchange_index": 12}, "client_response": {"status": 200, "content_type": "application/json", "body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}"}, "consumer": {"classified_as_refusal": true, "refusal_text": "REFUSALPROBE cannot help"}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "2765f46972bf89a96beb5b2158b0fc56a3a72288", "binary_version": "0.2.0", "binary_sha256": "749029d3e404deb7077ea76e00fdc73cd3e8f0d6756b9609755c1b6176c0a63f", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 3, "scenario": "chat-buffered-control", "client_request": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32}", "response_status": 200, "response_content_type": "application/json", "response_body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}", "exchange_index": 13}, "client_response": {"status": 200, "content_type": "application/json", "body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}"}, "consumer": {"classified_as_refusal": true, "refusal_text": "REFUSALPROBE cannot help"}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "2765f46972bf89a96beb5b2158b0fc56a3a72288", "binary_version": "0.2.0", "binary_sha256": "749029d3e404deb7077ea76e00fdc73cd3e8f0d6756b9609755c1b6176c0a63f", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 4, "scenario": "chat-buffered-control", "client_request": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32}", "response_status": 200, "response_content_type": "application/json", "response_body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}", "exchange_index": 14}, "client_response": {"status": 200, "content_type": "application/json", "body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}"}, "consumer": {"classified_as_refusal": true, "refusal_text": "REFUSALPROBE cannot help"}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "2765f46972bf89a96beb5b2158b0fc56a3a72288", "binary_version": "0.2.0", "binary_sha256": "749029d3e404deb7077ea76e00fdc73cd3e8f0d6756b9609755c1b6176c0a63f", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 5, "scenario": "chat-buffered-control", "client_request": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32}", "response_status": 200, "response_content_type": "application/json", "response_body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}", "exchange_index": 15}, "client_response": {"status": 200, "content_type": "application/json", "body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}"}, "consumer": {"classified_as_refusal": true, "refusal_text": "REFUSALPROBE cannot help"}} diff --git a/transcripts/069/switchyard-pr623-chat-stream-control.jsonl b/transcripts/069/switchyard-pr623-chat-stream-control.jsonl new file mode 100644 index 0000000..9edc120 --- /dev/null +++ b/transcripts/069/switchyard-pr623-chat-stream-control.jsonl @@ -0,0 +1,5 @@ +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "2765f46972bf89a96beb5b2158b0fc56a3a72288", "binary_version": "0.2.0", "binary_sha256": "749029d3e404deb7077ea76e00fdc73cd3e8f0d6756b9609755c1b6176c0a63f", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 1, "scenario": "chat-stream-control", "client_request": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32,\"stream\":true}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32,\"stream\":true,\"stream_options\":{\"include_usage\":true}}", "response_status": 200, "response_content_type": "text/event-stream; charset=utf-8", "response_body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "exchange_index": 16}, "client_response": {"status": 200, "content_type": "text/event-stream", "body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n"}, "consumer": {"classified_as_refusal": true, "refusal_text": "REFUSALPROBE cannot help"}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "2765f46972bf89a96beb5b2158b0fc56a3a72288", "binary_version": "0.2.0", "binary_sha256": "749029d3e404deb7077ea76e00fdc73cd3e8f0d6756b9609755c1b6176c0a63f", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 2, "scenario": "chat-stream-control", "client_request": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32,\"stream\":true}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32,\"stream\":true,\"stream_options\":{\"include_usage\":true}}", "response_status": 200, "response_content_type": "text/event-stream; charset=utf-8", "response_body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "exchange_index": 17}, "client_response": {"status": 200, "content_type": "text/event-stream", "body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n"}, "consumer": {"classified_as_refusal": true, "refusal_text": "REFUSALPROBE cannot help"}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "2765f46972bf89a96beb5b2158b0fc56a3a72288", "binary_version": "0.2.0", "binary_sha256": "749029d3e404deb7077ea76e00fdc73cd3e8f0d6756b9609755c1b6176c0a63f", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 3, "scenario": "chat-stream-control", "client_request": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32,\"stream\":true}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32,\"stream\":true,\"stream_options\":{\"include_usage\":true}}", "response_status": 200, "response_content_type": "text/event-stream; charset=utf-8", "response_body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "exchange_index": 18}, "client_response": {"status": 200, "content_type": "text/event-stream", "body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n"}, "consumer": {"classified_as_refusal": true, "refusal_text": "REFUSALPROBE cannot help"}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "2765f46972bf89a96beb5b2158b0fc56a3a72288", "binary_version": "0.2.0", "binary_sha256": "749029d3e404deb7077ea76e00fdc73cd3e8f0d6756b9609755c1b6176c0a63f", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 4, "scenario": "chat-stream-control", "client_request": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32,\"stream\":true}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32,\"stream\":true,\"stream_options\":{\"include_usage\":true}}", "response_status": 200, "response_content_type": "text/event-stream; charset=utf-8", "response_body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "exchange_index": 19}, "client_response": {"status": 200, "content_type": "text/event-stream", "body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n"}, "consumer": {"classified_as_refusal": true, "refusal_text": "REFUSALPROBE cannot help"}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "2765f46972bf89a96beb5b2158b0fc56a3a72288", "binary_version": "0.2.0", "binary_sha256": "749029d3e404deb7077ea76e00fdc73cd3e8f0d6756b9609755c1b6176c0a63f", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 5, "scenario": "chat-stream-control", "client_request": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32,\"stream\":true}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_tokens\":32,\"stream\":true,\"stream_options\":{\"include_usage\":true}}", "response_status": 200, "response_content_type": "text/event-stream; charset=utf-8", "response_body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "exchange_index": 20}, "client_response": {"status": 200, "content_type": "text/event-stream", "body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n"}, "consumer": {"classified_as_refusal": true, "refusal_text": "REFUSALPROBE cannot help"}} diff --git a/transcripts/069/switchyard-pr623-responses-buffered.jsonl b/transcripts/069/switchyard-pr623-responses-buffered.jsonl new file mode 100644 index 0000000..61b2d86 --- /dev/null +++ b/transcripts/069/switchyard-pr623-responses-buffered.jsonl @@ -0,0 +1,5 @@ +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "2765f46972bf89a96beb5b2158b0fc56a3a72288", "binary_version": "0.2.0", "binary_sha256": "749029d3e404deb7077ea76e00fdc73cd3e8f0d6756b9609755c1b6176c0a63f", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 1, "scenario": "responses-buffered", "client_request": {"method": "POST", "path": "/v1/responses", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"input\":\"REFUSALPROBE trigger\",\"max_output_tokens\":32}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_completion_tokens\":32}", "response_status": 200, "response_content_type": "application/json", "response_body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}", "exchange_index": 1}, "client_response": {"status": 200, "content_type": "application/json", "body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"model\":\"captured-model\",\"status\":\"completed\",\"incomplete_details\":null,\"output\":[{\"type\":\"message\",\"id\":\"msg_switchyard\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"REFUSALPROBE cannot help\",\"annotations\":[]}]}],\"usage\":{\"input_tokens\":4,\"output_tokens\":4,\"total_tokens\":8,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}},\"parallel_tool_calls\":true,\"tool_choice\":\"auto\",\"tools\":[]}"}, "consumer": {"classified_as_refusal": false, "refusal_text": "", "ordinary_output_text": "REFUSALPROBE cannot help", "content_types": ["output_text"]}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "2765f46972bf89a96beb5b2158b0fc56a3a72288", "binary_version": "0.2.0", "binary_sha256": "749029d3e404deb7077ea76e00fdc73cd3e8f0d6756b9609755c1b6176c0a63f", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 2, "scenario": "responses-buffered", "client_request": {"method": "POST", "path": "/v1/responses", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"input\":\"REFUSALPROBE trigger\",\"max_output_tokens\":32}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_completion_tokens\":32}", "response_status": 200, "response_content_type": "application/json", "response_body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}", "exchange_index": 2}, "client_response": {"status": 200, "content_type": "application/json", "body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"model\":\"captured-model\",\"status\":\"completed\",\"incomplete_details\":null,\"output\":[{\"type\":\"message\",\"id\":\"msg_switchyard\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"REFUSALPROBE cannot help\",\"annotations\":[]}]}],\"usage\":{\"input_tokens\":4,\"output_tokens\":4,\"total_tokens\":8,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}},\"parallel_tool_calls\":true,\"tool_choice\":\"auto\",\"tools\":[]}"}, "consumer": {"classified_as_refusal": false, "refusal_text": "", "ordinary_output_text": "REFUSALPROBE cannot help", "content_types": ["output_text"]}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "2765f46972bf89a96beb5b2158b0fc56a3a72288", "binary_version": "0.2.0", "binary_sha256": "749029d3e404deb7077ea76e00fdc73cd3e8f0d6756b9609755c1b6176c0a63f", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 3, "scenario": "responses-buffered", "client_request": {"method": "POST", "path": "/v1/responses", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"input\":\"REFUSALPROBE trigger\",\"max_output_tokens\":32}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_completion_tokens\":32}", "response_status": 200, "response_content_type": "application/json", "response_body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}", "exchange_index": 3}, "client_response": {"status": 200, "content_type": "application/json", "body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"model\":\"captured-model\",\"status\":\"completed\",\"incomplete_details\":null,\"output\":[{\"type\":\"message\",\"id\":\"msg_switchyard\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"REFUSALPROBE cannot help\",\"annotations\":[]}]}],\"usage\":{\"input_tokens\":4,\"output_tokens\":4,\"total_tokens\":8,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}},\"parallel_tool_calls\":true,\"tool_choice\":\"auto\",\"tools\":[]}"}, "consumer": {"classified_as_refusal": false, "refusal_text": "", "ordinary_output_text": "REFUSALPROBE cannot help", "content_types": ["output_text"]}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "2765f46972bf89a96beb5b2158b0fc56a3a72288", "binary_version": "0.2.0", "binary_sha256": "749029d3e404deb7077ea76e00fdc73cd3e8f0d6756b9609755c1b6176c0a63f", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 4, "scenario": "responses-buffered", "client_request": {"method": "POST", "path": "/v1/responses", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"input\":\"REFUSALPROBE trigger\",\"max_output_tokens\":32}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_completion_tokens\":32}", "response_status": 200, "response_content_type": "application/json", "response_body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}", "exchange_index": 4}, "client_response": {"status": 200, "content_type": "application/json", "body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"model\":\"captured-model\",\"status\":\"completed\",\"incomplete_details\":null,\"output\":[{\"type\":\"message\",\"id\":\"msg_switchyard\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"REFUSALPROBE cannot help\",\"annotations\":[]}]}],\"usage\":{\"input_tokens\":4,\"output_tokens\":4,\"total_tokens\":8,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}},\"parallel_tool_calls\":true,\"tool_choice\":\"auto\",\"tools\":[]}"}, "consumer": {"classified_as_refusal": false, "refusal_text": "", "ordinary_output_text": "REFUSALPROBE cannot help", "content_types": ["output_text"]}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "2765f46972bf89a96beb5b2158b0fc56a3a72288", "binary_version": "0.2.0", "binary_sha256": "749029d3e404deb7077ea76e00fdc73cd3e8f0d6756b9609755c1b6176c0a63f", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 5, "scenario": "responses-buffered", "client_request": {"method": "POST", "path": "/v1/responses", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"input\":\"REFUSALPROBE trigger\",\"max_output_tokens\":32}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_completion_tokens\":32}", "response_status": 200, "response_content_type": "application/json", "response_body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"REFUSALPROBE cannot help\",\"annotations\":[]},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":4,\"total_tokens\":8}}", "exchange_index": 5}, "client_response": {"status": 200, "content_type": "application/json", "body_raw": "{\"id\":\"chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"model\":\"captured-model\",\"status\":\"completed\",\"incomplete_details\":null,\"output\":[{\"type\":\"message\",\"id\":\"msg_switchyard\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"REFUSALPROBE cannot help\",\"annotations\":[]}]}],\"usage\":{\"input_tokens\":4,\"output_tokens\":4,\"total_tokens\":8,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}},\"parallel_tool_calls\":true,\"tool_choice\":\"auto\",\"tools\":[]}"}, "consumer": {"classified_as_refusal": false, "refusal_text": "", "ordinary_output_text": "REFUSALPROBE cannot help", "content_types": ["output_text"]}} diff --git a/transcripts/069/switchyard-pr623-responses-stream.jsonl b/transcripts/069/switchyard-pr623-responses-stream.jsonl new file mode 100644 index 0000000..5a9f876 --- /dev/null +++ b/transcripts/069/switchyard-pr623-responses-stream.jsonl @@ -0,0 +1,5 @@ +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "2765f46972bf89a96beb5b2158b0fc56a3a72288", "binary_version": "0.2.0", "binary_sha256": "749029d3e404deb7077ea76e00fdc73cd3e8f0d6756b9609755c1b6176c0a63f", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 1, "scenario": "responses-stream", "client_request": {"method": "POST", "path": "/v1/responses", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"input\":\"REFUSALPROBE trigger\",\"max_output_tokens\":32,\"stream\":true}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_completion_tokens\":32,\"stream\":true,\"stream_options\":{\"include_usage\":true}}", "response_status": 200, "response_content_type": "text/event-stream; charset=utf-8", "response_body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "exchange_index": 6}, "client_response": {"status": 200, "content_type": "text/event-stream", "body_raw": "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"completed_at\":null,\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"metadata\":null,\"model\":\"captured-model\",\"output\":[],\"parallel_tool_calls\":true,\"frequency_penalty\":null,\"presence_penalty\":null,\"status\":\"in_progress\",\"temperature\":null,\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0,\"total_tokens\":0,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}}},\"sequence_number\":0}\n\nevent: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"type\":\"message\",\"id\":\"msg_0\",\"role\":\"assistant\",\"status\":\"in_progress\",\"content\":[]},\"sequence_number\":1}\n\nevent: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"output_index\":0,\"content_index\":0,\"part\":{\"type\":\"output_text\",\"text\":\"\"},\"sequence_number\":2}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"content_index\":0,\"delta\":\"REFUSALPROBE cannot help\",\"sequence_number\":3}\n\nevent: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"output_index\":0,\"content_index\":0,\"part\":{\"type\":\"output_text\",\"text\":\"REFUSALPROBE cannot help\"},\"sequence_number\":4}\n\nevent: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"type\":\"message\",\"id\":\"msg_0\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"REFUSALPROBE cannot help\"}]},\"sequence_number\":5}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"completed_at\":null,\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"metadata\":null,\"model\":\"captured-model\",\"output\":[{\"type\":\"message\",\"id\":\"msg_0\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"REFUSALPROBE cannot help\"}]}],\"parallel_tool_calls\":true,\"frequency_penalty\":null,\"presence_penalty\":null,\"status\":\"completed\",\"temperature\":null,\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0,\"total_tokens\":0,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}}},\"sequence_number\":6}\n\n"}, "consumer": {"classified_as_refusal": false, "refusal_text": "", "ordinary_output_text": "REFUSALPROBE cannot help", "event_types": ["response.created", "response.output_item.added", "response.content_part.added", "response.output_text.delta", "response.content_part.done", "response.output_item.done", "response.completed"]}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "2765f46972bf89a96beb5b2158b0fc56a3a72288", "binary_version": "0.2.0", "binary_sha256": "749029d3e404deb7077ea76e00fdc73cd3e8f0d6756b9609755c1b6176c0a63f", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 2, "scenario": "responses-stream", "client_request": {"method": "POST", "path": "/v1/responses", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"input\":\"REFUSALPROBE trigger\",\"max_output_tokens\":32,\"stream\":true}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_completion_tokens\":32,\"stream\":true,\"stream_options\":{\"include_usage\":true}}", "response_status": 200, "response_content_type": "text/event-stream; charset=utf-8", "response_body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "exchange_index": 7}, "client_response": {"status": 200, "content_type": "text/event-stream", "body_raw": "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"completed_at\":null,\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"metadata\":null,\"model\":\"captured-model\",\"output\":[],\"parallel_tool_calls\":true,\"frequency_penalty\":null,\"presence_penalty\":null,\"status\":\"in_progress\",\"temperature\":null,\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0,\"total_tokens\":0,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}}},\"sequence_number\":0}\n\nevent: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"type\":\"message\",\"id\":\"msg_0\",\"role\":\"assistant\",\"status\":\"in_progress\",\"content\":[]},\"sequence_number\":1}\n\nevent: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"output_index\":0,\"content_index\":0,\"part\":{\"type\":\"output_text\",\"text\":\"\"},\"sequence_number\":2}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"content_index\":0,\"delta\":\"REFUSALPROBE cannot help\",\"sequence_number\":3}\n\nevent: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"output_index\":0,\"content_index\":0,\"part\":{\"type\":\"output_text\",\"text\":\"REFUSALPROBE cannot help\"},\"sequence_number\":4}\n\nevent: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"type\":\"message\",\"id\":\"msg_0\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"REFUSALPROBE cannot help\"}]},\"sequence_number\":5}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"completed_at\":null,\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"metadata\":null,\"model\":\"captured-model\",\"output\":[{\"type\":\"message\",\"id\":\"msg_0\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"REFUSALPROBE cannot help\"}]}],\"parallel_tool_calls\":true,\"frequency_penalty\":null,\"presence_penalty\":null,\"status\":\"completed\",\"temperature\":null,\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0,\"total_tokens\":0,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}}},\"sequence_number\":6}\n\n"}, "consumer": {"classified_as_refusal": false, "refusal_text": "", "ordinary_output_text": "REFUSALPROBE cannot help", "event_types": ["response.created", "response.output_item.added", "response.content_part.added", "response.output_text.delta", "response.content_part.done", "response.output_item.done", "response.completed"]}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "2765f46972bf89a96beb5b2158b0fc56a3a72288", "binary_version": "0.2.0", "binary_sha256": "749029d3e404deb7077ea76e00fdc73cd3e8f0d6756b9609755c1b6176c0a63f", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 3, "scenario": "responses-stream", "client_request": {"method": "POST", "path": "/v1/responses", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"input\":\"REFUSALPROBE trigger\",\"max_output_tokens\":32,\"stream\":true}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_completion_tokens\":32,\"stream\":true,\"stream_options\":{\"include_usage\":true}}", "response_status": 200, "response_content_type": "text/event-stream; charset=utf-8", "response_body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "exchange_index": 8}, "client_response": {"status": 200, "content_type": "text/event-stream", "body_raw": "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"completed_at\":null,\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"metadata\":null,\"model\":\"captured-model\",\"output\":[],\"parallel_tool_calls\":true,\"frequency_penalty\":null,\"presence_penalty\":null,\"status\":\"in_progress\",\"temperature\":null,\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0,\"total_tokens\":0,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}}},\"sequence_number\":0}\n\nevent: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"type\":\"message\",\"id\":\"msg_0\",\"role\":\"assistant\",\"status\":\"in_progress\",\"content\":[]},\"sequence_number\":1}\n\nevent: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"output_index\":0,\"content_index\":0,\"part\":{\"type\":\"output_text\",\"text\":\"\"},\"sequence_number\":2}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"content_index\":0,\"delta\":\"REFUSALPROBE cannot help\",\"sequence_number\":3}\n\nevent: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"output_index\":0,\"content_index\":0,\"part\":{\"type\":\"output_text\",\"text\":\"REFUSALPROBE cannot help\"},\"sequence_number\":4}\n\nevent: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"type\":\"message\",\"id\":\"msg_0\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"REFUSALPROBE cannot help\"}]},\"sequence_number\":5}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"completed_at\":null,\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"metadata\":null,\"model\":\"captured-model\",\"output\":[{\"type\":\"message\",\"id\":\"msg_0\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"REFUSALPROBE cannot help\"}]}],\"parallel_tool_calls\":true,\"frequency_penalty\":null,\"presence_penalty\":null,\"status\":\"completed\",\"temperature\":null,\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0,\"total_tokens\":0,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}}},\"sequence_number\":6}\n\n"}, "consumer": {"classified_as_refusal": false, "refusal_text": "", "ordinary_output_text": "REFUSALPROBE cannot help", "event_types": ["response.created", "response.output_item.added", "response.content_part.added", "response.output_text.delta", "response.content_part.done", "response.output_item.done", "response.completed"]}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "2765f46972bf89a96beb5b2158b0fc56a3a72288", "binary_version": "0.2.0", "binary_sha256": "749029d3e404deb7077ea76e00fdc73cd3e8f0d6756b9609755c1b6176c0a63f", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 4, "scenario": "responses-stream", "client_request": {"method": "POST", "path": "/v1/responses", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"input\":\"REFUSALPROBE trigger\",\"max_output_tokens\":32,\"stream\":true}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_completion_tokens\":32,\"stream\":true,\"stream_options\":{\"include_usage\":true}}", "response_status": 200, "response_content_type": "text/event-stream; charset=utf-8", "response_body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "exchange_index": 9}, "client_response": {"status": 200, "content_type": "text/event-stream", "body_raw": "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"completed_at\":null,\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"metadata\":null,\"model\":\"captured-model\",\"output\":[],\"parallel_tool_calls\":true,\"frequency_penalty\":null,\"presence_penalty\":null,\"status\":\"in_progress\",\"temperature\":null,\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0,\"total_tokens\":0,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}}},\"sequence_number\":0}\n\nevent: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"type\":\"message\",\"id\":\"msg_0\",\"role\":\"assistant\",\"status\":\"in_progress\",\"content\":[]},\"sequence_number\":1}\n\nevent: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"output_index\":0,\"content_index\":0,\"part\":{\"type\":\"output_text\",\"text\":\"\"},\"sequence_number\":2}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"content_index\":0,\"delta\":\"REFUSALPROBE cannot help\",\"sequence_number\":3}\n\nevent: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"output_index\":0,\"content_index\":0,\"part\":{\"type\":\"output_text\",\"text\":\"REFUSALPROBE cannot help\"},\"sequence_number\":4}\n\nevent: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"type\":\"message\",\"id\":\"msg_0\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"REFUSALPROBE cannot help\"}]},\"sequence_number\":5}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"completed_at\":null,\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"metadata\":null,\"model\":\"captured-model\",\"output\":[{\"type\":\"message\",\"id\":\"msg_0\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"REFUSALPROBE cannot help\"}]}],\"parallel_tool_calls\":true,\"frequency_penalty\":null,\"presence_penalty\":null,\"status\":\"completed\",\"temperature\":null,\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0,\"total_tokens\":0,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}}},\"sequence_number\":6}\n\n"}, "consumer": {"classified_as_refusal": false, "refusal_text": "", "ordinary_output_text": "REFUSALPROBE cannot help", "event_types": ["response.created", "response.output_item.added", "response.content_part.added", "response.output_text.delta", "response.content_part.done", "response.output_item.done", "response.completed"]}} +{"target": {"repository": "https://github.com/NVIDIA-NeMo/Switchyard", "commit": "2765f46972bf89a96beb5b2158b0fc56a3a72288", "binary_version": "0.2.0", "binary_sha256": "749029d3e404deb7077ea76e00fdc73cd3e8f0d6756b9609755c1b6176c0a63f", "rustc_version": "rustc 1.96.1 (31fca3adb 2026-06-26)", "configuration": "openai_chat backend, passthrough route, max_retries=0"}, "trial": 5, "scenario": "responses-stream", "client_request": {"method": "POST", "path": "/v1/responses", "content_type": "application/json", "body_raw": "{\"model\":\"main\",\"input\":\"REFUSALPROBE trigger\",\"max_output_tokens\":32,\"stream\":true}"}, "upstream_exchange": {"method": "POST", "path": "/v1/chat/completions", "content_type": "application/json", "body_raw": "{\"model\":\"captured-model\",\"messages\":[{\"role\":\"user\",\"content\":\"REFUSALPROBE trigger\"}],\"max_completion_tokens\":32,\"stream\":true,\"stream_options\":{\"include_usage\":true}}", "response_status": 200, "response_content_type": "text/event-stream; charset=utf-8", "response_body_raw": "data: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"refusal\":\"\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{\"refusal\":\"REFUSALPROBE cannot help\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-refusal-069\",\"object\":\"chat.completion.chunk\",\"created\":1788541200,\"model\":\"captured-model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "exchange_index": 10}, "client_response": {"status": 200, "content_type": "text/event-stream", "body_raw": "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"completed_at\":null,\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"metadata\":null,\"model\":\"captured-model\",\"output\":[],\"parallel_tool_calls\":true,\"frequency_penalty\":null,\"presence_penalty\":null,\"status\":\"in_progress\",\"temperature\":null,\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0,\"total_tokens\":0,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}}},\"sequence_number\":0}\n\nevent: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"type\":\"message\",\"id\":\"msg_0\",\"role\":\"assistant\",\"status\":\"in_progress\",\"content\":[]},\"sequence_number\":1}\n\nevent: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"output_index\":0,\"content_index\":0,\"part\":{\"type\":\"output_text\",\"text\":\"\"},\"sequence_number\":2}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"content_index\":0,\"delta\":\"REFUSALPROBE cannot help\",\"sequence_number\":3}\n\nevent: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"output_index\":0,\"content_index\":0,\"part\":{\"type\":\"output_text\",\"text\":\"REFUSALPROBE cannot help\"},\"sequence_number\":4}\n\nevent: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"type\":\"message\",\"id\":\"msg_0\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"REFUSALPROBE cannot help\"}]},\"sequence_number\":5}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_chatcmpl-refusal-069\",\"object\":\"response\",\"created_at\":0,\"completed_at\":null,\"error\":null,\"incomplete_details\":null,\"instructions\":null,\"metadata\":null,\"model\":\"captured-model\",\"output\":[{\"type\":\"message\",\"id\":\"msg_0\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"REFUSALPROBE cannot help\"}]}],\"parallel_tool_calls\":true,\"frequency_penalty\":null,\"presence_penalty\":null,\"status\":\"completed\",\"temperature\":null,\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0,\"total_tokens\":0,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}}},\"sequence_number\":6}\n\n"}, "consumer": {"classified_as_refusal": false, "refusal_text": "", "ordinary_output_text": "REFUSALPROBE cannot help", "event_types": ["response.created", "response.output_item.added", "response.content_part.added", "response.output_text.delta", "response.content_part.done", "response.output_item.done", "response.completed"]}} diff --git a/transcripts/069/switchyard-pr623-summary.json b/transcripts/069/switchyard-pr623-summary.json new file mode 100644 index 0000000..b3776fd --- /dev/null +++ b/transcripts/069/switchyard-pr623-summary.json @@ -0,0 +1,28 @@ +{ + "target_commit": "2765f46972bf89a96beb5b2158b0fc56a3a72288", + "binary_sha256": "749029d3e404deb7077ea76e00fdc73cd3e8f0d6756b9609755c1b6176c0a63f", + "label": "pr623", + "model": "captured-model", + "results": { + "responses-buffered": { + "runs": 5, + "typed_refusal_detected": 0, + "typed_refusal_missed": 5 + }, + "responses-stream": { + "runs": 5, + "typed_refusal_detected": 0, + "typed_refusal_missed": 5 + }, + "chat-buffered-control": { + "runs": 5, + "typed_refusal_detected": 5, + "typed_refusal_missed": 0 + }, + "chat-stream-control": { + "runs": 5, + "typed_refusal_detected": 5, + "typed_refusal_missed": 0 + } + } +}