From 4efb19339f31c7666e9c859a4d7491c195db35d9 Mon Sep 17 00:00:00 2001 From: Sean Lopp Date: Wed, 2 Sep 2026 15:03:40 -0600 Subject: [PATCH 1/6] refactor(stage-router): generalize tool activity categories Signed-off-by: Sean Lopp --- .../libsy/src/algorithms/util/tool_signals.rs | 89 +++++++++++-------- 1 file changed, 52 insertions(+), 37 deletions(-) diff --git a/crates/libsy/src/algorithms/util/tool_signals.rs b/crates/libsy/src/algorithms/util/tool_signals.rs index 3427357be..f24626a51 100644 --- a/crates/libsy/src/algorithms/util/tool_signals.rs +++ b/crates/libsy/src/algorithms/util/tool_signals.rs @@ -278,12 +278,18 @@ struct ObservedToolCall { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum ToolCategory { +enum MutationKind { Write, Edit, - Read, +} + +/// Domain-neutral meaning assigned to an observed tool call. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ToolSemantic { + Mutate(MutationKind), + Observe, Plan, - Other, + Unknown, } /// Request-side processor that extracts tool-result signals from each request @@ -314,38 +320,38 @@ impl Processor for ToolSignalProcessor { } } -fn classify_tool_call(name: &str, command: Option<&str>) -> ToolCategory { +fn classify_tool_call(name: &str, command: Option<&str>) -> ToolSemantic { let lower = name.to_lowercase(); if WRITE_TOOL_NAMES.contains(&lower.as_str()) { - return ToolCategory::Write; + return ToolSemantic::Mutate(MutationKind::Write); } if EDIT_TOOL_NAMES.contains(&lower.as_str()) { - return ToolCategory::Edit; + return ToolSemantic::Mutate(MutationKind::Edit); } if READ_TOOL_NAMES.contains(&lower.as_str()) { - return ToolCategory::Read; + return ToolSemantic::Observe; } if PLAN_TOOL_NAMES.contains(&lower.as_str()) { - return ToolCategory::Plan; + return ToolSemantic::Plan; } if BASH_TOOL_NAMES.contains(&lower.as_str()) && let Some(cmd) = command { // Write/edit redirection trumps read-like operands. if BASH_WRITE_PATTERNS.iter().any(|p| cmd.contains(p)) { - return ToolCategory::Write; + return ToolSemantic::Mutate(MutationKind::Write); } if cmd.contains("python") && PYTHON_WRITE_PATTERNS.iter().any(|p| cmd.contains(p)) { - return ToolCategory::Write; + return ToolSemantic::Mutate(MutationKind::Write); } if BASH_EDIT_PATTERNS.iter().any(|p| cmd.contains(p)) { - return ToolCategory::Edit; + return ToolSemantic::Mutate(MutationKind::Edit); } if BASH_READ_PATTERNS.iter().any(|p| cmd.contains(p)) { - return ToolCategory::Read; + return ToolSemantic::Observe; } } - ToolCategory::Other + ToolSemantic::Unknown } // ─── extraction entry point ─────────────────────────────────────────────────── @@ -474,7 +480,7 @@ fn build_signal( let no_error_streak = compute_no_error_streak(&tool_texts); // Single pass: cumulative + sliding-window counters together. Also tracks - // the trailing pure-bash streak (consecutive `Other`-category calls back + // the trailing pure-bash streak (consecutive `Unknown` calls back // from the end) — the build-pit proxy. let recent_start = tool_calls.len().saturating_sub(recent_window); let mut write_count = 0u32; @@ -490,38 +496,38 @@ fn build_signal( for (i, tc) in tool_calls.iter().enumerate().rev() { let cat = classify_tool_call(&tc.name, tc.command.as_deref()); if streak_open { - if matches!(cat, ToolCategory::Other) { + if matches!(cat, ToolSemantic::Unknown) { pure_bash_streak += 1; } else { streak_open = false; } } match cat { - ToolCategory::Write => { + ToolSemantic::Mutate(MutationKind::Write) => { write_count += 1; if i >= recent_start { recent_write_count += 1; } } - ToolCategory::Edit => { + ToolSemantic::Mutate(MutationKind::Edit) => { edit_count += 1; if i >= recent_start { recent_edit_count += 1; } } - ToolCategory::Read => { + ToolSemantic::Observe => { read_count += 1; if i >= recent_start { recent_read_count += 1; } } - ToolCategory::Plan => { + ToolSemantic::Plan => { todowrite_count += 1; if i >= recent_start { recent_todowrite_count += 1; } } - ToolCategory::Other => {} + ToolSemantic::Unknown => {} } } @@ -1099,13 +1105,13 @@ mod tests { #[test] fn todowrite_classifies_as_plan() { - assert_eq!(classify_tool_call("TodoWrite", None), ToolCategory::Plan); - assert_eq!(classify_tool_call("todo_write", None), ToolCategory::Plan); + assert_eq!(classify_tool_call("TodoWrite", None), ToolSemantic::Plan); + assert_eq!(classify_tool_call("todo_write", None), ToolSemantic::Plan); } #[test] fn codex_update_plan_classifies_as_plan() { - assert_eq!(classify_tool_call("update_plan", None), ToolCategory::Plan); + assert_eq!(classify_tool_call("update_plan", None), ToolSemantic::Plan); } #[test] @@ -1113,46 +1119,55 @@ mod tests { // shell_command + heredoc -> Write. assert_eq!( classify_tool_call("shell_command", Some("cat > /app/foo.py <<'eof'\nx=1\neof")), - ToolCategory::Write, + ToolSemantic::Mutate(MutationKind::Write), ); // shell_command + read-like inspection -> Read. assert_eq!( classify_tool_call("shell_command", Some("ls /app")), - ToolCategory::Read, + ToolSemantic::Observe, ); // shell_command without matching patterns -> Other. assert_eq!( classify_tool_call("shell_command", Some("./run_tests.sh")), - ToolCategory::Other, + ToolSemantic::Unknown, ); } #[test] fn read_tool_classifies_as_read() { - assert_eq!(classify_tool_call("Read", None), ToolCategory::Read); - assert_eq!(classify_tool_call("View", None), ToolCategory::Read); + assert_eq!(classify_tool_call("Read", None), ToolSemantic::Observe); + assert_eq!(classify_tool_call("View", None), ToolSemantic::Observe); } #[test] fn hermes_tool_names_classify() { // Hermes (NousResearch) file tools route by name. - assert_eq!(classify_tool_call("write_file", None), ToolCategory::Write); - assert_eq!(classify_tool_call("patch", None), ToolCategory::Edit); - assert_eq!(classify_tool_call("read_file", None), ToolCategory::Read); - assert_eq!(classify_tool_call("search_files", None), ToolCategory::Read); + assert_eq!( + classify_tool_call("write_file", None), + ToolSemantic::Mutate(MutationKind::Write) + ); + assert_eq!( + classify_tool_call("patch", None), + ToolSemantic::Mutate(MutationKind::Edit) + ); + assert_eq!(classify_tool_call("read_file", None), ToolSemantic::Observe); + assert_eq!( + classify_tool_call("search_files", None), + ToolSemantic::Observe + ); // Hermes runs shell through `terminal`, which carries a `command` arg, // so its intent comes from the Bash-pattern match like codex's shell_command. assert_eq!( classify_tool_call("terminal", Some("sed -i 's/a/b/' /app/x.py")), - ToolCategory::Edit, + ToolSemantic::Mutate(MutationKind::Edit), ); assert_eq!( classify_tool_call("terminal", Some("grep foo /app")), - ToolCategory::Read, + ToolSemantic::Observe, ); assert_eq!( classify_tool_call("terminal", Some("./run_tests.sh")), - ToolCategory::Other, + ToolSemantic::Unknown, ); } @@ -1167,7 +1182,7 @@ mod tests { for cmd in cases { assert_eq!( classify_tool_call("Bash", Some(cmd)), - ToolCategory::Read, + ToolSemantic::Observe, "expected Read for {cmd}" ); } @@ -1179,7 +1194,7 @@ mod tests { // write redirection must win. assert_eq!( classify_tool_call("Bash", Some("cat /etc/hosts > /tmp/out")), - ToolCategory::Write, + ToolSemantic::Mutate(MutationKind::Write), ); } From 4ced7311b174ca68086ab78cb00e25c836102cf5 Mon Sep 17 00:00:00 2001 From: Sean Lopp Date: Wed, 2 Sep 2026 15:17:14 -0600 Subject: [PATCH 2/6] feat(stage-router): configure custom tool semantics Signed-off-by: Sean Lopp --- crates/libsy/src/algorithms/stage.rs | 7 +- crates/libsy/src/algorithms/util/stage.rs | 24 +- .../libsy/src/algorithms/util/tool_signals.rs | 220 +++++++++++++++++- crates/libsy/src/lib.rs | 2 +- crates/switchyard-py/src/libsy_bindings.rs | 17 +- crates/switchyard-runner/src/algorithm.rs | 7 + crates/switchyard-runner/src/config.rs | 9 + crates/switchyard-server/tests/server.rs | 67 ++++++ .../stage_router_routing.md | 36 ++- tests/test_libsy_minimal_bindings.py | 28 +++ 10 files changed, 397 insertions(+), 20 deletions(-) diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs index 3fd4acc1e..2f9adf945 100644 --- a/crates/libsy/src/algorithms/stage.rs +++ b/crates/libsy/src/algorithms/stage.rs @@ -24,7 +24,7 @@ use super::util::stage::{ DecisionSource, HandoffNoteConfig, PickerMode, StageClassifier, StageTargets, Tier, fall_open_tier, record_decision_source, record_routing_decision, }; -use super::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSignalProcessor}; +use super::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSemantics, ToolSignalProcessor}; use crate::core::algorithm::{Algorithm, Driver}; use crate::core::classifier::{Classification, Classifier, Score}; use crate::core::state::State; @@ -113,6 +113,8 @@ pub struct StageRouterConfig { /// Trailing tool results the signals are computed over. `None` uses /// [`DEFAULT_RECENT_WINDOW`]. pub recent_window: Option, + /// Exact tool-name semantics added to the built-in coding vocabulary. + pub tool_semantics: ToolSemantics, /// Note handed to the model on a signal-driven escalation, and on a /// hand-back to the efficient tier when a de-escalation note is configured. pub handoff_notes: Option, @@ -133,6 +135,7 @@ impl StageRouterConfig { mode, confidence_threshold, recent_window: None, + tool_semantics: ToolSemantics::default(), handoff_notes: None, tier_prompts: TargetPrompts::default(), llm_fallback: None, @@ -190,6 +193,7 @@ pub(crate) fn build_stage_route( ), }); } + config.tool_semantics.validate()?; // The tiers are a fixed pair; their targets are whatever the deployment calls // them, and the classifier scores onto those names. let targets = StageTargets::new(capable.clone(), efficient.clone()); @@ -205,6 +209,7 @@ pub(crate) fn build_stage_route( } let signals = ToolSignalProcessor { recent_window: config.recent_window.unwrap_or(DEFAULT_RECENT_WINDOW), + tool_semantics: config.tool_semantics, }; let target_set = vec![capable.clone(), efficient.clone()]; let mut router = FallThrough::::new_with_state(target_set) diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rs index d5da79596..95b802b61 100644 --- a/crates/libsy/src/algorithms/util/stage.rs +++ b/crates/libsy/src/algorithms/util/stage.rs @@ -257,9 +257,9 @@ impl ScoreResult { pub struct CodingAgentDimensions { /// Windowed max error severity in `[0, 1]`. pub severity: f64, - /// `1.0` when a deep turn has no reads, plans, writes, or edits (pure churn). + /// `1.0` when a deep turn has no recognized observation, mutation, plan, or new activity. pub spinning: f64, - /// `1.0` when a deep turn reads/plans but does not write or edit. + /// `1.0` when a deep turn observes/plans but does not mutate or show new activity. pub exploring: f64, /// Fraction of recent tool ops that produced code (writes + edits). pub production_intensity: f64, @@ -302,10 +302,11 @@ pub fn dimensions_from_signal(signal: &ToolSignals) -> CodingAgentDimensions { let deep_enough = signal.turn_depth >= STALL_MIN_TURN_DEPTH; let no_production = signal.recent_write_count == 0 && signal.recent_edit_count == 0; let investigating = signal.recent_read_count >= 1 || signal.recent_todowrite_count >= 1; + let new_activity = signal.recent_new_count >= 1; // spinning vs exploring partition the "not producing" case by investigative // activity, so at most one fires — no double-counting on the production axis. - let spinning = deep_enough && no_production && !investigating; - let exploring = deep_enough && no_production && investigating; + let spinning = deep_enough && no_production && !investigating && !new_activity; + let exploring = deep_enough && no_production && investigating && !new_activity; CodingAgentDimensions { severity: f64::from(signal.severity), @@ -746,6 +747,21 @@ mod tests { } } + #[test] + fn new_activity_suppresses_stall_dimensions_without_biasing_the_score() { + let signal = ToolSignals { + turn_depth: STALL_MIN_TURN_DEPTH, + recent_new_count: 1, + ..Default::default() + }; + + let dimensions = dimensions_from_signal(&signal); + + assert_eq!(dimensions.spinning, 0.0); + assert_eq!(dimensions.exploring, 0.0); + assert_eq!(score_signal(&signal).score, 0.0); + } + // ─── StageClassifier ───────────────────────────────────────────────── /// Tiers named the way a deployment would name them. diff --git a/crates/libsy/src/algorithms/util/tool_signals.rs b/crates/libsy/src/algorithms/util/tool_signals.rs index f24626a51..711d197bc 100644 --- a/crates/libsy/src/algorithms/util/tool_signals.rs +++ b/crates/libsy/src/algorithms/util/tool_signals.rs @@ -13,10 +13,11 @@ #![allow(dead_code)] use async_trait::async_trait; +use serde::Deserialize; use serde_json::Value; use switchyard_protocol::{ContentBlock, Request, Role}; -use crate::Result; +use crate::{LibsyError, Result}; use crate::core::processor::{Event, Processor}; use crate::core::state::State; @@ -203,6 +204,83 @@ static NUMERIC_FAILURE_KEYWORDS: &[&str] = &["failed", "failure", "failures", "e /// passing a window to [`ToolSignals::from_request`]. pub const DEFAULT_RECENT_WINDOW: usize = 3; +/// Exact tool-name semantics added to the stage router's built-in vocabulary. +/// +/// Matching is ASCII case-insensitive. These lists are additive: built-in tool +/// names cannot be reclassified. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)] +#[serde(default, deny_unknown_fields)] +pub struct ToolSemantics { + /// Read-only lookup or inspection tools. + pub observe: Vec, + /// Tools that change task or external state. + pub mutate: Vec, + /// Explicit planning or task-decomposition tools. + pub plan: Vec, + /// Tools that demonstrate new forward activity without favoring either tier. + pub new: Vec, +} + +impl ToolSemantics { + /// Rejects ambiguous mappings and attempts to reclassify built-in tools. + pub fn validate(&self) -> Result<()> { + let mut seen: Vec<(String, &'static str)> = Vec::new(); + for (category, names) in [ + ("observe", &self.observe), + ("mutate", &self.mutate), + ("plan", &self.plan), + ("new", &self.new), + ] { + for name in names { + if name.trim().is_empty() { + return Err(tool_semantics_error(format!( + "tool_semantics.{category} contains an empty tool name" + ))); + } + let normalized = name.to_ascii_lowercase(); + if is_builtin_tool_name(&normalized) { + return Err(tool_semantics_error(format!( + "tool {name:?} already has built-in semantics and cannot be reclassified" + ))); + } + if let Some((_, previous)) = seen.iter().find(|(seen, _)| seen == &normalized) { + return Err(tool_semantics_error(format!( + "tool {name:?} appears in both tool_semantics.{previous} and tool_semantics.{category}" + ))); + } + seen.push((normalized, category)); + } + } + Ok(()) + } + + fn classify(&self, name: &str) -> Option { + if contains_name(&self.observe, name) { + Some(ToolSemantic::Observe) + } else if contains_name(&self.mutate, name) { + // The stage scorer treats writes and edits identically. Custom + // mutations use the write counter to preserve the public signal shape. + Some(ToolSemantic::Mutate(MutationKind::Write)) + } else if contains_name(&self.plan, name) { + Some(ToolSemantic::Plan) + } else if contains_name(&self.new, name) { + Some(ToolSemantic::New) + } else { + None + } + } +} + +fn contains_name(names: &[String], candidate: &str) -> bool { + names + .iter() + .any(|name| name.eq_ignore_ascii_case(candidate)) +} + +fn tool_semantics_error(message: String) -> LibsyError { + LibsyError::AlgorithmError { message } +} + // ─── output type ───────────────────────────────────────────────────────────── /// Tool-execution signals extracted from a normalized [`Request`]. @@ -237,7 +315,11 @@ pub struct ToolSignals { pub recent_read_count: u32, /// TodoWrite calls within the configured recent window (default: [`DEFAULT_RECENT_WINDOW`]). pub recent_todowrite_count: u32, - /// Consecutive trailing tool calls in the `Other` category (no Write/Edit/Read/ + /// Configured `new` tool calls across the full request history. + pub new_count: u32, + /// Configured `new` tool calls within the recent window. + pub recent_new_count: u32, + /// Consecutive trailing tool calls in the `Unknown` category (no Write/Edit/Read/ /// Plan match). Surfaced in the classifier state summary; not scored directly. pub pure_bash_streak: u32, /// At least one of the last three tool results matched a test-pass pattern. @@ -261,12 +343,25 @@ pub struct ToolSignals { } impl ToolSignals { - /// Extracts tool and progress signals from `request`. + /// Extracts tool and activity signals from `request`. /// /// `window_size` limits recent counters to the newest tool results. `None` /// uses [`DEFAULT_RECENT_WINDOW`]. pub fn from_request(request: &Request, window_size: Option) -> Self { - extract_tool_signals_with_window(request, window_size.unwrap_or(DEFAULT_RECENT_WINDOW)) + Self::from_request_with_semantics(request, window_size, &ToolSemantics::default()) + } + + /// Extracts signals using the built-in vocabulary plus additive semantics. + pub fn from_request_with_semantics( + request: &Request, + window_size: Option, + semantics: &ToolSemantics, + ) -> Self { + extract_tool_signals_with_window_and_semantics( + request, + window_size.unwrap_or(DEFAULT_RECENT_WINDOW), + semantics, + ) } } @@ -289,6 +384,7 @@ enum ToolSemantic { Mutate(MutationKind), Observe, Plan, + New, Unknown, } @@ -299,12 +395,15 @@ pub struct ToolSignalProcessor { /// Number of trailing tool results the `recent_*` counts and windowed /// severity are computed over. pub recent_window: usize, + /// Route-scoped additions to the built-in tool vocabulary. + pub tool_semantics: ToolSemantics, } impl Default for ToolSignalProcessor { fn default() -> Self { Self { recent_window: DEFAULT_RECENT_WINDOW, + tool_semantics: ToolSemantics::default(), } } } @@ -313,7 +412,11 @@ impl Default for ToolSignalProcessor { impl Processor for ToolSignalProcessor { async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> { if let Event::Request { request: req, .. } = event { - let tool_signal = ToolSignals::from_request(req, Some(self.recent_window)); + let tool_signal = ToolSignals::from_request_with_semantics( + req, + Some(self.recent_window), + &self.tool_semantics, + ); state.tool_signals = Some(tool_signal); } Ok(()) @@ -321,6 +424,14 @@ impl Processor for ToolSignalProcessor { } fn classify_tool_call(name: &str, command: Option<&str>) -> ToolSemantic { + classify_tool_call_with_semantics(name, command, &ToolSemantics::default()) +} + +fn classify_tool_call_with_semantics( + name: &str, + command: Option<&str>, + semantics: &ToolSemantics, +) -> ToolSemantic { let lower = name.to_lowercase(); if WRITE_TOOL_NAMES.contains(&lower.as_str()) { return ToolSemantic::Mutate(MutationKind::Write); @@ -351,7 +462,15 @@ fn classify_tool_call(name: &str, command: Option<&str>) -> ToolSemantic { return ToolSemantic::Observe; } } - ToolSemantic::Unknown + semantics.classify(&lower).unwrap_or(ToolSemantic::Unknown) +} + +fn is_builtin_tool_name(lower: &str) -> bool { + WRITE_TOOL_NAMES.contains(&lower) + || EDIT_TOOL_NAMES.contains(&lower) + || READ_TOOL_NAMES.contains(&lower) + || PLAN_TOOL_NAMES.contains(&lower) + || BASH_TOOL_NAMES.contains(&lower) } // ─── extraction entry point ─────────────────────────────────────────────────── @@ -361,6 +480,18 @@ fn classify_tool_call(name: &str, command: Option<&str>) -> ToolSemantic { /// Returns [`ToolSignals::default()`] when the message history contains no tool /// activity, so callers can always inspect the signal fields. fn extract_tool_signals_with_window(request: &Request, recent_window: usize) -> ToolSignals { + extract_tool_signals_with_window_and_semantics( + request, + recent_window, + &ToolSemantics::default(), + ) +} + +fn extract_tool_signals_with_window_and_semantics( + request: &Request, + recent_window: usize, + semantics: &ToolSemantics, +) -> ToolSignals { // Read the decoded conversation, not the raw body: every inbound format lands // in the same shape here, so the signals do not depend on knowing which one it // arrived as. @@ -407,7 +538,13 @@ fn extract_tool_signals_with_window(request: &Request, recent_window: usize) -> } } - let mut signal = build_signal(tool_texts, tool_calls, messages.len() as u32, recent_window); + let mut signal = build_signal( + tool_texts, + tool_calls, + messages.len() as u32, + recent_window, + semantics, + ); signal.compacted = compacted; signal.tool_result_count = u32::try_from(tool_result_count).unwrap_or(u32::MAX); signal.assistant_turn_count = u32::try_from(assistant_turn_count).unwrap_or(u32::MAX); @@ -462,6 +599,7 @@ fn build_signal( tool_calls: Vec, turn_depth: u32, recent_window: usize, + semantics: &ToolSemantics, ) -> ToolSignals { // Windowed severity: take the MAX severity across the last `recent_window` tool // results rather than only the last one. An error's severity then persists for @@ -491,10 +629,12 @@ fn build_signal( let mut recent_edit_count = 0u32; let mut recent_read_count = 0u32; let mut recent_todowrite_count = 0u32; + let mut new_count = 0u32; + let mut recent_new_count = 0u32; let mut pure_bash_streak = 0u32; let mut streak_open = true; for (i, tc) in tool_calls.iter().enumerate().rev() { - let cat = classify_tool_call(&tc.name, tc.command.as_deref()); + let cat = classify_tool_call_with_semantics(&tc.name, tc.command.as_deref(), semantics); if streak_open { if matches!(cat, ToolSemantic::Unknown) { pure_bash_streak += 1; @@ -527,6 +667,12 @@ fn build_signal( recent_todowrite_count += 1; } } + ToolSemantic::New => { + new_count += 1; + if i >= recent_start { + recent_new_count += 1; + } + } ToolSemantic::Unknown => {} } } @@ -544,6 +690,8 @@ fn build_signal( recent_write_count, recent_read_count, recent_todowrite_count, + new_count, + recent_new_count, pure_bash_streak, tests_passed, turn_depth, @@ -1126,7 +1274,7 @@ mod tests { classify_tool_call("shell_command", Some("ls /app")), ToolSemantic::Observe, ); - // shell_command without matching patterns -> Other. + // shell_command without matching patterns -> Unknown. assert_eq!( classify_tool_call("shell_command", Some("./run_tests.sh")), ToolSemantic::Unknown, @@ -1246,4 +1394,58 @@ mod tests { assert_eq!(sig.read_count, 1); assert_eq!(sig.recent_read_count, 1); } + + #[test] + fn configured_tool_semantics_extend_the_builtin_vocabulary() { + let semantics = ToolSemantics { + observe: vec!["KB_search".to_string()], + mutate: vec!["send_payment_request".to_string()], + plan: vec!["create_research_plan".to_string()], + new: vec!["send_message_to_user".to_string()], + }; + semantics.validate().expect("valid additive semantics"); + let request = with_messages(vec![ + tc("kb_SEARCH"), + tc("send_payment_request"), + tc("create_research_plan"), + tc("send_message_to_user"), + ]); + + let signal = ToolSignals::from_request_with_semantics(&request, None, &semantics); + + assert_eq!(signal.read_count, 1); + assert_eq!(signal.write_count, 1); + assert_eq!(signal.todowrite_count, 1); + assert_eq!(signal.new_count, 1); + assert_eq!(signal.recent_new_count, 1); + assert_eq!(signal.pure_bash_streak, 0); + } + + #[test] + fn tool_semantics_reject_duplicates_and_builtin_reclassification() { + let duplicate = ToolSemantics { + observe: vec!["lookup".to_string()], + mutate: vec!["LOOKUP".to_string()], + ..Default::default() + }; + assert!( + duplicate + .validate() + .expect_err("duplicate should fail") + .to_string() + .contains("appears in both") + ); + + let builtin = ToolSemantics { + new: vec!["write_file".to_string()], + ..Default::default() + }; + assert!( + builtin + .validate() + .expect_err("built-in should fail") + .to_string() + .contains("built-in semantics") + ); + } } diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs index ad70b8da2..7999f2428 100644 --- a/crates/libsy/src/lib.rs +++ b/crates/libsy/src/lib.rs @@ -33,7 +33,7 @@ pub use algorithms::util::classifier_contract::{ pub use algorithms::util::escalation::EscalationJudgeConfig; pub use algorithms::util::prompts::{SystemPromptProcessor, TargetPrompts, append_note}; pub use algorithms::util::subagent::{SubagentGate, SubagentOverride}; -pub use algorithms::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSignals}; +pub use algorithms::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSemantics, ToolSignals}; // Stage-router scoring and tier selection — the shared signal-driven routing // core (scorer, picker, and the `StageClassifier`). diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index 79adca287..cc7801a42 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -16,7 +16,7 @@ use switchyard_libsy::{ CustomClassifierConfig, CustomClassifierPolicy, EscalationJudgeConfig, HandoffNoteConfig, LibsyError as RustLibsyError, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, PickerMode, Random, RoutingOutcome, StageRouter, StageRouterConfig, Step as RustStep, - StepStream, TaskClassifierConfig, + StepStream, TaskClassifierConfig, ToolSemantics, }; use switchyard_protocol::{ LlmClientError, LlmResponse, LlmResponseStream, LlmResponseStreamEvent, Metadata, ModelId, @@ -815,6 +815,7 @@ fn build_llm_classifier(config: LlmClassifierConfig) -> PyResult { only_on_wrong_signal_escalation=true, capable_system_prompt=None, efficient_system_prompt=None, + tool_semantics=None, classifier=None ))] #[allow(clippy::too_many_arguments)] @@ -830,6 +831,7 @@ fn stage_router_algorithm( only_on_wrong_signal_escalation: bool, capable_system_prompt: Option, efficient_system_prompt: Option, + tool_semantics: Option>>, classifier: Option>, ) -> PyResult { let mode = match picker { @@ -864,6 +866,19 @@ fn stage_router_algorithm( if let Some(prompt) = efficient_system_prompt { config.tier_prompts = config.tier_prompts.with(efficient.clone(), prompt); } + if let Some(mut semantics) = tool_semantics { + config.tool_semantics = ToolSemantics { + observe: semantics.remove("observe").unwrap_or_default(), + mutate: semantics.remove("mutate").unwrap_or_default(), + plan: semantics.remove("plan").unwrap_or_default(), + new: semantics.remove("new").unwrap_or_default(), + }; + if let Some(category) = semantics.keys().next() { + return Err(PyValueError::new_err(format!( + "unknown tool_semantics category {category:?}; expected observe, mutate, plan, or new" + ))); + } + } config.llm_fallback = classifier .map(|classifier| classifier.bind(py).try_borrow()?.clone_core(py)) .transpose()?; diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index e5db6d477..21a8fa9a0 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -15,6 +15,7 @@ use libsy::{ CustomClassifierPolicy, EscalationJudgeConfig, GateTrigger, HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough, PickerMode, Random, StageRouter, StageRouterConfig, SubagentRouter, SubagentRouterConfig, TaskClassifierConfig, + ToolSemantics, }; use serde::Deserialize; use switchyard_protocol::ModelId; @@ -401,6 +402,9 @@ pub struct StageTierConfig { /// How many trailing tool results the signals are scored over. #[serde(default)] pub recent_turn_window: Option, + /// Exact tool-name semantics added to the built-in stage vocabulary. + #[serde(default)] + pub tool_semantics: ToolSemantics, /// Notes handed to a tier when the router switches to it. #[serde(default)] pub handoff_notes: Option, @@ -1004,6 +1008,7 @@ fn build_algorithm( efficient_target, confidence_threshold, recent_turn_window, + tool_semantics, handoff_notes, } = tiers; if matches!(picker, PickerMode::CapableFirst) { @@ -1015,6 +1020,7 @@ fn build_algorithm( let efficient = resolve_target_model_id(route_name, efficient_target, targets)?; let mut config = StageRouterConfig::new(*picker, *confidence_threshold); config.recent_window = *recent_turn_window; + config.tool_semantics = tool_semantics.clone(); config.handoff_notes = handoff_notes.clone(); // The judge is called through its own target, so it is not a routing // destination and stays out of the tier pair. @@ -1064,6 +1070,7 @@ fn build_algorithm( let mut stage_config = StageRouterConfig::new(PickerMode::EfficientFirst, stage.confidence_threshold); stage_config.recent_window = stage.recent_turn_window; + stage_config.tool_semantics = stage.tool_semantics.clone(); stage_config.handoff_notes = stage.handoff_notes.clone(); let config = CompositeRouterConfig { judge_target, diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index 5317df264..6a792df37 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -747,6 +747,12 @@ efficient_target = "weak" picker = "efficient_first" confidence_threshold = 1.0 +[routes.stage.tool_semantics] +observe = ["lookup_customer"] +mutate = ["send_payment"] +plan = ["create_workflow"] +new = ["send_message"] + [routes.stage.classifier] target = "stage_judge" base_threshold = 0.5 @@ -774,6 +780,9 @@ classify_trigger = "user_turn" capable_target = "strong" efficient_target = "weak" confidence_threshold = 0.5 + +[routes.composed.stage.tool_semantics] +new = ["send_message"] "# ) } diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 0f6567092..5bd6b5887 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -1425,6 +1425,73 @@ base_threshold = 0.5 Ok(()) } +#[tokio::test] +async fn stage_router_uses_configured_tool_semantics() -> TestResult { + let upstream = MockUpstream::start().await?; + let state = load_test_config(&format!( + r#" +schema_version = 1 + +[llm_clients.upstream] +format = "openai_chat" +base_url = "{base_url}" + +[targets.strong] +id = "model/strong" +llm_client = "upstream" + +[targets.weak] +id = "model/weak" +llm_client = "upstream" + +[routes.stage] +id = "switchyard/stage" +type = "stage_router" +capable_target = "strong" +efficient_target = "weak" +picker = "capable_first" +confidence_threshold = 0.3 + +[routes.stage.tool_semantics] +mutate = ["send_payment_request"] +"#, + base_url = upstream.base_url + ))?; + let app = build_switchyard_router(state); + + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "switchyard/stage", + "messages": [ + {"role": "user", "content": "pay the balance"}, + {"role": "assistant", "tool_calls": [{ + "id": "call_1", + "type": "function", + "function": { + "name": "send_payment_request", + "arguments": "{}" + } + }]}, + {"role": "tool", "tool_call_id": "call_1", "content": "payment sent"} + ] + })), + ) + .await?; + + assert_eq!(response.status, StatusCode::OK); + assert_eq!( + response + .headers + .get("x-model-router-selected-model") + .and_then(|value| value.to_str().ok()), + Some("model/weak") + ); + Ok(()) +} + #[tokio::test] async fn custom_classifier_routes_four_targets_and_falls_back_on_an_invalid_verdict() -> TestResult { diff --git a/docs/routing_algorithms/stage_router_routing.md b/docs/routing_algorithms/stage_router_routing.md index 95c484721..6e7ff9ac1 100644 --- a/docs/routing_algorithms/stage_router_routing.md +++ b/docs/routing_algorithms/stage_router_routing.md @@ -15,10 +15,9 @@ remaining targets in configured order for that request. See ## How it works -A coding agent's run moves through stages: early on it explores the codebase and -recovers from errors, and later it settles into more mechanical implementation. -Those stages call for different amounts of model capability, which is what the -router keys on. +A tool-using agent's run moves through stages that call for different amounts of +model capability. The built-in vocabulary is calibrated for coding agents: it +recognizes file observation, mutation, planning, shell activity, and test results. For each LLM call, stage-router estimates which stage the agent is in from the **tool-result history** on the conversation, scoring two axes: @@ -210,6 +209,35 @@ switchyard-server --config routes.toml --port 4000 This is the recommended default: routing on tool signals alone, no classifier. +### Optional: custom tool semantics + +Extend the built-in coding vocabulary when an agent uses domain-specific tool +names. Mappings are route-scoped, additive, and matched by exact name without +regard to ASCII case: + +```toml +[routes.stage.tool_semantics] +observe = ["KB_search", "get_customer_by_phone"] +mutate = ["send_payment_request", "update_inventory"] +plan = ["create_research_plan"] +new = ["start_conversation", "send_message_to_user"] +``` + +The categories affect existing stage signals: + +- `observe` counts as investigation, like the built-in read and search tools. +- `mutate` counts as production, like the built-in write and edit tools. +- `plan` counts as investigation, like the built-in planning tools. +- `new` records forward activity that suppresses false `spinning` and + `exploring` signals, but does not otherwise favor either tier. +- `unknown` remains the fallback for tools that match neither the built-in + vocabulary nor configured semantics. + +Configuration cannot reclassify a built-in tool. Empty names, duplicate names +across categories, and unknown category keys are rejected when the route is +loaded. Argument-aware wrapper tools, inferred semantics, and learned routing +rules are outside this exact-name configuration. + ### Optional: handoff notes Add a `[routes.stage.handoff_notes]` section to pass a contextual note to the diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py index b693d4d87..eb33f2cbd 100644 --- a/tests/test_libsy_minimal_bindings.py +++ b/tests/test_libsy_minimal_bindings.py @@ -436,3 +436,31 @@ async def call(self, request: dict[str, Any]) -> dict[str, Any]: assert selected_model == "fast" assert response["model"] == "strong" + + +def test_stage_router_accepts_additive_tool_semantics() -> None: + algorithm = algorithms.stage_router( + "strong", + "fast", + picker="efficient_first", + confidence_threshold=0.5, + tool_semantics={ + "observe": ["KB_search"], + "mutate": ["send_payment_request"], + "plan": ["create_research_plan"], + "new": ["send_message_to_user"], + }, + ) + + assert callable(algorithm.run_stream) + + +def test_stage_router_rejects_unknown_tool_semantics_category() -> None: + with pytest.raises(ValueError, match="unknown tool_semantics category"): + algorithms.stage_router( + "strong", + "fast", + picker="efficient_first", + confidence_threshold=0.5, + tool_semantics={"complete": ["end_conversation"]}, + ) From 13f8b857f9b11aac2d5dfe7442f6c3411fed40f4 Mon Sep 17 00:00:00 2001 From: Sean Lopp Date: Wed, 2 Sep 2026 15:46:54 -0600 Subject: [PATCH 3/6] docs(stage-router): document custom tool semantics Signed-off-by: Sean Lopp --- crates/switchyard-server/README.md | 11 +++++++++-- docs/getting_started.md | 2 +- docs/reference/toml_schema.md | 8 ++++++++ docs/routing_algorithms/composite_routing.md | 7 +++++++ docs/routing_algorithms/overview.md | 2 +- docs/routing_algorithms/subagent_routing.md | 3 +++ switchyard_rust/libsy.py | 1 + 7 files changed, 30 insertions(+), 4 deletions(-) diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index b14b7fa37..06c3808c9 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -52,6 +52,11 @@ capable_target = "model_a" efficient_target = "model_b" picker = "efficient_first" confidence_threshold = 0.5 + +[routes.stage.tool_semantics] +observe = ["lookup_customer"] +mutate = ["update_inventory"] +new = ["send_message"] ``` ```bash @@ -121,8 +126,10 @@ fallback produced while the judge was unreachable. `message_hash_fallback` keys content rather than a session id, so unrelated callers sending identical text share one assignment. -A `stage_router` route scores tool-result and agent-progress signals from recent turns to pick a -tier per turn, without an extra classifier call on every turn. `capable_target`, +A `stage_router` route scores tool-result and activity signals from recent turns to pick a +tier per turn, without an extra classifier call on every turn. Domain-specific exact tool names +can extend its built-in coding vocabulary through `tool_semantics.observe`, +`tool_semantics.mutate`, `tool_semantics.plan`, and neutral `tool_semantics.new`. `capable_target`, `efficient_target`, `picker` (`efficient_first` or `capable_first`), and `confidence_threshold` are required. Optional handoff notes, per-tier system prompts, and a capability-judge fallback are documented in [Stage-Router Routing](../../docs/routing_algorithms/stage_router_routing.md). diff --git a/docs/getting_started.md b/docs/getting_started.md index 7c1cb362b..5f61e8034 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -136,7 +136,7 @@ settings. The Rust server also supports: | Auto | You want a recommended default instead of picking a strategy yourself. | `auto` | | [Random](routing_algorithms/random_routing.md) | You need a weighted split for A/B tests or baselines. | `random` | | [LLM classifier](routing_algorithms/llm_classifier_routing.md) | Request content should decide whether to use the weak or strong target. | `llm_classifier` | -| [Stage router](routing_algorithms/stage_router_routing.md) | Tool-result and progress signals should select an efficient or capable target. | `stage_router` | +| [Stage router](routing_algorithms/stage_router_routing.md) | Built-in or configured tool-activity signals should select an efficient or capable target. | `stage_router` | A single TOML file can declare multiple routes. The table key, such as `routes.smart`, is a local configuration name; each route's `id` is exposed as a diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index a819a6178..93dcaa7b6 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -248,6 +248,10 @@ optional `handoff_notes` and `classifier` tables and for tuning. | `picker` | Yes | — | `efficient_first`, or `capable_first` (experimental, unbenchmarked). Tier used when the signals are not confident. | | `confidence_threshold` | Yes | — | Corroboration a decisive pick needs. In `[0, 1]`. | | `recent_turn_window` | No | `3` | Trailing tool results the signals are computed over. | +| `tool_semantics.observe` | No | `[]` | Exact domain tool names that count as read-only investigation. Matching ignores ASCII case. | +| `tool_semantics.mutate` | No | `[]` | Exact domain tool names that count as state-changing production. | +| `tool_semantics.plan` | No | `[]` | Exact domain tool names that count as planning or task decomposition. | +| `tool_semantics.new` | No | `[]` | Exact domain tool names that demonstrate forward activity without favoring either tier. | | `classifier.classify_trigger` | No | `every_request` | When the judge runs. See the `llm_classifier` route. `new_session` has no effect here. | | `classifier.response_format_type` | No | `json_schema` | Structured-output mode for the optional classifier judge. Use `json_object` when the classifier provider does not support JSON Schema; Switchyard adds the schema to the prompt and validates the verdict locally. | | `subagents` | No | unset | Nested `passthrough` or custom `llm_classifier` policy used only for delegated sub-agent work. See [Sub-Agent-Aware Routing](../routing_algorithms/subagent_routing.md). | @@ -283,6 +287,10 @@ configuration. Today a classifier sets the tier a stage router falls open to whe | `stage.efficient_target` | Yes | — | Efficient tier. | | `stage.confidence_threshold` | Yes | — | Corroboration a decisive signal needs. In `[0, 1]`. | | `stage.recent_turn_window` | No | `3` | Trailing tool results the signals are computed over. | +| `stage.tool_semantics.observe` | No | `[]` | Additional exact tool names that count as observation. | +| `stage.tool_semantics.mutate` | No | `[]` | Additional exact tool names that count as mutation. | +| `stage.tool_semantics.plan` | No | `[]` | Additional exact tool names that count as planning. | +| `stage.tool_semantics.new` | No | `[]` | Additional exact tool names that count as neutral forward activity. | | `subagents` | No | unset | Nested policy used only for delegated sub-agent work. | The tier is retained per session. A deployment that sends no session ID needs diff --git a/docs/routing_algorithms/composite_routing.md b/docs/routing_algorithms/composite_routing.md index f11226935..076f23bd0 100644 --- a/docs/routing_algorithms/composite_routing.md +++ b/docs/routing_algorithms/composite_routing.md @@ -33,6 +33,11 @@ classify_trigger = "user_turn" capable_target = "strong" efficient_target = "weak" confidence_threshold = 0.5 + +[routes.switchyard.stage.tool_semantics] +observe = ["lookup_customer"] +mutate = ["update_inventory"] +new = ["send_message"] ``` `[routes.switchyard.classifier]` takes the `stage_router` classifier fields. @@ -41,6 +46,8 @@ whenever the user speaks, `new_session` picks once and holds it. `[routes.switchyard.stage]` takes the `stage_router` fields except `picker`, whose job the classifier does per turn. Leave `classifier` out as well: that judge runs ahead of the fall-open tier, so it answers most of the turns this one decided. +Custom `tool_semantics` remain stage-local and have the same additive behavior as +on a standalone stage route. ## Sessions diff --git a/docs/routing_algorithms/overview.md b/docs/routing_algorithms/overview.md index 6350bc24e..074b3c151 100644 --- a/docs/routing_algorithms/overview.md +++ b/docs/routing_algorithms/overview.md @@ -15,7 +15,7 @@ configuration and tuning. For the vocabulary these pages use, see | [Sub-Agent-Aware Routing](subagent_routing.md) | Delegated sub-agents should use a separate routing policy from the parent agent. | `passthrough` or `stage_router` with `subagents` | | [Random Routing](random_routing.md) | You need a fixed traffic split for A/B tests, baselines, or cost experiments. | `random` | | [LLM Classifier Routing](llm_classifier_routing.md) | Request content should decide whether a turn needs the weak or strong tier. | `llm_classifier` | -| [Stage-Router Routing](stage_router_routing.md) | Tool-result and agent-progress signals should route most turns without an extra classifier call. | `stage_router` | +| [Stage-Router Routing](stage_router_routing.md) | Built-in or configured tool-activity signals should route most turns without an extra classifier call. | `stage_router` | | Auto Routing | You want a recommended default instead of picking a strategy yourself. For a deeper dive on the current default, see [Stage-Router Routing](stage_router_routing.md); for full control, pick one of the strategies above instead. | `auto` | | [Composite Routing](composite_routing.md) | Routing algorithms are composed, one setting the configuration of another before handing off. Today an LLM classifier sets a stage router's default tier. | `composite` | | [Escalation-Router Routing](escalation_router_routing.md) | Start every task on the weak tier and escalate to strong when an LLM judge detects trouble. | `llm_classifier` with `escalation` | diff --git a/docs/routing_algorithms/subagent_routing.md b/docs/routing_algorithms/subagent_routing.md index 2533e80ea..ab201f2a0 100644 --- a/docs/routing_algorithms/subagent_routing.md +++ b/docs/routing_algorithms/subagent_routing.md @@ -98,6 +98,9 @@ tool_calling = true reasoning = true ``` +The parent stage route may also declare `[routes.agent.tool_semantics]` to map +domain-specific tools; delegated sub-agent policy configuration is unaffected. + Clients must still request the route ID (`agent` above). An explicit model name that is not registered as a route is rejected before sub-agent classification. `message_hash_fallback` is not supported for sub-agent routing because affinity diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py index 9756b1c6b..0ce4968a1 100644 --- a/switchyard_rust/libsy.py +++ b/switchyard_rust/libsy.py @@ -261,6 +261,7 @@ def stage_router( only_on_wrong_signal_escalation: bool = True, capable_system_prompt: str | None = None, efficient_system_prompt: str | None = None, + tool_semantics: Mapping[str, Sequence[str]] | None = None, classifier: LlmFallback | None = None, ) -> Algorithm: ... From 288a9c3fa72ef9f69d38afac11e027f066d36fcd Mon Sep 17 00:00:00 2001 From: Sean Lopp Date: Wed, 2 Sep 2026 17:55:06 -0600 Subject: [PATCH 4/6] fix(stage-router): address tool semantics review Signed-off-by: Sean Lopp --- .../libsy/src/algorithms/util/tool_signals.rs | 20 +++++++- crates/switchyard-server/README.md | 5 +- crates/switchyard-server/tests/server.rs | 1 + docs/reference/toml_schema.md | 16 +++---- docs/routing_algorithms/composite_routing.md | 1 + switchyard_rust/libsy.py | 2 +- tests/test_libsy_minimal_bindings.py | 46 +++++++++++++++++-- 7 files changed, 75 insertions(+), 16 deletions(-) diff --git a/crates/libsy/src/algorithms/util/tool_signals.rs b/crates/libsy/src/algorithms/util/tool_signals.rs index 711d197bc..9f51c2db0 100644 --- a/crates/libsy/src/algorithms/util/tool_signals.rs +++ b/crates/libsy/src/algorithms/util/tool_signals.rs @@ -432,7 +432,8 @@ fn classify_tool_call_with_semantics( command: Option<&str>, semantics: &ToolSemantics, ) -> ToolSemantic { - let lower = name.to_lowercase(); + // Built-in names and Bash command inference take precedence over route-scoped mappings. + let lower = name.to_ascii_lowercase(); if WRITE_TOOL_NAMES.contains(&lower.as_str()) { return ToolSemantic::Mutate(MutationKind::Write); } @@ -1421,6 +1422,23 @@ mod tests { assert_eq!(signal.pure_bash_streak, 0); } + #[test] + fn configured_tool_semantics_only_fold_ascii_case() { + let semantics = ToolSemantics { + observe: vec!["kb_search".to_string()], + ..Default::default() + }; + + assert_eq!( + classify_tool_call_with_semantics("KB_SEARCH", None, &semantics), + ToolSemantic::Observe + ); + assert_eq!( + classify_tool_call_with_semantics("KB_SEARCH", None, &semantics), + ToolSemantic::Unknown + ); + } + #[test] fn tool_semantics_reject_duplicates_and_builtin_reclassification() { let duplicate = ToolSemantics { diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index 06c3808c9..b4631b26f 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -131,8 +131,9 @@ tier per turn, without an extra classifier call on every turn. Domain-specific e can extend its built-in coding vocabulary through `tool_semantics.observe`, `tool_semantics.mutate`, `tool_semantics.plan`, and neutral `tool_semantics.new`. `capable_target`, `efficient_target`, `picker` (`efficient_first` or `capable_first`), and `confidence_threshold` -are required. Optional handoff notes, per-tier system prompts, and a capability-judge fallback are -documented in [Stage-Router Routing](../../docs/routing_algorithms/stage_router_routing.md). +are required. All configured semantic names use exact ASCII case-insensitive matching. Optional +handoff notes, per-tier system prompts, and a capability-judge fallback are documented in +[Stage-Router Routing](../../docs/routing_algorithms/stage_router_routing.md). ## Endpoints diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 5bd6b5887..933b93379 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -1425,6 +1425,7 @@ base_threshold = 0.5 Ok(()) } +// A configured mutation must select the efficient tier through the HTTP configuration path. #[tokio::test] async fn stage_router_uses_configured_tool_semantics() -> TestResult { let upstream = MockUpstream::start().await?; diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 93dcaa7b6..3df685872 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -248,10 +248,10 @@ optional `handoff_notes` and `classifier` tables and for tuning. | `picker` | Yes | — | `efficient_first`, or `capable_first` (experimental, unbenchmarked). Tier used when the signals are not confident. | | `confidence_threshold` | Yes | — | Corroboration a decisive pick needs. In `[0, 1]`. | | `recent_turn_window` | No | `3` | Trailing tool results the signals are computed over. | -| `tool_semantics.observe` | No | `[]` | Exact domain tool names that count as read-only investigation. Matching ignores ASCII case. | -| `tool_semantics.mutate` | No | `[]` | Exact domain tool names that count as state-changing production. | -| `tool_semantics.plan` | No | `[]` | Exact domain tool names that count as planning or task decomposition. | -| `tool_semantics.new` | No | `[]` | Exact domain tool names that demonstrate forward activity without favoring either tier. | +| `tool_semantics.observe` | No | `[]` | Exact ASCII case-insensitive domain tool names that count as read-only investigation. | +| `tool_semantics.mutate` | No | `[]` | Exact ASCII case-insensitive domain tool names that count as state-changing production. | +| `tool_semantics.plan` | No | `[]` | Exact ASCII case-insensitive domain tool names that count as planning or task decomposition. | +| `tool_semantics.new` | No | `[]` | Exact ASCII case-insensitive domain tool names that demonstrate forward activity without favoring either tier. | | `classifier.classify_trigger` | No | `every_request` | When the judge runs. See the `llm_classifier` route. `new_session` has no effect here. | | `classifier.response_format_type` | No | `json_schema` | Structured-output mode for the optional classifier judge. Use `json_object` when the classifier provider does not support JSON Schema; Switchyard adds the schema to the prompt and validates the verdict locally. | | `subagents` | No | unset | Nested `passthrough` or custom `llm_classifier` policy used only for delegated sub-agent work. See [Sub-Agent-Aware Routing](../routing_algorithms/subagent_routing.md). | @@ -287,10 +287,10 @@ configuration. Today a classifier sets the tier a stage router falls open to whe | `stage.efficient_target` | Yes | — | Efficient tier. | | `stage.confidence_threshold` | Yes | — | Corroboration a decisive signal needs. In `[0, 1]`. | | `stage.recent_turn_window` | No | `3` | Trailing tool results the signals are computed over. | -| `stage.tool_semantics.observe` | No | `[]` | Additional exact tool names that count as observation. | -| `stage.tool_semantics.mutate` | No | `[]` | Additional exact tool names that count as mutation. | -| `stage.tool_semantics.plan` | No | `[]` | Additional exact tool names that count as planning. | -| `stage.tool_semantics.new` | No | `[]` | Additional exact tool names that count as neutral forward activity. | +| `stage.tool_semantics.observe` | No | `[]` | Additional exact ASCII case-insensitive tool names that count as observation. | +| `stage.tool_semantics.mutate` | No | `[]` | Additional exact ASCII case-insensitive tool names that count as mutation. | +| `stage.tool_semantics.plan` | No | `[]` | Additional exact ASCII case-insensitive tool names that count as planning. | +| `stage.tool_semantics.new` | No | `[]` | Additional exact ASCII case-insensitive tool names that count as neutral forward activity. | | `subagents` | No | unset | Nested policy used only for delegated sub-agent work. | The tier is retained per session. A deployment that sends no session ID needs diff --git a/docs/routing_algorithms/composite_routing.md b/docs/routing_algorithms/composite_routing.md index 076f23bd0..aaa6c3e9b 100644 --- a/docs/routing_algorithms/composite_routing.md +++ b/docs/routing_algorithms/composite_routing.md @@ -37,6 +37,7 @@ confidence_threshold = 0.5 [routes.switchyard.stage.tool_semantics] observe = ["lookup_customer"] mutate = ["update_inventory"] +plan = ["create_research_plan"] new = ["send_message"] ``` diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py index 0ce4968a1..5acbac3d4 100644 --- a/switchyard_rust/libsy.py +++ b/switchyard_rust/libsy.py @@ -261,7 +261,7 @@ def stage_router( only_on_wrong_signal_escalation: bool = True, capable_system_prompt: str | None = None, efficient_system_prompt: str | None = None, - tool_semantics: Mapping[str, Sequence[str]] | None = None, + tool_semantics: dict[str, Sequence[str]] | None = None, classifier: LlmFallback | None = None, ) -> Algorithm: ... diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py index eb33f2cbd..4d1fb78e6 100644 --- a/tests/test_libsy_minimal_bindings.py +++ b/tests/test_libsy_minimal_bindings.py @@ -438,12 +438,14 @@ async def call(self, request: dict[str, Any]) -> dict[str, Any]: assert response["model"] == "strong" -def test_stage_router_accepts_additive_tool_semantics() -> None: +async def test_stage_router_applies_additive_tool_semantics() -> None: + """Verify that configured mutation semantics reach stage-router scoring.""" + algorithm = algorithms.stage_router( "strong", "fast", - picker="efficient_first", - confidence_threshold=0.5, + picker="capable_first", + confidence_threshold=0.3, tool_semantics={ "observe": ["KB_search"], "mutate": ["send_payment_request"], @@ -452,7 +454,43 @@ def test_stage_router_accepts_additive_tool_semantics() -> None: }, ) - assert callable(algorithm.run_stream) + selected_model, _ = await run_algorithm( + algorithm, + {"strong": EchoClient("strong"), "fast": EchoClient("fast")}, + request={ + "model": "auto", + "messages": [ + { + "role": "user", + "content": [{"type": "text", "text": "pay the balance"}], + }, + { + "role": "assistant", + "content": [ + { + "type": "tool_call", + "id": "call_1", + "name": "send_payment_request", + "arguments": {}, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_call_id": "call_1", + "content": [{"type": "text", "text": "payment sent"}], + "is_error": None, + } + ], + }, + ], + }, + ) + + assert selected_model == "fast" def test_stage_router_rejects_unknown_tool_semantics_category() -> None: From 96927d33c9a33e20ae41a5a75185b52b61bebfa2 Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Thu, 10 Sep 2026 09:35:25 -0700 Subject: [PATCH 5/6] fix(stage-router): preserve built-in tool semantics Signed-off-by: Sabhatina Selvam --- crates/libsy/src/algorithms/util/stage.rs | 36 ++++++ .../libsy/src/algorithms/util/tool_signals.rs | 104 ++++++++++++++++-- crates/switchyard-runner/src/config.rs | 27 +++++ 3 files changed, 160 insertions(+), 7 deletions(-) diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rs index 95b802b61..f9e3ef806 100644 --- a/crates/libsy/src/algorithms/util/stage.rs +++ b/crates/libsy/src/algorithms/util/stage.rs @@ -762,6 +762,42 @@ mod tests { assert_eq!(score_signal(&signal).score, 0.0); } + #[test] + fn old_new_activity_does_not_suppress_a_current_stall() { + let signal = ToolSignals { + turn_depth: STALL_MIN_TURN_DEPTH, + new_count: 1, + recent_new_count: 0, + ..Default::default() + }; + + let dimensions = dimensions_from_signal(&signal); + + assert_eq!(dimensions.spinning, 1.0); + assert_eq!(dimensions.exploring, 0.0); + assert!(score_signal(&signal).score > 0.0); + } + + #[test] + fn new_activity_does_not_dilute_existing_production() { + let baseline = ToolSignals { + turn_depth: STALL_MIN_TURN_DEPTH, + recent_write_count: 1, + ..Default::default() + }; + let with_new = ToolSignals { + recent_new_count: 2, + new_count: 2, + ..baseline.clone() + }; + + assert_eq!( + dimensions_from_signal(&with_new).production_intensity, + dimensions_from_signal(&baseline).production_intensity + ); + assert_eq!(score_signal(&with_new), score_signal(&baseline)); + } + // ─── StageClassifier ───────────────────────────────────────────────── /// Tiers named the way a deployment would name them. diff --git a/crates/libsy/src/algorithms/util/tool_signals.rs b/crates/libsy/src/algorithms/util/tool_signals.rs index 9f51c2db0..2c91eccb3 100644 --- a/crates/libsy/src/algorithms/util/tool_signals.rs +++ b/crates/libsy/src/algorithms/util/tool_signals.rs @@ -238,7 +238,7 @@ impl ToolSemantics { ))); } let normalized = name.to_ascii_lowercase(); - if is_builtin_tool_name(&normalized) { + if is_builtin_tool_name(&name.to_lowercase()) { return Err(tool_semantics_error(format!( "tool {name:?} already has built-in semantics and cannot be reclassified" ))); @@ -433,7 +433,7 @@ fn classify_tool_call_with_semantics( semantics: &ToolSemantics, ) -> ToolSemantic { // Built-in names and Bash command inference take precedence over route-scoped mappings. - let lower = name.to_ascii_lowercase(); + let lower = name.to_lowercase(); if WRITE_TOOL_NAMES.contains(&lower.as_str()) { return ToolSemantic::Mutate(MutationKind::Write); } @@ -463,7 +463,7 @@ fn classify_tool_call_with_semantics( return ToolSemantic::Observe; } } - semantics.classify(&lower).unwrap_or(ToolSemantic::Unknown) + semantics.classify(name).unwrap_or(ToolSemantic::Unknown) } fn is_builtin_tool_name(lower: &str) -> bool { @@ -1406,20 +1406,24 @@ mod tests { }; semantics.validate().expect("valid additive semantics"); let request = with_messages(vec![ + tc("Read"), + tc("Write"), + tc("TodoWrite"), tc("kb_SEARCH"), tc("send_payment_request"), tc("create_research_plan"), tc("send_message_to_user"), + tc("unlisted_tool"), ]); let signal = ToolSignals::from_request_with_semantics(&request, None, &semantics); - assert_eq!(signal.read_count, 1); - assert_eq!(signal.write_count, 1); - assert_eq!(signal.todowrite_count, 1); + assert_eq!(signal.read_count, 2); + assert_eq!(signal.write_count, 2); + assert_eq!(signal.todowrite_count, 2); assert_eq!(signal.new_count, 1); assert_eq!(signal.recent_new_count, 1); - assert_eq!(signal.pure_bash_streak, 0); + assert_eq!(signal.pure_bash_streak, 1); } #[test] @@ -1433,12 +1437,86 @@ mod tests { classify_tool_call_with_semantics("KB_SEARCH", None, &semantics), ToolSemantic::Observe ); + // U+212A lowercases to ASCII `k` under Unicode rules, but custom names + // intentionally ignore only ASCII case. assert_eq!( classify_tool_call_with_semantics("KB_SEARCH", None, &semantics), ToolSemantic::Unknown ); } + #[test] + fn custom_semantics_preserve_builtin_unicode_lowercasing() { + let semantics = ToolSemantics { + observe: vec!["lookup_customer".to_string()], + ..Default::default() + }; + + // This matched the built-in `notebookedit` before custom semantics existed. + assert_eq!( + classify_tool_call_with_semantics("notebooKedit", None, &semantics), + ToolSemantic::Mutate(MutationKind::Edit) + ); + } + + #[test] + fn configured_semantics_never_replace_builtin_classifications() { + let semantics = ToolSemantics { + observe: vec!["lookup_customer".to_string()], + mutate: vec!["send_payment".to_string()], + plan: vec!["create_workflow".to_string()], + new: vec!["send_message".to_string()], + }; + + for name in WRITE_TOOL_NAMES { + assert_eq!( + classify_tool_call_with_semantics(name, None, &semantics), + ToolSemantic::Mutate(MutationKind::Write), + "write tool {name:?} changed classification" + ); + } + for name in EDIT_TOOL_NAMES { + assert_eq!( + classify_tool_call_with_semantics(name, None, &semantics), + ToolSemantic::Mutate(MutationKind::Edit), + "edit tool {name:?} changed classification" + ); + } + for name in READ_TOOL_NAMES { + assert_eq!( + classify_tool_call_with_semantics(name, None, &semantics), + ToolSemantic::Observe, + "read tool {name:?} changed classification" + ); + } + for name in PLAN_TOOL_NAMES { + assert_eq!( + classify_tool_call_with_semantics(name, None, &semantics), + ToolSemantic::Plan, + "plan tool {name:?} changed classification" + ); + } + + for (command, expected) in [ + ("cat /tmp/input", ToolSemantic::Observe), + ( + "cat /tmp/input > /tmp/output", + ToolSemantic::Mutate(MutationKind::Write), + ), + ( + "sed -i 's/a/b/' /tmp/file", + ToolSemantic::Mutate(MutationKind::Edit), + ), + ("./run_tests.sh", ToolSemantic::Unknown), + ] { + assert_eq!( + classify_tool_call_with_semantics("BASH", Some(command), &semantics), + expected, + "bash command {command:?} changed classification" + ); + } + } + #[test] fn tool_semantics_reject_duplicates_and_builtin_reclassification() { let duplicate = ToolSemantics { @@ -1465,5 +1543,17 @@ mod tests { .to_string() .contains("built-in semantics") ); + + let empty = ToolSemantics { + observe: vec![" \t".to_string()], + ..Default::default() + }; + assert!( + empty + .validate() + .expect_err("empty name should fail") + .to_string() + .contains("empty tool name") + ); } } diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index 6a792df37..a2ef9aea7 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -817,6 +817,33 @@ new = ["send_message"] Ok(()) } + #[test] + fn stage_rejects_ambiguous_or_non_additive_tool_semantics() { + for (configured, expected) in [ + ( + stage_config().replace( + "mutate = [\"send_payment\"]", + "mutate = [\"LOOKUP_CUSTOMER\"]", + ), + "appears in both tool_semantics.observe and tool_semantics.mutate", + ), + ( + stage_config().replace("new = [\"send_message\"]", "new = [\"Read\"]"), + "already has built-in semantics and cannot be reclassified", + ), + ( + stage_config().replace("observe = [\"lookup_customer\"]", "observe = [\" \"]"), + "contains an empty tool name", + ), + ] { + let message = error_message(&configured); + assert!( + message.contains(expected), + "expected {expected:?} in error, got: {message}" + ); + } + } + #[test] fn passthrough_and_stage_accept_subagent_routing() -> RunnerResult<()> { let stage = stage_config(); From acab87596c74dd07d8068b512925fdd2595ccfdd Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Thu, 10 Sep 2026 10:09:15 -0700 Subject: [PATCH 6/6] test(stage-router): cover custom semantics scoring Signed-off-by: Sabhatina Selvam --- .../libsy/src/algorithms/util/tool_signals.rs | 52 +++++++++++ crates/switchyard-server/tests/server.rs | 88 +++++++++++++++++++ 2 files changed, 140 insertions(+) diff --git a/crates/libsy/src/algorithms/util/tool_signals.rs b/crates/libsy/src/algorithms/util/tool_signals.rs index 2c91eccb3..43e1bd814 100644 --- a/crates/libsy/src/algorithms/util/tool_signals.rs +++ b/crates/libsy/src/algorithms/util/tool_signals.rs @@ -809,6 +809,7 @@ fn has_nonzero_failure_count(lower: &str) -> bool { #[cfg(test)] mod tests { use super::*; + use crate::algorithms::util::stage::score_signal; use serde_json::json; use switchyard_protocol::{ContentBlock, LlmRequest, Message, Role, ToolCall, ToolResult}; @@ -1517,6 +1518,57 @@ mod tests { } } + #[test] + fn configured_semantics_score_like_their_builtin_equivalents() { + let semantics = ToolSemantics { + observe: vec!["lookup_customer".to_string()], + mutate: vec!["send_payment".to_string()], + plan: vec!["create_workflow".to_string()], + ..Default::default() + }; + + for (builtin, configured) in [ + ("Read", "lookup_customer"), + ("Write", "send_payment"), + ("TodoWrite", "create_workflow"), + ] { + let messages_before_tool = || { + vec![ + Message::text(Role::User, "start"), + Message::text(Role::Assistant, "working"), + Message::text(Role::User, "continue"), + Message::text(Role::Assistant, "working"), + Message::text(Role::User, "continue"), + Message::text(Role::Assistant, "working"), + Message::text(Role::User, "continue"), + ] + }; + let mut builtin_messages = messages_before_tool(); + builtin_messages.push(tc(builtin)); + let mut configured_messages = messages_before_tool(); + configured_messages.push(tc(configured)); + + let builtin_score = score_signal(&ToolSignals::from_request( + &with_messages(builtin_messages), + None, + )); + let configured_score = score_signal(&ToolSignals::from_request_with_semantics( + &with_messages(configured_messages), + None, + &semantics, + )); + + assert_ne!( + builtin_score.score, 0.0, + "the {builtin:?} control must exercise a scoring dimension" + ); + assert_eq!( + configured_score, builtin_score, + "configured tool {configured:?} must score exactly like {builtin:?}" + ); + } + } + #[test] fn tool_semantics_reject_duplicates_and_builtin_reclassification() { let duplicate = ToolSemantics { diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 933b93379..60b73840c 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -1493,6 +1493,94 @@ mutate = ["send_payment_request"] Ok(()) } +// Composite TOML must pass custom stage semantics through to the nested stage router. +#[tokio::test] +async fn composite_router_uses_configured_tool_semantics() -> TestResult { + let upstream = MockUpstream::start().await?; + let state = load_test_config(&format!( + r#" +schema_version = 1 + +[llm_clients.upstream] +format = "openai_chat" +base_url = "{base_url}" + +[targets.classifier] +id = "model/classifier" +llm_client = "upstream" + +[targets.strong] +id = "model/strong" +llm_client = "upstream" + +[targets.weak] +id = "model/weak" +llm_client = "upstream" + +[routes.composite] +id = "switchyard/composite" +type = "composite" + +[routes.composite.classifier] +target = "classifier" +base_threshold = 0.5 +classify_trigger = "user_turn" + +[routes.composite.stage] +capable_target = "strong" +efficient_target = "weak" +confidence_threshold = 0.3 + +[routes.composite.stage.tool_semantics] +new = ["send_message_to_user"] +"#, + base_url = upstream.base_url + ))?; + let app = build_switchyard_router(state); + + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "switchyard/composite", + "messages": [ + {"role": "user", "content": "help the customer"}, + {"role": "assistant", "content": "working"}, + {"role": "user", "content": "continue"}, + {"role": "assistant", "content": "working"}, + {"role": "user", "content": "continue"}, + {"role": "assistant", "content": "working"}, + {"role": "assistant", "tool_calls": [{ + "id": "call_1", + "type": "function", + "function": { + "name": "send_message_to_user", + "arguments": "{}" + } + }]}, + {"role": "tool", "tool_call_id": "call_1", "content": "message sent"} + ] + })), + ) + .await?; + + assert_eq!(response.status, StatusCode::OK); + assert_eq!( + response + .headers + .get("x-model-router-selected-model") + .and_then(|value| value.to_str().ok()), + Some("model/weak") + ); + assert_eq!( + upstream.models().await, + ["model/weak"], + "configured new activity must suppress the deep-turn stall without consulting the judge" + ); + Ok(()) +} + #[tokio::test] async fn custom_classifier_routes_four_targets_and_falls_back_on_an_invalid_verdict() -> TestResult {