Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 24 additions & 6 deletions crates/switchyard-server/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -479,12 +479,18 @@ async fn stamp_request_start(mut request: HttpRequest, next: Next) -> Response {
next.run(request).await
}

/// Builds an Axum router for the supported LLM wire formats.
/// Builds an Axum router containing only the primary LLM inference endpoints.
///
/// The registered paths are `/v1/chat/completions`, `/v1/messages`, and `/v1/responses`.
/// The router excludes operational, discovery, auxiliary, and fallback proxy routes so an
/// embedding application can mount those capabilities under its own policies.
pub fn build_llm_router(state: ServerState) -> Router {
finish_router(primary_llm_routes(), state)
}

/// Builds the full Axum router used by the standalone Switchyard server.
pub fn build_switchyard_router(state: ServerState) -> Router {
let mut router = Router::new()
.route("/v1/chat/completions", post(openai_chat_completions))
.route("/v1/messages", post(anthropic_messages))
.route("/v1/responses", post(openai_responses))
let mut router = primary_llm_routes()
.route("/v1/decision", post(decision))
.route("/v1/messages/count_tokens", post(anthropic_count_tokens))
.route(
Expand All @@ -500,8 +506,20 @@ pub fn build_switchyard_router(state: ServerState) -> Router {
if state.routing_log.is_some() {
router = router.route("/v1/routing/session-stats", get(get_session_stats));
}
finish_router(router.fallback(proxy_unmatched), state)
}

// Keeps the embedded and standalone servers on the same primary route definitions.
fn primary_llm_routes() -> Router<ServerState> {
Router::new()
.route("/v1/chat/completions", post(openai_chat_completions))
.route("/v1/messages", post(anthropic_messages))
.route("/v1/responses", post(openai_responses))
}

// Applies the serving limits and request timing shared by both public router constructors.
fn finish_router(router: Router<ServerState>, state: ServerState) -> Router {
router
.fallback(proxy_unmatched)
.layer(DefaultBodyLimit::max(DEFAULT_MAX_REQUEST_BODY_BYTES))
// `layer` only wraps routes registered before it, so this stays last.
.layer(axum::middleware::from_fn(stamp_request_start))
Expand Down
40 changes: 39 additions & 1 deletion crates/switchyard-server/tests/server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,9 @@ use switchyard_llm_client::{
use switchyard_protocol::ModelId;
use switchyard_protocol::RoutedLlmClient;
use switchyard_server::config::load_server_state;
use switchyard_server::{DEFAULT_MAX_REQUEST_BODY_BYTES, ServerState, build_switchyard_router};
use switchyard_server::{
DEFAULT_MAX_REQUEST_BODY_BYTES, ServerState, build_llm_router, build_switchyard_router,
};
use tokio::net::TcpListener;
use tokio::sync::Mutex;
use tokio::task::JoinHandle;
Expand Down Expand Up @@ -528,6 +530,42 @@ async fn test_app(routes: &[(&str, &[&str])]) -> TestResult<(MockUpstream, Route
Ok((upstream, app))
}

// Embedders expose only the three primary inference endpoints and own every other route.
#[tokio::test]
async fn llm_router_exposes_only_primary_llm_endpoints() -> TestResult {
Comment thread
afourniernv marked this conversation as resolved.
let state = random_state("http://127.0.0.1:1/v1", &[(ROUTE_MODEL, &["model/weak"])])?;
let app = build_llm_router(state);

for path in ["/v1/chat/completions", "/v1/messages", "/v1/responses"] {
assert_eq!(
send(&app, "GET", path, None).await?.status,
StatusCode::METHOD_NOT_ALLOWED,
"{path} should be registered"
);
}

for path in [
"/v1/decision",
"/v1/messages/count_tokens",
"/v1/responses/input_tokens",
"/v1/responses/compact",
"/v1/models",
"/v1/stats",
"/v1/stats/reset",
"/v1/routing/session-stats",
"/metrics",
"/health",
"/future/provider/endpoint",
] {
assert_eq!(
send(&app, "POST", path, None).await?.status,
StatusCode::NOT_FOUND,
"{path} should not be registered"
);
}
Ok(())
}

fn empty_token_totals() -> Value {
json!({
"prompt": 0,
Expand Down
Loading