diff --git a/CONTEXT.md b/CONTEXT.md index db7d34b..f2dc7de 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -41,3 +41,17 @@ These three Microsoft offerings are distinct and must not all be called "Azure". `eagerEotThreshold`) that lets a downstream agent begin preparing a reply before the turn is confirmed. May be retracted by a `TurnResumed` event if the speaker continues. + +## Turn detection terms + +- **Turn Detection** — the provider-neutral configuration (`context_switch_core`) + that controls how a backend decides a turn has ended. Each backend converter + forwards only the fields it understands. Because the two backends model + end-of-turn aggressiveness differently, it is expressed twice rather than + mapped between forms: `threshold` / `eagerThreshold` (confidence floats) are + Deepgram-only, and `thresholdLevel` is Voice Live-only. + +- **Threshold Level** — the categorical end-of-turn aggressiveness consumed by + Voice Live (`low` / `medium` / `high`). Lower levels wait for stronger + evidence before ending a turn; higher levels end turns earlier. When unset, + Voice Live uses its built-in default behavior. diff --git a/README.md b/README.md index e6119a2..395ad2c 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,9 @@ cargo run --example transcribe -- azure # Run generic transcribe example with Deepgram provider cargo run --example transcribe -- deepgram +# Deepgram-specific end-of-turn tuning +cargo run --example transcribe -- deepgram --turn-threshold 0.7 --turn-timeout-ms 900 --turn-eager-threshold 0.6 + # Run generic transcribe example with ElevenLabs provider cargo run --example transcribe -- elevenlabs @@ -75,6 +78,9 @@ cargo run --example transcribe -- aristech # Run generic transcribe example with Microsoft Voice Live provider cargo run --example transcribe -- voice-live +# Voice Live end-of-turn tuning +cargo run --example transcribe -- voice-live --turn-threshold-level high --turn-timeout-ms 800 + # Run Azure synthesize example cargo run --example azure-synthesize diff --git a/core/src/lib.rs b/core/src/lib.rs index 6e24baa..2c0666c 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -8,6 +8,7 @@ mod protocol; mod registry; pub mod service; pub mod speech_gate; +mod turn_detection; use std::time; @@ -20,6 +21,7 @@ pub use duration::Duration; pub use protocol::*; pub use registry::*; pub use service::Service; +pub use turn_detection::{ThresholdLevel, TurnDetection}; /// A unidirectional audio message. Useful for implementing an audio transfer channel. #[derive(Debug)] diff --git a/core/src/turn_detection.rs b/core/src/turn_detection.rs new file mode 100644 index 0000000..388dbc4 --- /dev/null +++ b/core/src/turn_detection.rs @@ -0,0 +1,40 @@ +use serde::{Deserialize, Serialize}; + +/// Provider-neutral turn-detection configuration. +/// +/// Each backend converter forwards only the fields it understands and ignores the rest. +/// To avoid lossy numeric-to-categorical mapping, the end-of-turn aggressiveness is +/// expressed twice with distinct, provider-specific shapes: +/// - `threshold` (and `eager_threshold`) are confidence floats consumed by Deepgram Flux. +/// - `threshold_level` is the categorical level consumed by Voice Live. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TurnDetection { + /// End-of-turn confidence threshold (Deepgram Flux `eot_threshold`, valid range `0.5`–`0.9`). + /// Ignored by Voice Live. + #[serde(skip_serializing_if = "Option::is_none")] + pub threshold: Option, + /// Categorical end-of-turn aggressiveness (Voice Live `threshold_level`). + /// Ignored by Deepgram. + #[serde(skip_serializing_if = "Option::is_none")] + pub threshold_level: Option, + /// Silence (ms) before a turn is force-closed. Forwarded by both backends. + #[serde(skip_serializing_if = "Option::is_none")] + pub timeout_ms: Option, + /// Eager (early) end-of-turn confidence threshold (Deepgram Flux `eager_eot_threshold`, + /// valid range `0.3`–`0.9`). Ignored by Voice Live. + #[serde(skip_serializing_if = "Option::is_none")] + pub eager_threshold: Option, +} + +/// Categorical end-of-turn aggressiveness consumed by Voice Live. +/// +/// Lower levels wait for stronger evidence before ending a turn; higher levels end turns +/// earlier. When unset, Voice Live uses its built-in default behavior. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ThresholdLevel { + Low, + Medium, + High, +} diff --git a/docs/adr/0001-turn-detection-provider-specific-fields.md b/docs/adr/0001-turn-detection-provider-specific-fields.md new file mode 100644 index 0000000..6644047 --- /dev/null +++ b/docs/adr/0001-turn-detection-provider-specific-fields.md @@ -0,0 +1,23 @@ +# Provider-specific turn-detection fields instead of a mapped neutral value + +The shared `TurnDetection` abstraction in `context_switch_core` is used by both Deepgram Flux +and Voice Live, but the two backends model end-of-turn aggressiveness incompatibly: Deepgram +takes confidence floats (`eot_threshold` 0.5–0.9, `eager_eot_threshold` 0.3–0.9) while Voice Live +takes a categorical `threshold_level` (low/medium/high). Rather than invent a single neutral +value and translate it per backend, we expose both shapes side by side — `threshold` / +`eager_threshold` (Deepgram-only) and `threshold_level` (Voice Live-only) — and let each +converter forward only the fields it understands. + +## Considered Options + +- **Bucket a neutral float into Voice Live levels** (e.g. split Deepgram's 0.5–0.9 range into + thirds, inverted). Rejected: the cutoffs are arbitrary magic numbers, the direction is + inverted and confusing, and the mapping is lossy in both directions. +- **A richer enum that can express both shapes.** Rejected: heavier and couples core to both + providers' models without removing the underlying conceptual mismatch. + +## Consequences + +- Callers must know which field applies to their chosen backend; a field meant for the other + backend is silently ignored. +- New turn-based backends add their own field(s) here rather than reinterpreting existing ones. diff --git a/examples/transcribe.rs b/examples/transcribe.rs index 130c6b8..6c2b46d 100644 --- a/examples/transcribe.rs +++ b/examples/transcribe.rs @@ -8,10 +8,6 @@ use tokio::select; use tokio::sync::mpsc::{channel, unbounded_channel}; use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; -use openai_api_rs::realtime::types::{ - AzureSemanticVadConfig, EndOfUtteranceDetectionConfig, EndOfUtteranceDetectionModel, - EndOfUtteranceThresholdLevel, TurnDetection, -}; use rodio::DeviceSinkBuilder; use context_switch::services::{ @@ -21,7 +17,9 @@ use context_switch::services::{ use context_switch::{AudioConsumer, InputModality, OutputModality}; use context_switch_core::language::Languages; use context_switch_core::service::Service; -use context_switch_core::{AudioFormat, AudioFrame, Conversation, Input, audio}; +use context_switch_core::{ + AudioFormat, AudioFrame, Conversation, Input, ThresholdLevel, TurnDetection, audio, +}; const DEFAULT_LANGUAGE: &str = "en-US"; @@ -38,6 +36,14 @@ struct Args { region: Option, #[arg(long)] diarization: bool, + #[arg(long)] + turn_threshold: Option, + #[arg(long, value_enum)] + turn_threshold_level: Option, + #[arg(long)] + turn_timeout_ms: Option, + #[arg(long)] + turn_eager_threshold: Option, } #[derive(Debug, Clone, Copy, ValueEnum)] @@ -56,6 +62,31 @@ enum Provider { Deepgram, } +#[derive(Debug, Clone, Copy, ValueEnum)] +enum TurnThresholdLevel { + Low, + Medium, + High, +} + +impl From for ThresholdLevel { + fn from(value: TurnThresholdLevel) -> Self { + match value { + TurnThresholdLevel::Low => ThresholdLevel::Low, + TurnThresholdLevel::Medium => ThresholdLevel::Medium, + TurnThresholdLevel::High => ThresholdLevel::High, + } + } +} + +#[derive(Debug, Clone)] +struct ProviderArgs<'a> { + model: Option<&'a str>, + region: Option<&'a str>, + diarization: bool, + turn_detection: Option, +} + #[tokio::main] async fn main() -> Result<()> { dotenvy::dotenv_override()?; @@ -68,17 +99,29 @@ async fn main() -> Result<()> { args.language }; let languages = Languages::new(language)?; - let model = args.model.as_deref(); - let region = args.region.as_deref(); - let diarization = args.diarization; + let provider_args = ProviderArgs { + model: args.model.as_deref(), + region: args.region.as_deref(), + diarization: args.diarization, + turn_detection: if args.turn_threshold.is_some() + || args.turn_threshold_level.is_some() + || args.turn_timeout_ms.is_some() + || args.turn_eager_threshold.is_some() + { + Some(TurnDetection { + threshold: args.turn_threshold, + threshold_level: args.turn_threshold_level.map(ThresholdLevel::from), + timeout_ms: args.turn_timeout_ms, + eager_threshold: args.turn_eager_threshold, + }) + } else { + None + }, + }; match args.input.as_deref() { - Some(path) => { - recognize_from_wav(args.provider, path, &languages, model, region, diarization).await? - } - None => { - recognize_from_microphone(args.provider, &languages, model, region, diarization).await? - } + Some(path) => recognize_from_wav(args.provider, path, &languages, &provider_args).await?, + None => recognize_from_microphone(args.provider, &languages, &provider_args).await?, } Ok(()) @@ -88,9 +131,7 @@ async fn recognize_from_wav( provider: Provider, file: &Path, languages: &Languages, - model: Option<&str>, - region: Option<&str>, - diarization: bool, + provider_args: &ProviderArgs<'_>, ) -> Result<()> { let format = AudioFormat { channels: 1, @@ -107,24 +148,13 @@ async fn recognize_from_wav( producer.produce(frame)?; } - recognize( - provider, - format, - input_consumer, - languages, - model, - region, - diarization, - ) - .await + recognize(provider, format, input_consumer, languages, provider_args).await } async fn recognize_from_microphone( provider: Provider, languages: &Languages, - model: Option<&str>, - region: Option<&str>, - diarization: bool, + provider_args: &ProviderArgs<'_>, ) -> Result<()> { // Keep an output sink alive so Bluetooth headsets can switch to a bidirectional profile. let _output_sink = match DeviceSinkBuilder::open_default_sink() { @@ -170,16 +200,7 @@ async fn recognize_from_microphone( stream.play().expect("Failed to play stream"); - recognize( - provider, - format, - input_consumer, - languages, - model, - region, - diarization, - ) - .await + recognize(provider, format, input_consumer, languages, provider_args).await } async fn recognize( @@ -187,9 +208,7 @@ async fn recognize( format: AudioFormat, mut input_consumer: AudioConsumer, languages: &Languages, - model: Option<&str>, - region: Option<&str>, - diarization: bool, + provider_args: &ProviderArgs<'_>, ) -> Result<()> { let (output_producer, mut output_consumer) = unbounded_channel(); let (conversation_input_producer, conversation_input_consumer) = channel(16_384); @@ -197,9 +216,7 @@ async fn recognize( let conversation = start_conversation( provider, languages, - model, - region, - diarization, + provider_args, Conversation::new( InputModality::Audio { format }, [OutputModality::Text, OutputModality::InterimText], @@ -238,12 +255,16 @@ async fn recognize( async fn start_conversation( provider: Provider, languages: &Languages, - model: Option<&str>, - region: Option<&str>, - diarization: bool, + provider_args: &ProviderArgs<'_>, conversation: Conversation, ) -> Result<()> { - validate_provider_args(provider, model, region, diarization)?; + validate_provider_args( + provider, + provider_args.model, + provider_args.region, + provider_args.diarization, + provider_args.turn_detection.is_some(), + )?; match provider { Provider::Azure => { @@ -255,7 +276,7 @@ async fn start_conversation( subscription_key: env::var("AZURE_SUBSCRIPTION_KEY") .expect("AZURE_SUBSCRIPTION_KEY undefined"), language: languages.join_csv(), - diarization, + diarization: provider_args.diarization, speech_gate: false, }; AzureTranscribe.conversation(params, conversation).await @@ -284,7 +305,8 @@ async fn start_conversation( .await } Provider::Google => { - let region = region + let region = provider_args + .region .map(str::to_owned) .or_else(|| env::var("GOOGLE_TRANSCRIBE_REGION").ok()); @@ -303,11 +325,11 @@ async fn start_conversation( // https://docs.cloud.google.com/speech-to-text/docs/speech-to-text-supported-languages let params = google_transcribe::transcribe::Params { - model: model.map(str::to_owned).unwrap_or_else(|| { + model: provider_args.model.map(str::to_owned).unwrap_or_else(|| { env::var("GOOGLE_TRANSCRIBE_MODEL").unwrap_or_else(|_| "latest_long".to_owned()) }), language: languages.join_csv(), - diarization, + diarization: provider_args.diarization, region, }; GoogleTranscribe.conversation(params, conversation).await @@ -347,14 +369,13 @@ async fn start_conversation( .context("Voice Live provider supports exactly one --language value")? .clone(), ); - let vad_languages = language.clone().map(|lang| vec![lang]); let params = microsoft_voice_live::Params { api_key: env::var("MICROSOFT_VOICE_LIVE_API_KEY") .expect("MICROSOFT_VOICE_LIVE_API_KEY undefined"), endpoint: env::var("MICROSOFT_VOICE_LIVE_ENDPOINT") .expect("MICROSOFT_VOICE_LIVE_ENDPOINT undefined (must be wss://...)"), - model: model.map(str::to_owned).unwrap_or_else(|| { + model: provider_args.model.map(str::to_owned).unwrap_or_else(|| { env::var("MICROSOFT_VOICE_LIVE_MODEL").unwrap_or_else(|_| "gpt-4.1".to_owned()) }), api_version: env::var("MICROSOFT_VOICE_LIVE_API_VERSION").ok(), @@ -362,18 +383,9 @@ async fn start_conversation( .unwrap_or_else(|_| "azure-speech".to_owned()), language, noise_reduction: None, - turn_detection: Some(TurnDetection::AzureSemanticVadMultilingual( - AzureSemanticVadConfig { - end_of_utterance_detection: Some(EndOfUtteranceDetectionConfig { - model: EndOfUtteranceDetectionModel::SmartEndOfTurnDetection, - threshold_level: Some(EndOfUtteranceThresholdLevel::Default), - timeout_ms: Some(5000), - }), - // remove_filler_words: Some(true), - languages: vad_languages, - ..Default::default() - }, - )), + // When omitted, Voice Live defaults to Azure multilingual semantic VAD with + // smart end-of-turn detection. + turn_detection: provider_args.turn_detection.clone(), }; MicrosoftVoiceLiveTranscribe .conversation(params, conversation) @@ -386,7 +398,7 @@ async fn start_conversation( language: languages.join_csv(), profanity_filter: false, keyterm: vec![], - turn_detection: deepgram_service::transcribe::TurnDetection::default(), + turn_detection: provider_args.turn_detection.clone(), }; DeepgramTranscribe.conversation(params, conversation).await @@ -399,6 +411,7 @@ struct ProviderCapabilities { region: bool, diarization: bool, model: bool, + turn_detection: bool, } impl Provider { @@ -410,7 +423,9 @@ impl Provider { capabilities.diarization = true; capabilities.model = true; } - Provider::Deepgram => {} + Provider::Deepgram => { + capabilities.turn_detection = true; + } Provider::Elevenlabs => { capabilities.model = true; } @@ -424,6 +439,7 @@ impl Provider { } Provider::VoiceLive => { capabilities.model = true; + capabilities.turn_detection = true; } } @@ -455,6 +471,7 @@ fn validate_provider_args( model: Option<&str>, region: Option<&str>, diarization: bool, + turn_detection: bool, ) -> Result<()> { let capabilities = provider.capabilities(); @@ -465,5 +482,11 @@ fn validate_provider_args( diarization, capabilities.diarization, provider, + )?; + validate_capability( + "--turn-threshold/--turn-threshold-level/--turn-timeout-ms/--turn-eager-threshold", + turn_detection, + capabilities.turn_detection, + provider, ) } diff --git a/services/deepgram/src/transcribe.rs b/services/deepgram/src/transcribe.rs index a2d584f..1b744e0 100644 --- a/services/deepgram/src/transcribe.rs +++ b/services/deepgram/src/transcribe.rs @@ -15,7 +15,7 @@ use deepgram::common::options::{Encoding, Model, Options}; use context_switch_core::language::{Languages, bcp47_to_iso639_3}; use context_switch_core::{ - BillingRecord, BillingSchedule, Conversation, Input, OutputPath, Service, + BillingRecord, BillingSchedule, Conversation, Input, OutputPath, Service, TurnDetection, }; #[derive(Debug, Deserialize)] @@ -29,16 +29,11 @@ pub struct Params { pub profanity_filter: bool, #[serde(default)] pub keyterm: Vec, - #[serde(flatten)] - pub turn_detection: TurnDetection, -} - -#[derive(Debug, Default, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct TurnDetection { - pub threshold: Option, - pub timeout_ms: Option, - pub eager_threshold: Option, + /// Provider-neutral turn-detection configuration. Only `threshold`, `timeoutMs`, and + /// `eagerThreshold` are forwarded to Deepgram Flux; `thresholdLevel` is ignored. When + /// omitted, Flux applies its own built-in end-of-turn defaults. + #[serde(default)] + pub turn_detection: Option, } #[derive(Debug)] @@ -66,14 +61,16 @@ impl Service for DeepgramTranscribe { let (model, language_hints) = select_model_and_language_hints(&languages)?; let mut options_builder = Options::builder().model(model); - if let Some(eot_threshold) = params.turn_detection.threshold { - options_builder = options_builder.eot_threshold(eot_threshold); - } - if let Some(eot_timeout_ms) = params.turn_detection.timeout_ms { - options_builder = options_builder.eot_timeout_ms(eot_timeout_ms); - } - if let Some(eager_eot_threshold) = params.turn_detection.eager_threshold { - options_builder = options_builder.eager_eot_threshold(eager_eot_threshold); + if let Some(turn_detection) = ¶ms.turn_detection { + if let Some(eot_threshold) = turn_detection.threshold { + options_builder = options_builder.eot_threshold(eot_threshold); + } + if let Some(eot_timeout_ms) = turn_detection.timeout_ms { + options_builder = options_builder.eot_timeout_ms(eot_timeout_ms); + } + if let Some(eager_eot_threshold) = turn_detection.eager_threshold { + options_builder = options_builder.eager_eot_threshold(eager_eot_threshold); + } } if params.profanity_filter { options_builder = options_builder.profanity_filter(true); diff --git a/services/microsoft-voice-live/src/client.rs b/services/microsoft-voice-live/src/client.rs index 0596b91..d4809ee 100644 --- a/services/microsoft-voice-live/src/client.rs +++ b/services/microsoft-voice-live/src/client.rs @@ -6,7 +6,10 @@ use futures::stream::{SplitSink, SplitStream}; use futures::{SinkExt, StreamExt}; use openai_api_rs::realtime::client_event::{self, ClientEvent}; use openai_api_rs::realtime::server_event::ServerEvent; -use openai_api_rs::realtime::types::{self, AzureSemanticVadConfig, TurnDetection}; +use openai_api_rs::realtime::types::{ + self, AzureSemanticVadConfig, EndOfUtteranceDetectionConfig, EndOfUtteranceDetectionModel, + EndOfUtteranceThresholdLevel, TurnDetection, +}; use tokio::{net::TcpStream, select}; use tokio_tungstenite::tungstenite::{Bytes, protocol::Message}; use tokio_tungstenite::{MaybeTlsStream, WebSocketStream}; @@ -14,7 +17,7 @@ use tracing::{debug, info, trace, warn}; use context_switch_core::{ AudioFormat, AudioFrame, BillingRecord, BillingSchedule, ConversationInput, ConversationOutput, - Input, OutputPath, audio, + Input, OutputPath, ThresholdLevel, audio, }; use crate::transcribe::{Params, ServiceOutputEvent}; @@ -118,7 +121,7 @@ impl Client { model: params.transcription_model.clone(), prompt: None, }), - turn_detection: Some(transcription_turn_detection(params.turn_detection.clone())), + turn_detection: Some(transcription_turn_detection(params.turn_detection.as_ref())), }; log_requested_session_update(&session); @@ -284,25 +287,38 @@ impl Client { } } -/// Produces the turn-detection configuration for transcription. Responses are always suppressed -/// (`create_response = false`) because this service only transcribes; a missing configuration -/// defaults to Azure semantic VAD. -fn transcription_turn_detection(configured: Option) -> TurnDetection { - let mut detection = configured - .unwrap_or_else(|| TurnDetection::AzureSemanticVad(AzureSemanticVadConfig::default())); - match &mut detection { - TurnDetection::ServerVAD(config) => { - config.create_response = false; - } - TurnDetection::SemanticVAD(config) => config.create_response = false, - TurnDetection::AzureSemanticVad(config) => { - config.create_response = Some(false); - } - TurnDetection::AzureSemanticVadMultilingual(config) => { - config.create_response = Some(false); - } +/// Produces the Voice Live turn-detection configuration for transcription. +/// +/// The neutral configuration is realized as Azure multilingual semantic VAD with smart +/// end-of-turn detection. Only `threshold_level` and `timeout_ms` are honored; the Deepgram-only +/// float thresholds are ignored. Responses are always suppressed (`create_response = false`) +/// because this service only transcribes. A missing `threshold_level` falls back to the service +/// default behavior. +fn transcription_turn_detection( + configured: Option<&context_switch_core::TurnDetection>, +) -> TurnDetection { + let threshold_level = configured + .and_then(|detection| detection.threshold_level) + .map(eou_threshold_level); + let timeout_ms = configured.and_then(|detection| detection.timeout_ms); + + TurnDetection::AzureSemanticVadMultilingual(AzureSemanticVadConfig { + end_of_utterance_detection: Some(EndOfUtteranceDetectionConfig { + model: EndOfUtteranceDetectionModel::SmartEndOfTurnDetection, + threshold_level, + timeout_ms, + }), + create_response: Some(false), + ..Default::default() + }) +} + +fn eou_threshold_level(level: ThresholdLevel) -> EndOfUtteranceThresholdLevel { + match level { + ThresholdLevel::Low => EndOfUtteranceThresholdLevel::Low, + ThresholdLevel::Medium => EndOfUtteranceThresholdLevel::Medium, + ThresholdLevel::High => EndOfUtteranceThresholdLevel::High, } - detection } enum FlowControl { diff --git a/services/microsoft-voice-live/src/transcribe.rs b/services/microsoft-voice-live/src/transcribe.rs index 827e887..2598972 100644 --- a/services/microsoft-voice-live/src/transcribe.rs +++ b/services/microsoft-voice-live/src/transcribe.rs @@ -1,9 +1,9 @@ use anyhow::Result; use async_trait::async_trait; -use openai_api_rs::realtime::types::{NoiseReduction, TurnDetection}; +use openai_api_rs::realtime::types::NoiseReduction; use serde::{Deserialize, Serialize}; -use context_switch_core::{Conversation, Service}; +use context_switch_core::{Conversation, Service, TurnDetection}; use crate::host::Host; @@ -30,8 +30,9 @@ pub struct Params { pub language: Option, /// Input-audio noise reduction (Azure deep noise suppression, near/far field). pub noise_reduction: Option, - /// Turn-detection configuration (Azure semantic VAD, server VAD, ...). Defaults to Azure - /// semantic VAD with responses suppressed when omitted. + /// Provider-neutral turn-detection configuration. Only `threshold_level` and `timeout_ms` + /// are forwarded to Voice Live; the float thresholds are ignored. When omitted, Voice Live + /// defaults to Azure multilingual semantic VAD with smart end-of-turn detection. pub turn_detection: Option, }