From 294fe39282d51c54a3ed408e4f5e1e1c1aed3252 Mon Sep 17 00:00:00 2001 From: David Chen Date: Thu, 4 Jun 2026 23:52:18 -0700 Subject: [PATCH 1/5] adding ability to control local video track encode parameters --- .changeset/local-video-encoding-controls.md | 6 + Cargo.lock | 2 + examples/local_video/Cargo.toml | 2 + examples/local_video/src/publisher.rs | 141 +++++++-- examples/local_video/src/subscriber.rs | 321 +++++++++++++++++++- examples/local_video/src/video_display.rs | 28 +- libwebrtc/src/native/rtp_parameters.rs | 1 + libwebrtc/src/native/rtp_sender.rs | 20 +- libwebrtc/src/rtp_parameters.rs | 1 + livekit/src/prelude.rs | 3 +- livekit/src/room/publication/local.rs | 14 + livekit/src/room/track/local_video_track.rs | 45 +++ 12 files changed, 559 insertions(+), 25 deletions(-) create mode 100644 .changeset/local-video-encoding-controls.md diff --git a/.changeset/local-video-encoding-controls.md b/.changeset/local-video-encoding-controls.md new file mode 100644 index 000000000..9a957d8d6 --- /dev/null +++ b/.changeset/local-video-encoding-controls.md @@ -0,0 +1,6 @@ +--- +libwebrtc: patch +livekit: patch +--- + +Add runtime video encoding limit controls for local video tracks and wire them into the `local_video` publisher/subscriber example via RPC. diff --git a/Cargo.lock b/Cargo.lock index b85400203..173fcdf0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4189,6 +4189,8 @@ dependencies = [ "nokhwa", "objc2 0.6.4", "parking_lot", + "serde", + "serde_json", "tokio", "tokio-stream", "webrtc-sys", diff --git a/examples/local_video/Cargo.toml b/examples/local_video/Cargo.toml index dedbc76d6..0c12294b4 100644 --- a/examples/local_video/Cargo.toml +++ b/examples/local_video/Cargo.toml @@ -49,6 +49,8 @@ parking_lot = { workspace = true, features = ["deadlock_detection"] } anyhow = { workspace = true } chrono = "0.4" bytemuck = { version = "1.16", features = ["derive"] } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } nokhwa = { git = "https://github.com/l1npengtul/nokhwa", rev = "4923ecab7cf26f9dba83867a15a9d8662d021296", default-features = false, features = ["output-threaded"] } diff --git a/examples/local_video/src/publisher.rs b/examples/local_video/src/publisher.rs index 62b8030b0..52e1be38b 100644 --- a/examples/local_video/src/publisher.rs +++ b/examples/local_video/src/publisher.rs @@ -20,6 +20,7 @@ use nokhwa::utils::{ }; use nokhwa::Camera; use parking_lot::Mutex; +use serde::{Deserialize, Serialize}; use std::collections::{HashMap, VecDeque}; use std::env; use std::sync::{ @@ -214,6 +215,24 @@ fn unix_time_us_now() -> u64 { } const MAX_BACKEND_CAPTURE_TIMESTAMP_AGE_US: u64 = 5_000_000; +const SET_VIDEO_ENCODING_LIMITS_METHOD: &str = "set-video-encoding-limits"; + +#[derive(Debug, Deserialize, Serialize)] +struct SetEncodingLimitsRequest { + track_sid: String, + bitrate_bps: Option, + max_framerate: Option, + scale_resolution_down_by: Option, + reason: String, +} + +#[derive(Debug, Deserialize, Serialize)] +struct SetEncodingLimitsResponse { + applied_bitrate_bps: Option, + applied_max_framerate: Option, + applied_scale_resolution_down_by: Option, + track_sid: String, +} #[derive(Default)] struct CaptureTimestampLogState { @@ -329,21 +348,39 @@ struct PublisherTimingSummary { capture_to_webrtc_total_ms: RollingMs, } -fn find_video_outbound_encoder(stats: &[livekit::webrtc::stats::RtcStats]) -> Option<&str> { - let mut fallback = None; +#[derive(Default)] +struct PublisherVideoOutboundStats { + encoder_implementation: Option, + target_bitrate_bps: Option, +} + +fn find_video_outbound_stats( + stats: &[livekit::webrtc::stats::RtcStats], +) -> PublisherVideoOutboundStats { + let mut fallback = PublisherVideoOutboundStats::default(); for stat in stats { let livekit::webrtc::stats::RtcStats::OutboundRtp(outbound) = stat else { continue; }; - if outbound.stream.kind != "video" || outbound.outbound.encoder_implementation.is_empty() { + if outbound.stream.kind != "video" { continue; } - let implementation = outbound.outbound.encoder_implementation.as_str(); + let current = PublisherVideoOutboundStats { + encoder_implementation: (!outbound.outbound.encoder_implementation.is_empty()) + .then(|| outbound.outbound.encoder_implementation.clone()), + target_bitrate_bps: (outbound.outbound.target_bitrate > 0.0) + .then_some(outbound.outbound.target_bitrate), + }; if outbound.outbound.active { - return Some(implementation); + return current; + } + if fallback.encoder_implementation.is_none() { + fallback.encoder_implementation = current.encoder_implementation; + } + if fallback.target_bitrate_bps.is_none() { + fallback.target_bitrate_bps = current.target_bitrate_bps; } - fallback.get_or_insert(implementation); } fallback @@ -366,14 +403,20 @@ async fn update_publisher_encoder_overlay( match track.get_stats().await { Ok(stats) => { - if let Some(implementation) = find_video_outbound_encoder(&stats) { + let outbound = find_video_outbound_stats(&stats); + if let Some(implementation) = outbound.encoder_implementation { if implementation != last_implementation { info!("Publisher video encoder implementation: {implementation}"); - last_implementation = implementation.to_string(); + last_implementation = implementation.clone(); } let mut shared = shared.lock(); - shared.codec_implementation = implementation.to_string(); + shared.codec_implementation = implementation; + if let Some(target_bitrate_bps) = outbound.target_bitrate_bps { + shared.encode_bitrate_mbps = Some(target_bitrate_bps / 1_000_000.0); + } + } else if let Some(target_bitrate_bps) = outbound.target_bitrate_bps { + shared.lock().encode_bitrate_mbps = Some(target_bitrate_bps / 1_000_000.0); } logged_initial = true; } @@ -541,6 +584,66 @@ fn update_shared_timing_sample( } } +fn register_encoding_limits_rpc(room: &Arc, publication: LocalTrackPublication) { + room.local_participant().register_rpc_method( + SET_VIDEO_ENCODING_LIMITS_METHOD.to_string(), + move |data| { + let publication = publication.clone(); + Box::pin(async move { + let request: SetEncodingLimitsRequest = serde_json::from_str(&data.payload) + .map_err(|err| { + RpcError::new( + 400, + "invalid encoding limits request".to_string(), + Some(err.to_string()), + ) + })?; + + let publication_sid = publication.sid().to_string(); + if request.track_sid != publication_sid { + return Err(RpcError::new( + 404, + "track not found".to_string(), + Some(request.track_sid), + )); + } + + let limits = VideoEncodingLimits { + max_bitrate: request.bitrate_bps, + max_framerate: request.max_framerate, + scale_resolution_down_by: request.scale_resolution_down_by, + }; + publication.set_video_encoding_limits(limits).map_err(|err| { + RpcError::new(500, format!("set encoding limits failed: {err}"), None) + })?; + + info!( + "{} requested video encoding limits: {:?} bps, {:?} fps, {:?}x scale ({})", + data.caller_identity, + request.bitrate_bps, + request.max_framerate, + request.scale_resolution_down_by, + request.reason + ); + + serde_json::to_string(&SetEncodingLimitsResponse { + applied_bitrate_bps: request.bitrate_bps, + applied_max_framerate: request.max_framerate, + applied_scale_resolution_down_by: request.scale_resolution_down_by, + track_sid: publication_sid, + }) + .map_err(|err| { + RpcError::new( + 500, + "failed to serialize encoding limits response".to_string(), + Some(err.to_string()), + ) + }) + }) + }, + ); +} + #[cfg(test)] mod tests { use super::*; @@ -955,21 +1058,23 @@ async fn run(args: Args, ctrl_c_received: Arc) -> Result<()> { .publish_track(LocalTrack::Video(track.clone()), publish_opts(requested_codec)) .await; - let actual_codec = if let Err(e) = publish_result { - if matches!(requested_codec, VideoCodec::H265) { + let (publication, actual_codec) = match publish_result { + Ok(publication) => { + info!("Published camera track"); + (publication, requested_codec) + } + Err(e) if matches!(requested_codec, VideoCodec::H265) => { log::warn!("H.265 publish failed ({}). Falling back to H.264...", e); - room.local_participant() + let publication = room + .local_participant() .publish_track(LocalTrack::Video(track.clone()), publish_opts(VideoCodec::H264)) .await?; info!("Published camera track with H.264 fallback"); - VideoCodec::H264 - } else { - return Err(e.into()); + (publication, VideoCodec::H264) } - } else { - info!("Published camera track"); - requested_codec + Err(e) => return Err(e.into()), }; + register_encoding_limits_rpc(&room, publication); let capture_config = CaptureConfig { fps: args.fps, diff --git a/examples/local_video/src/subscriber.rs b/examples/local_video/src/subscriber.rs index 1f3a4df47..7b072ab26 100644 --- a/examples/local_video/src/subscriber.rs +++ b/examples/local_video/src/subscriber.rs @@ -12,6 +12,7 @@ use livekit::webrtc::video_stream::native::NativeVideoStream; use livekit_api::access_token; use log::{debug, info}; use parking_lot::Mutex; +use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, env, @@ -31,6 +32,15 @@ use codec_display::{codec_from_mime, codec_with_implementation}; use subscriber_timing::SubscriberTimingHandle; use viewport_aspect::AspectConstrainedViewport; +const SET_VIDEO_ENCODING_LIMITS_METHOD: &str = "set-video-encoding-limits"; +const DEFAULT_CONTROL_BITRATE_BPS: u64 = 1_500_000; +const DEFAULT_CONTROL_FRAMERATE: f64 = 30.0; +const DEFAULT_CONTROL_RESOLUTION_SCALE: f64 = 1.0; +const BITRATE_KEY_STEP_BPS: u64 = 100_000; +const MIN_CONTROL_BITRATE_BPS: u64 = BITRATE_KEY_STEP_BPS; +const MIN_FRAMERATE: f64 = 0.1; +const MAX_FRAMERATE: f64 = 60.0; + #[cfg(target_os = "macos")] mod macos_native_video { use std::ffi::c_void; @@ -409,6 +419,181 @@ struct Args { e2ee_key: Option, } +#[derive(Debug, Deserialize, Serialize)] +struct SetEncodingLimitsRequest { + track_sid: String, + bitrate_bps: Option, + max_framerate: Option, + scale_resolution_down_by: Option, + reason: String, +} + +#[derive(Debug, Deserialize, Serialize)] +struct SetEncodingLimitsResponse { + applied_bitrate_bps: Option, + applied_max_framerate: Option, + applied_scale_resolution_down_by: Option, + track_sid: String, +} + +#[derive(Clone, Copy, Debug)] +struct EncodingControlState { + bitrate_bps: u64, + max_framerate: f64, + scale_resolution_down_by: f64, +} + +impl Default for EncodingControlState { + fn default() -> Self { + Self { + bitrate_bps: DEFAULT_CONTROL_BITRATE_BPS, + max_framerate: DEFAULT_CONTROL_FRAMERATE, + scale_resolution_down_by: DEFAULT_CONTROL_RESOLUTION_SCALE, + } + } +} + +impl EncodingControlState { + fn limits(self) -> VideoEncodingLimits { + VideoEncodingLimits { + max_bitrate: Some(self.bitrate_bps), + max_framerate: Some(self.max_framerate), + scale_resolution_down_by: Some(self.scale_resolution_down_by), + } + } +} + +#[derive(Clone)] +struct EncodingControl { + inner: Arc, +} + +struct EncodingControlInner { + room: Arc, + target: Mutex>, + state: Mutex, + handle: tokio::runtime::Handle, +} + +#[derive(Clone)] +struct EncodingControlTarget { + publisher_identity: String, + track_sid: TrackSid, +} + +impl EncodingControl { + fn new(room: Arc) -> Self { + Self { + inner: Arc::new(EncodingControlInner { + room, + target: Mutex::new(None), + state: Mutex::new(EncodingControlState::default()), + handle: tokio::runtime::Handle::current(), + }), + } + } + + fn set_active_track(&self, publisher_identity: String, track_sid: TrackSid) { + *self.inner.target.lock() = Some(EncodingControlTarget { publisher_identity, track_sid }); + } + + fn clear_active_track(&self, track_sid: &TrackSid) { + let mut target = self.inner.target.lock(); + if target.as_ref().is_some_and(|target| &target.track_sid == track_sid) { + *target = None; + } + } + + fn active_state(&self) -> Option { + if self.inner.target.lock().is_none() { + return None; + } + Some(*self.inner.state.lock()) + } + + fn adjust_bitrate(&self, increase: bool) { + let reason = + if increase { "keyboard bitrate increase" } else { "keyboard bitrate decrease" }; + self.update_limits(reason, |state| { + state.bitrate_bps = if increase { + state.bitrate_bps.saturating_add(BITRATE_KEY_STEP_BPS) + } else { + state.bitrate_bps.saturating_sub(BITRATE_KEY_STEP_BPS).max(MIN_CONTROL_BITRATE_BPS) + }; + }); + } + + fn adjust_framerate(&self, increase: bool) { + let reason = + if increase { "keyboard framerate increase" } else { "keyboard framerate decrease" }; + self.update_limits(reason, |state| { + state.max_framerate = if increase { + next_higher_framerate(state.max_framerate) + } else { + next_lower_framerate(state.max_framerate) + }; + }); + } + + fn set_resolution_scale(&self, scale_resolution_down_by: f64) { + self.update_limits("keyboard resolution scale", |state| { + state.scale_resolution_down_by = scale_resolution_down_by; + }); + } + + fn update_limits(&self, reason: &'static str, update: impl FnOnce(&mut EncodingControlState)) { + let (limits, target) = { + let mut state = self.inner.state.lock(); + update(&mut state); + (state.limits(), self.inner.target.lock().clone()) + }; + let Some(target) = target else { + debug!("No active publisher video track for encoding control request"); + return; + }; + + let room = self.inner.room.clone(); + self.inner.handle.spawn(async move { + if let Err(err) = request_encoding_limits(&room, target, limits, reason).await { + log::warn!("encoding limits RPC failed: {err}"); + } + }); + } +} + +fn next_lower_framerate(fps: f64) -> f64 { + if fps <= 0.2 { + MIN_FRAMERATE + } else if fps <= 0.5 { + 0.2 + } else if fps <= 1.0 { + 0.5 + } else if fps <= 5.0 { + 1.0 + } else { + let next = fps - 5.0; + if next >= 5.0 { + next + } else { + 1.0 + } + } +} + +fn next_higher_framerate(fps: f64) -> f64 { + if fps < 0.2 { + 0.2 + } else if fps < 0.5 { + 0.5 + } else if fps < 1.0 { + 1.0 + } else if fps < 5.0 { + 5.0 + } else { + (fps + 5.0).min(MAX_FRAMERATE) + } +} + struct SharedYuv { width: u32, height: u32, @@ -777,7 +962,8 @@ fn video_status_line( simulcast: bool, ) -> String { let codec = codec_with_implementation(codec, codec_implementation); - let bitrate = bitrate_mbps.map(|mbps| format!(" {:.1}mbps", mbps.max(0.0))).unwrap_or_default(); + let bitrate = + bitrate_mbps.map(|mbps| format!(" {:.1} mbps", mbps.max(0.0))).unwrap_or_default(); if simulcast { format!("{}x{} {:.1}fps {codec}{bitrate} Simulcast", width, height, fps.max(0.0)) } else { @@ -785,6 +971,60 @@ fn video_status_line( } } +fn encoding_control_status_line(state: EncodingControlState) -> String { + format!( + "Encoding limit {:.1}mbps {:.1}fps {:.1}x scale", + state.bitrate_bps as f64 / 1_000_000.0, + state.max_framerate, + state.scale_resolution_down_by, + ) +} + +async fn request_encoding_limits( + room: &Arc, + target: EncodingControlTarget, + limits: VideoEncodingLimits, + reason: &'static str, +) -> Result<()> { + info!( + "Requesting video encoding limits from {}: {:?} bps, {:?} fps, {:?}x scale ({})", + target.publisher_identity, + limits.max_bitrate, + limits.max_framerate, + limits.scale_resolution_down_by, + reason, + ); + + let payload = serde_json::to_string(&SetEncodingLimitsRequest { + track_sid: target.track_sid.to_string(), + bitrate_bps: limits.max_bitrate, + max_framerate: limits.max_framerate, + scale_resolution_down_by: limits.scale_resolution_down_by, + reason: reason.to_string(), + })?; + let response = room + .local_participant() + .perform_rpc( + PerformRpcData::new(target.publisher_identity, SET_VIDEO_ENCODING_LIMITS_METHOD) + .with_payload(payload) + .with_response_timeout(Duration::from_millis(500)) + .with_max_round_trip_latency(Duration::from_millis(500)), + ) + .await + .map_err(|err| anyhow::anyhow!("encoding limits RPC failed: {err}"))?; + + let response: SetEncodingLimitsResponse = serde_json::from_str(&response)?; + info!( + "Publisher applied video encoding limits on {}: {:?} bps, {:?} fps, {:?}x scale", + response.track_sid, + response.applied_bitrate_bps, + response.applied_max_framerate, + response.applied_scale_resolution_down_by, + ); + + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -803,11 +1043,43 @@ mod tests { Arc::new(Mutex::new(SimulcastState { available: true, ..Default::default() })); let subscriber_timing = SubscriberTimingHandle::new(); - let lines = subscriber_overlay_lines(&shared, &simulcast, false, &subscriber_timing) + let lines = subscriber_overlay_lines(&shared, &simulcast, false, &subscriber_timing, None) .expect("overlay should render"); assert_eq!(lines, vec!["1280x720 29.6fps H264 NVDEC 1.2 mbps Simulcast"]); } + + #[test] + fn framerate_steps_drop_below_five_to_named_low_rates() { + assert_eq!(next_lower_framerate(30.0), 25.0); + assert_eq!(next_lower_framerate(10.0), 5.0); + assert_eq!(next_lower_framerate(5.0), 1.0); + assert_eq!(next_lower_framerate(1.0), 0.5); + assert_eq!(next_lower_framerate(0.5), 0.2); + assert_eq!(next_lower_framerate(0.2), 0.1); + assert_eq!(next_lower_framerate(0.1), 0.1); + } + + #[test] + fn framerate_steps_rise_through_named_low_rates() { + assert_eq!(next_higher_framerate(0.1), 0.2); + assert_eq!(next_higher_framerate(0.2), 0.5); + assert_eq!(next_higher_framerate(0.5), 1.0); + assert_eq!(next_higher_framerate(1.0), 5.0); + assert_eq!(next_higher_framerate(5.0), 10.0); + assert_eq!(next_higher_framerate(60.0), 60.0); + } + + #[test] + fn encoding_control_line_uses_compact_mbps() { + let line = encoding_control_status_line(EncodingControlState { + bitrate_bps: 1_250_000, + max_framerate: 0.5, + scale_resolution_down_by: 4.0, + }); + + assert_eq!(line, "Encoding limit 1.2mbps 0.5fps 4.0x scale"); + } } async fn handle_track_subscribed( @@ -822,6 +1094,7 @@ async fn handle_track_subscribed( ctrl_c_received: &Arc, simulcast: &Arc>, repaint_ctx: &Arc>, + encoding_control: &EncodingControl, subscriber_timing: SubscriberTimingHandle, ) { // If a participant filter is set, skip others @@ -872,6 +1145,7 @@ async fn handle_track_subscribed( let mut s = shared.lock(); s.codec = codec; } + encoding_control.set_active_track(participant.identity().to_string(), sid.clone()); let mut timing_events = video_track.subscribe_timing_events(); let subscriber_timing_events = subscriber_timing.clone(); @@ -890,6 +1164,7 @@ async fn handle_track_subscribed( let my_sid = sid.clone(); let ctrl_c_sink = ctrl_c_received.clone(); let repaint_ctx_sink = repaint_ctx.clone(); + let encoding_control_sink = encoding_control.clone(); let subscriber_timing_sink = subscriber_timing.clone(); // Initialize simulcast state for this publication { @@ -983,6 +1258,7 @@ async fn handle_track_subscribed( let mut active = active_sid2.lock(); if active.as_ref() == Some(&my_sid) { *active = None; + encoding_control_sink.clear_active_track(&my_sid); } }); @@ -1066,6 +1342,7 @@ fn subscriber_overlay_lines( simulcast: &Arc>, include_timing: bool, subscriber_timing: &SubscriberTimingHandle, + encoding_control_state: Option, ) -> Option> { let status_line = { let s = shared.lock(); @@ -1086,6 +1363,9 @@ fn subscriber_overlay_lines( }; let mut lines = vec![status_line]; + if let Some(state) = encoding_control_state { + lines.push(encoding_control_status_line(state)); + } if include_timing { if let Some(mut timing_lines) = subscriber_timing.display_overlay_lines(Instant::now()) { lines.append(&mut timing_lines); @@ -1128,6 +1408,7 @@ fn handle_track_unsubscribed( video_size: &Arc, active_sid: &Arc>>, simulcast: &Arc>, + encoding_control: &EncodingControl, subscriber_timing: &SubscriberTimingHandle, ) { let sid = publication.sid().clone(); @@ -1136,6 +1417,7 @@ fn handle_track_unsubscribed( info!("Video track unsubscribed ({}), clearing active sink", sid); *active = None; } + encoding_control.clear_active_track(&sid); clear_hud_and_simulcast(shared, frame_slot, video_size, simulcast, subscriber_timing); } @@ -1146,6 +1428,7 @@ fn handle_track_unpublished( video_size: &Arc, active_sid: &Arc>>, simulcast: &Arc>, + encoding_control: &EncodingControl, subscriber_timing: &SubscriberTimingHandle, ) { let sid = publication.sid().clone(); @@ -1154,6 +1437,7 @@ fn handle_track_unpublished( info!("Video track unpublished ({}), clearing active sink", sid); *active = None; } + encoding_control.clear_active_track(&sid); clear_hud_and_simulcast(shared, frame_slot, video_size, simulcast, subscriber_timing); } @@ -1163,6 +1447,7 @@ struct VideoApp { video_size: Arc, simulcast: Arc>, subscriber_timing: SubscriberTimingHandle, + encoding_control: EncodingControl, repaint_ctx: Arc>, ctrl_c_received: Arc, viewport: AspectConstrainedViewport, @@ -1181,6 +1466,28 @@ impl eframe::App for VideoApp { self.viewport.set_video_size(ctx, width, height); } + if ctx.input(|i| i.key_pressed(egui::Key::ArrowUp)) { + self.encoding_control.adjust_bitrate(true); + } + if ctx.input(|i| i.key_pressed(egui::Key::ArrowDown)) { + self.encoding_control.adjust_bitrate(false); + } + if ctx.input(|i| i.key_pressed(egui::Key::ArrowLeft)) { + self.encoding_control.adjust_framerate(false); + } + if ctx.input(|i| i.key_pressed(egui::Key::ArrowRight)) { + self.encoding_control.adjust_framerate(true); + } + if ctx.input(|i| i.key_pressed(egui::Key::Num1)) { + self.encoding_control.set_resolution_scale(1.0); + } + if ctx.input(|i| i.key_pressed(egui::Key::Num2)) { + self.encoding_control.set_resolution_scale(2.0); + } + if ctx.input(|i| i.key_pressed(egui::Key::Num3)) { + self.encoding_control.set_resolution_scale(4.0); + } + let render_frame = self.frame_slot.take(); if let Some(frame) = render_frame.as_ref() { if let Some(metadata) = frame.frame_metadata { @@ -1199,6 +1506,7 @@ impl eframe::App for VideoApp { &self.simulcast, self.display_timestamp, &self.subscriber_timing, + self.encoding_control.active_state(), ); egui::CentralPanel::default().frame(egui::Frame::NONE).show(ctx, |ui| { @@ -1326,6 +1634,7 @@ async fn run(args: Args, ctrl_c_received: Arc) -> Result<()> { let (room, _) = Room::connect(&url, &token, room_options).await?; let room = Arc::new(room); info!("Connected: {} - {}", room.name(), room.sid().await); + let encoding_control = EncodingControl::new(room.clone()); // Enable E2EE after connection if args.e2ee_key.is_some() { @@ -1360,10 +1669,12 @@ async fn run(args: Args, ctrl_c_received: Arc) -> Result<()> { let repaint_ctx_events = repaint_ctx.clone(); let ctrl_c_events = ctrl_c_received.clone(); let subscriber_timing_events = subscriber_timing.clone(); + let encoding_control_events = encoding_control.clone(); + let room_events = room.clone(); tokio::spawn(async move { let active_sid = active_sid.clone(); let simulcast = simulcast_events; - let mut events = room.subscribe(); + let mut events = room_events.subscribe(); info!("Subscribed to room events"); while let Some(evt) = events.recv().await { debug!("Room event: {:?}", evt); @@ -1381,6 +1692,7 @@ async fn run(args: Args, ctrl_c_received: Arc) -> Result<()> { &ctrl_c_events, &simulcast, &repaint_ctx_events, + &encoding_control_events, subscriber_timing_events.clone(), ) .await; @@ -1393,6 +1705,7 @@ async fn run(args: Args, ctrl_c_received: Arc) -> Result<()> { &video_size_events, &active_sid, &simulcast, + &encoding_control_events, &subscriber_timing_events, ); } @@ -1404,6 +1717,7 @@ async fn run(args: Args, ctrl_c_received: Arc) -> Result<()> { &video_size_events, &active_sid, &simulcast, + &encoding_control_events, &subscriber_timing_events, ); } @@ -1420,6 +1734,7 @@ async fn run(args: Args, ctrl_c_received: Arc) -> Result<()> { video_size, simulcast, subscriber_timing, + encoding_control, repaint_ctx, ctrl_c_received: ctrl_c_received.clone(), viewport, diff --git a/examples/local_video/src/video_display.rs b/examples/local_video/src/video_display.rs index 9acb7fd1e..dcd69e289 100644 --- a/examples/local_video/src/video_display.rs +++ b/examples/local_video/src/video_display.rs @@ -24,6 +24,7 @@ pub(crate) struct SharedYuv { pub(crate) v: Vec, pub(crate) codec: String, pub(crate) codec_implementation: String, + pub(crate) encode_bitrate_mbps: Option, pub(crate) fps: f32, pub(crate) simulcast: bool, pub(crate) dirty: bool, @@ -373,13 +374,16 @@ fn video_status_line( fps: f32, codec: &str, codec_implementation: &str, + encode_bitrate_mbps: Option, simulcast: bool, ) -> String { let codec = codec_with_implementation(codec, codec_implementation); + let bitrate = + encode_bitrate_mbps.map(|mbps| format!(" {:.1}mbps", mbps.max(0.0))).unwrap_or_default(); if simulcast { - format!("{}x{} {:.1}fps {codec} Simulcast", width, height, fps.max(0.0)) + format!("{}x{} {:.1}fps {codec}{bitrate} Simulcast", width, height, fps.max(0.0)) } else { - format!("{}x{} {:.1}fps {codec}", width, height, fps.max(0.0)) + format!("{}x{} {:.1}fps {codec}{bitrate}", width, height, fps.max(0.0)) } } @@ -401,6 +405,7 @@ fn publisher_overlay_lines( s.fps, &s.codec, &s.codec_implementation, + s.encode_bitrate_mbps, s.simulcast, ), s.timing_sample, @@ -453,6 +458,25 @@ mod tests { assert_eq!(lines, vec!["1280x720 29.6fps H264 NVENC Simulcast"]); } + #[test] + fn publisher_overlay_shows_encode_bitrate() { + let shared = Arc::new(Mutex::new(SharedYuv::default())); + { + let mut s = shared.lock(); + s.width = 1280; + s.height = 720; + s.codec = "H264".to_string(); + s.encode_bitrate_mbps = Some(1.25); + s.fps = 29.6; + } + + let mut overlay_state = PublisherTimingOverlayState::default(); + let lines = publisher_overlay_lines(&shared, &mut overlay_state, Instant::now()) + .expect("status overlay should render"); + + assert_eq!(lines, vec!["1280x720 29.6fps H264 1.2mbps"]); + } + #[test] fn preview_handoff_skips_unconsumed_frame() { let shared = Arc::new(Mutex::new(SharedYuv::default())); diff --git a/libwebrtc/src/native/rtp_parameters.rs b/libwebrtc/src/native/rtp_parameters.rs index fc6fc28b0..fd7e2976e 100644 --- a/libwebrtc/src/native/rtp_parameters.rs +++ b/libwebrtc/src/native/rtp_parameters.rs @@ -39,6 +39,7 @@ impl From for RtpParameters { Self { codecs: value.codecs.into_iter().map(Into::into).collect(), header_extensions: value.header_extensions.into_iter().map(Into::into).collect(), + encodings: value.encodings.into_iter().map(Into::into).collect(), rtcp: value.rtcp.into(), } } diff --git a/libwebrtc/src/native/rtp_sender.rs b/libwebrtc/src/native/rtp_sender.rs index f58b08484..cf8ca4371 100644 --- a/libwebrtc/src/native/rtp_sender.rs +++ b/libwebrtc/src/native/rtp_sender.rs @@ -76,8 +76,26 @@ impl RtpSender { } pub fn set_parameters(&self, parameters: RtpParameters) -> Result<(), RtcError> { + let mut native_parameters = self.sys_handle.get_parameters(); + for (native_encoding, encoding) in + native_parameters.encodings.iter_mut().zip(parameters.encodings) + { + native_encoding.active = encoding.active; + native_encoding.has_max_bitrate_bps = encoding.max_bitrate.is_some(); + native_encoding.max_bitrate_bps = encoding.max_bitrate.unwrap_or_default() as i32; + native_encoding.has_max_framerate = encoding.max_framerate.is_some(); + native_encoding.max_framerate = encoding.max_framerate.unwrap_or_default(); + native_encoding.network_priority = encoding.priority.into(); + native_encoding.has_scale_resolution_down_by = + encoding.scale_resolution_down_by.is_some(); + native_encoding.scale_resolution_down_by = + encoding.scale_resolution_down_by.unwrap_or_default(); + native_encoding.has_scalability_mode = encoding.scalability_mode.is_some(); + native_encoding.scalability_mode = encoding.scalability_mode.unwrap_or_default(); + } + self.sys_handle - .set_parameters(parameters.into()) + .set_parameters(native_parameters) .map_err(|e| unsafe { sys_err::ffi::RtcError::from(e.what()).into() }) } diff --git a/libwebrtc/src/rtp_parameters.rs b/libwebrtc/src/rtp_parameters.rs index 75d8b1830..da9a26170 100644 --- a/libwebrtc/src/rtp_parameters.rs +++ b/libwebrtc/src/rtp_parameters.rs @@ -33,6 +33,7 @@ pub struct RtpHeaderExtensionParameters { pub struct RtpParameters { pub codecs: Vec, pub header_extensions: Vec, + pub encodings: Vec, pub rtcp: RtcpParameters, } diff --git a/livekit/src/prelude.rs b/livekit/src/prelude.rs index 8c57f3108..cfc2bc006 100644 --- a/livekit/src/prelude.rs +++ b/livekit/src/prelude.rs @@ -30,7 +30,8 @@ pub use crate::{ AudioTrack, LocalAudioTrack, LocalTrack, LocalVideoTrack, PublishTimingEvent, PublishTimingEventStream, PublishTimingStage, RemoteAudioTrack, RemoteTrack, RemoteVideoTrack, StreamState, SubscribeTimingEvent, SubscribeTimingEventStream, - SubscribeTimingStage, Track, TrackDimension, TrackKind, TrackSource, VideoTrack, + SubscribeTimingStage, Track, TrackDimension, TrackKind, TrackSource, VideoEncodingLimits, + VideoTrack, }, ConnectionState, DataPacket, DataPacketKind, Room, RoomError, RoomEvent, RoomOptions, RoomResult, RoomSdkOptions, SipDTMF, Transcription, TranscriptionSegment, diff --git a/livekit/src/room/publication/local.rs b/livekit/src/room/publication/local.rs index 0a12e4e27..7a96d2886 100644 --- a/livekit/src/room/publication/local.rs +++ b/livekit/src/room/publication/local.rs @@ -78,6 +78,20 @@ impl LocalTrackPublication { self.local.publish_options.lock().clone() } + /// Sets runtime encoding limits for this publication's local video track. + /// + /// Pass `None` for an individual field to clear that explicit cap and + /// return control of that field to libwebrtc. + pub fn set_video_encoding_limits(&self, limits: VideoEncodingLimits) -> RoomResult<()> { + let Some(LocalTrack::Video(track)) = self.track() else { + return Err(RoomError::Internal( + "publication does not contain a local video track".into(), + )); + }; + + track.set_encoding_limits(limits) + } + pub fn mute(&self) { if let Some(track) = self.track() { track.mute(); diff --git a/livekit/src/room/track/local_video_track.rs b/livekit/src/room/track/local_video_track.rs index 65fc9596d..1676b2390 100644 --- a/livekit/src/room/track/local_video_track.rs +++ b/livekit/src/room/track/local_video_track.rs @@ -46,6 +46,17 @@ pub struct LocalVideoTrack { publish_timing_tx: Arc>>>, } +/// Runtime encoding limits for a published local video track. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct VideoEncodingLimits { + /// Maximum encoded bitrate in bits per second. + pub max_bitrate: Option, + /// Maximum encoded frame rate in frames per second. + pub max_framerate: Option, + /// Encoded resolution downscale factor. + pub scale_resolution_down_by: Option, +} + impl Debug for LocalVideoTrack { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("LocalVideoTrack") @@ -166,6 +177,40 @@ impl LocalVideoTrack { self.source.clone() } + /// Sets runtime encoding limits for all RTP encodings on this published video track. + /// + /// Pass `None` for an individual field to clear that explicit cap and + /// return control of that field to libwebrtc. The track must be published + /// before this method can update sender parameters. + pub(crate) fn set_encoding_limits(&self, limits: VideoEncodingLimits) -> RoomResult<()> { + self.update_encoding_parameters(|encoding| { + encoding.max_bitrate = limits.max_bitrate; + encoding.max_framerate = limits.max_framerate; + encoding.scale_resolution_down_by = limits.scale_resolution_down_by; + }) + } + + fn update_encoding_parameters( + &self, + mut update: impl FnMut(&mut RtpEncodingParameters), + ) -> RoomResult<()> { + let Some(transceiver) = self.transceiver() else { + return Err(RoomError::Rtc(RtcError { + error_type: RtcErrorType::InvalidState, + message: "track is not published".into(), + })); + }; + + let sender = transceiver.sender(); + let mut parameters = sender.parameters(); + for encoding in &mut parameters.encodings { + update(encoding); + } + sender.set_parameters(parameters)?; + + Ok(()) + } + /// Returns a stream of native local video publish-pipeline timing events. /// /// Multiple concurrent subscriptions are supported; each call returns an From 8c913cd822a8ac1884a8e101813980fcf0d1e4f3 Mon Sep 17 00:00:00 2001 From: David Chen Date: Mon, 8 Jun 2026 15:22:21 -0700 Subject: [PATCH 2/5] simulcast aware --- .changeset/local-video-encoding-controls.md | 2 +- examples/local_video/src/publisher.rs | 123 ++++-- examples/local_video/src/subscriber.rs | 20 +- libwebrtc/src/native/rtp_sender.rs | 11 + livekit/src/prelude.rs | 2 +- livekit/src/room/publication/local.rs | 24 +- livekit/src/room/track/local_video_track.rs | 397 +++++++++++++++++++- 7 files changed, 538 insertions(+), 41 deletions(-) diff --git a/.changeset/local-video-encoding-controls.md b/.changeset/local-video-encoding-controls.md index 9a957d8d6..cff1bdc4c 100644 --- a/.changeset/local-video-encoding-controls.md +++ b/.changeset/local-video-encoding-controls.md @@ -3,4 +3,4 @@ libwebrtc: patch livekit: patch --- -Add runtime video encoding limit controls for local video tracks and wire them into the `local_video` publisher/subscriber example via RPC. +Add simulcast-aware runtime video encoding limit controls for local video tracks, including quality-specific updates, and wire them into the `local_video` publisher/subscriber example via RPC. diff --git a/examples/local_video/src/publisher.rs b/examples/local_video/src/publisher.rs index 52e1be38b..7e14602ba 100644 --- a/examples/local_video/src/publisher.rs +++ b/examples/local_video/src/publisher.rs @@ -216,10 +216,30 @@ fn unix_time_us_now() -> u64 { const MAX_BACKEND_CAPTURE_TIMESTAMP_AGE_US: u64 = 5_000_000; const SET_VIDEO_ENCODING_LIMITS_METHOD: &str = "set-video-encoding-limits"; +const HIGH_RID: &str = "f"; + +#[derive(Clone, Copy, Debug, Deserialize, Serialize)] +#[serde(rename_all = "lowercase")] +enum EncodingQuality { + Low, + Medium, + High, +} + +impl From for VideoQuality { + fn from(quality: EncodingQuality) -> Self { + match quality { + EncodingQuality::Low => VideoQuality::Low, + EncodingQuality::Medium => VideoQuality::Medium, + EncodingQuality::High => VideoQuality::High, + } + } +} #[derive(Debug, Deserialize, Serialize)] struct SetEncodingLimitsRequest { track_sid: String, + quality: Option, bitrate_bps: Option, max_framerate: Option, scale_resolution_down_by: Option, @@ -231,6 +251,7 @@ struct SetEncodingLimitsResponse { applied_bitrate_bps: Option, applied_max_framerate: Option, applied_scale_resolution_down_by: Option, + quality: Option, track_sid: String, } @@ -348,16 +369,21 @@ struct PublisherTimingSummary { capture_to_webrtc_total_ms: RollingMs, } -#[derive(Default)] +#[derive(Clone, Default)] struct PublisherVideoOutboundStats { + rid: String, + active: bool, encoder_implementation: Option, target_bitrate_bps: Option, + frame_width: u32, + frame_height: u32, + frames_per_second: f64, } -fn find_video_outbound_stats( +fn collect_video_outbound_stats( stats: &[livekit::webrtc::stats::RtcStats], -) -> PublisherVideoOutboundStats { - let mut fallback = PublisherVideoOutboundStats::default(); +) -> Vec { + let mut outbounds = Vec::new(); for stat in stats { let livekit::webrtc::stats::RtcStats::OutboundRtp(outbound) = stat else { continue; @@ -366,24 +392,60 @@ fn find_video_outbound_stats( continue; } - let current = PublisherVideoOutboundStats { + outbounds.push(PublisherVideoOutboundStats { + rid: outbound.outbound.rid.clone(), + active: outbound.outbound.active, encoder_implementation: (!outbound.outbound.encoder_implementation.is_empty()) .then(|| outbound.outbound.encoder_implementation.clone()), target_bitrate_bps: (outbound.outbound.target_bitrate > 0.0) .then_some(outbound.outbound.target_bitrate), - }; - if outbound.outbound.active { - return current; - } - if fallback.encoder_implementation.is_none() { - fallback.encoder_implementation = current.encoder_implementation; - } - if fallback.target_bitrate_bps.is_none() { - fallback.target_bitrate_bps = current.target_bitrate_bps; - } + frame_width: outbound.outbound.frame_width, + frame_height: outbound.outbound.frame_height, + frames_per_second: outbound.outbound.frames_per_second, + }); + } + + outbounds +} + +fn find_video_outbound_stats( + outbounds: &[PublisherVideoOutboundStats], +) -> PublisherVideoOutboundStats { + outbounds + .iter() + .find(|outbound| outbound.active && outbound.rid == HIGH_RID) + .or_else(|| outbounds.iter().find(|outbound| outbound.active)) + .or_else(|| outbounds.iter().find(|outbound| outbound.rid == HIGH_RID)) + .or_else(|| outbounds.first()) + .cloned() + .unwrap_or_default() +} + +fn format_video_outbound_layers(outbounds: &[PublisherVideoOutboundStats]) -> Option { + if outbounds.len() <= 1 { + return None; } - fallback + let layers = outbounds + .iter() + .map(|outbound| { + let rid = if outbound.rid.is_empty() { "-" } else { outbound.rid.as_str() }; + let active = if outbound.active { "*" } else { "" }; + let bitrate = outbound + .target_bitrate_bps + .map(|bps| format!("{:.2}mbps", bps / 1_000_000.0)) + .unwrap_or_else(|| "?mbps".to_string()); + format!( + "{rid}{active}: {}x{} {:.1}fps {bitrate}", + outbound.frame_width, + outbound.frame_height, + outbound.frames_per_second.max(0.0), + ) + }) + .collect::>() + .join(", "); + + Some(format!("Publisher RTP encodings: {layers}")) } async fn update_publisher_encoder_overlay( @@ -393,6 +455,7 @@ async fn update_publisher_encoder_overlay( ) { let mut logged_initial = false; let mut last_implementation = String::new(); + let mut last_layers_line = String::new(); let mut interval = tokio::time::interval(Duration::from_secs(1)); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); @@ -403,7 +466,15 @@ async fn update_publisher_encoder_overlay( match track.get_stats().await { Ok(stats) => { - let outbound = find_video_outbound_stats(&stats); + let outbounds = collect_video_outbound_stats(&stats); + if let Some(layers_line) = format_video_outbound_layers(&outbounds) { + if layers_line != last_layers_line { + info!("{layers_line}"); + last_layers_line = layers_line; + } + } + + let outbound = find_video_outbound_stats(&outbounds); if let Some(implementation) = outbound.encoder_implementation { if implementation != last_implementation { info!("Publisher video encoder implementation: {implementation}"); @@ -613,13 +684,22 @@ fn register_encoding_limits_rpc(room: &Arc, publication: LocalTrackPublica max_framerate: request.max_framerate, scale_resolution_down_by: request.scale_resolution_down_by, }; - publication.set_video_encoding_limits(limits).map_err(|err| { - RpcError::new(500, format!("set encoding limits failed: {err}"), None) - })?; + if let Some(quality) = request.quality { + publication + .set_video_encoding_limits_for_quality(quality.into(), limits) + .map_err(|err| { + RpcError::new(500, format!("set encoding limits failed: {err}"), None) + })?; + } else { + publication.set_video_encoding_limits(limits).map_err(|err| { + RpcError::new(500, format!("set encoding limits failed: {err}"), None) + })?; + } info!( - "{} requested video encoding limits: {:?} bps, {:?} fps, {:?}x scale ({})", + "{} requested video encoding limits: quality {:?}, {:?} bps, {:?} fps, {:?}x scale ({})", data.caller_identity, + request.quality, request.bitrate_bps, request.max_framerate, request.scale_resolution_down_by, @@ -630,6 +710,7 @@ fn register_encoding_limits_rpc(room: &Arc, publication: LocalTrackPublica applied_bitrate_bps: request.bitrate_bps, applied_max_framerate: request.max_framerate, applied_scale_resolution_down_by: request.scale_resolution_down_by, + quality: request.quality, track_sid: publication_sid, }) .map_err(|err| { diff --git a/examples/local_video/src/subscriber.rs b/examples/local_video/src/subscriber.rs index 7b072ab26..b2072f623 100644 --- a/examples/local_video/src/subscriber.rs +++ b/examples/local_video/src/subscriber.rs @@ -41,6 +41,14 @@ const MIN_CONTROL_BITRATE_BPS: u64 = BITRATE_KEY_STEP_BPS; const MIN_FRAMERATE: f64 = 0.1; const MAX_FRAMERATE: f64 = 60.0; +#[derive(Clone, Copy, Debug, Deserialize, Serialize)] +#[serde(rename_all = "lowercase")] +enum EncodingQuality { + Low, + Medium, + High, +} + #[cfg(target_os = "macos")] mod macos_native_video { use std::ffi::c_void; @@ -422,6 +430,7 @@ struct Args { #[derive(Debug, Deserialize, Serialize)] struct SetEncodingLimitsRequest { track_sid: String, + quality: Option, bitrate_bps: Option, max_framerate: Option, scale_resolution_down_by: Option, @@ -433,6 +442,7 @@ struct SetEncodingLimitsResponse { applied_bitrate_bps: Option, applied_max_framerate: Option, applied_scale_resolution_down_by: Option, + quality: Option, track_sid: String, } @@ -554,7 +564,7 @@ impl EncodingControl { let room = self.inner.room.clone(); self.inner.handle.spawn(async move { - if let Err(err) = request_encoding_limits(&room, target, limits, reason).await { + if let Err(err) = request_encoding_limits(&room, target, None, limits, reason).await { log::warn!("encoding limits RPC failed: {err}"); } }); @@ -983,12 +993,14 @@ fn encoding_control_status_line(state: EncodingControlState) -> String { async fn request_encoding_limits( room: &Arc, target: EncodingControlTarget, + quality: Option, limits: VideoEncodingLimits, reason: &'static str, ) -> Result<()> { info!( - "Requesting video encoding limits from {}: {:?} bps, {:?} fps, {:?}x scale ({})", + "Requesting video encoding limits from {}: quality {:?}, {:?} bps, {:?} fps, {:?}x scale ({})", target.publisher_identity, + quality, limits.max_bitrate, limits.max_framerate, limits.scale_resolution_down_by, @@ -997,6 +1009,7 @@ async fn request_encoding_limits( let payload = serde_json::to_string(&SetEncodingLimitsRequest { track_sid: target.track_sid.to_string(), + quality, bitrate_bps: limits.max_bitrate, max_framerate: limits.max_framerate, scale_resolution_down_by: limits.scale_resolution_down_by, @@ -1015,8 +1028,9 @@ async fn request_encoding_limits( let response: SetEncodingLimitsResponse = serde_json::from_str(&response)?; info!( - "Publisher applied video encoding limits on {}: {:?} bps, {:?} fps, {:?}x scale", + "Publisher applied video encoding limits on {}: quality {:?}, {:?} bps, {:?} fps, {:?}x scale", response.track_sid, + response.quality, response.applied_bitrate_bps, response.applied_max_framerate, response.applied_scale_resolution_down_by, diff --git a/libwebrtc/src/native/rtp_sender.rs b/libwebrtc/src/native/rtp_sender.rs index cf8ca4371..735fe5202 100644 --- a/libwebrtc/src/native/rtp_sender.rs +++ b/libwebrtc/src/native/rtp_sender.rs @@ -77,6 +77,17 @@ impl RtpSender { pub fn set_parameters(&self, parameters: RtpParameters) -> Result<(), RtcError> { let mut native_parameters = self.sys_handle.get_parameters(); + if native_parameters.encodings.len() != parameters.encodings.len() { + return Err(RtcError { + error_type: RtcErrorType::InvalidState, + message: format!( + "encoding count changed from {} to {}", + native_parameters.encodings.len(), + parameters.encodings.len() + ), + }); + } + for (native_encoding, encoding) in native_parameters.encodings.iter_mut().zip(parameters.encodings) { diff --git a/livekit/src/prelude.rs b/livekit/src/prelude.rs index cfc2bc006..e6056963d 100644 --- a/livekit/src/prelude.rs +++ b/livekit/src/prelude.rs @@ -31,7 +31,7 @@ pub use crate::{ PublishTimingEventStream, PublishTimingStage, RemoteAudioTrack, RemoteTrack, RemoteVideoTrack, StreamState, SubscribeTimingEvent, SubscribeTimingEventStream, SubscribeTimingStage, Track, TrackDimension, TrackKind, TrackSource, VideoEncodingLimits, - VideoTrack, + VideoQuality, VideoTrack, }, ConnectionState, DataPacket, DataPacketKind, Room, RoomError, RoomEvent, RoomOptions, RoomResult, RoomSdkOptions, SipDTMF, Transcription, TranscriptionSegment, diff --git a/livekit/src/room/publication/local.rs b/livekit/src/room/publication/local.rs index 7a96d2886..1f51aa0e7 100644 --- a/livekit/src/room/publication/local.rs +++ b/livekit/src/room/publication/local.rs @@ -80,8 +80,9 @@ impl LocalTrackPublication { /// Sets runtime encoding limits for this publication's local video track. /// - /// Pass `None` for an individual field to clear that explicit cap and - /// return control of that field to libwebrtc. + /// Pass `None` for an individual field to restore the original publish-time + /// encoding value. For simulcasted video, `Some` values target the high + /// layer and lower layers preserve their original ratios. pub fn set_video_encoding_limits(&self, limits: VideoEncodingLimits) -> RoomResult<()> { let Some(LocalTrack::Video(track)) = self.track() else { return Err(RoomError::Internal( @@ -92,6 +93,25 @@ impl LocalTrackPublication { track.set_encoding_limits(limits) } + /// Sets runtime encoding limits for a specific video quality layer. + /// + /// For simulcasted video, [`VideoQuality::Low`], [`VideoQuality::Medium`], + /// and [`VideoQuality::High`] map to the standard LiveKit RIDs `q`, `h`, + /// and `f`. For non-simulcast video, only [`VideoQuality::High`] is valid. + pub fn set_video_encoding_limits_for_quality( + &self, + quality: VideoQuality, + limits: VideoEncodingLimits, + ) -> RoomResult<()> { + let Some(LocalTrack::Video(track)) = self.track() else { + return Err(RoomError::Internal( + "publication does not contain a local video track".into(), + )); + }; + + track.set_encoding_limits_for_quality(quality, limits) + } + pub fn mute(&self) { if let Some(track) = self.track() { track.mute(); diff --git a/livekit/src/room/track/local_video_track.rs b/livekit/src/room/track/local_video_track.rs index 1676b2390..c6202211e 100644 --- a/livekit/src/room/track/local_video_track.rs +++ b/livekit/src/room/track/local_video_track.rs @@ -31,17 +31,21 @@ use parking_lot::Mutex; use tokio::sync::broadcast; use tokio_stream::{wrappers::BroadcastStream, Stream}; -use super::TrackInner; +use super::{TrackInner, VideoQuality}; use crate::{prelude::*, rtc_engine::lk_runtime::LkRuntime}; pub use libwebrtc::native::packet_trailer::{PublishTimingEvent, PublishTimingStage}; const PUBLISH_TIMING_BUFFER: usize = 256; +const HIGH_RID: &str = "f"; +const MEDIUM_RID: &str = "h"; +const LOW_RID: &str = "q"; #[derive(Clone)] pub struct LocalVideoTrack { inner: Arc, source: RtcVideoSource, + baseline_encodings: Arc>>>, packet_trailer_handler: Arc>>, publish_timing_tx: Arc>>>, } @@ -98,6 +102,7 @@ impl LocalVideoTrack { MediaStreamTrack::Video(rtc_track), )), source, + baseline_encodings: Arc::new(Mutex::new(None)), packet_trailer_handler: Arc::new(Mutex::new(None)), publish_timing_tx: Arc::new(Mutex::new(None)), } @@ -177,22 +182,37 @@ impl LocalVideoTrack { self.source.clone() } - /// Sets runtime encoding limits for all RTP encodings on this published video track. + /// Sets runtime encoding limits for this published video track. /// - /// Pass `None` for an individual field to clear that explicit cap and - /// return control of that field to libwebrtc. The track must be published - /// before this method can update sender parameters. + /// Pass `None` for an individual field to restore the original publish-time + /// encoding value. The track must be published before this method can + /// update sender parameters. When the track is + /// simulcasted, `Some` values target the high layer and lower layers keep + /// the same ratios as the original publish-time encoding ladder. pub(crate) fn set_encoding_limits(&self, limits: VideoEncodingLimits) -> RoomResult<()> { - self.update_encoding_parameters(|encoding| { - encoding.max_bitrate = limits.max_bitrate; - encoding.max_framerate = limits.max_framerate; - encoding.scale_resolution_down_by = limits.scale_resolution_down_by; + self.update_encoding_parameters(|encodings, baseline| { + apply_track_encoding_limits(encodings, baseline, limits) + }) + } + + /// Sets runtime encoding limits for one RTP encoding on this published video track. + /// + /// On simulcasted tracks, the quality is mapped to the standard LiveKit + /// RIDs (`q`, `h`, `f`). On non-simulcast tracks, only [`VideoQuality::High`] + /// can be used. + pub(crate) fn set_encoding_limits_for_quality( + &self, + quality: VideoQuality, + limits: VideoEncodingLimits, + ) -> RoomResult<()> { + self.update_encoding_parameters(|encodings, baseline| { + apply_quality_encoding_limits(encodings, baseline, quality, limits) }) } fn update_encoding_parameters( &self, - mut update: impl FnMut(&mut RtpEncodingParameters), + update: impl FnOnce(&mut [RtpEncodingParameters], &[RtpEncodingParameters]) -> RoomResult<()>, ) -> RoomResult<()> { let Some(transceiver) = self.transceiver() else { return Err(RoomError::Rtc(RtcError { @@ -203,9 +223,12 @@ impl LocalVideoTrack { let sender = transceiver.sender(); let mut parameters = sender.parameters(); - for encoding in &mut parameters.encodings { - update(encoding); - } + let baseline = { + let mut baseline_encodings = self.baseline_encodings.lock(); + baseline_encodings.get_or_insert_with(|| parameters.encodings.clone()).clone() + }; + + update(&mut parameters.encodings, &baseline)?; sender.set_parameters(parameters)?; Ok(()) @@ -305,6 +328,11 @@ impl LocalVideoTrack { } pub(crate) fn set_transceiver(&self, transceiver: Option) { + let baseline = transceiver.as_ref().map(|transceiver| { + let sender = transceiver.sender(); + sender.parameters().encodings + }); + *self.baseline_encodings.lock() = baseline; self.inner.info.write().transceiver = transceiver; } @@ -312,3 +340,346 @@ impl LocalVideoTrack { super::update_info(&self.inner, &Track::LocalVideo(self.clone()), info); } } + +fn apply_track_encoding_limits( + encodings: &mut [RtpEncodingParameters], + baseline: &[RtpEncodingParameters], + limits: VideoEncodingLimits, +) -> RoomResult<()> { + validate_encoding_baseline(encodings, baseline)?; + + if encodings.len() == 1 { + encodings[0] = exact_encoding_limits(&encodings[0], &baseline[0], limits); + return Ok(()); + } + + let high_baseline = encoding_for_quality(baseline, VideoQuality::High)?; + let mut updated = Vec::with_capacity(encodings.len()); + + for encoding in encodings.iter() { + let quality = quality_for_rid(&encoding.rid).ok_or_else(|| { + invalid_state(format!("unsupported simulcast RID '{}'", encoding.rid)) + })?; + let encoding_baseline = encoding_for_quality(baseline, quality)?; + updated.push(scaled_encoding_limits(encoding, encoding_baseline, high_baseline, limits)?); + } + + encodings.clone_from_slice(&updated); + Ok(()) +} + +fn apply_quality_encoding_limits( + encodings: &mut [RtpEncodingParameters], + baseline: &[RtpEncodingParameters], + quality: VideoQuality, + limits: VideoEncodingLimits, +) -> RoomResult<()> { + validate_encoding_baseline(encodings, baseline)?; + + if encodings.len() == 1 { + if quality != VideoQuality::High { + return Err(invalid_state(format!( + "{quality:?} encoding limits require a simulcasted track" + ))); + } + + encodings[0] = exact_encoding_limits(&encodings[0], &baseline[0], limits); + return Ok(()); + } + + let rid = rid_for_quality(quality); + let index = encodings + .iter() + .position(|encoding| encoding.rid == rid) + .ok_or_else(|| invalid_state(format!("missing simulcast RID '{rid}'")))?; + let encoding_baseline = baseline + .iter() + .find(|encoding| encoding.rid == rid) + .ok_or_else(|| invalid_state(format!("missing baseline simulcast RID '{rid}'")))?; + + encodings[index] = exact_encoding_limits(&encodings[index], encoding_baseline, limits); + Ok(()) +} + +fn validate_encoding_baseline( + encodings: &[RtpEncodingParameters], + baseline: &[RtpEncodingParameters], +) -> RoomResult<()> { + if encodings.is_empty() { + return Err(invalid_state("track has no RTP encodings")); + } + if encodings.len() != baseline.len() { + return Err(invalid_state(format!( + "sender encoding count changed from {} to {}", + baseline.len(), + encodings.len() + ))); + } + Ok(()) +} + +fn scaled_encoding_limits( + encoding: &RtpEncodingParameters, + baseline: &RtpEncodingParameters, + high_baseline: &RtpEncodingParameters, + limits: VideoEncodingLimits, +) -> RoomResult { + let mut updated = encoding.clone(); + updated.max_bitrate = match limits.max_bitrate { + Some(max_bitrate) => Some(scale_u64( + max_bitrate, + required_u64(baseline.max_bitrate, "baseline max_bitrate")?, + required_u64(high_baseline.max_bitrate, "high baseline max_bitrate")?, + )?), + None => baseline.max_bitrate, + }; + updated.max_framerate = match limits.max_framerate { + Some(max_framerate) => Some(scale_f64( + max_framerate, + required_f64(baseline.max_framerate, "baseline max_framerate")?, + required_f64(high_baseline.max_framerate, "high baseline max_framerate")?, + )?), + None => baseline.max_framerate, + }; + updated.scale_resolution_down_by = match limits.scale_resolution_down_by { + Some(scale_resolution_down_by) => Some(scale_f64( + scale_resolution_down_by, + required_f64(baseline.scale_resolution_down_by, "baseline scale_resolution_down_by")?, + required_f64( + high_baseline.scale_resolution_down_by, + "high baseline scale_resolution_down_by", + )?, + )?), + None => baseline.scale_resolution_down_by, + }; + + Ok(updated) +} + +fn exact_encoding_limits( + encoding: &RtpEncodingParameters, + baseline: &RtpEncodingParameters, + limits: VideoEncodingLimits, +) -> RtpEncodingParameters { + let mut updated = encoding.clone(); + updated.max_bitrate = limits.max_bitrate.or(baseline.max_bitrate); + updated.max_framerate = limits.max_framerate.or(baseline.max_framerate); + updated.scale_resolution_down_by = + limits.scale_resolution_down_by.or(baseline.scale_resolution_down_by); + updated +} + +fn required_u64(value: Option, field: &'static str) -> RoomResult { + value.ok_or_else(|| invalid_state(format!("missing {field}"))) +} + +fn required_f64(value: Option, field: &'static str) -> RoomResult { + value.ok_or_else(|| invalid_state(format!("missing {field}"))) +} + +fn scale_u64(target_high: u64, baseline: u64, high_baseline: u64) -> RoomResult { + if high_baseline == 0 { + return Err(invalid_state("high baseline max_bitrate is zero")); + } + Ok(((target_high as f64 * baseline as f64 / high_baseline as f64).round() as u64).max(1)) +} + +fn scale_f64(target_high: f64, baseline: f64, high_baseline: f64) -> RoomResult { + if high_baseline <= 0.0 { + return Err(invalid_state("high baseline value must be greater than zero")); + } + Ok(target_high * baseline / high_baseline) +} + +fn encoding_for_quality( + encodings: &[RtpEncodingParameters], + quality: VideoQuality, +) -> RoomResult<&RtpEncodingParameters> { + let rid = rid_for_quality(quality); + encodings + .iter() + .find(|encoding| encoding.rid == rid) + .ok_or_else(|| invalid_state(format!("missing baseline simulcast RID '{rid}'"))) +} + +fn rid_for_quality(quality: VideoQuality) -> &'static str { + match quality { + VideoQuality::Low => LOW_RID, + VideoQuality::Medium => MEDIUM_RID, + VideoQuality::High => HIGH_RID, + } +} + +fn quality_for_rid(rid: &str) -> Option { + match rid { + LOW_RID => Some(VideoQuality::Low), + MEDIUM_RID => Some(VideoQuality::Medium), + HIGH_RID => Some(VideoQuality::High), + _ => None, + } +} + +fn invalid_state(message: impl Into) -> RoomError { + RoomError::Rtc(RtcError { error_type: RtcErrorType::InvalidState, message: message.into() }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn encoding( + rid: &str, + max_bitrate: u64, + max_framerate: f64, + scale_resolution_down_by: f64, + ) -> RtpEncodingParameters { + RtpEncodingParameters { + rid: rid.to_string(), + max_bitrate: Some(max_bitrate), + max_framerate: Some(max_framerate), + scale_resolution_down_by: Some(scale_resolution_down_by), + ..Default::default() + } + } + + fn simulcast_baseline() -> Vec { + vec![ + encoding(HIGH_RID, 1_700_000, 30.0, 1.0), + encoding(MEDIUM_RID, 450_000, 30.0, 2.0), + encoding(LOW_RID, 160_000, 30.0, 4.0), + ] + } + + fn assert_encoding_matches(encoding: &RtpEncodingParameters, expected: &RtpEncodingParameters) { + assert_eq!(encoding.rid, expected.rid); + assert_eq!(encoding.max_bitrate, expected.max_bitrate); + assert_eq!(encoding.max_framerate, expected.max_framerate); + assert_eq!(encoding.scale_resolution_down_by, expected.scale_resolution_down_by); + } + + fn assert_encodings_match( + encodings: &[RtpEncodingParameters], + expected: &[RtpEncodingParameters], + ) { + assert_eq!(encodings.len(), expected.len()); + for (encoding, expected) in encodings.iter().zip(expected) { + assert_encoding_matches(encoding, expected); + } + } + + #[test] + fn track_limits_preserve_simulcast_ratios() { + let baseline = simulcast_baseline(); + let mut encodings = baseline.clone(); + + apply_track_encoding_limits( + &mut encodings, + &baseline, + VideoEncodingLimits { + max_bitrate: Some(850_000), + max_framerate: Some(15.0), + scale_resolution_down_by: Some(2.0), + }, + ) + .expect("track limits should apply"); + + assert_eq!(encodings[0].max_bitrate, Some(850_000)); + assert_eq!(encodings[1].max_bitrate, Some(225_000)); + assert_eq!(encodings[2].max_bitrate, Some(80_000)); + assert_eq!(encodings[0].max_framerate, Some(15.0)); + assert_eq!(encodings[1].max_framerate, Some(15.0)); + assert_eq!(encodings[2].max_framerate, Some(15.0)); + assert_eq!(encodings[0].scale_resolution_down_by, Some(2.0)); + assert_eq!(encodings[1].scale_resolution_down_by, Some(4.0)); + assert_eq!(encodings[2].scale_resolution_down_by, Some(8.0)); + } + + #[test] + fn track_limits_restore_baseline_fields_when_cleared() { + let baseline = simulcast_baseline(); + let mut encodings = vec![ + encoding(HIGH_RID, 900_000, 10.0, 2.0), + encoding(MEDIUM_RID, 240_000, 10.0, 4.0), + encoding(LOW_RID, 85_000, 10.0, 8.0), + ]; + + apply_track_encoding_limits( + &mut encodings, + &baseline, + VideoEncodingLimits { + max_bitrate: None, + max_framerate: None, + scale_resolution_down_by: None, + }, + ) + .expect("clearing limits should apply"); + + assert_encodings_match(&encodings, &baseline); + } + + #[test] + fn quality_limits_only_update_requested_simulcast_layer() { + let baseline = simulcast_baseline(); + let mut encodings = baseline.clone(); + + apply_quality_encoding_limits( + &mut encodings, + &baseline, + VideoQuality::Medium, + VideoEncodingLimits { + max_bitrate: Some(300_000), + max_framerate: Some(12.0), + scale_resolution_down_by: Some(3.0), + }, + ) + .expect("quality limits should apply"); + + assert_encoding_matches(&encodings[0], &baseline[0]); + assert_encoding_matches(&encodings[2], &baseline[2]); + assert_eq!(encodings[1].max_bitrate, Some(300_000)); + assert_eq!(encodings[1].max_framerate, Some(12.0)); + assert_eq!(encodings[1].scale_resolution_down_by, Some(3.0)); + } + + #[test] + fn quality_limits_fail_when_quality_is_missing() { + let baseline = vec![encoding(HIGH_RID, 1_700_000, 30.0, 1.0)]; + let mut encodings = baseline.clone(); + + let err = apply_quality_encoding_limits( + &mut encodings, + &baseline, + VideoQuality::Low, + VideoEncodingLimits { max_bitrate: Some(80_000), ..Default::default() }, + ) + .expect_err("low quality requires a simulcast encoding"); + + assert!(matches!( + err, + RoomError::Rtc(RtcError { error_type: RtcErrorType::InvalidState, .. }) + )); + assert_encodings_match(&encodings, &baseline); + } + + #[test] + fn high_quality_limits_apply_to_single_encoding() { + let baseline = vec![encoding("", 1_700_000, 30.0, 1.0)]; + let mut encodings = baseline.clone(); + + apply_quality_encoding_limits( + &mut encodings, + &baseline, + VideoQuality::High, + VideoEncodingLimits { + max_bitrate: Some(900_000), + max_framerate: None, + scale_resolution_down_by: Some(2.0), + }, + ) + .expect("high quality should apply to one encoding"); + + assert_eq!(encodings[0].max_bitrate, Some(900_000)); + assert_eq!(encodings[0].max_framerate, Some(30.0)); + assert_eq!(encodings[0].scale_resolution_down_by, Some(2.0)); + } +} From af0adaf0ff082d55008ddcbd24390cd97e25ca45 Mon Sep 17 00:00:00 2001 From: David Chen Date: Mon, 8 Jun 2026 16:19:25 -0700 Subject: [PATCH 3/5] update subscriber example --- examples/local_video/src/subscriber.rs | 107 ++++++++++++++++++++----- 1 file changed, 89 insertions(+), 18 deletions(-) diff --git a/examples/local_video/src/subscriber.rs b/examples/local_video/src/subscriber.rs index b2072f623..a115ed97b 100644 --- a/examples/local_video/src/subscriber.rs +++ b/examples/local_video/src/subscriber.rs @@ -41,7 +41,7 @@ const MIN_CONTROL_BITRATE_BPS: u64 = BITRATE_KEY_STEP_BPS; const MIN_FRAMERATE: f64 = 0.1; const MAX_FRAMERATE: f64 = 60.0; -#[derive(Clone, Copy, Debug, Deserialize, Serialize)] +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(rename_all = "lowercase")] enum EncodingQuality { Low, @@ -49,6 +49,16 @@ enum EncodingQuality { High, } +impl From for EncodingQuality { + fn from(quality: livekit::track::VideoQuality) -> Self { + match quality { + livekit::track::VideoQuality::Low => Self::Low, + livekit::track::VideoQuality::Medium => Self::Medium, + livekit::track::VideoQuality::High => Self::High, + } + } +} + #[cfg(target_os = "macos")] mod macos_native_video { use std::ffi::c_void; @@ -480,6 +490,7 @@ struct EncodingControl { struct EncodingControlInner { room: Arc, + simulcast: Arc>, target: Mutex>, state: Mutex, handle: tokio::runtime::Handle, @@ -492,10 +503,11 @@ struct EncodingControlTarget { } impl EncodingControl { - fn new(room: Arc) -> Self { + fn new(room: Arc, simulcast: Arc>) -> Self { Self { inner: Arc::new(EncodingControlInner { room, + simulcast, target: Mutex::new(None), state: Mutex::new(EncodingControlState::default()), handle: tokio::runtime::Handle::current(), @@ -552,10 +564,14 @@ impl EncodingControl { } fn update_limits(&self, reason: &'static str, update: impl FnOnce(&mut EncodingControlState)) { - let (limits, target) = { + let (limits, target, quality) = { let mut state = self.inner.state.lock(); update(&mut state); - (state.limits(), self.inner.target.lock().clone()) + ( + state.limits(), + self.inner.target.lock().clone(), + simulcast_encoding_quality(&self.inner.simulcast.lock()), + ) }; let Some(target) = target else { debug!("No active publisher video track for encoding control request"); @@ -564,7 +580,8 @@ impl EncodingControl { let room = self.inner.room.clone(); self.inner.handle.spawn(async move { - if let Err(err) = request_encoding_limits(&room, target, None, limits, reason).await { + let result = request_encoding_limits(&room, target, quality, limits, reason).await; + if let Err(err) = result { log::warn!("encoding limits RPC failed: {err}"); } }); @@ -735,6 +752,16 @@ impl Default for SimulcastState { } } +fn simulcast_encoding_quality(state: &SimulcastState) -> Option { + state.available.then(|| { + state + .active_quality + .or(state.requested_quality) + .unwrap_or(livekit::track::VideoQuality::High) + .into() + }) +} + fn infer_quality_from_dims( full_w: u32, _full_h: u32, @@ -879,18 +906,28 @@ fn update_simulcast_quality_from_stats( let Some(inbound) = find_video_inbound_stats(stats) else { return; }; - let Some((fw, fh)) = simulcast_state_full_dims(simulcast) else { - return; - }; - let q = infer_quality_from_dims( - fw, - fh, + update_simulcast_quality_from_dimensions( inbound.inbound.frame_width as u32, inbound.inbound.frame_height as u32, + simulcast, ); +} + +fn update_simulcast_quality_from_dimensions( + current_width: u32, + current_height: u32, + simulcast: &Arc>, +) { let mut sc = simulcast.lock(); - sc.active_quality = Some(q); + if !sc.available { + return; + } + let Some((fw, fh)) = sc.full_dims else { + return; + }; + + sc.active_quality = Some(infer_quality_from_dims(fw, fh, current_width, current_height)); } fn update_decoder_implementation_from_stats( @@ -957,11 +994,6 @@ fn current_timestamp_us() -> u64 { anchor.unix_timestamp_us.saturating_add(anchor.instant.elapsed().as_micros() as u64) } -fn simulcast_state_full_dims(state: &Arc>) -> Option<(u32, u32)> { - let sc = state.lock(); - sc.full_dims -} - fn video_status_line( width: u32, height: u32, @@ -1094,6 +1126,43 @@ mod tests { assert_eq!(line, "Encoding limit 1.2mbps 0.5fps 4.0x scale"); } + + #[test] + fn encoding_control_skips_quality_for_non_simulcast_tracks() { + let state = SimulcastState::default(); + + assert_eq!(simulcast_encoding_quality(&state), None); + } + + #[test] + fn encoding_control_targets_active_simulcast_layer() { + let state = SimulcastState { + available: true, + requested_quality: Some(livekit::track::VideoQuality::High), + active_quality: Some(livekit::track::VideoQuality::Medium), + ..Default::default() + }; + + assert_eq!(simulcast_encoding_quality(&state), Some(EncodingQuality::Medium)); + } + + #[test] + fn encoding_control_targets_requested_quality_until_active_layer_is_known() { + let state = SimulcastState { + available: true, + requested_quality: Some(livekit::track::VideoQuality::Low), + ..Default::default() + }; + + assert_eq!(simulcast_encoding_quality(&state), Some(EncodingQuality::Low)); + } + + #[test] + fn encoding_control_defaults_to_high_for_new_simulcast_tracks() { + let state = SimulcastState { available: true, ..Default::default() }; + + assert_eq!(simulcast_encoding_quality(&state), Some(EncodingQuality::High)); + } } async fn handle_track_subscribed( @@ -1172,6 +1241,7 @@ async fn handle_track_subscribed( // Start background sink task immediately so stats lookup cannot delay first-frame handling. let rtc_track = video_track.rtc_track(); let shared2 = shared.clone(); + let simulcast_sink = simulcast.clone(); let frame_slot_sink = frame_slot.clone(); let video_size_sink = video_size.clone(); let active_sid2 = active_sid.clone(); @@ -1222,6 +1292,7 @@ async fn handle_track_subscribed( } let w = frame.buffer.width(); let h = frame.buffer.height(); + update_simulcast_quality_from_dimensions(w, h, &simulcast_sink); if !logged_first { debug!("First frame: {}x{}, type {:?}", w, h, frame.buffer.buffer_type()); @@ -1648,7 +1719,6 @@ async fn run(args: Args, ctrl_c_received: Arc) -> Result<()> { let (room, _) = Room::connect(&url, &token, room_options).await?; let room = Arc::new(room); info!("Connected: {} - {}", room.name(), room.sid().await); - let encoding_control = EncodingControl::new(room.clone()); // Enable E2EE after connection if args.e2ee_key.is_some() { @@ -1678,6 +1748,7 @@ async fn run(args: Args, ctrl_c_received: Arc) -> Result<()> { let active_sid = Arc::new(Mutex::new(None::)); // Shared simulcast UI/control state let simulcast = Arc::new(Mutex::new(SimulcastState::default())); + let encoding_control = EncodingControl::new(room.clone(), simulcast.clone()); let repaint_ctx = Arc::new(OnceLock::new()); let simulcast_events = simulcast.clone(); let repaint_ctx_events = repaint_ctx.clone(); From c5a458a1d6823f415e913ee4e88c2a91a080a854 Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 9 Jun 2026 13:52:33 -0700 Subject: [PATCH 4/5] only update the high simulcast layer and preserve scaling for the whole ladder --- .changeset/local-video-encoding-controls.md | 2 +- examples/local_video/src/publisher.rs | 60 ++++----- examples/local_video/src/subscriber.rs | 94 ++----------- livekit/src/room/publication/local.rs | 19 --- livekit/src/room/track/local_video_track.rs | 142 +++++++------------- 5 files changed, 90 insertions(+), 227 deletions(-) diff --git a/.changeset/local-video-encoding-controls.md b/.changeset/local-video-encoding-controls.md index cff1bdc4c..e65c1a7fa 100644 --- a/.changeset/local-video-encoding-controls.md +++ b/.changeset/local-video-encoding-controls.md @@ -3,4 +3,4 @@ libwebrtc: patch livekit: patch --- -Add simulcast-aware runtime video encoding limit controls for local video tracks, including quality-specific updates, and wire them into the `local_video` publisher/subscriber example via RPC. +Add simulcast-aware runtime video encoding limit controls for local video tracks, preserving the publish-time simulcast ladder while applying track-level caps through the `local_video` publisher/subscriber example RPC. diff --git a/examples/local_video/src/publisher.rs b/examples/local_video/src/publisher.rs index 7e14602ba..3cd32124c 100644 --- a/examples/local_video/src/publisher.rs +++ b/examples/local_video/src/publisher.rs @@ -218,28 +218,10 @@ const MAX_BACKEND_CAPTURE_TIMESTAMP_AGE_US: u64 = 5_000_000; const SET_VIDEO_ENCODING_LIMITS_METHOD: &str = "set-video-encoding-limits"; const HIGH_RID: &str = "f"; -#[derive(Clone, Copy, Debug, Deserialize, Serialize)] -#[serde(rename_all = "lowercase")] -enum EncodingQuality { - Low, - Medium, - High, -} - -impl From for VideoQuality { - fn from(quality: EncodingQuality) -> Self { - match quality { - EncodingQuality::Low => VideoQuality::Low, - EncodingQuality::Medium => VideoQuality::Medium, - EncodingQuality::High => VideoQuality::High, - } - } -} - #[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] struct SetEncodingLimitsRequest { track_sid: String, - quality: Option, bitrate_bps: Option, max_framerate: Option, scale_resolution_down_by: Option, @@ -251,7 +233,6 @@ struct SetEncodingLimitsResponse { applied_bitrate_bps: Option, applied_max_framerate: Option, applied_scale_resolution_down_by: Option, - quality: Option, track_sid: String, } @@ -468,6 +449,7 @@ async fn update_publisher_encoder_overlay( Ok(stats) => { let outbounds = collect_video_outbound_stats(&stats); if let Some(layers_line) = format_video_outbound_layers(&outbounds) { + debug!("{layers_line}"); if layers_line != last_layers_line { info!("{layers_line}"); last_layers_line = layers_line; @@ -661,6 +643,10 @@ fn register_encoding_limits_rpc(room: &Arc, publication: LocalTrackPublica move |data| { let publication = publication.clone(); Box::pin(async move { + debug!( + "Raw video encoding limits RPC from {}: {}", + data.caller_identity, data.payload + ); let request: SetEncodingLimitsRequest = serde_json::from_str(&data.payload) .map_err(|err| { RpcError::new( @@ -679,38 +665,38 @@ fn register_encoding_limits_rpc(room: &Arc, publication: LocalTrackPublica )); } + info!( + "{} requested video encoding limits: track={}, {:?} bps, {:?} fps, {:?}x scale ({})", + data.caller_identity, + request.track_sid, + request.bitrate_bps, + request.max_framerate, + request.scale_resolution_down_by, + request.reason + ); + let limits = VideoEncodingLimits { max_bitrate: request.bitrate_bps, max_framerate: request.max_framerate, scale_resolution_down_by: request.scale_resolution_down_by, }; - if let Some(quality) = request.quality { - publication - .set_video_encoding_limits_for_quality(quality.into(), limits) - .map_err(|err| { - RpcError::new(500, format!("set encoding limits failed: {err}"), None) - })?; - } else { - publication.set_video_encoding_limits(limits).map_err(|err| { - RpcError::new(500, format!("set encoding limits failed: {err}"), None) - })?; - } + debug!("Applying track-level publisher encoding limits"); + publication.set_video_encoding_limits(limits).map_err(|err| { + RpcError::new(500, format!("set encoding limits failed: {err}"), None) + })?; info!( - "{} requested video encoding limits: quality {:?}, {:?} bps, {:?} fps, {:?}x scale ({})", - data.caller_identity, - request.quality, + "Applied video encoding limits: track={}, {:?} bps, {:?} fps, {:?}x scale", + publication_sid, request.bitrate_bps, request.max_framerate, - request.scale_resolution_down_by, - request.reason + request.scale_resolution_down_by ); serde_json::to_string(&SetEncodingLimitsResponse { applied_bitrate_bps: request.bitrate_bps, applied_max_framerate: request.max_framerate, applied_scale_resolution_down_by: request.scale_resolution_down_by, - quality: request.quality, track_sid: publication_sid, }) .map_err(|err| { diff --git a/examples/local_video/src/subscriber.rs b/examples/local_video/src/subscriber.rs index a115ed97b..ca898078f 100644 --- a/examples/local_video/src/subscriber.rs +++ b/examples/local_video/src/subscriber.rs @@ -41,24 +41,6 @@ const MIN_CONTROL_BITRATE_BPS: u64 = BITRATE_KEY_STEP_BPS; const MIN_FRAMERATE: f64 = 0.1; const MAX_FRAMERATE: f64 = 60.0; -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] -#[serde(rename_all = "lowercase")] -enum EncodingQuality { - Low, - Medium, - High, -} - -impl From for EncodingQuality { - fn from(quality: livekit::track::VideoQuality) -> Self { - match quality { - livekit::track::VideoQuality::Low => Self::Low, - livekit::track::VideoQuality::Medium => Self::Medium, - livekit::track::VideoQuality::High => Self::High, - } - } -} - #[cfg(target_os = "macos")] mod macos_native_video { use std::ffi::c_void; @@ -438,9 +420,9 @@ struct Args { } #[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] struct SetEncodingLimitsRequest { track_sid: String, - quality: Option, bitrate_bps: Option, max_framerate: Option, scale_resolution_down_by: Option, @@ -452,7 +434,6 @@ struct SetEncodingLimitsResponse { applied_bitrate_bps: Option, applied_max_framerate: Option, applied_scale_resolution_down_by: Option, - quality: Option, track_sid: String, } @@ -564,14 +545,17 @@ impl EncodingControl { } fn update_limits(&self, reason: &'static str, update: impl FnOnce(&mut EncodingControlState)) { - let (limits, target, quality) = { + let (limits, target) = { let mut state = self.inner.state.lock(); update(&mut state); - ( - state.limits(), - self.inner.target.lock().clone(), - simulcast_encoding_quality(&self.inner.simulcast.lock()), - ) + let simulcast = self.inner.simulcast.lock(); + debug!( + "Encoding control ladder update: requested={:?}, active={:?}, simulcast={}, reason={reason}", + simulcast.requested_quality, + simulcast.active_quality, + simulcast.available + ); + (state.limits(), self.inner.target.lock().clone()) }; let Some(target) = target else { debug!("No active publisher video track for encoding control request"); @@ -580,7 +564,7 @@ impl EncodingControl { let room = self.inner.room.clone(); self.inner.handle.spawn(async move { - let result = request_encoding_limits(&room, target, quality, limits, reason).await; + let result = request_encoding_limits(&room, target, limits, reason).await; if let Err(err) = result { log::warn!("encoding limits RPC failed: {err}"); } @@ -752,16 +736,6 @@ impl Default for SimulcastState { } } -fn simulcast_encoding_quality(state: &SimulcastState) -> Option { - state.available.then(|| { - state - .active_quality - .or(state.requested_quality) - .unwrap_or(livekit::track::VideoQuality::High) - .into() - }) -} - fn infer_quality_from_dims( full_w: u32, _full_h: u32, @@ -1025,14 +999,12 @@ fn encoding_control_status_line(state: EncodingControlState) -> String { async fn request_encoding_limits( room: &Arc, target: EncodingControlTarget, - quality: Option, limits: VideoEncodingLimits, reason: &'static str, ) -> Result<()> { info!( - "Requesting video encoding limits from {}: quality {:?}, {:?} bps, {:?} fps, {:?}x scale ({})", + "Requesting video encoding limits from {}: {:?} bps, {:?} fps, {:?}x scale ({})", target.publisher_identity, - quality, limits.max_bitrate, limits.max_framerate, limits.scale_resolution_down_by, @@ -1041,7 +1013,6 @@ async fn request_encoding_limits( let payload = serde_json::to_string(&SetEncodingLimitsRequest { track_sid: target.track_sid.to_string(), - quality, bitrate_bps: limits.max_bitrate, max_framerate: limits.max_framerate, scale_resolution_down_by: limits.scale_resolution_down_by, @@ -1060,9 +1031,8 @@ async fn request_encoding_limits( let response: SetEncodingLimitsResponse = serde_json::from_str(&response)?; info!( - "Publisher applied video encoding limits on {}: quality {:?}, {:?} bps, {:?} fps, {:?}x scale", + "Publisher applied video encoding limits on {}: {:?} bps, {:?} fps, {:?}x scale", response.track_sid, - response.quality, response.applied_bitrate_bps, response.applied_max_framerate, response.applied_scale_resolution_down_by, @@ -1126,43 +1096,6 @@ mod tests { assert_eq!(line, "Encoding limit 1.2mbps 0.5fps 4.0x scale"); } - - #[test] - fn encoding_control_skips_quality_for_non_simulcast_tracks() { - let state = SimulcastState::default(); - - assert_eq!(simulcast_encoding_quality(&state), None); - } - - #[test] - fn encoding_control_targets_active_simulcast_layer() { - let state = SimulcastState { - available: true, - requested_quality: Some(livekit::track::VideoQuality::High), - active_quality: Some(livekit::track::VideoQuality::Medium), - ..Default::default() - }; - - assert_eq!(simulcast_encoding_quality(&state), Some(EncodingQuality::Medium)); - } - - #[test] - fn encoding_control_targets_requested_quality_until_active_layer_is_known() { - let state = SimulcastState { - available: true, - requested_quality: Some(livekit::track::VideoQuality::Low), - ..Default::default() - }; - - assert_eq!(simulcast_encoding_quality(&state), Some(EncodingQuality::Low)); - } - - #[test] - fn encoding_control_defaults_to_high_for_new_simulcast_tracks() { - let state = SimulcastState { available: true, ..Default::default() }; - - assert_eq!(simulcast_encoding_quality(&state), Some(EncodingQuality::High)); - } } async fn handle_track_subscribed( @@ -1644,6 +1577,7 @@ impl eframe::App for VideoApp { let resp = ui.selectable_label(is_selected, label); if resp.clicked() { if let Some(ref pub_remote) = sc.publication { + info!("Requesting subscriber simulcast quality {q:?}"); pub_remote.set_video_quality(q); sc.requested_quality = Some(q); } diff --git a/livekit/src/room/publication/local.rs b/livekit/src/room/publication/local.rs index 1f51aa0e7..fe14ad06f 100644 --- a/livekit/src/room/publication/local.rs +++ b/livekit/src/room/publication/local.rs @@ -93,25 +93,6 @@ impl LocalTrackPublication { track.set_encoding_limits(limits) } - /// Sets runtime encoding limits for a specific video quality layer. - /// - /// For simulcasted video, [`VideoQuality::Low`], [`VideoQuality::Medium`], - /// and [`VideoQuality::High`] map to the standard LiveKit RIDs `q`, `h`, - /// and `f`. For non-simulcast video, only [`VideoQuality::High`] is valid. - pub fn set_video_encoding_limits_for_quality( - &self, - quality: VideoQuality, - limits: VideoEncodingLimits, - ) -> RoomResult<()> { - let Some(LocalTrack::Video(track)) = self.track() else { - return Err(RoomError::Internal( - "publication does not contain a local video track".into(), - )); - }; - - track.set_encoding_limits_for_quality(quality, limits) - } - pub fn mute(&self) { if let Some(track) = self.track() { track.mute(); diff --git a/livekit/src/room/track/local_video_track.rs b/livekit/src/room/track/local_video_track.rs index c6202211e..17c3e9636 100644 --- a/livekit/src/room/track/local_video_track.rs +++ b/livekit/src/room/track/local_video_track.rs @@ -190,26 +190,12 @@ impl LocalVideoTrack { /// simulcasted, `Some` values target the high layer and lower layers keep /// the same ratios as the original publish-time encoding ladder. pub(crate) fn set_encoding_limits(&self, limits: VideoEncodingLimits) -> RoomResult<()> { + log::debug!("applying track-level local video encoding limits: {limits:?}"); self.update_encoding_parameters(|encodings, baseline| { apply_track_encoding_limits(encodings, baseline, limits) }) } - /// Sets runtime encoding limits for one RTP encoding on this published video track. - /// - /// On simulcasted tracks, the quality is mapped to the standard LiveKit - /// RIDs (`q`, `h`, `f`). On non-simulcast tracks, only [`VideoQuality::High`] - /// can be used. - pub(crate) fn set_encoding_limits_for_quality( - &self, - quality: VideoQuality, - limits: VideoEncodingLimits, - ) -> RoomResult<()> { - self.update_encoding_parameters(|encodings, baseline| { - apply_quality_encoding_limits(encodings, baseline, quality, limits) - }) - } - fn update_encoding_parameters( &self, update: impl FnOnce(&mut [RtpEncodingParameters], &[RtpEncodingParameters]) -> RoomResult<()>, @@ -227,8 +213,15 @@ impl LocalVideoTrack { let mut baseline_encodings = self.baseline_encodings.lock(); baseline_encodings.get_or_insert_with(|| parameters.encodings.clone()).clone() }; + let before = parameters.encodings.clone(); update(&mut parameters.encodings, &baseline)?; + log::debug!( + "local video sender encoding update: baseline=[{}], before=[{}], after=[{}]", + format_encoding_parameters(&baseline), + format_encoding_parameters(&before), + format_encoding_parameters(¶meters.encodings) + ); sender.set_parameters(parameters)?; Ok(()) @@ -332,6 +325,13 @@ impl LocalVideoTrack { let sender = transceiver.sender(); sender.parameters().encodings }); + log::debug!( + "local video sender baseline encodings: [{}]", + baseline + .as_deref() + .map(format_encoding_parameters) + .unwrap_or_else(|| "none".to_string()) + ); *self.baseline_encodings.lock() = baseline; self.inner.info.write().transceiver = transceiver; } @@ -368,39 +368,6 @@ fn apply_track_encoding_limits( Ok(()) } -fn apply_quality_encoding_limits( - encodings: &mut [RtpEncodingParameters], - baseline: &[RtpEncodingParameters], - quality: VideoQuality, - limits: VideoEncodingLimits, -) -> RoomResult<()> { - validate_encoding_baseline(encodings, baseline)?; - - if encodings.len() == 1 { - if quality != VideoQuality::High { - return Err(invalid_state(format!( - "{quality:?} encoding limits require a simulcasted track" - ))); - } - - encodings[0] = exact_encoding_limits(&encodings[0], &baseline[0], limits); - return Ok(()); - } - - let rid = rid_for_quality(quality); - let index = encodings - .iter() - .position(|encoding| encoding.rid == rid) - .ok_or_else(|| invalid_state(format!("missing simulcast RID '{rid}'")))?; - let encoding_baseline = baseline - .iter() - .find(|encoding| encoding.rid == rid) - .ok_or_else(|| invalid_state(format!("missing baseline simulcast RID '{rid}'")))?; - - encodings[index] = exact_encoding_limits(&encodings[index], encoding_baseline, limits); - Ok(()) -} - fn validate_encoding_baseline( encodings: &[RtpEncodingParameters], baseline: &[RtpEncodingParameters], @@ -519,6 +486,25 @@ fn quality_for_rid(rid: &str) -> Option { } } +fn format_encoding_parameters(encodings: &[RtpEncodingParameters]) -> String { + encodings + .iter() + .enumerate() + .map(|(index, encoding)| { + let rid = if encoding.rid.is_empty() { "-" } else { encoding.rid.as_str() }; + format!( + "#{index} rid={rid} active={} bitrate={:?} fps={:?} scale={:?} scalability={:?}", + encoding.active, + encoding.max_bitrate, + encoding.max_framerate, + encoding.scale_resolution_down_by, + encoding.scalability_mode + ) + }) + .collect::>() + .join(", ") +} + fn invalid_state(message: impl Into) -> RoomError { RoomError::Rtc(RtcError { error_type: RtcErrorType::InvalidState, message: message.into() }) } @@ -618,68 +604,44 @@ mod tests { } #[test] - fn quality_limits_only_update_requested_simulcast_layer() { - let baseline = simulcast_baseline(); + fn track_limits_apply_to_single_encoding() { + let baseline = vec![encoding("", 1_700_000, 30.0, 1.0)]; let mut encodings = baseline.clone(); - apply_quality_encoding_limits( + apply_track_encoding_limits( &mut encodings, &baseline, - VideoQuality::Medium, VideoEncodingLimits { - max_bitrate: Some(300_000), - max_framerate: Some(12.0), - scale_resolution_down_by: Some(3.0), + max_bitrate: Some(900_000), + max_framerate: None, + scale_resolution_down_by: Some(2.0), }, ) - .expect("quality limits should apply"); + .expect("track limits should apply to one encoding"); - assert_encoding_matches(&encodings[0], &baseline[0]); - assert_encoding_matches(&encodings[2], &baseline[2]); - assert_eq!(encodings[1].max_bitrate, Some(300_000)); - assert_eq!(encodings[1].max_framerate, Some(12.0)); - assert_eq!(encodings[1].scale_resolution_down_by, Some(3.0)); + assert_eq!(encodings[0].max_bitrate, Some(900_000)); + assert_eq!(encodings[0].max_framerate, Some(30.0)); + assert_eq!(encodings[0].scale_resolution_down_by, Some(2.0)); } #[test] - fn quality_limits_fail_when_quality_is_missing() { - let baseline = vec![encoding(HIGH_RID, 1_700_000, 30.0, 1.0)]; + fn track_limits_reject_unsupported_simulcast_rid_without_mutating() { + let baseline = simulcast_baseline(); let mut encodings = baseline.clone(); + encodings[1].rid = "unknown".to_string(); + let before = encodings.clone(); - let err = apply_quality_encoding_limits( + let err = apply_track_encoding_limits( &mut encodings, &baseline, - VideoQuality::Low, - VideoEncodingLimits { max_bitrate: Some(80_000), ..Default::default() }, + VideoEncodingLimits { max_bitrate: Some(850_000), ..Default::default() }, ) - .expect_err("low quality requires a simulcast encoding"); + .expect_err("unsupported simulcast RIDs should fail"); assert!(matches!( err, RoomError::Rtc(RtcError { error_type: RtcErrorType::InvalidState, .. }) )); - assert_encodings_match(&encodings, &baseline); - } - - #[test] - fn high_quality_limits_apply_to_single_encoding() { - let baseline = vec![encoding("", 1_700_000, 30.0, 1.0)]; - let mut encodings = baseline.clone(); - - apply_quality_encoding_limits( - &mut encodings, - &baseline, - VideoQuality::High, - VideoEncodingLimits { - max_bitrate: Some(900_000), - max_framerate: None, - scale_resolution_down_by: Some(2.0), - }, - ) - .expect("high quality should apply to one encoding"); - - assert_eq!(encodings[0].max_bitrate, Some(900_000)); - assert_eq!(encodings[0].max_framerate, Some(30.0)); - assert_eq!(encodings[0].scale_resolution_down_by, Some(2.0)); + assert_encodings_match(&encodings, &before); } } From b89addfcd172cd8df09aa2ed9287db45fba7a2de Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 9 Jun 2026 15:07:04 -0700 Subject: [PATCH 5/5] refactor SetEncodingLimitsRequest & SetEncodingLimitsResponse structs --- examples/local_video/src/encoding_control.rs | 26 ++++++++++++++++++++ examples/local_video/src/publisher.rs | 24 +++--------------- examples/local_video/src/subscriber.rs | 24 +++--------------- 3 files changed, 34 insertions(+), 40 deletions(-) create mode 100644 examples/local_video/src/encoding_control.rs diff --git a/examples/local_video/src/encoding_control.rs b/examples/local_video/src/encoding_control.rs new file mode 100644 index 000000000..fb849d097 --- /dev/null +++ b/examples/local_video/src/encoding_control.rs @@ -0,0 +1,26 @@ +use serde::{Deserialize, Serialize}; + +/// RPC method name used to negotiate video encoding limits between the +/// subscriber (caller) and the publisher (responder). +pub const SET_VIDEO_ENCODING_LIMITS_METHOD: &str = "set-video-encoding-limits"; + +/// Payload sent by the subscriber to request new video encoding limits on a +/// publisher's track. +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct SetEncodingLimitsRequest { + pub track_sid: String, + pub bitrate_bps: Option, + pub max_framerate: Option, + pub scale_resolution_down_by: Option, + pub reason: String, +} + +/// Payload returned by the publisher describing the encoding limits it applied. +#[derive(Debug, Deserialize, Serialize)] +pub struct SetEncodingLimitsResponse { + pub applied_bitrate_bps: Option, + pub applied_max_framerate: Option, + pub applied_scale_resolution_down_by: Option, + pub track_sid: String, +} diff --git a/examples/local_video/src/publisher.rs b/examples/local_video/src/publisher.rs index 3cd32124c..849019dec 100644 --- a/examples/local_video/src/publisher.rs +++ b/examples/local_video/src/publisher.rs @@ -20,7 +20,6 @@ use nokhwa::utils::{ }; use nokhwa::Camera; use parking_lot::Mutex; -use serde::{Deserialize, Serialize}; use std::collections::{HashMap, VecDeque}; use std::env; use std::sync::{ @@ -31,11 +30,15 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use yuv_sys; mod codec_display; +mod encoding_control; mod test_pattern; mod timestamp_burn; mod video_display; mod viewport_aspect; +use encoding_control::{ + SetEncodingLimitsRequest, SetEncodingLimitsResponse, SET_VIDEO_ENCODING_LIMITS_METHOD, +}; use test_pattern::TestPattern; use timestamp_burn::TimestampOverlay; use video_display::{align_up, PublisherTimingSample, SharedYuv}; @@ -215,27 +218,8 @@ fn unix_time_us_now() -> u64 { } const MAX_BACKEND_CAPTURE_TIMESTAMP_AGE_US: u64 = 5_000_000; -const SET_VIDEO_ENCODING_LIMITS_METHOD: &str = "set-video-encoding-limits"; const HIGH_RID: &str = "f"; -#[derive(Debug, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] -struct SetEncodingLimitsRequest { - track_sid: String, - bitrate_bps: Option, - max_framerate: Option, - scale_resolution_down_by: Option, - reason: String, -} - -#[derive(Debug, Deserialize, Serialize)] -struct SetEncodingLimitsResponse { - applied_bitrate_bps: Option, - applied_max_framerate: Option, - applied_scale_resolution_down_by: Option, - track_sid: String, -} - #[derive(Default)] struct CaptureTimestampLogState { logged_source: bool, diff --git a/examples/local_video/src/subscriber.rs b/examples/local_video/src/subscriber.rs index ca898078f..5fc317890 100644 --- a/examples/local_video/src/subscriber.rs +++ b/examples/local_video/src/subscriber.rs @@ -12,7 +12,6 @@ use livekit::webrtc::video_stream::native::NativeVideoStream; use livekit_api::access_token; use log::{debug, info}; use parking_lot::Mutex; -use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, env, @@ -25,14 +24,17 @@ use std::{ }; mod codec_display; +mod encoding_control; mod subscriber_timing; mod viewport_aspect; use codec_display::{codec_from_mime, codec_with_implementation}; +use encoding_control::{ + SetEncodingLimitsRequest, SetEncodingLimitsResponse, SET_VIDEO_ENCODING_LIMITS_METHOD, +}; use subscriber_timing::SubscriberTimingHandle; use viewport_aspect::AspectConstrainedViewport; -const SET_VIDEO_ENCODING_LIMITS_METHOD: &str = "set-video-encoding-limits"; const DEFAULT_CONTROL_BITRATE_BPS: u64 = 1_500_000; const DEFAULT_CONTROL_FRAMERATE: f64 = 30.0; const DEFAULT_CONTROL_RESOLUTION_SCALE: f64 = 1.0; @@ -419,24 +421,6 @@ struct Args { e2ee_key: Option, } -#[derive(Debug, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] -struct SetEncodingLimitsRequest { - track_sid: String, - bitrate_bps: Option, - max_framerate: Option, - scale_resolution_down_by: Option, - reason: String, -} - -#[derive(Debug, Deserialize, Serialize)] -struct SetEncodingLimitsResponse { - applied_bitrate_bps: Option, - applied_max_framerate: Option, - applied_scale_resolution_down_by: Option, - track_sid: String, -} - #[derive(Clone, Copy, Debug)] struct EncodingControlState { bitrate_bps: u64,