From d5bf74e48ec62308a099e4655ec6938997ae1dca Mon Sep 17 00:00:00 2001 From: IsaFxck Date: Mon, 12 Oct 2026 00:27:13 +0300 Subject: [PATCH] feat: stereo publish option, application playout delay, synchronous unbuffered audio capture - TrackPublishOptions::stereo sets TF_STEREO, so the server negotiates stereo Opus with the publisher and its subscribers. - livekit::setAudioPlayoutDelay(): applications that render remote audio themselves report the delay they add after the SDK. The synthetic audio device returns it as PlayoutDelay, which WebRTC's audio/video sync subtracts from the audio playout position. - AudioSource::captureFrame on an unbuffered source hands the frame to WebRTC in the calling thread (CaptureAudioFrameRequest.synchronous, NativeAudioSource::capture_frame_direct). It no longer waits for an async runtime task and an FFI callback, which stalled realtime capture threads for up to 440 ms under CPU load. Co-Authored-By: Claude Opus 5.5 --- include/livekit/livekit.h | 9 + include/livekit/room_event_types.h | 4 + .../0001-native-v2-runtime.patch | 362 +++++++++++++++++- src/audio_source.cpp | 13 + src/livekit.cpp | 7 + src/room_proto_converter.cpp | 6 + src/tests/unit/test_audio_source.cpp | 14 + src/tests/unit/test_room_event_types.cpp | 1 + src/tests/unit/test_video_frame_metadata.cpp | 14 + 9 files changed, 420 insertions(+), 10 deletions(-) diff --git a/include/livekit/livekit.h b/include/livekit/livekit.h index 267e1c1..365be3f 100644 --- a/include/livekit/livekit.h +++ b/include/livekit/livekit.h @@ -16,6 +16,8 @@ #pragma once +#include + #include "livekit/audio_frame.h" #include "livekit/audio_processing_module.h" #include "livekit/audio_source.h" @@ -63,4 +65,11 @@ LIVEKIT_API bool initialize(const LogLevel& level = LogLevel::Info); /// After shutdown, you may call initialize() again. LIVEKIT_API void shutdown(); +/// Delay of remote audio that the application renders itself after the SDK +/// hands it over (its own buffers and output device), for applications that +/// disable the SDK's platform playout. WebRTC's audio/video synchronization +/// subtracts it from the audio playout position, so pictures wait for the +/// sound actually heard. Process-wide; clamped to 0..1000 ms. +LIVEKIT_API void setAudioPlayoutDelay(std::chrono::milliseconds delay); + } // namespace livekit diff --git a/include/livekit/room_event_types.h b/include/livekit/room_event_types.h index 1e8e3b9..b78e757 100644 --- a/include/livekit/room_event_types.h +++ b/include/livekit/room_event_types.h @@ -381,6 +381,10 @@ struct TrackPublishOptions { /// Controls how the encoder trades off between resolution and framerate /// when bandwidth is constrained. If not set, the server defaults apply. std::optional degradation_preference; + + /// Encode a two-channel audio source as stereo Opus. The server then + /// negotiates stereo with the publisher and its subscribers. + std::optional stereo; }; // --------------------------------------------------------- diff --git a/patches/client-sdk-rust/0001-native-v2-runtime.patch b/patches/client-sdk-rust/0001-native-v2-runtime.patch index 8ca6006..0145d80 100644 --- a/patches/client-sdk-rust/0001-native-v2-runtime.patch +++ b/patches/client-sdk-rust/0001-native-v2-runtime.patch @@ -122,6 +122,36 @@ index 00000000..5896d522 +replacement screen track. Return an absent track safely for inactive senders. +This preserves bounded transceiver reuse while allowing the SFU to associate +new RTP media with the replacement publication. +diff --git a/.changeset/stereo-playout-delay.md b/.changeset/stereo-playout-delay.md +new file mode 100644 +index 00000000..83864652 +--- /dev/null ++++ b/.changeset/stereo-playout-delay.md +@@ -0,0 +1,8 @@ ++--- ++libwebrtc: patch ++livekit: patch ++livekit-ffi: patch ++webrtc-sys: patch ++--- ++ ++Add a `stereo` publish option (TF_STEREO); `livekit_ffi_set_audio_playout_delay`, reported by the synthetic audio device as its playout delay; and synchronous capture for unbuffered audio sources (`CaptureAudioFrameRequest.synchronous`, `NativeAudioSource::capture_frame_direct`). +diff --git a/libwebrtc/src/audio_source.rs b/libwebrtc/src/audio_source.rs +index aadf7e63..0b0afa48 100644 +--- a/libwebrtc/src/audio_source.rs ++++ b/libwebrtc/src/audio_source.rs +@@ -195,6 +195,11 @@ pub mod native { + self.handle.capture_frame(frame).await + } + ++ /// Unbuffered sources: delivers one 10 ms frame in the calling thread. ++ pub fn capture_frame_direct(&self, frame: &AudioFrame<'_>) -> Result<(), RtcError> { ++ self.handle.capture_frame_direct(frame) ++ } ++ + pub fn set_audio_options(&self, options: AudioSourceOptions) { + self.handle.set_audio_options(options) + } diff --git a/libwebrtc/src/lib.rs b/libwebrtc/src/lib.rs index 97611978..ee40bf25 100644 --- a/libwebrtc/src/lib.rs @@ -179,6 +209,135 @@ index a5c53fb9..f40b204d 100644 pub fn new( echo_canceller_enabled: bool, gain_controller_enabled: bool, +diff --git a/libwebrtc/src/native/audio_source.rs b/libwebrtc/src/native/audio_source.rs +index 6b0d8893..70167e97 100644 +--- a/libwebrtc/src/native/audio_source.rs ++++ b/libwebrtc/src/native/audio_source.rs +@@ -89,62 +89,81 @@ impl NativeAudioSource { + self.sys_handle.clear_buffer(); + } + +- pub async fn capture_frame(&self, frame: &AudioFrame<'_>) -> Result<(), RtcError> { ++ /// Delivers one 10 ms frame to WebRTC in the calling thread. Unbuffered ++ /// sources only: no runtime task or completion callback is involved, so ++ /// a busy async runtime cannot delay a realtime capture thread. ++ pub fn capture_frame_direct(&self, frame: &AudioFrame<'_>) -> Result<(), RtcError> { + if self.sample_rate != frame.sample_rate || self.num_channels != frame.num_channels { + return Err(RtcError { + error_type: RtcErrorType::InvalidState, + message: "sample_rate and num_channels don't match".to_owned(), + }); + } ++ if self.queue_size_samples != 0 { ++ return Err(RtcError { ++ error_type: RtcErrorType::InvalidState, ++ message: "direct capture requires an unbuffered source".to_owned(), ++ }); ++ } ++ // frame size must be 10ms for fast path ++ let expected_frames_per_ch = (self.sample_rate / 100) as usize; ++ if frame.data.len() % (self.num_channels as usize) != 0 { ++ return Err(RtcError { ++ error_type: RtcErrorType::InvalidState, ++ message: "frame.data length not divisible by channel count".to_owned(), ++ }); ++ } ++ let nb_frames = frame.data.len() / (self.num_channels as usize); ++ if nb_frames != expected_frames_per_ch { ++ return Err(RtcError { ++ error_type: RtcErrorType::InvalidState, ++ message: format!( ++ "direct capture requires 10ms frames: got {} frames, expected {}", ++ nb_frames, expected_frames_per_ch ++ ), ++ }); ++ } + +- // Fast path: no buffering +- if self.queue_size_samples == 0 { +- // frame size must be 10ms for fast path +- let expected_frames_per_ch = (self.sample_rate / 100) as usize; +- if frame.data.len() % (self.num_channels as usize) != 0 { +- return Err(RtcError { +- error_type: RtcErrorType::InvalidState, +- message: "frame.data length not divisible by channel count".to_owned(), +- }); +- } +- let nb_frames = frame.data.len() / (self.num_channels as usize); +- if nb_frames != expected_frames_per_ch { ++ // Define a no-op callback for fast path (queue_size_ms=0) ++ // This is safer than passing null, which can cause UB in release mode optimizations ++ extern "C" fn noop_complete_callback(_ctx: *const sys_at::SourceContext) { ++ // No-op: fast path completes synchronously, no callback needed ++ } ++ ++ unsafe { ++ let data: &[i16] = frame.data.as_ref(); ++ // Use a valid no-op callback instead of null for safety ++ // In release mode, transmuting null pointers can cause UB ++ let noop_callback = sys_at::CompleteCallback(noop_complete_callback); ++ let ok = self.sys_handle.capture_frame( ++ data, ++ self.sample_rate, ++ self.num_channels, ++ nb_frames, ++ std::ptr::null(), // Context is still null - callback won't use it ++ noop_callback, ++ ); ++ if !ok { + return Err(RtcError { + error_type: RtcErrorType::InvalidState, +- message: format!( +- "direct capture requires 10ms frames: got {} frames, expected {}", +- nb_frames, expected_frames_per_ch +- ), ++ message: "failed to capture frame without buffering".to_owned(), + }); + } ++ } ++ Ok(()) ++ } + +- // Define a no-op callback for fast path (queue_size_ms=0) +- // This is safer than passing null, which can cause UB in release mode optimizations +- extern "C" fn noop_complete_callback(_ctx: *const sys_at::SourceContext) { +- // No-op: fast path completes synchronously, no callback needed +- } ++ pub async fn capture_frame(&self, frame: &AudioFrame<'_>) -> Result<(), RtcError> { ++ if self.sample_rate != frame.sample_rate || self.num_channels != frame.num_channels { ++ return Err(RtcError { ++ error_type: RtcErrorType::InvalidState, ++ message: "sample_rate and num_channels don't match".to_owned(), ++ }); ++ } + +- unsafe { +- let data: &[i16] = frame.data.as_ref(); +- // Use a valid no-op callback instead of null for safety +- // In release mode, transmuting null pointers can cause UB +- let noop_callback = sys_at::CompleteCallback(noop_complete_callback); +- let ok = self.sys_handle.capture_frame( +- data, +- self.sample_rate, +- self.num_channels, +- nb_frames, +- std::ptr::null(), // Context is still null - callback won't use it +- noop_callback, +- ); +- if !ok { +- return Err(RtcError { +- error_type: RtcErrorType::InvalidState, +- message: "failed to capture frame without buffering".to_owned(), +- }); +- } +- } +- return Ok(()); ++ // Fast path: no buffering ++ if self.queue_size_samples == 0 { ++ return self.capture_frame_direct(frame); + } + + // Buffered path. diff --git a/libwebrtc/src/native/fresh_audio_stream.rs b/libwebrtc/src/native/fresh_audio_stream.rs new file mode 100644 index 00000000..5d90f21d @@ -672,18 +831,39 @@ index 7a4234e3..880e2825 100644 pub trait VideoFrameBufferExt: VideoBuffer { diff --git a/livekit-ffi/include/livekit_ffi.h b/livekit-ffi/include/livekit_ffi.h -index 15f64dfa..025438bc 100644 +index 15f64dfa..bf31c631 100644 --- a/livekit-ffi/include/livekit_ffi.h +++ b/livekit-ffi/include/livekit_ffi.h -@@ -37,6 +37,8 @@ FfiHandleId livekit_ffi_request(const uint8_t *data, size_t len, +@@ -37,6 +37,15 @@ FfiHandleId livekit_ffi_request(const uint8_t *data, size_t len, bool livekit_ffi_drop_handle(FfiHandleId handle_id); +void livekit_ffi_set_external_video_decoder(const void *decoder); ++ ++/** ++ * Delay of remote audio rendered by the application after the SDK hands it ++ * over (its buffers and output device). WebRTC's audio/video synchronization ++ * subtracts it from the audio playout position. Clamped to 1000 ms. ++ */ ++void livekit_ffi_set_audio_playout_delay(uint32_t delay_ms); + } // extern "C" #endif // livekit_ffi +diff --git a/livekit-ffi/protocol/audio_frame.proto b/livekit-ffi/protocol/audio_frame.proto +index 35143902..69cd9424 100644 +--- a/livekit-ffi/protocol/audio_frame.proto ++++ b/livekit-ffi/protocol/audio_frame.proto +@@ -91,6 +91,9 @@ message CaptureAudioFrameRequest { + required uint64 source_handle = 1; + required AudioFrameBufferInfo buffer = 2; + optional uint64 request_async_id = 3; ++ // Unbuffered (queue_size_ms = 0) sources only: deliver the frame in the ++ // calling thread. The response completes the capture; no callback follows. ++ optional bool synchronous = 4; + } + message CaptureAudioFrameResponse { + required uint64 async_id = 1; diff --git a/livekit-ffi/protocol/ffi.proto b/livekit-ffi/protocol/ffi.proto index 53ea782b..1d03c91c 100644 --- a/livekit-ffi/protocol/ffi.proto @@ -727,10 +907,19 @@ index 53ea782b..1d03c91c 100644 } diff --git a/livekit-ffi/protocol/room.proto b/livekit-ffi/protocol/room.proto -index df08ed5b..ab1068a5 100644 +index df08ed5b..b531af27 100644 --- a/livekit-ffi/protocol/room.proto +++ b/livekit-ffi/protocol/room.proto -@@ -335,6 +335,7 @@ enum VideoEncoderBackend { +@@ -326,6 +326,8 @@ message TrackPublishOptions { + // Controls how the encoder trades off between resolution and framerate + // when bandwidth is constrained. Default is MAINTAIN_RESOLUTION. + optional DegradationPreference degradation_preference = 13; ++ // Encode a two-channel audio source as stereo Opus. ++ optional bool stereo = 14; + } + + enum VideoEncoderBackend { +@@ -335,6 +337,7 @@ enum VideoEncoderBackend { ENCODER_BACKEND_NVENC = 3; ENCODER_BACKEND_VAAPI = 4; ENCODER_BACKEND_VIDEOTOOLBOX = 5; @@ -806,10 +995,10 @@ index a4b610e0..0e177673 100644 message VideoSourceInfo { diff --git a/livekit-ffi/src/cabi.rs b/livekit-ffi/src/cabi.rs -index d9e83f13..e8ab3b62 100644 +index d9e83f13..0ea4fa91 100644 --- a/livekit-ffi/src/cabi.rs +++ b/livekit-ffi/src/cabi.rs -@@ -108,6 +108,19 @@ pub extern "C" fn livekit_ffi_drop_handle(handle_id: FfiHandleId) -> bool { +@@ -108,6 +108,27 @@ pub extern "C" fn livekit_ffi_drop_handle(handle_id: FfiHandleId) -> bool { FFI_SERVER.drop_handle(handle_id) } @@ -825,12 +1014,20 @@ index d9e83f13..e8ab3b62 100644 +pub unsafe extern "C" fn livekit_ffi_set_external_video_decoder(decoder: *const std::ffi::c_void) { + webrtc_sys::external_video_decoder::lk_set_external_video_decoder(decoder); +} ++ ++/// Delay of remote audio rendered by the application after the SDK hands it ++/// over (its buffers and output device). WebRTC's audio/video synchronization ++/// subtracts it from the audio playout position. Clamped to 1000 ms. ++#[no_mangle] ++pub extern "C" fn livekit_ffi_set_audio_playout_delay(delay_ms: u32) { ++ unsafe { webrtc_sys::playout_delay::lk_set_playout_delay_ms(delay_ms) } ++} + #[no_mangle] pub extern "C" fn livekit_ffi_dispose() { FFI_SERVER.async_runtime.block_on(FFI_SERVER.dispose()); diff --git a/livekit-ffi/src/conversion/room.rs b/livekit-ffi/src/conversion/room.rs -index 4922add5..281dcfc7 100644 +index 4922add5..36763e4d 100644 --- a/livekit-ffi/src/conversion/room.rs +++ b/livekit-ffi/src/conversion/room.rs @@ -63,6 +63,9 @@ fn video_encoder_from_proto(backend: Option) -> Option @@ -843,7 +1040,32 @@ index 4922add5..281dcfc7 100644 } } -@@ -447,6 +450,7 @@ mod tests { +@@ -366,6 +369,7 @@ impl From for TrackPublishOptions { + .unwrap_or(default_publish_options.video_encoder), + scalability_mode: opts.scalability_mode, + degradation_preference: degradation_preference_from_proto(opts.degradation_preference), ++ stereo: opts.stereo.unwrap_or(default_publish_options.stereo), + } + } + } +@@ -428,6 +432,16 @@ mod tests { + assert!(options.sdk_options.other_sdks.is_none()); + } + ++ #[test] ++ fn stereo_publish_option_round_trips() { ++ assert!(!TrackPublishOptions::from(proto::TrackPublishOptions::default()).stereo); ++ let options = TrackPublishOptions::from(proto::TrackPublishOptions { ++ stereo: Some(true), ++ ..Default::default() ++ }); ++ assert!(options.stereo); ++ } ++ + #[test] + fn video_encoder_defaults_to_auto() { + let options = TrackPublishOptions::from(proto::TrackPublishOptions::default()); +@@ -447,6 +461,7 @@ mod tests { proto::VideoEncoderBackend::EncoderBackendVideotoolbox, VideoEncoderBackend::VideoToolbox, ), @@ -1106,6 +1328,39 @@ index 00000000..a13fa9f4 + } + 0 +} +diff --git a/livekit-ffi/src/server/audio_source.rs b/livekit-ffi/src/server/audio_source.rs +index fd4f06cf..5c472e20 100644 +--- a/livekit-ffi/src/server/audio_source.rs ++++ b/livekit-ffi/src/server/audio_source.rs +@@ -108,6 +108,28 @@ impl FfiAudioSource { + let source = self.source.clone(); + let async_id = server.resolve_async_id(capture.request_async_id); + ++ if capture.synchronous.unwrap_or(false) { ++ #[cfg(not(target_arch = "wasm32"))] ++ if let RtcAudioSource::Native(ref source) = source { ++ let data = unsafe { ++ let len = buffer.num_channels * buffer.samples_per_channel; ++ slice::from_raw_parts(buffer.data_ptr as *const i16, len as usize) ++ }; ++ source ++ .capture_frame_direct(&AudioFrame { ++ data: Cow::Borrowed(data), ++ sample_rate: buffer.sample_rate, ++ num_channels: buffer.num_channels, ++ samples_per_channel: buffer.samples_per_channel, ++ }) ++ .map_err(|e| FfiError::InvalidRequest(e.to_string().into()))?; ++ return Ok(proto::CaptureAudioFrameResponse { async_id }); ++ } ++ return Err(FfiError::InvalidRequest( ++ "synchronous capture needs a native source".into(), ++ )); ++ } ++ + let data = unsafe { + let len = buffer.num_channels * buffer.samples_per_channel; + slice::from_raw_parts(buffer.data_ptr as *const i16, len as usize) diff --git a/livekit-ffi/src/server/colorcvt/cvtimpl.rs b/livekit-ffi/src/server/colorcvt/cvtimpl.rs index 95a52e26..cb63a157 100644 --- a/livekit-ffi/src/server/colorcvt/cvtimpl.rs @@ -1436,6 +1691,43 @@ index b02d88e5..c00819f8 100644 let participant = if participant_identity.as_str() == inner.local_participant.identity().as_str() { +diff --git a/livekit/src/room/options.rs b/livekit/src/room/options.rs +index f443f608..98a3228c 100644 +--- a/livekit/src/room/options.rs ++++ b/livekit/src/room/options.rs +@@ -151,6 +151,9 @@ pub struct TrackPublishOptions { + /// - Screen share: `MaintainResolution` (clarity is critical for text/UI) + /// - Other/unknown: `Balanced` + pub degradation_preference: Option, ++ /// Encode a two-channel audio source as stereo Opus. The server then ++ /// negotiates stereo with the publisher and its subscribers. ++ pub stereo: bool, + } + + impl Default for TrackPublishOptions { +@@ -170,6 +173,7 @@ impl Default for TrackPublishOptions { + video_encoder: VideoEncoderBackend::Auto, + scalability_mode: None, + degradation_preference: None, ++ stereo: false, + } + } + } +diff --git a/livekit/src/room/participant/local_participant.rs b/livekit/src/room/participant/local_participant.rs +index f01c754c..d83a61c8 100644 +--- a/livekit/src/room/participant/local_participant.rs ++++ b/livekit/src/room/participant/local_participant.rs +@@ -398,6 +398,10 @@ impl LocalParticipant { + if options.preconnect_buffer { + req.audio_features.push(proto::AudioTrackFeature::TfPreconnectBuffer as i32); + } ++ if options.stereo && matches!(track, LocalTrack::Audio(_)) { ++ req.stereo = true; ++ req.audio_features.push(proto::AudioTrackFeature::TfStereo as i32); ++ } + + req.packet_trailer_features = + options.frame_metadata_features.to_proto().into_iter().map(|f| f as i32).collect(); diff --git a/livekit/src/rtc_engine/peer_transport.rs b/livekit/src/rtc_engine/peer_transport.rs index 7eccd2ef..0e41446d 100644 --- a/livekit/src/rtc_engine/peer_transport.rs @@ -2561,6 +2853,23 @@ index abf4f5ab..d49ab199 100644 }; std::shared_ptr create_peer_connection_factory(); +diff --git a/webrtc-sys/include/livekit/synthetic_audio_device.h b/webrtc-sys/include/livekit/synthetic_audio_device.h +index 0d0bb8db..3b31e137 100644 +--- a/webrtc-sys/include/livekit/synthetic_audio_device.h ++++ b/webrtc-sys/include/livekit/synthetic_audio_device.h +@@ -24,6 +24,12 @@ + #include "rtc_base/synchronization/mutex.h" + #include "rtc_base/task_utils/repeating_task.h" + ++// Delay of audio that the application renders after WebRTC hands it over ++// (its own buffers and output device). The synthetic device reports it as ++// PlayoutDelay, so audio/video synchronization and the playout estimates ++// account for it. Clamped to 1000 ms; process-wide. ++extern "C" void lk_set_playout_delay_ms(uint32_t delay_ms); ++ + namespace livekit_ffi { + + class SyntheticAudioDevice : public webrtc::AudioDeviceModule { diff --git a/webrtc-sys/include/livekit/video_frame_buffer.h b/webrtc-sys/include/livekit/video_frame_buffer.h index cb419fd3..4f09d876 100644 --- a/webrtc-sys/include/livekit/video_frame_buffer.h @@ -2894,10 +3203,10 @@ index 00000000..79dcb9c0 + +} // namespace livekit_ffi diff --git a/webrtc-sys/src/lib.rs b/webrtc-sys/src/lib.rs -index d33322f1..3ef31583 100644 +index d33322f1..29af5e31 100644 --- a/webrtc-sys/src/lib.rs +++ b/webrtc-sys/src/lib.rs -@@ -43,6 +43,16 @@ pub mod video_track; +@@ -43,6 +43,23 @@ pub mod video_track; pub mod webrtc; pub mod yuv_helper; @@ -2910,6 +3219,13 @@ index d33322f1..3ef31583 100644 + pub fn lk_set_external_video_decoder(decoder: *const std::ffi::c_void); + } +} ++ ++/// Application playout delay, see `include/livekit/synthetic_audio_device.h`. ++pub mod playout_delay { ++ extern "C" { ++ pub fn lk_set_playout_delay_ms(delay_ms: u32); ++ } ++} + pub const MEDIA_TYPE_VIDEO: &str = "video"; pub const MEDIA_TYPE_AUDIO: &str = "audio"; @@ -3364,6 +3680,32 @@ index e97bed2d..24ae4db9 100644 } uint32_t RtpSender::ssrc() const { +diff --git a/webrtc-sys/src/synthetic_audio_device.cpp b/webrtc-sys/src/synthetic_audio_device.cpp +index 8074b9d9..ec76aef1 100644 +--- a/webrtc-sys/src/synthetic_audio_device.cpp ++++ b/webrtc-sys/src/synthetic_audio_device.cpp +@@ -288,7 +288,12 @@ int32_t SyntheticAudioDevice::StereoRecording(bool* enabled) const { + return 0; + } + ++namespace { ++std::atomic g_playout_delay_ms{0}; ++} // namespace ++ + int32_t SyntheticAudioDevice::PlayoutDelay(uint16_t* delayMS) const { ++ *delayMS = g_playout_delay_ms.load(std::memory_order_relaxed); + return 0; + } + +@@ -333,3 +338,8 @@ int32_t SyntheticAudioDevice::SetObserver(webrtc::AudioDeviceObserver* observer) + } + + } // namespace livekit_ffi ++ ++extern "C" void lk_set_playout_delay_ms(uint32_t delay_ms) { ++ livekit_ffi::g_playout_delay_ms.store(static_cast(delay_ms > 1000 ? 1000 : delay_ms), ++ std::memory_order_relaxed); ++} diff --git a/webrtc-sys/src/video_decoder_factory.cpp b/webrtc-sys/src/video_decoder_factory.cpp index d22ae70d..676b8e5a 100644 --- a/webrtc-sys/src/video_decoder_factory.cpp diff --git a/src/audio_source.cpp b/src/audio_source.cpp index f5e647a..100847b 100644 --- a/src/audio_source.cpp +++ b/src/audio_source.cpp @@ -114,6 +114,19 @@ void AudioSource::captureFrame(const AudioFrame& frame, int timeout_ms) { // Build AudioFrameBufferInfo from the wrapper const proto::AudioFrameBufferInfo buf = frame.toProto(); + if (queue_size_ms_ == 0) { + // Unbuffered: WebRTC consumes the frame in this thread. No runtime task + // or callback thread can delay a realtime capture thread under CPU load. + proto::FfiRequest req; + auto* msg = req.mutable_capture_audio_frame(); + msg->set_source_handle(handle_.get()); + msg->set_synchronous(true); + msg->mutable_buffer()->CopyFrom(buf); + if (!FfiClient::instance().sendRequest(req).has_capture_audio_frame()) { + throw std::runtime_error("AudioSource::captureFrame: FFI response missing capture result"); + } + return; + } // Use async FFI API and block until the callback completes auto fut = FfiClient::instance().captureAudioFrameAsync(handle_.get(), buf); if (timeout_ms == 0) { diff --git a/src/livekit.cpp b/src/livekit.cpp index fb9870f..6ed6c47 100644 --- a/src/livekit.cpp +++ b/src/livekit.cpp @@ -16,6 +16,8 @@ #include "livekit/livekit.h" +#include + #include "ffi_client.h" #include "livekit_ffi.h" #include "lk_log.h" @@ -34,6 +36,11 @@ void setExternalVideoDecoder(const LkExternalVideoDecoder* decoder) { livekit_ffi_set_external_video_decoder(decoder); } +void setAudioPlayoutDelay(std::chrono::milliseconds delay) { + livekit_ffi_set_audio_playout_delay(static_cast(std::clamp( + delay.count(), 0, 1000))); +} + bool isInitialized() { return FfiClient::instance().isInitialized(); } void shutdown() { diff --git a/src/room_proto_converter.cpp b/src/room_proto_converter.cpp index 2faa7c9..c094af0 100644 --- a/src/room_proto_converter.cpp +++ b/src/room_proto_converter.cpp @@ -539,6 +539,9 @@ proto::TrackPublishOptions toProto(const TrackPublishOptions& in) { if (in.degradation_preference) { msg.set_degradation_preference(static_cast(*in.degradation_preference)); } + if (in.stereo) { + msg.set_stereo(*in.stereo); + } return msg; } @@ -582,6 +585,9 @@ TrackPublishOptions fromProto(const proto::TrackPublishOptions& in) { if (in.has_degradation_preference()) { out.degradation_preference = static_cast(in.degradation_preference()); } + if (in.has_stereo()) { + out.stereo = in.stereo(); + } return out; } diff --git a/src/tests/unit/test_audio_source.cpp b/src/tests/unit/test_audio_source.cpp index d7aae04..451ef16 100644 --- a/src/tests/unit/test_audio_source.cpp +++ b/src/tests/unit/test_audio_source.cpp @@ -68,6 +68,20 @@ TEST_F(AudioSourceTest, BufferedIngressTimeoutIsNotReportedAsSuccess) { source.clearQueue(); } +TEST_F(AudioSourceTest, UnbufferedCaptureCompletesInTheCallingThread) { + auto source = std::make_shared(48000, 2, 0); + auto track = LocalAudioTrack::createLocalAudioTrack("direct-input", source); + FreshAudioStream stream(track); + auto input = AudioFrame::create(48000, 2, 480); + // No timeout applies: the frame is in the sink when captureFrame returns. + source->captureFrame(input, 0); + FreshAudioFrame output; + EXPECT_EQ(stream.tryRead(output, 1'000'000), FreshAudioReadResult::frame); + // Errors surface as the call's own exception, not a later callback. + auto partial = AudioFrame::create(48000, 2, 240); + EXPECT_THROW(source->captureFrame(partial, 0), std::runtime_error); +} + TEST_F(AudioSourceTest, FreshSinkBoundsQueueAndPreservesStereo) { auto source = std::make_shared(48000, 2, 0); auto track = LocalAudioTrack::createLocalAudioTrack("fresh-input", source); diff --git a/src/tests/unit/test_room_event_types.cpp b/src/tests/unit/test_room_event_types.cpp index ce60b74..055546e 100644 --- a/src/tests/unit/test_room_event_types.cpp +++ b/src/tests/unit/test_room_event_types.cpp @@ -51,6 +51,7 @@ TEST(RoomEventTypesTest, TrackPublishOptionsDefaults) { EXPECT_FALSE(options.packet_trailer_features.frame_id); EXPECT_FALSE(options.packet_trailer_features.user_data); EXPECT_FALSE(options.degradation_preference.has_value()); + EXPECT_FALSE(options.stereo.has_value()); } TEST(RoomEventTypesTest, UserPacketDataDefaults) { diff --git a/src/tests/unit/test_video_frame_metadata.cpp b/src/tests/unit/test_video_frame_metadata.cpp index 7130076..df7ad06 100644 --- a/src/tests/unit/test_video_frame_metadata.cpp +++ b/src/tests/unit/test_video_frame_metadata.cpp @@ -152,6 +152,20 @@ TEST(TrackPublishOptionsTest, DegradationPreferenceRoundTrip) { EXPECT_EQ(*round_trip.degradation_preference, DegradationPreference::MaintainFramerateAndResolution); } +TEST(TrackPublishOptionsTest, StereoRoundTrip) { + EXPECT_FALSE(toProto(TrackPublishOptions{}).has_stereo()); + + TrackPublishOptions options; + options.stereo = true; + const proto::TrackPublishOptions proto_options = toProto(options); + ASSERT_TRUE(proto_options.has_stereo()); + EXPECT_TRUE(proto_options.stereo()); + + const TrackPublishOptions round_trip = fromProto(proto_options); + ASSERT_TRUE(round_trip.stereo.has_value()); + EXPECT_TRUE(*round_trip.stereo); +} + TEST(TrackPublishOptionsTest, PreEncodedBackendRoundTrip) { TrackPublishOptions options; options.video_encoder = VideoEncoderBackend::PreEncoded;