diff --git a/docs/realtime-microphone-dsp.md b/docs/realtime-microphone-dsp.md index bef0dc3..613bd39 100644 --- a/docs/realtime-microphone-dsp.md +++ b/docs/realtime-microphone-dsp.md @@ -5,15 +5,17 @@ the application fork's microphone stage (#127 in syrnike13/syrnike13-app). The existing `AudioProcessingModule` API remains unchanged. Create, use and destroy one instance on its sole owner thread. Construction -allocates two independent WebRTC processors and prewarms them with 200 fixed +allocates three independent WebRTC processors and prewarms them with 200 fixed frames. `process` accepts exactly 480 PCM16 mono samples at 48 kHz, an optional rendered reference of the same shape, and a bounded 0–500 ms delay. Processing -applies AEC followed by NS; either processor can be bypassed independently. +applies AEC followed by NS; each processor can be bypassed independently. +AGC2 is only available through `processGain`. To preserve gate thresholds, +measure the post-AEC/NS level, then call `processGain` on that frame. Absent reference bypasses AEC without disabling NS. The application owns -reference freshness, renderer epochs, gate/mute, gain and device/publication +reference freshness, renderer epochs, gate/mute and device/publication lifetime; this class has no Room, device, callback or global FFI handle. -The private native ABI has only create/process/destroy operations. Fixed buffers +The private native ABI has only create/process/process-gain/destroy operations. Fixed buffers and integer errors avoid protobuf requests and error-string allocations in ordinary processing. The owner contract forbids concurrent calls. A new echo timeline requires a newly prepared instance; there is intentionally no reset diff --git a/include/livekit/realtime_audio_processing.h b/include/livekit/realtime_audio_processing.h index 2d68416..1c92a81 100644 --- a/include/livekit/realtime_audio_processing.h +++ b/include/livekit/realtime_audio_processing.h @@ -28,15 +28,15 @@ namespace livekit { /// @brief Result of an exclusively owned fixed-frame DSP operation. enum class RealtimeAudioProcessingResult { ok, invalid_input, failed, wrong_thread }; -/// @brief Warm, fixed 48 kHz mono microphone AEC/NS, independent of AudioSource. +/// @brief Warm, fixed 48 kHz mono microphone AEC/NS/AGC2, independent of AudioSource. /// /// Construct, process and destroy on one owner thread. Construction -/// allocates and prewarms both processors; ordinary processing uses fixed -/// buffers and bypasses the protobuf FFI and global handle table. NS and AEC +/// allocates and prewarms all three processors; ordinary processing uses fixed +/// buffers and bypasses the protobuf FFI and global handle table. NS, AEC and AGC2 /// can be enabled independently per frame. No Room or device is owned here. class LIVEKIT_API RealtimeAudioProcessing final { public: - /// @brief Allocate and prewarm the two independent WebRTC processors. + /// @brief Allocate and prewarm the three independent WebRTC processors. /// @throws std::runtime_error If processor initialization fails. RealtimeAudioProcessing(); /// @brief Release the processors on their owner after processing has stopped. @@ -56,6 +56,11 @@ class LIVEKIT_API RealtimeAudioProcessing final { std::uint32_t stream_delay_ms, bool noise_suppression, bool echo_cancellation) noexcept; + /// @brief Apply only AGC2, allowing a gate to measure the frame after AEC/NS. + /// @param microphone Exactly 480 writable mono PCM16 samples. + /// @return Status under the same owner-thread contract as process(). + [[nodiscard]] RealtimeAudioProcessingResult processGain(std::array& microphone) noexcept; + private: struct Deleter { void operator()(void* value) const noexcept; diff --git a/patches/client-sdk-rust/0001-native-v2-runtime.patch b/patches/client-sdk-rust/0001-native-v2-runtime.patch index ac33c14..ccc5c35 100644 --- a/patches/client-sdk-rust/0001-native-v2-runtime.patch +++ b/patches/client-sdk-rust/0001-native-v2-runtime.patch @@ -121,6 +121,20 @@ index 00000000..9a1363bb +publisher transceiver. Apply the current AddTrack identity only to the offer +sent to the SFU, so a later unrelated negotiation does not remove the replacement +publication. Transceiver reuse and its resource bound remain enabled. +diff --git a/.changeset/realtime-agc2.md b/.changeset/realtime-agc2.md +new file mode 100644 +index 00000000..74933b96 +--- /dev/null ++++ b/.changeset/realtime-agc2.md +@@ -0,0 +1,8 @@ ++--- ++livekit-ffi: patch ++--- ++ ++Add independently prewarmed AGC2 adaptive digital gain to the fixed-frame ++microphone processing seam through a separate gain-only operation, allowing ++gate measurement after AEC and noise suppression without protobuf requests or ++processing-path buffer allocations. The AEC/NS operation has no gain flag. diff --git a/.changeset/reused-screen-sender.md b/.changeset/reused-screen-sender.md new file mode 100644 index 00000000..5896d522 @@ -984,10 +998,10 @@ index 7a4234e3..880e2825 100644 pub trait VideoFrameBufferExt: VideoBuffer { diff --git a/livekit-ffi/include/livekit_ffi.h b/livekit-ffi/include/livekit_ffi.h -index 15f64dfa..6da08d39 100644 +index 15f64dfa..3fb1022b 100644 --- a/livekit-ffi/include/livekit_ffi.h +++ b/livekit-ffi/include/livekit_ffi.h -@@ -37,6 +37,23 @@ FfiHandleId livekit_ffi_request(const uint8_t *data, size_t len, +@@ -37,6 +37,33 @@ FfiHandleId livekit_ffi_request(const uint8_t *data, size_t len, bool livekit_ffi_drop_handle(FfiHandleId handle_id); @@ -1007,6 +1021,16 @@ index 15f64dfa..6da08d39 100644 +/// Synchronously pulls one 10 ms frame for all active Rooms; false if not playing +/// or external playout is disabled. Call from the single playout owner thread. +bool livekit_ffi_pull_audio_playout(); ++ ++void *livekit_ffi_realtime_apm_create(); ++ ++void livekit_ffi_realtime_apm_destroy(void *handle); ++ ++int32_t livekit_ffi_realtime_apm_process(void *handle, int16_t *microphone, size_t samples, ++ const int16_t *reference, uint32_t delay_ms, ++ bool noise_suppression, bool echo_cancellation); ++ ++int32_t livekit_ffi_realtime_apm_process_gain(void *handle, int16_t *microphone, size_t samples); + } // extern "C" @@ -1385,10 +1409,10 @@ index 06695660..87cd8850 100644 pub mod proto; diff --git a/livekit-ffi/src/realtime_apm.rs b/livekit-ffi/src/realtime_apm.rs new file mode 100644 -index 00000000..a13fa9f4 +index 00000000..7e5cce40 --- /dev/null +++ b/livekit-ffi/src/realtime_apm.rs -@@ -0,0 +1,116 @@ +@@ -0,0 +1,223 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); @@ -1412,15 +1436,17 @@ index 00000000..a13fa9f4 +struct RealtimeApm { + echo: AudioProcessingModule, + noise: AudioProcessingModule, ++ gain: AudioProcessingModule, +} + -+/// Allocate and prewarm the two independent processors on their future owner. ++/// Allocate and prewarm the three independent processors on their future owner. +#[no_mangle] +pub extern "C" fn livekit_ffi_realtime_apm_create() -> *mut c_void { + panic::catch_unwind(|| { + let mut apm = Box::new(RealtimeApm { + echo: AudioProcessingModule::new(true, false, false, false), + noise: AudioProcessingModule::new(false, false, false, true), ++ gain: AudioProcessingModule::new(false, true, false, false), + }); + let mut reference = [0i16; 480]; + let mut microphone = [0i16; 480]; @@ -1435,6 +1461,7 @@ index 00000000..a13fa9f4 + || apm.echo.process_10ms(&mut reference, true) != 0 + || apm.echo.process_10ms(&mut microphone, false) != 0 + || apm.noise.process_10ms(&mut microphone, false) != 0 ++ || apm.gain.process_10ms(&mut microphone, false) != 0 + { + return ptr::null_mut(); + } @@ -1505,6 +1532,110 @@ index 00000000..a13fa9f4 + } + 0 +} ++ ++/// Apply AGC2 to one fixed frame. Returns 0=ok, 1=invalid input, 2=DSP failure. ++/// ++/// # Safety ++/// `handle` is a live, exclusively accessed create result. `microphone` points ++/// to exactly 480 writable i16 samples, valid for this synchronous call. ++#[no_mangle] ++pub unsafe extern "C" fn livekit_ffi_realtime_apm_process_gain( ++ handle: *mut c_void, ++ microphone: *mut i16, ++ samples: usize, ++) -> i32 { ++ if handle.is_null() ++ || microphone.is_null() ++ || samples != 480 ++ || microphone as usize % align_of::() != 0 ++ { ++ return 1; ++ } ++ // SAFETY: The caller exclusively owns the live handle and writable fixed frame. ++ let apm = unsafe { &mut *handle.cast::() }; ++ // SAFETY: Validated dimensions/alignment and exclusive caller-owned storage. ++ let microphone = unsafe { &mut *microphone.cast::<[i16; 480]>() }; ++ if apm.gain.process_10ms(microphone, false) != 0 { ++ return 2; ++ } ++ 0 ++} ++ ++#[cfg(test)] ++mod tests { ++ use super::*; ++ ++ #[test] ++ fn validates_input_before_processing() { ++ let handle = livekit_ffi_realtime_apm_create(); ++ assert!(!handle.is_null()); ++ let mut samples = [1234i16; 480]; ++ let original = samples; ++ // SAFETY: The handle and frame remain valid and exclusively owned here. ++ unsafe { ++ assert_eq!( ++ livekit_ffi_realtime_apm_process( ++ handle, ++ samples.as_mut_ptr(), ++ 479, ++ ptr::null(), ++ 0, ++ false, ++ false, ++ ), ++ 1 ++ ); ++ assert_eq!(samples, original); ++ assert_eq!( ++ livekit_ffi_realtime_apm_process( ++ handle, ++ samples.as_mut_ptr(), ++ 480, ++ ptr::null(), ++ 501, ++ false, ++ false, ++ ), ++ 1 ++ ); ++ assert_eq!(samples, original); ++ assert_eq!( ++ livekit_ffi_realtime_apm_process( ++ handle, ++ samples.as_mut_ptr(), ++ 480, ++ ptr::null(), ++ 0, ++ false, ++ true, ++ ), ++ 0 ++ ); ++ assert_eq!(samples, original); ++ livekit_ffi_realtime_apm_destroy(handle); ++ } ++ } ++ ++ #[test] ++ fn validates_input_before_gain() { ++ let handle = livekit_ffi_realtime_apm_create(); ++ assert!(!handle.is_null()); ++ let mut samples = [1234i16; 480]; ++ let original = samples; ++ // SAFETY: The handle and frame remain valid and exclusively owned here. ++ unsafe { ++ assert_eq!(livekit_ffi_realtime_apm_process_gain(handle, samples.as_mut_ptr(), 479), 1); ++ assert_eq!( ++ livekit_ffi_realtime_apm_process_gain(ptr::null_mut(), samples.as_mut_ptr(), 480), ++ 1 ++ ); ++ assert_eq!(livekit_ffi_realtime_apm_process_gain(handle, ptr::null_mut(), 480), 1); ++ assert_eq!(samples, original); ++ assert_eq!(livekit_ffi_realtime_apm_process_gain(handle, samples.as_mut_ptr(), 480), 0); ++ livekit_ffi_realtime_apm_destroy(handle); ++ } ++ } ++} diff --git a/livekit-ffi/src/server/audio_source.rs b/livekit-ffi/src/server/audio_source.rs index fd4f06cf..5c472e20 100644 --- a/livekit-ffi/src/server/audio_source.rs diff --git a/src/realtime_audio_processing.cpp b/src/realtime_audio_processing.cpp index 0750fa0..7b00809 100644 --- a/src/realtime_audio_processing.cpp +++ b/src/realtime_audio_processing.cpp @@ -25,6 +25,7 @@ void livekit_ffi_realtime_apm_destroy(void* handle); int livekit_ffi_realtime_apm_process(void* handle, std::int16_t* microphone, std::size_t samples, const std::int16_t* reference, std::uint32_t delay_ms, bool noise_suppression, bool echo_cancellation); +int livekit_ffi_realtime_apm_process_gain(void* handle, std::int16_t* microphone, std::size_t samples); } namespace livekit { @@ -54,4 +55,8 @@ RealtimeAudioProcessingResult RealtimeAudioProcessing::process(std::arraydata() : nullptr, stream_delay_ms, noise_suppression, echo_cancellation)); } +RealtimeAudioProcessingResult RealtimeAudioProcessing::processGain(std::array& microphone) noexcept { + if (owner_ != std::this_thread::get_id()) return RealtimeAudioProcessingResult::wrong_thread; + return resultOf(livekit_ffi_realtime_apm_process_gain(state_.get(), microphone.data(), microphone.size())); +} } // namespace livekit diff --git a/src/tests/unit/test_audio_processing_module.cpp b/src/tests/unit/test_audio_processing_module.cpp index 325cb27..e09ff35 100644 --- a/src/tests/unit/test_audio_processing_module.cpp +++ b/src/tests/unit/test_audio_processing_module.cpp @@ -897,7 +897,7 @@ TEST_F(AudioProcessingModuleTest, RealtimeBypassAndOwnerContract) { EXPECT_EQ(processor.process(samples, nullptr, 501, false, false), RealtimeAudioProcessingResult::invalid_input); EXPECT_EQ(samples, original); auto result = RealtimeAudioProcessingResult::ok; - std::thread other([&] { result = processor.process(samples, nullptr, 0, false, false); }); + std::thread other([&] { result = processor.processGain(samples); }); other.join(); EXPECT_EQ(result, RealtimeAudioProcessingResult::wrong_thread); EXPECT_EQ(samples, original); @@ -951,4 +951,48 @@ TEST_F(AudioProcessingModuleTest, RealtimeSyntheticEcho) { EXPECT_GE(erle_db, 10.0) << "AEC must suppress the delayed reference independently of NS"; } +TEST_F(AudioProcessingModuleTest, RealtimeGainRaisesQuietSpeech) { + RealtimeAudioProcessing processor; + double input_energy = 0; + double output_energy = 0; + int peak = 0; + for (int frame = 0; frame < 600; ++frame) { + std::array samples{}; + for (std::size_t index = 0; index < samples.size(); ++index) { + const double time = (frame * samples.size() + index) / 48000.0; + const double envelope = 0.6 + 0.4 * std::sin(2.0 * M_PI * 3.0 * time); + const double voiced = 0.5 * std::sin(2.0 * M_PI * 150.0 * time) + 0.3 * std::sin(2.0 * M_PI * 450.0 * time) + + 0.2 * std::sin(2.0 * M_PI * 750.0 * time); + samples[index] = static_cast(700.0 * envelope * voiced); + if (frame >= 400) input_energy += static_cast(samples[index]) * samples[index]; + } + ASSERT_EQ(processor.processGain(samples), RealtimeAudioProcessingResult::ok); + for (const auto sample : samples) { + peak = std::max(peak, std::abs(static_cast(sample))); + if (frame >= 400) output_energy += static_cast(sample) * sample; + } + } + EXPECT_GT(output_energy, input_energy * 4.0) << "AGC2 should raise quiet speech by at least 6 dB after adaptation"; + EXPECT_LT(peak, 32767) << "AGC2 must not clip the quiet input"; +} + +TEST_F(AudioProcessingModuleTest, RealtimeBypassDoesNotAdvanceGain) { + RealtimeAudioProcessing gain_only; + RealtimeAudioProcessing bypass_then_gain; + std::uint32_t seed = 123; + for (int frame = 0; frame < 300; ++frame) { + std::array samples{}; + for (auto& sample : samples) { + seed = seed * 1664525 + 1013904223; + sample = static_cast(static_cast(seed >> 20) - 2048); + } + auto bypass_samples = samples; + ASSERT_EQ(bypass_then_gain.process(bypass_samples, nullptr, 0, false, false), RealtimeAudioProcessingResult::ok); + EXPECT_EQ(bypass_samples, samples) << "AEC/NS bypass must not apply gain"; + ASSERT_EQ(gain_only.processGain(samples), RealtimeAudioProcessingResult::ok); + ASSERT_EQ(bypass_then_gain.processGain(bypass_samples), RealtimeAudioProcessingResult::ok); + EXPECT_EQ(bypass_samples, samples) << "AEC/NS processing must not advance AGC2 state"; + } +} + } // namespace livekit::test