From 77b2494b32ef2c6a12913e2725546090ea3e6d68 Mon Sep 17 00:00:00 2001 From: IsaFxck Date: Tue, 8 Sep 2026 01:29:14 +0300 Subject: [PATCH] Add fixed-frame microphone DSP seam for app issue 127 --- CMakeLists.txt | 1 + docs/realtime-microphone-dsp.md | 40 ++++ include/livekit/realtime_audio_processing.h | 66 ++++++ .../0001-native-v2-runtime.patch | 192 ++++++++++++++++++ src/realtime_audio_processing.cpp | 57 ++++++ .../unit/test_audio_processing_module.cpp | 66 ++++++ 6 files changed, 422 insertions(+) create mode 100644 docs/realtime-microphone-dsp.md create mode 100644 include/livekit/realtime_audio_processing.h create mode 100644 src/realtime_audio_processing.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index ffab479..a4c3a6b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -405,6 +405,7 @@ add_custom_target(build_rust_ffi add_library(livekit SHARED src/audio_frame.cpp src/audio_processing_module.cpp + src/realtime_audio_processing.cpp src/audio_source.cpp src/audio_stream.cpp src/data_track_frame.cpp diff --git a/docs/realtime-microphone-dsp.md b/docs/realtime-microphone-dsp.md new file mode 100644 index 0000000..bef0dc3 --- /dev/null +++ b/docs/realtime-microphone-dsp.md @@ -0,0 +1,40 @@ +# Fixed-frame microphone processing + +`RealtimeAudioProcessing` is a narrowly scoped native microphone DSP seam for +the application fork's microphone stage (#127 in syrnike13/syrnike13-app). +The existing `AudioProcessingModule` API remains unchanged. + +Create, use and destroy one instance on its sole owner thread. Construction +allocates two independent WebRTC processors and prewarms them with 200 fixed +frames. `process` accepts exactly 480 PCM16 mono samples at 48 kHz, an optional +rendered reference of the same shape, and a bounded 0–500 ms delay. Processing +applies AEC followed by NS; either processor can be bypassed independently. +Absent reference bypasses AEC without disabling NS. The application owns +reference freshness, renderer epochs, gate/mute, gain and device/publication +lifetime; this class has no Room, device, callback or global FFI handle. + +The private native ABI has only create/process/destroy operations. Fixed buffers +and integer errors avoid protobuf requests and error-string allocations in +ordinary processing. The owner contract forbids concurrent calls. A new echo +timeline requires a newly prepared instance; there is intentionally no reset +API because WebRTC `Initialize` allocates state. + +Local Windows x64 Release experiments against the pinned `.12` base: + +| Measurement | Result | +| --- | --- | +| Existing protobuf APM, 1,000 warmed forward/reverse/delay frames | 17,000 heap allocations / 489,000 bytes | +| Fixed-frame seam, 1,000 measured echo frames | 0 observed HeapAlloc/HeapReAlloc calls | +| Synthetic 30 ms delayed/convolved echo, NS disabled | 40.6 dB ERLE | +| Independent stationary-noise fixture | 13.8 dB attenuation, 0 measured heap calls | +| Application warm DSP worker, 200 mute/unmute cycles and config updates | 0 measured heap calls | +| Initial experiment with WebRTC Initialize at epoch change | 1,369 allocations; rejected and removed from the seam | + +Heap evidence uses disposable-process import interposition in the loaded SDK and +UCRT on the calling thread. It is not a complete ETW trace or proof about every +allocator, platform, physical echo path or double-talk condition. The native +application fixture records timing, epochs and whole-worker heap samples. + +SDK unit tests cover invalid delay, exact unavailable-reference bypass, owner +thread rejection, independent NS over 20 create/process/destroy cycles, and +synthetic AEC with an ERLE threshold of 10 dB. diff --git a/include/livekit/realtime_audio_processing.h b/include/livekit/realtime_audio_processing.h new file mode 100644 index 0000000..2d68416 --- /dev/null +++ b/include/livekit/realtime_audio_processing.h @@ -0,0 +1,66 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "livekit/visibility.h" + +namespace livekit { + +/// @brief Result of an exclusively owned fixed-frame DSP operation. +enum class RealtimeAudioProcessingResult { ok, invalid_input, failed, wrong_thread }; + +/// @brief Warm, fixed 48 kHz mono microphone AEC/NS, independent of AudioSource. +/// +/// Construct, process and destroy on one owner thread. Construction +/// allocates and prewarms both processors; ordinary processing uses fixed +/// buffers and bypasses the protobuf FFI and global handle table. NS and AEC +/// can be enabled independently per frame. No Room or device is owned here. +class LIVEKIT_API RealtimeAudioProcessing final { +public: + /// @brief Allocate and prewarm the two independent WebRTC processors. + /// @throws std::runtime_error If processor initialization fails. + RealtimeAudioProcessing(); + /// @brief Release the processors on their owner after processing has stopped. + ~RealtimeAudioProcessing(); + RealtimeAudioProcessing(const RealtimeAudioProcessing&) = delete; + RealtimeAudioProcessing& operator=(const RealtimeAudioProcessing&) = delete; + + /// @brief Process one fixed frame in place, AEC followed by NS. + /// @param microphone Exactly 480 writable mono PCM16 samples. + /// @param reference Rendered mono frame, or null to bypass unavailable AEC. + /// @param stream_delay_ms Estimated render/capture delay, between 0 and 500 ms. + /// @param noise_suppression Whether to apply the warm NS processor. + /// @param echo_cancellation Whether to apply AEC when reference is present. + /// @return Status; a failure does not change any device or publication. + [[nodiscard]] RealtimeAudioProcessingResult process(std::array& microphone, + const std::array* reference, + std::uint32_t stream_delay_ms, bool noise_suppression, + bool echo_cancellation) noexcept; + +private: + struct Deleter { + void operator()(void* value) const noexcept; + }; + std::unique_ptr state_; + std::thread::id owner_; +}; +} // namespace livekit diff --git a/patches/client-sdk-rust/0001-native-v2-runtime.patch b/patches/client-sdk-rust/0001-native-v2-runtime.patch index fa52a2b..7816739 100644 --- a/patches/client-sdk-rust/0001-native-v2-runtime.patch +++ b/patches/client-sdk-rust/0001-native-v2-runtime.patch @@ -1,3 +1,18 @@ +diff --git a/.changeset/microphone-realtime-apm.md b/.changeset/microphone-realtime-apm.md +new file mode 100644 +index 00000000..aea85aa7 +--- /dev/null ++++ b/.changeset/microphone-realtime-apm.md +@@ -0,0 +1,9 @@ ++--- ++livekit-ffi: patch ++libwebrtc: patch ++webrtc-sys: patch ++--- ++ ++Add an exclusively owned, fixed-frame microphone AEC/NS seam that avoids ++protobuf serialization, global FFI handles and error-string allocation on the ++processing path. Existing AudioProcessingModule APIs remain unchanged. diff --git a/.changeset/preencoded-bitrate-feedback.md b/.changeset/preencoded-bitrate-feedback.md new file mode 100644 index 00000000..0dffd207 @@ -78,6 +93,49 @@ index 00000000..5896d522 +replacement screen track. Return an absent track safely for inactive senders. +This preserves bounded transceiver reuse while allowing the SFU to associate +new RTP media with the replacement publication. +diff --git a/libwebrtc/src/native/apm.rs b/libwebrtc/src/native/apm.rs +index a5c53fb9..f40b204d 100644 +--- a/libwebrtc/src/native/apm.rs ++++ b/libwebrtc/src/native/apm.rs +@@ -22,6 +22,38 @@ pub struct AudioProcessingModule { + } + + impl AudioProcessingModule { ++ /// Process one fixed 48 kHz mono frame without constructing an error string. ++ pub fn process_10ms(&mut self, data: &mut [i16; 480], reverse: bool) -> i32 { ++ // SAFETY: The fixed array is aligned, valid for 480 i16 samples, and ++ // exclusively borrowed. WebRTC supports in-place processing. ++ unsafe { ++ if reverse { ++ self.sys_handle.pin_mut().process_reverse_stream( ++ data.as_ptr(), ++ 480, ++ data.as_mut_ptr(), ++ 480, ++ 48_000, ++ 1, ++ ) ++ } else { ++ self.sys_handle.pin_mut().process_stream( ++ data.as_ptr(), ++ 480, ++ data.as_mut_ptr(), ++ 480, ++ 48_000, ++ 1, ++ ) ++ } ++ } ++ } ++ ++ /// Set the fixed-frame processor delay without allocating an error value. ++ pub fn set_realtime_delay_ms(&mut self, delay_ms: i32) -> i32 { ++ self.sys_handle.pin_mut().set_stream_delay_ms(delay_ms) ++ } ++ + pub fn new( + echo_canceller_enabled: bool, + gain_controller_enabled: bool, diff --git a/libwebrtc/src/native/rtp_transceiver.rs b/libwebrtc/src/native/rtp_transceiver.rs index 3bd4fb59..a6a089bc 100644 --- a/libwebrtc/src/native/rtp_transceiver.rs @@ -260,6 +318,140 @@ index 4922add5..281dcfc7 100644 ]; for (proto_backend, expected) in cases { +diff --git a/livekit-ffi/src/lib.rs b/livekit-ffi/src/lib.rs +index 06695660..9c71e3fd 100644 +--- a/livekit-ffi/src/lib.rs ++++ b/livekit-ffi/src/lib.rs +@@ -19,6 +19,7 @@ use livekit::prelude::*; + use thiserror::Error; + + mod conversion; ++mod realtime_apm; + + pub mod cabi; + pub mod proto; +diff --git a/livekit-ffi/src/realtime_apm.rs b/livekit-ffi/src/realtime_apm.rs +new file mode 100644 +index 00000000..a13fa9f4 +--- /dev/null ++++ b/livekit-ffi/src/realtime_apm.rs +@@ -0,0 +1,116 @@ ++// Copyright 2026 LiveKit, Inc. ++// ++// Licensed under the Apache License, Version 2.0 (the "License"); ++// you may not use this file except in compliance with the License. ++// You may obtain a copy of the License at ++// ++// http://www.apache.org/licenses/LICENSE-2.0 ++// ++// Unless required by applicable law or agreed to in writing, software ++// distributed under the License is distributed on an "AS IS" BASIS, ++// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++// See the License for the specific language governing permissions and ++// limitations under the License. ++ ++//! Fixed-frame, exclusively owned microphone DSP seam. This has no FFI server ++//! handle table, protobuf, mutex, event, callback or asynchronous operation. ++ ++use livekit::webrtc::native::apm::AudioProcessingModule; ++use std::{ffi::c_void, mem::align_of, panic, ptr}; ++ ++struct RealtimeApm { ++ echo: AudioProcessingModule, ++ noise: AudioProcessingModule, ++} ++ ++/// Allocate and prewarm the two independent processors on their future owner. ++#[no_mangle] ++pub extern "C" fn livekit_ffi_realtime_apm_create() -> *mut c_void { ++ panic::catch_unwind(|| { ++ let mut apm = Box::new(RealtimeApm { ++ echo: AudioProcessingModule::new(true, false, false, false), ++ noise: AudioProcessingModule::new(false, false, false, true), ++ }); ++ let mut reference = [0i16; 480]; ++ let mut microphone = [0i16; 480]; ++ let mut seed = 1u32; ++ for _ in 0..200 { ++ for sample in &mut reference { ++ seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); ++ *sample = ((seed >> 20) as i16) - 2048; ++ } ++ microphone.copy_from_slice(&reference); ++ if apm.echo.set_realtime_delay_ms(0) != 0 ++ || apm.echo.process_10ms(&mut reference, true) != 0 ++ || apm.echo.process_10ms(&mut microphone, false) != 0 ++ || apm.noise.process_10ms(&mut microphone, false) != 0 ++ { ++ return ptr::null_mut(); ++ } ++ } ++ Box::into_raw(apm).cast() ++ }) ++ .unwrap_or(ptr::null_mut()) ++} ++ ++/// Release a processor on the same owner after all processing calls finish. ++/// ++/// # Safety ++/// `handle` must be null or a live pointer returned by create, consumed exactly ++/// once. It must not be accessed concurrently or after this call. ++#[no_mangle] ++pub unsafe extern "C" fn livekit_ffi_realtime_apm_destroy(handle: *mut c_void) { ++ if !handle.is_null() { ++ // SAFETY: Exclusive ownership is returned under the documented contract. ++ drop(unsafe { Box::from_raw(handle.cast::()) }); ++ } ++} ++ ++/// Process one 48 kHz mono/10 ms frame. Returns 0=ok, 1=invalid input, 2=DSP failure. ++/// ++/// # Safety ++/// `handle` is a live, exclusively accessed create result. `microphone` points ++/// to exactly 480 writable i16 samples. `reference` is null or 480 readable i16 ++/// samples. All storage remains valid for this synchronous call. ++#[no_mangle] ++pub unsafe extern "C" fn livekit_ffi_realtime_apm_process( ++ handle: *mut c_void, ++ microphone: *mut i16, ++ samples: usize, ++ reference: *const i16, ++ delay_ms: u32, ++ noise_suppression: bool, ++ echo_cancellation: bool, ++) -> i32 { ++ if handle.is_null() ++ || microphone.is_null() ++ || samples != 480 ++ || microphone as usize % align_of::() != 0 ++ || reference as usize % align_of::() != 0 ++ || delay_ms > 500 ++ { ++ return 1; ++ } ++ let mut reverse = [0i16; 480]; ++ if !reference.is_null() { ++ // SAFETY: Caller guarantees the 480 samples. Copy before borrowing the ++ // microphone, allowing callers to use the same storage without aliasing. ++ unsafe { ptr::copy_nonoverlapping(reference, reverse.as_mut_ptr(), 480) }; ++ } ++ // SAFETY: Validated dimensions/alignment and exclusive caller-owned storage. ++ let apm = unsafe { &mut *handle.cast::() }; ++ // SAFETY: The caller supplies exactly one writable fixed frame. ++ let microphone = unsafe { &mut *microphone.cast::<[i16; 480]>() }; ++ if echo_cancellation && !reference.is_null() { ++ if apm.echo.set_realtime_delay_ms(delay_ms as i32) != 0 ++ || apm.echo.process_10ms(&mut reverse, true) != 0 ++ || apm.echo.process_10ms(microphone, false) != 0 ++ { ++ return 2; ++ } ++ } ++ if noise_suppression && apm.noise.process_10ms(microphone, false) != 0 { ++ return 2; ++ } ++ 0 ++} diff --git a/livekit-ffi/src/server/requests.rs b/livekit-ffi/src/server/requests.rs index 8ee56988..95f1c87c 100644 --- a/livekit-ffi/src/server/requests.rs diff --git a/src/realtime_audio_processing.cpp b/src/realtime_audio_processing.cpp new file mode 100644 index 0000000..0750fa0 --- /dev/null +++ b/src/realtime_audio_processing.cpp @@ -0,0 +1,57 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "livekit/realtime_audio_processing.h" + +#include + +// Private native ABI: no generated protocol changes or public raw pointer API. +extern "C" { +void* livekit_ffi_realtime_apm_create(); +void livekit_ffi_realtime_apm_destroy(void* handle); +int livekit_ffi_realtime_apm_process(void* handle, std::int16_t* microphone, std::size_t samples, + const std::int16_t* reference, std::uint32_t delay_ms, bool noise_suppression, + bool echo_cancellation); +} + +namespace livekit { +namespace { +RealtimeAudioProcessingResult resultOf(int result) noexcept { + if (result == 0) return RealtimeAudioProcessingResult::ok; + if (result == 1) return RealtimeAudioProcessingResult::invalid_input; + return RealtimeAudioProcessingResult::failed; +} +} // namespace +void RealtimeAudioProcessing::Deleter::operator()(void* value) const noexcept { + livekit_ffi_realtime_apm_destroy(value); +} +RealtimeAudioProcessing::RealtimeAudioProcessing() + : state_(livekit_ffi_realtime_apm_create()), owner_(std::this_thread::get_id()) { + if (!state_) throw std::runtime_error("Realtime microphone DSP initialization failed"); +} +RealtimeAudioProcessing::~RealtimeAudioProcessing() { + if (owner_ != std::this_thread::get_id()) std::terminate(); +} +RealtimeAudioProcessingResult RealtimeAudioProcessing::process(std::array& microphone, + const std::array* reference, + std::uint32_t stream_delay_ms, bool noise_suppression, + bool echo_cancellation) noexcept { + if (owner_ != std::this_thread::get_id()) return RealtimeAudioProcessingResult::wrong_thread; + return resultOf(livekit_ffi_realtime_apm_process(state_.get(), microphone.data(), microphone.size(), + reference ? reference->data() : nullptr, stream_delay_ms, + noise_suppression, echo_cancellation)); +} +} // namespace livekit diff --git a/src/tests/unit/test_audio_processing_module.cpp b/src/tests/unit/test_audio_processing_module.cpp index 62d86d6..325cb27 100644 --- a/src/tests/unit/test_audio_processing_module.cpp +++ b/src/tests/unit/test_audio_processing_module.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -26,6 +27,7 @@ #include #include #include +#include #include namespace livekit::test { @@ -885,4 +887,68 @@ TEST_F(AudioProcessingModuleTest, AGCWithNoiseSuppressionCombined) { EXPECT_GT(avg_output_energy, 100.0) << "Combined AGC+NS should produce reasonable output"; } +TEST_F(AudioProcessingModuleTest, RealtimeBypassAndOwnerContract) { + RealtimeAudioProcessing processor; + std::array samples{}; + samples.fill(1234); + const auto original = samples; + EXPECT_EQ(processor.process(samples, nullptr, 0, false, true), RealtimeAudioProcessingResult::ok); + EXPECT_EQ(samples, original); + EXPECT_EQ(processor.process(samples, nullptr, 501, false, false), RealtimeAudioProcessingResult::invalid_input); + EXPECT_EQ(samples, original); + auto result = RealtimeAudioProcessingResult::ok; + std::thread other([&] { result = processor.process(samples, nullptr, 0, false, false); }); + other.join(); + EXPECT_EQ(result, RealtimeAudioProcessingResult::wrong_thread); + EXPECT_EQ(samples, original); +} + +TEST_F(AudioProcessingModuleTest, RealtimeIndependentNoiseAndLifecycle) { + std::uint32_t seed = 123; + for (int cycle = 0; cycle < 20; ++cycle) { + RealtimeAudioProcessing processor; + double input_energy = 0; + double output_energy = 0; + for (int frame = 0; frame < 300; ++frame) { + std::array samples{}; + for (auto& sample : samples) { + seed = seed * 1664525 + 1013904223; + sample = static_cast((seed >> 20) - 2048); + if (frame >= 150) input_energy += static_cast(sample) * sample; + } + ASSERT_EQ(processor.process(samples, nullptr, 0, true, true), RealtimeAudioProcessingResult::ok); + if (frame >= 150) + for (const auto sample : samples) output_energy += static_cast(sample) * sample; + } + EXPECT_LT(output_energy, input_energy * 0.5) << "NS must work with unavailable AEC reference"; + } +} + +TEST_F(AudioProcessingModuleTest, RealtimeSyntheticEcho) { + RealtimeAudioProcessing processor; + std::array, 4> history{}; + std::uint32_t seed = 0x12345678; + double input_energy = 0; + double output_energy = 0; + for (std::size_t frame = 0; frame < 1500; ++frame) { + auto& reference = history[frame % history.size()]; + for (auto& sample : reference) { + seed = seed * 1664525 + 1013904223; + sample = static_cast(static_cast(seed >> 16) * 12000 / 32768 - 12000); + } + const auto& delayed = history[(frame + 1) % history.size()]; + std::array microphone{}; + for (std::size_t index = 0; index < microphone.size(); ++index) { + const auto previous = index ? delayed[index - 1] : 0; + microphone[index] = static_cast(delayed[index] * 0.55 + previous * 0.2); + if (frame >= 500) input_energy += static_cast(microphone[index]) * microphone[index]; + } + ASSERT_EQ(processor.process(microphone, &reference, 30, false, true), RealtimeAudioProcessingResult::ok); + if (frame >= 500) + for (const auto sample : microphone) output_energy += static_cast(sample) * sample; + } + const auto erle_db = 10 * std::log10(input_energy / std::max(output_energy, 1.0)); + EXPECT_GE(erle_db, 10.0) << "AEC must suppress the delayed reference independently of NS"; +} + } // namespace livekit::test