Add fixed-frame microphone AEC/NS seam

CMakeLists.txt

@@ -405,6 +405,7 @@ add_custom_target(build_rust_ffi
 add_library(livekit SHARED
   src/audio_frame.cpp
   src/audio_processing_module.cpp
+  src/realtime_audio_processing.cpp
   src/audio_source.cpp
   src/audio_stream.cpp
   src/data_track_frame.cpp

docs/realtime-microphone-dsp.md

@@ -0,0 +1,40 @@
+# Fixed-frame microphone processing
+
+`RealtimeAudioProcessing` is a narrowly scoped native microphone DSP seam for
+the application fork's microphone stage (#127 in syrnike13/syrnike13-app).
+The existing `AudioProcessingModule` API remains unchanged.
+
+Create, use and destroy one instance on its sole owner thread. Construction
+allocates two independent WebRTC processors and prewarms them with 200 fixed
+frames. `process` accepts exactly 480 PCM16 mono samples at 48 kHz, an optional
+rendered reference of the same shape, and a bounded 0–500 ms delay. Processing
+applies AEC followed by NS; either processor can be bypassed independently.
+Absent reference bypasses AEC without disabling NS. The application owns
+reference freshness, renderer epochs, gate/mute, gain and device/publication
+lifetime; this class has no Room, device, callback or global FFI handle.
+
+The private native ABI has only create/process/destroy operations. Fixed buffers
+and integer errors avoid protobuf requests and error-string allocations in
+ordinary processing. The owner contract forbids concurrent calls. A new echo
+timeline requires a newly prepared instance; there is intentionally no reset
+API because WebRTC `Initialize` allocates state.
+
+Local Windows x64 Release experiments against the pinned `.12` base:
+
+| Measurement | Result |
+| --- | --- |
+| Existing protobuf APM, 1,000 warmed forward/reverse/delay frames | 17,000 heap allocations / 489,000 bytes |
+| Fixed-frame seam, 1,000 measured echo frames | 0 observed HeapAlloc/HeapReAlloc calls |
+| Synthetic 30 ms delayed/convolved echo, NS disabled | 40.6 dB ERLE |
+| Independent stationary-noise fixture | 13.8 dB attenuation, 0 measured heap calls |
+| Application warm DSP worker, 200 mute/unmute cycles and config updates | 0 measured heap calls |
+| Initial experiment with WebRTC Initialize at epoch change | 1,369 allocations; rejected and removed from the seam |
+
+Heap evidence uses disposable-process import interposition in the loaded SDK and
+UCRT on the calling thread. It is not a complete ETW trace or proof about every
+allocator, platform, physical echo path or double-talk condition. The native
+application fixture records timing, epochs and whole-worker heap samples.
+
+SDK unit tests cover invalid delay, exact unavailable-reference bypass, owner
+thread rejection, independent NS over 20 create/process/destroy cycles, and
+synthetic AEC with an ERLE threshold of 10 dB.

include/livekit/realtime_audio_processing.h

@@ -0,0 +1,66 @@
+/*
+ * Copyright 2026 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <array>
+#include <cstdint>
+#include <memory>
+#include <thread>
+
+#include "livekit/visibility.h"
+
+namespace livekit {
+
+/// @brief Result of an exclusively owned fixed-frame DSP operation.
+enum class RealtimeAudioProcessingResult { ok, invalid_input, failed, wrong_thread };
+
+/// @brief Warm, fixed 48 kHz mono microphone AEC/NS, independent of AudioSource.
+///
+/// Construct, process and destroy on one owner thread. Construction
+/// allocates and prewarms both processors; ordinary processing uses fixed
+/// buffers and bypasses the protobuf FFI and global handle table. NS and AEC
+/// can be enabled independently per frame. No Room or device is owned here.
+class LIVEKIT_API RealtimeAudioProcessing final {
+public:
+  /// @brief Allocate and prewarm the two independent WebRTC processors.
+  /// @throws std::runtime_error If processor initialization fails.
+  RealtimeAudioProcessing();
+  /// @brief Release the processors on their owner after processing has stopped.
+  ~RealtimeAudioProcessing();
+  RealtimeAudioProcessing(const RealtimeAudioProcessing&) = delete;
+  RealtimeAudioProcessing& operator=(const RealtimeAudioProcessing&) = delete;
+
+  /// @brief Process one fixed frame in place, AEC followed by NS.
+  /// @param microphone Exactly 480 writable mono PCM16 samples.
+  /// @param reference Rendered mono frame, or null to bypass unavailable AEC.
+  /// @param stream_delay_ms Estimated render/capture delay, between 0 and 500 ms.
+  /// @param noise_suppression Whether to apply the warm NS processor.
+  /// @param echo_cancellation Whether to apply AEC when reference is present.
+  /// @return Status; a failure does not change any device or publication.
+  [[nodiscard]] RealtimeAudioProcessingResult process(std::array<std::int16_t, 480>& microphone,
+                                                      const std::array<std::int16_t, 480>* reference,
+                                                      std::uint32_t stream_delay_ms, bool noise_suppression,
+                                                      bool echo_cancellation) noexcept;
+
+private:
+  struct Deleter {
+    void operator()(void* value) const noexcept;
+  };
+  std::unique_ptr<void, Deleter> state_;
+  std::thread::id owner_;
+};
+} // namespace livekit

patches/client-sdk-rust/0001-native-v2-runtime.patch

@@ -1,3 +1,18 @@
+diff --git a/.changeset/microphone-realtime-apm.md b/.changeset/microphone-realtime-apm.md
+new file mode 100644
+index 00000000..aea85aa7
+--- /dev/null
++++ b/.changeset/microphone-realtime-apm.md
+@@ -0,0 +1,9 @@
++---
++livekit-ffi: patch
++libwebrtc: patch
++webrtc-sys: patch
++---
++
++Add an exclusively owned, fixed-frame microphone AEC/NS seam that avoids
++protobuf serialization, global FFI handles and error-string allocation on the
++processing path. Existing AudioProcessingModule APIs remain unchanged.
 diff --git a/.changeset/preencoded-bitrate-feedback.md b/.changeset/preencoded-bitrate-feedback.md
 new file mode 100644
 index 00000000..0dffd207
@@ -78,6 +93,49 @@ index 00000000..5896d522
 +replacement screen track. Return an absent track safely for inactive senders.
 +This preserves bounded transceiver reuse while allowing the SFU to associate
 +new RTP media with the replacement publication.
+diff --git a/libwebrtc/src/native/apm.rs b/libwebrtc/src/native/apm.rs
+index a5c53fb9..f40b204d 100644
+--- a/libwebrtc/src/native/apm.rs
++++ b/libwebrtc/src/native/apm.rs
+@@ -22,6 +22,38 @@ pub struct AudioProcessingModule {
+ }
+
+ impl AudioProcessingModule {
++    /// Process one fixed 48 kHz mono frame without constructing an error string.
++    pub fn process_10ms(&mut self, data: &mut [i16; 480], reverse: bool) -> i32 {
++        // SAFETY: The fixed array is aligned, valid for 480 i16 samples, and
++        // exclusively borrowed. WebRTC supports in-place processing.
++        unsafe {
++            if reverse {
++                self.sys_handle.pin_mut().process_reverse_stream(
++                    data.as_ptr(),
++                    480,
++                    data.as_mut_ptr(),
++                    480,
++                    48_000,
++                    1,
++                )
++            } else {
++                self.sys_handle.pin_mut().process_stream(
++                    data.as_ptr(),
++                    480,
++                    data.as_mut_ptr(),
++                    480,
++                    48_000,
++                    1,
++                )
++            }
++        }
++    }
++
++    /// Set the fixed-frame processor delay without allocating an error value.
++    pub fn set_realtime_delay_ms(&mut self, delay_ms: i32) -> i32 {
++        self.sys_handle.pin_mut().set_stream_delay_ms(delay_ms)
++    }
++
+     pub fn new(
+         echo_canceller_enabled: bool,
+         gain_controller_enabled: bool,
 diff --git a/libwebrtc/src/native/rtp_transceiver.rs b/libwebrtc/src/native/rtp_transceiver.rs
 index 3bd4fb59..a6a089bc 100644
 --- a/libwebrtc/src/native/rtp_transceiver.rs
@@ -260,6 +318,140 @@ index 4922add5..281dcfc7 100644
          ];
 
          for (proto_backend, expected) in cases {
+diff --git a/livekit-ffi/src/lib.rs b/livekit-ffi/src/lib.rs
+index 06695660..9c71e3fd 100644
+--- a/livekit-ffi/src/lib.rs
++++ b/livekit-ffi/src/lib.rs
+@@ -19,6 +19,7 @@ use livekit::prelude::*;
+ use thiserror::Error;
+
+ mod conversion;
++mod realtime_apm;
+
+ pub mod cabi;
+ pub mod proto;
+diff --git a/livekit-ffi/src/realtime_apm.rs b/livekit-ffi/src/realtime_apm.rs
+new file mode 100644
+index 00000000..a13fa9f4
+--- /dev/null
++++ b/livekit-ffi/src/realtime_apm.rs
+@@ -0,0 +1,116 @@
++// Copyright 2026 LiveKit, Inc.
++//
++// Licensed under the Apache License, Version 2.0 (the "License");
++// you may not use this file except in compliance with the License.
++// You may obtain a copy of the License at
++//
++//     http://www.apache.org/licenses/LICENSE-2.0
++//
++// Unless required by applicable law or agreed to in writing, software
++// distributed under the License is distributed on an "AS IS" BASIS,
++// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++// See the License for the specific language governing permissions and
++// limitations under the License.
++
++//! Fixed-frame, exclusively owned microphone DSP seam. This has no FFI server
++//! handle table, protobuf, mutex, event, callback or asynchronous operation.
++
++use livekit::webrtc::native::apm::AudioProcessingModule;
++use std::{ffi::c_void, mem::align_of, panic, ptr};
++
++struct RealtimeApm {
++    echo: AudioProcessingModule,
++    noise: AudioProcessingModule,
++}
++
++/// Allocate and prewarm the two independent processors on their future owner.
++#[no_mangle]
++pub extern "C" fn livekit_ffi_realtime_apm_create() -> *mut c_void {
++    panic::catch_unwind(|| {
++        let mut apm = Box::new(RealtimeApm {
++            echo: AudioProcessingModule::new(true, false, false, false),
++            noise: AudioProcessingModule::new(false, false, false, true),
++        });
++        let mut reference = [0i16; 480];
++        let mut microphone = [0i16; 480];
++        let mut seed = 1u32;
++        for _ in 0..200 {
++            for sample in &mut reference {
++                seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
++                *sample = ((seed >> 20) as i16) - 2048;
++            }
++            microphone.copy_from_slice(&reference);
++            if apm.echo.set_realtime_delay_ms(0) != 0
++                || apm.echo.process_10ms(&mut reference, true) != 0
++                || apm.echo.process_10ms(&mut microphone, false) != 0
++                || apm.noise.process_10ms(&mut microphone, false) != 0
++            {
++                return ptr::null_mut();
++            }
++        }
++        Box::into_raw(apm).cast()
++    })
++    .unwrap_or(ptr::null_mut())
++}
++
++/// Release a processor on the same owner after all processing calls finish.
++///
++/// # Safety
++/// `handle` must be null or a live pointer returned by create, consumed exactly
++/// once. It must not be accessed concurrently or after this call.
++#[no_mangle]
++pub unsafe extern "C" fn livekit_ffi_realtime_apm_destroy(handle: *mut c_void) {
++    if !handle.is_null() {
++        // SAFETY: Exclusive ownership is returned under the documented contract.
++        drop(unsafe { Box::from_raw(handle.cast::<RealtimeApm>()) });
++    }
++}
++
++/// Process one 48 kHz mono/10 ms frame. Returns 0=ok, 1=invalid input, 2=DSP failure.
++///
++/// # Safety
++/// `handle` is a live, exclusively accessed create result. `microphone` points
++/// to exactly 480 writable i16 samples. `reference` is null or 480 readable i16
++/// samples. All storage remains valid for this synchronous call.
++#[no_mangle]
++pub unsafe extern "C" fn livekit_ffi_realtime_apm_process(
++    handle: *mut c_void,
++    microphone: *mut i16,
++    samples: usize,
++    reference: *const i16,
++    delay_ms: u32,
++    noise_suppression: bool,
++    echo_cancellation: bool,
++) -> i32 {
++    if handle.is_null()
++        || microphone.is_null()
++        || samples != 480
++        || microphone as usize % align_of::<i16>() != 0
++        || reference as usize % align_of::<i16>() != 0
++        || delay_ms > 500
++    {
++        return 1;
++    }
++    let mut reverse = [0i16; 480];
++    if !reference.is_null() {
++        // SAFETY: Caller guarantees the 480 samples. Copy before borrowing the
++        // microphone, allowing callers to use the same storage without aliasing.
++        unsafe { ptr::copy_nonoverlapping(reference, reverse.as_mut_ptr(), 480) };
++    }
++    // SAFETY: Validated dimensions/alignment and exclusive caller-owned storage.
++    let apm = unsafe { &mut *handle.cast::<RealtimeApm>() };
++    // SAFETY: The caller supplies exactly one writable fixed frame.
++    let microphone = unsafe { &mut *microphone.cast::<[i16; 480]>() };
++    if echo_cancellation && !reference.is_null() {
++        if apm.echo.set_realtime_delay_ms(delay_ms as i32) != 0
++            || apm.echo.process_10ms(&mut reverse, true) != 0
++            || apm.echo.process_10ms(microphone, false) != 0
++        {
++            return 2;
++        }
++    }
++    if noise_suppression && apm.noise.process_10ms(microphone, false) != 0 {
++        return 2;
++    }
++    0
++}
 diff --git a/livekit-ffi/src/server/requests.rs b/livekit-ffi/src/server/requests.rs
 index 8ee56988..95f1c87c 100644
 --- a/livekit-ffi/src/server/requests.rs

src/realtime_audio_processing.cpp

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2026 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "livekit/realtime_audio_processing.h"
+
+#include <stdexcept>
+
+// Private native ABI: no generated protocol changes or public raw pointer API.
+extern "C" {
+void* livekit_ffi_realtime_apm_create();
+void livekit_ffi_realtime_apm_destroy(void* handle);
+int livekit_ffi_realtime_apm_process(void* handle, std::int16_t* microphone, std::size_t samples,
+                                     const std::int16_t* reference, std::uint32_t delay_ms, bool noise_suppression,
+                                     bool echo_cancellation);
+}
+
+namespace livekit {
+namespace {
+RealtimeAudioProcessingResult resultOf(int result) noexcept {
+  if (result == 0) return RealtimeAudioProcessingResult::ok;
+  if (result == 1) return RealtimeAudioProcessingResult::invalid_input;
+  return RealtimeAudioProcessingResult::failed;
+}
+} // namespace
+void RealtimeAudioProcessing::Deleter::operator()(void* value) const noexcept {
+  livekit_ffi_realtime_apm_destroy(value);
+}
+RealtimeAudioProcessing::RealtimeAudioProcessing()
+    : state_(livekit_ffi_realtime_apm_create()), owner_(std::this_thread::get_id()) {
+  if (!state_) throw std::runtime_error("Realtime microphone DSP initialization failed");
+}
+RealtimeAudioProcessing::~RealtimeAudioProcessing() {
+  if (owner_ != std::this_thread::get_id()) std::terminate();
+}
+RealtimeAudioProcessingResult RealtimeAudioProcessing::process(std::array<std::int16_t, 480>& microphone,
+                                                               const std::array<std::int16_t, 480>* reference,
+                                                               std::uint32_t stream_delay_ms, bool noise_suppression,
+                                                               bool echo_cancellation) noexcept {
+  if (owner_ != std::this_thread::get_id()) return RealtimeAudioProcessingResult::wrong_thread;
+  return resultOf(livekit_ffi_realtime_apm_process(state_.get(), microphone.data(), microphone.size(),
+                                                   reference ? reference->data() : nullptr, stream_delay_ms,
+                                                   noise_suppression, echo_cancellation));
+}
+} // namespace livekit