feat: AGC2 adaptive digital gain in realtime microphone processing

docs/realtime-microphone-dsp.md

@@ -5,15 +5,17 @@ the application fork's microphone stage (#127 in syrnike13/syrnike13-app).
 The existing `AudioProcessingModule` API remains unchanged.
 
 Create, use and destroy one instance on its sole owner thread. Construction
-allocates two independent WebRTC processors and prewarms them with 200 fixed
+allocates three independent WebRTC processors and prewarms them with 200 fixed
 frames. `process` accepts exactly 480 PCM16 mono samples at 48 kHz, an optional
 rendered reference of the same shape, and a bounded 0–500 ms delay. Processing
-applies AEC followed by NS; either processor can be bypassed independently.
+applies AEC followed by NS; each processor can be bypassed independently.
+AGC2 is only available through `processGain`. To preserve gate thresholds,
+measure the post-AEC/NS level, then call `processGain` on that frame.
 Absent reference bypasses AEC without disabling NS. The application owns
-reference freshness, renderer epochs, gate/mute, gain and device/publication
+reference freshness, renderer epochs, gate/mute and device/publication
 lifetime; this class has no Room, device, callback or global FFI handle.
 
-The private native ABI has only create/process/destroy operations. Fixed buffers
+The private native ABI has only create/process/process-gain/destroy operations. Fixed buffers
 and integer errors avoid protobuf requests and error-string allocations in
 ordinary processing. The owner contract forbids concurrent calls. A new echo
 timeline requires a newly prepared instance; there is intentionally no reset

include/livekit/realtime_audio_processing.h

@@ -28,15 +28,15 @@ namespace livekit {
 /// @brief Result of an exclusively owned fixed-frame DSP operation.
 enum class RealtimeAudioProcessingResult { ok, invalid_input, failed, wrong_thread };
 
-/// @brief Warm, fixed 48 kHz mono microphone AEC/NS, independent of AudioSource.
+/// @brief Warm, fixed 48 kHz mono microphone AEC/NS/AGC2, independent of AudioSource.
 ///
 /// Construct, process and destroy on one owner thread. Construction
-/// allocates and prewarms both processors; ordinary processing uses fixed
-/// buffers and bypasses the protobuf FFI and global handle table. NS and AEC
+/// allocates and prewarms all three processors; ordinary processing uses fixed
+/// buffers and bypasses the protobuf FFI and global handle table. NS, AEC and AGC2
 /// can be enabled independently per frame. No Room or device is owned here.
 class LIVEKIT_API RealtimeAudioProcessing final {
 public:
-  /// @brief Allocate and prewarm the two independent WebRTC processors.
+  /// @brief Allocate and prewarm the three independent WebRTC processors.
   /// @throws std::runtime_error If processor initialization fails.
   RealtimeAudioProcessing();
   /// @brief Release the processors on their owner after processing has stopped.
@@ -56,6 +56,11 @@ class LIVEKIT_API RealtimeAudioProcessing final {
                                                       std::uint32_t stream_delay_ms, bool noise_suppression,
                                                       bool echo_cancellation) noexcept;
 
+  /// @brief Apply only AGC2, allowing a gate to measure the frame after AEC/NS.
+  /// @param microphone Exactly 480 writable mono PCM16 samples.
+  /// @return Status under the same owner-thread contract as process().
+  [[nodiscard]] RealtimeAudioProcessingResult processGain(std::array<std::int16_t, 480>& microphone) noexcept;
+
 private:
   struct Deleter {
     void operator()(void* value) const noexcept;

patches/client-sdk-rust/0001-native-v2-runtime.patch

@@ -121,6 +121,20 @@ index 00000000..9a1363bb
 +publisher transceiver. Apply the current AddTrack identity only to the offer
 +sent to the SFU, so a later unrelated negotiation does not remove the replacement
 +publication. Transceiver reuse and its resource bound remain enabled.
+diff --git a/.changeset/realtime-agc2.md b/.changeset/realtime-agc2.md
+new file mode 100644
+index 00000000..74933b96
+--- /dev/null
++++ b/.changeset/realtime-agc2.md
+@@ -0,0 +1,8 @@
++---
++livekit-ffi: patch
++---
++
++Add independently prewarmed AGC2 adaptive digital gain to the fixed-frame
++microphone processing seam through a separate gain-only operation, allowing
++gate measurement after AEC and noise suppression without protobuf requests or
++processing-path buffer allocations. The AEC/NS operation has no gain flag.
 diff --git a/.changeset/reused-screen-sender.md b/.changeset/reused-screen-sender.md
 new file mode 100644
 index 00000000..5896d522
@@ -984,10 +998,10 @@ index 7a4234e3..880e2825 100644
 
      pub trait VideoFrameBufferExt: VideoBuffer {
 diff --git a/livekit-ffi/include/livekit_ffi.h b/livekit-ffi/include/livekit_ffi.h
-index 15f64dfa..6da08d39 100644
+index 15f64dfa..3fb1022b 100644
 --- a/livekit-ffi/include/livekit_ffi.h
 +++ b/livekit-ffi/include/livekit_ffi.h
-@@ -37,6 +37,23 @@ FfiHandleId livekit_ffi_request(const uint8_t *data, size_t len,
+@@ -37,6 +37,33 @@ FfiHandleId livekit_ffi_request(const uint8_t *data, size_t len,
 
  bool livekit_ffi_drop_handle(FfiHandleId handle_id);
 
@@ -1007,6 +1021,16 @@ index 15f64dfa..6da08d39 100644
 +/// Synchronously pulls one 10 ms frame for all active Rooms; false if not playing
 +/// or external playout is disabled. Call from the single playout owner thread.
 +bool livekit_ffi_pull_audio_playout();
++
++void *livekit_ffi_realtime_apm_create();
++
++void livekit_ffi_realtime_apm_destroy(void *handle);
++
++int32_t livekit_ffi_realtime_apm_process(void *handle, int16_t *microphone, size_t samples,
++                                       const int16_t *reference, uint32_t delay_ms,
++                                       bool noise_suppression, bool echo_cancellation);
++
++int32_t livekit_ffi_realtime_apm_process_gain(void *handle, int16_t *microphone, size_t samples);
 +
  } // extern "C"
 
@@ -1385,10 +1409,10 @@ index 06695660..87cd8850 100644
  pub mod proto;
 diff --git a/livekit-ffi/src/realtime_apm.rs b/livekit-ffi/src/realtime_apm.rs
 new file mode 100644
-index 00000000..a13fa9f4
+index 00000000..7e5cce40
 --- /dev/null
 +++ b/livekit-ffi/src/realtime_apm.rs
-@@ -0,0 +1,116 @@
+@@ -0,0 +1,223 @@
 +// Copyright 2026 LiveKit, Inc.
 +//
 +// Licensed under the Apache License, Version 2.0 (the "License");
@@ -1412,15 +1436,17 @@ index 00000000..a13fa9f4
 +struct RealtimeApm {
 +    echo: AudioProcessingModule,
 +    noise: AudioProcessingModule,
++    gain: AudioProcessingModule,
 +}
 +
-+/// Allocate and prewarm the two independent processors on their future owner.
++/// Allocate and prewarm the three independent processors on their future owner.
 +#[no_mangle]
 +pub extern "C" fn livekit_ffi_realtime_apm_create() -> *mut c_void {
 +    panic::catch_unwind(|| {
 +        let mut apm = Box::new(RealtimeApm {
 +            echo: AudioProcessingModule::new(true, false, false, false),
 +            noise: AudioProcessingModule::new(false, false, false, true),
++            gain: AudioProcessingModule::new(false, true, false, false),
 +        });
 +        let mut reference = [0i16; 480];
 +        let mut microphone = [0i16; 480];
@@ -1435,6 +1461,7 @@ index 00000000..a13fa9f4
 +                || apm.echo.process_10ms(&mut reference, true) != 0
 +                || apm.echo.process_10ms(&mut microphone, false) != 0
 +                || apm.noise.process_10ms(&mut microphone, false) != 0
++                || apm.gain.process_10ms(&mut microphone, false) != 0
 +            {
 +                return ptr::null_mut();
 +            }
@@ -1505,6 +1532,110 @@ index 00000000..a13fa9f4
 +    }
 +    0
 +}
++
++/// Apply AGC2 to one fixed frame. Returns 0=ok, 1=invalid input, 2=DSP failure.
++///
++/// # Safety
++/// `handle` is a live, exclusively accessed create result. `microphone` points
++/// to exactly 480 writable i16 samples, valid for this synchronous call.
++#[no_mangle]
++pub unsafe extern "C" fn livekit_ffi_realtime_apm_process_gain(
++    handle: *mut c_void,
++    microphone: *mut i16,
++    samples: usize,
++) -> i32 {
++    if handle.is_null()
++        || microphone.is_null()
++        || samples != 480
++        || microphone as usize % align_of::<i16>() != 0
++    {
++        return 1;
++    }
++    // SAFETY: The caller exclusively owns the live handle and writable fixed frame.
++    let apm = unsafe { &mut *handle.cast::<RealtimeApm>() };
++    // SAFETY: Validated dimensions/alignment and exclusive caller-owned storage.
++    let microphone = unsafe { &mut *microphone.cast::<[i16; 480]>() };
++    if apm.gain.process_10ms(microphone, false) != 0 {
++        return 2;
++    }
++    0
++}
++
++#[cfg(test)]
++mod tests {
++    use super::*;
++
++    #[test]
++    fn validates_input_before_processing() {
++        let handle = livekit_ffi_realtime_apm_create();
++        assert!(!handle.is_null());
++        let mut samples = [1234i16; 480];
++        let original = samples;
++        // SAFETY: The handle and frame remain valid and exclusively owned here.
++        unsafe {
++            assert_eq!(
++                livekit_ffi_realtime_apm_process(
++                    handle,
++                    samples.as_mut_ptr(),
++                    479,
++                    ptr::null(),
++                    0,
++                    false,
++                    false,
++                ),
++                1
++            );
++            assert_eq!(samples, original);
++            assert_eq!(
++                livekit_ffi_realtime_apm_process(
++                    handle,
++                    samples.as_mut_ptr(),
++                    480,
++                    ptr::null(),
++                    501,
++                    false,
++                    false,
++                ),
++                1
++            );
++            assert_eq!(samples, original);
++            assert_eq!(
++                livekit_ffi_realtime_apm_process(
++                    handle,
++                    samples.as_mut_ptr(),
++                    480,
++                    ptr::null(),
++                    0,
++                    false,
++                    true,
++                ),
++                0
++            );
++            assert_eq!(samples, original);
++            livekit_ffi_realtime_apm_destroy(handle);
++        }
++    }
++
++    #[test]
++    fn validates_input_before_gain() {
++        let handle = livekit_ffi_realtime_apm_create();
++        assert!(!handle.is_null());
++        let mut samples = [1234i16; 480];
++        let original = samples;
++        // SAFETY: The handle and frame remain valid and exclusively owned here.
++        unsafe {
++            assert_eq!(livekit_ffi_realtime_apm_process_gain(handle, samples.as_mut_ptr(), 479), 1);
++            assert_eq!(
++                livekit_ffi_realtime_apm_process_gain(ptr::null_mut(), samples.as_mut_ptr(), 480),
++                1
++            );
++            assert_eq!(livekit_ffi_realtime_apm_process_gain(handle, ptr::null_mut(), 480), 1);
++            assert_eq!(samples, original);
++            assert_eq!(livekit_ffi_realtime_apm_process_gain(handle, samples.as_mut_ptr(), 480), 0);
++            livekit_ffi_realtime_apm_destroy(handle);
++        }
++    }
++}
 diff --git a/livekit-ffi/src/server/audio_source.rs b/livekit-ffi/src/server/audio_source.rs
 index fd4f06cf..5c472e20 100644
 --- a/livekit-ffi/src/server/audio_source.rs

src/realtime_audio_processing.cpp

@@ -25,6 +25,7 @@ void livekit_ffi_realtime_apm_destroy(void* handle);
 int livekit_ffi_realtime_apm_process(void* handle, std::int16_t* microphone, std::size_t samples,
                                      const std::int16_t* reference, std::uint32_t delay_ms, bool noise_suppression,
                                      bool echo_cancellation);
+int livekit_ffi_realtime_apm_process_gain(void* handle, std::int16_t* microphone, std::size_t samples);
 }
 
 namespace livekit {
@@ -54,4 +55,8 @@ RealtimeAudioProcessingResult RealtimeAudioProcessing::process(std::array<std::i
                                                    reference ? reference->data() : nullptr, stream_delay_ms,
                                                    noise_suppression, echo_cancellation));
 }
+RealtimeAudioProcessingResult RealtimeAudioProcessing::processGain(std::array<std::int16_t, 480>& microphone) noexcept {
+  if (owner_ != std::this_thread::get_id()) return RealtimeAudioProcessingResult::wrong_thread;
+  return resultOf(livekit_ffi_realtime_apm_process_gain(state_.get(), microphone.data(), microphone.size()));
+}
 } // namespace livekit

src/tests/unit/test_audio_processing_module.cpp

@@ -897,7 +897,7 @@ TEST_F(AudioProcessingModuleTest, RealtimeBypassAndOwnerContract) {
   EXPECT_EQ(processor.process(samples, nullptr, 501, false, false), RealtimeAudioProcessingResult::invalid_input);
   EXPECT_EQ(samples, original);
   auto result = RealtimeAudioProcessingResult::ok;
-  std::thread other([&] { result = processor.process(samples, nullptr, 0, false, false); });
+  std::thread other([&] { result = processor.processGain(samples); });
   other.join();
   EXPECT_EQ(result, RealtimeAudioProcessingResult::wrong_thread);
   EXPECT_EQ(samples, original);
@@ -951,4 +951,48 @@ TEST_F(AudioProcessingModuleTest, RealtimeSyntheticEcho) {
   EXPECT_GE(erle_db, 10.0) << "AEC must suppress the delayed reference independently of NS";
 }
 
+TEST_F(AudioProcessingModuleTest, RealtimeGainRaisesQuietSpeech) {
+  RealtimeAudioProcessing processor;
+  double input_energy = 0;
+  double output_energy = 0;
+  int peak = 0;
+  for (int frame = 0; frame < 600; ++frame) {
+    std::array<std::int16_t, 480> samples{};
+    for (std::size_t index = 0; index < samples.size(); ++index) {
+      const double time = (frame * samples.size() + index) / 48000.0;
+      const double envelope = 0.6 + 0.4 * std::sin(2.0 * M_PI * 3.0 * time);
+      const double voiced = 0.5 * std::sin(2.0 * M_PI * 150.0 * time) + 0.3 * std::sin(2.0 * M_PI * 450.0 * time) +
+                            0.2 * std::sin(2.0 * M_PI * 750.0 * time);
+      samples[index] = static_cast<std::int16_t>(700.0 * envelope * voiced);
+      if (frame >= 400) input_energy += static_cast<double>(samples[index]) * samples[index];
+    }
+    ASSERT_EQ(processor.processGain(samples), RealtimeAudioProcessingResult::ok);
+    for (const auto sample : samples) {
+      peak = std::max(peak, std::abs(static_cast<int>(sample)));
+      if (frame >= 400) output_energy += static_cast<double>(sample) * sample;
+    }
+  }
+  EXPECT_GT(output_energy, input_energy * 4.0) << "AGC2 should raise quiet speech by at least 6 dB after adaptation";
+  EXPECT_LT(peak, 32767) << "AGC2 must not clip the quiet input";
+}
+
+TEST_F(AudioProcessingModuleTest, RealtimeBypassDoesNotAdvanceGain) {
+  RealtimeAudioProcessing gain_only;
+  RealtimeAudioProcessing bypass_then_gain;
+  std::uint32_t seed = 123;
+  for (int frame = 0; frame < 300; ++frame) {
+    std::array<std::int16_t, 480> samples{};
+    for (auto& sample : samples) {
+      seed = seed * 1664525 + 1013904223;
+      sample = static_cast<std::int16_t>(static_cast<int>(seed >> 20) - 2048);
+    }
+    auto bypass_samples = samples;
+    ASSERT_EQ(bypass_then_gain.process(bypass_samples, nullptr, 0, false, false), RealtimeAudioProcessingResult::ok);
+    EXPECT_EQ(bypass_samples, samples) << "AEC/NS bypass must not apply gain";
+    ASSERT_EQ(gain_only.processGain(samples), RealtimeAudioProcessingResult::ok);
+    ASSERT_EQ(bypass_then_gain.processGain(bypass_samples), RealtimeAudioProcessingResult::ok);
+    EXPECT_EQ(bypass_samples, samples) << "AEC/NS processing must not advance AGC2 state";
+  }
+}
+
 } // namespace livekit::test