From 296aa0fe37d23200a26a73605468f83e3c4f3033 Mon Sep 17 00:00:00 2001 From: PA Dev Bot <344001113+pa-devbot@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:16:38 +0000 Subject: [PATCH] Audio pipeline: simplify the tree with WorkingAudio2WorkingAudio. The FFT and sink branches of AudioPassthroughPairQt used to hand-wire ChannelMixer and AudioResampler nodes per conversion need. They now build a single WorkingAudio2WorkingAudio node (mix + resample, in the order chosen by channel count) when a conversion is needed, and fall back to a direct connection when the source already matches the target shape (48 kHz mono for the FFT branch, the device's native shape for the sink branch), preserving today's no-conversion path bit-for-bit. The INTERLEAVE_RL_96000 swap keeps going through the node's weight matrix, exactly as the sink mixer did. Also hardens ChannelMixer::push_samples(): mix_channels() strides its output buffer at the buffer size, so pushes larger than the buffer previously corrupted adjacent channel regions and wrote past the end of the buffer. The decoder always chunked to its own buffer size, so the live tree never hit this, but the all-in-one node (and any future direct consumer) must not rely on that. push_samples() now chunks internally. Pinned by MixerOverBufferPushTest. Adds Working2WorkingOrderTest: feeds identical input through the node and through the hand-wired chain of the same kernels (in the node's internal ordering, mix -> resample or resample -> mix) and asserts bit-identical output, for both orderings, the mix-only cases, and the identity-matrix + resample case. Pushes deliberately exceed the mixer's buffer size to exercise the chunking. --- .../Backends/AudioPassthroughPairQt.cpp | 72 ++---- .../Backends/AudioPassthroughPairQt.h | 11 +- .../Kernels_AudioPipeline_Tests.cpp | 235 ++++++++++++++++++ .../AudioPipeline/Kernels_ChannelMixer.cpp | 28 ++- 4 files changed, 279 insertions(+), 67 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.cpp b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.cpp index a29e0f7f7a..c3c76c33af 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.cpp +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.cpp @@ -17,8 +17,7 @@ #include "CommonFramework/AudioPipeline/Spectrum/FFTStreamer.h" #include "Kernels/AudioPipeline/Kernels_RawAudio2WorkingAudio.h" #include "Kernels/AudioPipeline/Kernels_WorkingAudio2RawAudio.h" -#include "Kernels/AudioPipeline/Kernels_ChannelMixer.h" -#include "Kernels/AudioResampler/Kernels_AudioResampler.h" +#include "Kernels/AudioPipeline/Kernels_WorkingAudio2WorkingAudio.h" #include #if QT_VERSION_MAJOR == 5 #include @@ -155,8 +154,7 @@ void AudioPassthroughPairQt::teardown_tree(){ } m_fft_listener.reset(); m_fft_node.reset(); - m_fft_mixer.reset(); - m_fft_resampler.reset(); + m_fft_conv.reset(); teardown_sink_branch(); m_source_decoder.reset(); } @@ -164,10 +162,8 @@ void AudioPassthroughPairQt::teardown_tree(){ void AudioPassthroughPairQt::teardown_sink_branch(){ if (m_source_decoder){ - if (m_sink_mixer){ - m_source_decoder->remove_listener(*m_sink_mixer); - }else if (m_sink_resampler){ - m_source_decoder->remove_listener(*m_sink_resampler); + if (m_sink_conv){ + m_source_decoder->remove_listener(*m_sink_conv); }else if (m_sink_encoder){ m_source_decoder->remove_listener(*m_sink_encoder); } @@ -177,8 +173,7 @@ void AudioPassthroughPairQt::teardown_sink_branch(){ if (m_sink_encoder && m_sink_device){ m_sink_encoder->remove_listener(*m_sink_device); } - m_sink_mixer.reset(); - m_sink_resampler.reset(); + m_sink_conv.reset(); m_sink_encoder.reset(); } @@ -205,33 +200,22 @@ void AudioPassthroughPairQt::build_tree(float input_volume){ // in m_fft_node, the tree's FFT endpoint. const size_t fft_rate = 48000; m_fft_node.reset(new WorkingAudioToFFT(fft_rate)); - Kernels::AudioPipeline::WorkingAudioStreamListener* head = m_fft_node.get(); - if (raw_rate != fft_rate){ - m_fft_resampler.reset( - new Kernels::AudioResampler::AudioResampler( - m_logger, - raw_rate, - fft_rate, - 1 - ) - ); - m_fft_resampler->add_listener(*head); - head = m_fft_resampler.get(); - } - if (raw_channels != 1){ - auto matrix = build_channel_mixer_matrix( + if (raw_rate == fft_rate && raw_channels == 1){ + // No conversion needed: feed the FFT node directly. + m_source_decoder->add_listener(*m_fft_node); + }else{ + std::vector matrix = build_channel_mixer_matrix( channel_format(raw_channels), channel_format(1) ); - if (matrix.empty()){ + if (matrix.size() != raw_channels){ throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "No channel mixer matrix."); } - m_fft_mixer.reset(new Kernels::AudioPipeline::ChannelMixer( - raw_rate, raw_channels, 1, std::move(matrix) + m_fft_conv.reset(new Kernels::AudioPipeline::WorkingAudio2WorkingAudio( + m_logger, raw_rate, fft_rate, raw_channels, 1, std::move(matrix) )); - m_fft_mixer->add_listener(*head); - head = m_fft_mixer.get(); + m_fft_conv->add_listener(*m_fft_node); + m_source_decoder->add_listener(*m_fft_conv); } - m_source_decoder->add_listener(*head); // The sink branch may have been created by the init_audio_sink() call // that ran earlier in the same locked scope (which found no decoder @@ -264,23 +248,11 @@ void AudioPassthroughPairQt::rebuild_sink_branch(){ // so it runs before the FFT head. teardown_sink_branch(); - Kernels::AudioPipeline::WorkingAudioStreamListener* head = nullptr; m_sink_encoder.reset(new Kernels::AudioPipeline::WorkingAudio2RawAudio( sink_sample_format, sink_rate, sink_channels, convert_audio_volume_from_slider(m_output_volume) )); - head = m_sink_encoder.get(); - if (raw_rate != sink_rate){ - m_sink_resampler.reset(new Kernels::AudioResampler::AudioResampler( - m_logger, - raw_rate, - sink_rate, - sink_channels - )); - m_sink_resampler->add_listener(*head); - head = m_sink_resampler.get(); - } // INTERLEAVE_RL_96000 plays the card's channel 1 (the odd physical // samples) as the left channel; the input reinterpretation preserves // the physical order, so the sink branch swaps the two channels. @@ -293,7 +265,10 @@ void AudioPassthroughPairQt::rebuild_sink_branch(){ bool swap_channels = m_input_format == AudioChannelFormat::INTERLEAVE_RL_96000 && raw_channels == 2 && sink_channels == 2; - if (raw_channels != sink_channels || swap_channels){ + if (raw_rate == sink_rate && raw_channels == sink_channels && !swap_channels){ + // No conversion needed: feed the encoder directly. + m_source_decoder->push_front_listener(*m_sink_encoder); + }else{ std::vector matrix; if (swap_channels){ // out = [in 1, in 0]; weight(in i -> out o) at index i + in_ch * o. @@ -306,13 +281,12 @@ void AudioPassthroughPairQt::rebuild_sink_branch(){ if (matrix.size() != raw_channels * sink_channels){ throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "No channel mixer matrix."); } - m_sink_mixer.reset(new Kernels::AudioPipeline::ChannelMixer( - raw_rate, raw_channels, sink_channels, std::move(matrix) + m_sink_conv.reset(new Kernels::AudioPipeline::WorkingAudio2WorkingAudio( + m_logger, raw_rate, sink_rate, raw_channels, sink_channels, std::move(matrix) )); - m_sink_mixer->add_listener(*head); - head = m_sink_mixer.get(); + m_sink_conv->add_listener(*m_sink_encoder); + m_source_decoder->push_front_listener(*m_sink_conv); } - m_source_decoder->push_front_listener(*head); // The branch's open tail: raw bytes to the output device. m_sink_encoder->add_listener(*m_sink_device); diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.h b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.h index 6ce1b4560a..cf609419db 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.h +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.h @@ -26,10 +26,7 @@ namespace Kernels{ namespace AudioPipeline{ class RawAudio2WorkingAudio; class WorkingAudio2RawAudio; -class ChannelMixer; -} -namespace AudioResampler{ -class AudioResampler; +class WorkingAudio2WorkingAudio; } } @@ -106,11 +103,9 @@ class AudioPassthroughPairQt final : public QObject, public AudioPassthroughPair // channels reinterpreted from the physical 1ch x 96000 stream; // MONO_96000 keeps the physical shape.) std::unique_ptr m_source_decoder; - std::unique_ptr m_sink_mixer; // Source channels -> sink channels. - std::unique_ptr m_sink_resampler; + std::unique_ptr m_sink_conv; // Source shape -> sink shape (mix + resample). std::unique_ptr m_sink_encoder; - std::unique_ptr m_fft_mixer; // Source channels -> 1 channel. - std::unique_ptr m_fft_resampler; + std::unique_ptr m_fft_conv; // Source shape -> 48 kHz mono (mix + resample). std::unique_ptr m_fft_node; // The tree's FFT endpoint. AudioDeviceInfo m_output_device; diff --git a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp index bdc3d1fd45..a6b66e11ba 100644 --- a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp +++ b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp @@ -6,6 +6,9 @@ #include #include +#include +#include +#include #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/PrettyPrint.h" #include "Common/Cpp/Containers/AlignedVector.tpp" @@ -16,6 +19,10 @@ #include "Kernels_WorkingAudio2RawAudio.h" #include "Kernels_AudioPipeline_RoundTrip_TestInfra.h" #include "Kernels_ChannelMixer_TestInfra.h" +#include "CommonFramework/AudioPipeline/ChannelMixer.h" +#include "Kernels/AudioResampler/Kernels_AudioResampler.h" +#include "Kernels_ChannelMixer.h" +#include "Kernels_WorkingAudio2WorkingAudio.h" namespace PokemonAutomation{ namespace Kernels{ @@ -310,6 +317,220 @@ class FloatGainLiveUpdateTest : public UnitTest{ +// +// WorkingAudio2WorkingAudio vs. the hand-wired chain of the same kernels. +// +// The all-in-one node orders its internals by channel count: +// - in_channels >= out_channels: mixer (at the input rate) -> resampler, +// - in_channels < out_channels: resampler (at the input channel count) -> +// mixer (at the output rate). +// Identical input is fed through the node and through that exact hand-wired +// chain, and the outputs are asserted bit-identical. The two orderings are +// mathematically equivalent (both are LTI - mixing and FIR resampling +// commute), so this pins the node's wiring, the shared matrix construction +// and the stride forwarding. +// +class Working2WorkingOrderTest : public UnitTest{ +public: + Working2WorkingOrderTest(size_t in_rate, size_t out_rate, size_t in_ch, size_t out_ch) + : UnitTest( + "Kernels::AudioPipeline - Working2WorkingOrder(" + + std::to_string(in_rate) + "->" + std::to_string(out_rate) + "Hz, " + + std::to_string(in_ch) + "ch->" + std::to_string(out_ch) + "ch)" + ) + , m_in_rate(in_rate) + , m_out_rate(out_rate) + , m_in_ch(in_ch) + , m_out_ch(out_ch) + {} + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + const size_t frames = 8192; + + // Planar input: channel c lives at [c * frames .. c * frames + frames). + AlignedVector input(m_in_ch * frames); + std::mt19937 generator(42); + std::uniform_real_distribution distribution(-1.0f, 1.0f); + for (size_t c = 0; c < m_in_ch; c++){ + for (size_t i = 0; i < frames; i++){ + input[c * frames + i] = distribution(generator); + } + } + + auto channel_format = [](size_t channels){ + QAudioFormat format; + format.setChannelCount((int)channels); + format.setChannelConfig( + channels == 1 + ? QAudioFormat::ChannelConfig::ChannelConfigMono + : QAudioFormat::ChannelConfig::ChannelConfigStereo + ); + return format; + }; + const QAudioFormat in_format = channel_format(m_in_ch); + const QAudioFormat out_format = channel_format(m_out_ch); + + // The node under test. + std::vector node_matrix = build_channel_mixer_matrix(in_format, out_format); + if (node_matrix.size() != m_in_ch * m_out_ch){ + return "No channel mixer matrix for this conversion."; + } + WorkingAudio2WorkingAudio node( + logger, m_in_rate, m_out_rate, m_in_ch, m_out_ch, node_matrix + ); + RecordingWorkingAudioListener node_output(m_out_rate, m_out_ch, frames * 2); + node.add_listener(node_output); + + // The hand-wired chain, in the node's internal ordering. + std::vector chain_matrix = build_channel_mixer_matrix(in_format, out_format); + std::unique_ptr chain_mixer; + std::unique_ptr chain_resampler; + WorkingAudioNode* chain_head = nullptr; + WorkingAudioNode* chain_tail = nullptr; + if (m_in_ch >= m_out_ch){ + chain_mixer.reset(new ChannelMixer( + m_in_rate, m_in_ch, m_out_ch, chain_matrix + )); + chain_head = chain_mixer.get(); + if (m_in_rate != m_out_rate){ + chain_resampler.reset(new AudioResampler::AudioResampler( + logger, m_in_rate, m_out_rate, m_out_ch + )); + chain_mixer->add_listener(*chain_resampler); + chain_tail = chain_resampler.get(); + }else{ + chain_tail = chain_mixer.get(); + } + }else{ + chain_mixer.reset(new ChannelMixer( + m_out_rate, m_in_ch, m_out_ch, chain_matrix + )); + chain_tail = chain_mixer.get(); + if (m_in_rate != m_out_rate){ + chain_resampler.reset(new AudioResampler::AudioResampler( + logger, m_in_rate, m_out_rate, m_in_ch + )); + chain_resampler->add_listener(*chain_tail); + chain_head = chain_resampler.get(); + }else{ + chain_head = chain_mixer.get(); + } + } + RecordingWorkingAudioListener chain_output(m_out_rate, m_out_ch, frames * 2); + chain_tail->add_listener(chain_output); + + // Feed both in two chunks, each with a padded stride (stride > + // samples exercises the channel-block layout handling). + size_t offset = 0; + const size_t chunk_sizes[] = {5000, frames - 5000}; + for (size_t chunk : chunk_sizes){ + AlignedVector padded(m_in_ch * (chunk + 3)); + for (size_t c = 0; c < m_in_ch; c++){ + for (size_t i = 0; i < chunk; i++){ + padded[c * (chunk + 3) + i] = input[c * frames + offset + i]; + } + } + node.push_samples(chunk, chunk + 3, padded.data()); + chain_head->push_samples(chunk, chunk + 3, padded.data()); + offset += chunk; + } + + const size_t written = node_output.frames_written(); + if (written == 0){ + return "No output was produced."; + } + if (written != chain_output.frames_written()){ + return "Frame count mismatch: node wrote " + std::to_string(written) + + " frames, chain wrote " + std::to_string(chain_output.frames_written()) + "."; + } + const auto& a = node_output.data(); + const auto& b = chain_output.data(); + const size_t count = written * m_out_ch; + if (std::memcmp(a.data(), b.data(), count * sizeof(float)) != 0){ + for (size_t i = 0; i < count; i++){ + if (a[i] != b[i]){ + return "Mismatch at " + std::to_string(i) + ": " + tostr_default(a[i]) + + " != " + tostr_default(b[i]) + "."; + } + } + return "Bit-level mismatch with equal values (sign of zero or NaN payload)."; + } + + return true; + } + + +private: + const size_t m_in_rate; + const size_t m_out_rate; + const size_t m_in_ch; + const size_t m_out_ch; +}; + + + +// +// ChannelMixer with pushes larger than its internal buffer. +// +// mix_channels() strides the output buffer at the buffer size, so +// push_samples() must chunk larger pushes internally. An identity mix +// must pass every sample through unchanged, regardless of how the +// push is split up. +// +class MixerOverBufferPushTest : public UnitTest{ +public: + MixerOverBufferPushTest(size_t channels, size_t samples) + : UnitTest( + "Kernels::AudioPipeline - MixerOverBufferPush(" + + std::to_string(channels) + "ch x " + std::to_string(samples) + ")" + ) + , m_channels(channels) + , m_samples(samples) + {} + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + std::vector matrix(m_channels * m_channels, 0.0f); + for (size_t c = 0; c < m_channels; c++){ + matrix[c + m_channels * c] = 1.0f; + } + + GuardPageBuffer input = random_float_input(m_samples * m_channels, 42); + + const size_t sample_rate = 44100; + ChannelMixer mixer(sample_rate, m_channels, m_channels, std::move(matrix)); + RecordingWorkingAudioListener listener(sample_rate, m_channels, m_samples); + mixer.add_listener(listener); + + // A single push larger than the mixer's internal buffer (4096 + 256). + mixer.push_samples(m_samples, m_samples, input.data()); + + if (listener.frames_written() != m_samples){ + return "Received " + std::to_string(listener.frames_written()) + " frames, expected " + + std::to_string(m_samples) + "."; + } + + for (size_t i = 0; i < m_samples; i++){ + for (size_t c = 0; c < m_channels; c++){ + float expected = input[c * m_samples + i]; + if (expected != listener.data()[i * m_channels + c]){ + return "Mismatch at (" + std::to_string(i) + ", " + std::to_string(c) + "): " + + tostr_default(expected) + " != " + + tostr_default(listener.data()[i * m_channels + c]) + "."; + } + } + } + + return true; + } + + +private: + const size_t m_channels; + const size_t m_samples; +}; + + + void add_tests(UnitTestDatabase& database){ // Raw -> Working -> Raw round-trip. for (size_t channels = 1; channels <= 4; channels++){ @@ -345,6 +566,20 @@ void add_tests(UnitTestDatabase& database){ // 1 -> 2 -> 1 averaging round-trip. database.add(1000); + + // Pushes larger than the mixer's internal buffer get chunked. + for (size_t channels = 1; channels <= 3; channels++){ + database.add(channels, 10000); + } + + // WorkingAudio2WorkingAudio vs. the hand-wired chain: both internal + // orderings (mix -> resample, resample -> mix), plus the mix-only and + // identity-matrix + resample cases. + database.add(44100, 48000, 2, 1); + database.add(44100, 48000, 1, 2); + database.add(48000, 48000, 2, 1); + database.add(48000, 48000, 1, 2); + database.add(48000, 44100, 2, 2); } diff --git a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_ChannelMixer.cpp b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_ChannelMixer.cpp index be07615ec9..d541b9ea02 100644 --- a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_ChannelMixer.cpp +++ b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_ChannelMixer.cpp @@ -4,6 +4,7 @@ * */ +#include #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/Containers/AlignedVector.tpp" #include "Kernels/AudioChannelMixer/Kernels_AudioChannelMixer.h" @@ -47,16 +48,23 @@ void ChannelMixer::remove_listener(WorkingAudioStreamListener& listener){ } void ChannelMixer::push_samples(size_t samples, size_t stride, const float* data){ - AudioChannelMixer::mix_channels( - m_weight_matrix.data(), - samples, - out_channels(), m_buffer_size, m_buffer.data(), - in_channels(), stride, data - ); - m_listeners.run_method( - &WorkingAudioStreamListener::push_samples, - samples, m_buffer_size, m_buffer.data() - ); + // mix_channels() strides the output buffer at m_buffer_size, so it can + // mix at most m_buffer_size samples per call; chunk larger pushes. + while (samples > 0){ + const size_t block = std::min(samples, m_buffer_size); + AudioChannelMixer::mix_channels( + m_weight_matrix.data(), + block, + out_channels(), m_buffer_size, m_buffer.data(), + in_channels(), stride, data + ); + m_listeners.run_method( + &WorkingAudioStreamListener::push_samples, + block, m_buffer_size, m_buffer.data() + ); + data += block; + samples -= block; + } }