diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.cpp b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.cpp index a29e0f7f7a..c3c76c33af 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.cpp +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.cpp @@ -17,8 +17,7 @@ #include "CommonFramework/AudioPipeline/Spectrum/FFTStreamer.h" #include "Kernels/AudioPipeline/Kernels_RawAudio2WorkingAudio.h" #include "Kernels/AudioPipeline/Kernels_WorkingAudio2RawAudio.h" -#include "Kernels/AudioPipeline/Kernels_ChannelMixer.h" -#include "Kernels/AudioResampler/Kernels_AudioResampler.h" +#include "Kernels/AudioPipeline/Kernels_WorkingAudio2WorkingAudio.h" #include #if QT_VERSION_MAJOR == 5 #include @@ -155,8 +154,7 @@ void AudioPassthroughPairQt::teardown_tree(){ } m_fft_listener.reset(); m_fft_node.reset(); - m_fft_mixer.reset(); - m_fft_resampler.reset(); + m_fft_conv.reset(); teardown_sink_branch(); m_source_decoder.reset(); } @@ -164,10 +162,8 @@ void AudioPassthroughPairQt::teardown_tree(){ void AudioPassthroughPairQt::teardown_sink_branch(){ if (m_source_decoder){ - if (m_sink_mixer){ - m_source_decoder->remove_listener(*m_sink_mixer); - }else if (m_sink_resampler){ - m_source_decoder->remove_listener(*m_sink_resampler); + if (m_sink_conv){ + m_source_decoder->remove_listener(*m_sink_conv); }else if (m_sink_encoder){ m_source_decoder->remove_listener(*m_sink_encoder); } @@ -177,8 +173,7 @@ void AudioPassthroughPairQt::teardown_sink_branch(){ if (m_sink_encoder && m_sink_device){ m_sink_encoder->remove_listener(*m_sink_device); } - m_sink_mixer.reset(); - m_sink_resampler.reset(); + m_sink_conv.reset(); m_sink_encoder.reset(); } @@ -205,33 +200,22 @@ void AudioPassthroughPairQt::build_tree(float input_volume){ // in m_fft_node, the tree's FFT endpoint. const size_t fft_rate = 48000; m_fft_node.reset(new WorkingAudioToFFT(fft_rate)); - Kernels::AudioPipeline::WorkingAudioStreamListener* head = m_fft_node.get(); - if (raw_rate != fft_rate){ - m_fft_resampler.reset( - new Kernels::AudioResampler::AudioResampler( - m_logger, - raw_rate, - fft_rate, - 1 - ) - ); - m_fft_resampler->add_listener(*head); - head = m_fft_resampler.get(); - } - if (raw_channels != 1){ - auto matrix = build_channel_mixer_matrix( + if (raw_rate == fft_rate && raw_channels == 1){ + // No conversion needed: feed the FFT node directly. + m_source_decoder->add_listener(*m_fft_node); + }else{ + std::vector matrix = build_channel_mixer_matrix( channel_format(raw_channels), channel_format(1) ); - if (matrix.empty()){ + if (matrix.size() != raw_channels){ throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "No channel mixer matrix."); } - m_fft_mixer.reset(new Kernels::AudioPipeline::ChannelMixer( - raw_rate, raw_channels, 1, std::move(matrix) + m_fft_conv.reset(new Kernels::AudioPipeline::WorkingAudio2WorkingAudio( + m_logger, raw_rate, fft_rate, raw_channels, 1, std::move(matrix) )); - m_fft_mixer->add_listener(*head); - head = m_fft_mixer.get(); + m_fft_conv->add_listener(*m_fft_node); + m_source_decoder->add_listener(*m_fft_conv); } - m_source_decoder->add_listener(*head); // The sink branch may have been created by the init_audio_sink() call // that ran earlier in the same locked scope (which found no decoder @@ -264,23 +248,11 @@ void AudioPassthroughPairQt::rebuild_sink_branch(){ // so it runs before the FFT head. teardown_sink_branch(); - Kernels::AudioPipeline::WorkingAudioStreamListener* head = nullptr; m_sink_encoder.reset(new Kernels::AudioPipeline::WorkingAudio2RawAudio( sink_sample_format, sink_rate, sink_channels, convert_audio_volume_from_slider(m_output_volume) )); - head = m_sink_encoder.get(); - if (raw_rate != sink_rate){ - m_sink_resampler.reset(new Kernels::AudioResampler::AudioResampler( - m_logger, - raw_rate, - sink_rate, - sink_channels - )); - m_sink_resampler->add_listener(*head); - head = m_sink_resampler.get(); - } // INTERLEAVE_RL_96000 plays the card's channel 1 (the odd physical // samples) as the left channel; the input reinterpretation preserves // the physical order, so the sink branch swaps the two channels. @@ -293,7 +265,10 @@ void AudioPassthroughPairQt::rebuild_sink_branch(){ bool swap_channels = m_input_format == AudioChannelFormat::INTERLEAVE_RL_96000 && raw_channels == 2 && sink_channels == 2; - if (raw_channels != sink_channels || swap_channels){ + if (raw_rate == sink_rate && raw_channels == sink_channels && !swap_channels){ + // No conversion needed: feed the encoder directly. + m_source_decoder->push_front_listener(*m_sink_encoder); + }else{ std::vector matrix; if (swap_channels){ // out = [in 1, in 0]; weight(in i -> out o) at index i + in_ch * o. @@ -306,13 +281,12 @@ void AudioPassthroughPairQt::rebuild_sink_branch(){ if (matrix.size() != raw_channels * sink_channels){ throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "No channel mixer matrix."); } - m_sink_mixer.reset(new Kernels::AudioPipeline::ChannelMixer( - raw_rate, raw_channels, sink_channels, std::move(matrix) + m_sink_conv.reset(new Kernels::AudioPipeline::WorkingAudio2WorkingAudio( + m_logger, raw_rate, sink_rate, raw_channels, sink_channels, std::move(matrix) )); - m_sink_mixer->add_listener(*head); - head = m_sink_mixer.get(); + m_sink_conv->add_listener(*m_sink_encoder); + m_source_decoder->push_front_listener(*m_sink_conv); } - m_source_decoder->push_front_listener(*head); // The branch's open tail: raw bytes to the output device. m_sink_encoder->add_listener(*m_sink_device); diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.h b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.h index 6ce1b4560a..cf609419db 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.h +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.h @@ -26,10 +26,7 @@ namespace Kernels{ namespace AudioPipeline{ class RawAudio2WorkingAudio; class WorkingAudio2RawAudio; -class ChannelMixer; -} -namespace AudioResampler{ -class AudioResampler; +class WorkingAudio2WorkingAudio; } } @@ -106,11 +103,9 @@ class AudioPassthroughPairQt final : public QObject, public AudioPassthroughPair // channels reinterpreted from the physical 1ch x 96000 stream; // MONO_96000 keeps the physical shape.) std::unique_ptr m_source_decoder; - std::unique_ptr m_sink_mixer; // Source channels -> sink channels. - std::unique_ptr m_sink_resampler; + std::unique_ptr m_sink_conv; // Source shape -> sink shape (mix + resample). std::unique_ptr m_sink_encoder; - std::unique_ptr m_fft_mixer; // Source channels -> 1 channel. - std::unique_ptr m_fft_resampler; + std::unique_ptr m_fft_conv; // Source shape -> 48 kHz mono (mix + resample). std::unique_ptr m_fft_node; // The tree's FFT endpoint. AudioDeviceInfo m_output_device; diff --git a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp index bdc3d1fd45..a6b66e11ba 100644 --- a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp +++ b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_AudioPipeline_Tests.cpp @@ -6,6 +6,9 @@ #include #include +#include +#include +#include #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/PrettyPrint.h" #include "Common/Cpp/Containers/AlignedVector.tpp" @@ -16,6 +19,10 @@ #include "Kernels_WorkingAudio2RawAudio.h" #include "Kernels_AudioPipeline_RoundTrip_TestInfra.h" #include "Kernels_ChannelMixer_TestInfra.h" +#include "CommonFramework/AudioPipeline/ChannelMixer.h" +#include "Kernels/AudioResampler/Kernels_AudioResampler.h" +#include "Kernels_ChannelMixer.h" +#include "Kernels_WorkingAudio2WorkingAudio.h" namespace PokemonAutomation{ namespace Kernels{ @@ -310,6 +317,220 @@ class FloatGainLiveUpdateTest : public UnitTest{ +// +// WorkingAudio2WorkingAudio vs. the hand-wired chain of the same kernels. +// +// The all-in-one node orders its internals by channel count: +// - in_channels >= out_channels: mixer (at the input rate) -> resampler, +// - in_channels < out_channels: resampler (at the input channel count) -> +// mixer (at the output rate). +// Identical input is fed through the node and through that exact hand-wired +// chain, and the outputs are asserted bit-identical. The two orderings are +// mathematically equivalent (both are LTI - mixing and FIR resampling +// commute), so this pins the node's wiring, the shared matrix construction +// and the stride forwarding. +// +class Working2WorkingOrderTest : public UnitTest{ +public: + Working2WorkingOrderTest(size_t in_rate, size_t out_rate, size_t in_ch, size_t out_ch) + : UnitTest( + "Kernels::AudioPipeline - Working2WorkingOrder(" + + std::to_string(in_rate) + "->" + std::to_string(out_rate) + "Hz, " + + std::to_string(in_ch) + "ch->" + std::to_string(out_ch) + "ch)" + ) + , m_in_rate(in_rate) + , m_out_rate(out_rate) + , m_in_ch(in_ch) + , m_out_ch(out_ch) + {} + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + const size_t frames = 8192; + + // Planar input: channel c lives at [c * frames .. c * frames + frames). + AlignedVector input(m_in_ch * frames); + std::mt19937 generator(42); + std::uniform_real_distribution distribution(-1.0f, 1.0f); + for (size_t c = 0; c < m_in_ch; c++){ + for (size_t i = 0; i < frames; i++){ + input[c * frames + i] = distribution(generator); + } + } + + auto channel_format = [](size_t channels){ + QAudioFormat format; + format.setChannelCount((int)channels); + format.setChannelConfig( + channels == 1 + ? QAudioFormat::ChannelConfig::ChannelConfigMono + : QAudioFormat::ChannelConfig::ChannelConfigStereo + ); + return format; + }; + const QAudioFormat in_format = channel_format(m_in_ch); + const QAudioFormat out_format = channel_format(m_out_ch); + + // The node under test. + std::vector node_matrix = build_channel_mixer_matrix(in_format, out_format); + if (node_matrix.size() != m_in_ch * m_out_ch){ + return "No channel mixer matrix for this conversion."; + } + WorkingAudio2WorkingAudio node( + logger, m_in_rate, m_out_rate, m_in_ch, m_out_ch, node_matrix + ); + RecordingWorkingAudioListener node_output(m_out_rate, m_out_ch, frames * 2); + node.add_listener(node_output); + + // The hand-wired chain, in the node's internal ordering. + std::vector chain_matrix = build_channel_mixer_matrix(in_format, out_format); + std::unique_ptr chain_mixer; + std::unique_ptr chain_resampler; + WorkingAudioNode* chain_head = nullptr; + WorkingAudioNode* chain_tail = nullptr; + if (m_in_ch >= m_out_ch){ + chain_mixer.reset(new ChannelMixer( + m_in_rate, m_in_ch, m_out_ch, chain_matrix + )); + chain_head = chain_mixer.get(); + if (m_in_rate != m_out_rate){ + chain_resampler.reset(new AudioResampler::AudioResampler( + logger, m_in_rate, m_out_rate, m_out_ch + )); + chain_mixer->add_listener(*chain_resampler); + chain_tail = chain_resampler.get(); + }else{ + chain_tail = chain_mixer.get(); + } + }else{ + chain_mixer.reset(new ChannelMixer( + m_out_rate, m_in_ch, m_out_ch, chain_matrix + )); + chain_tail = chain_mixer.get(); + if (m_in_rate != m_out_rate){ + chain_resampler.reset(new AudioResampler::AudioResampler( + logger, m_in_rate, m_out_rate, m_in_ch + )); + chain_resampler->add_listener(*chain_tail); + chain_head = chain_resampler.get(); + }else{ + chain_head = chain_mixer.get(); + } + } + RecordingWorkingAudioListener chain_output(m_out_rate, m_out_ch, frames * 2); + chain_tail->add_listener(chain_output); + + // Feed both in two chunks, each with a padded stride (stride > + // samples exercises the channel-block layout handling). + size_t offset = 0; + const size_t chunk_sizes[] = {5000, frames - 5000}; + for (size_t chunk : chunk_sizes){ + AlignedVector padded(m_in_ch * (chunk + 3)); + for (size_t c = 0; c < m_in_ch; c++){ + for (size_t i = 0; i < chunk; i++){ + padded[c * (chunk + 3) + i] = input[c * frames + offset + i]; + } + } + node.push_samples(chunk, chunk + 3, padded.data()); + chain_head->push_samples(chunk, chunk + 3, padded.data()); + offset += chunk; + } + + const size_t written = node_output.frames_written(); + if (written == 0){ + return "No output was produced."; + } + if (written != chain_output.frames_written()){ + return "Frame count mismatch: node wrote " + std::to_string(written) + + " frames, chain wrote " + std::to_string(chain_output.frames_written()) + "."; + } + const auto& a = node_output.data(); + const auto& b = chain_output.data(); + const size_t count = written * m_out_ch; + if (std::memcmp(a.data(), b.data(), count * sizeof(float)) != 0){ + for (size_t i = 0; i < count; i++){ + if (a[i] != b[i]){ + return "Mismatch at " + std::to_string(i) + ": " + tostr_default(a[i]) + + " != " + tostr_default(b[i]) + "."; + } + } + return "Bit-level mismatch with equal values (sign of zero or NaN payload)."; + } + + return true; + } + + +private: + const size_t m_in_rate; + const size_t m_out_rate; + const size_t m_in_ch; + const size_t m_out_ch; +}; + + + +// +// ChannelMixer with pushes larger than its internal buffer. +// +// mix_channels() strides the output buffer at the buffer size, so +// push_samples() must chunk larger pushes internally. An identity mix +// must pass every sample through unchanged, regardless of how the +// push is split up. +// +class MixerOverBufferPushTest : public UnitTest{ +public: + MixerOverBufferPushTest(size_t channels, size_t samples) + : UnitTest( + "Kernels::AudioPipeline - MixerOverBufferPush(" + + std::to_string(channels) + "ch x " + std::to_string(samples) + ")" + ) + , m_channels(channels) + , m_samples(samples) + {} + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + std::vector matrix(m_channels * m_channels, 0.0f); + for (size_t c = 0; c < m_channels; c++){ + matrix[c + m_channels * c] = 1.0f; + } + + GuardPageBuffer input = random_float_input(m_samples * m_channels, 42); + + const size_t sample_rate = 44100; + ChannelMixer mixer(sample_rate, m_channels, m_channels, std::move(matrix)); + RecordingWorkingAudioListener listener(sample_rate, m_channels, m_samples); + mixer.add_listener(listener); + + // A single push larger than the mixer's internal buffer (4096 + 256). + mixer.push_samples(m_samples, m_samples, input.data()); + + if (listener.frames_written() != m_samples){ + return "Received " + std::to_string(listener.frames_written()) + " frames, expected " + + std::to_string(m_samples) + "."; + } + + for (size_t i = 0; i < m_samples; i++){ + for (size_t c = 0; c < m_channels; c++){ + float expected = input[c * m_samples + i]; + if (expected != listener.data()[i * m_channels + c]){ + return "Mismatch at (" + std::to_string(i) + ", " + std::to_string(c) + "): " + + tostr_default(expected) + " != " + + tostr_default(listener.data()[i * m_channels + c]) + "."; + } + } + } + + return true; + } + + +private: + const size_t m_channels; + const size_t m_samples; +}; + + + void add_tests(UnitTestDatabase& database){ // Raw -> Working -> Raw round-trip. for (size_t channels = 1; channels <= 4; channels++){ @@ -345,6 +566,20 @@ void add_tests(UnitTestDatabase& database){ // 1 -> 2 -> 1 averaging round-trip. database.add(1000); + + // Pushes larger than the mixer's internal buffer get chunked. + for (size_t channels = 1; channels <= 3; channels++){ + database.add(channels, 10000); + } + + // WorkingAudio2WorkingAudio vs. the hand-wired chain: both internal + // orderings (mix -> resample, resample -> mix), plus the mix-only and + // identity-matrix + resample cases. + database.add(44100, 48000, 2, 1); + database.add(44100, 48000, 1, 2); + database.add(48000, 48000, 2, 1); + database.add(48000, 48000, 1, 2); + database.add(48000, 44100, 2, 2); } diff --git a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_ChannelMixer.cpp b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_ChannelMixer.cpp index be07615ec9..d541b9ea02 100644 --- a/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_ChannelMixer.cpp +++ b/SerialPrograms/Source/Kernels/AudioPipeline/Kernels_ChannelMixer.cpp @@ -4,6 +4,7 @@ * */ +#include #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/Containers/AlignedVector.tpp" #include "Kernels/AudioChannelMixer/Kernels_AudioChannelMixer.h" @@ -47,16 +48,23 @@ void ChannelMixer::remove_listener(WorkingAudioStreamListener& listener){ } void ChannelMixer::push_samples(size_t samples, size_t stride, const float* data){ - AudioChannelMixer::mix_channels( - m_weight_matrix.data(), - samples, - out_channels(), m_buffer_size, m_buffer.data(), - in_channels(), stride, data - ); - m_listeners.run_method( - &WorkingAudioStreamListener::push_samples, - samples, m_buffer_size, m_buffer.data() - ); + // mix_channels() strides the output buffer at m_buffer_size, so it can + // mix at most m_buffer_size samples per call; chunk larger pushes. + while (samples > 0){ + const size_t block = std::min(samples, m_buffer_size); + AudioChannelMixer::mix_channels( + m_weight_matrix.data(), + block, + out_channels(), m_buffer_size, m_buffer.data(), + in_channels(), stride, data + ); + m_listeners.run_method( + &WorkingAudioStreamListener::push_samples, + block, m_buffer_size, m_buffer.data() + ); + data += block; + samples -= block; + } }