From 1000f49416a3f9551e1fa5c3bce4b00ef2b79d97 Mon Sep 17 00:00:00 2001 From: PA Dev Bot <344001113+pa-devbot@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:26:05 +0000 Subject: [PATCH 1/2] Audio pipeline: rebase the working-stream path onto the new node API. Rebased #1671 onto main (incl. #1674's AudioFormat / WorkingAudio2WorkingAudio and #1676's tree simplification), and rewrote the attach path around the per-listener conversion node: - AudioPassthroughPair gains add_listener(const AudioFormat&, WorkingAudioStreamListener&) / remove_listener(WorkingAudioStreamListener&). The Qt pair builds one WorkingAudio2WorkingAudio per attached listener (source shape -> the requested shape), plus a small forwarder that carries the requested shape as its nominal one (the node validates its listeners against that; the user's listener may have any nominal shape of its own). Entries survive tree rebuilds: the nodes are rebuilt for each new source shape in build_tree() and detached in teardown_tree(). The INTERLEAVE_RL_96000 channel swap is applied to the node's matrix, the same way as for the sink branch. Unresolvable layouts are logged and skipped rather than thrown from the calling thread. - StreamHistorySession now implements both listener interfaces: the legacy AudioFloatStreamListener::on_samples path stays exactly as it was (kept until #1563 switches off it), and the new WorkingAudioStreamListener::push_samples re-interleaves the planar working stream at the active preset's physical shape (the shape its tracker records at; the RL swap already applied by the pair's node). The session attaches at start() and re-attaches in post_input_change(), removing the old-shape attach first so no push in the old shape can ever race the shape update. While both paths are attached the tracker is fed by both (double-record) - inert in the active tracker, and #1563 drops the legacy attach when it switches. - The console wires it up: set_audio_session() before start(), and the working-listener remove in try_shutdown(). Verified: -Werror clean on aarch64; full command-line suite green. --- .../AudioPipeline/AudioPassthroughPair.h | 18 ++- .../AudioPipeline/AudioSession.cpp | 8 ++ .../AudioPipeline/AudioSession.h | 5 + .../Backends/AudioPassthroughPairQt.cpp | 128 ++++++++++++++++++ .../Backends/AudioPassthroughPairQt.h | 26 ++++ .../Backends/AudioPassthroughPairQtThread.cpp | 12 ++ .../Backends/AudioPassthroughPairQtThread.h | 3 + .../Recording/StreamHistorySession.cpp | 127 +++++++++++++++-- .../Recording/StreamHistorySession.h | 16 +++ .../Framework/ConsoleSystemSession.cpp | 2 + 10 files changed, 331 insertions(+), 14 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioPassthroughPair.h b/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioPassthroughPair.h index c0b4a2bd99..ac719a792f 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioPassthroughPair.h +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioPassthroughPair.h @@ -14,11 +14,18 @@ namespace PokemonAutomation{ struct AudioFloatStreamListener; struct FFTListener; +namespace Kernels{ +namespace AudioPipeline{ +class WorkingAudioStreamListener; +} +} + // Represents an audio source/sink pair where audio is passed through from // source -> sink with minimal latency. // -// If desired, listeners can be attached to receive the FFT spectrums. +// If desired, listeners can be attached to receive the FFT spectrums, or to +// receive the working audio stream converted to a requested shape. // // This class is fully thread-safe. Both source and sink can be changed // asynchronously. Listeners can be attached/detached asynchronously. @@ -31,6 +38,15 @@ class AudioPassthroughPair{ virtual void add_listener(FFTListener& listener) = 0; virtual void remove_listener(FFTListener& listener) = 0; + // Attach a listener that receives the working audio stream converted to + // "format" (sample rate + channel count; the channel layout in + // format.native_format() drives the channel mixing). The pair builds a + // conversion node per listener; the listener receives planar float + // samples in the requested shape, with the input volume applied. + // Re-adding a listener (e.g. with a new format) rebuilds its node. + virtual void add_listener(const AudioFormat& format, Kernels::AudioPipeline::WorkingAudioStreamListener& listener) = 0; + virtual void remove_listener(Kernels::AudioPipeline::WorkingAudioStreamListener& listener) = 0; + public: virtual ~AudioPassthroughPair() = default; diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioSession.cpp b/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioSession.cpp index 5483a94c9d..7478d1ed4c 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioSession.cpp +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioSession.cpp @@ -34,6 +34,14 @@ void AudioSession::add_stream_listener(AudioFloatStreamListener& listener){ void AudioSession::remove_stream_listener(AudioFloatStreamListener& listener){ m_devices->remove_listener(listener); } +void AudioSession::add_working_stream_listener( + const AudioFormat& format, Kernels::AudioPipeline::WorkingAudioStreamListener& listener +){ + m_devices->add_listener(format, listener); +} +void AudioSession::remove_working_stream_listener(Kernels::AudioPipeline::WorkingAudioStreamListener& listener){ + m_devices->remove_listener(listener); +} void AudioSession::add_spectrum_listener(AudioSpectrumHolder::Listener& listener){ m_spectrum_holder.add_listener(listener); } diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioSession.h b/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioSession.h index 73fd083920..da47606eff 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioSession.h +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioSession.h @@ -45,6 +45,11 @@ class AudioSession final : public AudioFeed, private FFTListener, private Watchd void add_stream_listener(AudioFloatStreamListener& listener); void remove_stream_listener(AudioFloatStreamListener& listener); + // Attach a listener that receives the working audio stream (float) + // converted to the requested format. See AudioPassthroughPair. + void add_working_stream_listener(const AudioFormat& format, Kernels::AudioPipeline::WorkingAudioStreamListener& listener); + void remove_working_stream_listener(Kernels::AudioPipeline::WorkingAudioStreamListener& listener); + void add_spectrum_listener(AudioSpectrumHolder::Listener& listener); void remove_spectrum_listener(AudioSpectrumHolder::Listener& listener); diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.cpp b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.cpp index c3c76c33af..a8ee89eff6 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.cpp +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.cpp @@ -5,6 +5,7 @@ */ #include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/Containers/Pimpl.tpp" #include "Common/Cpp/Logging/AbstractLogger.h" #include "Common/Cpp/PrettyPrint.h" #include "CommonFramework/GlobalSettingsPanel.h" @@ -86,6 +87,25 @@ void AudioPassthroughPairQt::remove_listener(FFTListener& listener){ WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); m_fft_listeners.remove(listener); } +void AudioPassthroughPairQt::add_listener( + const AudioFormat& format, Kernels::AudioPipeline::WorkingAudioStreamListener& listener +){ + auto scope_check = m_sanitizer.check_scope(); + WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); + WorkingListenerEntry& entry = m_working_entries[&listener]; + entry.m_format = format; + build_working_entry(entry, listener); +} +void AudioPassthroughPairQt::remove_listener(Kernels::AudioPipeline::WorkingAudioStreamListener& listener){ + auto scope_check = m_sanitizer.check_scope(); + WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); + auto iter = m_working_entries.find(&listener); + if (iter == m_working_entries.end()){ + return; + } + detach_working_entry(iter->second); + m_working_entries.erase(iter); +} @@ -133,6 +153,105 @@ class AudioPassthroughPairQt::InternalFFTListener final : public FFTListener{ }; +// +// Per-listener working-stream conversion. +// +// Each attached working-stream listener gets its own +// WorkingAudio2WorkingAudio node (source shape -> the listener's requested +// shape) attached to the source decoder, plus an InternalWorkingListener +// forwarder that carries the requested shape as its nominal one (the node +// validates its listeners against it; the user's listener may have any +// nominal shape of its own). +// +// The entries in m_working_entries survive tree rebuilds: when the input +// changes, build_tree() rebuilds each node for the new source shape, and +// teardown_tree() detaches them. All of these methods require the caller +// to hold m_lock. Detaching blocks on in-flight pushes, so no push can +// ever run through a node or forwarder that is about to be destroyed. +// + +class AudioPassthroughPairQt::InternalWorkingListener final : public Kernels::AudioPipeline::WorkingAudioStreamListener{ +public: + InternalWorkingListener( + Kernels::AudioPipeline::WorkingAudioStreamListener& target, + size_t sample_rate, size_t channels + ) + : Kernels::AudioPipeline::WorkingAudioStreamListener(sample_rate, channels) + , m_target(target) + {} + // The pair always detaches this from its node before destroying it + // (detach_working_entry). "m_target" is the user's listener, owned by + // the caller; it outlives the attach by contract. + virtual void push_samples(size_t samples, size_t stride, const float* data) override{ + m_target.push_samples(samples, stride, data); + } + +private: + Kernels::AudioPipeline::WorkingAudioStreamListener& m_target; +}; + + +void AudioPassthroughPairQt::build_working_entry( + WorkingListenerEntry& entry, Kernels::AudioPipeline::WorkingAudioStreamListener& listener +){ + if (!m_source_decoder || m_input_format == AudioChannelFormat::NONE){ + return; + } + // Re-adding with a new format: detach the old node first. + if (entry.m_node){ + detach_working_entry(entry); + } + + const size_t raw_rate = m_reader->raw_sample_rate(); + const size_t raw_channels = m_reader->raw_channels(); + const size_t out_rate = entry.m_format.sample_rate(); + const size_t out_channels = entry.m_format.channels(); + if (out_rate == 0 || out_channels == 0){ + m_logger.log("Working listener: invalid requested format; not attached.", COLOR_RED); + return; + } + + // INTERLEAVE_RL_96000 plays the card's channel 1 (the odd physical + // samples) as the left channel; the input reinterpretation preserves + // the physical order, so the swap is applied to the matrix (the same + // special case the sink branch uses). + std::vector matrix; + if (m_input_format == AudioChannelFormat::INTERLEAVE_RL_96000 + && raw_channels == 2 && out_channels == 2){ + // out = [in 1, in 0]; weight(in i -> out o) at index i + in_ch * o. + matrix = {0.0f, 1.0f, 1.0f, 0.0f}; + }else{ + matrix = build_channel_mixer_matrix( + channel_format(raw_channels), entry.m_format.native_format() + ); + } + if (matrix.size() != raw_channels * out_channels){ + m_logger.log("Working listener: no channel mixer matrix for this conversion; not attached.", COLOR_RED); + return; + } + + entry.m_node.reset(new Kernels::AudioPipeline::WorkingAudio2WorkingAudio( + m_logger, raw_rate, out_rate, raw_channels, out_channels, std::move(matrix) + )); + entry.m_forwarder.reset(new InternalWorkingListener(listener, out_rate, out_channels)); + entry.m_node->add_listener(*entry.m_forwarder); + m_source_decoder->add_listener(*entry.m_node); +} + + +void AudioPassthroughPairQt::detach_working_entry(WorkingListenerEntry& entry){ + if (entry.m_forwarder && entry.m_node){ + entry.m_node->remove_listener(*entry.m_forwarder); + } + if (entry.m_node && m_source_decoder){ + m_source_decoder->remove_listener(*entry.m_node); + } + entry.m_forwarder.reset(); + entry.m_node.reset(); +} + + + // // The new pipeline tree. // @@ -152,6 +271,9 @@ void AudioPassthroughPairQt::teardown_tree(){ if (m_source_decoder){ m_reader->remove_raw_listener(*m_source_decoder); } + for (auto& entry : m_working_entries){ + detach_working_entry(entry.second); + } m_fft_listener.reset(); m_fft_node.reset(); m_fft_conv.reset(); @@ -194,6 +316,12 @@ void AudioPassthroughPairQt::build_tree(float input_volume){ )); m_reader->add_raw_listener(*m_source_decoder); + // Rebuild the per-listener working-stream conversion nodes for the new + // source shape (the entries themselves survive the rebuild). + for (auto& entry : m_working_entries){ + build_working_entry(entry.second, *entry.first); + } + // FFT branch: source channels -> 1 channel, source rate -> FFT rate. // The tree runs the FFT at 48 kHz for all inputs (the post-migration // standard); a 44.1 kHz source is resampled up here. The branch ends diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.h b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.h index cf609419db..a4ad051e93 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.h +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.h @@ -8,6 +8,7 @@ #define PokemonAutomation_AudioPipeline_AudioPassthroughPairQt_H #include +#include //#include #include #include "Common/Cpp/ListenerSet.h" @@ -27,6 +28,7 @@ namespace AudioPipeline{ class RawAudio2WorkingAudio; class WorkingAudio2RawAudio; class WorkingAudio2WorkingAudio; +class WorkingAudioStreamListener; } } @@ -39,6 +41,9 @@ class AudioPassthroughPairQt final : public QObject, public AudioPassthroughPair virtual void add_listener(FFTListener& listener) override; virtual void remove_listener(FFTListener& listener) override; + virtual void add_listener(const AudioFormat& format, Kernels::AudioPipeline::WorkingAudioStreamListener& listener) override; + virtual void remove_listener(Kernels::AudioPipeline::WorkingAudioStreamListener& listener) override; + public: virtual ~AudioPassthroughPairQt(); @@ -66,6 +71,18 @@ class AudioPassthroughPairQt final : public QObject, public AudioPassthroughPair private: class SampleListener; class InternalFFTListener; + class InternalWorkingListener; + + // One entry per attached working-stream listener: the requested format + // plus the conversion node (source shape -> requested shape) and the + // forwarder that carries the requested shape as its nominal one. The + // node attaches to m_source_decoder; the forwarder attaches to the node. + // All of these are only touched while holding m_lock. + struct WorkingListenerEntry{ + AudioFormat m_format; + std::unique_ptr m_node; + std::unique_ptr m_forwarder; + }; void init_audio_sink(); @@ -82,6 +99,9 @@ class AudioPassthroughPairQt final : public QObject, public AudioPassthroughPair void build_tree(float input_volume); void rebuild_sink_branch(); + void build_working_entry(WorkingListenerEntry& entry, Kernels::AudioPipeline::WorkingAudioStreamListener& listener); + void detach_working_entry(WorkingListenerEntry& entry); + private: Logger& m_logger; @@ -108,6 +128,12 @@ class AudioPassthroughPairQt final : public QObject, public AudioPassthroughPair std::unique_ptr m_fft_conv; // Source shape -> 48 kHz mono (mix + resample). std::unique_ptr m_fft_node; // The tree's FFT endpoint. + // Per-listener working-stream conversion nodes. The entries survive + // tree rebuilds (the listeners stay attached across input changes); + // their nodes are rebuilt for each new source shape in build_tree() + // and detached in teardown_tree(). + std::map m_working_entries; + AudioDeviceInfo m_output_device; float m_file_input_multiplier = 1.0; float m_device_input_multiplier = 1.0; diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQtThread.cpp b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQtThread.cpp index b4a31a9520..a42dd6fbe1 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQtThread.cpp +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQtThread.cpp @@ -35,6 +35,18 @@ void AudioPassthroughPairQtThread::remove_listener(FFTListener& listener){ AudioPassthroughPairQt* body = m_body; body->remove_listener(listener); } +void AudioPassthroughPairQtThread::add_listener( + const AudioFormat& format, Kernels::AudioPipeline::WorkingAudioStreamListener& listener +){ + auto scope_check = m_sanitizer.check_scope(); + AudioPassthroughPairQt* body = m_body; + body->add_listener(format, listener); +} +void AudioPassthroughPairQtThread::remove_listener(Kernels::AudioPipeline::WorkingAudioStreamListener& listener){ + auto scope_check = m_sanitizer.check_scope(); + AudioPassthroughPairQt* body = m_body; + body->remove_listener(listener); +} AudioPassthroughPairQtThread::AudioPassthroughPairQtThread(Logger& logger) diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQtThread.h b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQtThread.h index baa4f18cf9..24fc6ef29a 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQtThread.h +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQtThread.h @@ -27,6 +27,9 @@ class AudioPassthroughPairQtThread : public AudioPassthroughPair{ virtual void add_listener(FFTListener& listener) override; virtual void remove_listener(FFTListener& listener) override; + virtual void add_listener(const AudioFormat& format, Kernels::AudioPipeline::WorkingAudioStreamListener& listener) override; + virtual void remove_listener(Kernels::AudioPipeline::WorkingAudioStreamListener& listener) override; + public: AudioPassthroughPairQtThread(Logger& logger); ~AudioPassthroughPairQtThread(); diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp index a692d6f7c6..0ed1f96414 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp @@ -4,10 +4,13 @@ * */ +#include +#include #include "Common/Cpp/Exceptions.h" -#include "Common/Cpp/Containers/Pimpl.tpp" +#include "Common/Cpp/Containers/Pimpl.tpp" // Pimpl (AudioFormat) instantiations #include "Common/Cpp/Concurrency/SpinLock.h" #include "CommonFramework/GlobalSettingsPanel.h" +#include "CommonFramework/AudioPipeline/AudioInfo.h" #include "CommonFramework/VideoPipeline/Backends/VideoFrameQt.h" #include "CommonFramework/Recording/StreamHistoryOption.h" @@ -36,6 +39,14 @@ struct StreamHistorySession::Data{ std::chrono::seconds m_window; AudioChannelFormat m_audio_format; bool m_has_video; + // The active input's physical working-stream shape (what the tree + // delivers through the working listener, and what the tracker records + // at). Set in initialize(). + size_t m_active_channels = 0; + size_t m_active_rate = 0; + std::vector m_interleaved; + // The audio session the working listener attaches to (set_audio_session). + AudioSession* m_audio_session = nullptr; std::shared_ptr m_current; Data(Logger& logger) @@ -51,17 +62,29 @@ struct StreamHistorySession::Data{ StreamHistorySession::~StreamHistorySession() = default; StreamHistorySession::StreamHistorySession(Logger& logger) + // The working-stream nominal shape is metadata only: the pair's + // per-listener forwarder carries the shape actually requested, and + // nothing validates this one. : AudioFloatStreamListener(1) + , Kernels::AudioPipeline::WorkingAudioStreamListener(48000, 2) , m_data(CONSTRUCT_TOKEN, logger) {} void StreamHistorySession::start(AudioChannelFormat format, bool has_video){ Data& data = *m_data; - WriteSpinLock lg(data.m_lock); - if (!data.m_current){ - data.m_audio_format = format; - data.m_has_video = has_video; - initialize(); + { + WriteSpinLock lg(data.m_lock); + if (!data.m_current){ + data.m_audio_format = format; + data.m_has_video = has_video; + initialize(); + } } + attach_working_listener(); +} +void StreamHistorySession::set_audio_session(AudioSession& session){ + Data& data = *m_data; + WriteSpinLock lg(data.m_lock); + data.m_audio_session = &session; } @@ -98,6 +121,27 @@ void StreamHistorySession::on_samples(const float* samples, size_t frames){ data.m_current->on_samples(samples, frames); } } +void StreamHistorySession::push_samples(size_t samples, size_t stride, const float* data){ + Data& state = *m_data; + WriteSpinLock lg(state.m_lock); + if (!state.m_current || samples == 0 || state.m_active_channels == 0){ + return; + } + // The working stream is split (planar) channels: channel "c" starts at + // "data[c * stride]" and runs for "samples" samples. (stride may be 0 + // for the single-channel pass-through.) Re-interleave for the tracker. + // The capture card's RL channel swap (INTERLEAVE_RL_96000) is already + // applied by the pair's conversion node matrix, the same way as for + // the sink branch. + const size_t channels = state.m_active_channels; + state.m_interleaved.resize(samples * channels); + for (size_t i = 0; i < samples; i++){ + for (size_t c = 0; c < channels; c++){ + state.m_interleaved[i * channels + c] = data[c * stride + i]; + } + } + state.m_current->on_samples(state.m_interleaved.data(), samples); +} void StreamHistorySession::on_frame(std::shared_ptr frame){ Data& data = *m_data; WriteSpinLock lg(data.m_lock); @@ -129,24 +173,48 @@ void StreamHistorySession::initialize(){ // cout << "video = " << data.m_has_video << endl; + // The tracker's audio shape is the input source's physical working- + // stream shape (what the tree delivers through the working listener), + // not the preset's legacy framing. (DUAL_44100 was previously + // misreported as mono; MONO_96000 was previously framed as 2-sample + // frames at 48000.) "expected_samples_per_frame" keeps the legacy + // framing values for the AudioFloatStreamListener path. switch (data.m_audio_format){ case AudioChannelFormat::NONE: expected_samples_per_frame = 0; + data.m_active_channels = 0; + data.m_active_rate = 0; data.m_current.reset(new StreamHistoryTracker(data.m_logger, data.m_window, 0, 0, data.m_has_video)); return; case AudioChannelFormat::MONO_48000: expected_samples_per_frame = 1; + data.m_active_channels = 1; + data.m_active_rate = 48000; data.m_current.reset(new StreamHistoryTracker(data.m_logger, data.m_window, 1, 48000, data.m_has_video)); return; case AudioChannelFormat::DUAL_44100: expected_samples_per_frame = 2; - data.m_current.reset(new StreamHistoryTracker(data.m_logger, data.m_window, 1, 44100, data.m_has_video)); + data.m_active_channels = 2; + data.m_active_rate = 44100; + data.m_current.reset(new StreamHistoryTracker(data.m_logger, data.m_window, 2, 44100, data.m_has_video)); return; case AudioChannelFormat::DUAL_48000: + expected_samples_per_frame = 2; + data.m_active_channels = 2; + data.m_active_rate = 48000; + data.m_current.reset(new StreamHistoryTracker(data.m_logger, data.m_window, 2, 48000, data.m_has_video)); + return; case AudioChannelFormat::MONO_96000: + expected_samples_per_frame = 2; + data.m_active_channels = 1; + data.m_active_rate = 96000; + data.m_current.reset(new StreamHistoryTracker(data.m_logger, data.m_window, 1, 96000, data.m_has_video)); + return; case AudioChannelFormat::INTERLEAVE_LR_96000: case AudioChannelFormat::INTERLEAVE_RL_96000: expected_samples_per_frame = 2; + data.m_active_channels = 2; + data.m_active_rate = 48000; data.m_current.reset(new StreamHistoryTracker(data.m_logger, data.m_window, 2, 48000, data.m_has_video)); return; default: @@ -165,13 +233,46 @@ void StreamHistorySession::pre_input_change(){ void StreamHistorySession::post_input_change(const std::string& file, const AudioDeviceInfo& device, AudioChannelFormat format){ // cout << "post_input_change()" << endl; Data& data = *m_data; - WriteSpinLock lg(data.m_lock); - if (device){ - data.m_audio_format = format; - }else{ - data.m_audio_format = AudioChannelFormat::NONE; + AudioSession* audio_session; + { + WriteSpinLock lg(data.m_lock); + audio_session = data.m_audio_session; } - initialize(); + // Detach the working listener first: after this returns, no pushes in + // the old shape can reach push_samples() (the pair quiesces in-flight + // pushes). Then update the shapes, then attach for the new shape. + if (audio_session){ + audio_session->remove_working_stream_listener(*this); + } + { + WriteSpinLock lg(data.m_lock); + if (device){ + data.m_audio_format = format; + }else{ + data.m_audio_format = AudioChannelFormat::NONE; + } + initialize(); + } + attach_working_listener(); +} +void StreamHistorySession::attach_working_listener(){ + Data& data = *m_data; + AudioSession* audio_session; + AudioChannelFormat target; + { + WriteSpinLock lg(data.m_lock); + audio_session = data.m_audio_session; + target = data.m_audio_format; + } + if (!audio_session || target == AudioChannelFormat::NONE){ + return; + } + // The remove quiesces any in-flight pushes in the old shape (and is a + // no-op if not attached yet); the node is then built for the new one. + audio_session->remove_working_stream_listener(*this); + QAudioFormat native_format; + set_format(native_format, target); + audio_session->add_working_stream_listener(AudioFormat(native_format), *this); } void StreamHistorySession::pre_shutdown(){ // cout << "pre_shutdown()" << endl; diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.h b/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.h index f6379d414c..dae2f2bcc8 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.h +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.h @@ -13,12 +13,14 @@ #include "Common/Cpp/Containers/Pimpl.h" #include "CommonFramework/AudioPipeline/AudioSession.h" #include "CommonFramework/VideoPipeline/VideoSession.h" +#include "Kernels/AudioPipeline/Kernels_AudioPipeline.h" namespace PokemonAutomation{ class StreamHistorySession : public AudioFloatStreamListener + , public Kernels::AudioPipeline::WorkingAudioStreamListener , public VideoFrameListener , public AudioSession::StateListener , public VideoSession::StateListener @@ -29,8 +31,17 @@ class StreamHistorySession void start(AudioChannelFormat format, bool has_video); bool save(const std::string& filename) const; + // The audio session this session's working-stream listener attaches to + // (set before start(); used to re-attach on input changes). + void set_audio_session(AudioSession& session); + public: + // Legacy float stream (kept until the remaining float consumers + // migrate; see the audio pipeline hand-off doc). virtual void on_samples(const float* data, size_t frames) override; + // Working stream, delivered in the shape requested at attach time + // (the active preset's physical shape), split (planar) channels. + virtual void push_samples(size_t samples, size_t stride, const float* data) override; virtual void on_frame(std::shared_ptr frame) override; public: @@ -47,6 +58,11 @@ class StreamHistorySession void clear(); void initialize(); + // Attaches this session to the audio session's working stream at the + // active preset's physical shape (removing any prior attach first). + // Must be called with no locks held. + void attach_working_listener(); + private: struct Data; Pimpl m_data; diff --git a/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp b/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp index 9bed1fa736..bc420dda97 100644 --- a/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp +++ b/SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp @@ -30,6 +30,7 @@ bool ConsoleSystemSession::try_shutdown() noexcept{ m_video.remove_state_listener(m_history); m_video.remove_state_listener(m_history); m_audio.remove_stream_listener(m_history); + m_audio.remove_working_stream_listener(m_history); m_audio.remove_state_listener(m_history); m_overlay.remove_stat(*m_main_thread_utilization); @@ -77,6 +78,7 @@ ConsoleSystemSession::ConsoleSystemSession( } update_status(); + m_history.set_audio_session(m_audio); m_history.start(m_audio.input_format(), m_video.current_source() != nullptr); try{ From 5ecc57463ced552dd4a7b2bedd026a3a70ed8f79 Mon Sep 17 00:00:00 2001 From: PA Dev Bot <344001113+pa-devbot@users.noreply.github.com> Date: Thu, 22 Oct 2026 18:44:18 +0000 Subject: [PATCH 2/2] Audio pipeline: define InternalWorkingListener before its first destruction. remove_listener() destroys a WorkingListenerEntry (m_working_entries.erase) at its call site, which instantiates ~unique_ptr() there. The nested class was defined further down in the .cpp, so on GCC 15 (x86 CI) the deleter's sizeof static_assert fired on the incomplete type. aarch64's older GCC instantiated it lazily enough to miss it. Moved the definition above the first member-function body so the type is complete at every destruction site in the translation unit. Pure reordering, no behavior change. --- .../Backends/AudioPassthroughPairQt.cpp | 76 ++++++++++--------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.cpp b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.cpp index a8ee89eff6..5c0184dea6 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.cpp +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.cpp @@ -68,6 +68,46 @@ QAudioFormat channel_format(size_t channels){ } +// +// Per-listener working-stream conversion. +// +// (Defined before the first use: remove_listener() destroys a +// WorkingListenerEntry - and with it the unique_ptr +// in it - at its call site, so the type must be complete there.) +// Each attached working-stream listener gets its own +// WorkingAudio2WorkingAudio node (source shape -> the listener's requested +// shape) attached to the source decoder, plus an InternalWorkingListener +// forwarder that carries the requested shape as its nominal one (the node +// validates its listeners against it; the user's listener may have any +// nominal shape of its own). +// +// The entries in m_working_entries survive tree rebuilds: when the input +// changes, build_tree() rebuilds each node for the new source shape, and +// teardown_tree() detaches them. All of these methods require the caller +// to hold m_lock. Detaching blocks on in-flight pushes, so no push can +// ever run through a node or forwarder that is about to be destroyed. +// + +class AudioPassthroughPairQt::InternalWorkingListener final : public Kernels::AudioPipeline::WorkingAudioStreamListener{ +public: + InternalWorkingListener( + Kernels::AudioPipeline::WorkingAudioStreamListener& target, + size_t sample_rate, size_t channels + ) + : Kernels::AudioPipeline::WorkingAudioStreamListener(sample_rate, channels) + , m_target(target) + {} + // The pair always detaches this from its node before destroying it + // (detach_working_entry). "m_target" is the user's listener, owned by + // the caller; it outlives the attach by contract. + virtual void push_samples(size_t samples, size_t stride, const float* data) override{ + m_target.push_samples(samples, stride, data); + } + +private: + Kernels::AudioPipeline::WorkingAudioStreamListener& m_target; +}; + void AudioPassthroughPairQt::add_listener(AudioFloatStreamListener& listener){ auto scope_check = m_sanitizer.check_scope(); @@ -153,42 +193,6 @@ class AudioPassthroughPairQt::InternalFFTListener final : public FFTListener{ }; -// -// Per-listener working-stream conversion. -// -// Each attached working-stream listener gets its own -// WorkingAudio2WorkingAudio node (source shape -> the listener's requested -// shape) attached to the source decoder, plus an InternalWorkingListener -// forwarder that carries the requested shape as its nominal one (the node -// validates its listeners against it; the user's listener may have any -// nominal shape of its own). -// -// The entries in m_working_entries survive tree rebuilds: when the input -// changes, build_tree() rebuilds each node for the new source shape, and -// teardown_tree() detaches them. All of these methods require the caller -// to hold m_lock. Detaching blocks on in-flight pushes, so no push can -// ever run through a node or forwarder that is about to be destroyed. -// - -class AudioPassthroughPairQt::InternalWorkingListener final : public Kernels::AudioPipeline::WorkingAudioStreamListener{ -public: - InternalWorkingListener( - Kernels::AudioPipeline::WorkingAudioStreamListener& target, - size_t sample_rate, size_t channels - ) - : Kernels::AudioPipeline::WorkingAudioStreamListener(sample_rate, channels) - , m_target(target) - {} - // The pair always detaches this from its node before destroying it - // (detach_working_entry). "m_target" is the user's listener, owned by - // the caller; it outlives the attach by contract. - virtual void push_samples(size_t samples, size_t stride, const float* data) override{ - m_target.push_samples(samples, stride, data); - } - -private: - Kernels::AudioPipeline::WorkingAudioStreamListener& m_target; -}; void AudioPassthroughPairQt::build_working_entry(