Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,18 @@ namespace PokemonAutomation{
struct AudioFloatStreamListener;
struct FFTListener;

namespace Kernels{
namespace AudioPipeline{
class WorkingAudioStreamListener;
}
}


// Represents an audio source/sink pair where audio is passed through from
// source -> sink with minimal latency.
//
// If desired, listeners can be attached to receive the FFT spectrums.
// If desired, listeners can be attached to receive the FFT spectrums, or to
// receive the working audio stream converted to a requested shape.
//
// This class is fully thread-safe. Both source and sink can be changed
// asynchronously. Listeners can be attached/detached asynchronously.
Expand All @@ -31,6 +38,15 @@ class AudioPassthroughPair{
virtual void add_listener(FFTListener& listener) = 0;
virtual void remove_listener(FFTListener& listener) = 0;

// Attach a listener that receives the working audio stream converted to
// "format" (sample rate + channel count; the channel layout in
// format.native_format() drives the channel mixing). The pair builds a
// conversion node per listener; the listener receives planar float
// samples in the requested shape, with the input volume applied.
// Re-adding a listener (e.g. with a new format) rebuilds its node.
virtual void add_listener(const AudioFormat& format, Kernels::AudioPipeline::WorkingAudioStreamListener& listener) = 0;
virtual void remove_listener(Kernels::AudioPipeline::WorkingAudioStreamListener& listener) = 0;

public:
virtual ~AudioPassthroughPair() = default;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,14 @@ void AudioSession::add_stream_listener(AudioFloatStreamListener& listener){
void AudioSession::remove_stream_listener(AudioFloatStreamListener& listener){
m_devices->remove_listener(listener);
}
void AudioSession::add_working_stream_listener(
const AudioFormat& format, Kernels::AudioPipeline::WorkingAudioStreamListener& listener
){
m_devices->add_listener(format, listener);
}
void AudioSession::remove_working_stream_listener(Kernels::AudioPipeline::WorkingAudioStreamListener& listener){
m_devices->remove_listener(listener);
}
void AudioSession::add_spectrum_listener(AudioSpectrumHolder::Listener& listener){
m_spectrum_holder.add_listener(listener);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,11 @@ class AudioSession final : public AudioFeed, private FFTListener, private Watchd
void add_stream_listener(AudioFloatStreamListener& listener);
void remove_stream_listener(AudioFloatStreamListener& listener);

// Attach a listener that receives the working audio stream (float)
// converted to the requested format. See AudioPassthroughPair.
void add_working_stream_listener(const AudioFormat& format, Kernels::AudioPipeline::WorkingAudioStreamListener& listener);
void remove_working_stream_listener(Kernels::AudioPipeline::WorkingAudioStreamListener& listener);

void add_spectrum_listener(AudioSpectrumHolder::Listener& listener);
void remove_spectrum_listener(AudioSpectrumHolder::Listener& listener);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
*/

#include "Common/Cpp/Exceptions.h"
#include "Common/Cpp/Containers/Pimpl.tpp"
#include "Common/Cpp/Logging/AbstractLogger.h"
#include "Common/Cpp/PrettyPrint.h"
#include "CommonFramework/GlobalSettingsPanel.h"
Expand Down Expand Up @@ -67,6 +68,46 @@ QAudioFormat channel_format(size_t channels){
}


//
// Per-listener working-stream conversion.
//
// (Defined before the first use: remove_listener() destroys a
// WorkingListenerEntry - and with it the unique_ptr<InternalWorkingListener>
// in it - at its call site, so the type must be complete there.)
// Each attached working-stream listener gets its own
// WorkingAudio2WorkingAudio node (source shape -> the listener's requested
// shape) attached to the source decoder, plus an InternalWorkingListener
// forwarder that carries the requested shape as its nominal one (the node
// validates its listeners against it; the user's listener may have any
// nominal shape of its own).
//
// The entries in m_working_entries survive tree rebuilds: when the input
// changes, build_tree() rebuilds each node for the new source shape, and
// teardown_tree() detaches them. All of these methods require the caller
// to hold m_lock. Detaching blocks on in-flight pushes, so no push can
// ever run through a node or forwarder that is about to be destroyed.
//

class AudioPassthroughPairQt::InternalWorkingListener final : public Kernels::AudioPipeline::WorkingAudioStreamListener{
public:
InternalWorkingListener(
Kernels::AudioPipeline::WorkingAudioStreamListener& target,
size_t sample_rate, size_t channels
)
: Kernels::AudioPipeline::WorkingAudioStreamListener(sample_rate, channels)
, m_target(target)
{}
// The pair always detaches this from its node before destroying it
// (detach_working_entry). "m_target" is the user's listener, owned by
// the caller; it outlives the attach by contract.
virtual void push_samples(size_t samples, size_t stride, const float* data) override{
m_target.push_samples(samples, stride, data);
}

private:
Kernels::AudioPipeline::WorkingAudioStreamListener& m_target;
};


void AudioPassthroughPairQt::add_listener(AudioFloatStreamListener& listener){
auto scope_check = m_sanitizer.check_scope();
Expand All @@ -86,6 +127,25 @@ void AudioPassthroughPairQt::remove_listener(FFTListener& listener){
WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION);
m_fft_listeners.remove(listener);
}
void AudioPassthroughPairQt::add_listener(
const AudioFormat& format, Kernels::AudioPipeline::WorkingAudioStreamListener& listener
){
auto scope_check = m_sanitizer.check_scope();
WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION);
WorkingListenerEntry& entry = m_working_entries[&listener];
entry.m_format = format;
build_working_entry(entry, listener);
}
void AudioPassthroughPairQt::remove_listener(Kernels::AudioPipeline::WorkingAudioStreamListener& listener){
auto scope_check = m_sanitizer.check_scope();
WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION);
auto iter = m_working_entries.find(&listener);
if (iter == m_working_entries.end()){
return;
}
detach_working_entry(iter->second);
m_working_entries.erase(iter);
}



Expand Down Expand Up @@ -133,6 +193,69 @@ class AudioPassthroughPairQt::InternalFFTListener final : public FFTListener{
};




void AudioPassthroughPairQt::build_working_entry(
WorkingListenerEntry& entry, Kernels::AudioPipeline::WorkingAudioStreamListener& listener
){
if (!m_source_decoder || m_input_format == AudioChannelFormat::NONE){
return;
}
// Re-adding with a new format: detach the old node first.
if (entry.m_node){
detach_working_entry(entry);
}

const size_t raw_rate = m_reader->raw_sample_rate();
const size_t raw_channels = m_reader->raw_channels();
const size_t out_rate = entry.m_format.sample_rate();
const size_t out_channels = entry.m_format.channels();
if (out_rate == 0 || out_channels == 0){
m_logger.log("Working listener: invalid requested format; not attached.", COLOR_RED);
return;
}

// INTERLEAVE_RL_96000 plays the card's channel 1 (the odd physical
// samples) as the left channel; the input reinterpretation preserves
// the physical order, so the swap is applied to the matrix (the same
// special case the sink branch uses).
std::vector<float> matrix;
if (m_input_format == AudioChannelFormat::INTERLEAVE_RL_96000
&& raw_channels == 2 && out_channels == 2){
// out = [in 1, in 0]; weight(in i -> out o) at index i + in_ch * o.
matrix = {0.0f, 1.0f, 1.0f, 0.0f};
}else{
matrix = build_channel_mixer_matrix(
channel_format(raw_channels), entry.m_format.native_format()
);
}
if (matrix.size() != raw_channels * out_channels){
m_logger.log("Working listener: no channel mixer matrix for this conversion; not attached.", COLOR_RED);
return;
}

entry.m_node.reset(new Kernels::AudioPipeline::WorkingAudio2WorkingAudio(
m_logger, raw_rate, out_rate, raw_channels, out_channels, std::move(matrix)
));
entry.m_forwarder.reset(new InternalWorkingListener(listener, out_rate, out_channels));
entry.m_node->add_listener(*entry.m_forwarder);
m_source_decoder->add_listener(*entry.m_node);
}


void AudioPassthroughPairQt::detach_working_entry(WorkingListenerEntry& entry){
if (entry.m_forwarder && entry.m_node){
entry.m_node->remove_listener(*entry.m_forwarder);
}
if (entry.m_node && m_source_decoder){
m_source_decoder->remove_listener(*entry.m_node);
}
entry.m_forwarder.reset();
entry.m_node.reset();
}



//
// The new pipeline tree.
//
Expand All @@ -152,6 +275,9 @@ void AudioPassthroughPairQt::teardown_tree(){
if (m_source_decoder){
m_reader->remove_raw_listener(*m_source_decoder);
}
for (auto& entry : m_working_entries){
detach_working_entry(entry.second);
}
m_fft_listener.reset();
m_fft_node.reset();
m_fft_conv.reset();
Expand Down Expand Up @@ -194,6 +320,12 @@ void AudioPassthroughPairQt::build_tree(float input_volume){
));
m_reader->add_raw_listener(*m_source_decoder);

// Rebuild the per-listener working-stream conversion nodes for the new
// source shape (the entries themselves survive the rebuild).
for (auto& entry : m_working_entries){
build_working_entry(entry.second, *entry.first);
}

// FFT branch: source channels -> 1 channel, source rate -> FFT rate.
// The tree runs the FFT at 48 kHz for all inputs (the post-migration
// standard); a 44.1 kHz source is resampled up here. The branch ends
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
#define PokemonAutomation_AudioPipeline_AudioPassthroughPairQt_H

#include <memory>
#include <map>
//#include <set>
#include <QObject>
#include "Common/Cpp/ListenerSet.h"
Expand All @@ -27,6 +28,7 @@ namespace AudioPipeline{
class RawAudio2WorkingAudio;
class WorkingAudio2RawAudio;
class WorkingAudio2WorkingAudio;
class WorkingAudioStreamListener;
}
}

Expand All @@ -39,6 +41,9 @@ class AudioPassthroughPairQt final : public QObject, public AudioPassthroughPair
virtual void add_listener(FFTListener& listener) override;
virtual void remove_listener(FFTListener& listener) override;

virtual void add_listener(const AudioFormat& format, Kernels::AudioPipeline::WorkingAudioStreamListener& listener) override;
virtual void remove_listener(Kernels::AudioPipeline::WorkingAudioStreamListener& listener) override;


public:
virtual ~AudioPassthroughPairQt();
Expand Down Expand Up @@ -66,6 +71,18 @@ class AudioPassthroughPairQt final : public QObject, public AudioPassthroughPair
private:
class SampleListener;
class InternalFFTListener;
class InternalWorkingListener;

// One entry per attached working-stream listener: the requested format
// plus the conversion node (source shape -> requested shape) and the
// forwarder that carries the requested shape as its nominal one. The
// node attaches to m_source_decoder; the forwarder attaches to the node.
// All of these are only touched while holding m_lock.
struct WorkingListenerEntry{
AudioFormat m_format;
std::unique_ptr<Kernels::AudioPipeline::WorkingAudio2WorkingAudio> m_node;
std::unique_ptr<InternalWorkingListener> m_forwarder;
};

void init_audio_sink();

Expand All @@ -82,6 +99,9 @@ class AudioPassthroughPairQt final : public QObject, public AudioPassthroughPair
void build_tree(float input_volume);
void rebuild_sink_branch();

void build_working_entry(WorkingListenerEntry& entry, Kernels::AudioPipeline::WorkingAudioStreamListener& listener);
void detach_working_entry(WorkingListenerEntry& entry);


private:
Logger& m_logger;
Expand All @@ -108,6 +128,12 @@ class AudioPassthroughPairQt final : public QObject, public AudioPassthroughPair
std::unique_ptr<Kernels::AudioPipeline::WorkingAudio2WorkingAudio> m_fft_conv; // Source shape -> 48 kHz mono (mix + resample).
std::unique_ptr<WorkingAudioToFFT> m_fft_node; // The tree's FFT endpoint.

// Per-listener working-stream conversion nodes. The entries survive
// tree rebuilds (the listeners stay attached across input changes);
// their nodes are rebuilt for each new source shape in build_tree()
// and detached in teardown_tree().
std::map<Kernels::AudioPipeline::WorkingAudioStreamListener*, WorkingListenerEntry> m_working_entries;

AudioDeviceInfo m_output_device;
float m_file_input_multiplier = 1.0;
float m_device_input_multiplier = 1.0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,18 @@ void AudioPassthroughPairQtThread::remove_listener(FFTListener& listener){
AudioPassthroughPairQt* body = m_body;
body->remove_listener(listener);
}
void AudioPassthroughPairQtThread::add_listener(
const AudioFormat& format, Kernels::AudioPipeline::WorkingAudioStreamListener& listener
){
auto scope_check = m_sanitizer.check_scope();
AudioPassthroughPairQt* body = m_body;
body->add_listener(format, listener);
}
void AudioPassthroughPairQtThread::remove_listener(Kernels::AudioPipeline::WorkingAudioStreamListener& listener){
auto scope_check = m_sanitizer.check_scope();
AudioPassthroughPairQt* body = m_body;
body->remove_listener(listener);
}


AudioPassthroughPairQtThread::AudioPassthroughPairQtThread(Logger& logger)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ class AudioPassthroughPairQtThread : public AudioPassthroughPair{
virtual void add_listener(FFTListener& listener) override;
virtual void remove_listener(FFTListener& listener) override;

virtual void add_listener(const AudioFormat& format, Kernels::AudioPipeline::WorkingAudioStreamListener& listener) override;
virtual void remove_listener(Kernels::AudioPipeline::WorkingAudioStreamListener& listener) override;

public:
AudioPassthroughPairQtThread(Logger& logger);
~AudioPassthroughPairQtThread();
Expand Down
Loading
Loading